Differential D46690 Diff 144949 sys/dev/gve/gve_adminq.c

Changeset View

Standalone View

sys/dev/gve/gve_adminq.c

/*-		/*-
* SPDX-License-Identifier: BSD-3-Clause		* SPDX-License-Identifier: BSD-3-Clause
*		*
* Copyright (c) 2023 Google LLC		* Copyright (c) 2023-2024 Google LLC
*		*
* Redistribution and use in source and binary forms, with or without modification,		* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:		* are permitted provided that the following conditions are met:
*		*
* 1. Redistributions of source code must retain the above copyright notice, this		* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.		* list of conditions and the following disclaimer.
*		*
* 2. Redistributions in binary form must reproduce the above copyright notice,		* 2. Redistributions in binary form must reproduce the above copyright notice,
Show All 39 Lines
#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected." \		#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected." \
" Possible older version of guest driver.\n"		" Possible older version of guest driver.\n"

static		static
void gve_parse_device_option(struct gve_priv *priv,		void gve_parse_device_option(struct gve_priv *priv,
struct gve_device_descriptor *device_descriptor,		struct gve_device_descriptor *device_descriptor,
struct gve_device_option *option,		struct gve_device_option *option,
struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,		struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
		struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)		struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
{		{
uint32_t req_feat_mask = be32toh(option->required_features_mask);		uint32_t req_feat_mask = be32toh(option->required_features_mask);
uint16_t option_length = be16toh(option->option_length);		uint16_t option_length = be16toh(option->option_length);
uint16_t option_id = be16toh(option->option_id);		uint16_t option_id = be16toh(option->option_id);

/*		/*
* If the length or feature mask doesn't match, continue without		* If the length or feature mask doesn't match, continue without
Show All 12 Lines	case GVE_DEV_OPT_ID_GQI_QPL:

if (option_length > sizeof(**dev_op_gqi_qpl)) {		if (option_length > sizeof(**dev_op_gqi_qpl)) {
device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,		device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
"GQI QPL");		"GQI QPL");
}		}
dev_op_gqi_qpl = (void )(option + 1);		dev_op_gqi_qpl = (void )(option + 1);
break;		break;

		case GVE_DEV_OPT_ID_DQO_RDA:
		if (option_length < sizeof(**dev_op_dqo_rda) \|\|
		req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
		device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
		"DQO RDA", (int)sizeof(**dev_op_dqo_rda),
		GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
		option_length, req_feat_mask);
		break;
		}

		if (option_length > sizeof(**dev_op_dqo_rda)) {
		device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
		"DQO RDA");
		}
		dev_op_dqo_rda = (void )(option + 1);
		break;

case GVE_DEV_OPT_ID_JUMBO_FRAMES:		case GVE_DEV_OPT_ID_JUMBO_FRAMES:
if (option_length < sizeof(**dev_op_jumbo_frames) \|\|		if (option_length < sizeof(**dev_op_jumbo_frames) \|\|
req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {		req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,		device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
"Jumbo Frames", (int)sizeof(**dev_op_jumbo_frames),		"Jumbo Frames", (int)sizeof(**dev_op_jumbo_frames),
GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,		GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
option_length, req_feat_mask);		option_length, req_feat_mask);
break;		break;
Show All 16 Lines	void gve_parse_device_option(struct gve_priv *priv,
}		}
}		}

/* Process all device options for a given describe device call. */		/* Process all device options for a given describe device call. */
static int		static int
gve_process_device_options(struct gve_priv *priv,		gve_process_device_options(struct gve_priv *priv,
struct gve_device_descriptor *descriptor,		struct gve_device_descriptor *descriptor,
struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,		struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
		struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)		struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
{		{
char desc_end = (char )descriptor + be16toh(descriptor->total_length);		char desc_end = (char )descriptor + be16toh(descriptor->total_length);
const int num_options = be16toh(descriptor->num_device_options);		const int num_options = be16toh(descriptor->num_device_options);
struct gve_device_option *dev_opt;		struct gve_device_option *dev_opt;
int i;		int i;

/* The options struct directly follows the device descriptor. */		/* The options struct directly follows the device descriptor. */
dev_opt = (void *)(descriptor + 1);		dev_opt = (void *)(descriptor + 1);
for (i = 0; i < num_options; i++) {		for (i = 0; i < num_options; i++) {
if ((char *)(dev_opt + 1) > desc_end \|\|		if ((char *)(dev_opt + 1) > desc_end \|\|
(char *)(dev_opt + 1) + be16toh(dev_opt->option_length) > desc_end) {		(char *)(dev_opt + 1) + be16toh(dev_opt->option_length) > desc_end) {
device_printf(priv->dev,		device_printf(priv->dev,
"options exceed device_descriptor's total length.\n");		"options exceed device descriptor's total length.\n");
return (EINVAL);		return (EINVAL);
}		}

gve_parse_device_option(priv, descriptor, dev_opt,		gve_parse_device_option(priv, descriptor, dev_opt,
dev_op_gqi_qpl, dev_op_jumbo_frames);		dev_op_gqi_qpl, dev_op_dqo_rda, dev_op_jumbo_frames);
dev_opt = (void )((char )(dev_opt + 1) + be16toh(dev_opt->option_length));		dev_opt = (void )((char )(dev_opt + 1) + be16toh(dev_opt->option_length));
}		}

return (0);		return (0);
}		}

static int gve_adminq_execute_cmd(struct gve_priv *priv,		static int gve_adminq_execute_cmd(struct gve_priv *priv,
struct gve_adminq_command *cmd);		struct gve_adminq_command *cmd);
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines	gve_adminq_create_rx_queue(struct gve_priv *priv, uint32_t queue_index)
struct gve_rx_ring *rx = &priv->rx[queue_index];		struct gve_rx_ring *rx = &priv->rx[queue_index];
struct gve_dma_handle *qres_dma = &rx->com.q_resources_mem;		struct gve_dma_handle *qres_dma = &rx->com.q_resources_mem;

bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);		bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);

cmd.opcode = htobe32(GVE_ADMINQ_CREATE_RX_QUEUE);		cmd.opcode = htobe32(GVE_ADMINQ_CREATE_RX_QUEUE);
cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {		cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
.queue_id = htobe32(queue_index),		.queue_id = htobe32(queue_index),
.index = htobe32(queue_index),
.ntfy_id = htobe32(rx->com.ntfy_id),		.ntfy_id = htobe32(rx->com.ntfy_id),
.queue_resources_addr = htobe64(qres_dma->bus_addr),		.queue_resources_addr = htobe64(qres_dma->bus_addr),
.rx_desc_ring_addr = htobe64(rx->desc_ring_mem.bus_addr),
.rx_data_ring_addr = htobe64(rx->data_ring_mem.bus_addr),
.queue_page_list_id = htobe32((rx->com.qpl)->id),
.rx_ring_size = htobe16(priv->rx_desc_cnt),		.rx_ring_size = htobe16(priv->rx_desc_cnt),
.packet_buffer_size = htobe16(GVE_DEFAULT_RX_BUFFER_SIZE),		.packet_buffer_size = htobe16(GVE_DEFAULT_RX_BUFFER_SIZE),
};		};

		if (gve_is_gqi(priv)) {
		cmd.create_rx_queue.rx_desc_ring_addr =
		htobe64(rx->desc_ring_mem.bus_addr);
		cmd.create_rx_queue.rx_data_ring_addr =
		htobe64(rx->data_ring_mem.bus_addr);
		cmd.create_rx_queue.index =
		htobe32(queue_index);
		cmd.create_rx_queue.queue_page_list_id =
		htobe32((rx->com.qpl)->id);
		} else {
		cmd.create_rx_queue.queue_page_list_id =
		htobe32(GVE_RAW_ADDRESSING_QPL_ID);
		cmd.create_rx_queue.rx_desc_ring_addr =
		htobe64(rx->dqo.compl_ring_mem.bus_addr);
		cmd.create_rx_queue.rx_data_ring_addr =
		htobe64(rx->desc_ring_mem.bus_addr);
		cmd.create_rx_queue.rx_buff_ring_size =
		htobe16(priv->rx_desc_cnt);
		cmd.create_rx_queue.enable_rsc =
		!!((if_getcapenable(priv->ifp) & IFCAP_LRO) &&
		!gve_disable_hw_lro);
		}

return (gve_adminq_execute_cmd(priv, &cmd));		return (gve_adminq_execute_cmd(priv, &cmd));
}		}

int		int
gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)		gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)
{		{
int err;		int err;
int i;		int i;
Show All 25 Lines	gve_adminq_create_tx_queue(struct gve_priv *priv, uint32_t queue_index)

bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);		bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);

cmd.opcode = htobe32(GVE_ADMINQ_CREATE_TX_QUEUE);		cmd.opcode = htobe32(GVE_ADMINQ_CREATE_TX_QUEUE);
cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {		cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
.queue_id = htobe32(queue_index),		.queue_id = htobe32(queue_index),
.queue_resources_addr = htobe64(qres_dma->bus_addr),		.queue_resources_addr = htobe64(qres_dma->bus_addr),
.tx_ring_addr = htobe64(tx->desc_ring_mem.bus_addr),		.tx_ring_addr = htobe64(tx->desc_ring_mem.bus_addr),
.queue_page_list_id = htobe32((tx->com.qpl)->id),
.ntfy_id = htobe32(tx->com.ntfy_id),		.ntfy_id = htobe32(tx->com.ntfy_id),
.tx_ring_size = htobe16(priv->tx_desc_cnt),		.tx_ring_size = htobe16(priv->tx_desc_cnt),
};		};

		if (gve_is_gqi(priv)) {
		cmd.create_tx_queue.queue_page_list_id =
		htobe32((tx->com.qpl)->id);
		} else {
		cmd.create_tx_queue.queue_page_list_id =
		htobe32(GVE_RAW_ADDRESSING_QPL_ID);
		cmd.create_tx_queue.tx_comp_ring_addr =
		htobe64(tx->dqo.compl_ring_mem.bus_addr);
		cmd.create_tx_queue.tx_comp_ring_size =
		htobe16(priv->tx_desc_cnt);
		}
return (gve_adminq_execute_cmd(priv, &cmd));		return (gve_adminq_execute_cmd(priv, &cmd));
}		}

int		int
gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)		gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)
{		{
int err;		int err;
int i;		int i;
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines

int		int
gve_adminq_describe_device(struct gve_priv *priv)		gve_adminq_describe_device(struct gve_priv *priv)
{		{
struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};		struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
struct gve_device_descriptor *desc;		struct gve_device_descriptor *desc;
struct gve_dma_handle desc_mem;		struct gve_dma_handle desc_mem;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;		struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
		struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;		struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
uint32_t supported_features_mask = 0;		uint32_t supported_features_mask = 0;
int rc;		int rc;
int i;		int i;

rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);		rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);
if (rc != 0) {		if (rc != 0) {
device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");		device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");
Show All 12 Lines	gve_adminq_describe_device(struct gve_priv *priv)
bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_PREWRITE);		bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_PREWRITE);

rc = gve_adminq_execute_cmd(priv, &aq_cmd);		rc = gve_adminq_execute_cmd(priv, &aq_cmd);
if (rc != 0)		if (rc != 0)
goto free_device_descriptor;		goto free_device_descriptor;

bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_POSTREAD);		bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_POSTREAD);

rc = gve_process_device_options(priv, desc, &dev_op_gqi_qpl,		rc = gve_process_device_options(priv, desc,
		&dev_op_gqi_qpl, &dev_op_dqo_rda,
&dev_op_jumbo_frames);		&dev_op_jumbo_frames);
if (rc != 0)		if (rc != 0)
goto free_device_descriptor;		goto free_device_descriptor;

if (dev_op_gqi_qpl != NULL) {		if (dev_op_dqo_rda != NULL) {
		snprintf(gve_queue_format, sizeof(gve_queue_format),
		"%s", "DQO RDA");
		priv->queue_format = GVE_DQO_RDA_FORMAT;
		supported_features_mask = be32toh(
		dev_op_dqo_rda->supported_features_mask);
		if (bootverbose)
		device_printf(priv->dev,
		"Driver is running with DQO RDA queue format.\n");
		} else if (dev_op_gqi_qpl != NULL) {
		snprintf(gve_queue_format, sizeof(gve_queue_format),
		"%s", "GQI QPL");
priv->queue_format = GVE_GQI_QPL_FORMAT;		priv->queue_format = GVE_GQI_QPL_FORMAT;
supported_features_mask = be32toh(		supported_features_mask = be32toh(
dev_op_gqi_qpl->supported_features_mask);		dev_op_gqi_qpl->supported_features_mask);
if (bootverbose)		if (bootverbose)
device_printf(priv->dev,		device_printf(priv->dev,
"Driver is running with GQI QPL queue format.\n");		"Driver is running with GQI QPL queue format.\n");
} else {		} else {
device_printf(priv->dev, "No compatible queue formats\n");		device_printf(priv->dev, "No compatible queue formats\n");
rc = (EINVAL);		rc = EINVAL;
goto free_device_descriptor;		goto free_device_descriptor;
}		}

priv->num_event_counters = be16toh(desc->counters);		priv->num_event_counters = be16toh(desc->counters);
priv->default_num_queues = be16toh(desc->default_num_queues);		priv->default_num_queues = be16toh(desc->default_num_queues);
priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);		priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);
priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);		priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);
priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);		priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	aq_cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
.driver_info_len = htobe64(driver_info_len),		.driver_info_len = htobe64(driver_info_len),
.driver_info_addr = htobe64(driver_info_addr),		.driver_info_addr = htobe64(driver_info_addr),
};		};

return (gve_adminq_execute_cmd(priv, &aq_cmd));		return (gve_adminq_execute_cmd(priv, &aq_cmd));
}		}

int		int
		gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
		struct gve_ptype_lut *ptype_lut_dqo)
		{
		struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
		struct gve_ptype_map *ptype_map;
		struct gve_dma_handle dma;
		int err = 0;
		int i;

		err = gve_dma_alloc_coherent(priv, sizeof(*ptype_map), PAGE_SIZE, &dma);
		if (err)
		return (err);
		ptype_map = dma.cpu_addr;

		aq_cmd.opcode = htobe32(GVE_ADMINQ_GET_PTYPE_MAP);
		aq_cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
		.ptype_map_len = htobe64(sizeof(*ptype_map)),
		.ptype_map_addr = htobe64(dma.bus_addr),
		};

		err = gve_adminq_execute_cmd(priv, &aq_cmd);
		if (err)
		goto err;

		/* Populate ptype_lut_dqo. */
		for (i = 0; i < GVE_NUM_PTYPES; i++) {
		ptype_lut_dqo->ptypes[i].l3_type = ptype_map->ptypes[i].l3_type;
		ptype_lut_dqo->ptypes[i].l4_type = ptype_map->ptypes[i].l4_type;
		}
		err:
		gve_dma_free_coherent(&dma);
		return (err);
		}

		int
gve_adminq_alloc(struct gve_priv *priv)		gve_adminq_alloc(struct gve_priv *priv)
{		{
int rc;		int rc;

if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))		if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))
return (0);		return (0);

if (priv->aq_mem.cpu_addr == NULL) {		if (priv->aq_mem.cpu_addr == NULL) {
Show All 20 Lines	gve_adminq_alloc(struct gve_priv *priv)
priv->adminq_register_page_list_cnt = 0;		priv->adminq_register_page_list_cnt = 0;
priv->adminq_unregister_page_list_cnt = 0;		priv->adminq_unregister_page_list_cnt = 0;
priv->adminq_create_tx_queue_cnt = 0;		priv->adminq_create_tx_queue_cnt = 0;
priv->adminq_create_rx_queue_cnt = 0;		priv->adminq_create_rx_queue_cnt = 0;
priv->adminq_destroy_tx_queue_cnt = 0;		priv->adminq_destroy_tx_queue_cnt = 0;
priv->adminq_destroy_rx_queue_cnt = 0;		priv->adminq_destroy_rx_queue_cnt = 0;
priv->adminq_dcfg_device_resources_cnt = 0;		priv->adminq_dcfg_device_resources_cnt = 0;
priv->adminq_set_driver_parameter_cnt = 0;		priv->adminq_set_driver_parameter_cnt = 0;
		priv->adminq_get_ptype_map_cnt = 0;

gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,		gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,
priv->adminq_bus_addr / ADMINQ_SIZE);		priv->adminq_bus_addr / ADMINQ_SIZE);

gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);		gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);
return (0);		return (0);
}		}

▲ Show 20 Lines • Show All 211 Lines • ▼ Show 20 Lines	case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
break;		break;

case GVE_ADMINQ_SET_DRIVER_PARAMETER:		case GVE_ADMINQ_SET_DRIVER_PARAMETER:
priv->adminq_set_driver_parameter_cnt++;		priv->adminq_set_driver_parameter_cnt++;
break;		break;

case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:		case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
priv->adminq_verify_driver_compatibility_cnt++;		priv->adminq_verify_driver_compatibility_cnt++;
		break;

		case GVE_ADMINQ_GET_PTYPE_MAP:
		priv->adminq_get_ptype_map_cnt++;
break;		break;

default:		default:
device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);		device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);
}		}

return (0);		return (0);
}		}
Show All 23 Lines