D59029.diff
No OneTemporary
Actions

Size

17 KB

Referenced Files

None

Subscribers

None

D59029.diff
View Options

	Index: share/man/man4/ice.4
	===================================================================
	--- share/man/man4/ice.4
	+++ share/man/man4/ice.4
	@@ -1185,6 +1185,16 @@
	A later VF reset can retry this recovery; a PF or device reset, or SR-IOV
	configuration recreation, can also recover the VF.
	.Pp
	+The driver protects the shared PF mailbox from a VF which submits requests
	+faster than the PF can service them.
	+E830 controllers enforce a per-VF limit of 64 outstanding messages in
	+hardware.
	+On older controllers, the driver detects a VF responsible for at least 63
	+messages in a congested mailbox snapshot, resets it with its queues disabled,
	+and discards its requests so that other VFs can continue to use the mailbox.
	+An externally initiated VF function-level reset, PF reset, or SR-IOV
	+configuration recreation releases the VF.
	+.Pp
	For each configured VF,
	.Xr ifconfig 8
	with the
	@@ -1195,8 +1205,9 @@
	The generic state reports whether malicious-driver detection has blocked the
	VF and whether its queues can currently carry traffic.
	The driver group reports cumulative transmit and receive malicious-driver
	-events, whether mirroring is configured, the source VSI when configured, and
	-whether the ingress and egress hardware mirror rules are active.
	+events, software mailbox-overflow isolation on pre-E830 controllers, whether
	+mirroring is configured, the source VSI when configured, and whether the
	+ingress and egress hardware mirror rules are active.
	.Pp
	Structured status consumers receive this information in the version 1
	.Cm driver.ice
	@@ -1218,6 +1229,15 @@
	Cumulative number of transmit malicious-driver latches attributed to the VF.
	.It Cm mdd-rx-events
	Cumulative number of receive malicious-driver latches attributed to the VF.
	+.It Cm mailbox-blocked
	+Boolean indicating that software mailbox-overflow detection has isolated the
	+VF.
	+This field is omitted on E830 controllers, which enforce the mailbox limit in
	+hardware without persistent software isolation.
	+.It Cm mailbox-overflow-events
	+Cumulative number of times software mailbox-overflow detection has isolated
	+the VF.
	+This field is omitted on E830 controllers.
	.El
	.Pp
	An up to date list of parameters and their defaults can be found by using
	Index: sys/dev/ice/ice_iflib.h
	===================================================================
	--- sys/dev/ice/ice_iflib.h
	+++ sys/dev/ice/ice_iflib.h
	@@ -348,6 +348,7 @@
	#ifdef PCI_IOV
	struct ice_vf *vfs;
	u16 num_vfs;
	+ u8 mbx_admin_passes;
	bool mdd_auto_reset_vf;
	#endif
	struct ice_resmgr os_imgr;
	Index: sys/dev/ice/ice_iov.h
	===================================================================
	--- sys/dev/ice/ice_iov.h
	+++ sys/dev/ice/ice_iov.h
	@@ -74,6 +74,7 @@
	VF_FLAG_REBUILD_FAILED = BIT(6),
	VF_FLAG_RESET_FAILED = BIT(7),
	VF_FLAG_MDD_BLOCKED = BIT(8),
	+ VF_FLAG_MBX_BLOCKED = BIT(9),
	};

	struct ice_vf_mac_filter {
	@@ -114,6 +115,8 @@

	u64 mdd_tx_events;
	u64 mdd_rx_events;
	+ struct ice_mbx_vf_info mbx_info;
	+ u64 mbx_overflow_events;
	struct timeval last_mdd_log;
	};

	@@ -144,8 +147,10 @@
	u32 ice_iov_handle_mdd(struct ice_softc *sc);
	void ice_iov_notify_vfs_reset(struct ice_softc *sc);
	int ice_iov_quiesce_vfs_for_reset(struct ice_softc *sc);
	+void ice_iov_reconfigure_mbx(struct ice_softc *sc);

	-void ice_vc_handle_vf_msg(struct ice_softc sc, struct ice_rq_event_info event);
	+void ice_vc_handle_vf_msg(struct ice_softc sc, struct ice_rq_event_info event,
	+ struct ice_mbx_data *mbx_data);
	void ice_vc_notify_all_vfs_link_state(struct ice_softc *sc);

	#endif /* _ICE_IOV_H_ */
	Index: sys/dev/ice/ice_iov.c
	===================================================================
	--- sys/dev/ice/ice_iov.c
	+++ sys/dev/ice/ice_iov.c
	@@ -51,6 +51,8 @@
	#define ICE_VF_STATUS_MIRROR_EGRESS_ACTIVE "mirror-egress-active"
	#define ICE_VF_STATUS_MDD_TX_EVENTS "mdd-tx-events"
	#define ICE_VF_STATUS_MDD_RX_EVENTS "mdd-rx-events"
	+#define ICE_VF_STATUS_MBX_BLOCKED "mailbox-blocked"
	+#define ICE_VF_STATUS_MBX_OVERFLOW_EVENTS "mailbox-overflow-events"

	#define ICE_VC_MAX_RX_BUFFER \
	((16 * 1024) - BIT(ICE_RLAN_CTX_DBUF_S))
	@@ -72,6 +74,7 @@
	static int ice_iov_restore_vf_host_config(struct ice_softc *sc,
	struct ice_vf *vf);
	static void ice_iov_clear_vf_queue_state(struct ice_vf *vf);
	+static void ice_iov_clear_vf_mbx(struct ice_softc sc, struct ice_vf vf);
	static void ice_iov_complete_vf_reset(struct ice_softc *sc,
	struct ice_vf *vf, bool restore_mapping);
	static void ice_iov_ready_vf(struct ice_softc sc, struct ice_vf vf);
	@@ -189,12 +192,38 @@
	"pci_iov_attach failed (error=%s)\n",
	ice_err_str(error));
	ice_clear_bit(ICE_FEATURE_SRIOV, sc->feat_en);
	- } else
	+ } else {
	ice_set_bit(ICE_FEATURE_SRIOV, sc->feat_en);
	+ if (ice_is_e830(&sc->hw))
	+ ice_iov_reconfigure_mbx(sc);
	+ else
	+ ice_mbx_init_snapshot(&sc->hw);
	+ }

	return (error);
	}

	+/**
	+ * ice_iov_reconfigure_mbx - Restore hardware mailbox flood protection
	+ * @sc: device softc structure
	+ *
	+ * E830 limits each VF's outstanding messages in hardware. The threshold
	+ * register is reset by a core reset and must be restored during rebuild.
	+ * Older devices use the software snapshot detector instead.
	+ */
	+void
	+ice_iov_reconfigure_mbx(struct ice_softc *sc)
	+{
	+ struct ice_hw *hw = &sc->hw;
	+
	+ if (!ice_is_e830(hw))
	+ return;
	+
	+ wr32(hw, E830_MBX_PF_IN_FLIGHT_VF_MSGS_THRESH,
	+ ICE_MBX_OVERFLOW_WATERMARK);
	+ ice_flush(hw);
	+}
	+
	/**
	* ice_iov_detach - Teardown SR-IOV PF host support
	* @sc: device softc structure
	@@ -238,8 +267,13 @@
	return (ENOMEM);

	/* Initialize each VF with basic information */
	- for (int i = 0; i < num_vfs; i++)
	+ for (int i = 0; i < num_vfs; i++) {
	sc->vfs[i].vf_num = i;
	+ if (ice_is_e830(&sc->hw))
	+ ice_mbx_vf_clear_cnt_e830(&sc->hw, i);
	+ else
	+ ice_mbx_init_vf_info(&sc->hw, &sc->vfs[i].mbx_info);
	+ }

	/* Save off number of configured VFs */
	sc->num_vfs = num_vfs;
	@@ -654,7 +688,8 @@
	(vf_flags & VF_FLAG_ENABLED) != 0 && vsi != NULL;
	info->initialized = (vf_flags & VF_FLAG_INITIALIZED) != 0;
	info->traffic_enabled = info->configured &&
	- (vf_flags & VF_FLAG_MDD_BLOCKED) == 0 &&
	+ (vf_flags & (VF_FLAG_MDD_BLOCKED \|
	+ VF_FLAG_MBX_BLOCKED)) == 0 &&
	(vf->txq_configured != 0 \|\| vf->rxq_enabled != 0);
	info->mdd_blocked = (vf_flags & VF_FLAG_MDD_BLOCKED) != 0;
	if (!ETHER_IS_ZERO(vf->mac)) {
	@@ -673,7 +708,8 @@
	ICE_INVALID_MIRROR_VSI;
	extension = if_vf_status_add_extension(info,
	ICE_VF_STATUS_NAMESPACE, ICE_VF_STATUS_VERSION,
	- mirror_configured ? 6 : 5);
	+ (mirror_configured ? 6 : 5) +
	+ (ice_is_e830(&sc->hw) ? 0 : 2));
	if (extension == NULL) {
	if_vf_status_free(status);
	return (ENOMEM);
	@@ -694,8 +730,16 @@
	field++;
	if_vf_extension_set_number(extension, field++,
	ICE_VF_STATUS_MDD_TX_EVENTS, vf->mdd_tx_events);
	- if_vf_extension_set_number(extension, field,
	+ if_vf_extension_set_number(extension, field++,
	ICE_VF_STATUS_MDD_RX_EVENTS, vf->mdd_rx_events);
	+ if (!ice_is_e830(&sc->hw)) {
	+ if_vf_extension_set_bool(extension, field++,
	+ ICE_VF_STATUS_MBX_BLOCKED,
	+ (vf_flags & VF_FLAG_MBX_BLOCKED) != 0);
	+ if_vf_extension_set_number(extension, field,
	+ ICE_VF_STATUS_MBX_OVERFLOW_EVENTS,
	+ vf->mbx_overflow_events);
	+ }
	}
	info->allow_set_mac = (vf_flags & VF_FLAG_SET_MAC_CAP) != 0;
	info->allow_set_vlan = (vf_flags & VF_FLAG_VLAN_CAP) != 0;
	@@ -720,6 +764,8 @@
	/* Release per-VF resources */
	for (int i = 0; i < sc->num_vfs; i++) {
	vf = &sc->vfs[i];
	+ if (!ice_is_e830(&sc->hw))
	+ LIST_DEL(&vf->mbx_info.list_entry);
	atomic_store_rel_32(&vf->vf_flags, 0);
	vsi = vf->vsi;
	free(vf->mac_filters, M_ICE);
	@@ -1029,10 +1075,26 @@
	ice_iov_clear_vf_queue_state(vf);
	ice_iov_clear_vf_mdd(sc, vf);
	atomic_clear_32(&vf->vf_flags, VF_FLAG_MDD_BLOCKED);
	+ ice_iov_clear_vf_mbx(sc, vf);

	ice_iov_complete_vf_reset(sc, vf, true);
	}

	+/**
	+ * ice_iov_clear_vf_mbx - Release mailbox isolation after a completed reset
	+ * @sc: device softc structure
	+ * @vf: VF whose mailbox state should be cleared
	+ */
	+static void
	+ice_iov_clear_vf_mbx(struct ice_softc sc, struct ice_vf vf)
	+{
	+ if (ice_is_e830(&sc->hw))
	+ ice_mbx_vf_clear_cnt_e830(&sc->hw, vf->vf_num);
	+ else
	+ ice_mbx_clear_malvf(&vf->mbx_info);
	+ atomic_clear_32(&vf->vf_flags, VF_FLAG_MBX_BLOCKED);
	+}
	+
	/**
	* ice_iov_rebuild_vf - Rebuild a VF VSI after a PF or device reset
	* @sc: device softc structure
	@@ -2806,17 +2868,97 @@
	VIRTCHNL_STATUS_SUCCESS, (u8 *)&event, sizeof(event), NULL);
	}

	+/**
	+ * ice_iov_mbx_overflow - Detect and isolate a VF flooding the PF mailbox
	+ * @sc: device private structure
	+ * @vf: VF which sent the current message
	+ * @mbx_data: software mailbox snapshot data, or NULL on E830
	+ *
	+ * E830 enforces the per-VF watermark in hardware. On older devices, reset
	+ * and block a VF after the Intel snapshot detector first attributes an
	+ * overflow. A later external VF reset, PF reset, or SR-IOV recreation
	+ * releases it.
	+ *
	+ * @returns true if the current message must be discarded.
	+ */
	+static bool
	+ice_iov_mbx_overflow(struct ice_softc sc, struct ice_vf vf,
	+ struct ice_mbx_data *mbx_data)
	+{
	+ struct ice_hw *hw = &sc->hw;
	+ bool report_malvf;
	+ u32 reg, vf_flags;
	+ int error, status;
	+
	+ if (mbx_data == NULL)
	+ return (false);
	+ if ((atomic_load_acq_32(&vf->vf_flags) & VF_FLAG_MBX_BLOCKED) != 0)
	+ return (true);
	+
	+ report_malvf = false;
	+ status = ice_mbx_vf_state_handler(hw, mbx_data, &vf->mbx_info,
	+ &report_malvf);
	+ ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice_iov,
	+ mailbox_overflow, ice_iov_fail_vf_matches(vf->vf_num),
	+ FAIL_POINT_NONSLEEPABLE, {
	+ status = 0;
	+ vf->mbx_info.malicious = 1;
	+ report_malvf = true;
	+ });
	+ if (status != 0) {
	+ device_printf(sc->dev,
	+ "Unable to check VF %u mailbox overflow, err %s\n",
	+ vf->vf_num, ice_status_str(status));
	+ return (false);
	+ }
	+ if (!report_malvf)
	+ return (vf->mbx_info.malicious != 0);
	+
	+ vf->mbx_overflow_events++;
	+ atomic_set_32(&vf->vf_flags, VF_FLAG_MBX_BLOCKED);
	+ device_printf(sc->dev,
	+ "VF %u exceeded the mailbox message limit; resetting and blocking it\n",
	+ vf->vf_num);
	+
	+ vf_flags = atomic_load_acq_32(&vf->vf_flags);
	+ if ((vf_flags & VF_FLAG_ENABLED) != 0 && vf->vsi != NULL) {
	+ error = ice_reset_vf(sc, vf, true, false);
	+ if (error != 0) {
	+ device_printf(sc->dev,
	+ "Unable to isolate VF %u after mailbox overflow: %s\n",
	+ vf->vf_num, ice_err_str(error));
	+ } else {
	+ /*
	+ * Leave queues and mailbox requests blocked, but complete VFR
	+ * so a later physical FLR can create a new reset edge and
	+ * recover the VF.
	+ */
	+ ice_iov_complete_vf_reset(sc, vf, false);
	+ }
	+ } else {
	+ /* An incompletely configured VF has no queues to drain. */
	+ reg = rd32(hw, VPGEN_VFRTRIG(vf->vf_num));
	+ reg \|= VPGEN_VFRTRIG_VFSWR_M;
	+ wr32(hw, VPGEN_VFRTRIG(vf->vf_num), reg);
	+ ice_flush(hw);
	+ }
	+
	+ return (true);
	+}
	+
	/**
	* ice_vc_handle_vf_msg - Handle a message from a VF
	* @sc: device private structure
	* @event: event received from the HW MBX queue
	+ * @mbx_data: software overflow-detection data, or NULL on E830
	*
	* Called whenever an event is received from a VF on the HW mailbox queue.
	* Responsible for handling these messages as well as responding to the
	* VF afterwards, depending on the received message type.
	*/
	void
	-ice_vc_handle_vf_msg(struct ice_softc sc, struct ice_rq_event_info event)
	+ice_vc_handle_vf_msg(struct ice_softc sc, struct ice_rq_event_info event,
	+ struct ice_mbx_data *mbx_data)
	{
	struct ice_hw *hw = &sc->hw;
	device_t dev = sc->dev;
	@@ -2836,6 +2978,8 @@
	}

	vf = &sc->vfs[v_id];
	+ if (ice_iov_mbx_overflow(sc, vf, mbx_data))
	+ return;

	/* Perform basic checks on the msg */
	err = virtchnl_vc_validate_vf_msg(&vf->version, v_opcode, msg, msglen);
	@@ -2849,8 +2993,8 @@
	vf_flags = atomic_load_acq_32(&vf->vf_flags);
	if ((vf_flags & VF_FLAG_ENABLED) == 0 \|\| vf->vsi == NULL)
	return;
	- /* Only a reset outside this dispatcher may release an MDD-blocked VF. */
	- if ((vf_flags & VF_FLAG_MDD_BLOCKED) != 0)
	+ /* Only a reset outside this dispatcher may release an isolated VF. */
	+ if ((vf_flags & (VF_FLAG_MDD_BLOCKED \| VF_FLAG_MBX_BLOCKED)) != 0)
	return;

	/*
	Index: sys/dev/ice/ice_lib.c
	===================================================================
	--- sys/dev/ice/ice_lib.c
	+++ sys/dev/ice/ice_lib.c
	@@ -108,7 +108,8 @@
	struct ice_ctl_q_info *cq);
	static void ice_process_link_event(struct ice_softc sc, struct ice_rq_event_info e);
	static void ice_process_ctrlq_event(struct ice_softc sc, const char qname,
	- struct ice_rq_event_info *event);
	+ struct ice_rq_event_info *event,
	+ struct ice_mbx_data *mbx_data);
	static void ice_nvm_version_str(struct ice_hw hw, struct sbuf buf);
	static void ice_update_port_oversize(struct ice_softc *sc, u64 rx_errors);
	static void ice_active_pkg_version_str(struct ice_hw hw, struct sbuf buf);
	@@ -2332,7 +2333,8 @@
	*/
	static void
	ice_process_ctrlq_event(struct ice_softc sc, const char qname,
	- struct ice_rq_event_info *event)
	+ struct ice_rq_event_info *event,
	+ struct ice_mbx_data *mbx_data)
	{
	u16 opcode;

	@@ -2344,7 +2346,7 @@
	break;
	#ifdef PCI_IOV
	case ice_mbx_opc_send_msg_to_pf:
	- ice_vc_handle_vf_msg(sc, event);
	+ ice_vc_handle_vf_msg(sc, event, mbx_data);
	break;
	#endif
	case ice_aqc_opc_fw_logs_event:
	@@ -2380,6 +2382,9 @@
	ice_process_ctrlq(struct ice_softc sc, enum ice_ctl_q q_type, u16 pending)
	{
	struct ice_rq_event_info event = { { 0 } };
	+#ifdef PCI_IOV
	+ struct ice_mbx_data mbx_data = { 0 };
	+#endif
	struct ice_hw *hw = &sc->hw;
	struct ice_ctl_q_info *cq;
	int status;
	@@ -2398,6 +2403,11 @@
	case ICE_CTL_Q_MAILBOX:
	cq = &hw->mailboxq;
	qname = "Mailbox";
	+#ifdef PCI_IOV
	+ if (!ice_is_e830(hw) && sc->num_vfs != 0)
	+ hw->mbx_snapshot.mbx_buf.state =
	+ ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
	+#endif
	break;
	default:
	device_printf(sc->dev,
	@@ -2433,10 +2443,31 @@
	return (EIO);
	}
	/* XXX should we separate this handler by controlq type? */
	- ice_process_ctrlq_event(sc, qname, &event);
	+#ifdef PCI_IOV
	+ if (q_type == ICE_CTL_Q_MAILBOX &&
	+ le16toh(event.desc.opcode) == ice_mbx_opc_send_msg_to_pf) {
	+ if (ice_is_e830(hw)) {
	+ ice_process_ctrlq_event(sc, qname, &event, NULL);
	+ ice_e830_mbx_vf_dec_trig(hw, &event);
	+ } else {
	+ mbx_data.max_num_msgs_mbx = cq->num_rq_entries;
	+ mbx_data.async_watermark_val =
	+ ICE_MBX_OVERFLOW_WATERMARK;
	+ mbx_data.num_msg_proc = loop;
	+ mbx_data.num_pending_arq = *pending;
	+ ice_process_ctrlq_event(sc, qname, &event,
	+ &mbx_data);
	+ }
	+ } else
	+#endif
	+ ice_process_ctrlq_event(sc, qname, &event, NULL);
	} while (*pending && (++loop < ICE_CTRLQ_WORK_LIMIT));

	free(event.msg_buf, M_ICE);
	+ ICE_FAIL_POINT_CODE_COND(sc, _debug_fail_point_ice, mailbox_pending,
	+ q_type == ICE_CTL_Q_MAILBOX, FAIL_POINT_NONSLEEPABLE, {
	+ *pending = 1;
	+ });

	return 0;
	}
	Index: sys/dev/ice/ice_vf_mbx.h
	===================================================================
	--- sys/dev/ice/ice_vf_mbx.h
	+++ sys/dev/ice/ice_vf_mbx.h
	@@ -44,6 +44,7 @@
	* MBX_VF_IN_FLIGHT_MSGS_AT_PF_CNT register.
	*/
	#define ICE_ASYNC_VF_MSG_THRESHOLD 63
	+#define ICE_MBX_OVERFLOW_WATERMARK (ICE_ASYNC_VF_MSG_THRESHOLD + 1)

	int
	ice_aq_send_msg_to_pf(struct ice_hw *hw, enum virtchnl_ops v_opcode,
	Index: sys/dev/ice/if_ice_iflib.c
	===================================================================
	--- sys/dev/ice/if_ice_iflib.c
	+++ sys/dev/ice/if_ice_iflib.c
	@@ -2420,7 +2420,7 @@
	{
	struct ice_softc sc = (struct ice_softc )iflib_get_softc(ctx);
	enum ice_fw_modes fw_mode;
	- bool reschedule = false;
	+ bool defer_mailbox = false, reschedule = false;
	u16 pending = 0;

	ASSERT_CTX_LOCKED(sc);
	@@ -2462,19 +2462,44 @@
	*/
	;
	} else if (ice_testandclear_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING)) {
	+ pending = 0;
	ice_process_ctrlq(sc, ICE_CTL_Q_ADMIN, &pending);
	if (pending > 0)
	reschedule = true;

	if (ice_is_generic_mac(&sc->hw)) {
	+ pending = 0;
	ice_process_ctrlq(sc, ICE_CTL_Q_SB, &pending);
	if (pending > 0)
	reschedule = true;
	}

	+ pending = 0;
	ice_process_ctrlq(sc, ICE_CTL_Q_MAILBOX, &pending);
	- if (pending > 0)
	+ if (pending > 0) {
	+#ifdef PCI_IOV
	+ /*
	+ * Two passes drain one initially full 512-entry mailbox.
	+ * If it remains nonempty, a VF is replenishing it faster
	+ * than this task can drain it. Leave the interrupt masked
	+ * and let the periodic admin timer schedule bounded work.
	+ */
	+ if (sc->mbx_admin_passes <
	+ howmany(ICE_MBXQ_LEN, ICE_CTRLQ_WORK_LIMIT))
	+ sc->mbx_admin_passes++;
	+ if (sc->mbx_admin_passes <
	+ howmany(ICE_MBXQ_LEN, ICE_CTRLQ_WORK_LIMIT))
	+ reschedule = true;
	+ else
	+ defer_mailbox = true;
	+#else
	reschedule = true;
	+#endif
	+ } else {
	+#ifdef PCI_IOV
	+ sc->mbx_admin_passes = 0;
	+#endif
	+ }
	}

	/* Poll for link up */
	@@ -2500,6 +2525,8 @@
	if (reschedule) {
	ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
	iflib_admin_intr_deferred(ctx);
	+ } else if (defer_mailbox) {
	+ ice_set_state(&sc->state, ICE_STATE_CONTROLQ_EVENT_PENDING);
	} else {
	ice_enable_intr(&sc->hw, sc->irqvs[0].me);
	}
	@@ -2712,6 +2739,10 @@
	goto err_shutdown_ctrlq;
	}

	+#ifdef PCI_IOV
	+ ice_iov_reconfigure_mbx(sc);
	+#endif
	+
	/* Query the allocated resources for Tx scheduler */
	status = ice_sched_query_res_alloc(hw);
	if (status) {

File Metadata

Mime Type: text/plain
Expires: Mon, Aug 31, 10:53 PM (11 h, 21 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 37728076
Default Alt Text: D59029.diff (17 KB)

D59029.diffNo OneTemporaryActions

D59029.diffView Options

File Metadata

Event Timeline

D59029.diff
No OneTemporary
Actions

D59029.diff
View Options