Index: head/sys/dev/sfxge/common/ef10_rx.c =================================================================== --- head/sys/dev/sfxge/common/ef10_rx.c (revision 341050) +++ head/sys/dev/sfxge/common/ef10_rx.c (revision 341051) @@ -1,1117 +1,1124 @@ /*- * Copyright (c) 2012-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 static __checkReturn efx_rc_t efx_mcdi_init_rxq( __in efx_nic_t *enp, __in uint32_t ndescs, __in uint32_t target_evq, __in uint32_t label, __in uint32_t instance, __in efsys_mem_t *esmp, __in boolean_t disable_scatter, __in boolean_t want_inner_classes, __in uint32_t ps_bufsize) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN, MC_CMD_INIT_RXQ_EXT_OUT_LEN)]; int npages = EFX_RXQ_NBUFS(ndescs); int i; efx_qword_t *dma_addr; uint64_t addr; efx_rc_t rc; uint32_t dma_mode; boolean_t want_outer_classes; EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS); + if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_RXQ_SIZE(ndescs))) { + rc = EINVAL; + goto fail1; + } + if (ps_bufsize > 0) dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; else dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; if (encp->enc_tunnel_encapsulations_supported != 0 && !want_inner_classes) { /* * WANT_OUTER_CLASSES can only be specified on hardware which * supports tunnel encapsulation offloads, even though it is * effectively the behaviour the hardware gives. * * Also, on hardware which does support such offloads, older * firmware rejects the flag if the offloads are not supported * by the current firmware variant, which means this may fail if * the capabilities are not updated when the firmware variant * changes. This is not an issue on newer firmware, as it was * changed in bug 69842 (v6.4.2.1007) to permit this flag to be * specified on all firmware variants. */ want_outer_classes = B_TRUE; } else { want_outer_classes = B_FALSE; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_INIT_RXQ; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN; MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs); MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq); MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS, INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, INIT_RXQ_EXT_IN_CRC_MODE, 0, INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, INIT_RXQ_EXT_IN_DMA_MODE, dma_mode, INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize, INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes); MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); addr = EFSYS_MEM_ADDR(esmp); for (i = 0; i < npages; i++) { EFX_POPULATE_QWORD_2(*dma_addr, EFX_DWORD_1, (uint32_t)(addr >> 32), EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); dma_addr++; addr += EFX_BUF_SIZE; } efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; - goto fail1; + goto fail2; } return (0); +fail2: + EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn efx_rc_t efx_mcdi_fini_rxq( __in efx_nic_t *enp, __in uint32_t instance) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, MC_CMD_FINI_RXQ_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_FINI_RXQ; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } return (0); fail1: /* * EALREADY is not an error, but indicates that the MC has rebooted and * that the RXQ has already been destroyed. */ if (rc != EALREADY) EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_RX_SCALE static __checkReturn efx_rc_t efx_mcdi_rss_context_alloc( __in efx_nic_t *enp, __in efx_rx_scale_context_type_t type, __in uint32_t num_queues, __out uint32_t *rss_contextp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; uint32_t rss_context; uint32_t context_type; efx_rc_t rc; if (num_queues > EFX_MAXRSS) { rc = EINVAL; goto fail1; } switch (type) { case EFX_RX_SCALE_EXCLUSIVE: context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; break; case EFX_RX_SCALE_SHARED: context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; break; default: rc = EINVAL; goto fail2; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, EVB_PORT_ID_ASSIGNED); MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); /* * For exclusive contexts, NUM_QUEUES is only used to validate * indirection table offsets. * For shared contexts, the provided context will spread traffic over * NUM_QUEUES many queues. */ MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail3; } if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { rc = EMSGSIZE; goto fail4; } rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); if (rss_context == EF10_RSS_CONTEXT_INVALID) { rc = ENOENT; goto fail5; } *rss_contextp = rss_context; return (0); fail5: EFSYS_PROBE(fail5); fail4: EFSYS_PROBE(fail4); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE static efx_rc_t efx_mcdi_rss_context_free( __in efx_nic_t *enp, __in uint32_t rss_context) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; efx_rc_t rc; if (rss_context == EF10_RSS_CONTEXT_INVALID) { rc = EINVAL; goto fail1; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail2; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE static efx_rc_t efx_mcdi_rss_context_set_flags( __in efx_nic_t *enp, __in uint32_t rss_context, __in efx_rx_hash_type_t type) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; efx_rc_t rc; if (rss_context == EF10_RSS_CONTEXT_INVALID) { rc = EINVAL; goto fail1; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, rss_context); MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, (type & EFX_RX_HASH_IPV4) ? 1 : 0, RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, (type & EFX_RX_HASH_IPV6) ? 1 : 0, RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail2; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE static efx_rc_t efx_mcdi_rss_context_set_key( __in efx_nic_t *enp, __in uint32_t rss_context, __in_ecount(n) uint8_t *key, __in size_t n) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; efx_rc_t rc; if (rss_context == EF10_RSS_CONTEXT_INVALID) { rc = EINVAL; goto fail1; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, rss_context); EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { rc = EINVAL; goto fail2; } memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), key, n); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail3; } return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE static efx_rc_t efx_mcdi_rss_context_set_table( __in efx_nic_t *enp, __in uint32_t rss_context, __in_ecount(n) unsigned int *table, __in size_t n) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; uint8_t *req_table; int i, rc; if (rss_context == EF10_RSS_CONTEXT_INVALID) { rc = EINVAL; goto fail1; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, rss_context); req_table = MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); for (i = 0; i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; i++) { req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; } efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail2; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ __checkReturn efx_rc_t ef10_rx_init( __in efx_nic_t *enp) { #if EFSYS_OPT_RX_SCALE if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, &enp->en_rss_context) == 0) { /* * Allocated an exclusive RSS context, which allows both the * indirection table and key to be modified. */ enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; enp->en_hash_support = EFX_RX_HASH_AVAILABLE; } else { /* * Failed to allocate an exclusive RSS context. Continue * operation without support for RSS. The pseudo-header in * received packets will not contain a Toeplitz hash value. */ enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; } #endif /* EFSYS_OPT_RX_SCALE */ return (0); } #if EFSYS_OPT_RX_SCATTER __checkReturn efx_rc_t ef10_rx_scatter_enable( __in efx_nic_t *enp, __in unsigned int buf_size) { _NOTE(ARGUNUSED(enp, buf_size)) return (0); } #endif /* EFSYS_OPT_RX_SCATTER */ #if EFSYS_OPT_RX_SCALE __checkReturn efx_rc_t ef10_rx_scale_context_alloc( __in efx_nic_t *enp, __in efx_rx_scale_context_type_t type, __in uint32_t num_queues, __out uint32_t *rss_contextp) { efx_rc_t rc; rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); if (rc != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE __checkReturn efx_rc_t ef10_rx_scale_context_free( __in efx_nic_t *enp, __in uint32_t rss_context) { efx_rc_t rc; rc = efx_mcdi_rss_context_free(enp, rss_context); if (rc != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE __checkReturn efx_rc_t ef10_rx_scale_mode_set( __in efx_nic_t *enp, __in uint32_t rss_context, __in efx_rx_hash_alg_t alg, __in efx_rx_hash_type_t type, __in boolean_t insert) { efx_rc_t rc; EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); EFSYS_ASSERT3U(insert, ==, B_TRUE); if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { rc = EINVAL; goto fail1; } if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { rc = ENOTSUP; goto fail2; } rss_context = enp->en_rss_context; } if ((rc = efx_mcdi_rss_context_set_flags(enp, rss_context, type)) != 0) goto fail3; return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE __checkReturn efx_rc_t ef10_rx_scale_key_set( __in efx_nic_t *enp, __in uint32_t rss_context, __in_ecount(n) uint8_t *key, __in size_t n) { efx_rc_t rc; EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { rc = ENOTSUP; goto fail1; } rss_context = enp->en_rss_context; } if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_SCALE __checkReturn efx_rc_t ef10_rx_scale_tbl_set( __in efx_nic_t *enp, __in uint32_t rss_context, __in_ecount(n) unsigned int *table, __in size_t n) { efx_rc_t rc; if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { rc = ENOTSUP; goto fail1; } rss_context = enp->en_rss_context; } if ((rc = efx_mcdi_rss_context_set_table(enp, rss_context, table, n)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_RX_SCALE */ /* * EF10 RX pseudo-header * --------------------- * * Receive packets are prefixed by an (optional) 14 byte pseudo-header: * * +00: Toeplitz hash value. * (32bit little-endian) * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. * (16bit big-endian) * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. * (16bit big-endian) * +08: Packet Length. Zero if the RX datapath was in cut-through mode. * (16bit little-endian) * +10: MAC timestamp. Zero if timestamping is not enabled. * (32bit little-endian) * * See "The RX Pseudo-header" in SF-109306-TC. */ __checkReturn efx_rc_t ef10_rx_prefix_pktlen( __in efx_nic_t *enp, __in uint8_t *buffer, __out uint16_t *lengthp) { _NOTE(ARGUNUSED(enp)) /* * The RX pseudo-header contains the packet length, excluding the * pseudo-header. If the hardware receive datapath was operating in * cut-through mode then the length in the RX pseudo-header will be * zero, and the packet length must be obtained from the DMA length * reported in the RX event. */ *lengthp = buffer[8] | (buffer[9] << 8); return (0); } #if EFSYS_OPT_RX_SCALE __checkReturn uint32_t ef10_rx_prefix_hash( __in efx_nic_t *enp, __in efx_rx_hash_alg_t func, __in uint8_t *buffer) { _NOTE(ARGUNUSED(enp)) switch (func) { case EFX_RX_HASHALG_TOEPLITZ: return (buffer[0] | (buffer[1] << 8) | (buffer[2] << 16) | (buffer[3] << 24)); default: EFSYS_ASSERT(0); return (0); } } #endif /* EFSYS_OPT_RX_SCALE */ #if EFSYS_OPT_RX_PACKED_STREAM /* * Fake length for RXQ descriptors in packed stream mode * to make hardware happy */ #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 #endif void ef10_rx_qpost( __in efx_rxq_t *erp, __in_ecount(ndescs) efsys_dma_addr_t *addrp, __in size_t size, __in unsigned int ndescs, __in unsigned int completed, __in unsigned int added) { efx_qword_t qword; unsigned int i; unsigned int offset; unsigned int id; _NOTE(ARGUNUSED(completed)) #if EFSYS_OPT_RX_PACKED_STREAM /* * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT * and equal to 0 after applying mask. Hardware does not like it. */ if (erp->er_ev_qstate->eers_rx_packed_stream) size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; #endif /* The client driver must not overfill the queue */ EFSYS_ASSERT3U(added - completed + ndescs, <=, EFX_RXQ_LIMIT(erp->er_mask + 1)); id = added & (erp->er_mask); for (i = 0; i < ndescs; i++) { EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, unsigned int, id, efsys_dma_addr_t, addrp[i], size_t, size); EFX_POPULATE_QWORD_3(qword, ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), ESF_DZ_RX_KER_BUF_ADDR_DW0, (uint32_t)(addrp[i] & 0xffffffff), ESF_DZ_RX_KER_BUF_ADDR_DW1, (uint32_t)(addrp[i] >> 32)); offset = id * sizeof (efx_qword_t); EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); id = (id + 1) & (erp->er_mask); } } void ef10_rx_qpush( __in efx_rxq_t *erp, __in unsigned int added, __inout unsigned int *pushedp) { efx_nic_t *enp = erp->er_enp; unsigned int pushed = *pushedp; uint32_t wptr; efx_dword_t dword; /* Hardware has alignment restriction for WPTR */ wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); if (pushed == wptr) return; *pushedp = wptr; /* Push the populated descriptors out */ wptr &= erp->er_mask; EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, wptr, pushed & erp->er_mask); EFSYS_PIO_WRITE_BARRIER(); EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, erp->er_index, &dword, B_FALSE); } #if EFSYS_OPT_RX_PACKED_STREAM void ef10_rx_qpush_ps_credits( __in efx_rxq_t *erp) { efx_nic_t *enp = erp->er_enp; efx_dword_t dword; efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; uint32_t credits; EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); if (rxq_state->eers_rx_packed_stream_credits == 0) return; /* * It is a bug if we think that FW has utilized more * credits than it is allowed to have (maximum). However, * make sure that we do not credit more than maximum anyway. */ credits = MIN(rxq_state->eers_rx_packed_stream_credits, EFX_RX_PACKED_STREAM_MAX_CREDITS); EFX_POPULATE_DWORD_3(dword, ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, ERF_DZ_RX_DESC_MAGIC_CMD, ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, ERF_DZ_RX_DESC_MAGIC_DATA, credits); EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, erp->er_index, &dword, B_FALSE); rxq_state->eers_rx_packed_stream_credits = 0; } /* * In accordance with SF-112241-TC the received data has the following layout: * - 8 byte pseudo-header which consist of: * - 4 byte little-endian timestamp * - 2 byte little-endian captured length in bytes * - 2 byte little-endian original packet length in bytes * - captured packet bytes * - optional padding to align to 64 bytes boundary * - 64 bytes scratch space for the host software */ __checkReturn uint8_t * ef10_rx_qps_packet_info( __in efx_rxq_t *erp, __in uint8_t *buffer, __in uint32_t buffer_length, __in uint32_t current_offset, __out uint16_t *lengthp, __out uint32_t *next_offsetp, __out uint32_t *timestamp) { uint16_t buf_len; uint8_t *pkt_start; efx_qword_t *qwordp; efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); buffer += current_offset; pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; qwordp = (efx_qword_t *)buffer; *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, EFX_RX_PACKED_STREAM_ALIGNMENT); *next_offsetp = current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); if ((*next_offsetp ^ current_offset) & EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) rxq_state->eers_rx_packed_stream_credits++; return (pkt_start); } #endif __checkReturn efx_rc_t ef10_rx_qflush( __in efx_rxq_t *erp) { efx_nic_t *enp = erp->er_enp; efx_rc_t rc; if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) goto fail1; return (0); fail1: /* * EALREADY is not an error, but indicates that the MC has rebooted and * that the RXQ has already been destroyed. Callers need to know that * the RXQ flush has completed to avoid waiting until timeout for a * flush done event that will not be delivered. */ if (rc != EALREADY) EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_rx_qenable( __in efx_rxq_t *erp) { /* FIXME */ _NOTE(ARGUNUSED(erp)) /* FIXME */ } __checkReturn efx_rc_t ef10_rx_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in unsigned int label, __in efx_rxq_type_t type, __in uint32_t type_data, __in efsys_mem_t *esmp, __in size_t ndescs, __in uint32_t id, __in unsigned int flags, __in efx_evq_t *eep, __in efx_rxq_t *erp) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_rc_t rc; boolean_t disable_scatter; boolean_t want_inner_classes; unsigned int ps_buf_size; _NOTE(ARGUNUSED(id, erp, type_data)) EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); if (!ISP2(ndescs) || (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) { rc = EINVAL; goto fail1; } if (index >= encp->enc_rxq_limit) { rc = EINVAL; goto fail2; } switch (type) { case EFX_RXQ_TYPE_DEFAULT: ps_buf_size = 0; break; #if EFSYS_OPT_RX_PACKED_STREAM case EFX_RXQ_TYPE_PACKED_STREAM: switch (type_data) { case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; break; case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; break; case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; break; case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; break; case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; break; default: rc = ENOTSUP; goto fail3; } break; #endif /* EFSYS_OPT_RX_PACKED_STREAM */ default: rc = ENOTSUP; goto fail4; } #if EFSYS_OPT_RX_PACKED_STREAM if (ps_buf_size != 0) { /* Check if datapath firmware supports packed stream mode */ if (encp->enc_rx_packed_stream_supported == B_FALSE) { rc = ENOTSUP; goto fail5; } /* Check if packed stream allows configurable buffer sizes */ if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { rc = ENOTSUP; goto fail6; } } #else /* EFSYS_OPT_RX_PACKED_STREAM */ EFSYS_ASSERT(ps_buf_size == 0); #endif /* EFSYS_OPT_RX_PACKED_STREAM */ /* Scatter can only be disabled if the firmware supports doing so */ if (flags & EFX_RXQ_FLAG_SCATTER) disable_scatter = B_FALSE; else disable_scatter = encp->enc_rx_disable_scatter_supported; if (flags & EFX_RXQ_FLAG_INNER_CLASSES) want_inner_classes = B_TRUE; else want_inner_classes = B_FALSE; if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index, esmp, disable_scatter, want_inner_classes, ps_buf_size)) != 0) goto fail7; erp->er_eep = eep; erp->er_label = label; ef10_ev_rxlabel_init(eep, erp, label, type); erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; return (0); fail7: EFSYS_PROBE(fail7); #if EFSYS_OPT_RX_PACKED_STREAM fail6: EFSYS_PROBE(fail6); fail5: EFSYS_PROBE(fail5); #endif /* EFSYS_OPT_RX_PACKED_STREAM */ fail4: EFSYS_PROBE(fail4); #if EFSYS_OPT_RX_PACKED_STREAM fail3: EFSYS_PROBE(fail3); #endif /* EFSYS_OPT_RX_PACKED_STREAM */ fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_rx_qdestroy( __in efx_rxq_t *erp) { efx_nic_t *enp = erp->er_enp; efx_evq_t *eep = erp->er_eep; unsigned int label = erp->er_label; ef10_ev_rxlabel_fini(eep, label); EFSYS_ASSERT(enp->en_rx_qcount != 0); --enp->en_rx_qcount; EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); } void ef10_rx_fini( __in efx_nic_t *enp) { #if EFSYS_OPT_RX_SCALE if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); enp->en_rss_context = 0; enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; #else _NOTE(ARGUNUSED(enp)) #endif /* EFSYS_OPT_RX_SCALE */ } #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ Index: head/sys/dev/sfxge/common/ef10_tx.c =================================================================== --- head/sys/dev/sfxge/common/ef10_tx.c (revision 341050) +++ head/sys/dev/sfxge/common/ef10_tx.c (revision 341051) @@ -1,796 +1,803 @@ /*- * Copyright (c) 2012-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 #if EFSYS_OPT_QSTATS #define EFX_TX_QSTAT_INCR(_etp, _stat) \ do { \ (_etp)->et_stat[_stat]++; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFX_TX_QSTAT_INCR(_etp, _stat) #endif static __checkReturn efx_rc_t efx_mcdi_init_txq( __in efx_nic_t *enp, __in uint32_t ndescs, __in uint32_t target_evq, __in uint32_t label, __in uint32_t instance, __in uint16_t flags, __in efsys_mem_t *esmp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_INIT_TXQ_IN_LEN(EFX_TXQ_MAX_BUFS), MC_CMD_INIT_TXQ_OUT_LEN)]; efx_qword_t *dma_addr; uint64_t addr; int npages; int i; efx_rc_t rc; EFSYS_ASSERT(EFX_TXQ_MAX_BUFS >= EFX_TXQ_NBUFS(enp->en_nic_cfg.enc_txq_max_ndescs)); + if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_TXQ_SIZE(ndescs))) { + rc = EINVAL; + goto fail1; + } + npages = EFX_TXQ_NBUFS(ndescs); if (MC_CMD_INIT_TXQ_IN_LEN(npages) > sizeof (payload)) { rc = EINVAL; - goto fail1; + goto fail2; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_INIT_TXQ; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_INIT_TXQ_IN_LEN(npages); req.emr_out_buf = payload; req.emr_out_length = MC_CMD_INIT_TXQ_OUT_LEN; MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_SIZE, ndescs); MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_TARGET_EVQ, target_evq); MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_LABEL, label); MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_INSTANCE, instance); MCDI_IN_POPULATE_DWORD_9(req, INIT_TXQ_IN_FLAGS, INIT_TXQ_IN_FLAG_BUFF_MODE, 0, INIT_TXQ_IN_FLAG_IP_CSUM_DIS, (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1, INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1, INIT_TXQ_EXT_IN_FLAG_INNER_IP_CSUM_EN, (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0, INIT_TXQ_EXT_IN_FLAG_INNER_TCP_CSUM_EN, (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0, INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, (flags & EFX_TXQ_FATSOV2) ? 1 : 0, INIT_TXQ_IN_FLAG_TCP_UDP_ONLY, 0, INIT_TXQ_IN_CRC_MODE, 0, INIT_TXQ_IN_FLAG_TIMESTAMP, 0); MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_OWNER_ID, 0); MCDI_IN_SET_DWORD(req, INIT_TXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); dma_addr = MCDI_IN2(req, efx_qword_t, INIT_TXQ_IN_DMA_ADDR); addr = EFSYS_MEM_ADDR(esmp); for (i = 0; i < npages; i++) { EFX_POPULATE_QWORD_2(*dma_addr, EFX_DWORD_1, (uint32_t)(addr >> 32), EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); dma_addr++; addr += EFX_BUF_SIZE; } efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; - goto fail2; + goto fail3; } return (0); +fail3: + EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn efx_rc_t efx_mcdi_fini_txq( __in efx_nic_t *enp, __in uint32_t instance) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_FINI_TXQ_IN_LEN, MC_CMD_FINI_TXQ_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_FINI_TXQ; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_FINI_TXQ_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_FINI_TXQ_OUT_LEN; MCDI_IN_SET_DWORD(req, FINI_TXQ_IN_INSTANCE, instance); efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } return (0); fail1: /* * EALREADY is not an error, but indicates that the MC has rebooted and * that the TXQ has already been destroyed. */ if (rc != EALREADY) EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t ef10_tx_init( __in efx_nic_t *enp) { _NOTE(ARGUNUSED(enp)) return (0); } void ef10_tx_fini( __in efx_nic_t *enp) { _NOTE(ARGUNUSED(enp)) } __checkReturn efx_rc_t ef10_tx_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in unsigned int label, __in efsys_mem_t *esmp, __in size_t ndescs, __in uint32_t id, __in uint16_t flags, __in efx_evq_t *eep, __in efx_txq_t *etp, __out unsigned int *addedp) { efx_nic_cfg_t *encp = &enp->en_nic_cfg; uint16_t inner_csum; efx_desc_t desc; efx_rc_t rc; _NOTE(ARGUNUSED(id)) inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; if (((flags & inner_csum) != 0) && (encp->enc_tunnel_encapsulations_supported == 0)) { rc = EINVAL; goto fail1; } if ((rc = efx_mcdi_init_txq(enp, ndescs, eep->ee_index, label, index, flags, esmp)) != 0) goto fail2; /* * A previous user of this TX queue may have written a descriptor to the * TX push collector, but not pushed the doorbell (e.g. after a crash). * The next doorbell write would then push the stale descriptor. * * Ensure the (per network port) TX push collector is cleared by writing * a no-op TX option descriptor. See bug29981 for details. */ *addedp = 1; ef10_tx_qdesc_checksum_create(etp, flags, &desc); EFSYS_MEM_WRITEQ(etp->et_esmp, 0, &desc.ed_eq); ef10_tx_qpush(etp, *addedp, 0); return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_tx_qdestroy( __in efx_txq_t *etp) { /* FIXME */ _NOTE(ARGUNUSED(etp)) /* FIXME */ } __checkReturn efx_rc_t ef10_tx_qpio_enable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; efx_piobuf_handle_t handle; efx_rc_t rc; if (etp->et_pio_size != 0) { rc = EALREADY; goto fail1; } /* Sub-allocate a PIO block from a piobuf */ if ((rc = ef10_nic_pio_alloc(enp, &etp->et_pio_bufnum, &handle, &etp->et_pio_blknum, &etp->et_pio_offset, &etp->et_pio_size)) != 0) { goto fail2; } EFSYS_ASSERT3U(etp->et_pio_size, !=, 0); /* Link the piobuf to this TXQ */ if ((rc = ef10_nic_pio_link(enp, etp->et_index, handle)) != 0) { goto fail3; } /* * et_pio_offset is the offset of the sub-allocated block within the * hardware PIO buffer. It is used as the buffer address in the PIO * option descriptor. * * et_pio_write_offset is the offset of the sub-allocated block from the * start of the write-combined memory mapping, and is used for writing * data into the PIO buffer. */ etp->et_pio_write_offset = (etp->et_pio_bufnum * ER_DZ_TX_PIOBUF_STEP) + ER_DZ_TX_PIOBUF_OFST + etp->et_pio_offset; return (0); fail3: EFSYS_PROBE(fail3); ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum); fail2: EFSYS_PROBE(fail2); etp->et_pio_size = 0; fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_tx_qpio_disable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; if (etp->et_pio_size != 0) { /* Unlink the piobuf from this TXQ */ ef10_nic_pio_unlink(enp, etp->et_index); /* Free the sub-allocated PIO block */ ef10_nic_pio_free(enp, etp->et_pio_bufnum, etp->et_pio_blknum); etp->et_pio_size = 0; etp->et_pio_write_offset = 0; } } __checkReturn efx_rc_t ef10_tx_qpio_write( __in efx_txq_t *etp, __in_ecount(length) uint8_t *buffer, __in size_t length, __in size_t offset) { efx_nic_t *enp = etp->et_enp; efsys_bar_t *esbp = enp->en_esbp; uint32_t write_offset; uint32_t write_offset_limit; efx_qword_t *eqp; efx_rc_t rc; EFSYS_ASSERT(length % sizeof (efx_qword_t) == 0); if (etp->et_pio_size == 0) { rc = ENOENT; goto fail1; } if (offset + length > etp->et_pio_size) { rc = ENOSPC; goto fail2; } /* * Writes to PIO buffers must be 64 bit aligned, and multiples of * 64 bits. */ write_offset = etp->et_pio_write_offset + offset; write_offset_limit = write_offset + length; eqp = (efx_qword_t *)buffer; while (write_offset < write_offset_limit) { EFSYS_BAR_WC_WRITEQ(esbp, write_offset, eqp); eqp++; write_offset += sizeof (efx_qword_t); } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t ef10_tx_qpio_post( __in efx_txq_t *etp, __in size_t pkt_length, __in unsigned int completed, __inout unsigned int *addedp) { efx_qword_t pio_desc; unsigned int id; size_t offset; unsigned int added = *addedp; efx_rc_t rc; if (added - completed + 1 > EFX_TXQ_LIMIT(etp->et_mask + 1)) { rc = ENOSPC; goto fail1; } if (etp->et_pio_size == 0) { rc = ENOENT; goto fail2; } id = added++ & etp->et_mask; offset = id * sizeof (efx_qword_t); EFSYS_PROBE4(tx_pio_post, unsigned int, etp->et_index, unsigned int, id, uint32_t, etp->et_pio_offset, size_t, pkt_length); EFX_POPULATE_QWORD_5(pio_desc, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, 1, ESF_DZ_TX_PIO_CONT, 0, ESF_DZ_TX_PIO_BYTE_CNT, pkt_length, ESF_DZ_TX_PIO_BUF_ADDR, etp->et_pio_offset); EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &pio_desc); EFX_TX_QSTAT_INCR(etp, TX_POST_PIO); *addedp = added; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t ef10_tx_qpost( __in efx_txq_t *etp, __in_ecount(ndescs) efx_buffer_t *eb, __in unsigned int ndescs, __in unsigned int completed, __inout unsigned int *addedp) { unsigned int added = *addedp; unsigned int i; efx_rc_t rc; if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { rc = ENOSPC; goto fail1; } for (i = 0; i < ndescs; i++) { efx_buffer_t *ebp = &eb[i]; efsys_dma_addr_t addr = ebp->eb_addr; size_t size = ebp->eb_size; boolean_t eop = ebp->eb_eop; unsigned int id; size_t offset; efx_qword_t qword; /* No limitations on boundary crossing */ EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); id = added++ & etp->et_mask; offset = id * sizeof (efx_qword_t); EFSYS_PROBE5(tx_post, unsigned int, etp->et_index, unsigned int, id, efsys_dma_addr_t, addr, size_t, size, boolean_t, eop); EFX_POPULATE_QWORD_5(qword, ESF_DZ_TX_KER_TYPE, 0, ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &qword); } EFX_TX_QSTAT_INCR(etp, TX_POST); *addedp = added; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } /* * This improves performance by, when possible, pushing a TX descriptor at the * same time as the doorbell. The descriptor must be added to the TXQ, so that * can be used if the hardware decides not to use the pushed descriptor. */ void ef10_tx_qpush( __in efx_txq_t *etp, __in unsigned int added, __in unsigned int pushed) { efx_nic_t *enp = etp->et_enp; unsigned int wptr; unsigned int id; size_t offset; efx_qword_t desc; efx_oword_t oword; wptr = added & etp->et_mask; id = pushed & etp->et_mask; offset = id * sizeof (efx_qword_t); EFSYS_MEM_READQ(etp->et_esmp, offset, &desc); /* * SF Bug 65776: TSO option descriptors cannot be pushed if pacer bypass * is enabled on the event queue this transmit queue is attached to. * * To ensure the code is safe, it is easiest to simply test the type of * the descriptor to push, and only push it is if it not a TSO option * descriptor. */ if ((EFX_QWORD_FIELD(desc, ESF_DZ_TX_DESC_IS_OPT) != 1) || (EFX_QWORD_FIELD(desc, ESF_DZ_TX_OPTION_TYPE) != ESE_DZ_TX_OPTION_DESC_TSO)) { /* Push the descriptor and update the wptr. */ EFX_POPULATE_OWORD_3(oword, ERF_DZ_TX_DESC_WPTR, wptr, ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); /* Ensure ordering of memory (descriptors) and PIO (doorbell) */ EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, wptr, id); EFSYS_PIO_WRITE_BARRIER(); EFX_BAR_VI_DOORBELL_WRITEO(enp, ER_DZ_TX_DESC_UPD_REG, etp->et_index, &oword); } else { efx_dword_t dword; /* * Only update the wptr. This is signalled to the hardware by * only writing one DWORD of the doorbell register. */ EFX_POPULATE_OWORD_1(oword, ERF_DZ_TX_DESC_WPTR, wptr); dword = oword.eo_dword[2]; /* Ensure ordering of memory (descriptors) and PIO (doorbell) */ EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, wptr, id); EFSYS_PIO_WRITE_BARRIER(); EFX_BAR_VI_WRITED2(enp, ER_DZ_TX_DESC_UPD_REG, etp->et_index, &dword, B_FALSE); } } __checkReturn efx_rc_t ef10_tx_qdesc_post( __in efx_txq_t *etp, __in_ecount(ndescs) efx_desc_t *ed, __in unsigned int ndescs, __in unsigned int completed, __inout unsigned int *addedp) { unsigned int added = *addedp; unsigned int i; efx_rc_t rc; if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { rc = ENOSPC; goto fail1; } for (i = 0; i < ndescs; i++) { efx_desc_t *edp = &ed[i]; unsigned int id; size_t offset; id = added++ & etp->et_mask; offset = id * sizeof (efx_desc_t); EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); } EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, unsigned int, added, unsigned int, ndescs); EFX_TX_QSTAT_INCR(etp, TX_POST); *addedp = added; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_tx_qdesc_dma_create( __in efx_txq_t *etp, __in efsys_dma_addr_t addr, __in size_t size, __in boolean_t eop, __out efx_desc_t *edp) { _NOTE(ARGUNUSED(etp)) /* No limitations on boundary crossing */ EFSYS_ASSERT(size <= etp->et_enp->en_nic_cfg.enc_tx_dma_desc_size_max); EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, efsys_dma_addr_t, addr, size_t, size, boolean_t, eop); EFX_POPULATE_QWORD_5(edp->ed_eq, ESF_DZ_TX_KER_TYPE, 0, ESF_DZ_TX_KER_CONT, (eop) ? 0 : 1, ESF_DZ_TX_KER_BYTE_CNT, (uint32_t)(size), ESF_DZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), ESF_DZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); } void ef10_tx_qdesc_tso_create( __in efx_txq_t *etp, __in uint16_t ipv4_id, __in uint32_t tcp_seq, __in uint8_t tcp_flags, __out efx_desc_t *edp) { _NOTE(ARGUNUSED(etp)) EFSYS_PROBE4(tx_desc_tso_create, unsigned int, etp->et_index, uint16_t, ipv4_id, uint32_t, tcp_seq, uint8_t, tcp_flags); EFX_POPULATE_QWORD_5(edp->ed_eq, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags, ESF_DZ_TX_TSO_IP_ID, ipv4_id, ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); } void ef10_tx_qdesc_tso2_create( __in efx_txq_t *etp, __in uint16_t ipv4_id, __in uint32_t tcp_seq, __in uint16_t tcp_mss, __out_ecount(count) efx_desc_t *edp, __in int count) { _NOTE(ARGUNUSED(etp, count)) EFSYS_PROBE4(tx_desc_tso2_create, unsigned int, etp->et_index, uint16_t, ipv4_id, uint32_t, tcp_seq, uint16_t, tcp_mss); EFSYS_ASSERT(count >= EFX_TX_FATSOV2_OPT_NDESCS); EFX_POPULATE_QWORD_5(edp[0].ed_eq, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, ESF_DZ_TX_TSO_OPTION_TYPE, ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, ESF_DZ_TX_TSO_IP_ID, ipv4_id, ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq); EFX_POPULATE_QWORD_4(edp[1].ed_eq, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, ESF_DZ_TX_TSO_OPTION_TYPE, ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, ESF_DZ_TX_TSO_TCP_MSS, tcp_mss); } void ef10_tx_qdesc_vlantci_create( __in efx_txq_t *etp, __in uint16_t tci, __out efx_desc_t *edp) { _NOTE(ARGUNUSED(etp)) EFSYS_PROBE2(tx_desc_vlantci_create, unsigned int, etp->et_index, uint16_t, tci); EFX_POPULATE_QWORD_4(edp->ed_eq, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_VLAN, ESF_DZ_TX_VLAN_OP, tci ? 1 : 0, ESF_DZ_TX_VLAN_TAG1, tci); } void ef10_tx_qdesc_checksum_create( __in efx_txq_t *etp, __in uint16_t flags, __out efx_desc_t *edp) { _NOTE(ARGUNUSED(etp)); EFSYS_PROBE2(tx_desc_checksum_create, unsigned int, etp->et_index, uint32_t, flags); EFX_POPULATE_QWORD_6(edp->ed_eq, ESF_DZ_TX_DESC_IS_OPT, 1, ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_CRC_CSUM, ESF_DZ_TX_OPTION_UDP_TCP_CSUM, (flags & EFX_TXQ_CKSUM_TCPUDP) ? 1 : 0, ESF_DZ_TX_OPTION_IP_CSUM, (flags & EFX_TXQ_CKSUM_IPV4) ? 1 : 0, ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM, (flags & EFX_TXQ_CKSUM_INNER_TCPUDP) ? 1 : 0, ESF_DZ_TX_OPTION_INNER_IP_CSUM, (flags & EFX_TXQ_CKSUM_INNER_IPV4) ? 1 : 0); } __checkReturn efx_rc_t ef10_tx_qpace( __in efx_txq_t *etp, __in unsigned int ns) { efx_rc_t rc; /* FIXME */ _NOTE(ARGUNUSED(etp, ns)) _NOTE(CONSTANTCONDITION) if (B_FALSE) { rc = ENOTSUP; goto fail1; } /* FIXME */ return (0); fail1: /* * EALREADY is not an error, but indicates that the MC has rebooted and * that the TXQ has already been destroyed. Callers need to know that * the TXQ flush has completed to avoid waiting until timeout for a * flush done event that will not be delivered. */ if (rc != EALREADY) EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t ef10_tx_qflush( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; efx_rc_t rc; if ((rc = efx_mcdi_fini_txq(enp, etp->et_index)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void ef10_tx_qenable( __in efx_txq_t *etp) { /* FIXME */ _NOTE(ARGUNUSED(etp)) /* FIXME */ } #if EFSYS_OPT_QSTATS void ef10_tx_qstats_update( __in efx_txq_t *etp, __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) { unsigned int id; for (id = 0; id < TX_NQSTATS; id++) { efsys_stat_t *essp = &stat[id]; EFSYS_STAT_INCR(essp, etp->et_stat[id]); etp->et_stat[id] = 0; } } #endif /* EFSYS_OPT_QSTATS */ #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ Index: head/sys/dev/sfxge/common/efsys.h =================================================================== --- head/sys/dev/sfxge/common/efsys.h (revision 341050) +++ head/sys/dev/sfxge/common/efsys.h (revision 341051) @@ -1,1208 +1,1212 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2016 Solarflare Communications Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. * * $FreeBSD$ */ #ifndef _SYS_EFSYS_H #define _SYS_EFSYS_H #ifdef __cplusplus extern "C" { #endif #include #include #include #include #include #include #include #include #include #include #include #include #define EFSYS_HAS_UINT64 1 #if defined(__x86_64__) #define EFSYS_USE_UINT64 1 #else #define EFSYS_USE_UINT64 0 #endif #define EFSYS_HAS_SSE2_M128 0 #if _BYTE_ORDER == _BIG_ENDIAN #define EFSYS_IS_BIG_ENDIAN 1 #define EFSYS_IS_LITTLE_ENDIAN 0 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define EFSYS_IS_BIG_ENDIAN 0 #define EFSYS_IS_LITTLE_ENDIAN 1 #endif #include "efx_types.h" /* Common code requires this */ #if __FreeBSD_version < 800068 #define memmove(d, s, l) bcopy(s, d, l) #endif /* FreeBSD equivalents of Solaris things */ #ifndef _NOTE #define _NOTE(s) #endif #ifndef B_FALSE #define B_FALSE FALSE #endif #ifndef B_TRUE #define B_TRUE TRUE #endif #ifndef IS_P2ALIGNED #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) #endif #ifndef P2ROUNDUP #define P2ROUNDUP(x, align) (-(-(x) & -(align))) #endif #ifndef P2ALIGN #define P2ALIGN(_x, _a) ((_x) & -(_a)) #endif #ifndef IS2P #define ISP2(x) (((x) & ((x) - 1)) == 0) #endif #if defined(__x86_64__) && __FreeBSD_version >= 1000000 #define SFXGE_USE_BUS_SPACE_8 1 #if !defined(bus_space_read_stream_8) #define bus_space_read_stream_8(t, h, o) \ bus_space_read_8((t), (h), (o)) #define bus_space_write_stream_8(t, h, o, v) \ bus_space_write_8((t), (h), (o), (v)) #endif #endif #define ENOTACTIVE EINVAL /* Memory type to use on FreeBSD */ MALLOC_DECLARE(M_SFXGE); /* Machine dependend prefetch wrappers */ #if defined(__i386__) || defined(__amd64__) static __inline void prefetch_read_many(void *addr) { __asm__( "prefetcht0 (%0)" : : "r" (addr)); } static __inline void prefetch_read_once(void *addr) { __asm__( "prefetchnta (%0)" : : "r" (addr)); } #elif defined(__sparc64__) static __inline void prefetch_read_many(void *addr) { __asm__( "prefetch [%0], 0" : : "r" (addr)); } static __inline void prefetch_read_once(void *addr) { __asm__( "prefetch [%0], 1" : : "r" (addr)); } #else static __inline void prefetch_read_many(void *addr) { } static __inline void prefetch_read_once(void *addr) { } #endif #if defined(__i386__) || defined(__amd64__) #include #include #endif static __inline void sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf *m, bus_dma_segment_t *seg) { #if defined(__i386__) || defined(__amd64__) seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); seg->ds_len = m->m_len; #else int nsegstmp; bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); #endif } /* Modifiers used for Windows builds */ #define __in #define __in_opt #define __in_ecount(_n) #define __in_ecount_opt(_n) #define __in_bcount(_n) #define __in_bcount_opt(_n) #define __out #define __out_opt #define __out_ecount(_n) #define __out_ecount_opt(_n) #define __out_bcount(_n) #define __out_bcount_opt(_n) #define __out_bcount_part(_n, _l) #define __out_bcount_part_opt(_n, _l) #define __deref_out #define __inout #define __inout_opt #define __inout_ecount(_n) #define __inout_ecount_opt(_n) #define __inout_bcount(_n) #define __inout_bcount_opt(_n) #define __inout_bcount_full_opt(_n) #define __deref_out_bcount_opt(n) #define __checkReturn #define __success(_x) #define __drv_when(_p, _c) /* Code inclusion options */ #define EFSYS_OPT_NAMES 1 #define EFSYS_OPT_SIENA 1 #define EFSYS_OPT_HUNTINGTON 1 #define EFSYS_OPT_MEDFORD 1 #define EFSYS_OPT_MEDFORD2 0 #ifdef DEBUG #define EFSYS_OPT_CHECK_REG 1 #else #define EFSYS_OPT_CHECK_REG 0 #endif #define EFSYS_OPT_MCDI 1 #define EFSYS_OPT_MCDI_LOGGING 0 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 #define EFSYS_OPT_MAC_STATS 1 #define EFSYS_OPT_LOOPBACK 0 #define EFSYS_OPT_MON_MCDI 0 #define EFSYS_OPT_MON_STATS 0 #define EFSYS_OPT_PHY_STATS 1 #define EFSYS_OPT_BIST 1 #define EFSYS_OPT_PHY_LED_CONTROL 1 #define EFSYS_OPT_PHY_FLAGS 0 #define EFSYS_OPT_VPD 1 #define EFSYS_OPT_NVRAM 1 #define EFSYS_OPT_BOOTCFG 0 #define EFSYS_OPT_DIAG 0 #define EFSYS_OPT_RX_SCALE 1 #define EFSYS_OPT_QSTATS 1 #define EFSYS_OPT_FILTER 1 #define EFSYS_OPT_RX_SCATTER 0 #define EFSYS_OPT_EV_PREFETCH 0 #define EFSYS_OPT_DECODE_INTR_FATAL 1 #define EFSYS_OPT_LICENSING 0 #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0 #define EFSYS_OPT_RX_PACKED_STREAM 0 #define EFSYS_OPT_TUNNEL 0 /* ID */ typedef struct __efsys_identifier_s efsys_identifier_t; /* PROBE */ #ifndef DTRACE_PROBE #define EFSYS_PROBE(_name) #define EFSYS_PROBE1(_name, _type1, _arg1) #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) #else /* DTRACE_PROBE */ #define EFSYS_PROBE(_name) \ DTRACE_PROBE(_name) #define EFSYS_PROBE1(_name, _type1, _arg1) \ DTRACE_PROBE1(_name, _type1, _arg1) #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) \ DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) \ DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #ifdef DTRACE_PROBE5 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) \ DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #else #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) \ DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #endif #ifdef DTRACE_PROBE6 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) \ DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #else #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) \ EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #endif #ifdef DTRACE_PROBE7 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) \ DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) #else #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) \ EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #endif #endif /* DTRACE_PROBE */ /* DMA */ typedef uint64_t efsys_dma_addr_t; typedef struct efsys_mem_s { bus_dma_tag_t esm_tag; bus_dmamap_t esm_map; caddr_t esm_base; efsys_dma_addr_t esm_addr; + size_t esm_size; } efsys_mem_t; +#define EFSYS_MEM_SIZE(_esmp) \ + ((_esmp)->esm_size) +#define EFSYS_MEM_ADDR(_esmp) \ + ((_esmp)->esm_addr) + +#define EFSYS_MEM_IS_NULL(_esmp) \ + ((_esmp)->esm_base == NULL) + + #define EFSYS_MEM_ZERO(_esmp, _size) \ do { \ (void) memset((_esmp)->esm_base, 0, (_size)); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_edp)->ed_u32[0] = *addr; \ \ EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #if defined(__x86_64__) #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ do { \ uint64_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eqp)->eq_u64[0] = *addr; \ \ EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eqp)->eq_u32[0] = *addr++; \ (_eqp)->eq_u32[1] = *addr; \ \ EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif #if defined(__x86_64__) #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ do { \ uint64_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eop)->eo_u64[0] = *addr++; \ (_eop)->eo_u64[1] = *addr; \ \ EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eop)->eo_u32[0] = *addr++; \ (_eop)->eo_u32[1] = *addr++; \ (_eop)->eo_u32[2] = *addr++; \ (_eop)->eo_u32[3] = *addr; \ \ EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr = (_edp)->ed_u32[0]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #if defined(__x86_64__) #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ do { \ uint64_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr = (_eqp)->eq_u64[0]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr++ = (_eqp)->eq_u32[0]; \ *addr = (_eqp)->eq_u32[1]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif #if defined(__x86_64__) #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ do { \ uint64_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr++ = (_eop)->eo_u64[0]; \ *addr = (_eop)->eo_u64[1]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr++ = (_eop)->eo_u32[0]; \ *addr++ = (_eop)->eo_u32[1]; \ *addr++ = (_eop)->eo_u32[2]; \ *addr = (_eop)->eo_u32[3]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif - -#define EFSYS_MEM_ADDR(_esmp) \ - ((_esmp)->esm_addr) - -#define EFSYS_MEM_IS_NULL(_esmp) \ - ((_esmp)->esm_base == NULL) /* BAR */ #define SFXGE_LOCK_NAME_MAX 16 typedef struct efsys_bar_s { struct mtx esb_lock; char esb_lock_name[SFXGE_LOCK_NAME_MAX]; bus_space_tag_t esb_tag; bus_space_handle_t esb_handle; int esb_rid; struct resource *esb_res; } efsys_bar_t; #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ do { \ snprintf((_esbp)->esb_lock_name, \ sizeof((_esbp)->esb_lock_name), \ "%s:bar", (_ifname)); \ mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ NULL, MTX_DEF); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ mtx_destroy(&(_esbp)->esb_lock) #define SFXGE_BAR_LOCK(_esbp) \ mtx_lock(&(_esbp)->esb_lock) #define SFXGE_BAR_UNLOCK(_esbp) \ mtx_unlock(&(_esbp)->esb_lock) #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ (_edp)->ed_u32[0] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset)); \ \ EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #if defined(SFXGE_USE_BUS_SPACE_8) #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ SFXGE_BAR_LOCK(_esbp); \ \ (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset)); \ \ EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ (_eop)->eo_u64[0] = bus_space_read_stream_8( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset)); \ (_eop)->eo_u64[1] = bus_space_read_stream_8( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset) + 8); \ \ EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ SFXGE_BAR_LOCK(_esbp); \ \ (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset)); \ (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset) + 4); \ \ EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ (_eop)->eo_u32[0] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset)); \ (_eop)->eo_u32[1] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset) + 4); \ (_eop)->eo_u32[2] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset) + 8); \ (_eop)->eo_u32[3] = bus_space_read_stream_4( \ (_esbp)->esb_tag, (_esbp)->esb_handle, \ (_offset) + 12); \ \ EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ /* \ * Make sure that previous writes to the dword have \ * been done. It should be cheaper than barrier just \ * after the write below. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_dword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset), (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #if defined(SFXGE_USE_BUS_SPACE_8) #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ /* \ * Make sure that previous writes to the qword have \ * been done. It should be cheaper than barrier just \ * after the write below. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_qword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_8((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset), (_eqp)->eq_u64[0]); \ \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ /* \ * Make sure that previous writes to the qword have \ * been done. It should be cheaper than barrier just \ * after the last write below. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_qword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset), (_eqp)->eq_u32[0]); \ /* \ * It should be guaranteed that the last dword comes \ * the last, so barrier entire qword to be sure that \ * neither above nor below writes are reordered. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_qword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset) + 4, (_eqp)->eq_u32[1]); \ \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif /* * Guarantees 64bit aligned 64bit writes to write combined BAR mapping * (required by PIO hardware) */ #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ (void) (_esbp); \ \ /* FIXME: Perform a 64-bit write */ \ KASSERT(0, ("not implemented")); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #if defined(SFXGE_USE_BUS_SPACE_8) #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ /* \ * Make sure that previous writes to the oword have \ * been done. It should be cheaper than barrier just \ * after the last write below. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_oword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_8((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset), (_eop)->eo_u64[0]); \ /* \ * It should be guaranteed that the last qword comes \ * the last, so barrier entire oword to be sure that \ * neither above nor below writes are reordered. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_oword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_8((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset) + 8, (_eop)->eo_u64[1]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ /* \ * Make sure that previous writes to the oword have \ * been done. It should be cheaper than barrier just \ * after the last write below. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_oword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset), (_eop)->eo_u32[0]); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset) + 4, (_eop)->eo_u32[1]); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset) + 8, (_eop)->eo_u32[2]); \ /* \ * It should be guaranteed that the last dword comes \ * the last, so barrier entire oword to be sure that \ * neither above nor below writes are reordered. \ */ \ bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), sizeof (efx_oword_t), \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_stream_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, \ (_offset) + 12, (_eop)->eo_u32[3]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif /* Use the standard octo-word write for doorbell writes */ #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ do { \ EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* SPIN */ #define EFSYS_SPIN(_us) \ do { \ DELAY(_us); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_SLEEP EFSYS_SPIN /* BARRIERS */ #define EFSYS_MEM_READ_BARRIER() rmb() #define EFSYS_PIO_WRITE_BARRIER() /* DMA SYNC */ #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ do { \ bus_dmamap_sync((_esmp)->esm_tag, \ (_esmp)->esm_map, \ BUS_DMASYNC_POSTREAD); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ do { \ bus_dmamap_sync((_esmp)->esm_tag, \ (_esmp)->esm_map, \ BUS_DMASYNC_PREWRITE); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* TIMESTAMP */ typedef clock_t efsys_timestamp_t; #define EFSYS_TIMESTAMP(_usp) \ do { \ clock_t now; \ \ now = ticks; \ *(_usp) = now * hz / 1000000; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* KMEM */ #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ do { \ (_esip) = (_esip); \ /* \ * The macro is used in non-sleepable contexts, for \ * example, holding a mutex. \ */ \ (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_KMEM_FREE(_esip, _size, _p) \ do { \ (void) (_esip); \ (void) (_size); \ free((_p), M_SFXGE); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* LOCK */ typedef struct efsys_lock_s { struct mtx lock; char lock_name[SFXGE_LOCK_NAME_MAX]; } efsys_lock_t; #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ do { \ efsys_lock_t *__eslp = (_eslp); \ \ snprintf((__eslp)->lock_name, \ sizeof((__eslp)->lock_name), \ "%s:%s", (_ifname), (_label)); \ mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ NULL, MTX_DEF); \ } while (B_FALSE) #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ mtx_destroy(&(_eslp)->lock) #define SFXGE_EFSYS_LOCK(_eslp) \ mtx_lock(&(_eslp)->lock) #define SFXGE_EFSYS_UNLOCK(_eslp) \ mtx_unlock(&(_eslp)->lock) #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ mtx_assert(&(_eslp)->lock, MA_OWNED) typedef int efsys_lock_state_t; #define EFSYS_LOCK_MAGIC 0x000010c4 #define EFSYS_LOCK(_lockp, _state) \ do { \ SFXGE_EFSYS_LOCK(_lockp); \ (_state) = EFSYS_LOCK_MAGIC; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_UNLOCK(_lockp, _state) \ do { \ if ((_state) != EFSYS_LOCK_MAGIC) \ KASSERT(B_FALSE, ("not locked")); \ SFXGE_EFSYS_UNLOCK(_lockp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* STAT */ typedef uint64_t efsys_stat_t; #define EFSYS_STAT_INCR(_knp, _delta) \ do { \ *(_knp) += (_delta); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_DECR(_knp, _delta) \ do { \ *(_knp) -= (_delta); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET(_knp, _val) \ do { \ *(_knp) = (_val); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ do { \ *(_knp) = le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ do { \ *(_knp) = le32toh((_valp)->ed_u32[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ do { \ *(_knp) += le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ do { \ *(_knp) -= le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* ERR */ extern void sfxge_err(efsys_identifier_t *, unsigned int, uint32_t, uint32_t); #if EFSYS_OPT_DECODE_INTR_FATAL #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ do { \ sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif /* ASSERT */ #define EFSYS_ASSERT(_exp) do { \ if (!(_exp)) \ panic("%s", #_exp); \ } while (0) #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ const _t __x = (_t)(_x); \ const _t __y = (_t)(_y); \ if (!(__x _op __y)) \ panic("assertion failed at %s:%u", __FILE__, __LINE__); \ } while(0) #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) /* ROTATE */ #define EFSYS_HAS_ROTL_DWORD 0 #ifdef __cplusplus } #endif #endif /* _SYS_EFSYS_H */ Index: head/sys/dev/sfxge/common/efx_intr.c =================================================================== --- head/sys/dev/sfxge/common/efx_intr.c (revision 341050) +++ head/sys/dev/sfxge/common/efx_intr.c (revision 341051) @@ -1,583 +1,594 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_intr_init( __in efx_nic_t *enp, __in efx_intr_type_t type, __in efsys_mem_t *esmp); static void siena_intr_enable( __in efx_nic_t *enp); static void siena_intr_disable( __in efx_nic_t *enp); static void siena_intr_disable_unlocked( __in efx_nic_t *enp); static __checkReturn efx_rc_t siena_intr_trigger( __in efx_nic_t *enp, __in unsigned int level); static void siena_intr_fini( __in efx_nic_t *enp); static void siena_intr_status_line( __in efx_nic_t *enp, __out boolean_t *fatalp, __out uint32_t *qmaskp); static void siena_intr_status_message( __in efx_nic_t *enp, __in unsigned int message, __out boolean_t *fatalp); static void siena_intr_fatal( __in efx_nic_t *enp); static __checkReturn boolean_t siena_intr_check_fatal( __in efx_nic_t *enp); #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_SIENA static const efx_intr_ops_t __efx_intr_siena_ops = { siena_intr_init, /* eio_init */ siena_intr_enable, /* eio_enable */ siena_intr_disable, /* eio_disable */ siena_intr_disable_unlocked, /* eio_disable_unlocked */ siena_intr_trigger, /* eio_trigger */ siena_intr_status_line, /* eio_status_line */ siena_intr_status_message, /* eio_status_message */ siena_intr_fatal, /* eio_fatal */ siena_intr_fini, /* eio_fini */ }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 static const efx_intr_ops_t __efx_intr_ef10_ops = { ef10_intr_init, /* eio_init */ ef10_intr_enable, /* eio_enable */ ef10_intr_disable, /* eio_disable */ ef10_intr_disable_unlocked, /* eio_disable_unlocked */ ef10_intr_trigger, /* eio_trigger */ ef10_intr_status_line, /* eio_status_line */ ef10_intr_status_message, /* eio_status_message */ ef10_intr_fatal, /* eio_fatal */ ef10_intr_fini, /* eio_fini */ }; #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ __checkReturn efx_rc_t efx_intr_init( __in efx_nic_t *enp, __in efx_intr_type_t type, __in efsys_mem_t *esmp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); if (enp->en_mod_flags & EFX_MOD_INTR) { rc = EINVAL; goto fail1; } eip->ei_esmp = esmp; eip->ei_type = type; eip->ei_level = 0; enp->en_mod_flags |= EFX_MOD_INTR; switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: eiop = &__efx_intr_siena_ops; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: eiop = &__efx_intr_ef10_ops; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: eiop = &__efx_intr_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD */ #if EFSYS_OPT_MEDFORD2 case EFX_FAMILY_MEDFORD2: eiop = &__efx_intr_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD2 */ default: EFSYS_ASSERT(B_FALSE); rc = ENOTSUP; goto fail2; } if ((rc = eiop->eio_init(enp, type, esmp)) != 0) goto fail3; eip->ei_eiop = eiop; return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_intr_fini( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_fini(enp); enp->en_mod_flags &= ~EFX_MOD_INTR; } void efx_intr_enable( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_enable(enp); } void efx_intr_disable( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_disable(enp); } void efx_intr_disable_unlocked( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_disable_unlocked(enp); } __checkReturn efx_rc_t efx_intr_trigger( __in efx_nic_t *enp, __in unsigned int level) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); return (eiop->eio_trigger(enp, level)); } void efx_intr_status_line( __in efx_nic_t *enp, __out boolean_t *fatalp, __out uint32_t *qmaskp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_status_line(enp, fatalp, qmaskp); } void efx_intr_status_message( __in efx_nic_t *enp, __in unsigned int message, __out boolean_t *fatalp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_status_message(enp, message, fatalp); } void efx_intr_fatal( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); const efx_intr_ops_t *eiop = eip->ei_eiop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); eiop->eio_fatal(enp); } /* ************************************************************************* */ /* ************************************************************************* */ /* ************************************************************************* */ #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_intr_init( __in efx_nic_t *enp, __in efx_intr_type_t type, __in efsys_mem_t *esmp) { efx_intr_t *eip = &(enp->en_intr); efx_oword_t oword; + efx_rc_t rc; + if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_INTR_SIZE)) { + rc = EINVAL; + goto fail1; + } + /* * bug17213 workaround. * * Under legacy interrupts, don't share a level between fatal * interrupts and event queue interrupts. Under MSI-X, they * must share, or we won't get an interrupt. */ if (enp->en_family == EFX_FAMILY_SIENA && eip->ei_type == EFX_INTR_LINE) eip->ei_level = 0x1f; else eip->ei_level = 0; /* Enable all the genuinely fatal interrupts */ EFX_SET_OWORD(oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_ILL_ADR_INT_KER_EN, 0); EFX_SET_OWORD_FIELD(oword, FRF_AZ_RBUF_OWN_INT_KER_EN, 0); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TBUF_OWN_INT_KER_EN, 0); if (enp->en_family >= EFX_FAMILY_SIENA) EFX_SET_OWORD_FIELD(oword, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 0); EFX_BAR_WRITEO(enp, FR_AZ_FATAL_INTR_REG_KER, &oword); /* Set up the interrupt address register */ EFX_POPULATE_OWORD_3(oword, FRF_AZ_NORM_INT_VEC_DIS_KER, (type == EFX_INTR_MESSAGE) ? 1 : 0, FRF_AZ_INT_ADR_KER_DW0, EFSYS_MEM_ADDR(esmp) & 0xffffffff, FRF_AZ_INT_ADR_KER_DW1, EFSYS_MEM_ADDR(esmp) >> 32); EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword); return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); } static void siena_intr_enable( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); efx_oword_t oword; EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level); EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 1); EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); } static void siena_intr_disable( __in efx_nic_t *enp) { efx_oword_t oword; EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0); EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); EFSYS_SPIN(10); } static void siena_intr_disable_unlocked( __in efx_nic_t *enp) { efx_oword_t oword; EFSYS_BAR_READO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST, &oword, B_FALSE); EFX_SET_OWORD_FIELD(oword, FRF_AZ_DRV_INT_EN_KER, 0); EFSYS_BAR_WRITEO(enp->en_esbp, FR_AZ_INT_EN_REG_KER_OFST, &oword, B_FALSE); } static __checkReturn efx_rc_t siena_intr_trigger( __in efx_nic_t *enp, __in unsigned int level) { efx_intr_t *eip = &(enp->en_intr); efx_oword_t oword; unsigned int count; uint32_t sel; efx_rc_t rc; /* bug16757: No event queues can be initialized */ EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_EV)); if (level >= EFX_NINTR_SIENA) { rc = EINVAL; goto fail1; } if (level > EFX_MASK32(FRF_AZ_KER_INT_LEVE_SEL)) return (ENOTSUP); /* avoid EFSYS_PROBE() */ sel = level; /* Trigger a test interrupt */ EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, sel); EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER, 1); EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); /* * Wait up to 100ms for the interrupt to be raised before restoring * KER_INT_LEVE_SEL. Ignore a failure to raise (the caller will * observe this soon enough anyway), but always reset KER_INT_LEVE_SEL */ count = 0; do { EFSYS_SPIN(100); /* 100us */ EFX_BAR_READO(enp, FR_AZ_INT_EN_REG_KER, &oword); } while (EFX_OWORD_FIELD(oword, FRF_AZ_KER_INT_KER) && ++count < 1000); EFX_SET_OWORD_FIELD(oword, FRF_AZ_KER_INT_LEVE_SEL, eip->ei_level); EFX_BAR_WRITEO(enp, FR_AZ_INT_EN_REG_KER, &oword); return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn boolean_t siena_intr_check_fatal( __in efx_nic_t *enp) { efx_intr_t *eip = &(enp->en_intr); efsys_mem_t *esmp = eip->ei_esmp; efx_oword_t oword; /* Read the syndrome */ EFSYS_MEM_READO(esmp, 0, &oword); if (EFX_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT) != 0) { EFSYS_PROBE(fatal); /* Clear the fatal interrupt condition */ EFX_SET_OWORD_FIELD(oword, FSF_AZ_NET_IVEC_FATAL_INT, 0); EFSYS_MEM_WRITEO(esmp, 0, &oword); return (B_TRUE); } return (B_FALSE); } static void siena_intr_status_line( __in efx_nic_t *enp, __out boolean_t *fatalp, __out uint32_t *qmaskp) { efx_intr_t *eip = &(enp->en_intr); efx_dword_t dword; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); /* * Read the queue mask and implicitly acknowledge the * interrupt. */ EFX_BAR_READD(enp, FR_BZ_INT_ISR0_REG, &dword, B_FALSE); *qmaskp = EFX_DWORD_FIELD(dword, EFX_DWORD_0); EFSYS_PROBE1(qmask, uint32_t, *qmaskp); if (*qmaskp & (1U << eip->ei_level)) *fatalp = siena_intr_check_fatal(enp); else *fatalp = B_FALSE; } static void siena_intr_status_message( __in efx_nic_t *enp, __in unsigned int message, __out boolean_t *fatalp) { efx_intr_t *eip = &(enp->en_intr); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); if (message == eip->ei_level) *fatalp = siena_intr_check_fatal(enp); else *fatalp = B_FALSE; } static void siena_intr_fatal( __in efx_nic_t *enp) { #if EFSYS_OPT_DECODE_INTR_FATAL efx_oword_t fatal; efx_oword_t mem_per; EFX_BAR_READO(enp, FR_AZ_FATAL_INTR_REG_KER, &fatal); EFX_ZERO_OWORD(mem_per); if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0 || EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0) EFX_BAR_READO(enp, FR_AZ_MEM_STAT_REG, &mem_per); if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRAM_OOB_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_OOB, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_BUFID_DC_OOB_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_BUFID_DC_OOB, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_MEM_PERR_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_MEM_PERR, EFX_OWORD_FIELD(mem_per, EFX_DWORD_0), EFX_OWORD_FIELD(mem_per, EFX_DWORD_1)); if (EFX_OWORD_FIELD(fatal, FRF_AZ_RBUF_OWN_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_RBUF_OWN, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_TBUF_OWN_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_TBUF_OWN, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_RDESCQ_OWN_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_RDESQ_OWN, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_TDESCQ_OWN_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_TDESQ_OWN, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVQ_OWN_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_EVQ_OWN, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_EVF_OFLO_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_EVFF_OFLO, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_ILL_ADR_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_ILL_ADDR, 0, 0); if (EFX_OWORD_FIELD(fatal, FRF_AZ_SRM_PERR_INT_KER) != 0) EFSYS_ERR(enp->en_esip, EFX_ERR_SRAM_PERR, EFX_OWORD_FIELD(mem_per, EFX_DWORD_0), EFX_OWORD_FIELD(mem_per, EFX_DWORD_1)); #else EFSYS_ASSERT(0); #endif } static void siena_intr_fini( __in efx_nic_t *enp) { efx_oword_t oword; /* Clear the interrupt address register */ EFX_ZERO_OWORD(oword); EFX_BAR_WRITEO(enp, FR_AZ_INT_ADR_REG_KER, &oword); } #endif /* EFSYS_OPT_SIENA */ Index: head/sys/dev/sfxge/common/efx_mcdi.c =================================================================== --- head/sys/dev/sfxge/common/efx_mcdi.c (revision 341050) +++ head/sys/dev/sfxge/common/efx_mcdi.c (revision 341051) @@ -1,2366 +1,2368 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_MCDI /* * There are three versions of the MCDI interface: * - MCDIv0: Siena BootROM. Transport uses MCDIv1 headers. * - MCDIv1: Siena firmware and Huntington BootROM. * - MCDIv2: EF10 firmware (Huntington/Medford) and Medford BootROM. * Transport uses MCDIv2 headers. * * MCDIv2 Header NOT_EPOCH flag * ---------------------------- * A new epoch begins at initial startup or after an MC reboot, and defines when * the MC should reject stale MCDI requests. * * The first MCDI request sent by the host should contain NOT_EPOCH=0, and all * subsequent requests (until the next MC reboot) should contain NOT_EPOCH=1. * * After rebooting the MC will fail all requests with NOT_EPOCH=1 by writing a * response with ERROR=1 and DATALEN=0 until a request is seen with NOT_EPOCH=0. */ #if EFSYS_OPT_SIENA static const efx_mcdi_ops_t __efx_mcdi_siena_ops = { siena_mcdi_init, /* emco_init */ siena_mcdi_send_request, /* emco_send_request */ siena_mcdi_poll_reboot, /* emco_poll_reboot */ siena_mcdi_poll_response, /* emco_poll_response */ siena_mcdi_read_response, /* emco_read_response */ siena_mcdi_fini, /* emco_fini */ siena_mcdi_feature_supported, /* emco_feature_supported */ siena_mcdi_get_timeout, /* emco_get_timeout */ }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 static const efx_mcdi_ops_t __efx_mcdi_ef10_ops = { ef10_mcdi_init, /* emco_init */ ef10_mcdi_send_request, /* emco_send_request */ ef10_mcdi_poll_reboot, /* emco_poll_reboot */ ef10_mcdi_poll_response, /* emco_poll_response */ ef10_mcdi_read_response, /* emco_read_response */ ef10_mcdi_fini, /* emco_fini */ ef10_mcdi_feature_supported, /* emco_feature_supported */ ef10_mcdi_get_timeout, /* emco_get_timeout */ }; #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ __checkReturn efx_rc_t efx_mcdi_init( __in efx_nic_t *enp, __in const efx_mcdi_transport_t *emtp) { const efx_mcdi_ops_t *emcop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, ==, 0); switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: emcop = &__efx_mcdi_siena_ops; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: emcop = &__efx_mcdi_ef10_ops; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: emcop = &__efx_mcdi_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD */ #if EFSYS_OPT_MEDFORD2 case EFX_FAMILY_MEDFORD2: emcop = &__efx_mcdi_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD2 */ default: EFSYS_ASSERT(0); rc = ENOTSUP; goto fail1; } if (enp->en_features & EFX_FEATURE_MCDI_DMA) { /* MCDI requires a DMA buffer in host memory */ if ((emtp == NULL) || (emtp->emt_dma_mem) == NULL) { rc = EINVAL; goto fail2; } } enp->en_mcdi.em_emtp = emtp; if (emcop != NULL && emcop->emco_init != NULL) { if ((rc = emcop->emco_init(enp, emtp)) != 0) goto fail3; } enp->en_mcdi.em_emcop = emcop; enp->en_mod_flags |= EFX_MOD_MCDI; return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); enp->en_mcdi.em_emcop = NULL; enp->en_mcdi.em_emtp = NULL; enp->en_mod_flags &= ~EFX_MOD_MCDI; return (rc); } void efx_mcdi_fini( __in efx_nic_t *enp) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, ==, EFX_MOD_MCDI); if (emcop != NULL && emcop->emco_fini != NULL) emcop->emco_fini(enp); emip->emi_port = 0; emip->emi_aborted = 0; enp->en_mcdi.em_emcop = NULL; enp->en_mod_flags &= ~EFX_MOD_MCDI; } void efx_mcdi_new_epoch( __in efx_nic_t *enp) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efsys_lock_state_t state; /* Start a new epoch (allow fresh MCDI requests to succeed) */ EFSYS_LOCK(enp->en_eslp, state); emip->emi_new_epoch = B_TRUE; EFSYS_UNLOCK(enp->en_eslp, state); } static void efx_mcdi_send_request( __in efx_nic_t *enp, __in void *hdrp, __in size_t hdr_len, __in void *sdup, __in size_t sdu_len) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; emcop->emco_send_request(enp, hdrp, hdr_len, sdup, sdu_len); } static efx_rc_t efx_mcdi_poll_reboot( __in efx_nic_t *enp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; efx_rc_t rc; rc = emcop->emco_poll_reboot(enp); return (rc); } static boolean_t efx_mcdi_poll_response( __in efx_nic_t *enp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; boolean_t available; available = emcop->emco_poll_response(enp); return (available); } static void efx_mcdi_read_response( __in efx_nic_t *enp, __out void *bufferp, __in size_t offset, __in size_t length) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; emcop->emco_read_response(enp, bufferp, offset, length); } void efx_mcdi_request_start( __in efx_nic_t *enp, __in efx_mcdi_req_t *emrp, __in boolean_t ev_cpl) { #if EFSYS_OPT_MCDI_LOGGING const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; #endif efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efx_dword_t hdr[2]; size_t hdr_len; unsigned int max_version; unsigned int seq; unsigned int xflags; boolean_t new_epoch; efsys_lock_state_t state; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); /* * efx_mcdi_request_start() is naturally serialised against both * efx_mcdi_request_poll() and efx_mcdi_ev_cpl()/efx_mcdi_ev_death(), * by virtue of there only being one outstanding MCDI request. * Unfortunately, upper layers may also call efx_mcdi_request_abort() * at any time, to timeout a pending mcdi request, That request may * then subsequently complete, meaning efx_mcdi_ev_cpl() or * efx_mcdi_ev_death() may end up running in parallel with * efx_mcdi_request_start(). This race is handled by ensuring that * %emi_pending_req, %emi_ev_cpl and %emi_seq are protected by the * en_eslp lock. */ EFSYS_LOCK(enp->en_eslp, state); EFSYS_ASSERT(emip->emi_pending_req == NULL); emip->emi_pending_req = emrp; emip->emi_ev_cpl = ev_cpl; emip->emi_poll_cnt = 0; seq = emip->emi_seq++ & EFX_MASK32(MCDI_HEADER_SEQ); new_epoch = emip->emi_new_epoch; max_version = emip->emi_max_version; EFSYS_UNLOCK(enp->en_eslp, state); xflags = 0; if (ev_cpl) xflags |= MCDI_HEADER_XFLAGS_EVREQ; /* * Huntington firmware supports MCDIv2, but the Huntington BootROM only * supports MCDIv1. Use MCDIv1 headers for MCDIv1 commands where * possible to support this. */ if ((max_version >= 2) && ((emrp->emr_cmd > MC_CMD_CMD_SPACE_ESCAPE_7) || (emrp->emr_in_length > MCDI_CTL_SDU_LEN_MAX_V1) || (emrp->emr_out_length > MCDI_CTL_SDU_LEN_MAX_V1))) { /* Construct MCDI v2 header */ hdr_len = sizeof (hdr); EFX_POPULATE_DWORD_8(hdr[0], MCDI_HEADER_CODE, MC_CMD_V2_EXTN, MCDI_HEADER_RESYNC, 1, MCDI_HEADER_DATALEN, 0, MCDI_HEADER_SEQ, seq, MCDI_HEADER_NOT_EPOCH, new_epoch ? 0 : 1, MCDI_HEADER_ERROR, 0, MCDI_HEADER_RESPONSE, 0, MCDI_HEADER_XFLAGS, xflags); EFX_POPULATE_DWORD_2(hdr[1], MC_CMD_V2_EXTN_IN_EXTENDED_CMD, emrp->emr_cmd, MC_CMD_V2_EXTN_IN_ACTUAL_LEN, emrp->emr_in_length); } else { /* Construct MCDI v1 header */ hdr_len = sizeof (hdr[0]); EFX_POPULATE_DWORD_8(hdr[0], MCDI_HEADER_CODE, emrp->emr_cmd, MCDI_HEADER_RESYNC, 1, MCDI_HEADER_DATALEN, emrp->emr_in_length, MCDI_HEADER_SEQ, seq, MCDI_HEADER_NOT_EPOCH, new_epoch ? 0 : 1, MCDI_HEADER_ERROR, 0, MCDI_HEADER_RESPONSE, 0, MCDI_HEADER_XFLAGS, xflags); } #if EFSYS_OPT_MCDI_LOGGING if (emtp->emt_logger != NULL) { emtp->emt_logger(emtp->emt_context, EFX_LOG_MCDI_REQUEST, &hdr, hdr_len, emrp->emr_in_buf, emrp->emr_in_length); } #endif /* EFSYS_OPT_MCDI_LOGGING */ efx_mcdi_send_request(enp, &hdr[0], hdr_len, emrp->emr_in_buf, emrp->emr_in_length); } static void efx_mcdi_read_response_header( __in efx_nic_t *enp, __inout efx_mcdi_req_t *emrp) { #if EFSYS_OPT_MCDI_LOGGING const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; #endif /* EFSYS_OPT_MCDI_LOGGING */ efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efx_dword_t hdr[2]; unsigned int hdr_len; unsigned int data_len; unsigned int seq; unsigned int cmd; unsigned int error; efx_rc_t rc; EFSYS_ASSERT(emrp != NULL); efx_mcdi_read_response(enp, &hdr[0], 0, sizeof (hdr[0])); hdr_len = sizeof (hdr[0]); cmd = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_CODE); seq = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_SEQ); error = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_ERROR); if (cmd != MC_CMD_V2_EXTN) { data_len = EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_DATALEN); } else { efx_mcdi_read_response(enp, &hdr[1], hdr_len, sizeof (hdr[1])); hdr_len += sizeof (hdr[1]); cmd = EFX_DWORD_FIELD(hdr[1], MC_CMD_V2_EXTN_IN_EXTENDED_CMD); data_len = EFX_DWORD_FIELD(hdr[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN); } if (error && (data_len == 0)) { /* The MC has rebooted since the request was sent. */ EFSYS_SPIN(EFX_MCDI_STATUS_SLEEP_US); efx_mcdi_poll_reboot(enp); rc = EIO; goto fail1; } if ((cmd != emrp->emr_cmd) || (seq != ((emip->emi_seq - 1) & EFX_MASK32(MCDI_HEADER_SEQ)))) { /* Response is for a different request */ rc = EIO; goto fail2; } if (error) { efx_dword_t err[2]; unsigned int err_len = MIN(data_len, sizeof (err)); int err_code = MC_CMD_ERR_EPROTO; int err_arg = 0; /* Read error code (and arg num for MCDI v2 commands) */ efx_mcdi_read_response(enp, &err, hdr_len, err_len); if (err_len >= (MC_CMD_ERR_CODE_OFST + sizeof (efx_dword_t))) err_code = EFX_DWORD_FIELD(err[0], EFX_DWORD_0); #ifdef WITH_MCDI_V2 if (err_len >= (MC_CMD_ERR_ARG_OFST + sizeof (efx_dword_t))) err_arg = EFX_DWORD_FIELD(err[1], EFX_DWORD_0); #endif emrp->emr_err_code = err_code; emrp->emr_err_arg = err_arg; #if EFSYS_OPT_MCDI_PROXY_AUTH if ((err_code == MC_CMD_ERR_PROXY_PENDING) && (err_len == sizeof (err))) { /* * The MCDI request would normally fail with EPERM, but * firmware has forwarded it to an authorization agent * attached to a privileged PF. * * Save the authorization request handle. The client * must wait for a PROXY_RESPONSE event, or timeout. */ emrp->emr_proxy_handle = err_arg; } #endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ #if EFSYS_OPT_MCDI_LOGGING if (emtp->emt_logger != NULL) { emtp->emt_logger(emtp->emt_context, EFX_LOG_MCDI_RESPONSE, &hdr, hdr_len, &err, err_len); } #endif /* EFSYS_OPT_MCDI_LOGGING */ if (!emrp->emr_quiet) { EFSYS_PROBE3(mcdi_err_arg, int, emrp->emr_cmd, int, err_code, int, err_arg); } rc = efx_mcdi_request_errcode(err_code); goto fail3; } emrp->emr_rc = 0; emrp->emr_out_length_used = data_len; #if EFSYS_OPT_MCDI_PROXY_AUTH emrp->emr_proxy_handle = 0; #endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ return; fail3: fail2: fail1: emrp->emr_rc = rc; emrp->emr_out_length_used = 0; } static void efx_mcdi_finish_response( __in efx_nic_t *enp, __in efx_mcdi_req_t *emrp) { #if EFSYS_OPT_MCDI_LOGGING const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; #endif /* EFSYS_OPT_MCDI_LOGGING */ efx_dword_t hdr[2]; unsigned int hdr_len; size_t bytes; if (emrp->emr_out_buf == NULL) return; /* Read the command header to detect MCDI response format */ hdr_len = sizeof (hdr[0]); efx_mcdi_read_response(enp, &hdr[0], 0, hdr_len); if (EFX_DWORD_FIELD(hdr[0], MCDI_HEADER_CODE) == MC_CMD_V2_EXTN) { /* * Read the actual payload length. The length given in the event * is only correct for responses with the V1 format. */ efx_mcdi_read_response(enp, &hdr[1], hdr_len, sizeof (hdr[1])); hdr_len += sizeof (hdr[1]); emrp->emr_out_length_used = EFX_DWORD_FIELD(hdr[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN); } /* Copy payload out into caller supplied buffer */ bytes = MIN(emrp->emr_out_length_used, emrp->emr_out_length); efx_mcdi_read_response(enp, emrp->emr_out_buf, hdr_len, bytes); #if EFSYS_OPT_MCDI_LOGGING if (emtp->emt_logger != NULL) { emtp->emt_logger(emtp->emt_context, EFX_LOG_MCDI_RESPONSE, &hdr, hdr_len, emrp->emr_out_buf, bytes); } #endif /* EFSYS_OPT_MCDI_LOGGING */ } __checkReturn boolean_t efx_mcdi_request_poll( __in efx_nic_t *enp) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efx_mcdi_req_t *emrp; efsys_lock_state_t state; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); /* Serialise against post-watchdog efx_mcdi_ev* */ EFSYS_LOCK(enp->en_eslp, state); EFSYS_ASSERT(emip->emi_pending_req != NULL); EFSYS_ASSERT(!emip->emi_ev_cpl); emrp = emip->emi_pending_req; /* Check for reboot atomically w.r.t efx_mcdi_request_start */ if (emip->emi_poll_cnt++ == 0) { if ((rc = efx_mcdi_poll_reboot(enp)) != 0) { emip->emi_pending_req = NULL; EFSYS_UNLOCK(enp->en_eslp, state); /* Reboot/Assertion */ if (rc == EIO || rc == EINTR) efx_mcdi_raise_exception(enp, emrp, rc); goto fail1; } } /* Check if a response is available */ if (efx_mcdi_poll_response(enp) == B_FALSE) { EFSYS_UNLOCK(enp->en_eslp, state); return (B_FALSE); } /* Read the response header */ efx_mcdi_read_response_header(enp, emrp); /* Request complete */ emip->emi_pending_req = NULL; /* Ensure stale MCDI requests fail after an MC reboot. */ emip->emi_new_epoch = B_FALSE; EFSYS_UNLOCK(enp->en_eslp, state); if ((rc = emrp->emr_rc) != 0) goto fail2; efx_mcdi_finish_response(enp, emrp); return (B_TRUE); fail2: if (!emrp->emr_quiet) EFSYS_PROBE(fail2); fail1: if (!emrp->emr_quiet) EFSYS_PROBE1(fail1, efx_rc_t, rc); return (B_TRUE); } __checkReturn boolean_t efx_mcdi_request_abort( __in efx_nic_t *enp) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efx_mcdi_req_t *emrp; boolean_t aborted; efsys_lock_state_t state; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); /* * efx_mcdi_ev_* may have already completed this event, and be * spinning/blocked on the upper layer lock. So it *is* legitimate * to for emi_pending_req to be NULL. If there is a pending event * completed request, then provide a "credit" to allow * efx_mcdi_ev_cpl() to accept a single spurious completion. */ EFSYS_LOCK(enp->en_eslp, state); emrp = emip->emi_pending_req; aborted = (emrp != NULL); if (aborted) { emip->emi_pending_req = NULL; /* Error the request */ emrp->emr_out_length_used = 0; emrp->emr_rc = ETIMEDOUT; /* Provide a credit for seqno/emr_pending_req mismatches */ if (emip->emi_ev_cpl) ++emip->emi_aborted; /* * The upper layer has called us, so we don't * need to complete the request. */ } EFSYS_UNLOCK(enp->en_eslp, state); return (aborted); } void efx_mcdi_get_timeout( __in efx_nic_t *enp, __in efx_mcdi_req_t *emrp, __out uint32_t *timeoutp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; emcop->emco_get_timeout(enp, emrp, timeoutp); } __checkReturn efx_rc_t efx_mcdi_request_errcode( __in unsigned int err) { switch (err) { /* MCDI v1 */ case MC_CMD_ERR_EPERM: return (EACCES); case MC_CMD_ERR_ENOENT: return (ENOENT); case MC_CMD_ERR_EINTR: return (EINTR); case MC_CMD_ERR_EACCES: return (EACCES); case MC_CMD_ERR_EBUSY: return (EBUSY); case MC_CMD_ERR_EINVAL: return (EINVAL); case MC_CMD_ERR_EDEADLK: return (EDEADLK); case MC_CMD_ERR_ENOSYS: return (ENOTSUP); case MC_CMD_ERR_ETIME: return (ETIMEDOUT); case MC_CMD_ERR_ENOTSUP: return (ENOTSUP); case MC_CMD_ERR_EALREADY: return (EALREADY); /* MCDI v2 */ case MC_CMD_ERR_EEXIST: return (EEXIST); #ifdef MC_CMD_ERR_EAGAIN case MC_CMD_ERR_EAGAIN: return (EAGAIN); #endif #ifdef MC_CMD_ERR_ENOSPC case MC_CMD_ERR_ENOSPC: return (ENOSPC); #endif case MC_CMD_ERR_ERANGE: return (ERANGE); case MC_CMD_ERR_ALLOC_FAIL: return (ENOMEM); case MC_CMD_ERR_NO_VADAPTOR: return (ENOENT); case MC_CMD_ERR_NO_EVB_PORT: return (ENOENT); case MC_CMD_ERR_NO_VSWITCH: return (ENODEV); case MC_CMD_ERR_VLAN_LIMIT: return (EINVAL); case MC_CMD_ERR_BAD_PCI_FUNC: return (ENODEV); case MC_CMD_ERR_BAD_VLAN_MODE: return (EINVAL); case MC_CMD_ERR_BAD_VSWITCH_TYPE: return (EINVAL); case MC_CMD_ERR_BAD_VPORT_TYPE: return (EINVAL); case MC_CMD_ERR_MAC_EXIST: return (EEXIST); case MC_CMD_ERR_PROXY_PENDING: return (EAGAIN); default: EFSYS_PROBE1(mc_pcol_error, int, err); return (EIO); } } void efx_mcdi_raise_exception( __in efx_nic_t *enp, __in_opt efx_mcdi_req_t *emrp, __in int rc) { const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; efx_mcdi_exception_t exception; /* Reboot or Assertion failure only */ EFSYS_ASSERT(rc == EIO || rc == EINTR); /* * If MC_CMD_REBOOT causes a reboot (dependent on parameters), * then the EIO is not worthy of an exception. */ if (emrp != NULL && emrp->emr_cmd == MC_CMD_REBOOT && rc == EIO) return; exception = (rc == EIO) ? EFX_MCDI_EXCEPTION_MC_REBOOT : EFX_MCDI_EXCEPTION_MC_BADASSERT; emtp->emt_exception(emtp->emt_context, exception); } void efx_mcdi_execute( __in efx_nic_t *enp, __inout efx_mcdi_req_t *emrp) { const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); emrp->emr_quiet = B_FALSE; emtp->emt_execute(emtp->emt_context, emrp); } void efx_mcdi_execute_quiet( __in efx_nic_t *enp, __inout efx_mcdi_req_t *emrp) { const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); emrp->emr_quiet = B_TRUE; emtp->emt_execute(emtp->emt_context, emrp); } void efx_mcdi_ev_cpl( __in efx_nic_t *enp, __in unsigned int seq, __in unsigned int outlen, __in int errcode) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; efx_mcdi_req_t *emrp; efsys_lock_state_t state; EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI); EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); /* * Serialise against efx_mcdi_request_poll()/efx_mcdi_request_start() * when we're completing an aborted request. */ EFSYS_LOCK(enp->en_eslp, state); if (emip->emi_pending_req == NULL || !emip->emi_ev_cpl || (seq != ((emip->emi_seq - 1) & EFX_MASK32(MCDI_HEADER_SEQ)))) { EFSYS_ASSERT(emip->emi_aborted > 0); if (emip->emi_aborted > 0) --emip->emi_aborted; EFSYS_UNLOCK(enp->en_eslp, state); return; } emrp = emip->emi_pending_req; emip->emi_pending_req = NULL; EFSYS_UNLOCK(enp->en_eslp, state); if (emip->emi_max_version >= 2) { /* MCDIv2 response details do not fit into an event. */ efx_mcdi_read_response_header(enp, emrp); } else { if (errcode != 0) { if (!emrp->emr_quiet) { EFSYS_PROBE2(mcdi_err, int, emrp->emr_cmd, int, errcode); } emrp->emr_out_length_used = 0; emrp->emr_rc = efx_mcdi_request_errcode(errcode); } else { emrp->emr_out_length_used = outlen; emrp->emr_rc = 0; } } if (emrp->emr_rc == 0) efx_mcdi_finish_response(enp, emrp); emtp->emt_ev_cpl(emtp->emt_context); } #if EFSYS_OPT_MCDI_PROXY_AUTH __checkReturn efx_rc_t efx_mcdi_get_proxy_handle( __in efx_nic_t *enp, __in efx_mcdi_req_t *emrp, __out uint32_t *handlep) { efx_rc_t rc; _NOTE(ARGUNUSED(enp)) /* * Return proxy handle from MCDI request that returned with error * MC_MCD_ERR_PROXY_PENDING. This handle is used to wait for a matching * PROXY_RESPONSE event. */ if ((emrp == NULL) || (handlep == NULL)) { rc = EINVAL; goto fail1; } if ((emrp->emr_rc != 0) && (emrp->emr_err_code == MC_CMD_ERR_PROXY_PENDING)) { *handlep = emrp->emr_proxy_handle; rc = 0; } else { *handlep = 0; rc = ENOENT; } return (rc); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_mcdi_ev_proxy_response( __in efx_nic_t *enp, __in unsigned int handle, __in unsigned int status) { const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; efx_rc_t rc; /* * Handle results of an authorization request for a privileged MCDI * command. If authorization was granted then we must re-issue the * original MCDI request. If authorization failed or timed out, * then the original MCDI request should be completed with the * result code from this event. */ rc = (status == 0) ? 0 : efx_mcdi_request_errcode(status); emtp->emt_ev_proxy_response(emtp->emt_context, handle, rc); } #endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ void efx_mcdi_ev_death( __in efx_nic_t *enp, __in int rc) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp; efx_mcdi_req_t *emrp = NULL; boolean_t ev_cpl; efsys_lock_state_t state; /* * The MCDI request (if there is one) has been terminated, either * by a BADASSERT or REBOOT event. * * If there is an outstanding event-completed MCDI operation, then we * will never receive the completion event (because both MCDI * completions and BADASSERT events are sent to the same evq). So * complete this MCDI op. * * This function might run in parallel with efx_mcdi_request_poll() * for poll completed mcdi requests, and also with * efx_mcdi_request_start() for post-watchdog completions. */ EFSYS_LOCK(enp->en_eslp, state); emrp = emip->emi_pending_req; ev_cpl = emip->emi_ev_cpl; if (emrp != NULL && emip->emi_ev_cpl) { emip->emi_pending_req = NULL; emrp->emr_out_length_used = 0; emrp->emr_rc = rc; ++emip->emi_aborted; } /* * Since we're running in parallel with a request, consume the * status word before dropping the lock. */ if (rc == EIO || rc == EINTR) { EFSYS_SPIN(EFX_MCDI_STATUS_SLEEP_US); (void) efx_mcdi_poll_reboot(enp); emip->emi_new_epoch = B_TRUE; } EFSYS_UNLOCK(enp->en_eslp, state); efx_mcdi_raise_exception(enp, emrp, rc); if (emrp != NULL && ev_cpl) emtp->emt_ev_cpl(emtp->emt_context); } __checkReturn efx_rc_t efx_mcdi_version( __in efx_nic_t *enp, __out_ecount_opt(4) uint16_t versionp[4], __out_opt uint32_t *buildp, __out_opt efx_mcdi_boot_t *statusp) { efx_mcdi_req_t req; uint8_t payload[MAX(MAX(MC_CMD_GET_VERSION_IN_LEN, MC_CMD_GET_VERSION_OUT_LEN), MAX(MC_CMD_GET_BOOT_STATUS_IN_LEN, MC_CMD_GET_BOOT_STATUS_OUT_LEN))]; efx_word_t *ver_words; uint16_t version[4]; uint32_t build; efx_mcdi_boot_t status; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_features, &, EFX_FEATURE_MCDI); (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_VERSION; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_VERSION_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_VERSION_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } /* bootrom support */ if (req.emr_out_length_used == MC_CMD_GET_VERSION_V0_OUT_LEN) { version[0] = version[1] = version[2] = version[3] = 0; build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE); goto version; } if (req.emr_out_length_used < MC_CMD_GET_VERSION_OUT_LEN) { rc = EMSGSIZE; goto fail2; } ver_words = MCDI_OUT2(req, efx_word_t, GET_VERSION_OUT_VERSION); version[0] = EFX_WORD_FIELD(ver_words[0], EFX_WORD_0); version[1] = EFX_WORD_FIELD(ver_words[1], EFX_WORD_0); version[2] = EFX_WORD_FIELD(ver_words[2], EFX_WORD_0); version[3] = EFX_WORD_FIELD(ver_words[3], EFX_WORD_0); build = MCDI_OUT_DWORD(req, GET_VERSION_OUT_FIRMWARE); version: /* The bootrom doesn't understand BOOT_STATUS */ if (MC_FW_VERSION_IS_BOOTLOADER(build)) { status = EFX_MCDI_BOOT_ROM; goto out; } (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_BOOT_STATUS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_BOOT_STATUS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_BOOT_STATUS_OUT_LEN; efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc == EACCES) { /* Unprivileged functions cannot access BOOT_STATUS */ status = EFX_MCDI_BOOT_PRIMARY; version[0] = version[1] = version[2] = version[3] = 0; build = 0; goto out; } if (req.emr_rc != 0) { rc = req.emr_rc; goto fail3; } if (req.emr_out_length_used < MC_CMD_GET_BOOT_STATUS_OUT_LEN) { rc = EMSGSIZE; goto fail4; } if (MCDI_OUT_DWORD_FIELD(req, GET_BOOT_STATUS_OUT_FLAGS, GET_BOOT_STATUS_OUT_FLAGS_PRIMARY)) status = EFX_MCDI_BOOT_PRIMARY; else status = EFX_MCDI_BOOT_SECONDARY; out: if (versionp != NULL) memcpy(versionp, version, sizeof (version)); if (buildp != NULL) *buildp = build; if (statusp != NULL) *statusp = status; return (0); fail4: EFSYS_PROBE(fail4); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_get_capabilities( __in efx_nic_t *enp, __out_opt uint32_t *flagsp, __out_opt uint16_t *rx_dpcpu_fw_idp, __out_opt uint16_t *tx_dpcpu_fw_idp, __out_opt uint32_t *flags2p, __out_opt uint32_t *tso2ncp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_CAPABILITIES_IN_LEN, MC_CMD_GET_CAPABILITIES_V2_OUT_LEN)]; boolean_t v2_capable; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_CAPABILITIES; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_CAPABILITIES_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_CAPABILITIES_V2_OUT_LEN; efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_CAPABILITIES_OUT_LEN) { rc = EMSGSIZE; goto fail2; } if (flagsp != NULL) *flagsp = MCDI_OUT_DWORD(req, GET_CAPABILITIES_OUT_FLAGS1); if (rx_dpcpu_fw_idp != NULL) *rx_dpcpu_fw_idp = MCDI_OUT_WORD(req, GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID); if (tx_dpcpu_fw_idp != NULL) *tx_dpcpu_fw_idp = MCDI_OUT_WORD(req, GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID); if (req.emr_out_length_used < MC_CMD_GET_CAPABILITIES_V2_OUT_LEN) v2_capable = B_FALSE; else v2_capable = B_TRUE; if (flags2p != NULL) { *flags2p = (v2_capable) ? MCDI_OUT_DWORD(req, GET_CAPABILITIES_V2_OUT_FLAGS2) : 0; } if (tso2ncp != NULL) { *tso2ncp = (v2_capable) ? MCDI_OUT_WORD(req, GET_CAPABILITIES_V2_OUT_TX_TSO_V2_N_CONTEXTS) : 0; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn efx_rc_t efx_mcdi_do_reboot( __in efx_nic_t *enp, __in boolean_t after_assertion) { uint8_t payload[MAX(MC_CMD_REBOOT_IN_LEN, MC_CMD_REBOOT_OUT_LEN)]; efx_mcdi_req_t req; efx_rc_t rc; /* * We could require the caller to have caused en_mod_flags=0 to * call this function. This doesn't help the other port though, * who's about to get the MC ripped out from underneath them. * Since they have to cope with the subsequent fallout of MCDI * failures, we should as well. */ EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_REBOOT; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_REBOOT_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_REBOOT_OUT_LEN; MCDI_IN_SET_DWORD(req, REBOOT_IN_FLAGS, (after_assertion ? MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION : 0)); efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc == EACCES) { /* Unprivileged functions cannot reboot the MC. */ goto out; } /* A successful reboot request returns EIO. */ if (req.emr_rc != 0 && req.emr_rc != EIO) { rc = req.emr_rc; goto fail1; } out: return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_reboot( __in efx_nic_t *enp) { return (efx_mcdi_do_reboot(enp, B_FALSE)); } __checkReturn efx_rc_t efx_mcdi_exit_assertion_handler( __in efx_nic_t *enp) { return (efx_mcdi_do_reboot(enp, B_TRUE)); } __checkReturn efx_rc_t efx_mcdi_read_assertion( __in efx_nic_t *enp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_ASSERTS_IN_LEN, MC_CMD_GET_ASSERTS_OUT_LEN)]; const char *reason; unsigned int flags; unsigned int index; unsigned int ofst; int retry; efx_rc_t rc; /* * Before we attempt to chat to the MC, we should verify that the MC * isn't in its assertion handler, either due to a previous reboot, * or because we're reinitializing due to an eec_exception(). * * Use GET_ASSERTS to read any assertion state that may be present. * Retry this command twice. Once because a boot-time assertion failure * might cause the 1st MCDI request to fail. And once again because * we might race with efx_mcdi_exit_assertion_handler() running on * partner port(s) on the same NIC. */ retry = 2; do { (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_ASSERTS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_ASSERTS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_ASSERTS_OUT_LEN; MCDI_IN_SET_DWORD(req, GET_ASSERTS_IN_CLEAR, 1); efx_mcdi_execute_quiet(enp, &req); } while ((req.emr_rc == EINTR || req.emr_rc == EIO) && retry-- > 0); if (req.emr_rc != 0) { if (req.emr_rc == EACCES) { /* Unprivileged functions cannot clear assertions. */ goto out; } rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_ASSERTS_OUT_LEN) { rc = EMSGSIZE; goto fail2; } /* Print out any assertion state recorded */ flags = MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_GLOBAL_FLAGS); if (flags == MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS) return (0); reason = (flags == MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL) ? "system-level assertion" : (flags == MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL) ? "thread-level assertion" : (flags == MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED) ? "watchdog reset" : (flags == MC_CMD_GET_ASSERTS_FLAGS_ADDR_TRAP) ? "illegal address trap" : "unknown assertion"; EFSYS_PROBE3(mcpu_assertion, const char *, reason, unsigned int, MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_SAVED_PC_OFFS), unsigned int, MCDI_OUT_DWORD(req, GET_ASSERTS_OUT_THREAD_OFFS)); /* Print out the registers (r1 ... r31) */ ofst = MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST; for (index = 1; index < 1 + MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_NUM; index++) { EFSYS_PROBE2(mcpu_register, unsigned int, index, unsigned int, EFX_DWORD_FIELD(*MCDI_OUT(req, efx_dword_t, ofst), EFX_DWORD_0)); ofst += sizeof (efx_dword_t); } EFSYS_ASSERT(ofst <= MC_CMD_GET_ASSERTS_OUT_LEN); out: return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } /* * Internal routines for for specific MCDI requests. */ __checkReturn efx_rc_t efx_mcdi_drv_attach( __in efx_nic_t *enp, __in boolean_t attach) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_DRV_ATTACH_IN_LEN, MC_CMD_DRV_ATTACH_EXT_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_DRV_ATTACH; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_DRV_ATTACH_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_DRV_ATTACH_EXT_OUT_LEN; /* * Use DONT_CARE for the datapath firmware type to ensure that the * driver can attach to an unprivileged function. The datapath firmware * type to use is controlled by the 'sfboot' utility. */ MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_NEW_STATE, attach ? 1 : 0); MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_UPDATE, 1); MCDI_IN_SET_DWORD(req, DRV_ATTACH_IN_FIRMWARE_ID, MC_CMD_FW_DONT_CARE); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_DRV_ATTACH_OUT_LEN) { rc = EMSGSIZE; goto fail2; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_get_board_cfg( __in efx_nic_t *enp, __out_opt uint32_t *board_typep, __out_opt efx_dword_t *capabilitiesp, __out_ecount_opt(6) uint8_t mac_addrp[6]) { efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip); efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_BOARD_CFG_IN_LEN, MC_CMD_GET_BOARD_CFG_OUT_LENMIN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_BOARD_CFG; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_BOARD_CFG_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_BOARD_CFG_OUT_LENMIN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_BOARD_CFG_OUT_LENMIN) { rc = EMSGSIZE; goto fail2; } if (mac_addrp != NULL) { uint8_t *addrp; if (emip->emi_port == 1) { addrp = MCDI_OUT2(req, uint8_t, GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0); } else if (emip->emi_port == 2) { addrp = MCDI_OUT2(req, uint8_t, GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1); } else { rc = EINVAL; goto fail3; } EFX_MAC_ADDR_COPY(mac_addrp, addrp); } if (capabilitiesp != NULL) { if (emip->emi_port == 1) { *capabilitiesp = *MCDI_OUT2(req, efx_dword_t, GET_BOARD_CFG_OUT_CAPABILITIES_PORT0); } else if (emip->emi_port == 2) { *capabilitiesp = *MCDI_OUT2(req, efx_dword_t, GET_BOARD_CFG_OUT_CAPABILITIES_PORT1); } else { rc = EINVAL; goto fail4; } } if (board_typep != NULL) { *board_typep = MCDI_OUT_DWORD(req, GET_BOARD_CFG_OUT_BOARD_TYPE); } return (0); fail4: EFSYS_PROBE(fail4); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_get_resource_limits( __in efx_nic_t *enp, __out_opt uint32_t *nevqp, __out_opt uint32_t *nrxqp, __out_opt uint32_t *ntxqp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_RESOURCE_LIMITS_IN_LEN, MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_RESOURCE_LIMITS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_RESOURCE_LIMITS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN) { rc = EMSGSIZE; goto fail2; } if (nevqp != NULL) *nevqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_EVQ); if (nrxqp != NULL) *nrxqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_RXQ); if (ntxqp != NULL) *ntxqp = MCDI_OUT_DWORD(req, GET_RESOURCE_LIMITS_OUT_TXQ); return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_get_phy_cfg( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_PHY_CFG_IN_LEN, MC_CMD_GET_PHY_CFG_OUT_LEN)]; #if EFSYS_OPT_NAMES const char *namep; size_t namelen; #endif uint32_t phy_media_type; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_PHY_CFG; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_PHY_CFG_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_PHY_CFG_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_PHY_CFG_OUT_LEN) { rc = EMSGSIZE; goto fail2; } encp->enc_phy_type = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_TYPE); #if EFSYS_OPT_NAMES namep = MCDI_OUT2(req, char, GET_PHY_CFG_OUT_NAME); namelen = MIN(sizeof (encp->enc_phy_name) - 1, strnlen(namep, MC_CMD_GET_PHY_CFG_OUT_NAME_LEN)); (void) memset(encp->enc_phy_name, 0, sizeof (encp->enc_phy_name)); memcpy(encp->enc_phy_name, namep, namelen); #endif /* EFSYS_OPT_NAMES */ (void) memset(encp->enc_phy_revision, 0, sizeof (encp->enc_phy_revision)); memcpy(encp->enc_phy_revision, MCDI_OUT2(req, char, GET_PHY_CFG_OUT_REVISION), MIN(sizeof (encp->enc_phy_revision) - 1, MC_CMD_GET_PHY_CFG_OUT_REVISION_LEN)); #if EFSYS_OPT_PHY_LED_CONTROL encp->enc_led_mask = ((1 << EFX_PHY_LED_DEFAULT) | (1 << EFX_PHY_LED_OFF) | (1 << EFX_PHY_LED_ON)); #endif /* EFSYS_OPT_PHY_LED_CONTROL */ /* Get the media type of the fixed port, if recognised. */ EFX_STATIC_ASSERT(MC_CMD_MEDIA_XAUI == EFX_PHY_MEDIA_XAUI); EFX_STATIC_ASSERT(MC_CMD_MEDIA_CX4 == EFX_PHY_MEDIA_CX4); EFX_STATIC_ASSERT(MC_CMD_MEDIA_KX4 == EFX_PHY_MEDIA_KX4); EFX_STATIC_ASSERT(MC_CMD_MEDIA_XFP == EFX_PHY_MEDIA_XFP); EFX_STATIC_ASSERT(MC_CMD_MEDIA_SFP_PLUS == EFX_PHY_MEDIA_SFP_PLUS); EFX_STATIC_ASSERT(MC_CMD_MEDIA_BASE_T == EFX_PHY_MEDIA_BASE_T); EFX_STATIC_ASSERT(MC_CMD_MEDIA_QSFP_PLUS == EFX_PHY_MEDIA_QSFP_PLUS); phy_media_type = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_MEDIA_TYPE); epp->ep_fixed_port_type = (efx_phy_media_type_t) phy_media_type; if (epp->ep_fixed_port_type >= EFX_PHY_MEDIA_NTYPES) epp->ep_fixed_port_type = EFX_PHY_MEDIA_INVALID; epp->ep_phy_cap_mask = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_SUPPORTED_CAP); #if EFSYS_OPT_PHY_FLAGS encp->enc_phy_flags_mask = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_FLAGS); #endif /* EFSYS_OPT_PHY_FLAGS */ encp->enc_port = (uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_PRT); /* Populate internal state */ encp->enc_mcdi_mdio_channel = (uint8_t)MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_CHANNEL); #if EFSYS_OPT_PHY_STATS encp->enc_mcdi_phy_stat_mask = MCDI_OUT_DWORD(req, GET_PHY_CFG_OUT_STATS_MASK); #endif /* EFSYS_OPT_PHY_STATS */ #if EFSYS_OPT_BIST encp->enc_bist_mask = 0; if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, GET_PHY_CFG_OUT_BIST_CABLE_SHORT)) encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_CABLE_SHORT); if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, GET_PHY_CFG_OUT_BIST_CABLE_LONG)) encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_CABLE_LONG); if (MCDI_OUT_DWORD_FIELD(req, GET_PHY_CFG_OUT_FLAGS, GET_PHY_CFG_OUT_BIST)) encp->enc_bist_mask |= (1 << EFX_BIST_TYPE_PHY_NORMAL); #endif /* EFSYS_OPT_BIST */ return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_firmware_update_supported( __in efx_nic_t *enp, __out boolean_t *supportedp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; efx_rc_t rc; if (emcop != NULL) { if ((rc = emcop->emco_feature_supported(enp, EFX_MCDI_FEATURE_FW_UPDATE, supportedp)) != 0) goto fail1; } else { /* Earlier devices always supported updates */ *supportedp = B_TRUE; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_macaddr_change_supported( __in efx_nic_t *enp, __out boolean_t *supportedp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; efx_rc_t rc; if (emcop != NULL) { if ((rc = emcop->emco_feature_supported(enp, EFX_MCDI_FEATURE_MACADDR_CHANGE, supportedp)) != 0) goto fail1; } else { /* Earlier devices always supported MAC changes */ *supportedp = B_TRUE; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_link_control_supported( __in efx_nic_t *enp, __out boolean_t *supportedp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; efx_rc_t rc; if (emcop != NULL) { if ((rc = emcop->emco_feature_supported(enp, EFX_MCDI_FEATURE_LINK_CONTROL, supportedp)) != 0) goto fail1; } else { /* Earlier devices always supported link control */ *supportedp = B_TRUE; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_mac_spoofing_supported( __in efx_nic_t *enp, __out boolean_t *supportedp) { const efx_mcdi_ops_t *emcop = enp->en_mcdi.em_emcop; efx_rc_t rc; if (emcop != NULL) { if ((rc = emcop->emco_feature_supported(enp, EFX_MCDI_FEATURE_MAC_SPOOFING, supportedp)) != 0) goto fail1; } else { /* Earlier devices always supported MAC spoofing */ *supportedp = B_TRUE; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_BIST #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 /* * Enter bist offline mode. This is a fw mode which puts the NIC into a state * where memory BIST tests can be run and not much else can interfere or happen. * A reboot is required to exit this mode. */ __checkReturn efx_rc_t efx_mcdi_bist_enable_offline( __in efx_nic_t *enp) { efx_mcdi_req_t req; efx_rc_t rc; EFX_STATIC_ASSERT(MC_CMD_ENABLE_OFFLINE_BIST_IN_LEN == 0); EFX_STATIC_ASSERT(MC_CMD_ENABLE_OFFLINE_BIST_OUT_LEN == 0); req.emr_cmd = MC_CMD_ENABLE_OFFLINE_BIST; req.emr_in_buf = NULL; req.emr_in_length = 0; req.emr_out_buf = NULL; req.emr_out_length = 0; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ __checkReturn efx_rc_t efx_mcdi_bist_start( __in efx_nic_t *enp, __in efx_bist_type_t type) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_START_BIST_IN_LEN, MC_CMD_START_BIST_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_START_BIST; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_START_BIST_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_START_BIST_OUT_LEN; switch (type) { case EFX_BIST_TYPE_PHY_NORMAL: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PHY_BIST); break; case EFX_BIST_TYPE_PHY_CABLE_SHORT: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PHY_BIST_CABLE_SHORT); break; case EFX_BIST_TYPE_PHY_CABLE_LONG: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PHY_BIST_CABLE_LONG); break; case EFX_BIST_TYPE_MC_MEM: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_MC_MEM_BIST); break; case EFX_BIST_TYPE_SAT_MEM: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_PORT_MEM_BIST); break; case EFX_BIST_TYPE_REG: MCDI_IN_SET_DWORD(req, START_BIST_IN_TYPE, MC_CMD_REG_BIST); break; default: EFSYS_ASSERT(0); } efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_BIST */ /* Enable logging of some events (e.g. link state changes) */ __checkReturn efx_rc_t efx_mcdi_log_ctrl( __in efx_nic_t *enp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_LOG_CTRL_IN_LEN, MC_CMD_LOG_CTRL_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_LOG_CTRL; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_LOG_CTRL_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_LOG_CTRL_OUT_LEN; MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST, MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ); MCDI_IN_SET_DWORD(req, LOG_CTRL_IN_LOG_DEST_EVQ, 0); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_MAC_STATS typedef enum efx_stats_action_e { EFX_STATS_CLEAR, EFX_STATS_UPLOAD, EFX_STATS_ENABLE_NOEVENTS, EFX_STATS_ENABLE_EVENTS, EFX_STATS_DISABLE, } efx_stats_action_t; static __checkReturn efx_rc_t efx_mcdi_mac_stats( __in efx_nic_t *enp, __in_opt efsys_mem_t *esmp, __in efx_stats_action_t action, __in uint16_t period_ms) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_MAC_STATS_IN_LEN, MC_CMD_MAC_STATS_OUT_DMA_LEN)]; int clear = (action == EFX_STATS_CLEAR); int upload = (action == EFX_STATS_UPLOAD); int enable = (action == EFX_STATS_ENABLE_NOEVENTS); int events = (action == EFX_STATS_ENABLE_EVENTS); int disable = (action == EFX_STATS_DISABLE); efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_MAC_STATS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_MAC_STATS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_MAC_STATS_OUT_DMA_LEN; MCDI_IN_POPULATE_DWORD_6(req, MAC_STATS_IN_CMD, MAC_STATS_IN_DMA, upload, MAC_STATS_IN_CLEAR, clear, MAC_STATS_IN_PERIODIC_CHANGE, enable | events | disable, MAC_STATS_IN_PERIODIC_ENABLE, enable | events, MAC_STATS_IN_PERIODIC_NOEVENT, !events, MAC_STATS_IN_PERIOD_MS, (enable | events) ? period_ms : 0); if (esmp != NULL) { - int bytes = MC_CMD_MAC_NSTATS * sizeof (uint64_t); + uint32_t bytes = MC_CMD_MAC_NSTATS * sizeof (uint64_t); EFX_STATIC_ASSERT(MC_CMD_MAC_NSTATS * sizeof (uint64_t) <= EFX_MAC_STATS_SIZE); + + EFSYS_ASSERT3U(bytes, <=, (uint32_t)EFSYS_MEM_SIZE(esmp)); MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_LO, EFSYS_MEM_ADDR(esmp) & 0xffffffff); MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_ADDR_HI, EFSYS_MEM_ADDR(esmp) >> 32); MCDI_IN_SET_DWORD(req, MAC_STATS_IN_DMA_LEN, bytes); } else { EFSYS_ASSERT(!upload && !enable && !events); } /* * NOTE: Do not use EVB_PORT_ID_ASSIGNED when disabling periodic stats, * as this may fail (and leave periodic DMA enabled) if the * vadapter has already been deleted. */ MCDI_IN_SET_DWORD(req, MAC_STATS_IN_PORT_ID, (disable ? EVB_PORT_ID_NULL : enp->en_vport_id)); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { /* EF10: Expect ENOENT if no DMA queues are initialised */ if ((req.emr_rc != ENOENT) || (enp->en_rx_qcount + enp->en_tx_qcount != 0)) { rc = req.emr_rc; goto fail1; } } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_mac_stats_clear( __in efx_nic_t *enp) { efx_rc_t rc; if ((rc = efx_mcdi_mac_stats(enp, NULL, EFX_STATS_CLEAR, 0)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_mac_stats_upload( __in efx_nic_t *enp, __in efsys_mem_t *esmp) { efx_rc_t rc; /* * The MC DMAs aggregate statistics for our convenience, so we can * avoid having to pull the statistics buffer into the cache to * maintain cumulative statistics. */ if ((rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_UPLOAD, 0)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_mac_stats_periodic( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __in uint16_t period_ms, __in boolean_t events) { efx_rc_t rc; /* * The MC DMAs aggregate statistics for our convenience, so we can * avoid having to pull the statistics buffer into the cache to * maintain cumulative statistics. * Huntington uses a fixed 1sec period. * Medford uses a fixed 1sec period before v6.2.1.1033 firmware. */ if (period_ms == 0) rc = efx_mcdi_mac_stats(enp, NULL, EFX_STATS_DISABLE, 0); else if (events) rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_ENABLE_EVENTS, period_ms); else rc = efx_mcdi_mac_stats(enp, esmp, EFX_STATS_ENABLE_NOEVENTS, period_ms); if (rc != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_MAC_STATS */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 /* * This function returns the pf and vf number of a function. If it is a pf the * vf number is 0xffff. The vf number is the index of the vf on that * function. So if you have 3 vfs on pf 0 the 3 vfs will return (pf=0,vf=0), * (pf=0,vf=1), (pf=0,vf=2) aand the pf will return (pf=0, vf=0xffff). */ __checkReturn efx_rc_t efx_mcdi_get_function_info( __in efx_nic_t *enp, __out uint32_t *pfp, __out_opt uint32_t *vfp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_FUNCTION_INFO_IN_LEN, MC_CMD_GET_FUNCTION_INFO_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_FUNCTION_INFO; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_FUNCTION_INFO_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_FUNCTION_INFO_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_FUNCTION_INFO_OUT_LEN) { rc = EMSGSIZE; goto fail2; } *pfp = MCDI_OUT_DWORD(req, GET_FUNCTION_INFO_OUT_PF); if (vfp != NULL) *vfp = MCDI_OUT_DWORD(req, GET_FUNCTION_INFO_OUT_VF); return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_privilege_mask( __in efx_nic_t *enp, __in uint32_t pf, __in uint32_t vf, __out uint32_t *maskp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_PRIVILEGE_MASK_IN_LEN, MC_CMD_PRIVILEGE_MASK_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_PRIVILEGE_MASK; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_PRIVILEGE_MASK_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_PRIVILEGE_MASK_OUT_LEN; MCDI_IN_POPULATE_DWORD_2(req, PRIVILEGE_MASK_IN_FUNCTION, PRIVILEGE_MASK_IN_FUNCTION_PF, pf, PRIVILEGE_MASK_IN_FUNCTION_VF, vf); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_PRIVILEGE_MASK_OUT_LEN) { rc = EMSGSIZE; goto fail2; } *maskp = MCDI_OUT_DWORD(req, PRIVILEGE_MASK_OUT_OLD_MASK); return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */ __checkReturn efx_rc_t efx_mcdi_set_workaround( __in efx_nic_t *enp, __in uint32_t type, __in boolean_t enabled, __out_opt uint32_t *flagsp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_WORKAROUND_IN_LEN, MC_CMD_WORKAROUND_EXT_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_WORKAROUND; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_WORKAROUND_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_WORKAROUND_OUT_LEN; MCDI_IN_SET_DWORD(req, WORKAROUND_IN_TYPE, type); MCDI_IN_SET_DWORD(req, WORKAROUND_IN_ENABLED, enabled ? 1 : 0); efx_mcdi_execute_quiet(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (flagsp != NULL) { if (req.emr_out_length_used >= MC_CMD_WORKAROUND_EXT_OUT_LEN) *flagsp = MCDI_OUT_DWORD(req, WORKAROUND_EXT_OUT_FLAGS); else *flagsp = 0; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mcdi_get_workarounds( __in efx_nic_t *enp, __out_opt uint32_t *implementedp, __out_opt uint32_t *enabledp) { efx_mcdi_req_t req; uint8_t payload[MC_CMD_GET_WORKAROUNDS_OUT_LEN]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_WORKAROUNDS; req.emr_in_buf = NULL; req.emr_in_length = 0; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_WORKAROUNDS_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (implementedp != NULL) { *implementedp = MCDI_OUT_DWORD(req, GET_WORKAROUNDS_OUT_IMPLEMENTED); } if (enabledp != NULL) { *enabledp = MCDI_OUT_DWORD(req, GET_WORKAROUNDS_OUT_ENABLED); } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } /* * Size of media information page in accordance with SFF-8472 and SFF-8436. * It is used in MCDI interface as well. */ #define EFX_PHY_MEDIA_INFO_PAGE_SIZE 0x80 static __checkReturn efx_rc_t efx_mcdi_get_phy_media_info( __in efx_nic_t *enp, __in uint32_t mcdi_page, __in uint8_t offset, __in uint8_t len, __out_bcount(len) uint8_t *data) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN, MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN( EFX_PHY_MEDIA_INFO_PAGE_SIZE))]; efx_rc_t rc; EFSYS_ASSERT((uint32_t)offset + len <= EFX_PHY_MEDIA_INFO_PAGE_SIZE); (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_PHY_MEDIA_INFO; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(EFX_PHY_MEDIA_INFO_PAGE_SIZE); MCDI_IN_SET_DWORD(req, GET_PHY_MEDIA_INFO_IN_PAGE, mcdi_page); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used != MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(EFX_PHY_MEDIA_INFO_PAGE_SIZE)) { rc = EMSGSIZE; goto fail2; } if (MCDI_OUT_DWORD(req, GET_PHY_MEDIA_INFO_OUT_DATALEN) != EFX_PHY_MEDIA_INFO_PAGE_SIZE) { rc = EIO; goto fail3; } memcpy(data, MCDI_OUT2(req, uint8_t, GET_PHY_MEDIA_INFO_OUT_DATA) + offset, len); return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } /* * 2-wire device address of the base information in accordance with SFF-8472 * Diagnostic Monitoring Interface for Optical Transceivers section * 4 Memory Organization. */ #define EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE 0xA0 /* * 2-wire device address of the digital diagnostics monitoring interface * in accordance with SFF-8472 Diagnostic Monitoring Interface for Optical * Transceivers section 4 Memory Organization. */ #define EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM 0xA2 /* * Hard wired 2-wire device address for QSFP+ in accordance with SFF-8436 * QSFP+ 10 Gbs 4X PLUGGABLE TRANSCEIVER section 7.4 Device Addressing and * Operation. */ #define EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP 0xA0 __checkReturn efx_rc_t efx_mcdi_phy_module_get_info( __in efx_nic_t *enp, __in uint8_t dev_addr, __in uint8_t offset, __in uint8_t len, __out_bcount(len) uint8_t *data) { efx_port_t *epp = &(enp->en_port); efx_rc_t rc; uint32_t mcdi_lower_page; uint32_t mcdi_upper_page; EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); /* * Map device address to MC_CMD_GET_PHY_MEDIA_INFO pages. * Offset plus length interface allows to access page 0 only. * I.e. non-zero upper pages are not accessible. * See SFF-8472 section 4 Memory Organization and SFF-8436 section 7.6 * QSFP+ Memory Map for details on how information is structured * and accessible. */ switch (epp->ep_fixed_port_type) { case EFX_PHY_MEDIA_SFP_PLUS: /* * In accordance with SFF-8472 Diagnostic Monitoring * Interface for Optical Transceivers section 4 Memory * Organization two 2-wire addresses are defined. */ switch (dev_addr) { /* Base information */ case EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_BASE: /* * MCDI page 0 should be used to access lower * page 0 (0x00 - 0x7f) at the device address 0xA0. */ mcdi_lower_page = 0; /* * MCDI page 1 should be used to access upper * page 0 (0x80 - 0xff) at the device address 0xA0. */ mcdi_upper_page = 1; break; /* Diagnostics */ case EFX_PHY_MEDIA_INFO_DEV_ADDR_SFP_DDM: /* * MCDI page 2 should be used to access lower * page 0 (0x00 - 0x7f) at the device address 0xA2. */ mcdi_lower_page = 2; /* * MCDI page 3 should be used to access upper * page 0 (0x80 - 0xff) at the device address 0xA2. */ mcdi_upper_page = 3; break; default: rc = ENOTSUP; goto fail1; } break; case EFX_PHY_MEDIA_QSFP_PLUS: switch (dev_addr) { case EFX_PHY_MEDIA_INFO_DEV_ADDR_QSFP: /* * MCDI page -1 should be used to access lower page 0 * (0x00 - 0x7f). */ mcdi_lower_page = (uint32_t)-1; /* * MCDI page 0 should be used to access upper page 0 * (0x80h - 0xff). */ mcdi_upper_page = 0; break; default: rc = ENOTSUP; goto fail1; } break; default: rc = ENOTSUP; goto fail1; } if (offset < EFX_PHY_MEDIA_INFO_PAGE_SIZE) { uint8_t read_len = MIN(len, EFX_PHY_MEDIA_INFO_PAGE_SIZE - offset); rc = efx_mcdi_get_phy_media_info(enp, mcdi_lower_page, offset, read_len, data); if (rc != 0) goto fail2; data += read_len; len -= read_len; offset = 0; } else { offset -= EFX_PHY_MEDIA_INFO_PAGE_SIZE; } if (len > 0) { EFSYS_ASSERT3U(len, <=, EFX_PHY_MEDIA_INFO_PAGE_SIZE); EFSYS_ASSERT3U(offset, <, EFX_PHY_MEDIA_INFO_PAGE_SIZE); rc = efx_mcdi_get_phy_media_info(enp, mcdi_upper_page, offset, len, data); if (rc != 0) goto fail3; } return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_MCDI */ Index: head/sys/dev/sfxge/common/siena_phy.c =================================================================== --- head/sys/dev/sfxge/common/siena_phy.c (revision 341050) +++ head/sys/dev/sfxge/common/siena_phy.c (revision 341051) @@ -1,804 +1,811 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_SIENA static void siena_phy_decode_cap( __in uint32_t mcdi_cap, __out uint32_t *maskp) { uint32_t mask; mask = 0; if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN)) mask |= (1 << EFX_PHY_CAP_10HDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10FDX_LBN)) mask |= (1 << EFX_PHY_CAP_10FDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100HDX_LBN)) mask |= (1 << EFX_PHY_CAP_100HDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_100FDX_LBN)) mask |= (1 << EFX_PHY_CAP_100FDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000HDX_LBN)) mask |= (1 << EFX_PHY_CAP_1000HDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN)) mask |= (1 << EFX_PHY_CAP_1000FDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN)) mask |= (1 << EFX_PHY_CAP_10000FDX); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_PAUSE_LBN)) mask |= (1 << EFX_PHY_CAP_PAUSE); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_ASYM_LBN)) mask |= (1 << EFX_PHY_CAP_ASYM); if (mcdi_cap & (1 << MC_CMD_PHY_CAP_AN_LBN)) mask |= (1 << EFX_PHY_CAP_AN); *maskp = mask; } static void siena_phy_decode_link_mode( __in efx_nic_t *enp, __in uint32_t link_flags, __in unsigned int speed, __in unsigned int fcntl, __out efx_link_mode_t *link_modep, __out unsigned int *fcntlp) { boolean_t fd = !!(link_flags & (1 << MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN)); boolean_t up = !!(link_flags & (1 << MC_CMD_GET_LINK_OUT_LINK_UP_LBN)); _NOTE(ARGUNUSED(enp)) if (!up) *link_modep = EFX_LINK_DOWN; else if (speed == 10000 && fd) *link_modep = EFX_LINK_10000FDX; else if (speed == 1000) *link_modep = fd ? EFX_LINK_1000FDX : EFX_LINK_1000HDX; else if (speed == 100) *link_modep = fd ? EFX_LINK_100FDX : EFX_LINK_100HDX; else if (speed == 10) *link_modep = fd ? EFX_LINK_10FDX : EFX_LINK_10HDX; else *link_modep = EFX_LINK_UNKNOWN; if (fcntl == MC_CMD_FCNTL_OFF) *fcntlp = 0; else if (fcntl == MC_CMD_FCNTL_RESPOND) *fcntlp = EFX_FCNTL_RESPOND; else if (fcntl == MC_CMD_FCNTL_BIDIR) *fcntlp = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; else { EFSYS_PROBE1(mc_pcol_error, int, fcntl); *fcntlp = 0; } } void siena_phy_link_ev( __in efx_nic_t *enp, __in efx_qword_t *eqp, __out efx_link_mode_t *link_modep) { efx_port_t *epp = &(enp->en_port); unsigned int link_flags; unsigned int speed; unsigned int fcntl; efx_link_mode_t link_mode; uint32_t lp_cap_mask; /* * Convert the LINKCHANGE speed enumeration into mbit/s, in the * same way as GET_LINK encodes the speed */ switch (MCDI_EV_FIELD(eqp, LINKCHANGE_SPEED)) { case MCDI_EVENT_LINKCHANGE_SPEED_100M: speed = 100; break; case MCDI_EVENT_LINKCHANGE_SPEED_1G: speed = 1000; break; case MCDI_EVENT_LINKCHANGE_SPEED_10G: speed = 10000; break; default: speed = 0; break; } link_flags = MCDI_EV_FIELD(eqp, LINKCHANGE_LINK_FLAGS); siena_phy_decode_link_mode(enp, link_flags, speed, MCDI_EV_FIELD(eqp, LINKCHANGE_FCNTL), &link_mode, &fcntl); siena_phy_decode_cap(MCDI_EV_FIELD(eqp, LINKCHANGE_LP_CAP), &lp_cap_mask); /* * It's safe to update ep_lp_cap_mask without the driver's port lock * because presumably any concurrently running efx_port_poll() is * only going to arrive at the same value. * * ep_fcntl has two meanings. It's either the link common fcntl * (if the PHY supports AN), or it's the forced link state. If * the former, it's safe to update the value for the same reason as * for ep_lp_cap_mask. If the latter, then just ignore the value, * because we can race with efx_mac_fcntl_set(). */ epp->ep_lp_cap_mask = lp_cap_mask; if (epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) epp->ep_fcntl = fcntl; *link_modep = link_mode; } __checkReturn efx_rc_t siena_phy_power( __in efx_nic_t *enp, __in boolean_t power) { efx_rc_t rc; if (!power) return (0); /* Check if the PHY is a zombie */ if ((rc = siena_phy_verify(enp)) != 0) goto fail1; enp->en_reset_flags |= EFX_RESET_PHY; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t siena_phy_get_link( __in efx_nic_t *enp, __out siena_link_state_t *slsp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_LINK_IN_LEN, MC_CMD_GET_LINK_OUT_LEN)]; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_LINK; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_LINK_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_LINK_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_LINK_OUT_LEN) { rc = EMSGSIZE; goto fail2; } siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_CAP), &slsp->sls_adv_cap_mask); siena_phy_decode_cap(MCDI_OUT_DWORD(req, GET_LINK_OUT_LP_CAP), &slsp->sls_lp_cap_mask); siena_phy_decode_link_mode(enp, MCDI_OUT_DWORD(req, GET_LINK_OUT_FLAGS), MCDI_OUT_DWORD(req, GET_LINK_OUT_LINK_SPEED), MCDI_OUT_DWORD(req, GET_LINK_OUT_FCNTL), &slsp->sls_link_mode, &slsp->sls_fcntl); #if EFSYS_OPT_LOOPBACK /* Assert the MC_CMD_LOOPBACK and EFX_LOOPBACK namespace agree */ EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_NONE == EFX_LOOPBACK_OFF); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_DATA == EFX_LOOPBACK_DATA); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMAC == EFX_LOOPBACK_GMAC); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGMII == EFX_LOOPBACK_XGMII); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGXS == EFX_LOOPBACK_XGXS); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI == EFX_LOOPBACK_XAUI); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII == EFX_LOOPBACK_GMII); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII == EFX_LOOPBACK_SGMII); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XGBR == EFX_LOOPBACK_XGBR); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI == EFX_LOOPBACK_XFI); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XAUI_FAR == EFX_LOOPBACK_XAUI_FAR); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GMII_FAR == EFX_LOOPBACK_GMII_FAR); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_SGMII_FAR == EFX_LOOPBACK_SGMII_FAR); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_XFI_FAR == EFX_LOOPBACK_XFI_FAR); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_GPHY == EFX_LOOPBACK_GPHY); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PHYXS == EFX_LOOPBACK_PHY_XS); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PCS == EFX_LOOPBACK_PCS); EFX_STATIC_ASSERT(MC_CMD_LOOPBACK_PMAPMD == EFX_LOOPBACK_PMA_PMD); slsp->sls_loopback = MCDI_OUT_DWORD(req, GET_LINK_OUT_LOOPBACK_MODE); #endif /* EFSYS_OPT_LOOPBACK */ slsp->sls_mac_up = MCDI_OUT_DWORD(req, GET_LINK_OUT_MAC_FAULT) == 0; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t siena_phy_reconfigure( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); efx_mcdi_req_t req; uint8_t payload[MAX(MAX(MC_CMD_SET_ID_LED_IN_LEN, MC_CMD_SET_ID_LED_OUT_LEN), MAX(MC_CMD_SET_LINK_IN_LEN, MC_CMD_SET_LINK_OUT_LEN))]; uint32_t cap_mask; #if EFSYS_OPT_PHY_LED_CONTROL unsigned int led_mode; #endif unsigned int speed; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_SET_LINK; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_SET_LINK_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_SET_LINK_OUT_LEN; cap_mask = epp->ep_adv_cap_mask; MCDI_IN_POPULATE_DWORD_10(req, SET_LINK_IN_CAP, PHY_CAP_10HDX, (cap_mask >> EFX_PHY_CAP_10HDX) & 0x1, PHY_CAP_10FDX, (cap_mask >> EFX_PHY_CAP_10FDX) & 0x1, PHY_CAP_100HDX, (cap_mask >> EFX_PHY_CAP_100HDX) & 0x1, PHY_CAP_100FDX, (cap_mask >> EFX_PHY_CAP_100FDX) & 0x1, PHY_CAP_1000HDX, (cap_mask >> EFX_PHY_CAP_1000HDX) & 0x1, PHY_CAP_1000FDX, (cap_mask >> EFX_PHY_CAP_1000FDX) & 0x1, PHY_CAP_10000FDX, (cap_mask >> EFX_PHY_CAP_10000FDX) & 0x1, PHY_CAP_PAUSE, (cap_mask >> EFX_PHY_CAP_PAUSE) & 0x1, PHY_CAP_ASYM, (cap_mask >> EFX_PHY_CAP_ASYM) & 0x1, PHY_CAP_AN, (cap_mask >> EFX_PHY_CAP_AN) & 0x1); #if EFSYS_OPT_LOOPBACK MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, epp->ep_loopback_type); switch (epp->ep_loopback_link_mode) { case EFX_LINK_100FDX: speed = 100; break; case EFX_LINK_1000FDX: speed = 1000; break; case EFX_LINK_10000FDX: speed = 10000; break; default: speed = 0; } #else MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_MODE, MC_CMD_LOOPBACK_NONE); speed = 0; #endif /* EFSYS_OPT_LOOPBACK */ MCDI_IN_SET_DWORD(req, SET_LINK_IN_LOOPBACK_SPEED, speed); #if EFSYS_OPT_PHY_FLAGS MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, epp->ep_phy_flags); #else MCDI_IN_SET_DWORD(req, SET_LINK_IN_FLAGS, 0); #endif /* EFSYS_OPT_PHY_FLAGS */ efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } /* And set the blink mode */ (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_SET_ID_LED; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_SET_ID_LED_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_SET_ID_LED_OUT_LEN; #if EFSYS_OPT_PHY_LED_CONTROL switch (epp->ep_phy_led_mode) { case EFX_PHY_LED_DEFAULT: led_mode = MC_CMD_LED_DEFAULT; break; case EFX_PHY_LED_OFF: led_mode = MC_CMD_LED_OFF; break; case EFX_PHY_LED_ON: led_mode = MC_CMD_LED_ON; break; default: EFSYS_ASSERT(0); led_mode = MC_CMD_LED_DEFAULT; } MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, led_mode); #else MCDI_IN_SET_DWORD(req, SET_ID_LED_IN_STATE, MC_CMD_LED_DEFAULT); #endif /* EFSYS_OPT_PHY_LED_CONTROL */ efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail2; } return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t siena_phy_verify( __in efx_nic_t *enp) { efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_GET_PHY_STATE_IN_LEN, MC_CMD_GET_PHY_STATE_OUT_LEN)]; uint32_t state; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_GET_PHY_STATE; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_GET_PHY_STATE_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_GET_PHY_STATE_OUT_LEN; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_GET_PHY_STATE_OUT_LEN) { rc = EMSGSIZE; goto fail2; } state = MCDI_OUT_DWORD(req, GET_PHY_STATE_OUT_STATE); if (state != MC_CMD_PHY_STATE_OK) { if (state != MC_CMD_PHY_STATE_ZOMBIE) EFSYS_PROBE1(mc_pcol_error, int, state); rc = ENOTACTIVE; goto fail3; } return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t siena_phy_oui_get( __in efx_nic_t *enp, __out uint32_t *ouip) { _NOTE(ARGUNUSED(enp, ouip)) return (ENOTSUP); } #if EFSYS_OPT_PHY_STATS #define SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat, \ _mc_record, _efx_record) \ if ((_vmask) & (1ULL << (_mc_record))) { \ (_smask) |= (1ULL << (_efx_record)); \ if ((_stat) != NULL && !EFSYS_MEM_IS_NULL(_esmp)) { \ efx_dword_t dword; \ EFSYS_MEM_READD(_esmp, (_mc_record) * 4, &dword);\ (_stat)[_efx_record] = \ EFX_DWORD_FIELD(dword, EFX_DWORD_0); \ } \ } #define SIENA_SIMPLE_STAT_SET2(_vmask, _esmp, _smask, _stat, _record) \ SIENA_SIMPLE_STAT_SET(_vmask, _esmp, _smask, _stat, \ MC_CMD_ ## _record, \ EFX_PHY_STAT_ ## _record) void siena_phy_decode_stats( __in efx_nic_t *enp, __in uint32_t vmask, __in_opt efsys_mem_t *esmp, __out_opt uint64_t *smaskp, __inout_ecount_opt(EFX_PHY_NSTATS) uint32_t *stat) { uint64_t smask = 0; _NOTE(ARGUNUSED(enp)) SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, OUI); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_LINK_UP); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_RX_FAULT); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PMA_PMD_TX_FAULT); if (vmask & (1 << MC_CMD_PMA_PMD_SIGNAL)) { smask |= ((1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_A) | (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_B) | (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_C) | (1ULL << EFX_PHY_STAT_PMA_PMD_SIGNAL_D)); if (stat != NULL && esmp != NULL && !EFSYS_MEM_IS_NULL(esmp)) { efx_dword_t dword; uint32_t sig; EFSYS_MEM_READD(esmp, 4 * MC_CMD_PMA_PMD_SIGNAL, &dword); sig = EFX_DWORD_FIELD(dword, EFX_DWORD_0); stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_A] = (sig >> 1) & 1; stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_B] = (sig >> 2) & 1; stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_C] = (sig >> 3) & 1; stat[EFX_PHY_STAT_PMA_PMD_SIGNAL_D] = (sig >> 4) & 1; } } SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_A, EFX_PHY_STAT_SNR_A); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_B, EFX_PHY_STAT_SNR_B); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_C, EFX_PHY_STAT_SNR_C); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PMA_PMD_SNR_D, EFX_PHY_STAT_SNR_D); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_LINK_UP); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_RX_FAULT); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_TX_FAULT); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BER); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, PCS_BLOCK_ERRORS); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_LINK_UP, EFX_PHY_STAT_PHY_XS_LINK_UP); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_RX_FAULT, EFX_PHY_STAT_PHY_XS_RX_FAULT); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_TX_FAULT, EFX_PHY_STAT_PHY_XS_TX_FAULT); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_PHYXS_ALIGN, EFX_PHY_STAT_PHY_XS_ALIGN); if (vmask & (1 << MC_CMD_PHYXS_SYNC)) { smask |= ((1 << EFX_PHY_STAT_PHY_XS_SYNC_A) | (1 << EFX_PHY_STAT_PHY_XS_SYNC_B) | (1 << EFX_PHY_STAT_PHY_XS_SYNC_C) | (1 << EFX_PHY_STAT_PHY_XS_SYNC_D)); if (stat != NULL && !EFSYS_MEM_IS_NULL(esmp)) { efx_dword_t dword; uint32_t sync; EFSYS_MEM_READD(esmp, 4 * MC_CMD_PHYXS_SYNC, &dword); sync = EFX_DWORD_FIELD(dword, EFX_DWORD_0); stat[EFX_PHY_STAT_PHY_XS_SYNC_A] = (sync >> 0) & 1; stat[EFX_PHY_STAT_PHY_XS_SYNC_B] = (sync >> 1) & 1; stat[EFX_PHY_STAT_PHY_XS_SYNC_C] = (sync >> 2) & 1; stat[EFX_PHY_STAT_PHY_XS_SYNC_D] = (sync >> 3) & 1; } } SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_LINK_UP); SIENA_SIMPLE_STAT_SET2(vmask, esmp, smask, stat, AN_COMPLETE); SIENA_SIMPLE_STAT_SET(vmask, esmp, smask, stat, MC_CMD_CL22_LINK_UP, EFX_PHY_STAT_CL22EXT_LINK_UP); if (smaskp != NULL) *smaskp = smask; } __checkReturn efx_rc_t siena_phy_stats_update( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); uint32_t vmask = encp->enc_mcdi_phy_stat_mask; uint64_t smask; efx_mcdi_req_t req; uint8_t payload[MAX(MC_CMD_PHY_STATS_IN_LEN, MC_CMD_PHY_STATS_OUT_DMA_LEN)]; efx_rc_t rc; + if ((esmp == NULL) || (EFSYS_MEM_SIZE(esmp) < EFX_PHY_STATS_SIZE)) { + rc = EINVAL; + goto fail1; + } + (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_PHY_STATS; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_PHY_STATS_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MC_CMD_PHY_STATS_OUT_DMA_LEN; MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_LO, EFSYS_MEM_ADDR(esmp) & 0xffffffff); MCDI_IN_SET_DWORD(req, PHY_STATS_IN_DMA_ADDR_HI, EFSYS_MEM_ADDR(esmp) >> 32); efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; - goto fail1; + goto fail2; } EFSYS_ASSERT3U(req.emr_out_length, ==, MC_CMD_PHY_STATS_OUT_DMA_LEN); siena_phy_decode_stats(enp, vmask, esmp, &smask, stat); EFSYS_ASSERT(smask == encp->enc_phy_stat_mask); return (0); +fail2: + EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (0); } #endif /* EFSYS_OPT_PHY_STATS */ #if EFSYS_OPT_BIST __checkReturn efx_rc_t siena_phy_bist_start( __in efx_nic_t *enp, __in efx_bist_type_t type) { efx_rc_t rc; if ((rc = efx_mcdi_bist_start(enp, type)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn unsigned long siena_phy_sft9001_bist_status( __in uint16_t code) { switch (code) { case MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY: return (EFX_PHY_CABLE_STATUS_BUSY); case MC_CMD_POLL_BIST_SFT9001_INTER_PAIR_SHORT: return (EFX_PHY_CABLE_STATUS_INTERPAIRSHORT); case MC_CMD_POLL_BIST_SFT9001_INTRA_PAIR_SHORT: return (EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT); case MC_CMD_POLL_BIST_SFT9001_PAIR_OPEN: return (EFX_PHY_CABLE_STATUS_OPEN); case MC_CMD_POLL_BIST_SFT9001_PAIR_OK: return (EFX_PHY_CABLE_STATUS_OK); default: return (EFX_PHY_CABLE_STATUS_INVALID); } } __checkReturn efx_rc_t siena_phy_bist_poll( __in efx_nic_t *enp, __in efx_bist_type_t type, __out efx_bist_result_t *resultp, __out_opt __drv_when(count > 0, __notnull) uint32_t *value_maskp, __out_ecount_opt(count) __drv_when(count > 0, __notnull) unsigned long *valuesp, __in size_t count) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); uint8_t payload[MAX(MC_CMD_POLL_BIST_IN_LEN, MCDI_CTL_SDU_LEN_MAX)]; uint32_t value_mask = 0; efx_mcdi_req_t req; uint32_t result; efx_rc_t rc; (void) memset(payload, 0, sizeof (payload)); req.emr_cmd = MC_CMD_POLL_BIST; req.emr_in_buf = payload; req.emr_in_length = MC_CMD_POLL_BIST_IN_LEN; req.emr_out_buf = payload; req.emr_out_length = MCDI_CTL_SDU_LEN_MAX; efx_mcdi_execute(enp, &req); if (req.emr_rc != 0) { rc = req.emr_rc; goto fail1; } if (req.emr_out_length_used < MC_CMD_POLL_BIST_OUT_RESULT_OFST + 4) { rc = EMSGSIZE; goto fail2; } if (count > 0) (void) memset(valuesp, '\0', count * sizeof (unsigned long)); result = MCDI_OUT_DWORD(req, POLL_BIST_OUT_RESULT); /* Extract PHY specific results */ if (result == MC_CMD_POLL_BIST_PASSED && encp->enc_phy_type == EFX_PHY_SFT9001B && req.emr_out_length_used >= MC_CMD_POLL_BIST_OUT_SFT9001_LEN && (type == EFX_BIST_TYPE_PHY_CABLE_SHORT || type == EFX_BIST_TYPE_PHY_CABLE_LONG)) { uint16_t word; if (count > EFX_BIST_PHY_CABLE_LENGTH_A) { if (valuesp != NULL) valuesp[EFX_BIST_PHY_CABLE_LENGTH_A] = MCDI_OUT_DWORD(req, POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A); value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_A); } if (count > EFX_BIST_PHY_CABLE_LENGTH_B) { if (valuesp != NULL) valuesp[EFX_BIST_PHY_CABLE_LENGTH_B] = MCDI_OUT_DWORD(req, POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B); value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_B); } if (count > EFX_BIST_PHY_CABLE_LENGTH_C) { if (valuesp != NULL) valuesp[EFX_BIST_PHY_CABLE_LENGTH_C] = MCDI_OUT_DWORD(req, POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C); value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_C); } if (count > EFX_BIST_PHY_CABLE_LENGTH_D) { if (valuesp != NULL) valuesp[EFX_BIST_PHY_CABLE_LENGTH_D] = MCDI_OUT_DWORD(req, POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D); value_mask |= (1 << EFX_BIST_PHY_CABLE_LENGTH_D); } if (count > EFX_BIST_PHY_CABLE_STATUS_A) { if (valuesp != NULL) { word = MCDI_OUT_WORD(req, POLL_BIST_OUT_SFT9001_CABLE_STATUS_A); valuesp[EFX_BIST_PHY_CABLE_STATUS_A] = siena_phy_sft9001_bist_status(word); } value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_A); } if (count > EFX_BIST_PHY_CABLE_STATUS_B) { if (valuesp != NULL) { word = MCDI_OUT_WORD(req, POLL_BIST_OUT_SFT9001_CABLE_STATUS_B); valuesp[EFX_BIST_PHY_CABLE_STATUS_B] = siena_phy_sft9001_bist_status(word); } value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_B); } if (count > EFX_BIST_PHY_CABLE_STATUS_C) { if (valuesp != NULL) { word = MCDI_OUT_WORD(req, POLL_BIST_OUT_SFT9001_CABLE_STATUS_C); valuesp[EFX_BIST_PHY_CABLE_STATUS_C] = siena_phy_sft9001_bist_status(word); } value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_C); } if (count > EFX_BIST_PHY_CABLE_STATUS_D) { if (valuesp != NULL) { word = MCDI_OUT_WORD(req, POLL_BIST_OUT_SFT9001_CABLE_STATUS_D); valuesp[EFX_BIST_PHY_CABLE_STATUS_D] = siena_phy_sft9001_bist_status(word); } value_mask |= (1 << EFX_BIST_PHY_CABLE_STATUS_D); } } else if (result == MC_CMD_POLL_BIST_FAILED && encp->enc_phy_type == EFX_PHY_QLX111V && req.emr_out_length >= MC_CMD_POLL_BIST_OUT_MRSFP_LEN && count > EFX_BIST_FAULT_CODE) { if (valuesp != NULL) valuesp[EFX_BIST_FAULT_CODE] = MCDI_OUT_DWORD(req, POLL_BIST_OUT_MRSFP_TEST); value_mask |= 1 << EFX_BIST_FAULT_CODE; } if (value_maskp != NULL) *value_maskp = value_mask; EFSYS_ASSERT(resultp != NULL); if (result == MC_CMD_POLL_BIST_RUNNING) *resultp = EFX_BIST_RESULT_RUNNING; else if (result == MC_CMD_POLL_BIST_PASSED) *resultp = EFX_BIST_RESULT_PASSED; else *resultp = EFX_BIST_RESULT_FAILED; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void siena_phy_bist_stop( __in efx_nic_t *enp, __in efx_bist_type_t type) { /* There is no way to stop BIST on Siena */ _NOTE(ARGUNUSED(enp, type)) } #endif /* EFSYS_OPT_BIST */ #endif /* EFSYS_OPT_SIENA */ Index: head/sys/dev/sfxge/sfxge_dma.c =================================================================== --- head/sys/dev/sfxge/sfxge_dma.c (revision 341050) +++ head/sys/dev/sfxge/sfxge_dma.c (revision 341051) @@ -1,218 +1,220 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010-2016 Solarflare Communications Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" static void sfxge_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *addr; addr = arg; if (error != 0) { *addr = 0; return; } *addr = segs[0].ds_addr; } int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs) { bus_dma_segment_t *psegs; struct mbuf *m; int seg_count; int defragged; int err; m = *mp; defragged = err = seg_count = 0; KASSERT(m->m_pkthdr.len, ("packet has zero header length")); retry: psegs = segs; seg_count = 0; if (m->m_next == NULL) { sfxge_map_mbuf_fast(tag, map, m, segs); *nsegs = 1; return (0); } #if defined(__i386__) || defined(__amd64__) while (m != NULL && seg_count < maxsegs) { /* * firmware doesn't like empty segments */ if (m->m_len != 0) { seg_count++; sfxge_map_mbuf_fast(tag, map, m, psegs); psegs++; } m = m->m_next; } #else err = bus_dmamap_load_mbuf_sg(tag, map, *mp, segs, &seg_count, 0); #endif if (seg_count == 0) { err = EFBIG; goto err_out; } else if (err == EFBIG || seg_count >= maxsegs) { if (!defragged) { m = m_defrag(*mp, M_NOWAIT); if (m == NULL) { err = ENOBUFS; goto err_out; } *mp = m; defragged = 1; goto retry; } err = EFBIG; goto err_out; } *nsegs = seg_count; err_out: return (err); } void sfxge_dma_free(efsys_mem_t *esmp) { bus_dmamap_unload(esmp->esm_tag, esmp->esm_map); bus_dmamem_free(esmp->esm_tag, esmp->esm_base, esmp->esm_map); bus_dma_tag_destroy(esmp->esm_tag); esmp->esm_addr = 0; esmp->esm_base = NULL; + esmp->esm_size = 0; } int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len, efsys_mem_t *esmp) { void *vaddr; /* Create the child DMA tag. */ if (bus_dma_tag_create(sc->parent_dma_tag, PAGE_SIZE, 0, MIN(0x3FFFFFFFFFFFUL, BUS_SPACE_MAXADDR), BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, &esmp->esm_tag) != 0) { device_printf(sc->dev, "Couldn't allocate txq DMA tag\n"); goto fail_tag_create; } /* Allocate kernel memory. */ if (bus_dmamem_alloc(esmp->esm_tag, (void **)&vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, &esmp->esm_map) != 0) { device_printf(sc->dev, "Couldn't allocate DMA memory\n"); goto fail_alloc; } /* Load map into device memory. */ if (bus_dmamap_load(esmp->esm_tag, esmp->esm_map, vaddr, len, sfxge_dma_cb, &esmp->esm_addr, 0) != 0) { device_printf(sc->dev, "Couldn't load DMA mapping\n"); goto fail_load; } /* * The callback gets error information about the mapping * and will have set esm_addr to 0 if something went * wrong. */ if (esmp->esm_addr == 0) goto fail_load_check; esmp->esm_base = vaddr; + esmp->esm_size = len; return (0); fail_load_check: fail_load: bus_dmamem_free(esmp->esm_tag, vaddr, esmp->esm_map); fail_alloc: bus_dma_tag_destroy(esmp->esm_tag); fail_tag_create: return (ENOMEM); } void sfxge_dma_fini(struct sfxge_softc *sc) { bus_dma_tag_destroy(sc->parent_dma_tag); } int sfxge_dma_init(struct sfxge_softc *sc) { /* Create the parent dma tag. */ if (bus_dma_tag_create(bus_get_dma_tag(sc->dev), /* parent */ 1, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lock, lockarg */ &sc->parent_dma_tag) != 0) { device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); return (ENOMEM); } return (0); }