Index: head/sys/dev/sfxge/common/efx_ev.c =================================================================== --- head/sys/dev/sfxge/common/efx_ev.c (revision 310689) +++ head/sys/dev/sfxge/common/efx_ev.c (revision 310690) @@ -1,1458 +1,1458 @@ /*- * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_MON_MCDI #include "mcdi_mon.h" #endif #if EFSYS_OPT_QSTATS #define EFX_EV_QSTAT_INCR(_eep, _stat) \ do { \ (_eep)->ee_stat[_stat]++; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFX_EV_QSTAT_INCR(_eep, _stat) #endif #define EFX_EV_PRESENT(_qword) \ (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \ EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff) #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_ev_init( __in efx_nic_t *enp); static void siena_ev_fini( __in efx_nic_t *enp); static __checkReturn efx_rc_t siena_ev_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint32_t us, __in efx_evq_t *eep); static void siena_ev_qdestroy( __in efx_evq_t *eep); static __checkReturn efx_rc_t siena_ev_qprime( __in efx_evq_t *eep, __in unsigned int count); static void siena_ev_qpoll( __in efx_evq_t *eep, __inout unsigned int *countp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg); static void siena_ev_qpost( __in efx_evq_t *eep, __in uint16_t data); static __checkReturn efx_rc_t siena_ev_qmoderate( __in efx_evq_t *eep, __in unsigned int us); #if EFSYS_OPT_QSTATS static void siena_ev_qstats_update( __in efx_evq_t *eep, __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); #endif #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_SIENA static const efx_ev_ops_t __efx_ev_siena_ops = { siena_ev_init, /* eevo_init */ siena_ev_fini, /* eevo_fini */ siena_ev_qcreate, /* eevo_qcreate */ siena_ev_qdestroy, /* eevo_qdestroy */ siena_ev_qprime, /* eevo_qprime */ siena_ev_qpost, /* eevo_qpost */ siena_ev_qmoderate, /* eevo_qmoderate */ #if EFSYS_OPT_QSTATS siena_ev_qstats_update, /* eevo_qstats_update */ #endif }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD static const efx_ev_ops_t __efx_ev_ef10_ops = { ef10_ev_init, /* eevo_init */ ef10_ev_fini, /* eevo_fini */ ef10_ev_qcreate, /* eevo_qcreate */ ef10_ev_qdestroy, /* eevo_qdestroy */ ef10_ev_qprime, /* eevo_qprime */ ef10_ev_qpost, /* eevo_qpost */ ef10_ev_qmoderate, /* eevo_qmoderate */ #if EFSYS_OPT_QSTATS ef10_ev_qstats_update, /* eevo_qstats_update */ #endif }; #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ __checkReturn efx_rc_t efx_ev_init( __in efx_nic_t *enp) { const efx_ev_ops_t *eevop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); if (enp->en_mod_flags & EFX_MOD_EV) { rc = EINVAL; goto fail1; } switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: eevop = &__efx_ev_siena_ops; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: eevop = &__efx_ev_ef10_ops; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: eevop = &__efx_ev_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD */ default: EFSYS_ASSERT(0); rc = ENOTSUP; goto fail1; } EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); if ((rc = eevop->eevo_init(enp)) != 0) goto fail2; enp->en_eevop = eevop; enp->en_mod_flags |= EFX_MOD_EV; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); enp->en_eevop = NULL; enp->en_mod_flags &= ~EFX_MOD_EV; return (rc); } void efx_ev_fini( __in efx_nic_t *enp) { const efx_ev_ops_t *eevop = enp->en_eevop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); eevop->eevo_fini(enp); enp->en_eevop = NULL; enp->en_mod_flags &= ~EFX_MOD_EV; } __checkReturn efx_rc_t efx_ev_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint32_t us, __deref_out efx_evq_t **eepp) { const efx_ev_ops_t *eevop = enp->en_eevop; efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_evq_t *eep; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit); /* Allocate an EVQ object */ EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep); if (eep == NULL) { rc = ENOMEM; goto fail1; } eep->ee_magic = EFX_EVQ_MAGIC; eep->ee_enp = enp; eep->ee_index = index; eep->ee_mask = n - 1; eep->ee_esmp = esmp; /* * Set outputs before the queue is created because interrupts may be * raised for events immediately after the queue is created, before the * function call below returns. See bug58606. * * The eepp pointer passed in by the client must therefore point to data * shared with the client's event processing context. */ enp->en_ev_qcount++; *eepp = eep; if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, eep)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); *eepp = NULL; enp->en_ev_qcount--; EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_ev_qdestroy( __in efx_evq_t *eep) { efx_nic_t *enp = eep->ee_enp; const efx_ev_ops_t *eevop = enp->en_eevop; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); EFSYS_ASSERT(enp->en_ev_qcount != 0); --enp->en_ev_qcount; eevop->eevo_qdestroy(eep); /* Free the EVQ object */ EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); } __checkReturn efx_rc_t efx_ev_qprime( __in efx_evq_t *eep, __in unsigned int count) { efx_nic_t *enp = eep->ee_enp; const efx_ev_ops_t *eevop = enp->en_eevop; efx_rc_t rc; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); if (!(enp->en_mod_flags & EFX_MOD_INTR)) { rc = EINVAL; goto fail1; } if ((rc = eevop->eevo_qprime(eep, count)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn boolean_t efx_ev_qpending( __in efx_evq_t *eep, __in unsigned int count) { size_t offset; efx_qword_t qword; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); offset = (count & eep->ee_mask) * sizeof (efx_qword_t); EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword); return (EFX_EV_PRESENT(qword)); } #if EFSYS_OPT_EV_PREFETCH void efx_ev_qprefetch( __in efx_evq_t *eep, __in unsigned int count) { unsigned int offset; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); offset = (count & eep->ee_mask) * sizeof (efx_qword_t); EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); } #endif /* EFSYS_OPT_EV_PREFETCH */ void efx_ev_qpoll( __in efx_evq_t *eep, __inout unsigned int *countp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); /* * FIXME: Huntington will require support for hardware event batching * and merging, which will need a different ev_qpoll implementation. * * Without those features the Falcon/Siena code can be used unchanged. */ EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN); EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH); EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV); EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV); EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV); EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV == FSE_AZ_EV_CODE_DRV_GEN_EV); #if EFSYS_OPT_MCDI EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV == FSE_AZ_EV_CODE_MCDI_EVRESPONSE); #endif siena_ev_qpoll(eep, countp, eecp, arg); } void efx_ev_qpost( __in efx_evq_t *eep, __in uint16_t data) { efx_nic_t *enp = eep->ee_enp; const efx_ev_ops_t *eevop = enp->en_eevop; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); EFSYS_ASSERT(eevop != NULL && eevop->eevo_qpost != NULL); eevop->eevo_qpost(eep, data); } __checkReturn efx_rc_t efx_ev_usecs_to_ticks( __in efx_nic_t *enp, __in unsigned int us, __out unsigned int *ticksp) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); unsigned int ticks; /* Convert microseconds to a timer tick count */ if (us == 0) ticks = 0; else if (us * 1000 < encp->enc_evq_timer_quantum_ns) ticks = 1; /* Never round down to zero */ else ticks = us * 1000 / encp->enc_evq_timer_quantum_ns; *ticksp = ticks; return (0); } __checkReturn efx_rc_t efx_ev_qmoderate( __in efx_evq_t *eep, __in unsigned int us) { efx_nic_t *enp = eep->ee_enp; const efx_ev_ops_t *eevop = enp->en_eevop; efx_rc_t rc; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); if ((rc = eevop->eevo_qmoderate(eep, us)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_QSTATS void efx_ev_qstats_update( __in efx_evq_t *eep, __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) { efx_nic_t *enp = eep->ee_enp; const efx_ev_ops_t *eevop = enp->en_eevop; EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); eevop->eevo_qstats_update(eep, stat); } #endif /* EFSYS_OPT_QSTATS */ #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_ev_init( __in efx_nic_t *enp) { efx_oword_t oword; /* * Program the event queue for receive and transmit queue * flush events. */ EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0); EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword); return (0); } static __checkReturn boolean_t siena_ev_rx_not_ok( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in uint32_t label, __in uint32_t id, __inout uint16_t *flagsp) { boolean_t ignore = B_FALSE; if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC); EFSYS_PROBE(tobe_disc); /* * Assume this is a unicast address mismatch, unless below * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or * EV_RX_PAUSE_FRM_ERR is set. */ (*flagsp) |= EFX_ADDR_MISMATCH; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) { EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id); EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); (*flagsp) |= EFX_DISCARD; #if EFSYS_OPT_RX_SCATTER /* * Lookout for payload queue ran dry errors and ignore them. * * Sadly for the header/data split cases, the descriptor * pointer in this event refers to the header queue and * therefore cannot be easily detected as duplicate. * So we drop these and rely on the receive processing seeing * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard * the partially received packet. */ if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) && (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) && (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0)) ignore = B_TRUE; #endif /* EFSYS_OPT_RX_SCATTER */ } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); EFSYS_PROBE(crc_err); (*flagsp) &= ~EFX_ADDR_MISMATCH; (*flagsp) |= EFX_DISCARD; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR); EFSYS_PROBE(pause_frm_err); (*flagsp) &= ~EFX_ADDR_MISMATCH; (*flagsp) |= EFX_DISCARD; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR); EFSYS_PROBE(owner_id_err); (*flagsp) |= EFX_DISCARD; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); EFSYS_PROBE(ipv4_err); (*flagsp) &= ~EFX_CKSUM_IPV4; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); EFSYS_PROBE(udp_chk_err); (*flagsp) &= ~EFX_CKSUM_TCPUDP; } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR); /* * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error * condition. */ (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP); } return (ignore); } static __checkReturn boolean_t siena_ev_rx( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { uint32_t id; uint32_t size; uint32_t label; boolean_t ok; #if EFSYS_OPT_RX_SCATTER boolean_t sop; boolean_t jumbo_cont; #endif /* EFSYS_OPT_RX_SCATTER */ uint32_t hdr_type; boolean_t is_v6; uint16_t flags; boolean_t ignore; boolean_t should_abort; EFX_EV_QSTAT_INCR(eep, EV_RX); /* Basic packet information */ id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR); size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT); label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL); ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0); #if EFSYS_OPT_RX_SCATTER sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0); jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0); #endif /* EFSYS_OPT_RX_SCATTER */ hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE); is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0); /* * If packet is marked as OK and packet type is TCP/IP or * UDP/IP or other IP, then we can rely on the hardware checksums. */ switch (hdr_type) { case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP: flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP; if (is_v6) { EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); flags |= EFX_PKT_IPV6; } else { EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; } break; case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP: flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP; if (is_v6) { EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); flags |= EFX_PKT_IPV6; } else { EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; } break; case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: if (is_v6) { EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); flags = EFX_PKT_IPV6; } else { EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4; } break; case FSE_AZ_RX_EV_HDR_TYPE_OTHER: EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); flags = 0; break; default: EFSYS_ASSERT(B_FALSE); flags = 0; break; } #if EFSYS_OPT_RX_SCATTER /* Report scatter and header/lookahead split buffer flags */ if (sop) flags |= EFX_PKT_START; if (jumbo_cont) flags |= EFX_PKT_CONT; #endif /* EFSYS_OPT_RX_SCATTER */ /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */ if (!ok) { ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags); if (ignore) { EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, uint32_t, size, uint16_t, flags); return (B_FALSE); } } /* If we're not discarding the packet then it is ok */ if (~flags & EFX_DISCARD) EFX_EV_QSTAT_INCR(eep, EV_RX_OK); /* Detect multicast packets that didn't match the filter */ if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT); if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) { EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH); } else { EFSYS_PROBE(mcast_mismatch); flags |= EFX_ADDR_MISMATCH; } } else { flags |= EFX_PKT_UNICAST; } /* * The packet parser in Siena can abort parsing packets under * certain error conditions, setting the PKT_NOT_PARSED bit * (which clears PKT_OK). If this is set, then don't trust * the PKT_TYPE field. */ if (!ok) { uint32_t parse_err; parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED); if (parse_err != 0) flags |= EFX_CHECK_VLAN; } if (~flags & EFX_CHECK_VLAN) { uint32_t pkt_type; pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE); if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN) flags |= EFX_PKT_VLAN_TAGGED; } EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, uint32_t, size, uint16_t, flags); EFSYS_ASSERT(eecp->eec_rx != NULL); should_abort = eecp->eec_rx(arg, label, id, size, flags); return (should_abort); } static __checkReturn boolean_t siena_ev_tx( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { uint32_t id; uint32_t label; boolean_t should_abort; EFX_EV_QSTAT_INCR(eep, EV_TX); if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 && EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 && EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 && EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) { id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR); label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL); EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); EFSYS_ASSERT(eecp->eec_tx != NULL); should_abort = eecp->eec_tx(arg, label, id); return (should_abort); } if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0) EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0) EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR); if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0) EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG); if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0) EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL); EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED); return (B_FALSE); } static __checkReturn boolean_t siena_ev_global( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { _NOTE(ARGUNUSED(eqp, eecp, arg)) EFX_EV_QSTAT_INCR(eep, EV_GLOBAL); return (B_FALSE); } static __checkReturn boolean_t siena_ev_driver( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { boolean_t should_abort; EFX_EV_QSTAT_INCR(eep, EV_DRIVER); should_abort = B_FALSE; switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) { case FSE_AZ_TX_DESCQ_FLS_DONE_EV: { uint32_t txq_index; EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); should_abort = eecp->eec_txq_flush_done(arg, txq_index); break; } case FSE_AZ_RX_DESCQ_FLS_DONE_EV: { uint32_t rxq_index; uint32_t failed; rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL); if (failed) { EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED); EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index); should_abort = eecp->eec_rxq_flush_failed(arg, rxq_index); } else { EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); } break; } case FSE_AZ_EVQ_INIT_DONE_EV: EFSYS_ASSERT(eecp->eec_initialized != NULL); should_abort = eecp->eec_initialized(arg); break; case FSE_AZ_EVQ_NOT_EN_EV: EFSYS_PROBE(evq_not_en); break; case FSE_AZ_SRM_UPD_DONE_EV: { uint32_t code; EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE); code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); EFSYS_ASSERT(eecp->eec_sram != NULL); should_abort = eecp->eec_sram(arg, code); break; } case FSE_AZ_WAKE_UP_EV: { uint32_t id; id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); EFSYS_ASSERT(eecp->eec_wake_up != NULL); should_abort = eecp->eec_wake_up(arg, id); break; } case FSE_AZ_TX_PKT_NON_TCP_UDP: EFSYS_PROBE(tx_pkt_non_tcp_udp); break; case FSE_AZ_TIMER_EV: { uint32_t id; id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); EFSYS_ASSERT(eecp->eec_timer != NULL); should_abort = eecp->eec_timer(arg, id); break; } case FSE_AZ_RX_DSC_ERROR_EV: EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR); EFSYS_PROBE(rx_dsc_error); EFSYS_ASSERT(eecp->eec_exception != NULL); should_abort = eecp->eec_exception(arg, EFX_EXCEPTION_RX_DSC_ERROR, 0); break; case FSE_AZ_TX_DSC_ERROR_EV: EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR); EFSYS_PROBE(tx_dsc_error); EFSYS_ASSERT(eecp->eec_exception != NULL); should_abort = eecp->eec_exception(arg, EFX_EXCEPTION_TX_DSC_ERROR, 0); break; default: break; } return (should_abort); } static __checkReturn boolean_t siena_ev_drv_gen( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { uint32_t data; boolean_t should_abort; EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0); if (data >= ((uint32_t)1 << 16)) { EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); return (B_TRUE); } EFSYS_ASSERT(eecp->eec_software != NULL); should_abort = eecp->eec_software(arg, (uint16_t)data); return (should_abort); } #if EFSYS_OPT_MCDI static __checkReturn boolean_t siena_ev_mcdi( __in efx_evq_t *eep, __in efx_qword_t *eqp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { efx_nic_t *enp = eep->ee_enp; unsigned code; boolean_t should_abort = B_FALSE; EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); if (enp->en_family != EFX_FAMILY_SIENA) goto out; EFSYS_ASSERT(eecp->eec_link_change != NULL); EFSYS_ASSERT(eecp->eec_exception != NULL); #if EFSYS_OPT_MON_STATS EFSYS_ASSERT(eecp->eec_monitor != NULL); #endif EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); switch (code) { case MCDI_EVENT_CODE_BADSSERT: efx_mcdi_ev_death(enp, EINTR); break; case MCDI_EVENT_CODE_CMDDONE: efx_mcdi_ev_cpl(enp, MCDI_EV_FIELD(eqp, CMDDONE_SEQ), MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); break; case MCDI_EVENT_CODE_LINKCHANGE: { efx_link_mode_t link_mode; siena_phy_link_ev(enp, eqp, &link_mode); should_abort = eecp->eec_link_change(arg, link_mode); break; } case MCDI_EVENT_CODE_SENSOREVT: { #if EFSYS_OPT_MON_STATS efx_mon_stat_t id; efx_mon_stat_value_t value; efx_rc_t rc; if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) should_abort = eecp->eec_monitor(arg, id, value); else if (rc == ENOTSUP) { should_abort = eecp->eec_exception(arg, EFX_EXCEPTION_UNKNOWN_SENSOREVT, MCDI_EV_FIELD(eqp, DATA)); } else EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ #else should_abort = B_FALSE; #endif break; } case MCDI_EVENT_CODE_SCHEDERR: /* Informational only */ break; case MCDI_EVENT_CODE_REBOOT: efx_mcdi_ev_death(enp, EIO); break; case MCDI_EVENT_CODE_MAC_STATS_DMA: #if EFSYS_OPT_MAC_STATS if (eecp->eec_mac_stats != NULL) { eecp->eec_mac_stats(arg, MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); } #endif break; case MCDI_EVENT_CODE_FWALERT: { uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) should_abort = eecp->eec_exception(arg, EFX_EXCEPTION_FWALERT_SRAM, MCDI_EV_FIELD(eqp, FWALERT_DATA)); else should_abort = eecp->eec_exception(arg, EFX_EXCEPTION_UNKNOWN_FWALERT, MCDI_EV_FIELD(eqp, DATA)); break; } default: EFSYS_PROBE1(mc_pcol_error, int, code); break; } out: return (should_abort); } #endif /* EFSYS_OPT_MCDI */ static __checkReturn efx_rc_t siena_ev_qprime( __in efx_evq_t *eep, __in unsigned int count) { efx_nic_t *enp = eep->ee_enp; uint32_t rptr; efx_dword_t dword; rptr = count & eep->ee_mask; EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr); EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index, &dword, B_FALSE); return (0); } #define EFX_EV_BATCH 8 static void siena_ev_qpoll( __in efx_evq_t *eep, __inout unsigned int *countp, __in const efx_ev_callbacks_t *eecp, __in_opt void *arg) { efx_qword_t ev[EFX_EV_BATCH]; unsigned int batch; unsigned int total; unsigned int count; unsigned int index; size_t offset; EFSYS_ASSERT(countp != NULL); EFSYS_ASSERT(eecp != NULL); count = *countp; do { /* Read up until the end of the batch period */ batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1)); offset = (count & eep->ee_mask) * sizeof (efx_qword_t); for (total = 0; total < batch; ++total) { EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total])); if (!EFX_EV_PRESENT(ev[total])) break; EFSYS_PROBE3(event, unsigned int, eep->ee_index, uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1), uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0)); offset += sizeof (efx_qword_t); } #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1) /* * Prefetch the next batch when we get within PREFETCH_PERIOD * of a completed batch. If the batch is smaller, then prefetch * immediately. */ if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD) EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); #endif /* EFSYS_OPT_EV_PREFETCH */ /* Process the batch of events */ for (index = 0; index < total; ++index) { boolean_t should_abort; uint32_t code; #if EFSYS_OPT_EV_PREFETCH /* Prefetch if we've now reached the batch period */ if (total == batch && index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) { offset = (count + batch) & eep->ee_mask; offset *= sizeof (efx_qword_t); EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); } #endif /* EFSYS_OPT_EV_PREFETCH */ EFX_EV_QSTAT_INCR(eep, EV_ALL); code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE); switch (code) { case FSE_AZ_EV_CODE_RX_EV: should_abort = eep->ee_rx(eep, &(ev[index]), eecp, arg); break; case FSE_AZ_EV_CODE_TX_EV: should_abort = eep->ee_tx(eep, &(ev[index]), eecp, arg); break; case FSE_AZ_EV_CODE_DRIVER_EV: should_abort = eep->ee_driver(eep, &(ev[index]), eecp, arg); break; case FSE_AZ_EV_CODE_DRV_GEN_EV: should_abort = eep->ee_drv_gen(eep, &(ev[index]), eecp, arg); break; #if EFSYS_OPT_MCDI case FSE_AZ_EV_CODE_MCDI_EVRESPONSE: should_abort = eep->ee_mcdi(eep, &(ev[index]), eecp, arg); break; #endif case FSE_AZ_EV_CODE_GLOBAL_EV: if (eep->ee_global) { should_abort = eep->ee_global(eep, &(ev[index]), eecp, arg); break; } /* else fallthrough */ default: EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, uint32_t, EFX_QWORD_FIELD(ev[index], EFX_DWORD_1), uint32_t, EFX_QWORD_FIELD(ev[index], EFX_DWORD_0)); EFSYS_ASSERT(eecp->eec_exception != NULL); (void) eecp->eec_exception(arg, EFX_EXCEPTION_EV_ERROR, code); should_abort = B_TRUE; } if (should_abort) { /* Ignore subsequent events */ total = index + 1; break; } } /* * Now that the hardware has most likely moved onto dma'ing * into the next cache line, clear the processed events. Take * care to only clear out events that we've processed */ EFX_SET_QWORD(ev[0]); offset = (count & eep->ee_mask) * sizeof (efx_qword_t); for (index = 0; index < total; ++index) { EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0])); offset += sizeof (efx_qword_t); } count += total; } while (total == batch); *countp = count; } static void siena_ev_qpost( __in efx_evq_t *eep, __in uint16_t data) { efx_nic_t *enp = eep->ee_enp; efx_qword_t ev; efx_oword_t oword; EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV, FSF_AZ_EV_DATA_DW0, (uint32_t)data); EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index, EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0), EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1)); EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword); } static __checkReturn efx_rc_t siena_ev_qmoderate( __in efx_evq_t *eep, __in unsigned int us) { efx_nic_t *enp = eep->ee_enp; efx_nic_cfg_t *encp = &(enp->en_nic_cfg); unsigned int locked; efx_dword_t dword; efx_rc_t rc; if (us > encp->enc_evq_timer_max_us) { rc = EINVAL; goto fail1; } /* If the value is zero then disable the timer */ if (us == 0) { EFX_POPULATE_DWORD_2(dword, FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS, FRF_CZ_TC_TIMER_VAL, 0); } else { unsigned int ticks; if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) goto fail2; EFSYS_ASSERT(ticks > 0); EFX_POPULATE_DWORD_2(dword, FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF, FRF_CZ_TC_TIMER_VAL, ticks - 1); } locked = (eep->ee_index == 0) ? 1 : 0; EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0, eep->ee_index, &dword, locked); return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn efx_rc_t siena_ev_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint32_t us, __in efx_evq_t *eep) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); uint32_t size; efx_oword_t oword; efx_rc_t rc; _NOTE(ARGUNUSED(esmp)) EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { rc = EINVAL; goto fail1; } if (index >= encp->enc_evq_limit) { rc = EINVAL; goto fail2; } #if EFSYS_OPT_RX_SCALE if (enp->en_intr.ei_type == EFX_INTR_LINE && index >= EFX_MAXRSS_LEGACY) { rc = EINVAL; goto fail3; } #endif for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS); size++) if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS)) break; if (id + (1 << size) >= encp->enc_buftbl_limit) { rc = EINVAL; goto fail4; } /* Set up the handler table */ eep->ee_rx = siena_ev_rx; eep->ee_tx = siena_ev_tx; eep->ee_driver = siena_ev_driver; eep->ee_global = siena_ev_global; eep->ee_drv_gen = siena_ev_drv_gen; #if EFSYS_OPT_MCDI eep->ee_mcdi = siena_ev_mcdi; #endif /* EFSYS_OPT_MCDI */ /* Set up the new event queue */ EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE); EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size, FRF_AZ_EVQ_BUF_BASE_ID, id); EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE); /* Set initial interrupt moderation */ siena_ev_qmoderate(eep, us); return (0); fail4: EFSYS_PROBE(fail4); #if EFSYS_OPT_RX_SCALE fail3: EFSYS_PROBE(fail3); #endif fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_QSTATS #if EFSYS_OPT_NAMES -/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock b693ddf85aee1bfd */ -static const char *__efx_ev_qstat_name[] = { +/* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock c0f3bc5083b40532 */ +static const char * const __efx_ev_qstat_name[] = { "all", "rx", "rx_ok", "rx_frm_trunc", "rx_tobe_disc", "rx_pause_frm_err", "rx_buf_owner_id_err", "rx_ipv4_hdr_chksum_err", "rx_tcp_udp_chksum_err", "rx_eth_crc_err", "rx_ip_frag_err", "rx_mcast_pkt", "rx_mcast_hash_match", "rx_tcp_ipv4", "rx_tcp_ipv6", "rx_udp_ipv4", "rx_udp_ipv6", "rx_other_ipv4", "rx_other_ipv6", "rx_non_ip", "rx_batch", "tx", "tx_wq_ff_full", "tx_pkt_err", "tx_pkt_too_big", "tx_unexpected", "global", "global_mnt", "driver", "driver_srm_upd_done", "driver_tx_descq_fls_done", "driver_rx_descq_fls_done", "driver_rx_descq_fls_failed", "driver_rx_dsc_error", "driver_tx_dsc_error", "drv_gen", "mcdi_response", }; /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */ const char * efx_ev_qstat_name( __in efx_nic_t *enp, __in unsigned int id) { EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(id, <, EV_NQSTATS); return (__efx_ev_qstat_name[id]); } #endif /* EFSYS_OPT_NAMES */ #endif /* EFSYS_OPT_QSTATS */ #if EFSYS_OPT_SIENA #if EFSYS_OPT_QSTATS static void siena_ev_qstats_update( __in efx_evq_t *eep, __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) { unsigned int id; for (id = 0; id < EV_NQSTATS; id++) { efsys_stat_t *essp = &stat[id]; EFSYS_STAT_INCR(essp, eep->ee_stat[id]); eep->ee_stat[id] = 0; } } #endif /* EFSYS_OPT_QSTATS */ static void siena_ev_qdestroy( __in efx_evq_t *eep) { efx_nic_t *enp = eep->ee_enp; efx_oword_t oword; /* Purge event queue */ EFX_ZERO_OWORD(oword); EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, eep->ee_index, &oword, B_TRUE); EFX_ZERO_OWORD(oword); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE); } static void siena_ev_fini( __in efx_nic_t *enp) { _NOTE(ARGUNUSED(enp)) } #endif /* EFSYS_OPT_SIENA */ Index: head/sys/dev/sfxge/common/efx_mac.c =================================================================== --- head/sys/dev/sfxge/common/efx_mac.c (revision 310689) +++ head/sys/dev/sfxge/common/efx_mac.c (revision 310690) @@ -1,828 +1,828 @@ /*- * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_mac_multicast_list_set( __in efx_nic_t *enp); #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_SIENA static const efx_mac_ops_t __efx_siena_mac_ops = { siena_mac_poll, /* emo_poll */ siena_mac_up, /* emo_up */ siena_mac_reconfigure, /* emo_addr_set */ siena_mac_reconfigure, /* emo_pdu_set */ siena_mac_pdu_get, /* emo_pdu_get */ siena_mac_reconfigure, /* emo_reconfigure */ siena_mac_multicast_list_set, /* emo_multicast_list_set */ NULL, /* emo_filter_set_default_rxq */ NULL, /* emo_filter_default_rxq_clear */ #if EFSYS_OPT_LOOPBACK siena_mac_loopback_set, /* emo_loopback_set */ #endif /* EFSYS_OPT_LOOPBACK */ #if EFSYS_OPT_MAC_STATS efx_mcdi_mac_stats_upload, /* emo_stats_upload */ efx_mcdi_mac_stats_periodic, /* emo_stats_periodic */ siena_mac_stats_update /* emo_stats_update */ #endif /* EFSYS_OPT_MAC_STATS */ }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD static const efx_mac_ops_t __efx_ef10_mac_ops = { ef10_mac_poll, /* emo_poll */ ef10_mac_up, /* emo_up */ ef10_mac_addr_set, /* emo_addr_set */ ef10_mac_pdu_set, /* emo_pdu_set */ ef10_mac_pdu_get, /* emo_pdu_get */ ef10_mac_reconfigure, /* emo_reconfigure */ ef10_mac_multicast_list_set, /* emo_multicast_list_set */ ef10_mac_filter_default_rxq_set, /* emo_filter_default_rxq_set */ ef10_mac_filter_default_rxq_clear, /* emo_filter_default_rxq_clear */ #if EFSYS_OPT_LOOPBACK ef10_mac_loopback_set, /* emo_loopback_set */ #endif /* EFSYS_OPT_LOOPBACK */ #if EFSYS_OPT_MAC_STATS efx_mcdi_mac_stats_upload, /* emo_stats_upload */ efx_mcdi_mac_stats_periodic, /* emo_stats_periodic */ ef10_mac_stats_update /* emo_stats_update */ #endif /* EFSYS_OPT_MAC_STATS */ }; #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ __checkReturn efx_rc_t efx_mac_pdu_set( __in efx_nic_t *enp, __in size_t pdu) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; uint32_t old_pdu; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); EFSYS_ASSERT(emop != NULL); if (pdu < EFX_MAC_PDU_MIN) { rc = EINVAL; goto fail1; } if (pdu > EFX_MAC_PDU_MAX) { rc = EINVAL; goto fail2; } old_pdu = epp->ep_mac_pdu; epp->ep_mac_pdu = (uint32_t)pdu; if ((rc = emop->emo_pdu_set(enp)) != 0) goto fail3; return (0); fail3: EFSYS_PROBE(fail3); epp->ep_mac_pdu = old_pdu; fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_pdu_get( __in efx_nic_t *enp, __out size_t *pdu) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; if ((rc = emop->emo_pdu_get(enp, pdu)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_addr_set( __in efx_nic_t *enp, __in uint8_t *addr) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; uint8_t old_addr[6]; uint32_t oui; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (EFX_MAC_ADDR_IS_MULTICAST(addr)) { rc = EINVAL; goto fail1; } oui = addr[0] << 16 | addr[1] << 8 | addr[2]; if (oui == 0x000000) { rc = EINVAL; goto fail2; } EFX_MAC_ADDR_COPY(old_addr, epp->ep_mac_addr); EFX_MAC_ADDR_COPY(epp->ep_mac_addr, addr); if ((rc = emop->emo_addr_set(enp)) != 0) goto fail3; return (0); fail3: EFSYS_PROBE(fail3); EFX_MAC_ADDR_COPY(epp->ep_mac_addr, old_addr); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_filter_set( __in efx_nic_t *enp, __in boolean_t all_unicst, __in boolean_t mulcst, __in boolean_t all_mulcst, __in boolean_t brdcst) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; boolean_t old_all_unicst; boolean_t old_mulcst; boolean_t old_all_mulcst; boolean_t old_brdcst; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); old_all_unicst = epp->ep_all_unicst; old_mulcst = epp->ep_mulcst; old_all_mulcst = epp->ep_all_mulcst; old_brdcst = epp->ep_brdcst; epp->ep_all_unicst = all_unicst; epp->ep_mulcst = mulcst; epp->ep_all_mulcst = all_mulcst; epp->ep_brdcst = brdcst; if ((rc = emop->emo_reconfigure(enp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); epp->ep_all_unicst = old_all_unicst; epp->ep_mulcst = old_mulcst; epp->ep_all_mulcst = old_all_mulcst; epp->ep_brdcst = old_brdcst; return (rc); } __checkReturn efx_rc_t efx_mac_drain( __in efx_nic_t *enp, __in boolean_t enabled) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); EFSYS_ASSERT(emop != NULL); if (epp->ep_mac_drain == enabled) return (0); epp->ep_mac_drain = enabled; if ((rc = emop->emo_reconfigure(enp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_up( __in efx_nic_t *enp, __out boolean_t *mac_upp) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if ((rc = emop->emo_up(enp, mac_upp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_fcntl_set( __in efx_nic_t *enp, __in unsigned int fcntl, __in boolean_t autoneg) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; const efx_phy_ops_t *epop = epp->ep_epop; unsigned int old_fcntl; boolean_t old_autoneg; unsigned int old_adv_cap; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if ((fcntl & ~(EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE)) != 0) { rc = EINVAL; goto fail1; } /* * Ignore a request to set flow control auto-negotiation * if the PHY doesn't support it. */ if (~epp->ep_phy_cap_mask & (1 << EFX_PHY_CAP_AN)) autoneg = B_FALSE; old_fcntl = epp->ep_fcntl; old_autoneg = epp->ep_fcntl_autoneg; old_adv_cap = epp->ep_adv_cap_mask; epp->ep_fcntl = fcntl; epp->ep_fcntl_autoneg = autoneg; /* * Always encode the flow control settings in the advertised * capabilities even if we are not trying to auto-negotiate * them and reconfigure both the PHY and the MAC. */ if (fcntl & EFX_FCNTL_RESPOND) epp->ep_adv_cap_mask |= (1 << EFX_PHY_CAP_PAUSE | 1 << EFX_PHY_CAP_ASYM); else epp->ep_adv_cap_mask &= ~(1 << EFX_PHY_CAP_PAUSE | 1 << EFX_PHY_CAP_ASYM); if (fcntl & EFX_FCNTL_GENERATE) epp->ep_adv_cap_mask ^= (1 << EFX_PHY_CAP_ASYM); if ((rc = epop->epo_reconfigure(enp)) != 0) goto fail2; if ((rc = emop->emo_reconfigure(enp)) != 0) goto fail3; return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); epp->ep_fcntl = old_fcntl; epp->ep_fcntl_autoneg = old_autoneg; epp->ep_adv_cap_mask = old_adv_cap; fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_mac_fcntl_get( __in efx_nic_t *enp, __out unsigned int *fcntl_wantedp, __out unsigned int *fcntl_linkp) { efx_port_t *epp = &(enp->en_port); unsigned int wanted = 0; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); /* * Decode the requested flow control settings from the PHY * advertised capabilities. */ if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_PAUSE)) wanted = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; if (epp->ep_adv_cap_mask & (1 << EFX_PHY_CAP_ASYM)) wanted ^= EFX_FCNTL_GENERATE; *fcntl_linkp = epp->ep_fcntl; *fcntl_wantedp = wanted; } __checkReturn efx_rc_t efx_mac_multicast_list_set( __in efx_nic_t *enp, __in_ecount(6*count) uint8_t const *addrs, __in int count) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; uint8_t *old_mulcst_addr_list = NULL; uint32_t old_mulcst_addr_count; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (count > EFX_MAC_MULTICAST_LIST_MAX) { rc = EINVAL; goto fail1; } old_mulcst_addr_count = epp->ep_mulcst_addr_count; if (old_mulcst_addr_count > 0) { /* Allocate memory to store old list (instead of using stack) */ EFSYS_KMEM_ALLOC(enp->en_esip, old_mulcst_addr_count * EFX_MAC_ADDR_LEN, old_mulcst_addr_list); if (old_mulcst_addr_list == NULL) { rc = ENOMEM; goto fail2; } /* Save the old list in case we need to rollback */ memcpy(old_mulcst_addr_list, epp->ep_mulcst_addr_list, old_mulcst_addr_count * EFX_MAC_ADDR_LEN); } /* Store the new list */ memcpy(epp->ep_mulcst_addr_list, addrs, count * EFX_MAC_ADDR_LEN); epp->ep_mulcst_addr_count = count; if ((rc = emop->emo_multicast_list_set(enp)) != 0) goto fail3; if (old_mulcst_addr_count > 0) { EFSYS_KMEM_FREE(enp->en_esip, old_mulcst_addr_count * EFX_MAC_ADDR_LEN, old_mulcst_addr_list); } return (0); fail3: EFSYS_PROBE(fail3); /* Restore original list on failure */ epp->ep_mulcst_addr_count = old_mulcst_addr_count; if (old_mulcst_addr_count > 0) { memcpy(epp->ep_mulcst_addr_list, old_mulcst_addr_list, old_mulcst_addr_count * EFX_MAC_ADDR_LEN); EFSYS_KMEM_FREE(enp->en_esip, old_mulcst_addr_count * EFX_MAC_ADDR_LEN, old_mulcst_addr_list); } fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_filter_default_rxq_set( __in efx_nic_t *enp, __in efx_rxq_t *erp, __in boolean_t using_rss) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (emop->emo_filter_default_rxq_set != NULL) { rc = emop->emo_filter_default_rxq_set(enp, erp, using_rss); if (rc != 0) goto fail1; } return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_mac_filter_default_rxq_clear( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (emop->emo_filter_default_rxq_clear != NULL) emop->emo_filter_default_rxq_clear(enp); } #if EFSYS_OPT_MAC_STATS #if EFSYS_OPT_NAMES -/* START MKCONFIG GENERATED EfxMacStatNamesBlock 054d43a31d2d7a45 */ -static const char *__efx_mac_stat_name[] = { +/* START MKCONFIG GENERATED EfxMacStatNamesBlock c11b91b42f922516 */ +static const char * const __efx_mac_stat_name[] = { "rx_octets", "rx_pkts", "rx_unicst_pkts", "rx_multicst_pkts", "rx_brdcst_pkts", "rx_pause_pkts", "rx_le_64_pkts", "rx_65_to_127_pkts", "rx_128_to_255_pkts", "rx_256_to_511_pkts", "rx_512_to_1023_pkts", "rx_1024_to_15xx_pkts", "rx_ge_15xx_pkts", "rx_errors", "rx_fcs_errors", "rx_drop_events", "rx_false_carrier_errors", "rx_symbol_errors", "rx_align_errors", "rx_internal_errors", "rx_jabber_pkts", "rx_lane0_char_err", "rx_lane1_char_err", "rx_lane2_char_err", "rx_lane3_char_err", "rx_lane0_disp_err", "rx_lane1_disp_err", "rx_lane2_disp_err", "rx_lane3_disp_err", "rx_match_fault", "rx_nodesc_drop_cnt", "tx_octets", "tx_pkts", "tx_unicst_pkts", "tx_multicst_pkts", "tx_brdcst_pkts", "tx_pause_pkts", "tx_le_64_pkts", "tx_65_to_127_pkts", "tx_128_to_255_pkts", "tx_256_to_511_pkts", "tx_512_to_1023_pkts", "tx_1024_to_15xx_pkts", "tx_ge_15xx_pkts", "tx_errors", "tx_sgl_col_pkts", "tx_mult_col_pkts", "tx_ex_col_pkts", "tx_late_col_pkts", "tx_def_pkts", "tx_ex_def_pkts", "pm_trunc_bb_overflow", "pm_discard_bb_overflow", "pm_trunc_vfifo_full", "pm_discard_vfifo_full", "pm_trunc_qbb", "pm_discard_qbb", "pm_discard_mapping", "rxdp_q_disabled_pkts", "rxdp_di_dropped_pkts", "rxdp_streaming_pkts", "rxdp_hlb_fetch", "rxdp_hlb_wait", "vadapter_rx_unicast_packets", "vadapter_rx_unicast_bytes", "vadapter_rx_multicast_packets", "vadapter_rx_multicast_bytes", "vadapter_rx_broadcast_packets", "vadapter_rx_broadcast_bytes", "vadapter_rx_bad_packets", "vadapter_rx_bad_bytes", "vadapter_rx_overflow", "vadapter_tx_unicast_packets", "vadapter_tx_unicast_bytes", "vadapter_tx_multicast_packets", "vadapter_tx_multicast_bytes", "vadapter_tx_broadcast_packets", "vadapter_tx_broadcast_bytes", "vadapter_tx_bad_packets", "vadapter_tx_bad_bytes", "vadapter_tx_overflow", }; /* END MKCONFIG GENERATED EfxMacStatNamesBlock */ __checkReturn const char * efx_mac_stat_name( __in efx_nic_t *enp, __in unsigned int id) { _NOTE(ARGUNUSED(enp)) EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(id, <, EFX_MAC_NSTATS); return (__efx_mac_stat_name[id]); } #endif /* EFSYS_OPT_NAMES */ __checkReturn efx_rc_t efx_mac_stats_upload( __in efx_nic_t *enp, __in efsys_mem_t *esmp) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); EFSYS_ASSERT(emop != NULL); /* * Don't assert !ep_mac_stats_pending, because the client might * have failed to finalise statistics when previously stopping * the port. */ if ((rc = emop->emo_stats_upload(enp, esmp)) != 0) goto fail1; epp->ep_mac_stats_pending = B_TRUE; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_stats_periodic( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __in uint16_t period_ms, __in boolean_t events) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); EFSYS_ASSERT(emop != NULL); if (emop->emo_stats_periodic == NULL) { rc = EINVAL; goto fail1; } if ((rc = emop->emo_stats_periodic(enp, esmp, period_ms, events)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_mac_stats_update( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __inout_ecount(EFX_MAC_NSTATS) efsys_stat_t *essp, __inout_opt uint32_t *generationp) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); EFSYS_ASSERT(emop != NULL); rc = emop->emo_stats_update(enp, esmp, essp, generationp); if (rc == 0) epp->ep_mac_stats_pending = B_FALSE; return (rc); } #endif /* EFSYS_OPT_MAC_STATS */ __checkReturn efx_rc_t efx_mac_select( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); efx_mac_type_t type = EFX_MAC_INVALID; const efx_mac_ops_t *emop; int rc = EINVAL; switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: emop = &__efx_siena_mac_ops; type = EFX_MAC_SIENA; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: emop = &__efx_ef10_mac_ops; type = EFX_MAC_HUNTINGTON; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: emop = &__efx_ef10_mac_ops; type = EFX_MAC_MEDFORD; break; #endif /* EFSYS_OPT_MEDFORD */ default: rc = EINVAL; goto fail1; } EFSYS_ASSERT(type != EFX_MAC_INVALID); EFSYS_ASSERT3U(type, <, EFX_MAC_NTYPES); EFSYS_ASSERT(emop != NULL); epp->ep_emop = emop; epp->ep_mac_type = type; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_SIENA #define EFX_MAC_HASH_BITS (1 << 8) /* Compute the multicast hash as used on Falcon and Siena. */ static void siena_mac_multicast_hash_compute( __in_ecount(6*count) uint8_t const *addrs, __in int count, __out efx_oword_t *hash_low, __out efx_oword_t *hash_high) { uint32_t crc, index; int i; EFSYS_ASSERT(hash_low != NULL); EFSYS_ASSERT(hash_high != NULL); EFX_ZERO_OWORD(*hash_low); EFX_ZERO_OWORD(*hash_high); for (i = 0; i < count; i++) { /* Calculate hash bucket (IEEE 802.3 CRC32 of the MAC addr) */ crc = efx_crc32_calculate(0xffffffff, addrs, EFX_MAC_ADDR_LEN); index = crc % EFX_MAC_HASH_BITS; if (index < 128) { EFX_SET_OWORD_BIT(*hash_low, index); } else { EFX_SET_OWORD_BIT(*hash_high, index - 128); } addrs += EFX_MAC_ADDR_LEN; } } static __checkReturn efx_rc_t siena_mac_multicast_list_set( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); const efx_mac_ops_t *emop = epp->ep_emop; efx_oword_t old_hash[2]; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); memcpy(old_hash, epp->ep_multicst_hash, sizeof (old_hash)); siena_mac_multicast_hash_compute( epp->ep_mulcst_addr_list, epp->ep_mulcst_addr_count, &epp->ep_multicst_hash[0], &epp->ep_multicst_hash[1]); if ((rc = emop->emo_reconfigure(enp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); memcpy(epp->ep_multicst_hash, old_hash, sizeof (old_hash)); return (rc); } #endif /* EFSYS_OPT_SIENA */ Index: head/sys/dev/sfxge/common/efx_mon.c =================================================================== --- head/sys/dev/sfxge/common/efx_mon.c (revision 310689) +++ head/sys/dev/sfxge/common/efx_mon.c (revision 310690) @@ -1,258 +1,258 @@ /*- * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_MON_MCDI #include "mcdi_mon.h" #endif #if EFSYS_OPT_NAMES static const char *__efx_mon_name[] = { "", "sfx90x0", "sfx91x0", "sfx92x0" }; const char * efx_mon_name( __in efx_nic_t *enp) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID); EFSYS_ASSERT3U(encp->enc_mon_type, <, EFX_MON_NTYPES); return (__efx_mon_name[encp->enc_mon_type]); } #endif /* EFSYS_OPT_NAMES */ #if EFSYS_OPT_MON_MCDI static const efx_mon_ops_t __efx_mon_mcdi_ops = { #if EFSYS_OPT_MON_STATS mcdi_mon_stats_update /* emo_stats_update */ #endif /* EFSYS_OPT_MON_STATS */ }; #endif __checkReturn efx_rc_t efx_mon_init( __in efx_nic_t *enp) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_mon_t *emp = &(enp->en_mon); const efx_mon_ops_t *emop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); if (enp->en_mod_flags & EFX_MOD_MON) { rc = EINVAL; goto fail1; } enp->en_mod_flags |= EFX_MOD_MON; emp->em_type = encp->enc_mon_type; EFSYS_ASSERT(encp->enc_mon_type != EFX_MON_INVALID); switch (emp->em_type) { #if EFSYS_OPT_MON_MCDI case EFX_MON_SFC90X0: case EFX_MON_SFC91X0: case EFX_MON_SFC92X0: emop = &__efx_mon_mcdi_ops; break; #endif default: rc = ENOTSUP; goto fail2; } emp->em_emop = emop; return (0); fail2: EFSYS_PROBE(fail2); emp->em_type = EFX_MON_INVALID; enp->en_mod_flags &= ~EFX_MOD_MON; fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_MON_STATS #if EFSYS_OPT_NAMES -/* START MKCONFIG GENERATED MonitorStatNamesBlock 31f437eafb0b0437 */ -static const char *__mon_stat_name[] = { +/* START MKCONFIG GENERATED MonitorStatNamesBlock 5daa2a5725ba734b */ +static const char * const __mon_stat_name[] = { "value_2_5v", "value_vccp1", "value_vcc", "value_5v", "value_12v", "value_vccp2", "value_ext_temp", "value_int_temp", "value_ain1", "value_ain2", "controller_cooling", "ext_cooling", "1v", "1_2v", "1_8v", "3_3v", "1_2va", "vref", "vaoe", "aoe_temperature", "psu_aoe_temperature", "psu_temperature", "fan0", "fan1", "fan2", "fan3", "fan4", "vaoe_in", "iaoe", "iaoe_in", "nic_power", "0_9v", "i0_9v", "i1_2v", "0_9v_adc", "controller_temperature2", "vreg_temperature", "vreg_0_9v_temperature", "vreg_1_2v_temperature", "int_vptat", "controller_internal_adc_temperature", "ext_vptat", "controller_external_adc_temperature", "ambient_temperature", "airflow", "vdd08d_vss08d_csr", "vdd08d_vss08d_csr_extadc", "hotpoint_temperature", "phy_power_switch_port0", "phy_power_switch_port1", "mum_vcc", "0v9_a", "i0v9_a", "0v9_a_temp", "0v9_b", "i0v9_b", "0v9_b_temp", "ccom_avreg_1v2_supply", "ccom_avreg_1v2_supply_ext_adc", "ccom_avreg_1v8_supply", "ccom_avreg_1v8_supply_ext_adc", "controller_master_vptat", "controller_master_internal_temp", "controller_master_vptat_ext_adc", "controller_master_internal_temp_ext_adc", "controller_slave_vptat", "controller_slave_internal_temp", "controller_slave_vptat_ext_adc", "controller_slave_internal_temp_ext_adc", "sodimm_vout", "sodimm_0_temp", "sodimm_1_temp", "phy0_vcc", "phy1_vcc", "controller_tdiode_temp", "board_front_temp", "board_back_temp", }; /* END MKCONFIG GENERATED MonitorStatNamesBlock */ extern const char * efx_mon_stat_name( __in efx_nic_t *enp, __in efx_mon_stat_t id) { _NOTE(ARGUNUSED(enp)) EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(id, <, EFX_MON_NSTATS); return (__mon_stat_name[id]); } #endif /* EFSYS_OPT_NAMES */ __checkReturn efx_rc_t efx_mon_stats_update( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __inout_ecount(EFX_MON_NSTATS) efx_mon_stat_value_t *values) { efx_mon_t *emp = &(enp->en_mon); const efx_mon_ops_t *emop = emp->em_emop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON); return (emop->emo_stats_update(enp, esmp, values)); } #endif /* EFSYS_OPT_MON_STATS */ void efx_mon_fini( __in efx_nic_t *enp) { efx_mon_t *emp = &(enp->en_mon); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MON); emp->em_emop = NULL; emp->em_type = EFX_MON_INVALID; enp->en_mod_flags &= ~EFX_MOD_MON; } Index: head/sys/dev/sfxge/common/efx_phy.c =================================================================== --- head/sys/dev/sfxge/common/efx_phy.c (revision 310689) +++ head/sys/dev/sfxge/common/efx_phy.c (revision 310690) @@ -1,565 +1,565 @@ /*- * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_SIENA static const efx_phy_ops_t __efx_phy_siena_ops = { siena_phy_power, /* epo_power */ NULL, /* epo_reset */ siena_phy_reconfigure, /* epo_reconfigure */ siena_phy_verify, /* epo_verify */ siena_phy_oui_get, /* epo_oui_get */ #if EFSYS_OPT_PHY_STATS siena_phy_stats_update, /* epo_stats_update */ #endif /* EFSYS_OPT_PHY_STATS */ #if EFSYS_OPT_BIST NULL, /* epo_bist_enable_offline */ siena_phy_bist_start, /* epo_bist_start */ siena_phy_bist_poll, /* epo_bist_poll */ siena_phy_bist_stop, /* epo_bist_stop */ #endif /* EFSYS_OPT_BIST */ }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD static const efx_phy_ops_t __efx_phy_ef10_ops = { ef10_phy_power, /* epo_power */ NULL, /* epo_reset */ ef10_phy_reconfigure, /* epo_reconfigure */ ef10_phy_verify, /* epo_verify */ ef10_phy_oui_get, /* epo_oui_get */ #if EFSYS_OPT_PHY_STATS ef10_phy_stats_update, /* epo_stats_update */ #endif /* EFSYS_OPT_PHY_STATS */ #if EFSYS_OPT_BIST /* FIXME: Are these BIST methods appropriate for Medford? */ hunt_bist_enable_offline, /* epo_bist_enable_offline */ hunt_bist_start, /* epo_bist_start */ hunt_bist_poll, /* epo_bist_poll */ hunt_bist_stop, /* epo_bist_stop */ #endif /* EFSYS_OPT_BIST */ }; #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ __checkReturn efx_rc_t efx_phy_probe( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); efx_nic_cfg_t *encp = &(enp->en_nic_cfg); const efx_phy_ops_t *epop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); epp->ep_port = encp->enc_port; epp->ep_phy_type = encp->enc_phy_type; /* Hook in operations structure */ switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: epop = &__efx_phy_siena_ops; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: epop = &__efx_phy_ef10_ops; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: epop = &__efx_phy_ef10_ops; break; #endif /* EFSYS_OPT_MEDFORD */ default: rc = ENOTSUP; goto fail1; } epp->ep_epop = epop; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); epp->ep_port = 0; epp->ep_phy_type = 0; return (rc); } __checkReturn efx_rc_t efx_phy_verify( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); return (epop->epo_verify(enp)); } #if EFSYS_OPT_PHY_LED_CONTROL __checkReturn efx_rc_t efx_phy_led_set( __in efx_nic_t *enp, __in efx_phy_led_mode_t mode) { efx_nic_cfg_t *encp = (&enp->en_nic_cfg); efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; uint32_t mask; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (epp->ep_phy_led_mode == mode) goto done; mask = (1 << EFX_PHY_LED_DEFAULT); mask |= encp->enc_led_mask; if (!((1 << mode) & mask)) { rc = ENOTSUP; goto fail1; } EFSYS_ASSERT3U(mode, <, EFX_PHY_LED_NMODES); epp->ep_phy_led_mode = mode; if ((rc = epop->epo_reconfigure(enp)) != 0) goto fail2; done: return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #endif /* EFSYS_OPT_PHY_LED_CONTROL */ void efx_phy_adv_cap_get( __in efx_nic_t *enp, __in uint32_t flag, __out uint32_t *maskp) { efx_port_t *epp = &(enp->en_port); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE); switch (flag) { case EFX_PHY_CAP_CURRENT: *maskp = epp->ep_adv_cap_mask; break; case EFX_PHY_CAP_DEFAULT: *maskp = epp->ep_default_adv_cap_mask; break; case EFX_PHY_CAP_PERM: *maskp = epp->ep_phy_cap_mask; break; default: EFSYS_ASSERT(B_FALSE); break; } } __checkReturn efx_rc_t efx_phy_adv_cap_set( __in efx_nic_t *enp, __in uint32_t mask) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; uint32_t old_mask; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if ((mask & ~epp->ep_phy_cap_mask) != 0) { rc = ENOTSUP; goto fail1; } if (epp->ep_adv_cap_mask == mask) goto done; old_mask = epp->ep_adv_cap_mask; epp->ep_adv_cap_mask = mask; if ((rc = epop->epo_reconfigure(enp)) != 0) goto fail2; done: return (0); fail2: EFSYS_PROBE(fail2); epp->ep_adv_cap_mask = old_mask; /* Reconfigure for robustness */ if (epop->epo_reconfigure(enp) != 0) { /* * We may have an inconsistent view of our advertised speed * capabilities. */ EFSYS_ASSERT(0); } fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_phy_lp_cap_get( __in efx_nic_t *enp, __out uint32_t *maskp) { efx_port_t *epp = &(enp->en_port); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); *maskp = epp->ep_lp_cap_mask; } __checkReturn efx_rc_t efx_phy_oui_get( __in efx_nic_t *enp, __out uint32_t *ouip) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); return (epop->epo_oui_get(enp, ouip)); } void efx_phy_media_type_get( __in efx_nic_t *enp, __out efx_phy_media_type_t *typep) { efx_port_t *epp = &(enp->en_port); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); if (epp->ep_module_type != EFX_PHY_MEDIA_INVALID) *typep = epp->ep_module_type; else *typep = epp->ep_fixed_port_type; } __checkReturn efx_rc_t efx_phy_module_get_info( __in efx_nic_t *enp, __in uint8_t dev_addr, __in uint8_t offset, __in uint8_t len, __out_bcount(len) uint8_t *data) { efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT(data != NULL); if ((uint32_t)offset + len > 0xff) { rc = EINVAL; goto fail1; } if ((rc = efx_mcdi_phy_module_get_info(enp, dev_addr, offset, len, data)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } #if EFSYS_OPT_PHY_STATS #if EFSYS_OPT_NAMES -/* START MKCONFIG GENERATED PhyStatNamesBlock d5f79b4bc2c050fe */ -static const char *__efx_phy_stat_name[] = { +/* START MKCONFIG GENERATED PhyStatNamesBlock af9ffa24da3bc100 */ +static const char * const __efx_phy_stat_name[] = { "oui", "pma_pmd_link_up", "pma_pmd_rx_fault", "pma_pmd_tx_fault", "pma_pmd_rev_a", "pma_pmd_rev_b", "pma_pmd_rev_c", "pma_pmd_rev_d", "pcs_link_up", "pcs_rx_fault", "pcs_tx_fault", "pcs_ber", "pcs_block_errors", "phy_xs_link_up", "phy_xs_rx_fault", "phy_xs_tx_fault", "phy_xs_align", "phy_xs_sync_a", "phy_xs_sync_b", "phy_xs_sync_c", "phy_xs_sync_d", "an_link_up", "an_master", "an_local_rx_ok", "an_remote_rx_ok", "cl22ext_link_up", "snr_a", "snr_b", "snr_c", "snr_d", "pma_pmd_signal_a", "pma_pmd_signal_b", "pma_pmd_signal_c", "pma_pmd_signal_d", "an_complete", "pma_pmd_rev_major", "pma_pmd_rev_minor", "pma_pmd_rev_micro", "pcs_fw_version_0", "pcs_fw_version_1", "pcs_fw_version_2", "pcs_fw_version_3", "pcs_fw_build_yy", "pcs_fw_build_mm", "pcs_fw_build_dd", "pcs_op_mode", }; /* END MKCONFIG GENERATED PhyStatNamesBlock */ const char * efx_phy_stat_name( __in efx_nic_t *enp, __in efx_phy_stat_t type) { _NOTE(ARGUNUSED(enp)) EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(type, <, EFX_PHY_NSTATS); return (__efx_phy_stat_name[type]); } #endif /* EFSYS_OPT_NAMES */ __checkReturn efx_rc_t efx_phy_stats_update( __in efx_nic_t *enp, __in efsys_mem_t *esmp, __inout_ecount(EFX_PHY_NSTATS) uint32_t *stat) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PORT); return (epop->epo_stats_update(enp, esmp, stat)); } #endif /* EFSYS_OPT_PHY_STATS */ #if EFSYS_OPT_BIST __checkReturn efx_rc_t efx_bist_enable_offline( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); if (epop->epo_bist_enable_offline == NULL) { rc = ENOTSUP; goto fail1; } if ((rc = epop->epo_bist_enable_offline(enp)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_bist_start( __in efx_nic_t *enp, __in efx_bist_type_t type) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); EFSYS_ASSERT3U(epp->ep_current_bist, ==, EFX_BIST_TYPE_UNKNOWN); if (epop->epo_bist_start == NULL) { rc = ENOTSUP; goto fail1; } if ((rc = epop->epo_bist_start(enp, type)) != 0) goto fail2; epp->ep_current_bist = type; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_bist_poll( __in efx_nic_t *enp, __in efx_bist_type_t type, __out efx_bist_result_t *resultp, __out_opt uint32_t *value_maskp, __out_ecount_opt(count) unsigned long *valuesp, __in size_t count) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); EFSYS_ASSERT3U(epp->ep_current_bist, ==, type); EFSYS_ASSERT(epop->epo_bist_poll != NULL); if (epop->epo_bist_poll == NULL) { rc = ENOTSUP; goto fail1; } if ((rc = epop->epo_bist_poll(enp, type, resultp, value_maskp, valuesp, count)) != 0) goto fail2; return (0); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_bist_stop( __in efx_nic_t *enp, __in efx_bist_type_t type) { efx_port_t *epp = &(enp->en_port); const efx_phy_ops_t *epop = epp->ep_epop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(type, !=, EFX_BIST_TYPE_UNKNOWN); EFSYS_ASSERT3U(type, <, EFX_BIST_TYPE_NTYPES); EFSYS_ASSERT3U(epp->ep_current_bist, ==, type); EFSYS_ASSERT(epop->epo_bist_stop != NULL); if (epop->epo_bist_stop != NULL) epop->epo_bist_stop(enp, type); epp->ep_current_bist = EFX_BIST_TYPE_UNKNOWN; } #endif /* EFSYS_OPT_BIST */ void efx_phy_unprobe( __in efx_nic_t *enp) { efx_port_t *epp = &(enp->en_port); EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); epp->ep_epop = NULL; epp->ep_adv_cap_mask = 0; epp->ep_port = 0; epp->ep_phy_type = 0; } Index: head/sys/dev/sfxge/common/efx_tx.c =================================================================== --- head/sys/dev/sfxge/common/efx_tx.c (revision 310689) +++ head/sys/dev/sfxge/common/efx_tx.c (revision 310690) @@ -1,1093 +1,1093 @@ /*- * Copyright (c) 2007-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include __FBSDID("$FreeBSD$"); #include "efx.h" #include "efx_impl.h" #if EFSYS_OPT_QSTATS #define EFX_TX_QSTAT_INCR(_etp, _stat) \ do { \ (_etp)->et_stat[_stat]++; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #else #define EFX_TX_QSTAT_INCR(_etp, _stat) #endif #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_tx_init( __in efx_nic_t *enp); static void siena_tx_fini( __in efx_nic_t *enp); static __checkReturn efx_rc_t siena_tx_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in unsigned int label, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint16_t flags, __in efx_evq_t *eep, __in efx_txq_t *etp, __out unsigned int *addedp); static void siena_tx_qdestroy( __in efx_txq_t *etp); static __checkReturn efx_rc_t siena_tx_qpost( __in efx_txq_t *etp, __in_ecount(n) efx_buffer_t *eb, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp); static void siena_tx_qpush( __in efx_txq_t *etp, __in unsigned int added, __in unsigned int pushed); static __checkReturn efx_rc_t siena_tx_qpace( __in efx_txq_t *etp, __in unsigned int ns); static __checkReturn efx_rc_t siena_tx_qflush( __in efx_txq_t *etp); static void siena_tx_qenable( __in efx_txq_t *etp); __checkReturn efx_rc_t siena_tx_qdesc_post( __in efx_txq_t *etp, __in_ecount(n) efx_desc_t *ed, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp); void siena_tx_qdesc_dma_create( __in efx_txq_t *etp, __in efsys_dma_addr_t addr, __in size_t size, __in boolean_t eop, __out efx_desc_t *edp); #if EFSYS_OPT_QSTATS static void siena_tx_qstats_update( __in efx_txq_t *etp, __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); #endif #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_SIENA static const efx_tx_ops_t __efx_tx_siena_ops = { siena_tx_init, /* etxo_init */ siena_tx_fini, /* etxo_fini */ siena_tx_qcreate, /* etxo_qcreate */ siena_tx_qdestroy, /* etxo_qdestroy */ siena_tx_qpost, /* etxo_qpost */ siena_tx_qpush, /* etxo_qpush */ siena_tx_qpace, /* etxo_qpace */ siena_tx_qflush, /* etxo_qflush */ siena_tx_qenable, /* etxo_qenable */ NULL, /* etxo_qpio_enable */ NULL, /* etxo_qpio_disable */ NULL, /* etxo_qpio_write */ NULL, /* etxo_qpio_post */ siena_tx_qdesc_post, /* etxo_qdesc_post */ siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ NULL, /* etxo_qdesc_tso_create */ NULL, /* etxo_qdesc_tso2_create */ NULL, /* etxo_qdesc_vlantci_create */ #if EFSYS_OPT_QSTATS siena_tx_qstats_update, /* etxo_qstats_update */ #endif }; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON static const efx_tx_ops_t __efx_tx_hunt_ops = { ef10_tx_init, /* etxo_init */ ef10_tx_fini, /* etxo_fini */ ef10_tx_qcreate, /* etxo_qcreate */ ef10_tx_qdestroy, /* etxo_qdestroy */ ef10_tx_qpost, /* etxo_qpost */ ef10_tx_qpush, /* etxo_qpush */ ef10_tx_qpace, /* etxo_qpace */ ef10_tx_qflush, /* etxo_qflush */ ef10_tx_qenable, /* etxo_qenable */ ef10_tx_qpio_enable, /* etxo_qpio_enable */ ef10_tx_qpio_disable, /* etxo_qpio_disable */ ef10_tx_qpio_write, /* etxo_qpio_write */ ef10_tx_qpio_post, /* etxo_qpio_post */ ef10_tx_qdesc_post, /* etxo_qdesc_post */ ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ #if EFSYS_OPT_QSTATS ef10_tx_qstats_update, /* etxo_qstats_update */ #endif }; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD static const efx_tx_ops_t __efx_tx_medford_ops = { ef10_tx_init, /* etxo_init */ ef10_tx_fini, /* etxo_fini */ ef10_tx_qcreate, /* etxo_qcreate */ ef10_tx_qdestroy, /* etxo_qdestroy */ ef10_tx_qpost, /* etxo_qpost */ ef10_tx_qpush, /* etxo_qpush */ ef10_tx_qpace, /* etxo_qpace */ ef10_tx_qflush, /* etxo_qflush */ ef10_tx_qenable, /* etxo_qenable */ ef10_tx_qpio_enable, /* etxo_qpio_enable */ ef10_tx_qpio_disable, /* etxo_qpio_disable */ ef10_tx_qpio_write, /* etxo_qpio_write */ ef10_tx_qpio_post, /* etxo_qpio_post */ ef10_tx_qdesc_post, /* etxo_qdesc_post */ ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ NULL, /* etxo_qdesc_tso_create */ ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ #if EFSYS_OPT_QSTATS ef10_tx_qstats_update, /* etxo_qstats_update */ #endif }; #endif /* EFSYS_OPT_MEDFORD */ __checkReturn efx_rc_t efx_tx_init( __in efx_nic_t *enp) { const efx_tx_ops_t *etxop; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); if (!(enp->en_mod_flags & EFX_MOD_EV)) { rc = EINVAL; goto fail1; } if (enp->en_mod_flags & EFX_MOD_TX) { rc = EINVAL; goto fail2; } switch (enp->en_family) { #if EFSYS_OPT_SIENA case EFX_FAMILY_SIENA: etxop = &__efx_tx_siena_ops; break; #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_HUNTINGTON case EFX_FAMILY_HUNTINGTON: etxop = &__efx_tx_hunt_ops; break; #endif /* EFSYS_OPT_HUNTINGTON */ #if EFSYS_OPT_MEDFORD case EFX_FAMILY_MEDFORD: etxop = &__efx_tx_medford_ops; break; #endif /* EFSYS_OPT_MEDFORD */ default: EFSYS_ASSERT(0); rc = ENOTSUP; goto fail3; } EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); if ((rc = etxop->etxo_init(enp)) != 0) goto fail4; enp->en_etxop = etxop; enp->en_mod_flags |= EFX_MOD_TX; return (0); fail4: EFSYS_PROBE(fail4); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); enp->en_etxop = NULL; enp->en_mod_flags &= ~EFX_MOD_TX; return (rc); } void efx_tx_fini( __in efx_nic_t *enp) { const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); etxop->etxo_fini(enp); enp->en_etxop = NULL; enp->en_mod_flags &= ~EFX_MOD_TX; } __checkReturn efx_rc_t efx_tx_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in unsigned int label, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint16_t flags, __in efx_evq_t *eep, __deref_out efx_txq_t **etpp, __out unsigned int *addedp) { const efx_tx_ops_t *etxop = enp->en_etxop; efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_txq_t *etp; efx_rc_t rc; EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); /* Allocate an TXQ object */ EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); if (etp == NULL) { rc = ENOMEM; goto fail1; } etp->et_magic = EFX_TXQ_MAGIC; etp->et_enp = enp; etp->et_index = index; etp->et_mask = n - 1; etp->et_esmp = esmp; /* Initial descriptor index may be modified by etxo_qcreate */ *addedp = 0; if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, n, id, flags, eep, etp, addedp)) != 0) goto fail2; enp->en_tx_qcount++; *etpp = etp; return (0); fail2: EFSYS_PROBE(fail2); EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_tx_qdestroy( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); EFSYS_ASSERT(enp->en_tx_qcount != 0); --enp->en_tx_qcount; etxop->etxo_qdestroy(etp); /* Free the TXQ object */ EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); } __checkReturn efx_rc_t efx_tx_qpost( __in efx_txq_t *etp, __in_ecount(n) efx_buffer_t *eb, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if ((rc = etxop->etxo_qpost(etp, eb, n, completed, addedp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_tx_qpush( __in efx_txq_t *etp, __in unsigned int added, __in unsigned int pushed) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); etxop->etxo_qpush(etp, added, pushed); } __checkReturn efx_rc_t efx_tx_qpace( __in efx_txq_t *etp, __in unsigned int ns) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if ((rc = etxop->etxo_qpace(etp, ns)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_tx_qflush( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if ((rc = etxop->etxo_qflush(etp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_tx_qenable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); etxop->etxo_qenable(etp); } __checkReturn efx_rc_t efx_tx_qpio_enable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { rc = ENOTSUP; goto fail1; } if (etxop->etxo_qpio_enable == NULL) { rc = ENOTSUP; goto fail2; } if ((rc = etxop->etxo_qpio_enable(etp)) != 0) goto fail3; return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_tx_qpio_disable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if (etxop->etxo_qpio_disable != NULL) etxop->etxo_qpio_disable(etp); } __checkReturn efx_rc_t efx_tx_qpio_write( __in efx_txq_t *etp, __in_ecount(buf_length) uint8_t *buffer, __in size_t buf_length, __in size_t pio_buf_offset) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if (etxop->etxo_qpio_write != NULL) { if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, pio_buf_offset)) != 0) goto fail1; return (0); } return (ENOTSUP); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_tx_qpio_post( __in efx_txq_t *etp, __in size_t pkt_length, __in unsigned int completed, __inout unsigned int *addedp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if (etxop->etxo_qpio_post != NULL) { if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, addedp)) != 0) goto fail1; return (0); } return (ENOTSUP); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t efx_tx_qdesc_post( __in efx_txq_t *etp, __in_ecount(n) efx_desc_t *ed, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; efx_rc_t rc; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); if ((rc = etxop->etxo_qdesc_post(etp, ed, n, completed, addedp)) != 0) goto fail1; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void efx_tx_qdesc_dma_create( __in efx_txq_t *etp, __in efsys_dma_addr_t addr, __in size_t size, __in boolean_t eop, __out efx_desc_t *edp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); } void efx_tx_qdesc_tso_create( __in efx_txq_t *etp, __in uint16_t ipv4_id, __in uint32_t tcp_seq, __in uint8_t tcp_flags, __out efx_desc_t *edp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); } void efx_tx_qdesc_tso2_create( __in efx_txq_t *etp, __in uint16_t ipv4_id, __in uint32_t tcp_seq, __in uint16_t mss, __out_ecount(count) efx_desc_t *edp, __in int count) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count); } void efx_tx_qdesc_vlantci_create( __in efx_txq_t *etp, __in uint16_t tci, __out efx_desc_t *edp) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); etxop->etxo_qdesc_vlantci_create(etp, tci, edp); } #if EFSYS_OPT_QSTATS void efx_tx_qstats_update( __in efx_txq_t *etp, __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) { efx_nic_t *enp = etp->et_enp; const efx_tx_ops_t *etxop = enp->en_etxop; EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); etxop->etxo_qstats_update(etp, stat); } #endif #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t siena_tx_init( __in efx_nic_t *enp) { efx_oword_t oword; /* * Disable the timer-based TX DMA backoff and allow TX DMA to be * controlled by the RX FIFO fill level (although always allow a * minimal trickle). */ EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); /* * Filter all packets less than 14 bytes to avoid parsing * errors. */ EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); /* * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 * descriptors (which is bad). */ EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); return (0); } #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ do { \ unsigned int id; \ size_t offset; \ efx_qword_t qword; \ \ id = (_added)++ & (_etp)->et_mask; \ offset = id * sizeof (efx_qword_t); \ \ EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ unsigned int, id, efsys_dma_addr_t, (_addr), \ size_t, (_size), boolean_t, (_eop)); \ \ EFX_POPULATE_QWORD_4(qword, \ FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ FSF_AZ_TX_KER_BUF_ADDR_DW0, \ (uint32_t)((_addr) & 0xffffffff), \ FSF_AZ_TX_KER_BUF_ADDR_DW1, \ (uint32_t)((_addr) >> 32)); \ EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) static __checkReturn efx_rc_t siena_tx_qpost( __in efx_txq_t *etp, __in_ecount(n) efx_buffer_t *eb, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp) { unsigned int added = *addedp; unsigned int i; int rc = ENOSPC; if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) goto fail1; for (i = 0; i < n; i++) { efx_buffer_t *ebp = &eb[i]; efsys_dma_addr_t start = ebp->eb_addr; size_t size = ebp->eb_size; efsys_dma_addr_t end = start + size; /* Fragments must not span 4k boundaries. */ EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); } EFX_TX_QSTAT_INCR(etp, TX_POST); *addedp = added; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static void siena_tx_qpush( __in efx_txq_t *etp, __in unsigned int added, __in unsigned int pushed) { efx_nic_t *enp = etp->et_enp; uint32_t wptr; efx_dword_t dword; efx_oword_t oword; /* Push the populated descriptors out */ wptr = added & etp->et_mask; EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); /* Only write the third DWORD */ EFX_POPULATE_DWORD_1(dword, EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, wptr, pushed & etp->et_mask); EFSYS_PIO_WRITE_BARRIER(); EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, etp->et_index, &dword, B_FALSE); } #define EFX_MAX_PACE_VALUE 20 #define EFX_TX_PACE_CLOCK_BASE 104 static __checkReturn efx_rc_t siena_tx_qpace( __in efx_txq_t *etp, __in unsigned int ns) { efx_nic_t *enp = etp->et_enp; efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_oword_t oword; unsigned int pace_val; unsigned int timer_period; efx_rc_t rc; if (ns == 0) { pace_val = 0; } else { /* * The pace_val to write into the table is s.t * ns <= timer_period * (2 ^ pace_val) */ timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { if ((timer_period << pace_val) >= ns) break; } } if (pace_val > EFX_MAX_PACE_VALUE) { rc = EINVAL; goto fail1; } /* Update the pacing table */ EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, &oword, B_TRUE); return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } static __checkReturn efx_rc_t siena_tx_qflush( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; efx_oword_t oword; uint32_t label; efx_tx_qpace(etp, 0); label = etp->et_index; /* Flush the queue */ EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, FRF_AZ_TX_FLUSH_DESCQ, label); EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); return (0); } static void siena_tx_qenable( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; efx_oword_t oword; EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, etp->et_index, &oword, B_TRUE); EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, etp->et_index, &oword, B_TRUE); } static __checkReturn efx_rc_t siena_tx_qcreate( __in efx_nic_t *enp, __in unsigned int index, __in unsigned int label, __in efsys_mem_t *esmp, __in size_t n, __in uint32_t id, __in uint16_t flags, __in efx_evq_t *eep, __in efx_txq_t *etp, __out unsigned int *addedp) { efx_nic_cfg_t *encp = &(enp->en_nic_cfg); efx_oword_t oword; uint32_t size; efx_rc_t rc; _NOTE(ARGUNUSED(esmp)) EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { rc = EINVAL; goto fail1; } if (index >= encp->enc_txq_limit) { rc = EINVAL; goto fail2; } for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); size++) if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) break; if (id + (1 << size) >= encp->enc_buftbl_limit) { rc = EINVAL; goto fail3; } /* Set up the new descriptor queue */ *addedp = 0; EFX_POPULATE_OWORD_6(oword, FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, FRF_AZ_TX_DESCQ_OWNER_ID, 0, FRF_AZ_TX_DESCQ_LABEL, label, FRF_AZ_TX_DESCQ_SIZE, size, FRF_AZ_TX_DESCQ_TYPE, 0); EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, etp->et_index, &oword, B_TRUE); return (0); fail3: EFSYS_PROBE(fail3); fail2: EFSYS_PROBE(fail2); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } __checkReturn efx_rc_t siena_tx_qdesc_post( __in efx_txq_t *etp, __in_ecount(n) efx_desc_t *ed, __in unsigned int n, __in unsigned int completed, __inout unsigned int *addedp) { unsigned int added = *addedp; unsigned int i; efx_rc_t rc; if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { rc = ENOSPC; goto fail1; } for (i = 0; i < n; i++) { efx_desc_t *edp = &ed[i]; unsigned int id; size_t offset; id = added++ & etp->et_mask; offset = id * sizeof (efx_desc_t); EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); } EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, unsigned int, added, unsigned int, n); EFX_TX_QSTAT_INCR(etp, TX_POST); *addedp = added; return (0); fail1: EFSYS_PROBE1(fail1, efx_rc_t, rc); return (rc); } void siena_tx_qdesc_dma_create( __in efx_txq_t *etp, __in efsys_dma_addr_t addr, __in size_t size, __in boolean_t eop, __out efx_desc_t *edp) { /* Fragments must not span 4k boundaries. */ EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, efsys_dma_addr_t, addr, size_t, size, boolean_t, eop); EFX_POPULATE_QWORD_4(edp->ed_eq, FSF_AZ_TX_KER_CONT, eop ? 0 : 1, FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, FSF_AZ_TX_KER_BUF_ADDR_DW0, (uint32_t)(addr & 0xffffffff), FSF_AZ_TX_KER_BUF_ADDR_DW1, (uint32_t)(addr >> 32)); } #endif /* EFSYS_OPT_SIENA */ #if EFSYS_OPT_QSTATS #if EFSYS_OPT_NAMES -/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ -static const char *__efx_tx_qstat_name[] = { +/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ +static const char * const __efx_tx_qstat_name[] = { "post", "post_pio", }; /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ const char * efx_tx_qstat_name( __in efx_nic_t *enp, __in unsigned int id) { _NOTE(ARGUNUSED(enp)) EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); EFSYS_ASSERT3U(id, <, TX_NQSTATS); return (__efx_tx_qstat_name[id]); } #endif /* EFSYS_OPT_NAMES */ #endif /* EFSYS_OPT_QSTATS */ #if EFSYS_OPT_SIENA #if EFSYS_OPT_QSTATS static void siena_tx_qstats_update( __in efx_txq_t *etp, __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) { unsigned int id; for (id = 0; id < TX_NQSTATS; id++) { efsys_stat_t *essp = &stat[id]; EFSYS_STAT_INCR(essp, etp->et_stat[id]); etp->et_stat[id] = 0; } } #endif /* EFSYS_OPT_QSTATS */ static void siena_tx_qdestroy( __in efx_txq_t *etp) { efx_nic_t *enp = etp->et_enp; efx_oword_t oword; /* Purge descriptor queue */ EFX_ZERO_OWORD(oword); EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, etp->et_index, &oword, B_TRUE); } static void siena_tx_fini( __in efx_nic_t *enp) { _NOTE(ARGUNUSED(enp)) } #endif /* EFSYS_OPT_SIENA */