Index: head/sys/dev/sfxge/common/efsys.h =================================================================== --- head/sys/dev/sfxge/common/efsys.h (revision 278220) +++ head/sys/dev/sfxge/common/efsys.h (revision 278221) @@ -1,831 +1,840 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_EFSYS_H #define _SYS_EFSYS_H #ifdef __cplusplus extern "C" { #endif #include #include #include #include #include #include #include #include #include #include #include #include #define EFSYS_HAS_UINT64 1 #define EFSYS_USE_UINT64 0 #if _BYTE_ORDER == _BIG_ENDIAN #define EFSYS_IS_BIG_ENDIAN 1 #define EFSYS_IS_LITTLE_ENDIAN 0 #elif _BYTE_ORDER == _LITTLE_ENDIAN #define EFSYS_IS_BIG_ENDIAN 0 #define EFSYS_IS_LITTLE_ENDIAN 1 #endif #include "efx_types.h" /* Common code requires this */ #if __FreeBSD_version < 800068 #define memmove(d, s, l) bcopy(s, d, l) #endif /* FreeBSD equivalents of Solaris things */ #ifndef _NOTE #define _NOTE(s) #endif #ifndef B_FALSE #define B_FALSE FALSE #endif #ifndef B_TRUE #define B_TRUE TRUE #endif #ifndef IS_P2ALIGNED #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) #endif #ifndef P2ROUNDUP #define P2ROUNDUP(x, align) (-(-(x) & -(align))) #endif #ifndef IS2P #define ISP2(x) (((x) & ((x) - 1)) == 0) #endif #define ENOTACTIVE EINVAL /* Memory type to use on FreeBSD */ MALLOC_DECLARE(M_SFXGE); /* Machine dependend prefetch wrappers */ #if defined(__i386__) || defined(__amd64__) static __inline void prefetch_read_many(void *addr) { __asm__( "prefetcht0 (%0)" : : "r" (addr)); } static __inline void prefetch_read_once(void *addr) { __asm__( "prefetchnta (%0)" : : "r" (addr)); } #elif defined(__sparc64__) static __inline void prefetch_read_many(void *addr) { __asm__( "prefetch [%0], 0" : : "r" (addr)); } static __inline void prefetch_read_once(void *addr) { __asm__( "prefetch [%0], 1" : : "r" (addr)); } #else static __inline void prefetch_read_many(void *addr) { } static __inline void prefetch_read_once(void *addr) { } #endif #if defined(__i386__) || defined(__amd64__) #include #include #endif static __inline void sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf *m, bus_dma_segment_t *seg) { #if defined(__i386__) || defined(__amd64__) seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); seg->ds_len = m->m_len; #else int nsegstmp; bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); #endif } /* Modifiers used for DOS builds */ #define __cs #define __far /* Modifiers used for Windows builds */ #define __in #define __in_opt #define __in_ecount(_n) #define __in_ecount_opt(_n) #define __in_bcount(_n) #define __in_bcount_opt(_n) #define __out #define __out_opt #define __out_ecount(_n) #define __out_ecount_opt(_n) #define __out_bcount(_n) #define __out_bcount_opt(_n) #define __deref_out #define __inout #define __inout_opt #define __inout_ecount(_n) #define __inout_ecount_opt(_n) #define __inout_bcount(_n) #define __inout_bcount_opt(_n) #define __inout_bcount_full_opt(_n) #define __deref_out_bcount_opt(n) #define __checkReturn #define __drv_when(_p, _c) /* Code inclusion options */ #define EFSYS_OPT_NAMES 1 #define EFSYS_OPT_FALCON 0 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 #define EFSYS_OPT_SIENA 1 #ifdef DEBUG #define EFSYS_OPT_CHECK_REG 1 #else #define EFSYS_OPT_CHECK_REG 0 #endif #define EFSYS_OPT_MCDI 1 #define EFSYS_OPT_MAC_FALCON_GMAC 0 #define EFSYS_OPT_MAC_FALCON_XMAC 0 #define EFSYS_OPT_MAC_STATS 1 #define EFSYS_OPT_LOOPBACK 0 #define EFSYS_OPT_MON_NULL 0 #define EFSYS_OPT_MON_LM87 0 #define EFSYS_OPT_MON_MAX6647 0 #define EFSYS_OPT_MON_SIENA 0 #define EFSYS_OPT_MON_STATS 0 #define EFSYS_OPT_PHY_NULL 0 #define EFSYS_OPT_PHY_QT2022C2 0 #define EFSYS_OPT_PHY_SFX7101 0 #define EFSYS_OPT_PHY_TXC43128 0 #define EFSYS_OPT_PHY_PM8358 0 #define EFSYS_OPT_PHY_SFT9001 0 #define EFSYS_OPT_PHY_QT2025C 0 #define EFSYS_OPT_PHY_STATS 1 #define EFSYS_OPT_PHY_PROPS 0 #define EFSYS_OPT_PHY_BIST 1 #define EFSYS_OPT_PHY_LED_CONTROL 1 #define EFSYS_OPT_PHY_FLAGS 0 #define EFSYS_OPT_VPD 1 #define EFSYS_OPT_NVRAM 1 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 #define EFSYS_OPT_NVRAM_SFT9001 0 #define EFSYS_OPT_NVRAM_SFX7101 0 #define EFSYS_OPT_BOOTCFG 0 #define EFSYS_OPT_PCIE_TUNE 0 #define EFSYS_OPT_DIAG 0 #define EFSYS_OPT_WOL 1 #define EFSYS_OPT_RX_SCALE 1 #define EFSYS_OPT_QSTATS 1 #define EFSYS_OPT_FILTER 0 #define EFSYS_OPT_RX_SCATTER 0 #define EFSYS_OPT_RX_HDR_SPLIT 0 #define EFSYS_OPT_EV_PREFETCH 0 #define EFSYS_OPT_DECODE_INTR_FATAL 1 /* ID */ typedef struct __efsys_identifier_s efsys_identifier_t; /* PROBE */ #ifndef DTRACE_PROBE #define EFSYS_PROBE(_name) #define EFSYS_PROBE1(_name, _type1, _arg1) #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) #else /* DTRACE_PROBE */ #define EFSYS_PROBE(_name) \ DTRACE_PROBE(_name) #define EFSYS_PROBE1(_name, _type1, _arg1) \ DTRACE_PROBE1(_name, _type1, _arg1) #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) \ DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3) #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) \ DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #ifdef DTRACE_PROBE5 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) \ DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #else #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) \ DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4) #endif #ifdef DTRACE_PROBE6 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) \ DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #else #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) \ EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5) #endif #ifdef DTRACE_PROBE7 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) \ DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) #else #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6, _type7, _arg7) \ EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ _type3, _arg3, _type4, _arg4, _type5, _arg5, \ _type6, _arg6) #endif #endif /* DTRACE_PROBE */ /* DMA */ typedef uint64_t efsys_dma_addr_t; typedef struct efsys_mem_s { bus_dma_tag_t esm_tag; bus_dmamap_t esm_map; caddr_t esm_base; efsys_dma_addr_t esm_addr; } efsys_mem_t; #define EFSYS_MEM_ZERO(_esmp, _size) \ do { \ (void) memset((_esmp)->esm_base, 0, (_size)); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_edp)->ed_u32[0] = *addr; \ \ EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eqp)->eq_u32[0] = *addr++; \ (_eqp)->eq_u32[1] = *addr; \ \ EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ (_eop)->eo_u32[0] = *addr++; \ (_eop)->eo_u32[1] = *addr++; \ (_eop)->eo_u32[2] = *addr++; \ (_eop)->eo_u32[3] = *addr; \ \ EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr = (_edp)->ed_u32[0]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr++ = (_eqp)->eq_u32[0]; \ *addr = (_eqp)->eq_u32[1]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ do { \ uint32_t *addr; \ \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ addr = (void *)((_esmp)->esm_base + (_offset)); \ \ *addr++ = (_eop)->eo_u32[0]; \ *addr++ = (_eop)->eo_u32[1]; \ *addr++ = (_eop)->eo_u32[2]; \ *addr = (_eop)->eo_u32[3]; \ \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_MEM_ADDR(_esmp) \ ((_esmp)->esm_addr) /* BAR */ typedef struct efsys_bar_s { struct mtx esb_lock; bus_space_tag_t esb_tag; bus_space_handle_t esb_handle; int esb_rid; struct resource *esb_res; } efsys_bar_t; +#define SFXGE_BAR_LOCK_INIT(_esbp, _name) \ + mtx_init(&(_esbp)->esb_lock, (_name), NULL, MTX_DEF) +#define SFXGE_BAR_LOCK_DESTROY(_esbp) \ + mtx_destroy(&(_esbp)->esb_lock) +#define SFXGE_BAR_LOCK(_esbp) \ + mtx_lock(&(_esbp)->esb_lock) +#define SFXGE_BAR_UNLOCK(_esbp) \ + mtx_unlock(&(_esbp)->esb_lock) + #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset)); \ \ EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset)); \ (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset+4)); \ \ EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset)); \ (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset+4)); \ (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset+8)); \ (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \ (_esbp)->esb_handle, (_offset+12)); \ \ EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ uint32_t, (_edp)->ed_u32[0]); \ \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), (_edp)->ed_u32[0]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ ("not power of 2 aligned")); \ \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ uint32_t, (_eqp)->eq_u32[1], \ uint32_t, (_eqp)->eq_u32[0]); \ \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), (_eqp)->eq_u32[0]); \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset+4), (_eqp)->eq_u32[1]); \ \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ do { \ _NOTE(CONSTANTCONDITION) \ KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ ("not power of 2 aligned")); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_lock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_LOCK(_esbp); \ \ EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ uint32_t, (_eop)->eo_u32[3], \ uint32_t, (_eop)->eo_u32[2], \ uint32_t, (_eop)->eo_u32[1], \ uint32_t, (_eop)->eo_u32[0]); \ \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset), (_eop)->eo_u32[0]); \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset+4), (_eop)->eo_u32[1]); \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset+8), (_eop)->eo_u32[2]); \ bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ (_offset+12), (_eop)->eo_u32[3]); \ \ _NOTE(CONSTANTCONDITION) \ if (_lock) \ - mtx_unlock(&((_esbp)->esb_lock)); \ + SFXGE_BAR_UNLOCK(_esbp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* SPIN */ #define EFSYS_SPIN(_us) \ do { \ DELAY(_us); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_SLEEP EFSYS_SPIN /* BARRIERS */ #define EFSYS_MEM_READ_BARRIER() rmb() #define EFSYS_PIO_WRITE_BARRIER() /* TIMESTAMP */ typedef clock_t efsys_timestamp_t; #define EFSYS_TIMESTAMP(_usp) \ do { \ clock_t now; \ \ now = ticks; \ *(_usp) = now * hz / 1000000; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* KMEM */ #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ do { \ (_esip) = (_esip); \ (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_KMEM_FREE(_esip, _size, _p) \ do { \ (void) (_esip); \ (void) (_size); \ free((_p), M_SFXGE); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* LOCK */ typedef struct mtx efsys_lock_t; #define EFSYS_LOCK_MAGIC 0x000010c4 #define EFSYS_LOCK(_lockp, _state) \ do { \ mtx_lock(_lockp); \ (_state) = EFSYS_LOCK_MAGIC; \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_UNLOCK(_lockp, _state) \ do { \ if ((_state) != EFSYS_LOCK_MAGIC) \ KASSERT(B_FALSE, ("not locked")); \ mtx_unlock(_lockp); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* PREEMPT */ #define EFSYS_PREEMPT_DISABLE(_state) \ do { \ (_state) = (_state); \ critical_enter(); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_PREEMPT_ENABLE(_state) \ do { \ (_state) = (_state); \ critical_exit(_state); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* STAT */ typedef uint64_t efsys_stat_t; #define EFSYS_STAT_INCR(_knp, _delta) \ do { \ *(_knp) += (_delta); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_DECR(_knp, _delta) \ do { \ *(_knp) -= (_delta); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET(_knp, _val) \ do { \ *(_knp) = (_val); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ do { \ *(_knp) = le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ do { \ *(_knp) = le32toh((_valp)->ed_u32[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ do { \ *(_knp) += le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ do { \ *(_knp) -= le64toh((_valp)->eq_u64[0]); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) /* ERR */ extern void sfxge_err(efsys_identifier_t *, unsigned int, uint32_t, uint32_t); #if EFSYS_OPT_DECODE_INTR_FATAL #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ do { \ sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ _NOTE(CONSTANTCONDITION) \ } while (B_FALSE) #endif /* ASSERT */ #define EFSYS_ASSERT(_exp) do { \ if (!(_exp)) \ panic(#_exp); \ } while (0) #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ const _t __x = (_t)(_x); \ const _t __y = (_t)(_y); \ if (!(__x _op __y)) \ panic("assertion failed at %s:%u", __FILE__, __LINE__); \ } while(0) #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) #ifdef __cplusplus } #endif #endif /* _SYS_EFSYS_H */ Index: head/sys/dev/sfxge/sfxge.c =================================================================== --- head/sys/dev/sfxge/sfxge.c (revision 278220) +++ head/sys/dev/sfxge/sfxge.c (revision 278221) @@ -1,821 +1,821 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" #include "sfxge_rx.h" #define SFXGE_CAP (IFCAP_VLAN_MTU | \ IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO | \ IFCAP_JUMBO_MTU | IFCAP_LRO | \ IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE) #define SFXGE_CAP_ENABLE SFXGE_CAP #define SFXGE_CAP_FIXED (IFCAP_VLAN_MTU | IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | \ IFCAP_JUMBO_MTU | IFCAP_LINKSTATE) MALLOC_DEFINE(M_SFXGE, "sfxge", "Solarflare 10GigE driver"); SYSCTL_NODE(_hw, OID_AUTO, sfxge, CTLFLAG_RD, 0, "SFXGE driver parameters"); #define SFXGE_PARAM_RX_RING SFXGE_PARAM(rx_ring) static int sfxge_rx_ring_entries = SFXGE_NDESCS; TUNABLE_INT(SFXGE_PARAM_RX_RING, &sfxge_rx_ring_entries); SYSCTL_INT(_hw_sfxge, OID_AUTO, rx_ring, CTLFLAG_RDTUN, &sfxge_rx_ring_entries, 0, "Maximum number of descriptors in a receive ring"); #define SFXGE_PARAM_TX_RING SFXGE_PARAM(tx_ring) static int sfxge_tx_ring_entries = SFXGE_NDESCS; TUNABLE_INT(SFXGE_PARAM_TX_RING, &sfxge_tx_ring_entries); SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_ring, CTLFLAG_RDTUN, &sfxge_tx_ring_entries, 0, "Maximum number of descriptors in a transmit ring"); static void sfxge_reset(void *arg, int npending); static int sfxge_start(struct sfxge_softc *sc) { int rc; - sx_assert(&sc->softc_lock, LA_XLOCKED); + SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc); if (sc->init_state == SFXGE_STARTED) return (0); if (sc->init_state != SFXGE_REGISTERED) { rc = EINVAL; goto fail; } if ((rc = efx_nic_init(sc->enp)) != 0) goto fail; /* Start processing interrupts. */ if ((rc = sfxge_intr_start(sc)) != 0) goto fail2; /* Start processing events. */ if ((rc = sfxge_ev_start(sc)) != 0) goto fail3; /* Start the receiver side. */ if ((rc = sfxge_rx_start(sc)) != 0) goto fail4; /* Start the transmitter side. */ if ((rc = sfxge_tx_start(sc)) != 0) goto fail5; /* Fire up the port. */ if ((rc = sfxge_port_start(sc)) != 0) goto fail6; sc->init_state = SFXGE_STARTED; /* Tell the stack we're running. */ sc->ifnet->if_drv_flags |= IFF_DRV_RUNNING; sc->ifnet->if_drv_flags &= ~IFF_DRV_OACTIVE; return (0); fail6: sfxge_tx_stop(sc); fail5: sfxge_rx_stop(sc); fail4: sfxge_ev_stop(sc); fail3: sfxge_intr_stop(sc); fail2: efx_nic_fini(sc->enp); fail: device_printf(sc->dev, "sfxge_start: %d\n", rc); return (rc); } static void sfxge_if_init(void *arg) { struct sfxge_softc *sc; sc = (struct sfxge_softc *)arg; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); (void)sfxge_start(sc); - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); } static void sfxge_stop(struct sfxge_softc *sc) { - sx_assert(&sc->softc_lock, LA_XLOCKED); + SFXGE_ADAPTER_LOCK_ASSERT_OWNED(sc); if (sc->init_state != SFXGE_STARTED) return; sc->init_state = SFXGE_REGISTERED; /* Stop the port. */ sfxge_port_stop(sc); /* Stop the transmitter. */ sfxge_tx_stop(sc); /* Stop the receiver. */ sfxge_rx_stop(sc); /* Stop processing events. */ sfxge_ev_stop(sc); /* Stop processing interrupts. */ sfxge_intr_stop(sc); efx_nic_fini(sc->enp); sc->ifnet->if_drv_flags &= ~IFF_DRV_RUNNING; } static int sfxge_if_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) { struct sfxge_softc *sc; struct ifreq *ifr; int error; ifr = (struct ifreq *)data; sc = ifp->if_softc; error = 0; switch (command) { case SIOCSIFFLAGS: - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); if (ifp->if_flags & IFF_UP) { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if ((ifp->if_flags ^ sc->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { sfxge_mac_filter_set(sc); } } else sfxge_start(sc); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) sfxge_stop(sc); sc->if_flags = ifp->if_flags; - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); break; case SIOCSIFMTU: if (ifr->ifr_mtu == ifp->if_mtu) { /* Nothing to do */ error = 0; } else if (ifr->ifr_mtu > SFXGE_MAX_MTU) { error = EINVAL; } else if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { ifp->if_mtu = ifr->ifr_mtu; error = 0; } else { /* Restart required */ - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); sfxge_stop(sc); ifp->if_mtu = ifr->ifr_mtu; error = sfxge_start(sc); - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); if (error != 0) { ifp->if_flags &= ~IFF_UP; ifp->if_drv_flags &= ~IFF_DRV_RUNNING; if_down(ifp); } } break; case SIOCADDMULTI: case SIOCDELMULTI: if (ifp->if_drv_flags & IFF_DRV_RUNNING) sfxge_mac_filter_set(sc); break; case SIOCSIFCAP: - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); /* * The networking core already rejects attempts to * enable capabilities we don't have. We still have * to reject attempts to disable capabilities that we * can't (yet) disable. */ if (~ifr->ifr_reqcap & SFXGE_CAP_FIXED) { error = EINVAL; - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); break; } ifp->if_capenable = ifr->ifr_reqcap; if (ifp->if_capenable & IFCAP_TXCSUM) ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); else ifp->if_hwassist &= ~(CSUM_IP | CSUM_TCP | CSUM_UDP); if (ifp->if_capenable & IFCAP_TSO) ifp->if_hwassist |= CSUM_TSO; else ifp->if_hwassist &= ~CSUM_TSO; - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->media, command); break; default: error = ether_ioctl(ifp, command, data); } return (error); } static void sfxge_ifnet_fini(struct ifnet *ifp) { struct sfxge_softc *sc = ifp->if_softc; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); sfxge_stop(sc); - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); ifmedia_removeall(&sc->media); ether_ifdetach(ifp); if_free(ifp); } static int sfxge_ifnet_init(struct ifnet *ifp, struct sfxge_softc *sc) { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sc->enp); device_t dev; int rc; dev = sc->dev; sc->ifnet = ifp; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_init = sfxge_if_init; ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = sfxge_if_ioctl; ifp->if_capabilities = SFXGE_CAP; ifp->if_capenable = SFXGE_CAP_ENABLE; ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO; ether_ifattach(ifp, encp->enc_mac_addr); #ifdef SFXGE_HAVE_MQ ifp->if_transmit = sfxge_if_transmit; ifp->if_qflush = sfxge_if_qflush; #else ifp->if_start = sfxge_if_start; IFQ_SET_MAXLEN(&ifp->if_snd, sc->txq_entries - 1); ifp->if_snd.ifq_drv_maxlen = sc->txq_entries - 1; IFQ_SET_READY(&ifp->if_snd); mtx_init(&sc->tx_lock, "txq", NULL, MTX_DEF); #endif if ((rc = sfxge_port_ifmedia_init(sc)) != 0) goto fail; return (0); fail: ether_ifdetach(sc->ifnet); return (rc); } void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp) { KASSERT(sc->buffer_table_next + n <= efx_nic_cfg_get(sc->enp)->enc_buftbl_limit, ("buffer table full")); *idp = sc->buffer_table_next; sc->buffer_table_next += n; } static int sfxge_bar_init(struct sfxge_softc *sc) { efsys_bar_t *esbp = &sc->bar; esbp->esb_rid = PCIR_BAR(EFX_MEM_BAR); if ((esbp->esb_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &esbp->esb_rid, RF_ACTIVE)) == NULL) { device_printf(sc->dev, "Cannot allocate BAR region %d\n", EFX_MEM_BAR); return (ENXIO); } esbp->esb_tag = rman_get_bustag(esbp->esb_res); esbp->esb_handle = rman_get_bushandle(esbp->esb_res); - mtx_init(&esbp->esb_lock, "sfxge_efsys_bar", NULL, MTX_DEF); + SFXGE_BAR_LOCK_INIT(esbp, "sfxge_efsys_bar"); return (0); } static void sfxge_bar_fini(struct sfxge_softc *sc) { efsys_bar_t *esbp = &sc->bar; bus_release_resource(sc->dev, SYS_RES_MEMORY, esbp->esb_rid, esbp->esb_res); - mtx_destroy(&esbp->esb_lock); + SFXGE_BAR_LOCK_DESTROY(esbp); } static int sfxge_create(struct sfxge_softc *sc) { device_t dev; efx_nic_t *enp; int error; char rss_param_name[sizeof(SFXGE_PARAM(%d.max_rss_channels))]; dev = sc->dev; - sx_init(&sc->softc_lock, "sfxge_softc"); + SFXGE_ADAPTER_LOCK_INIT(sc, "sfxge_softc"); sc->max_rss_channels = 0; snprintf(rss_param_name, sizeof(rss_param_name), SFXGE_PARAM(%d.max_rss_channels), (int)device_get_unit(dev)); TUNABLE_INT_FETCH(rss_param_name, &sc->max_rss_channels); sc->stats_node = SYSCTL_ADD_NODE( device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "stats", CTLFLAG_RD, NULL, "Statistics"); if (sc->stats_node == NULL) { error = ENOMEM; goto fail; } TASK_INIT(&sc->task_reset, 0, sfxge_reset, sc); (void) pci_enable_busmaster(dev); /* Initialize DMA mappings. */ if ((error = sfxge_dma_init(sc)) != 0) goto fail; /* Map the device registers. */ if ((error = sfxge_bar_init(sc)) != 0) goto fail; error = efx_family(pci_get_vendor(dev), pci_get_device(dev), &sc->family); KASSERT(error == 0, ("Family should be filtered by sfxge_probe()")); /* Create the common code nic object. */ mtx_init(&sc->enp_lock, "sfxge_nic", NULL, MTX_DEF); if ((error = efx_nic_create(sc->family, (efsys_identifier_t *)sc, &sc->bar, &sc->enp_lock, &enp)) != 0) goto fail3; sc->enp = enp; if (!ISP2(sfxge_rx_ring_entries) || !(sfxge_rx_ring_entries & EFX_RXQ_NDESCS_MASK)) { log(LOG_ERR, "%s=%d must be power of 2 from %u to %u", SFXGE_PARAM_RX_RING, sfxge_rx_ring_entries, EFX_RXQ_MINNDESCS, EFX_RXQ_MAXNDESCS); error = EINVAL; goto fail_rx_ring_entries; } sc->rxq_entries = sfxge_rx_ring_entries; if (!ISP2(sfxge_tx_ring_entries) || !(sfxge_tx_ring_entries & EFX_TXQ_NDESCS_MASK)) { log(LOG_ERR, "%s=%d must be power of 2 from %u to %u", SFXGE_PARAM_TX_RING, sfxge_tx_ring_entries, EFX_TXQ_MINNDESCS, EFX_TXQ_MAXNDESCS); error = EINVAL; goto fail_tx_ring_entries; } sc->txq_entries = sfxge_tx_ring_entries; /* Initialize MCDI to talk to the microcontroller. */ if ((error = sfxge_mcdi_init(sc)) != 0) goto fail4; /* Probe the NIC and build the configuration data area. */ if ((error = efx_nic_probe(enp)) != 0) goto fail5; /* Initialize the NVRAM. */ if ((error = efx_nvram_init(enp)) != 0) goto fail6; /* Initialize the VPD. */ if ((error = efx_vpd_init(enp)) != 0) goto fail7; /* Reset the NIC. */ if ((error = efx_nic_reset(enp)) != 0) goto fail8; /* Initialize buffer table allocation. */ sc->buffer_table_next = 0; /* Set up interrupts. */ if ((error = sfxge_intr_init(sc)) != 0) goto fail8; /* Initialize event processing state. */ if ((error = sfxge_ev_init(sc)) != 0) goto fail11; /* Initialize receive state. */ if ((error = sfxge_rx_init(sc)) != 0) goto fail12; /* Initialize transmit state. */ if ((error = sfxge_tx_init(sc)) != 0) goto fail13; /* Initialize port state. */ if ((error = sfxge_port_init(sc)) != 0) goto fail14; sc->init_state = SFXGE_INITIALIZED; return (0); fail14: sfxge_tx_fini(sc); fail13: sfxge_rx_fini(sc); fail12: sfxge_ev_fini(sc); fail11: sfxge_intr_fini(sc); fail8: efx_vpd_fini(enp); fail7: efx_nvram_fini(enp); fail6: efx_nic_unprobe(enp); fail5: sfxge_mcdi_fini(sc); fail4: fail_tx_ring_entries: fail_rx_ring_entries: sc->enp = NULL; efx_nic_destroy(enp); mtx_destroy(&sc->enp_lock); fail3: sfxge_bar_fini(sc); (void) pci_disable_busmaster(sc->dev); fail: sc->dev = NULL; - sx_destroy(&sc->softc_lock); + SFXGE_ADAPTER_LOCK_DESTROY(sc); return (error); } static void sfxge_destroy(struct sfxge_softc *sc) { efx_nic_t *enp; /* Clean up port state. */ sfxge_port_fini(sc); /* Clean up transmit state. */ sfxge_tx_fini(sc); /* Clean up receive state. */ sfxge_rx_fini(sc); /* Clean up event processing state. */ sfxge_ev_fini(sc); /* Clean up interrupts. */ sfxge_intr_fini(sc); /* Tear down common code subsystems. */ efx_nic_reset(sc->enp); efx_vpd_fini(sc->enp); efx_nvram_fini(sc->enp); efx_nic_unprobe(sc->enp); /* Tear down MCDI. */ sfxge_mcdi_fini(sc); /* Destroy common code context. */ enp = sc->enp; sc->enp = NULL; efx_nic_destroy(enp); /* Free DMA memory. */ sfxge_dma_fini(sc); /* Free mapped BARs. */ sfxge_bar_fini(sc); (void) pci_disable_busmaster(sc->dev); taskqueue_drain(taskqueue_thread, &sc->task_reset); /* Destroy the softc lock. */ - sx_destroy(&sc->softc_lock); + SFXGE_ADAPTER_LOCK_DESTROY(sc); } static int sfxge_vpd_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; efx_vpd_value_t value; int rc; value.evv_tag = arg2 >> 16; value.evv_keyword = arg2 & 0xffff; if ((rc = efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value)) != 0) return (rc); return (SYSCTL_OUT(req, value.evv_value, value.evv_length)); } static void sfxge_vpd_try_add(struct sfxge_softc *sc, struct sysctl_oid_list *list, efx_vpd_tag_t tag, const char *keyword) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); efx_vpd_value_t value; /* Check whether VPD tag/keyword is present */ value.evv_tag = tag; value.evv_keyword = EFX_VPD_KEYWORD(keyword[0], keyword[1]); if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) != 0) return; SYSCTL_ADD_PROC( ctx, list, OID_AUTO, keyword, CTLTYPE_STRING|CTLFLAG_RD, sc, tag << 16 | EFX_VPD_KEYWORD(keyword[0], keyword[1]), sfxge_vpd_handler, "A", ""); } static int sfxge_vpd_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid *vpd_node; struct sysctl_oid_list *vpd_list; char keyword[3]; efx_vpd_value_t value; int rc; if ((rc = efx_vpd_size(sc->enp, &sc->vpd_size)) != 0) goto fail; sc->vpd_data = malloc(sc->vpd_size, M_SFXGE, M_WAITOK); if ((rc = efx_vpd_read(sc->enp, sc->vpd_data, sc->vpd_size)) != 0) goto fail2; /* Copy ID (product name) into device description, and log it. */ value.evv_tag = EFX_VPD_ID; if (efx_vpd_get(sc->enp, sc->vpd_data, sc->vpd_size, &value) == 0) { value.evv_value[value.evv_length] = 0; device_set_desc_copy(sc->dev, value.evv_value); device_printf(sc->dev, "%s\n", value.evv_value); } vpd_node = SYSCTL_ADD_NODE( ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "vpd", CTLFLAG_RD, NULL, "Vital Product Data"); vpd_list = SYSCTL_CHILDREN(vpd_node); /* Add sysctls for all expected and any vendor-defined keywords. */ sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "PN"); sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "EC"); sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, "SN"); keyword[0] = 'V'; keyword[2] = 0; for (keyword[1] = '0'; keyword[1] <= '9'; keyword[1]++) sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword); for (keyword[1] = 'A'; keyword[1] <= 'Z'; keyword[1]++) sfxge_vpd_try_add(sc, vpd_list, EFX_VPD_RO, keyword); return (0); fail2: free(sc->vpd_data, M_SFXGE); fail: return (rc); } static void sfxge_vpd_fini(struct sfxge_softc *sc) { free(sc->vpd_data, M_SFXGE); } static void sfxge_reset(void *arg, int npending) { struct sfxge_softc *sc; int rc; (void)npending; sc = (struct sfxge_softc *)arg; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); if (sc->init_state != SFXGE_STARTED) goto done; sfxge_stop(sc); efx_nic_reset(sc->enp); if ((rc = sfxge_start(sc)) != 0) device_printf(sc->dev, "reset failed (%d); interface is now stopped\n", rc); done: - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); } void sfxge_schedule_reset(struct sfxge_softc *sc) { taskqueue_enqueue(taskqueue_thread, &sc->task_reset); } static int sfxge_attach(device_t dev) { struct sfxge_softc *sc; struct ifnet *ifp; int error; sc = device_get_softc(dev); sc->dev = dev; /* Allocate ifnet. */ ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "Couldn't allocate ifnet\n"); error = ENOMEM; goto fail; } sc->ifnet = ifp; /* Initialize hardware. */ if ((error = sfxge_create(sc)) != 0) goto fail2; /* Create the ifnet for the port. */ if ((error = sfxge_ifnet_init(ifp, sc)) != 0) goto fail3; if ((error = sfxge_vpd_init(sc)) != 0) goto fail4; sc->init_state = SFXGE_REGISTERED; return (0); fail4: sfxge_ifnet_fini(ifp); fail3: sfxge_destroy(sc); fail2: if_free(sc->ifnet); fail: return (error); } static int sfxge_detach(device_t dev) { struct sfxge_softc *sc; sc = device_get_softc(dev); sfxge_vpd_fini(sc); /* Destroy the ifnet. */ sfxge_ifnet_fini(sc->ifnet); /* Tear down hardware. */ sfxge_destroy(sc); return (0); } static int sfxge_probe(device_t dev) { uint16_t pci_vendor_id; uint16_t pci_device_id; efx_family_t family; int rc; pci_vendor_id = pci_get_vendor(dev); pci_device_id = pci_get_device(dev); rc = efx_family(pci_vendor_id, pci_device_id, &family); if (rc != 0) return (ENXIO); KASSERT(family == EFX_FAMILY_SIENA, ("impossible controller family")); device_set_desc(dev, "Solarflare SFC9000 family"); return (0); } static device_method_t sfxge_methods[] = { DEVMETHOD(device_probe, sfxge_probe), DEVMETHOD(device_attach, sfxge_attach), DEVMETHOD(device_detach, sfxge_detach), DEVMETHOD_END }; static devclass_t sfxge_devclass; static driver_t sfxge_driver = { "sfxge", sfxge_methods, sizeof(struct sfxge_softc) }; DRIVER_MODULE(sfxge, pci, sfxge_driver, sfxge_devclass, 0, 0); Index: head/sys/dev/sfxge/sfxge.h =================================================================== --- head/sys/dev/sfxge/sfxge.h (revision 278220) +++ head/sys/dev/sfxge/sfxge.h (revision 278221) @@ -1,317 +1,361 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SFXGE_H #define _SFXGE_H #include #include #include #include #include #include #include #include #include #include #include #include /* * Backward-compatibility */ #ifndef CACHE_LINE_SIZE /* This should be right on most machines the driver will be used on, and * we needn't care too much about wasting a few KB per interface. */ #define CACHE_LINE_SIZE 128 #endif #ifndef IFCAP_LINKSTATE #define IFCAP_LINKSTATE 0 #endif #ifndef IFCAP_VLAN_HWTSO #define IFCAP_VLAN_HWTSO 0 #endif #ifndef IFM_10G_T #define IFM_10G_T IFM_UNKNOWN #endif #ifndef IFM_10G_KX4 #define IFM_10G_KX4 IFM_10G_CX4 #endif #if __FreeBSD_version >= 800054 /* Networking core is multiqueue aware. We can manage our own TX * queues and use m_pkthdr.flowid. */ #define SFXGE_HAVE_MQ #endif #if (__FreeBSD_version >= 800501 && __FreeBSD_version < 900000) || \ __FreeBSD_version >= 900003 #define SFXGE_HAVE_DESCRIBE_INTR #endif #ifdef IFM_ETH_RXPAUSE #define SFXGE_HAVE_PAUSE_MEDIAOPTS #endif #ifndef CTLTYPE_U64 #define CTLTYPE_U64 CTLTYPE_QUAD #endif #include "sfxge_rx.h" #include "sfxge_tx.h" #define ROUNDUP_POW_OF_TWO(_n) (1ULL << flsl((_n) - 1)) #define SFXGE_IP_ALIGN 2 #define SFXGE_ETHERTYPE_LOOPBACK 0x9000 /* Xerox loopback */ enum sfxge_evq_state { SFXGE_EVQ_UNINITIALIZED = 0, SFXGE_EVQ_INITIALIZED, SFXGE_EVQ_STARTING, SFXGE_EVQ_STARTED }; #define SFXGE_EV_BATCH 16384 struct sfxge_evq { /* Structure members below are sorted by usage order */ struct sfxge_softc *sc; struct mtx lock; unsigned int index; enum sfxge_evq_state init_state; efsys_mem_t mem; efx_evq_t *common; unsigned int read_ptr; boolean_t exception; unsigned int rx_done; unsigned int tx_done; /* Linked list of TX queues with completions to process */ struct sfxge_txq *txq; struct sfxge_txq **txqs; /* Structure members not used on event processing path */ unsigned int buf_base_id; unsigned int entries; } __aligned(CACHE_LINE_SIZE); #define SFXGE_NDESCS 1024 #define SFXGE_MODERATION 30 enum sfxge_intr_state { SFXGE_INTR_UNINITIALIZED = 0, SFXGE_INTR_INITIALIZED, SFXGE_INTR_TESTING, SFXGE_INTR_STARTED }; struct sfxge_intr_hdl { int eih_rid; void *eih_tag; struct resource *eih_res; }; struct sfxge_intr { enum sfxge_intr_state state; struct resource *msix_res; struct sfxge_intr_hdl *table; int n_alloc; int type; efsys_mem_t status; uint32_t zero_count; }; enum sfxge_mcdi_state { SFXGE_MCDI_UNINITIALIZED = 0, SFXGE_MCDI_INITIALIZED, SFXGE_MCDI_BUSY, SFXGE_MCDI_COMPLETED }; struct sfxge_mcdi { struct mtx lock; struct cv cv; enum sfxge_mcdi_state state; efx_mcdi_transport_t transport; }; struct sfxge_hw_stats { clock_t update_time; efsys_mem_t dma_buf; void *decode_buf; }; enum sfxge_port_state { SFXGE_PORT_UNINITIALIZED = 0, SFXGE_PORT_INITIALIZED, SFXGE_PORT_STARTED }; struct sfxge_port { struct sfxge_softc *sc; struct mtx lock; enum sfxge_port_state init_state; #ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS unsigned int wanted_fc; #endif struct sfxge_hw_stats phy_stats; struct sfxge_hw_stats mac_stats; efx_link_mode_t link_mode; }; enum sfxge_softc_state { SFXGE_UNINITIALIZED = 0, SFXGE_INITIALIZED, SFXGE_REGISTERED, SFXGE_STARTED }; struct sfxge_softc { device_t dev; struct sx softc_lock; enum sfxge_softc_state init_state; struct ifnet *ifnet; unsigned int if_flags; struct sysctl_oid *stats_node; struct sysctl_oid *txqs_node; struct task task_reset; efx_family_t family; caddr_t vpd_data; size_t vpd_size; efx_nic_t *enp; struct mtx enp_lock; unsigned int rxq_entries; unsigned int txq_entries; bus_dma_tag_t parent_dma_tag; efsys_bar_t bar; struct sfxge_intr intr; struct sfxge_mcdi mcdi; struct sfxge_port port; uint32_t buffer_table_next; struct sfxge_evq *evq[SFXGE_RX_SCALE_MAX]; unsigned int ev_moderation; #if EFSYS_OPT_QSTATS clock_t ev_stats_update_time; uint64_t ev_stats[EV_NQSTATS]; #endif unsigned int max_rss_channels; uma_zone_t rxq_cache; struct sfxge_rxq *rxq[SFXGE_RX_SCALE_MAX]; unsigned int rx_indir_table[SFXGE_RX_SCALE_MAX]; #ifdef SFXGE_HAVE_MQ struct sfxge_txq *txq[SFXGE_TXQ_NTYPES + SFXGE_RX_SCALE_MAX]; #else struct sfxge_txq *txq[SFXGE_TXQ_NTYPES]; #endif struct ifmedia media; size_t rx_prefix_size; size_t rx_buffer_size; uma_zone_t rx_buffer_zone; #ifndef SFXGE_HAVE_MQ struct mtx tx_lock __aligned(CACHE_LINE_SIZE); #endif }; #define SFXGE_LINK_UP(sc) ((sc)->port.link_mode != EFX_LINK_DOWN) #define SFXGE_RUNNING(sc) ((sc)->ifnet->if_drv_flags & IFF_DRV_RUNNING) #define SFXGE_PARAM(_name) "hw.sfxge." #_name SYSCTL_DECL(_hw_sfxge); /* * From sfxge.c. */ extern void sfxge_schedule_reset(struct sfxge_softc *sc); extern void sfxge_sram_buf_tbl_alloc(struct sfxge_softc *sc, size_t n, uint32_t *idp); /* * From sfxge_dma.c. */ extern int sfxge_dma_init(struct sfxge_softc *sc); extern void sfxge_dma_fini(struct sfxge_softc *sc); extern int sfxge_dma_alloc(struct sfxge_softc *sc, bus_size_t len, efsys_mem_t *esmp); extern void sfxge_dma_free(efsys_mem_t *esmp); extern int sfxge_dma_map_sg_collapse(bus_dma_tag_t tag, bus_dmamap_t map, struct mbuf **mp, bus_dma_segment_t *segs, int *nsegs, int maxsegs); /* * From sfxge_ev.c. */ extern int sfxge_ev_init(struct sfxge_softc *sc); extern void sfxge_ev_fini(struct sfxge_softc *sc); extern int sfxge_ev_start(struct sfxge_softc *sc); extern void sfxge_ev_stop(struct sfxge_softc *sc); extern int sfxge_ev_qpoll(struct sfxge_evq *evq); /* * From sfxge_intr.c. */ extern int sfxge_intr_init(struct sfxge_softc *sc); extern void sfxge_intr_fini(struct sfxge_softc *sc); extern int sfxge_intr_start(struct sfxge_softc *sc); extern void sfxge_intr_stop(struct sfxge_softc *sc); /* * From sfxge_mcdi.c. */ extern int sfxge_mcdi_init(struct sfxge_softc *sc); extern void sfxge_mcdi_fini(struct sfxge_softc *sc); /* * From sfxge_port.c. */ extern int sfxge_port_init(struct sfxge_softc *sc); extern void sfxge_port_fini(struct sfxge_softc *sc); extern int sfxge_port_start(struct sfxge_softc *sc); extern void sfxge_port_stop(struct sfxge_softc *sc); extern void sfxge_mac_link_update(struct sfxge_softc *sc, efx_link_mode_t mode); extern int sfxge_mac_filter_set(struct sfxge_softc *sc); extern int sfxge_port_ifmedia_init(struct sfxge_softc *sc); #define SFXGE_MAX_MTU (9 * 1024) +#define SFXGE_ADAPTER_LOCK_INIT(_sc, _name) \ + sx_init(&(_sc)->softc_lock, (_name)) +#define SFXGE_ADAPTER_LOCK_DESTROY(_sc) \ + sx_destroy(&(_sc)->softc_lock) +#define SFXGE_ADAPTER_LOCK(_sc) \ + sx_xlock(&(_sc)->softc_lock) +#define SFXGE_ADAPTER_UNLOCK(_sc) \ + sx_xunlock(&(_sc)->softc_lock) +#define SFXGE_ADAPTER_LOCK_ASSERT_OWNED(_sc) \ + sx_assert(&(_sc)->softc_lock, LA_XLOCKED) + +#define SFXGE_PORT_LOCK_INIT(_port, _name) \ + mtx_init(&(_port)->lock, (_name), NULL, MTX_DEF) +#define SFXGE_PORT_LOCK_DESTROY(_port) \ + mtx_destroy(&(_port)->lock) +#define SFXGE_PORT_LOCK(_port) \ + mtx_lock(&(_port)->lock) +#define SFXGE_PORT_UNLOCK(_port) \ + mtx_unlock(&(_port)->lock) +#define SFXGE_PORT_LOCK_ASSERT_OWNED(_port) \ + mtx_assert(&(_port)->lock, MA_OWNED) + +#define SFXGE_MCDI_LOCK_INIT(_mcdi, _name) \ + mtx_init(&(_mcdi)->lock, (_name), NULL, MTX_DEF) +#define SFXGE_MCDI_LOCK_DESTROY(_mcdi) \ + mtx_destroy(&(_mcdi)->lock) +#define SFXGE_MCDI_LOCK(_mcdi) \ + mtx_lock(&(_mcdi)->lock) +#define SFXGE_MCDI_UNLOCK(_mcdi) \ + mtx_unlock(&(_mcdi)->lock) +#define SFXGE_MCDI_LOCK_ASSERT_OWNED(_mcdi) \ + mtx_assert(&(_mcdi)->lock, MA_OWNED) + +#define SFXGE_EVQ_LOCK_INIT(_evq, _name) \ + mtx_init(&(_evq)->lock, (_name), NULL, MTX_DEF) +#define SFXGE_EVQ_LOCK_DESTROY(_evq) \ + mtx_destroy(&(_evq)->lock) +#define SFXGE_EVQ_LOCK(_evq) \ + mtx_lock(&(_evq)->lock) +#define SFXGE_EVQ_UNLOCK(_evq) \ + mtx_unlock(&(_evq)->lock) +#define SFXGE_EVQ_LOCK_ASSERT_OWNED(_evq) \ + mtx_assert(&(_evq)->lock, MA_OWNED) + #endif /* _SFXGE_H */ Index: head/sys/dev/sfxge/sfxge_ev.c =================================================================== --- head/sys/dev/sfxge/sfxge_ev.c (revision 278220) +++ head/sys/dev/sfxge/sfxge_ev.c (revision 278221) @@ -1,903 +1,903 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" static void sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) { struct sfxge_softc *sc; unsigned int index; struct sfxge_rxq *rxq; struct sfxge_txq *txq; sc = evq->sc; index = evq->index; rxq = sc->rxq[index]; if ((txq = evq->txq) != NULL) { evq->txq = NULL; evq->txqs = &(evq->txq); do { struct sfxge_txq *next; next = txq->next; txq->next = NULL; KASSERT(txq->evq_index == index, ("txq->evq_index != index")); if (txq->pending != txq->completed) sfxge_tx_qcomplete(txq, evq); txq = next; } while (txq != NULL); } if (rxq->pending != rxq->completed) sfxge_rx_qcomplete(rxq, eop); } static boolean_t sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int expected; struct sfxge_rx_sw_desc *rx_desc; evq = arg; sc = evq->sc; if (evq->exception) goto done; rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); if (rxq->init_state != SFXGE_RXQ_STARTED) goto done; expected = rxq->pending++ & rxq->ptr_mask; if (id != expected) { evq->exception = B_TRUE; device_printf(sc->dev, "RX completion out of order" " (id=%#x expected=%#x flags=%#x); resetting\n", id, expected, flags); sfxge_schedule_reset(sc); goto done; } rx_desc = &rxq->queue[id]; KASSERT(rx_desc->flags == EFX_DISCARD, ("rx_desc->flags != EFX_DISCARD")); rx_desc->flags = flags; KASSERT(size < (1 << 16), ("size > (1 << 16)")); rx_desc->size = (uint16_t)size; prefetch_read_many(rx_desc->mbuf); evq->rx_done++; if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) sfxge_ev_qcomplete(evq, B_FALSE); done: return (evq->rx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; sc = evq->sc; evq->exception = B_TRUE; if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { device_printf(sc->dev, "hardware exception (code=%u); resetting\n", code); sfxge_schedule_reset(sc); } return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; evq = sc->evq[index]; label = rxq_index; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_rxq *rxq; unsigned int index; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; rxq = sc->rxq[rxq_index]; KASSERT(rxq != NULL, ("rxq == NULL")); /* Resend a software event on the correct queue */ index = rxq->index; evq = sc->evq[index]; label = rxq_index; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static struct sfxge_txq * sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) { unsigned int index; KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); return (evq->sc->txq[index]); } static boolean_t sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) { struct sfxge_evq *evq; struct sfxge_txq *txq; unsigned int stop; unsigned int delta; evq = (struct sfxge_evq *)arg; txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); if (txq->init_state != SFXGE_TXQ_STARTED) goto done; stop = (id + 1) & txq->ptr_mask; id = txq->pending & txq->ptr_mask; delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); txq->pending += delta; evq->tx_done++; if (txq->next == NULL && evq->txqs != &(txq->next)) { *(evq->txqs) = txq; evq->txqs = &(txq->next); } if (txq->pending - txq->completed >= SFXGE_TX_BATCH) sfxge_tx_qcomplete(txq, evq); done: return (evq->tx_done >= SFXGE_EV_BATCH); } static boolean_t sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) { struct sfxge_evq *evq; struct sfxge_softc *sc; struct sfxge_txq *txq; unsigned int label; uint16_t magic; evq = (struct sfxge_evq *)arg; sc = evq->sc; txq = sc->txq[txq_index]; KASSERT(txq != NULL, ("txq == NULL")); KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, ("txq not initialized")); /* Resend a software event on the correct queue */ evq = sc->evq[txq->evq_index]; label = txq->type; KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); efx_ev_qpost(evq->common, magic); return (B_FALSE); } static boolean_t sfxge_ev_software(void *arg, uint16_t magic) { struct sfxge_evq *evq; struct sfxge_softc *sc; unsigned int label; evq = (struct sfxge_evq *)arg; sc = evq->sc; label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; switch (magic) { case SFXGE_MAGIC_RX_QFLUSH_DONE: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qflush_done(rxq); break; } case SFXGE_MAGIC_RX_QFLUSH_FAILED: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qflush_failed(rxq); break; } case SFXGE_MAGIC_RX_QREFILL: { struct sfxge_rxq *rxq = sc->rxq[label]; KASSERT(rxq != NULL, ("rxq == NULL")); KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); sfxge_rx_qrefill(rxq); break; } case SFXGE_MAGIC_TX_QFLUSH_DONE: { struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); KASSERT(txq != NULL, ("txq == NULL")); KASSERT(evq->index == txq->evq_index, ("evq->index != txq->evq_index")); sfxge_tx_qflush_done(txq); break; } default: break; } return (B_FALSE); } static boolean_t sfxge_ev_sram(void *arg, uint32_t code) { (void)arg; (void)code; switch (code) { case EFX_SRAM_UPDATE: EFSYS_PROBE(sram_update); break; case EFX_SRAM_CLEAR: EFSYS_PROBE(sram_clear); break; case EFX_SRAM_ILLEGAL_CLEAR: EFSYS_PROBE(sram_illegal_clear); break; default: KASSERT(B_FALSE, ("Impossible SRAM event")); break; } return (B_FALSE); } static boolean_t sfxge_ev_timer(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } static boolean_t sfxge_ev_wake_up(void *arg, uint32_t index) { (void)arg; (void)index; return (B_FALSE); } #if EFSYS_OPT_QSTATS static void sfxge_ev_stat_update(struct sfxge_softc *sc) { struct sfxge_evq *evq; unsigned int index; clock_t now; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) goto out; now = ticks; if (now - sc->ev_stats_update_time < hz) goto out; sc->ev_stats_update_time = now; /* Add event counts from each event queue in turn */ for (index = 0; index < sc->intr.n_alloc; index++) { evq = sc->evq[index]; - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); efx_ev_qstats_update(evq->common, sc->ev_stats); - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); } out: - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); } static int sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; sfxge_ev_stat_update(sc); return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); } static void sfxge_ev_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; char name[40]; stat_list = SYSCTL_CHILDREN(sc->stats_node); for (id = 0; id < EV_NQSTATS; id++) { snprintf(name, sizeof(name), "ev_%s", efx_ev_qstat_name(sc->enp, id)); SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, sc, id, sfxge_ev_stat_handler, "Q", ""); } } #endif /* EFSYS_OPT_QSTATS */ static void sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) { struct sfxge_evq *evq; efx_evq_t *eep; evq = sc->evq[idx]; eep = evq->common; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); (void)efx_ev_qmoderate(eep, us); } static int sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; struct sfxge_intr *intr = &sc->intr; unsigned int moderation; int error; int index; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); if (req->newptr != NULL) { if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) != 0) goto out; /* We may not be calling efx_ev_qmoderate() now, * so we have to range-check the value ourselves. */ if (moderation > efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { error = EINVAL; goto out; } sc->ev_moderation = moderation; if (intr->state == SFXGE_INTR_STARTED) { for (index = 0; index < intr->n_alloc; index++) sfxge_ev_qmoderate(sc, index, moderation); } } else { error = SYSCTL_OUT(req, &sc->ev_moderation, sizeof(sc->ev_moderation)); } out: - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); return (error); } static boolean_t sfxge_ev_initialized(void *arg) { struct sfxge_evq *evq; evq = (struct sfxge_evq *)arg; KASSERT(evq->init_state == SFXGE_EVQ_STARTING, ("evq not starting")); evq->init_state = SFXGE_EVQ_STARTED; return (0); } static boolean_t sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) { struct sfxge_evq *evq; struct sfxge_softc *sc; evq = (struct sfxge_evq *)arg; sc = evq->sc; sfxge_mac_link_update(sc, link_mode); return (0); } static const efx_ev_callbacks_t sfxge_ev_callbacks = { .eec_initialized = sfxge_ev_initialized, .eec_rx = sfxge_ev_rx, .eec_tx = sfxge_ev_tx, .eec_exception = sfxge_ev_exception, .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, .eec_txq_flush_done = sfxge_ev_txq_flush_done, .eec_software = sfxge_ev_software, .eec_sram = sfxge_ev_sram, .eec_wake_up = sfxge_ev_wake_up, .eec_timer = sfxge_ev_timer, .eec_link_change = sfxge_ev_link_change, }; int sfxge_ev_qpoll(struct sfxge_evq *evq) { int rc; - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); if (evq->init_state != SFXGE_EVQ_STARTING && evq->init_state != SFXGE_EVQ_STARTED) { rc = EINVAL; goto fail; } /* Synchronize the DMA memory for reading */ bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, BUS_DMASYNC_POSTREAD); KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); KASSERT(evq->txq == NULL, ("evq->txq != NULL")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); /* Poll the queue */ efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); evq->rx_done = 0; evq->tx_done = 0; /* Perform any pending completion processing */ sfxge_ev_qcomplete(evq, B_TRUE); /* Re-prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail; - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); return (0); fail: - mtx_unlock(&(evq->lock)); + SFXGE_EVQ_UNLOCK(evq); return (rc); } static void sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; evq->read_ptr = 0; evq->exception = B_FALSE; #if EFSYS_OPT_QSTATS /* Add event counts before discarding the common evq state */ efx_ev_qstats_update(evq->common, sc->ev_stats); #endif efx_ev_qdestroy(evq->common); efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); } static int sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int count; int rc; evq = sc->evq[index]; esmp = &evq->mem; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); /* Clear all events. */ (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); /* Program the buffer table. */ if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, EFX_EVQ_NBUFS(evq->entries))) != 0) return (rc); /* Create the common code event queue. */ if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, evq->buf_base_id, &evq->common)) != 0) goto fail; - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); /* Set the default moderation */ (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); /* Prime the event queue for interrupts */ if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) goto fail2; evq->init_state = SFXGE_EVQ_STARTING; - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); /* Wait for the initialization event */ count = 0; do { /* Pause for 100 ms */ pause("sfxge evq init", hz / 10); /* Check to see if the test event has been processed */ if (evq->init_state == SFXGE_EVQ_STARTED) goto done; } while (++count < 20); rc = ETIMEDOUT; goto fail3; done: return (0); fail3: - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); evq->init_state = SFXGE_EVQ_INITIALIZED; fail2: - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); efx_ev_qdestroy(evq->common); fail: efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, EFX_EVQ_NBUFS(evq->entries)); return (rc); } void sfxge_ev_stop(struct sfxge_softc *sc) { struct sfxge_intr *intr; efx_nic_t *enp; int index; intr = &sc->intr; enp = sc->enp; KASSERT(intr->state == SFXGE_INTR_STARTED, ("Interrupts not started")); /* Stop the event queue(s) */ index = intr->n_alloc; while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(enp); } int sfxge_ev_start(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_STARTED, ("intr->state != SFXGE_INTR_STARTED")); /* Initialize the event module */ if ((rc = efx_ev_init(sc->enp)) != 0) return (rc); /* Start the event queues */ for (index = 0; index < intr->n_alloc; index++) { if ((rc = sfxge_ev_qstart(sc, index)) != 0) goto fail; } return (0); fail: /* Stop the event queue(s) */ while (--index >= 0) sfxge_ev_qstop(sc, index); /* Tear down the event module */ efx_ev_fini(sc->enp); return (rc); } static void sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; evq = sc->evq[index]; KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, ("evq->init_state != SFXGE_EVQ_INITIALIZED")); KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); sfxge_dma_free(&evq->mem); sc->evq[index] = NULL; - mtx_destroy(&evq->lock); + SFXGE_EVQ_LOCK_DESTROY(evq); free(evq, M_SFXGE); } static int sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) { struct sfxge_evq *evq; efsys_mem_t *esmp; int rc; KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); evq->sc = sc; evq->index = index; sc->evq[index] = evq; esmp = &evq->mem; /* Build an event queue with room for one event per tx and rx buffer, * plus some extra for link state events and MCDI completions. * There are three tx queues in the first event queue and one in * other. */ if (index == 0) evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + 3 * sc->txq_entries + 128); else evq->entries = ROUNDUP_POW_OF_TWO(sc->rxq_entries + sc->txq_entries + 128); /* Initialise TX completion list */ evq->txqs = &evq->txq; /* Allocate DMA space. */ if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) return (rc); /* Allocate buffer table entries. */ sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), &evq->buf_base_id); - mtx_init(&evq->lock, "evq", NULL, MTX_DEF); + SFXGE_EVQ_LOCK_INIT(evq, "evq"); evq->init_state = SFXGE_EVQ_INITIALIZED; return (0); } void sfxge_ev_fini(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); sc->ev_moderation = 0; /* Tear down the event queue(s). */ index = intr->n_alloc; while (--index >= 0) sfxge_ev_qfini(sc, index); } int sfxge_ev_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); /* Set default interrupt moderation; add a sysctl to * read and change it. */ sc->ev_moderation = SFXGE_MODERATION; SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, sc, 0, sfxge_int_mod_handler, "IU", "sfxge interrupt moderation (us)"); /* * Initialize the event queue(s) - one per interrupt. */ for (index = 0; index < intr->n_alloc; index++) { if ((rc = sfxge_ev_qinit(sc, index)) != 0) goto fail; } #if EFSYS_OPT_QSTATS sfxge_ev_stat_init(sc); #endif return (0); fail: while (--index >= 0) sfxge_ev_qfini(sc, index); return (rc); } Index: head/sys/dev/sfxge/sfxge_mcdi.c =================================================================== --- head/sys/dev/sfxge/sfxge_mcdi.c (revision 278220) +++ head/sys/dev/sfxge/sfxge_mcdi.c (revision 278221) @@ -1,250 +1,248 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "common/efx.h" #include "common/efx_mcdi.h" #include "common/efx_regs_mcdi.h" #include "sfxge.h" #define SFXGE_MCDI_POLL_INTERVAL_MIN 10 /* 10us in 1us units */ #define SFXGE_MCDI_POLL_INTERVAL_MAX 100000 /* 100ms in 1us units */ #define SFXGE_MCDI_WATCHDOG_INTERVAL 10000000 /* 10s in 1us units */ /* Acquire exclusive access to MCDI for the duration of a request. */ static void sfxge_mcdi_acquire(struct sfxge_mcdi *mcdi) { - - mtx_lock(&mcdi->lock); + SFXGE_MCDI_LOCK(mcdi); KASSERT(mcdi->state != SFXGE_MCDI_UNINITIALIZED, ("MCDI not initialized")); while (mcdi->state != SFXGE_MCDI_INITIALIZED) (void)cv_wait_sig(&mcdi->cv, &mcdi->lock); mcdi->state = SFXGE_MCDI_BUSY; - mtx_unlock(&mcdi->lock); + SFXGE_MCDI_UNLOCK(mcdi); } /* Release ownership of MCDI on request completion. */ static void sfxge_mcdi_release(struct sfxge_mcdi *mcdi) { - - mtx_lock(&mcdi->lock); + SFXGE_MCDI_LOCK(mcdi); KASSERT((mcdi->state == SFXGE_MCDI_BUSY || mcdi->state == SFXGE_MCDI_COMPLETED), ("MCDI not busy or task not completed")); mcdi->state = SFXGE_MCDI_INITIALIZED; cv_broadcast(&mcdi->cv); - mtx_unlock(&mcdi->lock); + SFXGE_MCDI_UNLOCK(mcdi); } static void sfxge_mcdi_timeout(struct sfxge_softc *sc) { device_t dev = sc->dev; log(LOG_WARNING, "[%s%d] MC_TIMEOUT", device_get_name(dev), device_get_unit(dev)); EFSYS_PROBE(mcdi_timeout); sfxge_schedule_reset(sc); } static void sfxge_mcdi_poll(struct sfxge_softc *sc) { efx_nic_t *enp; clock_t delay_total; clock_t delay_us; boolean_t aborted; delay_total = 0; delay_us = SFXGE_MCDI_POLL_INTERVAL_MIN; enp = sc->enp; do { if (efx_mcdi_request_poll(enp)) { EFSYS_PROBE1(mcdi_delay, clock_t, delay_total); return; } if (delay_total > SFXGE_MCDI_WATCHDOG_INTERVAL) { aborted = efx_mcdi_request_abort(enp); KASSERT(aborted, ("abort failed")); sfxge_mcdi_timeout(sc); return; } /* Spin or block depending on delay interval. */ if (delay_us < 1000000) DELAY(delay_us); else pause("mcdi wait", delay_us * hz / 1000000); delay_total += delay_us; /* Exponentially back off the poll frequency. */ delay_us = delay_us * 2; if (delay_us > SFXGE_MCDI_POLL_INTERVAL_MAX) delay_us = SFXGE_MCDI_POLL_INTERVAL_MAX; } while (1); } static void sfxge_mcdi_execute(void *arg, efx_mcdi_req_t *emrp) { struct sfxge_softc *sc; struct sfxge_mcdi *mcdi; sc = (struct sfxge_softc *)arg; mcdi = &sc->mcdi; sfxge_mcdi_acquire(mcdi); /* Issue request and poll for completion. */ efx_mcdi_request_start(sc->enp, emrp, B_FALSE); sfxge_mcdi_poll(sc); sfxge_mcdi_release(mcdi); } static void sfxge_mcdi_ev_cpl(void *arg) { struct sfxge_softc *sc; struct sfxge_mcdi *mcdi; sc = (struct sfxge_softc *)arg; mcdi = &sc->mcdi; - mtx_lock(&mcdi->lock); + SFXGE_MCDI_LOCK(mcdi); KASSERT(mcdi->state == SFXGE_MCDI_BUSY, ("MCDI not busy")); mcdi->state = SFXGE_MCDI_COMPLETED; cv_broadcast(&mcdi->cv); - mtx_unlock(&mcdi->lock); + SFXGE_MCDI_UNLOCK(mcdi); } static void sfxge_mcdi_exception(void *arg, efx_mcdi_exception_t eme) { struct sfxge_softc *sc; device_t dev; sc = (struct sfxge_softc *)arg; dev = sc->dev; log(LOG_WARNING, "[%s%d] MC_%s", device_get_name(dev), device_get_unit(dev), (eme == EFX_MCDI_EXCEPTION_MC_REBOOT) ? "REBOOT" : (eme == EFX_MCDI_EXCEPTION_MC_BADASSERT) ? "BADASSERT" : "UNKNOWN"); EFSYS_PROBE(mcdi_exception); sfxge_schedule_reset(sc); } int sfxge_mcdi_init(struct sfxge_softc *sc) { efx_nic_t *enp; struct sfxge_mcdi *mcdi; efx_mcdi_transport_t *emtp; int rc; enp = sc->enp; mcdi = &sc->mcdi; emtp = &mcdi->transport; KASSERT(mcdi->state == SFXGE_MCDI_UNINITIALIZED, ("MCDI already initialized")); - mtx_init(&mcdi->lock, "sfxge_mcdi", NULL, MTX_DEF); + SFXGE_MCDI_LOCK_INIT(mcdi, "sfxge_mcdi"); mcdi->state = SFXGE_MCDI_INITIALIZED; emtp->emt_context = sc; emtp->emt_execute = sfxge_mcdi_execute; emtp->emt_ev_cpl = sfxge_mcdi_ev_cpl; emtp->emt_exception = sfxge_mcdi_exception; cv_init(&mcdi->cv, "sfxge_mcdi"); if ((rc = efx_mcdi_init(enp, emtp)) != 0) goto fail; return (0); fail: - mtx_destroy(&mcdi->lock); + SFXGE_MCDI_LOCK_DESTROY(mcdi); mcdi->state = SFXGE_MCDI_UNINITIALIZED; return (rc); } void sfxge_mcdi_fini(struct sfxge_softc *sc) { struct sfxge_mcdi *mcdi; efx_nic_t *enp; efx_mcdi_transport_t *emtp; enp = sc->enp; mcdi = &sc->mcdi; emtp = &mcdi->transport; - mtx_lock(&mcdi->lock); + SFXGE_MCDI_LOCK(mcdi); KASSERT(mcdi->state == SFXGE_MCDI_INITIALIZED, ("MCDI not initialized")); efx_mcdi_fini(enp); bzero(emtp, sizeof(*emtp)); cv_destroy(&mcdi->cv); - mtx_unlock(&mcdi->lock); + SFXGE_MCDI_UNLOCK(mcdi); - mtx_destroy(&mcdi->lock); + SFXGE_MCDI_LOCK_DESTROY(mcdi); } Index: head/sys/dev/sfxge/sfxge_port.c =================================================================== --- head/sys/dev/sfxge/sfxge_port.c (revision 278220) +++ head/sys/dev/sfxge/sfxge_port.c (revision 278221) @@ -1,798 +1,798 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include "common/efx.h" #include "sfxge.h" static int sfxge_mac_stat_update(struct sfxge_softc *sc) { struct sfxge_port *port = &sc->port; efsys_mem_t *esmp = &(port->mac_stats.dma_buf); clock_t now; unsigned int count; int rc; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); if (port->init_state != SFXGE_PORT_STARTED) { rc = 0; goto out; } now = ticks; if (now - port->mac_stats.update_time < hz) { rc = 0; goto out; } port->mac_stats.update_time = now; /* If we're unlucky enough to read statistics wduring the DMA, wait * up to 10ms for it to finish (typically takes <500us) */ for (count = 0; count < 100; ++count) { EFSYS_PROBE1(wait, unsigned int, count); /* Synchronize the DMA memory for reading */ bus_dmamap_sync(esmp->esm_tag, esmp->esm_map, BUS_DMASYNC_POSTREAD); /* Try to update the cached counters */ if ((rc = efx_mac_stats_update(sc->enp, esmp, port->mac_stats.decode_buf, NULL)) != EAGAIN) goto out; DELAY(100); } rc = ETIMEDOUT; out: - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (rc); } static int sfxge_mac_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; int rc; if ((rc = sfxge_mac_stat_update(sc)) != 0) return (rc); return (SYSCTL_OUT(req, (uint64_t *)sc->port.mac_stats.decode_buf + id, sizeof(uint64_t))); } static void sfxge_mac_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; const char *name; stat_list = SYSCTL_CHILDREN(sc->stats_node); /* Initialise the named stats */ for (id = 0; id < EFX_MAC_NSTATS; id++) { name = efx_mac_stat_name(sc->enp, id); SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, sc, id, sfxge_mac_stat_handler, "Q", ""); } } #ifdef SFXGE_HAVE_PAUSE_MEDIAOPTS static unsigned int sfxge_port_wanted_fc(struct sfxge_softc *sc) { struct ifmedia_entry *ifm = sc->media.ifm_cur; if (ifm->ifm_media == (IFM_ETHER | IFM_AUTO)) return (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE); return (((ifm->ifm_media & IFM_ETH_RXPAUSE) ? EFX_FCNTL_RESPOND : 0) | ((ifm->ifm_media & IFM_ETH_TXPAUSE) ? EFX_FCNTL_GENERATE : 0)); } static unsigned int sfxge_port_link_fc_ifm(struct sfxge_softc *sc) { unsigned int wanted_fc, link_fc; efx_mac_fcntl_get(sc->enp, &wanted_fc, &link_fc); return ((link_fc & EFX_FCNTL_RESPOND) ? IFM_ETH_RXPAUSE : 0) | ((link_fc & EFX_FCNTL_GENERATE) ? IFM_ETH_TXPAUSE : 0); } #else /* !SFXGE_HAVE_PAUSE_MEDIAOPTS */ static unsigned int sfxge_port_wanted_fc(struct sfxge_softc *sc) { return (sc->port.wanted_fc); } static unsigned int sfxge_port_link_fc_ifm(struct sfxge_softc *sc) { return (0); } static int sfxge_port_wanted_fc_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc; struct sfxge_port *port; unsigned int fcntl; int error; sc = arg1; port = &sc->port; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); if (req->newptr != NULL) { if ((error = SYSCTL_IN(req, &fcntl, sizeof(fcntl))) != 0) goto out; if (port->wanted_fc == fcntl) goto out; port->wanted_fc = fcntl; if (port->init_state != SFXGE_PORT_STARTED) goto out; error = efx_mac_fcntl_set(sc->enp, port->wanted_fc, B_TRUE); } else { error = SYSCTL_OUT(req, &port->wanted_fc, sizeof(port->wanted_fc)); } out: - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (error); } static int sfxge_port_link_fc_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc; struct sfxge_port *port; unsigned int wanted_fc, link_fc; int error; sc = arg1; port = &sc->port; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); if (port->init_state == SFXGE_PORT_STARTED && SFXGE_LINK_UP(sc)) efx_mac_fcntl_get(sc->enp, &wanted_fc, &link_fc); else link_fc = 0; error = SYSCTL_OUT(req, &link_fc, sizeof(link_fc)); - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (error); } #endif /* SFXGE_HAVE_PAUSE_MEDIAOPTS */ static const uint64_t sfxge_link_baudrate[EFX_LINK_NMODES] = { [EFX_LINK_10HDX] = IF_Mbps(10), [EFX_LINK_10FDX] = IF_Mbps(10), [EFX_LINK_100HDX] = IF_Mbps(100), [EFX_LINK_100FDX] = IF_Mbps(100), [EFX_LINK_1000HDX] = IF_Gbps(1), [EFX_LINK_1000FDX] = IF_Gbps(1), [EFX_LINK_10000FDX] = IF_Gbps(10), }; void sfxge_mac_link_update(struct sfxge_softc *sc, efx_link_mode_t mode) { struct sfxge_port *port; int link_state; port = &sc->port; if (port->link_mode == mode) return; port->link_mode = mode; /* Push link state update to the OS */ link_state = (port->link_mode != EFX_LINK_DOWN ? LINK_STATE_UP : LINK_STATE_DOWN); sc->ifnet->if_baudrate = sfxge_link_baudrate[port->link_mode]; if_link_state_change(sc->ifnet, link_state); } static void sfxge_mac_poll_work(void *arg, int npending) { struct sfxge_softc *sc; efx_nic_t *enp; struct sfxge_port *port; efx_link_mode_t mode; sc = (struct sfxge_softc *)arg; enp = sc->enp; port = &sc->port; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); if (port->init_state != SFXGE_PORT_STARTED) goto done; /* This may sleep waiting for MCDI completion */ (void)efx_port_poll(enp, &mode); sfxge_mac_link_update(sc, mode); done: - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); } static int sfxge_mac_filter_set_locked(struct sfxge_softc *sc) { unsigned int bucket[EFX_MAC_HASH_BITS]; struct ifnet *ifp = sc->ifnet; struct ifmultiaddr *ifma; struct sockaddr_dl *sa; efx_nic_t *enp = sc->enp; unsigned int index; int rc; /* Set promisc-unicast and broadcast filter bits */ if ((rc = efx_mac_filter_set(enp, !!(ifp->if_flags & IFF_PROMISC), B_TRUE)) != 0) return (rc); /* Set multicast hash filter */ if (ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) { for (index = 0; index < EFX_MAC_HASH_BITS; index++) bucket[index] = 1; } else { /* Broadcast frames also go through the multicast * filter, and the broadcast address hashes to * 0xff. */ bucket[0xff] = 1; if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family == AF_LINK) { sa = (struct sockaddr_dl *)ifma->ifma_addr; index = ether_crc32_le(LLADDR(sa), 6) & 0xff; bucket[index] = 1; } } if_maddr_runlock(ifp); } return (efx_mac_hash_set(enp, bucket)); } int sfxge_mac_filter_set(struct sfxge_softc *sc) { struct sfxge_port *port = &sc->port; int rc; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); /* * The function may be called without softc_lock held in the * case of SIOCADDMULTI and SIOCDELMULTI ioctls. ioctl handler * checks IFF_DRV_RUNNING flag which implies port started, but * it is not guaranteed to remain. softc_lock shared lock can't * be held in the case of these ioctls processing, since it * results in failure where kernel complains that non-sleepable * lock is held in sleeping thread. Both problems are repeatable * on LAG with LACP proto bring up. */ if (port->init_state == SFXGE_PORT_STARTED) rc = sfxge_mac_filter_set_locked(sc); else rc = 0; - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (rc); } void sfxge_port_stop(struct sfxge_softc *sc) { struct sfxge_port *port; efx_nic_t *enp; port = &sc->port; enp = sc->enp; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); KASSERT(port->init_state == SFXGE_PORT_STARTED, ("port not started")); port->init_state = SFXGE_PORT_INITIALIZED; port->mac_stats.update_time = 0; /* This may call MCDI */ (void)efx_mac_drain(enp, B_TRUE); (void)efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf, 0, B_FALSE); port->link_mode = EFX_LINK_UNKNOWN; /* Destroy the common code port object. */ efx_port_fini(sc->enp); - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); } int sfxge_port_start(struct sfxge_softc *sc) { uint8_t mac_addr[ETHER_ADDR_LEN]; struct ifnet *ifp = sc->ifnet; struct sfxge_port *port; efx_nic_t *enp; size_t pdu; int rc; port = &sc->port; enp = sc->enp; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); KASSERT(port->init_state == SFXGE_PORT_INITIALIZED, ("port not initialized")); /* Initialize the port object in the common code. */ if ((rc = efx_port_init(sc->enp)) != 0) goto fail; /* Set the SDU */ pdu = EFX_MAC_PDU(ifp->if_mtu); if ((rc = efx_mac_pdu_set(enp, pdu)) != 0) goto fail2; if ((rc = efx_mac_fcntl_set(enp, sfxge_port_wanted_fc(sc), B_TRUE)) != 0) goto fail2; /* Set the unicast address */ if_addr_rlock(ifp); bcopy(LLADDR((struct sockaddr_dl *)ifp->if_addr->ifa_addr), mac_addr, sizeof(mac_addr)); if_addr_runlock(ifp); if ((rc = efx_mac_addr_set(enp, mac_addr)) != 0) goto fail; sfxge_mac_filter_set_locked(sc); /* Update MAC stats by DMA every second */ if ((rc = efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf, 1000, B_FALSE)) != 0) goto fail2; if ((rc = efx_mac_drain(enp, B_FALSE)) != 0) goto fail3; if ((rc = efx_phy_adv_cap_set(sc->enp, sc->media.ifm_cur->ifm_data)) != 0) goto fail4; port->init_state = SFXGE_PORT_STARTED; /* Single poll in case there were missing initial events */ - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); sfxge_mac_poll_work(sc, 0); return (0); fail4: (void)efx_mac_drain(enp, B_TRUE); fail3: (void)efx_mac_stats_periodic(enp, &port->mac_stats.dma_buf, 0, B_FALSE); fail2: efx_port_fini(sc->enp); fail: - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (rc); } static int sfxge_phy_stat_update(struct sfxge_softc *sc) { struct sfxge_port *port = &sc->port; efsys_mem_t *esmp = &port->phy_stats.dma_buf; clock_t now; unsigned int count; int rc; - mtx_lock(&port->lock); + SFXGE_PORT_LOCK(port); if (port->init_state != SFXGE_PORT_STARTED) { rc = 0; goto out; } now = ticks; if (now - port->phy_stats.update_time < hz) { rc = 0; goto out; } port->phy_stats.update_time = now; /* If we're unlucky enough to read statistics wduring the DMA, wait * up to 10ms for it to finish (typically takes <500us) */ for (count = 0; count < 100; ++count) { EFSYS_PROBE1(wait, unsigned int, count); /* Synchronize the DMA memory for reading */ bus_dmamap_sync(esmp->esm_tag, esmp->esm_map, BUS_DMASYNC_POSTREAD); /* Try to update the cached counters */ if ((rc = efx_phy_stats_update(sc->enp, esmp, port->phy_stats.decode_buf)) != EAGAIN) goto out; DELAY(100); } rc = ETIMEDOUT; out: - mtx_unlock(&port->lock); + SFXGE_PORT_UNLOCK(port); return (rc); } static int sfxge_phy_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; int rc; if ((rc = sfxge_phy_stat_update(sc)) != 0) return (rc); return (SYSCTL_OUT(req, (uint32_t *)sc->port.phy_stats.decode_buf + id, sizeof(uint32_t))); } static void sfxge_phy_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; const char *name; uint64_t stat_mask = efx_nic_cfg_get(sc->enp)->enc_phy_stat_mask; stat_list = SYSCTL_CHILDREN(sc->stats_node); /* Initialise the named stats */ for (id = 0; id < EFX_PHY_NSTATS; id++) { if (!(stat_mask & ((uint64_t)1 << id))) continue; name = efx_phy_stat_name(sc->enp, id); SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, name, CTLTYPE_UINT|CTLFLAG_RD, sc, id, sfxge_phy_stat_handler, id == EFX_PHY_STAT_OUI ? "IX" : "IU", ""); } } void sfxge_port_fini(struct sfxge_softc *sc) { struct sfxge_port *port; efsys_mem_t *esmp; port = &sc->port; esmp = &port->mac_stats.dma_buf; KASSERT(port->init_state == SFXGE_PORT_INITIALIZED, ("Port not initialized")); port->init_state = SFXGE_PORT_UNINITIALIZED; port->link_mode = EFX_LINK_UNKNOWN; /* Finish with PHY DMA memory */ sfxge_dma_free(&port->phy_stats.dma_buf); free(port->phy_stats.decode_buf, M_SFXGE); sfxge_dma_free(esmp); free(port->mac_stats.decode_buf, M_SFXGE); - mtx_destroy(&port->lock); + SFXGE_PORT_LOCK_DESTROY(port); port->sc = NULL; } int sfxge_port_init(struct sfxge_softc *sc) { struct sfxge_port *port; struct sysctl_ctx_list *sysctl_ctx; struct sysctl_oid *sysctl_tree; efsys_mem_t *mac_stats_buf, *phy_stats_buf; int rc; port = &sc->port; mac_stats_buf = &port->mac_stats.dma_buf; phy_stats_buf = &port->phy_stats.dma_buf; KASSERT(port->init_state == SFXGE_PORT_UNINITIALIZED, ("Port already initialized")); port->sc = sc; - mtx_init(&port->lock, "sfxge_port", NULL, MTX_DEF); + SFXGE_PORT_LOCK_INIT(port, "sfxge_port"); port->phy_stats.decode_buf = malloc(EFX_PHY_NSTATS * sizeof(uint32_t), M_SFXGE, M_WAITOK | M_ZERO); if ((rc = sfxge_dma_alloc(sc, EFX_PHY_STATS_SIZE, phy_stats_buf)) != 0) goto fail; sfxge_phy_stat_init(sc); sysctl_ctx = device_get_sysctl_ctx(sc->dev); sysctl_tree = device_get_sysctl_tree(sc->dev); #ifndef SFXGE_HAVE_PAUSE_MEDIAOPTS /* If flow control cannot be configured or reported through * ifmedia, provide sysctls for it. */ port->wanted_fc = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE; SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "wanted_fc", CTLTYPE_UINT|CTLFLAG_RW, sc, 0, sfxge_port_wanted_fc_handler, "IU", "wanted flow control mode"); SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "link_fc", CTLTYPE_UINT|CTLFLAG_RD, sc, 0, sfxge_port_link_fc_handler, "IU", "link flow control mode"); #endif port->mac_stats.decode_buf = malloc(EFX_MAC_NSTATS * sizeof(uint64_t), M_SFXGE, M_WAITOK | M_ZERO); if ((rc = sfxge_dma_alloc(sc, EFX_MAC_STATS_SIZE, mac_stats_buf)) != 0) goto fail2; sfxge_mac_stat_init(sc); port->init_state = SFXGE_PORT_INITIALIZED; return (0); fail2: free(port->mac_stats.decode_buf, M_SFXGE); sfxge_dma_free(phy_stats_buf); fail: free(port->phy_stats.decode_buf, M_SFXGE); - (void)mtx_destroy(&port->lock); + SFXGE_PORT_LOCK_DESTROY(port); port->sc = NULL; return (rc); } static int sfxge_link_mode[EFX_PHY_MEDIA_NTYPES][EFX_LINK_NMODES] = { [EFX_PHY_MEDIA_CX4] = { [EFX_LINK_10000FDX] = IFM_ETHER | IFM_FDX | IFM_10G_CX4, }, [EFX_PHY_MEDIA_KX4] = { [EFX_LINK_10000FDX] = IFM_ETHER | IFM_FDX | IFM_10G_KX4, }, [EFX_PHY_MEDIA_XFP] = { /* Don't know the module type, but assume SR for now. */ [EFX_LINK_10000FDX] = IFM_ETHER | IFM_FDX | IFM_10G_SR, }, [EFX_PHY_MEDIA_SFP_PLUS] = { /* Don't know the module type, but assume SX/SR for now. */ [EFX_LINK_1000FDX] = IFM_ETHER | IFM_FDX | IFM_1000_SX, [EFX_LINK_10000FDX] = IFM_ETHER | IFM_FDX | IFM_10G_SR, }, [EFX_PHY_MEDIA_BASE_T] = { [EFX_LINK_10HDX] = IFM_ETHER | IFM_HDX | IFM_10_T, [EFX_LINK_10FDX] = IFM_ETHER | IFM_FDX | IFM_10_T, [EFX_LINK_100HDX] = IFM_ETHER | IFM_HDX | IFM_100_TX, [EFX_LINK_100FDX] = IFM_ETHER | IFM_FDX | IFM_100_TX, [EFX_LINK_1000HDX] = IFM_ETHER | IFM_HDX | IFM_1000_T, [EFX_LINK_1000FDX] = IFM_ETHER | IFM_FDX | IFM_1000_T, [EFX_LINK_10000FDX] = IFM_ETHER | IFM_FDX | IFM_10G_T, }, }; static void sfxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct sfxge_softc *sc; efx_phy_media_type_t medium_type; efx_link_mode_t mode; sc = ifp->if_softc; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (SFXGE_RUNNING(sc) && SFXGE_LINK_UP(sc)) { ifmr->ifm_status |= IFM_ACTIVE; efx_phy_media_type_get(sc->enp, &medium_type); mode = sc->port.link_mode; ifmr->ifm_active |= sfxge_link_mode[medium_type][mode]; ifmr->ifm_active |= sfxge_port_link_fc_ifm(sc); } - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); } static int sfxge_media_change(struct ifnet *ifp) { struct sfxge_softc *sc; struct ifmedia_entry *ifm; int rc; sc = ifp->if_softc; ifm = sc->media.ifm_cur; - sx_xlock(&sc->softc_lock); + SFXGE_ADAPTER_LOCK(sc); if (!SFXGE_RUNNING(sc)) { rc = 0; goto out; } rc = efx_mac_fcntl_set(sc->enp, sfxge_port_wanted_fc(sc), B_TRUE); if (rc != 0) goto out; rc = efx_phy_adv_cap_set(sc->enp, ifm->ifm_data); out: - sx_xunlock(&sc->softc_lock); + SFXGE_ADAPTER_UNLOCK(sc); return (rc); } int sfxge_port_ifmedia_init(struct sfxge_softc *sc) { efx_phy_media_type_t medium_type; uint32_t cap_mask, mode_cap_mask; efx_link_mode_t mode; int mode_ifm, best_mode_ifm = 0; int rc; /* We need port state to initialise the ifmedia list. */ if ((rc = efx_nic_init(sc->enp)) != 0) goto out; if ((rc = efx_port_init(sc->enp)) != 0) goto out2; /* * Register ifconfig callbacks for querying and setting the * link mode and link status. */ ifmedia_init(&sc->media, IFM_IMASK, sfxge_media_change, sfxge_media_status); /* * Map firmware medium type and capabilities to ifmedia types. * ifmedia does not distinguish between forcing the link mode * and disabling auto-negotiation. 1000BASE-T and 10GBASE-T * require AN even if only one link mode is enabled, and for * 100BASE-TX it is useful even if the link mode is forced. * Therefore we never disable auto-negotiation. * * Also enable and advertise flow control by default. */ efx_phy_media_type_get(sc->enp, &medium_type); efx_phy_adv_cap_get(sc->enp, EFX_PHY_CAP_PERM, &cap_mask); EFX_STATIC_ASSERT(EFX_LINK_10HDX == EFX_PHY_CAP_10HDX + 1); EFX_STATIC_ASSERT(EFX_LINK_10FDX == EFX_PHY_CAP_10FDX + 1); EFX_STATIC_ASSERT(EFX_LINK_100HDX == EFX_PHY_CAP_100HDX + 1); EFX_STATIC_ASSERT(EFX_LINK_100FDX == EFX_PHY_CAP_100FDX + 1); EFX_STATIC_ASSERT(EFX_LINK_1000HDX == EFX_PHY_CAP_1000HDX + 1); EFX_STATIC_ASSERT(EFX_LINK_1000FDX == EFX_PHY_CAP_1000FDX + 1); EFX_STATIC_ASSERT(EFX_LINK_10000FDX == EFX_PHY_CAP_10000FDX + 1); for (mode = EFX_LINK_10HDX; mode <= EFX_LINK_10000FDX; mode++) { mode_cap_mask = 1 << (mode - 1); mode_ifm = sfxge_link_mode[medium_type][mode]; if ((cap_mask & mode_cap_mask) && mode_ifm) { mode_cap_mask |= cap_mask & (1 << EFX_PHY_CAP_AN); #ifdef SFXGE_HAVE_PAUSE_MEDIAOPTS /* No flow-control */ ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL); /* Respond-only. If using AN, we implicitly * offer symmetric as well, but that doesn't * mean we *have* to generate pause frames. */ mode_cap_mask |= cap_mask & ((1 << EFX_PHY_CAP_PAUSE) | (1 << EFX_PHY_CAP_ASYM)); mode_ifm |= IFM_ETH_RXPAUSE; ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL); /* Symmetric */ mode_cap_mask &= ~(1 << EFX_PHY_CAP_ASYM); mode_ifm |= IFM_ETH_TXPAUSE; #else /* !SFXGE_HAVE_PAUSE_MEDIAOPTS */ mode_cap_mask |= cap_mask & (1 << EFX_PHY_CAP_PAUSE); #endif ifmedia_add(&sc->media, mode_ifm, mode_cap_mask, NULL); /* Link modes are numbered in order of speed, * so assume the last one available is the best. */ best_mode_ifm = mode_ifm; } } if (cap_mask & (1 << EFX_PHY_CAP_AN)) { /* Add autoselect mode. */ mode_ifm = IFM_ETHER | IFM_AUTO; ifmedia_add(&sc->media, mode_ifm, cap_mask & ~(1 << EFX_PHY_CAP_ASYM), NULL); best_mode_ifm = mode_ifm; } if (best_mode_ifm != 0) ifmedia_set(&sc->media, best_mode_ifm); /* Now discard port state until interface is started. */ efx_port_fini(sc->enp); out2: efx_nic_fini(sc->enp); out: return (rc); } Index: head/sys/dev/sfxge/sfxge_rx.c =================================================================== --- head/sys/dev/sfxge/sfxge_rx.c (revision 278220) +++ head/sys/dev/sfxge/sfxge_rx.c (revision 278221) @@ -1,1234 +1,1234 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" #include "sfxge_rx.h" #define RX_REFILL_THRESHOLD(_entries) (EFX_RXQ_LIMIT(_entries) * 9 / 10) /* Size of the LRO hash table. Must be a power of 2. A larger table * means we can accelerate a larger number of streams. */ static unsigned lro_table_size = 128; /* Maximum length of a hash chain. If chains get too long then the lookup * time increases and may exceed the benefit of LRO. */ static unsigned lro_chain_max = 20; /* Maximum time (in ticks) that a connection can be idle before it's LRO * state is discarded. */ static unsigned lro_idle_ticks; /* initialised in sfxge_rx_init() */ /* Number of packets with payload that must arrive in-order before a * connection is eligible for LRO. The idea is we should avoid coalescing * segments when the sender is in slow-start because reducing the ACK rate * can damage performance. */ static int lro_slow_start_packets = 2000; /* Number of packets with payload that must arrive in-order following loss * before a connection is eligible for LRO. The idea is we should avoid * coalescing segments when the sender is recovering from loss, because * reducing the ACK rate can damage performance. */ static int lro_loss_packets = 20; /* Flags for sfxge_lro_conn::l2_id; must not collide with EVL_VLID_MASK */ #define SFXGE_LRO_L2_ID_VLAN 0x4000 #define SFXGE_LRO_L2_ID_IPV6 0x8000 #define SFXGE_LRO_CONN_IS_VLAN_ENCAP(c) ((c)->l2_id & SFXGE_LRO_L2_ID_VLAN) #define SFXGE_LRO_CONN_IS_TCPIPV4(c) (!((c)->l2_id & SFXGE_LRO_L2_ID_IPV6)) /* Compare IPv6 addresses, avoiding conditional branches */ static __inline unsigned long ipv6_addr_cmp(const struct in6_addr *left, const struct in6_addr *right) { #if LONG_BIT == 64 const uint64_t *left64 = (const uint64_t *)left; const uint64_t *right64 = (const uint64_t *)right; return (left64[0] - right64[0]) | (left64[1] - right64[1]); #else return (left->s6_addr32[0] - right->s6_addr32[0]) | (left->s6_addr32[1] - right->s6_addr32[1]) | (left->s6_addr32[2] - right->s6_addr32[2]) | (left->s6_addr32[3] - right->s6_addr32[3]); #endif } void sfxge_rx_qflush_done(struct sfxge_rxq *rxq) { rxq->flush_state = SFXGE_FLUSH_DONE; } void sfxge_rx_qflush_failed(struct sfxge_rxq *rxq) { rxq->flush_state = SFXGE_FLUSH_FAILED; } static uint8_t toep_key[] = { 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa }; static void sfxge_rx_post_refill(void *arg) { struct sfxge_rxq *rxq = arg; struct sfxge_softc *sc; unsigned int index; struct sfxge_evq *evq; uint16_t magic; sc = rxq->sc; index = rxq->index; evq = sc->evq[index]; magic = SFXGE_MAGIC_RX_QREFILL | index; /* This is guaranteed due to the start/stop order of rx and ev */ KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq not started")); KASSERT(rxq->init_state == SFXGE_RXQ_STARTED, ("rxq not started")); efx_ev_qpost(evq->common, magic); } static void sfxge_rx_schedule_refill(struct sfxge_rxq *rxq, boolean_t retrying) { /* Initially retry after 100 ms, but back off in case of * repeated failures as we probably have to wait for the * administrator to raise the pool limit. */ if (retrying) rxq->refill_delay = min(rxq->refill_delay * 2, 10 * hz); else rxq->refill_delay = hz / 10; callout_reset_curcpu(&rxq->refill_callout, rxq->refill_delay, sfxge_rx_post_refill, rxq); } static inline struct mbuf *sfxge_rx_alloc_mbuf(struct sfxge_softc *sc) { struct mb_args args; struct mbuf *m; /* Allocate mbuf structure */ args.flags = M_PKTHDR; args.type = MT_DATA; m = (struct mbuf *)uma_zalloc_arg(zone_mbuf, &args, M_NOWAIT); /* Allocate (and attach) packet buffer */ if (m != NULL && !uma_zalloc_arg(sc->rx_buffer_zone, m, M_NOWAIT)) { uma_zfree(zone_mbuf, m); m = NULL; } return (m); } #define SFXGE_REFILL_BATCH 64 static void sfxge_rx_qfill(struct sfxge_rxq *rxq, unsigned int target, boolean_t retrying) { struct sfxge_softc *sc; unsigned int index; struct sfxge_evq *evq; unsigned int batch; unsigned int rxfill; unsigned int mblksize; int ntodo; efsys_dma_addr_t addr[SFXGE_REFILL_BATCH]; sc = rxq->sc; index = rxq->index; evq = sc->evq[index]; prefetch_read_many(sc->enp); prefetch_read_many(rxq->common); - mtx_assert(&evq->lock, MA_OWNED); + SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); if (rxq->init_state != SFXGE_RXQ_STARTED) return; rxfill = rxq->added - rxq->completed; KASSERT(rxfill <= EFX_RXQ_LIMIT(rxq->entries), ("rxfill > EFX_RXQ_LIMIT(rxq->entries)")); ntodo = min(EFX_RXQ_LIMIT(rxq->entries) - rxfill, target); KASSERT(ntodo <= EFX_RXQ_LIMIT(rxq->entries), ("ntodo > EFX_RQX_LIMIT(rxq->entries)")); if (ntodo == 0) return; batch = 0; mblksize = sc->rx_buffer_size; while (ntodo-- > 0) { unsigned int id; struct sfxge_rx_sw_desc *rx_desc; bus_dma_segment_t seg; struct mbuf *m; id = (rxq->added + batch) & rxq->ptr_mask; rx_desc = &rxq->queue[id]; KASSERT(rx_desc->mbuf == NULL, ("rx_desc->mbuf != NULL")); rx_desc->flags = EFX_DISCARD; m = rx_desc->mbuf = sfxge_rx_alloc_mbuf(sc); if (m == NULL) break; sfxge_map_mbuf_fast(rxq->mem.esm_tag, rxq->mem.esm_map, m, &seg); addr[batch++] = seg.ds_addr; if (batch == SFXGE_REFILL_BATCH) { efx_rx_qpost(rxq->common, addr, mblksize, batch, rxq->completed, rxq->added); rxq->added += batch; batch = 0; } } if (ntodo != 0) sfxge_rx_schedule_refill(rxq, retrying); if (batch != 0) { efx_rx_qpost(rxq->common, addr, mblksize, batch, rxq->completed, rxq->added); rxq->added += batch; } /* Make the descriptors visible to the hardware */ bus_dmamap_sync(rxq->mem.esm_tag, rxq->mem.esm_map, BUS_DMASYNC_PREWRITE); efx_rx_qpush(rxq->common, rxq->added); } void sfxge_rx_qrefill(struct sfxge_rxq *rxq) { if (rxq->init_state != SFXGE_RXQ_STARTED) return; /* Make sure the queue is full */ sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_TRUE); } static void __sfxge_rx_deliver(struct sfxge_softc *sc, struct mbuf *m) { struct ifnet *ifp = sc->ifnet; m->m_pkthdr.rcvif = ifp; m->m_pkthdr.csum_data = 0xffff; ifp->if_input(ifp, m); } static void sfxge_rx_deliver(struct sfxge_softc *sc, struct sfxge_rx_sw_desc *rx_desc) { struct mbuf *m = rx_desc->mbuf; int csum_flags; /* Convert checksum flags */ csum_flags = (rx_desc->flags & EFX_CKSUM_IPV4) ? (CSUM_IP_CHECKED | CSUM_IP_VALID) : 0; if (rx_desc->flags & EFX_CKSUM_TCPUDP) csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; #ifdef SFXGE_HAVE_MQ /* The hash covers a 4-tuple for TCP only */ if (rx_desc->flags & EFX_PKT_TCP) { m->m_pkthdr.flowid = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ, mtod(m, uint8_t *)); M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); } #endif m->m_data += sc->rx_prefix_size; m->m_len = rx_desc->size - sc->rx_prefix_size; m->m_pkthdr.len = m->m_len; m->m_pkthdr.csum_flags = csum_flags; __sfxge_rx_deliver(sc, rx_desc->mbuf); rx_desc->flags = EFX_DISCARD; rx_desc->mbuf = NULL; } static void sfxge_lro_deliver(struct sfxge_lro_state *st, struct sfxge_lro_conn *c) { struct sfxge_softc *sc = st->sc; struct mbuf *m = c->mbuf; struct tcphdr *c_th; int csum_flags; KASSERT(m, ("no mbuf to deliver")); ++st->n_bursts; /* Finish off packet munging and recalculate IP header checksum. */ if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) { struct ip *iph = c->nh; iph->ip_len = htons(iph->ip_len); iph->ip_sum = 0; iph->ip_sum = in_cksum_hdr(iph); c_th = (struct tcphdr *)(iph + 1); csum_flags = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID); } else { struct ip6_hdr *iph = c->nh; iph->ip6_plen = htons(iph->ip6_plen); c_th = (struct tcphdr *)(iph + 1); csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; } c_th->th_win = c->th_last->th_win; c_th->th_ack = c->th_last->th_ack; if (c_th->th_off == c->th_last->th_off) { /* Copy TCP options (take care to avoid going negative). */ int optlen = ((c_th->th_off - 5) & 0xf) << 2u; memcpy(c_th + 1, c->th_last + 1, optlen); } #ifdef SFXGE_HAVE_MQ m->m_pkthdr.flowid = c->conn_hash; M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); #endif m->m_pkthdr.csum_flags = csum_flags; __sfxge_rx_deliver(sc, m); c->mbuf = NULL; c->delivered = 1; } /* Drop the given connection, and add it to the free list. */ static void sfxge_lro_drop(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c) { unsigned bucket; KASSERT(!c->mbuf, ("found orphaned mbuf")); if (c->next_buf.mbuf != NULL) { sfxge_rx_deliver(rxq->sc, &c->next_buf); LIST_REMOVE(c, active_link); } bucket = c->conn_hash & rxq->lro.conns_mask; KASSERT(rxq->lro.conns_n[bucket] > 0, ("LRO: bucket fill level wrong")); --rxq->lro.conns_n[bucket]; TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link); TAILQ_INSERT_HEAD(&rxq->lro.free_conns, c, link); } /* Stop tracking connections that have gone idle in order to keep hash * chains short. */ static void sfxge_lro_purge_idle(struct sfxge_rxq *rxq, unsigned now) { struct sfxge_lro_conn *c; unsigned i; KASSERT(LIST_EMPTY(&rxq->lro.active_conns), ("found active connections")); rxq->lro.last_purge_ticks = now; for (i = 0; i <= rxq->lro.conns_mask; ++i) { if (TAILQ_EMPTY(&rxq->lro.conns[i])) continue; c = TAILQ_LAST(&rxq->lro.conns[i], sfxge_lro_tailq); if (now - c->last_pkt_ticks > lro_idle_ticks) { ++rxq->lro.n_drop_idle; sfxge_lro_drop(rxq, c); } } } static void sfxge_lro_merge(struct sfxge_lro_state *st, struct sfxge_lro_conn *c, struct mbuf *mbuf, struct tcphdr *th) { struct tcphdr *c_th; /* Tack the new mbuf onto the chain. */ KASSERT(!mbuf->m_next, ("mbuf already chained")); c->mbuf_tail->m_next = mbuf; c->mbuf_tail = mbuf; /* Increase length appropriately */ c->mbuf->m_pkthdr.len += mbuf->m_len; /* Update the connection state flags */ if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) { struct ip *iph = c->nh; iph->ip_len += mbuf->m_len; c_th = (struct tcphdr *)(iph + 1); } else { struct ip6_hdr *iph = c->nh; iph->ip6_plen += mbuf->m_len; c_th = (struct tcphdr *)(iph + 1); } c_th->th_flags |= (th->th_flags & TH_PUSH); c->th_last = th; ++st->n_merges; /* Pass packet up now if another segment could overflow the IP * length. */ if (c->mbuf->m_pkthdr.len > 65536 - 9200) sfxge_lro_deliver(st, c); } static void sfxge_lro_start(struct sfxge_lro_state *st, struct sfxge_lro_conn *c, struct mbuf *mbuf, void *nh, struct tcphdr *th) { /* Start the chain */ c->mbuf = mbuf; c->mbuf_tail = c->mbuf; c->nh = nh; c->th_last = th; mbuf->m_pkthdr.len = mbuf->m_len; /* Mangle header fields for later processing */ if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) { struct ip *iph = nh; iph->ip_len = ntohs(iph->ip_len); } else { struct ip6_hdr *iph = nh; iph->ip6_plen = ntohs(iph->ip6_plen); } } /* Try to merge or otherwise hold or deliver (as appropriate) the * packet buffered for this connection (c->next_buf). Return a flag * indicating whether the connection is still active for LRO purposes. */ static int sfxge_lro_try_merge(struct sfxge_rxq *rxq, struct sfxge_lro_conn *c) { struct sfxge_rx_sw_desc *rx_buf = &c->next_buf; char *eh = c->next_eh; int data_length, hdr_length, dont_merge; unsigned th_seq, pkt_length; struct tcphdr *th; unsigned now; if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) { struct ip *iph = c->next_nh; th = (struct tcphdr *)(iph + 1); pkt_length = ntohs(iph->ip_len) + (char *) iph - eh; } else { struct ip6_hdr *iph = c->next_nh; th = (struct tcphdr *)(iph + 1); pkt_length = ntohs(iph->ip6_plen) + (char *) th - eh; } hdr_length = (char *) th + th->th_off * 4 - eh; data_length = (min(pkt_length, rx_buf->size - rxq->sc->rx_prefix_size) - hdr_length); th_seq = ntohl(th->th_seq); dont_merge = ((data_length <= 0) | (th->th_flags & (TH_URG | TH_SYN | TH_RST | TH_FIN))); /* Check for options other than aligned timestamp. */ if (th->th_off != 5) { const uint32_t *opt_ptr = (const uint32_t *) (th + 1); if (th->th_off == 8 && opt_ptr[0] == ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) { /* timestamp option -- okay */ } else { dont_merge = 1; } } if (__predict_false(th_seq != c->next_seq)) { /* Out-of-order, so start counting again. */ if (c->mbuf != NULL) sfxge_lro_deliver(&rxq->lro, c); c->n_in_order_pkts -= lro_loss_packets; c->next_seq = th_seq + data_length; ++rxq->lro.n_misorder; goto deliver_buf_out; } c->next_seq = th_seq + data_length; now = ticks; if (now - c->last_pkt_ticks > lro_idle_ticks) { ++rxq->lro.n_drop_idle; if (c->mbuf != NULL) sfxge_lro_deliver(&rxq->lro, c); sfxge_lro_drop(rxq, c); return (0); } c->last_pkt_ticks = ticks; if (c->n_in_order_pkts < lro_slow_start_packets) { /* May be in slow-start, so don't merge. */ ++rxq->lro.n_slow_start; ++c->n_in_order_pkts; goto deliver_buf_out; } if (__predict_false(dont_merge)) { if (c->mbuf != NULL) sfxge_lro_deliver(&rxq->lro, c); if (th->th_flags & (TH_FIN | TH_RST)) { ++rxq->lro.n_drop_closed; sfxge_lro_drop(rxq, c); return (0); } goto deliver_buf_out; } rx_buf->mbuf->m_data += rxq->sc->rx_prefix_size; if (__predict_true(c->mbuf != NULL)) { /* Remove headers and any padding */ rx_buf->mbuf->m_data += hdr_length; rx_buf->mbuf->m_len = data_length; sfxge_lro_merge(&rxq->lro, c, rx_buf->mbuf, th); } else { /* Remove any padding */ rx_buf->mbuf->m_len = pkt_length; sfxge_lro_start(&rxq->lro, c, rx_buf->mbuf, c->next_nh, th); } rx_buf->mbuf = NULL; return (1); deliver_buf_out: sfxge_rx_deliver(rxq->sc, rx_buf); return (1); } static void sfxge_lro_new_conn(struct sfxge_lro_state *st, uint32_t conn_hash, uint16_t l2_id, void *nh, struct tcphdr *th) { unsigned bucket = conn_hash & st->conns_mask; struct sfxge_lro_conn *c; if (st->conns_n[bucket] >= lro_chain_max) { ++st->n_too_many; return; } if (!TAILQ_EMPTY(&st->free_conns)) { c = TAILQ_FIRST(&st->free_conns); TAILQ_REMOVE(&st->free_conns, c, link); } else { c = malloc(sizeof(*c), M_SFXGE, M_NOWAIT); if (c == NULL) return; c->mbuf = NULL; c->next_buf.mbuf = NULL; } /* Create the connection tracking data */ ++st->conns_n[bucket]; TAILQ_INSERT_HEAD(&st->conns[bucket], c, link); c->l2_id = l2_id; c->conn_hash = conn_hash; c->source = th->th_sport; c->dest = th->th_dport; c->n_in_order_pkts = 0; c->last_pkt_ticks = *(volatile int *)&ticks; c->delivered = 0; ++st->n_new_stream; /* NB. We don't initialise c->next_seq, and it doesn't matter what * value it has. Most likely the next packet received for this * connection will not match -- no harm done. */ } /* Process mbuf and decide whether to dispatch it to the stack now or * later. */ static void sfxge_lro(struct sfxge_rxq *rxq, struct sfxge_rx_sw_desc *rx_buf) { struct sfxge_softc *sc = rxq->sc; struct mbuf *m = rx_buf->mbuf; struct ether_header *eh; struct sfxge_lro_conn *c; uint16_t l2_id; uint16_t l3_proto; void *nh; struct tcphdr *th; uint32_t conn_hash; unsigned bucket; /* Get the hardware hash */ conn_hash = EFX_RX_HASH_VALUE(EFX_RX_HASHALG_TOEPLITZ, mtod(m, uint8_t *)); eh = (struct ether_header *)(m->m_data + sc->rx_prefix_size); if (eh->ether_type == htons(ETHERTYPE_VLAN)) { struct ether_vlan_header *veh = (struct ether_vlan_header *)eh; l2_id = EVL_VLANOFTAG(ntohs(veh->evl_tag)) | SFXGE_LRO_L2_ID_VLAN; l3_proto = veh->evl_proto; nh = veh + 1; } else { l2_id = 0; l3_proto = eh->ether_type; nh = eh + 1; } /* Check whether this is a suitable packet (unfragmented * TCP/IPv4 or TCP/IPv6). If so, find the TCP header and * length, and compute a hash if necessary. If not, return. */ if (l3_proto == htons(ETHERTYPE_IP)) { struct ip *iph = nh; if ((iph->ip_p - IPPROTO_TCP) | (iph->ip_hl - (sizeof(*iph) >> 2u)) | (iph->ip_off & htons(IP_MF | IP_OFFMASK))) goto deliver_now; th = (struct tcphdr *)(iph + 1); } else if (l3_proto == htons(ETHERTYPE_IPV6)) { struct ip6_hdr *iph = nh; if (iph->ip6_nxt != IPPROTO_TCP) goto deliver_now; l2_id |= SFXGE_LRO_L2_ID_IPV6; th = (struct tcphdr *)(iph + 1); } else { goto deliver_now; } bucket = conn_hash & rxq->lro.conns_mask; TAILQ_FOREACH(c, &rxq->lro.conns[bucket], link) { if ((c->l2_id - l2_id) | (c->conn_hash - conn_hash)) continue; if ((c->source - th->th_sport) | (c->dest - th->th_dport)) continue; if (c->mbuf != NULL) { if (SFXGE_LRO_CONN_IS_TCPIPV4(c)) { struct ip *c_iph, *iph = nh; c_iph = c->nh; if ((c_iph->ip_src.s_addr - iph->ip_src.s_addr) | (c_iph->ip_dst.s_addr - iph->ip_dst.s_addr)) continue; } else { struct ip6_hdr *c_iph, *iph = nh; c_iph = c->nh; if (ipv6_addr_cmp(&c_iph->ip6_src, &iph->ip6_src) | ipv6_addr_cmp(&c_iph->ip6_dst, &iph->ip6_dst)) continue; } } /* Re-insert at head of list to reduce lookup time. */ TAILQ_REMOVE(&rxq->lro.conns[bucket], c, link); TAILQ_INSERT_HEAD(&rxq->lro.conns[bucket], c, link); if (c->next_buf.mbuf != NULL) { if (!sfxge_lro_try_merge(rxq, c)) goto deliver_now; } else { LIST_INSERT_HEAD(&rxq->lro.active_conns, c, active_link); } c->next_buf = *rx_buf; c->next_eh = eh; c->next_nh = nh; rx_buf->mbuf = NULL; rx_buf->flags = EFX_DISCARD; return; } sfxge_lro_new_conn(&rxq->lro, conn_hash, l2_id, nh, th); deliver_now: sfxge_rx_deliver(sc, rx_buf); } static void sfxge_lro_end_of_burst(struct sfxge_rxq *rxq) { struct sfxge_lro_state *st = &rxq->lro; struct sfxge_lro_conn *c; unsigned t; while (!LIST_EMPTY(&st->active_conns)) { c = LIST_FIRST(&st->active_conns); if (!c->delivered && c->mbuf != NULL) sfxge_lro_deliver(st, c); if (sfxge_lro_try_merge(rxq, c)) { if (c->mbuf != NULL) sfxge_lro_deliver(st, c); LIST_REMOVE(c, active_link); } c->delivered = 0; } t = *(volatile int *)&ticks; if (__predict_false(t != st->last_purge_ticks)) sfxge_lro_purge_idle(rxq, t); } void sfxge_rx_qcomplete(struct sfxge_rxq *rxq, boolean_t eop) { struct sfxge_softc *sc = rxq->sc; int lro_enabled = sc->ifnet->if_capenable & IFCAP_LRO; unsigned int index; struct sfxge_evq *evq; unsigned int completed; unsigned int level; struct mbuf *m; struct sfxge_rx_sw_desc *prev = NULL; index = rxq->index; evq = sc->evq[index]; - mtx_assert(&evq->lock, MA_OWNED); + SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); completed = rxq->completed; while (completed != rxq->pending) { unsigned int id; struct sfxge_rx_sw_desc *rx_desc; id = completed++ & rxq->ptr_mask; rx_desc = &rxq->queue[id]; m = rx_desc->mbuf; if (rxq->init_state != SFXGE_RXQ_STARTED) goto discard; if (rx_desc->flags & (EFX_ADDR_MISMATCH | EFX_DISCARD)) goto discard; prefetch_read_many(mtod(m, caddr_t)); /* Check for loopback packets */ if (!(rx_desc->flags & EFX_PKT_IPV4) && !(rx_desc->flags & EFX_PKT_IPV6)) { struct ether_header *etherhp; /*LINTED*/ etherhp = mtod(m, struct ether_header *); if (etherhp->ether_type == htons(SFXGE_ETHERTYPE_LOOPBACK)) { EFSYS_PROBE(loopback); rxq->loopback++; goto discard; } } /* Pass packet up the stack or into LRO (pipelined) */ if (prev != NULL) { if (lro_enabled) sfxge_lro(rxq, prev); else sfxge_rx_deliver(sc, prev); } prev = rx_desc; continue; discard: /* Return the packet to the pool */ m_free(m); rx_desc->mbuf = NULL; } rxq->completed = completed; level = rxq->added - rxq->completed; /* Pass last packet up the stack or into LRO */ if (prev != NULL) { if (lro_enabled) sfxge_lro(rxq, prev); else sfxge_rx_deliver(sc, prev); } /* * If there are any pending flows and this is the end of the * poll then they must be completed. */ if (eop) sfxge_lro_end_of_burst(rxq); /* Top up the queue if necessary */ if (level < rxq->refill_threshold) sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(rxq->entries), B_FALSE); } static void sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index) { struct sfxge_rxq *rxq; struct sfxge_evq *evq; unsigned int count; rxq = sc->rxq[index]; evq = sc->evq[index]; - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); KASSERT(rxq->init_state == SFXGE_RXQ_STARTED, ("rxq not started")); rxq->init_state = SFXGE_RXQ_INITIALIZED; callout_stop(&rxq->refill_callout); again: rxq->flush_state = SFXGE_FLUSH_PENDING; /* Flush the receive queue */ efx_rx_qflush(rxq->common); - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); count = 0; do { /* Spin for 100 ms */ DELAY(100000); if (rxq->flush_state != SFXGE_FLUSH_PENDING) break; } while (++count < 20); - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); if (rxq->flush_state == SFXGE_FLUSH_FAILED) goto again; rxq->flush_state = SFXGE_FLUSH_DONE; rxq->pending = rxq->added; sfxge_rx_qcomplete(rxq, B_TRUE); KASSERT(rxq->completed == rxq->pending, ("rxq->completed != rxq->pending")); rxq->added = 0; rxq->pending = 0; rxq->completed = 0; rxq->loopback = 0; /* Destroy the common code receive queue. */ efx_rx_qdestroy(rxq->common); efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id, EFX_RXQ_NBUFS(sc->rxq_entries)); - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); } static int sfxge_rx_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_rxq *rxq; efsys_mem_t *esmp; struct sfxge_evq *evq; int rc; rxq = sc->rxq[index]; esmp = &rxq->mem; evq = sc->evq[index]; KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED, ("rxq->init_state != SFXGE_RXQ_INITIALIZED")); KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); /* Program the buffer table. */ if ((rc = efx_sram_buf_tbl_set(sc->enp, rxq->buf_base_id, esmp, EFX_RXQ_NBUFS(sc->rxq_entries))) != 0) return (rc); /* Create the common code receive queue. */ if ((rc = efx_rx_qcreate(sc->enp, index, index, EFX_RXQ_TYPE_DEFAULT, esmp, sc->rxq_entries, rxq->buf_base_id, evq->common, &rxq->common)) != 0) goto fail; - mtx_lock(&evq->lock); + SFXGE_EVQ_LOCK(evq); /* Enable the receive queue. */ efx_rx_qenable(rxq->common); rxq->init_state = SFXGE_RXQ_STARTED; /* Try to fill the queue from the pool. */ sfxge_rx_qfill(rxq, EFX_RXQ_LIMIT(sc->rxq_entries), B_FALSE); - mtx_unlock(&evq->lock); + SFXGE_EVQ_UNLOCK(evq); return (0); fail: efx_sram_buf_tbl_clear(sc->enp, rxq->buf_base_id, EFX_RXQ_NBUFS(sc->rxq_entries)); return (rc); } void sfxge_rx_stop(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; intr = &sc->intr; /* Stop the receive queue(s) */ index = intr->n_alloc; while (--index >= 0) sfxge_rx_qstop(sc, index); sc->rx_prefix_size = 0; sc->rx_buffer_size = 0; efx_rx_fini(sc->enp); } int sfxge_rx_start(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; /* Initialize the common code receive module. */ if ((rc = efx_rx_init(sc->enp)) != 0) return (rc); /* Calculate the receive packet buffer size. */ sc->rx_prefix_size = EFX_RX_PREFIX_SIZE; sc->rx_buffer_size = (EFX_MAC_PDU(sc->ifnet->if_mtu) + sc->rx_prefix_size); /* Select zone for packet buffers */ if (sc->rx_buffer_size <= MCLBYTES) sc->rx_buffer_zone = zone_clust; else if (sc->rx_buffer_size <= MJUMPAGESIZE) sc->rx_buffer_zone = zone_jumbop; else if (sc->rx_buffer_size <= MJUM9BYTES) sc->rx_buffer_zone = zone_jumbo9; else sc->rx_buffer_zone = zone_jumbo16; /* * Set up the scale table. Enable all hash types and hash insertion. */ for (index = 0; index < SFXGE_RX_SCALE_MAX; index++) sc->rx_indir_table[index] = index % sc->intr.n_alloc; if ((rc = efx_rx_scale_tbl_set(sc->enp, sc->rx_indir_table, SFXGE_RX_SCALE_MAX)) != 0) goto fail; (void)efx_rx_scale_mode_set(sc->enp, EFX_RX_HASHALG_TOEPLITZ, (1 << EFX_RX_HASH_IPV4) | (1 << EFX_RX_HASH_TCPIPV4) | (1 << EFX_RX_HASH_IPV6) | (1 << EFX_RX_HASH_TCPIPV6), B_TRUE); if ((rc = efx_rx_scale_toeplitz_ipv4_key_set(sc->enp, toep_key, sizeof(toep_key))) != 0) goto fail; /* Start the receive queue(s). */ for (index = 0; index < intr->n_alloc; index++) { if ((rc = sfxge_rx_qstart(sc, index)) != 0) goto fail2; } return (0); fail2: while (--index >= 0) sfxge_rx_qstop(sc, index); fail: efx_rx_fini(sc->enp); return (rc); } static void sfxge_lro_init(struct sfxge_rxq *rxq) { struct sfxge_lro_state *st = &rxq->lro; unsigned i; st->conns_mask = lro_table_size - 1; KASSERT(!((st->conns_mask + 1) & st->conns_mask), ("lro_table_size must be a power of 2")); st->sc = rxq->sc; st->conns = malloc((st->conns_mask + 1) * sizeof(st->conns[0]), M_SFXGE, M_WAITOK); st->conns_n = malloc((st->conns_mask + 1) * sizeof(st->conns_n[0]), M_SFXGE, M_WAITOK); for (i = 0; i <= st->conns_mask; ++i) { TAILQ_INIT(&st->conns[i]); st->conns_n[i] = 0; } LIST_INIT(&st->active_conns); TAILQ_INIT(&st->free_conns); } static void sfxge_lro_fini(struct sfxge_rxq *rxq) { struct sfxge_lro_state *st = &rxq->lro; struct sfxge_lro_conn *c; unsigned i; /* Return cleanly if sfxge_lro_init() has not been called. */ if (st->conns == NULL) return; KASSERT(LIST_EMPTY(&st->active_conns), ("found active connections")); for (i = 0; i <= st->conns_mask; ++i) { while (!TAILQ_EMPTY(&st->conns[i])) { c = TAILQ_LAST(&st->conns[i], sfxge_lro_tailq); sfxge_lro_drop(rxq, c); } } while (!TAILQ_EMPTY(&st->free_conns)) { c = TAILQ_FIRST(&st->free_conns); TAILQ_REMOVE(&st->free_conns, c, link); KASSERT(!c->mbuf, ("found orphaned mbuf")); free(c, M_SFXGE); } free(st->conns_n, M_SFXGE); free(st->conns, M_SFXGE); st->conns = NULL; } static void sfxge_rx_qfini(struct sfxge_softc *sc, unsigned int index) { struct sfxge_rxq *rxq; rxq = sc->rxq[index]; KASSERT(rxq->init_state == SFXGE_RXQ_INITIALIZED, ("rxq->init_state != SFXGE_RXQ_INITIALIZED")); /* Free the context array and the flow table. */ free(rxq->queue, M_SFXGE); sfxge_lro_fini(rxq); /* Release DMA memory. */ sfxge_dma_free(&rxq->mem); sc->rxq[index] = NULL; free(rxq, M_SFXGE); } static int sfxge_rx_qinit(struct sfxge_softc *sc, unsigned int index) { struct sfxge_rxq *rxq; struct sfxge_evq *evq; efsys_mem_t *esmp; int rc; KASSERT(index < sc->intr.n_alloc, ("index >= %d", sc->intr.n_alloc)); rxq = malloc(sizeof(struct sfxge_rxq), M_SFXGE, M_ZERO | M_WAITOK); rxq->sc = sc; rxq->index = index; rxq->entries = sc->rxq_entries; rxq->ptr_mask = rxq->entries - 1; rxq->refill_threshold = RX_REFILL_THRESHOLD(rxq->entries); sc->rxq[index] = rxq; esmp = &rxq->mem; evq = sc->evq[index]; /* Allocate and zero DMA space. */ if ((rc = sfxge_dma_alloc(sc, EFX_RXQ_SIZE(sc->rxq_entries), esmp)) != 0) return (rc); (void)memset(esmp->esm_base, 0, EFX_RXQ_SIZE(sc->rxq_entries)); /* Allocate buffer table entries. */ sfxge_sram_buf_tbl_alloc(sc, EFX_RXQ_NBUFS(sc->rxq_entries), &rxq->buf_base_id); /* Allocate the context array and the flow table. */ rxq->queue = malloc(sizeof(struct sfxge_rx_sw_desc) * sc->rxq_entries, M_SFXGE, M_WAITOK | M_ZERO); sfxge_lro_init(rxq); callout_init(&rxq->refill_callout, B_TRUE); rxq->init_state = SFXGE_RXQ_INITIALIZED; return (0); } static const struct { const char *name; size_t offset; } sfxge_rx_stats[] = { #define SFXGE_RX_STAT(name, member) \ { #name, offsetof(struct sfxge_rxq, member) } SFXGE_RX_STAT(lro_merges, lro.n_merges), SFXGE_RX_STAT(lro_bursts, lro.n_bursts), SFXGE_RX_STAT(lro_slow_start, lro.n_slow_start), SFXGE_RX_STAT(lro_misorder, lro.n_misorder), SFXGE_RX_STAT(lro_too_many, lro.n_too_many), SFXGE_RX_STAT(lro_new_stream, lro.n_new_stream), SFXGE_RX_STAT(lro_drop_idle, lro.n_drop_idle), SFXGE_RX_STAT(lro_drop_closed, lro.n_drop_closed) }; static int sfxge_rx_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; unsigned int sum, index; /* Sum across all RX queues */ sum = 0; for (index = 0; index < sc->intr.n_alloc; index++) sum += *(unsigned int *)((caddr_t)sc->rxq[index] + sfxge_rx_stats[id].offset); return (SYSCTL_OUT(req, &sum, sizeof(sum))); } static void sfxge_rx_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; stat_list = SYSCTL_CHILDREN(sc->stats_node); for (id = 0; id < sizeof(sfxge_rx_stats) / sizeof(sfxge_rx_stats[0]); id++) { SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, sfxge_rx_stats[id].name, CTLTYPE_UINT|CTLFLAG_RD, sc, id, sfxge_rx_stat_handler, "IU", ""); } } void sfxge_rx_fini(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; intr = &sc->intr; index = intr->n_alloc; while (--index >= 0) sfxge_rx_qfini(sc, index); } int sfxge_rx_init(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; if (lro_idle_ticks == 0) lro_idle_ticks = hz / 10 + 1; /* 100 ms */ intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); /* Initialize the receive queue(s) - one per interrupt. */ for (index = 0; index < intr->n_alloc; index++) { if ((rc = sfxge_rx_qinit(sc, index)) != 0) goto fail; } sfxge_rx_stat_init(sc); return (0); fail: /* Tear down the receive queue(s). */ while (--index >= 0) sfxge_rx_qfini(sc, index); return (rc); } Index: head/sys/dev/sfxge/sfxge_tx.c =================================================================== --- head/sys/dev/sfxge/sfxge_tx.c (revision 278220) +++ head/sys/dev/sfxge/sfxge_tx.c (revision 278221) @@ -1,1634 +1,1633 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Theory of operation: * * Tx queues allocation and mapping * * One Tx queue with enabled checksum offload is allocated per Rx channel * (event queue). Also 2 Tx queues (one without checksum offload and one * with IP checksum offload only) are allocated and bound to event queue 0. * sfxge_txq_type is used as Tx queue label. * * So, event queue plus label mapping to Tx queue index is: * if event queue index is 0, TxQ-index = TxQ-label * [0..SFXGE_TXQ_NTYPES) * else TxQ-index = SFXGE_TXQ_NTYPES + EvQ-index - 1 * See sfxge_get_txq_by_label() sfxge_ev.c */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/efx.h" #include "sfxge.h" #include "sfxge_tx.h" /* Set the block level to ensure there is space to generate a * large number of descriptors for TSO. With minimum MSS and * maximum mbuf length we might need more than a ring-ful of * descriptors, but this should not happen in practice except * due to deliberate attack. In that case we will truncate * the output at a packet boundary. Allow for a reasonable * minimum MSS of 512. */ #define SFXGE_TSO_MAX_DESC ((65535 / 512) * 2 + SFXGE_TX_MAPPING_MAX_SEG - 1) #define SFXGE_TXQ_BLOCK_LEVEL(_entries) ((_entries) - SFXGE_TSO_MAX_DESC) #ifdef SFXGE_HAVE_MQ #define SFXGE_PARAM_TX_DPL_GET_MAX SFXGE_PARAM(tx_dpl_get_max) static int sfxge_tx_dpl_get_max = SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT; TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_MAX, &sfxge_tx_dpl_get_max); SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_max, CTLFLAG_RDTUN, &sfxge_tx_dpl_get_max, 0, "Maximum number of any packets in deferred packet get-list"); #define SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX \ SFXGE_PARAM(tx_dpl_get_non_tcp_max) static int sfxge_tx_dpl_get_non_tcp_max = SFXGE_TX_DPL_GET_NON_TCP_PKT_LIMIT_DEFAULT; TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX, &sfxge_tx_dpl_get_non_tcp_max); SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_non_tcp_max, CTLFLAG_RDTUN, &sfxge_tx_dpl_get_non_tcp_max, 0, "Maximum number of non-TCP packets in deferred packet get-list"); #define SFXGE_PARAM_TX_DPL_PUT_MAX SFXGE_PARAM(tx_dpl_put_max) static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT; TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max); SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN, &sfxge_tx_dpl_put_max, 0, "Maximum number of any packets in deferred packet put-list"); #endif /* Forward declarations. */ static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq); static void sfxge_tx_qlist_post(struct sfxge_txq *txq); static void sfxge_tx_qunblock(struct sfxge_txq *txq); static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, const bus_dma_segment_t *dma_seg, int n_dma_seg); void sfxge_tx_qcomplete(struct sfxge_txq *txq, struct sfxge_evq *evq) { unsigned int completed; - mtx_assert(&evq->lock, MA_OWNED); + SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); completed = txq->completed; while (completed != txq->pending) { struct sfxge_tx_mapping *stmp; unsigned int id; id = completed++ & txq->ptr_mask; stmp = &txq->stmp[id]; if (stmp->flags & TX_BUF_UNMAP) { bus_dmamap_unload(txq->packet_dma_tag, stmp->map); if (stmp->flags & TX_BUF_MBUF) { struct mbuf *m = stmp->u.mbuf; do m = m_free(m); while (m != NULL); } else { free(stmp->u.heap_buf, M_SFXGE); } stmp->flags = 0; } } txq->completed = completed; /* Check whether we need to unblock the queue. */ mb(); if (txq->blocked) { unsigned int level; level = txq->added - txq->completed; if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries)) sfxge_tx_qunblock(txq); } } #ifdef SFXGE_HAVE_MQ static inline unsigned int sfxge_is_mbuf_non_tcp(struct mbuf *mbuf) { /* Absense of TCP checksum flags does not mean that it is non-TCP * but it should be true if user wants to achieve high throughput. */ return (!(mbuf->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))); } /* * Reorder the put list and append it to the get list. */ static void sfxge_tx_qdpl_swizzle(struct sfxge_txq *txq) { struct sfxge_tx_dpl *stdp; struct mbuf *mbuf, *get_next, **get_tailp; volatile uintptr_t *putp; uintptr_t put; unsigned int count; unsigned int non_tcp_count; - mtx_assert(&txq->lock, MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); stdp = &txq->dpl; /* Acquire the put list. */ putp = &stdp->std_put; put = atomic_readandclear_ptr(putp); mbuf = (void *)put; if (mbuf == NULL) return; /* Reverse the put list. */ get_tailp = &mbuf->m_nextpkt; get_next = NULL; count = 0; non_tcp_count = 0; do { struct mbuf *put_next; non_tcp_count += sfxge_is_mbuf_non_tcp(mbuf); put_next = mbuf->m_nextpkt; mbuf->m_nextpkt = get_next; get_next = mbuf; mbuf = put_next; count++; } while (mbuf != NULL); /* Append the reversed put list to the get list. */ KASSERT(*get_tailp == NULL, ("*get_tailp != NULL")); *stdp->std_getp = get_next; stdp->std_getp = get_tailp; stdp->std_get_count += count; stdp->std_get_non_tcp_count += non_tcp_count; } #endif /* SFXGE_HAVE_MQ */ static void sfxge_tx_qreap(struct sfxge_txq *txq) { - mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); txq->reaped = txq->completed; } static void sfxge_tx_qlist_post(struct sfxge_txq *txq) { unsigned int old_added; unsigned int level; int rc; - mtx_assert(SFXGE_TXQ_LOCK(txq), MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); KASSERT(txq->n_pend_desc != 0, ("txq->n_pend_desc == 0")); KASSERT(txq->n_pend_desc <= SFXGE_TSO_MAX_DESC, ("txq->n_pend_desc too large")); KASSERT(!txq->blocked, ("txq->blocked")); old_added = txq->added; /* Post the fragment list. */ rc = efx_tx_qpost(txq->common, txq->pend_desc, txq->n_pend_desc, txq->reaped, &txq->added); KASSERT(rc == 0, ("efx_tx_qpost() failed")); /* If efx_tx_qpost() had to refragment, our information about * buffers to free may be associated with the wrong * descriptors. */ KASSERT(txq->added - old_added == txq->n_pend_desc, ("efx_tx_qpost() refragmented descriptors")); level = txq->added - txq->reaped; KASSERT(level <= txq->entries, ("overfilled TX queue")); /* Clear the fragment list. */ txq->n_pend_desc = 0; /* Have we reached the block level? */ if (level < SFXGE_TXQ_BLOCK_LEVEL(txq->entries)) return; /* Reap, and check again */ sfxge_tx_qreap(txq); level = txq->added - txq->reaped; if (level < SFXGE_TXQ_BLOCK_LEVEL(txq->entries)) return; txq->blocked = 1; /* * Avoid a race with completion interrupt handling that could leave * the queue blocked. */ mb(); sfxge_tx_qreap(txq); level = txq->added - txq->reaped; if (level < SFXGE_TXQ_BLOCK_LEVEL(txq->entries)) { mb(); txq->blocked = 0; } } static int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf) { bus_dmamap_t *used_map; bus_dmamap_t map; bus_dma_segment_t dma_seg[SFXGE_TX_MAPPING_MAX_SEG]; unsigned int id; struct sfxge_tx_mapping *stmp; efx_buffer_t *desc; int n_dma_seg; int rc; int i; KASSERT(!txq->blocked, ("txq->blocked")); if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) prefetch_read_many(mbuf->m_data); if (txq->init_state != SFXGE_TXQ_STARTED) { rc = EINTR; goto reject; } /* Load the packet for DMA. */ id = txq->added & txq->ptr_mask; stmp = &txq->stmp[id]; rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, stmp->map, mbuf, dma_seg, &n_dma_seg, 0); if (rc == EFBIG) { /* Try again. */ struct mbuf *new_mbuf = m_collapse(mbuf, M_NOWAIT, SFXGE_TX_MAPPING_MAX_SEG); if (new_mbuf == NULL) goto reject; ++txq->collapses; mbuf = new_mbuf; rc = bus_dmamap_load_mbuf_sg(txq->packet_dma_tag, stmp->map, mbuf, dma_seg, &n_dma_seg, 0); } if (rc != 0) goto reject; /* Make the packet visible to the hardware. */ bus_dmamap_sync(txq->packet_dma_tag, stmp->map, BUS_DMASYNC_PREWRITE); used_map = &stmp->map; if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) { rc = sfxge_tx_queue_tso(txq, mbuf, dma_seg, n_dma_seg); if (rc < 0) goto reject_mapped; stmp = &txq->stmp[rc]; } else { /* Add the mapping to the fragment list, and set flags * for the buffer. */ i = 0; for (;;) { desc = &txq->pend_desc[i]; desc->eb_addr = dma_seg[i].ds_addr; desc->eb_size = dma_seg[i].ds_len; if (i == n_dma_seg - 1) { desc->eb_eop = 1; break; } desc->eb_eop = 0; i++; stmp->flags = 0; if (__predict_false(stmp == &txq->stmp[txq->ptr_mask])) stmp = &txq->stmp[0]; else stmp++; } txq->n_pend_desc = n_dma_seg; } /* * If the mapping required more than one descriptor * then we need to associate the DMA map with the last * descriptor, not the first. */ if (used_map != &stmp->map) { map = stmp->map; stmp->map = *used_map; *used_map = map; } stmp->u.mbuf = mbuf; stmp->flags = TX_BUF_UNMAP | TX_BUF_MBUF; /* Post the fragment list. */ sfxge_tx_qlist_post(txq); return (0); reject_mapped: bus_dmamap_unload(txq->packet_dma_tag, *used_map); reject: /* Drop the packet on the floor. */ m_freem(mbuf); ++txq->drops; return (rc); } #ifdef SFXGE_HAVE_MQ /* * Drain the deferred packet list into the transmit queue. */ static void sfxge_tx_qdpl_drain(struct sfxge_txq *txq) { struct sfxge_softc *sc; struct sfxge_tx_dpl *stdp; struct mbuf *mbuf, *next; unsigned int count; unsigned int non_tcp_count; unsigned int pushed; int rc; - mtx_assert(&txq->lock, MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); sc = txq->sc; stdp = &txq->dpl; pushed = txq->added; prefetch_read_many(sc->enp); prefetch_read_many(txq->common); mbuf = stdp->std_get; count = stdp->std_get_count; non_tcp_count = stdp->std_get_non_tcp_count; if (count > stdp->std_get_hiwat) stdp->std_get_hiwat = count; while (count != 0) { KASSERT(mbuf != NULL, ("mbuf == NULL")); next = mbuf->m_nextpkt; mbuf->m_nextpkt = NULL; ETHER_BPF_MTAP(sc->ifnet, mbuf); /* packet capture */ if (next != NULL) prefetch_read_many(next); rc = sfxge_tx_queue_mbuf(txq, mbuf); --count; non_tcp_count -= sfxge_is_mbuf_non_tcp(mbuf); mbuf = next; if (rc != 0) continue; if (txq->blocked) break; /* Push the fragments to the hardware in batches. */ if (txq->added - pushed >= SFXGE_TX_BATCH) { efx_tx_qpush(txq->common, txq->added); pushed = txq->added; } } if (count == 0) { KASSERT(mbuf == NULL, ("mbuf != NULL")); KASSERT(non_tcp_count == 0, ("inconsistent TCP/non-TCP detection")); stdp->std_get = NULL; stdp->std_get_count = 0; stdp->std_get_non_tcp_count = 0; stdp->std_getp = &stdp->std_get; } else { stdp->std_get = mbuf; stdp->std_get_count = count; stdp->std_get_non_tcp_count = non_tcp_count; } if (txq->added != pushed) efx_tx_qpush(txq->common, txq->added); KASSERT(txq->blocked || stdp->std_get_count == 0, ("queue unblocked but count is non-zero")); } #define SFXGE_TX_QDPL_PENDING(_txq) \ ((_txq)->dpl.std_put != 0) /* * Service the deferred packet list. * * NOTE: drops the txq mutex! */ static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq) { - mtx_assert(&txq->lock, MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); do { if (SFXGE_TX_QDPL_PENDING(txq)) sfxge_tx_qdpl_swizzle(txq); if (!txq->blocked) sfxge_tx_qdpl_drain(txq); - mtx_unlock(&txq->lock); + SFXGE_TXQ_UNLOCK(txq); } while (SFXGE_TX_QDPL_PENDING(txq) && - mtx_trylock(&txq->lock)); + SFXGE_TXQ_TRYLOCK(txq)); } /* * Put a packet on the deferred packet list. * * If we are called with the txq lock held, we put the packet on the "get * list", otherwise we atomically push it on the "put list". The swizzle * function takes care of ordering. * * The length of the put list is bounded by SFXGE_TX_MAX_DEFFERED. We * overload the csum_data field in the mbuf to keep track of this length * because there is no cheap alternative to avoid races. */ static inline int sfxge_tx_qdpl_put(struct sfxge_txq *txq, struct mbuf *mbuf, int locked) { struct sfxge_tx_dpl *stdp; stdp = &txq->dpl; KASSERT(mbuf->m_nextpkt == NULL, ("mbuf->m_nextpkt != NULL")); if (locked) { - mtx_assert(&txq->lock, MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); sfxge_tx_qdpl_swizzle(txq); if (stdp->std_get_count >= stdp->std_get_max) { txq->get_overflow++; return (ENOBUFS); } if (sfxge_is_mbuf_non_tcp(mbuf)) { if (stdp->std_get_non_tcp_count >= stdp->std_get_non_tcp_max) { txq->get_non_tcp_overflow++; return (ENOBUFS); } stdp->std_get_non_tcp_count++; } *(stdp->std_getp) = mbuf; stdp->std_getp = &mbuf->m_nextpkt; stdp->std_get_count++; } else { volatile uintptr_t *putp; uintptr_t old; uintptr_t new; unsigned old_len; putp = &stdp->std_put; new = (uintptr_t)mbuf; do { old = *putp; if (old != 0) { struct mbuf *mp = (struct mbuf *)old; old_len = mp->m_pkthdr.csum_data; } else old_len = 0; if (old_len >= stdp->std_put_max) { atomic_add_long(&txq->put_overflow, 1); return (ENOBUFS); } mbuf->m_pkthdr.csum_data = old_len + 1; mbuf->m_nextpkt = (void *)old; } while (atomic_cmpset_ptr(putp, old, new) == 0); } return (0); } /* * Called from if_transmit - will try to grab the txq lock and enqueue to the * put list if it succeeds, otherwise will push onto the defer list. */ int sfxge_tx_packet_add(struct sfxge_txq *txq, struct mbuf *m) { int locked; int rc; if (!SFXGE_LINK_UP(txq->sc)) { rc = ENETDOWN; atomic_add_long(&txq->netdown_drops, 1); goto fail; } /* * Try to grab the txq lock. If we are able to get the lock, * the packet will be appended to the "get list" of the deferred * packet list. Otherwise, it will be pushed on the "put list". */ - locked = mtx_trylock(&txq->lock); + locked = SFXGE_TXQ_TRYLOCK(txq); if (sfxge_tx_qdpl_put(txq, m, locked) != 0) { if (locked) - mtx_unlock(&txq->lock); + SFXGE_TXQ_UNLOCK(txq); rc = ENOBUFS; goto fail; } /* * Try to grab the lock again. * * If we are able to get the lock, we need to process the deferred * packet list. If we are not able to get the lock, another thread * is processing the list. */ if (!locked) - locked = mtx_trylock(&txq->lock); + locked = SFXGE_TXQ_TRYLOCK(txq); if (locked) { /* Try to service the list. */ sfxge_tx_qdpl_service(txq); /* Lock has been dropped. */ } return (0); fail: m_freem(m); return (rc); } static void sfxge_tx_qdpl_flush(struct sfxge_txq *txq) { struct sfxge_tx_dpl *stdp = &txq->dpl; struct mbuf *mbuf, *next; - mtx_lock(&txq->lock); + SFXGE_TXQ_LOCK(txq); sfxge_tx_qdpl_swizzle(txq); for (mbuf = stdp->std_get; mbuf != NULL; mbuf = next) { next = mbuf->m_nextpkt; m_freem(mbuf); } stdp->std_get = NULL; stdp->std_get_count = 0; stdp->std_get_non_tcp_count = 0; stdp->std_getp = &stdp->std_get; - mtx_unlock(&txq->lock); + SFXGE_TXQ_UNLOCK(txq); } void sfxge_if_qflush(struct ifnet *ifp) { struct sfxge_softc *sc; int i; sc = ifp->if_softc; for (i = 0; i < SFXGE_TX_SCALE(sc); i++) sfxge_tx_qdpl_flush(sc->txq[i]); } /* * TX start -- called by the stack. */ int sfxge_if_transmit(struct ifnet *ifp, struct mbuf *m) { struct sfxge_softc *sc; struct sfxge_txq *txq; int rc; sc = (struct sfxge_softc *)ifp->if_softc; KASSERT(ifp->if_flags & IFF_UP, ("interface not up")); /* Pick the desired transmit queue. */ if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) { int index = 0; /* check if flowid is set */ if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { uint32_t hash = m->m_pkthdr.flowid; index = sc->rx_indir_table[hash % SFXGE_RX_SCALE_MAX]; } txq = sc->txq[SFXGE_TXQ_IP_TCP_UDP_CKSUM + index]; } else if (m->m_pkthdr.csum_flags & CSUM_DELAY_IP) { txq = sc->txq[SFXGE_TXQ_IP_CKSUM]; } else { txq = sc->txq[SFXGE_TXQ_NON_CKSUM]; } rc = sfxge_tx_packet_add(txq, m); return (rc); } #else /* !SFXGE_HAVE_MQ */ static void sfxge_if_start_locked(struct ifnet *ifp) { struct sfxge_softc *sc = ifp->if_softc; struct sfxge_txq *txq; struct mbuf *mbuf; unsigned int pushed[SFXGE_TXQ_NTYPES]; unsigned int q_index; if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!sc->port.link_up) return; for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) { txq = sc->txq[q_index]; pushed[q_index] = txq->added; } while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, mbuf); if (mbuf == NULL) break; ETHER_BPF_MTAP(ifp, mbuf); /* packet capture */ /* Pick the desired transmit queue. */ if (mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_TSO)) q_index = SFXGE_TXQ_IP_TCP_UDP_CKSUM; else if (mbuf->m_pkthdr.csum_flags & CSUM_DELAY_IP) q_index = SFXGE_TXQ_IP_CKSUM; else q_index = SFXGE_TXQ_NON_CKSUM; txq = sc->txq[q_index]; if (sfxge_tx_queue_mbuf(txq, mbuf) != 0) continue; if (txq->blocked) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } /* Push the fragments to the hardware in batches. */ if (txq->added - pushed[q_index] >= SFXGE_TX_BATCH) { efx_tx_qpush(txq->common, txq->added); pushed[q_index] = txq->added; } } for (q_index = 0; q_index < SFXGE_TXQ_NTYPES; q_index++) { txq = sc->txq[q_index]; if (txq->added != pushed[q_index]) efx_tx_qpush(txq->common, txq->added); } } void sfxge_if_start(struct ifnet *ifp) { struct sfxge_softc *sc = ifp->if_softc; - mtx_lock(&sc->tx_lock); + SFXGE_TXQ_LOCK(sc->txq[0]); sfxge_if_start_locked(ifp); - mtx_unlock(&sc->tx_lock); + SFXGE_TXQ_UNLOCK(sc->txq[0]); } static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq) { - struct sfxge_softc *sc = txq->sc; - struct ifnet *ifp = sc->ifnet; + struct ifnet *ifp = txq->sc->ifnet; - mtx_assert(&sc->tx_lock, MA_OWNED); + SFXGE_TXQ_LOCK_ASSERT_OWNED(txq); ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sfxge_if_start_locked(ifp); - mtx_unlock(&sc->tx_lock); + SFXGE_TXQ_UNLOCK(txq); } #endif /* SFXGE_HAVE_MQ */ /* * Software "TSO". Not quite as good as doing it in hardware, but * still faster than segmenting in the stack. */ struct sfxge_tso_state { /* Output position */ unsigned out_len; /* Remaining length in current segment */ unsigned seqnum; /* Current sequence number */ unsigned packet_space; /* Remaining space in current packet */ /* Input position */ unsigned dma_seg_i; /* Current DMA segment number */ uint64_t dma_addr; /* DMA address of current position */ unsigned in_len; /* Remaining length in current mbuf */ const struct mbuf *mbuf; /* Input mbuf (head of chain) */ u_short protocol; /* Network protocol (after VLAN decap) */ ssize_t nh_off; /* Offset of network header */ ssize_t tcph_off; /* Offset of TCP header */ unsigned header_len; /* Number of bytes of header */ int full_packet_size; /* Number of bytes to put in each outgoing * segment */ }; static inline const struct ip *tso_iph(const struct sfxge_tso_state *tso) { KASSERT(tso->protocol == htons(ETHERTYPE_IP), ("tso_iph() in non-IPv4 state")); return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); } static inline const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) { KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), ("tso_ip6h() in non-IPv6 state")); return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); } static inline const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) { return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); } /* Size of preallocated TSO header buffers. Larger blocks must be * allocated from the heap. */ #define TSOH_STD_SIZE 128 /* At most half the descriptors in the queue at any time will refer to * a TSO header buffer, since they must always be followed by a * payload descriptor referring to an mbuf. */ #define TSOH_COUNT(_txq_entries) ((_txq_entries) / 2u) #define TSOH_PER_PAGE (PAGE_SIZE / TSOH_STD_SIZE) #define TSOH_PAGE_COUNT(_txq_entries) \ ((TSOH_COUNT(_txq_entries) + TSOH_PER_PAGE - 1) / TSOH_PER_PAGE) static int tso_init(struct sfxge_txq *txq) { struct sfxge_softc *sc = txq->sc; unsigned int tsoh_page_count = TSOH_PAGE_COUNT(sc->txq_entries); int i, rc; /* Allocate TSO header buffers */ txq->tsoh_buffer = malloc(tsoh_page_count * sizeof(txq->tsoh_buffer[0]), M_SFXGE, M_WAITOK); for (i = 0; i < tsoh_page_count; i++) { rc = sfxge_dma_alloc(sc, PAGE_SIZE, &txq->tsoh_buffer[i]); if (rc != 0) goto fail; } return (0); fail: while (i-- > 0) sfxge_dma_free(&txq->tsoh_buffer[i]); free(txq->tsoh_buffer, M_SFXGE); txq->tsoh_buffer = NULL; return (rc); } static void tso_fini(struct sfxge_txq *txq) { int i; if (txq->tsoh_buffer != NULL) { for (i = 0; i < TSOH_PAGE_COUNT(txq->sc->txq_entries); i++) sfxge_dma_free(&txq->tsoh_buffer[i]); free(txq->tsoh_buffer, M_SFXGE); } } static void tso_start(struct sfxge_tso_state *tso, struct mbuf *mbuf) { struct ether_header *eh = mtod(mbuf, struct ether_header *); tso->mbuf = mbuf; /* Find network protocol and header */ tso->protocol = eh->ether_type; if (tso->protocol == htons(ETHERTYPE_VLAN)) { struct ether_vlan_header *veh = mtod(mbuf, struct ether_vlan_header *); tso->protocol = veh->evl_proto; tso->nh_off = sizeof(*veh); } else { tso->nh_off = sizeof(*eh); } /* Find TCP header */ if (tso->protocol == htons(ETHERTYPE_IP)) { KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP, ("TSO required on non-TCP packet")); tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl; } else { KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), ("TSO required on non-IP packet")); KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP, ("TSO required on non-TCP packet")); tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr); } tso->header_len = tso->tcph_off + 4 * tso_tcph(tso)->th_off; tso->full_packet_size = tso->header_len + mbuf->m_pkthdr.tso_segsz; tso->seqnum = ntohl(tso_tcph(tso)->th_seq); /* These flags must not be duplicated */ KASSERT(!(tso_tcph(tso)->th_flags & (TH_URG | TH_SYN | TH_RST)), ("incompatible TCP flag on TSO packet")); tso->out_len = mbuf->m_pkthdr.len - tso->header_len; } /* * tso_fill_packet_with_fragment - form descriptors for the current fragment * * Form descriptors for the current fragment, until we reach the end * of fragment or end-of-packet. Return 0 on success, 1 if not enough * space. */ static void tso_fill_packet_with_fragment(struct sfxge_txq *txq, struct sfxge_tso_state *tso) { efx_buffer_t *desc; int n; if (tso->in_len == 0 || tso->packet_space == 0) return; KASSERT(tso->in_len > 0, ("TSO input length went negative")); KASSERT(tso->packet_space > 0, ("TSO packet space went negative")); n = min(tso->in_len, tso->packet_space); tso->packet_space -= n; tso->out_len -= n; tso->in_len -= n; desc = &txq->pend_desc[txq->n_pend_desc++]; desc->eb_addr = tso->dma_addr; desc->eb_size = n; desc->eb_eop = tso->out_len == 0 || tso->packet_space == 0; tso->dma_addr += n; } /* Callback from bus_dmamap_load() for long TSO headers. */ static void tso_map_long_header(void *dma_addr_ret, bus_dma_segment_t *segs, int nseg, int error) { *(uint64_t *)dma_addr_ret = ((__predict_true(error == 0) && __predict_true(nseg == 1)) ? segs->ds_addr : 0); } /* * tso_start_new_packet - generate a new header and prepare for the new packet * * Generate a new header and prepare for the new packet. Return 0 on * success, or an error code if failed to alloc header. */ static int tso_start_new_packet(struct sfxge_txq *txq, struct sfxge_tso_state *tso, unsigned int id) { struct sfxge_tx_mapping *stmp = &txq->stmp[id]; struct tcphdr *tsoh_th; unsigned ip_length; caddr_t header; uint64_t dma_addr; bus_dmamap_t map; efx_buffer_t *desc; int rc; /* Allocate a DMA-mapped header buffer. */ if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) { unsigned int page_index = (id / 2) / TSOH_PER_PAGE; unsigned int buf_index = (id / 2) % TSOH_PER_PAGE; header = (txq->tsoh_buffer[page_index].esm_base + buf_index * TSOH_STD_SIZE); dma_addr = (txq->tsoh_buffer[page_index].esm_addr + buf_index * TSOH_STD_SIZE); map = txq->tsoh_buffer[page_index].esm_map; stmp->flags = 0; } else { /* We cannot use bus_dmamem_alloc() as that may sleep */ header = malloc(tso->header_len, M_SFXGE, M_NOWAIT); if (__predict_false(!header)) return (ENOMEM); rc = bus_dmamap_load(txq->packet_dma_tag, stmp->map, header, tso->header_len, tso_map_long_header, &dma_addr, BUS_DMA_NOWAIT); if (__predict_false(dma_addr == 0)) { if (rc == 0) { /* Succeeded but got >1 segment */ bus_dmamap_unload(txq->packet_dma_tag, stmp->map); rc = EINVAL; } free(header, M_SFXGE); return (rc); } map = stmp->map; txq->tso_long_headers++; stmp->u.heap_buf = header; stmp->flags = TX_BUF_UNMAP; } tsoh_th = (struct tcphdr *)(header + tso->tcph_off); /* Copy and update the headers. */ m_copydata(tso->mbuf, 0, tso->header_len, header); tsoh_th->th_seq = htonl(tso->seqnum); tso->seqnum += tso->mbuf->m_pkthdr.tso_segsz; if (tso->out_len > tso->mbuf->m_pkthdr.tso_segsz) { /* This packet will not finish the TSO burst. */ ip_length = tso->full_packet_size - tso->nh_off; tsoh_th->th_flags &= ~(TH_FIN | TH_PUSH); } else { /* This packet will be the last in the TSO burst. */ ip_length = tso->header_len - tso->nh_off + tso->out_len; } if (tso->protocol == htons(ETHERTYPE_IP)) { struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off); tsoh_iph->ip_len = htons(ip_length); /* XXX We should increment ip_id, but FreeBSD doesn't * currently allocate extra IDs for multiple segments. */ } else { struct ip6_hdr *tsoh_iph = (struct ip6_hdr *)(header + tso->nh_off); tsoh_iph->ip6_plen = htons(ip_length - sizeof(*tsoh_iph)); } /* Make the header visible to the hardware. */ bus_dmamap_sync(txq->packet_dma_tag, map, BUS_DMASYNC_PREWRITE); tso->packet_space = tso->mbuf->m_pkthdr.tso_segsz; txq->tso_packets++; /* Form a descriptor for this header. */ desc = &txq->pend_desc[txq->n_pend_desc++]; desc->eb_addr = dma_addr; desc->eb_size = tso->header_len; desc->eb_eop = 0; return (0); } static int sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, const bus_dma_segment_t *dma_seg, int n_dma_seg) { struct sfxge_tso_state tso; unsigned int id, next_id; unsigned skipped = 0; tso_start(&tso, mbuf); while (dma_seg->ds_len + skipped <= tso.header_len) { skipped += dma_seg->ds_len; --n_dma_seg; KASSERT(n_dma_seg, ("no payload found in TSO packet")); ++dma_seg; } tso.in_len = dma_seg->ds_len + (tso.header_len - skipped); tso.dma_addr = dma_seg->ds_addr + (tso.header_len - skipped); id = txq->added & txq->ptr_mask; if (__predict_false(tso_start_new_packet(txq, &tso, id))) return (-1); while (1) { id = (id + 1) & txq->ptr_mask; tso_fill_packet_with_fragment(txq, &tso); /* Move onto the next fragment? */ if (tso.in_len == 0) { --n_dma_seg; if (n_dma_seg == 0) break; ++dma_seg; tso.in_len = dma_seg->ds_len; tso.dma_addr = dma_seg->ds_addr; } /* End of packet? */ if (tso.packet_space == 0) { /* If the queue is now full due to tiny MSS, * or we can't create another header, discard * the remainder of the input mbuf but do not * roll back the work we have done. */ if (txq->n_pend_desc > SFXGE_TSO_MAX_DESC - (1 + SFXGE_TX_MAPPING_MAX_SEG)) break; next_id = (id + 1) & txq->ptr_mask; if (__predict_false(tso_start_new_packet(txq, &tso, next_id))) break; id = next_id; } } txq->tso_bursts++; return (id); } static void sfxge_tx_qunblock(struct sfxge_txq *txq) { struct sfxge_softc *sc; struct sfxge_evq *evq; sc = txq->sc; evq = sc->evq[txq->evq_index]; - mtx_assert(&evq->lock, MA_OWNED); + SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); if (txq->init_state != SFXGE_TXQ_STARTED) return; - mtx_lock(SFXGE_TXQ_LOCK(txq)); + SFXGE_TXQ_LOCK(txq); if (txq->blocked) { unsigned int level; level = txq->added - txq->completed; if (level <= SFXGE_TXQ_UNBLOCK_LEVEL(txq->entries)) txq->blocked = 0; } sfxge_tx_qdpl_service(txq); /* note: lock has been dropped */ } void sfxge_tx_qflush_done(struct sfxge_txq *txq) { txq->flush_state = SFXGE_FLUSH_DONE; } static void sfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index) { struct sfxge_txq *txq; struct sfxge_evq *evq; unsigned int count; txq = sc->txq[index]; evq = sc->evq[txq->evq_index]; - mtx_lock(SFXGE_TXQ_LOCK(txq)); + SFXGE_TXQ_LOCK(txq); KASSERT(txq->init_state == SFXGE_TXQ_STARTED, ("txq->init_state != SFXGE_TXQ_STARTED")); txq->init_state = SFXGE_TXQ_INITIALIZED; txq->flush_state = SFXGE_FLUSH_PENDING; /* Flush the transmit queue. */ efx_tx_qflush(txq->common); - mtx_unlock(SFXGE_TXQ_LOCK(txq)); + SFXGE_TXQ_UNLOCK(txq); count = 0; do { /* Spin for 100ms. */ DELAY(100000); if (txq->flush_state != SFXGE_FLUSH_PENDING) break; } while (++count < 20); - mtx_lock(&evq->lock); - mtx_lock(SFXGE_TXQ_LOCK(txq)); + SFXGE_EVQ_LOCK(evq); + SFXGE_TXQ_LOCK(txq); KASSERT(txq->flush_state != SFXGE_FLUSH_FAILED, ("txq->flush_state == SFXGE_FLUSH_FAILED")); txq->flush_state = SFXGE_FLUSH_DONE; txq->blocked = 0; txq->pending = txq->added; sfxge_tx_qcomplete(txq, evq); KASSERT(txq->completed == txq->added, ("txq->completed != txq->added")); sfxge_tx_qreap(txq); KASSERT(txq->reaped == txq->completed, ("txq->reaped != txq->completed")); txq->added = 0; txq->pending = 0; txq->completed = 0; txq->reaped = 0; /* Destroy the common code transmit queue. */ efx_tx_qdestroy(txq->common); txq->common = NULL; efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id, EFX_TXQ_NBUFS(sc->txq_entries)); - mtx_unlock(&evq->lock); - mtx_unlock(SFXGE_TXQ_LOCK(txq)); + SFXGE_EVQ_UNLOCK(evq); + SFXGE_TXQ_UNLOCK(txq); } static int sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_txq *txq; efsys_mem_t *esmp; uint16_t flags; struct sfxge_evq *evq; int rc; txq = sc->txq[index]; esmp = &txq->mem; evq = sc->evq[txq->evq_index]; KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, ("txq->init_state != SFXGE_TXQ_INITIALIZED")); KASSERT(evq->init_state == SFXGE_EVQ_STARTED, ("evq->init_state != SFXGE_EVQ_STARTED")); /* Program the buffer table. */ if ((rc = efx_sram_buf_tbl_set(sc->enp, txq->buf_base_id, esmp, EFX_TXQ_NBUFS(sc->txq_entries))) != 0) return (rc); /* Determine the kind of queue we are creating. */ switch (txq->type) { case SFXGE_TXQ_NON_CKSUM: flags = 0; break; case SFXGE_TXQ_IP_CKSUM: flags = EFX_CKSUM_IPV4; break; case SFXGE_TXQ_IP_TCP_UDP_CKSUM: flags = EFX_CKSUM_IPV4 | EFX_CKSUM_TCPUDP; break; default: KASSERT(0, ("Impossible TX queue")); flags = 0; break; } /* Create the common code transmit queue. */ if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp, sc->txq_entries, txq->buf_base_id, flags, evq->common, &txq->common)) != 0) goto fail; - mtx_lock(SFXGE_TXQ_LOCK(txq)); + SFXGE_TXQ_LOCK(txq); /* Enable the transmit queue. */ efx_tx_qenable(txq->common); txq->init_state = SFXGE_TXQ_STARTED; - mtx_unlock(SFXGE_TXQ_LOCK(txq)); + SFXGE_TXQ_UNLOCK(txq); return (0); fail: efx_sram_buf_tbl_clear(sc->enp, txq->buf_base_id, EFX_TXQ_NBUFS(sc->txq_entries)); return (rc); } void sfxge_tx_stop(struct sfxge_softc *sc) { const efx_nic_cfg_t *encp; int index; index = SFXGE_TX_SCALE(sc); while (--index >= 0) sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM); encp = efx_nic_cfg_get(sc->enp); sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM); /* Tear down the transmit module */ efx_tx_fini(sc->enp); } int sfxge_tx_start(struct sfxge_softc *sc) { int index; int rc; /* Initialize the common code transmit module. */ if ((rc = efx_tx_init(sc->enp)) != 0) return (rc); if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_NON_CKSUM)) != 0) goto fail; if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_CKSUM)) != 0) goto fail2; for (index = 0; index < SFXGE_TX_SCALE(sc); index++) { if ((rc = sfxge_tx_qstart(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index)) != 0) goto fail3; } return (0); fail3: while (--index >= 0) sfxge_tx_qstop(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); sfxge_tx_qstop(sc, SFXGE_TXQ_IP_CKSUM); fail2: sfxge_tx_qstop(sc, SFXGE_TXQ_NON_CKSUM); fail: efx_tx_fini(sc->enp); return (rc); } /** * Destroy a transmit queue. */ static void sfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index) { struct sfxge_txq *txq; unsigned int nmaps; txq = sc->txq[index]; KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, ("txq->init_state != SFXGE_TXQ_INITIALIZED")); if (txq->type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) tso_fini(txq); /* Free the context arrays. */ free(txq->pend_desc, M_SFXGE); nmaps = sc->txq_entries; while (nmaps-- != 0) bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map); free(txq->stmp, M_SFXGE); /* Release DMA memory mapping. */ sfxge_dma_free(&txq->mem); sc->txq[index] = NULL; #ifdef SFXGE_HAVE_MQ - mtx_destroy(&txq->lock); + SFXGE_TXQ_LOCK_DESTROY(txq); #endif free(txq, M_SFXGE); } static int sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index, enum sfxge_txq_type type, unsigned int evq_index) { char name[16]; struct sysctl_oid *txq_node; struct sfxge_txq *txq; struct sfxge_evq *evq; #ifdef SFXGE_HAVE_MQ struct sfxge_tx_dpl *stdp; #endif efsys_mem_t *esmp; unsigned int nmaps; int rc; txq = malloc(sizeof(struct sfxge_txq), M_SFXGE, M_ZERO | M_WAITOK); txq->sc = sc; txq->entries = sc->txq_entries; txq->ptr_mask = txq->entries - 1; sc->txq[txq_index] = txq; esmp = &txq->mem; evq = sc->evq[evq_index]; /* Allocate and zero DMA space for the descriptor ring. */ if ((rc = sfxge_dma_alloc(sc, EFX_TXQ_SIZE(sc->txq_entries), esmp)) != 0) return (rc); (void)memset(esmp->esm_base, 0, EFX_TXQ_SIZE(sc->txq_entries)); /* Allocate buffer table entries. */ sfxge_sram_buf_tbl_alloc(sc, EFX_TXQ_NBUFS(sc->txq_entries), &txq->buf_base_id); /* Create a DMA tag for packet mappings. */ if (bus_dma_tag_create(sc->parent_dma_tag, 1, 0x1000, MIN(0x3FFFFFFFFFFFUL, BUS_SPACE_MAXADDR), BUS_SPACE_MAXADDR, NULL, NULL, 0x11000, SFXGE_TX_MAPPING_MAX_SEG, 0x1000, 0, NULL, NULL, &txq->packet_dma_tag) != 0) { device_printf(sc->dev, "Couldn't allocate txq DMA tag\n"); rc = ENOMEM; goto fail; } /* Allocate pending descriptor array for batching writes. */ txq->pend_desc = malloc(sizeof(efx_buffer_t) * sc->txq_entries, M_SFXGE, M_ZERO | M_WAITOK); /* Allocate and initialise mbuf DMA mapping array. */ txq->stmp = malloc(sizeof(struct sfxge_tx_mapping) * sc->txq_entries, M_SFXGE, M_ZERO | M_WAITOK); for (nmaps = 0; nmaps < sc->txq_entries; nmaps++) { rc = bus_dmamap_create(txq->packet_dma_tag, 0, &txq->stmp[nmaps].map); if (rc != 0) goto fail2; } snprintf(name, sizeof(name), "%u", txq_index); txq_node = SYSCTL_ADD_NODE( device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->txqs_node), OID_AUTO, name, CTLFLAG_RD, NULL, ""); if (txq_node == NULL) { rc = ENOMEM; goto fail_txq_node; } if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM && (rc = tso_init(txq)) != 0) goto fail3; #ifdef SFXGE_HAVE_MQ if (sfxge_tx_dpl_get_max <= 0) { log(LOG_ERR, "%s=%d must be greater than 0", SFXGE_PARAM_TX_DPL_GET_MAX, sfxge_tx_dpl_get_max); rc = EINVAL; goto fail_tx_dpl_get_max; } if (sfxge_tx_dpl_get_non_tcp_max <= 0) { log(LOG_ERR, "%s=%d must be greater than 0", SFXGE_PARAM_TX_DPL_GET_NON_TCP_MAX, sfxge_tx_dpl_get_non_tcp_max); rc = EINVAL; goto fail_tx_dpl_get_max; } if (sfxge_tx_dpl_put_max < 0) { log(LOG_ERR, "%s=%d must be greater or equal to 0", SFXGE_PARAM_TX_DPL_PUT_MAX, sfxge_tx_dpl_put_max); rc = EINVAL; goto fail_tx_dpl_put_max; } /* Initialize the deferred packet list. */ stdp = &txq->dpl; stdp->std_put_max = sfxge_tx_dpl_put_max; stdp->std_get_max = sfxge_tx_dpl_get_max; stdp->std_get_non_tcp_max = sfxge_tx_dpl_get_non_tcp_max; stdp->std_getp = &stdp->std_get; - mtx_init(&txq->lock, "txq", NULL, MTX_DEF); + SFXGE_TXQ_LOCK_INIT(txq, "txq"); SYSCTL_ADD_UINT(device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(txq_node), OID_AUTO, "dpl_get_count", CTLFLAG_RD | CTLFLAG_STATS, &stdp->std_get_count, 0, ""); SYSCTL_ADD_UINT(device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(txq_node), OID_AUTO, "dpl_get_non_tcp_count", CTLFLAG_RD | CTLFLAG_STATS, &stdp->std_get_non_tcp_count, 0, ""); SYSCTL_ADD_UINT(device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(txq_node), OID_AUTO, "dpl_get_hiwat", CTLFLAG_RD | CTLFLAG_STATS, &stdp->std_get_hiwat, 0, ""); #endif txq->type = type; txq->evq_index = evq_index; txq->txq_index = txq_index; txq->init_state = SFXGE_TXQ_INITIALIZED; return (0); fail_tx_dpl_put_max: fail_tx_dpl_get_max: fail3: fail_txq_node: free(txq->pend_desc, M_SFXGE); fail2: while (nmaps-- != 0) bus_dmamap_destroy(txq->packet_dma_tag, txq->stmp[nmaps].map); free(txq->stmp, M_SFXGE); bus_dma_tag_destroy(txq->packet_dma_tag); fail: sfxge_dma_free(esmp); return (rc); } static const struct { const char *name; size_t offset; } sfxge_tx_stats[] = { #define SFXGE_TX_STAT(name, member) \ { #name, offsetof(struct sfxge_txq, member) } SFXGE_TX_STAT(tso_bursts, tso_bursts), SFXGE_TX_STAT(tso_packets, tso_packets), SFXGE_TX_STAT(tso_long_headers, tso_long_headers), SFXGE_TX_STAT(tx_collapses, collapses), SFXGE_TX_STAT(tx_drops, drops), SFXGE_TX_STAT(tx_get_overflow, get_overflow), SFXGE_TX_STAT(tx_get_non_tcp_overflow, get_non_tcp_overflow), SFXGE_TX_STAT(tx_put_overflow, put_overflow), SFXGE_TX_STAT(tx_netdown_drops, netdown_drops), }; static int sfxge_tx_stat_handler(SYSCTL_HANDLER_ARGS) { struct sfxge_softc *sc = arg1; unsigned int id = arg2; unsigned long sum; unsigned int index; /* Sum across all TX queues */ sum = 0; for (index = 0; index < SFXGE_TXQ_IP_TCP_UDP_CKSUM + SFXGE_TX_SCALE(sc); index++) sum += *(unsigned long *)((caddr_t)sc->txq[index] + sfxge_tx_stats[id].offset); return (SYSCTL_OUT(req, &sum, sizeof(sum))); } static void sfxge_tx_stat_init(struct sfxge_softc *sc) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); struct sysctl_oid_list *stat_list; unsigned int id; stat_list = SYSCTL_CHILDREN(sc->stats_node); for (id = 0; id < sizeof(sfxge_tx_stats) / sizeof(sfxge_tx_stats[0]); id++) { SYSCTL_ADD_PROC( ctx, stat_list, OID_AUTO, sfxge_tx_stats[id].name, CTLTYPE_ULONG|CTLFLAG_RD, sc, id, sfxge_tx_stat_handler, "LU", ""); } } void sfxge_tx_fini(struct sfxge_softc *sc) { int index; index = SFXGE_TX_SCALE(sc); while (--index >= 0) sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM); sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM); } int sfxge_tx_init(struct sfxge_softc *sc) { struct sfxge_intr *intr; int index; int rc; intr = &sc->intr; KASSERT(intr->state == SFXGE_INTR_INITIALIZED, ("intr->state != SFXGE_INTR_INITIALIZED")); sc->txqs_node = SYSCTL_ADD_NODE( device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "txq", CTLFLAG_RD, NULL, "Tx queues"); if (sc->txqs_node == NULL) { rc = ENOMEM; goto fail_txq_node; } /* Initialize the transmit queues */ if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_NON_CKSUM, SFXGE_TXQ_NON_CKSUM, 0)) != 0) goto fail; if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_CKSUM, SFXGE_TXQ_IP_CKSUM, 0)) != 0) goto fail2; for (index = 0; index < SFXGE_TX_SCALE(sc); index++) { if ((rc = sfxge_tx_qinit(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index, SFXGE_TXQ_IP_TCP_UDP_CKSUM, index)) != 0) goto fail3; } sfxge_tx_stat_init(sc); return (0); fail3: sfxge_tx_qfini(sc, SFXGE_TXQ_IP_CKSUM); while (--index >= 0) sfxge_tx_qfini(sc, SFXGE_TXQ_IP_TCP_UDP_CKSUM + index); fail2: sfxge_tx_qfini(sc, SFXGE_TXQ_NON_CKSUM); fail: fail_txq_node: return (rc); } Index: head/sys/dev/sfxge/sfxge_tx.h =================================================================== --- head/sys/dev/sfxge/sfxge_tx.h (revision 278220) +++ head/sys/dev/sfxge/sfxge_tx.h (revision 278221) @@ -1,206 +1,220 @@ /*- * Copyright (c) 2010-2011 Solarflare Communications, Inc. * All rights reserved. * * This software was developed in part by Philip Paeps under contract for * Solarflare Communications, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SFXGE_TX_H #define _SFXGE_TX_H #include #include #include /* Maximum number of DMA segments needed to map an mbuf chain. With * TSO, the mbuf length may be just over 64K, divided into 2K mbuf * clusters. (The chain could be longer than this initially, but can * be shortened with m_collapse().) */ #define SFXGE_TX_MAPPING_MAX_SEG (64 / 2 + 1) /* Maximum number of DMA segments needed to map an output packet. It * could overlap all mbufs in the chain and also require an extra * segment for a TSO header. */ #define SFXGE_TX_PACKET_MAX_SEG (SFXGE_TX_MAPPING_MAX_SEG + 1) /* * Buffer mapping flags. * * Buffers and DMA mappings must be freed when the last descriptor * referring to them is completed. Set the TX_BUF_UNMAP and * TX_BUF_MBUF flags on the last descriptor generated for an mbuf * chain. Set only the TX_BUF_UNMAP flag on a descriptor referring to * a heap buffer. */ enum sfxge_tx_buf_flags { TX_BUF_UNMAP = 1, TX_BUF_MBUF = 2, }; /* * Buffer mapping information for descriptors in flight. */ struct sfxge_tx_mapping { union { struct mbuf *mbuf; caddr_t heap_buf; } u; bus_dmamap_t map; enum sfxge_tx_buf_flags flags; }; #define SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT (64 * 1024) #define SFXGE_TX_DPL_GET_NON_TCP_PKT_LIMIT_DEFAULT 1024 #define SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT 64 /* * Deferred packet list. */ struct sfxge_tx_dpl { unsigned int std_get_max; /* Maximum number of packets * in get list */ unsigned int std_get_non_tcp_max; /* Maximum number * of non-TCP packets * in get list */ unsigned int std_put_max; /* Maximum number of packets * in put list */ uintptr_t std_put; /* Head of put list. */ struct mbuf *std_get; /* Head of get list. */ struct mbuf **std_getp; /* Tail of get list. */ unsigned int std_get_count; /* Packets in get list. */ unsigned int std_get_non_tcp_count; /* Non-TCP packets * in get list */ unsigned int std_get_hiwat; /* Packets in get list * high watermark */ }; #define SFXGE_TX_BUFFER_SIZE 0x400 #define SFXGE_TX_HEADER_SIZE 0x100 #define SFXGE_TX_COPY_THRESHOLD 0x200 enum sfxge_txq_state { SFXGE_TXQ_UNINITIALIZED = 0, SFXGE_TXQ_INITIALIZED, SFXGE_TXQ_STARTED }; enum sfxge_txq_type { SFXGE_TXQ_NON_CKSUM = 0, SFXGE_TXQ_IP_CKSUM, SFXGE_TXQ_IP_TCP_UDP_CKSUM, SFXGE_TXQ_NTYPES }; #define SFXGE_TXQ_UNBLOCK_LEVEL(_entries) (EFX_TXQ_LIMIT(_entries) / 4) #define SFXGE_TX_BATCH 64 #ifdef SFXGE_HAVE_MQ -#define SFXGE_TXQ_LOCK(txq) (&(txq)->lock) +#define SFXGE_TX_LOCK(txq) (&(txq)->lock) #define SFXGE_TX_SCALE(sc) ((sc)->intr.n_alloc) #else -#define SFXGE_TXQ_LOCK(txq) (&(txq)->sc->tx_lock) +#define SFXGE_TX_LOCK(txq) (&(txq)->sc->tx_lock) #define SFXGE_TX_SCALE(sc) 1 #endif + +#define SFXGE_TXQ_LOCK_INIT(_txq, _name) \ + mtx_init(&(_txq)->lock, (_name), NULL, MTX_DEF) +#define SFXGE_TXQ_LOCK_DESTROY(_txq) \ + mtx_destroy(&(_txq)->lock) +#define SFXGE_TXQ_LOCK(_txq) \ + mtx_lock(SFXGE_TX_LOCK(_txq)) +#define SFXGE_TXQ_TRYLOCK(_txq) \ + mtx_trylock(SFXGE_TX_LOCK(_txq)) +#define SFXGE_TXQ_UNLOCK(_txq) \ + mtx_unlock(SFXGE_TX_LOCK(_txq)) +#define SFXGE_TXQ_LOCK_ASSERT_OWNED(_txq) \ + mtx_assert(SFXGE_TX_LOCK(_txq), MA_OWNED) + struct sfxge_txq { /* The following fields should be written very rarely */ struct sfxge_softc *sc; enum sfxge_txq_state init_state; enum sfxge_flush_state flush_state; enum sfxge_txq_type type; unsigned int txq_index; unsigned int evq_index; efsys_mem_t mem; unsigned int buf_base_id; unsigned int entries; unsigned int ptr_mask; struct sfxge_tx_mapping *stmp; /* Packets in flight. */ bus_dma_tag_t packet_dma_tag; efx_buffer_t *pend_desc; efx_txq_t *common; efsys_mem_t *tsoh_buffer; /* This field changes more often and is read regularly on both * the initiation and completion paths */ int blocked __aligned(CACHE_LINE_SIZE); /* The following fields change more often, and are used mostly * on the initiation path */ #ifdef SFXGE_HAVE_MQ struct mtx lock __aligned(CACHE_LINE_SIZE); struct sfxge_tx_dpl dpl; /* Deferred packet list. */ unsigned int n_pend_desc; #else unsigned int n_pend_desc __aligned(CACHE_LINE_SIZE); #endif unsigned int added; unsigned int reaped; /* Statistics */ unsigned long tso_bursts; unsigned long tso_packets; unsigned long tso_long_headers; unsigned long collapses; unsigned long drops; unsigned long get_overflow; unsigned long get_non_tcp_overflow; unsigned long put_overflow; unsigned long netdown_drops; /* The following fields change more often, and are used mostly * on the completion path */ unsigned int pending __aligned(CACHE_LINE_SIZE); unsigned int completed; struct sfxge_txq *next; }; struct sfxge_evq; extern int sfxge_tx_packet_add(struct sfxge_txq *, struct mbuf *); extern int sfxge_tx_init(struct sfxge_softc *sc); extern void sfxge_tx_fini(struct sfxge_softc *sc); extern int sfxge_tx_start(struct sfxge_softc *sc); extern void sfxge_tx_stop(struct sfxge_softc *sc); extern void sfxge_tx_qcomplete(struct sfxge_txq *txq, struct sfxge_evq *evq); extern void sfxge_tx_qflush_done(struct sfxge_txq *txq); #ifdef SFXGE_HAVE_MQ extern void sfxge_if_qflush(struct ifnet *ifp); extern int sfxge_if_transmit(struct ifnet *ifp, struct mbuf *m); #else extern void sfxge_if_start(struct ifnet *ifp); #endif #endif