Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
#include <sys/syslog.h> | #include <sys/syslog.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_media.h> | #include <net/if_media.h> | ||||
#include <net/if_vxlan.h> | |||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/mp_ring.h> | #include <net/mp_ring.h> | ||||
#include <net/debugnet.h> | #include <net/debugnet.h> | ||||
#include <net/pfil.h> | #include <net/pfil.h> | ||||
#include <net/vnet.h> | #include <net/vnet.h> | ||||
#include <netinet/in.h> | #include <netinet/in.h> | ||||
#include <netinet/in_pcb.h> | #include <netinet/in_pcb.h> | ||||
#include <netinet/tcp_lro.h> | #include <netinet/tcp_lro.h> | ||||
#include <netinet/in_systm.h> | #include <netinet/in_systm.h> | ||||
#include <netinet/if_ether.h> | #include <netinet/if_ether.h> | ||||
#include <netinet/ip.h> | #include <netinet/ip.h> | ||||
#include <netinet/ip6.h> | #include <netinet/ip6.h> | ||||
#include <netinet/tcp.h> | #include <netinet/tcp.h> | ||||
#include <netinet/udp.h> | |||||
#include <netinet/ip_var.h> | #include <netinet/ip_var.h> | ||||
#include <netinet6/ip6_var.h> | #include <netinet6/ip6_var.h> | ||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/in_cksum.h> | #include <machine/in_cksum.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | |||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
#define isc_rxd_available ifc_txrx.ift_rxd_available | #define isc_rxd_available ifc_txrx.ift_rxd_available | ||||
#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_rxd_flush ifc_txrx.ift_rxd_flush | #define isc_rxd_flush ifc_txrx.ift_rxd_flush | ||||
#define isc_legacy_intr ifc_txrx.ift_legacy_intr | #define isc_legacy_intr ifc_txrx.ift_legacy_intr | ||||
eventhandler_tag ifc_vlan_attach_event; | eventhandler_tag ifc_vlan_attach_event; | ||||
eventhandler_tag ifc_vlan_detach_event; | eventhandler_tag ifc_vlan_detach_event; | ||||
eventhandler_tag ifc_vxlan_attach_event; | |||||
eventhandler_tag ifc_vxlan_detach_event; | |||||
struct ether_addr ifc_mac; | struct ether_addr ifc_mac; | ||||
}; | }; | ||||
void * | void * | ||||
iflib_get_softc(if_ctx_t ctx) | iflib_get_softc(if_ctx_t ctx) | ||||
{ | { | ||||
return (ctx->ifc_softc); | return (ctx->ifc_softc); | ||||
▲ Show 20 Lines • Show All 481 Lines • ▼ Show 20 Lines | |||||
static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | ||||
static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | ||||
#ifdef ALTQ | #ifdef ALTQ | ||||
static void iflib_altq_if_start(if_t ifp); | static void iflib_altq_if_start(if_t ifp); | ||||
static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m); | static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m); | ||||
#endif | #endif | ||||
static int iflib_register(if_ctx_t); | static int iflib_register(if_ctx_t); | ||||
static void iflib_deregister(if_ctx_t); | static void iflib_deregister(if_ctx_t); | ||||
static void iflib_unregister_vlan_handlers(if_ctx_t ctx); | static void iflib_unregister_event_handlers(if_ctx_t ctx); | ||||
static uint16_t iflib_get_mbuf_size_for(unsigned int size); | static uint16_t iflib_get_mbuf_size_for(unsigned int size); | ||||
static void iflib_init_locked(if_ctx_t ctx); | static void iflib_init_locked(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_post(if_ctx_t ctx); | static void iflib_add_device_sysctl_post(if_ctx_t ctx); | ||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static void iflib_free_intr_mem(if_ctx_t ctx); | static void iflib_free_intr_mem(if_ctx_t ctx); | ||||
▲ Show 20 Lines • Show All 1,739 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
iflib_rxq_t rxq; | iflib_rxq_t rxq; | ||||
int i, j, tx_ip_csum_flags, tx_ip6_csum_flags; | int i, j, tx_ip_csum_flags, tx_ip6_csum_flags; | ||||
int vxlan_hwcsum_flags, vxlan_hwtso_flags; | |||||
if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
/* | /* | ||||
* See iflib_stop(). Useful in case iflib_init_locked() is | * See iflib_stop(). Useful in case iflib_init_locked() is | ||||
* called without first calling iflib_stop(). | * called without first calling iflib_stop(). | ||||
*/ | */ | ||||
netmap_disable_all_rings(ifp); | netmap_disable_all_rings(ifp); | ||||
tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); | tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); | ||||
tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); | tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); | ||||
/* CSUM_ENCAP_VXLAN has to be set if any of VXLAN offloads is enabled */ | |||||
vxlan_hwcsum_flags = scctx->isc_tx_csum_flags & (CSUM_INNER_IP6_UDP | | |||||
CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP | | |||||
CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_ENCAP_VXLAN); | |||||
vxlan_hwtso_flags = scctx->isc_tx_csum_flags & (CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO | | |||||
CSUM_ENCAP_VXLAN); | |||||
/* Set hardware offload abilities */ | /* Set hardware offload abilities */ | ||||
if_clearhwassist(ifp); | if_clearhwassist(ifp); | ||||
if (if_getcapenable(ifp) & IFCAP_TXCSUM) | if (if_getcapenable(ifp) & IFCAP_TXCSUM) | ||||
if_sethwassistbits(ifp, tx_ip_csum_flags, 0); | if_sethwassistbits(ifp, tx_ip_csum_flags, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) | if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) | ||||
if_sethwassistbits(ifp, tx_ip6_csum_flags, 0); | if_sethwassistbits(ifp, tx_ip6_csum_flags, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TSO4) | if (if_getcapenable(ifp) & IFCAP_TSO4) | ||||
if_sethwassistbits(ifp, CSUM_IP_TSO, 0); | if_sethwassistbits(ifp, CSUM_IP_TSO, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TSO6) | if (if_getcapenable(ifp) & IFCAP_TSO6) | ||||
if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); | if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_VXLAN_HWCSUM) | |||||
if_sethwassistbits(ifp, vxlan_hwcsum_flags, 0); | |||||
if (if_getcapenable(ifp) & IFCAP_VXLAN_HWTSO) | |||||
if_sethwassistbits(ifp, vxlan_hwtso_flags, 0); | |||||
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { | for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
callout_stop(&txq->ift_netmap_timer); | callout_stop(&txq->ift_netmap_timer); | ||||
#endif /* DEV_NETMAP */ | #endif /* DEV_NETMAP */ | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
▲ Show 20 Lines • Show All 655 Lines • ▼ Show 20 Lines | |||||
print_pkt(if_pkt_info_t pi) | print_pkt(if_pkt_info_t pi) | ||||
{ | { | ||||
printf("pi len: %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n", | printf("pi len: %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n", | ||||
pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx); | pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx); | ||||
printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n", | printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n", | ||||
pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); | pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); | ||||
printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n", | printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n", | ||||
pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); | pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); | ||||
printf("pi outer_etype: %d outer_ip_len: %d tun_len: %d\n", | |||||
pi->ipi_outer_etype, pi->ipi_outer_ip_hlen, pi->ipi_tun_hlen); | |||||
} | } | ||||
#endif | #endif | ||||
#define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | ||||
#define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP_TSO)) | #define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP_TSO)) | ||||
#define IS_TX_INNER_OFFLOAD4(pi) ((pi)->ipi_csum_flags & \ | |||||
(CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO)) | |||||
#define IS_INNER_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_INNER_IP_TSO) | |||||
#define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | ||||
#define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_TSO)) | #define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_TSO)) | ||||
#define IS_TX_INNER_OFFLOAD6(pi) ((pi)->ipi_csum_flags & \ | |||||
(CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO)) | |||||
#define IS_INNER_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_INNER_IP6_TSO) | |||||
static int | static int | ||||
iflib_parse_inner_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | |||||
{ | |||||
if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | |||||
struct ether_vlan_header *eh; | |||||
struct ip *outer_ip; | |||||
int ehdrlen; | |||||
struct mbuf *m; | |||||
size_t off; | |||||
if (pi->ipi_ipproto != IPPROTO_UDP) | |||||
return (ENXIO); | |||||
m = *mp; | |||||
/* | |||||
* Save outer frame info and reuse etype and ip_hlen for inner frame. | |||||
*/ | |||||
pi->ipi_outer_etype = pi->ipi_etype; | |||||
pi->ipi_outer_ip_hlen = pi->ipi_ip_hlen; | |||||
pi->ipi_tun_hlen = sizeof(struct udphdr) + sizeof(struct vxlan_header); | |||||
/* size of outer frame header */ | |||||
off = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tun_hlen; | |||||
outer_ip = (struct ip *)((caddr_t)m->m_data + pi->ipi_ehdrlen); | |||||
/* For VXLAN first mbuf usually contains only outer frame headers */ | |||||
if (m->m_len == off) { | |||||
m = m->m_next; | |||||
off = 0; | |||||
} | |||||
if (__predict_false((size_t)m->m_len < off + sizeof(*eh))) | |||||
return (ENOMEM); | |||||
eh = (struct ether_vlan_header *)((caddr_t)m->m_data + off); | |||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | |||||
pi->ipi_etype = ntohs(eh->evl_proto); | |||||
ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | |||||
} else { | |||||
pi->ipi_etype = ntohs(eh->evl_encap_proto); | |||||
ehdrlen = ETHER_HDR_LEN; | |||||
} | |||||
pi->ipi_tun_hlen += ehdrlen; | |||||
switch (pi->ipi_etype) { | |||||
#ifdef INET | |||||
case ETHERTYPE_IP: | |||||
{ | |||||
struct ip *ip = NULL; | |||||
struct tcphdr *th = NULL; | |||||
int minhlen = off + ehdrlen + sizeof(*ip); | |||||
if (m->m_pkthdr.csum_flags & (CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO)) | |||||
minhlen += sizeof(*th); | |||||
minhlen = min(m->m_pkthdr.len, minhlen); | |||||
if (__predict_false(m->m_len < minhlen)) { | |||||
txq->ift_pullups++; | |||||
if (__predict_false((m = m_pullup(m, minhlen)) == NULL)) | |||||
return (ENOMEM); | |||||
} | |||||
ip = (struct ip *)(m->m_data + ehdrlen); | |||||
if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) | |||||
th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); | |||||
pi->ipi_ip_hlen = ip->ip_hl << 2; | |||||
pi->ipi_ipproto = ip->ip_p; | |||||
/* TCP checksum offload may require TCP header length */ | |||||
if (IS_TX_INNER_OFFLOAD4(pi)) { | |||||
if (__predict_false(ip->ip_p != IPPROTO_TCP)) | |||||
return (ENXIO); | |||||
if (__predict_false(th == NULL)) { | |||||
txq->ift_pullups++; | |||||
if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) | |||||
return (ENOMEM); | |||||
th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); | |||||
} | |||||
pi->ipi_tcp_hflags = th->th_flags; | |||||
pi->ipi_tcp_hlen = th->th_off << 2; | |||||
pi->ipi_tcp_seq = th->th_seq; | |||||
if (IS_INNER_TSO4(pi)) { | |||||
/* | |||||
* TSO always requires hardware checksum offload. | |||||
*/ | |||||
pi->ipi_csum_flags |= (CSUM_INNER_IP_TCP | CSUM_INNER_IP); | |||||
th->th_sum = in_pseudo(ip->ip_src.s_addr, | |||||
ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | |||||
pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | |||||
if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { | |||||
ip->ip_sum = 0; | |||||
ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); | |||||
} | |||||
} | |||||
} | |||||
if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_INNER_IP)) { | |||||
ip->ip_sum = 0; | |||||
outer_ip->ip_sum = 0; | |||||
} | |||||
break; | |||||
} | |||||
#endif | |||||
#ifdef INET6 | |||||
case ETHERTYPE_IPV6: | |||||
{ | |||||
struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen); | |||||
struct tcphdr *th; | |||||
pi->ipi_ip_hlen = sizeof(struct ip6_hdr); | |||||
if (__predict_false(m->m_len < ehdrlen + sizeof(struct ip6_hdr))) { | |||||
txq->ift_pullups++; | |||||
if (__predict_false((m = m_pullup(m, ehdrlen + sizeof(struct ip6_hdr))) == NULL)) | |||||
return (ENOMEM); | |||||
} | |||||
th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); | |||||
/* XXX-BZ this will go badly in case of ext hdrs. */ | |||||
pi->ipi_ipproto = ip6->ip6_nxt; | |||||
pi->ipi_flags |= IPI_TX_IPV6; | |||||
/* TCP checksum offload may require TCP header length */ | |||||
if (IS_TX_INNER_OFFLOAD6(pi)) { | |||||
if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) | |||||
return (ENXIO); | |||||
if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { | |||||
txq->ift_pullups++; | |||||
if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL)) | |||||
return (ENOMEM); | |||||
} | |||||
pi->ipi_tcp_hflags = th->th_flags; | |||||
pi->ipi_tcp_hlen = th->th_off << 2; | |||||
pi->ipi_tcp_seq = th->th_seq; | |||||
if (IS_INNER_TSO6(pi)) { | |||||
/* | |||||
* TSO always requires hardware checksum offload. | |||||
*/ | |||||
pi->ipi_csum_flags |= CSUM_INNER_IP6_TCP; | |||||
th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); | |||||
pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | |||||
} | |||||
} | |||||
break; | |||||
} | |||||
#endif | |||||
default: | |||||
pi->ipi_csum_flags &= ~CSUM_OFFLOAD; | |||||
pi->ipi_ip_hlen = 0; | |||||
break; | |||||
} | |||||
*mp = m; | |||||
return (0); | |||||
} | |||||
static int | |||||
iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | ||||
{ | { | ||||
if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | ||||
struct ether_vlan_header *eh; | struct ether_vlan_header *eh; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
m = *mp; | m = *mp; | ||||
if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) && | if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) && | ||||
Show All 28 Lines | iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | ||||
switch (pi->ipi_etype) { | switch (pi->ipi_etype) { | ||||
#ifdef INET | #ifdef INET | ||||
case ETHERTYPE_IP: | case ETHERTYPE_IP: | ||||
{ | { | ||||
struct mbuf *n; | struct mbuf *n; | ||||
struct ip *ip = NULL; | struct ip *ip = NULL; | ||||
struct tcphdr *th = NULL; | struct tcphdr *th = NULL; | ||||
int minthlen; | int minthlen = pi->ipi_ehdrlen + sizeof(*ip); | ||||
minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th)); | if ((m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN) == 0) | ||||
minthlen += sizeof(*th); | |||||
minthlen = min(m->m_pkthdr.len, minthlen); | |||||
if (__predict_false(m->m_len < minthlen)) { | if (__predict_false(m->m_len < minthlen)) { | ||||
/* | /* | ||||
* if this code bloat is causing too much of a hit | * if this code bloat is causing too much of a hit | ||||
* move it to a separate function and mark it noinline | * move it to a separate function and mark it noinline | ||||
*/ | */ | ||||
if (m->m_len == pi->ipi_ehdrlen) { | if (m->m_len == pi->ipi_ehdrlen) { | ||||
n = m->m_next; | n = m->m_next; | ||||
MPASS(n); | MPASS(n); | ||||
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
default: | default: | ||||
pi->ipi_csum_flags &= ~CSUM_OFFLOAD; | pi->ipi_csum_flags &= ~CSUM_OFFLOAD; | ||||
pi->ipi_ip_hlen = 0; | pi->ipi_ip_hlen = 0; | ||||
break; | break; | ||||
} | } | ||||
*mp = m; | *mp = m; | ||||
if ((m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN) != 0) | |||||
return iflib_parse_inner_header(txq, pi, mp); | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* If dodgy hardware rejects the scatter gather chain we've handed it | * If dodgy hardware rejects the scatter gather chain we've handed it | ||||
* we'll need to remove the mbuf chain from ifsg_m[] before we can add the | * we'll need to remove the mbuf chain from ifsg_m[] before we can add the | ||||
* m_defrag'd mbufs | * m_defrag'd mbufs | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 923 Lines • ▼ Show 20 Lines | iflib_if_qflush(if_t ifp) | ||||
* ALTQ queue(s). | * ALTQ queue(s). | ||||
*/ | */ | ||||
if_qflush(ifp); | if_qflush(ifp); | ||||
} | } | ||||
#define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \ | #define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \ | ||||
IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \ | IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \ | ||||
IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | \ | IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | \ | ||||
IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM | IFCAP_MEXTPG) | IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM | IFCAP_MEXTPG | \ | ||||
IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO) | |||||
static int | static int | ||||
iflib_if_ioctl(if_t ifp, u_long command, caddr_t data) | iflib_if_ioctl(if_t ifp, u_long command, caddr_t data) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
struct ifreq *ifr = (struct ifreq *)data; | struct ifreq *ifr = (struct ifreq *)data; | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
struct ifaddr *ifa = (struct ifaddr *)data; | struct ifaddr *ifa = (struct ifaddr *)data; | ||||
▲ Show 20 Lines • Show All 226 Lines • ▼ Show 20 Lines | iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) | ||||
IFDI_VLAN_UNREGISTER(ctx, vtag); | IFDI_VLAN_UNREGISTER(ctx, vtag); | ||||
/* Re-init to load the changes, if required */ | /* Re-init to load the changes, if required */ | ||||
if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) | if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
static void | static void | ||||
iflib_vxlan_register(void *arg, if_t ifp, sa_family_t family, uint16_t port) | |||||
{ | |||||
if_ctx_t ctx = arg; | |||||
MPASS(family == AF_INET || family == AF_INET6); | |||||
if (iflib_in_detach(ctx)) | |||||
return; | |||||
/* Check if interface has VXLAN offloads enabled */ | |||||
if (!(if_getcapenable(ctx->ifc_ifp) & | |||||
(IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO))) | |||||
return; | |||||
CTX_LOCK(ctx); | |||||
/* Driver may need to stop traffic before enabling VXLAN offload */ | |||||
if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VXLAN_CONFIG)) | |||||
iflib_stop(ctx); | |||||
IFDI_VXLAN_REGISTER(ctx, port); | |||||
/* Re-init to load the changes, if required */ | |||||
if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VXLAN_CONFIG)) | |||||
iflib_init_locked(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_vxlan_unregister(void *arg, if_t ifp, sa_family_t family, uint16_t port) | |||||
{ | |||||
if_ctx_t ctx = arg; | |||||
MPASS(family == AF_INET || family == AF_INET6); | |||||
CTX_LOCK(ctx); | |||||
/* Driver may need all tagged packets to be flushed */ | |||||
if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VXLAN_CONFIG)) | |||||
iflib_stop(ctx); | |||||
IFDI_VXLAN_UNREGISTER(ctx, port); | |||||
/* Re-init to load the changes, if required */ | |||||
if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VXLAN_CONFIG)) | |||||
iflib_init_locked(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_led_func(void *arg, int onoff) | iflib_led_func(void *arg, int onoff) | ||||
{ | { | ||||
if_ctx_t ctx = arg; | if_ctx_t ctx = arg; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_LED_FUNC(ctx, onoff); | IFDI_LED_FUNC(ctx, onoff); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 926 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
iflib_pseudo_deregister(if_ctx_t ctx) | iflib_pseudo_deregister(if_ctx_t ctx) | ||||
{ | { | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
/* Unregister VLAN event handlers early */ | /* Unregister VLAN event handlers early */ | ||||
iflib_unregister_vlan_handlers(ctx); | iflib_unregister_event_handlers(ctx); | ||||
if ((sctx->isc_flags & IFLIB_PSEUDO) && | if ((sctx->isc_flags & IFLIB_PSEUDO) && | ||||
(sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0) { | (sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0) { | ||||
bpfdetach(ifp); | bpfdetach(ifp); | ||||
if_detach(ifp); | if_detach(ifp); | ||||
} else { | } else { | ||||
ether_ifdetach(ifp); | ether_ifdetach(ifp); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | if (!CTX_IS_VF(ctx) && pci_iov_detach(dev) != 0) { | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
#endif | #endif | ||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
ctx->ifc_flags |= IFC_IN_DETACH; | ctx->ifc_flags |= IFC_IN_DETACH; | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
/* Unregister VLAN handlers before calling iflib_stop() */ | /* Unregister VLAN and VXLAN handlers before calling iflib_stop() */ | ||||
iflib_unregister_vlan_handlers(ctx); | iflib_unregister_event_handlers(ctx); | ||||
iflib_netmap_detach(ifp); | iflib_netmap_detach(ifp); | ||||
ether_ifdetach(ifp); | ether_ifdetach(ifp); | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
iflib_stop(ctx); | iflib_stop(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
▲ Show 20 Lines • Show All 299 Lines • ▼ Show 20 Lines | else | ||||
iflags |= IFF_BROADCAST | IFF_SIMPLEX; | iflags |= IFF_BROADCAST | IFF_SIMPLEX; | ||||
if_setflags(ifp, iflags); | if_setflags(ifp, iflags); | ||||
ctx->ifc_vlan_attach_event = | ctx->ifc_vlan_attach_event = | ||||
EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, | EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, | ||||
EVENTHANDLER_PRI_FIRST); | EVENTHANDLER_PRI_FIRST); | ||||
ctx->ifc_vlan_detach_event = | ctx->ifc_vlan_detach_event = | ||||
EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx, | EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx, | ||||
EVENTHANDLER_PRI_FIRST); | EVENTHANDLER_PRI_FIRST); | ||||
ctx->ifc_vxlan_attach_event = | |||||
EVENTHANDLER_REGISTER(vxlan_start, iflib_vxlan_register, ctx, | |||||
EVENTHANDLER_PRI_FIRST); | |||||
ctx->ifc_vxlan_detach_event = | |||||
EVENTHANDLER_REGISTER(vxlan_stop, iflib_vxlan_unregister, ctx, | |||||
EVENTHANDLER_PRI_FIRST); | |||||
if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { | if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { | ||||
ctx->ifc_mediap = &ctx->ifc_media; | ctx->ifc_mediap = &ctx->ifc_media; | ||||
ifmedia_init(ctx->ifc_mediap, IFM_IMASK, | ifmedia_init(ctx->ifc_mediap, IFM_IMASK, | ||||
iflib_media_change, iflib_media_status); | iflib_media_change, iflib_media_status); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
iflib_unregister_vlan_handlers(if_ctx_t ctx) | iflib_unregister_event_handlers(if_ctx_t ctx) | ||||
{ | { | ||||
/* Unregister VLAN events */ | /* Unregister VLAN events */ | ||||
if (ctx->ifc_vlan_attach_event != NULL) { | if (ctx->ifc_vlan_attach_event != NULL) { | ||||
EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | ||||
ctx->ifc_vlan_attach_event = NULL; | ctx->ifc_vlan_attach_event = NULL; | ||||
} | } | ||||
if (ctx->ifc_vlan_detach_event != NULL) { | if (ctx->ifc_vlan_detach_event != NULL) { | ||||
EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | ||||
ctx->ifc_vlan_detach_event = NULL; | ctx->ifc_vlan_detach_event = NULL; | ||||
} | } | ||||
/* Unregister VxLAN events */ | |||||
if (ctx->ifc_vxlan_attach_event != NULL) { | |||||
EVENTHANDLER_DEREGISTER(vxlan_start, ctx->ifc_vxlan_attach_event); | |||||
ctx->ifc_vxlan_attach_event = NULL; | |||||
} | } | ||||
if (ctx->ifc_vxlan_detach_event != NULL) { | |||||
EVENTHANDLER_DEREGISTER(vxlan_stop, ctx->ifc_vxlan_detach_event); | |||||
ctx->ifc_vxlan_detach_event = NULL; | |||||
} | |||||
} | |||||
static void | static void | ||||
iflib_deregister(if_ctx_t ctx) | iflib_deregister(if_ctx_t ctx) | ||||
{ | { | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
/* Remove all media */ | /* Remove all media */ | ||||
ifmedia_removeall(&ctx->ifc_media); | ifmedia_removeall(&ctx->ifc_media); | ||||
/* Ensure that VLAN event handlers are unregistered */ | /* Ensure that VLAN and VXLAN event handlers are unregistered */ | ||||
iflib_unregister_vlan_handlers(ctx); | iflib_unregister_event_handlers(ctx); | ||||
/* Release kobject reference */ | /* Release kobject reference */ | ||||
kobj_delete((kobj_t) ctx, NULL); | kobj_delete((kobj_t) ctx, NULL); | ||||
/* Free the ifnet structure */ | /* Free the ifnet structure */ | ||||
if_free(ifp); | if_free(ifp); | ||||
STATE_LOCK_DESTROY(ctx); | STATE_LOCK_DESTROY(ctx); | ||||
▲ Show 20 Lines • Show All 1,284 Lines • Show Last 20 Lines |