Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 2,209 Lines • ▼ Show 20 Lines | for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { | ||||
} | } | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | ||||
if (iflib_fl_setup(fl)) { | if (iflib_fl_setup(fl)) { | ||||
device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n"); | device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n"); | ||||
goto done; | goto done; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
done: | done: | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
txq = ctx->ifc_txqs; | txq = ctx->ifc_txqs; | ||||
for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, | callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, | ||||
txq->ift_timer.c_cpu); | txq->ift_timer.c_cpu); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 530 Lines • ▼ Show 20 Lines | print_pkt(if_pkt_info_t pi) | ||||
printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n", | printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n", | ||||
pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); | pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag); | ||||
printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n", | printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n", | ||||
pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); | pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto); | ||||
} | } | ||||
#endif | #endif | ||||
#define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | ||||
#define IS_IP4_TCP(pi) ((pi)->ipi_csum_flags & CSUM_IP_TCP) | |||||
#define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | ||||
#define IS_IP6_TCP(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TCP) | |||||
static int | static int | ||||
iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | ||||
{ | { | ||||
if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | ||||
struct ether_vlan_header *eh; | struct ether_vlan_header *eh; | ||||
struct mbuf *m, *n; | struct mbuf *m, *n; | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | case ETHERTYPE_IP: | ||||
} | } | ||||
pi->ipi_ip_hlen = ip->ip_hl << 2; | pi->ipi_ip_hlen = ip->ip_hl << 2; | ||||
pi->ipi_ipproto = ip->ip_p; | pi->ipi_ipproto = ip->ip_p; | ||||
pi->ipi_flags |= IPI_TX_IPV4; | pi->ipi_flags |= IPI_TX_IPV4; | ||||
if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP)) | if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP)) | ||||
ip->ip_sum = 0; | ip->ip_sum = 0; | ||||
if (IS_TSO4(pi)) { | /* TCP checksum offload may require TCP header length */ | ||||
if (pi->ipi_ipproto == IPPROTO_TCP) { | if (IS_IP4_TCP(pi) || IS_TSO4(pi)) { | ||||
/* Can this just be assumed? It's possible non-TCP TSO could be supported */ | |||||
if (__predict_true(pi->ipi_ipproto == IPPROTO_TCP)) { | |||||
if (__predict_false(th == NULL)) { | if (__predict_false(th == NULL)) { | ||||
txq->ift_pullups++; | txq->ift_pullups++; | ||||
if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) | if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); | th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); | ||||
} | } | ||||
pi->ipi_tcp_hflags = th->th_flags; | pi->ipi_tcp_hflags = th->th_flags; | ||||
pi->ipi_tcp_hlen = th->th_off << 2; | pi->ipi_tcp_hlen = th->th_off << 2; | ||||
pi->ipi_tcp_seq = th->th_seq; | pi->ipi_tcp_seq = th->th_seq; | ||||
} | } | ||||
} | |||||
if (IS_TSO4(pi)) { | |||||
if (__predict_false(ip->ip_p != IPPROTO_TCP)) | if (__predict_false(ip->ip_p != IPPROTO_TCP)) | ||||
return (ENXIO); | return (ENXIO); | ||||
th->th_sum = in_pseudo(ip->ip_src.s_addr, | th->th_sum = in_pseudo(ip->ip_src.s_addr, | ||||
ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | ||||
pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | ||||
if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { | if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { | ||||
ip->ip_sum = 0; | ip->ip_sum = 0; | ||||
ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); | ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); | ||||
Show All 14 Lines | if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); | th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); | ||||
/* XXX-BZ this will go badly in case of ext hdrs. */ | /* XXX-BZ this will go badly in case of ext hdrs. */ | ||||
pi->ipi_ipproto = ip6->ip6_nxt; | pi->ipi_ipproto = ip6->ip6_nxt; | ||||
pi->ipi_flags |= IPI_TX_IPV6; | pi->ipi_flags |= IPI_TX_IPV6; | ||||
if (IS_TSO6(pi)) { | /* TCP checksum offload may require TCP header length */ | ||||
if (IS_IP6_TCP(pi) || IS_TSO6(pi)) { | |||||
if (pi->ipi_ipproto == IPPROTO_TCP) { | if (pi->ipi_ipproto == IPPROTO_TCP) { | ||||
if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { | if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { | ||||
if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL)) | if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL)) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
txq->ift_pullups++; | |||||
} | } | ||||
pi->ipi_tcp_hflags = th->th_flags; | pi->ipi_tcp_hflags = th->th_flags; | ||||
pi->ipi_tcp_hlen = th->th_off << 2; | pi->ipi_tcp_hlen = th->th_off << 2; | ||||
pi->ipi_tcp_seq = th->th_seq; | |||||
} | } | ||||
} | |||||
if (IS_TSO6(pi)) { | |||||
if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) | if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) | ||||
return (ENXIO); | return (ENXIO); | ||||
/* | /* | ||||
* The corresponding flag is set by the stack in the IPv4 | * The corresponding flag is set by the stack in the IPv4 | ||||
* TSO case, but not in IPv6 (at least in FreeBSD 10.2). | * TSO case, but not in IPv6 (at least in FreeBSD 10.2). | ||||
* So, set it here because the rest of the flow requires it. | * So, set it here because the rest of the flow requires it. | ||||
*/ | */ | ||||
pi->ipi_csum_flags |= CSUM_TCP_IPV6; | pi->ipi_csum_flags |= CSUM_IP6_TSO; | ||||
th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); | th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); | ||||
pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
#endif | #endif | ||||
default: | default: | ||||
pi->ipi_csum_flags &= ~CSUM_OFFLOAD; | pi->ipi_csum_flags &= ~CSUM_OFFLOAD; | ||||
▲ Show 20 Lines • Show All 1,003 Lines • ▼ Show 20 Lines | if (ifa->ifa_addr->sa_family == AF_INET6) | ||||
avoid_reset = TRUE; | avoid_reset = TRUE; | ||||
#endif | #endif | ||||
/* | /* | ||||
** Calling init results in link renegotiation, | ** Calling init results in link renegotiation, | ||||
** so we avoid doing it when possible. | ** so we avoid doing it when possible. | ||||
*/ | */ | ||||
if (avoid_reset) { | if (avoid_reset) { | ||||
if_setflagbits(ifp, IFF_UP,0); | if_setflagbits(ifp, IFF_UP,0); | ||||
if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) | ||||
reinit = 1; | reinit = 1; | ||||
#ifdef INET | #ifdef INET | ||||
if (!(if_getflags(ifp) & IFF_NOARP)) | if (!(if_getflags(ifp) & IFF_NOARP)) | ||||
arp_ifinit(ifp, ifa); | arp_ifinit(ifp, ifa); | ||||
#endif | #endif | ||||
} else | } else | ||||
err = ether_ioctl(ifp, command, data); | err = ether_ioctl(ifp, command, data); | ||||
break; | break; | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | case SIOCSIFCAP: | ||||
mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); | mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); | ||||
setmask = 0; | setmask = 0; | ||||
#ifdef TCP_OFFLOAD | #ifdef TCP_OFFLOAD | ||||
setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6); | setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6); | ||||
#endif | #endif | ||||
setmask |= (mask & IFCAP_FLAGS); | setmask |= (mask & IFCAP_FLAGS); | ||||
if (setmask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) | if (setmask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) | ||||
setmask |= (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); | setmask |= (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); | ||||
if ((mask & IFCAP_WOL) && | if ((mask & IFCAP_WOL) && | ||||
(if_getcapabilities(ifp) & IFCAP_WOL) != 0) | (if_getcapabilities(ifp) & IFCAP_WOL) != 0) | ||||
setmask |= (mask & (IFCAP_WOL_MCAST|IFCAP_WOL_MAGIC)); | setmask |= (mask & (IFCAP_WOL_MCAST|IFCAP_WOL_MAGIC)); | ||||
if_vlancap(ifp); | if_vlancap(ifp); | ||||
/* | /* | ||||
* want to ensure that traffic has stopped before we change any of the flags | * want to ensure that traffic has stopped before we change any of the flags | ||||
*/ | */ | ||||
if (setmask) { | if (setmask) { | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
bits = if_getdrvflags(ifp); | bits = if_getdrvflags(ifp); | ||||
if (bits & IFF_DRV_RUNNING) | if (bits & IFF_DRV_RUNNING) | ||||
iflib_stop(ctx); | iflib_stop(ctx); | ||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
if_togglecapenable(ifp, setmask); | if_togglecapenable(ifp, setmask); | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
if (bits & IFF_DRV_RUNNING) | if (bits & IFF_DRV_RUNNING) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
if_setdrvflags(ifp, bits); | if_setdrvflags(ifp, bits); | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
case SIOCGPRIVATE_0: | case SIOCGPRIVATE_0: | ||||
case SIOCSDRVSPEC: | case SIOCSDRVSPEC: | ||||
case SIOCGDRVSPEC: | case SIOCGDRVSPEC: | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
err = IFDI_PRIV_IOCTL(ctx, command, data); | err = IFDI_PRIV_IOCTL(ctx, command, data); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
break; | break; | ||||
default: | default: | ||||
▲ Show 20 Lines • Show All 700 Lines • ▼ Show 20 Lines | iflib_queues_alloc(if_ctx_t ctx) | ||||
KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1")); | KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1")); | ||||
KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1")); | KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1")); | ||||
brscp = NULL; | brscp = NULL; | ||||
txq = NULL; | txq = NULL; | ||||
rxq = NULL; | rxq = NULL; | ||||
/* Allocate the TX ring struct memory */ | /* Allocate the TX ring struct memory */ | ||||
if (!(txq = | if (!(txq = | ||||
(iflib_txq_t) malloc(sizeof(struct iflib_txq) * | (iflib_txq_t) malloc(sizeof(struct iflib_txq) * | ||||
ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) { | ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate TX ring memory\n"); | device_printf(dev, "Unable to allocate TX ring memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 500 Lines • ▼ Show 20 Lines | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
} else { | } else { | ||||
taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, char *name) | iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
void *q; | void *q; | ||||
int irq_num = -1; | int irq_num = -1; | ||||
int err; | int err; | ||||
▲ Show 20 Lines • Show All 695 Lines • Show Last 20 Lines |