Changeset View
Changeset View
Standalone View
Standalone View
sys/net/if_epair.c
Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | #define DPRINTF(fmt, arg...) \ | ||||
if (epair_debug) \ | if (epair_debug) \ | ||||
printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) | printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) | ||||
#else | #else | ||||
#define DPRINTF(fmt, arg...) | #define DPRINTF(fmt, arg...) | ||||
#endif | #endif | ||||
static void epair_nh_sintr(struct mbuf *); | static void epair_nh_sintr(struct mbuf *); | ||||
static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); | static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); | ||||
#ifdef EPAIR_DRAIN | |||||
static void epair_nh_drainedcpu(u_int); | static void epair_nh_drainedcpu(u_int); | ||||
#endif | |||||
static void epair_start_locked(struct ifnet *); | static void epair_start_locked(struct ifnet *); | ||||
static int epair_media_change(struct ifnet *); | static int epair_media_change(struct ifnet *); | ||||
static void epair_media_status(struct ifnet *, struct ifmediareq *); | static void epair_media_status(struct ifnet *, struct ifmediareq *); | ||||
static int epair_clone_match(struct if_clone *, const char *); | static int epair_clone_match(struct if_clone *, const char *); | ||||
static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); | static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); | ||||
static int epair_clone_destroy(struct if_clone *, struct ifnet *); | static int epair_clone_destroy(struct if_clone *, struct ifnet *); | ||||
static const char epairname[] = "epair"; | static const char epairname[] = "epair"; | ||||
static unsigned int next_index = 0; | static unsigned int next_index = 0; | ||||
/* Netisr related definitions and sysctl. */ | /* Netisr related definitions and sysctl. */ | ||||
static struct netisr_handler epair_nh = { | static struct netisr_handler epair_nh = { | ||||
.nh_name = epairname, | .nh_name = epairname, | ||||
.nh_proto = NETISR_EPAIR, | .nh_proto = NETISR_EPAIR, | ||||
.nh_policy = NETISR_POLICY_CPU, | .nh_policy = NETISR_POLICY_CPU, | ||||
.nh_handler = epair_nh_sintr, | .nh_handler = epair_nh_sintr, | ||||
.nh_m2cpuid = epair_nh_m2cpuid, | .nh_m2cpuid = epair_nh_m2cpuid, | ||||
#ifdef EPAIR_DRAIN | |||||
.nh_drainedcpu = epair_nh_drainedcpu, | .nh_drainedcpu = epair_nh_drainedcpu, | ||||
#endif | |||||
}; | }; | ||||
static int | static int | ||||
sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) | sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
int error, qlimit; | int error, qlimit; | ||||
netisr_getqlimit(&epair_nh, &qlimit); | netisr_getqlimit(&epair_nh, &qlimit); | ||||
Show All 14 Lines | struct epair_softc { | ||||
struct ifnet *oifp; /* other ifp of pair. */ | struct ifnet *oifp; /* other ifp of pair. */ | ||||
struct ifmedia media; /* Media config (fake). */ | struct ifmedia media; /* Media config (fake). */ | ||||
u_int refcount; /* # of mbufs in flight. */ | u_int refcount; /* # of mbufs in flight. */ | ||||
u_int cpuid; /* CPU ID assigned upon creation. */ | u_int cpuid; /* CPU ID assigned upon creation. */ | ||||
void (*if_qflush)(struct ifnet *); | void (*if_qflush)(struct ifnet *); | ||||
/* Original if_qflush routine. */ | /* Original if_qflush routine. */ | ||||
}; | }; | ||||
#ifdef EPAIR_DRAIN | |||||
/* | /* | ||||
* Per-CPU list of ifps with data in the ifq that needs to be flushed | * Per-CPU list of ifps with data in the ifq that needs to be flushed | ||||
* to the netisr ``hw'' queue before we allow any further direct queuing | * to the netisr ``hw'' queue before we allow any further direct queuing | ||||
* to the ``hw'' queue. | * to the ``hw'' queue. | ||||
*/ | */ | ||||
struct epair_ifp_drain { | struct epair_ifp_drain { | ||||
STAILQ_ENTRY(epair_ifp_drain) ifp_next; | STAILQ_ENTRY(epair_ifp_drain) ifp_next; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
}; | }; | ||||
STAILQ_HEAD(eid_list, epair_ifp_drain); | STAILQ_HEAD(eid_list, epair_ifp_drain); | ||||
#endif | |||||
#define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ | #define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ | ||||
"if_epair", NULL, MTX_DEF) | "if_epair", NULL, MTX_DEF) | ||||
#define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) | #define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) | ||||
#define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ | #define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ | ||||
MA_OWNED) | MA_OWNED) | ||||
#define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) | #define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) | ||||
#define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) | #define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) | ||||
Show All 17 Lines | |||||
#define V_epair_cloner VNET(epair_cloner) | #define V_epair_cloner VNET(epair_cloner) | ||||
/* | /* | ||||
* DPCPU area and functions. | * DPCPU area and functions. | ||||
*/ | */ | ||||
struct epair_dpcpu { | struct epair_dpcpu { | ||||
struct mtx if_epair_mtx; /* Per-CPU locking. */ | struct mtx if_epair_mtx; /* Per-CPU locking. */ | ||||
int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ | int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ | ||||
#ifdef EPAIR_DRAIN | |||||
struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with | struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with | ||||
* data in the ifq. */ | * data in the ifq. */ | ||||
#endif | |||||
}; | }; | ||||
DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); | DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); | ||||
static void | static void | ||||
epair_dpcpu_init(void) | epair_dpcpu_init(void) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
#ifdef EPAIR_DRAIN | |||||
struct eid_list *s; | struct eid_list *s; | ||||
#endif | |||||
u_int cpuid; | u_int cpuid; | ||||
CPU_FOREACH(cpuid) { | CPU_FOREACH(cpuid) { | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | ||||
/* Initialize per-cpu lock. */ | /* Initialize per-cpu lock. */ | ||||
EPAIR_LOCK_INIT(epair_dpcpu); | EPAIR_LOCK_INIT(epair_dpcpu); | ||||
/* Driver flags are per-cpu as are our netisr "hw" queues. */ | /* Driver flags are per-cpu as are our netisr "hw" queues. */ | ||||
epair_dpcpu->epair_drv_flags = 0; | epair_dpcpu->epair_drv_flags = 0; | ||||
#ifdef EPAIR_DRAIN | |||||
/* | /* | ||||
* Initialize per-cpu drain list. | * Initialize per-cpu drain list. | ||||
* Manually do what STAILQ_HEAD_INITIALIZER would do. | * Manually do what STAILQ_HEAD_INITIALIZER would do. | ||||
*/ | */ | ||||
s = &epair_dpcpu->epair_ifp_drain_list; | s = &epair_dpcpu->epair_ifp_drain_list; | ||||
s->stqh_first = NULL; | s->stqh_first = NULL; | ||||
s->stqh_last = &s->stqh_first; | s->stqh_last = &s->stqh_first; | ||||
#endif | |||||
} | } | ||||
} | } | ||||
static void | static void | ||||
epair_dpcpu_detach(void) | epair_dpcpu_detach(void) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
u_int cpuid; | u_int cpuid; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) | epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) | ||||
{ | { | ||||
*cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); | *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); | ||||
return (m); | return (m); | ||||
} | } | ||||
#ifdef EPAIR_DRAIN | |||||
static void | static void | ||||
epair_nh_drainedcpu(u_int cpuid) | epair_nh_drainedcpu(u_int cpuid) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
struct epair_ifp_drain *elm, *tvar; | struct epair_ifp_drain *elm, *tvar; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | ||||
Show All 32 Lines | if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { | ||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | ||||
DPRINTF("hw queue length overflow at %u\n", | DPRINTF("hw queue length overflow at %u\n", | ||||
epair_nh.nh_qlimit); | epair_nh.nh_qlimit); | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
EPAIR_UNLOCK(epair_dpcpu); | EPAIR_UNLOCK(epair_dpcpu); | ||||
} | } | ||||
#endif | |||||
/* | /* | ||||
* Network interface (`if') related functions. | * Network interface (`if') related functions. | ||||
*/ | */ | ||||
#ifdef EPAIR_DRAIN | |||||
static void | static void | ||||
epair_remove_ifp_from_draining(struct ifnet *ifp) | epair_remove_ifp_from_draining(struct ifnet *ifp) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
struct epair_ifp_drain *elm, *tvar; | struct epair_ifp_drain *elm, *tvar; | ||||
u_int cpuid; | u_int cpuid; | ||||
CPU_FOREACH(cpuid) { | CPU_FOREACH(cpuid) { | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | epair_add_ifp_for_draining(struct ifnet *ifp) | ||||
elm->ifp = ifp; | elm->ifp = ifp; | ||||
/* Add a reference for the ifp pointer on the list. */ | /* Add a reference for the ifp pointer on the list. */ | ||||
EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | ||||
STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); | STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); | ||||
return (0); | return (0); | ||||
} | } | ||||
#endif | |||||
static void | static void | ||||
epair_start_locked(struct ifnet *ifp) | epair_start_locked(struct ifnet *ifp) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
struct epair_softc *sc; | struct epair_softc *sc; | ||||
struct ifnet *oifp; | struct ifnet *oifp; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | for (;;) { | ||||
error = netisr_queue(NETISR_EPAIR, m); | error = netisr_queue(NETISR_EPAIR, m); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
if (!error) { | if (!error) { | ||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | ||||
/* Someone else received the packet. */ | /* Someone else received the packet. */ | ||||
if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | ||||
} else { | } else { | ||||
/* The packet was freed already. */ | /* The packet was freed already. */ | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
#ifdef EPAIR_DRAIN | |||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | ||||
ifp->if_drv_flags |= IFF_DRV_OACTIVE; | ifp->if_drv_flags |= IFF_DRV_OACTIVE; | ||||
(void) epair_add_ifp_for_draining(ifp); | (void) epair_add_ifp_for_draining(ifp); | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | EPAIR_REFCOUNT_RELEASE(&sc->refcount); | ||||
kp: Should we be releasing references? | |||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | ||||
("%s: ifp=%p sc->refcount not >= 1: %d", | ("%s: ifp=%p sc->refcount not >= 1: %d", | ||||
__func__, oifp, sc->refcount)); | __func__, oifp, sc->refcount)); | ||||
#endif | |||||
} | } | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
epair_start(struct ifnet *ifp) | epair_start(struct ifnet *ifp) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct epair_dpcpu *epair_dpcpu; | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | if (ALTQ_IS_ENABLED(&ifp->if_snd)) { | ||||
if (error) | if (error) | ||||
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); | if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); | ||||
IF_UNLOCK(&ifp->if_snd); | IF_UNLOCK(&ifp->if_snd); | ||||
if (!error) { | if (!error) { | ||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | ||||
if (mflags & (M_BCAST|M_MCAST)) | if (mflags & (M_BCAST|M_MCAST)) | ||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | ||||
#ifdef EPAIR_DRAIN | |||||
if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) | if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) | ||||
#endif | |||||
epair_start_locked(ifp); | epair_start_locked(ifp); | ||||
#ifdef EPAIR_DRAIN | |||||
else | else | ||||
(void)epair_add_ifp_for_draining(ifp); | (void)epair_add_ifp_for_draining(ifp); | ||||
#endif | |||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
IF_UNLOCK(&ifp->if_snd); | IF_UNLOCK(&ifp->if_snd); | ||||
#endif | #endif | ||||
#ifdef EPAIR_DRAIN | |||||
if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { | if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { | ||||
/* | /* | ||||
* Our hardware queue is full, try to fall back | * Our hardware queue is full, try to fall back | ||||
* queuing to the ifq but do not call ifp->if_start. | * queuing to the ifq but do not call ifp->if_start. | ||||
* Either we are lucky or the packet is gone. | * Either we are lucky or the packet is gone. | ||||
*/ | */ | ||||
IFQ_ENQUEUE(&ifp->if_snd, m, error); | IFQ_ENQUEUE(&ifp->if_snd, m, error); | ||||
if (!error) | if (!error) | ||||
(void)epair_add_ifp_for_draining(ifp); | (void)epair_add_ifp_for_draining(ifp); | ||||
return (error); | return (error); | ||||
} | } | ||||
#endif | |||||
sc = oifp->if_softc; | sc = oifp->if_softc; | ||||
/* | /* | ||||
* Add a reference so the interface cannot go while the | * Add a reference so the interface cannot go while the | ||||
* packet is in transit as we rely on rcvif to stay valid. | * packet is in transit as we rely on rcvif to stay valid. | ||||
*/ | */ | ||||
EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | ||||
m->m_pkthdr.rcvif = oifp; | m->m_pkthdr.rcvif = oifp; | ||||
CURVNET_SET_QUIET(oifp->if_vnet); | CURVNET_SET_QUIET(oifp->if_vnet); | ||||
error = netisr_queue(NETISR_EPAIR, m); | error = netisr_queue(NETISR_EPAIR, m); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
if (!error) { | if (!error) { | ||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | ||||
/* | /* | ||||
* IFQ_HANDOFF_ADJ/ip_handoff() update statistics, | * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, | ||||
* but as we bypass all this we have to duplicate | * but as we bypass all this we have to duplicate | ||||
* the logic another time. | * the logic another time. | ||||
*/ | */ | ||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | ||||
if (mflags & (M_BCAST|M_MCAST)) | if (mflags & (M_BCAST|M_MCAST)) | ||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | ||||
/* Someone else received the packet. */ | /* Someone else received the packet. */ | ||||
if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | ||||
} else { | } else { | ||||
/* The packet was freed already. */ | /* The packet was freed already. */ | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
#ifdef EPAIR_DRAIN | |||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | ||||
ifp->if_drv_flags |= IFF_DRV_OACTIVE; | ifp->if_drv_flags |= IFF_DRV_OACTIVE; | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | EPAIR_REFCOUNT_RELEASE(&sc->refcount); | ||||
kpUnsubmitted Not Done Inline ActionsShould we not be releasing references here as well? kp: Should we not be releasing references here as well? | |||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | ||||
("%s: ifp=%p sc->refcount not >= 1: %d", | ("%s: ifp=%p sc->refcount not >= 1: %d", | ||||
__func__, oifp, sc->refcount)); | __func__, oifp, sc->refcount)); | ||||
#endif | |||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
epair_transmit(struct ifnet *ifp, struct mbuf *m) | epair_transmit(struct ifnet *ifp, struct mbuf *m) | ||||
{ | { | ||||
Show All 10 Lines | |||||
static void | static void | ||||
epair_qflush(struct ifnet *ifp) | epair_qflush(struct ifnet *ifp) | ||||
{ | { | ||||
struct epair_softc *sc; | struct epair_softc *sc; | ||||
sc = ifp->if_softc; | sc = ifp->if_softc; | ||||
KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", | KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", | ||||
__func__, ifp, sc)); | __func__, ifp, sc)); | ||||
#ifdef EPAIR_DRAIN | |||||
/* | /* | ||||
* Remove this ifp from all backpointer lists. The interface will not | * Remove this ifp from all backpointer lists. The interface will not | ||||
* usable for flushing anyway nor should it have anything to flush | * usable for flushing anyway nor should it have anything to flush | ||||
* after if_qflush(). | * after if_qflush(). | ||||
*/ | */ | ||||
epair_remove_ifp_from_draining(ifp); | epair_remove_ifp_from_draining(ifp); | ||||
#endif | |||||
if (sc->if_qflush) | if (sc->if_qflush) | ||||
sc->if_qflush(ifp); | sc->if_qflush(ifp); | ||||
} | } | ||||
static int | static int | ||||
epair_media_change(struct ifnet *ifp __unused) | epair_media_change(struct ifnet *ifp __unused) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 426 Lines • Show Last 20 Lines |
Should we be releasing references?