Changeset View
Standalone View
sys/net/if_epair.c
/*- | /*- | ||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | * SPDX-License-Identifier: BSD-2-Clause-FreeBSD | ||||
* | * | ||||
* Copyright (c) 2008 The FreeBSD Foundation | * Copyright (c) 2008 The FreeBSD Foundation | ||||
* Copyright (c) 2009-2010 Bjoern A. Zeeb <bz@FreeBSD.org> | |||||
* All rights reserved. | * All rights reserved. | ||||
imp: I think the foundation has a blanket 'remove all rights reserved' policy. Maybe you can ask? | |||||
rgrimesUnsubmitted Done Inline ActionsMy understanding is the foundation has removed the "All rights reserved" stanza from they contracts with indivisuals, and the project has removed the stanza from the templates. Further it was my undersanding that at "some point" Ed Maste said they, the foundation, would look into doing a tree sweep to clean up the Foundations copyrights. rgrimes: My understanding is the foundation has removed the "All rights reserved" stanza from they… | |||||
* Copyright (c) 2009-2010,2020-2021 Bjoern A. Zeeb <bz@FreeBSD.org> | |||||
impUnsubmitted Done Inline ActionsCopyright law is such that you can also do 2009-2021 as well. There's no requirement to list individual years. Only the oldest and youngest years are relevant. imp: Copyright law is such that you can also do 2009-2021 as well. There's no requirement to list… | |||||
bzUnsubmitted Done Inline ActionsWell, copyright law here doesn't require me to put the line down at all and the only year which is relevant are 70 years after my death. The world is round. bz: Well, copyright law here doesn't require me to put the line down at all and the only year which… | |||||
bzUnsubmitted Done Inline ActionsSorry, that wasn't polite of me to answer that way. bz: Sorry, that wasn't polite of me to answer that way. | |||||
impUnsubmitted Done Inline ActionsI was just saying that a date range is legally acceptable and easier to do if you wanted to. The final choice is yours, of course, and I'm sorry that I failed to make that clear. imp: I was just saying that a date range is legally acceptable and easier to do if you wanted to. | |||||
rgrimesUnsubmitted Done Inline ActionsThough Warner and myself stronly disagree on which is correct on how to specify years, I can assure you the law itself does not state " Form of Notice.—If a notice appears on the copies, it shall consist of the following three elements: (1) the symbol © (the letter C in a circle), or the word “Copyright”, or the abbreviation “Copr.”; and (2) the year of first publication of the work; in the case of compilations or derivative works incorporating previously published material, the year date of first publication of the compilation or derivative work is sufficient. The year date may be omitted where a pictorial, graphic, or sculptural work, with accompanying text matter, if any, is reproduced in or on greeting cards, postcards, stationery, jewelry, dolls, toys, or any useful articles; and (3) the name of the owner of copyright in the work, or an abbreviation by which the name can be recognized, or a generally known alternative designation of the owner. rgrimes: Though Warner and myself stronly disagree on which is correct on how to specify years, I can… | |||||
impUnsubmitted Done Inline ActionsAgreed. Legal opinion does vary, and often is influenced by international concerns. And rod is right: wording was imprecise. The range of years is the most common advice I've seen in this areas, though the copyright holder is free to do what they want. imp: Agreed. Legal opinion does vary, and often is influenced by international concerns. And rod is… | |||||
* | * | ||||
* This software was developed by CK Software GmbH under sponsorship | * This software was developed by CK Software GmbH under sponsorship | ||||
* from the FreeBSD Foundation. | * from the FreeBSD Foundation. | ||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
Show All 17 Lines | |||||
/* | /* | ||||
* A pair of virtual back-to-back connected ethernet like interfaces | * A pair of virtual back-to-back connected ethernet like interfaces | ||||
* (``two interfaces with a virtual cross-over cable''). | * (``two interfaces with a virtual cross-over cable''). | ||||
* | * | ||||
* This is mostly intended to be used to provide connectivity between | * This is mostly intended to be used to provide connectivity between | ||||
* different virtual network stack instances. | * different virtual network stack instances. | ||||
*/ | */ | ||||
/* | |||||
* Things to re-think once we have more experience: | |||||
* - ifp->if_reassign function once we can test with vimage. Depending on | |||||
* how if_vmove() is going to be improved. | |||||
* - Real random etheraddrs that are checked to be uniquish; we would need | |||||
* to re-do them in case we move the interface between network stacks | |||||
* in a private if_reassign function. | |||||
* In case we bridge to a real interface/network or between indepedent | |||||
* epairs on multiple stacks/machines, we may need this. | |||||
* For now let the user handle that case. | |||||
*/ | |||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/hash.h> | #include <sys/hash.h> | ||||
#include <sys/jail.h> | #include <sys/jail.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/libkern.h> | #include <sys/libkern.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mbuf.h> | #include <sys/mbuf.h> | ||||
#include <sys/module.h> | #include <sys/module.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/refcount.h> | |||||
#include <sys/queue.h> | #include <sys/queue.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/socket.h> | #include <sys/socket.h> | ||||
#include <sys/sockio.h> | #include <sys/sockio.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/types.h> | #include <sys/types.h> | ||||
#include <sys/bus.h> | |||||
#include <sys/interrupt.h> | |||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_clone.h> | #include <net/if_clone.h> | ||||
#include <net/if_media.h> | #include <net/if_media.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/netisr.h> | #include <net/netisr.h> | ||||
#include <net/vnet.h> | #include <net/vnet.h> | ||||
SYSCTL_DECL(_net_link); | |||||
static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | |||||
"epair sysctl"); | |||||
#ifdef EPAIR_DEBUG | |||||
static int epair_debug = 0; | |||||
SYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW, | |||||
&epair_debug, 0, "if_epair(4) debugging."); | |||||
#define DPRINTF(fmt, arg...) \ | |||||
if (epair_debug) \ | |||||
printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) | |||||
#else | |||||
#define DPRINTF(fmt, arg...) | |||||
#endif | |||||
static void epair_nh_sintr(struct mbuf *); | |||||
static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); | |||||
static void epair_nh_drainedcpu(u_int); | |||||
static void epair_start_locked(struct ifnet *); | |||||
static int epair_media_change(struct ifnet *); | |||||
static void epair_media_status(struct ifnet *, struct ifmediareq *); | |||||
static int epair_clone_match(struct if_clone *, const char *); | static int epair_clone_match(struct if_clone *, const char *); | ||||
static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); | static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); | ||||
static int epair_clone_destroy(struct if_clone *, struct ifnet *); | static int epair_clone_destroy(struct if_clone *, struct ifnet *); | ||||
static const char epairname[] = "epair"; | static const char epairname[] = "epair"; | ||||
static unsigned int next_index = 0; | static unsigned int maxqlen = 256; /* IFQ_MAXLEN is too low these days. */ | ||||
/* Netisr related definitions and sysctl. */ | SYSCTL_DECL(_net_link); | ||||
static struct netisr_handler epair_nh = { | static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | ||||
.nh_name = epairname, | "epair sysctl"); | ||||
.nh_proto = NETISR_EPAIR, | SYSCTL_UINT(_net_link_epair, OID_AUTO, maxqlen, | ||||
.nh_policy = NETISR_POLICY_CPU, | CTLFLAG_RW | CTLFLAG_MPSAFE, &maxqlen, 0, | ||||
.nh_handler = epair_nh_sintr, | "Maximum if_epair(4) \"hw\" queue length set at interface clone time."); | ||||
.nh_m2cpuid = epair_nh_m2cpuid, | |||||
.nh_drainedcpu = epair_nh_drainedcpu, | |||||
}; | |||||
static int | static MALLOC_DEFINE(M_EPAIR, epairname, | ||||
sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) | "Pair of virtual cross-over connected Ethernet-like interfaces"); | ||||
{ | |||||
int error, qlimit; | |||||
netisr_getqlimit(&epair_nh, &qlimit); | VNET_DEFINE_STATIC(struct if_clone *, epair_cloner); | ||||
error = sysctl_handle_int(oidp, &qlimit, 0, req); | #define V_epair_cloner VNET(epair_cloner) | ||||
if (error || !req->newptr) | |||||
return (error); | |||||
if (qlimit < 1) | |||||
return (EINVAL); | |||||
return (netisr_setqlimit(&epair_nh, qlimit)); | |||||
} | |||||
SYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen, | |||||
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, | |||||
sysctl_epair_netisr_maxqlen, "I", | |||||
"Maximum if_epair(4) netisr \"hw\" queue length"); | |||||
static unsigned int next_index = 0; | |||||
#define EPAIR_LOCK_INIT() mtx_init(&epair_n_index_mtx, "epairidx", \ | |||||
NULL, MTX_DEF) | |||||
#define EPAIR_LOCK_DESTROY() mtx_destroy(&epair_n_index_mtx) | |||||
#define EPAIR_LOCK() mtx_lock(&epair_n_index_mtx) | |||||
#define EPAIR_UNLOCK() mtx_unlock(&epair_n_index_mtx) | |||||
#define GLOBAL_SWI | |||||
#ifdef GLOBAL_SWI | |||||
static void *swi_cookie[MAXCPU]; /* swi(9). */ | |||||
static STAILQ_HEAD(, epair_softc) swi_sc[MAXCPU]; | |||||
#endif | |||||
static struct mtx epair_n_index_mtx; | |||||
struct epair_softc { | struct epair_softc { | ||||
struct ifnet *ifp; /* This ifp. */ | struct ifnet *ifp; /* This ifp. */ | ||||
struct ifnet *oifp; /* other ifp of pair. */ | struct ifnet *oifp; /* other ifp of pair. */ | ||||
void *swi_cookie; /* swi(9). */ | |||||
struct mbufq rxq; /* Receive queue. */ | |||||
struct mtx sc_mtx; /* Lock for this pair-half. */ | |||||
struct ifmedia media; /* Media config (fake). */ | struct ifmedia media; /* Media config (fake). */ | ||||
u_int refcount; /* # of mbufs in flight. */ | #ifdef GLOBAL_SWI | ||||
u_int cpuid; /* CPU ID assigned upon creation. */ | uint32_t cpuidx; | ||||
void (*if_qflush)(struct ifnet *); | STAILQ_ENTRY(epair_softc) entry; | ||||
/* Original if_qflush routine. */ | |||||
}; | |||||
/* | |||||
* Per-CPU list of ifps with data in the ifq that needs to be flushed | |||||
* to the netisr ``hw'' queue before we allow any further direct queuing | |||||
* to the ``hw'' queue. | |||||
*/ | |||||
struct epair_ifp_drain { | |||||
STAILQ_ENTRY(epair_ifp_drain) ifp_next; | |||||
struct ifnet *ifp; | |||||
}; | |||||
STAILQ_HEAD(eid_list, epair_ifp_drain); | |||||
#define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ | |||||
"if_epair", NULL, MTX_DEF) | |||||
#define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) | |||||
#define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ | |||||
MA_OWNED) | |||||
#define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) | |||||
#define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) | |||||
#ifdef INVARIANTS | |||||
#define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v)) | |||||
#define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r)) | |||||
#define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r)) | |||||
#define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p) | |||||
#else | |||||
#define EPAIR_REFCOUNT_INIT(r, v) | |||||
#define EPAIR_REFCOUNT_AQUIRE(r) | |||||
#define EPAIR_REFCOUNT_RELEASE(r) | |||||
#define EPAIR_REFCOUNT_ASSERT(a, p) | |||||
#endif | #endif | ||||
static MALLOC_DEFINE(M_EPAIR, epairname, | |||||
"Pair of virtual cross-over connected Ethernet-like interfaces"); | |||||
VNET_DEFINE_STATIC(struct if_clone *, epair_cloner); | |||||
#define V_epair_cloner VNET(epair_cloner) | |||||
/* | |||||
* DPCPU area and functions. | |||||
*/ | |||||
struct epair_dpcpu { | |||||
struct mtx if_epair_mtx; /* Per-CPU locking. */ | |||||
int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ | |||||
struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with | |||||
* data in the ifq. */ | |||||
}; | }; | ||||
DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); | |||||
#define EPAIR_SC_LOCK_INIT(_sc) mtx_init(&(_sc)->sc_mtx, "if_epair", \ | |||||
NULL, MTX_DEF) | |||||
#define EPAIR_SC_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) | |||||
#define EPAIR_SC_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) | |||||
#define EPAIR_SC_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) | |||||
#define EPAIR_SC_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) | |||||
static void | static void | ||||
epair_dpcpu_init(void) | epair_if_input(struct ifnet *ifp, struct mbufq *mq) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct mbuf *m; | ||||
struct eid_list *s; | |||||
u_int cpuid; | |||||
CPU_FOREACH(cpuid) { | CURVNET_SET_QUIET(ifp->if_vnet); | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | m = mbufq_dequeue(mq); | ||||
while (m != NULL) { | |||||
/* Initialize per-cpu lock. */ | (*ifp->if_input)(ifp, m); | ||||
EPAIR_LOCK_INIT(epair_dpcpu); | m = mbufq_dequeue(mq); | ||||
/* Driver flags are per-cpu as are our netisr "hw" queues. */ | |||||
epair_dpcpu->epair_drv_flags = 0; | |||||
/* | |||||
* Initialize per-cpu drain list. | |||||
* Manually do what STAILQ_HEAD_INITIALIZER would do. | |||||
*/ | |||||
s = &epair_dpcpu->epair_ifp_drain_list; | |||||
s->stqh_first = NULL; | |||||
s->stqh_last = &s->stqh_first; | |||||
} | } | ||||
CURVNET_RESTORE(); | |||||
} | } | ||||
static void | static void | ||||
epair_dpcpu_detach(void) | epair_sintr(struct epair_softc *sc) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct ifnet *ifp; | ||||
u_int cpuid; | struct mbufq mq; | ||||
CPU_FOREACH(cpuid) { | EPAIR_SC_LOCK(sc); | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | ifp = sc->ifp; | ||||
if_ref(ifp); | |||||
memcpy(&mq, &sc->rxq, sizeof(mq)); | |||||
/* This will also pickup changes from cysctl. */ | |||||
afedorovUnsubmitted Done Inline Actionscysctl -> sysctl afedorov: cysctl -> sysctl | |||||
mbufq_init(&sc->rxq, maxqlen); | |||||
EPAIR_SC_UNLOCK(sc); | |||||
/* Destroy per-cpu lock. */ | epair_if_input(ifp, &mq); | ||||
EPAIR_LOCK_DESTROY(epair_dpcpu); | |||||
} | |||||
} | |||||
/* | if_rele(ifp); | ||||
* Helper functions. | |||||
*/ | |||||
static u_int | |||||
cpuid_from_ifp(struct ifnet *ifp) | |||||
{ | |||||
struct epair_softc *sc; | |||||
if (ifp == NULL) | |||||
return (0); | |||||
sc = ifp->if_softc; | |||||
return (sc->cpuid); | |||||
} | } | ||||
/* | |||||
* Netisr handler functions. | |||||
*/ | |||||
static void | static void | ||||
epair_nh_sintr(struct mbuf *m) | epair_intr(void *arg) | ||||
{ | { | ||||
struct ifnet *ifp; | struct epair_softc *sc; | ||||
struct epair_softc *sc __unused; | #ifdef GLOBAL_SWI | ||||
uint32_t cpuidx; | |||||
ifp = m->m_pkthdr.rcvif; | cpuidx = (uintptr_t)arg; | ||||
(*ifp->if_input)(ifp, m); | EPAIR_LOCK(); | ||||
sc = ifp->if_softc; | STAILQ_FOREACH(sc, &swi_sc[cpuidx], entry) { | ||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | /* Do this lockless. */ | ||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | if (mbufq_len(&sc->rxq) == 0) | ||||
("%s: ifp=%p sc->refcount not >= 1: %d", | continue; | ||||
__func__, ifp, sc->refcount)); | epair_sintr(sc); | ||||
DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount); | |||||
} | } | ||||
EPAIR_UNLOCK(); | |||||
#else | |||||
sc = (struct epair_softc *)arg; | |||||
epair_sintr(sc); | |||||
#endif | |||||
static struct mbuf * | return; | ||||
epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) | |||||
{ | |||||
*cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); | |||||
return (m); | |||||
} | } | ||||
static void | static int | ||||
epair_nh_drainedcpu(u_int cpuid) | epair_menq(struct mbuf *m, struct epair_softc *osc) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | struct ifnet *ifp, *oifp; | ||||
struct epair_ifp_drain *elm, *tvar; | int error, len; | ||||
struct ifnet *ifp; | short mflags; | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | |||||
EPAIR_LOCK(epair_dpcpu); | |||||
/* | /* | ||||
* Assume our "hw" queue and possibly ifq will be emptied | * I know this looks weird. We pass the "other sc" as we need that one | ||||
* again. In case we will overflow the "hw" queue while | * and can get both ifps from it as well. | ||||
* draining, epair_start_locked will set IFF_DRV_OACTIVE | |||||
* again and we will stop and return. | |||||
*/ | */ | ||||
STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, | oifp = osc->ifp; | ||||
ifp_next, tvar) { | ifp = osc->oifp; | ||||
ifp = elm->ifp; | |||||
epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE; | |||||
ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; | |||||
epair_start_locked(ifp); | |||||
IFQ_LOCK(&ifp->if_snd); | /* Save values as once the mbuf is queued, it's not ours anymore. */ | ||||
if (IFQ_IS_EMPTY(&ifp->if_snd)) { | len = m->m_pkthdr.len; | ||||
struct epair_softc *sc __unused; | mflags = m->m_flags; | ||||
STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, | m->m_pkthdr.rcvif = oifp; | ||||
elm, epair_ifp_drain, ifp_next); | M_SETFIB(m, oifp->if_fib); | ||||
/* The cached ifp goes off the list. */ | EPAIR_SC_LOCK(osc); | ||||
sc = ifp->if_softc; | /* | ||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | * If the queue is full, free the oldest packet | ||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | * (think "DMA ring wraparound-overwrite"). | ||||
("%s: ifp=%p sc->refcount not >= 1: %d", | */ | ||||
__func__, ifp, sc->refcount)); | if (mbufq_full(&osc->rxq)) { | ||||
free(elm, M_EPAIR); | struct mbuf *m2; | ||||
} | |||||
IFQ_UNLOCK(&ifp->if_snd); | |||||
if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { | m2 = mbufq_dequeue(&osc->rxq); | ||||
/* Our "hw"q overflew again. */ | KASSERT(m2 != NULL, ("%s: m2 is NULL osc %p\n", __func__, osc)); | ||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | if_inc_counter(oifp, IFCOUNTER_IQDROPS, 1); | ||||
DPRINTF("hw queue length overflow at %u\n", | m_freem(m2); | ||||
epair_nh.nh_qlimit); | |||||
break; | |||||
} | } | ||||
} | error = mbufq_enqueue(&osc->rxq, m); | ||||
EPAIR_UNLOCK(epair_dpcpu); | EPAIR_SC_UNLOCK(osc); | ||||
} | if (error) { | ||||
m_freem(m); | |||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
} else { | |||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | |||||
/* | /* | ||||
* Network interface (`if') related functions. | * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, | ||||
* but as we bypass all this we have to duplicate | |||||
* the logic another time. | |||||
*/ | */ | ||||
static void | if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | ||||
epair_remove_ifp_from_draining(struct ifnet *ifp) | if (mflags & (M_BCAST|M_MCAST)) | ||||
{ | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | ||||
struct epair_dpcpu *epair_dpcpu; | /* Someone else received the packet. */ | ||||
struct epair_ifp_drain *elm, *tvar; | if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | ||||
u_int cpuid; | |||||
CPU_FOREACH(cpuid) { | /* Kick the interrupt handler. */ | ||||
epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); | if (osc->swi_cookie != NULL) | ||||
EPAIR_LOCK(epair_dpcpu); | swi_sched(osc->swi_cookie, 0); | ||||
STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, | |||||
ifp_next, tvar) { | |||||
if (ifp == elm->ifp) { | |||||
struct epair_softc *sc __unused; | |||||
STAILQ_REMOVE( | |||||
&epair_dpcpu->epair_ifp_drain_list, elm, | |||||
epair_ifp_drain, ifp_next); | |||||
/* The cached ifp goes off the list. */ | |||||
sc = ifp->if_softc; | |||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | |||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | |||||
("%s: ifp=%p sc->refcount not >= 1: %d", | |||||
__func__, ifp, sc->refcount)); | |||||
free(elm, M_EPAIR); | |||||
} | } | ||||
} | |||||
EPAIR_UNLOCK(epair_dpcpu); | |||||
} | |||||
} | |||||
static int | return (error); | ||||
Not Done Inline ActionsThis is wrong, and causes if_epair to not work. kp: This is wrong, and causes if_epair to not work.
It wants to only `swi_sched()` the first time… | |||||
epair_add_ifp_for_draining(struct ifnet *ifp) | |||||
{ | |||||
struct epair_dpcpu *epair_dpcpu; | |||||
struct epair_softc *sc; | |||||
struct epair_ifp_drain *elm = NULL; | |||||
sc = ifp->if_softc; | |||||
epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); | |||||
EPAIR_LOCK_ASSERT(epair_dpcpu); | |||||
STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next) | |||||
if (elm->ifp == ifp) | |||||
break; | |||||
/* If the ifp is there already, return success. */ | |||||
if (elm != NULL) | |||||
return (0); | |||||
elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO); | |||||
if (elm == NULL) | |||||
return (ENOMEM); | |||||
elm->ifp = ifp; | |||||
/* Add a reference for the ifp pointer on the list. */ | |||||
EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | |||||
STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); | |||||
return (0); | |||||
} | } | ||||
static void | static void | ||||
epair_start_locked(struct ifnet *ifp) | epair_start(struct ifnet *ifp) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | |||||
struct mbuf *m; | struct mbuf *m; | ||||
struct epair_softc *sc; | struct epair_softc *sc; | ||||
struct ifnet *oifp; | struct ifnet *oifp; | ||||
int error; | |||||
DPRINTF("ifp=%p\n", ifp); | |||||
sc = ifp->if_softc; | |||||
epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); | |||||
EPAIR_LOCK_ASSERT(epair_dpcpu); | |||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) | |||||
return; | |||||
if ((ifp->if_flags & IFF_UP) == 0) | |||||
return; | |||||
/* | /* | ||||
* We get packets here from ether_output via if_handoff() | * We get packets here from ether_output via if_handoff() | ||||
* and need to put them into the input queue of the oifp | * and need to put them into the input queue of the oifp | ||||
* and call oifp->if_input() via netisr/epair_sintr(). | * and will put the packet into the receive-queue (rxq) of the | ||||
* other interface (oifp) of our pair. | |||||
*/ | */ | ||||
sc = ifp->if_softc; | |||||
oifp = sc->oifp; | oifp = sc->oifp; | ||||
sc = oifp->if_softc; | sc = oifp->if_softc; | ||||
for (;;) { | for (;;) { | ||||
IFQ_DEQUEUE(&ifp->if_snd, m); | IFQ_DEQUEUE(&ifp->if_snd, m); | ||||
if (m == NULL) | if (m == NULL) | ||||
break; | break; | ||||
M_ASSERTPKTHDR(m); | |||||
BPF_MTAP(ifp, m); | BPF_MTAP(ifp, m); | ||||
/* | /* In case either interface is not usable drop the packet. */ | ||||
* In case the outgoing interface is not usable, | if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || | ||||
* drop the packet. | (ifp->if_flags & IFF_UP) == 0 || | ||||
*/ | (oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || | ||||
if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || | |||||
(oifp->if_flags & IFF_UP) ==0) { | (oifp->if_flags & IFF_UP) == 0) { | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
m_freem(m); | m_freem(m); | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
continue; | continue; | ||||
} | } | ||||
DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); | |||||
/* | (void) epair_menq(m, sc); | ||||
* Add a reference so the interface cannot go while the | |||||
* packet is in transit as we rely on rcvif to stay valid. | |||||
*/ | |||||
EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | |||||
m->m_pkthdr.rcvif = oifp; | |||||
CURVNET_SET_QUIET(oifp->if_vnet); | |||||
error = netisr_queue(NETISR_EPAIR, m); | |||||
CURVNET_RESTORE(); | |||||
if (!error) { | |||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | |||||
/* Someone else received the packet. */ | |||||
if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | |||||
} else { | |||||
/* The packet was freed already. */ | |||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | |||||
ifp->if_drv_flags |= IFF_DRV_OACTIVE; | |||||
(void) epair_add_ifp_for_draining(ifp); | |||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | |||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | |||||
("%s: ifp=%p sc->refcount not >= 1: %d", | |||||
__func__, oifp, sc->refcount)); | |||||
} | } | ||||
} | } | ||||
} | |||||
static void | |||||
epair_start(struct ifnet *ifp) | |||||
{ | |||||
struct epair_dpcpu *epair_dpcpu; | |||||
epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); | |||||
EPAIR_LOCK(epair_dpcpu); | |||||
epair_start_locked(ifp); | |||||
EPAIR_UNLOCK(epair_dpcpu); | |||||
} | |||||
static int | static int | ||||
epair_transmit_locked(struct ifnet *ifp, struct mbuf *m) | epair_transmit(struct ifnet *ifp, struct mbuf *m) | ||||
{ | { | ||||
struct epair_dpcpu *epair_dpcpu; | |||||
struct epair_softc *sc; | struct epair_softc *sc; | ||||
struct ifnet *oifp; | struct ifnet *oifp; | ||||
int error, len; | int error, len; | ||||
short mflags; | short mflags; | ||||
DPRINTF("ifp=%p m=%p\n", ifp, m); | |||||
sc = ifp->if_softc; | |||||
epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); | |||||
EPAIR_LOCK_ASSERT(epair_dpcpu); | |||||
if (m == NULL) | if (m == NULL) | ||||
return (0); | return (0); | ||||
M_ASSERTPKTHDR(m); | |||||
/* | /* | ||||
* We are not going to use the interface en/dequeue mechanism | * We are not going to use the interface en/dequeue mechanism | ||||
* on the TX side. We are called from ether_output_frame() | * on the TX side. We are called from ether_output_frame() | ||||
* and will put the packet into the incoming queue of the | * and will put the packet into the receive-queue (rxq) of the | ||||
* other interface of our pair via the netsir. | * other interface (oifp) of our pair. | ||||
*/ | */ | ||||
if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { | if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { | ||||
m_freem(m); | m_freem(m); | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
if ((ifp->if_flags & IFF_UP) == 0) { | if ((ifp->if_flags & IFF_UP) == 0) { | ||||
m_freem(m); | m_freem(m); | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
return (ENETDOWN); | return (ENETDOWN); | ||||
} | } | ||||
BPF_MTAP(ifp, m); | BPF_MTAP(ifp, m); | ||||
/* | /* | ||||
* In case the outgoing interface is not usable, | * In case the outgoing interface is not usable, | ||||
* drop the packet. | * drop the packet. | ||||
*/ | */ | ||||
sc = ifp->if_softc; | |||||
oifp = sc->oifp; | oifp = sc->oifp; | ||||
if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || | if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || | ||||
(oifp->if_flags & IFF_UP) ==0) { | (oifp->if_flags & IFF_UP) == 0) { | ||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | ||||
m_freem(m); | m_freem(m); | ||||
return (0); | return (0); | ||||
} | } | ||||
len = m->m_pkthdr.len; | len = m->m_pkthdr.len; | ||||
mflags = m->m_flags; | mflags = m->m_flags; | ||||
DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); | |||||
#ifdef ALTQ | #ifdef ALTQ | ||||
/* Support ALTQ via the classic if_start() path. */ | /* Support ALTQ via the classic if_start() path. */ | ||||
IF_LOCK(&ifp->if_snd); | IF_LOCK(&ifp->if_snd); | ||||
if (ALTQ_IS_ENABLED(&ifp->if_snd)) { | if (ALTQ_IS_ENABLED(&ifp->if_snd)) { | ||||
ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); | ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); | ||||
if (error) | if (error) | ||||
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); | if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); | ||||
IF_UNLOCK(&ifp->if_snd); | IF_UNLOCK(&ifp->if_snd); | ||||
if (!error) { | if (!error) { | ||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | ||||
if (mflags & (M_BCAST|M_MCAST)) | if (mflags & (M_BCAST|M_MCAST)) | ||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | ||||
epair_start(ifp); | |||||
if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) | |||||
epair_start_locked(ifp); | |||||
else | |||||
(void)epair_add_ifp_for_draining(ifp); | |||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
IF_UNLOCK(&ifp->if_snd); | IF_UNLOCK(&ifp->if_snd); | ||||
#endif | #endif | ||||
if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { | error = epair_menq(m, oifp->if_softc); | ||||
/* | |||||
* Our hardware queue is full, try to fall back | |||||
* queuing to the ifq but do not call ifp->if_start. | |||||
* Either we are lucky or the packet is gone. | |||||
*/ | |||||
IFQ_ENQUEUE(&ifp->if_snd, m, error); | |||||
if (!error) | |||||
(void)epair_add_ifp_for_draining(ifp); | |||||
return (error); | return (error); | ||||
} | } | ||||
sc = oifp->if_softc; | |||||
/* | |||||
* Add a reference so the interface cannot go while the | |||||
* packet is in transit as we rely on rcvif to stay valid. | |||||
*/ | |||||
EPAIR_REFCOUNT_AQUIRE(&sc->refcount); | |||||
m->m_pkthdr.rcvif = oifp; | |||||
CURVNET_SET_QUIET(oifp->if_vnet); | |||||
error = netisr_queue(NETISR_EPAIR, m); | |||||
CURVNET_RESTORE(); | |||||
if (!error) { | |||||
if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); | |||||
/* | |||||
* IFQ_HANDOFF_ADJ/ip_handoff() update statistics, | |||||
* but as we bypass all this we have to duplicate | |||||
* the logic another time. | |||||
*/ | |||||
if_inc_counter(ifp, IFCOUNTER_OBYTES, len); | |||||
if (mflags & (M_BCAST|M_MCAST)) | |||||
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | |||||
/* Someone else received the packet. */ | |||||
if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); | |||||
} else { | |||||
/* The packet was freed already. */ | |||||
epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; | |||||
ifp->if_drv_flags |= IFF_DRV_OACTIVE; | |||||
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); | |||||
EPAIR_REFCOUNT_RELEASE(&sc->refcount); | |||||
EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, | |||||
("%s: ifp=%p sc->refcount not >= 1: %d", | |||||
__func__, oifp, sc->refcount)); | |||||
} | |||||
return (error); | |||||
} | |||||
static int | static int | ||||
epair_transmit(struct ifnet *ifp, struct mbuf *m) | |||||
{ | |||||
struct epair_dpcpu *epair_dpcpu; | |||||
int error; | |||||
epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); | |||||
EPAIR_LOCK(epair_dpcpu); | |||||
error = epair_transmit_locked(ifp, m); | |||||
EPAIR_UNLOCK(epair_dpcpu); | |||||
return (error); | |||||
} | |||||
static void | |||||
epair_qflush(struct ifnet *ifp) | |||||
{ | |||||
struct epair_softc *sc; | |||||
sc = ifp->if_softc; | |||||
KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", | |||||
__func__, ifp, sc)); | |||||
/* | |||||
* Remove this ifp from all backpointer lists. The interface will not | |||||
* usable for flushing anyway nor should it have anything to flush | |||||
* after if_qflush(). | |||||
*/ | |||||
epair_remove_ifp_from_draining(ifp); | |||||
if (sc->if_qflush) | |||||
sc->if_qflush(ifp); | |||||
} | |||||
static int | |||||
epair_media_change(struct ifnet *ifp __unused) | epair_media_change(struct ifnet *ifp __unused) | ||||
{ | { | ||||
/* Do nothing. */ | /* Do nothing. */ | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
* We use our private ones so that we can create/destroy our secondary | * We use our private ones so that we can create/destroy our secondary | ||||
* device along with the primary one. | * device along with the primary one. | ||||
*/ | */ | ||||
static int | static int | ||||
epair_clone_match(struct if_clone *ifc, const char *name) | epair_clone_match(struct if_clone *ifc, const char *name) | ||||
{ | { | ||||
const char *cp; | const char *cp; | ||||
DPRINTF("name='%s'\n", name); | |||||
/* | /* | ||||
* Our base name is epair. | * Our base name is epair. | ||||
* Our interfaces will be named epair<n>[ab]. | * Our interfaces will be named epair<n>[ab]. | ||||
* So accept anything of the following list: | * So accept anything of the following list: | ||||
* - epair | * - epair | ||||
* - epair<n> | * - epair<n> | ||||
* but not the epair<n>[ab] versions. | * but not the epair<n>[ab] versions. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) | ||||
if (ifunit(name) != NULL) | if (ifunit(name) != NULL) | ||||
return (EEXIST); | return (EEXIST); | ||||
*dp = 'a'; | *dp = 'a'; | ||||
if (ifunit(name) != NULL) | if (ifunit(name) != NULL) | ||||
return (EEXIST); | return (EEXIST); | ||||
/* Allocate memory for both [ab] interfaces */ | /* Allocate memory for both [ab] interfaces */ | ||||
sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); | sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); | ||||
EPAIR_REFCOUNT_INIT(&sca->refcount, 1); | EPAIR_SC_LOCK_INIT(sca); | ||||
mbufq_init(&sca->rxq, maxqlen); | |||||
sca->ifp = if_alloc(IFT_ETHER); | sca->ifp = if_alloc(IFT_ETHER); | ||||
if (sca->ifp == NULL) { | if (sca->ifp == NULL) { | ||||
EPAIR_SC_LOCK_DESTROY(sca); | |||||
free(sca, M_EPAIR); | free(sca, M_EPAIR); | ||||
ifc_free_unit(ifc, unit); | ifc_free_unit(ifc, unit); | ||||
return (ENOSPC); | return (ENOSPC); | ||||
} | } | ||||
scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); | scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); | ||||
EPAIR_REFCOUNT_INIT(&scb->refcount, 1); | EPAIR_SC_LOCK_INIT(scb); | ||||
mbufq_init(&scb->rxq, maxqlen); | |||||
scb->ifp = if_alloc(IFT_ETHER); | scb->ifp = if_alloc(IFT_ETHER); | ||||
if (scb->ifp == NULL) { | if (scb->ifp == NULL) { | ||||
EPAIR_SC_LOCK_DESTROY(scb); | |||||
free(scb, M_EPAIR); | free(scb, M_EPAIR); | ||||
if_free(sca->ifp); | if_free(sca->ifp); | ||||
EPAIR_SC_LOCK_DESTROY(sca); | |||||
free(sca, M_EPAIR); | free(sca, M_EPAIR); | ||||
ifc_free_unit(ifc, unit); | ifc_free_unit(ifc, unit); | ||||
return (ENOSPC); | return (ENOSPC); | ||||
} | } | ||||
/* | /* | ||||
* Cross-reference the interfaces so we will be able to free both. | * Cross-reference the interfaces so we will be able to free both. | ||||
*/ | */ | ||||
sca->oifp = scb->ifp; | sca->oifp = scb->ifp; | ||||
scb->oifp = sca->ifp; | scb->oifp = sca->ifp; | ||||
/* | #ifdef GLOBAL_SWI | ||||
* Calculate the cpuid for netisr queueing based on the | EPAIR_LOCK(); | ||||
* ifIndex of the interfaces. As long as we cannot configure | #ifdef SMP | ||||
* this or use cpuset information easily we cannot guarantee | /* Get an approximate distribution. */ | ||||
* cache locality but we can at least allow parallelism. | hash = next_index % mp_ncpus; | ||||
*/ | #else | ||||
sca->cpuid = | hash = 0; | ||||
netisr_get_cpuid(sca->ifp->if_index); | #endif | ||||
scb->cpuid = | if (swi_cookie[hash] == NULL) { | ||||
netisr_get_cpuid(scb->ifp->if_index); | void *cookie; | ||||
EPAIR_UNLOCK(); | |||||
error = swi_add(NULL, epairname, | |||||
epair_intr, (void *)(uintptr_t)hash, | |||||
SWI_NET, INTR_MPSAFE, &cookie); | |||||
if (error) { | |||||
if_free(scb->ifp); | |||||
EPAIR_SC_LOCK_DESTROY(scb); | |||||
free(scb, M_EPAIR); | |||||
if_free(sca->ifp); | |||||
EPAIR_SC_LOCK_DESTROY(sca); | |||||
free(sca, M_EPAIR); | |||||
ifc_free_unit(ifc, unit); | |||||
return (ENOSPC); | |||||
} | |||||
EPAIR_LOCK(); | |||||
/* Recheck under lock even though a race is very unlikely. */ | |||||
if (swi_cookie[hash] == NULL) { | |||||
swi_cookie[hash] = cookie; | |||||
} else { | |||||
EPAIR_UNLOCK(); | |||||
(void) swi_remove(cookie); | |||||
EPAIR_LOCK(); | |||||
} | |||||
} | |||||
sca->cpuidx = hash; | |||||
STAILQ_INSERT_TAIL(&swi_sc[hash], sca, entry); | |||||
sca->swi_cookie = swi_cookie[hash]; | |||||
scb->cpuidx = hash; | |||||
STAILQ_INSERT_TAIL(&swi_sc[hash], scb, entry); | |||||
scb->swi_cookie = swi_cookie[hash]; | |||||
EPAIR_UNLOCK(); | |||||
#else | |||||
error = swi_add(NULL, epairname, epair_intr, sca, SWI_NET, | |||||
INTR_MPSAFE, &sca->swi_cookie); | |||||
error |= swi_add(NULL, epairname, epair_intr, scb, SWI_NET, | |||||
INTR_MPSAFE, &scb->swi_cookie); | |||||
if (error != 0) { | |||||
if (scb->swi_cookie != NULL) | |||||
(void) swi_remove(scb->swi_cookie); | |||||
if_free(scb->ifp); | |||||
EPAIR_SC_LOCK_DESTROY(scb); | |||||
free(scb, M_EPAIR); | |||||
if (sca->swi_cookie != NULL) | |||||
(void) swi_remove(sca->swi_cookie); | |||||
if_free(sca->ifp); | |||||
EPAIR_SC_LOCK_DESTROY(sca); | |||||
free(sca, M_EPAIR); | |||||
ifc_free_unit(ifc, unit); | |||||
return (ENOSPC); | |||||
} | |||||
#endif | |||||
/* Initialise pseudo media types. */ | /* Initialise pseudo media types. */ | ||||
ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status); | ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status); | ||||
ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL); | ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL); | ||||
ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T); | ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T); | ||||
ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status); | ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status); | ||||
ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL); | ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL); | ||||
ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T); | ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T); | ||||
Show All 19 Lines | #endif | ||||
* to a different VNET after creation. In that case its index | * to a different VNET after creation. In that case its index | ||||
* will be freed and the index can get reused by new epair instance. | * will be freed and the index can get reused by new epair instance. | ||||
* Make sure we do not create same etheraddr again. | * Make sure we do not create same etheraddr again. | ||||
*/ | */ | ||||
getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); | getcredhostid(curthread->td_ucred, (unsigned long *)&hostid); | ||||
if (hostid == 0) | if (hostid == 0) | ||||
arc4rand(&hostid, sizeof(hostid), 0); | arc4rand(&hostid, sizeof(hostid), 0); | ||||
EPAIR_LOCK(); | |||||
if (ifp->if_index > next_index) | if (ifp->if_index > next_index) | ||||
next_index = ifp->if_index; | next_index = ifp->if_index; | ||||
else | else | ||||
next_index++; | next_index++; | ||||
key[0] = (uint32_t)next_index; | key[0] = (uint32_t)next_index; | ||||
EPAIR_UNLOCK(); | |||||
key[1] = (uint32_t)(hostid & 0xffffffff); | key[1] = (uint32_t)(hostid & 0xffffffff); | ||||
key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); | key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff); | ||||
hash = jenkins_hash32(key, 3, 0); | hash = jenkins_hash32(key, 3, 0); | ||||
eaddr[0] = 0x02; | eaddr[0] = 0x02; | ||||
memcpy(&eaddr[1], &hash, 4); | memcpy(&eaddr[1], &hash, 4); | ||||
eaddr[5] = 0x0a; | eaddr[5] = 0x0a; | ||||
ether_ifattach(ifp, eaddr); | ether_ifattach(ifp, eaddr); | ||||
sca->if_qflush = ifp->if_qflush; | |||||
ifp->if_qflush = epair_qflush; | |||||
ifp->if_transmit = epair_transmit; | |||||
ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ | ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ | ||||
ifp->if_transmit = epair_transmit; | |||||
/* Swap the name and finish initialization of interface <n>b. */ | /* Swap the name and finish initialization of interface <n>b. */ | ||||
*dp = 'b'; | *dp = 'b'; | ||||
ifp = scb->ifp; | ifp = scb->ifp; | ||||
ifp->if_softc = scb; | ifp->if_softc = scb; | ||||
strlcpy(ifp->if_xname, name, IFNAMSIZ); | strlcpy(ifp->if_xname, name, IFNAMSIZ); | ||||
ifp->if_dname = epairname; | ifp->if_dname = epairname; | ||||
ifp->if_dunit = unit; | ifp->if_dunit = unit; | ||||
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; | ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; | ||||
ifp->if_capabilities = IFCAP_VLAN_MTU; | ifp->if_capabilities = IFCAP_VLAN_MTU; | ||||
ifp->if_capenable = IFCAP_VLAN_MTU; | ifp->if_capenable = IFCAP_VLAN_MTU; | ||||
ifp->if_start = epair_start; | ifp->if_start = epair_start; | ||||
ifp->if_ioctl = epair_ioctl; | ifp->if_ioctl = epair_ioctl; | ||||
ifp->if_init = epair_init; | ifp->if_init = epair_init; | ||||
if_setsendqlen(ifp, ifqmaxlen); | if_setsendqlen(ifp, ifqmaxlen); | ||||
if_setsendqready(ifp); | if_setsendqready(ifp); | ||||
/* We need to play some tricks here for the second interface. */ | /* We need to play some tricks here for the second interface. */ | ||||
strlcpy(name, epairname, len); | strlcpy(name, epairname, len); | ||||
/* Correctly set the name for the cloner list. */ | /* Correctly set the name for the cloner list. */ | ||||
strlcpy(name, scb->ifp->if_xname, len); | strlcpy(name, scb->ifp->if_xname, len); | ||||
epair_clone_add(ifc, scb); | epair_clone_add(ifc, scb); | ||||
scb->if_qflush = ifp->if_qflush; | |||||
ifp->if_qflush = epair_qflush; | |||||
ifp->if_transmit = epair_transmit; | |||||
ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ | ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */ | ||||
ifp->if_transmit = epair_transmit; | |||||
/* | /* | ||||
* Restore name to <n>a as the ifp for this will go into the | * Restore name to <n>a as the ifp for this will go into the | ||||
* cloner list for the initial call. | * cloner list for the initial call. | ||||
*/ | */ | ||||
strlcpy(name, sca->ifp->if_xname, len); | strlcpy(name, sca->ifp->if_xname, len); | ||||
DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb); | |||||
/* Tell the world, that we are ready to rock. */ | /* Tell the world, that we are ready to rock. */ | ||||
EPAIR_SC_LOCK(sca); | |||||
sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; | sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; | ||||
scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; | |||||
if_link_state_change(sca->ifp, LINK_STATE_UP); | if_link_state_change(sca->ifp, LINK_STATE_UP); | ||||
EPAIR_SC_UNLOCK(sca); | |||||
EPAIR_SC_LOCK(scb); | |||||
scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; | |||||
if_link_state_change(scb->ifp, LINK_STATE_UP); | if_link_state_change(scb->ifp, LINK_STATE_UP); | ||||
EPAIR_SC_UNLOCK(scb); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) | epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) | ||||
{ | { | ||||
struct ifnet *oifp; | struct ifnet *oifp; | ||||
struct epair_softc *sca, *scb; | struct epair_softc *sca, *scb; | ||||
int unit, error; | int unit, error; | ||||
DPRINTF("ifp=%p\n", ifp); | |||||
/* | /* | ||||
* In case we called into if_clone_destroyif() ourselves | * In case we called into if_clone_destroyif() ourselves | ||||
* again to remove the second interface, the softc will be | * again to remove the second interface, the softc will be | ||||
* NULL. In that case so not do anything but return success. | * NULL. In that case so not do anything but return success. | ||||
*/ | */ | ||||
if (ifp->if_softc == NULL) | if (ifp->if_softc == NULL) | ||||
return (0); | return (0); | ||||
unit = ifp->if_dunit; | unit = ifp->if_dunit; | ||||
sca = ifp->if_softc; | sca = ifp->if_softc; | ||||
EPAIR_SC_LOCK(sca); | |||||
oifp = sca->oifp; | oifp = sca->oifp; | ||||
scb = oifp->if_softc; | scb = oifp->if_softc; | ||||
DPRINTF("ifp=%p oifp=%p\n", ifp, oifp); | /* Frist get the interfaces down and detached. */ | ||||
afedorovUnsubmitted Not Done Inline ActionsFrist -> First afedorov: Frist -> First | |||||
if_link_state_change(ifp, LINK_STATE_DOWN); | if_link_state_change(ifp, LINK_STATE_DOWN); | ||||
if_link_state_change(oifp, LINK_STATE_DOWN); | |||||
ifp->if_drv_flags &= ~IFF_DRV_RUNNING; | ifp->if_drv_flags &= ~IFF_DRV_RUNNING; | ||||
EPAIR_SC_UNLOCK(sca); | |||||
EPAIR_SC_LOCK(scb); | |||||
if_link_state_change(oifp, LINK_STATE_DOWN); | |||||
oifp->if_drv_flags &= ~IFF_DRV_RUNNING; | oifp->if_drv_flags &= ~IFF_DRV_RUNNING; | ||||
EPAIR_SC_UNLOCK(scb); | |||||
/* | ether_ifdetach(ifp); | ||||
* Get rid of our second half. As the other of the two | |||||
* interfaces may reside in a different vnet, we need to | |||||
* switch before freeing them. | |||||
*/ | |||||
CURVNET_SET_QUIET(oifp->if_vnet); | |||||
ether_ifdetach(oifp); | ether_ifdetach(oifp); | ||||
/* | |||||
* Wait for all packets to be dispatched to if_input. | /* Second stop interrupt handler. */ | ||||
* The numbers can only go down as the interface is | #ifdef GLOBAL_SWI | ||||
* detached so there is no need to use atomics. | EPAIR_LOCK(); | ||||
*/ | STAILQ_REMOVE(&swi_sc[sca->cpuidx], sca, epair_softc, entry); | ||||
DPRINTF("scb refcnt=%u\n", scb->refcount); | STAILQ_REMOVE(&swi_sc[scb->cpuidx], scb, epair_softc, entry); | ||||
EPAIR_REFCOUNT_ASSERT(scb->refcount == 1, | EPAIR_UNLOCK(); | ||||
("%s: ifp=%p scb->refcount!=1: %d", __func__, oifp, scb->refcount)); | #else | ||||
if (sca->swi_cookie != NULL) | |||||
(void) swi_remove(sca->swi_cookie); | |||||
if (scb->swi_cookie != NULL) | |||||
(void) swi_remove(scb->swi_cookie); | |||||
#endif | |||||
sca->swi_cookie = NULL; | |||||
scb->swi_cookie = NULL; | |||||
/* Third free any queued packets and all the resources. */ | |||||
CURVNET_SET_QUIET(oifp->if_vnet); | |||||
mbufq_drain(&scb->rxq); | |||||
oifp->if_softc = NULL; | oifp->if_softc = NULL; | ||||
error = if_clone_destroyif(ifc, oifp); | error = if_clone_destroyif(ifc, oifp); | ||||
if (error) | if (error) | ||||
panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", | panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", | ||||
__func__, error); | __func__, error); | ||||
if_free(oifp); | if_free(oifp); | ||||
ifmedia_removeall(&scb->media); | ifmedia_removeall(&scb->media); | ||||
EPAIR_SC_LOCK_DESTROY(scb); | |||||
free(scb, M_EPAIR); | free(scb, M_EPAIR); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
ether_ifdetach(ifp); | mbufq_drain(&sca->rxq); | ||||
/* | |||||
* Wait for all packets to be dispatched to if_input. | |||||
*/ | |||||
DPRINTF("sca refcnt=%u\n", sca->refcount); | |||||
EPAIR_REFCOUNT_ASSERT(sca->refcount == 1, | |||||
("%s: ifp=%p sca->refcount!=1: %d", __func__, ifp, sca->refcount)); | |||||
if_free(ifp); | if_free(ifp); | ||||
ifmedia_removeall(&sca->media); | ifmedia_removeall(&sca->media); | ||||
EPAIR_SC_LOCK_DESTROY(sca); | |||||
free(sca, M_EPAIR); | free(sca, M_EPAIR); | ||||
/* Last free the cloner unit. */ | |||||
ifc_free_unit(ifc, unit); | ifc_free_unit(ifc, unit); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
vnet_epair_init(const void *unused __unused) | vnet_epair_init(const void *unused __unused) | ||||
{ | { | ||||
V_epair_cloner = if_clone_advanced(epairname, 0, | V_epair_cloner = if_clone_advanced(epairname, 0, | ||||
epair_clone_match, epair_clone_create, epair_clone_destroy); | epair_clone_match, epair_clone_create, epair_clone_destroy); | ||||
#ifdef VIMAGE | |||||
netisr_register_vnet(&epair_nh); | |||||
#endif | |||||
} | } | ||||
VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY, | VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY, | ||||
vnet_epair_init, NULL); | vnet_epair_init, NULL); | ||||
static void | static void | ||||
vnet_epair_uninit(const void *unused __unused) | vnet_epair_uninit(const void *unused __unused) | ||||
{ | { | ||||
#ifdef VIMAGE | |||||
netisr_unregister_vnet(&epair_nh); | |||||
#endif | |||||
if_clone_detach(V_epair_cloner); | if_clone_detach(V_epair_cloner); | ||||
} | } | ||||
VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, | VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, | ||||
vnet_epair_uninit, NULL); | vnet_epair_uninit, NULL); | ||||
static void | |||||
epair_uninit(const void *unused __unused) | |||||
{ | |||||
netisr_unregister(&epair_nh); | |||||
epair_dpcpu_detach(); | |||||
if (bootverbose) | |||||
printf("%s unloaded.\n", epairname); | |||||
} | |||||
SYSUNINIT(epair_uninit, SI_SUB_INIT_IF, SI_ORDER_MIDDLE, | |||||
epair_uninit, NULL); | |||||
static int | static int | ||||
epair_modevent(module_t mod, int type, void *data) | epair_modevent(module_t mod, int type, void *data) | ||||
{ | { | ||||
int qlimit; | #ifdef GLOBAL_SWI | ||||
int i; | |||||
#endif | |||||
switch (type) { | switch (type) { | ||||
case MOD_LOAD: | case MOD_LOAD: | ||||
/* For now limit us to one global mutex and one inq. */ | #ifdef GLOBAL_SWI | ||||
epair_dpcpu_init(); | for (i = 0; i < MAXCPU; i++) { | ||||
epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */ | swi_cookie[i] = NULL; | ||||
if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit)) | STAILQ_INIT(&swi_sc[i]); | ||||
epair_nh.nh_qlimit = qlimit; | } | ||||
netisr_register(&epair_nh); | #endif | ||||
EPAIR_LOCK_INIT(); | |||||
if (bootverbose) | if (bootverbose) | ||||
printf("%s initialized.\n", epairname); | printf("%s: %s initialized.\n", __func__, epairname); | ||||
break; | break; | ||||
case MOD_UNLOAD: | case MOD_UNLOAD: | ||||
/* Handled in epair_uninit() */ | #ifdef GLOBAL_SWI | ||||
EPAIR_LOCK(); | |||||
for (i = 0; i < MAXCPU; i++) { | |||||
if (!STAILQ_EMPTY(&swi_sc[i])) { | |||||
printf("%s: swi_sc[%d] active\n", __func__, i); | |||||
EPAIR_UNLOCK(); | |||||
return (EBUSY); | |||||
} | |||||
} | |||||
EPAIR_UNLOCK(); | |||||
for (i = 0; i < MAXCPU; i++) | |||||
if (swi_cookie[i] != NULL) | |||||
(void) swi_remove(swi_cookie[i]); | |||||
#endif | |||||
EPAIR_LOCK_DESTROY(); | |||||
if (bootverbose) | |||||
printf("%s: %s unloaded.\n", __func__, epairname); | |||||
break; | break; | ||||
default: | default: | ||||
return (EOPNOTSUPP); | return (EOPNOTSUPP); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static moduledata_t epair_mod = { | static moduledata_t epair_mod = { | ||||
"if_epair", | "if_epair", | ||||
epair_modevent, | epair_modevent, | ||||
0 | 0 | ||||
}; | }; | ||||
DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE); | DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE); | ||||
MODULE_VERSION(if_epair, 1); | MODULE_VERSION(if_epair, 2); |
I think the foundation has a blanket 'remove all rights reserved' policy. Maybe you can ask?