Index: sys/contrib/ipfilter/netinet/ip_fil_freebsd.c =================================================================== --- sys/contrib/ipfilter/netinet/ip_fil_freebsd.c +++ sys/contrib/ipfilter/netinet/ip_fil_freebsd.c @@ -1307,8 +1307,10 @@ fr_info_t *fin; mb_t *m; { + struct epoch_tracker et; int error = 0; + NET_EPOCH_ENTER(et); if (fin->fin_out == 0) { netisr_dispatch(NETISR_IP, m); } else { @@ -1316,6 +1318,7 @@ fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off); error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL); } + NET_EPOCH_EXIT(et); return error; } Index: sys/dev/firewire/if_fwip.c =================================================================== --- sys/dev/firewire/if_fwip.c +++ sys/dev/firewire/if_fwip.c @@ -708,6 +708,7 @@ static void fwip_stream_input(struct fw_xferq *xferq) { + struct epoch_tracker et; struct mbuf *m, *m0; struct m_tag *mtag; struct ifnet *ifp; @@ -717,10 +718,10 @@ uint16_t src; uint32_t *p; - fwip = (struct fwip_softc *)xferq->sc; ifp = fwip->fw_softc.fwip_ifp; + NET_EPOCH_ENTER(et); while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) { STAILQ_REMOVE_HEAD(&xferq->stvalid, link); fp = mtod(sxfer->mbuf, struct fw_pkt *); @@ -809,6 +810,7 @@ firewire_input(ifp, m, src); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); } + NET_EPOCH_EXIT(et); if (STAILQ_FIRST(&xferq->stfree) != NULL) fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch); } Index: sys/dev/iicbus/if_ic.c =================================================================== --- sys/dev/iicbus/if_ic.c +++ sys/dev/iicbus/if_ic.c @@ -309,9 +309,13 @@ BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN); top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0); if (top) { + struct epoch_tracker et; + mtx_unlock(&sc->ic_lock); M_SETFIB(top, sc->ic_ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, top); + NET_EPOCH_EXIT(et); mtx_lock(&sc->ic_lock); } break; Index: sys/dev/usb/net/if_usie.c =================================================================== --- sys/dev/usb/net/if_usie.c +++ sys/dev/usb/net/if_usie.c @@ -772,6 +772,7 @@ static void usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error) { + struct epoch_tracker et; struct usie_softc *sc = usbd_xfer_softc(xfer); struct ifnet *ifp = sc->sc_ifp; struct mbuf *m0; @@ -851,6 +852,7 @@ err = pkt = 0; /* HW can aggregate multiple frames in a single USB xfer */ + NET_EPOCH_ENTER(et); for (;;) { rxd = mtod(m, struct usie_desc *); @@ -917,6 +919,7 @@ m->m_data += diff; m->m_pkthdr.len = (m->m_len -= diff); } + NET_EPOCH_EXIT(et); mtx_lock(&sc->sc_mtx); Index: sys/dev/usb/net/uhso.c =================================================================== --- sys/dev/usb/net/uhso.c +++ sys/dev/usb/net/uhso.c @@ -1664,6 +1664,7 @@ static void uhso_if_rxflush(void *arg) { + struct epoch_tracker et; struct uhso_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint8_t *cp; @@ -1677,6 +1678,7 @@ m = NULL; mwait = sc->sc_mwait; + NET_EPOCH_ENTER(et); for (;;) { if (m == NULL) { if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL) @@ -1787,6 +1789,7 @@ m = m0 != NULL ? m0 : NULL; mtx_lock(&sc->sc_mtx); } + NET_EPOCH_EXIT(et); sc->sc_mwait = mwait; } Index: sys/kern/uipc_socket.c =================================================================== --- sys/kern/uipc_socket.c +++ sys/kern/uipc_socket.c @@ -143,6 +143,8 @@ #include #include +#include /* XXXGL: net_epoch should move out there */ +#include /* XXXGL: net_epoch should move out there */ #include Index: sys/net/if.c =================================================================== --- sys/net/if.c +++ sys/net/if.c @@ -351,17 +351,14 @@ struct ifnet * ifnet_byindex_ref(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); - if (ifp == NULL || (ifp->if_flags & IFF_DYING)) { - NET_EPOCH_EXIT(et); + if (ifp == NULL || (ifp->if_flags & IFF_DYING)) return (NULL); - } if_ref(ifp); - NET_EPOCH_EXIT(et); return (ifp); } @@ -425,15 +422,14 @@ struct ifaddr * ifaddr_byindex(u_short idx) { - struct epoch_tracker et; struct ifnet *ifp; struct ifaddr *ifa = NULL; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifp = ifnet_byindex_locked(idx); if (ifp != NULL && (ifa = ifp->if_addr) != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (ifa); } @@ -1613,39 +1609,32 @@ static int if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp) { - struct epoch_tracker et; int len, error; struct ifg_list *ifgl; struct ifg_req ifgrq, *ifgp; + NET_EPOCH_ASSERT(); + if (ifgr->ifgr_len == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) ifgr->ifgr_len += sizeof(struct ifg_req); - NET_EPOCH_EXIT(et); return (0); } len = ifgr->ifgr_len; ifgp = ifgr_groups_get(ifgr); /* XXX: wire */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) { - if (len < sizeof(ifgrq)) { - NET_EPOCH_EXIT(et); + if (len < sizeof(ifgrq)) return (EINVAL); - } bzero(&ifgrq, sizeof ifgrq); strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group, sizeof(ifgrq.ifgrq_group)); - if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) { - NET_EPOCH_EXIT(et); + if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) return (error); - } len -= sizeof(ifgrq); ifgp++; } - NET_EPOCH_EXIT(et); return (0); } @@ -1865,10 +1854,11 @@ ifa_maintain_loopback_route(int cmd, const char *otype, struct ifaddr *ifa, struct sockaddr *ia) { - int error; + struct epoch_tracker et; struct rt_addrinfo info; struct sockaddr_dl null_sdl; struct ifnet *ifp; + int error; ifp = ifa->ifa_ifp; @@ -1880,7 +1870,9 @@ info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&null_sdl; link_init_sdl(ifp, (struct sockaddr *)&null_sdl, ifp->if_type); + NET_EPOCH_ENTER(et); error = rtrequest1_fib(cmd, &info, NULL, ifp->if_fib); + NET_EPOCH_EXIT(et); if (error != 0 && !(cmd == RTM_ADD && error == EEXIST) && @@ -1934,7 +1926,8 @@ struct ifnet *ifp; struct ifaddr *ifa; - MPASS(in_epoch(net_epoch_preempt)); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != addr->sa_family) @@ -2193,15 +2186,15 @@ static void link_rtrequest(int cmd, struct rtentry *rt, struct rt_addrinfo *info) { - struct epoch_tracker et; struct ifaddr *ifa, *oifa; struct sockaddr *dst; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + if (cmd != RTM_ADD || ((ifa = rt->rt_ifa) == NULL) || ((ifp = ifa->ifa_ifp) == NULL) || ((dst = rt_key(rt)) == NULL)) return; - NET_EPOCH_ENTER(et); ifa = ifaof_ifpforaddr(dst, ifp); if (ifa) { oifa = rt->rt_ifa; @@ -2213,7 +2206,6 @@ if (ifa->ifa_rtrequest && ifa->ifa_rtrequest != link_rtrequest) ifa->ifa_rtrequest(cmd, rt, info); } - NET_EPOCH_EXIT(et); } struct sockaddr_dl * @@ -2319,17 +2311,22 @@ ifp->if_link_state = link_state; + /* XXXGL: reference ifp? */ taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask); } static void do_link_state_change(void *arg, int pending) { - struct ifnet *ifp = (struct ifnet *)arg; - int link_state = ifp->if_link_state; - CURVNET_SET(ifp->if_vnet); + struct epoch_tracker et; + struct ifnet *ifp; + int link_state; + + NET_EPOCH_ENTER(et); + ifp = arg; + link_state = ifp->if_link_state; - /* Notify that the link state has changed. */ + CURVNET_SET(ifp->if_vnet); rt_ifmsg(ifp); if (ifp->if_vlantrunk != NULL) (*vlan_link_state_p)(ifp); @@ -2355,6 +2352,7 @@ (link_state == LINK_STATE_UP) ? "UP" : "DOWN" ); EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -2904,9 +2902,14 @@ break; case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP): - if ((error = if_getgroup((struct ifgroupreq *)data, ifp))) - return (error); + { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); + error = if_getgroup((struct ifgroupreq *)data, ifp); + NET_EPOCH_EXIT(et); break; + } case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP): error = priv_check(td, PRIV_NET_DELIFGROUP); @@ -2975,7 +2978,7 @@ #ifdef COMPAT_FREEBSD32 caddr_t saved_data = NULL; struct ifmediareq ifmr; - struct ifmediareq *ifmrp; + struct ifmediareq *ifmrp = NULL; #endif struct ifnet *ifp; struct ifreq *ifr; @@ -2996,12 +2999,10 @@ } #endif - switch (cmd) { case SIOCGIFCONF: error = ifconf(cmd, data); - CURVNET_RESTORE(); - return (error); + goto out_noref; #ifdef COMPAT_FREEBSD32 case SIOCGIFCONF32: @@ -3014,16 +3015,14 @@ ifc.ifc_buf = PTRIN(ifc32->ifc_buf); error = ifconf(SIOCGIFCONF, (void *)&ifc); - CURVNET_RESTORE(); if (error == 0) ifc32->ifc_len = ifc.ifc_len; - return (error); + goto out_noref; } #endif } #ifdef COMPAT_FREEBSD32 - ifmrp = NULL; switch (cmd) { case SIOCGIFMEDIA32: case SIOCGIFXMEDIA32: @@ -3615,16 +3614,15 @@ struct ifmultiaddr *ifma; int lastref; #ifdef INVARIANTS - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); KASSERT(ifp != NULL, ("%s: ifnet went away", __func__)); #endif @@ -3690,16 +3688,15 @@ if (ifp == NULL) { printf("%s: ifma_ifp seems to be detached\n", __func__); } else { - struct epoch_tracker et; struct ifnet *oifp; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link) if (ifp == oifp) break; if (ifp != oifp) ifp = NULL; - NET_EPOCH_EXIT(et); } #endif /* @@ -3823,11 +3820,11 @@ struct sockaddr_dl *sdl; struct ifaddr *ifa; struct ifreq ifr; - struct epoch_tracker et; int rc; + NET_EPOCH_ASSERT(); + rc = 0; - NET_EPOCH_ENTER(et); ifa = ifp->if_addr; if (ifa == NULL) { rc = EINVAL; @@ -3861,7 +3858,6 @@ * to re-init it in order to reprogram its * address filter. */ - NET_EPOCH_EXIT(et); if ((ifp->if_flags & IFF_UP) != 0) { if (ifp->if_ioctl) { ifp->if_flags &= ~IFF_UP; @@ -3876,8 +3872,7 @@ } EVENTHANDLER_INVOKE(iflladdr_event, ifp); return (0); - out: - NET_EPOCH_EXIT(et); +out: return (rc); } Index: sys/net/if_ethersubr.c =================================================================== --- sys/net/if_ethersubr.c +++ sys/net/if_ethersubr.c @@ -795,7 +795,7 @@ static void ether_input(struct ifnet *ifp, struct mbuf *m) { - + struct epoch_tracker et; struct mbuf *mn; /* @@ -803,21 +803,23 @@ * m_nextpkt. We split them up into separate packets here and pass * them up. This allows the drivers to amortize the receive lock. */ + CURVNET_SET_QUIET(ifp->if_vnet); + NET_EPOCH_ENTER(et); while (m) { mn = m->m_nextpkt; m->m_nextpkt = NULL; /* - * We will rely on rcvif being set properly in the deferred context, - * so assert it is correct here. + * We will rely on rcvif being set properly in the deferred + * context, so assert it is correct here. */ KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p " "rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp)); - CURVNET_SET_QUIET(ifp->if_vnet); netisr_dispatch(NETISR_ETHER, m); - CURVNET_RESTORE(); m = mn; } + NET_EPOCH_EXIT(et); + CURVNET_RESTORE(); } /* @@ -830,6 +832,7 @@ int i, isr; u_short ether_type; + NET_EPOCH_ASSERT(); KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__)); /* Do not grab PROMISC frames in case we are re-entered. */ Index: sys/net/if_gif.c =================================================================== --- sys/net/if_gif.c +++ sys/net/if_gif.c @@ -415,6 +415,8 @@ struct ifnet *oldifp; int isr, n, af; + NET_EPOCH_ASSERT(); + if (ifp == NULL) { /* just in case */ m_freem(m); Index: sys/net/if_me.c =================================================================== --- sys/net/if_me.c +++ sys/net/if_me.c @@ -451,6 +451,8 @@ struct ip *ip; int hlen; + NET_EPOCH_ASSERT(); + ifp = ME2IFP(sc); /* checks for short packets */ hlen = sizeof(struct mobhdr); Index: sys/net/if_stf.c =================================================================== --- sys/net/if_stf.c +++ sys/net/if_stf.c @@ -613,6 +613,8 @@ u_int8_t otos, itos; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + if (proto != IPPROTO_IPV6) { m_freem(m); return (IPPROTO_DONE); Index: sys/net/if_tun.c =================================================================== --- sys/net/if_tun.c +++ sys/net/if_tun.c @@ -906,6 +906,7 @@ static int tunwrite(struct cdev *dev, struct uio *uio, int flag) { + struct epoch_tracker et; struct tun_softc *tp = dev->si_drv1; struct ifnet *ifp = TUN2IFP(tp); struct mbuf *m; @@ -975,7 +976,9 @@ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return (0); } Index: sys/net/if_vlan.c =================================================================== --- sys/net/if_vlan.c +++ sys/net/if_vlan.c @@ -232,7 +232,6 @@ #define VLAN_XLOCK_ASSERT() sx_assert(&_VLAN_SX_ID, SA_XLOCKED) #define VLAN_SXLOCK_ASSERT() sx_assert(&_VLAN_SX_ID, SA_LOCKED) - /* * We also have a per-trunk mutex that should be acquired when changing * its state. @@ -241,7 +240,6 @@ #define TRUNK_LOCK_DESTROY(trunk) mtx_destroy(&(trunk)->lock) #define TRUNK_WLOCK(trunk) mtx_lock(&(trunk)->lock) #define TRUNK_WUNLOCK(trunk) mtx_unlock(&(trunk)->lock) -#define TRUNK_LOCK_ASSERT(trunk) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock)) #define TRUNK_WLOCK_ASSERT(trunk) mtx_assert(&(trunk)->lock, MA_OWNED); /* @@ -684,18 +682,17 @@ static struct ifnet * vlan_trunkdev(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlan *ifv; + NET_EPOCH_ASSERT(); + if (ifp->if_type != IFT_L2VLAN) return (NULL); - NET_EPOCH_ENTER(et); ifv = ifp->if_softc; ifp = NULL; if (ifv->ifv_trunk) ifp = PARENT(ifv); - NET_EPOCH_EXIT(et); return (ifp); } @@ -767,21 +764,18 @@ static struct ifnet * vlan_devat(struct ifnet *ifp, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return (NULL); - } ifp = NULL; ifv = vlan_gethash(trunk, vid); if (ifv) ifp = ifv->ifv_ifp; - NET_EPOCH_EXIT(et); return (ifp); } @@ -1118,16 +1112,15 @@ static int vlan_transmit(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlan *ifv; struct ifnet *p; int error, len, mcast; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + ifv = ifp->if_softc; if (TRUNK(ifv) == NULL) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } @@ -1143,14 +1136,12 @@ */ if (!UP_AND_RUNNING(p)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return (ENETDOWN); } if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (0); } @@ -1164,7 +1155,6 @@ if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast); } else if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); - NET_EPOCH_EXIT(et); return (error); } @@ -1179,16 +1169,15 @@ static void vlan_input(struct ifnet *ifp, struct mbuf *m) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; struct m_tag *mtag; uint16_t vid, tag; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; if (trunk == NULL) { - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1211,7 +1200,6 @@ if (m->m_len < sizeof(*evl) && (m = m_pullup(m, sizeof(*evl))) == NULL) { if_printf(ifp, "cannot pullup VLAN header\n"); - NET_EPOCH_EXIT(et); return; } evl = mtod(m, struct ether_vlan_header *); @@ -1234,7 +1222,6 @@ __func__, ifp->if_xname, ifp->if_type); #endif if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1244,7 +1231,6 @@ ifv = vlan_gethash(trunk, vid); if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) { - NET_EPOCH_EXIT(et); if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1); m_freem(m); return; @@ -1264,7 +1250,6 @@ sizeof(uint8_t), M_NOWAIT); if (mtag == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); - NET_EPOCH_EXIT(et); m_freem(m); return; } @@ -1275,7 +1260,6 @@ m->m_pkthdr.rcvif = ifv->ifv_ifp; if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1); - NET_EPOCH_EXIT(et); /* Pass it back through the parent's input routine. */ (*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m); @@ -1301,11 +1285,12 @@ static int vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifnet *ifp; int error = 0; + NET_EPOCH_ASSERT(); + /* * We can handle non-ethernet hardware types as long as * they handle the tagging and headers themselves. @@ -1401,9 +1386,7 @@ ifp->if_link_state = p->if_link_state; - NET_EPOCH_ENTER(et); vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); /* * Set up our interface address to reflect the underlying @@ -1575,17 +1558,14 @@ static void vlan_link_state(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; - /* Called from a taskqueue_swi task, so we cannot sleep. */ - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + trunk = ifp->if_vlantrunk; - if (trunk == NULL) { - NET_EPOCH_EXIT(et); + if (trunk == NULL) return; - } TRUNK_WLOCK(trunk); VLAN_FOREACH(ifv, trunk) { @@ -1594,7 +1574,6 @@ trunk->parent->if_link_state); } TRUNK_WUNLOCK(trunk); - NET_EPOCH_EXIT(et); } static void @@ -1606,8 +1585,9 @@ int cap = 0, ena = 0, mena; u_long hwa = 0; - VLAN_SXLOCK_ASSERT(); NET_EPOCH_ASSERT(); + VLAN_SXLOCK_ASSERT(); + p = PARENT(ifv); ifp = ifv->ifv_ifp; @@ -1699,7 +1679,6 @@ static void vlan_trunk_capabilities(struct ifnet *ifp) { - struct epoch_tracker et; struct ifvlantrunk *trunk; struct ifvlan *ifv; @@ -1709,11 +1688,8 @@ VLAN_SUNLOCK(); return; } - NET_EPOCH_ENTER(et); - VLAN_FOREACH(ifv, trunk) { + VLAN_FOREACH(ifv, trunk) vlan_capabilities(ifv); - } - NET_EPOCH_EXIT(et); VLAN_SUNLOCK(); } @@ -1728,6 +1704,8 @@ struct vlanreq vlr; int error = 0; + NET_EPOCH_ASSERT(); + ifr = (struct ifreq *)data; ifa = (struct ifaddr *) data; ifv = ifp->if_softc; @@ -1904,13 +1882,8 @@ VLAN_SLOCK(); ifv->ifv_capenable = ifr->ifr_reqcap; trunk = TRUNK(ifv); - if (trunk != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if (trunk != NULL) vlan_capabilities(ifv); - NET_EPOCH_EXIT(et); - } VLAN_SUNLOCK(); break; Index: sys/net/netisr.c =================================================================== --- sys/net/netisr.c +++ sys/net/netisr.c @@ -860,6 +860,7 @@ netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) { struct netisr_work local_npw, *npwp; + struct epoch_tracker et; u_int handled; struct mbuf *m; @@ -889,6 +890,7 @@ npwp->nw_len = 0; nwsp->nws_pendingbits &= ~(1 << proto); NWS_UNLOCK(nwsp); + NET_EPOCH_ENTER(et); while ((m = local_npw.nw_head) != NULL) { local_npw.nw_head = m->m_nextpkt; m->m_nextpkt = NULL; @@ -901,6 +903,7 @@ netisr_proto[proto].np_handler(m); CURVNET_RESTORE(); } + NET_EPOCH_EXIT(et); KASSERT(local_npw.nw_len == 0, ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); if (netisr_proto[proto].np_drainedcpu) @@ -1087,6 +1090,7 @@ int dosignal, error; u_int cpuid, dispatch_policy; + NET_EPOCH_ASSERT(); KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING Index: sys/net/route.c =================================================================== --- sys/net/route.c +++ sys/net/route.c @@ -593,12 +593,12 @@ int error = 0; short *stat = NULL; struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct rib_head *rnh; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) { error = EAFNOSUPPORT; @@ -693,7 +693,6 @@ if (rt) RTFREE_LOCKED(rt); out: - NET_EPOCH_EXIT(et); if (error) V_rtstat.rts_badredirect++; else if (stat != NULL) Index: sys/net/rtsock.c =================================================================== --- sys/net/rtsock.c +++ sys/net/rtsock.c @@ -554,6 +554,7 @@ struct rib_head *rnh; struct rt_addrinfo info; struct sockaddr_storage ss; + struct epoch_tracker et; #ifdef INET6 struct sockaddr_in6 *sin6; int i, rti_need_deembed = 0; @@ -573,6 +574,7 @@ return (ENOBUFS); if ((m->m_flags & M_PKTHDR) == 0) panic("route_output"); + NET_EPOCH_ENTER(et); len = m->m_pkthdr.len; if (len < sizeof(*rtm) || len != mtod(m, struct rt_msghdr *)->rtm_msglen) @@ -795,11 +797,11 @@ NET_EPOCH_ENTER(et); ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1, RT_ALL_FIBS); + NET_EPOCH_EXIT(et); if (ifa != NULL) rt_maskedcopy(ifa->ifa_addr, &laddr, ifa->ifa_netmask); - NET_EPOCH_EXIT(et); } else rt_maskedcopy(rt->rt_ifa->ifa_addr, &laddr, @@ -890,6 +892,7 @@ } flush: + NET_EPOCH_EXIT(et); if (rt != NULL) RTFREE(rt); /* @@ -1753,11 +1756,9 @@ struct rt_addrinfo info; int len, error = 0; struct sockaddr_storage ss; - struct epoch_tracker et; bzero((caddr_t)&info, sizeof(info)); bzero(&ifd, sizeof(ifd)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1807,7 +1808,6 @@ info.rti_info[RTAX_BRD] = NULL; } done: - NET_EPOCH_EXIT(et); return (error); } @@ -1815,16 +1815,16 @@ sysctl_ifmalist(int af, struct walkarg *w) { struct rt_addrinfo info; - struct epoch_tracker et; struct ifaddr *ifa; struct ifmultiaddr *ifma; struct ifnet *ifp; int error, len; + NET_EPOCH_ASSERT(); + error = 0; bzero((caddr_t)&info, sizeof(info)); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (w->w_arg && w->w_arg != ifp->if_index) continue; @@ -1859,7 +1859,6 @@ if (error != 0) break; } - NET_EPOCH_EXIT(et); return (error); } @@ -1867,6 +1866,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS) { RIB_RLOCK_TRACKER; + struct epoch_tracker et; int *name = (int *)arg1; u_int namelen = arg2; struct rib_head *rnh = NULL; /* silence compiler. */ @@ -1910,8 +1910,8 @@ w.w_tmemsize = 65536; w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK); + NET_EPOCH_ENTER(et); switch (w.w_op) { - case NET_RT_DUMP: case NET_RT_FLAGS: if (af == 0) { /* dump all tables */ @@ -1938,13 +1938,9 @@ for (error = 0; error == 0 && i <= lim; i++) { rnh = rt_tables_get_rnh(fib, i); if (rnh != NULL) { - struct epoch_tracker et; - RIB_RLOCK(rnh); - NET_EPOCH_ENTER(et); error = rnh->rnh_walktree(&rnh->head, sysctl_dumpentry, &w); - NET_EPOCH_EXIT(et); RIB_RUNLOCK(rnh); } else if (af != 0) error = EAFNOSUPPORT; @@ -1960,6 +1956,7 @@ error = sysctl_ifmalist(af, &w); break; } + NET_EPOCH_EXIT(et); free(w.w_tmem, M_TEMP); return (error); Index: sys/netgraph/ng_ether.c =================================================================== --- sys/netgraph/ng_ether.c +++ sys/netgraph/ng_ether.c @@ -711,6 +711,7 @@ static int ng_ether_rcv_upper(hook_p hook, item_p item) { + struct epoch_tracker et; struct mbuf *m; const node_p node = NG_HOOK_NODE(hook); const priv_p priv = NG_NODE_PRIVATE(node); @@ -738,7 +739,9 @@ } /* Route packet back in */ + NET_EPOCH_ENTER(et); ether_demux(ifp, m); + NET_EPOCH_EXIT(et); return (0); } Index: sys/netgraph/ng_iface.c =================================================================== --- sys/netgraph/ng_iface.c +++ sys/netgraph/ng_iface.c @@ -690,6 +690,7 @@ const priv_p priv = NG_NODE_PRIVATE(NG_HOOK_NODE(hook)); const iffam_p iffam = get_iffam_from_hook(priv, hook); struct ifnet *const ifp = priv->ifp; + struct epoch_tracker et; struct mbuf *m; int isr; @@ -731,7 +732,9 @@ } random_harvest_queue(m, sizeof(*m), RANDOM_NET_NG); M_SETFIB(m, ifp->if_fib); + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return (0); } Index: sys/netgraph/ng_ip_input.c =================================================================== --- sys/netgraph/ng_ip_input.c +++ sys/netgraph/ng_ip_input.c @@ -125,8 +125,13 @@ NG_FREE_ITEM(item); if (curthread->td_ng_outbound) netisr_queue(NETISR_IP, m); - else + else { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); netisr_dispatch(NETISR_IP, m); + NET_EPOCH_EXIT(et); + } return 0; } Index: sys/netinet/if_ether.c =================================================================== --- sys/netinet/if_ether.c +++ sys/netinet/if_ether.c @@ -257,12 +257,16 @@ if (r_skip_req == 0 && lle->la_preempt > 0) { /* Entry was used, issue refresh request */ + struct epoch_tracker et; struct in_addr dst; + dst = lle->r_l3addr.addr4; lle->la_preempt--; callout_schedule(&lle->lle_timer, hz * V_arpt_rexmit); LLE_WUNLOCK(lle); + NET_EPOCH_ENTER(et); arprequest(ifp, NULL, &dst, NULL); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); return; } @@ -361,15 +365,15 @@ struct route ro; int error; + NET_EPOCH_ASSERT(); + if (sip == NULL) { /* * The caller did not supply a source address, try to find * a compatible one among those assigned to this interface. */ - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -387,7 +391,6 @@ IA_MASKSIN(ifa)->sin_addr.s_addr)) break; /* found it. */ } - NET_EPOCH_EXIT(et); if (sip == NULL) { printf("%s: cannot find matching address\n", __func__); return (EADDRNOTAVAIL); @@ -474,18 +477,15 @@ char *lladdr; int ll_len; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) *plle = NULL; - if ((flags & LLE_CREATE) == 0) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); + if ((flags & LLE_CREATE) == 0) la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); - } if (la == NULL && (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { la = lltable_alloc_entry(LLTABLE(ifp), 0, dst); if (la == NULL) { @@ -622,9 +622,10 @@ const struct sockaddr *dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *la = NULL; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; if (plle != NULL) @@ -644,7 +645,6 @@ } } - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, dst); if (la != NULL && (la->r_flags & RLLE_VALID) != 0) { /* Entry found, let's copy lle info */ @@ -658,12 +658,10 @@ *plle = la; LLE_WUNLOCK(la); } - NET_EPOCH_EXIT(et); return (0); } if (plle && la) LLE_WUNLOCK(la); - NET_EPOCH_EXIT(et); return (arpresolve_full(ifp, is_gw, la == NULL ? LLE_CREATE : 0, m, dst, desten, pflags, plle)); @@ -808,7 +806,8 @@ int lladdr_off; int error; char addrbuf[INET_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); sin.sin_len = sizeof(struct sockaddr_in); sin.sin_family = AF_INET; @@ -901,17 +900,14 @@ * No match, use the first inet address on the receive interface * as a dummy address for the rest of the function. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && (ifa->ifa_carp == NULL || (*carp_iamatch_p)(ifa, &enaddr))) { ia = ifatoia(ifa); ifa_ref(ifa); - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); /* * If bridging, fall back to using any inet address. @@ -968,9 +964,7 @@ sin.sin_family = AF_INET; sin.sin_addr = isaddr; dst = (struct sockaddr *)&sin; - NET_EPOCH_ENTER(et); la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, dst); - NET_EPOCH_EXIT(et); if (la != NULL) arp_check_update_lle(ah, isaddr, ifp, bridged, la); else if (itaddr.s_addr == myaddr.s_addr) { @@ -1048,9 +1042,7 @@ struct llentry *lle = NULL; sin.sin_addr = itaddr; - NET_EPOCH_ENTER(et); lle = lla_lookup(LLTABLE(ifp), 0, (struct sockaddr *)&sin); - NET_EPOCH_EXIT(et); if ((lle != NULL) && (lle->la_flags & LLE_PUB)) { (void)memcpy(ar_tha(ah), ar_sha(ah), ah->ar_hln); @@ -1429,6 +1421,7 @@ void arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) { + struct epoch_tracker et; const struct sockaddr_in *dst_in; const struct sockaddr *dst; @@ -1440,7 +1433,9 @@ if (ntohl(dst_in->sin_addr.s_addr) == INADDR_ANY) return; + NET_EPOCH_ENTER(et); arp_announce_ifaddr(ifp, dst_in->sin_addr, IF_LLADDR(ifp)); + NET_EPOCH_EXIT(et); if (garp_rexmit_count > 0) { garp_timer_start(ifa); } Index: sys/netinet/igmp.c =================================================================== --- sys/netinet/igmp.c +++ sys/netinet/igmp.c @@ -487,8 +487,10 @@ static void igmp_dispatch_queue(struct mbufq *mq, int limit, const int loop) { + struct epoch_tracker et; struct mbuf *m; + NET_EPOCH_ENTER(et); while ((m = mbufq_dequeue(mq)) != NULL) { CTR3(KTR_IGMPV3, "%s: dispatch %p from %p", __func__, mq, m); if (loop) @@ -497,6 +499,7 @@ if (--limit == 0) break; } + NET_EPOCH_EXIT(et); } /* @@ -692,11 +695,12 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; + NET_EPOCH_ASSERT(); + /* * IGMPv1 Host Mmembership Queries SHOULD always be addressed to * 224.0.0.1. They are always treated as General Queries. @@ -734,7 +738,6 @@ * for the interface on which the query arrived, * except those which are already running. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -762,7 +765,6 @@ break; } } - NET_EPOCH_EXIT(et); out_locked: IGMP_UNLOCK(); @@ -778,13 +780,14 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip, const struct igmp *igmp) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct igmp_ifsoftc *igi; struct in_multi *inm; int is_general_query; uint16_t timer; + NET_EPOCH_ASSERT(); + is_general_query = 0; /* @@ -836,7 +839,6 @@ */ CTR2(KTR_IGMPV3, "process v2 general query on ifp %p(%s)", ifp, ifp->if_xname); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -844,7 +846,6 @@ inm = (struct in_multi *)ifma->ifma_protospec; igmp_v2_update_group(inm, timer); } - NET_EPOCH_EXIT(et); } else { /* * Group-specific IGMPv2 query, we need only @@ -1220,13 +1221,9 @@ * Replace 0.0.0.0 with the subnet address if told to do so. */ if (V_igmp_recvifkludge && in_nullhost(ip->ip_src)) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); - NET_EPOCH_EXIT(et); } CTR3(KTR_IGMPV3, "process v1 report 0x%08x on ifp %p(%s)", @@ -1311,7 +1308,6 @@ /*const*/ struct igmp *igmp) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct in_ifaddr *ia; struct in_multi *inm; @@ -1320,23 +1316,19 @@ * leave requires knowing that we are the only member of a * group. */ - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr)) { - NET_EPOCH_EXIT(et); return (0); } IGMPSTAT_INC(igps_rcv_reports); if (ifp->if_flags & IFF_LOOPBACK) { - NET_EPOCH_EXIT(et); return (0); } if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) || !in_hosteq(igmp->igmp_group, ip->ip_dst)) { - NET_EPOCH_EXIT(et); IGMPSTAT_INC(igps_rcv_badreports); return (EINVAL); } @@ -1352,7 +1344,6 @@ if (ia != NULL) ip->ip_src.s_addr = htonl(ia->ia_subnet); } - NET_EPOCH_EXIT(et); CTR3(KTR_IGMPV3, "process v2 report 0x%08x on ifp %p(%s)", ntohl(igmp->igmp_group.s_addr), ifp, ifp->if_xname); @@ -1994,13 +1985,14 @@ struct ifnet *ifp; struct in_multi *inm; struct in_multi_head inm_free_tmp; - struct epoch_tracker et; CTR3(KTR_IGMPV3, "%s: cancel v3 timers on ifp %p(%s)", __func__, igi->igi_ifp, igi->igi_ifp->if_xname); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); + SLIST_INIT(&inm_free_tmp); /* @@ -2015,7 +2007,6 @@ * for all memberships scoped to this link. */ ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -2060,7 +2051,6 @@ inm->inm_timer = 0; mbufq_drain(&inm->inm_scq); } - NET_EPOCH_EXIT(et); inm_release_list_deferred(&inm_free_tmp); } @@ -2189,6 +2179,7 @@ struct ip *ip; struct mbuf *m; + NET_EPOCH_ASSERT(); IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); @@ -3303,7 +3294,6 @@ static void igmp_v3_dispatch_general_query(struct igmp_ifsoftc *igi) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in_multi *inm; @@ -3311,6 +3301,7 @@ IN_MULTI_LIST_LOCK_ASSERT(); IGMP_LOCK_ASSERT(); + NET_EPOCH_ASSERT(); KASSERT(igi->igi_version == IGMP_VERSION_3, ("%s: called when version %d", __func__, igi->igi_version)); @@ -3326,7 +3317,6 @@ ifp = igi->igi_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -3357,7 +3347,6 @@ break; } } - NET_EPOCH_EXIT(et); send: loop = (igi->igi_flags & IGIF_LOOPBACK) ? 1 : 0; @@ -3529,14 +3518,11 @@ ip->ip_src.s_addr = INADDR_ANY; if (m->m_flags & M_IGMP_LOOP) { - struct epoch_tracker et; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) ip->ip_src = ia->ia_addr.sin_addr; - NET_EPOCH_EXIT(et); } ip->ip_dst.s_addr = htonl(INADDR_ALLRPTS_GROUP); Index: sys/netinet/in.c =================================================================== --- sys/netinet/in.c +++ sys/netinet/in.c @@ -139,21 +139,18 @@ int in_ifhasaddr(struct ifnet *ifp, struct in_addr in) { - struct epoch_tracker et; struct ifaddr *ifa; struct in_ifaddr *ia; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = (struct in_ifaddr *)ifa; - if (ia->ia_addr.sin_addr.s_addr == in.s_addr) { - NET_EPOCH_EXIT(et); + if (ia->ia_addr.sin_addr.s_addr == in.s_addr) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -970,10 +967,11 @@ int in_broadcast(struct in_addr in, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa; int found; + NET_EPOCH_ASSERT(); + if (in.s_addr == INADDR_BROADCAST || in.s_addr == INADDR_ANY) return (1); @@ -984,14 +982,12 @@ * Look through the list of addresses for a match * with a broadcast address. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (ifa->ifa_addr->sa_family == AF_INET && in_ifaddr_broadcast(in, (struct in_ifaddr *)ifa)) { found = 1; break; } - NET_EPOCH_EXIT(et); return (found); } Index: sys/netinet/in_mcast.c =================================================================== --- sys/netinet/in_mcast.c +++ sys/netinet/in_mcast.c @@ -339,13 +339,13 @@ * The IN_MULTI_LIST_LOCK and IF_ADDR_LOCK on ifp must be held. */ struct in_multi * -inm_lookup_locked(struct ifnet *ifp, const struct in_addr ina) +inm_lookup(struct ifnet *ifp, const struct in_addr ina) { struct ifmultiaddr *ifma; struct in_multi *inm; IN_MULTI_LIST_LOCK_ASSERT(); - IF_ADDR_LOCK_ASSERT(ifp); + NET_EPOCH_ASSERT(); inm = NULL; CK_STAILQ_FOREACH(ifma, &((ifp)->if_multiaddrs), ifma_link) { @@ -360,24 +360,6 @@ return (inm); } -/* - * Wrapper for inm_lookup_locked(). - * The IF_ADDR_LOCK will be taken on ifp and released on return. - */ -struct in_multi * -inm_lookup(struct ifnet *ifp, const struct in_addr ina) -{ - struct epoch_tracker et; - struct in_multi *inm; - - IN_MULTI_LIST_LOCK_ASSERT(); - NET_EPOCH_ENTER(et); - inm = inm_lookup_locked(ifp, ina); - NET_EPOCH_EXIT(et); - - return (inm); -} - /* * Resize the ip_moptions vector to the next power-of-two minus 1. * May be called with locks held; do not sleep. @@ -1246,11 +1228,14 @@ in_joingroup(struct ifnet *ifp, const struct in_addr *gina, /*const*/ struct in_mfilter *imf, struct in_multi **pinm) { + struct epoch_tracker et; int error; + NET_EPOCH_ENTER(et); IN_MULTI_LOCK(); error = in_joingroup_locked(ifp, gina, imf, pinm); IN_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (error); } @@ -1272,6 +1257,7 @@ struct in_multi *inm; int error; + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); @@ -1329,11 +1315,14 @@ int in_leavegroup(struct in_multi *inm, /*const*/ struct in_mfilter *imf) { + struct epoch_tracker et; int error; + NET_EPOCH_ENTER(et); IN_MULTI_LOCK(); error = in_leavegroup_locked(inm, imf); IN_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (error); } @@ -1357,11 +1346,12 @@ struct in_mfilter timf; int error; - error = 0; - + NET_EPOCH_ASSERT(); IN_MULTI_LOCK_ASSERT(); IN_MULTI_LIST_UNLOCK_ASSERT(); + error = 0; + CTR5(KTR_IGMPV3, "%s: leave inm %p, 0x%08x/%s, imf %p", __func__, inm, ntohl(inm->inm_addr.s_addr), (inm_is_ifp_detached(inm) ? "null" : inm->inm_ifp->if_xname), @@ -1888,15 +1878,11 @@ if (!in_nullhost(imo->imo_multicast_addr)) { mreqn.imr_address = imo->imo_multicast_addr; } else if (ifp != NULL) { - struct epoch_tracker et; - mreqn.imr_ifindex = ifp->if_index; - NET_EPOCH_ENTER(et); IFP_TO_IA(ifp, ia, &in_ifa_tracker); if (ia != NULL) mreqn.imr_address = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); } } INP_WUNLOCK(inp); @@ -3011,8 +2997,10 @@ return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_IGMPV3, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -3020,12 +3008,13 @@ retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in_mcast_maxgrpsrc * sizeof(struct in_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_INET || ifma->ifma_protospec == NULL) @@ -3054,9 +3043,9 @@ break; } } - NET_EPOCH_EXIT(et); IN_MULTI_LIST_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } Index: sys/netinet/in_rmx.c =================================================================== --- sys/netinet/in_rmx.c +++ sys/netinet/in_rmx.c @@ -80,12 +80,17 @@ * dubious since it's so easy to inspect the address). */ if (rt->rt_flags & RTF_HOST) { - if (in_broadcast(sin->sin_addr, rt->rt_ifp)) { + struct epoch_tracker et; + bool bcast; + + NET_EPOCH_ENTER(et); + bcast = in_broadcast(sin->sin_addr, rt->rt_ifp); + NET_EPOCH_EXIT(et); + if (bcast) rt->rt_flags |= RTF_BROADCAST; - } else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == - sin->sin_addr.s_addr) { + else if (satosin(rt->rt_ifa->ifa_addr)->sin_addr.s_addr == + sin->sin_addr.s_addr) rt->rt_flags |= RTF_LOCAL; - } } if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) rt->rt_flags |= RTF_MULTICAST; Index: sys/netinet/in_var.h =================================================================== --- sys/netinet/in_var.h +++ sys/netinet/in_var.h @@ -171,6 +171,7 @@ /* struct in_ifaddr *ia; */ \ /* struct rm_priotracker *t; */ \ do { \ + NET_EPOCH_ASSERT(); \ IN_IFADDR_RLOCK((t)); \ for ((ia) = CK_STAILQ_FIRST(&V_in_ifaddrhead); \ (ia) != NULL && (ia)->ia_ifp != (ifp); \ @@ -387,7 +388,6 @@ struct route; struct ip_moptions; -struct in_multi *inm_lookup_locked(struct ifnet *, const struct in_addr); struct in_multi *inm_lookup(struct ifnet *, const struct in_addr); int imo_multi_filter(const struct ip_moptions *, const struct ifnet *, const struct sockaddr *, const struct sockaddr *); Index: sys/netinet/ip_carp.c =================================================================== --- sys/netinet/ip_carp.c +++ sys/netinet/ip_carp.c @@ -644,9 +644,10 @@ struct carp_softc *sc; uint64_t tmp_counter; struct timeval sc_tv, ch_tv; - struct epoch_tracker et; int error; + NET_EPOCH_ASSERT(); + /* * Verify that the VHID is valid on the receiving interface. * @@ -658,7 +659,6 @@ * (these should never happen, and as noted above, we may * miss real loops; this is just a double-check). */ - NET_EPOCH_ENTER(et); error = 0; match = NULL; IFNET_FOREACH_IFA(ifp, ifa) { @@ -672,7 +672,6 @@ ifa = error ? NULL : match; if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); if (ifa == NULL) { if (error == ELOOP) { @@ -880,19 +879,18 @@ static struct ifaddr * carp_best_ifa(int af, struct ifnet *ifp) { - struct epoch_tracker et; struct ifaddr *ifa, *best; + NET_EPOCH_ASSERT(); + if (af >= AF_MAX) return (NULL); best = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family == af && (best == NULL || ifa_preferred(best, ifa))) best = ifa; } - NET_EPOCH_EXIT(et); if (best != NULL) ifa_ref(best); return (best); @@ -1169,11 +1167,11 @@ struct ifaddr * carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + ifa = NULL; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1185,7 +1183,6 @@ ifa_ref(ifa); break; } - NET_EPOCH_EXIT(et); return (ifa); } @@ -1193,18 +1190,16 @@ caddr_t carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + IFNET_FOREACH_IFA(ifp, ifa) if (ifa->ifa_addr->sa_family == AF_INET6 && IN6_ARE_ADDR_EQUAL(taddr, IFA_IN6(ifa))) { struct carp_softc *sc = ifa->ifa_carp; struct m_tag *mtag; - NET_EPOCH_EXIT(et); - mtag = m_tag_get(PACKET_TAG_CARP, sizeof(struct carp_softc *), M_NOWAIT); if (mtag == NULL) @@ -1216,7 +1211,6 @@ return (LLADDR(&sc->sc_addr)); } - NET_EPOCH_EXIT(et); return (NULL); } Index: sys/netinet/ip_encap.c =================================================================== --- sys/netinet/ip_encap.c +++ sys/netinet/ip_encap.c @@ -125,11 +125,9 @@ #define ENCAP_WLOCK() mtx_lock(&encapmtx) #define ENCAP_WUNLOCK() mtx_unlock(&encapmtx) #define ENCAP_RLOCK_TRACKER struct epoch_tracker encap_et -#define ENCAP_RLOCK() \ - epoch_enter_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_RUNLOCK() \ - epoch_exit_preempt(net_epoch_preempt, &encap_et) -#define ENCAP_WAIT() epoch_wait_preempt(net_epoch_preempt) +#define ENCAP_RLOCK() NET_EPOCH_ENTER(encap_et) +#define ENCAP_RUNLOCK() NET_EPOCH_EXIT(encap_et) +#define ENCAP_WAIT() NET_EPOCH_WAIT() #define SRCADDR_WLOCK() mtx_lock(&srcaddrmtx) #define SRCADDR_WUNLOCK() mtx_unlock(&srcaddrmtx) Index: sys/netinet/ip_icmp.c =================================================================== --- sys/netinet/ip_icmp.c +++ sys/netinet/ip_icmp.c @@ -393,7 +393,6 @@ int icmp_input(struct mbuf **mp, int *offp, int proto) { - struct epoch_tracker et; struct icmp *icp; struct in_ifaddr *ia; struct mbuf *m = *mp; @@ -405,6 +404,8 @@ void (*ctlfunc)(int, struct sockaddr *, void *); int fibnum; + NET_EPOCH_ASSERT(); + *mp = NULL; /* @@ -421,7 +422,6 @@ inet_ntoa_r(ip->ip_dst, dstbuf), icmplen); } #endif - NET_EPOCH_ENTER(et); if (icmplen < ICMP_MINLEN) { ICMPSTAT_INC(icps_tooshort); goto freeit; @@ -429,7 +429,6 @@ i = hlen + min(icmplen, ICMP_ADVLENMIN); if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -547,7 +546,6 @@ if (m->m_len < i && (m = m_pullup(m, i)) == NULL) { /* This should actually not happen */ ICMPSTAT_INC(icps_tooshort); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); } ip = mtod(m, struct ip *); @@ -640,7 +638,6 @@ ICMPSTAT_INC(icps_reflect); ICMPSTAT_INC(icps_outhist[icp->icmp_type]); icmp_reflect(m); - NET_EPOCH_EXIT(et); return (IPPROTO_DONE); case ICMP_REDIRECT: @@ -717,13 +714,11 @@ } raw: - NET_EPOCH_EXIT(et); *mp = m; rip_input(mp, offp, proto); return (IPPROTO_DONE); freeit: - NET_EPOCH_EXIT(et); m_freem(m); return (IPPROTO_DONE); } @@ -735,7 +730,6 @@ icmp_reflect(struct mbuf *m) { struct rm_priotracker in_ifa_tracker; - struct epoch_tracker et; struct ip *ip = mtod(m, struct ip *); struct ifaddr *ifa; struct ifnet *ifp; @@ -745,6 +739,8 @@ struct mbuf *opts = NULL; int optlen = (ip->ip_hl << 2) - sizeof(struct ip); + NET_EPOCH_ASSERT(); + if (IN_MULTICAST(ntohl(ip->ip_src.s_addr)) || IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) || IN_ZERONET(ntohl(ip->ip_src.s_addr)) ) { @@ -779,7 +775,6 @@ */ ifp = m->m_pkthdr.rcvif; if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -787,11 +782,9 @@ if (satosin(&ia->ia_broadaddr)->sin_addr.s_addr == t.s_addr) { t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of @@ -800,16 +793,13 @@ * criteria apply. */ if (V_icmp_rfi && ifp != NULL) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the incoming packet was not addressed directly to us, use @@ -818,16 +808,13 @@ * with normal source selection. */ if (V_reply_src[0] != '\0' && (ifp = ifunit(V_reply_src))) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; ia = ifatoia(ifa); t = IA_SIN(ia)->sin_addr; - NET_EPOCH_EXIT(et); goto match; } - NET_EPOCH_EXIT(et); } /* * If the packet was transiting through us, use the address of Index: sys/netinet/ip_input.c =================================================================== --- sys/netinet/ip_input.c +++ sys/netinet/ip_input.c @@ -457,6 +457,7 @@ struct in_addr odst; /* original dst address */ M_ASSERTPKTHDR(m); + NET_EPOCH_ASSERT(); if (m->m_flags & M_FASTFWD_OURS) { m->m_flags &= ~M_FASTFWD_OURS; @@ -708,9 +709,6 @@ * into the stack for SIMPLEX interfaces handled by ether_output(). */ if (ifp != NULL && ifp->if_flags & IFF_BROADCAST) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET) continue; @@ -720,7 +718,6 @@ counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #ifdef BOOTP_COMPAT @@ -728,12 +725,10 @@ counter_u64_add(ia->ia_ifa.ifa_ipackets, 1); counter_u64_add(ia->ia_ifa.ifa_ibytes, m->m_pkthdr.len); - NET_EPOCH_EXIT(et); goto ours; } #endif } - NET_EPOCH_EXIT(et); ia = NULL; } /* RFC 3927 2.7: Do not forward datagrams for 169.254.0.0/16. */ @@ -953,9 +948,10 @@ struct sockaddr_in *sin; struct in_addr dest; struct route ro; - struct epoch_tracker et; int error, type = 0, code = 0, mtu = 0; + NET_EPOCH_ASSERT(); + if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(ip->ip_dst) == 0) { IPSTAT_INC(ips_cantforward); m_freem(m); @@ -982,7 +978,6 @@ #else in_rtalloc_ign(&ro, 0, M_GETFIB(m)); #endif - NET_EPOCH_ENTER(et); if (ro.ro_rt != NULL) { ia = ifatoia(ro.ro_rt->rt_ifa); } else @@ -1030,7 +1025,7 @@ m_freem(mcopy); if (error != EINPROGRESS) IPSTAT_INC(ips_cantforward); - goto out; + return; } /* No IPsec processing required */ } @@ -1083,11 +1078,11 @@ else { if (mcopy) m_freem(mcopy); - goto out; + return; } } if (mcopy == NULL) - goto out; + return; switch (error) { @@ -1130,11 +1125,9 @@ case ENOBUFS: case EACCES: /* ipfw denied packet */ m_freem(mcopy); - goto out; + return; } icmp_error(mcopy, type, code, dest.s_addr, mtu); - out: - NET_EPOCH_EXIT(et); } #define CHECK_SO_CT(sp, ct) \ Index: sys/netinet/ip_mroute.c =================================================================== --- sys/netinet/ip_mroute.c +++ sys/netinet/ip_mroute.c @@ -874,18 +874,13 @@ */ ifp = NULL; } else { - struct epoch_tracker et; - sin.sin_addr = vifcp->vifc_lcl_addr; - NET_EPOCH_ENTER(et); ifa = ifa_ifwithaddr((struct sockaddr *)&sin); if (ifa == NULL) { - NET_EPOCH_EXIT(et); VIF_UNLOCK(); return EADDRNOTAVAIL; } ifp = ifa->ifa_ifp; - NET_EPOCH_EXIT(et); } if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) { Index: sys/netinet/ip_options.c =================================================================== --- sys/netinet/ip_options.c +++ sys/netinet/ip_options.c @@ -109,7 +109,8 @@ uint32_t ntime; struct nhop4_extended nh_ext; struct sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET }; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); /* Ignore or reject packets with IP options. */ if (V_ip_doopts == 0) @@ -117,10 +118,9 @@ else if (V_ip_doopts == 2) { type = ICMP_UNREACH; code = ICMP_UNREACH_FILTER_PROHIB; - goto bad_unlocked; + goto bad; } - NET_EPOCH_ENTER(et); dst = ip->ip_dst; cp = (u_char *)(ip + 1); cnt = (ip->ip_hl << 2) - sizeof (struct ip); @@ -226,7 +226,6 @@ #endif IPSTAT_INC(ips_cantforward); m_freem(m); - NET_EPOCH_EXIT(et); return (1); } } @@ -381,15 +380,12 @@ cp[IPOPT_OFFSET] += sizeof(uint32_t); } } - NET_EPOCH_EXIT(et); if (forward && V_ipforwarding) { ip_forward(m, 1); return (1); } return (0); bad: - NET_EPOCH_EXIT(et); -bad_unlocked: icmp_error(m, type, code, 0, 0); IPSTAT_INC(ips_badoptions); return (1); Index: sys/netinet/ip_output.c =================================================================== --- sys/netinet/ip_output.c +++ sys/netinet/ip_output.c @@ -968,7 +968,8 @@ int ip_ctloutput(struct socket *so, struct sockopt *sopt) { - struct inpcb *inp = sotoinpcb(so); + struct inpcb *inp = sotoinpcb(so); + struct epoch_tracker et; int error, optval; #ifdef RSS uint32_t rss_bucket; @@ -1407,7 +1408,9 @@ case IP_MULTICAST_TTL: case IP_MULTICAST_LOOP: case IP_MSFILTER: + NET_EPOCH_ENTER(et); error = inp_getmoptions(inp, sopt); + NET_EPOCH_EXIT(et); break; #if defined(IPSEC) || defined(IPSEC_SUPPORT) Index: sys/netinet/sctp_bsd_addr.c =================================================================== --- sys/netinet/sctp_bsd_addr.c +++ sys/netinet/sctp_bsd_addr.c @@ -198,6 +198,7 @@ * make sure we lock any IFA that exists as we float through the * list of IFA's */ + struct epoch_tracker et; struct ifnet *ifn; struct ifaddr *ifa; struct sctp_ifa *sctp_ifa; @@ -206,15 +207,13 @@ struct in6_ifaddr *ifa6; #endif + NET_EPOCH_ENTER(et); IFNET_RLOCK(); CK_STAILQ_FOREACH(ifn, &MODULE_GLOBAL(ifnet), if_link) { - struct epoch_tracker et; - if (sctp_is_desired_interface_type(ifn) == 0) { /* non desired type */ continue; } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifn->if_addrhead, ifa_link) { if (ifa->ifa_addr == NULL) { continue; @@ -267,9 +266,9 @@ sctp_ifa->localifa_flags &= ~SCTP_ADDR_DEFER_USE; } } - NET_EPOCH_EXIT(et); } IFNET_RUNLOCK(); + NET_EPOCH_EXIT(et); } void Index: sys/netinet6/icmp6.c =================================================================== --- sys/netinet6/icmp6.c +++ sys/netinet6/icmp6.c @@ -1655,13 +1655,14 @@ ni6_addrs(struct icmp6_nodeinfo *ni6, struct mbuf *m, struct ifnet **ifpp, struct in6_addr *subj) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; int addrs = 0, addrsofif, iffound = 0; int niflags = ni6->ni_flags; + NET_EPOCH_ASSERT(); + if ((niflags & NI_NODEADDR_FLAG_ALL) == 0) { switch (ni6->ni_code) { case ICMP6_NI_SUBJ_IPV6: @@ -1677,7 +1678,6 @@ } } - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { addrsofif = 0; CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { @@ -1732,13 +1732,11 @@ } if (iffound) { *ifpp = ifp; - NET_EPOCH_EXIT(et); return (addrsofif); } addrs += addrsofif; } - NET_EPOCH_EXIT(et); return (addrs); } @@ -1747,7 +1745,6 @@ ni6_store_addrs(struct icmp6_nodeinfo *ni6, struct icmp6_nodeinfo *nni6, struct ifnet *ifp0, int resid) { - struct epoch_tracker et; struct ifnet *ifp; struct in6_ifaddr *ifa6; struct ifaddr *ifa; @@ -1757,10 +1754,11 @@ int niflags = ni6->ni_flags; u_int32_t ltime; + NET_EPOCH_ASSERT(); + if (ifp0 == NULL && !(niflags & NI_NODEADDR_FLAG_ALL)) return (0); /* needless to copy */ - NET_EPOCH_ENTER(et); ifp = ifp0 ? ifp0 : CK_STAILQ_FIRST(&V_ifnet); again: @@ -1824,7 +1822,6 @@ * Set the truncate flag and return. */ nni6->ni_flags |= NI_NODEADDR_FLAG_TRUNCATE; - NET_EPOCH_EXIT(et); return (copied); } @@ -1876,8 +1873,6 @@ goto again; } - NET_EPOCH_EXIT(et); - return (copied); } @@ -2558,14 +2553,11 @@ { /* target lladdr option */ - struct epoch_tracker et; int len; struct nd_opt_hdr *nd_opt; char *lladdr; - NET_EPOCH_ENTER(et); ln = nd6_lookup(router_ll6, 0, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) goto nolladdropt; Index: sys/netinet6/in6.c =================================================================== --- sys/netinet6/in6.c +++ sys/netinet6/in6.c @@ -1474,10 +1474,10 @@ struct in6_ifaddr * in6ifa_ifpforlinklocal(struct ifnet *ifp, int ignoreflags) { - struct epoch_tracker et; struct ifaddr *ifa; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1489,7 +1489,6 @@ break; } } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1549,13 +1548,13 @@ struct in6_ifaddr * in6ifa_llaonifp(struct ifnet *ifp) { - struct epoch_tracker et; struct sockaddr_in6 *sin6; struct ifaddr *ifa; + NET_EPOCH_ASSERT(); + if (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) return (NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1565,7 +1564,6 @@ IN6_IS_ADDR_MC_NODELOCAL(&sin6->sin6_addr)) break; } - NET_EPOCH_EXIT(et); return ((struct in6_ifaddr *)ifa); } @@ -1702,26 +1700,23 @@ in6_ifhasaddr(struct ifnet *ifp, struct in6_addr *addr) { struct in6_addr in6; - struct epoch_tracker et; struct ifaddr *ifa; struct in6_ifaddr *ia6; + NET_EPOCH_ASSERT(); + in6 = *addr; if (in6_clearscope(&in6)) return (0); in6_setscope(&in6, ifp, NULL); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; ia6 = (struct in6_ifaddr *)ifa; - if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) { - NET_EPOCH_EXIT(et); + if (IN6_ARE_ADDR_EQUAL(&ia6->ia_addr.sin6_addr, &in6)) return (1); - } } - NET_EPOCH_EXIT(et); return (0); } @@ -1825,12 +1820,13 @@ struct in6_ifaddr * in6_ifawithifp(struct ifnet *ifp, struct in6_addr *dst) { - struct epoch_tracker et; int dst_scope = in6_addrscope(dst), blen = -1, tlen; struct ifaddr *ifa; struct in6_ifaddr *besta = NULL; struct in6_ifaddr *dep[2]; /* last-resort: deprecated */ + NET_EPOCH_ASSERT(); + dep[0] = dep[1] = NULL; /* @@ -1839,7 +1835,6 @@ * If two or more, return one which matches the dst longest. * If none, return one of global addresses assigned other ifs. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1873,7 +1868,6 @@ } if (besta) { ifa_ref(&besta->ia_ifa); - NET_EPOCH_EXIT(et); return (besta); } @@ -1894,23 +1888,19 @@ if (ifa != NULL) ifa_ref(ifa); - NET_EPOCH_EXIT(et); return (struct in6_ifaddr *)ifa; } /* use the last-resort values, that are, deprecated addresses */ if (dep[0]) { ifa_ref((struct ifaddr *)dep[0]); - NET_EPOCH_EXIT(et); return dep[0]; } if (dep[1]) { ifa_ref((struct ifaddr *)dep[1]); - NET_EPOCH_EXIT(et); return dep[1]; } - NET_EPOCH_EXIT(et); return NULL; } @@ -1940,12 +1930,12 @@ arc4random() % (MAX_RTR_SOLICITATION_DELAY * hz)); } } - NET_EPOCH_EXIT(et); /* * special cases, like 6to4, are handled in in6_ifattach */ in6_ifattach(ifp, NULL); + NET_EPOCH_EXIT(et); } int @@ -2164,6 +2154,7 @@ char ip6buf[INET6_ADDRSTRLEN]; int fibnum; + NET_EPOCH_ASSERT(); KASSERT(l3addr->sa_family == AF_INET6, ("sin_family %d", l3addr->sa_family)); @@ -2172,19 +2163,15 @@ fibnum = V_rt_add_addr_allfibs ? RT_DEFAULT_FIB : ifp->if_fib; error = fib6_lookup_nh_basic(fibnum, &dst, scopeid, 0, 0, &nh6); if (error != 0 || (nh6.nh_flags & NHF_GATEWAY) || nh6.nh_ifp != ifp) { - struct epoch_tracker et; struct ifaddr *ifa; /* * Create an ND6 cache for an IPv6 neighbor * that is not covered by our own prefix. */ - NET_EPOCH_ENTER(et); ifa = ifaof_ifpforaddr(l3addr, ifp); if (ifa != NULL) { - NET_EPOCH_EXIT(et); return 0; } - NET_EPOCH_EXIT(et); log(LOG_INFO, "IPv6 address: \"%s\" is not on the network\n", ip6_sprintf(ip6buf, &sin6->sin6_addr)); return EINVAL; Index: sys/netinet6/in6_ifattach.c =================================================================== --- sys/netinet6/in6_ifattach.c +++ sys/netinet6/in6_ifattach.c @@ -240,7 +240,6 @@ int in6_get_hw_ifid(struct ifnet *ifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifaddr *ifa; struct sockaddr_dl *sdl; u_int8_t *addr; @@ -249,7 +248,8 @@ static u_int8_t allone[8] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_LINK) continue; @@ -261,12 +261,10 @@ goto found; } - NET_EPOCH_EXIT(et); return -1; found: - IF_ADDR_LOCK_ASSERT(ifp); addr = LLADDR(sdl); addrlen = sdl->sdl_alen; @@ -283,24 +281,18 @@ addrlen = 8; /* look at IEEE802/EUI64 only */ - if (addrlen != 8 && addrlen != 6) { - NET_EPOCH_EXIT(et); + if (addrlen != 8 && addrlen != 6) return -1; - } /* * check for invalid MAC address - on bsdi, we see it a lot * since wildboar configures all-zero MAC on pccard before * card insertion. */ - if (bcmp(addr, allzero, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allzero, addrlen) == 0) return -1; - } - if (bcmp(addr, allone, addrlen) == 0) { - NET_EPOCH_EXIT(et); + if (bcmp(addr, allone, addrlen) == 0) return -1; - } /* make EUI64 address */ if (addrlen == 8) @@ -325,19 +317,15 @@ * identifier source (can be renumbered). * we don't do this. */ - NET_EPOCH_EXIT(et); return -1; default: - NET_EPOCH_EXIT(et); return -1; } /* sanity check: g bit must not indicate "group" */ - if (EUI64_GROUP(in6)) { - NET_EPOCH_EXIT(et); + if (EUI64_GROUP(in6)) return -1; - } /* convert EUI64 into IPv6 interface identifier */ EUI64_TO_IFID(in6); @@ -347,12 +335,9 @@ * subnet router anycast */ if ((in6->s6_addr[8] & ~(EUI64_GBIT | EUI64_UBIT)) == 0x00 && - bcmp(&in6->s6_addr[9], allzero, 7) == 0) { - NET_EPOCH_EXIT(et); + bcmp(&in6->s6_addr[9], allzero, 7) == 0) return -1; - } - NET_EPOCH_EXIT(et); return 0; } @@ -367,9 +352,10 @@ get_ifid(struct ifnet *ifp0, struct ifnet *altifp, struct in6_addr *in6) { - struct epoch_tracker et; struct ifnet *ifp; + NET_EPOCH_ASSERT(); + /* first, try to get it from the interface itself */ if (in6_get_hw_ifid(ifp0, in6) == 0) { nd6log((LOG_DEBUG, "%s: got interface identifier from itself\n", @@ -385,7 +371,6 @@ } /* next, try to get it from some other hardware interface */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { if (ifp == ifp0) continue; @@ -400,11 +385,9 @@ nd6log((LOG_DEBUG, "%s: borrow interface identifier from %s\n", if_name(ifp0), if_name(ifp))); - NET_EPOCH_EXIT(et); goto success; } } - NET_EPOCH_EXIT(et); /* last resort: get from random number source */ if (get_rand_ifid(ifp, in6) == 0) { @@ -678,6 +661,8 @@ { struct in6_ifaddr *ia; + NET_EPOCH_ASSERT(); + if (ifp->if_afdata[AF_INET6] == NULL) return; /* Index: sys/netinet6/in6_mcast.c =================================================================== --- sys/netinet6/in6_mcast.c +++ sys/netinet6/in6_mcast.c @@ -1270,7 +1270,6 @@ KASSERT(mcaddr->s6_addr16[1] != 0, ("%s: scope zone ID not set", __func__)); } - IN6_MULTI_LOCK_ASSERT(); IN6_MULTI_LIST_UNLOCK_ASSERT(); @@ -2857,8 +2856,10 @@ return (EINVAL); } + NET_EPOCH_ENTER(et); ifp = ifnet_byindex(ifindex); if (ifp == NULL) { + NET_EPOCH_EXIT(et); CTR2(KTR_MLD, "%s: no ifp for ifindex %u", __func__, ifindex); return (ENOENT); @@ -2870,12 +2871,13 @@ retval = sysctl_wire_old_buffer(req, sizeof(uint32_t) + (in6_mcast_maxgrpsrc * sizeof(struct in6_addr))); - if (retval) + if (retval) { + NET_EPOCH_EXIT(et); return (retval); + } IN6_MULTI_LOCK(); IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -2903,10 +2905,9 @@ break; } } - NET_EPOCH_EXIT(et); - IN6_MULTI_LIST_UNLOCK(); IN6_MULTI_UNLOCK(); + NET_EPOCH_EXIT(et); return (retval); } Index: sys/netinet6/in6_var.h =================================================================== --- sys/netinet6/in6_var.h +++ sys/netinet6/in6_var.h @@ -733,6 +733,8 @@ * Look up an in6_multi record for an IPv6 multicast address * on the interface ifp. * If no record found, return NULL. + * + * SMPng: The IN6_MULTI_LOCK and must be held and must be in network epoch. */ static __inline struct in6_multi * in6m_lookup_locked(struct ifnet *ifp, const struct in6_addr *mcaddr) @@ -753,18 +755,17 @@ /* * Wrapper for in6m_lookup_locked(). * - * SMPng: Assumes that neithr the IN6_MULTI_LOCK() or IF_ADDR_LOCK() are held. + * SMPng: Assumes network epoch entered and that IN6_MULTI_LOCK() isn't held. */ static __inline struct in6_multi * in6m_lookup(struct ifnet *ifp, const struct in6_addr *mcaddr) { - struct epoch_tracker et; struct in6_multi *inm; + NET_EPOCH_ASSERT(); + IN6_MULTI_LIST_LOCK(); - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, mcaddr); - NET_EPOCH_EXIT(et); IN6_MULTI_LIST_UNLOCK(); return (inm); Index: sys/netinet6/ip6_output.c =================================================================== --- sys/netinet6/ip6_output.c +++ sys/netinet6/ip6_output.c @@ -294,6 +294,7 @@ struct mbuf *m = m0; struct mbuf *mprev = NULL; int hlen, tlen, len; + struct epoch_tracker et; struct route_in6 ip6route; struct rtentry *rt = NULL; struct sockaddr_in6 *dst, src_sa, dst_sa; @@ -493,6 +494,7 @@ ro = &opt->ip6po_route; dst = (struct sockaddr_in6 *)&ro->ro_dst; fibnum = (inp != NULL) ? inp->inp_inc.inc_fibnum : M_GETFIB(m); + NET_EPOCH_ENTER(et); again: /* * if specified, try to fill in the traffic class field. @@ -1094,6 +1096,7 @@ IP6STAT_INC(ip6s_fragmented); done: + NET_EPOCH_EXIT(et); if (ro == &ip6route) RO_RTFREE(ro); return (error); Index: sys/netinet6/mld6.c =================================================================== --- sys/netinet6/mld6.c +++ sys/netinet6/mld6.c @@ -635,7 +635,6 @@ mld_v1_input_query(struct ifnet *ifp, const struct ip6_hdr *ip6, /*const*/ struct mld_hdr *mld) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct mld_ifsoftc *mli; struct in6_multi *inm; @@ -645,6 +644,8 @@ char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + is_general_query = 0; if (!mld_v1enable) { @@ -703,7 +704,6 @@ if (timer == 0) timer = 1; - NET_EPOCH_ENTER(et); if (is_general_query) { /* * For each reporting group joined on this @@ -734,7 +734,6 @@ in6_clearscope(&mld->mld_addr); } - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -823,6 +822,8 @@ char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v2enable) { CTR3(KTR_MLD, "ignore v2 query src %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &ip6->ip6_src), @@ -938,8 +939,6 @@ V_interface_timers_running6 = 1; } } else { - struct epoch_tracker et; - /* * MLDv2 Group-specific or Group-and-source-specific Query. * @@ -948,18 +947,14 @@ * Queries for groups we are not a member of on this * link are simply ignored. */ - NET_EPOCH_ENTER(et); inm = in6m_lookup_locked(ifp, &mld->mld_addr); - if (inm == NULL) { - NET_EPOCH_EXIT(et); + if (inm == NULL) goto out_locked; - } if (nsrc > 0) { if (!ratecheck(&inm->in6m_lastgsrtv, &V_mld_gsrdelay)) { CTR1(KTR_MLD, "%s: GS query throttled.", __func__); - NET_EPOCH_EXIT(et); goto out_locked; } } @@ -977,7 +972,6 @@ /* XXX Clear embedded scope ID as userland won't expect it. */ in6_clearscope(&mld->mld_addr); - NET_EPOCH_EXIT(et); } out_locked: @@ -1112,13 +1106,14 @@ /*const*/ struct mld_hdr *mld) { struct in6_addr src, dst; - struct epoch_tracker et; struct in6_ifaddr *ia; struct in6_multi *inm; #ifdef KTR char ip6tbuf[INET6_ADDRSTRLEN]; #endif + NET_EPOCH_ASSERT(); + if (!mld_v1enable) { CTR3(KTR_MLD, "ignore v1 report %s on ifp %p(%s)", ip6_sprintf(ip6tbuf, &mld->mld_addr), @@ -1188,7 +1183,6 @@ IN6_MULTI_LIST_LOCK(); MLD_LOCK(); - NET_EPOCH_ENTER(et); /* * MLDv1 report suppression. @@ -1236,7 +1230,6 @@ } out_locked: - NET_EPOCH_EXIT(et); MLD_UNLOCK(); IN6_MULTI_LIST_UNLOCK(); @@ -2995,12 +2988,12 @@ static void mld_v2_dispatch_general_query(struct mld_ifsoftc *mli) { - struct epoch_tracker et; struct ifmultiaddr *ifma; struct ifnet *ifp; struct in6_multi *inm; int retval; + NET_EPOCH_ASSERT(); IN6_MULTI_LIST_LOCK_ASSERT(); MLD_LOCK_ASSERT(); @@ -3018,7 +3011,6 @@ ifp = mli->mli_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { inm = in6m_ifmultiaddr_get_inm(ifma); if (inm == NULL) @@ -3047,7 +3039,6 @@ break; } } - NET_EPOCH_EXIT(et); send: mld_dispatch_queue(&mli->mli_gq, MLD_MAX_RESPONSE_BURST); Index: sys/netinet6/nd6.c =================================================================== --- sys/netinet6/nd6.c +++ sys/netinet6/nd6.c @@ -731,6 +731,7 @@ static __noinline void nd6_llinfo_timer(void *arg) { + struct epoch_tracker et; struct llentry *ln; struct in6_addr *dst, *pdst, *psrc, src; struct ifnet *ifp; @@ -767,6 +768,7 @@ CURVNET_RESTORE(); return; } + NET_EPOCH_ENTER(et); ndi = ND_IFINFO(ifp); send_ns = 0; dst = &ln->r_l3addr.addr6; @@ -882,9 +884,9 @@ ln = NULL; nd6_ns_output(ifp, psrc, pdst, dst, NULL); } - if (ln != NULL) LLE_FREE_LOCKED(ln); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } @@ -896,6 +898,7 @@ nd6_timer(void *arg) { CURVNET_SET((struct vnet *) arg); + struct epoch_tracker et; struct nd_drhead drq; struct nd_prhead prl; struct nd_defrouter *dr, *ndr; @@ -925,6 +928,7 @@ * * XXXRW: in6_ifaddrhead locking. */ + NET_EPOCH_ENTER(et); addrloop: CK_STAILQ_FOREACH_SAFE(ia6, &V_in6_ifaddrhead, ia_link, nia6) { /* check address lifetime */ @@ -1011,6 +1015,7 @@ ia6->ia6_flags &= ~IN6_IFF_DEPRECATED; } } + NET_EPOCH_EXIT(et); ND6_WLOCK(); restart: @@ -1066,13 +1071,13 @@ static int regen_tmpaddr(struct in6_ifaddr *ia6) { - struct epoch_tracker et; struct ifaddr *ifa; struct ifnet *ifp; struct in6_ifaddr *public_ifa6 = NULL; + NET_EPOCH_ASSERT(); + ifp = ia6->ia_ifa.ifa_ifp; - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *it6; @@ -1113,7 +1118,6 @@ } if (public_ifa6 != NULL) ifa_ref(&public_ifa6->ia_ifa); - NET_EPOCH_EXIT(et); if (public_ifa6 != NULL) { int e; @@ -1384,10 +1388,10 @@ int nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp) { - struct epoch_tracker et; struct llentry *lle; int rc = 0; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); if (nd6_is_new_addr_neighbor(addr, ifp)) return (1); @@ -1396,12 +1400,10 @@ * Even if the address matches none of our addresses, it might be * in the neighbor cache. */ - NET_EPOCH_ENTER(et); if ((lle = nd6_lookup(&addr->sin6_addr, 0, ifp)) != NULL) { LLE_RUNLOCK(lle); rc = 1; } - NET_EPOCH_EXIT(et); return (rc); } @@ -1685,6 +1687,7 @@ struct ifaddr *ifa; struct in6_ifaddr *ia; + NET_EPOCH_ENTER(et); if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) && !(ND.flags & ND6_IFF_IFDISABLED)) { /* ifdisabled 1->0 transision */ @@ -1695,7 +1698,6 @@ * do not clear ND6_IFF_IFDISABLED. * See RFC 4862, Section 5.4.5. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != AF_INET6) continue; @@ -1704,7 +1706,6 @@ IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) { /* LLA is duplicated. */ @@ -1725,7 +1726,6 @@ ND_IFINFO(ifp)->flags |= ND6_IFF_IFDISABLED; if (V_ip6_dad_count > 0 && (ND_IFINFO(ifp)->flags & ND6_IFF_NO_DAD) == 0) { - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1734,7 +1734,6 @@ ia = (struct in6_ifaddr *)ifa; ia->ia6_flags |= IN6_IFF_TENTATIVE; } - NET_EPOCH_EXIT(et); } } @@ -1753,7 +1752,6 @@ * address is assigned, and IFF_UP, try to * assign one. */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { if (ifa->ifa_addr->sa_family != @@ -1763,15 +1761,15 @@ if (IN6_IS_ADDR_LINKLOCAL(IA6_IN6(ia))) break; } - NET_EPOCH_EXIT(et); if (ifa != NULL) /* No LLA is configured. */ in6_ifattach(ifp, NULL); } } - } ND_IFINFO(ifp)->flags = ND.flags; + NET_EPOCH_EXIT(et); break; + } #undef ND case SIOCSNDFLUSH_IN6: /* XXX: the ioctl name is confusing... */ /* sync kernel routing table with the default router list */ @@ -1967,12 +1965,12 @@ int flags; uint16_t router = 0; struct sockaddr_in6 sin6; - struct epoch_tracker et; struct mbuf *chain = NULL; u_char linkhdr[LLE_MAX_LINKHDR]; size_t linkhdrsize; int lladdr_off; + NET_EPOCH_ASSERT(); IF_AFDATA_UNLOCK_ASSERT(ifp); KASSERT(ifp != NULL, ("%s: ifp == NULL", __func__)); @@ -1992,9 +1990,7 @@ * description on it in NS section (RFC 2461 7.2.3). */ flags = lladdr ? LLE_EXCLUSIVE : 0; - NET_EPOCH_ENTER(et); ln = nd6_lookup(from, flags, ifp); - NET_EPOCH_EXIT(et); is_newentry = 0; if (ln == NULL) { flags |= LLE_EXCLUSIVE; @@ -2253,10 +2249,11 @@ const struct sockaddr *sa_dst, u_char *desten, uint32_t *pflags, struct llentry **plle) { - struct epoch_tracker et; struct llentry *ln = NULL; const struct sockaddr_in6 *dst6; + NET_EPOCH_ASSERT(); + if (pflags != NULL) *pflags = 0; @@ -2282,7 +2279,6 @@ } } - NET_EPOCH_ENTER(et); ln = nd6_lookup(&dst6->sin6_addr, plle ? LLE_EXCLUSIVE : LLE_UNLOCKED, ifp); if (ln != NULL && (ln->r_flags & RLLE_VALID) != 0) { @@ -2302,11 +2298,9 @@ *plle = ln; LLE_WUNLOCK(ln); } - NET_EPOCH_EXIT(et); return (0); } else if (plle && ln) LLE_WUNLOCK(ln); - NET_EPOCH_EXIT(et); return (nd6_resolve_slow(ifp, 0, m, dst6, desten, pflags, plle)); } @@ -2333,6 +2327,8 @@ int send_ns, ll_len; char *lladdr; + NET_EPOCH_ASSERT(); + /* * Address resolution or Neighbor Unreachability Detection * for the next hop. @@ -2340,11 +2336,7 @@ * or an anycast address(i.e. not a multicast). */ if (lle == NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); lle = nd6_lookup(&dst->sin6_addr, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if ((lle == NULL) && nd6_is_addr_neighbor(dst, ifp)) { /* * Since nd6_is_addr_neighbor() internally calls nd6_lookup(), Index: sys/netinet6/nd6_nbr.c =================================================================== --- sys/netinet6/nd6_nbr.c +++ sys/netinet6/nd6_nbr.c @@ -613,7 +613,6 @@ void nd6_na_input(struct mbuf *m, int off, int icmp6len) { - struct epoch_tracker et; struct ifnet *ifp = m->m_pkthdr.rcvif; struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); struct nd_neighbor_advert *nd_na; @@ -636,6 +635,8 @@ int lladdr_off; char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN]; + NET_EPOCH_ASSERT(); + /* RFC 6980: Nodes MUST silently ignore fragments */ if(m->m_flags & M_FRAGMENTED) goto freeit; @@ -741,9 +742,7 @@ * If no neighbor cache entry is found, NA SHOULD silently be * discarded. */ - NET_EPOCH_ENTER(et); ln = nd6_lookup(&taddr6, LLE_EXCLUSIVE, ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) { goto freeit; } @@ -1192,8 +1191,13 @@ nd6_dad_starttimer(struct dadq *dp, int ticks, int send_ns) { - if (send_ns != 0) + if (send_ns != 0) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); nd6_dad_ns_output(dp); + NET_EPOCH_EXIT(et); + } callout_reset(&dp->dad_timer_ch, ticks, (void (*)(void *))nd6_dad_timer, (void *)dp); } Index: sys/netinet6/nd6_rtr.c =================================================================== --- sys/netinet6/nd6_rtr.c +++ sys/netinet6/nd6_rtr.c @@ -868,10 +868,11 @@ void defrouter_select_fib(int fibnum) { - struct epoch_tracker et; struct nd_defrouter *dr, *selected_dr, *installed_dr; struct llentry *ln = NULL; + NET_EPOCH_ASSERT(); + if (fibnum == RT_ALL_FIBS) { for (fibnum = 0; fibnum < rt_numfibs; fibnum++) { defrouter_select_fib(fibnum); @@ -895,14 +896,12 @@ */ selected_dr = installed_dr = NULL; TAILQ_FOREACH(dr, &V_nd_defrouter, dr_entry) { - NET_EPOCH_ENTER(et); if (selected_dr == NULL && dr->ifp->if_fib == fibnum && (ln = nd6_lookup(&dr->rtaddr, 0, dr->ifp)) && ND6_IS_LLINFO_PROBREACH(ln)) { selected_dr = dr; defrouter_ref(selected_dr); } - NET_EPOCH_EXIT(et); if (ln != NULL) { LLE_RUNLOCK(ln); ln = NULL; @@ -946,7 +945,6 @@ } } } else if (installed_dr != NULL) { - NET_EPOCH_ENTER(et); if ((ln = nd6_lookup(&installed_dr->rtaddr, 0, installed_dr->ifp)) && ND6_IS_LLINFO_PROBREACH(ln) && @@ -955,7 +953,6 @@ defrouter_rele(selected_dr); selected_dr = installed_dr; } - NET_EPOCH_EXIT(et); if (ln != NULL) LLE_RUNLOCK(ln); } @@ -1260,7 +1257,10 @@ /* ND_OPT_PI_FLAG_ONLINK processing */ if (new->ndpr_raf_onlink) { + struct epoch_tracker et; + ND6_ONLINK_LOCK(); + NET_EPOCH_ENTER(et); if ((error = nd6_prefix_onlink(new)) != 0) { nd6log((LOG_ERR, "nd6_prelist_add: failed to make " "the prefix %s/%d on-link on %s (errno=%d)\n", @@ -1268,6 +1268,7 @@ pr->ndpr_plen, if_name(pr->ndpr_ifp), error)); /* proceed anyway. XXX: is it correct? */ } + NET_EPOCH_EXIT(et); ND6_ONLINK_UNLOCK(); } @@ -1351,7 +1352,8 @@ int auth; struct in6_addrlifetime lt6_tmp; char ip6buf[INET6_ADDRSTRLEN]; - struct epoch_tracker et; + + NET_EPOCH_ASSERT(); auth = 0; if (m) { @@ -1465,7 +1467,6 @@ * consider autoconfigured addresses while RFC2462 simply said * "address". */ - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { struct in6_ifaddr *ifa6; u_int32_t remaininglifetime; @@ -1588,7 +1589,6 @@ ifa6->ia6_lifetime = lt6_tmp; ifa6->ia6_updatetime = time_uptime; } - NET_EPOCH_EXIT(et); if (ia6_match == NULL && new->ndpr_vltime) { int ifidlen; @@ -1684,10 +1684,9 @@ ND6_LOCK_ASSERT(); + NET_EPOCH_ENTER(et); LIST_FOREACH(pfxrtr, &pr->ndpr_advrtrs, pfr_entry) { - NET_EPOCH_ENTER(et); ln = nd6_lookup(&pfxrtr->router->rtaddr, 0, pfxrtr->router->ifp); - NET_EPOCH_EXIT(et); if (ln == NULL) continue; canreach = ND6_IS_LLINFO_PROBREACH(ln); @@ -1695,6 +1694,7 @@ if (canreach) break; } + NET_EPOCH_EXIT(et); return (pfxrtr); } Index: sys/netinet6/raw_ip6.c =================================================================== --- sys/netinet6/raw_ip6.c +++ sys/netinet6/raw_ip6.c @@ -727,12 +727,12 @@ static int rip6_bind(struct socket *so, struct sockaddr *nam, struct thread *td) { - struct epoch_tracker et; struct inpcb *inp; struct sockaddr_in6 *addr = (struct sockaddr_in6 *)nam; struct ifaddr *ifa = NULL; int error = 0; + NET_EPOCH_ASSERT(); inp = sotoinpcb(so); KASSERT(inp != NULL, ("rip6_bind: inp == NULL")); @@ -745,20 +745,14 @@ if ((error = sa6_embedscope(addr, V_ip6_use_defzone)) != 0) return (error); - NET_EPOCH_ENTER(et); if (!IN6_IS_ADDR_UNSPECIFIED(&addr->sin6_addr) && - (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) { - NET_EPOCH_EXIT(et); + (ifa = ifa_ifwithaddr((struct sockaddr *)addr)) == NULL) return (EADDRNOTAVAIL); - } if (ifa != NULL && ((struct in6_ifaddr *)ifa)->ia6_flags & (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY| - IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) { - NET_EPOCH_EXIT(et); + IN6_IFF_DETACHED|IN6_IFF_DEPRECATED)) return (EADDRNOTAVAIL); - } - NET_EPOCH_EXIT(et); INP_INFO_WLOCK(&V_ripcbinfo); INP_WLOCK(inp); inp->in6p_laddr = addr->sin6_addr; Index: sys/netipsec/xform_ipcomp.c =================================================================== --- sys/netipsec/xform_ipcomp.c +++ sys/netipsec/xform_ipcomp.c @@ -52,6 +52,8 @@ #include #include +#include /* XXXGL: net_epoch should move out there */ +#include /* XXXGL: net_epoch should move out there */ #include #include @@ -122,6 +124,8 @@ { int isr; + NET_EPOCH_ASSERT(); + switch (proto) { #ifdef INET case IPPROTO_IPV4: Index: sys/netpfil/ipfw/ip_dn_io.c =================================================================== --- sys/netpfil/ipfw/ip_dn_io.c +++ sys/netpfil/ipfw/ip_dn_io.c @@ -50,6 +50,7 @@ #include #include /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ +#include /* NET_EPOCH_... */ #include #include @@ -741,6 +742,8 @@ { struct mbuf *n; + NET_EPOCH_ASSERT(); + for (; m != NULL; m = n) { struct ifnet *ifp = NULL; /* gcc 3.4.6 complains */ struct m_tag *tag; Index: sys/netpfil/pf/pf_if.c =================================================================== --- sys/netpfil/pf/pf_if.c +++ sys/netpfil/pf/pf_if.c @@ -76,8 +76,8 @@ eventhandler_tag pfi_detach_group_cookie; eventhandler_tag pfi_ifaddr_event_cookie; -static void pfi_attach_ifnet(struct ifnet *); -static void pfi_attach_ifgroup(struct ifg_group *); +static void pfi_attach_ifnet(struct ifnet *, struct pfi_kif *); +static void pfi_attach_ifgroup(struct ifg_group *, struct pfi_kif *); static void pfi_kif_update(struct pfi_kif *); static void pfi_dynaddr_update(struct pfi_dynaddr *dyn); @@ -114,25 +114,49 @@ void pfi_initialize_vnet(void) { + struct pfi_list kifs = LIST_HEAD_INITIALIZER(); + struct epoch_tracker et; + struct pfi_kif *kif; struct ifg_group *ifg; struct ifnet *ifp; - struct pfi_kif *kif; + int nkifs; V_pfi_buffer_max = 64; V_pfi_buffer = malloc(V_pfi_buffer_max * sizeof(*V_pfi_buffer), PFI_MTYPE, M_WAITOK); - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); - PF_RULES_WLOCK(); - V_pfi_all = pfi_kif_attach(kif, IFG_ALL); - PF_RULES_WUNLOCK(); - + nkifs = 1; /* one for V_pfi_all */ IFNET_RLOCK(); CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) - pfi_attach_ifgroup(ifg); + nkifs++; CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) - pfi_attach_ifnet(ifp); + nkifs++; + + for (int n = 0; n < nkifs; n++) { + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + LIST_INSERT_HEAD(&kifs, kif, pfik_list); + } + + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + V_pfi_all = pfi_kif_attach(kif, IFG_ALL); + CK_STAILQ_FOREACH(ifg, &V_ifg_head, ifg_next) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifgroup(ifg, kif); + } + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) { + kif = LIST_FIRST(&kifs); + LIST_REMOVE(kif, pfik_list); + pfi_attach_ifnet(ifp, kif); + } + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); IFNET_RUNLOCK(); + + MPASS(LIST_EMPTY(&kifs)); } void @@ -296,59 +320,44 @@ { struct ifg_list *p; + NET_EPOCH_ASSERT(); + if (rule_kif == NULL || rule_kif == packet_kif) return (1); if (rule_kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(p, &packet_kif->pfik_ifp->if_groups, ifgl_next) - if (p->ifgl_group == rule_kif->pfik_group) { - NET_EPOCH_EXIT(et); + if (p->ifgl_group == rule_kif->pfik_group) return (1); - } - NET_EPOCH_EXIT(et); } - return (0); } static void -pfi_attach_ifnet(struct ifnet *ifp) +pfi_attach_ifnet(struct ifnet *ifp, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifp->if_xname); - if_ref(ifp); - kif->pfik_ifp = ifp; ifp->if_pf_kif = kif; - pfi_kif_update(kif); - PF_RULES_WUNLOCK(); } static void -pfi_attach_ifgroup(struct ifg_group *ifg) +pfi_attach_ifgroup(struct ifg_group *ifg, struct pfi_kif *kif) { - struct pfi_kif *kif; - kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + PF_RULES_WASSERT(); - PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, ifg->ifg_group); - kif->pfik_group = ifg; ifg->ifg_pf_kif = kif; - PF_RULES_WUNLOCK(); } int @@ -389,6 +398,7 @@ int pfi_dynaddr_setup(struct pf_addr_wrap *aw, sa_family_t af) { + struct epoch_tracker et; struct pfi_dynaddr *dyn; char tblname[PF_TABLE_NAME_SIZE]; struct pf_ruleset *ruleset = NULL; @@ -445,7 +455,9 @@ TAILQ_INSERT_TAIL(&dyn->pfid_kif->pfik_dynaddrs, dyn, entry); aw->p.dyn = dyn; + NET_EPOCH_ENTER(et); pfi_kif_update(dyn->pfid_kif); + NET_EPOCH_EXIT(et); return (0); @@ -467,6 +479,7 @@ struct ifg_list *ifgl; struct pfi_dynaddr *p; + NET_EPOCH_ASSERT(); PF_RULES_WASSERT(); /* update all dynaddr */ @@ -475,13 +488,9 @@ /* again for all groups kif is member of */ if (kif->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgl, &kif->pfik_ifp->if_groups, ifgl_next) pfi_kif_update((struct pfi_kif *) ifgl->ifgl_group->ifg_pf_kif); - NET_EPOCH_EXIT(et); } } @@ -512,17 +521,15 @@ int e, size2 = 0; struct ifg_member *ifgm; + NET_EPOCH_ASSERT(); + V_pfi_buffer_cnt = 0; if (kif->pfik_ifp != NULL) pfi_instance_add(kif->pfik_ifp, net, flags); else if (kif->pfik_group != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); CK_STAILQ_FOREACH(ifgm, &kif->pfik_group->ifg_members, ifgm_next) pfi_instance_add(ifgm->ifgm_ifp, net, flags); - NET_EPOCH_EXIT(et); } if ((e = pfr_set_addrs(&kt->pfrkt_t, V_pfi_buffer, V_pfi_buffer_cnt, &size2, @@ -534,12 +541,12 @@ static void pfi_instance_add(struct ifnet *ifp, int net, int flags) { - struct epoch_tracker et; struct ifaddr *ia; int got4 = 0, got6 = 0; int net2, af; - NET_EPOCH_ENTER(et); + NET_EPOCH_ASSERT(); + CK_STAILQ_FOREACH(ia, &ifp->if_addrhead, ifa_link) { if (ia->ifa_addr == NULL) continue; @@ -597,7 +604,6 @@ else pfi_address_add(ia->ifa_addr, af, net2); } - NET_EPOCH_EXIT(et); } static void @@ -732,9 +738,11 @@ void pfi_get_ifaces(const char *name, struct pfi_kif *buf, int *size) { + struct epoch_tracker et; struct pfi_kif *p, *nextp; int n = 0; + NET_EPOCH_ENTER(et); for (p = RB_MIN(pfi_ifhead, &V_pfi_ifs); p; p = nextp) { nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); if (pfi_skip_if(name, p)) @@ -747,6 +755,7 @@ nextp = RB_NEXT(pfi_ifhead, &V_pfi_ifs, p); } *size = n; + NET_EPOCH_EXIT(et); } static int @@ -755,6 +764,8 @@ struct ifg_list *i; int n; + NET_EPOCH_ASSERT(); + if (filter == NULL || !*filter) return (0); if (!strcmp(p->pfik_name, filter)) @@ -764,45 +775,43 @@ return (1); /* sanity check */ if (filter[n-1] >= '0' && filter[n-1] <= '9') return (1); /* group names may not end in a digit */ - if (p->pfik_ifp != NULL) { - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); - CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) { - if (!strncmp(i->ifgl_group->ifg_group, filter, - IFNAMSIZ)) { - NET_EPOCH_EXIT(et); - return (0); /* iface is in group "filter" */ - } - } - NET_EPOCH_EXIT(et); - } + if (p->pfik_ifp == NULL) + return (1); + CK_STAILQ_FOREACH(i, &p->pfik_ifp->if_groups, ifgl_next) + if (!strncmp(i->ifgl_group->ifg_group, filter, IFNAMSIZ)) + return (0); /* iface is in group "filter" */ return (1); } int pfi_set_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags |= flags; } + NET_EPOCH_EXIT(et); return (0); } int pfi_clear_flags(const char *name, int flags) { + struct epoch_tracker et; struct pfi_kif *p; + NET_EPOCH_ENTER(et); RB_FOREACH(p, pfi_ifhead, &V_pfi_ifs) { if (pfi_skip_if(name, p)) continue; p->pfik_flags &= ~flags; } + NET_EPOCH_EXIT(et); return (0); } @@ -829,22 +838,28 @@ static void pfi_attach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifnet(ifp); -#ifdef ALTQ + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); + pfi_attach_ifnet(ifp, kif); +#ifdef ALTQ pf_altq_ifnet_event(ifp, 0); - PF_RULES_WUNLOCK(); #endif + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_detach_ifnet_event(void *arg __unused, struct ifnet *ifp) { + struct epoch_tracker et; struct pfi_kif *kif = (struct pfi_kif *)ifp->if_pf_kif; if (pfsync_detach_ifnet_ptr) @@ -858,6 +873,7 @@ return; } + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; pfi_kif_update(kif); @@ -871,22 +887,31 @@ pf_altq_ifnet_event(ifp, 1); #endif PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_attach_group_event(void *arg __unused, struct ifg_group *ifg) { + struct epoch_tracker et; + struct pfi_kif *kif; if (V_pf_vnet_active == 0) { /* Avoid teardown race in the least expensive way. */ return; } - pfi_attach_ifgroup(ifg); + kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); + PF_RULES_WLOCK(); + pfi_attach_ifgroup(ifg, kif); + PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void pfi_change_group_event(void *arg __unused, char *gname) { + struct epoch_tracker et; struct pfi_kif *kif; if (V_pf_vnet_active == 0) { @@ -895,11 +920,13 @@ } kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK); + NET_EPOCH_ENTER(et); PF_RULES_WLOCK(); V_pfi_update++; kif = pfi_kif_attach(kif, gname); pfi_kif_update(kif); PF_RULES_WUNLOCK(); + NET_EPOCH_EXIT(et); } static void @@ -937,8 +964,12 @@ } PF_RULES_WLOCK(); if (ifp->if_pf_kif) { + struct epoch_tracker et; + V_pfi_update++; + NET_EPOCH_ENTER(et); pfi_kif_update(ifp->if_pf_kif); + NET_EPOCH_EXIT(et); } PF_RULES_WUNLOCK(); } Index: sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c =================================================================== --- sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -1582,6 +1582,7 @@ void ipoib_demux(struct ifnet *ifp, struct mbuf *m, u_short proto) { + struct epoch_tracker et; int isr; #ifdef MAC @@ -1623,7 +1624,9 @@ default: goto discard; } + NET_EPOCH_ENTER(et); netisr_dispatch(isr, m); + NET_EPOCH_EXIT(et); return; discard: