Index: sys/net/if_bridge.c =================================================================== --- sys/net/if_bridge.c +++ sys/net/if_bridge.c @@ -229,7 +229,7 @@ * Bridge interface list entry. */ struct bridge_iflist { - LIST_ENTRY(bridge_iflist) bif_next; + CK_LIST_ENTRY(bridge_iflist) bif_next; struct ifnet *bif_ifp; /* member if */ struct bstp_port bif_stp; /* STP state */ uint32_t bif_flags; /* member if flags */ @@ -237,19 +237,24 @@ uint32_t bif_addrmax; /* max # of addresses */ uint32_t bif_addrcnt; /* cur. # of addresses */ uint32_t bif_addrexceeded;/* # of address violations */ + + struct epoch_context bif_epoch_ctx; }; /* * Bridge route node. */ struct bridge_rtnode { - LIST_ENTRY(bridge_rtnode) brt_hash; /* hash table linkage */ - LIST_ENTRY(bridge_rtnode) brt_list; /* list linkage */ + CK_LIST_ENTRY(bridge_rtnode) brt_hash; /* hash table linkage */ + CK_LIST_ENTRY(bridge_rtnode) brt_list; /* list linkage */ struct bridge_iflist *brt_dst; /* destination if */ unsigned long brt_expire; /* expiration time */ uint8_t brt_flags; /* address flags */ uint8_t brt_addr[ETHER_ADDR_LEN]; uint16_t brt_vlan; /* vlan id */ + + struct vnet *brt_vnet; + struct epoch_context brt_epoch_ctx; }; #define brt_ifp brt_dst->bif_ifp @@ -267,15 +272,17 @@ struct callout sc_brcallout; /* bridge callout */ uint32_t sc_iflist_ref; /* refcount for sc_iflist */ uint32_t sc_iflist_xcnt; /* refcount for sc_iflist */ - LIST_HEAD(, bridge_iflist) sc_iflist; /* member interface list */ - LIST_HEAD(, bridge_rtnode) *sc_rthash; /* our forwarding table */ - LIST_HEAD(, bridge_rtnode) sc_rtlist; /* list version of above */ + CK_LIST_HEAD(, bridge_iflist) sc_iflist; /* member interface list */ + CK_LIST_HEAD(, bridge_rtnode) *sc_rthash; /* our forwarding table */ + CK_LIST_HEAD(, bridge_rtnode) sc_rtlist; /* list version of above */ uint32_t sc_rthash_key; /* key for hash */ - LIST_HEAD(, bridge_iflist) sc_spanlist; /* span ports list */ + CK_LIST_HEAD(, bridge_iflist) sc_spanlist; /* span ports list */ struct bstp_state sc_stp; /* STP state */ uint32_t sc_brtexceeded; /* # of cache drops */ struct ifnet *sc_ifaddr; /* member mac copied from */ struct ether_addr sc_defaddr; /* Default MAC address */ + + struct epoch_context sc_epoch_ctx; }; VNET_DEFINE_STATIC(struct mtx, bridge_list_mtx); @@ -596,6 +603,11 @@ if_clone_detach(V_bridge_cloner); V_bridge_cloner = NULL; BRIDGE_LIST_LOCK_DESTROY(); + + /* Before we can destroy the uma zone, because there are callbacks that + * use it. */ + NET_EPOCH_WAIT(); + uma_zdestroy(V_bridge_rtnode_zone); } VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, @@ -697,8 +709,8 @@ callout_init_mtx(&sc->sc_brcallout, &sc->sc_mtx, 0); - LIST_INIT(&sc->sc_iflist); - LIST_INIT(&sc->sc_spanlist); + CK_LIST_INIT(&sc->sc_iflist); + CK_LIST_INIT(&sc->sc_spanlist); ifp->if_softc = sc; if_initname(ifp, bridge_name, unit); @@ -758,6 +770,17 @@ return (0); } +static void +bridge_clone_destroy_cb(struct epoch_context *ctx) +{ + struct bridge_softc *sc; + + sc = __containerof(ctx, struct bridge_softc, sc_epoch_ctx); + + BRIDGE_LOCK_DESTROY(sc); + free(sc, M_DEVBUF); +} + /* * bridge_clone_destroy: * @@ -768,16 +791,18 @@ { struct bridge_softc *sc = ifp->if_softc; struct bridge_iflist *bif; + struct epoch_tracker et; + NET_EPOCH_ENTER(et); BRIDGE_LOCK(sc); bridge_stop(ifp, 1); ifp->if_flags &= ~IFF_UP; - while ((bif = LIST_FIRST(&sc->sc_iflist)) != NULL) + while ((bif = CK_LIST_FIRST(&sc->sc_iflist)) != NULL) bridge_delete_member(sc, bif, 0); - while ((bif = LIST_FIRST(&sc->sc_spanlist)) != NULL) { + while ((bif = CK_LIST_FIRST(&sc->sc_spanlist)) != NULL) { bridge_delete_span(sc, bif); } @@ -793,11 +818,12 @@ BRIDGE_LIST_UNLOCK(); bstp_detach(&sc->sc_stp); + NET_EPOCH_EXIT(et); + ether_ifdetach(ifp); if_free(ifp); - BRIDGE_LOCK_DESTROY(sc); - free(sc, M_DEVBUF); + NET_EPOCH_CALL(bridge_clone_destroy_cb, &sc->sc_epoch_ctx); } /* @@ -823,6 +849,9 @@ struct ifdrv *ifd = (struct ifdrv *) data; const struct bridge_control *bc; int error = 0, oldmtu; + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); switch (cmd) { @@ -916,12 +945,12 @@ error = EINVAL; break; } - if (LIST_EMPTY(&sc->sc_iflist)) { + if (CK_LIST_EMPTY(&sc->sc_iflist)) { sc->sc_ifp->if_mtu = ifr->ifr_mtu; break; } BRIDGE_LOCK(sc); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (bif->bif_ifp->if_mtu != ifr->ifr_mtu) { log(LOG_NOTICE, "%s: invalid MTU: %u(%s)" " != %d\n", sc->sc_ifp->if_xname, @@ -944,6 +973,8 @@ break; } + NET_EPOCH_EXIT(et); + return (error); } @@ -958,16 +989,18 @@ struct bridge_iflist *bif; int enabled, mask; + BRIDGE_LOCK_ASSERT(sc); + /* Initial bitmask of capabilities to test */ mask = BRIDGE_IFCAPS_MASK; - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { /* Every member must support it or its disabled */ mask &= bif->bif_savedcaps; } BRIDGE_XLOCK(sc); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { enabled = bif->bif_ifp->if_capenable; enabled &= ~BRIDGE_IFCAPS_STRIP; /* strip off mask bits and enable them again if allowed */ @@ -1019,9 +1052,9 @@ struct bridge_iflist *bif; struct ifnet *ifp; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { ifp = bif->bif_ifp; if (strcmp(ifp->if_xname, name) == 0) return (bif); @@ -1040,9 +1073,9 @@ { struct bridge_iflist *bif; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (bif->bif_ifp == member_ifp) return (bif); } @@ -1050,6 +1083,16 @@ return (NULL); } +static void +bridge_delete_member_cb(struct epoch_context *ctx) +{ + struct bridge_iflist *bif; + + bif = __containerof(ctx, struct bridge_iflist, bif_epoch_ctx); + + free(bif, M_DEVBUF); +} + /* * bridge_delete_member: * @@ -1061,6 +1104,7 @@ { struct ifnet *ifs = bif->bif_ifp; struct ifnet *fif = NULL; + struct bridge_iflist *bifl; BRIDGE_LOCK_ASSERT(sc); @@ -1069,7 +1113,7 @@ ifs->if_bridge = NULL; BRIDGE_XLOCK(sc); - LIST_REMOVE(bif, bif_next); + CK_LIST_REMOVE(bif, bif_next); BRIDGE_XDROP(sc); /* @@ -1078,12 +1122,13 @@ * to its default address if no members are left. */ if (V_bridge_inherit_mac && sc->sc_ifaddr == ifs) { - if (LIST_EMPTY(&sc->sc_iflist)) { + if (CK_LIST_EMPTY(&sc->sc_iflist)) { bcopy(&sc->sc_defaddr, IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); sc->sc_ifaddr = NULL; } else { - fif = LIST_FIRST(&sc->sc_iflist)->bif_ifp; + bifl = CK_LIST_FIRST(&sc->sc_iflist); + fif = bifl->bif_ifp; bcopy(IF_LLADDR(fif), IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); sc->sc_ifaddr = fif; @@ -1128,7 +1173,8 @@ } bstp_destroy(&bif->bif_stp); /* prepare to free */ BRIDGE_LOCK(sc); - free(bif, M_DEVBUF); + + NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx); } /* @@ -1144,8 +1190,9 @@ KASSERT(bif->bif_ifp->if_bridge == NULL, ("%s: not a span interface", __func__)); - LIST_REMOVE(bif, bif_next); - free(bif, M_DEVBUF); + CK_LIST_REMOVE(bif, bif_next); + + NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx); } static int @@ -1163,7 +1210,7 @@ return (EINVAL); /* If it's in the span list, it can't be a member. */ - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) if (ifs == bif->bif_ifp) return (EBUSY); @@ -1202,7 +1249,7 @@ * interfaces. */ BRIDGE_XLOCK(sc); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (in6ifa_llaonifp(bif->bif_ifp)) { BRIDGE_UNLOCK(sc); in6_ifdetach(bif->bif_ifp); @@ -1228,7 +1275,7 @@ } #endif /* Allow the first Ethernet member to define the MTU */ - if (LIST_EMPTY(&sc->sc_iflist)) + if (CK_LIST_EMPTY(&sc->sc_iflist)) sc->sc_ifp->if_mtu = ifs->if_mtu; else if (sc->sc_ifp->if_mtu != ifs->if_mtu) { if_printf(sc->sc_ifp, "invalid MTU: %u(%s) != %u\n", @@ -1249,7 +1296,7 @@ * member and the MAC address of the bridge has not been changed from * the default randomly generated one. */ - if (V_bridge_inherit_mac && LIST_EMPTY(&sc->sc_iflist) && + if (V_bridge_inherit_mac && CK_LIST_EMPTY(&sc->sc_iflist) && !memcmp(IF_LLADDR(sc->sc_ifp), sc->sc_defaddr.octet, ETHER_ADDR_LEN)) { bcopy(IF_LLADDR(ifs), IF_LLADDR(sc->sc_ifp), ETHER_ADDR_LEN); sc->sc_ifaddr = ifs; @@ -1266,7 +1313,7 @@ * * NOTE: insert_***HEAD*** should be safe for the traversals. */ - LIST_INSERT_HEAD(&sc->sc_iflist, bif, bif_next); + CK_LIST_INSERT_HEAD(&sc->sc_iflist, bif, bif_next); /* Set interface capabilities to the intersection set of all members */ bridge_mutecaps(sc); @@ -1413,9 +1460,9 @@ int count, buflen, len, error = 0; count = 0; - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) count++; - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) count++; buflen = sizeof(breq) * count; @@ -1431,7 +1478,7 @@ buf = outbuf; len = min(bifc->ifbic_len, buflen); bzero(&breq, sizeof(breq)); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (len < sizeof(breq)) break; @@ -1446,7 +1493,7 @@ buf += sizeof(breq); len -= sizeof(breq); } - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) { if (len < sizeof(breq)) break; @@ -1481,7 +1528,7 @@ return (0); count = 0; - LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) + CK_LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) count++; buflen = sizeof(bareq) * count; @@ -1493,7 +1540,7 @@ buf = outbuf; len = min(bac->ifbac_len, buflen); bzero(&bareq, sizeof(bareq)); - LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { + CK_LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { if (len < sizeof(bareq)) goto out; strlcpy(bareq.ifba_ifsname, brt->brt_ifp->if_xname, @@ -1528,12 +1575,17 @@ struct bridge_iflist *bif; int error; + NET_EPOCH_ASSERT(); + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); + /* bridge_rtupdate() may acquire the lock. */ + BRIDGE_UNLOCK(sc); error = bridge_rtupdate(sc, req->ifba_dst, req->ifba_vlan, bif, 1, req->ifba_flags); + BRIDGE_LOCK(sc); return (error); } @@ -1696,7 +1748,7 @@ if (ifs == NULL) return (ENOENT); - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) if (ifs == bif->bif_ifp) return (EBUSY); @@ -1719,7 +1771,7 @@ bif->bif_ifp = ifs; bif->bif_flags = IFBIF_SPAN; - LIST_INSERT_HEAD(&sc->sc_spanlist, bif, bif_next); + CK_LIST_INSERT_HEAD(&sc->sc_spanlist, bif, bif_next); return (0); } @@ -1735,7 +1787,7 @@ if (ifs == NULL) return (ENOENT); - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) if (ifs == bif->bif_ifp) break; @@ -1797,7 +1849,7 @@ int count, buflen, len, error = 0; count = 0; - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if ((bif->bif_flags & IFBIF_STP) != 0) count++; } @@ -1816,7 +1868,7 @@ buf = outbuf; len = min(bifstp->ifbpstp_len, buflen); bzero(&bpreq, sizeof(bpreq)); - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (len < sizeof(bpreq)) break; @@ -1872,6 +1924,7 @@ { struct bridge_softc *sc = ifp->if_bridge; struct bridge_iflist *bif; + struct epoch_tracker et; if (ifp->if_flags & IFF_RENAMING) return; @@ -1882,6 +1935,7 @@ */ return; } + NET_EPOCH_ENTER(et); /* Check if the interface is a bridge member */ if (sc != NULL) { BRIDGE_LOCK(sc); @@ -1891,6 +1945,7 @@ bridge_delete_member(sc, bif, 1); BRIDGE_UNLOCK(sc); + NET_EPOCH_EXIT(et); return; } @@ -1898,7 +1953,7 @@ BRIDGE_LIST_LOCK(); LIST_FOREACH(sc, &V_bridge_list, sc_list) { BRIDGE_LOCK(sc); - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) if (ifp == bif->bif_ifp) { bridge_delete_span(sc, bif); break; @@ -1907,6 +1962,7 @@ BRIDGE_UNLOCK(sc); } BRIDGE_LIST_UNLOCK(); + NET_EPOCH_EXIT(et); } /* @@ -2066,6 +2122,8 @@ struct bridge_softc *sc; uint16_t vlan; + NET_EPOCH_ASSERT(); + if (m->m_len < ETHER_HDR_LEN) { m = m_pullup(m, ETHER_HDR_LEN); if (m == NULL) @@ -2076,7 +2134,6 @@ sc = ifp->if_bridge; vlan = VLANTAGOF(m); - BRIDGE_LOCK(sc); bifp = sc->sc_ifp; /* @@ -2103,17 +2160,11 @@ if (dst_if == NULL) { struct bridge_iflist *bif; struct mbuf *mc; - int error = 0, used = 0; + int used = 0; bridge_span(sc, m); - BRIDGE_LOCK2REF(sc, error); - if (error) { - m_freem(m); - return (0); - } - - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { dst_if = bif->bif_ifp; if (dst_if->if_type == IFT_GIF) @@ -2131,7 +2182,7 @@ bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) continue; - if (LIST_NEXT(bif, bif_next) == NULL) { + if (CK_LIST_NEXT(bif, bif_next) == NULL) { used = 1; mc = m; } else { @@ -2146,7 +2197,6 @@ } if (used == 0) m_freem(m); - BRIDGE_UNREF(sc); return (0); } @@ -2158,11 +2208,9 @@ bridge_span(sc, m); if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) { m_freem(m); - BRIDGE_UNLOCK(sc); return (0); } - BRIDGE_UNLOCK(sc); bridge_enqueue(sc, dst_if, m); return (0); } @@ -2187,10 +2235,8 @@ eh = mtod(m, struct ether_header *); - BRIDGE_LOCK(sc); if (((m->m_flags & (M_BCAST|M_MCAST)) == 0) && (dst_if = bridge_rtlookup(sc, eh->ether_dhost, 1)) != NULL) { - BRIDGE_UNLOCK(sc); error = bridge_enqueue(sc, dst_if, m); } else bridge_broadcast(sc, ifp, m, 0); @@ -2224,6 +2270,8 @@ uint8_t *dst; int error; + NET_EPOCH_ASSERT(); + src_if = m->m_pkthdr.rcvif; ifp = sc->sc_ifp; @@ -2302,12 +2350,10 @@ || PFIL_HOOKED_IN(V_inet6_pfil_head) #endif ) { - BRIDGE_UNLOCK(sc); if (bridge_pfil(&m, ifp, src_if, PFIL_IN) != 0) return; if (m == NULL) return; - BRIDGE_LOCK(sc); } if (dst_if == NULL) { @@ -2335,8 +2381,6 @@ dbif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) goto drop; - BRIDGE_UNLOCK(sc); - if (PFIL_HOOKED_OUT(V_inet_pfil_head) #ifdef INET6 || PFIL_HOOKED_OUT(V_inet6_pfil_head) @@ -2352,7 +2396,6 @@ return; drop: - BRIDGE_UNLOCK(sc); m_freem(m); } @@ -2373,6 +2416,8 @@ uint16_t vlan; int error; + NET_EPOCH_ASSERT(); + if ((sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return (m); @@ -2393,10 +2438,8 @@ m_freem(m); return (NULL); } - BRIDGE_LOCK(sc); bif = bridge_lookup_member_if(sc, ifp); if (bif == NULL) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2409,13 +2452,11 @@ if (memcmp(eh->ether_dhost, bstp_etheraddr, ETHER_ADDR_LEN) == 0) { bstp_input(&bif->bif_stp, ifp, m); /* consumes mbuf */ - BRIDGE_UNLOCK(sc); return (NULL); } if ((bif->bif_flags & IFBIF_STP) && bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2426,7 +2467,6 @@ */ mc = m_dup(m, M_NOWAIT); if (mc == NULL) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2458,7 +2498,6 @@ if ((bif->bif_flags & IFBIF_STP) && bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2492,7 +2531,6 @@ error = bridge_rtupdate(sc, eh->ether_shost, \ vlan, bif, 0, IFBAF_DYNAMIC); \ if (error && bif->bif_addrmax) { \ - BRIDGE_UNLOCK(sc); \ m_freem(m); \ return (NULL); \ } \ @@ -2500,7 +2538,6 @@ m->m_pkthdr.rcvif = iface; \ if ((iface) == ifp) { \ /* Skip bridge processing... src == dest */ \ - BRIDGE_UNLOCK(sc); \ return (m); \ } \ /* It's passing over or to the bridge, locally. */ \ @@ -2512,13 +2549,11 @@ OR_PFIL_HOOKED_INET6)) { \ if (bridge_pfil(&m, NULL, ifp, \ PFIL_IN) != 0 || m == NULL) { \ - BRIDGE_UNLOCK(sc); \ return (NULL); \ } \ } \ if ((iface) != bifp) \ ETHER_BPF_MTAP(iface, m); \ - BRIDGE_UNLOCK(sc); \ return (m); \ } \ \ @@ -2526,7 +2561,6 @@ if (memcmp(IF_LLADDR((iface)), eh->ether_shost, ETHER_ADDR_LEN) == 0 \ OR_CARP_CHECK_WE_ARE_SRC((iface)) \ ) { \ - BRIDGE_UNLOCK(sc); \ m_freem(m); \ return (NULL); \ } @@ -2546,7 +2580,7 @@ do { GRAB_OUR_PACKETS(ifp) } while (0); /* Now check the all bridge members. */ - LIST_FOREACH(bif2, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif2, &sc->sc_iflist, bif_next) { GRAB_OUR_PACKETS(bif2->bif_ifp) } @@ -2577,15 +2611,11 @@ struct bridge_iflist *dbif, *sbif; struct mbuf *mc; struct ifnet *dst_if; - int error = 0, used = 0, i; + int used = 0, i; - sbif = bridge_lookup_member_if(sc, src_if); + NET_EPOCH_ASSERT(); - BRIDGE_LOCK2REF(sc, error); - if (error) { - m_freem(m); - return; - } + sbif = bridge_lookup_member_if(sc, src_if); /* Filter on the bridge interface before broadcasting */ if (runfilt && (PFIL_HOOKED_OUT(V_inet_pfil_head) @@ -2594,12 +2624,12 @@ #endif )) { if (bridge_pfil(&m, sc->sc_ifp, NULL, PFIL_OUT) != 0) - goto out; + return; if (m == NULL) - goto out; + return; } - LIST_FOREACH(dbif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(dbif, &sc->sc_iflist, bif_next) { dst_if = dbif->bif_ifp; if (dst_if == src_if) continue; @@ -2619,7 +2649,7 @@ if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) continue; - if (LIST_NEXT(dbif, bif_next) == NULL) { + if (CK_LIST_NEXT(dbif, bif_next) == NULL) { mc = m; used = 1; } else { @@ -2659,9 +2689,6 @@ } if (used == 0) m_freem(m); - -out: - BRIDGE_UNREF(sc); } /* @@ -2677,10 +2704,12 @@ struct ifnet *dst_if; struct mbuf *mc; - if (LIST_EMPTY(&sc->sc_spanlist)) + NET_EPOCH_ASSERT(); + + if (CK_LIST_EMPTY(&sc->sc_spanlist)) return; - LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_spanlist, bif_next) { dst_if = bif->bif_ifp; if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) @@ -2708,7 +2737,8 @@ struct bridge_rtnode *brt; int error; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); + BRIDGE_UNLOCK_ASSERT(sc); /* Check the source address is valid and not multicast. */ if (ETHER_IS_MULTICAST(dst) || @@ -2725,13 +2755,24 @@ * update it, otherwise create a new one. */ if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) == NULL) { + BRIDGE_LOCK(sc); + + /* Check again, now that we have the lock. There could have + * been a race and we only want to insert this once. */ + if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) != NULL) { + BRIDGE_UNLOCK(sc); + return (0); + } + if (sc->sc_brtcnt >= sc->sc_brtmax) { sc->sc_brtexceeded++; + BRIDGE_UNLOCK(sc); return (ENOSPC); } /* Check per interface address limits (if enabled) */ if (bif->bif_addrmax && bif->bif_addrcnt >= bif->bif_addrmax) { bif->bif_addrexceeded++; + BRIDGE_UNLOCK(sc); return (ENOSPC); } @@ -2741,8 +2782,11 @@ * address. */ brt = uma_zalloc(V_bridge_rtnode_zone, M_NOWAIT | M_ZERO); - if (brt == NULL) + if (brt == NULL) { + BRIDGE_UNLOCK(sc); return (ENOMEM); + } + brt->brt_vnet = curvnet; if (bif->bif_flags & IFBIF_STICKY) brt->brt_flags = IFBAF_STICKY; @@ -2754,17 +2798,22 @@ if ((error = bridge_rtnode_insert(sc, brt)) != 0) { uma_zfree(V_bridge_rtnode_zone, brt); + BRIDGE_UNLOCK(sc); return (error); } brt->brt_dst = bif; bif->bif_addrcnt++; + + BRIDGE_UNLOCK(sc); } if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC && brt->brt_dst != bif) { + BRIDGE_LOCK(sc); brt->brt_dst->bif_addrcnt--; brt->brt_dst = bif; brt->brt_dst->bif_addrcnt++; + BRIDGE_UNLOCK(sc); } if ((flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) @@ -2785,7 +2834,7 @@ { struct bridge_rtnode *brt; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); if ((brt = bridge_rtnode_lookup(sc, addr, vlan)) == NULL) return (NULL); @@ -2816,7 +2865,7 @@ if (sc->sc_brtcnt <= sc->sc_brtmax) return; - LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { + CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { bridge_rtnode_destroy(sc, brt); if (sc->sc_brtcnt <= sc->sc_brtmax) @@ -2859,7 +2908,7 @@ BRIDGE_LOCK_ASSERT(sc); - LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { + CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { if (time_uptime >= brt->brt_expire) bridge_rtnode_destroy(sc, brt); @@ -2879,7 +2928,7 @@ BRIDGE_LOCK_ASSERT(sc); - LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { + CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if (full || (brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) bridge_rtnode_destroy(sc, brt); } @@ -2922,7 +2971,7 @@ BRIDGE_LOCK_ASSERT(sc); - LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { + CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if (brt->brt_ifp == ifp && (full || (brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC)) bridge_rtnode_destroy(sc, brt); @@ -2943,10 +2992,10 @@ M_DEVBUF, M_WAITOK); for (i = 0; i < BRIDGE_RTHASH_SIZE; i++) - LIST_INIT(&sc->sc_rthash[i]); + CK_LIST_INIT(&sc->sc_rthash[i]); sc->sc_rthash_key = arc4random(); - LIST_INIT(&sc->sc_rtlist); + CK_LIST_INIT(&sc->sc_rtlist); } /* @@ -3024,10 +3073,10 @@ uint32_t hash; int dir; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); hash = bridge_rthash(sc, addr); - LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { + CK_LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { dir = bridge_rtnode_addr_cmp(addr, brt->brt_addr); if (dir == 0 && (brt->brt_vlan == vlan || vlan == 0)) return (brt); @@ -3055,9 +3104,9 @@ hash = bridge_rthash(sc, brt->brt_addr); - lbrt = LIST_FIRST(&sc->sc_rthash[hash]); + lbrt = CK_LIST_FIRST(&sc->sc_rthash[hash]); if (lbrt == NULL) { - LIST_INSERT_HEAD(&sc->sc_rthash[hash], brt, brt_hash); + CK_LIST_INSERT_HEAD(&sc->sc_rthash[hash], brt, brt_hash); goto out; } @@ -3066,14 +3115,14 @@ if (dir == 0 && brt->brt_vlan == lbrt->brt_vlan) return (EEXIST); if (dir > 0) { - LIST_INSERT_BEFORE(lbrt, brt, brt_hash); + CK_LIST_INSERT_BEFORE(lbrt, brt, brt_hash); goto out; } - if (LIST_NEXT(lbrt, brt_hash) == NULL) { - LIST_INSERT_AFTER(lbrt, brt, brt_hash); + if (CK_LIST_NEXT(lbrt, brt_hash) == NULL) { + CK_LIST_INSERT_AFTER(lbrt, brt, brt_hash); goto out; } - lbrt = LIST_NEXT(lbrt, brt_hash); + lbrt = CK_LIST_NEXT(lbrt, brt_hash); } while (lbrt != NULL); #ifdef DIAGNOSTIC @@ -3081,12 +3130,24 @@ #endif out: - LIST_INSERT_HEAD(&sc->sc_rtlist, brt, brt_list); + CK_LIST_INSERT_HEAD(&sc->sc_rtlist, brt, brt_list); sc->sc_brtcnt++; return (0); } +static void +bridge_rtnode_destroy_cb(struct epoch_context *ctx) +{ + struct bridge_rtnode *brt; + + brt = __containerof(ctx, struct bridge_rtnode, brt_epoch_ctx); + + CURVNET_SET(brt->brt_vnet); + uma_zfree(V_bridge_rtnode_zone, brt); + CURVNET_RESTORE(); +} + /* * bridge_rtnode_destroy: * @@ -3097,12 +3158,13 @@ { BRIDGE_LOCK_ASSERT(sc); - LIST_REMOVE(brt, brt_hash); + CK_LIST_REMOVE(brt, brt_hash); - LIST_REMOVE(brt, brt_list); + CK_LIST_REMOVE(brt, brt_list); sc->sc_brtcnt--; brt->brt_dst->bif_addrcnt--; - uma_zfree(V_bridge_rtnode_zone, brt); + + NET_EPOCH_CALL(bridge_rtnode_destroy_cb, &brt->brt_epoch_ctx); } /* @@ -3126,7 +3188,7 @@ if (age == 0) bridge_rtdelete(sc, ifp, IFBF_FLUSHDYN); else { - LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { + CK_LIST_FOREACH(brt, &sc->sc_rtlist, brt_list) { /* Cap the expiry time to 'age' */ if (brt->brt_ifp == ifp && brt->brt_expire > time_uptime + age && @@ -3641,17 +3703,20 @@ { struct bridge_softc *sc = ifp->if_bridge; struct bridge_iflist *bif; + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); - BRIDGE_LOCK(sc); bif = bridge_lookup_member_if(sc, ifp); if (bif == NULL) { - BRIDGE_UNLOCK(sc); + NET_EPOCH_EXIT(et); return; } bridge_linkcheck(sc); - BRIDGE_UNLOCK(sc); bstp_linkstate(&bif->bif_stp); + + NET_EPOCH_EXIT(et); } static void @@ -3660,11 +3725,12 @@ struct bridge_iflist *bif; int new_link, hasls; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); + new_link = LINK_STATE_DOWN; hasls = 0; /* Our link is considered up if at least one of our ports is active */ - LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { + CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (bif->bif_ifp->if_capabilities & IFCAP_LINKSTATE) hasls++; if (bif->bif_ifp->if_link_state == LINK_STATE_UP) { @@ -3672,7 +3738,7 @@ break; } } - if (!LIST_EMPTY(&sc->sc_iflist) && !hasls) { + if (!CK_LIST_EMPTY(&sc->sc_iflist) && !hasls) { /* If no interfaces support link-state then we default to up */ new_link = LINK_STATE_UP; }