Index: sys/net/if_bridge.c =================================================================== --- sys/net/if_bridge.c +++ sys/net/if_bridge.c @@ -189,41 +189,14 @@ */ #define BRIDGE_LOCK_INIT(_sc) do { \ mtx_init(&(_sc)->sc_mtx, "if_bridge", NULL, MTX_DEF); \ - cv_init(&(_sc)->sc_cv, "if_bridge_cv"); \ } while (0) #define BRIDGE_LOCK_DESTROY(_sc) do { \ mtx_destroy(&(_sc)->sc_mtx); \ - cv_destroy(&(_sc)->sc_cv); \ } while (0) #define BRIDGE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define BRIDGE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define BRIDGE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) #define BRIDGE_UNLOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED) -#define BRIDGE_LOCK2REF(_sc, _err) do { \ - mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \ - if ((_sc)->sc_iflist_xcnt > 0) \ - (_err) = EBUSY; \ - else \ - (_sc)->sc_iflist_ref++; \ - mtx_unlock(&(_sc)->sc_mtx); \ -} while (0) -#define BRIDGE_UNREF(_sc) do { \ - mtx_lock(&(_sc)->sc_mtx); \ - (_sc)->sc_iflist_ref--; \ - if (((_sc)->sc_iflist_xcnt > 0) && ((_sc)->sc_iflist_ref == 0)) \ - cv_broadcast(&(_sc)->sc_cv); \ - mtx_unlock(&(_sc)->sc_mtx); \ -} while (0) -#define BRIDGE_XLOCK(_sc) do { \ - mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \ - (_sc)->sc_iflist_xcnt++; \ - while ((_sc)->sc_iflist_ref > 0) \ - cv_wait(&(_sc)->sc_cv, &(_sc)->sc_mtx); \ -} while (0) -#define BRIDGE_XDROP(_sc) do { \ - mtx_assert(&(_sc)->sc_mtx, MA_OWNED); \ - (_sc)->sc_iflist_xcnt--; \ -} while (0) /* * Bridge interface list entry. @@ -237,6 +210,7 @@ uint32_t bif_addrmax; /* max # of addresses */ uint32_t bif_addrcnt; /* cur. # of addresses */ uint32_t bif_addrexceeded;/* # of address violations */ + struct epoch_context bif_epoch_ctx; }; /* @@ -250,6 +224,8 @@ uint8_t brt_flags; /* address flags */ uint8_t brt_addr[ETHER_ADDR_LEN]; uint16_t brt_vlan; /* vlan id */ + struct vnet *brt_vnet; + struct epoch_context brt_epoch_ctx; }; #define brt_ifp brt_dst->bif_ifp @@ -260,13 +236,10 @@ struct ifnet *sc_ifp; /* make this an interface */ LIST_ENTRY(bridge_softc) sc_list; struct mtx sc_mtx; - struct cv sc_cv; uint32_t sc_brtmax; /* max # of addresses */ uint32_t sc_brtcnt; /* cur. # of addresses */ uint32_t sc_brttimeout; /* rt timeout in seconds */ struct callout sc_brcallout; /* bridge callout */ - uint32_t sc_iflist_ref; /* refcount for sc_iflist */ - uint32_t sc_iflist_xcnt; /* refcount for sc_iflist */ CK_LIST_HEAD(, bridge_iflist) sc_iflist; /* member interface list */ CK_LIST_HEAD(, bridge_rtnode) *sc_rthash; /* our forwarding table */ CK_LIST_HEAD(, bridge_rtnode) sc_rtlist; /* list version of above */ @@ -276,6 +249,7 @@ uint32_t sc_brtexceeded; /* # of cache drops */ struct ifnet *sc_ifaddr; /* member mac copied from */ struct ether_addr sc_defaddr; /* Default MAC address */ + struct epoch_context sc_epoch_ctx; }; VNET_DEFINE_STATIC(struct mtx, bridge_list_mtx); @@ -596,6 +570,10 @@ if_clone_detach(V_bridge_cloner); V_bridge_cloner = NULL; BRIDGE_LIST_LOCK_DESTROY(); + + /* Callbacks may use the UMA zone. */ + epoch_drain_callbacks(net_epoch_preempt); + uma_zdestroy(V_bridge_rtnode_zone); } VNET_SYSUNINIT(vnet_bridge_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, @@ -758,6 +736,17 @@ return (0); } +static void +bridge_clone_destroy_cb(struct epoch_context *ctx) +{ + struct bridge_softc *sc; + + sc = __containerof(ctx, struct bridge_softc, sc_epoch_ctx); + + BRIDGE_LOCK_DESTROY(sc); + free(sc, M_DEVBUF); +} + /* * bridge_clone_destroy: * @@ -768,7 +757,9 @@ { struct bridge_softc *sc = ifp->if_softc; struct bridge_iflist *bif; + struct epoch_tracker et; + NET_EPOCH_ENTER(et); BRIDGE_LOCK(sc); bridge_stop(ifp, 1); @@ -793,11 +784,12 @@ BRIDGE_LIST_UNLOCK(); bstp_detach(&sc->sc_stp); + NET_EPOCH_EXIT(et); + ether_ifdetach(ifp); if_free(ifp); - BRIDGE_LOCK_DESTROY(sc); - free(sc, M_DEVBUF); + NET_EPOCH_CALL(bridge_clone_destroy_cb, &sc->sc_epoch_ctx); } /* @@ -823,6 +815,9 @@ struct ifdrv *ifd = (struct ifdrv *) data; const struct bridge_control *bc; int error = 0, oldmtu; + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); switch (cmd) { @@ -944,6 +939,8 @@ break; } + NET_EPOCH_EXIT(et); + return (error); } @@ -958,6 +955,8 @@ struct bridge_iflist *bif; int enabled, mask; + BRIDGE_LOCK_ASSERT(sc); + /* Initial bitmask of capabilities to test */ mask = BRIDGE_IFCAPS_MASK; @@ -966,7 +965,6 @@ mask &= bif->bif_savedcaps; } - BRIDGE_XLOCK(sc); CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { enabled = bif->bif_ifp->if_capenable; enabled &= ~BRIDGE_IFCAPS_STRIP; @@ -977,8 +975,6 @@ bridge_set_ifcap(sc, bif, enabled); BRIDGE_LOCK(sc); } - BRIDGE_XDROP(sc); - } static void @@ -1019,7 +1015,7 @@ struct bridge_iflist *bif; struct ifnet *ifp; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { ifp = bif->bif_ifp; @@ -1040,7 +1036,7 @@ { struct bridge_iflist *bif; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (bif->bif_ifp == member_ifp) @@ -1050,6 +1046,16 @@ return (NULL); } +static void +bridge_delete_member_cb(struct epoch_context *ctx) +{ + struct bridge_iflist *bif; + + bif = __containerof(ctx, struct bridge_iflist, bif_epoch_ctx); + + free(bif, M_DEVBUF); +} + /* * bridge_delete_member: * @@ -1069,9 +1075,7 @@ bstp_disable(&bif->bif_stp); ifs->if_bridge = NULL; - BRIDGE_XLOCK(sc); CK_LIST_REMOVE(bif, bif_next); - BRIDGE_XDROP(sc); /* * If removing the interface that gave the bridge its mac address, set @@ -1130,7 +1134,8 @@ } bstp_destroy(&bif->bif_stp); /* prepare to free */ BRIDGE_LOCK(sc); - free(bif, M_DEVBUF); + + NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx); } /* @@ -1147,7 +1152,8 @@ ("%s: not a span interface", __func__)); CK_LIST_REMOVE(bif, bif_next); - free(bif, M_DEVBUF); + + NET_EPOCH_CALL(bridge_delete_member_cb, &bif->bif_epoch_ctx); } static int @@ -1203,7 +1209,6 @@ * If any, remove all inet6 addresses from the member * interfaces. */ - BRIDGE_XLOCK(sc); CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { if (in6ifa_llaonifp(bif->bif_ifp)) { BRIDGE_UNLOCK(sc); @@ -1216,7 +1221,6 @@ bif->bif_ifp->if_xname); } } - BRIDGE_XDROP(sc); if (in6ifa_llaonifp(ifs)) { BRIDGE_UNLOCK(sc); in6_ifdetach(ifs); @@ -1530,12 +1534,17 @@ struct bridge_iflist *bif; int error; + NET_EPOCH_ASSERT(); + bif = bridge_lookup_member(sc, req->ifba_ifsname); if (bif == NULL) return (ENOENT); + /* bridge_rtupdate() may acquire the lock. */ + BRIDGE_UNLOCK(sc); error = bridge_rtupdate(sc, req->ifba_dst, req->ifba_vlan, bif, 1, req->ifba_flags); + BRIDGE_LOCK(sc); return (error); } @@ -1874,6 +1883,7 @@ { struct bridge_softc *sc = ifp->if_bridge; struct bridge_iflist *bif; + struct epoch_tracker et; if (ifp->if_flags & IFF_RENAMING) return; @@ -1884,6 +1894,7 @@ */ return; } + NET_EPOCH_ENTER(et); /* Check if the interface is a bridge member */ if (sc != NULL) { BRIDGE_LOCK(sc); @@ -1893,6 +1904,7 @@ bridge_delete_member(sc, bif, 1); BRIDGE_UNLOCK(sc); + NET_EPOCH_EXIT(et); return; } @@ -1909,6 +1921,7 @@ BRIDGE_UNLOCK(sc); } BRIDGE_LIST_UNLOCK(); + NET_EPOCH_EXIT(et); } /* @@ -1945,6 +1958,7 @@ { struct bridge_softc *sc = ifp->if_softc; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) @@ -2068,6 +2082,8 @@ struct bridge_softc *sc; uint16_t vlan; + NET_EPOCH_ASSERT(); + if (m->m_len < ETHER_HDR_LEN) { m = m_pullup(m, ETHER_HDR_LEN); if (m == NULL) @@ -2078,7 +2094,6 @@ sc = ifp->if_bridge; vlan = VLANTAGOF(m); - BRIDGE_LOCK(sc); bifp = sc->sc_ifp; /* @@ -2105,16 +2120,10 @@ if (dst_if == NULL) { struct bridge_iflist *bif; struct mbuf *mc; - int error = 0, used = 0; + int used = 0; bridge_span(sc, m); - BRIDGE_LOCK2REF(sc, error); - if (error) { - m_freem(m); - return (0); - } - CK_LIST_FOREACH(bif, &sc->sc_iflist, bif_next) { dst_if = bif->bif_ifp; @@ -2148,7 +2157,6 @@ } if (used == 0) m_freem(m); - BRIDGE_UNREF(sc); return (0); } @@ -2160,11 +2168,9 @@ bridge_span(sc, m); if ((dst_if->if_drv_flags & IFF_DRV_RUNNING) == 0) { m_freem(m); - BRIDGE_UNLOCK(sc); return (0); } - BRIDGE_UNLOCK(sc); bridge_enqueue(sc, dst_if, m); return (0); } @@ -2189,10 +2195,8 @@ eh = mtod(m, struct ether_header *); - BRIDGE_LOCK(sc); if (((m->m_flags & (M_BCAST|M_MCAST)) == 0) && (dst_if = bridge_rtlookup(sc, eh->ether_dhost, 1)) != NULL) { - BRIDGE_UNLOCK(sc); error = bridge_enqueue(sc, dst_if, m); } else bridge_broadcast(sc, ifp, m, 0); @@ -2226,6 +2230,8 @@ uint8_t *dst; int error; + NET_EPOCH_ASSERT(); + src_if = m->m_pkthdr.rcvif; ifp = sc->sc_ifp; @@ -2304,12 +2310,10 @@ || PFIL_HOOKED_IN(V_inet6_pfil_head) #endif ) { - BRIDGE_UNLOCK(sc); if (bridge_pfil(&m, ifp, src_if, PFIL_IN) != 0) return; if (m == NULL) return; - BRIDGE_LOCK(sc); } if (dst_if == NULL) { @@ -2337,8 +2341,6 @@ dbif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) goto drop; - BRIDGE_UNLOCK(sc); - if (PFIL_HOOKED_OUT(V_inet_pfil_head) #ifdef INET6 || PFIL_HOOKED_OUT(V_inet6_pfil_head) @@ -2354,7 +2356,6 @@ return; drop: - BRIDGE_UNLOCK(sc); m_freem(m); } @@ -2375,6 +2376,8 @@ uint16_t vlan; int error; + NET_EPOCH_ASSERT(); + if ((sc->sc_ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return (m); @@ -2395,10 +2398,8 @@ m_freem(m); return (NULL); } - BRIDGE_LOCK(sc); bif = bridge_lookup_member_if(sc, ifp); if (bif == NULL) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2411,13 +2412,11 @@ if (memcmp(eh->ether_dhost, bstp_etheraddr, ETHER_ADDR_LEN) == 0) { bstp_input(&bif->bif_stp, ifp, m); /* consumes mbuf */ - BRIDGE_UNLOCK(sc); return (NULL); } if ((bif->bif_flags & IFBIF_STP) && bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2428,7 +2427,6 @@ */ mc = m_dup(m, M_NOWAIT); if (mc == NULL) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2460,7 +2458,6 @@ if ((bif->bif_flags & IFBIF_STP) && bif->bif_stp.bp_state == BSTP_IFSTATE_DISCARDING) { - BRIDGE_UNLOCK(sc); return (m); } @@ -2494,7 +2491,6 @@ error = bridge_rtupdate(sc, eh->ether_shost, \ vlan, bif, 0, IFBAF_DYNAMIC); \ if (error && bif->bif_addrmax) { \ - BRIDGE_UNLOCK(sc); \ m_freem(m); \ return (NULL); \ } \ @@ -2502,7 +2498,6 @@ m->m_pkthdr.rcvif = iface; \ if ((iface) == ifp) { \ /* Skip bridge processing... src == dest */ \ - BRIDGE_UNLOCK(sc); \ return (m); \ } \ /* It's passing over or to the bridge, locally. */ \ @@ -2514,13 +2509,11 @@ OR_PFIL_HOOKED_INET6)) { \ if (bridge_pfil(&m, NULL, ifp, \ PFIL_IN) != 0 || m == NULL) { \ - BRIDGE_UNLOCK(sc); \ return (NULL); \ } \ } \ if ((iface) != bifp) \ ETHER_BPF_MTAP(iface, m); \ - BRIDGE_UNLOCK(sc); \ return (m); \ } \ \ @@ -2528,7 +2521,6 @@ if (memcmp(IF_LLADDR((iface)), eh->ether_shost, ETHER_ADDR_LEN) == 0 \ OR_CARP_CHECK_WE_ARE_SRC((iface)) \ ) { \ - BRIDGE_UNLOCK(sc); \ m_freem(m); \ return (NULL); \ } @@ -2579,15 +2571,11 @@ struct bridge_iflist *dbif, *sbif; struct mbuf *mc; struct ifnet *dst_if; - int error = 0, used = 0, i; + int used = 0, i; - sbif = bridge_lookup_member_if(sc, src_if); + NET_EPOCH_ASSERT(); - BRIDGE_LOCK2REF(sc, error); - if (error) { - m_freem(m); - return; - } + sbif = bridge_lookup_member_if(sc, src_if); /* Filter on the bridge interface before broadcasting */ if (runfilt && (PFIL_HOOKED_OUT(V_inet_pfil_head) @@ -2596,9 +2584,9 @@ #endif )) { if (bridge_pfil(&m, sc->sc_ifp, NULL, PFIL_OUT) != 0) - goto out; + return; if (m == NULL) - goto out; + return; } CK_LIST_FOREACH(dbif, &sc->sc_iflist, bif_next) { @@ -2661,9 +2649,6 @@ } if (used == 0) m_freem(m); - -out: - BRIDGE_UNREF(sc); } /* @@ -2679,6 +2664,8 @@ struct ifnet *dst_if; struct mbuf *mc; + NET_EPOCH_ASSERT(); + if (CK_LIST_EMPTY(&sc->sc_spanlist)) return; @@ -2710,7 +2697,8 @@ struct bridge_rtnode *brt; int error; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); + BRIDGE_UNLOCK_ASSERT(sc); /* Check the source address is valid and not multicast. */ if (ETHER_IS_MULTICAST(dst) || @@ -2727,13 +2715,24 @@ * update it, otherwise create a new one. */ if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) == NULL) { + BRIDGE_LOCK(sc); + + /* Check again, now that we have the lock. There could have + * been a race and we only want to insert this once. */ + if ((brt = bridge_rtnode_lookup(sc, dst, vlan)) != NULL) { + BRIDGE_UNLOCK(sc); + return (0); + } + if (sc->sc_brtcnt >= sc->sc_brtmax) { sc->sc_brtexceeded++; + BRIDGE_UNLOCK(sc); return (ENOSPC); } /* Check per interface address limits (if enabled) */ if (bif->bif_addrmax && bif->bif_addrcnt >= bif->bif_addrmax) { bif->bif_addrexceeded++; + BRIDGE_UNLOCK(sc); return (ENOSPC); } @@ -2743,8 +2742,11 @@ * address. */ brt = uma_zalloc(V_bridge_rtnode_zone, M_NOWAIT | M_ZERO); - if (brt == NULL) + if (brt == NULL) { + BRIDGE_UNLOCK(sc); return (ENOMEM); + } + brt->brt_vnet = curvnet; if (bif->bif_flags & IFBIF_STICKY) brt->brt_flags = IFBAF_STICKY; @@ -2756,17 +2758,22 @@ if ((error = bridge_rtnode_insert(sc, brt)) != 0) { uma_zfree(V_bridge_rtnode_zone, brt); + BRIDGE_UNLOCK(sc); return (error); } brt->brt_dst = bif; bif->bif_addrcnt++; + + BRIDGE_UNLOCK(sc); } if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC && brt->brt_dst != bif) { + BRIDGE_LOCK(sc); brt->brt_dst->bif_addrcnt--; brt->brt_dst = bif; brt->brt_dst->bif_addrcnt++; + BRIDGE_UNLOCK(sc); } if ((flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) @@ -2787,7 +2794,7 @@ { struct bridge_rtnode *brt; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); if ((brt = bridge_rtnode_lookup(sc, addr, vlan)) == NULL) return (NULL); @@ -2807,6 +2814,7 @@ { struct bridge_rtnode *brt, *nbrt; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); /* Make sure we actually need to do this. */ @@ -2836,7 +2844,9 @@ bridge_timer(void *arg) { struct bridge_softc *sc = arg; + struct epoch_tracker et; + NET_EPOCH_ENTER(et); BRIDGE_LOCK_ASSERT(sc); /* Destruction of rtnodes requires a proper vnet context */ @@ -2847,6 +2857,7 @@ callout_reset(&sc->sc_brcallout, bridge_rtable_prune_period * hz, bridge_timer, sc); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -2859,6 +2870,7 @@ { struct bridge_rtnode *brt, *nbrt; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { @@ -2879,6 +2891,7 @@ { struct bridge_rtnode *brt, *nbrt; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { @@ -2898,6 +2911,7 @@ struct bridge_rtnode *brt; int found = 0; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); /* @@ -2922,6 +2936,7 @@ { struct bridge_rtnode *brt, *nbrt; + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { @@ -3026,7 +3041,7 @@ uint32_t hash; int dir; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); hash = bridge_rthash(sc, addr); CK_LIST_FOREACH(brt, &sc->sc_rthash[hash], brt_hash) { @@ -3089,6 +3104,18 @@ return (0); } +static void +bridge_rtnode_destroy_cb(struct epoch_context *ctx) +{ + struct bridge_rtnode *brt; + + brt = __containerof(ctx, struct bridge_rtnode, brt_epoch_ctx); + + CURVNET_SET(brt->brt_vnet); + uma_zfree(V_bridge_rtnode_zone, brt); + CURVNET_RESTORE(); +} + /* * bridge_rtnode_destroy: * @@ -3097,6 +3124,7 @@ static void bridge_rtnode_destroy(struct bridge_softc *sc, struct bridge_rtnode *brt) { + NET_EPOCH_ASSERT(); BRIDGE_LOCK_ASSERT(sc); CK_LIST_REMOVE(brt, brt_hash); @@ -3104,7 +3132,8 @@ CK_LIST_REMOVE(brt, brt_list); sc->sc_brtcnt--; brt->brt_dst->bif_addrcnt--; - uma_zfree(V_bridge_rtnode_zone, brt); + + NET_EPOCH_CALL(bridge_rtnode_destroy_cb, &brt->brt_epoch_ctx); } /* @@ -3117,7 +3146,9 @@ { struct bridge_softc *sc = ifp->if_bridge; struct bridge_rtnode *brt; + struct epoch_tracker et; + NET_EPOCH_ENTER(et); CURVNET_SET(ifp->if_vnet); BRIDGE_LOCK(sc); @@ -3138,6 +3169,7 @@ } BRIDGE_UNLOCK(sc); CURVNET_RESTORE(); + NET_EPOCH_EXIT(et); } /* @@ -3643,17 +3675,20 @@ { struct bridge_softc *sc = ifp->if_bridge; struct bridge_iflist *bif; + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); - BRIDGE_LOCK(sc); bif = bridge_lookup_member_if(sc, ifp); if (bif == NULL) { - BRIDGE_UNLOCK(sc); + NET_EPOCH_EXIT(et); return; } bridge_linkcheck(sc); - BRIDGE_UNLOCK(sc); bstp_linkstate(&bif->bif_stp); + + NET_EPOCH_EXIT(et); } static void @@ -3662,7 +3697,8 @@ struct bridge_iflist *bif; int new_link, hasls; - BRIDGE_LOCK_ASSERT(sc); + NET_EPOCH_ASSERT(); + new_link = LINK_STATE_DOWN; hasls = 0; /* Our link is considered up if at least one of our ports is active */