diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -4184,6 +4184,7 @@ net/route/nhop_ctl.c standard net/route/nhop_utils.c standard net/route/fib_algo.c optional fib_algo +net/route/route_cache.c optional inet | inet6 net/route/route_ctl.c standard net/route/route_ddb.c optional ddb net/route/route_helpers.c standard diff --git a/sys/net/route/route_cache.h b/sys/net/route/route_cache.h new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache.h @@ -0,0 +1,109 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NET_ROUTE_ROUTE_CACHE_H_ +#define _NET_ROUTE_ROUTE_CACHE_H_ + +struct route_cache_pcpu { + struct mtx mtx; + union { + struct route ro; +#ifdef INET + struct route_in ro4; +#endif +#ifdef INET6 + struct route_in6 ro6; +#endif + }; +} __aligned(CACHE_LINE_SIZE); + +struct route_cache { + struct route_cache_pcpu *pcpu; + struct rib_subscription *rs; +}; + +void route_cache_init(struct route_cache *); +void route_cache_uninit(struct route_cache *); +void route_cache_invalidate(struct route_cache *); +void route_cache_subscribe_rib_event(struct route_cache *, int, uint32_t); +void route_cache_unsubscribe_rib_event(struct route_cache *); + +static inline struct route * +route_cache_acquire(struct route_cache *rc) +{ + struct route_cache_pcpu *pcpu; + struct route *ro = NULL; + + pcpu = zpcpu_get(rc->pcpu); + if (mtx_trylock(&pcpu->mtx)) + ro = &pcpu->ro; + + return ro; +} + +static inline void +route_cache_release(struct route *ro) +{ + struct route_cache_pcpu *pcpu; + + if (ro != NULL) { + pcpu = __containerof(ro, struct route_cache_pcpu, ro); + mtx_assert(&pcpu->mtx, MA_OWNED); + mtx_unlock(&pcpu->mtx); + } +} + +#ifdef INET6 +static inline struct route_in6 * +route_cache_acquire6(struct route_cache *rc) +{ + struct route_cache_pcpu *pcpu; + struct route_in6 *ro = NULL; + + pcpu = zpcpu_get(rc->pcpu); + if (mtx_trylock(&pcpu->mtx)) + ro = &pcpu->ro6; + + return ro; +} + +static inline void +route_cache_release6(struct route_in6 *ro) +{ + struct route_cache_pcpu *pcpu; + + if (ro != NULL) { + pcpu = __containerof(ro, struct route_cache_pcpu, ro6); + mtx_assert(&pcpu->mtx, MA_OWNED); + mtx_unlock(&pcpu->mtx); + } +} +#endif + +#endif diff --git a/sys/net/route/route_cache.c b/sys/net/route/route_cache.c new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache.c @@ -0,0 +1,166 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + + +static uma_zone_t route_cache_pcpu_zone; + +static void +route_cache_pcpu_zone_init(void) +{ + route_cache_pcpu_zone = uma_zcreate("route-cache-pcpu", + sizeof(struct route_cache_pcpu), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} + +SYSINIT(route_cache_pcpu_zone_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, + route_cache_pcpu_zone_init, NULL); + +static void +route_cache_pcpu_zone_uninit(void) +{ + uma_zdestroy(route_cache_pcpu_zone); +} + +SYSUNINIT(route_cache_pcpu_zone_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, + route_cache_pcpu_zone_uninit, NULL); + +void +route_cache_init(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu, *rc_pcpu; + + KASSERT((rc->pcpu == NULL), ("route cache has been inited")); + KASSERT((rc->rs == NULL), ("route cache has subscribed rib event")); + rc_pcpu = uma_zalloc_pcpu(route_cache_pcpu_zone, M_WAITOK | M_ZERO); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc_pcpu, cpu); + mtx_init(&pcpu->mtx, "route_cache_pcpu_mtx", NULL, MTX_DEF); + pcpu->ro.ro_flags = RT_LLE_CACHE; /* Cache L2 as well */ + } + rc->pcpu = rc_pcpu; +} + +void +route_cache_uninit(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu; + + KASSERT((rc->pcpu != NULL), ("route cache is not inited")); + KASSERT((rc->rs == NULL), ("should unsubscribe rib event before uninit")); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu, cpu); + mtx_assert(&pcpu->mtx, MA_NOTOWNED); + // XXX use atomic_thread_fence_acq ? + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro); + mtx_unlock(&pcpu->mtx); + mtx_destroy(&pcpu->mtx); + } + uma_zfree_pcpu(route_cache_pcpu_zone, rc->pcpu); + rc->pcpu = NULL; +} + +void +route_cache_invalidate(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu; + + KASSERT((rc->pcpu != NULL), ("route cache is not inited")); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu, cpu); + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro); + mtx_unlock(&pcpu->mtx); + } +} + +static void +route_cache_subscription_cb(struct rib_head *rnh __unused, + struct rib_cmd_info *rci __unused, void *arg) +{ + struct route_cache *rc = arg; + + route_cache_invalidate(rc); +} + +void +route_cache_subscribe_rib_event(struct route_cache *rc, int family, + uint32_t fibnum) +{ + KASSERT((rc->rs == NULL), ("already subscribed rib event")); + rc->rs = rib_subscribe(fibnum, family, route_cache_subscription_cb, + rc, RIB_NOTIFY_IMMEDIATE, true); +} + +void +route_cache_unsubscribe_rib_event(struct route_cache *rc) +{ + struct epoch_tracker et; + + KASSERT((rc->rs != NULL), ("not subscribed rib event")); + NET_EPOCH_ENTER(et); + rib_unsubscribe(rc->rs); + NET_EPOCH_EXIT(et); + rc->rs = NULL; +}