diff --git a/sys/conf/files b/sys/conf/files --- a/sys/conf/files +++ b/sys/conf/files @@ -4182,6 +4182,9 @@ net/route/nhop_ctl.c standard net/route/nhop_utils.c standard net/route/fib_algo.c optional fib_algo +net/route/route_cache.c optional inet | inet6 +net/route/route_cache_in.c optional inet +net/route/route_cache_in6.c optional inet6 net/route/route_ctl.c standard net/route/route_ddb.c optional ddb net/route/route_helpers.c standard diff --git a/sys/net/route/route_cache.h b/sys/net/route/route_cache.h new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache.h @@ -0,0 +1,129 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NET_ROUTE_ROUTE_CACHE_H_ +#define _NET_ROUTE_ROUTE_CACHE_H_ + +#ifdef INET +struct route_cache_pcpu { + struct mtx mtx; + struct route ro; +} __aligned(CACHE_LINE_SIZE); +#endif + +#ifdef INET6 +struct route_cache_pcpu6 { + struct mtx mtx; + struct route_in6 ro6; +} __aligned(CACHE_LINE_SIZE); +#endif + +struct route_cache { + union { +#ifdef INET + struct route_cache_pcpu *pcpu; +#endif +#ifdef INET6 + struct route_cache_pcpu6 *pcpu6; +#endif + }; + struct rib_subscription *rs; +}; + +void route_cache_init(struct route_cache *, int); +void route_cache_uninit(struct route_cache *, int); +void route_cache_invalidate(struct route_cache *, int); +void route_cache_subscribe_rib_event(struct route_cache *, int, uint32_t); +void route_cache_unsubscribe_rib_event(struct route_cache *); + +#ifdef INET +void route_cache_init_in(struct route_cache *); +void route_cache_uninit_in(struct route_cache *); +void route_cache_invalidate_in(struct route_cache *); +void route_cache_subscribe_rib_event_in(struct route_cache *, uint32_t); + +static inline struct route * +route_cache_acquire(struct route_cache *rc) +{ + struct route_cache_pcpu *pcpu; + struct route *ro = NULL; + + pcpu = zpcpu_get(rc->pcpu); + if (mtx_trylock(&pcpu->mtx)) + ro = &pcpu->ro; + + return ro; +} + +static inline void +route_cache_release(struct route *ro) +{ + struct route_cache_pcpu *pcpu; + + if (ro != NULL) { + pcpu = __containerof(ro, struct route_cache_pcpu, ro); + mtx_assert(&pcpu->mtx, MA_OWNED); + mtx_unlock(&pcpu->mtx); + } +} +#endif + +#ifdef INET6 +void route_cache_init_in6(struct route_cache *); +void route_cache_uninit_in6(struct route_cache *); +void route_cache_invalidate_in6(struct route_cache *); +void route_cache_subscribe_rib_event_in6(struct route_cache *, uint32_t); + +static inline struct route_in6 * +route_cache_acquire6(struct route_cache *rc) +{ + struct route_cache_pcpu6 *pcpu; + struct route_in6 *ro = NULL; + + pcpu = zpcpu_get(rc->pcpu6); + if (mtx_trylock(&pcpu->mtx)) + ro = &pcpu->ro6; + + return ro; +} + +static inline void +route_cache_release6(struct route_in6 *ro) +{ + struct route_cache_pcpu6 *pcpu; + + if (ro != NULL) { + pcpu = __containerof(ro, struct route_cache_pcpu6, ro6); + mtx_assert(&pcpu->mtx, MA_OWNED); + mtx_unlock(&pcpu->mtx); + } +} +#endif + +#endif diff --git a/sys/net/route/route_cache.c b/sys/net/route/route_cache.c new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache.c @@ -0,0 +1,153 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + + +void +route_cache_init(struct route_cache *rc, int family) +{ + switch (family) { +#ifdef INET + case AF_INET: + route_cache_init_in(rc); + break; +#endif +#ifdef INET6 + case AF_INET6: + route_cache_init_in6(rc); + break; +#endif + default: + // Unreachable + panic("Unsupported af: %d", family); + } +} + +void +route_cache_uninit(struct route_cache *rc, int family) +{ + switch (family) { +#ifdef INET + case AF_INET: + route_cache_uninit_in(rc); + break; +#endif +#ifdef INET6 + case AF_INET6: + route_cache_uninit_in6(rc); + break; +#endif + default: + // Unreachable + panic("Unsupported af: %d", family); + } +} + +void +route_cache_invalidate(struct route_cache *rc, int family) +{ + switch (family) { +#ifdef INET + case AF_INET: + route_cache_invalidate_in(rc); + break; +#endif +#ifdef INET6 + case AF_INET6: + route_cache_invalidate_in6(rc); + break; +#endif + default: + // Unreachable + panic("Unsupported af: %d", family); + } +} + +void +route_cache_subscribe_rib_event(struct route_cache *rc, int family, + uint32_t fibnum) +{ + switch (family) { +#ifdef INET + case AF_INET: + route_cache_subscribe_rib_event_in(rc, fibnum); + break; +#endif +#ifdef INET6 + case AF_INET6: + route_cache_subscribe_rib_event_in6(rc, fibnum); + break; +#endif + default: + // Unreachable + panic("Unsupported af: %d", family); + } +} + +void +route_cache_unsubscribe_rib_event(struct route_cache *rc) +{ + struct epoch_tracker et; + + KASSERT((rc->rs != NULL), ("not subscribed rib event")); + NET_EPOCH_ENTER(et); + rib_unsubscribe(rc->rs); + NET_EPOCH_EXIT(et); + rc->rs = NULL; +} diff --git a/sys/net/route/route_cache_in.c b/sys/net/route/route_cache_in.c new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache_in.c @@ -0,0 +1,149 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#include "opt_inet.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + + +static uma_zone_t route_cache_pcpu_zone; + +static void +route_cache_pcpu_zone_init(void) +{ + route_cache_pcpu_zone = uma_zcreate("route-cache-pcpu", + sizeof(struct route_cache_pcpu), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} + +SYSINIT(route_cache_pcpu_zone_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, + route_cache_pcpu_zone_init, NULL); + +static void +route_cache_pcpu_zone_uninit(void) +{ + uma_zdestroy(route_cache_pcpu_zone); +} + +SYSUNINIT(route_cache_pcpu_zone_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, + route_cache_pcpu_zone_uninit, NULL); + + +void +route_cache_init_in(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu, *rc_pcpu; + + KASSERT((rc->pcpu == NULL), ("route cache has been inited")); + KASSERT((rc->rs == NULL), ("route cache has subscribed rib event")); + rc_pcpu = uma_zalloc_pcpu(route_cache_pcpu_zone, M_WAITOK | M_ZERO); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc_pcpu, cpu); + mtx_init(&pcpu->mtx, "route_cache_pcpu_mtx", NULL, MTX_DEF); + pcpu->ro.ro_flags = RT_LLE_CACHE; /* Cache L2 as well */ + } + rc->pcpu = rc_pcpu; +} + +void +route_cache_uninit_in(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu; + + KASSERT((rc->rs == NULL), ("should unsubscribe rib event before uninit")); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu, cpu); + mtx_assert(&pcpu->mtx, MA_NOTOWNED); + // XXX use atomic_thread_fence_acq ? + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro); + mtx_unlock(&pcpu->mtx); + mtx_destroy(&pcpu->mtx); + } + uma_zfree_pcpu(route_cache_pcpu_zone, rc->pcpu); + rc->pcpu = NULL; +} + +void +route_cache_invalidate_in(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu *pcpu; + + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu, cpu); + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro); + mtx_unlock(&pcpu->mtx); + } +} + +static void +route_cache_subscription_cb_in(struct rib_head *rnh __unused, + struct rib_cmd_info *rci __unused, void *arg) +{ + struct route_cache *rc = arg; + // XXX revalidate should be enough, NH_VALIDATE + route_cache_invalidate_in(rc); +} + +void +route_cache_subscribe_rib_event_in(struct route_cache *rc, uint32_t fibnum) +{ + KASSERT((rc->rs == NULL), ("already subscribed rib event")); + rc->rs = rib_subscribe(fibnum, AF_INET, route_cache_subscription_cb_in, + rc, RIB_NOTIFY_IMMEDIATE, true); +} diff --git a/sys/net/route/route_cache_in6.c b/sys/net/route/route_cache_in6.c new file mode 100644 --- /dev/null +++ b/sys/net/route/route_cache_in6.c @@ -0,0 +1,150 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2022 Zhenlei Huang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#include "opt_inet6.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + + +static uma_zone_t route_cache_pcpu6_zone; + +static void +route_cache_pcpu6_zone_init(void) +{ + route_cache_pcpu6_zone = uma_zcreate("route-cache-pcpu6", + sizeof(struct route_cache_pcpu6), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} + +SYSINIT(route_cache_pcpu6_zone_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, + route_cache_pcpu6_zone_init, NULL); + +static void +route_cache_pcpu6_zone_uninit(void) +{ + uma_zdestroy(route_cache_pcpu6_zone); +} + +SYSUNINIT(route_cache_pcpu6_zone_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, + route_cache_pcpu6_zone_uninit, NULL); + + +void +route_cache_init_in6(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu6 *pcpu, *rc_pcpu; + + KASSERT((rc->pcpu6 == NULL), ("route cache has been inited")); + KASSERT((rc->rs == NULL), ("route cache has subscribed rib event")); + rc_pcpu = uma_zalloc_pcpu(route_cache_pcpu6_zone, M_WAITOK | M_ZERO); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc_pcpu, cpu); + mtx_init(&pcpu->mtx, "route_cache_pcpu6_mtx", NULL, MTX_DEF); + pcpu->ro6.ro_flags = RT_LLE_CACHE; /* Cache L2 as well */ + } + rc->pcpu6 = rc_pcpu; +} + +void +route_cache_uninit_in6(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu6 *pcpu; + + KASSERT((rc->rs == NULL), ("should unsubscribe rib event before uninit")); + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu6, cpu); + mtx_assert(&pcpu->mtx, MA_NOTOWNED); + // XXX use atomic_thread_fence_acq ? + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro6); + mtx_unlock(&pcpu->mtx); + mtx_destroy(&pcpu->mtx); + } + uma_zfree_pcpu(route_cache_pcpu6_zone, rc->pcpu6); + rc->pcpu6 = NULL; +} + +void +route_cache_invalidate_in6(struct route_cache *rc) +{ + int cpu; + struct route_cache_pcpu6 *pcpu; + + CPU_FOREACH(cpu) { + pcpu = zpcpu_get_cpu(rc->pcpu6, cpu); + mtx_lock(&pcpu->mtx); + RO_INVALIDATE_CACHE(&pcpu->ro6); + mtx_unlock(&pcpu->mtx); + } +} + +static void +route_cache_subscription_cb_in6(struct rib_head *rnh __unused, + struct rib_cmd_info *rci __unused, void *arg) +{ + struct route_cache *rc = arg; + // XXX revalidate should be enough, NH_VALIDATE + route_cache_invalidate_in6(rc); +} + +void +route_cache_subscribe_rib_event_in6(struct route_cache *rc, uint32_t fibnum) +{ + KASSERT((rc->rs == NULL), ("already subscribed rib event")); + rc->rs = rib_subscribe(fibnum, AF_INET6, route_cache_subscription_cb_in6, + rc, RIB_NOTIFY_IMMEDIATE, true); +}