diff --git a/sys/net/route/mpath_ctl.c b/sys/net/route/mpath_ctl.c index b3f716875fa9..ffaaca918eec 100644 --- a/sys/net/route/mpath_ctl.c +++ b/sys/net/route/mpath_ctl.c @@ -1,194 +1,143 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_inet.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This file contains the supporting functions for adding/deleting/updating * multipath routes to the routing table. */ SYSCTL_DECL(_net_route); VNET_DEFINE(u_int, fib_hash_outbound) = 0; SYSCTL_UINT(_net_route, OID_AUTO, hash_outbound, CTLFLAG_RD | CTLFLAG_VNET, &VNET_NAME(fib_hash_outbound), 0, "Compute flowid for locally-originated packets"); /* Default entropy to add to the hash calculation for the outbound connections*/ uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN] = { 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, }; /* * Tries to add @rnd_add nhop to the existing set of nhops (@nh_orig) for the * prefix specified by @rt. * * Return 0 ans consumes rt / rnd_add nhop references. @rc gets populated * with the operation result. * Otherwise errno is returned. * * caller responsibility is to unlock/free rt and * rt->rt_nhop. */ int add_route_mpath(struct rib_head *rnh, struct rt_addrinfo *info, struct rtentry *rt, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc) { RIB_RLOCK_TRACKER; struct route_nhop_data rnd_new; int error = 0; /* * It is possible that multiple rtsock speakers will try to update * the same route simultaneously. Reduce the chance of failing the * request by retrying the cycle multiple times. */ for (int i = 0; i < RIB_MAX_RETRIES; i++) { error = nhgrp_get_addition_group(rnh, rnd_orig, rnd_add, &rnd_new); if (error != 0) { if (error != EAGAIN) break; /* * Group creation failed, most probably because * @rnd_orig data got scheduled for deletion. * Refresh @rnd_orig data and retry. */ RIB_RLOCK(rnh); lookup_prefix_rt(rnh, rt, rnd_orig); RIB_RUNLOCK(rnh); continue; } error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); if (error != EAGAIN) break; RTSTAT_INC(rts_add_retry); } if (V_fib_hash_outbound == 0 && error == 0 && NH_IS_NHGRP(rc->rc_nh_new)) { /* * First multipath route got installed. Enable local * outbound connections hashing. */ if (bootverbose) printf("FIB: enabled flowid calculation for locally-originated packets\n"); V_fib_hash_outbound = 1; } return (error); } - -struct rt_match_info { - struct rt_addrinfo *info; - struct rtentry *rt; -}; - -static bool -gw_filter_func(const struct nhop_object *nh, void *_data) -{ - struct rt_match_info *ri = (struct rt_match_info *)_data; - - return (check_info_match_nhop(ri->info, ri->rt, nh) == 0); -} - -/* - * Tries to delete matching paths from @nhg. - * Returns 0 on success and updates operation result in @rc. - */ -int -del_route_mpath(struct rib_head *rh, struct rt_addrinfo *info, - struct rtentry *rt, struct nhgrp_object *nhg, - struct rib_cmd_info *rc) -{ - struct route_nhop_data rnd; - struct rt_match_info ri = { .info = info, .rt = rt }; - int error; - - RIB_WLOCK_ASSERT(rh); - - /* - * Require gateway to delete multipath routes, to forbid - * deleting all paths at once. - * If the filter function is provided, skip gateway check to - * allow rib_walk_del() delete routes for any criteria based - * on provided callback. - */ - if ((info->rti_info[RTAX_GATEWAY] == NULL) && (info->rti_filter == NULL)) - return (ESRCH); - - error = nhgrp_get_filtered_group(rh, nhg, gw_filter_func, (void *)&ri, &rnd); - if (error == 0) { - if (rnd.rnd_nhgrp == nhg) { - /* No gateway match, unreference new group and return. */ - nhop_free_any(rnd.rnd_nhop); - return (ESRCH); - } - error = change_route(rh, rt, &rnd, rc); - } - return (error); -} - diff --git a/sys/net/route/nhgrp_ctl.c b/sys/net/route/nhgrp_ctl.c index f0b26916136c..3a7ebee8def2 100644 --- a/sys/net/route/nhgrp_ctl.c +++ b/sys/net/route/nhgrp_ctl.c @@ -1,891 +1,892 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_inet.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME nhgrp_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains the supporting functions for creating multipath groups * and compiling their dataplane parts. */ /* MPF_MULTIPATH must be the same as NHF_MULTIPATH for nhop selection to work */ _Static_assert(MPF_MULTIPATH == NHF_MULTIPATH, "MPF_MULTIPATH must be the same as NHF_MULTIPATH"); /* Offset and size of flags field has to be the same for nhop/nhop groups */ CHK_STRUCT_FIELD_GENERIC(struct nhop_object, nh_flags, struct nhgrp_object, nhg_flags); /* Cap multipath to 64, as the larger values would break rib_cmd_info bmasks */ CTASSERT(RIB_MAX_MPATH_WIDTH <= 64); static int wn_cmp_idx(const void *a, const void *b); static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops); static struct nhgrp_priv *get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, int *perror); static void destroy_nhgrp(struct nhgrp_priv *nhg_priv); static void destroy_nhgrp_epoch(epoch_context_t ctx); static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv); static int wn_cmp_idx(const void *a, const void *b) { const struct weightened_nhop *w_a = a; const struct weightened_nhop *w_b = b; uint32_t a_idx = w_a->nh->nh_priv->nh_idx; uint32_t b_idx = w_b->nh->nh_priv->nh_idx; if (a_idx < b_idx) return (-1); else if (a_idx > b_idx) return (1); else return (0); } /* * Perform in-place sorting for array of nexthops in @wn. * Sort by nexthop index ascending. */ static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops) { qsort(wn, num_nhops, sizeof(struct weightened_nhop), wn_cmp_idx); } /* * In order to determine the minimum weight difference in the array * of weights, create a sorted array of weights, using spare "storage" * field in the `struct weightened_nhop`. * Assume weights to be (mostly) the same and use insertion sort to * make it sorted. */ static void sort_weightened_nhops_weights(struct weightened_nhop *wn, int num_items) { wn[0].storage = wn[0].weight; for (int i = 1, j = 0; i < num_items; i++) { uint32_t weight = wn[i].weight; // read from 'weight' as it's not reordered /* Move all weights > weight 1 position right */ for (j = i - 1; j >= 0 && wn[j].storage > weight; j--) wn[j + 1].storage = wn[j].storage; wn[j + 1].storage = weight; } } /* * Calculate minimum number of slots required to fit the existing * set of weights in the common use case where weights are "easily" * comparable. * Assumes @wn is sorted by weight ascending and each weight is > 0. * Returns number of slots or 0 if precise calculation failed. * * Some examples: * note: (i, X) pair means (nhop=i, weight=X): * (1, 1) (2, 2) -> 3 slots [1, 2, 2] * (1, 100), (2, 200) -> 3 slots [1, 2, 2] * (1, 100), (2, 200), (3, 400) -> 7 slots [1, 2, 2, 3, 3, 3] */ static uint32_t calc_min_mpath_slots_fast(struct weightened_nhop *wn, size_t num_items, uint64_t *ptotal) { uint32_t i, last, xmin; uint64_t total = 0; // Get sorted array of weights in .storage field sort_weightened_nhops_weights(wn, num_items); last = 0; xmin = wn[0].storage; for (i = 0; i < num_items; i++) { total += wn[i].storage; if ((wn[i].storage != last) && ((wn[i].storage - last < xmin) || xmin == 0)) { xmin = wn[i].storage - last; } last = wn[i].storage; } *ptotal = total; /* xmin is the minimum unit of desired capacity */ if ((total % xmin) != 0) return (0); for (i = 0; i < num_items; i++) { if ((wn[i].weight % xmin) != 0) return (0); } return ((uint32_t)(total / xmin)); } /* * Calculate minimum number of slots required to fit the existing * set of weights while maintaining weight coefficients. * * Assume @wn is sorted by weight ascending and each weight is > 0. * * Tries to find simple precise solution first and falls back to * RIB_MAX_MPATH_WIDTH in case of any failure. */ static uint32_t calc_min_mpath_slots(struct weightened_nhop *wn, size_t num_items) { uint32_t v; uint64_t total; v = calc_min_mpath_slots_fast(wn, num_items, &total); if (total == 0) return (0); if ((v == 0) || (v > RIB_MAX_MPATH_WIDTH)) v = RIB_MAX_MPATH_WIDTH; return (v); } /* * Nexthop group data consists of * 1) dataplane part, with nhgrp_object as a header followed by an * arbitrary number of nexthop pointers. * 2) control plane part, with nhgrp_priv as a header, followed by * an arbirtrary number of 'struct weightened_nhop' object. * * Given nexthop groups are (mostly) immutable, allocate all data * in one go. * */ __noinline static size_t get_nhgrp_alloc_size(uint32_t nhg_size, uint32_t num_nhops) { size_t sz; sz = sizeof(struct nhgrp_object); sz += nhg_size * sizeof(struct nhop_object *); sz += sizeof(struct nhgrp_priv); sz += num_nhops * sizeof(struct weightened_nhop); return (sz); } /* * Compile actual list of nexthops to be used by datapath from * the nexthop group @dst. * * For example, compiling control plane list of 2 nexthops * [(200, A), (100, B)] would result in the datapath array * [A, A, B] */ static void compile_nhgrp(struct nhgrp_priv *dst_priv, const struct weightened_nhop *x, uint32_t num_slots) { struct nhgrp_object *dst; int i, slot_idx, remaining_slots; uint64_t remaining_sum, nh_weight, nh_slots; slot_idx = 0; dst = dst_priv->nhg; /* Calculate sum of all weights */ remaining_sum = 0; for (i = 0; i < dst_priv->nhg_nh_count; i++) remaining_sum += x[i].weight; remaining_slots = num_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, "sum: %lu, slots: %d", remaining_sum, remaining_slots); for (i = 0; i < dst_priv->nhg_nh_count; i++) { /* Calculate number of slots for the current nexthop */ if (remaining_sum > 0) { nh_weight = (uint64_t)x[i].weight; nh_slots = (nh_weight * remaining_slots / remaining_sum); } else nh_slots = 0; remaining_sum -= x[i].weight; remaining_slots -= nh_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, " rem_sum: %lu, rem_slots: %d nh_slots: %d, slot_idx: %d", remaining_sum, remaining_slots, (int)nh_slots, slot_idx); KASSERT((slot_idx + nh_slots <= num_slots), ("index overflow during nhg compilation")); while (nh_slots-- > 0) dst->nhops[slot_idx++] = x[i].nh; } } /* * Allocates new nexthop group for the list of weightened nexthops. * Assume sorted list. * Does NOT reference any nexthops in the group. * Returns group with refcount=1 or NULL. */ static struct nhgrp_priv * alloc_nhgrp(struct weightened_nhop *wn, int num_nhops) { uint32_t nhgrp_size; struct nhgrp_object *nhg; struct nhgrp_priv *nhg_priv; nhgrp_size = calc_min_mpath_slots(wn, num_nhops); if (nhgrp_size == 0) { /* Zero weights, abort */ return (NULL); } size_t sz = get_nhgrp_alloc_size(nhgrp_size, num_nhops); nhg = malloc(sz, M_NHOP, M_NOWAIT | M_ZERO); if (nhg == NULL) { FIB_NH_LOG(LOG_INFO, wn[0].nh, "unable to allocate group with num_nhops %d (compiled %u)", num_nhops, nhgrp_size); return (NULL); } /* Has to be the first to make NHGRP_PRIV() work */ nhg->nhg_size = nhgrp_size; nhg->nhg_flags = MPF_MULTIPATH; nhg_priv = NHGRP_PRIV(nhg); nhg_priv->nhg_nh_count = num_nhops; refcount_init(&nhg_priv->nhg_refcount, 1); /* Please see nhgrp_free() comments on the initial value */ refcount_init(&nhg_priv->nhg_linked, 2); nhg_priv->nhg = nhg; memcpy(&nhg_priv->nhg_nh_weights[0], wn, num_nhops * sizeof(struct weightened_nhop)); FIB_NH_LOG(LOG_DEBUG, wn[0].nh, "num_nhops: %d, compiled_nhop: %u", num_nhops, nhgrp_size); compile_nhgrp(nhg_priv, wn, nhg->nhg_size); return (nhg_priv); } void nhgrp_ref_object(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; u_int old __diagused; nhg_priv = NHGRP_PRIV(nhg); old = refcount_acquire(&nhg_priv->nhg_refcount); KASSERT(old > 0, ("%s: nhgrp object %p has 0 refs", __func__, nhg)); } void nhgrp_free(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; struct nh_control *ctl; struct epoch_tracker et; nhg_priv = NHGRP_PRIV(nhg); if (!refcount_release(&nhg_priv->nhg_refcount)) return; /* * group objects don't have an explicit lock attached to it. * As groups are reclaimed based on reference count, it is possible * that some groups will persist after vnet destruction callback * called. Given that, handle scenario with nhgrp_free_group() being * called either after or simultaneously with nhgrp_ctl_unlink_all() * by using another reference counter: nhg_linked. * * There are only 2 places, where nhg_linked can be decreased: * rib destroy (nhgrp_ctl_unlink_all) and this function. * nhg_link can never be increased. * * Hence, use initial value of 2 to make use of * refcount_release_if_not_last(). * * There can be two scenarious when calling this function: * * 1) nhg_linked value is 2. This means that either * nhgrp_ctl_unlink_all() has not been called OR it is running, * but we are guaranteed that nh_control won't be freed in * this epoch. Hence, nexthop can be safely unlinked. * * 2) nh_linked value is 1. In that case, nhgrp_ctl_unlink_all() * has been called and nhgrp unlink can be skipped. */ NET_EPOCH_ENTER(et); if (refcount_release_if_not_last(&nhg_priv->nhg_linked)) { ctl = nhg_priv->nh_control; if (unlink_nhgrp(ctl, nhg_priv) == NULL) { /* Do not try to reclaim */ RT_LOG(LOG_INFO, "Failed to unlink nexhop group %p", nhg_priv); NET_EPOCH_EXIT(et); return; } } NET_EPOCH_EXIT(et); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); epoch_call(net_epoch_preempt, destroy_nhgrp_epoch, &nhg_priv->nhg_epoch_ctx); } /* * Destroys all local resources belonging to @nhg_priv. */ __noinline static void destroy_nhgrp_int(struct nhgrp_priv *nhg_priv) { free(nhg_priv->nhg, M_NHOP); } __noinline static void destroy_nhgrp(struct nhgrp_priv *nhg_priv) { KASSERT((nhg_priv->nhg_refcount == 0), ("nhg_refcount != 0")); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); #if DEBUG_MAX_LEVEL >= LOG_DEBUG char nhgbuf[NHOP_PRINT_BUFSIZE]; FIB_NH_LOG(LOG_DEBUG, nhg_priv->nhg_nh_weights[0].nh, "destroying %s", nhgrp_print_buf(nhg_priv->nhg, nhgbuf, sizeof(nhgbuf))); #endif free_nhgrp_nhops(nhg_priv); destroy_nhgrp_int(nhg_priv); } /* * Epoch callback indicating group is safe to destroy */ static void destroy_nhgrp_epoch(epoch_context_t ctx) { struct nhgrp_priv *nhg_priv; nhg_priv = __containerof(ctx, struct nhgrp_priv, nhg_epoch_ctx); destroy_nhgrp(nhg_priv); } static bool ref_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { if (nhop_try_ref_object(nhg_priv->nhg_nh_weights[i].nh) != 0) continue; /* * Failed to ref the nexthop, b/c it's deleted. * Need to rollback references back. */ for (int j = 0; j < i; j++) nhop_free(nhg_priv->nhg_nh_weights[j].nh); return (false); } return (true); } static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) nhop_free(nhg_priv->nhg_nh_weights[i].nh); } /* * Creates or looks up an existing nexthop group based on @wn and @num_nhops. * * Returns referenced nhop group or NULL, passing error code in @perror. */ struct nhgrp_priv * get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, int *perror) { struct nhgrp_priv *key, *nhg_priv; if (num_nhops > RIB_MAX_MPATH_WIDTH) { *perror = E2BIG; return (NULL); } if (ctl->gr_head.hash_size == 0) { /* First multipath request. Bootstrap mpath datastructures. */ if (nhgrp_ctl_alloc_default(ctl, M_NOWAIT) == 0) { *perror = ENOMEM; return (NULL); } } /* Sort nexthops & check there are no duplicates */ sort_weightened_nhops(wn, num_nhops); uint32_t last_id = 0; for (int i = 0; i < num_nhops; i++) { if (wn[i].nh->nh_priv->nh_idx == last_id) { *perror = EEXIST; return (NULL); } last_id = wn[i].nh->nh_priv->nh_idx; } if ((key = alloc_nhgrp(wn, num_nhops)) == NULL) { *perror = ENOMEM; return (NULL); } nhg_priv = find_nhgrp(ctl, key); if (nhg_priv != NULL) { /* * Free originally-created group. As it hasn't been linked * and the dependent nexhops haven't been referenced, just free * the group. */ destroy_nhgrp_int(key); *perror = 0; return (nhg_priv); } else { /* No existing group, try to link the new one */ if (!ref_nhgrp_nhops(key)) { /* * Some of the nexthops have been scheduled for deletion. * As the group hasn't been linked / no nexhops have been * referenced, call the final destructor immediately. */ destroy_nhgrp_int(key); *perror = EAGAIN; return (NULL); } if (link_nhgrp(ctl, key) == 0) { /* Unable to allocate index? */ *perror = EAGAIN; free_nhgrp_nhops(key); destroy_nhgrp_int(key); return (NULL); } *perror = 0; return (key); } /* NOTREACHED */ } /* * Appends one or more nexthops denoted by @wm to the nexthop group @gr_orig. * * Returns referenced nexthop group or NULL. In the latter case, @perror is * filled with an error code. * Note that function does NOT care if the next nexthops already exists * in the @gr_orig. As a result, they will be added, resulting in the * same nexthop being present multiple times in the new group. */ static struct nhgrp_priv * append_nhops(struct nh_control *ctl, const struct nhgrp_object *gr_orig, struct weightened_nhop *wn, int num_nhops, int *perror) { char storage[64]; struct weightened_nhop *pnhops; struct nhgrp_priv *nhg_priv; const struct nhgrp_priv *src_priv; size_t sz; int curr_nhops; src_priv = NHGRP_PRIV_CONST(gr_orig); curr_nhops = src_priv->nhg_nh_count; *perror = 0; sz = (src_priv->nhg_nh_count + num_nhops) * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { pnhops = malloc(sz, M_TEMP, M_NOWAIT); if (pnhops == NULL) { *perror = ENOMEM; return (NULL); } } /* Copy nhops from original group first */ memcpy(pnhops, src_priv->nhg_nh_weights, curr_nhops * sizeof(struct weightened_nhop)); memcpy(&pnhops[curr_nhops], wn, num_nhops * sizeof(struct weightened_nhop)); curr_nhops += num_nhops; nhg_priv = get_nhgrp(ctl, pnhops, curr_nhops, perror); if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); if (nhg_priv == NULL) return (NULL); return (nhg_priv); } /* * Creates/finds nexthop group based on @wn and @num_nhops. * Returns 0 on success with referenced group in @rnd, or * errno. * * If the error is EAGAIN, then the operation can be retried. */ int nhgrp_get_group(struct rib_head *rh, struct weightened_nhop *wn, int num_nhops, struct nhgrp_object **pnhg) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; int error; nhg_priv = get_nhgrp(ctl, wn, num_nhops, &error); if (nhg_priv != NULL) *pnhg = nhg_priv->nhg; return (error); } /* * Creates new nexthop group based on @src group without the nexthops * chosen by @flt_func. * Returns 0 on success, storring the reference nhop group/object in @rnd. */ int -nhgrp_get_filtered_group(struct rib_head *rh, const struct nhgrp_object *src, - nhgrp_filter_cb_t flt_func, void *flt_data, struct route_nhop_data *rnd) +nhgrp_get_filtered_group(struct rib_head *rh, const struct rtentry *rt, + const struct nhgrp_object *src, rib_filter_f_t flt_func, void *flt_data, + struct route_nhop_data *rnd) { char storage[64]; struct nh_control *ctl = rh->nh_control; struct weightened_nhop *pnhops; const struct nhgrp_priv *mp_priv, *src_priv; size_t sz; int error, i, num_nhops; src_priv = NHGRP_PRIV_CONST(src); sz = src_priv->nhg_nh_count * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { if ((pnhops = malloc(sz, M_TEMP, M_NOWAIT)) == NULL) return (ENOMEM); } /* Filter nexthops */ error = 0; num_nhops = 0; for (i = 0; i < src_priv->nhg_nh_count; i++) { - if (flt_func(src_priv->nhg_nh_weights[i].nh, flt_data)) + if (flt_func(rt, src_priv->nhg_nh_weights[i].nh, flt_data)) continue; memcpy(&pnhops[num_nhops++], &src_priv->nhg_nh_weights[i], sizeof(struct weightened_nhop)); } if (num_nhops == 0) { rnd->rnd_nhgrp = NULL; rnd->rnd_weight = 0; } else if (num_nhops == 1) { rnd->rnd_nhop = pnhops[0].nh; rnd->rnd_weight = pnhops[0].weight; if (nhop_try_ref_object(rnd->rnd_nhop) == 0) error = EAGAIN; } else { mp_priv = get_nhgrp(ctl, pnhops, num_nhops, &error); if (mp_priv != NULL) rnd->rnd_nhgrp = mp_priv->nhg; rnd->rnd_weight = 0; } if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); return (error); } /* * Creates new multipath group based on existing group/nhop in @rnd_orig and * to-be-added nhop @wn_add. * Returns 0 on success and stores result in @rnd_new. */ int nhgrp_get_addition_group(struct rib_head *rh, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_new) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; struct weightened_nhop wn[2] = {}; int error; if (rnd_orig->rnd_nhop == NULL) { /* No paths to add to, just reference current nhop */ *rnd_new = *rnd_add; if (nhop_try_ref_object(rnd_new->rnd_nhop) == 0) return (EAGAIN); return (0); } wn[0].nh = rnd_add->rnd_nhop; wn[0].weight = rnd_add->rnd_weight; if (!NH_IS_NHGRP(rnd_orig->rnd_nhop)) { /* Simple merge of 2 non-multipath nexthops */ wn[1].nh = rnd_orig->rnd_nhop; wn[1].weight = rnd_orig->rnd_weight; nhg_priv = get_nhgrp(ctl, wn, 2, &error); } else { /* Get new nhop group with @rt->rt_nhop as an additional nhop */ nhg_priv = append_nhops(ctl, rnd_orig->rnd_nhgrp, &wn[0], 1, &error); } if (nhg_priv == NULL) return (error); rnd_new->rnd_nhgrp = nhg_priv->nhg; rnd_new->rnd_weight = 0; return (0); } /* * Returns pointer to array of nexthops with weights for * given @nhg. Stores number of items in the array into @pnum_nhops. */ const struct weightened_nhop * nhgrp_get_nhops(const struct nhgrp_object *nhg, uint32_t *pnum_nhops) { const struct nhgrp_priv *nhg_priv; KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); nhg_priv = NHGRP_PRIV_CONST(nhg); *pnum_nhops = nhg_priv->nhg_nh_count; return (nhg_priv->nhg_nh_weights); } /* * Prints nexhop group @nhg data in the provided @buf. * Example: nhg#33/sz=3:[#1:100,#2:100,#3:100] * Example: nhg#33/sz=5:[#1:100,#2:100,..] */ char * nhgrp_print_buf(const struct nhgrp_object *nhg, char *buf, size_t bufsize) { const struct nhgrp_priv *nhg_priv = NHGRP_PRIV_CONST(nhg); int off = snprintf(buf, bufsize, "nhg#%u/sz=%u:[", nhg_priv->nhg_idx, nhg_priv->nhg_nh_count); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { const struct weightened_nhop *wn = &nhg_priv->nhg_nh_weights[i]; int len = snprintf(&buf[off], bufsize - off, "#%u:%u,", wn->nh->nh_priv->nh_idx, wn->weight); if (len + off + 3 >= bufsize) { int len = snprintf(&buf[off], bufsize - off, "..."); off += len; break; } off += len; } if (off > 0) off--; // remove last "," if (off + 1 < bufsize) snprintf(&buf[off], bufsize - off, "]"); return buf; } __noinline static int dump_nhgrp_entry(struct rib_head *rh, const struct nhgrp_priv *nhg_priv, char *buffer, size_t buffer_size, struct sysctl_req *w) { struct rt_msghdr *rtm; struct nhgrp_external *nhge; struct nhgrp_container *nhgc; const struct nhgrp_object *nhg; struct nhgrp_nhop_external *ext; int error; size_t sz; nhg = nhg_priv->nhg; sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); /* controlplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; /* dataplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; KASSERT(sz <= buffer_size, ("increase nhgrp buffer size")); bzero(buffer, sz); rtm = (struct rt_msghdr *)buffer; rtm->rtm_msglen = sz; rtm->rtm_version = RTM_VERSION; rtm->rtm_type = RTM_GET; nhge = (struct nhgrp_external *)(rtm + 1); nhge->nhg_idx = nhg_priv->nhg_idx; nhge->nhg_refcount = nhg_priv->nhg_refcount; /* fill in control plane nexthops firs */ nhgc = (struct nhgrp_container *)(nhge + 1); nhgc->nhgc_type = NHG_C_TYPE_CNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; nhgc->nhgc_count = nhg_priv->nhg_nh_count; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { ext[i].nh_idx = nhg_priv->nhg_nh_weights[i].nh->nh_priv->nh_idx; ext[i].nh_weight = nhg_priv->nhg_nh_weights[i].weight; } /* fill in dataplane nexthops */ nhgc = (struct nhgrp_container *)(&ext[nhg_priv->nhg_nh_count]); nhgc->nhgc_type = NHG_C_TYPE_DNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; nhgc->nhgc_count = nhg->nhg_size; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg->nhg_size; i++) { ext[i].nh_idx = nhg->nhops[i]->nh_priv->nh_idx; ext[i].nh_weight = 0; } error = SYSCTL_OUT(w, buffer, sz); return (error); } uint32_t nhgrp_get_idx(const struct nhgrp_object *nhg) { const struct nhgrp_priv *nhg_priv; nhg_priv = NHGRP_PRIV_CONST(nhg); return (nhg_priv->nhg_idx); } uint32_t nhgrp_get_count(struct rib_head *rh) { struct nh_control *ctl; uint32_t count; ctl = rh->nh_control; NHOPS_RLOCK(ctl); count = ctl->gr_head.items_count; NHOPS_RUNLOCK(ctl); return (count); } int nhgrp_dump_sysctl(struct rib_head *rh, struct sysctl_req *w) { struct nh_control *ctl = rh->nh_control; struct epoch_tracker et; struct nhgrp_priv *nhg_priv; char *buffer; size_t sz; int error = 0; if (ctl->gr_head.items_count == 0) return (0); /* Calculate the maximum nhop group size in bytes */ sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); sz += 2 * sizeof(struct nhgrp_container); sz += 2 * sizeof(struct nhgrp_nhop_external) * RIB_MAX_MPATH_WIDTH; buffer = malloc(sz, M_TEMP, M_NOWAIT); if (buffer == NULL) return (ENOMEM); NET_EPOCH_ENTER(et); NHOPS_RLOCK(ctl); CHT_SLIST_FOREACH(&ctl->gr_head, mpath, nhg_priv) { error = dump_nhgrp_entry(rh, nhg_priv, buffer, sz, w); if (error != 0) break; } CHT_SLIST_FOREACH_END; NHOPS_RUNLOCK(ctl); NET_EPOCH_EXIT(et); free(buffer, M_TEMP); return (error); } diff --git a/sys/net/route/route_ctl.c b/sys/net/route/route_ctl.c index 133f9d34d854..34a029746fa1 100644 --- a/sys/net/route/route_ctl.c +++ b/sys/net/route/route_ctl.c @@ -1,1593 +1,1627 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME route_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains control plane routing tables functions. * * All functions assumes they are called in net epoch. */ struct rib_subscription { CK_STAILQ_ENTRY(rib_subscription) next; rib_subscription_cb_t *func; void *arg; struct rib_head *rnh; enum rib_subscription_type type; struct epoch_context epoch_ctx; }; static int add_route_byinfo(struct rib_head *rnh, struct rt_addrinfo *info, struct rib_cmd_info *rc); static int change_route_byinfo(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *nhd_orig, struct rib_cmd_info *rc); static int add_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc); static int delete_route(struct rib_head *rnh, struct rtentry *rt, struct rib_cmd_info *rc); -static int rt_unlinkrte(struct rib_head *rnh, struct rt_addrinfo *info, - struct rib_cmd_info *rc); +static int rt_delete_conditional(struct rib_head *rnh, struct rtentry *rt, + int prio, rib_filter_f_t *cb, void *cbdata, struct rib_cmd_info *rc); static void rib_notify(struct rib_head *rnh, enum rib_subscription_type type, struct rib_cmd_info *rc); +static int get_prio_from_info(const struct rt_addrinfo *info); +static int nhop_get_prio(const struct nhop_object *nh); + static void destroy_subscription_epoch(epoch_context_t ctx); #ifdef ROUTE_MPATH static bool rib_can_multipath(struct rib_head *rh); #endif /* Per-vnet multipath routing configuration */ SYSCTL_DECL(_net_route); #define V_rib_route_multipath VNET(rib_route_multipath) #ifdef ROUTE_MPATH #define _MP_FLAGS CTLFLAG_RW #else #define _MP_FLAGS CTLFLAG_RD #endif VNET_DEFINE(u_int, rib_route_multipath) = 1; SYSCTL_UINT(_net_route, OID_AUTO, multipath, _MP_FLAGS | CTLFLAG_VNET, &VNET_NAME(rib_route_multipath), 0, "Enable route multipath"); #undef _MP_FLAGS #if defined(INET) && defined(INET6) FEATURE(ipv4_rfc5549_support, "Route IPv4 packets via IPv6 nexthops"); #define V_rib_route_ipv6_nexthop VNET(rib_route_ipv6_nexthop) VNET_DEFINE(u_int, rib_route_ipv6_nexthop) = 1; SYSCTL_UINT(_net_route, OID_AUTO, ipv6_nexthop, CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(rib_route_ipv6_nexthop), 0, "Enable IPv4 route via IPv6 Next Hop address"); #endif /* Routing table UMA zone */ VNET_DEFINE_STATIC(uma_zone_t, rtzone); #define V_rtzone VNET(rtzone) /* Debug bits */ SYSCTL_NODE(_net_route, OID_AUTO, debug, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); void vnet_rtzone_init(void) { V_rtzone = uma_zcreate("rtentry", sizeof(struct rtentry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); } #ifdef VIMAGE void vnet_rtzone_destroy(void) { uma_zdestroy(V_rtzone); } #endif static void destroy_rtentry(struct rtentry *rt) { #ifdef VIMAGE struct nhop_object *nh = rt->rt_nhop; /* * At this moment rnh, nh_control may be already freed. * nhop interface may have been migrated to a different vnet. * Use vnet stored in the nexthop to delete the entry. */ #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh)) { const struct weightened_nhop *wn; uint32_t num_nhops; wn = nhgrp_get_nhops((struct nhgrp_object *)nh, &num_nhops); nh = wn[0].nh; } #endif CURVNET_SET(nhop_get_vnet(nh)); #endif /* Unreference nexthop */ nhop_free_any(rt->rt_nhop); uma_zfree(V_rtzone, rt); CURVNET_RESTORE(); } /* * Epoch callback indicating rtentry is safe to destroy */ static void destroy_rtentry_epoch(epoch_context_t ctx) { struct rtentry *rt; rt = __containerof(ctx, struct rtentry, rt_epoch_ctx); destroy_rtentry(rt); } /* * Schedule rtentry deletion */ static void rtfree(struct rtentry *rt) { KASSERT(rt != NULL, ("%s: NULL rt", __func__)); epoch_call(net_epoch_preempt, destroy_rtentry_epoch, &rt->rt_epoch_ctx); } static struct rib_head * get_rnh(uint32_t fibnum, const struct rt_addrinfo *info) { struct rib_head *rnh; struct sockaddr *dst; KASSERT((fibnum < rt_numfibs), ("rib_add_route: bad fibnum")); dst = info->rti_info[RTAX_DST]; rnh = rt_tables_get_rnh(fibnum, dst->sa_family); return (rnh); } #if defined(INET) && defined(INET6) static bool rib_can_ipv6_nexthop_address(struct rib_head *rh) { int result; CURVNET_SET(rh->rib_vnet); result = !!V_rib_route_ipv6_nexthop; CURVNET_RESTORE(); return (result); } #endif #ifdef ROUTE_MPATH static bool rib_can_multipath(struct rib_head *rh) { int result; CURVNET_SET(rh->rib_vnet); result = !!V_rib_route_multipath; CURVNET_RESTORE(); return (result); } /* * Check is nhop is multipath-eligible. * Avoid nhops without gateways and redirects. * * Returns 1 for multipath-eligible nexthop, * 0 otherwise. */ bool nhop_can_multipath(const struct nhop_object *nh) { if ((nh->nh_flags & NHF_MULTIPATH) != 0) return (1); if ((nh->nh_flags & NHF_GATEWAY) == 0) return (0); if ((nh->nh_flags & NHF_REDIRECT) != 0) return (0); return (1); } #endif static int get_info_weight(const struct rt_addrinfo *info, uint32_t default_weight) { uint32_t weight; if (info->rti_mflags & RTV_WEIGHT) weight = info->rti_rmx->rmx_weight; else weight = default_weight; /* Keep upper 1 byte for adm distance purposes */ if (weight > RT_MAX_WEIGHT) weight = RT_MAX_WEIGHT; else if (weight == 0) weight = default_weight; return (weight); } bool rt_is_host(const struct rtentry *rt) { return (rt->rte_flags & RTF_HOST); } sa_family_t rt_get_family(const struct rtentry *rt) { const struct sockaddr *dst; dst = (const struct sockaddr *)rt_key_const(rt); return (dst->sa_family); } /* * Returns pointer to nexthop or nexthop group * associated with @rt */ struct nhop_object * rt_get_raw_nhop(const struct rtentry *rt) { return (rt->rt_nhop); } #ifdef INET /* * Stores IPv4 address and prefix length of @rt inside * @paddr and @plen. * @pscopeid is currently always set to 0. */ void rt_get_inet_prefix_plen(const struct rtentry *rt, struct in_addr *paddr, int *plen, uint32_t *pscopeid) { const struct sockaddr_in *dst; dst = (const struct sockaddr_in *)rt_key_const(rt); KASSERT((dst->sin_family == AF_INET), ("rt family is %d, not inet", dst->sin_family)); *paddr = dst->sin_addr; dst = (const struct sockaddr_in *)rt_mask_const(rt); if (dst == NULL) *plen = 32; else *plen = bitcount32(dst->sin_addr.s_addr); *pscopeid = 0; } /* * Stores IPv4 address and prefix mask of @rt inside * @paddr and @pmask. Sets mask to INADDR_ANY for host routes. * @pscopeid is currently always set to 0. */ void rt_get_inet_prefix_pmask(const struct rtentry *rt, struct in_addr *paddr, struct in_addr *pmask, uint32_t *pscopeid) { const struct sockaddr_in *dst; dst = (const struct sockaddr_in *)rt_key_const(rt); KASSERT((dst->sin_family == AF_INET), ("rt family is %d, not inet", dst->sin_family)); *paddr = dst->sin_addr; dst = (const struct sockaddr_in *)rt_mask_const(rt); if (dst == NULL) pmask->s_addr = INADDR_BROADCAST; else *pmask = dst->sin_addr; *pscopeid = 0; } #endif #ifdef INET6 static int inet6_get_plen(const struct in6_addr *addr) { return (bitcount32(addr->s6_addr32[0]) + bitcount32(addr->s6_addr32[1]) + bitcount32(addr->s6_addr32[2]) + bitcount32(addr->s6_addr32[3])); } /* * Stores IPv6 address and prefix length of @rt inside * @paddr and @plen. Addresses are returned in de-embedded form. * Scopeid is set to 0 for non-LL addresses. */ void rt_get_inet6_prefix_plen(const struct rtentry *rt, struct in6_addr *paddr, int *plen, uint32_t *pscopeid) { const struct sockaddr_in6 *dst; dst = (const struct sockaddr_in6 *)rt_key_const(rt); KASSERT((dst->sin6_family == AF_INET6), ("rt family is %d, not inet6", dst->sin6_family)); if (IN6_IS_SCOPE_LINKLOCAL(&dst->sin6_addr)) in6_splitscope(&dst->sin6_addr, paddr, pscopeid); else *paddr = dst->sin6_addr; dst = (const struct sockaddr_in6 *)rt_mask_const(rt); if (dst == NULL) *plen = 128; else *plen = inet6_get_plen(&dst->sin6_addr); } /* * Stores IPv6 address and prefix mask of @rt inside * @paddr and @pmask. Addresses are returned in de-embedded form. * Scopeid is set to 0 for non-LL addresses. */ void rt_get_inet6_prefix_pmask(const struct rtentry *rt, struct in6_addr *paddr, struct in6_addr *pmask, uint32_t *pscopeid) { const struct sockaddr_in6 *dst; dst = (const struct sockaddr_in6 *)rt_key_const(rt); KASSERT((dst->sin6_family == AF_INET6), ("rt family is %d, not inet", dst->sin6_family)); if (IN6_IS_SCOPE_LINKLOCAL(&dst->sin6_addr)) in6_splitscope(&dst->sin6_addr, paddr, pscopeid); else *paddr = dst->sin6_addr; dst = (const struct sockaddr_in6 *)rt_mask_const(rt); if (dst == NULL) memset(pmask, 0xFF, sizeof(struct in6_addr)); else *pmask = dst->sin6_addr; } #endif /* * Check if specified @gw matches gw data in the nexthop @nh. * * Returns true if matches, false otherwise. */ bool match_nhop_gw(const struct nhop_object *nh, const struct sockaddr *gw) { if (nh->gw_sa.sa_family != gw->sa_family) return (false); switch (gw->sa_family) { case AF_INET: return (nh->gw4_sa.sin_addr.s_addr == ((const struct sockaddr_in *)gw)->sin_addr.s_addr); case AF_INET6: { const struct sockaddr_in6 *gw6; gw6 = (const struct sockaddr_in6 *)gw; /* * Currently (2020-09) IPv6 gws in kernel have their * scope embedded. Once this becomes false, this code * has to be revisited. */ if (IN6_ARE_ADDR_EQUAL(&nh->gw6_sa.sin6_addr, &gw6->sin6_addr)) return (true); return (false); } case AF_LINK: { const struct sockaddr_dl *sdl; sdl = (const struct sockaddr_dl *)gw; return (nh->gwl_sa.sdl_index == sdl->sdl_index); } default: return (memcmp(&nh->gw_sa, gw, nh->gw_sa.sa_len) == 0); } /* NOTREACHED */ return (false); } +struct gw_filter_data { + const struct sockaddr *gw; + int count; +}; + +static int +gw_filter_func(const struct rtentry *rt, const struct nhop_object *nh, void *_data) +{ + struct gw_filter_data *gwd = (struct gw_filter_data *)_data; + + /* Return only first match to make rtsock happy */ + if (match_nhop_gw(nh, gwd->gw) && gwd->count++ == 0) + return (1); + return (0); +} + /* * Checks if data in @info matches nexhop @nh. * * Returns 0 on success, * ESRCH if not matched, * ENOENT if filter function returned false */ int check_info_match_nhop(const struct rt_addrinfo *info, const struct rtentry *rt, const struct nhop_object *nh) { const struct sockaddr *gw = info->rti_info[RTAX_GATEWAY]; if (info->rti_filter != NULL) { if (info->rti_filter(rt, nh, info->rti_filterdata) == 0) return (ENOENT); else return (0); } if ((gw != NULL) && !match_nhop_gw(nh, gw)) return (ESRCH); return (0); } -/* - * Checks if nexhop @nh can be rewritten by data in @info because - * of higher "priority". Currently the only case for such scenario - * is kernel installing interface routes, marked by RTF_PINNED flag. - * - * Returns: - * 1 if @info data has higher priority - * 0 if priority is the same - * -1 if priority is lower - */ -int -can_override_nhop(const struct rt_addrinfo *info, const struct nhop_object *nh) -{ - - if (info->rti_flags & RTF_PINNED) { - return (NH_IS_PINNED(nh)) ? 0 : 1; - } else { - return (NH_IS_PINNED(nh)) ? -1 : 0; - } -} - /* * Runs exact prefix match based on @dst and @netmask. * Returns matched @rtentry if found or NULL. * If rtentry was found, saves nexthop / weight value into @rnd. */ static struct rtentry * lookup_prefix_bysa(struct rib_head *rnh, const struct sockaddr *dst, const struct sockaddr *netmask, struct route_nhop_data *rnd) { struct rtentry *rt; RIB_LOCK_ASSERT(rnh); rt = (struct rtentry *)rnh->rnh_lookup(dst, netmask, &rnh->head); if (rt != NULL) { rnd->rnd_nhop = rt->rt_nhop; rnd->rnd_weight = rt->rt_weight; } else { rnd->rnd_nhop = NULL; rnd->rnd_weight = 0; } return (rt); } struct rtentry * lookup_prefix_rt(struct rib_head *rnh, const struct rtentry *rt, struct route_nhop_data *rnd) { return (lookup_prefix_bysa(rnh, rt_key_const(rt), rt_mask_const(rt), rnd)); } /* * Runs exact prefix match based on dst/netmask from @info. * Assumes RIB lock is held. * Returns matched @rtentry if found or NULL. * If rtentry was found, saves nexthop / weight value into @rnd. */ struct rtentry * lookup_prefix(struct rib_head *rnh, const struct rt_addrinfo *info, struct route_nhop_data *rnd) { struct rtentry *rt; rt = lookup_prefix_bysa(rnh, info->rti_info[RTAX_DST], info->rti_info[RTAX_NETMASK], rnd); return (rt); } /* * Adds route defined by @info into the kernel table specified by @fibnum and * sa_family in @info->rti_info[RTAX_DST]. * * Returns 0 on success and fills in operation metadata into @rc. */ int rib_add_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct rib_head *rnh; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); /* * Check consistency between RTF_HOST flag and netmask * existence. */ if (info->rti_flags & RTF_HOST) info->rti_info[RTAX_NETMASK] = NULL; else if (info->rti_info[RTAX_NETMASK] == NULL) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: no RTF_HOST and empty netmask"); return (EINVAL); } bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_ADD; error = add_route_byinfo(rnh, info, rc); if (error == 0) rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); return (error); } /* * Checks if @dst and @gateway is valid combination. * * Returns true if is valid, false otherwise. */ static bool check_gateway(struct rib_head *rnh, struct sockaddr *dst, struct sockaddr *gateway) { if (dst->sa_family == gateway->sa_family) return (true); else if (gateway->sa_family == AF_UNSPEC) return (true); else if (gateway->sa_family == AF_LINK) return (true); #if defined(INET) && defined(INET6) else if (dst->sa_family == AF_INET && gateway->sa_family == AF_INET6 && rib_can_ipv6_nexthop_address(rnh)) return (true); #endif else return (false); } /* * Creates rtentry and nexthop based on @info data. * Return 0 and fills in rtentry into @prt on success, * Note: rtentry mask will be set to RTAX_NETMASK, thus its pointer is required * to be stable till the end of the operation (radix rt insertion/change/removal). * return errno otherwise. */ static int create_rtentry(struct rib_head *rnh, struct rt_addrinfo *info, struct rtentry **prt) { struct sockaddr *dst, *ndst, *gateway, *netmask; struct rtentry *rt; struct nhop_object *nh; int error, flags; dst = info->rti_info[RTAX_DST]; gateway = info->rti_info[RTAX_GATEWAY]; netmask = info->rti_info[RTAX_NETMASK]; flags = info->rti_flags; if ((flags & RTF_GATEWAY) && !gateway) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: RTF_GATEWAY set with empty gw"); return (EINVAL); } if (dst && gateway && !check_gateway(rnh, dst, gateway)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: invalid dst/gateway family combination (%d, %d)", dst->sa_family, gateway->sa_family); return (EINVAL); } if (dst->sa_len > sizeof(((struct rtentry *)NULL)->rt_dstb)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: dst->sa_len too large: %d", dst->sa_len); return (EINVAL); } if (info->rti_ifa == NULL) { error = rt_getifa_fib(info, rnh->rib_fibnum); if (error) return (error); } error = nhop_create_from_info(rnh, info, &nh); if (error != 0) return (error); rt = uma_zalloc(V_rtzone, M_NOWAIT | M_ZERO); if (rt == NULL) { nhop_free(nh); return (ENOBUFS); } rt->rte_flags = (RTF_UP | flags) & RTE_RT_FLAG_MASK; rt->rt_nhop = nh; /* Fill in dst */ memcpy(&rt->rt_dst, dst, dst->sa_len); rt_key(rt) = &rt->rt_dst; /* * point to the (possibly newly malloc'd) dest address. */ ndst = (struct sockaddr *)rt_key(rt); /* * make sure it contains the value we want (masked if needed). */ if (netmask) { rt_maskedcopy(dst, ndst, netmask); } else bcopy(dst, ndst, dst->sa_len); /* Set netmask to the storage from info. It will be updated upon insertion */ rt_mask(rt) = netmask; /* * We use the ifa reference returned by rt_getifa_fib(). * This moved from below so that rnh->rnh_addaddr() can * examine the ifa and ifa->ifa_ifp if it so desires. */ rt->rt_weight = get_info_weight(info, RT_DEFAULT_WEIGHT); *prt = rt; return (0); } static int add_route_byinfo(struct rib_head *rnh, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct nhop_object *nh_orig; struct route_nhop_data rnd_orig, rnd_add; struct nhop_object *nh; struct rtentry *rt, *rt_orig; int error; error = create_rtentry(rnh, info, &rt); if (error != 0) return (error); rnd_add.rnd_nhop = rt->rt_nhop; rnd_add.rnd_weight = rt->rt_weight; nh = rt->rt_nhop; RIB_WLOCK(rnh); error = add_route(rnh, rt, &rnd_add, rc); if (error == 0) { RIB_WUNLOCK(rnh); return (0); } /* addition failed. Lookup prefix in the rib to determine the cause */ rt_orig = lookup_prefix(rnh, info, &rnd_orig); if (rt_orig == NULL) { /* No prefix -> rnh_addaddr() failed to allocate memory */ RIB_WUNLOCK(rnh); nhop_free(nh); uma_zfree(V_rtzone, rt); return (ENOMEM); } /* We have existing route in the RIB. */ nh_orig = rnd_orig.rnd_nhop; /* Check if new route has higher preference */ - if (can_override_nhop(info, nh_orig) > 0) { + if (get_prio_from_info(info) > nhop_get_prio(nh_orig)) { /* Update nexthop to the new route */ change_route(rnh, rt_orig, &rnd_add, rc); RIB_WUNLOCK(rnh); uma_zfree(V_rtzone, rt); nhop_free(nh_orig); return (0); } RIB_WUNLOCK(rnh); #ifdef ROUTE_MPATH if (rib_can_multipath(rnh) && nhop_can_multipath(rnd_add.rnd_nhop) && nhop_can_multipath(rnd_orig.rnd_nhop)) error = add_route_mpath(rnh, info, rt, &rnd_add, &rnd_orig, rc); else #endif /* Unable to add - another route with the same preference exists */ error = EEXIST; /* * ROUTE_MPATH disabled: failed to add route, free both nhop and rt. * ROUTE_MPATH enabled: original nhop reference is unused in any case, * free rt only if not _adding_ new route to rib (e.g. the case * when initial lookup returned existing route, but then it got * deleted prior to multipath group insertion, leading to a simple * non-multipath add as a result). */ nhop_free(nh); if ((error != 0) || rc->rc_cmd != RTM_ADD) uma_zfree(V_rtzone, rt); return (error); } /* * Removes route defined by @info from the kernel table specified by @fibnum and * sa_family in @info->rti_info[RTAX_DST]. * * Returns 0 on success and fills in operation metadata into @rc. */ int rib_del_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct rib_head *rnh; - struct sockaddr *dst_orig, *netmask; + struct sockaddr *dst, *netmask; struct sockaddr_storage mdst; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_DELETE; - dst_orig = info->rti_info[RTAX_DST]; + dst = info->rti_info[RTAX_DST]; netmask = info->rti_info[RTAX_NETMASK]; if (netmask != NULL) { /* Ensure @dst is always properly masked */ - if (dst_orig->sa_len > sizeof(mdst)) { + if (dst->sa_len > sizeof(mdst)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: dst->sa_len too large"); return (EINVAL); } - rt_maskedcopy(dst_orig, (struct sockaddr *)&mdst, netmask); - info->rti_info[RTAX_DST] = (struct sockaddr *)&mdst; + rt_maskedcopy(dst, (struct sockaddr *)&mdst, netmask); + dst = (struct sockaddr *)&mdst; } + rib_filter_f_t *filter_func = NULL; + void *filter_arg = NULL; + struct gw_filter_data gwd = { .gw = info->rti_info[RTAX_GATEWAY] }; + + if (info->rti_filter != NULL) { + filter_func = info->rti_filter; + filter_arg = info->rti_filterdata; + } else if (gwd.gw != NULL) { + filter_func = gw_filter_func; + filter_arg = &gwd; + } + + int prio = get_prio_from_info(info); + RIB_WLOCK(rnh); - error = rt_unlinkrte(rnh, info, rc); + struct route_nhop_data rnd; + struct rtentry *rt = lookup_prefix_bysa(rnh, dst, netmask, &rnd); + if (rt != NULL) { + error = rt_delete_conditional(rnh, rt, prio, filter_func, + filter_arg, rc); + } else + error = ESRCH; RIB_WUNLOCK(rnh); - info->rti_info[RTAX_DST] = dst_orig; - if (error != 0) return (error); rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); if (rc->rc_cmd == RTM_DELETE) rtfree(rc->rc_rt); #ifdef ROUTE_MPATH else { /* * Deleting 1 path may result in RTM_CHANGE to * a different mpath group/nhop. * Free old mpath group. */ nhop_free_any(rc->rc_nh_old); } #endif return (0); } /* - * Conditionally unlinks rtentry matching data inside @info from @rnh. + * File-local concept for distingushing between the normal and + * RTF_PINNED routes tha can override the "normal" one. + */ +#define NH_PRIORITY_HIGH 2 +#define NH_PRIORITY_NORMAL 1 +static int +get_prio_from_info(const struct rt_addrinfo *info) +{ + if (info->rti_flags & RTF_PINNED) + return (NH_PRIORITY_HIGH); + return (NH_PRIORITY_NORMAL); +} + +static int +nhop_get_prio(const struct nhop_object *nh) +{ + if (NH_IS_PINNED(nh)) + return (NH_PRIORITY_HIGH); + return (NH_PRIORITY_NORMAL); +} + +/* + * Conditionally unlinks rtentry paths from @rnh matching @cb. * Returns 0 on success with operation result stored in @rc. * On error, returns: - * ESRCH - if prefix was not found, + * ESRCH - if prefix was not found or filter function failed to match * EADDRINUSE - if trying to delete higher priority route. - * ENOENT - if supplied filter function returned 0 (not matched). */ static int -rt_unlinkrte(struct rib_head *rnh, struct rt_addrinfo *info, struct rib_cmd_info *rc) +rt_delete_conditional(struct rib_head *rnh, struct rtentry *rt, + int prio, rib_filter_f_t *cb, void *cbdata, struct rib_cmd_info *rc) { - struct rtentry *rt; - struct nhop_object *nh; + struct nhop_object *nh = rt->rt_nhop; struct route_nhop_data rnd; - int error; - - rt = lookup_prefix(rnh, info, &rnd); - if (rt == NULL) - return (ESRCH); - nh = rt->rt_nhop; #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh)) { - error = del_route_mpath(rnh, info, rt, - (struct nhgrp_object *)nh, rc); + struct nhgrp_object *nhg = (struct nhgrp_object *)nh; + int error; + + if (cb == NULL) + return (ESRCH); + error = nhgrp_get_filtered_group(rnh, rt, nhg, cb, cbdata, &rnd); + if (error == 0) { + if (rnd.rnd_nhgrp == nhg) { + /* No match, unreference new group and return. */ + nhop_free_any(rnd.rnd_nhop); + return (ESRCH); + } + error = change_route(rnh, rt, &rnd, rc); + } return (error); } #endif - error = check_info_match_nhop(info, rt, nh); - if (error != 0) - return (error); + if (cb != NULL && !cb(rt, nh, cbdata)) + return (ESRCH); - if (can_override_nhop(info, nh) < 0) + if (prio < nhop_get_prio(nh)) return (EADDRINUSE); return (delete_route(rnh, rt, rc)); } int rib_change_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { RIB_RLOCK_TRACKER; struct route_nhop_data rnd_orig; struct rib_head *rnh; struct rtentry *rt; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_CHANGE; /* Check if updated gateway exists */ if ((info->rti_flags & RTF_GATEWAY) && (info->rti_info[RTAX_GATEWAY] == NULL)) { /* * route(8) adds RTF_GATEWAY flag if -interface is not set. * Remove RTF_GATEWAY to enforce consistency and maintain * compatibility.. */ info->rti_flags &= ~RTF_GATEWAY; } /* * route change is done in multiple steps, with dropping and * reacquiring lock. In the situations with multiple processes * changes the same route in can lead to the case when route * is changed between the steps. Address it by retrying the operation * multiple times before failing. */ RIB_RLOCK(rnh); rt = (struct rtentry *)rnh->rnh_lookup(info->rti_info[RTAX_DST], info->rti_info[RTAX_NETMASK], &rnh->head); if (rt == NULL) { RIB_RUNLOCK(rnh); return (ESRCH); } rnd_orig.rnd_nhop = rt->rt_nhop; rnd_orig.rnd_weight = rt->rt_weight; RIB_RUNLOCK(rnh); for (int i = 0; i < RIB_MAX_RETRIES; i++) { error = change_route_byinfo(rnh, rt, info, &rnd_orig, rc); if (error != EAGAIN) break; } return (error); } static int change_nhop(struct rib_head *rnh, struct rt_addrinfo *info, struct nhop_object *nh_orig, struct nhop_object **nh_new) { int error; /* * New gateway could require new ifaddr, ifp; * flags may also be different; ifp may be specified * by ll sockaddr when protocol address is ambiguous */ if (((nh_orig->nh_flags & NHF_GATEWAY) && info->rti_info[RTAX_GATEWAY] != NULL) || info->rti_info[RTAX_IFP] != NULL || (info->rti_info[RTAX_IFA] != NULL && !sa_equal(info->rti_info[RTAX_IFA], nh_orig->nh_ifa->ifa_addr))) { error = rt_getifa_fib(info, rnh->rib_fibnum); if (error != 0) { info->rti_ifa = NULL; return (error); } } error = nhop_create_from_nhop(rnh, nh_orig, info, nh_new); info->rti_ifa = NULL; return (error); } #ifdef ROUTE_MPATH static int change_mpath_route(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc) { int error = 0, found_idx = 0; struct nhop_object *nh_orig = NULL, *nh_new; struct route_nhop_data rnd_new = {}; const struct weightened_nhop *wn = NULL; struct weightened_nhop *wn_new; uint32_t num_nhops; wn = nhgrp_get_nhops(rnd_orig->rnd_nhgrp, &num_nhops); for (int i = 0; i < num_nhops; i++) { if (check_info_match_nhop(info, NULL, wn[i].nh) == 0) { nh_orig = wn[i].nh; found_idx = i; break; } } if (nh_orig == NULL) return (ESRCH); error = change_nhop(rnh, info, nh_orig, &nh_new); if (error != 0) return (error); wn_new = mallocarray(num_nhops, sizeof(struct weightened_nhop), M_TEMP, M_NOWAIT | M_ZERO); if (wn_new == NULL) { nhop_free(nh_new); return (EAGAIN); } memcpy(wn_new, wn, num_nhops * sizeof(struct weightened_nhop)); wn_new[found_idx].nh = nh_new; wn_new[found_idx].weight = get_info_weight(info, wn[found_idx].weight); error = nhgrp_get_group(rnh, wn_new, num_nhops, &rnd_new.rnd_nhgrp); nhop_free(nh_new); free(wn_new, M_TEMP); if (error != 0) return (error); error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); return (error); } #endif static int change_route_byinfo(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc) { int error = 0; struct nhop_object *nh_orig; struct route_nhop_data rnd_new; nh_orig = rnd_orig->rnd_nhop; if (nh_orig == NULL) return (ESRCH); #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh_orig)) return (change_mpath_route(rnh, rt, info, rnd_orig, rc)); #endif rnd_new.rnd_weight = get_info_weight(info, rnd_orig->rnd_weight); error = change_nhop(rnh, info, nh_orig, &rnd_new.rnd_nhop); if (error != 0) return (error); error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); return (error); } /* * Insert @rt with nhop data from @rnd_new to @rnh. * Returns 0 on success and stores operation results in @rc. */ static int add_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc) { struct radix_node *rn; RIB_WLOCK_ASSERT(rnh); rt->rt_nhop = rnd->rnd_nhop; rt->rt_weight = rnd->rnd_weight; rn = rnh->rnh_addaddr(rt_key(rt), rt_mask_const(rt), &rnh->head, rt->rt_nodes); if (rn != NULL) { if (!NH_IS_NHGRP(rnd->rnd_nhop) && nhop_get_expire(rnd->rnd_nhop)) tmproutes_update(rnh, rt, rnd->rnd_nhop); /* Finalize notification */ rib_bump_gen(rnh); rnh->rnh_prefixes++; rc->rc_cmd = RTM_ADD; rc->rc_rt = rt; rc->rc_nh_old = NULL; rc->rc_nh_new = rnd->rnd_nhop; rc->rc_nh_weight = rnd->rnd_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* Existing route or memory allocation failure. */ return (EEXIST); } /* * Unconditionally deletes @rt from @rnh. */ static int delete_route(struct rib_head *rnh, struct rtentry *rt, struct rib_cmd_info *rc) { RIB_WLOCK_ASSERT(rnh); /* Route deletion requested. */ struct radix_node *rn; rn = rnh->rnh_deladdr(rt_key_const(rt), rt_mask_const(rt), &rnh->head); if (rn == NULL) return (ESRCH); rt = RNTORT(rn); rt->rte_flags &= ~RTF_UP; rib_bump_gen(rnh); rnh->rnh_prefixes--; rc->rc_cmd = RTM_DELETE; rc->rc_rt = rt; rc->rc_nh_old = rt->rt_nhop; rc->rc_nh_new = NULL; rc->rc_nh_weight = rt->rt_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* * Switch @rt nhop/weigh to the ones specified in @rnd. * Returns 0 on success. */ int change_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc) { struct nhop_object *nh_orig; RIB_WLOCK_ASSERT(rnh); nh_orig = rt->rt_nhop; if (rnd->rnd_nhop == NULL) return (delete_route(rnh, rt, rc)); /* Changing nexthop & weight to a new one */ rt->rt_nhop = rnd->rnd_nhop; rt->rt_weight = rnd->rnd_weight; if (!NH_IS_NHGRP(rnd->rnd_nhop) && nhop_get_expire(rnd->rnd_nhop)) tmproutes_update(rnh, rt, rnd->rnd_nhop); /* Finalize notification */ rib_bump_gen(rnh); rc->rc_cmd = RTM_CHANGE; rc->rc_rt = rt; rc->rc_nh_old = nh_orig; rc->rc_nh_new = rnd->rnd_nhop; rc->rc_nh_weight = rnd->rnd_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* * Conditionally update route nhop/weight IFF data in @nhd_orig is * consistent with the current route data. * Nexthop in @nhd_new is consumed. */ int change_route_conditional(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_new, struct rib_cmd_info *rc) { struct rtentry *rt_new; int error = 0; #if DEBUG_MAX_LEVEL >= LOG_DEBUG2 { char buf_old[NHOP_PRINT_BUFSIZE], buf_new[NHOP_PRINT_BUFSIZE]; nhop_print_buf_any(rnd_orig->rnd_nhop, buf_old, NHOP_PRINT_BUFSIZE); nhop_print_buf_any(rnd_new->rnd_nhop, buf_new, NHOP_PRINT_BUFSIZE); FIB_LOG(LOG_DEBUG2, rnh->rib_fibnum, rnh->rib_family, "trying change %s -> %s", buf_old, buf_new); } #endif RIB_WLOCK(rnh); struct route_nhop_data rnd; rt_new = lookup_prefix_rt(rnh, rt, &rnd); if (rt_new == NULL) { if (rnd_orig->rnd_nhop == NULL) error = add_route(rnh, rt, rnd_new, rc); else { /* * Prefix does not exist, which was not our assumption. * Update @rnd_orig with the new data and return */ rnd_orig->rnd_nhop = NULL; rnd_orig->rnd_weight = 0; error = EAGAIN; } } else { /* Prefix exists, try to update */ if (rnd_orig->rnd_nhop == rt_new->rt_nhop) { /* * Nhop/mpath group hasn't changed. Flip * to the new precalculated one and return */ error = change_route(rnh, rt_new, rnd_new, rc); } else { /* Update and retry */ rnd_orig->rnd_nhop = rt_new->rt_nhop; rnd_orig->rnd_weight = rt_new->rt_weight; error = EAGAIN; } } RIB_WUNLOCK(rnh); if (error == 0) { rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); if (rnd_orig->rnd_nhop != NULL) nhop_free_any(rnd_orig->rnd_nhop); } else { if (rnd_new->rnd_nhop != NULL) nhop_free_any(rnd_new->rnd_nhop); } return (error); } /* * Performs modification of routing table specificed by @action. * Table is specified by @fibnum and sa_family in @info->rti_info[RTAX_DST]. * Needs to be run in network epoch. * * Returns 0 on success and fills in @rc with action result. */ int rib_action(uint32_t fibnum, int action, struct rt_addrinfo *info, struct rib_cmd_info *rc) { int error; switch (action) { case RTM_ADD: error = rib_add_route(fibnum, info, rc); break; case RTM_DELETE: error = rib_del_route(fibnum, info, rc); break; case RTM_CHANGE: error = rib_change_route(fibnum, info, rc); break; default: error = ENOTSUP; } return (error); } struct rt_delinfo { - struct rt_addrinfo info; struct rib_head *rnh; struct rtentry *head; + rib_filter_f_t *filter_f; + void *filter_arg; + int prio; struct rib_cmd_info rc; }; /* - * Conditionally unlinks @rn from radix tree based - * on info data passed in @arg. + * Conditionally unlinks rtenties or paths from radix tree based + * on the callback data passed in @arg. */ static int rt_checkdelroute(struct radix_node *rn, void *arg) { - struct rt_delinfo *di; - struct rt_addrinfo *info; - struct rtentry *rt; - - di = (struct rt_delinfo *)arg; - rt = (struct rtentry *)rn; - info = &di->info; + struct rt_delinfo *di = (struct rt_delinfo *)arg; + struct rtentry *rt = (struct rtentry *)rn; - info->rti_info[RTAX_DST] = rt_key(rt); - info->rti_info[RTAX_NETMASK] = rt_mask(rt); - - if (rt_unlinkrte(di->rnh, info, &di->rc) != 0) + if (rt_delete_conditional(di->rnh, rt, di->prio, + di->filter_f, di->filter_arg, &di->rc) != 0) return (0); /* * Add deleted rtentries to the list to GC them * after dropping the lock. * * XXX: Delayed notifications not implemented * for nexthop updates. */ if (di->rc.rc_cmd == RTM_DELETE) { /* Add to the list and return */ rt->rt_chain = di->head; di->head = rt; #ifdef ROUTE_MPATH } else { /* - * RTM_CHANGE to a diferent nexthop or nexthop group. + * RTM_CHANGE to a different nexthop or nexthop group. * Free old multipath group. */ nhop_free_any(di->rc.rc_nh_old); #endif } return (0); } /* * Iterates over a routing table specified by @fibnum and @family and * deletes elements marked by @filter_f. * @fibnum: rtable id * @family: AF_ address family * @filter_f: function returning non-zero value for items to delete * @arg: data to pass to the @filter_f function * @report: true if rtsock notification is needed. */ void -rib_walk_del(u_int fibnum, int family, rib_filter_f_t *filter_f, void *arg, bool report) +rib_walk_del(u_int fibnum, int family, rib_filter_f_t *filter_f, void *filter_arg, + bool report) { struct rib_head *rnh; - struct rt_delinfo di; struct rtentry *rt; struct nhop_object *nh; struct epoch_tracker et; rnh = rt_tables_get_rnh(fibnum, family); if (rnh == NULL) return; - bzero(&di, sizeof(di)); - di.info.rti_filter = filter_f; - di.info.rti_filterdata = arg; - di.rnh = rnh; - di.rc.rc_cmd = RTM_DELETE; + struct rt_delinfo di = { + .rnh = rnh, + .filter_f = filter_f, + .filter_arg = filter_arg, + .prio = NH_PRIORITY_NORMAL, + }; NET_EPOCH_ENTER(et); RIB_WLOCK(rnh); rnh->rnh_walktree(&rnh->head, rt_checkdelroute, &di); RIB_WUNLOCK(rnh); /* We might have something to reclaim. */ bzero(&di.rc, sizeof(di.rc)); di.rc.rc_cmd = RTM_DELETE; while (di.head != NULL) { rt = di.head; di.head = rt->rt_chain; rt->rt_chain = NULL; nh = rt->rt_nhop; di.rc.rc_rt = rt; di.rc.rc_nh_old = nh; rib_notify(rnh, RIB_NOTIFY_DELAYED, &di.rc); - /* TODO std rt -> rt_addrinfo export */ - di.info.rti_info[RTAX_DST] = rt_key(rt); - di.info.rti_info[RTAX_NETMASK] = rt_mask(rt); - if (report) { #ifdef ROUTE_MPATH struct nhgrp_object *nhg; const struct weightened_nhop *wn; uint32_t num_nhops; if (NH_IS_NHGRP(nh)) { nhg = (struct nhgrp_object *)nh; wn = nhgrp_get_nhops(nhg, &num_nhops); for (int i = 0; i < num_nhops; i++) rt_routemsg(RTM_DELETE, rt, wn[i].nh, fibnum); } else #endif rt_routemsg(RTM_DELETE, rt, nh, fibnum); } rtfree(rt); } NET_EPOCH_EXIT(et); } static int rt_delete_unconditional(struct radix_node *rn, void *arg) { struct rtentry *rt = RNTORT(rn); struct rib_head *rnh = (struct rib_head *)arg; rn = rnh->rnh_deladdr(rt_key(rt), rt_mask(rt), &rnh->head); if (RNTORT(rn) == rt) rtfree(rt); return (0); } /* * Removes all routes from the routing table without executing notifications. * rtentres will be removed after the end of a current epoch. */ static void rib_flush_routes(struct rib_head *rnh) { RIB_WLOCK(rnh); rnh->rnh_walktree(&rnh->head, rt_delete_unconditional, rnh); RIB_WUNLOCK(rnh); } void rib_flush_routes_family(int family) { struct rib_head *rnh; for (uint32_t fibnum = 0; fibnum < rt_numfibs; fibnum++) { if ((rnh = rt_tables_get_rnh(fibnum, family)) != NULL) rib_flush_routes(rnh); } } const char * rib_print_family(int family) { switch (family) { case AF_INET: return ("inet"); case AF_INET6: return ("inet6"); case AF_LINK: return ("link"); } return ("unknown"); } static void rib_notify(struct rib_head *rnh, enum rib_subscription_type type, struct rib_cmd_info *rc) { struct rib_subscription *rs; CK_STAILQ_FOREACH(rs, &rnh->rnh_subscribers, next) { if (rs->type == type) rs->func(rnh, rc, rs->arg); } } static struct rib_subscription * allocate_subscription(rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type, bool waitok) { struct rib_subscription *rs; int flags = M_ZERO | (waitok ? M_WAITOK : M_NOWAIT); rs = malloc(sizeof(struct rib_subscription), M_RTABLE, flags); if (rs == NULL) return (NULL); rs->func = f; rs->arg = arg; rs->type = type; return (rs); } /* * Subscribe for the changes in the routing table specified by @fibnum and * @family. * * Returns pointer to the subscription structure on success. */ struct rib_subscription * rib_subscribe(uint32_t fibnum, int family, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type, bool waitok) { struct rib_head *rnh; struct epoch_tracker et; NET_EPOCH_ENTER(et); KASSERT((fibnum < rt_numfibs), ("%s: bad fibnum", __func__)); rnh = rt_tables_get_rnh(fibnum, family); NET_EPOCH_EXIT(et); return (rib_subscribe_internal(rnh, f, arg, type, waitok)); } struct rib_subscription * rib_subscribe_internal(struct rib_head *rnh, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type, bool waitok) { struct rib_subscription *rs; struct epoch_tracker et; if ((rs = allocate_subscription(f, arg, type, waitok)) == NULL) return (NULL); rs->rnh = rnh; NET_EPOCH_ENTER(et); RIB_WLOCK(rnh); CK_STAILQ_INSERT_HEAD(&rnh->rnh_subscribers, rs, next); RIB_WUNLOCK(rnh); NET_EPOCH_EXIT(et); return (rs); } struct rib_subscription * rib_subscribe_locked(struct rib_head *rnh, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type) { struct rib_subscription *rs; NET_EPOCH_ASSERT(); RIB_WLOCK_ASSERT(rnh); if ((rs = allocate_subscription(f, arg, type, false)) == NULL) return (NULL); rs->rnh = rnh; CK_STAILQ_INSERT_HEAD(&rnh->rnh_subscribers, rs, next); return (rs); } /* * Remove rtable subscription @rs from the routing table. * Needs to be run in network epoch. */ void rib_unsubscribe(struct rib_subscription *rs) { struct rib_head *rnh = rs->rnh; NET_EPOCH_ASSERT(); RIB_WLOCK(rnh); CK_STAILQ_REMOVE(&rnh->rnh_subscribers, rs, rib_subscription, next); RIB_WUNLOCK(rnh); epoch_call(net_epoch_preempt, destroy_subscription_epoch, &rs->epoch_ctx); } void rib_unsubscribe_locked(struct rib_subscription *rs) { struct rib_head *rnh = rs->rnh; NET_EPOCH_ASSERT(); RIB_WLOCK_ASSERT(rnh); CK_STAILQ_REMOVE(&rnh->rnh_subscribers, rs, rib_subscription, next); epoch_call(net_epoch_preempt, destroy_subscription_epoch, &rs->epoch_ctx); } /* * Epoch callback indicating subscription is safe to destroy */ static void destroy_subscription_epoch(epoch_context_t ctx) { struct rib_subscription *rs; rs = __containerof(ctx, struct rib_subscription, epoch_ctx); free(rs, M_RTABLE); } void rib_init_subscriptions(struct rib_head *rnh) { CK_STAILQ_INIT(&rnh->rnh_subscribers); } void rib_destroy_subscriptions(struct rib_head *rnh) { struct rib_subscription *rs; struct epoch_tracker et; NET_EPOCH_ENTER(et); RIB_WLOCK(rnh); while ((rs = CK_STAILQ_FIRST(&rnh->rnh_subscribers)) != NULL) { CK_STAILQ_REMOVE_HEAD(&rnh->rnh_subscribers, next); epoch_call(net_epoch_preempt, destroy_subscription_epoch, &rs->epoch_ctx); } RIB_WUNLOCK(rnh); NET_EPOCH_EXIT(et); } diff --git a/sys/net/route/route_ctl.h b/sys/net/route/route_ctl.h index 7a1f7f04be9b..4371e3b19bfc 100644 --- a/sys/net/route/route_ctl.h +++ b/sys/net/route/route_ctl.h @@ -1,163 +1,163 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This header file contains public functions and structures used for * routing table manipulations. */ #ifndef _NET_ROUTE_ROUTE_CTL_H_ #define _NET_ROUTE_ROUTE_CTL_H_ struct rib_cmd_info { uint8_t rc_cmd; /* RTM_ADD|RTM_DEL|RTM_CHANGE */ uint8_t spare[3]; uint32_t rc_nh_weight; /* new nhop weight */ struct rtentry *rc_rt; /* Target entry */ struct nhop_object *rc_nh_old; /* Target nhop OR mpath */ struct nhop_object *rc_nh_new; /* Target nhop OR mpath */ }; int rib_add_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc); int rib_del_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc); int rib_change_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc); int rib_action(uint32_t fibnum, int action, struct rt_addrinfo *info, struct rib_cmd_info *rc); int rib_handle_ifaddr_info(uint32_t fibnum, int cmd, struct rt_addrinfo *info); typedef void route_notification_t(struct rib_cmd_info *rc, void *); void rib_decompose_notification(struct rib_cmd_info *rc, route_notification_t *cb, void *cbdata); int rib_add_redirect(u_int fibnum, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr *author, struct ifnet *ifp, int flags, int expire_sec); /* common flags for the functions below */ #define RIB_FLAG_WLOCK 0x01 /* Need exclusive rnh lock */ #define RIB_FLAG_LOCKED 0x02 /* Do not explicitly acquire rnh lock */ enum rib_walk_hook { RIB_WALK_HOOK_PRE, /* Hook is called before iteration */ RIB_WALK_HOOK_POST, /* Hook is called after iteration */ }; typedef int rib_walktree_f_t(struct rtentry *, void *); typedef void rib_walk_hook_f_t(struct rib_head *rnh, enum rib_walk_hook stage, void *arg); void rib_walk(uint32_t fibnum, int af, bool wlock, rib_walktree_f_t *wa_f, void *arg); void rib_walk_ext(uint32_t fibnum, int af, bool wlock, rib_walktree_f_t *wa_f, rib_walk_hook_f_t *hook_f, void *arg); void rib_walk_ext_internal(struct rib_head *rnh, bool wlock, rib_walktree_f_t *wa_f, rib_walk_hook_f_t *hook_f, void *arg); void rib_walk_ext_locked(struct rib_head *rnh, rib_walktree_f_t *wa_f, rib_walk_hook_f_t *hook_f, void *arg); void rib_walk_from(uint32_t fibnum, int family, uint32_t flags, struct sockaddr *prefix, struct sockaddr *mask, rib_walktree_f_t *wa_f, void *arg); void rib_walk_del(u_int fibnum, int family, rib_filter_f_t *filter_f, - void *arg, bool report); + void *filter_arg, bool report); void rib_foreach_table_walk(int family, bool wlock, rib_walktree_f_t *wa_f, rib_walk_hook_f_t *hook_f, void *arg); void rib_foreach_table_walk_del(int family, rib_filter_f_t *filter_f, void *arg); struct nhop_object; struct nhgrp_object; struct route_nhop_data { union { struct nhop_object *rnd_nhop; struct nhgrp_object *rnd_nhgrp; }; uint32_t rnd_weight; }; const struct rtentry *rib_lookup_prefix(uint32_t fibnum, int family, const struct sockaddr *dst, const struct sockaddr *netmask, struct route_nhop_data *rnd); const struct rtentry *rib_lookup_lpm(uint32_t fibnum, int family, const struct sockaddr *dst, struct route_nhop_data *rnd); /* rtentry accessors */ bool rt_is_host(const struct rtentry *rt); sa_family_t rt_get_family(const struct rtentry *); struct nhop_object *rt_get_raw_nhop(const struct rtentry *rt); #ifdef INET struct in_addr; void rt_get_inet_prefix_plen(const struct rtentry *rt, struct in_addr *paddr, int *plen, uint32_t *pscopeid); void rt_get_inet_prefix_pmask(const struct rtentry *rt, struct in_addr *paddr, struct in_addr *pmask, uint32_t *pscopeid); struct rtentry *rt_get_inet_parent(uint32_t fibnum, struct in_addr addr, int plen); #endif #ifdef INET6 struct in6_addr; void rt_get_inet6_prefix_plen(const struct rtentry *rt, struct in6_addr *paddr, int *plen, uint32_t *pscopeid); void rt_get_inet6_prefix_pmask(const struct rtentry *rt, struct in6_addr *paddr, struct in6_addr *pmask, uint32_t *pscopeid); struct rtentry *rt_get_inet6_parent(uint32_t fibnum, const struct in6_addr *paddr, int plen); #endif /* Nexthops */ uint32_t nhops_get_count(struct rib_head *rh); /* Multipath */ struct weightened_nhop; const struct weightened_nhop *nhgrp_get_nhops(const struct nhgrp_object *nhg, uint32_t *pnum_nhops); uint32_t nhgrp_get_count(struct rib_head *rh); /* Route subscriptions */ enum rib_subscription_type { RIB_NOTIFY_IMMEDIATE, RIB_NOTIFY_DELAYED }; struct rib_subscription; typedef void rib_subscription_cb_t(struct rib_head *rnh, struct rib_cmd_info *rc, void *arg); struct rib_subscription *rib_subscribe(uint32_t fibnum, int family, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type, bool waitok); struct rib_subscription *rib_subscribe_internal(struct rib_head *rnh, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type, bool waitok); struct rib_subscription *rib_subscribe_locked(struct rib_head *rnh, rib_subscription_cb_t *f, void *arg, enum rib_subscription_type type); void rib_unsubscribe(struct rib_subscription *rs); void rib_unsubscribe_locked(struct rib_subscription *rs); #endif diff --git a/sys/net/route/route_var.h b/sys/net/route/route_var.h index 12bf1b4093cb..df6adb66cfc1 100644 --- a/sys/net/route/route_var.h +++ b/sys/net/route/route_var.h @@ -1,330 +1,326 @@ /*- * Copyright (c) 2015-2016 * Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _NET_ROUTE_VAR_H_ #define _NET_ROUTE_VAR_H_ #ifndef RNF_NORMAL #include #endif #include #include #include /* struct sockaddr_in */ #include #include struct nh_control; /* Sets prefix-specific nexthop flags (NHF_DEFAULT, RTF/NHF_HOST, RTF_BROADCAST,..) */ typedef int rnh_set_nh_pfxflags_f_t(u_int fibnum, const struct sockaddr *addr, const struct sockaddr *mask, struct nhop_object *nh); /* Fills in family-specific details that are not yet set up (mtu, nhop type, ..) */ typedef int rnh_augment_nh_f_t(u_int fibnum, struct nhop_object *nh); struct rib_head { struct radix_head head; rn_matchaddr_f_t *rnh_matchaddr; /* longest match for sockaddr */ rn_addaddr_f_t *rnh_addaddr; /* add based on sockaddr*/ rn_deladdr_f_t *rnh_deladdr; /* remove based on sockaddr */ rn_lookup_f_t *rnh_lookup; /* exact match for sockaddr */ rn_walktree_t *rnh_walktree; /* traverse tree */ rn_walktree_from_t *rnh_walktree_from; /* traverse tree below a */ rnh_set_nh_pfxflags_f_t *rnh_set_nh_pfxflags; /* hook to alter record prior to insertion */ rt_gen_t rnh_gen; /* datapath generation counter */ int rnh_multipath; /* multipath capable ? */ struct radix_node rnh_nodes[3]; /* empty tree for common case */ struct rmlock rib_lock; /* config/data path lock */ struct radix_mask_head rmhead; /* masks radix head */ struct vnet *rib_vnet; /* vnet pointer */ int rib_family; /* AF of the rtable */ u_int rib_fibnum; /* fib number */ struct callout expire_callout; /* Callout for expiring dynamic routes */ time_t next_expire; /* Next expire run ts */ uint32_t rnh_prefixes; /* Number of prefixes */ rt_gen_t rnh_gen_rib; /* fib algo: rib generation counter */ uint32_t rib_dying:1; /* rib is detaching */ uint32_t rib_algo_fixed:1;/* fixed algorithm */ uint32_t rib_algo_init:1;/* algo init done */ struct nh_control *nh_control; /* nexthop subsystem data */ rnh_augment_nh_f_t *rnh_augment_nh;/* hook to alter nexthop prior to insertion */ CK_STAILQ_HEAD(, rib_subscription) rnh_subscribers;/* notification subscribers */ }; #define RIB_RLOCK_TRACKER struct rm_priotracker _rib_tracker #define RIB_LOCK_INIT(rh) rm_init(&(rh)->rib_lock, "rib head lock") #define RIB_LOCK_DESTROY(rh) rm_destroy(&(rh)->rib_lock) #define RIB_RLOCK(rh) rm_rlock(&(rh)->rib_lock, &_rib_tracker) #define RIB_RUNLOCK(rh) rm_runlock(&(rh)->rib_lock, &_rib_tracker) #define RIB_WLOCK(rh) rm_wlock(&(rh)->rib_lock) #define RIB_WUNLOCK(rh) rm_wunlock(&(rh)->rib_lock) #define RIB_LOCK_ASSERT(rh) rm_assert(&(rh)->rib_lock, RA_LOCKED) #define RIB_WLOCK_ASSERT(rh) rm_assert(&(rh)->rib_lock, RA_WLOCKED) /* Constants */ #define RIB_MAX_RETRIES 3 #define RT_MAXFIBS UINT16_MAX #define RIB_MAX_MPATH_WIDTH 64 /* Macro for verifying fields in af-specific 'struct route' structures */ #define CHK_STRUCT_FIELD_GENERIC(_s1, _f1, _s2, _f2) \ _Static_assert(sizeof(((_s1 *)0)->_f1) == sizeof(((_s2 *)0)->_f2), \ "Fields " #_f1 " and " #_f2 " size differs"); \ _Static_assert(__offsetof(_s1, _f1) == __offsetof(_s2, _f2), \ "Fields " #_f1 " and " #_f2 " offset differs"); #define _CHK_ROUTE_FIELD(_route_new, _field) \ CHK_STRUCT_FIELD_GENERIC(struct route, _field, _route_new, _field) #define CHK_STRUCT_ROUTE_FIELDS(_route_new) \ _CHK_ROUTE_FIELD(_route_new, ro_nh) \ _CHK_ROUTE_FIELD(_route_new, ro_lle) \ _CHK_ROUTE_FIELD(_route_new, ro_prepend)\ _CHK_ROUTE_FIELD(_route_new, ro_plen) \ _CHK_ROUTE_FIELD(_route_new, ro_flags) \ _CHK_ROUTE_FIELD(_route_new, ro_mtu) \ _CHK_ROUTE_FIELD(_route_new, spare) #define CHK_STRUCT_ROUTE_COMPAT(_ro_new, _dst_new) \ CHK_STRUCT_ROUTE_FIELDS(_ro_new); \ _Static_assert(__offsetof(struct route, ro_dst) == __offsetof(_ro_new, _dst_new),\ "ro_dst and " #_dst_new " are at different offset") static inline void rib_bump_gen(struct rib_head *rnh) { #ifdef FIB_ALGO rnh->rnh_gen_rib++; #else rnh->rnh_gen++; #endif } struct rib_head *rt_tables_get_rnh(uint32_t table, sa_family_t family); int rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum); struct rib_cmd_info; VNET_PCPUSTAT_DECLARE(struct rtstat, rtstat); #define RTSTAT_ADD(name, val) \ VNET_PCPUSTAT_ADD(struct rtstat, rtstat, name, (val)) #define RTSTAT_INC(name) RTSTAT_ADD(name, 1) /* * Convert a 'struct radix_node *' to a 'struct rtentry *'. * The operation can be done safely (in this code) because a * 'struct rtentry' starts with two 'struct radix_node''s, the first * one representing leaf nodes in the routing tree, which is * what the code in radix.c passes us as a 'struct radix_node'. * * But because there are a lot of assumptions in this conversion, * do not cast explicitly, but always use the macro below. */ #define RNTORT(p) ((struct rtentry *)(p)) struct rtentry { struct radix_node rt_nodes[2]; /* tree glue, and other values */ /* * XXX struct rtentry must begin with a struct radix_node (or two!) * because the code does some casts of a 'struct radix_node *' * to a 'struct rtentry *' */ #define rt_key(r) (*((struct sockaddr **)(&(r)->rt_nodes->rn_key))) #define rt_mask(r) (*((struct sockaddr **)(&(r)->rt_nodes->rn_mask))) #define rt_key_const(r) (*((const struct sockaddr * const *)(&(r)->rt_nodes->rn_key))) #define rt_mask_const(r) (*((const struct sockaddr * const *)(&(r)->rt_nodes->rn_mask))) /* * 2 radix_node structurs above consists of 2x6 pointers, leaving * 4 pointers (32 bytes) of the second cache line on amd64. * */ struct nhop_object *rt_nhop; /* nexthop data */ union { /* * Destination address storage. * sizeof(struct sockaddr_in6) == 28, however * the dataplane-relevant part (e.g. address) lies * at offset 8..24, making the address not crossing * cacheline boundary. */ struct sockaddr_in rt_dst4; struct sockaddr_in6 rt_dst6; struct sockaddr rt_dst; char rt_dstb[28]; }; int rte_flags; /* up/down?, host/net */ u_long rt_weight; /* absolute weight */ struct rtentry *rt_chain; /* pointer to next rtentry to delete */ struct epoch_context rt_epoch_ctx; /* net epoch tracker */ }; /* * With the split between the routing entry and the nexthop, * rt_flags has to be split between these 2 entries. As rtentry * mostly contains prefix data and is thought to be generic enough * so one can transparently change the nexthop pointer w/o requiring * any other rtentry changes, most of rt_flags shifts to the particular nexthop. * / * * RTF_UP: rtentry, as an indication that it is linked. * RTF_HOST: rtentry, nhop. The latter indication is needed for the datapath * RTF_DYNAMIC: nhop, to make rtentry generic. * RTF_MODIFIED: nhop, to make rtentry generic. (legacy) * -- "native" path (nhop) properties: * RTF_GATEWAY, RTF_STATIC, RTF_PROTO1, RTF_PROTO2, RTF_PROTO3, RTF_FIXEDMTU, * RTF_PINNED, RTF_REJECT, RTF_BLACKHOLE, RTF_BROADCAST */ /* rtentry rt flag mask */ #define RTE_RT_FLAG_MASK (RTF_UP | RTF_HOST) /* route_temporal.c */ void tmproutes_update(struct rib_head *rnh, struct rtentry *rt, struct nhop_object *nh); void tmproutes_init(struct rib_head *rh); void tmproutes_destroy(struct rib_head *rh); /* route_ctl.c */ struct route_nhop_data; int change_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc); int change_route_conditional(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *nhd_orig, struct route_nhop_data *nhd_new, struct rib_cmd_info *rc); struct rtentry *lookup_prefix(struct rib_head *rnh, const struct rt_addrinfo *info, struct route_nhop_data *rnd); struct rtentry *lookup_prefix_rt(struct rib_head *rnh, const struct rtentry *rt, struct route_nhop_data *rnd); bool nhop_can_multipath(const struct nhop_object *nh); bool match_nhop_gw(const struct nhop_object *nh, const struct sockaddr *gw); int check_info_match_nhop(const struct rt_addrinfo *info, const struct rtentry *rt, const struct nhop_object *nh); -int can_override_nhop(const struct rt_addrinfo *info, - const struct nhop_object *nh); void vnet_rtzone_init(void); void vnet_rtzone_destroy(void); /* subscriptions */ void rib_init_subscriptions(struct rib_head *rnh); void rib_destroy_subscriptions(struct rib_head *rnh); /* Nexhops */ void nhops_init(void); int nhops_init_rib(struct rib_head *rh); void nhops_destroy_rib(struct rib_head *rh); void nhop_ref_object(struct nhop_object *nh); int nhop_try_ref_object(struct nhop_object *nh); void nhop_ref_any(struct nhop_object *nh); void nhop_free_any(struct nhop_object *nh); int nhop_create_from_info(struct rib_head *rnh, struct rt_addrinfo *info, struct nhop_object **nh_ret); int nhop_create_from_nhop(struct rib_head *rnh, const struct nhop_object *nh_orig, struct rt_addrinfo *info, struct nhop_object **pnh_priv); void nhops_update_ifmtu(struct rib_head *rh, struct ifnet *ifp, uint32_t mtu); int nhops_dump_sysctl(struct rib_head *rh, struct sysctl_req *w); /* MULTIPATH */ #define MPF_MULTIPATH 0x08 /* need to be consistent with NHF_MULTIPATH */ struct nhgrp_object { uint16_t nhg_flags; /* nexthop group flags */ uint8_t nhg_size; /* dataplain group size */ uint8_t spare; struct nhop_object *nhops[0]; /* nhops */ }; static inline struct nhop_object * nhop_select(struct nhop_object *nh, uint32_t flowid) { #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh)) { struct nhgrp_object *nhg = (struct nhgrp_object *)nh; nh = nhg->nhops[flowid % nhg->nhg_size]; } #endif return (nh); } struct weightened_nhop; /* mpath_ctl.c */ int add_route_mpath(struct rib_head *rnh, struct rt_addrinfo *info, struct rtentry *rt, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc); -int del_route_mpath(struct rib_head *rh, struct rt_addrinfo *info, - struct rtentry *rt, struct nhgrp_object *nhg, struct rib_cmd_info *rc); /* nhgrp.c */ int nhgrp_ctl_init(struct nh_control *ctl); void nhgrp_ctl_free(struct nh_control *ctl); void nhgrp_ctl_unlink_all(struct nh_control *ctl); /* nhgrp_ctl.c */ int nhgrp_dump_sysctl(struct rib_head *rh, struct sysctl_req *w); int nhgrp_get_group(struct rib_head *rh, struct weightened_nhop *wn, int num_nhops, struct nhgrp_object **pnhg); -typedef bool nhgrp_filter_cb_t(const struct nhop_object *nh, void *data); -int nhgrp_get_filtered_group(struct rib_head *rh, const struct nhgrp_object *src, - nhgrp_filter_cb_t flt_func, void *flt_data, struct route_nhop_data *rnd); +int nhgrp_get_filtered_group(struct rib_head *rh, const struct rtentry *rt, + const struct nhgrp_object *src, rib_filter_f_t flt_func, void *flt_data, + struct route_nhop_data *rnd); int nhgrp_get_addition_group(struct rib_head *rnh, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_new); void nhgrp_ref_object(struct nhgrp_object *nhg); uint32_t nhgrp_get_idx(const struct nhgrp_object *nhg); void nhgrp_free(struct nhgrp_object *nhg); /* rtsock */ int rtsock_routemsg(int cmd, struct rtentry *rt, struct nhop_object *nh, int fibnum); int rtsock_routemsg_info(int cmd, struct rt_addrinfo *info, int fibnum); int rtsock_addrmsg(int cmd, struct ifaddr *ifa, int fibnum); /* lookup_framework.c */ void fib_grow_rtables(uint32_t new_num_tables); void fib_setup_family(int family, uint32_t num_tables); void fib_destroy_rib(struct rib_head *rh); void vnet_fib_init(void); void vnet_fib_destroy(void); /* Entropy data used for outbound hashing */ #define MPATH_ENTROPY_KEY_LEN 40 extern uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN]; #endif