diff --git a/sys/net/route/nhgrp_ctl.c b/sys/net/route/nhgrp_ctl.c index 842f9d5376bc..5d6fb219f877 100644 --- a/sys/net/route/nhgrp_ctl.c +++ b/sys/net/route/nhgrp_ctl.c @@ -1,916 +1,978 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_inet.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME nhgrp_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains the supporting functions for creating multipath groups * and compiling their dataplane parts. */ /* MPF_MULTIPATH must be the same as NHF_MULTIPATH for nhop selection to work */ _Static_assert(MPF_MULTIPATH == NHF_MULTIPATH, "MPF_MULTIPATH must be the same as NHF_MULTIPATH"); /* Offset and size of flags field has to be the same for nhop/nhop groups */ CHK_STRUCT_FIELD_GENERIC(struct nhop_object, nh_flags, struct nhgrp_object, nhg_flags); /* Cap multipath to 64, as the larger values would break rib_cmd_info bmasks */ CTASSERT(RIB_MAX_MPATH_WIDTH <= 64); static int wn_cmp_idx(const void *a, const void *b); static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops); static struct nhgrp_priv *get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, uint32_t uidx, int *perror); static void destroy_nhgrp(struct nhgrp_priv *nhg_priv); static void destroy_nhgrp_epoch(epoch_context_t ctx); static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv); static int wn_cmp_idx(const void *a, const void *b) { const struct weightened_nhop *w_a = a; const struct weightened_nhop *w_b = b; uint32_t a_idx = w_a->nh->nh_priv->nh_idx; uint32_t b_idx = w_b->nh->nh_priv->nh_idx; if (a_idx < b_idx) return (-1); else if (a_idx > b_idx) return (1); else return (0); } /* * Perform in-place sorting for array of nexthops in @wn. * Sort by nexthop index ascending. */ static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops) { qsort(wn, num_nhops, sizeof(struct weightened_nhop), wn_cmp_idx); } /* * In order to determine the minimum weight difference in the array * of weights, create a sorted array of weights, using spare "storage" * field in the `struct weightened_nhop`. * Assume weights to be (mostly) the same and use insertion sort to * make it sorted. */ static void sort_weightened_nhops_weights(struct weightened_nhop *wn, int num_items) { wn[0].storage = wn[0].weight; for (int i = 1, j = 0; i < num_items; i++) { uint32_t weight = wn[i].weight; // read from 'weight' as it's not reordered /* Move all weights > weight 1 position right */ for (j = i - 1; j >= 0 && wn[j].storage > weight; j--) wn[j + 1].storage = wn[j].storage; wn[j + 1].storage = weight; } } /* * Calculate minimum number of slots required to fit the existing * set of weights in the common use case where weights are "easily" * comparable. * Assumes @wn is sorted by weight ascending and each weight is > 0. * Returns number of slots or 0 if precise calculation failed. * * Some examples: * note: (i, X) pair means (nhop=i, weight=X): * (1, 1) (2, 2) -> 3 slots [1, 2, 2] * (1, 100), (2, 200) -> 3 slots [1, 2, 2] * (1, 100), (2, 200), (3, 400) -> 7 slots [1, 2, 2, 3, 3, 3] */ static uint32_t calc_min_mpath_slots_fast(struct weightened_nhop *wn, size_t num_items, uint64_t *ptotal) { uint32_t i, last, xmin; uint64_t total = 0; // Get sorted array of weights in .storage field sort_weightened_nhops_weights(wn, num_items); last = 0; xmin = wn[0].storage; for (i = 0; i < num_items; i++) { total += wn[i].storage; if ((wn[i].storage != last) && ((wn[i].storage - last < xmin) || xmin == 0)) { xmin = wn[i].storage - last; } last = wn[i].storage; } *ptotal = total; /* xmin is the minimum unit of desired capacity */ if ((total % xmin) != 0) return (0); for (i = 0; i < num_items; i++) { if ((wn[i].weight % xmin) != 0) return (0); } return ((uint32_t)(total / xmin)); } /* * Calculate minimum number of slots required to fit the existing * set of weights while maintaining weight coefficients. * * Assume @wn is sorted by weight ascending and each weight is > 0. * * Tries to find simple precise solution first and falls back to * RIB_MAX_MPATH_WIDTH in case of any failure. */ static uint32_t calc_min_mpath_slots(struct weightened_nhop *wn, size_t num_items) { uint32_t v; uint64_t total; v = calc_min_mpath_slots_fast(wn, num_items, &total); if (total == 0) return (0); if ((v == 0) || (v > RIB_MAX_MPATH_WIDTH)) v = RIB_MAX_MPATH_WIDTH; return (v); } /* * Nexthop group data consists of * 1) dataplane part, with nhgrp_object as a header followed by an * arbitrary number of nexthop pointers. * 2) control plane part, with nhgrp_priv as a header, followed by * an arbirtrary number of 'struct weightened_nhop' object. * * Given nexthop groups are (mostly) immutable, allocate all data * in one go. * */ __noinline static size_t get_nhgrp_alloc_size(uint32_t nhg_size, uint32_t num_nhops) { size_t sz; sz = sizeof(struct nhgrp_object); sz += nhg_size * sizeof(struct nhop_object *); sz += sizeof(struct nhgrp_priv); sz += num_nhops * sizeof(struct weightened_nhop); return (sz); } /* * Compile actual list of nexthops to be used by datapath from * the nexthop group @dst. * * For example, compiling control plane list of 2 nexthops * [(200, A), (100, B)] would result in the datapath array * [A, A, B] */ static void compile_nhgrp(struct nhgrp_priv *dst_priv, const struct weightened_nhop *x, uint32_t num_slots) { struct nhgrp_object *dst; int i, slot_idx, remaining_slots; uint64_t remaining_sum, nh_weight, nh_slots; slot_idx = 0; dst = dst_priv->nhg; /* Calculate sum of all weights */ remaining_sum = 0; for (i = 0; i < dst_priv->nhg_nh_count; i++) remaining_sum += x[i].weight; remaining_slots = num_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, "sum: %lu, slots: %d", remaining_sum, remaining_slots); for (i = 0; i < dst_priv->nhg_nh_count; i++) { /* Calculate number of slots for the current nexthop */ if (remaining_sum > 0) { nh_weight = (uint64_t)x[i].weight; nh_slots = (nh_weight * remaining_slots / remaining_sum); } else nh_slots = 0; remaining_sum -= x[i].weight; remaining_slots -= nh_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, " rem_sum: %lu, rem_slots: %d nh_slots: %d, slot_idx: %d", remaining_sum, remaining_slots, (int)nh_slots, slot_idx); KASSERT((slot_idx + nh_slots <= num_slots), ("index overflow during nhg compilation")); while (nh_slots-- > 0) dst->nhops[slot_idx++] = x[i].nh; } } /* * Allocates new nexthop group for the list of weightened nexthops. * Assume sorted list. * Does NOT reference any nexthops in the group. * Returns group with refcount=1 or NULL. */ static struct nhgrp_priv * alloc_nhgrp(struct weightened_nhop *wn, int num_nhops) { uint32_t nhgrp_size; struct nhgrp_object *nhg; struct nhgrp_priv *nhg_priv; nhgrp_size = calc_min_mpath_slots(wn, num_nhops); if (nhgrp_size == 0) { /* Zero weights, abort */ return (NULL); } size_t sz = get_nhgrp_alloc_size(nhgrp_size, num_nhops); nhg = malloc(sz, M_NHOP, M_NOWAIT | M_ZERO); if (nhg == NULL) { FIB_NH_LOG(LOG_INFO, wn[0].nh, "unable to allocate group with num_nhops %d (compiled %u)", num_nhops, nhgrp_size); return (NULL); } /* Has to be the first to make NHGRP_PRIV() work */ nhg->nhg_size = nhgrp_size; nhg->nhg_flags = MPF_MULTIPATH; nhg_priv = NHGRP_PRIV(nhg); nhg_priv->nhg_nh_count = num_nhops; refcount_init(&nhg_priv->nhg_refcount, 1); /* Please see nhgrp_free() comments on the initial value */ refcount_init(&nhg_priv->nhg_linked, 2); nhg_priv->nhg = nhg; memcpy(&nhg_priv->nhg_nh_weights[0], wn, num_nhops * sizeof(struct weightened_nhop)); FIB_NH_LOG(LOG_DEBUG, wn[0].nh, "num_nhops: %d, compiled_nhop: %u", num_nhops, nhgrp_size); compile_nhgrp(nhg_priv, wn, nhg->nhg_size); return (nhg_priv); } void nhgrp_ref_object(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; u_int old __diagused; nhg_priv = NHGRP_PRIV(nhg); old = refcount_acquire(&nhg_priv->nhg_refcount); KASSERT(old > 0, ("%s: nhgrp object %p has 0 refs", __func__, nhg)); } void nhgrp_free(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; struct nh_control *ctl; struct epoch_tracker et; nhg_priv = NHGRP_PRIV(nhg); if (!refcount_release(&nhg_priv->nhg_refcount)) return; /* * group objects don't have an explicit lock attached to it. * As groups are reclaimed based on reference count, it is possible * that some groups will persist after vnet destruction callback * called. Given that, handle scenario with nhgrp_free_group() being * called either after or simultaneously with nhgrp_ctl_unlink_all() * by using another reference counter: nhg_linked. * * There are only 2 places, where nhg_linked can be decreased: * rib destroy (nhgrp_ctl_unlink_all) and this function. * nhg_link can never be increased. * * Hence, use initial value of 2 to make use of * refcount_release_if_not_last(). * * There can be two scenarious when calling this function: * * 1) nhg_linked value is 2. This means that either * nhgrp_ctl_unlink_all() has not been called OR it is running, * but we are guaranteed that nh_control won't be freed in * this epoch. Hence, nexthop can be safely unlinked. * * 2) nh_linked value is 1. In that case, nhgrp_ctl_unlink_all() * has been called and nhgrp unlink can be skipped. */ NET_EPOCH_ENTER(et); if (refcount_release_if_not_last(&nhg_priv->nhg_linked)) { ctl = nhg_priv->nh_control; if (unlink_nhgrp(ctl, nhg_priv) == NULL) { /* Do not try to reclaim */ RT_LOG(LOG_INFO, "Failed to unlink nexhop group %p", nhg_priv); NET_EPOCH_EXIT(et); return; } } NET_EPOCH_EXIT(et); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); epoch_call(net_epoch_preempt, destroy_nhgrp_epoch, &nhg_priv->nhg_epoch_ctx); } /* * Destroys all local resources belonging to @nhg_priv. */ __noinline static void destroy_nhgrp_int(struct nhgrp_priv *nhg_priv) { free(nhg_priv->nhg, M_NHOP); } __noinline static void destroy_nhgrp(struct nhgrp_priv *nhg_priv) { KASSERT((nhg_priv->nhg_refcount == 0), ("nhg_refcount != 0")); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); IF_DEBUG_LEVEL(LOG_DEBUG2) { char nhgbuf[NHOP_PRINT_BUFSIZE] __unused; FIB_NH_LOG(LOG_DEBUG2, nhg_priv->nhg_nh_weights[0].nh, "destroying %s", nhgrp_print_buf(nhg_priv->nhg, nhgbuf, sizeof(nhgbuf))); } free_nhgrp_nhops(nhg_priv); destroy_nhgrp_int(nhg_priv); } /* * Epoch callback indicating group is safe to destroy */ static void destroy_nhgrp_epoch(epoch_context_t ctx) { struct nhgrp_priv *nhg_priv; nhg_priv = __containerof(ctx, struct nhgrp_priv, nhg_epoch_ctx); destroy_nhgrp(nhg_priv); } static bool ref_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { if (nhop_try_ref_object(nhg_priv->nhg_nh_weights[i].nh) != 0) continue; /* * Failed to ref the nexthop, b/c it's deleted. * Need to rollback references back. */ for (int j = 0; j < i; j++) nhop_free(nhg_priv->nhg_nh_weights[j].nh); return (false); } return (true); } static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) nhop_free(nhg_priv->nhg_nh_weights[i].nh); } /* - * Creates or looks up an existing nexthop group based on @wn and @num_nhops. - * - * Returns referenced nhop group or NULL, passing error code in @perror. + * Allocate nexthop group of size @num_nhops with nexthops specified by + * @wn. Nexthops have to be unique and match the fibnum/family of the group. + * Returns unlinked nhgrp object on success or NULL and non-zero perror. */ -struct nhgrp_priv * -get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, - uint32_t uidx, int *perror) +struct nhgrp_object * +nhgrp_alloc(uint32_t fibnum, int family, struct weightened_nhop *wn, int num_nhops, + int *perror) { - struct nhgrp_priv *key, *nhg_priv; + struct rib_head *rh = rt_tables_get_rnh(fibnum, family); + struct nhgrp_priv *nhg_priv; + struct nh_control *ctl; + + if (rh == NULL) { + *perror = E2BIG; + return (NULL); + } + + ctl = rh->nh_control; if (num_nhops > RIB_MAX_MPATH_WIDTH) { *perror = E2BIG; return (NULL); } if (ctl->gr_head.hash_size == 0) { /* First multipath request. Bootstrap mpath datastructures. */ if (nhgrp_ctl_alloc_default(ctl, M_NOWAIT) == 0) { *perror = ENOMEM; return (NULL); } } /* Sort nexthops & check there are no duplicates */ sort_weightened_nhops(wn, num_nhops); uint32_t last_id = 0; for (int i = 0; i < num_nhops; i++) { + if (wn[i].nh->nh_priv->nh_control != ctl) { + *perror = EINVAL; + return (NULL); + } if (wn[i].nh->nh_priv->nh_idx == last_id) { *perror = EEXIST; return (NULL); } last_id = wn[i].nh->nh_priv->nh_idx; } - if ((key = alloc_nhgrp(wn, num_nhops)) == NULL) { + if ((nhg_priv = alloc_nhgrp(wn, num_nhops)) == NULL) { *perror = ENOMEM; return (NULL); } - key->nhg_uidx = uidx; + nhg_priv->nh_control = ctl; + + *perror = 0; + return (nhg_priv->nhg); +} + +/* + * Finds an existing group matching @nhg or links @nhg to the tree. + * Returns the referenced group or NULL and non-zero @perror. + */ +struct nhgrp_object * +nhgrp_get_nhgrp(struct nhgrp_object *nhg, int *perror) +{ + struct nhgrp_priv *nhg_priv, *key = NHGRP_PRIV(nhg); + struct nh_control *ctl = key->nh_control; nhg_priv = find_nhgrp(ctl, key); if (nhg_priv != NULL) { /* * Free originally-created group. As it hasn't been linked * and the dependent nexhops haven't been referenced, just free * the group. */ destroy_nhgrp_int(key); *perror = 0; - return (nhg_priv); + return (nhg_priv->nhg); } else { /* No existing group, try to link the new one */ if (!ref_nhgrp_nhops(key)) { /* * Some of the nexthops have been scheduled for deletion. * As the group hasn't been linked / no nexhops have been * referenced, call the final destructor immediately. */ destroy_nhgrp_int(key); *perror = EAGAIN; return (NULL); } if (link_nhgrp(ctl, key) == 0) { /* Unable to allocate index? */ *perror = EAGAIN; free_nhgrp_nhops(key); destroy_nhgrp_int(key); return (NULL); } *perror = 0; - return (key); + return (nhg); } /* NOTREACHED */ } +/* + * Creates or looks up an existing nexthop group based on @wn and @num_nhops. + * + * Returns referenced nhop group or NULL, passing error code in @perror. + */ +struct nhgrp_priv * +get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, + uint32_t uidx, int *perror) +{ + struct nhgrp_object *nhg; + + nhg = nhgrp_alloc(ctl->ctl_rh->rib_fibnum, ctl->ctl_rh->rib_family, + wn, num_nhops, perror); + if (nhg == NULL) + return (NULL); + nhgrp_set_uidx(nhg, uidx); + nhg = nhgrp_get_nhgrp(nhg, perror); + if (nhg != NULL) + return (NHGRP_PRIV(nhg)); + return (NULL); +} + + /* * Appends one or more nexthops denoted by @wm to the nexthop group @gr_orig. * * Returns referenced nexthop group or NULL. In the latter case, @perror is * filled with an error code. * Note that function does NOT care if the next nexthops already exists * in the @gr_orig. As a result, they will be added, resulting in the * same nexthop being present multiple times in the new group. */ static struct nhgrp_priv * append_nhops(struct nh_control *ctl, const struct nhgrp_object *gr_orig, struct weightened_nhop *wn, int num_nhops, int *perror) { char storage[64]; struct weightened_nhop *pnhops; struct nhgrp_priv *nhg_priv; const struct nhgrp_priv *src_priv; size_t sz; int curr_nhops; src_priv = NHGRP_PRIV_CONST(gr_orig); curr_nhops = src_priv->nhg_nh_count; *perror = 0; sz = (src_priv->nhg_nh_count + num_nhops) * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { pnhops = malloc(sz, M_TEMP, M_NOWAIT); if (pnhops == NULL) { *perror = ENOMEM; return (NULL); } } /* Copy nhops from original group first */ memcpy(pnhops, src_priv->nhg_nh_weights, curr_nhops * sizeof(struct weightened_nhop)); memcpy(&pnhops[curr_nhops], wn, num_nhops * sizeof(struct weightened_nhop)); curr_nhops += num_nhops; nhg_priv = get_nhgrp(ctl, pnhops, curr_nhops, 0, perror); if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); if (nhg_priv == NULL) return (NULL); return (nhg_priv); } /* * Creates/finds nexthop group based on @wn and @num_nhops. * Returns 0 on success with referenced group in @rnd, or * errno. * * If the error is EAGAIN, then the operation can be retried. */ int nhgrp_get_group(struct rib_head *rh, struct weightened_nhop *wn, int num_nhops, uint32_t uidx, struct nhgrp_object **pnhg) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; int error; nhg_priv = get_nhgrp(ctl, wn, num_nhops, uidx, &error); if (nhg_priv != NULL) *pnhg = nhg_priv->nhg; return (error); } /* * Creates new nexthop group based on @src group without the nexthops * chosen by @flt_func. * Returns 0 on success, storring the reference nhop group/object in @rnd. */ int nhgrp_get_filtered_group(struct rib_head *rh, const struct rtentry *rt, const struct nhgrp_object *src, rib_filter_f_t flt_func, void *flt_data, struct route_nhop_data *rnd) { char storage[64]; struct nh_control *ctl = rh->nh_control; struct weightened_nhop *pnhops; const struct nhgrp_priv *mp_priv, *src_priv; size_t sz; int error, i, num_nhops; src_priv = NHGRP_PRIV_CONST(src); sz = src_priv->nhg_nh_count * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { if ((pnhops = malloc(sz, M_TEMP, M_NOWAIT)) == NULL) return (ENOMEM); } /* Filter nexthops */ error = 0; num_nhops = 0; for (i = 0; i < src_priv->nhg_nh_count; i++) { if (flt_func(rt, src_priv->nhg_nh_weights[i].nh, flt_data)) continue; memcpy(&pnhops[num_nhops++], &src_priv->nhg_nh_weights[i], sizeof(struct weightened_nhop)); } if (num_nhops == 0) { rnd->rnd_nhgrp = NULL; rnd->rnd_weight = 0; } else if (num_nhops == 1) { rnd->rnd_nhop = pnhops[0].nh; rnd->rnd_weight = pnhops[0].weight; if (nhop_try_ref_object(rnd->rnd_nhop) == 0) error = EAGAIN; } else { mp_priv = get_nhgrp(ctl, pnhops, num_nhops, 0, &error); if (mp_priv != NULL) rnd->rnd_nhgrp = mp_priv->nhg; rnd->rnd_weight = 0; } if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); return (error); } /* * Creates new multipath group based on existing group/nhop in @rnd_orig and * to-be-added nhop @wn_add. * Returns 0 on success and stores result in @rnd_new. */ int nhgrp_get_addition_group(struct rib_head *rh, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_new) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; struct weightened_nhop wn[2] = {}; int error; if (rnd_orig->rnd_nhop == NULL) { /* No paths to add to, just reference current nhop */ *rnd_new = *rnd_add; if (nhop_try_ref_object(rnd_new->rnd_nhop) == 0) return (EAGAIN); return (0); } wn[0].nh = rnd_add->rnd_nhop; wn[0].weight = rnd_add->rnd_weight; if (!NH_IS_NHGRP(rnd_orig->rnd_nhop)) { /* Simple merge of 2 non-multipath nexthops */ wn[1].nh = rnd_orig->rnd_nhop; wn[1].weight = rnd_orig->rnd_weight; nhg_priv = get_nhgrp(ctl, wn, 2, 0, &error); } else { /* Get new nhop group with @rt->rt_nhop as an additional nhop */ nhg_priv = append_nhops(ctl, rnd_orig->rnd_nhgrp, &wn[0], 1, &error); } if (nhg_priv == NULL) return (error); rnd_new->rnd_nhgrp = nhg_priv->nhg; rnd_new->rnd_weight = 0; return (0); } /* * Returns pointer to array of nexthops with weights for * given @nhg. Stores number of items in the array into @pnum_nhops. */ const struct weightened_nhop * nhgrp_get_nhops(const struct nhgrp_object *nhg, uint32_t *pnum_nhops) { const struct nhgrp_priv *nhg_priv; KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); nhg_priv = NHGRP_PRIV_CONST(nhg); *pnum_nhops = nhg_priv->nhg_nh_count; return (nhg_priv->nhg_nh_weights); } +void +nhgrp_set_uidx(struct nhgrp_object *nhg, uint32_t uidx) +{ + struct nhgrp_priv *nhg_priv; + + KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); + + nhg_priv = NHGRP_PRIV(nhg); + + nhg_priv->nhg_uidx = uidx; +} + uint32_t nhgrp_get_uidx(const struct nhgrp_object *nhg) { const struct nhgrp_priv *nhg_priv; KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); nhg_priv = NHGRP_PRIV_CONST(nhg); return (nhg_priv->nhg_uidx); } /* * Prints nexhop group @nhg data in the provided @buf. * Example: nhg#33/sz=3:[#1:100,#2:100,#3:100] * Example: nhg#33/sz=5:[#1:100,#2:100,..] */ char * nhgrp_print_buf(const struct nhgrp_object *nhg, char *buf, size_t bufsize) { const struct nhgrp_priv *nhg_priv = NHGRP_PRIV_CONST(nhg); int off = snprintf(buf, bufsize, "nhg#%u/sz=%u:[", nhg_priv->nhg_idx, nhg_priv->nhg_nh_count); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { const struct weightened_nhop *wn = &nhg_priv->nhg_nh_weights[i]; int len = snprintf(&buf[off], bufsize - off, "#%u:%u,", wn->nh->nh_priv->nh_idx, wn->weight); if (len + off + 3 >= bufsize) { int len = snprintf(&buf[off], bufsize - off, "..."); off += len; break; } off += len; } if (off > 0) off--; // remove last "," if (off + 1 < bufsize) snprintf(&buf[off], bufsize - off, "]"); return buf; } __noinline static int dump_nhgrp_entry(struct rib_head *rh, const struct nhgrp_priv *nhg_priv, char *buffer, size_t buffer_size, struct sysctl_req *w) { struct rt_msghdr *rtm; struct nhgrp_external *nhge; struct nhgrp_container *nhgc; const struct nhgrp_object *nhg; struct nhgrp_nhop_external *ext; int error; size_t sz; nhg = nhg_priv->nhg; sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); /* controlplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; /* dataplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; KASSERT(sz <= buffer_size, ("increase nhgrp buffer size")); bzero(buffer, sz); rtm = (struct rt_msghdr *)buffer; rtm->rtm_msglen = sz; rtm->rtm_version = RTM_VERSION; rtm->rtm_type = RTM_GET; nhge = (struct nhgrp_external *)(rtm + 1); nhge->nhg_idx = nhg_priv->nhg_idx; nhge->nhg_refcount = nhg_priv->nhg_refcount; /* fill in control plane nexthops firs */ nhgc = (struct nhgrp_container *)(nhge + 1); nhgc->nhgc_type = NHG_C_TYPE_CNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; nhgc->nhgc_count = nhg_priv->nhg_nh_count; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { ext[i].nh_idx = nhg_priv->nhg_nh_weights[i].nh->nh_priv->nh_idx; ext[i].nh_weight = nhg_priv->nhg_nh_weights[i].weight; } /* fill in dataplane nexthops */ nhgc = (struct nhgrp_container *)(&ext[nhg_priv->nhg_nh_count]); nhgc->nhgc_type = NHG_C_TYPE_DNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; nhgc->nhgc_count = nhg->nhg_size; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg->nhg_size; i++) { ext[i].nh_idx = nhg->nhops[i]->nh_priv->nh_idx; ext[i].nh_weight = 0; } error = SYSCTL_OUT(w, buffer, sz); return (error); } uint32_t nhgrp_get_idx(const struct nhgrp_object *nhg) { const struct nhgrp_priv *nhg_priv; nhg_priv = NHGRP_PRIV_CONST(nhg); return (nhg_priv->nhg_idx); } uint8_t nhgrp_get_origin(const struct nhgrp_object *nhg) { return (NHGRP_PRIV_CONST(nhg)->nhg_origin); } void nhgrp_set_origin(struct nhgrp_object *nhg, uint8_t origin) { NHGRP_PRIV(nhg)->nhg_origin = origin; } uint32_t nhgrp_get_count(struct rib_head *rh) { struct nh_control *ctl; uint32_t count; ctl = rh->nh_control; NHOPS_RLOCK(ctl); count = ctl->gr_head.items_count; NHOPS_RUNLOCK(ctl); return (count); } int nhgrp_dump_sysctl(struct rib_head *rh, struct sysctl_req *w) { struct nh_control *ctl = rh->nh_control; struct epoch_tracker et; struct nhgrp_priv *nhg_priv; char *buffer; size_t sz; int error = 0; if (ctl->gr_head.items_count == 0) return (0); /* Calculate the maximum nhop group size in bytes */ sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); sz += 2 * sizeof(struct nhgrp_container); sz += 2 * sizeof(struct nhgrp_nhop_external) * RIB_MAX_MPATH_WIDTH; buffer = malloc(sz, M_TEMP, M_NOWAIT); if (buffer == NULL) return (ENOMEM); NET_EPOCH_ENTER(et); NHOPS_RLOCK(ctl); CHT_SLIST_FOREACH(&ctl->gr_head, mpath, nhg_priv) { error = dump_nhgrp_entry(rh, nhg_priv, buffer, sz, w); if (error != 0) break; } CHT_SLIST_FOREACH_END; NHOPS_RUNLOCK(ctl); NET_EPOCH_EXIT(et); free(buffer, M_TEMP); return (error); } diff --git a/sys/net/route/nhop.h b/sys/net/route/nhop.h index f0feccd6c0f2..f79cdb3459e9 100644 --- a/sys/net/route/nhop.h +++ b/sys/net/route/nhop.h @@ -1,292 +1,296 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This header file contains public definitions for the nexthop routing subsystem. */ #ifndef _NET_ROUTE_NHOP_H_ #define _NET_ROUTE_NHOP_H_ #include /* sockaddr_in && sockaddr_in6 */ #include enum nhop_type { NH_TYPE_IPV4_ETHER_RSLV = 1, /* IPv4 ethernet without GW */ NH_TYPE_IPV4_ETHER_NHOP = 2, /* IPv4 with pre-calculated ethernet encap */ NH_TYPE_IPV6_ETHER_RSLV = 3, /* IPv6 ethernet, without GW */ NH_TYPE_IPV6_ETHER_NHOP = 4 /* IPv6 with pre-calculated ethernet encap*/ }; #ifdef _KERNEL /* * Define shorter version of AF_LINK sockaddr. * * Currently the only use case of AF_LINK gateway is storing * interface index of the interface of the source IPv6 address. * This is used by the IPv6 code for the connections over loopback * interface. * * The structure below copies 'struct sockaddr_dl', reducing the * size of sdl_data buffer, as it is not used. This change * allows to store the AF_LINK gateways in the nhop gateway itself, * simplifying control plane handling. */ struct sockaddr_dl_short { u_char sdl_len; /* Total length of sockaddr */ u_char sdl_family; /* AF_LINK */ u_short sdl_index; /* if != 0, system given index for interface */ u_char sdl_type; /* interface type */ u_char sdl_nlen; /* interface name length, no trailing 0 reqd. */ u_char sdl_alen; /* link level address length */ u_char sdl_slen; /* link layer selector length */ char sdl_data[8]; /* unused */ }; #define NHOP_RELATED_FLAGS \ (RTF_GATEWAY | RTF_HOST | RTF_REJECT | RTF_BLACKHOLE | \ RTF_FIXEDMTU | RTF_LOCAL | RTF_BROADCAST | RTF_MULTICAST) struct nh_control; struct nhop_priv; /* * Struct 'nhop_object' field description: * * nh_flags: NHF_ flags used in the dataplane code. NHF_GATEWAY or NHF_BLACKHOLE * can be examples of such flags. * nh_mtu: ready-to-use nexthop mtu. Already accounts for the link-level header, * interface MTU and protocol-specific limitations. * nh_prepend_len: link-level prepend length. Currently unused. * nh_ifp: logical transmit interface. The one from which if_transmit() will be * called. Guaranteed to be non-NULL. * nh_aifp: ifnet of the source address. Same as nh_ifp except IPv6 loopback * routes. See the example below. * nh_ifa: interface address to use. Guaranteed to be non-NULL. * nh_pksent: counter(9) reflecting the number of packets transmitted. * * gw_: storage suitable to hold AF_INET, AF_INET6 or AF_LINK gateway. More * details ara available in the examples below. * * Examples: * * Direct routes (routes w/o gateway): * NHF_GATEWAY is NOT set. * nh_ifp denotes the logical transmit interface (). * nh_aifp is the same as nh_ifp * gw_sa contains AF_LINK sa with nh_aifp ifindex (compat) * Loopback routes: * NHF_GATEWAY is NOT set. * nh_ifp points to the loopback interface (lo0). * nh_aifp points to the interface where the destination address belongs to. * This is useful in IPv6 link-local-over-loopback communications. * gw_sa contains AF_LINK sa with nh_aifp ifindex (compat) * GW routes: * NHF_GATEWAY is set. * nh_ifp denotes the logical transmit interface. * nh_aifp is the same as nh_ifp * gw_sa contains L3 address (either AF_INET or AF_INET6). * * * Note: struct nhop_object fields are ordered in a way that * supports memcmp-based comparisons. * */ #define NHOP_END_CMP (__offsetof(struct nhop_object, nh_pksent)) struct nhop_object { uint16_t nh_flags; /* nhop flags */ uint16_t nh_mtu; /* nexthop mtu */ union { struct sockaddr_in gw4_sa; /* GW accessor as IPv4 */ struct sockaddr_in6 gw6_sa; /* GW accessor as IPv6 */ struct sockaddr gw_sa; struct sockaddr_dl_short gwl_sa; /* AF_LINK gw (compat) */ char gw_buf[28]; }; struct ifnet *nh_ifp; /* Logical egress interface. Always != NULL */ struct ifaddr *nh_ifa; /* interface address to use. Always != NULL */ struct ifnet *nh_aifp; /* ifnet of the source address. Always != NULL */ counter_u64_t nh_pksent; /* packets sent using this nhop */ /* 32 bytes + 4xPTR == 64(amd64) / 48(i386) */ uint8_t nh_prepend_len; /* length of prepend data */ uint8_t spare[3]; uint32_t spare1; /* alignment */ char nh_prepend[48]; /* L2 prepend */ struct nhop_priv *nh_priv; /* control plane data */ /* -- 128 bytes -- */ }; /* * Nhop validness. * * Currently we verify whether link is up or not on every packet, which can be * quite costy. * TODO: subscribe for the interface notifications and update the nexthops * with NHF_INVALID flag. */ #define NH_IS_VALID(_nh) RT_LINK_IS_UP((_nh)->nh_ifp) #define NH_IS_NHGRP(_nh) ((_nh)->nh_flags & NHF_MULTIPATH) #define NH_FREE(_nh) do { \ nhop_free(_nh); \ /* guard against invalid refs */ \ _nh = NULL; \ } while (0) struct weightened_nhop { struct nhop_object *nh; uint32_t weight; uint32_t storage; }; void nhop_free(struct nhop_object *nh); struct sysctl_req; struct sockaddr_dl; struct rib_head; /* flags that can be set using nhop_set_rtflags() */ #define RT_SET_RTFLAGS_MASK (RTF_PROTO1 | RTF_PROTO2 | RTF_PROTO3 | RTF_STATIC) #define RT_CHANGE_RTFLAGS_MASK RT_SET_RTFLAGS_MASK struct nhop_object *nhop_alloc(uint32_t fibnum, int family); void nhop_copy(struct nhop_object *nh, const struct nhop_object *nh_orig); struct nhop_object *nhop_get_nhop(struct nhop_object *nh, int *perror); int nhop_get_unlinked(struct nhop_object *nh); void nhop_set_direct_gw(struct nhop_object *nh, struct ifnet *ifp); bool nhop_set_gw(struct nhop_object *nh, const struct sockaddr *sa, bool is_gw); void nhop_set_mtu(struct nhop_object *nh, uint32_t mtu, bool from_user); void nhop_set_rtflags(struct nhop_object *nh, int rt_flags); void nhop_set_pxtype_flag(struct nhop_object *nh, int nh_flag); void nhop_set_broadcast(struct nhop_object *nh, bool is_broadcast); void nhop_set_blackhole(struct nhop_object *nh, int blackhole_rt_flag); void nhop_set_pinned(struct nhop_object *nh, bool is_pinned); void nhop_set_redirect(struct nhop_object *nh, bool is_redirect); void nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type); void nhop_set_src(struct nhop_object *nh, struct ifaddr *ifa); void nhop_set_transmit_ifp(struct nhop_object *nh, struct ifnet *ifp); #define NH_ORIGIN_UNSPEC 0 /* not set */ #define NH_ORIGIN_REDIRECT 1 /* kernel-originated redirect */ #define NH_ORIGIN_KERNEL 2 /* kernel (interface) routes */ #define NH_ORIGIN_BOOT 3 /* kernel-originated routes at boot */ #define NH_ORIGIN_STATIC 4 /* route(8) routes */ void nhop_set_origin(struct nhop_object *nh, uint8_t origin); uint8_t nhop_get_origin(const struct nhop_object *nh); uint32_t nhop_get_idx(const struct nhop_object *nh); uint32_t nhop_get_uidx(const struct nhop_object *nh); void nhop_set_uidx(struct nhop_object *nh, uint32_t uidx); enum nhop_type nhop_get_type(const struct nhop_object *nh); int nhop_get_rtflags(const struct nhop_object *nh); struct vnet *nhop_get_vnet(const struct nhop_object *nh); struct nhop_object *nhop_select_func(struct nhop_object *nh, uint32_t flowid); int nhop_get_upper_family(const struct nhop_object *nh); bool nhop_set_upper_family(struct nhop_object *nh, int family); int nhop_get_neigh_family(const struct nhop_object *nh); uint32_t nhop_get_fibnum(const struct nhop_object *nh); void nhop_set_fibnum(struct nhop_object *nh, uint32_t fibnum); uint32_t nhop_get_expire(const struct nhop_object *nh); void nhop_set_expire(struct nhop_object *nh, uint32_t expire); struct rib_head *nhop_get_rh(const struct nhop_object *nh); struct nhgrp_object; +struct nhgrp_object *nhgrp_alloc(uint32_t fibnum, int family, + struct weightened_nhop *wn, int num_nhops, int *perror); +struct nhgrp_object *nhgrp_get_nhgrp(struct nhgrp_object *nhg, int *perror); +void nhgrp_set_uidx(struct nhgrp_object *nhg, uint32_t uidx); uint32_t nhgrp_get_uidx(const struct nhgrp_object *nhg); uint8_t nhgrp_get_origin(const struct nhgrp_object *nhg); void nhgrp_set_origin(struct nhgrp_object *nhg, uint8_t origin); #endif /* _KERNEL */ /* Kernel <> userland structures */ /* Structure usage and layout are described in dump_nhop_entry() */ struct nhop_external { uint32_t nh_len; /* length of the datastructure */ uint32_t nh_idx; /* Nexthop index */ uint32_t nh_fib; /* Fib nexhop is attached to */ uint32_t ifindex; /* transmit interface ifindex */ uint32_t aifindex; /* address ifindex */ uint8_t prepend_len; /* length of the prepend */ uint8_t nh_family; /* address family */ uint16_t nh_type; /* nexthop type */ uint16_t nh_mtu; /* nexthop mtu */ uint16_t nh_flags; /* nhop flags */ struct in_addr nh_addr; /* GW/DST IPv4 address */ struct in_addr nh_src; /* default source IPv4 address */ uint64_t nh_pksent; /* control plane */ /* lookup key: address, family, type */ char nh_prepend[64]; /* L2 prepend */ uint64_t nh_refcount; /* number of references */ }; struct nhop_addrs { uint32_t na_len; /* length of the datastructure */ uint16_t gw_sa_off; /* offset of gateway SA */ uint16_t src_sa_off; /* offset of src address SA */ }; #define NHG_C_TYPE_CNHOPS 0x1 /* Control plane nhops list */ #define NHG_C_TYPE_DNHOPS 0x2 /* Dataplane nhops list */ struct nhgrp_container { uint32_t nhgc_len; /* container length */ uint16_t nhgc_count; /* number of items */ uint8_t nhgc_type; /* container type */ uint8_t nhgc_subtype; /* container subtype */ }; struct nhgrp_nhop_external { uint32_t nh_idx; uint32_t nh_weight; }; /* * Layout: * - nhgrp_external * - nhgrp_container (control plane nhops list) * - nhgrp_nhop_external * - nhgrp_nhop_external * .. * - nhgrp_container (dataplane nhops list) * - nhgrp_nhop_external * - nhgrp_nhop_external */ struct nhgrp_external { uint32_t nhg_idx; /* Nexthop group index */ uint32_t nhg_refcount; /* number of references */ }; #endif