Differential D26449 Diff 77161 sys/net/route/nhgrp_ctl.c

Changeset View

Standalone View

sys/net/route/nhgrp_ctl.c

This file was added.

				/*-
				* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
				*
				* Copyright (c) 2020 Alexander V. Chernikov
				*
				* Redistribution and use in source and binary forms, with or without
				* modification, are permitted provided that the following conditions
				* are met:
				* 1. Redistributions of source code must retain the above copyright
				* notice, this list of conditions and the following disclaimer.
				* 2. Redistributions in binary form must reproduce the above copyright
				* notice, this list of conditions and the following disclaimer in the
				* documentation and/or other materials provided with the distribution.
				*
				* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
				* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
				* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
				* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
				* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
				* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
				* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
				* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
				* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
				* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
				* SUCH DAMAGE.
				*
				* $FreeBSD$
				*/

				#include "opt_inet.h"
				#include "opt_route.h"

				#include <sys/cdefs.h>
				#include <sys/param.h>
				#include <sys/systm.h>
				#include <sys/lock.h>
				#include <sys/rmlock.h>
				#include <sys/malloc.h>
				#include <sys/mbuf.h>
				#include <sys/refcount.h>
				#include <sys/socket.h>
				#include <sys/sysctl.h>
				#include <sys/kernel.h>
				#include <sys/epoch.h>

				#include <net/if.h>
				#include <net/if_var.h>
				#include <net/route.h>
				#include <net/route/route_ctl.h>
				#include <net/route/route_var.h>
				#include <net/vnet.h>

				#include <netinet/in.h>
				#include <netinet/in_var.h>
				#include <netinet/in_fib.h>

				#include <net/route/nhop_utils.h>
				#include <net/route/nhop.h>
				#include <net/route/nhop_var.h>
				#include <net/route/nhgrp_var.h>

				/*
				* This file contains the supporting functions for creating multipath groups
				* and compiling their dataplane parts.
				*/

				/* MPF_MULTIPATH must be the same as NHF_MULTIPATH for nhop selection to work */
				CTASSERT(MPF_MULTIPATH == NHF_MULTIPATH);
				/* Offset and size of flags field has to be the same for nhop/nhop groups */
				glebiusUnsubmitted Done Inline Actions New style suggest to use _Static_assert() compiler builtin. glebius: New style suggest to use _Static_assert() compiler builtin.
				CHK_STRUCT_FIELD_GENERIC(struct nhop_object, nh_flags, struct nhgrp_object, mp_flags);
				/* Cap multipath to 64, as the larger values would break rib_cmd_info bmasks */
				CTASSERT(RIB_MAX_MPATH_WIDTH <= 64);

				static int wn_cmp(const void a, const void b);
				static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops);

				static struct nhgrp_priv get_nhgrp(struct nh_control ctl,
				struct weightened_nhop wn, int num_nhops, int perror);
				static void destroy_nhgrp(struct nhgrp_priv *nhg_priv);
				static void destroy_nhgrp_epoch(epoch_context_t ctx);
				static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv);


				glebiusUnsubmitted Done Inline Actions Extra newline. glebius: Extra newline.
				static int
				wn_cmp(const void a, const void b)
				{
				const struct weightened_nhop *wa = a;
				const struct weightened_nhop *wb = b;

				if (wa->weight > wb->weight)
				return (1);
				else if (wa->weight < wb->weight)
				return (-1);

				/* Compare nexthops by pointer */
				if (wa->nh > wb->nh)
				return (1);
				else if (wa->nh < wb->nh)
				return (-1);
				else
				return (0);
				}

				/*
				* Perform in-place sorting for array of nexthops in @wn.
				*
				* To avoid nh groups duplication, nexthops/weights in the
				* @wn need to be ordered deterministically.
				* As this sorting is needed only for the control plane functionality,
				* there are no specific external requirements.
				*
				* Sort by weight first, to ease calculation of the slot sizes.
				*/
				static void
				sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops)
				{

				qsort(wn, num_nhops, sizeof(struct weightened_nhop), wn_cmp);
				}

				/*
				* Calculate minimum number of slots required to fit the existing
				* set of weights while maintaining weight coefficints.
				*
				* Assume @wn is sorted by weight ascending and each weight is > 0.
				*
				* Some examples:
				* nh=1,weight=1 nh=2,weight=2 -> 3 slots [1, 2, 2]
				* nh=1,weight=1000 nh=2,weight=2000 -> 3 slots: [1, 2, 2]
				* nh=1,weight=17 nh=2,weight=37 -> 3 slots: [1, 2, 2]
				* nh=1,weight=1 nh=2,weight=70 -> 64 slots: [1, 2, 2, ..]
				*/
				static uint32_t
				calc_min_mpath_slots(const struct weightened_nhop *wn, size_t num_items)
				{
				uint32_t i, last, xmin;
				uint64_t v, total = 0;

				last = 0;
				xmin = wn[0].weight;
				for (i = 0; i < num_items; i++) {
				total += wn[i].weight;
				if ((wn[i].weight - last < xmin) && (wn[i].weight != last))
				xmin = wn[i].weight - last;
				last = wn[i].weight;
				}
				// got minimum unit of desired accuracy
				v = total / xmin;
				if (v > RIB_MAX_MPATH_WIDTH) {
				/*
				* TODO: round to the MAX_MPATH and
				* see if this reduces the group size.
				*/
				v = RIB_MAX_MPATH_WIDTH;
				}

				return (uint32_t)v;
				}

				/*
				* Nexthop group data consists of
				* 1) dataplane part, with nhgrp_object as a header followed by an
				* arbitrary number of nexthop pointers.
				* 2) control plane part, with nhgrp_priv as a header, followed by
				* an arbirtrary number of 'struct weightened_nhop' object.
				*
				* Given nexthop groups are (mostly) immutable, allocate all data
				* in one go.
				*
				*/
				__noinline static size_t
				get_nhgrp_alloc_size(uint32_t mp_size, uint32_t num_nhops)
				{
				size_t sz;

				sz = sizeof(struct nhgrp_object);
				sz += mp_size * sizeof(struct nhop_object *);
				sz += sizeof(struct nhgrp_priv);
				sz += num_nhops * sizeof(struct weightened_nhop);
				return (sz);
				}

				/*
				* Compile actual list of nexthops to be used by datapath from
				* the nexthop group @dst.
				*
				* For example, compiling control plane list of 2 nexthops
				* [(200, A), (100, B)] would result in the datapath array
				* [A, A, B]
				*/
				static void
				compile_nhgrp(struct nhgrp_priv dst_priv, const struct weightened_nhop x,
				uint32_t num_slots)
				{
				struct nhgrp_object *dst;
				int i, slot_idx, remaining_slots;
				uint64_t remaining_sum, nh_weight, nh_slots;

				slot_idx = 0;
				dst = dst_priv->nhg;
				/* Calculate sum of all weights */
				remaining_sum = 0;
				for (i = 0; i < dst_priv->nhg_nh_count; i++)
				remaining_sum += x[i].weight;
				remaining_slots = num_slots;
				DPRINTF("O: %u/%u", (uint32_t)remaining_sum, remaining_slots);
				for (i = 0; i < dst_priv->nhg_nh_count; i++) {
				/* Calculate number of slots for the current nexthop */
				if (remaining_sum > 0) {
				nh_weight = (uint64_t)x[i].weight;
				nh_slots = (nh_weight * remaining_slots / remaining_sum);
				} else
				nh_slots = 0;

				remaining_sum -= x[i].weight;
				remaining_slots -= nh_slots;

				DPRINTF(" OO[%d]: %u/%u curr=%d slot_idx=%d", i,
				(uint32_t)remaining_sum, remaining_slots,
				(int)nh_slots, slot_idx);

				while (nh_slots-- > 0)
				dst->nhops[slot_idx++] = x[i].nh;
				}
				}

				/*
				* Allocates new nexthop group for the list of weightened nexthops.
				* Assume sorted list.
				* Does NOT reference any nexthops in the group.
				* Returns group with refcount=1 or NULL.
				*/
				static struct nhgrp_priv *
				alloc_nhgrp(struct weightened_nhop *wn, int num_nhops)
				{
				uint32_t mpath_size;
				int flags = M_NOWAIT;
				struct nhgrp_object *nhg;
				struct nhgrp_priv *nhg_priv;

				mpath_size = calc_min_mpath_slots(wn, num_nhops);
				if (mpath_size == 0) {
				/* Zero weights, abort */
				return (NULL);
				glebiusUnsubmitted Done Inline Actions May be add KASSERT to assure we don't overflow dst->nhg_size? glebius: May be add KASSERT to assure we don't overflow dst->nhg_size?
				}

				size_t sz = get_nhgrp_alloc_size(mpath_size, num_nhops);
				nhg = malloc(sz, M_NHOP, flags \| M_ZERO);
				if (nhg == NULL) {
				return (NULL);
				}

				/* Has to be the first to make NHGRP_PRIV() work */
				nhg->mp_size = mpath_size;
				DPRINTF("new mpath group: num_nhops: %u", (uint32_t)mpath_size);
				nhg->mp_flags = MPF_MULTIPATH;

				nhg_priv = NHGRP_PRIV(nhg);
				nhg_priv->nhg_nh_count = num_nhops;
				refcount_init(&nhg_priv->nhg_refcount, 1);

				/* Please see nhgrp_free() comments on the initial value */
				refcount_init(&nhg_priv->nhg_linked, 2);

				nhg_priv->nhg = nhg;
				memcpy(&nhg_priv->nhg_nh_weights[0], wn,
				num_nhops * sizeof(struct weightened_nhop));

				compile_nhgrp(nhg_priv, wn, nhg->mp_size);

				return (nhg_priv);
				}

				void
				nhgrp_free(struct nhgrp_object *nhg)
				{
				struct nhgrp_priv *nhg_priv;
				struct nh_control *ctl;
				struct epoch_tracker et;

				nhg_priv = NHGRP_PRIV(nhg);

				if (!refcount_release(&nhg_priv->nhg_refcount))
				return;

				/*
				* group objects don't have an explicit lock attached to it.
				* As groups are reclaimed based on reference count, it is possible
				* that some groups will persist after vnet destruction callback
				* called. Given that, handle scenario with nhgrp_free_group() being
				* called either after or simultaneously with nhgrp_ctl_unlink_all()
				* by using another reference counter: nhg_linked.
				*
				* There are only 2 places, where nhg_linked can be decreased:
				* rib destroy (nhgrp_ctl_unlink_all) and this function.
				* nhg_link can never be increased.
				*
				* Hence, use initial value of 2 to make use of
				* refcount_release_if_not_last().
				*
				* There can be two scenarious when calling this function:
				*
				* 1) nhg_linked value is 2. This means that either
				* nhgrp_ctl_unlink_all() has not been called OR it is running,
				* but we are guaranteed that nh_control won't be freed in
				* this epoch. Hence, nexthop can be safely unlinked.
				*
				* 2) nh_linked value is 1. In that case, nhgrp_ctl_unlink_all()
				* has been called and nhgrp unlink can be skipped.
				*/

				NET_EPOCH_ENTER(et);
				if (refcount_release_if_not_last(&nhg_priv->nhg_linked)) {
				ctl = nhg_priv->nh_control;
				if (unlink_nhgrp(ctl, nhg_priv) == NULL) {
				/* Do not try to reclaim */
				DPRINTF("Failed to unlink nexhop group %p", nhg_priv);
				NET_EPOCH_EXIT(et);
				return;
				}
				}
				NET_EPOCH_EXIT(et);

				epoch_call(net_epoch_preempt, destroy_nhgrp_epoch,
				&nhg_priv->nhg_epoch_ctx);
				}

				/*
				* Destroys all local resources belonging to @nhg_priv.
				*/
				__noinline static void
				destroy_nhgrp_int(struct nhgrp_priv *nhg_priv)
				{

				free(nhg_priv->nhg, M_NHOP);
				}

				__noinline static void
				destroy_nhgrp(struct nhgrp_priv *nhg_priv)
				{

				KASSERT((nhg_priv->nhg_refcount == 0), ("nhg_refcount != 0"));

				DPRINTF("DEL MPATH %p", nhg_priv);

				KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0"));

				free_nhgrp_nhops(nhg_priv);

				destroy_nhgrp_int(nhg_priv);
				}

				/*
				* Epoch callback indicating group is safe to destroy
				*/
				static void
				destroy_nhgrp_epoch(epoch_context_t ctx)
				{
				struct nhgrp_priv *nhg_priv;

				nhg_priv = __containerof(ctx, struct nhgrp_priv, nhg_epoch_ctx);

				destroy_nhgrp(nhg_priv);
				}

				static bool
				ref_nhgrp_nhops(struct nhgrp_priv *nhg_priv)
				{

				for (int i = 0; i < nhg_priv->nhg_nh_count; i++) {
				if (nhop_try_ref_object(nhg_priv->nhg_nh_weights[i].nh) != 0)
				continue;

				/*
				* Failed to ref the nexthop, b/c it's deleted.
				* Need to rollback references back.
				*/
				for (int j = 0; j < i; j++)
				nhop_free(nhg_priv->nhg_nh_weights[j].nh);
				return (false);
				}

				return (true);
				}

				static void
				free_nhgrp_nhops(struct nhgrp_priv *nhg_priv)
				{

				for (int i = 0; i < nhg_priv->nhg_nh_count; i++)
				nhop_free(nhg_priv->nhg_nh_weights[i].nh);
				}

				/*
				* Creates or looks up an existing nexthop group based on @wn and @num_nhops.
				*
				* Returns referenced nhop group or NULL, passing error code in @perror.
				*/
				struct nhgrp_priv *
				get_nhgrp(struct nh_control ctl, struct weightened_nhop wn, int num_nhops,
				int *perror)
				{
				struct nhgrp_priv key, nhg_priv;

				if (ctl->gr_head.hash_size == 0) {
				/* First multipath request. Bootstrap mpath datastructures. */
				if (nhgrp_ctl_alloc_default(ctl, M_NOWAIT) == 0) {
				*perror = ENOMEM;
				return (NULL);
				}
				}

				/* Sort nexthops & check there are no duplicates */
				sort_weightened_nhops(wn, num_nhops);
				uint32_t last_id = 0;
				for (int i = 0; i < num_nhops; i++) {
				if (wn[i].nh->nh_priv->nh_idx == last_id) {
				*perror = EEXIST;
				return (NULL);
				}
				last_id = wn[i].nh->nh_priv->nh_idx;
				}

				if ((key = alloc_nhgrp(wn, num_nhops)) == NULL) {
				*perror = ENOMEM;
				return (NULL);
				}

				nhg_priv = find_nhgrp(ctl, key);
				if (nhg_priv != NULL) {
				/*
				* Free originally-created group. As it hasn't been linked
				* and the dependent nexhops haven't been referenced, just free
				* the group.
				*/
				destroy_nhgrp_int(key);
				*perror = 0;
				return (nhg_priv);
				} else {
				/* No existing group, try to link the new one */
				if (!ref_nhgrp_nhops(key)) {
				/*
				* Some of the nexthops have been sheduled for deletion.
				* As the group hasn't been linked / no nexhops have been
				* referenced, call the final destructor immediately.
				*/
				destroy_nhgrp_int(key);
				*perror = EAGAIN;
				return (NULL);
				}
				if (link_nhgrp(ctl, key) == 0) {
				/* Unable to allocate index? */
				*perror = EAGAIN;
				destroy_nhgrp(key);
				}
				*perror = 0;
				return (key);
				}

				/* NOTREACHED */
				}

				/*
				* Appends one or more nexthops denoted by @wm to the nexthop group @gr_orig.
				*
				* Returns referenced nexthop group or NULL. In the latter case, @perror is
				* filled with an error code.
				* Note that function does NOT care if the next nexthops already exists
				* in the @gr_orig. As a result, they will be added, resulting in the
				* same nexthop being present multiple times in the new group.
				*/
				static struct nhgrp_priv *
				append_nhops(struct nh_control ctl, const struct nhgrp_object gr_orig,
				struct weightened_nhop wn, int num_nhops, int perror)
				{
				char storage[64];
				struct weightened_nhop *pnhops;
				struct nhgrp_priv *nhg_priv;
				const struct nhgrp_priv *src_priv;
				size_t sz;
				int curr_nhops;

				src_priv = NHGRP_PRIV_CONST(gr_orig);
				curr_nhops = src_priv->nhg_nh_count;

				*perror = 0;

				sz = (src_priv->nhg_nh_count + num_nhops) * (sizeof(struct weightened_nhop));
				/* optimize for <= 4 paths, each path=16 bytes */
				if (sz <= sizeof(storage))
				pnhops = (struct weightened_nhop *)&storage[0];
				else {
				pnhops = malloc(sz, M_TEMP, M_NOWAIT);
				if (pnhops == NULL) {
				*perror = ENOMEM;
				return (NULL);
				}
				}

				/* Copy nhops from original group first */
				memcpy(pnhops, src_priv->nhg_nh_weights,
				curr_nhops * sizeof(struct weightened_nhop));
				memcpy(&pnhops[curr_nhops], wn, num_nhops * sizeof(struct weightened_nhop));
				curr_nhops += num_nhops;

				nhg_priv = get_nhgrp(ctl, pnhops, curr_nhops, perror);

				if (pnhops != (struct weightened_nhop *)&storage[0])
				free(pnhops, M_TEMP);

				if (nhg_priv == NULL)
				return (NULL);

				return (nhg_priv);
				}


				/*
				* Creates/finds nexthop group based on @wn and @num_nhops.
				* Returns referenced group or NULL, with an error in @perror.
				*
				* If the error is EAGAIN, then the operation can be retried.
				*/
				int
				nhgrp_get_group(struct rib_head rh, struct weightened_nhop wn, int num_nhops,
				struct route_nhop_data *rnd)
				{
				struct nh_control *ctl = rh->nh_control;
				struct nhgrp_priv *nhg_priv;
				int error;

				nhg_priv = get_nhgrp(ctl, wn, num_nhops, &error);
				if (nhg_priv != NULL)
				rnd->rnd_nhgrp = nhg_priv->nhg;
				rnd->rnd_weight = 0;

				return (error);
				}

				/*
				* Creates new nexthop group based on @src group with the nexthops defined in bitmask
				* @nhop_mask removed.
				* Returns referenced nexthop group or NULL on failure.
				*/
				int
				nhgrp_get_filtered_group(struct rib_head rh, const struct nhgrp_object src,
				nhgrp_filter_cb_t flt_func, void flt_data, struct route_nhop_data rnd)
				{
				char storage[64];
				struct nh_control *ctl = rh->nh_control;
				struct weightened_nhop *pnhops;
				const struct nhgrp_priv mp_priv, src_priv;
				size_t sz;
				int error, i, num_nhops;

				src_priv = NHGRP_PRIV_CONST(src);

				sz = src_priv->nhg_nh_count * (sizeof(struct weightened_nhop));
				/* optimize for <= 4 paths, each path=16 bytes */
				if (sz <= sizeof(storage))
				pnhops = (struct weightened_nhop *)&storage[0];
				else {
				if ((pnhops = malloc(sz, M_TEMP, M_NOWAIT)) == NULL)
				return (ENOMEM);
				}

				/* Filter nexthops */
				error = 0;
				num_nhops = 0;
				for (i = 0; i < src_priv->nhg_nh_count; i++) {
				if (flt_func(src_priv->nhg_nh_weights[i].nh, flt_data))
				continue;
				memcpy(&pnhops[num_nhops++], &src_priv->nhg_nh_weights[i],
				sizeof(struct weightened_nhop));
				}

				if (num_nhops == 0) {
				rnd->rnd_nhgrp = NULL;
				rnd->rnd_weight = 0;
				} else if (num_nhops == 1) {
				rnd->rnd_nhop = pnhops[0].nh;
				rnd->rnd_weight = pnhops[0].weight;
				if (nhop_try_ref_object(rnd->rnd_nhop) == 0)
				error = EAGAIN;
				} else {
				mp_priv = get_nhgrp(ctl, pnhops, num_nhops, &error);
				if (mp_priv != NULL)
				rnd->rnd_nhgrp = mp_priv->nhg;
				rnd->rnd_weight = 0;
				}

				if (pnhops != (struct weightened_nhop *)&storage[0])
				free(pnhops, M_TEMP);

				return (error);
				}

				/*
				* Creates new multipath group based on existing group/nhop in @rnd_orig and
				* to-be-added nhop @wn_add.
				* Returns 0 on success and stores result in @rnd_new.
				*/
				int
				nhgrp_get_addition_group(struct rib_head rh, struct route_nhop_data rnd_orig,
				struct route_nhop_data rnd_add, struct route_nhop_data rnd_new)
				{
				struct nh_control *ctl = rh->nh_control;
				struct nhgrp_priv *nhg_priv;
				struct weightened_nhop wn[2];
				int error;

				if (rnd_orig->rnd_nhop == NULL) {
				/* No paths to add to, just reference current nhop */
				rnd_new = rnd_add;
				if (nhop_try_ref_object(rnd_new->rnd_nhop) == 0)
				return (EAGAIN);
				return (0);
				}

				wn[0].nh = rnd_add->rnd_nhop;
				wn[0].weight = rnd_add->rnd_weight;

				if (!NH_IS_NHGRP(rnd_orig->rnd_nhop)) {
				/* Simple merge of 2 non-multipath nexthops */
				wn[1].nh = rnd_orig->rnd_nhop;
				wn[1].weight = rnd_orig->rnd_weight;
				nhg_priv = get_nhgrp(ctl, wn, 2, &error);
				} else {
				/* Get new nhop group with @rt->rt_nhop as an additional nhop */
				nhg_priv = append_nhops(ctl, rnd_orig->rnd_nhgrp, &wn[0], 1,
				&error);
				}

				if (nhg_priv == NULL)
				return (error);
				rnd_new->rnd_nhgrp = nhg_priv->nhg;
				rnd_new->rnd_weight = 0;

				return (0);
				}

				/*
				* Returns pointer to array of nexthops with weights for
				* given @nhg. Stores number of items in the array into @pnum_nhops.
				*/
				struct weightened_nhop *
				nhgrp_get_nhops(struct nhgrp_object nhg, uint32_t pnum_nhops)
				{
				struct nhgrp_priv *nhg_priv;

				KASSERT(((nhg->mp_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath"));

				nhg_priv = NHGRP_PRIV(nhg);
				*pnum_nhops = nhg_priv->nhg_nh_count;

				return (nhg_priv->nhg_nh_weights);
				}

				__noinline static int
				dump_nhgrp_entry(struct rib_head rh, const struct nhgrp_priv nhg_priv,
				char buffer, struct sysctl_req w)
				{

				struct rt_msghdr *rtm;
				struct mpath_external *mpe;
				struct nhgrp_object *mp;
				struct mpath_nhop_external *ext;
				uint32_t *pidx;
				int error;
				size_t sz;

				//DPRINTF("Dumping: head %p nh %p flags %X req %p\n", rh, nh, nh->nh_flags, w);

				mp = nhg_priv->nhg;

				sz = sizeof(struct rt_msghdr) + sizeof(struct mpath_external);
				sz += sizeof(struct mpath_nhop_external) * nhg_priv->nhg_nh_count;
				sz += sizeof(uint32_t) * mp->mp_size;

				bzero(buffer, sz);

				rtm = (struct rt_msghdr *)buffer;
				rtm->rtm_msglen = sz;
				rtm->rtm_version = RTM_VERSION;
				rtm->rtm_type = RTM_GET;

				mpe = (struct mpath_external *)(rtm + 1);

				mpe->mp_idx = nhg_priv->nhg_idx;
				mpe->mp_refcount = nhg_priv->nhg_refcount;
				mpe->mp_nh_count = nhg_priv->nhg_nh_count;
				mpe->mp_group_size = mp->mp_size;

				ext = (struct mpath_nhop_external *)(mpe + 1);
				for (int i = 0; i < nhg_priv->nhg_nh_count; i++) {
				ext[i].nh_idx = nhg_priv->nhg_nh_weights[i].nh->nh_priv->nh_idx;
				ext[i].nh_weight = nhg_priv->nhg_nh_weights[i].weight;
				}

				pidx = (uint32_t *)&ext[nhg_priv->nhg_nh_count];
				for (int i = 0; i < mp->mp_size; i++)
				pidx[i] = mp->nhops[i]->nh_priv->nh_idx;

				error = SYSCTL_OUT(w, buffer, sz);

				/*
				DPRINTF("Exported %d ifindex %d family %d type %d error %d\n", nh->nh_priv->nh_idx, pnhe->ifindex,
				pnhe->nh_family, pnhe->nh_type, error);
				*/

				return (error);
				}

				int
				nhgrp_dump_sysctl(struct rib_head rh, struct sysctl_req w)
				{
				struct nh_control *ctl = rh->nh_control;
				struct epoch_tracker et;
				struct nhgrp_priv *nhg_priv;
				char *buffer;
				size_t sz;
				int error;

				if (ctl->gr_head.items_count == 0)
				return (0);

				sz = sizeof(struct mpath_external);
				sz += (sizeof(struct mpath_nhop_external) + sizeof(uint32_t)) *
				RIB_MAX_MPATH_WIDTH;
				buffer = malloc(sz, M_TEMP, M_WAITOK);

				DPRINTF("NHGRP DUMP: count=%u", ctl->gr_head.items_count);
				NET_EPOCH_ENTER(et);
				NHOPS_RLOCK(ctl);
				CHT_SLIST_FOREACH(&ctl->gr_head, mpath, nhg_priv) {
				error = dump_nhgrp_entry(rh, nhg_priv, buffer, w);
				if (error != 0) {
				NHOPS_RUNLOCK(ctl);
				NET_EPOCH_EXIT(et);
				free(buffer, M_TEMP);
				return (error);
				}
				} CHT_SLIST_FOREACH_END;
				NHOPS_RUNLOCK(ctl);
				NET_EPOCH_EXIT(et);

				free(buffer, M_TEMP);

				return (0);
				}