diff --git a/sys/net/route.c b/sys/net/route.c index 7190d93d787d..b74256207e4a 100644 --- a/sys/net/route.c +++ b/sys/net/route.c @@ -1,813 +1,814 @@ /*- * SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 1980, 1986, 1991, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)route.c 8.3.1.1 (Berkeley) 2/23/95 * $FreeBSD$ */ /************************************************************************ * Note: In this file a 'fib' is a "forwarding information base" * * Which is the new name for an in kernel routing (next hop) table. * ***********************************************************************/ #include "opt_inet.h" #include "opt_inet6.h" #include "opt_mrouting.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include VNET_PCPUSTAT_DEFINE(struct rtstat, rtstat); VNET_PCPUSTAT_SYSINIT(rtstat); #ifdef VIMAGE VNET_PCPUSTAT_SYSUNINIT(rtstat); #endif EVENTHANDLER_LIST_DEFINE(rt_addrmsg); static int rt_ifdelroute(const struct rtentry *rt, const struct nhop_object *, void *arg); static int rt_exportinfo(struct rtentry *rt, struct nhop_object *nh, struct rt_addrinfo *info, int flags); /* * route initialization must occur before ip6_init2(), which happenas at * SI_ORDER_MIDDLE. */ static void route_init(void) { nhops_init(); } SYSINIT(route_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, route_init, NULL); struct rib_head * rt_table_init(int offset, int family, u_int fibnum) { struct rib_head *rh; rh = malloc(sizeof(struct rib_head), M_RTABLE, M_WAITOK | M_ZERO); /* TODO: These details should be hidded inside radix.c */ /* Init masks tree */ rn_inithead_internal(&rh->head, rh->rnh_nodes, offset); rn_inithead_internal(&rh->rmhead.head, rh->rmhead.mask_nodes, 0); rh->head.rnh_masks = &rh->rmhead; /* Save metadata associated with this routing table. */ rh->rib_family = family; rh->rib_fibnum = fibnum; #ifdef VIMAGE rh->rib_vnet = curvnet; #endif tmproutes_init(rh); /* Init locks */ RIB_LOCK_INIT(rh); nhops_init_rib(rh); /* Init subscription system */ rib_init_subscriptions(rh); /* Finally, set base callbacks */ rh->rnh_addaddr = rn_addroute; rh->rnh_deladdr = rn_delete; rh->rnh_matchaddr = rn_match; rh->rnh_lookup = rn_lookup; rh->rnh_walktree = rn_walktree; rh->rnh_walktree_from = rn_walktree_from; return (rh); } static int rt_freeentry(struct radix_node *rn, void *arg) { struct radix_head * const rnh = arg; struct radix_node *x; x = (struct radix_node *)rn_delete(rn + 2, NULL, rnh); if (x != NULL) R_Free(x); return (0); } void rt_table_destroy(struct rib_head *rh) { RIB_WLOCK(rh); rh->rib_dying = true; RIB_WUNLOCK(rh); #ifdef FIB_ALGO fib_destroy_rib(rh); #endif tmproutes_destroy(rh); rn_walktree(&rh->rmhead.head, rt_freeentry, &rh->rmhead.head); nhops_destroy_rib(rh); rib_destroy_subscriptions(rh); /* Assume table is already empty */ RIB_LOCK_DESTROY(rh); free(rh, M_RTABLE); } /* * Adds a temporal redirect entry to the routing table. * @fibnum: fib number * @dst: destination to install redirect to * @gateway: gateway to go via * @author: sockaddr of originating router, can be NULL * @ifp: interface to use for the redirected route * @flags: set of flags to add. Allowed: RTF_GATEWAY * @lifetime_sec: time in seconds to expire this redirect. * * Retuns 0 on success, errno otherwise. */ int rib_add_redirect(u_int fibnum, struct sockaddr *dst, struct sockaddr *gateway, struct sockaddr *author, struct ifnet *ifp, int flags, int lifetime_sec) { struct route_nhop_data rnd = { .rnd_weight = RT_DEFAULT_WEIGHT }; struct rib_cmd_info rc; struct ifaddr *ifa; int error; NET_EPOCH_ASSERT(); if (rt_tables_get_rnh(fibnum, dst->sa_family) == NULL) return (EAFNOSUPPORT); /* Verify the allowed flag mask. */ KASSERT(((flags & ~(RTF_GATEWAY)) == 0), ("invalid redirect flags: %x", flags)); flags |= RTF_HOST | RTF_DYNAMIC; /* Get the best ifa for the given interface and gateway. */ if ((ifa = ifaof_ifpforaddr(gateway, ifp)) == NULL) return (ENETUNREACH); struct nhop_object *nh = nhop_alloc(fibnum, dst->sa_family); if (nh == NULL) return (ENOMEM); nhop_set_gw(nh, gateway, flags & RTF_GATEWAY); nhop_set_transmit_ifp(nh, ifp); nhop_set_src(nh, ifa); nhop_set_pxtype_flag(nh, NHF_HOST); nhop_set_expire(nh, lifetime_sec + time_uptime); nhop_set_redirect(nh, true); + nhop_set_origin(nh, NH_ORIGIN_REDIRECT); rnd.rnd_nhop = nhop_get_nhop(nh, &error); if (error == 0) { error = rib_add_route_px(fibnum, dst, -1, &rnd, RTM_F_CREATE, &rc); } if (error != 0) { /* TODO: add per-fib redirect stats. */ return (error); } RTSTAT_INC(rts_dynamic); /* Send notification of a route addition to userland. */ struct rt_addrinfo info = { .rti_info[RTAX_DST] = dst, .rti_info[RTAX_GATEWAY] = gateway, .rti_info[RTAX_AUTHOR] = author, }; rt_missmsg_fib(RTM_REDIRECT, &info, flags | RTF_UP, error, fibnum); return (0); } /* * Routing table ioctl interface. */ int rtioctl_fib(u_long req, caddr_t data, u_int fibnum) { /* * If more ioctl commands are added here, make sure the proper * super-user checks are being performed because it is possible for * prison-root to make it this far if raw sockets have been enabled * in jails. */ #ifdef INET /* Multicast goop, grrr... */ return mrt_ioctl ? mrt_ioctl(req, data, fibnum) : EOPNOTSUPP; #else /* INET */ return ENXIO; #endif /* INET */ } struct ifaddr * ifa_ifwithroute(int flags, const struct sockaddr *dst, const struct sockaddr *gateway, u_int fibnum) { struct ifaddr *ifa; NET_EPOCH_ASSERT(); if ((flags & RTF_GATEWAY) == 0) { /* * If we are adding a route to an interface, * and the interface is a pt to pt link * we should search for the destination * as our clue to the interface. Otherwise * we can use the local address. */ ifa = NULL; if (flags & RTF_HOST) ifa = ifa_ifwithdstaddr(dst, fibnum); if (ifa == NULL) ifa = ifa_ifwithaddr(gateway); } else { /* * If we are adding a route to a remote net * or host, the gateway may still be on the * other end of a pt to pt link. */ ifa = ifa_ifwithdstaddr(gateway, fibnum); } if (ifa == NULL) ifa = ifa_ifwithnet(gateway, 0, fibnum); if (ifa == NULL) { struct nhop_object *nh; nh = rib_lookup(fibnum, gateway, NHR_NONE, 0); /* * dismiss a gateway that is reachable only * through the default router */ if ((nh == NULL) || (nh->nh_flags & NHF_DEFAULT)) return (NULL); ifa = nh->nh_ifa; } if (ifa->ifa_addr->sa_family != dst->sa_family) { struct ifaddr *oifa = ifa; ifa = ifaof_ifpforaddr(dst, ifa->ifa_ifp); if (ifa == NULL) ifa = oifa; } return (ifa); } /* * Copy most of @rt data into @info. * * If @flags contains NHR_COPY, copies dst,netmask and gw to the * pointers specified by @info structure. Assume such pointers * are zeroed sockaddr-like structures with sa_len field initialized * to reflect size of the provided buffer. if no NHR_COPY is specified, * point dst,netmask and gw @info fields to appropriate @rt values. * * if @flags contains NHR_REF, do refcouting on rt_ifp and rt_ifa. * * Returns 0 on success. */ static int rt_exportinfo(struct rtentry *rt, struct nhop_object *nh, struct rt_addrinfo *info, int flags) { struct rt_metrics *rmx; struct sockaddr *src, *dst; int sa_len; if (flags & NHR_COPY) { /* Copy destination if dst is non-zero */ src = rt_key(rt); dst = info->rti_info[RTAX_DST]; sa_len = src->sa_len; if (dst != NULL) { if (src->sa_len > dst->sa_len) return (ENOMEM); memcpy(dst, src, src->sa_len); info->rti_addrs |= RTA_DST; } /* Copy mask if set && dst is non-zero */ src = rt_mask(rt); dst = info->rti_info[RTAX_NETMASK]; if (src != NULL && dst != NULL) { /* * Radix stores different value in sa_len, * assume rt_mask() to have the same length * as rt_key() */ if (sa_len > dst->sa_len) return (ENOMEM); memcpy(dst, src, src->sa_len); info->rti_addrs |= RTA_NETMASK; } /* Copy gateway is set && dst is non-zero */ src = &nh->gw_sa; dst = info->rti_info[RTAX_GATEWAY]; if ((nhop_get_rtflags(nh) & RTF_GATEWAY) && src != NULL && dst != NULL) { if (src->sa_len > dst->sa_len) return (ENOMEM); memcpy(dst, src, src->sa_len); info->rti_addrs |= RTA_GATEWAY; } } else { info->rti_info[RTAX_DST] = rt_key(rt); info->rti_addrs |= RTA_DST; if (rt_mask(rt) != NULL) { info->rti_info[RTAX_NETMASK] = rt_mask(rt); info->rti_addrs |= RTA_NETMASK; } if (nhop_get_rtflags(nh) & RTF_GATEWAY) { info->rti_info[RTAX_GATEWAY] = &nh->gw_sa; info->rti_addrs |= RTA_GATEWAY; } } rmx = info->rti_rmx; if (rmx != NULL) { info->rti_mflags |= RTV_MTU; rmx->rmx_mtu = nh->nh_mtu; } info->rti_flags = rt->rte_flags | nhop_get_rtflags(nh); info->rti_ifp = nh->nh_ifp; info->rti_ifa = nh->nh_ifa; if (flags & NHR_REF) { if_ref(info->rti_ifp); ifa_ref(info->rti_ifa); } return (0); } /* * Lookups up route entry for @dst in RIB database for fib @fibnum. * Exports entry data to @info using rt_exportinfo(). * * If @flags contains NHR_REF, refcouting is performed on rt_ifp and rt_ifa. * All references can be released later by calling rib_free_info(). * * Returns 0 on success. * Returns ENOENT for lookup failure, ENOMEM for export failure. */ int rib_lookup_info(uint32_t fibnum, const struct sockaddr *dst, uint32_t flags, uint32_t flowid, struct rt_addrinfo *info) { RIB_RLOCK_TRACKER; struct rib_head *rh; struct radix_node *rn; struct rtentry *rt; struct nhop_object *nh; int error; KASSERT((fibnum < rt_numfibs), ("rib_lookup_rte: bad fibnum")); rh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rh == NULL) return (ENOENT); RIB_RLOCK(rh); rn = rh->rnh_matchaddr(__DECONST(void *, dst), &rh->head); if (rn != NULL && ((rn->rn_flags & RNF_ROOT) == 0)) { rt = RNTORT(rn); nh = nhop_select(rt->rt_nhop, flowid); /* Ensure route & ifp is UP */ if (RT_LINK_IS_UP(nh->nh_ifp)) { flags = (flags & NHR_REF) | NHR_COPY; error = rt_exportinfo(rt, nh, info, flags); RIB_RUNLOCK(rh); return (error); } } RIB_RUNLOCK(rh); return (ENOENT); } /* * Releases all references acquired by rib_lookup_info() when * called with NHR_REF flags. */ void rib_free_info(struct rt_addrinfo *info) { ifa_free(info->rti_ifa); if_rele(info->rti_ifp); } /* * Delete Routes for a Network Interface * * Called for each routing entry via the rnh->rnh_walktree() call above * to delete all route entries referencing a detaching network interface. * * Arguments: * rt pointer to rtentry * nh pointer to nhop * arg argument passed to rnh->rnh_walktree() - detaching interface * * Returns: * 0 successful * errno failed - reason indicated */ static int rt_ifdelroute(const struct rtentry *rt, const struct nhop_object *nh, void *arg) { struct ifnet *ifp = arg; if (nh->nh_ifp != ifp) return (0); /* * Protect (sorta) against walktree recursion problems * with cloned routes */ if ((rt->rte_flags & RTF_UP) == 0) return (0); return (1); } void rt_flushifroutes(struct ifnet *ifp) { rib_foreach_table_walk_del(AF_UNSPEC, rt_ifdelroute, ifp); } /* * Tries to extract interface from RTAX_IFP passed in rt_addrinfo. * Interface can be specified ether as interface index (sdl_index) or * the interface name (sdl_data). * * Returns found ifp or NULL */ static struct ifnet * info_get_ifp(struct rt_addrinfo *info) { const struct sockaddr_dl *sdl; sdl = (const struct sockaddr_dl *)info->rti_info[RTAX_IFP]; if (sdl->sdl_family != AF_LINK) return (NULL); if (sdl->sdl_index != 0) return (ifnet_byindex(sdl->sdl_index)); if (sdl->sdl_nlen > 0) { char if_name[IF_NAMESIZE]; if (sdl->sdl_nlen + offsetof(struct sockaddr_dl, sdl_data) > sdl->sdl_len) return (NULL); if (sdl->sdl_nlen >= IF_NAMESIZE) return (NULL); bzero(if_name, sizeof(if_name)); memcpy(if_name, sdl->sdl_data, sdl->sdl_nlen); return (ifunit(if_name)); } return (NULL); } /* * Calculates proper ifa/ifp for the cases when gateway AF is different * from dst AF. * * Returns 0 on success. */ __noinline static int rt_getifa_family(struct rt_addrinfo *info, uint32_t fibnum) { if (info->rti_ifp == NULL) { struct ifaddr *ifa = NULL; /* * No transmit interface specified. Guess it by checking gw sa. */ const struct sockaddr *gw = info->rti_info[RTAX_GATEWAY]; ifa = ifa_ifwithroute(RTF_GATEWAY, gw, gw, fibnum); if (ifa == NULL) return (ENETUNREACH); info->rti_ifp = ifa->ifa_ifp; } /* Prefer address from outgoing interface */ info->rti_ifa = ifaof_ifpforaddr(info->rti_info[RTAX_DST], info->rti_ifp); #ifdef INET if (info->rti_ifa == NULL) { /* Use first found IPv4 address */ bool loopback_ok = info->rti_ifp->if_flags & IFF_LOOPBACK; info->rti_ifa = (struct ifaddr *)in_findlocal(fibnum, loopback_ok); } #endif if (info->rti_ifa == NULL) return (ENETUNREACH); return (0); } /* * Fills in rti_ifp and rti_ifa for the provided fib. * * Assume basic consistency checks are executed by callers: * RTAX_DST exists, if RTF_GATEWAY is set, RTAX_GATEWAY exists as well. */ int rt_getifa_fib(struct rt_addrinfo *info, u_int fibnum) { const struct sockaddr *dst, *gateway, *ifaaddr; int error, flags; dst = info->rti_info[RTAX_DST]; gateway = info->rti_info[RTAX_GATEWAY]; ifaaddr = info->rti_info[RTAX_IFA]; flags = info->rti_flags; /* * ifp may be specified by sockaddr_dl * when protocol address is ambiguous. */ error = 0; /* If we have interface specified by RTAX_IFP address, try to use it */ if ((info->rti_ifp == NULL) && (info->rti_info[RTAX_IFP] != NULL)) info->rti_ifp = info_get_ifp(info); /* * If we have source address specified, try to find it * TODO: avoid enumerating all ifas on all interfaces. */ if (info->rti_ifa == NULL && ifaaddr != NULL) info->rti_ifa = ifa_ifwithaddr(ifaaddr); if ((info->rti_ifa == NULL) && ((info->rti_flags & RTF_GATEWAY) != 0) && (gateway->sa_family != dst->sa_family)) return (rt_getifa_family(info, fibnum)); if (info->rti_ifa == NULL) { const struct sockaddr *sa; /* * Most common use case for the userland-supplied routes. * * Choose sockaddr to select ifa. * -- if ifp is set -- * Order of preference: * 1) IFA address * 2) gateway address * Note: for interface routes link-level gateway address * is specified to indicate the interface index without * specifying RTF_GATEWAY. In this case, ignore gateway * Note: gateway AF may be different from dst AF. In this case, * ignore gateway * 3) final destination. * 4) if all of these fails, try to get at least link-level ifa. * -- else -- * try to lookup gateway or dst in the routing table to get ifa */ if (info->rti_info[RTAX_IFA] != NULL) sa = info->rti_info[RTAX_IFA]; else if ((info->rti_flags & RTF_GATEWAY) != 0 && gateway->sa_family == dst->sa_family) sa = gateway; else sa = dst; if (info->rti_ifp != NULL) { info->rti_ifa = ifaof_ifpforaddr(sa, info->rti_ifp); /* Case 4 */ if (info->rti_ifa == NULL && gateway != NULL) info->rti_ifa = ifaof_ifpforaddr(gateway, info->rti_ifp); } else if (dst != NULL && gateway != NULL) info->rti_ifa = ifa_ifwithroute(flags, dst, gateway, fibnum); else if (sa != NULL) info->rti_ifa = ifa_ifwithroute(flags, sa, sa, fibnum); } if (info->rti_ifa != NULL) { if (info->rti_ifp == NULL) info->rti_ifp = info->rti_ifa->ifa_ifp; } else error = ENETUNREACH; return (error); } void rt_updatemtu(struct ifnet *ifp) { struct rib_head *rnh; int mtu; int i, j; /* * Try to update rt_mtu for all routes using this interface * Unfortunately the only way to do this is to traverse all * routing tables in all fibs/domains. */ for (i = 1; i <= AF_MAX; i++) { mtu = if_getmtu_family(ifp, i); for (j = 0; j < rt_numfibs; j++) { rnh = rt_tables_get_rnh(j, i); if (rnh == NULL) continue; nhops_update_ifmtu(rnh, ifp, mtu); } } } #if 0 int p_sockaddr(char *buf, int buflen, struct sockaddr *s); int rt_print(char *buf, int buflen, struct rtentry *rt); int p_sockaddr(char *buf, int buflen, struct sockaddr *s) { void *paddr = NULL; switch (s->sa_family) { case AF_INET: paddr = &((struct sockaddr_in *)s)->sin_addr; break; case AF_INET6: paddr = &((struct sockaddr_in6 *)s)->sin6_addr; break; } if (paddr == NULL) return (0); if (inet_ntop(s->sa_family, paddr, buf, buflen) == NULL) return (0); return (strlen(buf)); } int rt_print(char *buf, int buflen, struct rtentry *rt) { struct sockaddr *addr, *mask; int i = 0; addr = rt_key(rt); mask = rt_mask(rt); i = p_sockaddr(buf, buflen, addr); if (!(rt->rt_flags & RTF_HOST)) { buf[i++] = '/'; i += p_sockaddr(buf + i, buflen - i, mask); } if (rt->rt_flags & RTF_GATEWAY) { buf[i++] = '>'; i += p_sockaddr(buf + i, buflen - i, &rt->rt_nhop->gw_sa); } return (i); } #endif void rt_maskedcopy(const struct sockaddr *src, struct sockaddr *dst, const struct sockaddr *netmask) { const u_char *cp1 = (const u_char *)src; u_char *cp2 = (u_char *)dst; const u_char *cp3 = (const u_char *)netmask; u_char *cplim = cp2 + *cp3; u_char *cplim2 = cp2 + *cp1; *cp2++ = *cp1++; *cp2++ = *cp1++; /* copies sa_len & sa_family */ cp3 += 2; if (cplim > cplim2) cplim = cplim2; while (cp2 < cplim) *cp2++ = *cp1++ & *cp3++; if (cp2 < cplim2) bzero((caddr_t)cp2, (unsigned)(cplim2 - cp2)); } /* * Announce interface address arrival/withdraw * Returns 0 on success. */ int rt_addrmsg(int cmd, struct ifaddr *ifa, int fibnum) { KASSERT(cmd == RTM_ADD || cmd == RTM_DELETE, ("unexpected cmd %d", cmd)); KASSERT((fibnum >= 0 && fibnum < rt_numfibs), ("%s: fib out of range 0 <=%d<%d", __func__, fibnum, rt_numfibs)); EVENTHANDLER_DIRECT_INVOKE(rt_addrmsg, ifa, cmd); if (V_rt_add_addr_allfibs) fibnum = RT_ALL_FIBS; return (rtsock_addrmsg(cmd, ifa, fibnum)); } /* * Announce kernel-originated route addition/removal to rtsock based on @rt data. * cmd: RTM_ cmd * @rt: valid rtentry * @nh: nhop object to announce * @fibnum: fib id or RT_ALL_FIBS * * Returns 0 on success. */ int rt_routemsg(int cmd, struct rtentry *rt, struct nhop_object *nh, int fibnum) { KASSERT(cmd == RTM_ADD || cmd == RTM_DELETE, ("unexpected cmd %d", cmd)); KASSERT(fibnum == RT_ALL_FIBS || (fibnum >= 0 && fibnum < rt_numfibs), ("%s: fib out of range 0 <=%d<%d", __func__, fibnum, rt_numfibs)); KASSERT(rt_key(rt) != NULL, (":%s: rt_key must be supplied", __func__)); return (rtsock_routemsg(cmd, rt, nh, fibnum)); } /* * Announce kernel-originated route addition/removal to rtsock based on @rt data. * cmd: RTM_ cmd * @info: addrinfo structure with valid data. * @fibnum: fib id or RT_ALL_FIBS * * Returns 0 on success. */ int rt_routemsg_info(int cmd, struct rt_addrinfo *info, int fibnum) { KASSERT(cmd == RTM_ADD || cmd == RTM_DELETE || cmd == RTM_CHANGE, ("unexpected cmd %d", cmd)); KASSERT(fibnum == RT_ALL_FIBS || (fibnum >= 0 && fibnum < rt_numfibs), ("%s: fib out of range 0 <=%d<%d", __func__, fibnum, rt_numfibs)); KASSERT(info->rti_info[RTAX_DST] != NULL, (":%s: RTAX_DST must be supplied", __func__)); return (rtsock_routemsg_info(cmd, info, fibnum)); } diff --git a/sys/net/route/nhgrp_ctl.c b/sys/net/route/nhgrp_ctl.c index ca9e01ed0077..06f46746be4b 100644 --- a/sys/net/route/nhgrp_ctl.c +++ b/sys/net/route/nhgrp_ctl.c @@ -1,904 +1,916 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_inet.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME nhgrp_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains the supporting functions for creating multipath groups * and compiling their dataplane parts. */ /* MPF_MULTIPATH must be the same as NHF_MULTIPATH for nhop selection to work */ _Static_assert(MPF_MULTIPATH == NHF_MULTIPATH, "MPF_MULTIPATH must be the same as NHF_MULTIPATH"); /* Offset and size of flags field has to be the same for nhop/nhop groups */ CHK_STRUCT_FIELD_GENERIC(struct nhop_object, nh_flags, struct nhgrp_object, nhg_flags); /* Cap multipath to 64, as the larger values would break rib_cmd_info bmasks */ CTASSERT(RIB_MAX_MPATH_WIDTH <= 64); static int wn_cmp_idx(const void *a, const void *b); static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops); static struct nhgrp_priv *get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, uint32_t uidx, int *perror); static void destroy_nhgrp(struct nhgrp_priv *nhg_priv); static void destroy_nhgrp_epoch(epoch_context_t ctx); static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv); static int wn_cmp_idx(const void *a, const void *b) { const struct weightened_nhop *w_a = a; const struct weightened_nhop *w_b = b; uint32_t a_idx = w_a->nh->nh_priv->nh_idx; uint32_t b_idx = w_b->nh->nh_priv->nh_idx; if (a_idx < b_idx) return (-1); else if (a_idx > b_idx) return (1); else return (0); } /* * Perform in-place sorting for array of nexthops in @wn. * Sort by nexthop index ascending. */ static void sort_weightened_nhops(struct weightened_nhop *wn, int num_nhops) { qsort(wn, num_nhops, sizeof(struct weightened_nhop), wn_cmp_idx); } /* * In order to determine the minimum weight difference in the array * of weights, create a sorted array of weights, using spare "storage" * field in the `struct weightened_nhop`. * Assume weights to be (mostly) the same and use insertion sort to * make it sorted. */ static void sort_weightened_nhops_weights(struct weightened_nhop *wn, int num_items) { wn[0].storage = wn[0].weight; for (int i = 1, j = 0; i < num_items; i++) { uint32_t weight = wn[i].weight; // read from 'weight' as it's not reordered /* Move all weights > weight 1 position right */ for (j = i - 1; j >= 0 && wn[j].storage > weight; j--) wn[j + 1].storage = wn[j].storage; wn[j + 1].storage = weight; } } /* * Calculate minimum number of slots required to fit the existing * set of weights in the common use case where weights are "easily" * comparable. * Assumes @wn is sorted by weight ascending and each weight is > 0. * Returns number of slots or 0 if precise calculation failed. * * Some examples: * note: (i, X) pair means (nhop=i, weight=X): * (1, 1) (2, 2) -> 3 slots [1, 2, 2] * (1, 100), (2, 200) -> 3 slots [1, 2, 2] * (1, 100), (2, 200), (3, 400) -> 7 slots [1, 2, 2, 3, 3, 3] */ static uint32_t calc_min_mpath_slots_fast(struct weightened_nhop *wn, size_t num_items, uint64_t *ptotal) { uint32_t i, last, xmin; uint64_t total = 0; // Get sorted array of weights in .storage field sort_weightened_nhops_weights(wn, num_items); last = 0; xmin = wn[0].storage; for (i = 0; i < num_items; i++) { total += wn[i].storage; if ((wn[i].storage != last) && ((wn[i].storage - last < xmin) || xmin == 0)) { xmin = wn[i].storage - last; } last = wn[i].storage; } *ptotal = total; /* xmin is the minimum unit of desired capacity */ if ((total % xmin) != 0) return (0); for (i = 0; i < num_items; i++) { if ((wn[i].weight % xmin) != 0) return (0); } return ((uint32_t)(total / xmin)); } /* * Calculate minimum number of slots required to fit the existing * set of weights while maintaining weight coefficients. * * Assume @wn is sorted by weight ascending and each weight is > 0. * * Tries to find simple precise solution first and falls back to * RIB_MAX_MPATH_WIDTH in case of any failure. */ static uint32_t calc_min_mpath_slots(struct weightened_nhop *wn, size_t num_items) { uint32_t v; uint64_t total; v = calc_min_mpath_slots_fast(wn, num_items, &total); if (total == 0) return (0); if ((v == 0) || (v > RIB_MAX_MPATH_WIDTH)) v = RIB_MAX_MPATH_WIDTH; return (v); } /* * Nexthop group data consists of * 1) dataplane part, with nhgrp_object as a header followed by an * arbitrary number of nexthop pointers. * 2) control plane part, with nhgrp_priv as a header, followed by * an arbirtrary number of 'struct weightened_nhop' object. * * Given nexthop groups are (mostly) immutable, allocate all data * in one go. * */ __noinline static size_t get_nhgrp_alloc_size(uint32_t nhg_size, uint32_t num_nhops) { size_t sz; sz = sizeof(struct nhgrp_object); sz += nhg_size * sizeof(struct nhop_object *); sz += sizeof(struct nhgrp_priv); sz += num_nhops * sizeof(struct weightened_nhop); return (sz); } /* * Compile actual list of nexthops to be used by datapath from * the nexthop group @dst. * * For example, compiling control plane list of 2 nexthops * [(200, A), (100, B)] would result in the datapath array * [A, A, B] */ static void compile_nhgrp(struct nhgrp_priv *dst_priv, const struct weightened_nhop *x, uint32_t num_slots) { struct nhgrp_object *dst; int i, slot_idx, remaining_slots; uint64_t remaining_sum, nh_weight, nh_slots; slot_idx = 0; dst = dst_priv->nhg; /* Calculate sum of all weights */ remaining_sum = 0; for (i = 0; i < dst_priv->nhg_nh_count; i++) remaining_sum += x[i].weight; remaining_slots = num_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, "sum: %lu, slots: %d", remaining_sum, remaining_slots); for (i = 0; i < dst_priv->nhg_nh_count; i++) { /* Calculate number of slots for the current nexthop */ if (remaining_sum > 0) { nh_weight = (uint64_t)x[i].weight; nh_slots = (nh_weight * remaining_slots / remaining_sum); } else nh_slots = 0; remaining_sum -= x[i].weight; remaining_slots -= nh_slots; FIB_NH_LOG(LOG_DEBUG3, x[0].nh, " rem_sum: %lu, rem_slots: %d nh_slots: %d, slot_idx: %d", remaining_sum, remaining_slots, (int)nh_slots, slot_idx); KASSERT((slot_idx + nh_slots <= num_slots), ("index overflow during nhg compilation")); while (nh_slots-- > 0) dst->nhops[slot_idx++] = x[i].nh; } } /* * Allocates new nexthop group for the list of weightened nexthops. * Assume sorted list. * Does NOT reference any nexthops in the group. * Returns group with refcount=1 or NULL. */ static struct nhgrp_priv * alloc_nhgrp(struct weightened_nhop *wn, int num_nhops) { uint32_t nhgrp_size; struct nhgrp_object *nhg; struct nhgrp_priv *nhg_priv; nhgrp_size = calc_min_mpath_slots(wn, num_nhops); if (nhgrp_size == 0) { /* Zero weights, abort */ return (NULL); } size_t sz = get_nhgrp_alloc_size(nhgrp_size, num_nhops); nhg = malloc(sz, M_NHOP, M_NOWAIT | M_ZERO); if (nhg == NULL) { FIB_NH_LOG(LOG_INFO, wn[0].nh, "unable to allocate group with num_nhops %d (compiled %u)", num_nhops, nhgrp_size); return (NULL); } /* Has to be the first to make NHGRP_PRIV() work */ nhg->nhg_size = nhgrp_size; nhg->nhg_flags = MPF_MULTIPATH; nhg_priv = NHGRP_PRIV(nhg); nhg_priv->nhg_nh_count = num_nhops; refcount_init(&nhg_priv->nhg_refcount, 1); /* Please see nhgrp_free() comments on the initial value */ refcount_init(&nhg_priv->nhg_linked, 2); nhg_priv->nhg = nhg; memcpy(&nhg_priv->nhg_nh_weights[0], wn, num_nhops * sizeof(struct weightened_nhop)); FIB_NH_LOG(LOG_DEBUG, wn[0].nh, "num_nhops: %d, compiled_nhop: %u", num_nhops, nhgrp_size); compile_nhgrp(nhg_priv, wn, nhg->nhg_size); return (nhg_priv); } void nhgrp_ref_object(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; u_int old __diagused; nhg_priv = NHGRP_PRIV(nhg); old = refcount_acquire(&nhg_priv->nhg_refcount); KASSERT(old > 0, ("%s: nhgrp object %p has 0 refs", __func__, nhg)); } void nhgrp_free(struct nhgrp_object *nhg) { struct nhgrp_priv *nhg_priv; struct nh_control *ctl; struct epoch_tracker et; nhg_priv = NHGRP_PRIV(nhg); if (!refcount_release(&nhg_priv->nhg_refcount)) return; /* * group objects don't have an explicit lock attached to it. * As groups are reclaimed based on reference count, it is possible * that some groups will persist after vnet destruction callback * called. Given that, handle scenario with nhgrp_free_group() being * called either after or simultaneously with nhgrp_ctl_unlink_all() * by using another reference counter: nhg_linked. * * There are only 2 places, where nhg_linked can be decreased: * rib destroy (nhgrp_ctl_unlink_all) and this function. * nhg_link can never be increased. * * Hence, use initial value of 2 to make use of * refcount_release_if_not_last(). * * There can be two scenarious when calling this function: * * 1) nhg_linked value is 2. This means that either * nhgrp_ctl_unlink_all() has not been called OR it is running, * but we are guaranteed that nh_control won't be freed in * this epoch. Hence, nexthop can be safely unlinked. * * 2) nh_linked value is 1. In that case, nhgrp_ctl_unlink_all() * has been called and nhgrp unlink can be skipped. */ NET_EPOCH_ENTER(et); if (refcount_release_if_not_last(&nhg_priv->nhg_linked)) { ctl = nhg_priv->nh_control; if (unlink_nhgrp(ctl, nhg_priv) == NULL) { /* Do not try to reclaim */ RT_LOG(LOG_INFO, "Failed to unlink nexhop group %p", nhg_priv); NET_EPOCH_EXIT(et); return; } } NET_EPOCH_EXIT(et); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); epoch_call(net_epoch_preempt, destroy_nhgrp_epoch, &nhg_priv->nhg_epoch_ctx); } /* * Destroys all local resources belonging to @nhg_priv. */ __noinline static void destroy_nhgrp_int(struct nhgrp_priv *nhg_priv) { free(nhg_priv->nhg, M_NHOP); } __noinline static void destroy_nhgrp(struct nhgrp_priv *nhg_priv) { KASSERT((nhg_priv->nhg_refcount == 0), ("nhg_refcount != 0")); KASSERT((nhg_priv->nhg_idx == 0), ("gr_idx != 0")); IF_DEBUG_LEVEL(LOG_DEBUG2) { char nhgbuf[NHOP_PRINT_BUFSIZE] __unused; FIB_NH_LOG(LOG_DEBUG2, nhg_priv->nhg_nh_weights[0].nh, "destroying %s", nhgrp_print_buf(nhg_priv->nhg, nhgbuf, sizeof(nhgbuf))); } free_nhgrp_nhops(nhg_priv); destroy_nhgrp_int(nhg_priv); } /* * Epoch callback indicating group is safe to destroy */ static void destroy_nhgrp_epoch(epoch_context_t ctx) { struct nhgrp_priv *nhg_priv; nhg_priv = __containerof(ctx, struct nhgrp_priv, nhg_epoch_ctx); destroy_nhgrp(nhg_priv); } static bool ref_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { if (nhop_try_ref_object(nhg_priv->nhg_nh_weights[i].nh) != 0) continue; /* * Failed to ref the nexthop, b/c it's deleted. * Need to rollback references back. */ for (int j = 0; j < i; j++) nhop_free(nhg_priv->nhg_nh_weights[j].nh); return (false); } return (true); } static void free_nhgrp_nhops(struct nhgrp_priv *nhg_priv) { for (int i = 0; i < nhg_priv->nhg_nh_count; i++) nhop_free(nhg_priv->nhg_nh_weights[i].nh); } /* * Creates or looks up an existing nexthop group based on @wn and @num_nhops. * * Returns referenced nhop group or NULL, passing error code in @perror. */ struct nhgrp_priv * get_nhgrp(struct nh_control *ctl, struct weightened_nhop *wn, int num_nhops, uint32_t uidx, int *perror) { struct nhgrp_priv *key, *nhg_priv; if (num_nhops > RIB_MAX_MPATH_WIDTH) { *perror = E2BIG; return (NULL); } if (ctl->gr_head.hash_size == 0) { /* First multipath request. Bootstrap mpath datastructures. */ if (nhgrp_ctl_alloc_default(ctl, M_NOWAIT) == 0) { *perror = ENOMEM; return (NULL); } } /* Sort nexthops & check there are no duplicates */ sort_weightened_nhops(wn, num_nhops); uint32_t last_id = 0; for (int i = 0; i < num_nhops; i++) { if (wn[i].nh->nh_priv->nh_idx == last_id) { *perror = EEXIST; return (NULL); } last_id = wn[i].nh->nh_priv->nh_idx; } if ((key = alloc_nhgrp(wn, num_nhops)) == NULL) { *perror = ENOMEM; return (NULL); } key->nhg_uidx = uidx; nhg_priv = find_nhgrp(ctl, key); if (nhg_priv != NULL) { /* * Free originally-created group. As it hasn't been linked * and the dependent nexhops haven't been referenced, just free * the group. */ destroy_nhgrp_int(key); *perror = 0; return (nhg_priv); } else { /* No existing group, try to link the new one */ if (!ref_nhgrp_nhops(key)) { /* * Some of the nexthops have been scheduled for deletion. * As the group hasn't been linked / no nexhops have been * referenced, call the final destructor immediately. */ destroy_nhgrp_int(key); *perror = EAGAIN; return (NULL); } if (link_nhgrp(ctl, key) == 0) { /* Unable to allocate index? */ *perror = EAGAIN; free_nhgrp_nhops(key); destroy_nhgrp_int(key); return (NULL); } *perror = 0; return (key); } /* NOTREACHED */ } /* * Appends one or more nexthops denoted by @wm to the nexthop group @gr_orig. * * Returns referenced nexthop group or NULL. In the latter case, @perror is * filled with an error code. * Note that function does NOT care if the next nexthops already exists * in the @gr_orig. As a result, they will be added, resulting in the * same nexthop being present multiple times in the new group. */ static struct nhgrp_priv * append_nhops(struct nh_control *ctl, const struct nhgrp_object *gr_orig, struct weightened_nhop *wn, int num_nhops, int *perror) { char storage[64]; struct weightened_nhop *pnhops; struct nhgrp_priv *nhg_priv; const struct nhgrp_priv *src_priv; size_t sz; int curr_nhops; src_priv = NHGRP_PRIV_CONST(gr_orig); curr_nhops = src_priv->nhg_nh_count; *perror = 0; sz = (src_priv->nhg_nh_count + num_nhops) * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { pnhops = malloc(sz, M_TEMP, M_NOWAIT); if (pnhops == NULL) { *perror = ENOMEM; return (NULL); } } /* Copy nhops from original group first */ memcpy(pnhops, src_priv->nhg_nh_weights, curr_nhops * sizeof(struct weightened_nhop)); memcpy(&pnhops[curr_nhops], wn, num_nhops * sizeof(struct weightened_nhop)); curr_nhops += num_nhops; nhg_priv = get_nhgrp(ctl, pnhops, curr_nhops, 0, perror); if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); if (nhg_priv == NULL) return (NULL); return (nhg_priv); } /* * Creates/finds nexthop group based on @wn and @num_nhops. * Returns 0 on success with referenced group in @rnd, or * errno. * * If the error is EAGAIN, then the operation can be retried. */ int nhgrp_get_group(struct rib_head *rh, struct weightened_nhop *wn, int num_nhops, uint32_t uidx, struct nhgrp_object **pnhg) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; int error; nhg_priv = get_nhgrp(ctl, wn, num_nhops, uidx, &error); if (nhg_priv != NULL) *pnhg = nhg_priv->nhg; return (error); } /* * Creates new nexthop group based on @src group without the nexthops * chosen by @flt_func. * Returns 0 on success, storring the reference nhop group/object in @rnd. */ int nhgrp_get_filtered_group(struct rib_head *rh, const struct rtentry *rt, const struct nhgrp_object *src, rib_filter_f_t flt_func, void *flt_data, struct route_nhop_data *rnd) { char storage[64]; struct nh_control *ctl = rh->nh_control; struct weightened_nhop *pnhops; const struct nhgrp_priv *mp_priv, *src_priv; size_t sz; int error, i, num_nhops; src_priv = NHGRP_PRIV_CONST(src); sz = src_priv->nhg_nh_count * (sizeof(struct weightened_nhop)); /* optimize for <= 4 paths, each path=16 bytes */ if (sz <= sizeof(storage)) pnhops = (struct weightened_nhop *)&storage[0]; else { if ((pnhops = malloc(sz, M_TEMP, M_NOWAIT)) == NULL) return (ENOMEM); } /* Filter nexthops */ error = 0; num_nhops = 0; for (i = 0; i < src_priv->nhg_nh_count; i++) { if (flt_func(rt, src_priv->nhg_nh_weights[i].nh, flt_data)) continue; memcpy(&pnhops[num_nhops++], &src_priv->nhg_nh_weights[i], sizeof(struct weightened_nhop)); } if (num_nhops == 0) { rnd->rnd_nhgrp = NULL; rnd->rnd_weight = 0; } else if (num_nhops == 1) { rnd->rnd_nhop = pnhops[0].nh; rnd->rnd_weight = pnhops[0].weight; if (nhop_try_ref_object(rnd->rnd_nhop) == 0) error = EAGAIN; } else { mp_priv = get_nhgrp(ctl, pnhops, num_nhops, 0, &error); if (mp_priv != NULL) rnd->rnd_nhgrp = mp_priv->nhg; rnd->rnd_weight = 0; } if (pnhops != (struct weightened_nhop *)&storage[0]) free(pnhops, M_TEMP); return (error); } /* * Creates new multipath group based on existing group/nhop in @rnd_orig and * to-be-added nhop @wn_add. * Returns 0 on success and stores result in @rnd_new. */ int nhgrp_get_addition_group(struct rib_head *rh, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_new) { struct nh_control *ctl = rh->nh_control; struct nhgrp_priv *nhg_priv; struct weightened_nhop wn[2] = {}; int error; if (rnd_orig->rnd_nhop == NULL) { /* No paths to add to, just reference current nhop */ *rnd_new = *rnd_add; if (nhop_try_ref_object(rnd_new->rnd_nhop) == 0) return (EAGAIN); return (0); } wn[0].nh = rnd_add->rnd_nhop; wn[0].weight = rnd_add->rnd_weight; if (!NH_IS_NHGRP(rnd_orig->rnd_nhop)) { /* Simple merge of 2 non-multipath nexthops */ wn[1].nh = rnd_orig->rnd_nhop; wn[1].weight = rnd_orig->rnd_weight; nhg_priv = get_nhgrp(ctl, wn, 2, 0, &error); } else { /* Get new nhop group with @rt->rt_nhop as an additional nhop */ nhg_priv = append_nhops(ctl, rnd_orig->rnd_nhgrp, &wn[0], 1, &error); } if (nhg_priv == NULL) return (error); rnd_new->rnd_nhgrp = nhg_priv->nhg; rnd_new->rnd_weight = 0; return (0); } /* * Returns pointer to array of nexthops with weights for * given @nhg. Stores number of items in the array into @pnum_nhops. */ const struct weightened_nhop * nhgrp_get_nhops(const struct nhgrp_object *nhg, uint32_t *pnum_nhops) { const struct nhgrp_priv *nhg_priv; KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); nhg_priv = NHGRP_PRIV_CONST(nhg); *pnum_nhops = nhg_priv->nhg_nh_count; return (nhg_priv->nhg_nh_weights); } uint32_t nhgrp_get_uidx(const struct nhgrp_object *nhg) { const struct nhgrp_priv *nhg_priv; KASSERT(((nhg->nhg_flags & MPF_MULTIPATH) != 0), ("nhop is not mpath")); nhg_priv = NHGRP_PRIV_CONST(nhg); return (nhg_priv->nhg_uidx); } /* * Prints nexhop group @nhg data in the provided @buf. * Example: nhg#33/sz=3:[#1:100,#2:100,#3:100] * Example: nhg#33/sz=5:[#1:100,#2:100,..] */ char * nhgrp_print_buf(const struct nhgrp_object *nhg, char *buf, size_t bufsize) { const struct nhgrp_priv *nhg_priv = NHGRP_PRIV_CONST(nhg); int off = snprintf(buf, bufsize, "nhg#%u/sz=%u:[", nhg_priv->nhg_idx, nhg_priv->nhg_nh_count); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { const struct weightened_nhop *wn = &nhg_priv->nhg_nh_weights[i]; int len = snprintf(&buf[off], bufsize - off, "#%u:%u,", wn->nh->nh_priv->nh_idx, wn->weight); if (len + off + 3 >= bufsize) { int len = snprintf(&buf[off], bufsize - off, "..."); off += len; break; } off += len; } if (off > 0) off--; // remove last "," if (off + 1 < bufsize) snprintf(&buf[off], bufsize - off, "]"); return buf; } __noinline static int dump_nhgrp_entry(struct rib_head *rh, const struct nhgrp_priv *nhg_priv, char *buffer, size_t buffer_size, struct sysctl_req *w) { struct rt_msghdr *rtm; struct nhgrp_external *nhge; struct nhgrp_container *nhgc; const struct nhgrp_object *nhg; struct nhgrp_nhop_external *ext; int error; size_t sz; nhg = nhg_priv->nhg; sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); /* controlplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; /* dataplane nexthops */ sz += sizeof(struct nhgrp_container); sz += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; KASSERT(sz <= buffer_size, ("increase nhgrp buffer size")); bzero(buffer, sz); rtm = (struct rt_msghdr *)buffer; rtm->rtm_msglen = sz; rtm->rtm_version = RTM_VERSION; rtm->rtm_type = RTM_GET; nhge = (struct nhgrp_external *)(rtm + 1); nhge->nhg_idx = nhg_priv->nhg_idx; nhge->nhg_refcount = nhg_priv->nhg_refcount; /* fill in control plane nexthops firs */ nhgc = (struct nhgrp_container *)(nhge + 1); nhgc->nhgc_type = NHG_C_TYPE_CNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg_priv->nhg_nh_count; nhgc->nhgc_count = nhg_priv->nhg_nh_count; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg_priv->nhg_nh_count; i++) { ext[i].nh_idx = nhg_priv->nhg_nh_weights[i].nh->nh_priv->nh_idx; ext[i].nh_weight = nhg_priv->nhg_nh_weights[i].weight; } /* fill in dataplane nexthops */ nhgc = (struct nhgrp_container *)(&ext[nhg_priv->nhg_nh_count]); nhgc->nhgc_type = NHG_C_TYPE_DNHOPS; nhgc->nhgc_subtype = 0; nhgc->nhgc_len = sizeof(struct nhgrp_container); nhgc->nhgc_len += sizeof(struct nhgrp_nhop_external) * nhg->nhg_size; nhgc->nhgc_count = nhg->nhg_size; ext = (struct nhgrp_nhop_external *)(nhgc + 1); for (int i = 0; i < nhg->nhg_size; i++) { ext[i].nh_idx = nhg->nhops[i]->nh_priv->nh_idx; ext[i].nh_weight = 0; } error = SYSCTL_OUT(w, buffer, sz); return (error); } uint32_t nhgrp_get_idx(const struct nhgrp_object *nhg) { const struct nhgrp_priv *nhg_priv; nhg_priv = NHGRP_PRIV_CONST(nhg); return (nhg_priv->nhg_idx); } +uint8_t +nhgrp_get_origin(struct nhgrp_object *nhg) +{ + return (NHGRP_PRIV(nhg)->nhg_origin); +} + +void +nhgrp_set_origin(struct nhgrp_object *nhg, uint8_t origin) +{ + NHGRP_PRIV(nhg)->nhg_origin = origin; +} + uint32_t nhgrp_get_count(struct rib_head *rh) { struct nh_control *ctl; uint32_t count; ctl = rh->nh_control; NHOPS_RLOCK(ctl); count = ctl->gr_head.items_count; NHOPS_RUNLOCK(ctl); return (count); } int nhgrp_dump_sysctl(struct rib_head *rh, struct sysctl_req *w) { struct nh_control *ctl = rh->nh_control; struct epoch_tracker et; struct nhgrp_priv *nhg_priv; char *buffer; size_t sz; int error = 0; if (ctl->gr_head.items_count == 0) return (0); /* Calculate the maximum nhop group size in bytes */ sz = sizeof(struct rt_msghdr) + sizeof(struct nhgrp_external); sz += 2 * sizeof(struct nhgrp_container); sz += 2 * sizeof(struct nhgrp_nhop_external) * RIB_MAX_MPATH_WIDTH; buffer = malloc(sz, M_TEMP, M_NOWAIT); if (buffer == NULL) return (ENOMEM); NET_EPOCH_ENTER(et); NHOPS_RLOCK(ctl); CHT_SLIST_FOREACH(&ctl->gr_head, mpath, nhg_priv) { error = dump_nhgrp_entry(rh, nhg_priv, buffer, sz, w); if (error != 0) break; } CHT_SLIST_FOREACH_END; NHOPS_RUNLOCK(ctl); NET_EPOCH_EXIT(et); free(buffer, M_TEMP); return (error); } diff --git a/sys/net/route/nhgrp_var.h b/sys/net/route/nhgrp_var.h index 3d894857558d..c1540fbb33cf 100644 --- a/sys/net/route/nhgrp_var.h +++ b/sys/net/route/nhgrp_var.h @@ -1,73 +1,74 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This header file contains private definitions for the nexthop groups. * * Header is not intended to be included by the code external to the * routing subsystem. */ #ifndef _NET_ROUTE_NHGRP_VAR_H_ #define _NET_ROUTE_NHGRP_VAR_H_ /* nhgrp hash definition */ /* produce hash value for an object */ #define mpath_hash_obj(_obj) (hash_nhgrp(_obj)) /* compare two objects */ #define mpath_cmp(_one, _two) (cmp_nhgrp(_one, _two)) /* next object accessor */ #define mpath_next(_obj) (_obj)->nhg_priv_next struct nhgrp_priv { uint32_t nhg_idx; uint32_t nhg_uidx; uint8_t nhg_nh_count; /* number of items in nh_weights */ - uint8_t nhg_spare[3]; + uint8_t nhg_origin; /* protocol which created the group */ + uint8_t nhg_spare[2]; u_int nhg_refcount; /* use refcount */ u_int nhg_linked; /* refcount(9), == 2 if linked to the list */ struct nh_control *nh_control; /* parent control structure */ struct nhgrp_priv *nhg_priv_next; struct nhgrp_object *nhg; struct epoch_context nhg_epoch_ctx; /* epoch data for nhop */ struct weightened_nhop nhg_nh_weights[0]; }; #define _NHGRP_PRIV(_src) (&(_src)->nhops[(_src)->nhg_size]) #define NHGRP_PRIV(_src) ((struct nhgrp_priv *)_NHGRP_PRIV(_src)) #define NHGRP_PRIV_CONST(_src) ((const struct nhgrp_priv *)_NHGRP_PRIV(_src)) /* nhgrp.c */ bool nhgrp_ctl_alloc_default(struct nh_control *ctl, int malloc_flags); struct nhgrp_priv *find_nhgrp(struct nh_control *ctl, const struct nhgrp_priv *key); int link_nhgrp(struct nh_control *ctl, struct nhgrp_priv *grp_priv); struct nhgrp_priv *unlink_nhgrp(struct nh_control *ctl, struct nhgrp_priv *key); #endif diff --git a/sys/net/route/nhop.h b/sys/net/route/nhop.h index 24ddb692efb8..669284e0ac62 100644 --- a/sys/net/route/nhop.h +++ b/sys/net/route/nhop.h @@ -1,282 +1,292 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This header file contains public definitions for the nexthop routing subsystem. */ #ifndef _NET_ROUTE_NHOP_H_ #define _NET_ROUTE_NHOP_H_ #include /* sockaddr_in && sockaddr_in6 */ #include enum nhop_type { NH_TYPE_IPV4_ETHER_RSLV = 1, /* IPv4 ethernet without GW */ NH_TYPE_IPV4_ETHER_NHOP = 2, /* IPv4 with pre-calculated ethernet encap */ NH_TYPE_IPV6_ETHER_RSLV = 3, /* IPv6 ethernet, without GW */ NH_TYPE_IPV6_ETHER_NHOP = 4 /* IPv6 with pre-calculated ethernet encap*/ }; #ifdef _KERNEL /* * Define shorter version of AF_LINK sockaddr. * * Currently the only use case of AF_LINK gateway is storing * interface index of the interface of the source IPv6 address. * This is used by the IPv6 code for the connections over loopback * interface. * * The structure below copies 'struct sockaddr_dl', reducing the * size of sdl_data buffer, as it is not used. This change * allows to store the AF_LINK gateways in the nhop gateway itself, * simplifying control plane handling. */ struct sockaddr_dl_short { u_char sdl_len; /* Total length of sockaddr */ u_char sdl_family; /* AF_LINK */ u_short sdl_index; /* if != 0, system given index for interface */ u_char sdl_type; /* interface type */ u_char sdl_nlen; /* interface name length, no trailing 0 reqd. */ u_char sdl_alen; /* link level address length */ u_char sdl_slen; /* link layer selector length */ char sdl_data[8]; /* unused */ }; #define NHOP_RELATED_FLAGS \ (RTF_GATEWAY | RTF_HOST | RTF_REJECT | RTF_BLACKHOLE | \ RTF_FIXEDMTU | RTF_LOCAL | RTF_BROADCAST | RTF_MULTICAST) struct nh_control; struct nhop_priv; /* * Struct 'nhop_object' field description: * * nh_flags: NHF_ flags used in the dataplane code. NHF_GATEWAY or NHF_BLACKHOLE * can be examples of such flags. * nh_mtu: ready-to-use nexthop mtu. Already accounts for the link-level header, * interface MTU and protocol-specific limitations. * nh_prepend_len: link-level prepend length. Currently unused. * nh_ifp: logical transmit interface. The one from which if_transmit() will be * called. Guaranteed to be non-NULL. * nh_aifp: ifnet of the source address. Same as nh_ifp except IPv6 loopback * routes. See the example below. * nh_ifa: interface address to use. Guaranteed to be non-NULL. * nh_pksent: counter(9) reflecting the number of packets transmitted. * * gw_: storage suitable to hold AF_INET, AF_INET6 or AF_LINK gateway. More * details ara available in the examples below. * * Examples: * * Direct routes (routes w/o gateway): * NHF_GATEWAY is NOT set. * nh_ifp denotes the logical transmit interface (). * nh_aifp is the same as nh_ifp * gw_sa contains AF_LINK sa with nh_aifp ifindex (compat) * Loopback routes: * NHF_GATEWAY is NOT set. * nh_ifp points to the loopback interface (lo0). * nh_aifp points to the interface where the destination address belongs to. * This is useful in IPv6 link-local-over-loopback communications. * gw_sa contains AF_LINK sa with nh_aifp ifindex (compat) * GW routes: * NHF_GATEWAY is set. * nh_ifp denotes the logical transmit interface. * nh_aifp is the same as nh_ifp * gw_sa contains L3 address (either AF_INET or AF_INET6). * * * Note: struct nhop_object fields are ordered in a way that * supports memcmp-based comparisons. * */ #define NHOP_END_CMP (__offsetof(struct nhop_object, nh_pksent)) struct nhop_object { uint16_t nh_flags; /* nhop flags */ uint16_t nh_mtu; /* nexthop mtu */ union { struct sockaddr_in gw4_sa; /* GW accessor as IPv4 */ struct sockaddr_in6 gw6_sa; /* GW accessor as IPv6 */ struct sockaddr gw_sa; struct sockaddr_dl_short gwl_sa; /* AF_LINK gw (compat) */ char gw_buf[28]; }; struct ifnet *nh_ifp; /* Logical egress interface. Always != NULL */ struct ifaddr *nh_ifa; /* interface address to use. Always != NULL */ struct ifnet *nh_aifp; /* ifnet of the source address. Always != NULL */ counter_u64_t nh_pksent; /* packets sent using this nhop */ /* 32 bytes + 4xPTR == 64(amd64) / 48(i386) */ uint8_t nh_prepend_len; /* length of prepend data */ uint8_t spare[3]; uint32_t spare1; /* alignment */ char nh_prepend[48]; /* L2 prepend */ struct nhop_priv *nh_priv; /* control plane data */ /* -- 128 bytes -- */ }; /* * Nhop validness. * * Currently we verify whether link is up or not on every packet, which can be * quite costy. * TODO: subscribe for the interface notifications and update the nexthops * with NHF_INVALID flag. */ #define NH_IS_VALID(_nh) RT_LINK_IS_UP((_nh)->nh_ifp) #define NH_IS_NHGRP(_nh) ((_nh)->nh_flags & NHF_MULTIPATH) #define NH_FREE(_nh) do { \ nhop_free(_nh); \ /* guard against invalid refs */ \ _nh = NULL; \ } while (0) struct weightened_nhop { struct nhop_object *nh; uint32_t weight; uint32_t storage; }; void nhop_free(struct nhop_object *nh); struct sysctl_req; struct sockaddr_dl; struct rib_head; /* flags that can be set using nhop_set_rtflags() */ #define RT_SET_RTFLAGS_MASK (RTF_PROTO1 | RTF_PROTO2 | RTF_PROTO3 | RTF_STATIC) #define RT_CHANGE_RTFLAGS_MASK RT_SET_RTFLAGS_MASK struct nhop_object *nhop_alloc(uint32_t fibnum, int family); void nhop_copy(struct nhop_object *nh, const struct nhop_object *nh_orig); struct nhop_object *nhop_get_nhop(struct nhop_object *nh, int *perror); int nhop_get_unlinked(struct nhop_object *nh); void nhop_set_direct_gw(struct nhop_object *nh, struct ifnet *ifp); bool nhop_set_gw(struct nhop_object *nh, const struct sockaddr *sa, bool is_gw); void nhop_set_mtu(struct nhop_object *nh, uint32_t mtu, bool from_user); void nhop_set_rtflags(struct nhop_object *nh, int rt_flags); void nhop_set_pxtype_flag(struct nhop_object *nh, int nh_flag); void nhop_set_broadcast(struct nhop_object *nh, bool is_broadcast); void nhop_set_blackhole(struct nhop_object *nh, int blackhole_rt_flag); void nhop_set_pinned(struct nhop_object *nh, bool is_pinned); void nhop_set_redirect(struct nhop_object *nh, bool is_redirect); void nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type); void nhop_set_src(struct nhop_object *nh, struct ifaddr *ifa); void nhop_set_transmit_ifp(struct nhop_object *nh, struct ifnet *ifp); +#define NH_ORIGIN_UNSPEC 0 /* not set */ +#define NH_ORIGIN_REDIRECT 1 /* kernel-originated redirect */ +#define NH_ORIGIN_KERNEL 2 /* kernel (interface) routes */ +#define NH_ORIGIN_BOOT 3 /* kernel-originated routes at boot */ +#define NH_ORIGIN_STATIC 4 /* route(8) routes */ +void nhop_set_origin(struct nhop_object *nh, uint8_t origin); +uint8_t nhop_get_origin(struct nhop_object *nh); + uint32_t nhop_get_idx(const struct nhop_object *nh); uint32_t nhop_get_uidx(const struct nhop_object *nh); void nhop_set_uidx(struct nhop_object *nh, uint32_t uidx); enum nhop_type nhop_get_type(const struct nhop_object *nh); int nhop_get_rtflags(const struct nhop_object *nh); struct vnet *nhop_get_vnet(const struct nhop_object *nh); struct nhop_object *nhop_select_func(struct nhop_object *nh, uint32_t flowid); int nhop_get_upper_family(const struct nhop_object *nh); bool nhop_set_upper_family(struct nhop_object *nh, int family); int nhop_get_neigh_family(const struct nhop_object *nh); uint32_t nhop_get_fibnum(const struct nhop_object *nh); void nhop_set_fibnum(struct nhop_object *nh, uint32_t fibnum); uint32_t nhop_get_expire(const struct nhop_object *nh); void nhop_set_expire(struct nhop_object *nh, uint32_t expire); struct rib_head *nhop_get_rh(const struct nhop_object *nh); struct nhgrp_object; uint32_t nhgrp_get_uidx(const struct nhgrp_object *nhg); +uint8_t nhgrp_get_origin(struct nhgrp_object *nhg); +void nhgrp_set_origin(struct nhgrp_object *nhg, uint8_t origin); #endif /* _KERNEL */ /* Kernel <> userland structures */ /* Structure usage and layout are described in dump_nhop_entry() */ struct nhop_external { uint32_t nh_len; /* length of the datastructure */ uint32_t nh_idx; /* Nexthop index */ uint32_t nh_fib; /* Fib nexhop is attached to */ uint32_t ifindex; /* transmit interface ifindex */ uint32_t aifindex; /* address ifindex */ uint8_t prepend_len; /* length of the prepend */ uint8_t nh_family; /* address family */ uint16_t nh_type; /* nexthop type */ uint16_t nh_mtu; /* nexthop mtu */ uint16_t nh_flags; /* nhop flags */ struct in_addr nh_addr; /* GW/DST IPv4 address */ struct in_addr nh_src; /* default source IPv4 address */ uint64_t nh_pksent; /* control plane */ /* lookup key: address, family, type */ char nh_prepend[64]; /* L2 prepend */ uint64_t nh_refcount; /* number of references */ }; struct nhop_addrs { uint32_t na_len; /* length of the datastructure */ uint16_t gw_sa_off; /* offset of gateway SA */ uint16_t src_sa_off; /* offset of src address SA */ }; #define NHG_C_TYPE_CNHOPS 0x1 /* Control plane nhops list */ #define NHG_C_TYPE_DNHOPS 0x2 /* Dataplane nhops list */ struct nhgrp_container { uint32_t nhgc_len; /* container length */ uint16_t nhgc_count; /* number of items */ uint8_t nhgc_type; /* container type */ uint8_t nhgc_subtype; /* container subtype */ }; struct nhgrp_nhop_external { uint32_t nh_idx; uint32_t nh_weight; }; /* * Layout: * - nhgrp_external * - nhgrp_container (control plane nhops list) * - nhgrp_nhop_external * - nhgrp_nhop_external * .. * - nhgrp_container (dataplane nhops list) * - nhgrp_nhop_external * - nhgrp_nhop_external */ struct nhgrp_external { uint32_t nhg_idx; /* Nexthop group index */ uint32_t nhg_refcount; /* number of references */ }; #endif diff --git a/sys/net/route/nhop_ctl.c b/sys/net/route/nhop_ctl.c index e4195d106d01..033a99acab1b 100644 --- a/sys/net/route/nhop_ctl.c +++ b/sys/net/route/nhop_ctl.c @@ -1,1205 +1,1217 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME nhop_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains core functionality for the nexthop ("nhop") route subsystem. * The business logic needed to create nexhop objects is implemented here. * * Nexthops in the original sense are the objects containing all the necessary * information to forward the packet to the selected destination. * In particular, nexthop is defined by a combination of * ifp, ifa, aifp, mtu, gw addr(if set), nh_type, nh_upper_family, mask of rt_flags and * NHF_DEFAULT * * Additionally, each nexthop gets assigned its unique index (nexthop index). * It serves two purposes: first one is to ease the ability of userland programs to * reference nexthops by their index. The second one allows lookup algorithms to * to store index instead of pointer (2 bytes vs 8) as a lookup result. * All nexthops are stored in the resizable hash table. * * Basically, this file revolves around supporting 3 functions: * 1) nhop_create_from_info / nhop_create_from_nhop, which contains all * business logic on filling the nexthop fields based on the provided request. * 2) nhop_get(), which gets a usable referenced nexthops. * * Conventions: * 1) non-exported functions start with verb * 2) exported function starts with the subsystem prefix: "nhop" */ static int dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w); static int finalize_nhop(struct nh_control *ctl, struct nhop_object *nh, bool link); static struct ifnet *get_aifp(const struct nhop_object *nh); static void fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp); static void destroy_nhop_epoch(epoch_context_t ctx); static void destroy_nhop(struct nhop_object *nh); _Static_assert(__offsetof(struct nhop_object, nh_ifp) == 32, "nhop_object: wrong nh_ifp offset"); _Static_assert(sizeof(struct nhop_object) <= 128, "nhop_object: size exceeds 128 bytes"); static uma_zone_t nhops_zone; /* Global zone for each and every nexthop */ #define NHOP_OBJECT_ALIGNED_SIZE roundup2(sizeof(struct nhop_object), \ 2 * CACHE_LINE_SIZE) #define NHOP_PRIV_ALIGNED_SIZE roundup2(sizeof(struct nhop_priv), \ 2 * CACHE_LINE_SIZE) void nhops_init(void) { nhops_zone = uma_zcreate("routing nhops", NHOP_OBJECT_ALIGNED_SIZE + NHOP_PRIV_ALIGNED_SIZE, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); } /* * Fetches the interface of source address used by the route. * In all cases except interface-address-route it would be the * same as the transmit interfaces. * However, for the interface address this function will return * this interface ifp instead of loopback. This is needed to support * link-local IPv6 loopback communications. * * Returns found ifp. */ static struct ifnet * get_aifp(const struct nhop_object *nh) { struct ifnet *aifp = NULL; /* * Adjust the "outgoing" interface. If we're going to loop * the packet back to ourselves, the ifp would be the loopback * interface. However, we'd rather know the interface associated * to the destination address (which should probably be one of * our own addresses). */ if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) && nh->gw_sa.sa_family == AF_LINK) { aifp = ifnet_byindex(nh->gwl_sa.sdl_index); if (aifp == NULL) { FIB_NH_LOG(LOG_WARNING, nh, "unable to get aifp for %s index %d", if_name(nh->nh_ifp), nh->gwl_sa.sdl_index); } } if (aifp == NULL) aifp = nh->nh_ifp; return (aifp); } int cmp_priv(const struct nhop_priv *_one, const struct nhop_priv *_two) { if (memcmp(_one->nh, _two->nh, NHOP_END_CMP) != 0) return (0); if (memcmp(_one, _two, NH_PRIV_END_CMP) != 0) return (0); return (1); } /* * Conditionally sets @nh mtu data based on the @info data. */ static void set_nhop_mtu_from_info(struct nhop_object *nh, const struct rt_addrinfo *info) { if (info->rti_mflags & RTV_MTU) nhop_set_mtu(nh, info->rti_rmx->rmx_mtu, true); } /* * Fills in shorted link-level sockadd version suitable to be stored inside the * nexthop gateway buffer. */ static void fill_sdl_from_ifp(struct sockaddr_dl_short *sdl, const struct ifnet *ifp) { bzero(sdl, sizeof(struct sockaddr_dl_short)); sdl->sdl_family = AF_LINK; sdl->sdl_len = sizeof(struct sockaddr_dl_short); sdl->sdl_index = ifp->if_index; sdl->sdl_type = ifp->if_type; } static int set_nhop_gw_from_info(struct nhop_object *nh, struct rt_addrinfo *info) { struct sockaddr *gw; gw = info->rti_info[RTAX_GATEWAY]; MPASS(gw != NULL); bool is_gw = info->rti_flags & RTF_GATEWAY; if ((gw->sa_family == AF_LINK) && !is_gw) { /* * Interface route with interface specified by the interface * index in sockadd_dl structure. It is used in the IPv6 loopback * output code, where we need to preserve the original interface * to maintain proper scoping. * Despite the fact that nexthop code stores original interface * in the separate field (nh_aifp, see below), write AF_LINK * compatible sa with shorter total length. */ struct sockaddr_dl *sdl = (struct sockaddr_dl *)gw; struct ifnet *ifp = ifnet_byindex(sdl->sdl_index); if (ifp == NULL) { FIB_NH_LOG(LOG_DEBUG, nh, "error: invalid ifindex %d", sdl->sdl_index); return (EINVAL); } nhop_set_direct_gw(nh, ifp); } else { /* * Multiple options here: * * 1) RTF_GATEWAY with IPv4/IPv6 gateway data * 2) Interface route with IPv4/IPv6 address of the * matching interface. Some routing daemons do that * instead of specifying ifindex in AF_LINK. * * In both cases, save the original nexthop to make the callers * happy. */ if (!nhop_set_gw(nh, gw, is_gw)) return (EINVAL); } return (0); } static void set_nhop_expire_from_info(struct nhop_object *nh, const struct rt_addrinfo *info) { uint32_t nh_expire = 0; /* Kernel -> userland timebase conversion. */ if ((info->rti_mflags & RTV_EXPIRE) && (info->rti_rmx->rmx_expire > 0)) nh_expire = info->rti_rmx->rmx_expire - time_second + time_uptime; nhop_set_expire(nh, nh_expire); } /* * Creates a new nexthop based on the information in @info. * * Returns: * 0 on success, filling @nh_ret with the desired nexthop object ptr * errno otherwise */ int nhop_create_from_info(struct rib_head *rnh, struct rt_addrinfo *info, struct nhop_object **nh_ret) { int error; NET_EPOCH_ASSERT(); MPASS(info->rti_ifa != NULL); MPASS(info->rti_ifp != NULL); if (info->rti_info[RTAX_GATEWAY] == NULL) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: empty gateway"); return (EINVAL); } struct nhop_object *nh = nhop_alloc(rnh->rib_fibnum, rnh->rib_family); if (nh == NULL) return (ENOMEM); if ((error = set_nhop_gw_from_info(nh, info)) != 0) { nhop_free(nh); return (error); } nhop_set_transmit_ifp(nh, info->rti_ifp); nhop_set_blackhole(nh, info->rti_flags & (RTF_BLACKHOLE | RTF_REJECT)); error = rnh->rnh_set_nh_pfxflags(rnh->rib_fibnum, info->rti_info[RTAX_DST], info->rti_info[RTAX_NETMASK], nh); nhop_set_redirect(nh, info->rti_flags & RTF_DYNAMIC); nhop_set_pinned(nh, info->rti_flags & RTF_PINNED); set_nhop_expire_from_info(nh, info); nhop_set_rtflags(nh, info->rti_flags); set_nhop_mtu_from_info(nh, info); nhop_set_src(nh, info->rti_ifa); /* * The remaining fields are either set from nh_preadd hook * or are computed from the provided data */ *nh_ret = nhop_get_nhop(nh, &error); return (error); } /* * Gets linked nhop using the provided @nh nexhop data. * If linked nhop is found, returns it, freeing the provided one. * If there is no such nexthop, attaches the remaining data to the * provided nexthop and links it. * * Returns 0 on success, storing referenced nexthop in @pnh. * Otherwise, errno is returned. */ struct nhop_object * nhop_get_nhop(struct nhop_object *nh, int *perror) { struct rib_head *rnh = nhop_get_rh(nh); if (__predict_false(rnh == NULL)) { *perror = EAFNOSUPPORT; nhop_free(nh); return (NULL); } return (nhop_get_nhop_internal(rnh, nh, perror)); } struct nhop_object * nhop_get_nhop_internal(struct rib_head *rnh, struct nhop_object *nh, int *perror) { struct nhop_priv *tmp_priv; int error; nh->nh_aifp = get_aifp(nh); /* Give the protocols chance to augment nexthop properties */ error = rnh->rnh_augment_nh(rnh->rib_fibnum, nh); if (error != 0) { nhop_free(nh); *perror = error; return (NULL); } tmp_priv = find_nhop(rnh->nh_control, nh->nh_priv); if (tmp_priv != NULL) { nhop_free(nh); *perror = 0; return (tmp_priv->nh); } /* * Existing nexthop not found, need to create new one. * Note: multiple simultaneous requests * can result in multiple equal nexhops existing in the * nexthop table. This is not a not a problem until the * relative number of such nexthops is significant, which * is extremely unlikely. */ *perror = finalize_nhop(rnh->nh_control, nh, true); return (*perror == 0 ? nh : NULL); } /* * Gets referenced but unlinked nhop. * Alocates/references the remaining bits of the nexthop data, so * it can be safely linked later or used as a clone source. * * Returns 0 on success. */ int nhop_get_unlinked(struct nhop_object *nh) { struct rib_head *rnh = nhop_get_rh(nh); if (__predict_false(rnh == NULL)) { nhop_free(nh); return (EAFNOSUPPORT); } nh->nh_aifp = get_aifp(nh); return (finalize_nhop(rnh->nh_control, nh, false)); } /* * Update @nh with data supplied in @info. * This is a helper function to support route changes. * * It limits the changes that can be done to the route to the following: * 1) all combination of gateway changes * 2) route flags (FLAG[123],STATIC) * 3) route MTU * * Returns: * 0 on success, errno otherwise */ static int alter_nhop_from_info(struct nhop_object *nh, struct rt_addrinfo *info) { struct sockaddr *info_gw; int error; /* Update MTU if set in the request*/ set_nhop_mtu_from_info(nh, info); /* Only RTF_FLAG[123] and RTF_STATIC */ uint32_t rt_flags = nhop_get_rtflags(nh) & ~RT_CHANGE_RTFLAGS_MASK; rt_flags |= info->rti_flags & RT_CHANGE_RTFLAGS_MASK; nhop_set_rtflags(nh, rt_flags); /* Consider gateway change */ info_gw = info->rti_info[RTAX_GATEWAY]; if (info_gw != NULL) { error = set_nhop_gw_from_info(nh, info); if (error != 0) return (error); } if (info->rti_ifa != NULL) nhop_set_src(nh, info->rti_ifa); if (info->rti_ifp != NULL) nhop_set_transmit_ifp(nh, info->rti_ifp); return (0); } /* * Creates new nexthop based on @nh_orig and augmentation data from @info. * Helper function used in the route changes, please see * alter_nhop_from_info() comments for more details. * * Returns: * 0 on success, filling @nh_ret with the desired nexthop object * errno otherwise */ int nhop_create_from_nhop(struct rib_head *rnh, const struct nhop_object *nh_orig, struct rt_addrinfo *info, struct nhop_object **pnh) { struct nhop_object *nh; int error; NET_EPOCH_ASSERT(); nh = nhop_alloc(rnh->rib_fibnum, rnh->rib_family); if (nh == NULL) return (ENOMEM); nhop_copy(nh, nh_orig); error = alter_nhop_from_info(nh, info); if (error != 0) { nhop_free(nh); return (error); } *pnh = nhop_get_nhop(nh, &error); return (error); } static bool reference_nhop_deps(struct nhop_object *nh) { if (!ifa_try_ref(nh->nh_ifa)) return (false); nh->nh_aifp = get_aifp(nh); if (!if_try_ref(nh->nh_aifp)) { ifa_free(nh->nh_ifa); return (false); } FIB_NH_LOG(LOG_DEBUG2, nh, "nh_aifp: %s nh_ifp %s", if_name(nh->nh_aifp), if_name(nh->nh_ifp)); if (!if_try_ref(nh->nh_ifp)) { ifa_free(nh->nh_ifa); if_rele(nh->nh_aifp); return (false); } return (true); } /* * Alocates/references the remaining bits of nexthop data and links * it to the hash table. * Returns 0 if successful, * errno otherwise. @nh_priv is freed in case of error. */ static int finalize_nhop(struct nh_control *ctl, struct nhop_object *nh, bool link) { /* Allocate per-cpu packet counter */ nh->nh_pksent = counter_u64_alloc(M_NOWAIT); if (nh->nh_pksent == NULL) { nhop_free(nh); RTSTAT_INC(rts_nh_alloc_failure); FIB_NH_LOG(LOG_WARNING, nh, "counter_u64_alloc() failed"); return (ENOMEM); } if (!reference_nhop_deps(nh)) { counter_u64_free(nh->nh_pksent); nhop_free(nh); RTSTAT_INC(rts_nh_alloc_failure); FIB_NH_LOG(LOG_WARNING, nh, "interface reference failed"); return (EAGAIN); } /* Save vnet to ease destruction */ nh->nh_priv->nh_vnet = curvnet; /* Please see nhop_free() comments on the initial value */ refcount_init(&nh->nh_priv->nh_linked, 2); MPASS(nh->nh_priv->nh_fibnum == ctl->ctl_rh->rib_fibnum); if (!link) { refcount_release(&nh->nh_priv->nh_linked); NHOPS_WLOCK(ctl); nh->nh_priv->nh_finalized = 1; NHOPS_WUNLOCK(ctl); } else if (link_nhop(ctl, nh->nh_priv) == 0) { /* * Adding nexthop to the datastructures * failed. Call destructor w/o waiting for * the epoch end, as nexthop is not used * and return. */ char nhbuf[NHOP_PRINT_BUFSIZE]; FIB_NH_LOG(LOG_WARNING, nh, "failed to link %s", nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); destroy_nhop(nh); return (ENOBUFS); } IF_DEBUG_LEVEL(LOG_DEBUG) { char nhbuf[NHOP_PRINT_BUFSIZE] __unused; FIB_NH_LOG(LOG_DEBUG, nh, "finalized: %s", nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); } return (0); } static void destroy_nhop(struct nhop_object *nh) { if_rele(nh->nh_ifp); if_rele(nh->nh_aifp); ifa_free(nh->nh_ifa); counter_u64_free(nh->nh_pksent); uma_zfree(nhops_zone, nh); } /* * Epoch callback indicating nhop is safe to destroy */ static void destroy_nhop_epoch(epoch_context_t ctx) { struct nhop_priv *nh_priv; nh_priv = __containerof(ctx, struct nhop_priv, nh_epoch_ctx); destroy_nhop(nh_priv->nh); } void nhop_ref_object(struct nhop_object *nh) { u_int old __diagused; old = refcount_acquire(&nh->nh_priv->nh_refcnt); KASSERT(old > 0, ("%s: nhop object %p has 0 refs", __func__, nh)); } int nhop_try_ref_object(struct nhop_object *nh) { return (refcount_acquire_if_not_zero(&nh->nh_priv->nh_refcnt)); } void nhop_free(struct nhop_object *nh) { struct nh_control *ctl; struct nhop_priv *nh_priv = nh->nh_priv; struct epoch_tracker et; if (!refcount_release(&nh_priv->nh_refcnt)) return; /* allows to use nhop_free() during nhop init */ if (__predict_false(nh_priv->nh_finalized == 0)) { uma_zfree(nhops_zone, nh); return; } IF_DEBUG_LEVEL(LOG_DEBUG) { char nhbuf[NHOP_PRINT_BUFSIZE] __unused; FIB_NH_LOG(LOG_DEBUG, nh, "deleting %s", nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); } /* * There are only 2 places, where nh_linked can be decreased: * rib destroy (nhops_destroy_rib) and this function. * nh_link can never be increased. * * Hence, use initial value of 2 to make use of * refcount_release_if_not_last(). * * There can be two scenarious when calling this function: * * 1) nh_linked value is 2. This means that either * nhops_destroy_rib() has not been called OR it is running, * but we are guaranteed that nh_control won't be freed in * this epoch. Hence, nexthop can be safely unlinked. * * 2) nh_linked value is 1. In that case, nhops_destroy_rib() * has been called and nhop unlink can be skipped. */ NET_EPOCH_ENTER(et); if (refcount_release_if_not_last(&nh_priv->nh_linked)) { ctl = nh_priv->nh_control; if (unlink_nhop(ctl, nh_priv) == NULL) { /* Do not try to reclaim */ char nhbuf[NHOP_PRINT_BUFSIZE]; FIB_NH_LOG(LOG_WARNING, nh, "failed to unlink %s", nhop_print_buf(nh, nhbuf, sizeof(nhbuf))); NET_EPOCH_EXIT(et); return; } } NET_EPOCH_EXIT(et); epoch_call(net_epoch_preempt, destroy_nhop_epoch, &nh_priv->nh_epoch_ctx); } void nhop_ref_any(struct nhop_object *nh) { #ifdef ROUTE_MPATH if (!NH_IS_NHGRP(nh)) nhop_ref_object(nh); else nhgrp_ref_object((struct nhgrp_object *)nh); #else nhop_ref_object(nh); #endif } void nhop_free_any(struct nhop_object *nh) { #ifdef ROUTE_MPATH if (!NH_IS_NHGRP(nh)) nhop_free(nh); else nhgrp_free((struct nhgrp_object *)nh); #else nhop_free(nh); #endif } /* Nhop-related methods */ /* * Allocates an empty unlinked nhop object. * Returns object pointer or NULL on failure */ struct nhop_object * nhop_alloc(uint32_t fibnum, int family) { struct nhop_object *nh; struct nhop_priv *nh_priv; nh = (struct nhop_object *)uma_zalloc(nhops_zone, M_NOWAIT | M_ZERO); if (__predict_false(nh == NULL)) return (NULL); nh_priv = (struct nhop_priv *)((char *)nh + NHOP_OBJECT_ALIGNED_SIZE); nh->nh_priv = nh_priv; nh_priv->nh = nh; nh_priv->nh_upper_family = family; nh_priv->nh_fibnum = fibnum; /* Setup refcount early to allow nhop_free() to work */ refcount_init(&nh_priv->nh_refcnt, 1); return (nh); } void nhop_copy(struct nhop_object *nh, const struct nhop_object *nh_orig) { struct nhop_priv *nh_priv = nh->nh_priv; nh->nh_flags = nh_orig->nh_flags; nh->nh_mtu = nh_orig->nh_mtu; memcpy(&nh->gw_sa, &nh_orig->gw_sa, nh_orig->gw_sa.sa_len); nh->nh_ifp = nh_orig->nh_ifp; nh->nh_ifa = nh_orig->nh_ifa; nh->nh_aifp = nh_orig->nh_aifp; nh_priv->nh_upper_family = nh_orig->nh_priv->nh_upper_family; nh_priv->nh_neigh_family = nh_orig->nh_priv->nh_neigh_family; nh_priv->nh_type = nh_orig->nh_priv->nh_type; nh_priv->rt_flags = nh_orig->nh_priv->rt_flags; nh_priv->nh_fibnum = nh_orig->nh_priv->nh_fibnum; } void nhop_set_direct_gw(struct nhop_object *nh, struct ifnet *ifp) { nh->nh_flags &= ~NHF_GATEWAY; nh->nh_priv->rt_flags &= ~RTF_GATEWAY; nh->nh_priv->nh_neigh_family = nh->nh_priv->nh_upper_family; fill_sdl_from_ifp(&nh->gwl_sa, ifp); memset(&nh->gw_buf[nh->gw_sa.sa_len], 0, sizeof(nh->gw_buf) - nh->gw_sa.sa_len); } bool nhop_check_gateway(int upper_family, int neigh_family) { if (upper_family == neigh_family) return (true); else if (neigh_family == AF_UNSPEC || neigh_family == AF_LINK) return (true); #if defined(INET) && defined(INET6) else if (upper_family == AF_INET && neigh_family == AF_INET6 && rib_can_4o6_nhop()) return (true); #endif else return (false); } /* * Sets gateway for the nexthop. * It can be "normal" gateway with is_gw set or a special form of * adding interface route, refering to it by specifying local interface * address. In that case is_gw is set to false. */ bool nhop_set_gw(struct nhop_object *nh, const struct sockaddr *gw, bool is_gw) { if (gw->sa_len > sizeof(nh->gw_buf)) { FIB_NH_LOG(LOG_DEBUG, nh, "nhop SA size too big: AF %d len %u", gw->sa_family, gw->sa_len); return (false); } if (!nhop_check_gateway(nh->nh_priv->nh_upper_family, gw->sa_family)) { FIB_NH_LOG(LOG_DEBUG, nh, "error: invalid dst/gateway family combination (%d, %d)", nh->nh_priv->nh_upper_family, gw->sa_family); return (false); } memcpy(&nh->gw_sa, gw, gw->sa_len); memset(&nh->gw_buf[gw->sa_len], 0, sizeof(nh->gw_buf) - gw->sa_len); if (is_gw) { nh->nh_flags |= NHF_GATEWAY; nh->nh_priv->rt_flags |= RTF_GATEWAY; nh->nh_priv->nh_neigh_family = gw->sa_family; } else { nh->nh_flags &= ~NHF_GATEWAY; nh->nh_priv->rt_flags &= ~RTF_GATEWAY; nh->nh_priv->nh_neigh_family = nh->nh_priv->nh_upper_family; } return (true); } bool nhop_set_upper_family(struct nhop_object *nh, int family) { if (!nhop_check_gateway(nh->nh_priv->nh_upper_family, family)) { FIB_NH_LOG(LOG_DEBUG, nh, "error: invalid upper/neigh family combination (%d, %d)", nh->nh_priv->nh_upper_family, family); return (false); } nh->nh_priv->nh_upper_family = family; return (true); } void nhop_set_broadcast(struct nhop_object *nh, bool is_broadcast) { if (is_broadcast) { nh->nh_flags |= NHF_BROADCAST; nh->nh_priv->rt_flags |= RTF_BROADCAST; } else { nh->nh_flags &= ~NHF_BROADCAST; nh->nh_priv->rt_flags &= ~RTF_BROADCAST; } } void nhop_set_blackhole(struct nhop_object *nh, int blackhole_rt_flag) { nh->nh_flags &= ~(NHF_BLACKHOLE | NHF_REJECT); nh->nh_priv->rt_flags &= ~(RTF_BLACKHOLE | RTF_REJECT); switch (blackhole_rt_flag) { case RTF_BLACKHOLE: nh->nh_flags |= NHF_BLACKHOLE; nh->nh_priv->rt_flags |= RTF_BLACKHOLE; break; case RTF_REJECT: nh->nh_flags |= NHF_REJECT; nh->nh_priv->rt_flags |= RTF_REJECT; break; } } void nhop_set_redirect(struct nhop_object *nh, bool is_redirect) { if (is_redirect) { nh->nh_priv->rt_flags |= RTF_DYNAMIC; nh->nh_flags |= NHF_REDIRECT; } else { nh->nh_priv->rt_flags &= ~RTF_DYNAMIC; nh->nh_flags &= ~NHF_REDIRECT; } } void nhop_set_pinned(struct nhop_object *nh, bool is_pinned) { if (is_pinned) nh->nh_priv->rt_flags |= RTF_PINNED; else nh->nh_priv->rt_flags &= ~RTF_PINNED; } uint32_t nhop_get_idx(const struct nhop_object *nh) { return (nh->nh_priv->nh_idx); } uint32_t nhop_get_uidx(const struct nhop_object *nh) { return (nh->nh_priv->nh_uidx); } void nhop_set_uidx(struct nhop_object *nh, uint32_t uidx) { nh->nh_priv->nh_uidx = uidx; } enum nhop_type nhop_get_type(const struct nhop_object *nh) { return (nh->nh_priv->nh_type); } void nhop_set_type(struct nhop_object *nh, enum nhop_type nh_type) { nh->nh_priv->nh_type = nh_type; } int nhop_get_rtflags(const struct nhop_object *nh) { return (nh->nh_priv->rt_flags); } /* * Sets generic rtflags that are not covered by other functions. */ void nhop_set_rtflags(struct nhop_object *nh, int rt_flags) { nh->nh_priv->rt_flags &= ~RT_SET_RTFLAGS_MASK; nh->nh_priv->rt_flags |= (rt_flags & RT_SET_RTFLAGS_MASK); } /* * Sets flags that are specific to the prefix (NHF_HOST or NHF_DEFAULT). */ void nhop_set_pxtype_flag(struct nhop_object *nh, int nh_flag) { if (nh_flag == NHF_HOST) { nh->nh_flags |= NHF_HOST; nh->nh_flags &= ~NHF_DEFAULT; nh->nh_priv->rt_flags |= RTF_HOST; } else if (nh_flag == NHF_DEFAULT) { nh->nh_flags |= NHF_DEFAULT; nh->nh_flags &= ~NHF_HOST; nh->nh_priv->rt_flags &= ~RTF_HOST; } else { nh->nh_flags &= ~(NHF_HOST | NHF_DEFAULT); nh->nh_priv->rt_flags &= ~RTF_HOST; } } /* * Sets nhop MTU. Sets RTF_FIXEDMTU if mtu is explicitly * specified by userland. */ void nhop_set_mtu(struct nhop_object *nh, uint32_t mtu, bool from_user) { if (from_user) { if (mtu != 0) nh->nh_priv->rt_flags |= RTF_FIXEDMTU; else nh->nh_priv->rt_flags &= ~RTF_FIXEDMTU; } nh->nh_mtu = mtu; } void nhop_set_src(struct nhop_object *nh, struct ifaddr *ifa) { nh->nh_ifa = ifa; } void nhop_set_transmit_ifp(struct nhop_object *nh, struct ifnet *ifp) { nh->nh_ifp = ifp; } struct vnet * nhop_get_vnet(const struct nhop_object *nh) { return (nh->nh_priv->nh_vnet); } struct nhop_object * nhop_select_func(struct nhop_object *nh, uint32_t flowid) { return (nhop_select(nh, flowid)); } /* * Returns address family of the traffic uses the nexthop. */ int nhop_get_upper_family(const struct nhop_object *nh) { return (nh->nh_priv->nh_upper_family); } /* * Returns address family of the LLE or gateway that is used * to forward the traffic to. */ int nhop_get_neigh_family(const struct nhop_object *nh) { return (nh->nh_priv->nh_neigh_family); } uint32_t nhop_get_fibnum(const struct nhop_object *nh) { return (nh->nh_priv->nh_fibnum); } void nhop_set_fibnum(struct nhop_object *nh, uint32_t fibnum) { nh->nh_priv->nh_fibnum = fibnum; } uint32_t nhop_get_expire(const struct nhop_object *nh) { return (nh->nh_priv->nh_expire); } void nhop_set_expire(struct nhop_object *nh, uint32_t expire) { MPASS(!NH_IS_LINKED(nh)); nh->nh_priv->nh_expire = expire; } struct rib_head * nhop_get_rh(const struct nhop_object *nh) { uint32_t fibnum = nhop_get_fibnum(nh); int family = nhop_get_neigh_family(nh); return (rt_tables_get_rnh(fibnum, family)); } +uint8_t +nhop_get_origin(struct nhop_object *nh) +{ + return (nh->nh_priv->nh_origin); +} + +void +nhop_set_origin(struct nhop_object *nh, uint8_t origin) +{ + nh->nh_priv->nh_origin = origin; +} + void nhops_update_ifmtu(struct rib_head *rh, struct ifnet *ifp, uint32_t mtu) { struct nh_control *ctl; struct nhop_priv *nh_priv; struct nhop_object *nh; ctl = rh->nh_control; NHOPS_WLOCK(ctl); CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) { nh = nh_priv->nh; if (nh->nh_ifp == ifp) { if ((nh_priv->rt_flags & RTF_FIXEDMTU) == 0 || nh->nh_mtu > mtu) { /* Update MTU directly */ nh->nh_mtu = mtu; } } } CHT_SLIST_FOREACH_END; NHOPS_WUNLOCK(ctl); } /* * Prints nexthop @nh data in the provided @buf. * Example: nh#33/inet/em0/192.168.0.1 */ char * nhop_print_buf(const struct nhop_object *nh, char *buf, size_t bufsize) { #if defined(INET) || defined(INET6) char abuf[INET6_ADDRSTRLEN]; #endif struct nhop_priv *nh_priv = nh->nh_priv; const char *upper_str = rib_print_family(nh->nh_priv->nh_upper_family); switch (nh->gw_sa.sa_family) { #ifdef INET case AF_INET: inet_ntop(AF_INET, &nh->gw4_sa.sin_addr, abuf, sizeof(abuf)); snprintf(buf, bufsize, "nh#%d/%s/%s/%s", nh_priv->nh_idx, upper_str, if_name(nh->nh_ifp), abuf); break; #endif #ifdef INET6 case AF_INET6: inet_ntop(AF_INET6, &nh->gw6_sa.sin6_addr, abuf, sizeof(abuf)); snprintf(buf, bufsize, "nh#%d/%s/%s/%s", nh_priv->nh_idx, upper_str, if_name(nh->nh_ifp), abuf); break; #endif case AF_LINK: snprintf(buf, bufsize, "nh#%d/%s/%s/resolve", nh_priv->nh_idx, upper_str, if_name(nh->nh_ifp)); break; default: snprintf(buf, bufsize, "nh#%d/%s/%s/????", nh_priv->nh_idx, upper_str, if_name(nh->nh_ifp)); break; } return (buf); } char * nhop_print_buf_any(const struct nhop_object *nh, char *buf, size_t bufsize) { #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh)) return (nhgrp_print_buf((const struct nhgrp_object *)nh, buf, bufsize)); else #endif return (nhop_print_buf(nh, buf, bufsize)); } /* * Dumps a single entry to sysctl buffer. * * Layout: * rt_msghdr - generic RTM header to allow users to skip non-understood messages * nhop_external - nexhop description structure (with length) * nhop_addrs - structure encapsulating GW/SRC sockaddrs */ static int dump_nhop_entry(struct rib_head *rh, struct nhop_object *nh, struct sysctl_req *w) { struct { struct rt_msghdr rtm; struct nhop_external nhe; struct nhop_addrs na; } arpc; struct nhop_external *pnhe; struct sockaddr *gw_sa, *src_sa; struct sockaddr_storage ss; size_t addrs_len; int error; memset(&arpc, 0, sizeof(arpc)); arpc.rtm.rtm_msglen = sizeof(arpc); arpc.rtm.rtm_version = RTM_VERSION; arpc.rtm.rtm_type = RTM_GET; //arpc.rtm.rtm_flags = RTF_UP; arpc.rtm.rtm_flags = nh->nh_priv->rt_flags; /* nhop_external */ pnhe = &arpc.nhe; pnhe->nh_len = sizeof(struct nhop_external); pnhe->nh_idx = nh->nh_priv->nh_idx; pnhe->nh_fib = rh->rib_fibnum; pnhe->ifindex = nh->nh_ifp->if_index; pnhe->aifindex = nh->nh_aifp->if_index; pnhe->nh_family = nh->nh_priv->nh_upper_family; pnhe->nh_type = nh->nh_priv->nh_type; pnhe->nh_mtu = nh->nh_mtu; pnhe->nh_flags = nh->nh_flags; memcpy(pnhe->nh_prepend, nh->nh_prepend, sizeof(nh->nh_prepend)); pnhe->prepend_len = nh->nh_prepend_len; pnhe->nh_refcount = nh->nh_priv->nh_refcnt; pnhe->nh_pksent = counter_u64_fetch(nh->nh_pksent); /* sockaddr container */ addrs_len = sizeof(struct nhop_addrs); arpc.na.gw_sa_off = addrs_len; gw_sa = (struct sockaddr *)&nh->gw4_sa; addrs_len += gw_sa->sa_len; src_sa = nh->nh_ifa->ifa_addr; if (src_sa->sa_family == AF_LINK) { /* Shorten structure */ memset(&ss, 0, sizeof(struct sockaddr_storage)); fill_sdl_from_ifp((struct sockaddr_dl_short *)&ss, nh->nh_ifa->ifa_ifp); src_sa = (struct sockaddr *)&ss; } arpc.na.src_sa_off = addrs_len; addrs_len += src_sa->sa_len; /* Write total container length */ arpc.na.na_len = addrs_len; arpc.rtm.rtm_msglen += arpc.na.na_len - sizeof(struct nhop_addrs); error = SYSCTL_OUT(w, &arpc, sizeof(arpc)); if (error == 0) error = SYSCTL_OUT(w, gw_sa, gw_sa->sa_len); if (error == 0) error = SYSCTL_OUT(w, src_sa, src_sa->sa_len); return (error); } uint32_t nhops_get_count(struct rib_head *rh) { struct nh_control *ctl; uint32_t count; ctl = rh->nh_control; NHOPS_RLOCK(ctl); count = ctl->nh_head.items_count; NHOPS_RUNLOCK(ctl); return (count); } int nhops_dump_sysctl(struct rib_head *rh, struct sysctl_req *w) { struct nh_control *ctl; struct nhop_priv *nh_priv; int error; ctl = rh->nh_control; NHOPS_RLOCK(ctl); FIB_RH_LOG(LOG_DEBUG, rh, "dump %u items", ctl->nh_head.items_count); CHT_SLIST_FOREACH(&ctl->nh_head, nhops, nh_priv) { error = dump_nhop_entry(rh, nh_priv->nh, w); if (error != 0) { NHOPS_RUNLOCK(ctl); return (error); } } CHT_SLIST_FOREACH_END; NHOPS_RUNLOCK(ctl); return (0); } diff --git a/sys/net/route/nhop_var.h b/sys/net/route/nhop_var.h index c3c442a4bfa3..ace14556a14c 100644 --- a/sys/net/route/nhop_var.h +++ b/sys/net/route/nhop_var.h @@ -1,112 +1,113 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This header file contains private definitions for nexthop routing. * * Header is not intended to be included by the code external to the * routing subsystem. */ #ifndef _NET_ROUTE_NHOP_VAR_H_ #define _NET_ROUTE_NHOP_VAR_H_ MALLOC_DECLARE(M_NHOP); /* define nhop hash table */ struct nhop_priv; CHT_SLIST_DEFINE(nhops, struct nhop_priv); /* produce hash value for an object */ #define nhops_hash_obj(_obj) hash_priv(_obj) /* compare two objects */ #define nhops_cmp(_one, _two) cmp_priv(_one, _two) /* next object accessor */ #define nhops_next(_obj) (_obj)->nh_next /* define multipath hash table */ struct nhgrp_priv; CHT_SLIST_DEFINE(nhgroups, struct nhgrp_priv); struct nh_control { struct nhops_head nh_head; /* hash table head */ struct bitmask_head nh_idx_head; /* nhop index head */ struct nhgroups_head gr_head; /* nhgrp hash table head */ struct rwlock ctl_lock; /* overall ctl lock */ struct rib_head *ctl_rh; /* pointer back to rnh */ struct epoch_context ctl_epoch_ctx; /* epoch ctl helper */ }; #define NHOPS_WLOCK(ctl) rw_wlock(&(ctl)->ctl_lock) #define NHOPS_RLOCK(ctl) rw_rlock(&(ctl)->ctl_lock) #define NHOPS_WUNLOCK(ctl) rw_wunlock(&(ctl)->ctl_lock) #define NHOPS_RUNLOCK(ctl) rw_runlock(&(ctl)->ctl_lock) #define NHOPS_LOCK_INIT(ctl) rw_init(&(ctl)->ctl_lock, "nhop_ctl") #define NHOPS_LOCK_DESTROY(ctl) rw_destroy(&(ctl)->ctl_lock) #define NHOPS_WLOCK_ASSERT(ctl) rw_assert(&(ctl)->ctl_lock, RA_WLOCKED) /* Control plane-only nhop data */ struct nhop_object; struct nhop_priv { /* nhop lookup comparison start */ uint8_t nh_upper_family;/* address family of the lookup */ uint8_t nh_neigh_family;/* neighbor address family */ uint16_t nh_type; /* nexthop type */ uint32_t rt_flags; /* routing flags for the control plane */ uint32_t nh_expire; /* path expiration time */ uint32_t nh_uidx; /* userland-provided index */ /* nhop lookup comparison end */ uint32_t nh_idx; /* nexthop index */ uint32_t nh_fibnum; /* nexthop fib */ void *cb_func; /* function handling additional rewrite caps */ u_int nh_refcnt; /* number of references, refcount(9) */ u_int nh_linked; /* refcount(9), == 2 if linked to the list */ int nh_finalized; /* non-zero if finalized() was called */ + uint8_t nh_origin; /* protocol that originated the nexthop */ struct nhop_object *nh; /* backreference to the dataplane nhop */ struct nh_control *nh_control; /* backreference to the rnh */ struct nhop_priv *nh_next; /* hash table membership */ struct vnet *nh_vnet; /* vnet nhop belongs to */ struct epoch_context nh_epoch_ctx; /* epoch data for nhop */ }; #define NH_PRIV_END_CMP (__offsetof(struct nhop_priv, nh_idx)) #define NH_IS_PINNED(_nh) ((!NH_IS_NHGRP(_nh)) && \ ((_nh)->nh_priv->rt_flags & RTF_PINNED)) #define NH_IS_LINKED(_nh) ((_nh)->nh_priv->nh_idx != 0) /* nhop.c */ struct nhop_priv *find_nhop(struct nh_control *ctl, const struct nhop_priv *nh_priv); int link_nhop(struct nh_control *ctl, struct nhop_priv *nh_priv); struct nhop_priv *unlink_nhop(struct nh_control *ctl, struct nhop_priv *nh_priv); /* nhop_ctl.c */ int cmp_priv(const struct nhop_priv *_one, const struct nhop_priv *_two); #endif diff --git a/sys/net/route/route_ctl.c b/sys/net/route/route_ctl.c index a9ea2ad49103..d6c382735902 100644 --- a/sys/net/route/route_ctl.c +++ b/sys/net/route/route_ctl.c @@ -1,1550 +1,1551 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2020 Alexander V. Chernikov * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #include "opt_route.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEBUG_MOD_NAME route_ctl #define DEBUG_MAX_LEVEL LOG_DEBUG #include _DECLARE_DEBUG(LOG_INFO); /* * This file contains control plane routing tables functions. * * All functions assumes they are called in net epoch. */ union sockaddr_union { struct sockaddr sa; struct sockaddr_in sin; struct sockaddr_in6 sin6; char _buf[32]; }; static int add_route_byinfo(struct rib_head *rnh, struct rt_addrinfo *info, struct rib_cmd_info *rc); static int change_route_byinfo(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *nhd_orig, struct rib_cmd_info *rc); static int add_route_flags(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_add, int op_flags, struct rib_cmd_info *rc); #ifdef ROUTE_MPATH static int add_route_flags_mpath(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_orig, int op_flags, struct rib_cmd_info *rc); #endif static int add_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc); static int delete_route(struct rib_head *rnh, struct rtentry *rt, struct rib_cmd_info *rc); static int rt_delete_conditional(struct rib_head *rnh, struct rtentry *rt, int prio, rib_filter_f_t *cb, void *cbdata, struct rib_cmd_info *rc); static int get_prio_from_info(const struct rt_addrinfo *info); static int nhop_get_prio(const struct nhop_object *nh); #ifdef ROUTE_MPATH static bool rib_can_multipath(struct rib_head *rh); #endif /* Per-vnet multipath routing configuration */ SYSCTL_DECL(_net_route); #define V_rib_route_multipath VNET(rib_route_multipath) #ifdef ROUTE_MPATH #define _MP_FLAGS CTLFLAG_RW #else #define _MP_FLAGS CTLFLAG_RD #endif VNET_DEFINE(u_int, rib_route_multipath) = 1; SYSCTL_UINT(_net_route, OID_AUTO, multipath, _MP_FLAGS | CTLFLAG_VNET, &VNET_NAME(rib_route_multipath), 0, "Enable route multipath"); #undef _MP_FLAGS #ifdef ROUTE_MPATH VNET_DEFINE(u_int, fib_hash_outbound) = 0; SYSCTL_UINT(_net_route, OID_AUTO, hash_outbound, CTLFLAG_RD | CTLFLAG_VNET, &VNET_NAME(fib_hash_outbound), 0, "Compute flowid for locally-originated packets"); /* Default entropy to add to the hash calculation for the outbound connections*/ uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN] = { 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, }; #endif #if defined(INET) && defined(INET6) FEATURE(ipv4_rfc5549_support, "Route IPv4 packets via IPv6 nexthops"); #define V_rib_route_ipv6_nexthop VNET(rib_route_ipv6_nexthop) VNET_DEFINE_STATIC(u_int, rib_route_ipv6_nexthop) = 1; SYSCTL_UINT(_net_route, OID_AUTO, ipv6_nexthop, CTLFLAG_RW | CTLFLAG_VNET, &VNET_NAME(rib_route_ipv6_nexthop), 0, "Enable IPv4 route via IPv6 Next Hop address"); #endif /* Debug bits */ SYSCTL_NODE(_net_route, OID_AUTO, debug, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); static struct rib_head * get_rnh(uint32_t fibnum, const struct rt_addrinfo *info) { struct rib_head *rnh; struct sockaddr *dst; KASSERT((fibnum < rt_numfibs), ("rib_add_route: bad fibnum")); dst = info->rti_info[RTAX_DST]; rnh = rt_tables_get_rnh(fibnum, dst->sa_family); return (rnh); } #if defined(INET) && defined(INET6) bool rib_can_4o6_nhop(void) { return (!!V_rib_route_ipv6_nexthop); } #endif #ifdef ROUTE_MPATH static bool rib_can_multipath(struct rib_head *rh) { int result; CURVNET_SET(rh->rib_vnet); result = !!V_rib_route_multipath; CURVNET_RESTORE(); return (result); } /* * Check is nhop is multipath-eligible. * Avoid nhops without gateways and redirects. * * Returns 1 for multipath-eligible nexthop, * 0 otherwise. */ bool nhop_can_multipath(const struct nhop_object *nh) { if ((nh->nh_flags & NHF_MULTIPATH) != 0) return (1); if ((nh->nh_flags & NHF_GATEWAY) == 0) return (0); if ((nh->nh_flags & NHF_REDIRECT) != 0) return (0); return (1); } #endif static int get_info_weight(const struct rt_addrinfo *info, uint32_t default_weight) { uint32_t weight; if (info->rti_mflags & RTV_WEIGHT) weight = info->rti_rmx->rmx_weight; else weight = default_weight; /* Keep upper 1 byte for adm distance purposes */ if (weight > RT_MAX_WEIGHT) weight = RT_MAX_WEIGHT; else if (weight == 0) weight = default_weight; return (weight); } /* * File-local concept for distingushing between the normal and * RTF_PINNED routes tha can override the "normal" one. */ #define NH_PRIORITY_HIGH 2 #define NH_PRIORITY_NORMAL 1 static int get_prio_from_info(const struct rt_addrinfo *info) { if (info->rti_flags & RTF_PINNED) return (NH_PRIORITY_HIGH); return (NH_PRIORITY_NORMAL); } static int nhop_get_prio(const struct nhop_object *nh) { if (NH_IS_PINNED(nh)) return (NH_PRIORITY_HIGH); return (NH_PRIORITY_NORMAL); } /* * Check if specified @gw matches gw data in the nexthop @nh. * * Returns true if matches, false otherwise. */ bool match_nhop_gw(const struct nhop_object *nh, const struct sockaddr *gw) { if (nh->gw_sa.sa_family != gw->sa_family) return (false); switch (gw->sa_family) { case AF_INET: return (nh->gw4_sa.sin_addr.s_addr == ((const struct sockaddr_in *)gw)->sin_addr.s_addr); case AF_INET6: { const struct sockaddr_in6 *gw6; gw6 = (const struct sockaddr_in6 *)gw; /* * Currently (2020-09) IPv6 gws in kernel have their * scope embedded. Once this becomes false, this code * has to be revisited. */ if (IN6_ARE_ADDR_EQUAL(&nh->gw6_sa.sin6_addr, &gw6->sin6_addr)) return (true); return (false); } case AF_LINK: { const struct sockaddr_dl *sdl; sdl = (const struct sockaddr_dl *)gw; return (nh->gwl_sa.sdl_index == sdl->sdl_index); } default: return (memcmp(&nh->gw_sa, gw, nh->gw_sa.sa_len) == 0); } /* NOTREACHED */ return (false); } /* * Matches all nexthop with given @gw. * Can be used as rib_filter_f callback. */ int rib_match_gw(const struct rtentry *rt, const struct nhop_object *nh, void *gw_sa) { const struct sockaddr *gw = (const struct sockaddr *)gw_sa; return (match_nhop_gw(nh, gw)); } struct gw_filter_data { const struct sockaddr *gw; int count; }; /* * Matches first occurence of the gateway provided in @gwd */ static int match_gw_one(const struct rtentry *rt, const struct nhop_object *nh, void *_data) { struct gw_filter_data *gwd = (struct gw_filter_data *)_data; /* Return only first match to make rtsock happy */ if (match_nhop_gw(nh, gwd->gw) && gwd->count++ == 0) return (1); return (0); } /* * Checks if data in @info matches nexhop @nh. * * Returns 0 on success, * ESRCH if not matched, * ENOENT if filter function returned false */ int check_info_match_nhop(const struct rt_addrinfo *info, const struct rtentry *rt, const struct nhop_object *nh) { const struct sockaddr *gw = info->rti_info[RTAX_GATEWAY]; if (info->rti_filter != NULL) { if (info->rti_filter(rt, nh, info->rti_filterdata) == 0) return (ENOENT); else return (0); } if ((gw != NULL) && !match_nhop_gw(nh, gw)) return (ESRCH); return (0); } /* * Runs exact prefix match based on @dst and @netmask. * Returns matched @rtentry if found or NULL. * If rtentry was found, saves nexthop / weight value into @rnd. */ static struct rtentry * lookup_prefix_bysa(struct rib_head *rnh, const struct sockaddr *dst, const struct sockaddr *netmask, struct route_nhop_data *rnd) { struct rtentry *rt; RIB_LOCK_ASSERT(rnh); rt = (struct rtentry *)rnh->rnh_lookup(dst, netmask, &rnh->head); if (rt != NULL) { rnd->rnd_nhop = rt->rt_nhop; rnd->rnd_weight = rt->rt_weight; } else { rnd->rnd_nhop = NULL; rnd->rnd_weight = 0; } return (rt); } struct rtentry * lookup_prefix_rt(struct rib_head *rnh, const struct rtentry *rt, struct route_nhop_data *rnd) { return (lookup_prefix_bysa(rnh, rt_key_const(rt), rt_mask_const(rt), rnd)); } /* * Runs exact prefix match based on dst/netmask from @info. * Assumes RIB lock is held. * Returns matched @rtentry if found or NULL. * If rtentry was found, saves nexthop / weight value into @rnd. */ struct rtentry * lookup_prefix(struct rib_head *rnh, const struct rt_addrinfo *info, struct route_nhop_data *rnd) { struct rtentry *rt; rt = lookup_prefix_bysa(rnh, info->rti_info[RTAX_DST], info->rti_info[RTAX_NETMASK], rnd); return (rt); } static bool fill_pxmask_family(int family, int plen, struct sockaddr *_dst, struct sockaddr **pmask) { if (plen == -1) { *pmask = NULL; return (true); } switch (family) { #ifdef INET case AF_INET: { struct sockaddr_in *mask = (struct sockaddr_in *)(*pmask); struct sockaddr_in *dst= (struct sockaddr_in *)_dst; memset(mask, 0, sizeof(*mask)); mask->sin_family = family; mask->sin_len = sizeof(*mask); if (plen == 32) *pmask = NULL; else if (plen > 32 || plen < 0) return (false); else { uint32_t daddr, maddr; maddr = htonl(plen ? ~((1 << (32 - plen)) - 1) : 0); mask->sin_addr.s_addr = maddr; daddr = dst->sin_addr.s_addr; daddr = htonl(ntohl(daddr) & ntohl(maddr)); dst->sin_addr.s_addr = daddr; } return (true); } break; #endif #ifdef INET6 case AF_INET6: { struct sockaddr_in6 *mask = (struct sockaddr_in6 *)(*pmask); struct sockaddr_in6 *dst = (struct sockaddr_in6 *)_dst; memset(mask, 0, sizeof(*mask)); mask->sin6_family = family; mask->sin6_len = sizeof(*mask); if (plen == 128) *pmask = NULL; else if (plen > 128 || plen < 0) return (false); else { ip6_writemask(&mask->sin6_addr, plen); IN6_MASK_ADDR(&dst->sin6_addr, &mask->sin6_addr); } return (true); } break; #endif } return (false); } /* * Attempts to add @dst/plen prefix with nexthop/nexhopgroup data @rnd * to the routing table. * * @fibnum: rtable id to insert route to * @dst: verified kernel-originated sockaddr, can be masked if plen non-empty * @plen: prefix length (or -1 if host route or not applicable for AF) * @op_flags: combination of RTM_F_ flags * @rc: storage to report operation result * * Returns 0 on success. */ int rib_add_route_px(uint32_t fibnum, struct sockaddr *dst, int plen, struct route_nhop_data *rnd, int op_flags, struct rib_cmd_info *rc) { union sockaddr_union mask_storage; struct sockaddr *netmask = &mask_storage.sa; struct rtentry *rt = NULL; NET_EPOCH_ASSERT(); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_ADD; struct rib_head *rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) return (EAFNOSUPPORT); if (!fill_pxmask_family(dst->sa_family, plen, dst, &netmask)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: invalid plen %d", plen); return (EINVAL); } if (op_flags & RTM_F_CREATE) { if ((rt = rt_alloc(rnh, dst, netmask)) == NULL) { FIB_RH_LOG(LOG_INFO, rnh, "rtentry allocation failed"); return (ENOMEM); } } return (add_route_flags(rnh, rt, rnd, op_flags, rc)); } /* * Attempts to delete @dst/plen prefix matching gateway @gw from the * routing rable. * * @fibnum: rtable id to remove route from * @dst: verified kernel-originated sockaddr, can be masked if plen non-empty * @plen: prefix length (or -1 if host route or not applicable for AF) * @gw: gateway to match * @op_flags: combination of RTM_F_ flags * @rc: storage to report operation result * * Returns 0 on success. */ int rib_del_route_px_gw(uint32_t fibnum, struct sockaddr *dst, int plen, const struct sockaddr *gw, int op_flags, struct rib_cmd_info *rc) { struct gw_filter_data gwd = { .gw = gw }; return (rib_del_route_px(fibnum, dst, plen, match_gw_one, &gwd, op_flags, rc)); } /* * Attempts to delete @dst/plen prefix matching @filter_func from the * routing rable. * * @fibnum: rtable id to remove route from * @dst: verified kernel-originated sockaddr, can be masked if plen non-empty * @plen: prefix length (or -1 if host route or not applicable for AF) * @filter_func: func to be called for each nexthop of the prefix for matching * @filter_arg: argument to pass to @filter_func * @op_flags: combination of RTM_F_ flags * @rc: storage to report operation result * * Returns 0 on success. */ int rib_del_route_px(uint32_t fibnum, struct sockaddr *dst, int plen, rib_filter_f_t *filter_func, void *filter_arg, int op_flags, struct rib_cmd_info *rc) { union sockaddr_union mask_storage; struct sockaddr *netmask = &mask_storage.sa; int error; NET_EPOCH_ASSERT(); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_DELETE; struct rib_head *rnh = rt_tables_get_rnh(fibnum, dst->sa_family); if (rnh == NULL) return (EAFNOSUPPORT); if (dst->sa_len > sizeof(mask_storage)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: dst->sa_len too big: %d", dst->sa_len); return (EINVAL); } if (!fill_pxmask_family(dst->sa_family, plen, dst, &netmask)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: invalid plen %d", plen); return (EINVAL); } int prio = (op_flags & RTM_F_FORCE) ? NH_PRIORITY_HIGH : NH_PRIORITY_NORMAL; RIB_WLOCK(rnh); struct route_nhop_data rnd; struct rtentry *rt = lookup_prefix_bysa(rnh, dst, netmask, &rnd); if (rt != NULL) { error = rt_delete_conditional(rnh, rt, prio, filter_func, filter_arg, rc); } else error = ESRCH; RIB_WUNLOCK(rnh); if (error != 0) return (error); rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); if (rc->rc_cmd == RTM_DELETE) rt_free(rc->rc_rt); #ifdef ROUTE_MPATH else { /* * Deleting 1 path may result in RTM_CHANGE to * a different mpath group/nhop. * Free old mpath group. */ nhop_free_any(rc->rc_nh_old); } #endif return (0); } /* * Tries to copy route @rt from one rtable to the rtable specified by @dst_rh. * @rt: route to copy. * @rnd_src: nhop and weight. Multipath routes are not supported * @rh_dst: target rtable. * @rc: operation result storage * * Return 0 on success. */ int rib_copy_route(struct rtentry *rt, const struct route_nhop_data *rnd_src, struct rib_head *rh_dst, struct rib_cmd_info *rc) { struct nhop_object __diagused *nh_src = rnd_src->rnd_nhop; int error; MPASS((nh_src->nh_flags & NHF_MULTIPATH) == 0); IF_DEBUG_LEVEL(LOG_DEBUG2) { char nhbuf[NHOP_PRINT_BUFSIZE], rtbuf[NHOP_PRINT_BUFSIZE]; nhop_print_buf_any(nh_src, nhbuf, sizeof(nhbuf)); rt_print_buf(rt, rtbuf, sizeof(rtbuf)); FIB_RH_LOG(LOG_DEBUG2, rh_dst, "copying %s -> %s from fib %u", rtbuf, nhbuf, nhop_get_fibnum(nh_src)); } struct nhop_object *nh = nhop_alloc(rh_dst->rib_fibnum, rh_dst->rib_family); if (nh == NULL) { FIB_RH_LOG(LOG_INFO, rh_dst, "unable to allocate new nexthop"); return (ENOMEM); } nhop_copy(nh, rnd_src->rnd_nhop); + nhop_set_origin(nh, nhop_get_origin(rnd_src->rnd_nhop)); nhop_set_fibnum(nh, rh_dst->rib_fibnum); nh = nhop_get_nhop_internal(rh_dst, nh, &error); if (error != 0) { FIB_RH_LOG(LOG_INFO, rh_dst, "unable to finalize new nexthop: error %d", error); return (ENOMEM); } struct rtentry *rt_new = rt_alloc(rh_dst, rt_key(rt), rt_mask(rt)); if (rt_new == NULL) { FIB_RH_LOG(LOG_INFO, rh_dst, "unable to create new rtentry"); nhop_free(nh); return (ENOMEM); } struct route_nhop_data rnd = { .rnd_nhop = nh, .rnd_weight = rnd_src->rnd_weight }; int op_flags = RTM_F_CREATE | (NH_IS_PINNED(nh) ? RTM_F_FORCE : 0); error = add_route_flags(rh_dst, rt_new, &rnd, op_flags, rc); if (error != 0) { IF_DEBUG_LEVEL(LOG_DEBUG2) { char buf[NHOP_PRINT_BUFSIZE]; rt_print_buf(rt_new, buf, sizeof(buf)); FIB_RH_LOG(LOG_DEBUG, rh_dst, "Unable to add route %s: error %d", buf, error); } nhop_free(nh); rt_free_immediate(rt_new); } return (error); } /* * Adds route defined by @info into the kernel table specified by @fibnum and * sa_family in @info->rti_info[RTAX_DST]. * * Returns 0 on success and fills in operation metadata into @rc. */ int rib_add_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct rib_head *rnh; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); /* * Check consistency between RTF_HOST flag and netmask * existence. */ if (info->rti_flags & RTF_HOST) info->rti_info[RTAX_NETMASK] = NULL; else if (info->rti_info[RTAX_NETMASK] == NULL) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: no RTF_HOST and empty netmask"); return (EINVAL); } bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_ADD; error = add_route_byinfo(rnh, info, rc); if (error == 0) rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); return (error); } static int add_route_byinfo(struct rib_head *rnh, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct route_nhop_data rnd_add; struct nhop_object *nh; struct rtentry *rt; struct sockaddr *dst, *gateway, *netmask; int error; dst = info->rti_info[RTAX_DST]; gateway = info->rti_info[RTAX_GATEWAY]; netmask = info->rti_info[RTAX_NETMASK]; if ((info->rti_flags & RTF_GATEWAY) && !gateway) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: RTF_GATEWAY set with empty gw"); return (EINVAL); } if (dst && gateway && !nhop_check_gateway(dst->sa_family, gateway->sa_family)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: invalid dst/gateway family combination (%d, %d)", dst->sa_family, gateway->sa_family); return (EINVAL); } if (dst->sa_len > sizeof(((struct rtentry *)NULL)->rt_dstb)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: dst->sa_len too large: %d", dst->sa_len); return (EINVAL); } if (info->rti_ifa == NULL) { error = rt_getifa_fib(info, rnh->rib_fibnum); if (error) return (error); } if ((rt = rt_alloc(rnh, dst, netmask)) == NULL) return (ENOBUFS); error = nhop_create_from_info(rnh, info, &nh); if (error != 0) { rt_free_immediate(rt); return (error); } rnd_add.rnd_nhop = nh; rnd_add.rnd_weight = get_info_weight(info, RT_DEFAULT_WEIGHT); int op_flags = RTM_F_CREATE; if (get_prio_from_info(info) == NH_PRIORITY_HIGH) op_flags |= RTM_F_FORCE; else op_flags |= RTM_F_APPEND; return (add_route_flags(rnh, rt, &rnd_add, op_flags, rc)); } static int add_route_flags(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_add, int op_flags, struct rib_cmd_info *rc) { struct route_nhop_data rnd_orig; struct nhop_object *nh; struct rtentry *rt_orig; int error = 0; nh = rnd_add->rnd_nhop; RIB_WLOCK(rnh); rt_orig = lookup_prefix_rt(rnh, rt, &rnd_orig); if (rt_orig == NULL) { if (op_flags & RTM_F_CREATE) error = add_route(rnh, rt, rnd_add, rc); else error = ESRCH; /* no entry but creation was not required */ RIB_WUNLOCK(rnh); if (error != 0) goto out; return (0); } if (op_flags & RTM_F_EXCL) { /* We have existing route in the RIB but not allowed to replace. */ RIB_WUNLOCK(rnh); error = EEXIST; goto out; } /* Now either append or replace */ if (op_flags & RTM_F_REPLACE) { if (nhop_get_prio(rnd_orig.rnd_nhop) > nhop_get_prio(rnd_add->rnd_nhop)) { /* Old path is "better" (e.g. has PINNED flag set) */ error = EEXIST; goto out; } change_route(rnh, rt_orig, rnd_add, rc); RIB_WUNLOCK(rnh); nh = rc->rc_nh_old; goto out; } RIB_WUNLOCK(rnh); #ifdef ROUTE_MPATH if ((op_flags & RTM_F_APPEND) && rib_can_multipath(rnh) && nhop_can_multipath(rnd_add->rnd_nhop) && nhop_can_multipath(rnd_orig.rnd_nhop)) { for (int i = 0; i < RIB_MAX_RETRIES; i++) { error = add_route_flags_mpath(rnh, rt_orig, rnd_add, &rnd_orig, op_flags, rc); if (error != EAGAIN) break; RTSTAT_INC(rts_add_retry); } /* * Original nhop reference is unused in any case. */ nhop_free_any(rnd_add->rnd_nhop); if (op_flags & RTM_F_CREATE) { if (error != 0 || rc->rc_cmd != RTM_ADD) rt_free_immediate(rt); } return (error); } #endif /* Out of options - free state and return error */ error = EEXIST; out: if (op_flags & RTM_F_CREATE) rt_free_immediate(rt); nhop_free_any(nh); return (error); } #ifdef ROUTE_MPATH static int add_route_flags_mpath(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_add, struct route_nhop_data *rnd_orig, int op_flags, struct rib_cmd_info *rc) { RIB_RLOCK_TRACKER; struct route_nhop_data rnd_new; int error = 0; error = nhgrp_get_addition_group(rnh, rnd_orig, rnd_add, &rnd_new); if (error != 0) { if (error == EAGAIN) { /* * Group creation failed, most probably because * @rnd_orig data got scheduled for deletion. * Refresh @rnd_orig data and retry. */ RIB_RLOCK(rnh); lookup_prefix_rt(rnh, rt, rnd_orig); RIB_RUNLOCK(rnh); if (rnd_orig == NULL && !(op_flags & RTM_F_CREATE)) { /* In this iteration route doesn't exist */ error = ENOENT; } } return (error); } error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); if (error != 0) return (error); if (V_fib_hash_outbound == 0 && NH_IS_NHGRP(rc->rc_nh_new)) { /* * First multipath route got installed. Enable local * outbound connections hashing. */ if (bootverbose) printf("FIB: enabled flowid calculation for locally-originated packets\n"); V_fib_hash_outbound = 1; } return (0); } #endif /* * Removes route defined by @info from the kernel table specified by @fibnum and * sa_family in @info->rti_info[RTAX_DST]. * * Returns 0 on success and fills in operation metadata into @rc. */ int rib_del_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { struct rib_head *rnh; struct sockaddr *dst, *netmask; struct sockaddr_storage mdst; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_DELETE; dst = info->rti_info[RTAX_DST]; netmask = info->rti_info[RTAX_NETMASK]; if (netmask != NULL) { /* Ensure @dst is always properly masked */ if (dst->sa_len > sizeof(mdst)) { FIB_RH_LOG(LOG_DEBUG, rnh, "error: dst->sa_len too large"); return (EINVAL); } rt_maskedcopy(dst, (struct sockaddr *)&mdst, netmask); dst = (struct sockaddr *)&mdst; } rib_filter_f_t *filter_func = NULL; void *filter_arg = NULL; struct gw_filter_data gwd = { .gw = info->rti_info[RTAX_GATEWAY] }; if (info->rti_filter != NULL) { filter_func = info->rti_filter; filter_arg = info->rti_filterdata; } else if (gwd.gw != NULL) { filter_func = match_gw_one; filter_arg = &gwd; } int prio = get_prio_from_info(info); RIB_WLOCK(rnh); struct route_nhop_data rnd; struct rtentry *rt = lookup_prefix_bysa(rnh, dst, netmask, &rnd); if (rt != NULL) { error = rt_delete_conditional(rnh, rt, prio, filter_func, filter_arg, rc); } else error = ESRCH; RIB_WUNLOCK(rnh); if (error != 0) return (error); rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); if (rc->rc_cmd == RTM_DELETE) rt_free(rc->rc_rt); #ifdef ROUTE_MPATH else { /* * Deleting 1 path may result in RTM_CHANGE to * a different mpath group/nhop. * Free old mpath group. */ nhop_free_any(rc->rc_nh_old); } #endif return (0); } /* * Conditionally unlinks rtentry paths from @rnh matching @cb. * Returns 0 on success with operation result stored in @rc. * On error, returns: * ESRCH - if prefix was not found or filter function failed to match * EADDRINUSE - if trying to delete higher priority route. */ static int rt_delete_conditional(struct rib_head *rnh, struct rtentry *rt, int prio, rib_filter_f_t *cb, void *cbdata, struct rib_cmd_info *rc) { struct nhop_object *nh = rt->rt_nhop; #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh)) { struct nhgrp_object *nhg = (struct nhgrp_object *)nh; struct route_nhop_data rnd; int error; if (cb == NULL) return (ESRCH); error = nhgrp_get_filtered_group(rnh, rt, nhg, cb, cbdata, &rnd); if (error == 0) { if (rnd.rnd_nhgrp == nhg) { /* No match, unreference new group and return. */ nhop_free_any(rnd.rnd_nhop); return (ESRCH); } error = change_route(rnh, rt, &rnd, rc); } return (error); } #endif if (cb != NULL && !cb(rt, nh, cbdata)) return (ESRCH); if (prio < nhop_get_prio(nh)) return (EADDRINUSE); return (delete_route(rnh, rt, rc)); } int rib_change_route(uint32_t fibnum, struct rt_addrinfo *info, struct rib_cmd_info *rc) { RIB_RLOCK_TRACKER; struct route_nhop_data rnd_orig; struct rib_head *rnh; struct rtentry *rt; int error; NET_EPOCH_ASSERT(); rnh = get_rnh(fibnum, info); if (rnh == NULL) return (EAFNOSUPPORT); bzero(rc, sizeof(struct rib_cmd_info)); rc->rc_cmd = RTM_CHANGE; /* Check if updated gateway exists */ if ((info->rti_flags & RTF_GATEWAY) && (info->rti_info[RTAX_GATEWAY] == NULL)) { /* * route(8) adds RTF_GATEWAY flag if -interface is not set. * Remove RTF_GATEWAY to enforce consistency and maintain * compatibility.. */ info->rti_flags &= ~RTF_GATEWAY; } /* * route change is done in multiple steps, with dropping and * reacquiring lock. In the situations with multiple processes * changes the same route in can lead to the case when route * is changed between the steps. Address it by retrying the operation * multiple times before failing. */ RIB_RLOCK(rnh); rt = (struct rtentry *)rnh->rnh_lookup(info->rti_info[RTAX_DST], info->rti_info[RTAX_NETMASK], &rnh->head); if (rt == NULL) { RIB_RUNLOCK(rnh); return (ESRCH); } rnd_orig.rnd_nhop = rt->rt_nhop; rnd_orig.rnd_weight = rt->rt_weight; RIB_RUNLOCK(rnh); for (int i = 0; i < RIB_MAX_RETRIES; i++) { error = change_route_byinfo(rnh, rt, info, &rnd_orig, rc); if (error != EAGAIN) break; } return (error); } static int change_nhop(struct rib_head *rnh, struct rt_addrinfo *info, struct nhop_object *nh_orig, struct nhop_object **nh_new) { int error; /* * New gateway could require new ifaddr, ifp; * flags may also be different; ifp may be specified * by ll sockaddr when protocol address is ambiguous */ if (((nh_orig->nh_flags & NHF_GATEWAY) && info->rti_info[RTAX_GATEWAY] != NULL) || info->rti_info[RTAX_IFP] != NULL || (info->rti_info[RTAX_IFA] != NULL && !sa_equal(info->rti_info[RTAX_IFA], nh_orig->nh_ifa->ifa_addr))) { error = rt_getifa_fib(info, rnh->rib_fibnum); if (error != 0) { info->rti_ifa = NULL; return (error); } } error = nhop_create_from_nhop(rnh, nh_orig, info, nh_new); info->rti_ifa = NULL; return (error); } #ifdef ROUTE_MPATH static int change_mpath_route(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc) { int error = 0, found_idx = 0; struct nhop_object *nh_orig = NULL, *nh_new; struct route_nhop_data rnd_new = {}; const struct weightened_nhop *wn = NULL; struct weightened_nhop *wn_new; uint32_t num_nhops; wn = nhgrp_get_nhops(rnd_orig->rnd_nhgrp, &num_nhops); for (int i = 0; i < num_nhops; i++) { if (check_info_match_nhop(info, NULL, wn[i].nh) == 0) { nh_orig = wn[i].nh; found_idx = i; break; } } if (nh_orig == NULL) return (ESRCH); error = change_nhop(rnh, info, nh_orig, &nh_new); if (error != 0) return (error); wn_new = mallocarray(num_nhops, sizeof(struct weightened_nhop), M_TEMP, M_NOWAIT | M_ZERO); if (wn_new == NULL) { nhop_free(nh_new); return (EAGAIN); } memcpy(wn_new, wn, num_nhops * sizeof(struct weightened_nhop)); wn_new[found_idx].nh = nh_new; wn_new[found_idx].weight = get_info_weight(info, wn[found_idx].weight); error = nhgrp_get_group(rnh, wn_new, num_nhops, 0, &rnd_new.rnd_nhgrp); nhop_free(nh_new); free(wn_new, M_TEMP); if (error != 0) return (error); error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); return (error); } #endif static int change_route_byinfo(struct rib_head *rnh, struct rtentry *rt, struct rt_addrinfo *info, struct route_nhop_data *rnd_orig, struct rib_cmd_info *rc) { int error = 0; struct nhop_object *nh_orig; struct route_nhop_data rnd_new; nh_orig = rnd_orig->rnd_nhop; if (nh_orig == NULL) return (ESRCH); #ifdef ROUTE_MPATH if (NH_IS_NHGRP(nh_orig)) return (change_mpath_route(rnh, rt, info, rnd_orig, rc)); #endif rnd_new.rnd_weight = get_info_weight(info, rnd_orig->rnd_weight); error = change_nhop(rnh, info, nh_orig, &rnd_new.rnd_nhop); if (error != 0) return (error); error = change_route_conditional(rnh, rt, rnd_orig, &rnd_new, rc); return (error); } /* * Insert @rt with nhop data from @rnd_new to @rnh. * Returns 0 on success and stores operation results in @rc. */ static int add_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc) { struct radix_node *rn; RIB_WLOCK_ASSERT(rnh); rt->rt_nhop = rnd->rnd_nhop; rt->rt_weight = rnd->rnd_weight; rn = rnh->rnh_addaddr(rt_key(rt), rt_mask_const(rt), &rnh->head, rt->rt_nodes); if (rn != NULL) { if (!NH_IS_NHGRP(rnd->rnd_nhop) && nhop_get_expire(rnd->rnd_nhop)) tmproutes_update(rnh, rt, rnd->rnd_nhop); /* Finalize notification */ rib_bump_gen(rnh); rnh->rnh_prefixes++; rc->rc_cmd = RTM_ADD; rc->rc_rt = rt; rc->rc_nh_old = NULL; rc->rc_nh_new = rnd->rnd_nhop; rc->rc_nh_weight = rnd->rnd_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* Existing route or memory allocation failure. */ return (EEXIST); } /* * Unconditionally deletes @rt from @rnh. */ static int delete_route(struct rib_head *rnh, struct rtentry *rt, struct rib_cmd_info *rc) { RIB_WLOCK_ASSERT(rnh); /* Route deletion requested. */ struct radix_node *rn; rn = rnh->rnh_deladdr(rt_key_const(rt), rt_mask_const(rt), &rnh->head); if (rn == NULL) return (ESRCH); rt = RNTORT(rn); rt->rte_flags &= ~RTF_UP; rib_bump_gen(rnh); rnh->rnh_prefixes--; rc->rc_cmd = RTM_DELETE; rc->rc_rt = rt; rc->rc_nh_old = rt->rt_nhop; rc->rc_nh_new = NULL; rc->rc_nh_weight = rt->rt_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* * Switch @rt nhop/weigh to the ones specified in @rnd. * Returns 0 on success. */ int change_route(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd, struct rib_cmd_info *rc) { struct nhop_object *nh_orig; RIB_WLOCK_ASSERT(rnh); nh_orig = rt->rt_nhop; if (rnd->rnd_nhop == NULL) return (delete_route(rnh, rt, rc)); /* Changing nexthop & weight to a new one */ rt->rt_nhop = rnd->rnd_nhop; rt->rt_weight = rnd->rnd_weight; if (!NH_IS_NHGRP(rnd->rnd_nhop) && nhop_get_expire(rnd->rnd_nhop)) tmproutes_update(rnh, rt, rnd->rnd_nhop); /* Finalize notification */ rib_bump_gen(rnh); rc->rc_cmd = RTM_CHANGE; rc->rc_rt = rt; rc->rc_nh_old = nh_orig; rc->rc_nh_new = rnd->rnd_nhop; rc->rc_nh_weight = rnd->rnd_weight; rib_notify(rnh, RIB_NOTIFY_IMMEDIATE, rc); return (0); } /* * Conditionally update route nhop/weight IFF data in @nhd_orig is * consistent with the current route data. * Nexthop in @nhd_new is consumed. */ int change_route_conditional(struct rib_head *rnh, struct rtentry *rt, struct route_nhop_data *rnd_orig, struct route_nhop_data *rnd_new, struct rib_cmd_info *rc) { struct rtentry *rt_new; int error = 0; IF_DEBUG_LEVEL(LOG_DEBUG2) { char buf_old[NHOP_PRINT_BUFSIZE], buf_new[NHOP_PRINT_BUFSIZE]; nhop_print_buf_any(rnd_orig->rnd_nhop, buf_old, NHOP_PRINT_BUFSIZE); nhop_print_buf_any(rnd_new->rnd_nhop, buf_new, NHOP_PRINT_BUFSIZE); FIB_LOG(LOG_DEBUG2, rnh->rib_fibnum, rnh->rib_family, "trying change %s -> %s", buf_old, buf_new); } RIB_WLOCK(rnh); struct route_nhop_data rnd; rt_new = lookup_prefix_rt(rnh, rt, &rnd); if (rt_new == NULL) { if (rnd_orig->rnd_nhop == NULL) error = add_route(rnh, rt, rnd_new, rc); else { /* * Prefix does not exist, which was not our assumption. * Update @rnd_orig with the new data and return */ rnd_orig->rnd_nhop = NULL; rnd_orig->rnd_weight = 0; error = EAGAIN; } } else { /* Prefix exists, try to update */ if (rnd_orig->rnd_nhop == rt_new->rt_nhop) { /* * Nhop/mpath group hasn't changed. Flip * to the new precalculated one and return */ error = change_route(rnh, rt_new, rnd_new, rc); } else { /* Update and retry */ rnd_orig->rnd_nhop = rt_new->rt_nhop; rnd_orig->rnd_weight = rt_new->rt_weight; error = EAGAIN; } } RIB_WUNLOCK(rnh); if (error == 0) { rib_notify(rnh, RIB_NOTIFY_DELAYED, rc); if (rnd_orig->rnd_nhop != NULL) nhop_free_any(rnd_orig->rnd_nhop); } else { if (rnd_new->rnd_nhop != NULL) nhop_free_any(rnd_new->rnd_nhop); } return (error); } /* * Performs modification of routing table specificed by @action. * Table is specified by @fibnum and sa_family in @info->rti_info[RTAX_DST]. * Needs to be run in network epoch. * * Returns 0 on success and fills in @rc with action result. */ int rib_action(uint32_t fibnum, int action, struct rt_addrinfo *info, struct rib_cmd_info *rc) { int error; switch (action) { case RTM_ADD: error = rib_add_route(fibnum, info, rc); break; case RTM_DELETE: error = rib_del_route(fibnum, info, rc); break; case RTM_CHANGE: error = rib_change_route(fibnum, info, rc); break; default: error = ENOTSUP; } return (error); } struct rt_delinfo { struct rib_head *rnh; struct rtentry *head; rib_filter_f_t *filter_f; void *filter_arg; int prio; struct rib_cmd_info rc; }; /* * Conditionally unlinks rtenties or paths from radix tree based * on the callback data passed in @arg. */ static int rt_checkdelroute(struct radix_node *rn, void *arg) { struct rt_delinfo *di = (struct rt_delinfo *)arg; struct rtentry *rt = (struct rtentry *)rn; if (rt_delete_conditional(di->rnh, rt, di->prio, di->filter_f, di->filter_arg, &di->rc) != 0) return (0); /* * Add deleted rtentries to the list to GC them * after dropping the lock. * * XXX: Delayed notifications not implemented * for nexthop updates. */ if (di->rc.rc_cmd == RTM_DELETE) { /* Add to the list and return */ rt->rt_chain = di->head; di->head = rt; #ifdef ROUTE_MPATH } else { /* * RTM_CHANGE to a different nexthop or nexthop group. * Free old multipath group. */ nhop_free_any(di->rc.rc_nh_old); #endif } return (0); } /* * Iterates over a routing table specified by @fibnum and @family and * deletes elements marked by @filter_f. * @fibnum: rtable id * @family: AF_ address family * @filter_f: function returning non-zero value for items to delete * @arg: data to pass to the @filter_f function * @report: true if rtsock notification is needed. */ void rib_walk_del(u_int fibnum, int family, rib_filter_f_t *filter_f, void *filter_arg, bool report) { struct rib_head *rnh; struct rtentry *rt; struct nhop_object *nh; struct epoch_tracker et; rnh = rt_tables_get_rnh(fibnum, family); if (rnh == NULL) return; struct rt_delinfo di = { .rnh = rnh, .filter_f = filter_f, .filter_arg = filter_arg, .prio = NH_PRIORITY_NORMAL, }; NET_EPOCH_ENTER(et); RIB_WLOCK(rnh); rnh->rnh_walktree(&rnh->head, rt_checkdelroute, &di); RIB_WUNLOCK(rnh); /* We might have something to reclaim. */ bzero(&di.rc, sizeof(di.rc)); di.rc.rc_cmd = RTM_DELETE; while (di.head != NULL) { rt = di.head; di.head = rt->rt_chain; rt->rt_chain = NULL; nh = rt->rt_nhop; di.rc.rc_rt = rt; di.rc.rc_nh_old = nh; rib_notify(rnh, RIB_NOTIFY_DELAYED, &di.rc); if (report) { #ifdef ROUTE_MPATH struct nhgrp_object *nhg; const struct weightened_nhop *wn; uint32_t num_nhops; if (NH_IS_NHGRP(nh)) { nhg = (struct nhgrp_object *)nh; wn = nhgrp_get_nhops(nhg, &num_nhops); for (int i = 0; i < num_nhops; i++) rt_routemsg(RTM_DELETE, rt, wn[i].nh, fibnum); } else #endif rt_routemsg(RTM_DELETE, rt, nh, fibnum); } rt_free(rt); } NET_EPOCH_EXIT(et); } static int rt_delete_unconditional(struct radix_node *rn, void *arg) { struct rtentry *rt = RNTORT(rn); struct rib_head *rnh = (struct rib_head *)arg; rn = rnh->rnh_deladdr(rt_key(rt), rt_mask(rt), &rnh->head); if (RNTORT(rn) == rt) rt_free(rt); return (0); } /* * Removes all routes from the routing table without executing notifications. * rtentres will be removed after the end of a current epoch. */ static void rib_flush_routes(struct rib_head *rnh) { RIB_WLOCK(rnh); rnh->rnh_walktree(&rnh->head, rt_delete_unconditional, rnh); RIB_WUNLOCK(rnh); } void rib_flush_routes_family(int family) { struct rib_head *rnh; for (uint32_t fibnum = 0; fibnum < rt_numfibs; fibnum++) { if ((rnh = rt_tables_get_rnh(fibnum, family)) != NULL) rib_flush_routes(rnh); } } const char * rib_print_family(int family) { switch (family) { case AF_INET: return ("inet"); case AF_INET6: return ("inet6"); case AF_LINK: return ("link"); } return ("unknown"); }