D34340.id105429.diff
No OneTemporary
Actions

Size

58 KB

Referenced Files

None

Subscribers

None

D34340.id105429.diff
View Options

	Index: share/man/man4/Makefile
	===================================================================
	--- share/man/man4/Makefile
	+++ share/man/man4/Makefile
	@@ -418,6 +418,7 @@
	ow.4 \
	ow_temp.4 \
	owc.4 \
	+ ovpn.4 \
	${_padlock.4} \
	pass.4 \
	pca954x.4 \
	Index: share/man/man4/ovpn.4
	===================================================================
	--- /dev/null
	+++ share/man/man4/ovpn.4
	@@ -0,0 +1,54 @@
	+.\" Copyright (c) 2022 Rubicon Communications, LLC ("Netgate")
	+.\"
	+.\" Redistribution and use in source and binary forms, with or without
	+.\" modification, are permitted provided that the following conditions
	+.\" are met:
	+.\" 1. Redistributions of source code must retain the above copyright
	+.\" notice, this list of conditions and the following disclaimer.
	+.\" 2. Redistributions in binary form must reproduce the above copyright
	+.\" notice, this list of conditions and the following disclaimer in the
	+.\" documentation and/or other materials provided with the distribution.
	+.\"
	+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+.\" SUCH DAMAGE.
	+.\"
	+.Dd April 22, 2022
	+.Dt OVPN 4
	+.Os
	+.Sh NAME
	+.Nm ovpn
	+.Nd OpenVPN DCO driver
	+.Sh SYNOPSIS
	+To compile this driver into the kernel,
	+place the following lines in your
	+kernel configuration file:
	+.Bd -ragged -offset indent
	+.Cd "device ovpn"
	+.Ed
	+.Pp
	+Alternatively, to load the driver as a
	+module at boot time, place the following line in
	+.Xr loader.conf 5 :
	+.Bd -literal -offset indent
	+if_ovpn_load="YES"
	+.Ed
	+.Sh DESCRIPTION
	+The
	+.Nm
	+device driver provides support for OpenVPN DCO.
	+DCO, or Data Channel Offload, moves the OpenVPN data path into the kernel.
	+This can improve performance.
	+.Pp
	+The
	+.Nm
	+interface is created automatically by the OpenVPN daemon.
	+It requires no configuration other than that done by OpenVPN.
	Index: sys/conf/files
	===================================================================
	--- sys/conf/files
	+++ sys/conf/files
	@@ -4140,6 +4140,7 @@
	net/if_me.c optional me inet
	net/if_media.c standard
	net/if_mib.c standard
	+net/if_ovpn.c optional ovpn inet \| ovpn inet6
	net/if_stf.c optional stf inet inet6
	net/if_tuntap.c optional tuntap
	net/if_vlan.c optional vlan
	Index: sys/kern/kern_jail.c
	===================================================================
	--- sys/kern/kern_jail.c
	+++ sys/kern/kern_jail.c
	@@ -3675,6 +3675,7 @@
	case PRIV_NET_GIF:
	case PRIV_NET_SETIFVNET:
	case PRIV_NET_SETIFFIB:
	+ case PRIV_NET_OVPN:

	/*
	* 802.11-related privileges.
	Index: sys/modules/Makefile
	===================================================================
	--- sys/modules/Makefile
	+++ sys/modules/Makefile
	@@ -157,6 +157,7 @@
	${_if_me} \
	if_infiniband \
	if_lagg \
	+ if_ovpn \
	${_if_stf} \
	if_tuntap \
	if_vlan \
	Index: sys/modules/if_ovpn/Makefile
	===================================================================
	--- /dev/null
	+++ sys/modules/if_ovpn/Makefile
	@@ -0,0 +1,6 @@
	+.PATH: ${SRCTOP}/sys/net
	+
	+KMOD= if_ovpn
	+SRCS= if_ovpn.c opt_inet.h opt_inet6.h
	+
	+.include <bsd.kmod.mk>
	Index: sys/net/if_ovpn.h
	===================================================================
	--- /dev/null
	+++ sys/net/if_ovpn.h
	@@ -0,0 +1,64 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2021 Rubicon Communications, LLC (Netgate)
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ */
	+
	+#ifndef _NET_IF_OVPN_H_
	+#define _NET_IF_OVPN_H_
	+
	+#include <sys/types.h>
	+#include <netinet/in.h>
	+
	+/* Maximum size of an ioctl request. */
	+#define OVPN_MAX_REQUEST_SIZE 4096
	+
	+enum ovpn_notif_type {
	+ OVPN_NOTIF_DEL_PEER,
	+};
	+
	+enum ovpn_key_slot {
	+ OVPN_KEY_SLOT_PRIMARY = 0,
	+ OVPN_KEY_SLOT_SECONDARY = 1
	+};
	+
	+enum ovpn_key_cipher {
	+ OVPN_CIPHER_ALG_NONE = 0,
	+ OVPN_CIPHER_ALG_AES_GCM = 1,
	+ OVPN_CIPHER_ALG_CHACHA20_POLY1305 = 2
	+};
	+
	+#define OVPN_NEW_PEER _IO ('D', 1)
	+#define OVPN_DEL_PEER _IO ('D', 2)
	+#define OVPN_GET_STATS _IO ('D', 3)
	+#define OVPN_NEW_KEY _IO ('D', 4)
	+#define OVPN_SWAP_KEYS _IO ('D', 5)
	+#define OVPN_DEL_KEY _IO ('D', 6)
	+#define OVPN_SET_PEER _IO ('D', 7)
	+#define OVPN_START_VPN _IO ('D', 8)
	+#define OVPN_SEND_PKT _IO ('D', 9)
	+#define OVPN_POLL_PKT _IO ('D', 10)
	+#define OVPN_GET_PKT _IO ('D', 11)
	+
	+#endif
	Index: sys/net/if_ovpn.c
	===================================================================
	--- /dev/null
	+++ sys/net/if_ovpn.c
	@@ -0,0 +1,2273 @@
	+/*-
	+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	+ *
	+ * Copyright (c) 2021 Rubicon Communications, LLC (Netgate)
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ */
	+#include "opt_inet.h"
	+#include "opt_inet6.h"
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/buf_ring.h>
	+#include <sys/epoch.h>
	+#include <sys/file.h>
	+#include <sys/filedesc.h>
	+#include <sys/kernel.h>
	+#include <sys/malloc.h>
	+#include <sys/mbuf.h>
	+#include <sys/module.h>
	+#include <sys/nv.h>
	+#include <sys/priv.h>
	+#include <sys/socket.h>
	+#include <sys/socketvar.h>
	+#include <sys/sockio.h>
	+#include <sys/sysctl.h>
	+#include <sys/protosw.h>
	+#include <sys/rmlock.h>
	+
	+#include <machine/atomic.h>
	+
	+#include <net/bpf.h>
	+#include <net/if.h>
	+#include <net/if_clone.h>
	+#include <net/if_types.h>
	+#include <net/if_var.h>
	+#include <net/netisr.h>
	+#include <net/route/nhop.h>
	+
	+#include <netinet/in.h>
	+#include <netinet/in_fib.h>
	+#include <netinet/ip.h>
	+#include <netinet/ip6.h>
	+#include <netinet/ip_var.h>
	+#include <netinet/udp.h>
	+#include <netinet/udp_var.h>
	+
	+#include <netinet6/ip6_var.h>
	+#include <netinet6/in6_fib.h>
	+
	+#include <machine/in_cksum.h>
	+
	+#include <opencrypto/cryptodev.h>
	+
	+#include "if_ovpn.h"
	+
	+struct ovpn_kkey_dir {
	+ uint8_t key[32];
	+ uint8_t keylen;
	+ uint8_t nonce[8];
	+ uint8_t noncelen;
	+ enum ovpn_key_cipher cipher;
	+ crypto_session_t cryptoid;
	+
	+ struct mtx replay_mtx;
	+ /*
	+ * Last seen gapless sequence number. New rx seq numbers must be
	+ * strictly higher than this.
	+ */
	+ uint32_t rx_seq;
	+ /* Seen packets, relative to rx_seq. bit(0) will always be 0. */
	+ uint64_t rx_window;
	+};
	+
	+struct ovpn_kkey {
	+ struct ovpn_kkey_dir *encrypt;
	+ struct ovpn_kkey_dir *decrypt;
	+ uint8_t keyid;
	+ uint32_t peerid;
	+};
	+
	+struct ovpn_keepalive {
	+ uint32_t interval;
	+ uint32_t timeout;
	+};
	+
	+struct ovpn_wire_header {
	+ uint32_t opcode; /* opcode, key id, peer id */
	+ uint32_t seq;
	+ uint8_t auth_tag[16];
	+};
	+
	+struct ovpn_notification {
	+ enum ovpn_notif_type type;
	+ uint32_t peerid;
	+};
	+
	+struct ovpn_softc;
	+
	+struct ovpn_kpeer {
	+ uint32_t peerid;
	+
	+ struct ovpn_softc *sc;
	+ struct sockaddr_storage local;
	+ struct sockaddr_storage remote;
	+
	+ struct in_addr vpn4;
	+ struct in6_addr vpn6;
	+
	+ struct ovpn_kkey keys[2];
	+ uint32_t tx_seq;
	+
	+ struct ovpn_keepalive keepalive;
	+ struct callout ping_send;
	+ struct callout ping_rcv;
	+};
	+
	+#define OVPN_MAX_PEERS 128
	+
	+struct ovpn_softc {
	+ struct rmlock lock;
	+ struct ifnet *ifp;
	+ struct socket *so;
	+ struct ovpn_kpeer peers[OVPN_MAX_PEERS]; / XXX Hard limit for now? */
	+
	+ /* Pending packets */
	+ struct buf_ring *rxring;
	+ struct buf_ring *notifring;
	+
	+ /* Counters */
	+ counter_u64_t lost_ctrl_pkts_in;
	+ counter_u64_t lost_ctrl_pkts_out;
	+ counter_u64_t lost_data_pkts_in;
	+ counter_u64_t lost_data_pkts_out;
	+ counter_u64_t received_ctrl_pkts;
	+ counter_u64_t received_data_pkts;
	+ counter_u64_t sent_ctrl_pkts;
	+ counter_u64_t sent_data_pkts;
	+
	+ counter_u64_t transport_bytes_sent;
	+ counter_u64_t transport_bytes_received;
	+ counter_u64_t tunnel_bytes_sent;
	+ counter_u64_t tunnel_bytes_received;
	+
	+ struct epoch_context epoch_ctx;
	+};
	+
	+static struct ovpn_kpeer ovpn_find_peer(struct ovpn_softc , uint32_t);
	+static bool ovpn_udp_input(struct mbuf , int, struct inpcb ,
	+ const struct sockaddr , void );
	+static int ovpn_transmit_to_peer(struct ifnet , struct mbuf ,
	+ struct ovpn_kpeer , struct rm_priotracker );
	+static int ovpn_encap(struct ovpn_softc , uint32_t, struct mbuf );
	+static int ovpn_get_af(struct mbuf *);
	+static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
	+static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
	+
	+#define OVPN_MTU_MIN 576
	+#define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \
	+ sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
	+
	+#define OVPN_OP_DATA_V2 0x09
	+#define OVPN_OP_SHIFT 3
	+
	+VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
	+#define V_ovpn_cloner VNET(ovpn_cloner)
	+
	+#define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \
	+ struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
	+#define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
	+#define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
	+#define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock)
	+#define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock)
	+#define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED)
	+#define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED)
	+#define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED)
	+#define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED)
	+
	+#define TO_IN(x) ((struct sockaddr_in *)(x))
	+#define TO_IN6(x) ((struct sockaddr_in6 *)(x))
	+
	+static const char ovpnname[] = "ovpn";
	+static const char ovpngroupname[] = "openvpn";
	+
	+static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
	+
	+SYSCTL_DECL(_net_link);
	+static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	+ "OpenVPN DCO Interface");
	+VNET_DEFINE_STATIC(int, replay_protection) = 0;
	+#define V_replay_protection VNET(replay_protection)
	+SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET \| CTLFLAG_RW,
	+ &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
	+
	+static struct ovpn_kpeer *
	+ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
	+{
	+ struct ovpn_kpeer *p = NULL;
	+
	+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
	+ p = sc->peers[i];
	+ if (p == NULL)
	+ continue;
	+
	+ if (p->peerid == peerid) {
	+ MPASS(p->sc == sc);
	+ break;
	+ }
	+ }
	+
	+ return (p);
	+}
	+
	+static uint16_t
	+ovpn_get_port(struct sockaddr_storage *s)
	+{
	+ switch (s->ss_family) {
	+ case AF_INET: {
	+ struct sockaddr_in in = (struct sockaddr_in )s;
	+ return (in->sin_port);
	+ }
	+ case AF_INET6: {
	+ struct sockaddr_in6 in6 = (struct sockaddr_in6 )s;
	+ return (in6->sin6_port);
	+ }
	+ default:
	+ panic("Unsupported address family %d", s->ss_family);
	+ }
	+}
	+
	+static int
	+ovpn_nvlist_to_sockaddr(const nvlist_t nvl, struct sockaddr_storage sa)
	+{
	+ int af;
	+
	+ if (! nvlist_exists_number(nvl, "af"))
	+ return (EINVAL);
	+ if (! nvlist_exists_binary(nvl, "address"))
	+ return (EINVAL);
	+ if (! nvlist_exists_number(nvl, "port"))
	+ return (EINVAL);
	+
	+ af = nvlist_get_number(nvl, "af");
	+
	+ switch (af) {
	+#ifdef INET
	+ case AF_INET: {
	+ struct sockaddr_in in = (struct sockaddr_in )sa;
	+ size_t len;
	+ const void *addr = nvlist_get_binary(nvl, "address", &len);
	+ in->sin_family = af;
	+ if (len != sizeof(in->sin_addr))
	+ return (EINVAL);
	+
	+ memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
	+ in->sin_port = nvlist_get_number(nvl, "port");
	+ break;
	+ }
	+#endif
	+#ifdef INET6
	+ case AF_INET6: {
	+ struct sockaddr_in6 in6 = (struct sockaddr_in6 )sa;
	+ size_t len;
	+ const void *addr = nvlist_get_binary(nvl, "address", &len);
	+ in6->sin6_family = af;
	+ if (len != sizeof(in6->sin6_addr))
	+ return (EINVAL);
	+
	+ memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
	+ in6->sin6_port = nvlist_get_number(nvl, "port");
	+ break;
	+ }
	+#endif
	+ default:
	+ return (EINVAL);
	+ }
	+
	+ return (0);
	+}
	+
	+static bool
	+ovpn_has_peers(struct ovpn_softc *sc)
	+{
	+ int i;
	+
	+ OVPN_ASSERT(sc);
	+
	+ for (i = 0; i < OVPN_MAX_PEERS; i++) {
	+ if (sc->peers[i] != NULL)
	+ break;
	+ }
	+
	+ return (i != OVPN_MAX_PEERS);
	+}
	+
	+static void
	+ovpn_rele_so(struct ovpn_softc sc, struct ovpn_kpeer peer)
	+{
	+ bool has_peers;
	+
	+ OVPN_WASSERT(sc);
	+
	+ if (sc->so == NULL)
	+ return;
	+
	+ has_peers = ovpn_has_peers(sc);
	+
	+ /* Only remove the tunnel function if we're releasing the socket for
	+ * the last peer. */
	+ if (! has_peers)
	+ (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
	+
	+ sorele(sc->so);
	+
	+ if (! has_peers)
	+ sc->so = NULL;
	+}
	+
	+static int
	+ovpn_new_peer(struct ifnet ifp, const nvlist_t nvl)
	+{
	+#ifdef INET6
	+ struct epoch_tracker et;
	+#endif
	+ struct sockaddr_storage remote;
	+ struct ovpn_kpeer *peer = NULL;
	+ struct file *fp = NULL;
	+ struct sockaddr *name = NULL;
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ struct thread *td = curthread;
	+ struct socket *so = NULL;
	+ u_int fflag;
	+ int fd;
	+ uint32_t peerid;
	+ int ret = 0, i;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "fd"))
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_nvlist(nvl, "remote"))
	+ return (EINVAL);
	+
	+ peerid = nvlist_get_number(nvl, "peerid");
	+
	+ /* Disallow peer id re-use. */
	+ if (ovpn_find_peer(sc, peerid) != NULL)
	+ return (EEXIST);
	+
	+ ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
	+ &remote);
	+ if (ret != 0)
	+ return (ret);
	+
	+ fd = nvlist_get_number(nvl, "fd");
	+
	+ /* Look up the userspace process and use the fd to find the socket. */
	+ ret = getsock_cap(td, fd, &cap_connect_rights, &fp,
	+ &fflag, NULL);
	+ if (ret != 0)
	+ return (ret);
	+
	+ so = fp->f_data;
	+
	+ peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK \| M_ZERO);
	+ peer->peerid = peerid;
	+ peer->sc = sc;
	+ peer->tx_seq = 1;
	+
	+ if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
	+ size_t len;
	+ const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
	+ if (len != sizeof(peer->vpn4)) {
	+ ret = EINVAL;
	+ goto error;
	+ }
	+ memcpy(&peer->vpn4, addr, len);
	+ }
	+
	+ if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
	+ size_t len;
	+ const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
	+ if (len != sizeof(peer->vpn6)) {
	+ ret = EINVAL;
	+ goto error;
	+ }
	+ memcpy(&peer->vpn6, addr, len);
	+ }
	+
	+ callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
	+ callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
	+
	+ ret = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &name);
	+ if (ret)
	+ goto error;
	+
	+ if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
	+ ret = EINVAL;
	+ goto error;
	+ }
	+ if (name->sa_family != remote.ss_family) {
	+ ret = EINVAL;
	+ goto error;
	+ }
	+
	+ memcpy(&peer->local, name, name->sa_len);
	+ memcpy(&peer->remote, &remote, sizeof(remote));
	+ free(name, M_SONAME);
	+ name = NULL;
	+
	+#ifdef INET6
	+ if (peer->local.ss_family == AF_INET6 &&
	+ IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
	+ NET_EPOCH_ENTER(et);
	+ ret = in6_selectsrc_addr(0,
	+ &TO_IN6(&peer->remote)->sin6_addr,
	+ 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
	+ NET_EPOCH_EXIT(et);
	+ if (ret != 0) {
	+ goto error;
	+ }
	+ }
	+#endif
	+ OVPN_WLOCK(sc);
	+
	+ /* Must be the same socket as for other peers on this interface. */
	+ if (sc->so != NULL && so != sc->so)
	+ goto error_locked;
	+
	+ if (sc->so == NULL)
	+ sc->so = so;
	+
	+ /* Insert the peer into the list. */
	+ for (i = 0; i < OVPN_MAX_PEERS; i++) {
	+ if (sc->peers[i] != NULL)
	+ continue;
	+
	+ MPASS(sc->peers[i] == NULL);
	+ sc->peers[i] = peer;
	+ soref(sc->so);
	+ break;
	+ }
	+ if (i == OVPN_MAX_PEERS) {
	+ OVPN_WUNLOCK(sc);
	+ ret = ENOSPC;
	+ goto error_locked;
	+ }
	+
	+ ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
	+ if (ret == EBUSY) {
	+ /* Fine, another peer already set the input function. */
	+ ret = 0;
	+ }
	+ if (ret != 0) {
	+ sc->peers[i] = NULL;
	+ goto error_locked;
	+ }
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ goto done;
	+
	+error_locked:
	+ OVPN_WUNLOCK(sc);
	+error:
	+ free(name, M_SONAME);
	+ free(peer, M_OVPN);
	+done:
	+ if (fp != NULL)
	+ fdrop(fp, td);
	+
	+ return (ret);
	+}
	+
	+static void
	+ovpn_notify_del_peer(struct ovpn_softc sc, struct ovpn_kpeer peer)
	+{
	+ struct ovpn_notification *n;
	+
	+ OVPN_WASSERT(sc);
	+
	+ n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
	+ if (n == NULL)
	+ return;
	+
	+ n->peerid = peer->peerid;
	+ n->type = OVPN_NOTIF_DEL_PEER;
	+ if (buf_ring_enqueue(sc->notifring, n) != 0) {
	+ free(n, M_OVPN);
	+ } else if (sc->so != NULL) {
	+ printf("KP: %s() wakeup for del peer notif\n", __func__);
	+ /* Wake up userspace */
	+ sc->so->so_error = EAGAIN;
	+ sorwakeup(sc->so);
	+ sowwakeup(sc->so);
	+ }
	+}
	+
	+static int
	+_ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid)
	+{
	+ struct ovpn_kpeer *peer;
	+ int i;
	+
	+ OVPN_WASSERT(sc);
	+
	+ for (i = 0; i < OVPN_MAX_PEERS; i++) {
	+ if (sc->peers[i] == NULL)
	+ continue;
	+ if (sc->peers[i]->peerid != peerid)
	+ continue;
	+
	+ peer = sc->peers[i];
	+ break;
	+ }
	+ if (i == OVPN_MAX_PEERS)
	+ return (ENOENT);
	+
	+ sc->peers[i] = NULL;
	+
	+ ovpn_notify_del_peer(sc, peer);
	+
	+ for (int i = 0; i < 2; i++) {
	+ ovpn_free_kkey_dir(peer->keys[i].encrypt);
	+ ovpn_free_kkey_dir(peer->keys[i].decrypt);
	+ }
	+
	+ ovpn_rele_so(sc, peer);
	+
	+ callout_stop(&peer->ping_send);
	+ callout_stop(&peer->ping_rcv);
	+ free(peer, M_OVPN);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_del_peer(struct ifnet ifp, nvlist_t nvl)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ uint32_t peerid;
	+ int ret;
	+
	+ OVPN_WASSERT(sc);
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ peerid = nvlist_get_number(nvl, "peerid");
	+
	+ ret = _ovpn_del_peer(sc, peerid);
	+
	+ return (ret);
	+}
	+
	+static int
	+ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
	+ const nvlist_t *nvl)
	+{
	+ struct crypto_session_params csp;
	+ struct ovpn_kkey_dir *kdir;
	+ const char *ciphername;
	+ enum ovpn_key_cipher cipher;
	+ const void key, iv;
	+ size_t keylen = 0, ivlen = 0;
	+ int error;
	+
	+ if (! nvlist_exists_string(nvl, "cipher"))
	+ return (EINVAL);
	+ ciphername = nvlist_get_string(nvl, "cipher");
	+
	+ if (strcmp(ciphername, "none") == 0)
	+ cipher = OVPN_CIPHER_ALG_NONE;
	+ else if (strcmp(ciphername, "AES-256-GCM") == 0)
	+ cipher = OVPN_CIPHER_ALG_AES_GCM;
	+ else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
	+ cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
	+ else
	+ return (EINVAL);
	+
	+ if (cipher != OVPN_CIPHER_ALG_NONE) {
	+ if (! nvlist_exists_binary(nvl, "key"))
	+ return (EINVAL);
	+ key = nvlist_get_binary(nvl, "key", &keylen);
	+ if (keylen > sizeof(kdir->key))
	+ return (E2BIG);
	+
	+ if (! nvlist_exists_binary(nvl, "iv"))
	+ return (EINVAL);
	+ iv = nvlist_get_binary(nvl, "iv", &ivlen);
	+ if (ivlen != 8)
	+ return (E2BIG);
	+ }
	+
	+ kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
	+ M_WAITOK \| M_ZERO);
	+
	+ kdir->cipher = cipher;
	+ kdir->keylen = keylen;
	+ memcpy(kdir->key, key, keylen);
	+ kdir->noncelen = ivlen;
	+ memcpy(kdir->nonce, iv, ivlen);
	+
	+ if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
	+ /* Crypto init */
	+ bzero(&csp, sizeof(csp));
	+ csp.csp_mode = CSP_MODE_AEAD;
	+
	+ if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
	+ csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
	+ else
	+ csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
	+
	+ csp.csp_flags \|= CSP_F_SEPARATE_AAD;
	+
	+ csp.csp_cipher_klen = kdir->keylen;
	+ csp.csp_cipher_key = kdir->key;
	+ csp.csp_ivlen = 96 / 8;
	+
	+ error = crypto_newsession(&kdir->cryptoid, &csp,
	+ CRYPTOCAP_F_HARDWARE \| CRYPTOCAP_F_SOFTWARE);
	+ if (error) {
	+ free(kdir, M_OVPN);
	+ return (error);
	+ }
	+ }
	+
	+ mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
	+ *kdirp = kdir;
	+
	+ return (0);
	+}
	+
	+static void
	+ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
	+{
	+ if (kdir == NULL)
	+ return;
	+
	+ mtx_destroy(&kdir->replay_mtx);
	+
	+ crypto_freesession(kdir->cryptoid);
	+ free(kdir, M_OVPN);
	+}
	+
	+static int
	+ovpn_set_key(struct ifnet ifp, const nvlist_t nvl)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ struct ovpn_kkey_dir enc, dec;
	+ struct ovpn_kpeer *peer;
	+ int slot, keyid, peerid;
	+ int error;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "slot"))
	+ return (EINVAL);
	+ slot = nvlist_get_number(nvl, "slot");
	+
	+ if (! nvlist_exists_number(nvl, "keyid"))
	+ return (EINVAL);
	+ keyid = nvlist_get_number(nvl, "keyid");
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+ peerid = nvlist_get_number(nvl, "peerid");
	+
	+ peer = ovpn_find_peer(sc, peerid);
	+ if (peer == NULL)
	+ return (ENOENT);
	+
	+ if (slot != OVPN_KEY_SLOT_PRIMARY &&
	+ slot != OVPN_KEY_SLOT_SECONDARY)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_nvlist(nvl, "encrypt") \|\|
	+ ! nvlist_exists_nvlist(nvl, "decrypt"))
	+ return (EINVAL);
	+
	+ error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
	+ if (error)
	+ return (error);
	+
	+ error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
	+ if (error) {
	+ ovpn_free_kkey_dir(enc);
	+ return (error);
	+ }
	+
	+ OVPN_WLOCK(sc);
	+
	+ ovpn_free_kkey_dir(peer->keys[slot].encrypt);
	+ ovpn_free_kkey_dir(peer->keys[slot].decrypt);
	+
	+ peer->keys[slot].encrypt = enc;
	+ peer->keys[slot].decrypt = dec;
	+
	+ peer->keys[slot].keyid = keyid;
	+ peer->keys[slot].peerid = peerid;
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_check_key(struct ovpn_softc sc, struct ovpn_kpeer peer, enum ovpn_key_slot slot)
	+{
	+ OVPN_ASSERT(sc);
	+
	+ if (peer->keys[slot].encrypt == NULL)
	+ return (ENOLINK);
	+
	+ if (peer->keys[slot].decrypt == NULL)
	+ return (ENOLINK);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_start(struct ifnet *ifp)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+
	+ OVPN_WLOCK(sc);
	+
	+ ifp->if_flags \|= IFF_UP;
	+ ifp->if_drv_flags \|= IFF_DRV_RUNNING;
	+ if_link_state_change(ifp, LINK_STATE_UP);
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_swap_keys(struct ifnet ifp, nvlist_t nvl)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ struct ovpn_kpeer *peer;
	+ struct ovpn_kkey tmpkey;
	+ int error;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ OVPN_WLOCK(sc);
	+
	+ peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
	+ if (peer == NULL) {
	+ OVPN_WUNLOCK(sc);
	+ return (ENOENT);
	+ }
	+
	+ /* Check that we have a second key to swap to. */
	+ error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY);
	+ if (error) {
	+ OVPN_WUNLOCK(sc);
	+ return (error);
	+ }
	+
	+ tmpkey = peer->keys[0];
	+ peer->keys[0] = peer->keys[1];
	+ peer->keys[1] = tmpkey;
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_del_key(struct ifnet ifp, const nvlist_t nvl)
	+{
	+ enum ovpn_key_slot slot;
	+ struct ovpn_kpeer *peer;
	+ struct ovpn_softc *sc = ifp->if_softc;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "slot"))
	+ return (EINVAL);
	+ slot = nvlist_get_number(nvl, "slot");
	+
	+ if (slot != OVPN_KEY_SLOT_PRIMARY &&
	+ slot != OVPN_KEY_SLOT_SECONDARY)
	+ return (EINVAL);
	+
	+ OVPN_WLOCK(sc);
	+
	+ peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
	+ if (peer == NULL) {
	+ OVPN_WUNLOCK(sc);
	+ return (ENOENT);
	+ }
	+
	+ ovpn_free_kkey_dir(peer->keys[slot].encrypt);
	+ ovpn_free_kkey_dir(peer->keys[slot].decrypt);
	+
	+ peer->keys[slot].encrypt = NULL;
	+ peer->keys[slot].decrypt = NULL;
	+
	+ peer->keys[slot].keyid = 0;
	+ peer->keys[slot].peerid = 0;
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_send_pkt(struct ifnet ifp, const nvlist_t nvl)
	+{
	+ struct epoch_tracker et;
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ struct mbuf *m;
	+ const uint8_t *pkt;
	+ size_t pktlen;
	+ uint32_t peerid;
	+ int ret;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_binary(nvl, "packet"))
	+ return (EINVAL);
	+ pkt = nvlist_get_binary(nvl, "packet", &pktlen);
	+
	+ if (! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ peerid = nvlist_get_number(nvl, "peerid");
	+
	+ /*
	+ * Check that userspace isn't giving us a data packet. That might lead
	+ * to IV re-use, which would be bad.
	+ */
	+ if ((pkt[0] >> OVPN_OP_SHIFT) == OVPN_OP_DATA_V2)
	+ return (EINVAL);
	+
	+ m = m_get2(pktlen, M_WAITOK, MT_DATA, M_PKTHDR);
	+ if (m == NULL)
	+ return (ENOMEM);
	+
	+ m->m_len = m->m_pkthdr.len = pktlen;
	+ m_copyback(m, 0, m->m_len, pkt);
	+
	+ /* Now prepend IP/UDP headers and transmit the mbuf. */
	+ NET_EPOCH_ENTER(et);
	+ ret = ovpn_encap(sc, peerid, m);
	+ NET_EPOCH_EXIT(et);
	+ if (ret == 0)
	+ counter_u64_add(sc->sent_ctrl_pkts, 1);
	+ else
	+ counter_u64_add(sc->lost_ctrl_pkts_out, 1);
	+
	+ return (ret);
	+}
	+
	+static void
	+ovpn_send_ping(void *arg)
	+{
	+ static const uint8_t ping_str[] = {
	+ 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb,
	+ 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48
	+ };
	+
	+ struct epoch_tracker et;
	+ struct ovpn_kpeer *peer = arg;
	+ struct ovpn_softc *sc = peer->sc;
	+ struct mbuf *m;
	+
	+ OVPN_RASSERT(sc);
	+
	+ /* Ensure we repeat! */
	+ callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
	+ ovpn_send_ping, peer);
	+
	+ m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR);
	+ if (m == NULL)
	+ return;
	+
	+ m_copyback(m, 0, sizeof(ping_str), ping_str);
	+ m->m_len = m->m_pkthdr.len = sizeof(ping_str);
	+
	+ CURVNET_SET(sc->ifp->if_vnet);
	+ NET_EPOCH_ENTER(et);
	+ (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL);
	+ NET_EPOCH_EXIT(et);
	+ CURVNET_RESTORE();
	+}
	+
	+static void
	+ovpn_timeout(void *arg)
	+{
	+ struct ovpn_kpeer *peer = arg;
	+ struct ovpn_softc *sc = peer->sc;
	+ int ret __diagused;
	+
	+ OVPN_WASSERT(sc);
	+
	+ CURVNET_SET(sc->ifp->if_vnet);
	+ ret = _ovpn_del_peer(sc, peer->peerid);
	+ MPASS(ret == 0);
	+ CURVNET_RESTORE();
	+}
	+
	+static int
	+ovpn_set_peer(struct ifnet ifp, const nvlist_t nvl)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ struct ovpn_kpeer *peer;
	+
	+ if (nvl == NULL)
	+ return (EINVAL);
	+
	+ if (! nvlist_exists_number(nvl, "interval") \|\|
	+ ! nvlist_exists_number(nvl, "timeout") \|\|
	+ ! nvlist_exists_number(nvl, "peerid"))
	+ return (EINVAL);
	+
	+ OVPN_WLOCK(sc);
	+
	+ peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid"));
	+ if (peer == NULL) {
	+ OVPN_WUNLOCK(sc);
	+ return (ENOENT);
	+ }
	+
	+ peer->keepalive.interval = nvlist_get_number(nvl, "interval");
	+ peer->keepalive.timeout = nvlist_get_number(nvl, "timeout");
	+
	+ callout_reset(&peer->ping_send, peer->keepalive.interval * hz,
	+ ovpn_send_ping, peer);
	+ callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
	+ ovpn_timeout, peer);
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_ioctl_set(struct ifnet ifp, struct ifdrv ifd)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ uint8_t *buf = NULL;
	+ nvlist_t *nvl = NULL;
	+ int ret;
	+
	+ if (ifd->ifd_len != 0) {
	+ if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE)
	+ return (E2BIG);
	+
	+ buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK);
	+
	+ ret = copyin(ifd->ifd_data, buf, ifd->ifd_len);
	+ if (ret != 0) {
	+ free(buf, M_OVPN);
	+ return (ret);
	+ }
	+
	+ nvl = nvlist_unpack(buf, ifd->ifd_len, 0);
	+ free(buf, M_OVPN);
	+ if (nvl == NULL) {
	+ return (EINVAL);
	+ }
	+ }
	+
	+ switch (ifd->ifd_cmd) {
	+ case OVPN_NEW_PEER:
	+ ret = ovpn_new_peer(ifp, nvl);
	+ break;
	+ case OVPN_DEL_PEER:
	+ OVPN_WLOCK(sc);
	+ ret = ovpn_del_peer(ifp, nvl);
	+ OVPN_WUNLOCK(sc);
	+ break;
	+ case OVPN_NEW_KEY:
	+ ret = ovpn_set_key(ifp, nvl);
	+ break;
	+ case OVPN_START_VPN:
	+ ret = ovpn_start(ifp);
	+ break;
	+ case OVPN_SWAP_KEYS:
	+ ret = ovpn_swap_keys(ifp, nvl);
	+ break;
	+ case OVPN_DEL_KEY:
	+ ret = ovpn_del_key(ifp, nvl);
	+ break;
	+ case OVPN_SEND_PKT:
	+ ret = ovpn_send_pkt(ifp, nvl);
	+ break;
	+ case OVPN_SET_PEER:
	+ ret = ovpn_set_peer(ifp, nvl);
	+ break;
	+ default:
	+ ret = ENOTSUP;
	+ }
	+
	+ nvlist_destroy(nvl);
	+ return (ret);
	+}
	+
	+static int
	+ovpn_add_counters(nvlist_t parent, const char name, counter_u64_t in,
	+ counter_u64_t out)
	+{
	+ nvlist_t *nvl;
	+
	+ nvl = nvlist_create(0);
	+ if (nvl == NULL)
	+ return (ENOMEM);
	+
	+ nvlist_add_number(nvl, "in", counter_u64_fetch(in));
	+ nvlist_add_number(nvl, "out", counter_u64_fetch(out));
	+
	+ nvlist_add_nvlist(parent, name, nvl);
	+
	+ nvlist_destroy(nvl);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_get_stats(struct ovpn_softc sc, nvlist_t *onvl)
	+{
	+ nvlist_t *nvl;
	+ int ret;
	+
	+ nvl = nvlist_create(0);
	+ if (nvl == NULL)
	+ return (ENOMEM);
	+
	+ ret = ovpn_add_counters(nvl, "lost_ctrl", sc->lost_ctrl_pkts_in,
	+ sc->lost_ctrl_pkts_out);
	+ if (ret != 0)
	+ goto error;
	+ ret = ovpn_add_counters(nvl, "lost_data", sc->lost_data_pkts_in,
	+ sc->lost_data_pkts_out);
	+ if (ret != 0)
	+ goto error;
	+
	+ ret = ovpn_add_counters(nvl, "data", sc->received_data_pkts,
	+ sc->sent_data_pkts);
	+ if (ret != 0)
	+ goto error;
	+ ret = ovpn_add_counters(nvl, "ctrl", sc->received_ctrl_pkts,
	+ sc->sent_ctrl_pkts);
	+ if (ret != 0)
	+ goto error;
	+
	+ ret = ovpn_add_counters(nvl, "tunnel", sc->tunnel_bytes_received,
	+ sc->tunnel_bytes_received);
	+ if (ret != 0)
	+ goto error;
	+ ret = ovpn_add_counters(nvl, "transport", sc->transport_bytes_received,
	+ sc->transport_bytes_received);
	+ if (ret != 0)
	+ goto error;
	+
	+ *onvl = nvl;
	+
	+ return (0);
	+
	+error:
	+ nvlist_destroy(nvl);
	+ return (ret);
	+}
	+
	+static int
	+ovpn_poll_pkt(struct ovpn_softc sc, nvlist_t *onvl)
	+{
	+ nvlist_t *nvl;
	+
	+ nvl = nvlist_create(0);
	+ if (nvl == NULL)
	+ return (ENOMEM);
	+
	+ nvlist_add_number(nvl, "pending",
	+ buf_ring_count(sc->rxring) + buf_ring_count(sc->notifring));
	+
	+ *onvl = nvl;
	+
	+ return (0);
	+}
	+
	+static int
	+opvn_get_pkt(struct ovpn_softc sc, nvlist_t *onvl)
	+{
	+ struct ovpn_notification *n;
	+ struct ovpn_wire_header *ohdr;
	+ struct mbuf *m;
	+ uint8_t *buf;
	+ nvlist_t *nvl;
	+ uint32_t peerid;
	+ u_int mlength;
	+
	+ /* Check if we have notifications pending. */
	+ n = buf_ring_dequeue_mc(sc->notifring);
	+ if (n != NULL) {
	+ nvl = nvlist_create(0);
	+ if (nvl == NULL) {
	+ free(n, M_OVPN);
	+ return (ENOMEM);
	+ }
	+ nvlist_add_number(nvl, "peerid", n->peerid);
	+ nvlist_add_number(nvl, "notification", n->type);
	+ free(n, M_OVPN);
	+
	+ *onvl = nvl;
	+
	+ return (0);
	+ }
	+
	+ /* Queued packet. */
	+ m = buf_ring_dequeue_mc(sc->rxring);
	+ if (m == NULL)
	+ return (ENOENT);
	+
	+ mlength = m_length(m, NULL);
	+ buf = malloc(mlength, M_NVLIST, M_WAITOK);
	+ m_copydata(m, 0, mlength, buf);
	+ ohdr = (struct ovpn_wire_header *)buf;
	+ peerid = ntohl(ohdr->opcode) & 0x00ffffff;
	+
	+ nvl = nvlist_create(0);
	+ if (nvl == NULL) {
	+ counter_u64_add(sc->lost_ctrl_pkts_in, 1);
	+ m_freem(m);
	+ free(buf, M_NVLIST);
	+ return (ENOMEM);
	+ }
	+
	+ nvlist_move_binary(nvl, "packet", buf, mlength);
	+ buf = NULL;
	+ nvlist_add_number(nvl, "peerid", peerid);
	+
	+ *onvl = nvl;
	+
	+ m_freem(m);
	+
	+ return (0);
	+}
	+
	+static int
	+ovpn_ioctl_get(struct ifnet ifp, struct ifdrv ifd)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ nvlist_t *nvl = NULL;
	+ int error;
	+
	+ switch (ifd->ifd_cmd) {
	+ case OVPN_GET_STATS:
	+ error = ovpn_get_stats(sc, &nvl);
	+ break;
	+ case OVPN_POLL_PKT:
	+ error = ovpn_poll_pkt(sc, &nvl);
	+ break;
	+ case OVPN_GET_PKT:
	+ error = opvn_get_pkt(sc, &nvl);
	+ break;
	+ default:
	+ error = ENOTSUP;
	+ break;
	+ }
	+
	+ if (error == 0) {
	+ void *packed = NULL;
	+ size_t len;
	+
	+ MPASS(nvl != NULL);
	+
	+ packed = nvlist_pack(nvl, &len);
	+ if (! packed) {
	+ nvlist_destroy(nvl);
	+ return (ENOMEM);
	+ }
	+
	+ if (len > ifd->ifd_len) {
	+ free(packed, M_NVLIST);
	+ nvlist_destroy(nvl);
	+ return (ENOSPC);
	+ }
	+
	+ error = copyout(packed, ifd->ifd_data, len);
	+ ifd->ifd_len = len;
	+
	+ free(packed, M_NVLIST);
	+ nvlist_destroy(nvl);
	+ }
	+
	+ return (error);
	+}
	+
	+static int
	+ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
	+{
	+ struct ifdrv *ifd;
	+ int error;
	+
	+ switch (cmd) {
	+ case SIOCSDRVSPEC:
	+ case SIOCGDRVSPEC:
	+ error = priv_check(curthread, PRIV_NET_OVPN);
	+ if (error)
	+ return (error);
	+ break;
	+ }
	+
	+ switch (cmd) {
	+ case SIOCSDRVSPEC:
	+ ifd = (struct ifdrv *)data;
	+ error = ovpn_ioctl_set(ifp, ifd);
	+ break;
	+ case SIOCGDRVSPEC:
	+ ifd = (struct ifdrv *)data;
	+ error = ovpn_ioctl_get(ifp, ifd);
	+ break;
	+ case SIOCSIFMTU: {
	+ struct ifreq ifr = (struct ifreq )data;
	+ if (ifr->ifr_mtu < OVPN_MTU_MIN \|\| ifr->ifr_mtu > OVPN_MTU_MAX)
	+ return (EINVAL);
	+
	+ ifp->if_mtu = ifr->ifr_mtu;
	+ return (0);
	+ }
	+ case SIOCSIFADDR:
	+ case SIOCADDMULTI:
	+ case SIOCDELMULTI:
	+ case SIOCGIFMTU:
	+ case SIOCSIFFLAGS:
	+ return (0);
	+ default:
	+ error = EINVAL;
	+ }
	+
	+ return (error);
	+}
	+
	+static int
	+ovpn_encrypt_tx_cb(struct cryptop *crp)
	+{
	+ struct ovpn_kpeer *peer = crp->crp_opaque;
	+ struct ovpn_softc *sc = peer->sc;
	+ struct mbuf *m = crp->crp_buf.cb_mbuf;
	+ int ret;
	+
	+ if (crp->crp_etype != 0) {
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ m_freem(m);
	+ return (0);
	+ }
	+
	+ CURVNET_SET(sc->ifp->if_vnet);
	+
	+ MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
	+
	+ ret = ovpn_encap(sc, peer->peerid, m);
	+ if (ret == 0) {
	+ counter_u64_add(sc->sent_data_pkts, 1);
	+ counter_u64_add(sc->tunnel_bytes_sent, m->m_pkthdr.len -
	+ sizeof(struct ovpn_wire_header));
	+ }
	+
	+ CURVNET_RESTORE();
	+
	+ crypto_freereq(crp);
	+ return (0);
	+}
	+
	+static void
	+ovpn_finish_rx(struct ovpn_softc sc, struct mbuf m,
	+ struct ovpn_kpeer peer, struct ovpn_kkey key, uint32_t seq,
	+ struct rm_priotracker *_ovpn_lock_trackerp)
	+{
	+ uint32_t af;
	+ int ret __diagused;
	+
	+ OVPN_RASSERT(sc);
	+
	+ /* Replay protection. */
	+ if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ return;
	+ }
	+
	+ /* Valid packet, kick the timeout watchdog. */
	+ if (peer->keepalive.timeout)
	+ callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz,
	+ ovpn_timeout, peer);
	+
	+ OVPN_RUNLOCK(sc);
	+
	+ counter_u64_add(sc->received_data_pkts, 1);
	+ counter_u64_add(sc->tunnel_bytes_received, m->m_pkthdr.len);
	+
	+ /* Receive the packet on our interface. */
	+ m->m_pkthdr.rcvif = sc->ifp;
	+
	+ /* Ensure we can read the first byte. */
	+ m = m_pullup(m, 1);
	+
	+ /*
	+ * Check for address family, and disregard any control packets (e.g.
	+ * keepalive).
	+ */
	+ af = ovpn_get_af(m);
	+ if (af != 0) {
	+ BPF_MTAP2(sc->ifp, &af, sizeof(af), m);
	+ ret = netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6,
	+ m);
	+ MPASS(ret == 0);
	+ } else {
	+ m_freem(m);
	+ }
	+}
	+
	+static struct ovpn_kkey *
	+ovpn_find_key(struct ovpn_softc sc, struct ovpn_kpeer peer,
	+ const struct ovpn_wire_header *ohdr)
	+{
	+ struct ovpn_kkey *key = NULL;
	+ uint8_t keyid;
	+
	+ OVPN_RASSERT(sc);
	+
	+ keyid = (ntohl(ohdr->opcode) >> 24) & 0x07;
	+
	+ if (peer->keys[0].keyid == keyid)
	+ key = &peer->keys[0];
	+ else if (peer->keys[1].keyid == keyid)
	+ key = &peer->keys[1];
	+
	+ return (key);
	+}
	+
	+static int
	+ovpn_encrypt_rx_cb(struct cryptop *crp)
	+{
	+ struct ovpn_softc *sc = crp->crp_opaque;
	+ struct mbuf *m = crp->crp_buf.cb_mbuf;
	+ struct ovpn_kkey *key;
	+ struct ovpn_kpeer *peer;
	+ struct ovpn_wire_header *ohdr;
	+ uint32_t peerid;
	+
	+ OVPN_RLOCK_TRACKER;
	+
	+ MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF);
	+
	+ if (crp->crp_etype != 0) {
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ return (0);
	+ }
	+
	+ OVPN_RLOCK(sc);
	+ CURVNET_SET(sc->ifp->if_vnet);
	+
	+ ohdr = mtodo(m, sizeof(struct udphdr));
	+
	+ peerid = ntohl(ohdr->opcode) & 0x00ffffff;
	+ peer = ovpn_find_peer(sc, peerid);
	+ if (peer == NULL) {
	+ /* No such peer. Drop packet. */
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ CURVNET_RESTORE();
	+ return (0);
	+ }
	+
	+ key = ovpn_find_key(sc, peer, ohdr);
	+ if (key == NULL) {
	+ /*
	+ * Has this key been removed between us starting the decrypt
	+ * and finishing it?
	+ */
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ CURVNET_RESTORE();
	+ return (0);
	+ }
	+
	+ /* Now remove the outer headers */
	+ m_adj_decap(m, sizeof(struct udphdr) +
	+ sizeof(struct ovpn_wire_header));
	+
	+ ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp);
	+ OVPN_UNLOCK_ASSERT(sc);
	+
	+ CURVNET_RESTORE();
	+
	+ crypto_freereq(crp);
	+ return (0);
	+}
	+
	+static uint8_t EMPTY_BUFFER[AES_BLOCK_LEN];
	+
	+static int
	+ovpn_get_af(struct mbuf *m)
	+{
	+ struct ip *ip;
	+ struct ip6_hdr *ip6;
	+
	+ /*
	+ * We should pullup, but we're only interested in the first byte, so
	+ * that'll always be contiguous.
	+ */
	+ ip = mtod(m, struct ip *);
	+ if (ip->ip_v == IPVERSION)
	+ return (AF_INET);
	+
	+ ip6 = mtod(m, struct ip6_hdr *);
	+ if (ip6->ip6_vfc == IPV6_VERSION)
	+ return (AF_INET6);
	+
	+ return (0);
	+}
	+
	+static struct ovpn_kpeer *
	+ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr)
	+{
	+ struct ovpn_kpeer *peer = NULL;
	+
	+ OVPN_ASSERT(sc);
	+
	+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
	+ if (sc->peers[i] == NULL)
	+ continue;
	+ if (addr.s_addr == sc->peers[i]->vpn4.s_addr) {
	+ peer = sc->peers[i];
	+ break;
	+ }
	+ }
	+
	+ return (peer);
	+}
	+
	+static struct ovpn_kpeer *
	+ovpn_find_peer_by_ip6(struct ovpn_softc sc, const struct in6_addr addr)
	+{
	+ struct ovpn_kpeer *peer = NULL;
	+
	+ OVPN_ASSERT(sc);
	+
	+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
	+ if (sc->peers[i] == NULL)
	+ continue;
	+ if (memcmp(addr, &sc->peers[i]->vpn6, sizeof(*addr)) == 0) {
	+ peer = sc->peers[i];
	+ break;
	+ }
	+ }
	+
	+ return (peer);
	+}
	+
	+static struct ovpn_kpeer *
	+ovpn_route_peer(struct ovpn_softc sc, struct mbuf *m0)
	+{
	+ struct ovpn_kpeer *peer = NULL;
	+ int af;
	+
	+ NET_EPOCH_ASSERT();
	+
	+ af = ovpn_get_af(*m0);
	+ switch (af) {
	+#ifdef INET
	+ case AF_INET: {
	+ struct nhop_object *nh;
	+ struct ip *ip;
	+ m0 = m_pullup(m0, sizeof(struct ip));
	+ if (*m0 == NULL)
	+ return (NULL);
	+
	+ ip = mtod(m0, struct ip );
	+ peer = ovpn_find_peer_by_ip(sc, ip->ip_dst);
	+ if (peer == NULL) {
	+ nh = fib4_lookup(M_GETFIB(*m0), ip->ip_dst, 0, 0, 0);
	+ if (nh && (nh->nh_flags & NHF_GATEWAY))
	+ peer = ovpn_find_peer_by_ip(sc,
	+ nh->gw4_sa.sin_addr);
	+ }
	+ break;
	+ }
	+#endif
	+#ifdef INET6
	+ case AF_INET6: {
	+ struct nhop_object *nh;
	+ struct ip6_hdr *ip6;
	+ m0 = m_pullup(m0, sizeof(struct ip6_hdr));
	+ if (*m0 == NULL)
	+ return (NULL);
	+
	+ ip6 = mtod(m0, struct ip6_hdr );
	+ peer = ovpn_find_peer_by_ip6(sc, &ip6->ip6_dst);
	+ if (peer == NULL) {
	+ nh = fib6_lookup(M_GETFIB(*m0), &ip6->ip6_dst, 0, 0, 0);
	+ if (nh && (nh->nh_flags & NHF_GATEWAY))
	+ peer = ovpn_find_peer_by_ip6(sc,
	+ &nh->gw6_sa.sin6_addr);
	+ }
	+ break;
	+ }
	+#endif
	+ }
	+
	+ return (peer);
	+}
	+
	+static int
	+ovpn_transmit(struct ifnet ifp, struct mbuf m)
	+{
	+ struct ovpn_softc *sc;
	+ struct ovpn_kpeer *peer;
	+
	+ OVPN_RLOCK_TRACKER;
	+
	+ sc = ifp->if_softc;
	+
	+ if (ifp->if_link_state != LINK_STATE_UP) {
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ return (ENETDOWN);
	+ }
	+
	+ OVPN_RLOCK(sc);
	+
	+ peer = ovpn_route_peer(sc, &m);
	+ if (peer == NULL) {
	+ /* No destination. */
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ OVPN_RUNLOCK(sc);
	+ return (ENETDOWN);
	+ }
	+
	+ return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp));
	+}
	+
	+static int
	+ovpn_transmit_to_peer(struct ifnet ifp, struct mbuf m,
	+ struct ovpn_kpeer peer, struct rm_priotracker _ovpn_lock_trackerp)
	+{
	+ struct ovpn_wire_header *ohdr;
	+ struct ovpn_kkey *key;
	+ struct ovpn_softc *sc;
	+ struct cryptop *crp;
	+ uint32_t af, seq;
	+ size_t len, real_len, ovpn_hdr_len;
	+ int tunnel_len;
	+ int ret;
	+
	+ sc = ifp->if_softc;
	+
	+ OVPN_RASSERT(sc);
	+
	+ tunnel_len = m->m_pkthdr.len;
	+
	+ key = &peer->keys[OVPN_KEY_SLOT_PRIMARY];
	+ if (key->encrypt == NULL)
	+ return (ENOLINK);
	+
	+ af = ovpn_get_af(m);
	+ /* Don't capture control packets. */
	+ if (af != 0)
	+ BPF_MTAP2(ifp, &af, sizeof(af), m);
	+
	+ real_len = len = m->m_pkthdr.len;
	+ MPASS(real_len <= ifp->if_mtu);
	+
	+ ovpn_hdr_len = sizeof(struct ovpn_wire_header);
	+ if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE)
	+ ovpn_hdr_len -= 16; /* No auth tag. */
	+ else {
	+ /* Round up the len to a multiple of our block size. */
	+ len = roundup2(real_len, AES_BLOCK_LEN);
	+
	+ /* Now extend the mbuf. */
	+ m_append(m, len - real_len, EMPTY_BUFFER);
	+ }
	+
	+ M_PREPEND(m, ovpn_hdr_len, M_NOWAIT);
	+ if (m == NULL) {
	+ if (_ovpn_lock_trackerp != NULL)
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ return (ENOBUFS);
	+ }
	+ ohdr = mtod(m, struct ovpn_wire_header *);
	+ ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) \| key->keyid;
	+ ohdr->opcode <<= 24;
	+ ohdr->opcode \|= key->peerid;
	+ ohdr->opcode = htonl(ohdr->opcode);
	+
	+ seq = atomic_fetchadd_32(&peer->tx_seq, 1);
	+ seq = htonl(seq);
	+ ohdr->seq = seq;
	+
	+ if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) {
	+ ret = ovpn_encap(sc, peer->peerid, m);
	+ if (_ovpn_lock_trackerp != NULL)
	+ OVPN_RUNLOCK(sc);
	+ if (ret == 0) {
	+ counter_u64_add(sc->sent_data_pkts, 1);
	+ counter_u64_add(sc->tunnel_bytes_sent, tunnel_len);
	+ }
	+ return (ret);
	+ }
	+
	+ crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT);
	+ if (crp == NULL) {
	+ if (_ovpn_lock_trackerp != NULL)
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ m_freem(m);
	+ return (ENOBUFS);
	+ }
	+
	+ /* Encryption covers only the payload, not the header. */
	+ crp->crp_payload_start = sizeof(*ohdr);
	+ crp->crp_payload_length = len;
	+ crp->crp_op = CRYPTO_OP_ENCRYPT;
	+
	+ /*
	+ * AAD data covers the ovpn_wire_header minus the auth
	+ * tag.
	+ */
	+ crp->crp_aad_length = sizeof(*ohdr) -
	+ sizeof(ohdr->auth_tag);
	+ crp->crp_aad = ohdr;
	+ crp->crp_aad_start = 0;
	+ crp->crp_op \|= CRYPTO_OP_COMPUTE_DIGEST;
	+ crp->crp_digest_start =
	+ offsetof(struct ovpn_wire_header, auth_tag);
	+
	+ crp->crp_flags \|= CRYPTO_F_IV_SEPARATE;
	+ memcpy(crp->crp_iv, &seq, sizeof(seq));
	+ memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce,
	+ key->encrypt->noncelen);
	+
	+ crypto_use_mbuf(crp, m);
	+ crp->crp_flags \|= CRYPTO_F_CBIFSYNC;
	+ crp->crp_callback = ovpn_encrypt_tx_cb;
	+ crp->crp_opaque = peer;
	+
	+ if (_ovpn_lock_trackerp != NULL)
	+ OVPN_RUNLOCK(sc);
	+ ret = crypto_dispatch(crp);
	+ if (ret) {
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ m_freem(m);
	+ }
	+
	+ return (ret);
	+}
	+
	+/*
	+ * Note: Expects to hold the read lock on entry, and will release it itself.
	+ */
	+static int
	+ovpn_encap(struct ovpn_softc sc, uint32_t peerid, struct mbuf m)
	+{
	+ struct udphdr *udp;
	+ struct ovpn_kpeer *peer;
	+ int len;
	+
	+ OVPN_RLOCK_TRACKER;
	+
	+ OVPN_RLOCK(sc);
	+ NET_EPOCH_ASSERT();
	+
	+ peer = ovpn_find_peer(sc, peerid);
	+ if (peer == NULL \|\| sc->ifp->if_link_state != LINK_STATE_UP) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ m_freem(m);
	+ return (ENETDOWN);
	+ }
	+
	+ len = m->m_pkthdr.len;
	+
	+ M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT);
	+ if (m == NULL) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ m_freem(m);
	+ return (ENOBUFS);
	+ }
	+ udp = mtod(m, struct udphdr *);
	+
	+ MPASS(peer->local.ss_family == peer->remote.ss_family);
	+
	+ udp->uh_sport = ovpn_get_port(&peer->local);
	+ udp->uh_dport = ovpn_get_port(&peer->remote);
	+ udp->uh_ulen = htons(sizeof(struct udphdr) + len);
	+
	+ switch (peer->remote.ss_family) {
	+#ifdef INET
	+ case AF_INET: {
	+ struct sockaddr_in *in_local = TO_IN(&peer->local);
	+ struct sockaddr_in *in_remote = TO_IN(&peer->remote);
	+ struct ip *ip;
	+
	+ /*
	+ * This requires knowing the source IP, which we don't. Happily
	+ * we're allowed to keep this at 0, and the checksum won't do
	+ * anything the crypto won't already do.
	+ */
	+ udp->uh_sum = 0;
	+
	+ /* Set the checksum flags so we recalculate checksums. */
	+ m->m_pkthdr.csum_flags \|= CSUM_IP;
	+ m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
	+
	+ M_PREPEND(m, sizeof(struct ip), M_NOWAIT);
	+ if (m == NULL) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ return (ENOBUFS);
	+ }
	+ ip = mtod(m, struct ip *);
	+
	+ ip->ip_tos = 0;
	+ ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) +
	+ len);
	+ ip->ip_off = 0;
	+ ip->ip_ttl = V_ip_defttl;
	+ ip->ip_p = IPPROTO_UDP;
	+ ip->ip_sum = 0;
	+ if (in_local->sin_port != 0)
	+ ip->ip_src = in_local->sin_addr;
	+ else
	+ ip->ip_src.s_addr = INADDR_ANY;
	+ ip->ip_dst = in_remote->sin_addr;
	+
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->transport_bytes_sent, m->m_pkthdr.len);
	+
	+ return (ip_output(m, NULL, NULL, 0, NULL, NULL));
	+ }
	+#endif
	+#ifdef INET6
	+ case AF_INET6: {
	+ struct sockaddr_in6 *in6_local = TO_IN6(&peer->local);
	+ struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote);
	+ struct ip6_hdr *ip6;
	+
	+ M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT);
	+ if (m == NULL) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_out, 1);
	+ return (ENOBUFS);
	+ }
	+
	+ ip6 = mtod(m, struct ip6_hdr *);
	+
	+ ip6->ip6_vfc = IPV6_VERSION;
	+ ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK;
	+ ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) +
	+ len);
	+ ip6->ip6_nxt = IPPROTO_UDP;
	+ ip6->ip6_hlim = V_ip6_defhlim;
	+
	+ memcpy(&ip6->ip6_src, &in6_local->sin6_addr,
	+ sizeof(ip6->ip6_src));
	+ memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr,
	+ sizeof(ip6->ip6_dst));
	+
	+ udp->uh_sum = in6_cksum_pseudo(ip6, sizeof(struct udphdr),
	+ IPPROTO_UDP, 0);
	+
	+ m->m_pkthdr.csum_flags \|= CSUM_UDP_IPV6;
	+ m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
	+
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->transport_bytes_sent, m->m_pkthdr.len);
	+
	+ return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL,
	+ NULL));
	+ }
	+#endif
	+ default:
	+ panic("Unsupported address family %d",
	+ peer->remote.ss_family);
	+ }
	+}
	+
	+static int
	+ovpn_output(struct ifnet ifp, struct mbuf m, const struct sockaddr *dst,
	+ struct route *ro)
	+{
	+ return (ifp->if_transmit(ifp, m));
	+}
	+
	+static void
	+ovpn_rcv_ctrl(struct ovpn_softc sc, struct mbuf m, int off)
	+{
	+ /* Lop off the IP and UDP headers */
	+ m_adj_decap(m, off);
	+
	+ /* Keep in the local ring until userspace fetches it. */
	+ if (buf_ring_enqueue(sc->rxring, m) != 0) {
	+ counter_u64_add(sc->lost_ctrl_pkts_in, 1);
	+ m_freem(m);
	+ return;
	+ }
	+
	+ counter_u64_add(sc->received_ctrl_pkts, 1);
	+}
	+
	+static bool
	+ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq)
	+{
	+ uint32_t d;
	+
	+ mtx_lock(&key->replay_mtx);
	+
	+ /* Sequence number must be strictly greater than rx_seq */
	+ if (seq <= key->rx_seq) {
	+ mtx_unlock(&key->replay_mtx);
	+ return (false);
	+ }
	+
	+ /* Large jump. The packet authenticated okay, so just accept that. */
	+ if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) {
	+ key->rx_seq = seq;
	+ key->rx_window = 0;
	+ mtx_unlock(&key->replay_mtx);
	+ return (true);
	+ }
	+
	+ /* Happy case. */
	+ if ((seq == key->rx_seq + 1) && key->rx_window == 0) {
	+ key->rx_seq++;
	+ mtx_unlock(&key->replay_mtx);
	+ return (true);
	+ }
	+
	+ d = seq - key->rx_seq - 1;
	+
	+ if (key->rx_window & (1 << d)) {
	+ /* Dupe! */
	+ mtx_unlock(&key->replay_mtx);
	+ return (false);
	+ }
	+
	+ key->rx_window \|= 1 << d;
	+
	+ while (key->rx_window & 1) {
	+ key->rx_seq++;
	+ key->rx_window >>= 1;
	+ }
	+
	+ mtx_unlock(&key->replay_mtx);
	+
	+ return (true);
	+}
	+
	+static bool
	+ovpn_udp_input(struct mbuf m, int off, struct inpcb inp,
	+ const struct sockaddr sa, void ctx)
	+{
	+ struct ovpn_softc *sc = ctx;
	+ struct ovpn_wire_header ohdr;
	+ struct udphdr uhdr;
	+ struct ovpn_kkey *key;
	+ struct cryptop *crp;
	+ struct ovpn_kpeer *peer;
	+ size_t ohdrlen;
	+ uint32_t peerid;
	+ int ret;
	+ uint8_t op;
	+
	+ M_ASSERTPKTHDR(m);
	+
	+ OVPN_RLOCK_TRACKER;
	+
	+ counter_u64_add(sc->transport_bytes_received, m->m_pkthdr.len - off);
	+
	+ ohdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag);
	+
	+ if (m_length(m, NULL) < (off + sizeof(uhdr) + ohdrlen))
	+ return (false);
	+
	+ /* Copy the UDP and OVPN headers. */
	+ m_copydata(m, off, sizeof(uhdr), (caddr_t)&uhdr);
	+ m_copydata(m, off + sizeof(uhdr), ohdrlen, (caddr_t)&ohdr);
	+
	+ op = ntohl(ohdr.opcode) >> 24 >> OVPN_OP_SHIFT;
	+ peerid = ntohl(ohdr.opcode) & 0x00ffffff;
	+
	+ OVPN_RLOCK(sc);
	+
	+ peer = ovpn_find_peer(sc, peerid);
	+ if (peer == NULL) {
	+ OVPN_RUNLOCK(sc);
	+ return (false);
	+ }
	+
	+ /*
	+ * Simplify things by getting rid of the preceding headers, we don't
	+ * care about them.
	+ */
	+ m_adj_decap(m, off);
	+
	+ if (op != OVPN_OP_DATA_V2) {
	+ OVPN_RUNLOCK(sc);
	+ ovpn_rcv_ctrl(sc, m, sizeof(struct udphdr));
	+ INP_WLOCK(inp);
	+ udp_notify(inp, EAGAIN);
	+ INP_WUNLOCK(inp);
	+ return (true);
	+ }
	+
	+ key = ovpn_find_key(sc, peer, &ohdr);
	+ if (key == NULL \|\| key->decrypt == NULL) {
	+ OVPN_RUNLOCK(sc);
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ return (true);
	+ }
	+
	+ if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) {
	+ /* Now remove the outer headers */
	+ m_adj_decap(m, sizeof(struct udphdr) + ohdrlen);
	+
	+ ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr.seq),
	+ _ovpn_lock_trackerp);
	+ OVPN_UNLOCK_ASSERT(sc);
	+ return (true);
	+ }
	+
	+ ohdrlen += sizeof(ohdr.auth_tag);
	+ /* Now make sure we have the auth tag. */
	+ m_copydata(m, sizeof(uhdr), ohdrlen, (caddr_t)&ohdr);
	+
	+ /* Decrypt */
	+ crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT);
	+ if (crp == NULL) {
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ OVPN_RUNLOCK(sc);
	+ m_freem(m);
	+ return (true);
	+ }
	+
	+ crp->crp_payload_start = sizeof(struct udphdr) + sizeof(ohdr);
	+ crp->crp_payload_length = ntohs(uhdr.uh_ulen) -
	+ sizeof(uhdr) - sizeof(ohdr);
	+ crp->crp_op = CRYPTO_OP_DECRYPT;
	+
	+ /* AAD validation. */
	+ crp->crp_aad_length = sizeof(ohdr) -
	+ sizeof(ohdr.auth_tag);
	+ crp->crp_aad = &ohdr;
	+ crp->crp_aad_start = 0;
	+ crp->crp_op \|= CRYPTO_OP_VERIFY_DIGEST;
	+ crp->crp_digest_start = sizeof(struct udphdr) +
	+ offsetof(struct ovpn_wire_header, auth_tag);
	+
	+ crp->crp_flags \|= CRYPTO_F_IV_SEPARATE;
	+ memcpy(crp->crp_iv, &ohdr.seq, sizeof(ohdr.seq));
	+ memcpy(crp->crp_iv + sizeof(ohdr.seq), key->decrypt->nonce,
	+ key->decrypt->noncelen);
	+
	+ crypto_use_mbuf(crp, m);
	+ crp->crp_flags \|= CRYPTO_F_CBIFSYNC;
	+ crp->crp_callback = ovpn_encrypt_rx_cb;
	+ crp->crp_opaque = sc;
	+
	+ OVPN_RUNLOCK(sc);
	+ ret = crypto_dispatch(crp);
	+ if (ret != 0) {
	+ counter_u64_add(sc->lost_data_pkts_in, 1);
	+ m_freem(m);
	+ }
	+
	+ return (true);
	+}
	+
	+static void
	+ovpn_qflush(struct ifnet *ifp __unused)
	+{
	+
	+}
	+
	+static void
	+ovpn_flush_rxring(struct ovpn_softc *sc)
	+{
	+ struct mbuf *m;
	+ struct ovpn_notification *n;
	+
	+ OVPN_WASSERT(sc);
	+
	+ while (! buf_ring_empty(sc->rxring)) {
	+ m = buf_ring_dequeue_sc(sc->rxring);
	+ m_freem(m);
	+ }
	+
	+ while (! buf_ring_empty(sc->notifring)) {
	+ n = buf_ring_dequeue_sc(sc->notifring);
	+ free(n, M_OVPN);
	+ }
	+}
	+
	+#ifdef VIMAGE
	+static void
	+ovpn_reassign(struct ifnet ifp, struct vnet new_vnet __unused,
	+ char *unused __unused)
	+{
	+ struct ovpn_softc *sc = ifp->if_softc;
	+ int i;
	+ int ret __diagused;
	+
	+ i = 0;
	+
	+ OVPN_WLOCK(sc);
	+
	+ /* Flush keys & configuration. */
	+ do {
	+ if (sc->peers[i] != NULL) {
	+ ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
	+ MPASS(ret == 0);
	+ }
	+ i++;
	+ } while (i < OVPN_MAX_PEERS);
	+
	+ ovpn_flush_rxring(sc);
	+
	+ OVPN_WUNLOCK(sc);
	+}
	+#endif
	+
	+static int
	+ovpn_clone_match(struct if_clone ifc, const char name)
	+{
	+ /*
	+ * Allow all names that start with 'ovpn', specifically because pfSense
	+ * uses ovpnc1 / ovpns2
	+ */
	+ return (strncmp(ovpnname, name, strlen(ovpnname)) == 0);
	+}
	+
	+static int
	+ovpn_clone_create(struct if_clone ifc, char name, size_t len, caddr_t params)
	+{
	+ struct ovpn_softc *sc;
	+ struct ifnet *ifp;
	+ char *dp;
	+ int error, unit, wildcard;
	+
	+ /* Try to see if a special unit was requested. */
	+ error = ifc_name2unit(name, &unit);
	+ if (error != 0)
	+ return (error);
	+ wildcard = (unit < 0);
	+
	+ error = ifc_alloc_unit(ifc, &unit);
	+ if (error != 0)
	+ return (error);
	+
	+ /*
	+ * If no unit had been given, we need to adjust the ifName.
	+ */
	+ for (dp = name; *dp != '\0'; dp++);
	+ if (wildcard) {
	+ error = snprintf(dp, len - (dp - name), "%d", unit);
	+ if (error > len - (dp - name)) {
	+ /* ifName too long. */
	+ ifc_free_unit(ifc, unit);
	+ return (ENOSPC);
	+ }
	+ dp += error;
	+ }
	+
	+ /* Make sure it doesn't already exist. */
	+ if (ifunit(name) != NULL)
	+ return (EEXIST);
	+
	+ sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK \| M_ZERO);
	+ sc->ifp = if_alloc(IFT_ENC);
	+ rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE);
	+
	+ sc->rxring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
	+ sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL);
	+
	+ sc->lost_ctrl_pkts_in = counter_u64_alloc(M_WAITOK);
	+ sc->lost_ctrl_pkts_out = counter_u64_alloc(M_WAITOK);
	+ sc->lost_data_pkts_in = counter_u64_alloc(M_WAITOK);
	+ sc->lost_data_pkts_out = counter_u64_alloc(M_WAITOK);
	+ sc->received_ctrl_pkts = counter_u64_alloc(M_WAITOK);
	+ sc->received_data_pkts = counter_u64_alloc(M_WAITOK);
	+ sc->sent_ctrl_pkts = counter_u64_alloc(M_WAITOK);
	+ sc->sent_data_pkts = counter_u64_alloc(M_WAITOK);
	+ sc->transport_bytes_sent = counter_u64_alloc(M_WAITOK);
	+ sc->transport_bytes_received = counter_u64_alloc(M_WAITOK);
	+ sc->tunnel_bytes_sent = counter_u64_alloc(M_WAITOK);
	+ sc->tunnel_bytes_received = counter_u64_alloc(M_WAITOK);
	+
	+ ifp = sc->ifp;
	+ ifp->if_softc = sc;
	+ strlcpy(ifp->if_xname, name, IFNAMSIZ);
	+ ifp->if_dname = ovpngroupname;
	+ ifp->if_dunit = unit;
	+
	+ ifp->if_addrlen = 0;
	+ ifp->if_mtu = 1428;
	+ ifp->if_flags = IFF_POINTOPOINT \| IFF_MULTICAST;
	+ ifp->if_ioctl = ovpn_ioctl;
	+ ifp->if_transmit = ovpn_transmit;
	+ ifp->if_output = ovpn_output;
	+ ifp->if_qflush = ovpn_qflush;
	+#ifdef VIMAGE
	+ ifp->if_reassign = ovpn_reassign;
	+#endif
	+ ifp->if_capabilities \|= IFCAP_LINKSTATE;
	+ ifp->if_capenable \|= IFCAP_LINKSTATE;
	+
	+ if_attach(ifp);
	+ bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
	+
	+ return (0);
	+}
	+
	+static void
	+ovpn_clone_destroy_cb(struct epoch_context *ctx)
	+{
	+ struct ovpn_softc *sc;
	+
	+ sc = __containerof(ctx, struct ovpn_softc, epoch_ctx);
	+
	+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
	+ MPASS(sc->peers[i] == NULL);
	+ }
	+
	+ counter_u64_free(sc->lost_ctrl_pkts_in);
	+ counter_u64_free(sc->lost_ctrl_pkts_out);
	+ counter_u64_free(sc->lost_data_pkts_in);
	+ counter_u64_free(sc->lost_data_pkts_out);
	+ counter_u64_free(sc->received_ctrl_pkts);
	+ counter_u64_free(sc->received_data_pkts);
	+ counter_u64_free(sc->sent_ctrl_pkts);
	+ counter_u64_free(sc->sent_data_pkts);
	+ counter_u64_free(sc->transport_bytes_sent);
	+ counter_u64_free(sc->transport_bytes_received);
	+ counter_u64_free(sc->tunnel_bytes_sent);
	+ counter_u64_free(sc->tunnel_bytes_received);
	+
	+ if_free(sc->ifp);
	+ free(sc, M_OVPN);
	+}
	+
	+static int
	+ovpn_clone_destroy(struct if_clone ifc, struct ifnet ifp)
	+{
	+ struct ovpn_softc *sc;
	+ int unit;
	+ int i;
	+ int ret __diagused;
	+
	+ sc = ifp->if_softc;
	+ unit = ifp->if_dunit;
	+
	+ OVPN_WLOCK(sc);
	+ i = 0;
	+ do {
	+
	+ if (sc->peers[i] != NULL) {
	+ ret = _ovpn_del_peer(sc, sc->peers[i]->peerid);
	+ MPASS(ret == 0);
	+ }
	+ i++;
	+ } while (i < OVPN_MAX_PEERS);
	+
	+ ovpn_flush_rxring(sc);
	+ buf_ring_free(sc->rxring, M_OVPN);
	+ buf_ring_free(sc->notifring, M_OVPN);
	+
	+ OVPN_WUNLOCK(sc);
	+
	+ bpfdetach(ifp);
	+ if_detach(ifp);
	+ ifp->if_softc = NULL;
	+
	+ NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx);
	+
	+ if (unit != IF_DUNIT_NONE)
	+ ifc_free_unit(ifc, unit);
	+
	+ return (0);
	+}
	+
	+static void
	+vnet_ovpn_init(const void *unused __unused)
	+{
	+ V_ovpn_cloner = if_clone_advanced(ovpngroupname, 0, ovpn_clone_match,
	+ ovpn_clone_create, ovpn_clone_destroy);
	+}
	+VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
	+ vnet_ovpn_init, NULL);
	+
	+static void
	+vnet_ovpn_uninit(const void *unused __unused)
	+{
	+ if_clone_detach(V_ovpn_cloner);
	+}
	+VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY,
	+ vnet_ovpn_uninit, NULL);
	+
	+static int
	+ovpnmodevent(module_t mod, int type, void *data)
	+{
	+ switch (type) {
	+ case MOD_LOAD:
	+ /* Done in vnet_ovpn_init() */
	+ break;
	+ case MOD_UNLOAD:
	+ /* Done in vnet_ovpn_uninit() */
	+ break;
	+ default:
	+ return (EOPNOTSUPP);
	+ }
	+
	+ return (0);
	+}
	+
	+static moduledata_t ovpn_mod = {
	+ "if_ovpn",
	+ ovpnmodevent,
	+ 0
	+};
	+
	+DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
	+MODULE_VERSION(if_ovpn, 1);
	Index: sys/sys/priv.h
	===================================================================
	--- sys/sys/priv.h
	+++ sys/sys/priv.h
	@@ -348,6 +348,7 @@
	#define PRIV_NET_VXLAN 420 /* Administer vxlan. */
	#define PRIV_NET_SETLANPCP 421 /* Set LAN priority. */
	#define PRIV_NET_SETVLANPCP PRIV_NET_SETLANPCP /* Alias Set VLAN priority */
	+#define PRIV_NET_OVPN 422 /* Administer OpenVPN DCO. */

	/*
	* 802.11-related privileges.

File Metadata

Mime Type: text/plain
Expires: Wed, Feb 26, 9:06 AM (18 h, 35 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 16849892
Default Alt Text: D34340.id105429.diff (58 KB)

D34340.id105429.diffNo OneTemporaryActions

D34340.id105429.diffView Options

File Metadata

Event Timeline

D34340.id105429.diff
No OneTemporary
Actions

D34340.id105429.diff
View Options