Index: sys/net/netisr.h =================================================================== --- sys/net/netisr.h +++ sys/net/netisr.h @@ -59,6 +59,7 @@ #define NETISR_EPAIR 8 /* if_epair(4) */ #define NETISR_IP_DIRECT 9 /* direct-dispatch IPv4 */ #define NETISR_IPV6_DIRECT 10 /* direct-dispatch IPv6 */ +#define NETISR_IPSEC 11 /* IPsec processing queue */ /* * Protocol ordering and affinity policy constants. See the detailed Index: sys/netipsec/ipsec.h =================================================================== --- sys/netipsec/ipsec.h +++ sys/netipsec/ipsec.h @@ -256,7 +256,9 @@ int ipsec_run_hhooks(struct ipsec_ctx_data *ctx, int direction); VNET_DECLARE(int, ipsec_debug); +VNET_DECLARE(int, ipsec_use_netisr); #define V_ipsec_debug VNET(ipsec_debug) +#define V_ipsec_use_netisr VNET(ipsec_use_netisr) #ifdef REGRESSION VNET_DECLARE(int, ipsec_replay); @@ -336,6 +338,8 @@ int ipsec_process_done(struct mbuf *, struct secpolicy *, struct secasvar *, u_int); +void ipsec_netisr_output(struct mbuf *); + extern void m_checkalignment(const char* where, struct mbuf *m0, int off, int len); extern struct mbuf *m_makespace(struct mbuf *m0, int skip, int hlen, int *off); Index: sys/netipsec/ipsec.c =================================================================== --- sys/netipsec/ipsec.c +++ sys/netipsec/ipsec.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include @@ -141,6 +142,27 @@ return (error); } +VNET_DEFINE(int, ipsec_use_netisr) = 0; +static struct netisr_handler ipsec_output_nh = { + .nh_name = "ipsec", + .nh_handler = ipsec_netisr_output, + .nh_proto = NETISR_IPSEC, + .nh_policy = NETISR_POLICY_SOURCE, +}; + +static int +sysctl_ipsec_queue_maxlen(SYSCTL_HANDLER_ARGS) +{ + int error, qlimit; + + netisr_getqlimit(&ipsec_output_nh, &qlimit); + error = sysctl_handle_int(oidp, &qlimit, 0, req); + if (error || !req->newptr) + return (error); + if (qlimit < 1) + return (EINVAL); + return (netisr_setqlimit(&ipsec_output_nh, qlimit)); +} /* * Crypto support requirements: * @@ -204,6 +226,12 @@ SYSCTL_INT(_net_inet_ipsec, OID_AUTO, filtertunnel, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip4_filtertunnel), 0, "If set, filter packets from an IPsec tunnel."); +SYSCTL_INT(_net_inet_ipsec, OID_AUTO, use_netisr, + CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipsec_use_netisr), 0, + "If set, use deferred IPsec processing for outbound packets."); +SYSCTL_PROC(_net_inet_ipsec, OID_AUTO, intr_queue_maxlen, + CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_ipsec_queue_maxlen, "I", + "Maximum size of the IPsec output queue"); SYSCTL_VNET_PCPUSTAT(_net_inet_ipsec, OID_AUTO, ipsecstats, struct ipsecstat, ipsec4stat, "IPsec IPv4 statistics."); @@ -267,6 +295,12 @@ SYSCTL_INT(_net_inet6_ipsec6, OID_AUTO, filtertunnel, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ip6_filtertunnel), 0, "If set, filter packets from an IPsec tunnel."); +SYSCTL_INT(_net_inet6_ipsec6, OID_AUTO, use_netisr, + CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipsec_use_netisr), 0, + "If set, use deferred IPsec processing for outbound packets."); +SYSCTL_PROC(_net_inet6_ipsec6, OID_AUTO, intr_queue_maxlen, + CTLTYPE_INT | CTLFLAG_RW, 0, 0, sysctl_ipsec_queue_maxlen, "I", + "Maximum size of the IPsec output queue"); SYSCTL_VNET_PCPUSTAT(_net_inet6_ipsec6, IPSECCTL_STATS, ipsecstats, struct ipsecstat, ipsec6stat, "IPsec IPv6 statistics."); #endif /* INET6 */ @@ -1374,6 +1408,17 @@ key_bumpspgen(); } else printf("%s: failed to initialize default policy\n", __func__); +#ifdef VIMAGE + if (!IS_DEFAULT_VNET(curvnet)) + netisr_register_vnet(&ipsec_output_nh); + else +#endif + netisr_register(&ipsec_output_nh); + /* + * By default use deferred IPsec processing when in the system is + * configured less than 4 pages for the kernel stack. + */ + V_ipsec_use_netisr = (kstack_pages < 4); } @@ -1381,6 +1426,12 @@ def_policy_uninit(const void *unused __unused) { +#ifdef VIMAGE + if (!IS_DEFAULT_VNET(curvnet)) + netisr_unregister_vnet(&ipsec_output_nh); + else +#endif + netisr_unregister(&ipsec_output_nh); if (V_def_policy != NULL) { key_freesp(&V_def_policy); key_bumpspgen(); Index: sys/netipsec/ipsec_output.c =================================================================== --- sys/netipsec/ipsec_output.c +++ sys/netipsec/ipsec_output.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -92,6 +93,13 @@ #include +#define MTAG_IPSEC 1487673374 +struct ipsec_nh_ctx { + struct secpolicy *sp; + struct secasvar *sav; + u_int idx; +}; + #define IPSEC_OSTAT_INC(proto, name) do { \ if ((proto) == IPPROTO_ESP) \ ESPSTAT_INC(esps_##name); \ @@ -101,7 +109,9 @@ IPCOMPSTAT_INC(ipcomps_##name); \ } while (0) -static int ipsec_encap(struct mbuf **mp, struct secasindex *saidx); +static int ipsec_encap(struct mbuf **, struct secasindex *); +static int ipsec_queue_output(struct mbuf *, struct secpolicy *, + struct secasvar *, u_int); #ifdef INET static struct secasvar * @@ -181,34 +191,14 @@ * IPsec output logic for IPv4. */ static int -ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp, u_int idx) +ipsec4_xform_output(struct mbuf *m, struct secpolicy *sp, + struct secasvar *sav, u_int idx) { - char sbuf[IPSEC_ADDRSTRLEN], dbuf[IPSEC_ADDRSTRLEN]; struct ipsec_ctx_data ctx; union sockaddr_union *dst; - struct secasvar *sav; struct ip *ip; int error, i, off; - IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); - - /* - * We hold the reference to SP. Content of SP couldn't be changed. - * Craft secasindex and do lookup for suitable SA. - * Then do encapsulation if needed and call xform's output. - * We need to store SP in the xform callback parameters. - * In xform callback we will extract SP and it can be used to - * determine next transform. At the end of transform we can - * release reference to SP. - */ - sav = ipsec4_allocsa(m, sp, &idx, &error); - if (sav == NULL) { - if (error == EJUSTRETURN) { /* No IPsec required */ - key_freesp(&sp); - return (error); - } - goto bad; - } /* * XXXAE: most likely ip_sum at this point is wrong. */ @@ -230,12 +220,8 @@ ip->ip_sum = in_cksum(m, ip->ip_hl << 2); error = ipsec_encap(&m, &sav->sah->saidx); if (error != 0) { - DPRINTF(("%s: encapsulation for SA %s->%s " - "SPI 0x%08x failed with error %d\n", __func__, - ipsec_address(&sav->sah->saidx.src, sbuf, - sizeof(sbuf)), - ipsec_address(&sav->sah->saidx.dst, dbuf, - sizeof(dbuf)), ntohl(sav->spi), error)); + DPRINTF(("%s: encapsulation for SPI 0x%08x failed\n", + __func__, ntohl(sav->spi))); /* XXXAE: IPSEC_OSTAT_INC(tunnel); */ goto bad; } @@ -273,12 +259,50 @@ goto bad; } error = (*sav->tdb_xform->xf_output)(m, sp, sav, idx, i, off); - if (error != 0) { - key_freesav(&sav); - key_freesp(&sp); - } + /* mbuf was consumed by xform_output */ return (error); bad: + if (m != NULL) + m_freem(m); + return (error); +} + +static int +ipsec4_perform_request(struct mbuf *m, struct secpolicy *sp, u_int idx) +{ + struct secasvar *sav; + int error; + + IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); + /* + * We hold the reference to SP. Content of SP couldn't be changed. + * Craft secasindex and do lookup for suitable SA. + * Then do encapsulation if needed and call xform's output. + * We need to store SP in the xform callback parameters. + * In xform callback we will extract SP and it can be used to + * determine next transform. At the end of transform we can + * release reference to SP. + */ + sav = ipsec4_allocsa(m, sp, &idx, &error); + if (sav == NULL) { + if (error == EJUSTRETURN) { /* No IPsec required */ + key_freesp(&sp); + return (error); + } + goto bad; + } + + if (V_ipsec_use_netisr != 0) + error = ipsec_queue_output(m, sp, sav, idx); + else + error = ipsec4_xform_output(m, sp, sav, idx); + + if (error == 0) + return (error); + if (error == ENOMEM) + IPSECSTAT_INC(ips_out_nomem); + m = NULL; /* mbuf was consumed by netisr/xform_output */ +bad: IPSECSTAT_INC(ips_out_inval); if (m != NULL) m_freem(m); @@ -499,26 +523,14 @@ * IPsec output logic for IPv6. */ static int -ipsec6_perform_request(struct mbuf *m, struct secpolicy *sp, u_int idx) +ipsec6_xform_output(struct mbuf *m, struct secpolicy *sp, + struct secasvar *sav, u_int idx) { - char sbuf[IPSEC_ADDRSTRLEN], dbuf[IPSEC_ADDRSTRLEN]; struct ipsec_ctx_data ctx; union sockaddr_union *dst; - struct secasvar *sav; struct ip6_hdr *ip6; int error, i, off; - IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); - - sav = ipsec6_allocsa(m, sp, &idx, &error); - if (sav == NULL) { - if (error == EJUSTRETURN) { /* No IPsec required */ - key_freesp(&sp); - return (error); - } - goto bad; - } - /* Fix IP length in case if it is not set yet. */ ip6 = mtod(m, struct ip6_hdr *); ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); @@ -543,12 +555,8 @@ } error = ipsec_encap(&m, &sav->sah->saidx); if (error != 0) { - DPRINTF(("%s: encapsulation for SA %s->%s " - "SPI 0x%08x failed with error %d\n", __func__, - ipsec_address(&sav->sah->saidx.src, sbuf, - sizeof(sbuf)), - ipsec_address(&sav->sah->saidx.dst, dbuf, - sizeof(dbuf)), ntohl(sav->spi), error)); + DPRINTF(("%s: encapsulation for SPI 0x%08x failed\n", + __func__, ntohl(sav->spi))); /* XXXAE: IPSEC_OSTAT_INC(tunnel); */ goto bad; } @@ -581,12 +589,42 @@ goto bad; } error = (*sav->tdb_xform->xf_output)(m, sp, sav, idx, i, off); - if (error != 0) { - key_freesav(&sav); - key_freesp(&sp); - } + /* mbuf was consumed by xform_output */ return (error); bad: + if (m != NULL) + m_freem(m); + return (error); +} + +static int +ipsec6_perform_request(struct mbuf *m, struct secpolicy *sp, u_int idx) +{ + struct secasvar *sav; + int error; + + IPSEC_ASSERT(idx < sp->tcount, ("Wrong IPsec request index %d", idx)); + + sav = ipsec6_allocsa(m, sp, &idx, &error); + if (sav == NULL) { + if (error == EJUSTRETURN) { /* No IPsec required */ + key_freesp(&sp); + return (error); + } + goto bad; + } + + if (V_ipsec_use_netisr != 0) + error = ipsec_queue_output(m, sp, sav, idx); + else + error = ipsec6_xform_output(m, sp, sav, idx); + + if (error == 0) + return (error); + if (error == ENOMEM) + IPSEC6STAT_INC(ips_out_nomem); + m = NULL; /* mbuf was consumed by netisr/xform_output */ +bad: IPSEC6STAT_INC(ips_out_inval); if (m != NULL) m_freem(m); @@ -969,3 +1007,61 @@ return (0); } +static int +ipsec_queue_output(struct mbuf *m, struct secpolicy *sp, + struct secasvar *sav, u_int idx) +{ + struct ipsec_nh_ctx *ctx; + struct m_tag *mtag; + + mtag = m_tag_alloc(MTAG_IPSEC, 0, sizeof(*ctx), M_NOWAIT); + if (mtag == NULL) { + m_freem(m); + return (ENOMEM); + } + m_tag_prepend(m, mtag); + ctx = (struct ipsec_nh_ctx *)(mtag + 1); + ctx->sp = sp; + ctx->sav = sav; + ctx->idx = idx; + return (netisr_queue_src(NETISR_IPSEC, (uintptr_t)sav->spi, m)); +} + +void +ipsec_netisr_output(struct mbuf *m) +{ + struct ipsec_nh_ctx *ctx; + struct secpolicy *sp; + struct secasvar *sav; + struct m_tag *mtag; + + mtag = m_tag_locate(m, MTAG_IPSEC, 0, NULL); + if (mtag == NULL) { + m_freem(m); + return; + } + ctx = (struct ipsec_nh_ctx *)(mtag + 1); + sp = ctx->sp; + sav = ctx->sav; + switch (sav->sah->saidx.dst.sa.sa_family) { +#ifdef INET + case AF_INET: + if (ipsec4_xform_output(m, sp, sav, ctx->idx) == 0) + return; + IPSECSTAT_INC(ips_out_inval); + break; +#endif +#ifdef INET6 + case AF_INET6: + if (ipsec6_xform_output(m, sp, sav, ctx->idx) == 0) + return; + IPSEC6STAT_INC(ips_out_inval); + break; +#endif + default: + m_freem(m); + } + key_freesav(&sav); + key_freesp(&sp); +} +