Changeset View
Standalone View
sys/kern/uipc_ktls.c
Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | |||||
#include <vm/vm_pageout.h> | #include <vm/vm_pageout.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
struct ktls_wq { | struct ktls_wq { | ||||
struct mtx mtx; | struct mtx mtx; | ||||
STAILQ_HEAD(, mbuf) m_head; | STAILQ_HEAD(, mbuf) m_head; | ||||
STAILQ_HEAD(, socket) so_head; | STAILQ_HEAD(, socket) so_head; | ||||
bool running; | bool running; | ||||
int lastallocfail; | |||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
struct ktls_domain_info { | struct ktls_domain_info { | ||||
int count; | int count; | ||||
int cpu[MAXCPU]; | int cpu[MAXCPU]; | ||||
}; | }; | ||||
struct ktls_domain_info ktls_domains[MAXMEMDOM]; | struct ktls_domain_info ktls_domains[MAXMEMDOM]; | ||||
static struct ktls_wq *ktls_wq; | static struct ktls_wq *ktls_wq; | ||||
static struct proc *ktls_proc; | static struct proc *ktls_proc; | ||||
LIST_HEAD(, ktls_crypto_backend) ktls_backends; | LIST_HEAD(, ktls_crypto_backend) ktls_backends; | ||||
static struct rmlock ktls_backends_lock; | static struct rmlock ktls_backends_lock; | ||||
static uma_zone_t ktls_session_zone; | static uma_zone_t ktls_session_zone; | ||||
static uma_zone_t ktls_buffer_zone; | |||||
static uint16_t ktls_cpuid_lookup[MAXCPU]; | static uint16_t ktls_cpuid_lookup[MAXCPU]; | ||||
SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | ||||
"Kernel TLS offload"); | "Kernel TLS offload"); | ||||
SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, | ||||
"Kernel TLS offload stats"); | "Kernel TLS offload stats"); | ||||
static int ktls_allow_unload; | static int ktls_allow_unload; | ||||
SYSCTL_INT(_kern_ipc_tls, OID_AUTO, allow_unload, CTLFLAG_RDTUN, | SYSCTL_INT(_kern_ipc_tls, OID_AUTO, allow_unload, CTLFLAG_RDTUN, | ||||
&ktls_allow_unload, 0, "Allow software crypto modules to unload"); | &ktls_allow_unload, 0, "Allow software crypto modules to unload"); | ||||
#ifdef RSS | #ifdef RSS | ||||
static int ktls_bind_threads = 1; | static int ktls_bind_threads = 1; | ||||
#else | #else | ||||
static int ktls_bind_threads; | static int ktls_bind_threads; | ||||
#endif | #endif | ||||
SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN, | SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN, | ||||
&ktls_bind_threads, 0, | &ktls_bind_threads, 0, | ||||
"Bind crypto threads to cores (1) or cores and domains (2) at boot"); | "Bind crypto threads to cores (1) or cores and domains (2) at boot"); | ||||
static u_int ktls_maxlen = 16384; | static u_int ktls_maxlen = 16384; | ||||
SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RWTUN, | SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, | ||||
&ktls_maxlen, 0, "Maximum TLS record size"); | &ktls_maxlen, 0, "Maximum TLS record size"); | ||||
static int ktls_number_threads; | static int ktls_number_threads; | ||||
SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, | SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, | ||||
&ktls_number_threads, 0, | &ktls_number_threads, 0, | ||||
"Number of TLS threads in thread-pool"); | "Number of TLS threads in thread-pool"); | ||||
static bool ktls_offload_enable; | static bool ktls_offload_enable; | ||||
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, | SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, | ||||
&ktls_offload_enable, 0, | &ktls_offload_enable, 0, | ||||
"Enable support for kernel TLS offload"); | "Enable support for kernel TLS offload"); | ||||
static bool ktls_cbc_enable = true; | static bool ktls_cbc_enable = true; | ||||
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, | SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, | ||||
&ktls_cbc_enable, 1, | &ktls_cbc_enable, 1, | ||||
"Enable Support of AES-CBC crypto for kernel TLS"); | "Enable Support of AES-CBC crypto for kernel TLS"); | ||||
static bool ktls_sw_buffer_cache = true; | |||||
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, sw_buffer_cache, CTLFLAG_RDTUN, | |||||
&ktls_sw_buffer_cache, 1, | |||||
"Enable caching of output buffers for SW encryption"); | |||||
static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active); | static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD, | ||||
&ktls_tasks_active, "Number of active tasks"); | &ktls_tasks_active, "Number of active tasks"); | ||||
static COUNTER_U64_DEFINE_EARLY(ktls_cnt_tx_queued); | static COUNTER_U64_DEFINE_EARLY(ktls_cnt_tx_queued); | ||||
SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, sw_tx_inqueue, CTLFLAG_RD, | SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, sw_tx_inqueue, CTLFLAG_RD, | ||||
&ktls_cnt_tx_queued, | &ktls_cnt_tx_queued, | ||||
"Number of TLS records in queue to tasks for SW encryption"); | "Number of TLS records in queue to tasks for SW encryption"); | ||||
▲ Show 20 Lines • Show All 216 Lines • ▼ Show 20 Lines | if (ktls_bind_threads > 1 && inp->inp_numa_domain != M_NODOM) { | ||||
cpuid = di->cpu[inp->inp_flowid % di->count]; | cpuid = di->cpu[inp->inp_flowid % di->count]; | ||||
} else | } else | ||||
#endif | #endif | ||||
cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads]; | cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads]; | ||||
return (cpuid); | return (cpuid); | ||||
} | } | ||||
#endif | #endif | ||||
static int | |||||
ktls_buffer_import(void *arg, void **store, int count, int domain, int flags) | |||||
{ | |||||
vm_page_t m; | |||||
int i; | |||||
KASSERT((ktls_maxlen & PAGE_MASK) == 0, | |||||
("%s: ktls max length %d is not page size-aligned", | |||||
__func__, ktls_maxlen)); | |||||
for (i = 0; i < count; i++) { | |||||
m = vm_page_alloc_contig_domain(NULL, 0, domain, | |||||
VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_NODUMP | malloc2vm_flags(flags), | |||||
atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0, | |||||
VM_MEMATTR_DEFAULT); | |||||
if (m == NULL) | |||||
break; | |||||
store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); | |||||
} | |||||
return (i); | |||||
} | |||||
static void | static void | ||||
ktls_buffer_release(void *arg __unused, void **store, int count) | |||||
{ | |||||
vm_page_t m; | |||||
int i, j; | |||||
for (i = 0; i < count; i++) { | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); | |||||
for (j = 0; j < atop(ktls_maxlen); j++) { | |||||
(void)vm_page_unwire_noq(m + j); | |||||
vm_page_free(m + j); | |||||
} | |||||
} | |||||
} | |||||
static void | |||||
ktls_free_mext_contig(struct mbuf *m) | |||||
{ | |||||
M_ASSERTEXTPG(m); | |||||
uma_zfree(ktls_buffer_zone, (void *)PHYS_TO_DMAP(m->m_epg_pa[0])); | |||||
} | |||||
static void | |||||
ktls_init(void *dummy __unused) | ktls_init(void *dummy __unused) | ||||
{ | { | ||||
struct thread *td; | struct thread *td; | ||||
struct pcpu *pc; | struct pcpu *pc; | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int count, domain, error, i; | int count, domain, error, i; | ||||
rm_init(&ktls_backends_lock, "ktls backends"); | rm_init(&ktls_backends_lock, "ktls backends"); | ||||
LIST_INIT(&ktls_backends); | LIST_INIT(&ktls_backends); | ||||
ktls_wq = malloc(sizeof(*ktls_wq) * (mp_maxid + 1), M_KTLS, | ktls_wq = malloc(sizeof(*ktls_wq) * (mp_maxid + 1), M_KTLS, | ||||
M_WAITOK | M_ZERO); | M_WAITOK | M_ZERO); | ||||
ktls_session_zone = uma_zcreate("ktls_session", | ktls_session_zone = uma_zcreate("ktls_session", | ||||
sizeof(struct ktls_session), | sizeof(struct ktls_session), | ||||
NULL, NULL, NULL, NULL, | NULL, NULL, NULL, NULL, | ||||
UMA_ALIGN_CACHE, 0); | UMA_ALIGN_CACHE, 0); | ||||
if (ktls_sw_buffer_cache) { | |||||
ktls_buffer_zone = uma_zcache_create("ktls_buffers", | |||||
roundup2(ktls_maxlen, PAGE_SIZE), NULL, NULL, NULL, NULL, | |||||
ktls_buffer_import, ktls_buffer_release, NULL, | |||||
UMA_ZONE_FIRSTTOUCH); | |||||
} | |||||
/* | /* | ||||
* Initialize the workqueues to run the TLS work. We create a | * Initialize the workqueues to run the TLS work. We create a | ||||
* work queue for each CPU. | * work queue for each CPU. | ||||
*/ | */ | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
STAILQ_INIT(&ktls_wq[i].m_head); | STAILQ_INIT(&ktls_wq[i].m_head); | ||||
STAILQ_INIT(&ktls_wq[i].so_head); | STAILQ_INIT(&ktls_wq[i].so_head); | ||||
mtx_init(&ktls_wq[i].mtx, "ktls work queue", NULL, MTX_DEF); | mtx_init(&ktls_wq[i].mtx, "ktls work queue", NULL, MTX_DEF); | ||||
▲ Show 20 Lines • Show All 1,573 Lines • ▼ Show 20 Lines | ktls_enqueue_to_free(struct mbuf *m) | ||||
mtx_lock(&wq->mtx); | mtx_lock(&wq->mtx); | ||||
STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq); | STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq); | ||||
running = wq->running; | running = wq->running; | ||||
mtx_unlock(&wq->mtx); | mtx_unlock(&wq->mtx); | ||||
if (!running) | if (!running) | ||||
wakeup(wq); | wakeup(wq); | ||||
} | } | ||||
static void * | |||||
ktls_buffer_alloc(struct ktls_wq *wq, struct mbuf *m) | |||||
{ | |||||
void *buf; | |||||
if (m->m_epg_npgs <= 2) | |||||
return (NULL); | |||||
if (ktls_buffer_zone == NULL) | |||||
return (NULL); | |||||
if ((u_int)(ticks - wq->lastallocfail) < hz) { | |||||
/* | |||||
* Rate-limit allocation attempts after a failure. | |||||
* ktls_buffer_import() will acquire a per-domain mutex to check | |||||
* the free page queues and may fail consistently if memory is | |||||
* fragmented. | |||||
*/ | |||||
return (NULL); | |||||
} | |||||
buf = uma_zalloc(ktls_buffer_zone, M_NOWAIT | M_NORECLAIM); | |||||
if (buf == NULL) | |||||
wq->lastallocfail = ticks; | |||||
return (buf); | |||||
} | |||||
void | void | ||||
ktls_enqueue(struct mbuf *m, struct socket *so, int page_count) | ktls_enqueue(struct mbuf *m, struct socket *so, int page_count) | ||||
{ | { | ||||
struct ktls_wq *wq; | struct ktls_wq *wq; | ||||
bool running; | bool running; | ||||
KASSERT(((m->m_flags & (M_EXTPG | M_NOTREADY)) == | KASSERT(((m->m_flags & (M_EXTPG | M_NOTREADY)) == | ||||
(M_EXTPG | M_NOTREADY)), | (M_EXTPG | M_NOTREADY)), | ||||
Show All 16 Lines | ktls_enqueue(struct mbuf *m, struct socket *so, int page_count) | ||||
running = wq->running; | running = wq->running; | ||||
mtx_unlock(&wq->mtx); | mtx_unlock(&wq->mtx); | ||||
if (!running) | if (!running) | ||||
wakeup(wq); | wakeup(wq); | ||||
counter_u64_add(ktls_cnt_tx_queued, 1); | counter_u64_add(ktls_cnt_tx_queued, 1); | ||||
} | } | ||||
static __noinline void | static __noinline void | ||||
ktls_encrypt(struct mbuf *top) | ktls_encrypt(struct ktls_wq *wq, struct mbuf *top) | ||||
{ | { | ||||
struct ktls_session *tls; | struct ktls_session *tls; | ||||
struct socket *so; | struct socket *so; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | ||||
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | ||||
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)]; | ||||
vm_page_t pg; | vm_page_t pg; | ||||
void *cbuf; | |||||
int error, i, len, npages, off, total_pages; | int error, i, len, npages, off, total_pages; | ||||
bool is_anon; | bool is_anon; | ||||
so = top->m_epg_so; | so = top->m_epg_so; | ||||
tls = top->m_epg_tls; | tls = top->m_epg_tls; | ||||
KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); | KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top)); | ||||
KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); | KASSERT(so != NULL, ("so = NULL, top = %p\n", top)); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
Show All 25 Lines | KASSERT(m->m_epg_tls == tls, | ||||
("different TLS sessions in a single mbuf chain: %p vs %p", | ("different TLS sessions in a single mbuf chain: %p vs %p", | ||||
tls, m->m_epg_tls)); | tls, m->m_epg_tls)); | ||||
KASSERT((m->m_flags & (M_EXTPG | M_NOTREADY)) == | KASSERT((m->m_flags & (M_EXTPG | M_NOTREADY)) == | ||||
(M_EXTPG | M_NOTREADY), | (M_EXTPG | M_NOTREADY), | ||||
("%p not unready & nomap mbuf (top = %p)\n", m, top)); | ("%p not unready & nomap mbuf (top = %p)\n", m, top)); | ||||
KASSERT(npages + m->m_epg_npgs <= total_pages, | KASSERT(npages + m->m_epg_npgs <= total_pages, | ||||
("page count mismatch: top %p, total_pages %d, m %p", top, | ("page count mismatch: top %p, total_pages %d, m %p", top, | ||||
total_pages, m)); | total_pages, m)); | ||||
KASSERT(ptoa(m->m_epg_npgs) <= ktls_maxlen, | |||||
("page count %d larger than maximum frame length %d", | |||||
m->m_epg_npgs, ktls_maxlen)); | |||||
/* | /* | ||||
* Generate source and destination ivoecs to pass to | * Generate source and destination ivoecs to pass to | ||||
* the SW encryption backend. For writable mbufs, the | * the SW encryption backend. For writable mbufs, the | ||||
* destination iovec is a copy of the source and | * destination iovec is a copy of the source and | ||||
* encryption is done in place. For file-backed mbufs | * encryption is done in place. For file-backed mbufs | ||||
* (from sendfile), anonymous wired pages are | * (from sendfile), anonymous wired pages are | ||||
* allocated and assigned to the destination iovec. | * allocated and assigned to the destination iovec. | ||||
*/ | */ | ||||
is_anon = (m->m_epg_flags & EPG_FLAG_ANON) != 0; | is_anon = (m->m_epg_flags & EPG_FLAG_ANON) != 0; | ||||
off = m->m_epg_1st_off; | off = m->m_epg_1st_off; | ||||
for (i = 0; i < m->m_epg_npgs; i++, off = 0) { | for (i = 0; i < m->m_epg_npgs; i++, off = 0) { | ||||
len = m_epg_pagelen(m, i, off); | len = m_epg_pagelen(m, i, off); | ||||
src_iov[i].iov_len = len; | src_iov[i].iov_len = len; | ||||
src_iov[i].iov_base = | src_iov[i].iov_base = | ||||
(char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) + | (char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) + off; | ||||
off; | } | ||||
if (is_anon) { | if (is_anon) { | ||||
dst_iov[i].iov_base = src_iov[i].iov_base; | memcpy(dst_iov, src_iov, i * sizeof(struct iovec)); | ||||
dst_iov[i].iov_len = src_iov[i].iov_len; | } else if ((cbuf = ktls_buffer_alloc(wq, m)) != NULL) { | ||||
gallatin: Would it make sense to test for m_epg_npgs > 2 in order to avoid using a large page for a small… | |||||
Done Inline ActionsSo I think it would be better to always use the cache zone, i.e., we should go in the opposite direction. Suppose there's some severe memory shortage that causes the cache to be drained, and after that memory is too fragmented for contig allocations to succeed. Then, for each request we will miss in the cache, try to allocate a contig run and fail, and fall back to page-by-page allocations. So each KTLS thread will be acquiring multiple global locks per request with nothing to show for it. Instead, ktls_buffer_import() should allocate and map a non-contiguous run of pages if vm_page_alloc_contig_domain() fails. Do you think the extra memory usage for small requests is significant? Another possibility might be to stop using the cache zone entirely if allocations start failing, but IMO it is nicer to keep the complexity in ktls_buffer_import(). markj: So I think it would be better to always use the cache zone, i.e., we should go in the opposite… | |||||
Not Done Inline ActionsYou're talking about memory exhaustion causing zones to be drained. I'm just concerned about just more efficient use of memory in general, assuming there is something else besides this workload that could make effective use of the remaining memory. For the Netflix workload (large media files), I think the risk of wasting memory due to small requests is minimal. However, I could imagine other people using KTLS on a more generic web workload and typically sending files which are smaller than 16K. For example, loading a few random pages in Firefox with the Developer->Network thing running, I see lots (> 50%) of requests for files smaller than 16K. Would "right sizing" allocations and avoiding the 16k zone for small allocations would be less efficient than what we have today? gallatin: You're talking about memory exhaustion causing zones to be drained. I'm just concerned about… | |||||
Done Inline ActionsI had it in my head that these issues are related, but indeed, they're orthogonal. I agree now that we should fall back to page-by-page allocation for small requests. markj: I had it in my head that these issues are related, but indeed, they're orthogonal. I agree now… | |||||
Done Inline ActionsI ended up adding a primitive rate-limiting mechanism. We can't allocate KVA and do page-by-page allocation in ktls_buffer_import() very easily because the mapping is not saved in the mbuf, so ktls_free_mext_contig() doesn't know which buffer to free. It occurred to me while I was updating the revision that we could stash the mapping in the first vm_page structure, by adding a field to the plinks union. That has the disadvantage of requiring a PHYS_TO_VM_PAGE() call in ktls_free_mext_contig() but that's not too bad. That can be done in a separate diff though. markj: I ended up adding a primitive rate-limiting mechanism. We can't allocate KVA and do page-by… | |||||
continue; | len = ptoa(m->m_epg_npgs - 1) + m->m_epg_last_len - | ||||
} | m->m_epg_1st_off; | ||||
retry_page: | dst_iov[0].iov_base = (char *)cbuf + m->m_epg_1st_off; | ||||
pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | dst_iov[0].iov_len = len; | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP | VM_ALLOC_WIRED); | parray[0] = DMAP_TO_PHYS((vm_offset_t)cbuf); | ||||
if (pg == NULL) { | i = 1; | ||||
vm_wait(NULL); | } else { | ||||
goto retry_page; | cbuf = NULL; | ||||
} | off = m->m_epg_1st_off; | ||||
for (i = 0; i < m->m_epg_npgs; i++, off = 0) { | |||||
do { | |||||
pg = vm_page_alloc(NULL, 0, | |||||
VM_ALLOC_NORMAL | | |||||
VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_NODUMP | | |||||
VM_ALLOC_WIRED | | |||||
VM_ALLOC_WAITFAIL); | |||||
} while (pg == NULL); | |||||
len = m_epg_pagelen(m, i, off); | |||||
parray[i] = VM_PAGE_TO_PHYS(pg); | parray[i] = VM_PAGE_TO_PHYS(pg); | ||||
dst_iov[i].iov_base = | dst_iov[i].iov_base = | ||||
(char *)(void *)PHYS_TO_DMAP(parray[i]) + off; | (char *)(void *)PHYS_TO_DMAP( | ||||
parray[i]) + off; | |||||
dst_iov[i].iov_len = len; | dst_iov[i].iov_len = len; | ||||
} | } | ||||
} | |||||
if (__predict_false(m->m_epg_npgs == 0)) { | if (__predict_false(m->m_epg_npgs == 0)) { | ||||
/* TLS 1.0 empty fragment. */ | /* TLS 1.0 empty fragment. */ | ||||
npages++; | npages++; | ||||
} else | } else | ||||
npages += i; | npages += m->m_epg_npgs; | ||||
error = (*tls->sw_encrypt)(tls, | error = (*tls->sw_encrypt)(tls, | ||||
(const struct tls_record_layer *)m->m_epg_hdr, | (const struct tls_record_layer *)m->m_epg_hdr, | ||||
m->m_epg_trail, src_iov, dst_iov, i, m->m_epg_seqno, | m->m_epg_trail, src_iov, dst_iov, m->m_epg_npgs, i, | ||||
m->m_epg_record_type); | m->m_epg_seqno, m->m_epg_record_type); | ||||
if (error) { | if (error) { | ||||
counter_u64_add(ktls_offload_failed_crypto, 1); | counter_u64_add(ktls_offload_failed_crypto, 1); | ||||
break; | break; | ||||
} | } | ||||
/* | /* | ||||
* For file-backed mbufs, release the file-backed | * For file-backed mbufs, release the file-backed | ||||
* pages and replace them in the ext_pgs array with | * pages and replace them in the ext_pgs array with | ||||
* the anonymous wired pages allocated above. | * the anonymous wired pages allocated above. | ||||
*/ | */ | ||||
if (!is_anon) { | if (!is_anon) { | ||||
/* Free the old pages. */ | /* Free the old pages. */ | ||||
m->m_ext.ext_free(m); | m->m_ext.ext_free(m); | ||||
/* Replace them with the new pages. */ | /* Replace them with the new pages. */ | ||||
if (cbuf != NULL) { | |||||
for (i = 0; i < m->m_epg_npgs; i++) | for (i = 0; i < m->m_epg_npgs; i++) | ||||
m->m_epg_pa[i] = parray[0] + ptoa(i); | |||||
/* Contig pages should go back to the cache. */ | |||||
m->m_ext.ext_free = ktls_free_mext_contig; | |||||
} else { | |||||
for (i = 0; i < m->m_epg_npgs; i++) | |||||
m->m_epg_pa[i] = parray[i]; | m->m_epg_pa[i] = parray[i]; | ||||
/* Use the basic free routine. */ | /* Use the basic free routine. */ | ||||
m->m_ext.ext_free = mb_free_mext_pgs; | m->m_ext.ext_free = mb_free_mext_pgs; | ||||
} | |||||
/* Pages are now writable. */ | /* Pages are now writable. */ | ||||
m->m_epg_flags |= EPG_FLAG_ANON; | m->m_epg_flags |= EPG_FLAG_ANON; | ||||
} | } | ||||
/* | /* | ||||
* Drop a reference to the session now that it is no | * Drop a reference to the session now that it is no | ||||
* longer needed. Existing code depends on encrypted | * longer needed. Existing code depends on encrypted | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | for (;;) { | ||||
STAILQ_CONCAT(&local_so_head, &wq->so_head); | STAILQ_CONCAT(&local_so_head, &wq->so_head); | ||||
mtx_unlock(&wq->mtx); | mtx_unlock(&wq->mtx); | ||||
STAILQ_FOREACH_SAFE(m, &local_m_head, m_epg_stailq, n) { | STAILQ_FOREACH_SAFE(m, &local_m_head, m_epg_stailq, n) { | ||||
if (m->m_epg_flags & EPG_FLAG_2FREE) { | if (m->m_epg_flags & EPG_FLAG_2FREE) { | ||||
ktls_free(m->m_epg_tls); | ktls_free(m->m_epg_tls); | ||||
uma_zfree(zone_mbuf, m); | uma_zfree(zone_mbuf, m); | ||||
} else { | } else { | ||||
ktls_encrypt(m); | ktls_encrypt(wq, m); | ||||
counter_u64_add(ktls_cnt_tx_queued, -1); | counter_u64_add(ktls_cnt_tx_queued, -1); | ||||
} | } | ||||
} | } | ||||
STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) { | STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) { | ||||
ktls_decrypt(so); | ktls_decrypt(so); | ||||
counter_u64_add(ktls_cnt_rx_queued, -1); | counter_u64_add(ktls_cnt_rx_queued, -1); | ||||
} | } | ||||
} | } | ||||
} | } |
Would it make sense to test for m_epg_npgs > 2 in order to avoid using a large page for a small amount of data?