Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/uipc_mbuf.c
Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mbuf.h> | #include <sys/mbuf.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/domain.h> | #include <sys/domain.h> | ||||
#include <sys/protosw.h> | #include <sys/protosw.h> | ||||
#include <sys/uio.h> | #include <sys/uio.h> | ||||
#include <sys/sdt.h> | #include <sys/sdt.h> | ||||
#include <vm/vm.h> | |||||
#include <vm/vm_pageout.h> | |||||
#include <vm/vm_page.h> | |||||
#include <sys/vmmeter.h> | |||||
SDT_PROBE_DEFINE5_XLATE(sdt, , , m__init, | SDT_PROBE_DEFINE5_XLATE(sdt, , , m__init, | ||||
"struct mbuf *", "mbufinfo_t *", | "struct mbuf *", "mbufinfo_t *", | ||||
"uint32_t", "uint32_t", | "uint32_t", "uint32_t", | ||||
"uint16_t", "uint16_t", | "uint16_t", "uint16_t", | ||||
"uint32_t", "uint32_t", | "uint32_t", "uint32_t", | ||||
"uint32_t", "uint32_t"); | "uint32_t", "uint32_t"); | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | mb_dupcl(struct mbuf *n, struct mbuf *m) | ||||
* free routine and its arguments. Exclusion is EXT_EXTREF, | * free routine and its arguments. Exclusion is EXT_EXTREF, | ||||
* where 'ext_cnt' doesn't point into mbuf at all. | * where 'ext_cnt' doesn't point into mbuf at all. | ||||
*/ | */ | ||||
if (m->m_ext.ext_type == EXT_EXTREF) | if (m->m_ext.ext_type == EXT_EXTREF) | ||||
bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext)); | bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext)); | ||||
else | else | ||||
bcopy(&m->m_ext, &n->m_ext, m_ext_copylen); | bcopy(&m->m_ext, &n->m_ext, m_ext_copylen); | ||||
n->m_flags |= M_EXT; | n->m_flags |= M_EXT; | ||||
n->m_flags |= m->m_flags & M_RDONLY; | n->m_flags |= m->m_flags & (M_RDONLY | M_NOMAP); | ||||
/* See if this is the mbuf that holds the embedded refcount. */ | /* See if this is the mbuf that holds the embedded refcount. */ | ||||
if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { | if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { | ||||
refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count; | refcnt = n->m_ext.ext_cnt = &m->m_ext.ext_count; | ||||
n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF; | n->m_ext.ext_flags &= ~EXT_FLAG_EMBREF; | ||||
} else { | } else { | ||||
KASSERT(m->m_ext.ext_cnt != NULL, | KASSERT(m->m_ext.ext_cnt != NULL, | ||||
("%s: no refcounting pointer on %p", __func__, m)); | ("%s: no refcounting pointer on %p", __func__, m)); | ||||
Show All 27 Lines | |||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { | for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { | ||||
KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", | KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", | ||||
__func__, m, m0)); | __func__, m, m0)); | ||||
if (m->m_flags & M_PKTHDR) | if (m->m_flags & M_PKTHDR) | ||||
m_demote_pkthdr(m); | m_demote_pkthdr(m); | ||||
m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags); | m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | | ||||
M_NOMAP | flags); | |||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Sanity checks on mbuf (chain) for use in KASSERT() and general | * Sanity checks on mbuf (chain) for use in KASSERT() and general | ||||
* debugging. | * debugging. | ||||
* Returns 0 or panics when bad and 1 on all tests passed. | * Returns 0 or panics when bad and 1 on all tests passed. | ||||
* Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they | * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they | ||||
▲ Show 20 Lines • Show All 113 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
#ifdef MAC | #ifdef MAC | ||||
/* | /* | ||||
* XXXMAC: It could be this should also occur for non-MAC? | * XXXMAC: It could be this should also occur for non-MAC? | ||||
*/ | */ | ||||
if (to->m_flags & M_PKTHDR) | if (to->m_flags & M_PKTHDR) | ||||
m_tag_delete_chain(to, NULL); | m_tag_delete_chain(to, NULL); | ||||
#endif | #endif | ||||
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); | to->m_flags = (from->m_flags & M_COPYFLAGS) | | ||||
(to->m_flags & (M_EXT | M_NOMAP)); | |||||
if ((to->m_flags & M_EXT) == 0) | if ((to->m_flags & M_EXT) == 0) | ||||
to->m_data = to->m_pktdat; | to->m_data = to->m_pktdat; | ||||
to->m_pkthdr = from->m_pkthdr; /* especially tags */ | to->m_pkthdr = from->m_pkthdr; /* especially tags */ | ||||
SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ | SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ | ||||
from->m_flags &= ~M_PKTHDR; | from->m_flags &= ~M_PKTHDR; | ||||
if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) { | if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) { | ||||
from->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; | from->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; | ||||
from->m_pkthdr.snd_tag = NULL; | from->m_pkthdr.snd_tag = NULL; | ||||
Show All 21 Lines | #if 0 | ||||
/* Note: with MAC, this may not be a good assertion. */ | /* Note: with MAC, this may not be a good assertion. */ | ||||
KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); | KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); | ||||
#endif | #endif | ||||
MBUF_CHECKSLEEP(how); | MBUF_CHECKSLEEP(how); | ||||
#ifdef MAC | #ifdef MAC | ||||
if (to->m_flags & M_PKTHDR) | if (to->m_flags & M_PKTHDR) | ||||
m_tag_delete_chain(to, NULL); | m_tag_delete_chain(to, NULL); | ||||
#endif | #endif | ||||
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); | to->m_flags = (from->m_flags & M_COPYFLAGS) | | ||||
(to->m_flags & (M_EXT | M_NOMAP)); | |||||
if ((to->m_flags & M_EXT) == 0) | if ((to->m_flags & M_EXT) == 0) | ||||
to->m_data = to->m_pktdat; | to->m_data = to->m_pktdat; | ||||
to->m_pkthdr = from->m_pkthdr; | to->m_pkthdr = from->m_pkthdr; | ||||
if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) | if (from->m_pkthdr.csum_flags & CSUM_SND_TAG) | ||||
m_snd_tag_ref(from->m_pkthdr.snd_tag); | m_snd_tag_ref(from->m_pkthdr.snd_tag); | ||||
SLIST_INIT(&to->m_pkthdr.tags); | SLIST_INIT(&to->m_pkthdr.tags); | ||||
return (m_tag_copy_chain(to, from, how)); | return (m_tag_copy_chain(to, from, how)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 148 Lines • ▼ Show 20 Lines | while (m) { | ||||
m = m->m_next; | m = m->m_next; | ||||
} | } | ||||
return top; | return top; | ||||
nospace: | nospace: | ||||
m_freem(top); | m_freem(top); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
static void | |||||
m_copyfromunmapped(const struct mbuf *m, int off, int len, caddr_t cp) | |||||
{ | |||||
struct iovec iov; | |||||
struct uio uio; | |||||
int error; | |||||
KASSERT(off >= 0, ("m_copyfromunmapped: negative off %d", off)); | |||||
KASSERT(len >= 0, ("m_copyfromunmapped: negative len %d", len)); | |||||
KASSERT(off < m->m_len, | |||||
("m_copyfromunmapped: len exceeds mbuf length")); | |||||
iov.iov_base = cp; | |||||
iov.iov_len = len; | |||||
uio.uio_resid = len; | |||||
uio.uio_iov = &iov; | |||||
uio.uio_segflg = UIO_SYSSPACE; | |||||
uio.uio_iovcnt = 1; | |||||
uio.uio_offset = 0; | |||||
uio.uio_rw = UIO_READ; | |||||
error = m_unmappedtouio(m, off, &uio, len); | |||||
KASSERT(error == 0, ("m_unmappedtouio failed: off %d, len %d", off, | |||||
len)); | |||||
} | |||||
/* | /* | ||||
* Copy data from an mbuf chain starting "off" bytes from the beginning, | * Copy data from an mbuf chain starting "off" bytes from the beginning, | ||||
* continuing for "len" bytes, into the indicated buffer. | * continuing for "len" bytes, into the indicated buffer. | ||||
*/ | */ | ||||
void | void | ||||
m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) | m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) | ||||
{ | { | ||||
u_int count; | u_int count; | ||||
KASSERT(off >= 0, ("m_copydata, negative off %d", off)); | KASSERT(off >= 0, ("m_copydata, negative off %d", off)); | ||||
KASSERT(len >= 0, ("m_copydata, negative len %d", len)); | KASSERT(len >= 0, ("m_copydata, negative len %d", len)); | ||||
while (off > 0) { | while (off > 0) { | ||||
KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); | KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); | ||||
if (off < m->m_len) | if (off < m->m_len) | ||||
break; | break; | ||||
off -= m->m_len; | off -= m->m_len; | ||||
m = m->m_next; | m = m->m_next; | ||||
} | } | ||||
while (len > 0) { | while (len > 0) { | ||||
KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); | KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); | ||||
count = min(m->m_len - off, len); | count = min(m->m_len - off, len); | ||||
if ((m->m_flags & M_NOMAP) != 0) | |||||
m_copyfromunmapped(m, off, count, cp); | |||||
else | |||||
bcopy(mtod(m, caddr_t) + off, cp, count); | bcopy(mtod(m, caddr_t) + off, cp, count); | ||||
len -= count; | len -= count; | ||||
cp += count; | cp += count; | ||||
off = 0; | off = 0; | ||||
m = m->m_next; | m = m->m_next; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
void | void | ||||
m_cat(struct mbuf *m, struct mbuf *n) | m_cat(struct mbuf *m, struct mbuf *n) | ||||
{ | { | ||||
while (m->m_next) | while (m->m_next) | ||||
m = m->m_next; | m = m->m_next; | ||||
while (n) { | while (n) { | ||||
if (!M_WRITABLE(m) || | if (!M_WRITABLE(m) || | ||||
(n->m_flags & M_NOMAP) != 0 || | |||||
M_TRAILINGSPACE(m) < n->m_len) { | M_TRAILINGSPACE(m) < n->m_len) { | ||||
/* just join the two chains */ | /* just join the two chains */ | ||||
m->m_next = n; | m->m_next = n; | ||||
return; | return; | ||||
} | } | ||||
/* splat the data from one into the other */ | /* splat the data from one into the other */ | ||||
bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, | bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, | ||||
(u_int)n->m_len); | (u_int)n->m_len); | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
struct mbuf * | struct mbuf * | ||||
m_pullup(struct mbuf *n, int len) | m_pullup(struct mbuf *n, int len) | ||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int count; | int count; | ||||
int space; | int space; | ||||
KASSERT((n->m_flags & M_NOMAP) == 0, | |||||
("%s: unmapped mbuf %p", __func__, n)); | |||||
/* | /* | ||||
* If first mbuf has no cluster, and has room for len bytes | * If first mbuf has no cluster, and has room for len bytes | ||||
* without shifting current data, pullup into it, | * without shifting current data, pullup into it, | ||||
* otherwise allocate a new mbuf to prepend to the chain. | * otherwise allocate a new mbuf to prepend to the chain. | ||||
*/ | */ | ||||
if ((n->m_flags & M_EXT) == 0 && | if ((n->m_flags & M_EXT) == 0 && | ||||
n->m_data + len < &n->m_dat[MLEN] && n->m_next) { | n->m_data + len < &n->m_dat[MLEN] && n->m_next) { | ||||
if (n->m_len >= len) | if (n->m_len >= len) | ||||
▲ Show 20 Lines • Show All 537 Lines • ▼ Show 20 Lines | #ifdef MBUF_STRESS_TEST | ||||
m_defragfailure++; | m_defragfailure++; | ||||
#endif | #endif | ||||
if (m_final) | if (m_final) | ||||
m_freem(m_final); | m_freem(m_final); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* Return the number of fragments an mbuf will use. This is usually | |||||
* used as a proxy for the number of scatter/gather elements needed by | |||||
* a DMA engine to access an mbuf. In general mapped mbufs are | |||||
* assumed to be backed by physically contiguous buffers that only | |||||
* need a single fragment. Unmapped mbufs, on the other hand, can | |||||
* span disjoint physical pages. | |||||
*/ | |||||
static int | |||||
frags_per_mbuf(struct mbuf *m) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
int frags; | |||||
if ((m->m_flags & M_NOMAP) == 0) | |||||
return (1); | |||||
/* | |||||
* The header and trailer are counted as a single fragment | |||||
* each when present. | |||||
* | |||||
* XXX: This overestimates the number of fragments by assuming | |||||
* all the backing physical pages are disjoint. | |||||
*/ | |||||
ext_pgs = m->m_ext.ext_pgs; | |||||
frags = 0; | |||||
if (ext_pgs->hdr_len != 0) | |||||
frags++; | |||||
frags += ext_pgs->npgs; | |||||
if (ext_pgs->trail_len != 0) | |||||
frags++; | |||||
return (frags); | |||||
} | |||||
/* | |||||
* Defragment an mbuf chain, returning at most maxfrags separate | * Defragment an mbuf chain, returning at most maxfrags separate | ||||
* mbufs+clusters. If this is not possible NULL is returned and | * mbufs+clusters. If this is not possible NULL is returned and | ||||
* the original mbuf chain is left in its present (potentially | * the original mbuf chain is left in its present (potentially | ||||
* modified) state. We use two techniques: collapsing consecutive | * modified) state. We use two techniques: collapsing consecutive | ||||
* mbufs and replacing consecutive mbufs by a cluster. | * mbufs and replacing consecutive mbufs by a cluster. | ||||
* | * | ||||
* NB: this should really be named m_defrag but that name is taken | * NB: this should really be named m_defrag but that name is taken | ||||
*/ | */ | ||||
struct mbuf * | struct mbuf * | ||||
m_collapse(struct mbuf *m0, int how, int maxfrags) | m_collapse(struct mbuf *m0, int how, int maxfrags) | ||||
{ | { | ||||
struct mbuf *m, *n, *n2, **prev; | struct mbuf *m, *n, *n2, **prev; | ||||
u_int curfrags; | u_int curfrags; | ||||
/* | /* | ||||
* Calculate the current number of frags. | * Calculate the current number of frags. | ||||
*/ | */ | ||||
curfrags = 0; | curfrags = 0; | ||||
for (m = m0; m != NULL; m = m->m_next) | for (m = m0; m != NULL; m = m->m_next) | ||||
curfrags++; | curfrags += frags_per_mbuf(m); | ||||
/* | /* | ||||
* First, try to collapse mbufs. Note that we always collapse | * First, try to collapse mbufs. Note that we always collapse | ||||
* towards the front so we don't need to deal with moving the | * towards the front so we don't need to deal with moving the | ||||
* pkthdr. This may be suboptimal if the first mbuf has much | * pkthdr. This may be suboptimal if the first mbuf has much | ||||
* less data than the following. | * less data than the following. | ||||
*/ | */ | ||||
m = m0; | m = m0; | ||||
again: | again: | ||||
for (;;) { | for (;;) { | ||||
n = m->m_next; | n = m->m_next; | ||||
if (n == NULL) | if (n == NULL) | ||||
break; | break; | ||||
if (M_WRITABLE(m) && | if (M_WRITABLE(m) && | ||||
n->m_len < M_TRAILINGSPACE(m)) { | n->m_len < M_TRAILINGSPACE(m)) { | ||||
bcopy(mtod(n, void *), mtod(m, char *) + m->m_len, | m_copydata(n, 0, n->m_len, | ||||
n->m_len); | mtod(m, char *) + m->m_len); | ||||
m->m_len += n->m_len; | m->m_len += n->m_len; | ||||
m->m_next = n->m_next; | m->m_next = n->m_next; | ||||
curfrags -= frags_per_mbuf(n); | |||||
m_free(n); | m_free(n); | ||||
if (--curfrags <= maxfrags) | if (curfrags <= maxfrags) | ||||
return m0; | return m0; | ||||
} else | } else | ||||
m = n; | m = n; | ||||
} | } | ||||
KASSERT(maxfrags > 1, | KASSERT(maxfrags > 1, | ||||
("maxfrags %u, but normal collapse failed", maxfrags)); | ("maxfrags %u, but normal collapse failed", maxfrags)); | ||||
/* | /* | ||||
* Collapse consecutive mbufs to a cluster. | * Collapse consecutive mbufs to a cluster. | ||||
*/ | */ | ||||
prev = &m0->m_next; /* NB: not the first mbuf */ | prev = &m0->m_next; /* NB: not the first mbuf */ | ||||
while ((n = *prev) != NULL) { | while ((n = *prev) != NULL) { | ||||
if ((n2 = n->m_next) != NULL && | if ((n2 = n->m_next) != NULL && | ||||
n->m_len + n2->m_len < MCLBYTES) { | n->m_len + n2->m_len < MCLBYTES) { | ||||
m = m_getcl(how, MT_DATA, 0); | m = m_getcl(how, MT_DATA, 0); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto bad; | goto bad; | ||||
bcopy(mtod(n, void *), mtod(m, void *), n->m_len); | m_copydata(n, 0, n->m_len, mtod(m, char *)); | ||||
bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len, | m_copydata(n2, 0, n2->m_len, | ||||
n2->m_len); | mtod(m, char *) + n->m_len); | ||||
m->m_len = n->m_len + n2->m_len; | m->m_len = n->m_len + n2->m_len; | ||||
m->m_next = n2->m_next; | m->m_next = n2->m_next; | ||||
*prev = m; | *prev = m; | ||||
curfrags += 1; /* For the new cluster */ | |||||
curfrags -= frags_per_mbuf(n); | |||||
curfrags -= frags_per_mbuf(n2); | |||||
m_free(n); | m_free(n); | ||||
m_free(n2); | m_free(n2); | ||||
if (--curfrags <= maxfrags) /* +1 cl -2 mbufs */ | if (curfrags <= maxfrags) | ||||
return m0; | return m0; | ||||
/* | /* | ||||
* Still not there, try the normal collapse | * Still not there, try the normal collapse | ||||
* again before we allocate another cluster. | * again before we allocate another cluster. | ||||
*/ | */ | ||||
goto again; | goto again; | ||||
} | } | ||||
prev = &n->m_next; | prev = &n->m_next; | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | if (m_first) | ||||
m_freem(m_first); | m_freem(m_first); | ||||
/* Return the original chain on failure */ | /* Return the original chain on failure */ | ||||
return (m0); | return (m0); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Free pages from mbuf_ext_pgs, assuming they were allocated via | |||||
* vm_page_alloc() and aren't associated with any object. Complement | |||||
* to allocator from m_uiotombuf_nomap(). | |||||
*/ | |||||
void | |||||
mb_free_mext_pgs(struct mbuf *m) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
vm_page_t pg; | |||||
int wire_adj; | |||||
MBUF_EXT_PGS_ASSERT(m); | |||||
ext_pgs = m->m_ext.ext_pgs; | |||||
wire_adj = 0; | |||||
for (int i = 0; i < ext_pgs->npgs; i++) { | |||||
pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]); | |||||
/* | |||||
* Note: page is not locked, as it has no | |||||
* object and is not on any queues. | |||||
*/ | |||||
vm_page_free_toq(pg); | |||||
wire_adj++; | |||||
} | |||||
if (wire_adj) | |||||
vm_wire_sub(wire_adj); | |||||
} | |||||
static struct mbuf * | |||||
m_uiotombuf_nomap(struct uio *uio, int how, int len, int maxseg, int flags) | |||||
{ | |||||
struct mbuf *m, *mb, *prev; | |||||
struct mbuf_ext_pgs *pgs; | |||||
vm_page_t pg_array[MBUF_PEXT_MAX_PGS]; | |||||
int error, length, i, needed, wire_adj = 0; | |||||
ssize_t total; | |||||
int pflags = malloc2vm_flags(how) | VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP; | |||||
/* | |||||
* len can be zero or an arbitrary large value bound by | |||||
* the total data supplied by the uio. | |||||
*/ | |||||
if (len > 0) | |||||
total = MIN(uio->uio_resid, len); | |||||
else | |||||
total = uio->uio_resid; | |||||
if (maxseg == 0) | |||||
maxseg = MBUF_PEXT_MAX_PGS * PAGE_SIZE; | |||||
/* | |||||
* Allocate the pages | |||||
*/ | |||||
m = NULL; | |||||
while (total > 0) { | |||||
mb = mb_alloc_ext_pgs(how, (flags & M_PKTHDR), | |||||
mb_free_mext_pgs); | |||||
if (mb == NULL) | |||||
goto failed; | |||||
if (m == NULL) | |||||
m = mb; | |||||
else | |||||
prev->m_next = mb; | |||||
prev = mb; | |||||
pgs = mb->m_ext.ext_pgs; | |||||
needed = length = MIN(maxseg, total); | |||||
for (i = 0; needed > 0; i++, needed -= PAGE_SIZE) { | |||||
retry_page: | |||||
pg_array[i] = vm_page_alloc(NULL, 0, pflags); | |||||
if (pg_array[i] == NULL) { | |||||
if (wire_adj) | |||||
vm_wire_add(wire_adj); | |||||
wire_adj = 0; | |||||
if (how & M_NOWAIT) { | |||||
goto failed; | |||||
} else { | |||||
vm_wait(NULL); | |||||
goto retry_page; | |||||
} | |||||
} | |||||
wire_adj++; | |||||
pg_array[i]->flags &= ~PG_ZERO; | |||||
pgs->pa[i] = VM_PAGE_TO_PHYS(pg_array[i]); | |||||
pgs->npgs++; | |||||
} | |||||
pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1); | |||||
MBUF_EXT_PGS_ASSERT_SANITY(pgs); | |||||
vm_wire_add(wire_adj); | |||||
wire_adj = 0; | |||||
total -= length; | |||||
error = uiomove_fromphys(pg_array, 0, length, uio); | |||||
if (error != 0) | |||||
goto failed; | |||||
mb->m_len = length; | |||||
mb->m_ext.ext_size += PAGE_SIZE * pgs->npgs; | |||||
if (flags & M_PKTHDR) | |||||
m->m_pkthdr.len += length; | |||||
} | |||||
return (m); | |||||
failed: | |||||
m_freem(m); | |||||
return (NULL); | |||||
} | |||||
/* | |||||
* Copy the contents of uio into a properly sized mbuf chain. | * Copy the contents of uio into a properly sized mbuf chain. | ||||
*/ | */ | ||||
struct mbuf * | struct mbuf * | ||||
m_uiotombuf(struct uio *uio, int how, int len, int align, int flags) | m_uiotombuf(struct uio *uio, int how, int len, int align, int flags) | ||||
{ | { | ||||
struct mbuf *m, *mb; | struct mbuf *m, *mb; | ||||
int error, length; | int error, length; | ||||
ssize_t total; | ssize_t total; | ||||
int progress = 0; | int progress = 0; | ||||
if (flags & M_NOMAP) | |||||
return (m_uiotombuf_nomap(uio, how, len, align, flags)); | |||||
/* | /* | ||||
* len can be zero or an arbitrary large value bound by | * len can be zero or an arbitrary large value bound by | ||||
* the total data supplied by the uio. | * the total data supplied by the uio. | ||||
*/ | */ | ||||
if (len > 0) | if (len > 0) | ||||
total = (uio->uio_resid < len) ? uio->uio_resid : len; | total = (uio->uio_resid < len) ? uio->uio_resid : len; | ||||
else | else | ||||
total = uio->uio_resid; | total = uio->uio_resid; | ||||
Show All 30 Lines | if (flags & M_PKTHDR) | ||||
m->m_pkthdr.len += length; | m->m_pkthdr.len += length; | ||||
} | } | ||||
KASSERT(progress == total, ("%s: progress != total", __func__)); | KASSERT(progress == total, ("%s: progress != total", __func__)); | ||||
return (m); | return (m); | ||||
} | } | ||||
/* | /* | ||||
* Copy data from an unmapped mbuf into a uio limited by len if set. | |||||
*/ | |||||
int | |||||
m_unmappedtouio(const struct mbuf *m, int m_off, struct uio *uio, int len) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
vm_page_t pg; | |||||
int error, i, off, pglen, pgoff, seglen, segoff; | |||||
MBUF_EXT_PGS_ASSERT(m); | |||||
ext_pgs = m->m_ext.ext_pgs; | |||||
error = 0; | |||||
/* Skip over any data removed from the front. */ | |||||
off = mtod(m, vm_offset_t); | |||||
off += m_off; | |||||
if (ext_pgs->hdr_len != 0) { | |||||
if (off >= ext_pgs->hdr_len) { | |||||
off -= ext_pgs->hdr_len; | |||||
} else { | |||||
seglen = ext_pgs->hdr_len - off; | |||||
segoff = off; | |||||
seglen = min(seglen, len); | |||||
off = 0; | |||||
len -= seglen; | |||||
error = uiomove(&ext_pgs->hdr[segoff], seglen, uio); | |||||
} | |||||
} | |||||
pgoff = ext_pgs->first_pg_off; | |||||
for (i = 0; i < ext_pgs->npgs && error == 0 && len > 0; i++) { | |||||
pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); | |||||
if (off >= pglen) { | |||||
off -= pglen; | |||||
pgoff = 0; | |||||
continue; | |||||
} | |||||
seglen = pglen - off; | |||||
segoff = pgoff + off; | |||||
off = 0; | |||||
seglen = min(seglen, len); | |||||
len -= seglen; | |||||
pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]); | |||||
error = uiomove_fromphys(&pg, segoff, seglen, uio); | |||||
pgoff = 0; | |||||
}; | |||||
if (len != 0 && error == 0) { | |||||
KASSERT((off + len) <= ext_pgs->trail_len, | |||||
("off + len > trail (%d + %d > %d, m_off = %d)", off, len, | |||||
ext_pgs->trail_len, m_off)); | |||||
error = uiomove(&ext_pgs->trail[off], len, uio); | |||||
} | |||||
return (error); | |||||
} | |||||
/* | |||||
* Copy an mbuf chain into a uio limited by len if set. | * Copy an mbuf chain into a uio limited by len if set. | ||||
*/ | */ | ||||
int | int | ||||
m_mbuftouio(struct uio *uio, const struct mbuf *m, int len) | m_mbuftouio(struct uio *uio, const struct mbuf *m, int len) | ||||
{ | { | ||||
int error, length, total; | int error, length, total; | ||||
int progress = 0; | int progress = 0; | ||||
if (len > 0) | if (len > 0) | ||||
total = min(uio->uio_resid, len); | total = min(uio->uio_resid, len); | ||||
else | else | ||||
total = uio->uio_resid; | total = uio->uio_resid; | ||||
/* Fill the uio with data from the mbufs. */ | /* Fill the uio with data from the mbufs. */ | ||||
for (; m != NULL; m = m->m_next) { | for (; m != NULL; m = m->m_next) { | ||||
length = min(m->m_len, total - progress); | length = min(m->m_len, total - progress); | ||||
if ((m->m_flags & M_NOMAP) != 0) | |||||
error = m_unmappedtouio(m, 0, uio, length); | |||||
else | |||||
error = uiomove(mtod(m, void *), length, uio); | error = uiomove(mtod(m, void *), length, uio); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
progress += length; | progress += length; | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 277 Lines • Show Last 20 Lines |