Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/uipc_sockbuf.c
Show First 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | if (flags & PRUS_NOTREADY) | ||||
mask |= M_NOTREADY; | mask |= M_NOTREADY; | ||||
while (m) { | while (m) { | ||||
m->m_flags &= mask; | m->m_flags &= mask; | ||||
m = m->m_next; | m = m->m_next; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Mark ready "count" mbufs starting with "m". | * Compress M_NOTREADY mbufs after they have been readied by sbready(). | ||||
* | |||||
* sbcompress() skips M_NOTREADY mbufs since the data is not available to | |||||
* be copied at the time of sbcompress(). This function combines small | |||||
* mbufs similar to sbcompress() once mbufs are ready. 'm0' is the first | |||||
* mbuf sbready() marked ready, and 'end' is the first mbuf still not | |||||
* ready. | |||||
*/ | */ | ||||
static void | |||||
sbready_compress(struct sockbuf *sb, struct mbuf *m0, struct mbuf *end) | |||||
{ | |||||
struct mbuf *m, *n; | |||||
int ext_size; | |||||
SOCKBUF_LOCK_ASSERT(sb); | |||||
if ((sb->sb_flags & SB_NOCOALESCE) != 0) | |||||
return; | |||||
for (m = m0; m != end; m = m->m_next) { | |||||
MPASS((m->m_flags & M_NOTREADY) == 0); | |||||
/* Compress small unmapped mbufs into plain mbufs. */ | |||||
if ((m->m_flags & M_NOMAP) && m->m_len <= MLEN) { | |||||
MPASS(m->m_flags & M_EXT); | |||||
ext_size = m->m_ext.ext_size; | |||||
if (mb_unmapped_compress(m) == 0) { | |||||
sb->sb_mbcnt -= ext_size; | |||||
sb->sb_ccnt -= 1; | |||||
} | |||||
} | |||||
/* | |||||
* NB: In sbcompress(), 'n' is the last mbuf in the | |||||
* socket buffer and 'm' is the new mbuf being copied | |||||
* into the trailing space of 'n'. Here, the roles | |||||
* are reversed and 'n' is the next mbuf after 'm' | |||||
* that is being copied into the trailing space of | |||||
* 'm'. | |||||
*/ | |||||
n = m->m_next; | |||||
while ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 && | |||||
M_WRITABLE(m) && | |||||
(m->m_flags & M_NOMAP) == 0 && | |||||
n->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ | |||||
n->m_len <= M_TRAILINGSPACE(m) && | |||||
m->m_type == n->m_type) { | |||||
KASSERT(sb->sb_lastrecord != n, | |||||
("%s: merging start of record (%p) into previous mbuf (%p)", | |||||
__func__, n, m)); | |||||
m_copydata(n, 0, n->m_len, mtodo(m, m->m_len)); | |||||
m->m_len += n->m_len; | |||||
m->m_next = n->m_next; | |||||
m->m_flags |= n->m_flags & M_EOR; | |||||
if (sb->sb_mbtail == n) | |||||
sb->sb_mbtail = m; | |||||
sb->sb_mbcnt -= MSIZE; | |||||
sb->sb_mcnt -= 1; | |||||
if (n->m_flags & M_EXT) { | |||||
sb->sb_mbcnt -= n->m_ext.ext_size; | |||||
sb->sb_ccnt -= 1; | |||||
} | |||||
m_free(n); | |||||
n = m->m_next; | |||||
} | |||||
} | |||||
SBLASTRECORDCHK(sb); | |||||
SBLASTMBUFCHK(sb); | |||||
} | |||||
/* | |||||
* Mark ready "count" units of I/O starting with "m". Most mbufs | |||||
* count as a single unit of I/O except for EXT_PGS-backed mbufs which | |||||
* can be backed by multiple pages. | |||||
*/ | |||||
int | int | ||||
sbready(struct sockbuf *sb, struct mbuf *m, int count) | sbready(struct sockbuf *sb, struct mbuf *m0, int count) | ||||
{ | { | ||||
struct mbuf *m; | |||||
u_int blocker; | u_int blocker; | ||||
SOCKBUF_LOCK_ASSERT(sb); | SOCKBUF_LOCK_ASSERT(sb); | ||||
KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); | KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); | ||||
KASSERT(count > 0, ("%s: invalid count %d", __func__, count)); | |||||
m = m0; | |||||
blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0; | blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0; | ||||
for (int i = 0; i < count; i++, m = m->m_next) { | while (count > 0) { | ||||
KASSERT(m->m_flags & M_NOTREADY, | KASSERT(m->m_flags & M_NOTREADY, | ||||
("%s: m %p !M_NOTREADY", __func__, m)); | ("%s: m %p !M_NOTREADY", __func__, m)); | ||||
if ((m->m_flags & M_EXT) != 0 && | |||||
m->m_ext.ext_type == EXT_PGS) { | |||||
if (count < m->m_ext.ext_pgs->nrdy) { | |||||
m->m_ext.ext_pgs->nrdy -= count; | |||||
count = 0; | |||||
break; | |||||
} | |||||
count -= m->m_ext.ext_pgs->nrdy; | |||||
m->m_ext.ext_pgs->nrdy = 0; | |||||
} else | |||||
count--; | |||||
m->m_flags &= ~(M_NOTREADY | blocker); | m->m_flags &= ~(M_NOTREADY | blocker); | ||||
if (blocker) | if (blocker) | ||||
sb->sb_acc += m->m_len; | sb->sb_acc += m->m_len; | ||||
m = m->m_next; | |||||
} | } | ||||
if (!blocker) | /* | ||||
* If the first mbuf is still not fully ready because only | |||||
* some of its backing pages were readied, no further progress | |||||
* can be made. | |||||
*/ | |||||
if (m0 == m) { | |||||
MPASS(m->m_flags & M_NOTREADY); | |||||
return (EINPROGRESS); | return (EINPROGRESS); | ||||
} | |||||
if (!blocker) { | |||||
sbready_compress(sb, m0, m); | |||||
return (EINPROGRESS); | |||||
} | |||||
/* This one was blocking all the queue. */ | /* This one was blocking all the queue. */ | ||||
for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) { | for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) { | ||||
KASSERT(m->m_flags & M_BLOCKED, | KASSERT(m->m_flags & M_BLOCKED, | ||||
("%s: m %p !M_BLOCKED", __func__, m)); | ("%s: m %p !M_BLOCKED", __func__, m)); | ||||
m->m_flags &= ~M_BLOCKED; | m->m_flags &= ~M_BLOCKED; | ||||
sb->sb_acc += m->m_len; | sb->sb_acc += m->m_len; | ||||
} | } | ||||
sb->sb_fnrdy = m; | sb->sb_fnrdy = m; | ||||
sbready_compress(sb, m0, m); | |||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Adjust sockbuf state reflecting allocation of m. | * Adjust sockbuf state reflecting allocation of m. | ||||
*/ | */ | ||||
void | void | ||||
▲ Show 20 Lines • Show All 893 Lines • ▼ Show 20 Lines | if (m->m_len == 0 && | ||||
sb->sb_lastrecord = m->m_next; | sb->sb_lastrecord = m->m_next; | ||||
m = m_free(m); | m = m_free(m); | ||||
continue; | continue; | ||||
} | } | ||||
if (n && (n->m_flags & M_EOR) == 0 && | if (n && (n->m_flags & M_EOR) == 0 && | ||||
M_WRITABLE(n) && | M_WRITABLE(n) && | ||||
((sb->sb_flags & SB_NOCOALESCE) == 0) && | ((sb->sb_flags & SB_NOCOALESCE) == 0) && | ||||
!(m->m_flags & M_NOTREADY) && | !(m->m_flags & M_NOTREADY) && | ||||
!(n->m_flags & M_NOTREADY) && | !(n->m_flags & (M_NOTREADY | M_NOMAP)) && | ||||
m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ | m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ | ||||
m->m_len <= M_TRAILINGSPACE(n) && | m->m_len <= M_TRAILINGSPACE(n) && | ||||
n->m_type == m->m_type) { | n->m_type == m->m_type) { | ||||
bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, | m_copydata(m, 0, m->m_len, mtodo(n, n->m_len)); | ||||
(unsigned)m->m_len); | |||||
n->m_len += m->m_len; | n->m_len += m->m_len; | ||||
sb->sb_ccc += m->m_len; | sb->sb_ccc += m->m_len; | ||||
if (sb->sb_fnrdy == NULL) | if (sb->sb_fnrdy == NULL) | ||||
sb->sb_acc += m->m_len; | sb->sb_acc += m->m_len; | ||||
if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) | if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) | ||||
/* XXX: Probably don't need.*/ | /* XXX: Probably don't need.*/ | ||||
sb->sb_ctl += m->m_len; | sb->sb_ctl += m->m_len; | ||||
m = m_free(m); | m = m_free(m); | ||||
continue; | continue; | ||||
} | } | ||||
if (m->m_len <= MLEN && (m->m_flags & M_NOMAP) && | |||||
(m->m_flags & M_NOTREADY) == 0) | |||||
(void)mb_unmapped_compress(m); | |||||
if (n) | if (n) | ||||
n->m_next = m; | n->m_next = m; | ||||
else | else | ||||
sb->sb_mb = m; | sb->sb_mb = m; | ||||
sb->sb_mbtail = m; | sb->sb_mbtail = m; | ||||
sballoc(sb, m); | sballoc(sb, m); | ||||
n = m; | n = m; | ||||
m->m_flags &= ~M_EOR; | m->m_flags &= ~M_EOR; | ||||
▲ Show 20 Lines • Show All 350 Lines • Show Last 20 Lines |