diff --git a/sys/kern/subr_memdesc.c b/sys/kern/subr_memdesc.c index 9ba9d7fe031b..ff8aad7731cd 100644 --- a/sys/kern/subr_memdesc.c +++ b/sys/kern/subr_memdesc.c @@ -1,325 +1,799 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Chelsio Communications, Inc. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include +#include #include #include +/* + * memdesc_copyback copies data from a source buffer into a buffer + * described by a memory descriptor. + */ static void phys_copyback(vm_paddr_t pa, int off, int size, const void *src) { const char *cp; u_int page_off; int todo; void *p; KASSERT(PMAP_HAS_DMAP, ("direct-map required")); cp = src; pa += off; page_off = pa & PAGE_MASK; while (size > 0) { todo = min(PAGE_SIZE - page_off, size); p = (void *)PHYS_TO_DMAP(pa); memcpy(p, cp, todo); size -= todo; cp += todo; pa += todo; page_off = 0; } } static void vlist_copyback(struct bus_dma_segment *vlist, int sglist_cnt, int off, int size, const void *src) { const char *p; int todo; while (vlist->ds_len <= off) { KASSERT(sglist_cnt > 1, ("out of sglist entries")); off -= vlist->ds_len; vlist++; sglist_cnt--; } p = src; while (size > 0) { KASSERT(sglist_cnt >= 1, ("out of sglist entries")); todo = size; if (todo > vlist->ds_len - off) todo = vlist->ds_len - off; memcpy((char *)(uintptr_t)vlist->ds_addr + off, p, todo); off = 0; vlist++; sglist_cnt--; size -= todo; p += todo; } } static void plist_copyback(struct bus_dma_segment *plist, int sglist_cnt, int off, int size, const void *src) { const char *p; int todo; while (plist->ds_len <= off) { KASSERT(sglist_cnt > 1, ("out of sglist entries")); off -= plist->ds_len; plist++; sglist_cnt--; } p = src; while (size > 0) { KASSERT(sglist_cnt >= 1, ("out of sglist entries")); todo = size; if (todo > plist->ds_len - off) todo = plist->ds_len - off; phys_copyback(plist->ds_addr, off, todo, p); off = 0; plist++; sglist_cnt--; size -= todo; p += todo; } } static void vmpages_copyback(vm_page_t *m, int off, int size, const void *src) { struct iovec iov[1]; struct uio uio; int error __diagused; iov[0].iov_base = __DECONST(void *, src); iov[0].iov_len = size; uio.uio_iov = iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = size; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; error = uiomove_fromphys(m, off, size, &uio); KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed")); } void memdesc_copyback(struct memdesc *mem, int off, int size, const void *src) { KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); KASSERT(size >= 0, ("%s: invalid size %d", __func__, off)); switch (mem->md_type) { case MEMDESC_VADDR: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); memcpy((char *)mem->u.md_vaddr + off, src, size); break; case MEMDESC_PADDR: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); phys_copyback(mem->u.md_paddr, off, size, src); break; case MEMDESC_VLIST: vlist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); break; case MEMDESC_PLIST: plist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); break; case MEMDESC_UIO: panic("Use uiomove instead"); break; case MEMDESC_MBUF: m_copyback(mem->u.md_mbuf, off, size, src); break; case MEMDESC_VMPAGES: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); vmpages_copyback(mem->u.md_ma, mem->md_offset + off, size, src); break; default: __assert_unreachable(); } } +/* + * memdesc_copydata copies data from a buffer described by a memory + * descriptor into a destination buffer. + */ static void phys_copydata(vm_paddr_t pa, int off, int size, void *dst) { char *cp; u_int page_off; int todo; const void *p; KASSERT(PMAP_HAS_DMAP, ("direct-map required")); cp = dst; pa += off; page_off = pa & PAGE_MASK; while (size > 0) { todo = min(PAGE_SIZE - page_off, size); p = (const void *)PHYS_TO_DMAP(pa); memcpy(cp, p, todo); size -= todo; cp += todo; pa += todo; page_off = 0; } } static void vlist_copydata(struct bus_dma_segment *vlist, int sglist_cnt, int off, int size, void *dst) { char *p; int todo; while (vlist->ds_len <= off) { KASSERT(sglist_cnt > 1, ("out of sglist entries")); off -= vlist->ds_len; vlist++; sglist_cnt--; } p = dst; while (size > 0) { KASSERT(sglist_cnt >= 1, ("out of sglist entries")); todo = size; if (todo > vlist->ds_len - off) todo = vlist->ds_len - off; memcpy(p, (char *)(uintptr_t)vlist->ds_addr + off, todo); off = 0; vlist++; sglist_cnt--; size -= todo; p += todo; } } static void plist_copydata(struct bus_dma_segment *plist, int sglist_cnt, int off, int size, void *dst) { char *p; int todo; while (plist->ds_len <= off) { KASSERT(sglist_cnt > 1, ("out of sglist entries")); off -= plist->ds_len; plist++; sglist_cnt--; } p = dst; while (size > 0) { KASSERT(sglist_cnt >= 1, ("out of sglist entries")); todo = size; if (todo > plist->ds_len - off) todo = plist->ds_len - off; phys_copydata(plist->ds_addr, off, todo, p); off = 0; plist++; sglist_cnt--; size -= todo; p += todo; } } static void vmpages_copydata(vm_page_t *m, int off, int size, void *dst) { struct iovec iov[1]; struct uio uio; int error __diagused; iov[0].iov_base = dst; iov[0].iov_len = size; uio.uio_iov = iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = size; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; error = uiomove_fromphys(m, off, size, &uio); KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed")); } void memdesc_copydata(struct memdesc *mem, int off, int size, void *dst) { KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); KASSERT(size >= 0, ("%s: invalid size %d", __func__, off)); switch (mem->md_type) { case MEMDESC_VADDR: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); memcpy(dst, (const char *)mem->u.md_vaddr + off, size); break; case MEMDESC_PADDR: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); phys_copydata(mem->u.md_paddr, off, size, dst); break; case MEMDESC_VLIST: vlist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); break; case MEMDESC_PLIST: plist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); break; case MEMDESC_UIO: panic("Use uiomove instead"); break; case MEMDESC_MBUF: m_copydata(mem->u.md_mbuf, off, size, dst); break; case MEMDESC_VMPAGES: KASSERT(off + size <= mem->md_len, ("copy out of bounds")); vmpages_copydata(mem->u.md_ma, mem->md_offset + off, size, dst); break; default: __assert_unreachable(); } } + +/* + * memdesc_alloc_ext_mbufs allocates a chain of external mbufs backed + * by the storage of a memory descriptor's data buffer. + */ +static struct mbuf * +vaddr_ext_mbuf(memdesc_alloc_ext_mbuf_t *ext_alloc, void *cb_arg, int how, + void *buf, size_t len, size_t *actual_len) +{ + *actual_len = len; + return (ext_alloc(cb_arg, how, buf, len)); +} + +static bool +can_append_paddr(struct mbuf *m, vm_paddr_t pa) +{ + u_int last_len; + + /* Can always append to an empty mbuf. */ + if (m->m_epg_npgs == 0) + return (true); + + /* Can't append to a full mbuf. */ + if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) + return (false); + + /* Can't append a non-page-aligned address to a non-empty mbuf. */ + if ((pa & PAGE_MASK) != 0) + return (false); + + /* Can't append if the last page is not a full page. */ + last_len = m->m_epg_last_len; + if (m->m_epg_npgs == 1) + last_len += m->m_epg_1st_off; + return (last_len == PAGE_SIZE); +} + +/* + * Returns amount of data added to an M_EXTPG mbuf. + */ +static size_t +append_paddr_range(struct mbuf *m, vm_paddr_t pa, size_t len) +{ + size_t appended; + + appended = 0; + + /* Append the first page. */ + if (m->m_epg_npgs == 0) { + m->m_epg_pa[0] = trunc_page(pa); + m->m_epg_npgs = 1; + m->m_epg_1st_off = pa & PAGE_MASK; + m->m_epg_last_len = PAGE_SIZE - m->m_epg_1st_off; + if (m->m_epg_last_len > len) + m->m_epg_last_len = len; + m->m_len = m->m_epg_last_len; + len -= m->m_epg_last_len; + pa += m->m_epg_last_len; + appended += m->m_epg_last_len; + } + KASSERT(len == 0 || (pa & PAGE_MASK) == 0, + ("PA not aligned before full pages")); + + /* Full pages. */ + while (len >= PAGE_SIZE && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) { + m->m_epg_pa[m->m_epg_npgs] = pa; + m->m_epg_npgs++; + m->m_epg_last_len = PAGE_SIZE; + m->m_len += PAGE_SIZE; + pa += PAGE_SIZE; + len -= PAGE_SIZE; + appended += PAGE_SIZE; + } + + /* Final partial page. */ + if (len > 0 && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) { + KASSERT(len < PAGE_SIZE, ("final page is full page")); + m->m_epg_pa[m->m_epg_npgs] = pa; + m->m_epg_npgs++; + m->m_epg_last_len = len; + m->m_len += len; + appended += len; + } + + return (appended); +} + +static struct mbuf * +paddr_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how, + vm_paddr_t pa, size_t len, size_t *actual_len, bool can_truncate) +{ + struct mbuf *m, *tail; + size_t appended; + + if (can_truncate) { + vm_paddr_t end; + + /* + * Trim any partial page at the end, but not if it's + * the only page. + */ + end = trunc_page(pa + len); + if (end > pa) + len = end - pa; + } + *actual_len = len; + + m = tail = extpg_alloc(cb_arg, how); + if (m == NULL) + return (NULL); + while (len > 0) { + if (!can_append_paddr(tail, pa)) { + MBUF_EXT_PGS_ASSERT_SANITY(tail); + tail->m_next = extpg_alloc(cb_arg, how); + if (tail->m_next == NULL) + goto error; + tail = tail->m_next; + } + + appended = append_paddr_range(tail, pa, len); + KASSERT(appended > 0, ("did not append anything")); + KASSERT(appended <= len, ("appended too much")); + + pa += appended; + len -= appended; + } + + MBUF_EXT_PGS_ASSERT_SANITY(tail); + return (m); +error: + m_freem(m); + return (NULL); +} + +static struct mbuf * +vlist_ext_mbuf(memdesc_alloc_ext_mbuf_t *ext_alloc, void *cb_arg, int how, + struct bus_dma_segment *vlist, u_int sglist_cnt, size_t offset, + size_t len, size_t *actual_len) +{ + struct mbuf *m, *n, *tail; + size_t todo; + + *actual_len = len; + + while (vlist->ds_len <= offset) { + KASSERT(sglist_cnt > 1, ("out of sglist entries")); + + offset -= vlist->ds_len; + vlist++; + sglist_cnt--; + } + + m = tail = NULL; + while (len > 0) { + KASSERT(sglist_cnt >= 1, ("out of sglist entries")); + + todo = len; + if (todo > vlist->ds_len - offset) + todo = vlist->ds_len - offset; + + n = ext_alloc(cb_arg, how, (char *)(uintptr_t)vlist->ds_addr + + offset, todo); + if (n == NULL) + goto error; + + if (m == NULL) { + m = n; + tail = m; + } else { + tail->m_next = n; + tail = n; + } + + offset = 0; + vlist++; + sglist_cnt--; + len -= todo; + } + + return (m); +error: + m_freem(m); + return (NULL); +} + +static struct mbuf * +plist_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how, + struct bus_dma_segment *plist, u_int sglist_cnt, size_t offset, size_t len, + size_t *actual_len, bool can_truncate) +{ + vm_paddr_t pa; + struct mbuf *m, *tail; + size_t appended, totlen, todo; + + while (plist->ds_len <= offset) { + KASSERT(sglist_cnt > 1, ("out of sglist entries")); + + offset -= plist->ds_len; + plist++; + sglist_cnt--; + } + + totlen = 0; + m = tail = extpg_alloc(cb_arg, how); + if (m == NULL) + return (NULL); + while (len > 0) { + KASSERT(sglist_cnt >= 1, ("out of sglist entries")); + + pa = plist->ds_addr + offset; + todo = len; + if (todo > plist->ds_len - offset) + todo = plist->ds_len - offset; + + /* + * If truncation is enabled, avoid sending a final + * partial page, but only if there is more data + * available in the current segment. Also, at least + * some data must be sent, so only drop the final page + * for this segment if the segment spans multiple + * pages or some other data is already queued. + */ + else if (can_truncate) { + vm_paddr_t end; + + end = trunc_page(pa + len); + if (end <= pa && totlen != 0) { + /* + * This last segment is only a partial + * page. + */ + len = 0; + break; + } + todo = end - pa; + } + + offset = 0; + len -= todo; + totlen += todo; + + while (todo > 0) { + if (!can_append_paddr(tail, pa)) { + MBUF_EXT_PGS_ASSERT_SANITY(tail); + tail->m_next = extpg_alloc(cb_arg, how); + if (tail->m_next == NULL) + goto error; + tail = tail->m_next; + } + + appended = append_paddr_range(tail, pa, todo); + KASSERT(appended > 0, ("did not append anything")); + + pa += appended; + todo -= appended; + } + } + + MBUF_EXT_PGS_ASSERT_SANITY(tail); + *actual_len = totlen; + return (m); +error: + m_freem(m); + return (NULL); +} + +static struct mbuf * +vmpages_ext_mbuf(memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how, + vm_page_t *ma, size_t offset, size_t len, size_t *actual_len, + bool can_truncate) +{ + struct mbuf *m, *tail; + + while (offset >= PAGE_SIZE) { + ma++; + offset -= PAGE_SIZE; + } + + if (can_truncate) { + size_t end; + + /* + * Trim any partial page at the end, but not if it's + * the only page. + */ + end = trunc_page(offset + len); + if (end > offset) + len = end - offset; + } + *actual_len = len; + + m = tail = extpg_alloc(cb_arg, how); + if (m == NULL) + return (NULL); + + /* First page. */ + m->m_epg_pa[0] = VM_PAGE_TO_PHYS(*ma); + ma++; + m->m_epg_npgs = 1; + m->m_epg_1st_off = offset; + m->m_epg_last_len = PAGE_SIZE - offset; + if (m->m_epg_last_len > len) + m->m_epg_last_len = len; + m->m_len = m->m_epg_last_len; + len -= m->m_epg_last_len; + + /* Full pages. */ + while (len >= PAGE_SIZE) { + if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) { + MBUF_EXT_PGS_ASSERT_SANITY(tail); + tail->m_next = extpg_alloc(cb_arg, how); + if (tail->m_next == NULL) + goto error; + tail = tail->m_next; + } + + tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma); + ma++; + tail->m_epg_npgs++; + tail->m_epg_last_len = PAGE_SIZE; + tail->m_len += PAGE_SIZE; + len -= PAGE_SIZE; + } + + /* Last partial page. */ + if (len > 0) { + if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) { + MBUF_EXT_PGS_ASSERT_SANITY(tail); + tail->m_next = extpg_alloc(cb_arg, how); + if (tail->m_next == NULL) + goto error; + tail = tail->m_next; + } + + tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma); + ma++; + tail->m_epg_npgs++; + tail->m_epg_last_len = len; + tail->m_len += len; + } + + MBUF_EXT_PGS_ASSERT_SANITY(tail); + return (m); +error: + m_freem(m); + return (NULL); +} + +/* + * Somewhat similar to m_copym but optionally avoids a partial mbuf at + * the end. + */ +static struct mbuf * +mbuf_subchain(struct mbuf *m0, size_t offset, size_t len, + size_t *actual_len, bool can_truncate, int how) +{ + struct mbuf *m, *tail; + size_t totlen; + + while (offset >= m0->m_len) { + offset -= m0->m_len; + m0 = m0->m_next; + } + + /* Always return at least one mbuf. */ + totlen = m0->m_len - offset; + if (totlen > len) + totlen = len; + + m = m_get(how, MT_DATA); + if (m == NULL) + return (NULL); + m->m_len = totlen; + if (m0->m_flags & (M_EXT | M_EXTPG)) { + m->m_data = m0->m_data + offset; + mb_dupcl(m, m0); + } else + memcpy(mtod(m, void *), mtodo(m0, offset), m->m_len); + + tail = m; + m0 = m0->m_next; + len -= totlen; + while (len > 0) { + /* + * If truncation is enabled, don't send any partial + * mbufs besides the first one. + */ + if (can_truncate && m0->m_len > len) + break; + + tail->m_next = m_get(how, MT_DATA); + if (tail->m_next == NULL) + goto error; + tail = tail->m_next; + tail->m_len = m0->m_len; + if (m0->m_flags & (M_EXT | M_EXTPG)) { + tail->m_data = m0->m_data; + mb_dupcl(tail, m0); + } else + memcpy(mtod(tail, void *), mtod(m0, void *), + tail->m_len); + + totlen += tail->m_len; + m0 = m0->m_next; + len -= tail->m_len; + } + *actual_len = totlen; + return (m); +error: + m_freem(m); + return (NULL); +} + +struct mbuf * +memdesc_alloc_ext_mbufs(struct memdesc *mem, + memdesc_alloc_ext_mbuf_t *ext_alloc, + memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how, + size_t offset, size_t len, size_t *actual_len, bool can_truncate) +{ + struct mbuf *m; + size_t done; + + switch (mem->md_type) { + case MEMDESC_VADDR: + m = vaddr_ext_mbuf(ext_alloc, cb_arg, how, + (char *)mem->u.md_vaddr + offset, len, &done); + break; + case MEMDESC_PADDR: + m = paddr_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_paddr + + offset, len, &done, can_truncate); + break; + case MEMDESC_VLIST: + m = vlist_ext_mbuf(ext_alloc, cb_arg, how, mem->u.md_list, + mem->md_nseg, offset, len, &done); + break; + case MEMDESC_PLIST: + m = plist_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_list, + mem->md_nseg, offset, len, &done, can_truncate); + break; + case MEMDESC_UIO: + panic("uio not supported"); + case MEMDESC_MBUF: + m = mbuf_subchain(mem->u.md_mbuf, offset, len, &done, + can_truncate, how); + break; + case MEMDESC_VMPAGES: + m = vmpages_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_ma, + mem->md_offset + offset, len, &done, can_truncate); + break; + default: + __assert_unreachable(); + } + if (m == NULL) + return (NULL); + + if (can_truncate) { + KASSERT(done <= len, ("chain too long")); + } else { + KASSERT(done == len, ("short chain with no limit")); + } + KASSERT(m_length(m, NULL) == done, ("length mismatch")); + if (actual_len != NULL) + *actual_len = done; + return (m); +} diff --git a/sys/sys/memdesc.h b/sys/sys/memdesc.h index 24a2c2da6347..f1880fee2cda 100644 --- a/sys/sys/memdesc.h +++ b/sys/sys/memdesc.h @@ -1,166 +1,202 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 EMC Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_MEMDESC_H_ #define _SYS_MEMDESC_H_ struct bio; struct bus_dma_segment; struct uio; struct mbuf; struct vm_page; union ccb; /* * struct memdesc encapsulates various memory descriptors and provides * abstract access to them. */ struct memdesc { union { void *md_vaddr; vm_paddr_t md_paddr; struct bus_dma_segment *md_list; struct uio *md_uio; struct mbuf *md_mbuf; struct vm_page **md_ma; } u; union { /* type specific data. */ size_t md_len; /* VADDR, PADDR, VMPAGES */ int md_nseg; /* VLIST, PLIST */ }; union { uint32_t md_offset; /* VMPAGES */ }; uint32_t md_type; /* Type of memory. */ }; #define MEMDESC_VADDR 1 /* Contiguous virtual address. */ #define MEMDESC_PADDR 2 /* Contiguous physical address. */ #define MEMDESC_VLIST 3 /* scatter/gather list of kva addresses. */ #define MEMDESC_PLIST 4 /* scatter/gather list of physical addresses. */ #define MEMDESC_UIO 6 /* Pointer to a uio (any io). */ #define MEMDESC_MBUF 7 /* Pointer to a mbuf (network io). */ #define MEMDESC_VMPAGES 8 /* Pointer to array of VM pages. */ static inline struct memdesc memdesc_vaddr(void *vaddr, size_t len) { struct memdesc mem; mem.u.md_vaddr = vaddr; mem.md_len = len; mem.md_type = MEMDESC_VADDR; return (mem); } static inline struct memdesc memdesc_paddr(vm_paddr_t paddr, size_t len) { struct memdesc mem; mem.u.md_paddr = paddr; mem.md_len = len; mem.md_type = MEMDESC_PADDR; return (mem); } static inline struct memdesc memdesc_vlist(struct bus_dma_segment *vlist, int sglist_cnt) { struct memdesc mem; mem.u.md_list = vlist; mem.md_nseg = sglist_cnt; mem.md_type = MEMDESC_VLIST; return (mem); } static inline struct memdesc memdesc_plist(struct bus_dma_segment *plist, int sglist_cnt) { struct memdesc mem; mem.u.md_list = plist; mem.md_nseg = sglist_cnt; mem.md_type = MEMDESC_PLIST; return (mem); } static inline struct memdesc memdesc_uio(struct uio *uio) { struct memdesc mem; mem.u.md_uio = uio; mem.md_type = MEMDESC_UIO; return (mem); } static inline struct memdesc memdesc_mbuf(struct mbuf *mbuf) { struct memdesc mem; mem.u.md_mbuf = mbuf; mem.md_type = MEMDESC_MBUF; return (mem); } static inline struct memdesc memdesc_vmpages(struct vm_page **ma, size_t len, u_int ma_offset) { struct memdesc mem; mem.u.md_ma = ma; mem.md_len = len; mem.md_type = MEMDESC_VMPAGES; mem.md_offset = ma_offset; return (mem); } struct memdesc memdesc_bio(struct bio *bio); struct memdesc memdesc_ccb(union ccb *ccb); /* * Similar to m_copyback/data, *_copyback copy data from the 'src' * buffer into the memory descriptor's data buffer while *_copydata * copy data from the memory descriptor's data buffer into the the * 'dst' buffer. */ void memdesc_copyback(struct memdesc *mem, int off, int size, const void *src); void memdesc_copydata(struct memdesc *mem, int off, int size, void *dst); +/* + * This routine constructs a chain of M_EXT mbufs backed by a data + * buffer described by a memory descriptor. Some buffers may require + * multiple mbufs. For memory descriptors using unmapped storage + * (e.g. memdesc_vmpages), M_EXTPG mbufs are used. + * + * Since memory descriptors are not an actual buffer, just a + * description of the buffer, the caller is required to supply a + * couple of helper routines to manage allocation of the raw mbufs and + * associate them with a reference to the underlying buffer. + * + * The memdesc_alloc_ext_mbuf_t callback is passed the callback + * argument as its first argument, the how flag as its second + * argument, and the pointer and length of a KVA buffer. This + * callback should allocate an mbuf for the KVA buffer, either by + * making a copy of the data or using m_extaddref(). + * + * The memdesc_alloc_extpg_mbuf_t callback is passed the callback + * argument as its first argument and the how flag as its second + * argument. It should return an empty mbuf allocated by + * mb_alloc_ext_pgs. + * + * If either of the callbacks returns NULL, any partially allocated + * chain is freed and this routine returns NULL. + * + * If can_truncate is true, then this function might return a short + * chain to avoid gratuitously splitting up a page. + */ +typedef struct mbuf *memdesc_alloc_ext_mbuf_t(void *, int, void *, size_t); +typedef struct mbuf *memdesc_alloc_extpg_mbuf_t(void *, int); + +struct mbuf *memdesc_alloc_ext_mbufs(struct memdesc *mem, + memdesc_alloc_ext_mbuf_t *ext_alloc, + memdesc_alloc_extpg_mbuf_t *extpg_alloc, void *cb_arg, int how, + size_t offset, size_t len, size_t *actual_len, bool can_truncate); + #endif /* _SYS_MEMDESC_H_ */