Index: share/man/man9/mbpool.9 =================================================================== --- share/man/man9/mbpool.9 +++ share/man/man9/mbpool.9 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd July 15, 2003 +.Dd September 27, 2017 .Dt MBPOOL 9 .Os .Sh NAME @@ -50,7 +50,7 @@ .Ft void .Fn mbp_free "struct mbpool *mbp" "void *p" .Ft void -.Fn mbp_ext_free "void *" "void *" +.Fn mbp_ext_free "struct mbuf *" .Ft void .Fn mbp_card_free "struct mbpool *mbp" .Ft void @@ -223,8 +223,6 @@ can be given to .Fn m_extadd as the free function. -The user argument must be the pointer to -the pool. .Pp Before using the contents of a buffer returned by the card, the driver must call Index: share/man/man9/mbuf.9 =================================================================== --- share/man/man9/mbuf.9 +++ share/man/man9/mbuf.9 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd October 10, 2016 +.Dd September 27, 2017 .Dt MBUF 9 .Os .\" @@ -44,12 +44,12 @@ .Fn MCLGET "struct mbuf *mbuf" "int how" .Fo MEXTADD .Fa "struct mbuf *mbuf" -.Fa "caddr_t buf" +.Fa "char *buf" .Fa "u_int size" -.Fa "void (*free)(void *opt_arg1, void *opt_arg2)" +.Fa "void (*free)(struct mbuf *)" .Fa "void *opt_arg1" .Fa "void *opt_arg2" -.Fa "short flags" +.Fa "int flags" .Fa "int type" .Fc .\" @@ -416,8 +416,13 @@ .Fa opt_arg1 and .Fa opt_arg2 -arguments will be passed unmodified to -.Fa free . +arguments will be saved in +.Va ext_arg1 +and +.Va ext_arg2 +fields of the +.Va struct m_ext +of the mbuf. The .Fa flags argument specifies additional Index: sys/compat/ndis/kern_ndis.c =================================================================== --- sys/compat/ndis/kern_ndis.c +++ sys/compat/ndis/kern_ndis.c @@ -495,17 +495,21 @@ KeReleaseSpinLock(&block->nmb_returnlock, irql); } +static void +ndis_ext_free(struct mbuf *m) +{ + + return (ndis_return_packet(m->m_ext.ext_arg1)); +} + void -ndis_return_packet(struct mbuf *m, void *buf, void *arg) +ndis_return_packet(ndis_packet *p) { - ndis_packet *p; ndis_miniport_block *block; - if (arg == NULL) + if (p == NULL) return; - p = arg; - /* Decrement refcount. */ p->np_refcnt--; @@ -676,9 +680,8 @@ return (ENOBUFS); } m->m_len = MmGetMdlByteCount(buf); - m->m_data = MmGetMdlVirtualAddress(buf); - MEXTADD(m, m->m_data, m->m_len, ndis_return_packet, - m->m_data, p, 0, EXT_NDIS); + m_extadd(m, MmGetMdlVirtualAddress(buf), m->m_len, + ndis_ext_free, p, NULL, 0, EXT_NDIS); p->np_refcnt++; totlen += m->m_len; Index: sys/compat/ndis/ndis_var.h =================================================================== --- sys/compat/ndis/ndis_var.h +++ sys/compat/ndis/ndis_var.h @@ -1743,7 +1743,7 @@ extern int ndis_shutdown_nic(void *); extern int ndis_pnpevent_nic(void *, int); extern int ndis_init_nic(void *); -extern void ndis_return_packet(struct mbuf *, void *, void *); +extern void ndis_return_packet(ndis_packet *); extern int ndis_init_dma(void *); extern int ndis_destroy_dma(void *); extern int ndis_create_sysctls(void *); Index: sys/dev/cas/if_cas.c =================================================================== --- sys/dev/cas/if_cas.c +++ sys/dev/cas/if_cas.c @@ -133,7 +133,7 @@ static int cas_disable_rx(struct cas_softc *sc); static int cas_disable_tx(struct cas_softc *sc); static void cas_eint(struct cas_softc *sc, u_int status); -static void cas_free(struct mbuf *m, void *arg1, void* arg2); +static void cas_free(struct mbuf *m); static void cas_init(void *xsc); static void cas_init_locked(struct cas_softc *sc); static void cas_init_regs(struct cas_softc *sc); @@ -1732,16 +1732,10 @@ refcount_acquire(&rxds->rxds_refcount); bus_dmamap_sync(sc->sc_rdmatag, rxds->rxds_dmamap, BUS_DMASYNC_POSTREAD); -#if __FreeBSD_version < 800016 - MEXTADD(m, (caddr_t)rxds->rxds_buf + - off * 256 + ETHER_ALIGN, len, cas_free, - rxds, M_RDONLY, EXT_NET_DRV); -#else - MEXTADD(m, (caddr_t)rxds->rxds_buf + + m_extadd(m, (char *)rxds->rxds_buf + off * 256 + ETHER_ALIGN, len, cas_free, sc, (void *)(uintptr_t)idx, M_RDONLY, EXT_NET_DRV); -#endif if ((m->m_flags & M_EXT) == 0) { m_freem(m); m = NULL; @@ -1779,16 +1773,10 @@ m->m_len = min(CAS_PAGE_SIZE - off, len); bus_dmamap_sync(sc->sc_rdmatag, rxds->rxds_dmamap, BUS_DMASYNC_POSTREAD); -#if __FreeBSD_version < 800016 - MEXTADD(m, (caddr_t)rxds->rxds_buf + off, - m->m_len, cas_free, rxds, M_RDONLY, - EXT_NET_DRV); -#else - MEXTADD(m, (caddr_t)rxds->rxds_buf + off, + m_extadd(m, (char *)rxds->rxds_buf + off, m->m_len, cas_free, sc, (void *)(uintptr_t)idx, M_RDONLY, EXT_NET_DRV); -#endif if ((m->m_flags & M_EXT) == 0) { m_freem(m); m = NULL; @@ -1818,19 +1806,11 @@ sc->sc_rdmatag, rxds2->rxds_dmamap, BUS_DMASYNC_POSTREAD); -#if __FreeBSD_version < 800016 - MEXTADD(m2, - (caddr_t)rxds2->rxds_buf, - m2->m_len, cas_free, - rxds2, M_RDONLY, - EXT_NET_DRV); -#else - MEXTADD(m2, - (caddr_t)rxds2->rxds_buf, + m_extadd(m2, + (char *)rxds2->rxds_buf, m2->m_len, cas_free, sc, (void *)(uintptr_t)idx2, M_RDONLY, EXT_NET_DRV); -#endif if ((m2->m_flags & M_EXT) == 0) { m_freem(m2); @@ -1889,21 +1869,15 @@ } static void -cas_free(struct mbuf *m, void *arg1, void *arg2) +cas_free(struct mbuf *m) { struct cas_rxdsoft *rxds; struct cas_softc *sc; u_int idx, locked; -#if __FreeBSD_version < 800016 - rxds = arg2; - sc = rxds->rxds_sc; - idx = rxds->rxds_idx; -#else - sc = arg1; - idx = (uintptr_t)arg2; + sc = m->m_ext.ext_arg1; + idx = (uintptr_t)m->m_ext.ext_arg2; rxds = &sc->sc_rxdsoft[idx]; -#endif if (refcount_release(&rxds->rxds_refcount) == 0) return; Index: sys/dev/cas/if_casvar.h =================================================================== --- sys/dev/cas/if_casvar.h +++ sys/dev/cas/if_casvar.h @@ -119,10 +119,6 @@ void *rxds_buf; /* receive buffer */ bus_dmamap_t rxds_dmamap; /* our DMA map */ bus_addr_t rxds_paddr; /* physical address of the segment */ -#if __FreeBSD_version < 800016 - struct cas_softc *rxds_sc; /* softc pointer */ - u_int rxds_idx; /* our index */ -#endif u_int rxds_refcount; /* hardware + mbuf references */ }; @@ -239,18 +235,7 @@ __CAS_UPDATE_RXDESC(&(sc)->sc_rxdescs[(d)], \ &(sc)->sc_rxdsoft[(s)], (s)) -#if __FreeBSD_version < 800016 -#define CAS_INIT_RXDESC(sc, d, s) \ -do { \ - struct cas_rxdsoft *__rxds = &(sc)->sc_rxdsoft[(s)]; \ - \ - __rxds->rxds_sc = (sc); \ - __rxds->rxds_idx = (s); \ - __CAS_UPDATE_RXDESC(&(sc)->sc_rxdescs[(d)], __rxds, (s)); \ -} while (0) -#else #define CAS_INIT_RXDESC(sc, d, s) CAS_UPDATE_RXDESC(sc, d, s) -#endif #define CAS_LOCK_INIT(_sc, _name) \ mtx_init(&(_sc)->sc_mtx, _name, MTX_NETWORK_LOCK, MTX_DEF) Index: sys/dev/cxgbe/t4_sge.c =================================================================== --- sys/dev/cxgbe/t4_sge.c +++ sys/dev/cxgbe/t4_sge.c @@ -1670,10 +1670,10 @@ } static void -rxb_free(struct mbuf *m, void *arg1, void *arg2) +rxb_free(struct mbuf *m) { - uma_zone_t zone = arg1; - caddr_t cl = arg2; + uma_zone_t zone = m->m_ext.ext_arg1; + void *cl = m->m_ext.ext_arg2; uma_zfree(zone, cl); counter_u64_add(extfree_rels, 1); Index: sys/dev/cxgbe/tom/t4_cpl_io.c =================================================================== --- sys/dev/cxgbe/tom/t4_cpl_io.c +++ sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1979,9 +1979,9 @@ } static void -t4_aiotx_mbuf_free(struct mbuf *m, void *buffer, void *arg) +t4_aiotx_mbuf_free(struct mbuf *m) { - struct aiotx_buffer *ab = buffer; + struct aiotx_buffer *ab = m->m_ext.ext_arg1; #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, Index: sys/dev/dpaa/if_dtsec_rm.c =================================================================== --- sys/dev/dpaa/if_dtsec_rm.c +++ sys/dev/dpaa/if_dtsec_rm.c @@ -337,11 +337,13 @@ * @{ */ static void -dtsec_rm_fqr_mext_free(struct mbuf *m, void *buffer, void *arg) +dtsec_rm_fqr_mext_free(struct mbuf *m) { struct dtsec_softc *sc; + void *buffer; - sc = arg; + buffer = m->m_ext.ext_arg1; + sc = m->m_ext.ext_arg2; if (bman_count(sc->sc_rx_pool) <= DTSEC_RM_POOL_RX_MAX_SIZE) bman_put_buffer(sc->sc_rx_pool, buffer); else Index: sys/dev/if_ndis/if_ndis.c =================================================================== --- sys/dev/if_ndis/if_ndis.c +++ sys/dev/if_ndis/if_ndis.c @@ -1418,7 +1418,7 @@ p = packets[i]; if (p->np_oob.npo_status == NDIS_STATUS_SUCCESS) { p->np_refcnt++; - (void)ndis_return_packet(NULL ,p, block); + ndis_return_packet(p); } } return; @@ -1431,7 +1431,7 @@ if (ndis_ptom(&m0, p)) { device_printf(sc->ndis_dev, "ptom failed\n"); if (p->np_oob.npo_status == NDIS_STATUS_SUCCESS) - (void)ndis_return_packet(NULL, p, block); + ndis_return_packet(p); } else { #ifdef notdef if (p->np_oob.npo_status == NDIS_STATUS_RESOURCES) { Index: sys/dev/iscsi_initiator/isc_soc.c =================================================================== --- sys/dev/iscsi_initiator/isc_soc.c +++ sys/dev/iscsi_initiator/isc_soc.c @@ -70,12 +70,13 @@ | function for freeing external storage for mbuf */ static void -ext_free(struct mbuf *m, void *a, void *b) +ext_free(struct mbuf *m) { - pduq_t *pq = b; + pduq_t *pq = m->m_ext.ext_arg1; if(pq->buf != NULL) { - debug(3, "ou_refcnt=%d a=%p b=%p", ou_refcnt, a, pq->buf); + debug(3, "ou_refcnt=%d a=%p b=%p", + ou_refcnt, m->m_ext.ext_buf, pq->buf); free(pq->buf, M_ISCSIBUF); pq->buf = NULL; } @@ -137,11 +138,8 @@ md->m_ext.ext_cnt = &ou_refcnt; l = min(MCLBYTES, len); debug(4, "setting ext_free(arg=%p len/l=%d/%d)", pq->buf, len, l); - MEXTADD(md, pp->ds_addr + off, l, ext_free, -#if __FreeBSD_version >= 800000 - pp->ds_addr + off, -#endif - pq, 0, EXT_EXTREF); + m_extadd(md, pp->ds_addr + off, l, ext_free, pq, NULL, 0, + EXT_EXTREF); md->m_len = l; md->m_next = NULL; mh->m_pkthdr.len += l; Index: sys/dev/lge/if_lge.c =================================================================== --- sys/dev/lge/if_lge.c +++ sys/dev/lge/if_lge.c @@ -123,7 +123,7 @@ static int lge_alloc_jumbo_mem(struct lge_softc *); static void lge_free_jumbo_mem(struct lge_softc *); static void *lge_jalloc(struct lge_softc *); -static void lge_jfree(struct mbuf *, void *, void *); +static void lge_jfree(struct mbuf *); static int lge_newbuf(struct lge_softc *, struct lge_rx_desc *, struct mbuf *); static int lge_encap(struct lge_softc *, struct mbuf *, u_int32_t *); @@ -689,7 +689,7 @@ struct mbuf *m; { struct mbuf *m_new = NULL; - caddr_t *buf = NULL; + char *buf = NULL; if (m == NULL) { MGETHDR(m_new, M_NOWAIT, MT_DATA); @@ -710,10 +710,9 @@ return(ENOBUFS); } /* Attach the buffer to the mbuf */ - m_new->m_data = (void *)buf; m_new->m_len = m_new->m_pkthdr.len = LGE_JUMBO_FRAMELEN; - MEXTADD(m_new, buf, LGE_JUMBO_FRAMELEN, lge_jfree, - buf, (struct lge_softc *)sc, 0, EXT_NET_DRV); + m_extadd(m_new, buf, LGE_JUMBO_FRAMELEN, lge_jfree, sc, NULL, + 0, EXT_NET_DRV); } else { m_new = m; m_new->m_len = m_new->m_pkthdr.len = LGE_JUMBO_FRAMELEN; @@ -848,20 +847,20 @@ * Release a jumbo buffer. */ static void -lge_jfree(struct mbuf *m, void *buf, void *args) +lge_jfree(struct mbuf *m) { struct lge_softc *sc; int i; struct lge_jpool_entry *entry; /* Extract the softc struct pointer. */ - sc = args; + sc = m->m_ext.ext_arg1; if (sc == NULL) panic("lge_jfree: can't find softc pointer!"); /* calculate the slot this buffer belongs to */ - i = ((vm_offset_t)buf + i = ((vm_offset_t)m->m_ext.ext_buf - (vm_offset_t)sc->lge_cdata.lge_jumbo_buf) / LGE_JLEN; if ((i < 0) || (i >= LGE_JSLOTS)) Index: sys/dev/mwl/if_mwl.c =================================================================== --- sys/dev/mwl/if_mwl.c +++ sys/dev/mwl/if_mwl.c @@ -2522,12 +2522,12 @@ } static void -mwl_ext_free(struct mbuf *m, void *data, void *arg) +mwl_ext_free(struct mbuf *m) { - struct mwl_softc *sc = arg; + struct mwl_softc *sc = m->m_ext.ext_arg1; /* XXX bounds check data */ - mwl_putrxdma(sc, data); + mwl_putrxdma(sc, m->m_ext.ext_buf); /* * If we were previously blocked by a lack of rx dma buffers * check if we now have enough to restart rx interrupt handling. @@ -2746,8 +2746,8 @@ * descriptor using the replacement dma * buffer we just installed above. */ - MEXTADD(m, data, MWL_AGGR_SIZE, mwl_ext_free, - data, sc, 0, EXT_NET_DRV); + m_extadd(m, data, MWL_AGGR_SIZE, mwl_ext_free, sc, NULL, 0, + EXT_NET_DRV); m->m_data += off - hdrlen; m->m_pkthdr.len = m->m_len = pktlen; /* NB: dma buffer assumed read-only */ Index: sys/dev/netmap/netmap_generic.c =================================================================== --- sys/dev/netmap/netmap_generic.c +++ sys/dev/netmap/netmap_generic.c @@ -166,7 +166,7 @@ * has a KASSERT(), checking that the mbuf dtor function is not NULL. */ -static void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { } +static void void_mbuf_dtor(struct mbuf *m) { } #define SET_MBUF_DESTRUCTOR(m, fn) do { \ (m)->m_ext.ext_free = (fn != NULL) ? \ @@ -624,7 +624,7 @@ * txsync. */ netmap_generic_irq(na, r, NULL); #ifdef __FreeBSD__ - void_mbuf_dtor(m, NULL, NULL); + void_mbuf_dtor(m); #endif } Index: sys/dev/wb/if_wb.c =================================================================== --- sys/dev/wb/if_wb.c +++ sys/dev/wb/if_wb.c @@ -143,7 +143,7 @@ static int wb_attach(device_t); static int wb_detach(device_t); -static void wb_bfree(struct mbuf *, void *addr, void *args); +static void wb_bfree(struct mbuf *); static int wb_newbuf(struct wb_softc *, struct wb_chain_onefrag *, struct mbuf *); static int wb_encap(struct wb_softc *, struct wb_chain *, struct mbuf *); @@ -824,7 +824,7 @@ } static void -wb_bfree(struct mbuf *m, void *buf, void *args) +wb_bfree(struct mbuf *m) { } @@ -843,10 +843,9 @@ MGETHDR(m_new, M_NOWAIT, MT_DATA); if (m_new == NULL) return(ENOBUFS); - m_new->m_data = c->wb_buf; m_new->m_pkthdr.len = m_new->m_len = WB_BUFBYTES; - MEXTADD(m_new, c->wb_buf, WB_BUFBYTES, wb_bfree, c->wb_buf, - NULL, 0, EXT_NET_DRV); + m_extadd(m_new, c->wb_buf, WB_BUFBYTES, wb_bfree, NULL, NULL, + 0, EXT_NET_DRV); } else { m_new = m; m_new->m_len = m_new->m_pkthdr.len = WB_BUFBYTES; Index: sys/kern/kern_mbuf.c =================================================================== --- sys/kern/kern_mbuf.c +++ sys/kern/kern_mbuf.c @@ -504,7 +504,7 @@ #endif m = (struct mbuf *)arg; if (m != NULL) { - m->m_ext.ext_buf = (caddr_t)mem; + m->m_ext.ext_buf = (char *)mem; m->m_data = m->m_ext.ext_buf; m->m_flags |= M_EXT; m->m_ext.ext_free = NULL; @@ -675,31 +675,21 @@ uma_zfree(zone_mbuf, mref); break; case EXT_SFBUF: - sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2); - uma_zfree(zone_mbuf, mref); - break; - case EXT_SFBUF_NOCACHE: - sf_ext_free_nocache(m->m_ext.ext_arg1, - m->m_ext.ext_arg2); - uma_zfree(zone_mbuf, mref); - break; case EXT_NET_DRV: case EXT_MOD_TYPE: case EXT_DISPOSABLE: - KASSERT(m->m_ext.ext_free != NULL, + KASSERT(mref->m_ext.ext_free != NULL, ("%s: ext_free not set", __func__)); - (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, - m->m_ext.ext_arg2); + mref->m_ext.ext_free(mref); uma_zfree(zone_mbuf, mref); break; case EXT_EXTREF: KASSERT(m->m_ext.ext_free != NULL, ("%s: ext_free not set", __func__)); - (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, - m->m_ext.ext_arg2); + m->m_ext.ext_free(m); break; default: - KASSERT(m->m_ext.ext_type == 0, + KASSERT(mref->m_ext.ext_type == 0, ("%s: unknown ext_type", __func__)); } } @@ -918,9 +908,8 @@ * Nothing. */ void -m_extadd(struct mbuf *mb, caddr_t buf, u_int size, - void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2, - int flags, int type) +m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef, + void *arg1, void *arg2, int flags, int type) { KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); Index: sys/kern/kern_sendfile.c =================================================================== --- sys/kern/kern_sendfile.c +++ sys/kern/kern_sendfile.c @@ -62,6 +62,9 @@ #include #include +#define EXT_FLAG_SYNC EXT_FLAG_VENDOR1 +#define EXT_FLAG_NOCACHE EXT_FLAG_VENDOR2 + /* * Structure describing a single sendfile(2) I/O, which may consist of * several underlying pager I/Os. @@ -122,63 +125,53 @@ * Detach mapped page and release resources back to the system. Called * by mbuf(9) code when last reference to a page is freed. */ -void -sf_ext_free(void *arg1, void *arg2) +static void +sendfile_free_page(vm_page_t pg, bool nocache) { - struct sf_buf *sf = arg1; - struct sendfile_sync *sfs = arg2; - vm_page_t pg = sf_buf_page(sf); - - sf_buf_free(sf); vm_page_lock(pg); /* - * Check for the object going away on us. This can - * happen since we don't hold a reference to it. - * If so, we're responsible for freeing the page. + * In either case check for the object going away on us. This can + * happen since we don't hold a reference to it. If so, we're + * responsible for freeing the page. In 'noncache' case try to free + * the page, but only if it is cheap to. */ - if (vm_page_unwire(pg, PQ_INACTIVE) && pg->object == NULL) - vm_page_free(pg); - vm_page_unlock(pg); + if (vm_page_unwire(pg, nocache ? PQ_NONE : PQ_INACTIVE)) { + vm_object_t obj; - if (sfs != NULL) { - mtx_lock(&sfs->mtx); - KASSERT(sfs->count > 0, ("Sendfile sync botchup count == 0")); - if (--sfs->count == 0) - cv_signal(&sfs->cv); - mtx_unlock(&sfs->mtx); + if ((obj = pg->object) == NULL) + vm_page_free(pg); + else if (nocache) { + if (!vm_page_xbusied(pg) && VM_OBJECT_TRYWLOCK(obj)) { + vm_page_free(pg); + VM_OBJECT_WUNLOCK(obj); + } else + vm_page_deactivate(pg); + } } + vm_page_unlock(pg); } -/* - * Same as above, but forces the page to be detached from the object - * and go into free pool. - */ -void -sf_ext_free_nocache(void *arg1, void *arg2) +static void +sendfile_free_mext(struct mbuf *m) { - struct sf_buf *sf = arg1; - struct sendfile_sync *sfs = arg2; - vm_page_t pg = sf_buf_page(sf); + struct sf_buf *sf; + vm_page_t pg; + bool nocache; - sf_buf_free(sf); + KASSERT(m->m_flags & M_EXT && m->m_ext.ext_type == EXT_SFBUF, + ("%s: m %p !M_EXT or !EXT_SFBUF", __func__, m)); - vm_page_lock(pg); - if (vm_page_unwire(pg, PQ_NONE)) { - vm_object_t obj; + sf = m->m_ext.ext_arg1; + pg = sf_buf_page(sf); + nocache = m->m_ext.ext_flags & EXT_FLAG_NOCACHE; - /* Try to free the page, but only if it is cheap to. */ - if ((obj = pg->object) == NULL) - vm_page_free(pg); - else if (!vm_page_xbusied(pg) && VM_OBJECT_TRYWLOCK(obj)) { - vm_page_free(pg); - VM_OBJECT_WUNLOCK(obj); - } else - vm_page_deactivate(pg); - } - vm_page_unlock(pg); + sf_buf_free(sf); + sendfile_free_page(pg, nocache); + + if (m->m_ext.ext_flags & EXT_FLAG_SYNC) { + struct sendfile_sync *sfs = m->m_ext.ext_arg2; - if (sfs != NULL) { mtx_lock(&sfs->mtx); KASSERT(sfs->count > 0, ("Sendfile sync botchup count == 0")); if (--sfs->count == 0) @@ -777,7 +770,9 @@ m0->m_ext.ext_buf = (char *)sf_buf_kva(sf); m0->m_ext.ext_size = PAGE_SIZE; m0->m_ext.ext_arg1 = sf; - m0->m_ext.ext_arg2 = sfs; + m0->m_ext.ext_type = EXT_SFBUF; + m0->m_ext.ext_flags = EXT_FLAG_EMBREF; + m0->m_ext.ext_free = sendfile_free_mext; /* * SF_NOCACHE sets the page as being freed upon send. * However, we ignore it for the last page in 'space', @@ -785,14 +780,18 @@ * send (rem > space), or if we have readahead * configured (rhpages > 0). */ - if ((flags & SF_NOCACHE) == 0 || - (i == npages - 1 && - ((off + space) & PAGE_MASK) && - (rem > space || rhpages > 0))) - m0->m_ext.ext_type = EXT_SFBUF; - else - m0->m_ext.ext_type = EXT_SFBUF_NOCACHE; - m0->m_ext.ext_flags = EXT_FLAG_EMBREF; + if ((flags & SF_NOCACHE) && + (i != npages - 1 || + !((off + space) & PAGE_MASK) || + !(rem > space || rhpages > 0))) + m0->m_ext.ext_flags |= EXT_FLAG_NOCACHE; + if (sfs != NULL) { + m0->m_ext.ext_flags |= EXT_FLAG_SYNC; + m0->m_ext.ext_arg2 = sfs; + mtx_lock(&sfs->mtx); + sfs->count++; + mtx_unlock(&sfs->mtx); + } m0->m_ext.ext_count = 1; m0->m_flags |= (M_EXT | M_RDONLY); if (nios) @@ -810,12 +809,6 @@ else m = m0; mtail = m0; - - if (sfs != NULL) { - mtx_lock(&sfs->mtx); - sfs->count++; - mtx_unlock(&sfs->mtx); - } } if (vp != NULL) Index: sys/kern/subr_mbpool.c =================================================================== --- sys/kern/subr_mbpool.c +++ sys/kern/subr_mbpool.c @@ -281,10 +281,10 @@ * Mbuf system external mbuf free routine */ void -mbp_ext_free(struct mbuf *m, void *buf, void *arg) +mbp_ext_free(struct mbuf *m) { - mbp_free(arg, buf); + mbp_free(m->m_ext.ext_arg2, m->m_ext.ext_arg1); } /* Index: sys/kern/uipc_mbuf.c =================================================================== --- sys/kern/uipc_mbuf.c +++ sys/kern/uipc_mbuf.c @@ -188,7 +188,17 @@ KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n)); - n->m_ext = m->m_ext; + /* + * Cache access optimization. For most kinds of external + * storage we don't need full copy of m_ext, since the + * holder of the 'ext_count' is responsible to carry the + * free routine and its arguments. Exclusion is EXT_EXTREF, + * where 'ext_cnt' doesn't point into mbuf at all. + */ + if (m->m_ext.ext_type == EXT_EXTREF) + bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext)); + else + bcopy(&m->m_ext, &n->m_ext, m_ext_copylen); n->m_flags |= M_EXT; n->m_flags |= m->m_flags & M_RDONLY; Index: sys/sys/mbpool.h =================================================================== --- sys/sys/mbpool.h +++ sys/sys/mbpool.h @@ -69,7 +69,7 @@ void mbp_free(struct mbpool *, void *); /* free a chunk that is an external mbuf */ -void mbp_ext_free(struct mbuf *, void *, void *); +void mbp_ext_free(struct mbuf *); /* free all buffers that are marked to be on the card */ void mbp_card_free(struct mbpool *); Index: sys/sys/mbuf.h =================================================================== --- sys/sys/mbuf.h +++ sys/sys/mbuf.h @@ -197,17 +197,33 @@ * Compile-time assertions in uipc_mbuf.c test these values to ensure that * they are correct. */ +typedef void m_ext_free_t(struct mbuf *); struct m_ext { union { - volatile u_int ext_count; /* value of ref count info */ - volatile u_int *ext_cnt; /* pointer to ref count info */ + /* + * If EXT_FLAG_EMBREF is set, then we use refcount in the + * mbuf, the 'ext_count' member. Otherwise, we have a + * shadow copy and we use pointer 'ext_cnt'. The original + * mbuf is responsible to carry the pointer to free routine + * and its arguments. They aren't copied into shadows in + * mb_dupcl() to avoid dereferencing next cachelines. + */ + volatile u_int ext_count; + volatile u_int *ext_cnt; }; - caddr_t ext_buf; /* start of buffer */ + char *ext_buf; /* start of buffer */ uint32_t ext_size; /* size of buffer, for ext_free */ uint32_t ext_type:8, /* type of external storage */ ext_flags:24; /* external storage mbuf flags */ - void (*ext_free) /* free routine if not the usual */ - (struct mbuf *, void *, void *); + /* + * Fields below store the free context for the external storage. + * They are valid only in the refcount carrying mbuf, the one with + * EXT_FLAG_EMBREF flag, with exclusion for EXT_EXTREF type, where + * the free context is copied into all mbufs that use same external + * storage. + */ +#define m_ext_copylen offsetof(struct m_ext, ext_free) + m_ext_free_t *ext_free; /* free routine if not the usual */ void *ext_arg1; /* optional argument pointer */ void *ext_arg2; /* optional argument pointer */ }; @@ -410,7 +426,6 @@ #define EXT_JUMBO16 5 /* jumbo cluster 16184 bytes */ #define EXT_PACKET 6 /* mbuf+cluster from packet zone */ #define EXT_MBUF 7 /* external mbuf reference */ -#define EXT_SFBUF_NOCACHE 8 /* sendfile(2)'s sf_buf not to be cached */ #define EXT_VENDOR1 224 /* for vendor-internal use */ #define EXT_VENDOR2 225 /* for vendor-internal use */ @@ -436,10 +451,10 @@ #define EXT_FLAG_NOFREE 0x000010 /* don't free mbuf to pool, notyet */ -#define EXT_FLAG_VENDOR1 0x010000 /* for vendor-internal use */ -#define EXT_FLAG_VENDOR2 0x020000 /* for vendor-internal use */ -#define EXT_FLAG_VENDOR3 0x040000 /* for vendor-internal use */ -#define EXT_FLAG_VENDOR4 0x080000 /* for vendor-internal use */ +#define EXT_FLAG_VENDOR1 0x010000 /* These flags are vendor */ +#define EXT_FLAG_VENDOR2 0x020000 /* or submodule specific, */ +#define EXT_FLAG_VENDOR3 0x040000 /* not used by mbuf code. */ +#define EXT_FLAG_VENDOR4 0x080000 /* Set/read by submodule. */ #define EXT_FLAG_EXP1 0x100000 /* for experimental use */ #define EXT_FLAG_EXP2 0x200000 /* for experimental use */ @@ -455,12 +470,6 @@ "\24EXT_FLAG_VENDOR4\25EXT_FLAG_EXP1\26EXT_FLAG_EXP2\27EXT_FLAG_EXP3" \ "\30EXT_FLAG_EXP4" -/* - * External reference/free functions. - */ -void sf_ext_free(void *, void *); -void sf_ext_free_nocache(void *, void *); - /* * Flags indicating checksum, segmentation and other offload work to be * done, or already done, by hardware or lower layers. It is split into @@ -610,9 +619,8 @@ void (*)(char *, caddr_t, u_int)); struct mbuf *m_dup(const struct mbuf *, int); int m_dup_pkthdr(struct mbuf *, const struct mbuf *, int); -void m_extadd(struct mbuf *, caddr_t, u_int, - void (*)(struct mbuf *, void *, void *), void *, void *, - int, int); +void m_extadd(struct mbuf *, char *, u_int, m_ext_free_t, + void *, void *, int, int); u_int m_fixhdr(struct mbuf *); struct mbuf *m_fragment(struct mbuf *, int, int); void m_freem(struct mbuf *); @@ -667,8 +675,8 @@ * Associated an external reference counted buffer with an mbuf. */ static __inline void -m_extaddref(struct mbuf *m, caddr_t buf, u_int size, u_int *ref_cnt, - void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2) +m_extaddref(struct mbuf *m, char *buf, u_int size, u_int *ref_cnt, + m_ext_free_t freef, void *arg1, void *arg2) { KASSERT(ref_cnt != NULL, ("%s: ref_cnt not provided", __func__)); @@ -864,7 +872,7 @@ #define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) #define MCLGET(m, how) m_clget((m), (how)) #define MEXTADD(m, buf, size, free, arg1, arg2, flags, type) \ - m_extadd((m), (caddr_t)(buf), (size), (free), (arg1), (arg2), \ + m_extadd((m), (char *)(buf), (size), (free), (arg1), (arg2), \ (flags), (type)) #define m_getm(m, len, how, type) \ m_getm2((m), (len), (how), (type), M_PKTHDR)