Page MenuHomeFreeBSD
Differential D27726 Diff 81195 sys/kern/kern_sendfile.c

Changeset View

Standalone View

View Options

sys/kern/kern_sendfile.c

Show First 20 Lines • Show All 407 Lines • ▼ Show 20 Lines#endif
free(sfio, M_SENDFILE); free(sfio, M_SENDFILE);
} }
/* /*
* Iterate through pages vector and request paging for non-valid pages. * Iterate through pages vector and request paging for non-valid pages.
*/ */
static int static int
sendfile_swapin(vm_object_t obj, struct sf_io *sfio, int *nios, off_t off, sendfile_swapin(vm_object_t obj, struct sf_io *sfio, int *nios, off_t off,
off_t len, int npages, int rhpages, int flags) off_t len, int rhpages, int flags)
{ {
vm_page_t *pa; vm_page_t *pa;
int a, count, count1, grabbed, i, j, rv; int a, count, count1, grabbed, i, j, npages, rv;
pa = sfio->pa; pa = sfio->pa;
npages = sfio->npages;
*nios = 0; *nios = 0;
flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0; flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0;
sfio->pindex0 = OFF_TO_IDX(off); sfio->pindex0 = OFF_TO_IDX(off);
/* /*
* First grab all the pages and wire them. Note that we grab * First grab all the pages and wire them. Note that we grab
* only required pages. Readahead pages are dealt with later. * only required pages. Readahead pages are dealt with later.
*/ */
▲ Show 20 Lines • Show All 470 Lines • ▼ Show 20 Lines rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -
npages, rhpages); npages, rhpages);
sfio = malloc(sizeof(struct sf_io) + sfio = malloc(sizeof(struct sf_io) +
npages * sizeof(vm_page_t), M_SENDFILE, M_WAITOK); npages * sizeof(vm_page_t), M_SENDFILE, M_WAITOK);
refcount_init(&sfio->nios, 1); refcount_init(&sfio->nios, 1);
sfio->obj = obj; sfio->obj = obj;
sfio->error = 0; sfio->error = 0;
sfio->m = NULL; sfio->m = NULL;
sfio->npages = npages;
#ifdef KERN_TLS #ifdef KERN_TLS
/* /*
* This doesn't use ktls_hold() because sfio->m will * This doesn't use ktls_hold() because sfio->m will
* also have a reference on 'tls' that will be valid * also have a reference on 'tls' that will be valid
* for all of sfio's lifetime. * for all of sfio's lifetime.
*/ */
sfio->tls = tls; sfio->tls = tls;
#endif #endif
vm_object_pip_add(obj, 1); vm_object_pip_add(obj, 1);
error = sendfile_swapin(obj, sfio, &nios, off, space, npages, error = sendfile_swapin(obj, sfio, &nios, off, space, rhpages,
rhpages, flags); flags);
if (error != 0) { if (error != 0) {
if (vp != NULL) if (vp != NULL)
VOP_UNLOCK(vp); VOP_UNLOCK(vp);
sendfile_iodone(sfio, NULL, 0, error); sendfile_iodone(sfio, NULL, 0, error);
goto done; goto done;
} }
/* /*
Show All 31 Lines#endif
for (int i = 0; i < npages; i++) { for (int i = 0; i < npages; i++) {
/* /*
* If a page wasn't grabbed successfully, then * If a page wasn't grabbed successfully, then
* trim the array. Can happen only with SF_NODISKIO. * trim the array. Can happen only with SF_NODISKIO.
*/ */
if (pa[i] == NULL) { if (pa[i] == NULL) {
SFSTAT_INC(sf_busy); SFSTAT_INC(sf_busy);
fixspace(npages, i, off, &space); fixspace(npages, i, off, &space);
npages = i; sfio->npages = i;
softerr = EBUSY; softerr = EBUSY;
break; break;
} }
pga = pa[i]; pga = pa[i];
if (pga == bogus_page) if (pga == bogus_page)
pga = vm_page_relookup(obj, sfio->pindex0 + i); pga = vm_page_relookup(obj, sfio->pindex0 + i);
if (use_ext_pgs) { if (use_ext_pgs) {
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines for (int i = 0; i < npages; i++) {
* threads might exhaust the buffers and then * threads might exhaust the buffers and then
* deadlock. * deadlock.
*/ */
sf = sf_buf_alloc(pga, sf = sf_buf_alloc(pga,
m != NULL ? SFB_NOWAIT : SFB_CATCH); m != NULL ? SFB_NOWAIT : SFB_CATCH);
if (sf == NULL) { if (sf == NULL) {
SFSTAT_INC(sf_allocfail); SFSTAT_INC(sf_allocfail);
sendfile_iowait(sfio, "sfnosf"); sendfile_iowait(sfio, "sfnosf");
for (int j = i; j < npages; j++) for (int j = i; j < npages; j++) {
vm_page_unwire(pa[j], PQ_INACTIVE); vm_page_unwire(pa[j], PQ_INACTIVE);
pa[j] = NULL;
}
if (m == NULL) if (m == NULL)
softerr = ENOBUFS; softerr = ENOBUFS;
fixspace(npages, i, off, &space); fixspace(npages, i, off, &space);
npages = i; sfio->npages = i;
break; break;
} }
m0 = m_get(M_WAITOK, MT_DATA); m0 = m_get(M_WAITOK, MT_DATA);
m0->m_ext.ext_buf = (char *)sf_buf_kva(sf); m0->m_ext.ext_buf = (char *)sf_buf_kva(sf);
m0->m_ext.ext_size = PAGE_SIZE; m0->m_ext.ext_size = PAGE_SIZE;
m0->m_ext.ext_arg1 = sf; m0->m_ext.ext_arg1 = sf;
m0->m_ext.ext_type = EXT_SFBUF; m0->m_ext.ext_type = EXT_SFBUF;
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines#ifdef KERN_TLS
ktls_enqueue(m, so, tls_enq_cnt); ktls_enqueue(m, so, tls_enq_cnt);
} else } else
#endif #endif
error = (*so->so_proto->pr_usrreqs->pru_send) error = (*so->so_proto->pr_usrreqs->pru_send)
(so, 0, m, NULL, NULL, td); (so, 0, m, NULL, NULL, td);
} else { } else {
sfio->so = so; sfio->so = so;
sfio->m = m0; sfio->m = m0;
sfio->npages = npages;
soref(so); soref(so);
error = (*so->so_proto->pr_usrreqs->pru_send) error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td); (so, PRUS_NOTREADY, m, NULL, NULL, td);
sendfile_iodone(sfio, NULL, 0, 0); sendfile_iodone(sfio, NULL, 0, 0);
} }
CURVNET_RESTORE(); CURVNET_RESTORE();
m = NULL; /* pru_send always consumes */ m = NULL; /* pru_send always consumes */
▲ Show 20 Lines • Show All 173 Lines • Show Last 20 Lines