Differential D27726 Diff 81063 sys/kern/kern_sendfile.c

Changeset View

Standalone View

sys/kern/kern_sendfile.c

Show First 20 Lines • Show All 407 Lines • ▼ Show 20 Lines	#endif
free(sfio, M_SENDFILE);		free(sfio, M_SENDFILE);
}		}

/*		/*
* Iterate through pages vector and request paging for non-valid pages.		* Iterate through pages vector and request paging for non-valid pages.
*/		*/
static int		static int
sendfile_swapin(vm_object_t obj, struct sf_io sfio, int nios, off_t off,		sendfile_swapin(vm_object_t obj, struct sf_io sfio, int nios, off_t off,
off_t len, int npages, int rhpages, int flags)		off_t len, int rhpages, int flags)
{		{
vm_page_t *pa;		vm_page_t *pa;
int a, count, count1, grabbed, i, j, rv;		int a, count, count1, grabbed, i, j, npages, rv;

pa = sfio->pa;		pa = sfio->pa;
		npages = sfio->npages;
*nios = 0;		*nios = 0;
flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0;		flags = (flags & SF_NODISKIO) ? VM_ALLOC_NOWAIT : 0;
sfio->pindex0 = OFF_TO_IDX(off);		sfio->pindex0 = OFF_TO_IDX(off);

/*		/*
* First grab all the pages and wire them. Note that we grab		* First grab all the pages and wire them. Note that we grab
* only required pages. Readahead pages are dealt with later.		* only required pages. Readahead pages are dealt with later.
*/		*/
▲ Show 20 Lines • Show All 470 Lines • ▼ Show 20 Lines	rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -
npages, rhpages);		npages, rhpages);

sfio = malloc(sizeof(struct sf_io) +		sfio = malloc(sizeof(struct sf_io) +
npages * sizeof(vm_page_t), M_SENDFILE, M_WAITOK);		npages * sizeof(vm_page_t), M_SENDFILE, M_WAITOK);
refcount_init(&sfio->nios, 1);		refcount_init(&sfio->nios, 1);
sfio->obj = obj;		sfio->obj = obj;
sfio->error = 0;		sfio->error = 0;
sfio->m = NULL;		sfio->m = NULL;
		sfio->npages = npages;
#ifdef KERN_TLS		#ifdef KERN_TLS
/*		/*
* This doesn't use ktls_hold() because sfio->m will		* This doesn't use ktls_hold() because sfio->m will
* also have a reference on 'tls' that will be valid		* also have a reference on 'tls' that will be valid
* for all of sfio's lifetime.		* for all of sfio's lifetime.
*/		*/
sfio->tls = tls;		sfio->tls = tls;
#endif		#endif
vm_object_pip_add(obj, 1);		vm_object_pip_add(obj, 1);
error = sendfile_swapin(obj, sfio, &nios, off, space, npages,		error = sendfile_swapin(obj, sfio, &nios, off, space, rhpages,
rhpages, flags);		flags);
if (error != 0) {		if (error != 0) {
if (vp != NULL)		if (vp != NULL)
VOP_UNLOCK(vp);		VOP_UNLOCK(vp);
sendfile_iodone(sfio, NULL, 0, error);		sendfile_iodone(sfio, NULL, 0, error);
goto done;		goto done;
}		}

/*		/*
Show All 31 Lines	#endif
for (int i = 0; i < npages; i++) {		for (int i = 0; i < npages; i++) {
/*		/*
* If a page wasn't grabbed successfully, then		* If a page wasn't grabbed successfully, then
* trim the array. Can happen only with SF_NODISKIO.		* trim the array. Can happen only with SF_NODISKIO.
*/		*/
if (pa[i] == NULL) {		if (pa[i] == NULL) {
SFSTAT_INC(sf_busy);		SFSTAT_INC(sf_busy);
fixspace(npages, i, off, &space);		fixspace(npages, i, off, &space);
npages = i;		sfio->npages = i;
softerr = EBUSY;		softerr = EBUSY;
break;		break;
}		}
pga = pa[i];		pga = pa[i];
if (pga == bogus_page)		if (pga == bogus_page)
pga = vm_page_relookup(obj, sfio->pindex0 + i);		pga = vm_page_relookup(obj, sfio->pindex0 + i);

if (use_ext_pgs) {		if (use_ext_pgs) {
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines	for (int i = 0; i < npages; i++) {
* threads might exhaust the buffers and then		* threads might exhaust the buffers and then
* deadlock.		* deadlock.
*/		*/
sf = sf_buf_alloc(pga,		sf = sf_buf_alloc(pga,
m != NULL ? SFB_NOWAIT : SFB_CATCH);		m != NULL ? SFB_NOWAIT : SFB_CATCH);
if (sf == NULL) {		if (sf == NULL) {
SFSTAT_INC(sf_allocfail);		SFSTAT_INC(sf_allocfail);
sendfile_iowait(sfio, "sfnosf");		sendfile_iowait(sfio, "sfnosf");
for (int j = i; j < npages; j++)		for (int j = i; j < npages; j++) {
vm_page_unwire(pa[j], PQ_INACTIVE);		vm_page_unwire(pa[j], PQ_INACTIVE);
		pa[j] = NULL;
		}
if (m == NULL)		if (m == NULL)
softerr = ENOBUFS;		softerr = ENOBUFS;
fixspace(npages, i, off, &space);		fixspace(npages, i, off, &space);
npages = i;		sfio->npages = i;
break;		break;
}		}

m0 = m_get(M_WAITOK, MT_DATA);		m0 = m_get(M_WAITOK, MT_DATA);
m0->m_ext.ext_buf = (char *)sf_buf_kva(sf);		m0->m_ext.ext_buf = (char *)sf_buf_kva(sf);
m0->m_ext.ext_size = PAGE_SIZE;		m0->m_ext.ext_size = PAGE_SIZE;
m0->m_ext.ext_arg1 = sf;		m0->m_ext.ext_arg1 = sf;
m0->m_ext.ext_type = EXT_SFBUF;		m0->m_ext.ext_type = EXT_SFBUF;
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines	#ifdef KERN_TLS
ktls_enqueue(m, so, tls_enq_cnt);		ktls_enqueue(m, so, tls_enq_cnt);
} else		} else
#endif		#endif
error = (*so->so_proto->pr_usrreqs->pru_send)		error = (*so->so_proto->pr_usrreqs->pru_send)
(so, 0, m, NULL, NULL, td);		(so, 0, m, NULL, NULL, td);
} else {		} else {
sfio->so = so;		sfio->so = so;
sfio->m = m0;		sfio->m = m0;
sfio->npages = npages;
soref(so);		soref(so);
error = (*so->so_proto->pr_usrreqs->pru_send)		error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);		(so, PRUS_NOTREADY, m, NULL, NULL, td);
sendfile_iodone(sfio, NULL, 0, 0);		sendfile_iodone(sfio, NULL, 0, 0);
}		}
CURVNET_RESTORE();		CURVNET_RESTORE();

m = NULL; /* pru_send always consumes */		m = NULL; /* pru_send always consumes */
▲ Show 20 Lines • Show All 173 Lines • Show Last 20 Lines