Differential D23033 Diff 67031 head/sys/kern/kern_sendfile.c

Changeset View

Standalone View

head/sys/kern/kern_sendfile.c

Show First 20 Lines • Show All 83 Lines • ▼ Show 20 Lines
* of the data.		* of the data.
*/		*/
struct sf_io {		struct sf_io {
volatile u_int nios;		volatile u_int nios;
u_int error;		u_int error;
int npages;		int npages;
struct socket *so;		struct socket *so;
struct mbuf *m;		struct mbuf *m;
		vm_object_t obj;
#ifdef KERN_TLS		#ifdef KERN_TLS
struct ktls_session *tls;		struct ktls_session *tls;
#endif		#endif
vm_page_t pa[];		vm_page_t pa[];
};		};

/*		/*
* Structure used to track requests with SF_SYNC flag.		* Structure used to track requests with SF_SYNC flag.
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines	if (pg[i] != bogus_page)
vm_page_xunbusy_unchecked(pg[i]);		vm_page_xunbusy_unchecked(pg[i]);

if (error)		if (error)
sfio->error = error;		sfio->error = error;

if (!refcount_release(&sfio->nios))		if (!refcount_release(&sfio->nios))
return;		return;

		vm_object_pip_wakeup(sfio->obj);

if (__predict_false(sfio->error && sfio->m == NULL)) {		if (__predict_false(sfio->error && sfio->m == NULL)) {
/*		/*
* I/O operation failed, but pru_send hadn't been executed -		* I/O operation failed, but pru_send hadn't been executed -
* nothing had been sent to the socket. The syscall has		* nothing had been sent to the socket. The syscall has
* returned error to the user.		* returned error to the user.
*/		*/
free(sfio, M_TEMP);		free(sfio, M_TEMP);
return;		return;
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines	for (j = i + 1; j < i + count - 1; j++)
xfsize(j, npages, off, len))) {		xfsize(j, npages, off, len))) {
vm_page_xunbusy(pa[j]);		vm_page_xunbusy(pa[j]);
SFSTAT_INC(sf_pages_valid);		SFSTAT_INC(sf_pages_valid);
SFSTAT_INC(sf_pages_bogus);		SFSTAT_INC(sf_pages_bogus);
pa[j] = bogus_page;		pa[j] = bogus_page;
}		}

refcount_acquire(&sfio->nios);		refcount_acquire(&sfio->nios);
		VM_OBJECT_WUNLOCK(obj);
rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,		rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
i + count == npages ? &rhpages : NULL,		i + count == npages ? &rhpages : NULL,
&sendfile_iodone, sfio);		&sendfile_iodone, sfio);
		VM_OBJECT_WLOCK(obj);
if (__predict_false(rv != VM_PAGER_OK)) {		if (__predict_false(rv != VM_PAGER_OK)) {
/*		/*
* Perform full pages recovery before returning EIO.		* Perform full pages recovery before returning EIO.
* Pages from 0 to npages are wired.		* Pages from 0 to npages are wired.
* Pages from i to npages are also busied.		* Pages from i to npages are also busied.
* Pages from (i + 1) to (i + count - 1) may be		* Pages from (i + 1) to (i + count - 1) may be
* substituted to bogus page, and not busied.		* substituted to bogus page, and not busied.
*/		*/
▲ Show 20 Lines • Show All 375 Lines • ▼ Show 20 Lines	#endif
rhpages = min(howmany(MAXPHYS, PAGE_SIZE), rhpages);		rhpages = min(howmany(MAXPHYS, PAGE_SIZE), rhpages);
rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -		rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) -
npages, rhpages);		npages, rhpages);

sfio = malloc(sizeof(struct sf_io) +		sfio = malloc(sizeof(struct sf_io) +
npages * sizeof(vm_page_t), M_TEMP, M_WAITOK);		npages * sizeof(vm_page_t), M_TEMP, M_WAITOK);
refcount_init(&sfio->nios, 1);		refcount_init(&sfio->nios, 1);
sfio->so = so;		sfio->so = so;
		sfio->obj = obj;
sfio->error = 0;		sfio->error = 0;
		vm_object_pip_add(obj, 1);

#ifdef KERN_TLS		#ifdef KERN_TLS
/*		/*
* This doesn't use ktls_hold() because sfio->m will		* This doesn't use ktls_hold() because sfio->m will
* also have a reference on 'tls' that will be valid		* also have a reference on 'tls' that will be valid
* for all of sfio's lifetime.		* for all of sfio's lifetime.
*/		*/
sfio->tls = tls;		sfio->tls = tls;
▲ Show 20 Lines • Show All 221 Lines • ▼ Show 20 Lines
#endif		#endif
if (nios == 0) {		if (nios == 0) {
/*		/*
* If sendfile_swapin() didn't initiate any I/Os,		* If sendfile_swapin() didn't initiate any I/Os,
* which happens if all data is cached in VM, then		* which happens if all data is cached in VM, then
* we can send data right now without the		* we can send data right now without the
* PRUS_NOTREADY flag.		* PRUS_NOTREADY flag.
*/		*/
		vm_object_pip_wakeup(sfio->obj);
free(sfio, M_TEMP);		free(sfio, M_TEMP);
#ifdef KERN_TLS		#ifdef KERN_TLS
if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {		if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
error = (*so->so_proto->pr_usrreqs->pru_send)		error = (*so->so_proto->pr_usrreqs->pru_send)
(so, PRUS_NOTREADY, m, NULL, NULL, td);		(so, PRUS_NOTREADY, m, NULL, NULL, td);
soref(so);		soref(so);
ktls_enqueue(m, so, tls_enq_cnt);		ktls_enqueue(m, so, tls_enq_cnt);
} else		} else
▲ Show 20 Lines • Show All 184 Lines • Show Last 20 Lines