Index: sys/kern/kern_sendfile.c =================================================================== --- sys/kern/kern_sendfile.c +++ sys/kern/kern_sendfile.c @@ -52,6 +52,8 @@ #include #include +#include +#include #include #include @@ -317,7 +319,7 @@ */ static int sendfile_swapin(vm_object_t obj, struct sf_io *sfio, off_t off, off_t len, - int npages, int rhpages, int flags) + int npages, int rhpages, int flags, int domain) { vm_page_t *pa = sfio->pa; int grabbed, nios; @@ -331,8 +333,13 @@ */ VM_OBJECT_WLOCK(obj); - grabbed = vm_page_grab_pages(obj, OFF_TO_IDX(off), - VM_ALLOC_NORMAL | VM_ALLOC_WIRED | flags, pa, npages); + if (domain != M_NODOM) + grabbed = vm_page_grab_pages_domain(obj, OFF_TO_IDX(off), + VM_ALLOC_NORMAL | VM_ALLOC_WIRED | flags, pa, npages, domain); + else + grabbed = vm_page_grab_pages(obj, OFF_TO_IDX(off), + VM_ALLOC_NORMAL | VM_ALLOC_WIRED | flags, pa, npages); + if (grabbed < npages) { for (int i = grabbed; i < npages; i++) pa[i] = NULL; @@ -545,8 +552,9 @@ struct shmfd *shmfd; struct sendfile_sync *sfs; struct vattr va; + struct inpcb *inp; off_t off, sbytes, rem, obj_size; - int error, softerr, bsize, hdrlen; + int domain, error, softerr, bsize, hdrlen; obj = NULL; so = NULL; @@ -563,6 +571,11 @@ if (error != 0) goto out; + inp = sotoinpcb(so); + if (inp != NULL) + domain = inp->inp_numa_domain; + else + domain = M_NODOM; #ifdef MAC error = mac_socket_check_send(td->td_ucred, so); if (error != 0) @@ -743,8 +756,12 @@ sfio->so = so; sfio->error = 0; + /* + * If these pages are private, then put them on the + * correct numa domain for this connection + */ nios = sendfile_swapin(obj, sfio, off, space, npages, rhpages, - flags); + flags, (flags & SF_NOCACHE) ? domain : M_NODOM); /* * Loop and construct maximum sized mbuf chain to be bulk Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -537,6 +537,8 @@ vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int); int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count); +int vm_page_grab_pages_domain(vm_object_t object, vm_pindex_t pindex, int allocflags, + vm_page_t *ma, int count, int domain); void vm_page_deactivate(vm_page_t); void vm_page_deactivate_noreuse(vm_page_t); void vm_page_dequeue(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -3955,8 +3955,8 @@ * may return a partial prefix of the requested range. */ int -vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, - vm_page_t *ma, int count) +vm_page_grab_pages_domain(vm_object_t object, vm_pindex_t pindex, int allocflags, + vm_page_t *ma, int count, int domain) { vm_page_t m, mpred; int pflags; @@ -4017,8 +4017,13 @@ if ((allocflags & VM_ALLOC_SBUSY) != 0) vm_page_sbusy(m); } else { - m = vm_page_alloc_after(object, pindex + i, - pflags | VM_ALLOC_COUNT(count - i), mpred); + if (domain != -1) + m = vm_page_alloc_domain_after(object, + pindex + i, domain, + pflags | VM_ALLOC_COUNT(count - i), mpred); + else + m = vm_page_alloc_after(object, pindex + i, + pflags | VM_ALLOC_COUNT(count - i), mpred); if (m == NULL) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) break; @@ -4035,7 +4040,13 @@ } return (i); } - +int +vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, + vm_page_t *ma, int count) +{ + return (vm_page_grab_pages_domain(object, pindex, allocflags, + ma, count, -1)); +} /* * Mapping function for valid or dirty bits in a page. *