Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/vm_domainset.c
Property | Old Value | New Value |
---|---|---|
svn:eol-style | null | native \ No newline at end of property |
svn:keywords | null | FreeBSD=%H \ No newline at end of property |
svn:mime-type | null | text/plain \ No newline at end of property |
/*- | |||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | |||||
* | |||||
* Copyright (c) 2017, Jeffrey Roberson <jeff@freebsd.org> | |||||
* All rights reserved. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice unmodified, this list of conditions, and the following | |||||
* disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | |||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | |||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | |||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
* | |||||
*/ | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD$"); | |||||
#include "opt_vm.h" | |||||
#include <sys/param.h> | |||||
#include <sys/systm.h> | |||||
#include <sys/bitset.h> | |||||
#include <sys/domainset.h> | |||||
#include <sys/proc.h> | |||||
#include <sys/lock.h> | |||||
#include <sys/mutex.h> | |||||
#include <sys/malloc.h> | |||||
#include <sys/vmmeter.h> | |||||
#include <vm/vm.h> | |||||
#include <vm/vm_param.h> | |||||
#include <vm/vm_domainset.h> | |||||
#include <vm/vm_object.h> | |||||
#include <vm/vm_page.h> | |||||
#include <vm/vm_phys.h> | |||||
/* | |||||
* Iterators are written such that the first nowait pass has as short a | |||||
* codepath as possible to eliminate bloat from the allocator. It is | |||||
* assumed that most allocations are successful. | |||||
*/ | |||||
/* | |||||
* Determine which policy is to be used for this allocation. | |||||
*/ | |||||
static void | |||||
vm_domainset_iter_domain(struct vm_domainset_iter *di, struct vm_object *obj) | |||||
{ | |||||
struct domainset *domain; | |||||
/* | |||||
* object policy takes precedence over thread policy. The policies | |||||
* are immutable and unsynchronized. Updates can race but pointer | |||||
* loads are assumed to be atomic. | |||||
*/ | |||||
if (obj != NULL && (domain = obj->domain.dr_policy) != NULL) { | |||||
di->di_domain = domain; | |||||
di->di_iter = &obj->domain.dr_iterator; | |||||
} else { | |||||
di->di_domain = curthread->td_domain.dr_policy; | |||||
di->di_iter = &curthread->td_domain.dr_iterator; | |||||
} | |||||
} | |||||
static void | |||||
vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain) | |||||
{ | |||||
int d; | |||||
d = *di->di_iter; | |||||
do { | |||||
d = (d + 1) % di->di_domain->ds_max; | |||||
} while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask)); | |||||
*di->di_iter = *domain = d; | |||||
} | |||||
static void | |||||
vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain) | |||||
{ | |||||
int d; | |||||
d = *di->di_iter; | |||||
do { | |||||
d = (d + 1) % di->di_domain->ds_max; | |||||
} while (!DOMAINSET_ISSET(d, &di->di_domain->ds_mask) || | |||||
d == di->di_domain->ds_prefer); | |||||
*di->di_iter = *domain = d; | |||||
} | |||||
static void | |||||
vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain) | |||||
{ | |||||
KASSERT(di->di_n > 0, | |||||
("vm_domainset_iter_first: Invalid n %d", di->di_n)); | |||||
switch (di->di_domain->ds_policy) { | |||||
case DOMAINSET_POLICY_FIRSTTOUCH: | |||||
/* | |||||
* To prevent impossible allocations we convert an invalid | |||||
* first-touch to round-robin. | |||||
*/ | |||||
/* FALLTHROUGH */ | |||||
case DOMAINSET_POLICY_ROUNDROBIN: | |||||
vm_domainset_iter_rr(di, domain); | |||||
break; | |||||
case DOMAINSET_POLICY_PREFER: | |||||
vm_domainset_iter_prefer(di, domain); | |||||
break; | |||||
default: | |||||
panic("vm_domainset_iter_first: Unknown policy %d", | |||||
di->di_domain->ds_policy); | |||||
} | |||||
KASSERT(*domain < vm_ndomains, | |||||
("vm_domainset_iter_next: Invalid domain %d", *domain)); | |||||
} | |||||
static void | |||||
vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain) | |||||
{ | |||||
switch (di->di_domain->ds_policy) { | |||||
case DOMAINSET_POLICY_FIRSTTOUCH: | |||||
*domain = PCPU_GET(domain); | |||||
if (DOMAINSET_ISSET(*domain, &di->di_domain->ds_mask)) { | |||||
di->di_n = 1; | |||||
break; | |||||
} | |||||
/* | |||||
* To prevent impossible allocations we convert an invalid | |||||
* first-touch to round-robin. | |||||
*/ | |||||
/* FALLTHROUGH */ | |||||
case DOMAINSET_POLICY_ROUNDROBIN: | |||||
di->di_n = di->di_domain->ds_cnt; | |||||
vm_domainset_iter_rr(di, domain); | |||||
break; | |||||
case DOMAINSET_POLICY_PREFER: | |||||
*domain = di->di_domain->ds_prefer; | |||||
di->di_n = di->di_domain->ds_cnt; | |||||
break; | |||||
default: | |||||
panic("vm_domainset_iter_first: Unknown policy %d", | |||||
di->di_domain->ds_policy); | |||||
} | |||||
KASSERT(di->di_n > 0, | |||||
("vm_domainset_iter_first: Invalid n %d", di->di_n)); | |||||
KASSERT(*domain < vm_ndomains, | |||||
("vm_domainset_iter_first: Invalid domain %d", *domain)); | |||||
} | |||||
void | |||||
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj, | |||||
int *domain, int *req) | |||||
{ | |||||
vm_domainset_iter_domain(di, obj); | |||||
di->di_flags = *req; | |||||
*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) | | |||||
VM_ALLOC_NOWAIT; | |||||
vm_domainset_iter_first(di, domain); | |||||
} | |||||
int | |||||
vm_domainset_iter_page(struct vm_domainset_iter *di, int *domain, int *req) | |||||
{ | |||||
/* | |||||
* If we exhausted all options with NOWAIT and did a WAITFAIL it | |||||
* is time to return an error to the caller. | |||||
*/ | |||||
if ((*req & VM_ALLOC_WAITFAIL) != 0) | |||||
return (ENOMEM); | |||||
/* If there are more domains to visit we run the iterator. */ | |||||
if (--di->di_n != 0) { | |||||
vm_domainset_iter_next(di, domain); | |||||
return (0); | |||||
} | |||||
/* If we visited all domains and this was a NOWAIT we return error. */ | |||||
if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0) | |||||
return (ENOMEM); | |||||
/* | |||||
* We have visited all domains with non-blocking allocations, try | |||||
* from the beginning with a blocking allocation. | |||||
*/ | |||||
vm_domainset_iter_first(di, domain); | |||||
*req = di->di_flags; | |||||
return (0); | |||||
} | |||||
void | |||||
vm_domainset_iter_malloc_init(struct vm_domainset_iter *di, | |||||
struct vm_object *obj, int *domain, int *flags) | |||||
{ | |||||
vm_domainset_iter_domain(di, obj); | |||||
di->di_flags = *flags; | |||||
*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT; | |||||
vm_domainset_iter_first(di, domain); | |||||
} | |||||
int | |||||
vm_domainset_iter_malloc(struct vm_domainset_iter *di, int *domain, int *flags) | |||||
{ | |||||
/* If there are more domains to visit we run the iterator. */ | |||||
if (--di->di_n != 0) { | |||||
vm_domainset_iter_next(di, domain); | |||||
return (0); | |||||
} | |||||
/* If we visited all domains and this was a NOWAIT we return error. */ | |||||
if ((di->di_flags & M_WAITOK) == 0) | |||||
return (ENOMEM); | |||||
/* | |||||
* We have visited all domains with non-blocking allocations, try | |||||
* from the beginning with a blocking allocation. | |||||
*/ | |||||
vm_domainset_iter_first(di, domain); | |||||
*flags = di->di_flags; | |||||
return (0); | |||||
} |