Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_cpuset.c
Show First 20 Lines • Show All 113 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* A simple application should not concern itself with sets at all and | * A simple application should not concern itself with sets at all and | ||||
* rather apply masks to its own threads via CPU_WHICH_TID and a -1 id | * rather apply masks to its own threads via CPU_WHICH_TID and a -1 id | ||||
* meaning 'curthread'. It may query available cpus for that tid with a | * meaning 'curthread'. It may query available cpus for that tid with a | ||||
* getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). | * getaffinity call using (CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, ...). | ||||
*/ | */ | ||||
LIST_HEAD(domainlist, domainset); | LIST_HEAD(domainlist, domainset); | ||||
struct domainset __read_mostly domainset_firsttouch; | |||||
struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; | struct domainset __read_mostly domainset_fixed[MAXMEMDOM]; | ||||
struct domainset __read_mostly domainset_interleave; | |||||
struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; | struct domainset __read_mostly domainset_prefer[MAXMEMDOM]; | ||||
struct domainset __read_mostly domainset_roundrobin; | struct domainset __read_mostly domainset_roundrobin; | ||||
static uma_zone_t cpuset_zone; | static uma_zone_t cpuset_zone; | ||||
static uma_zone_t domainset_zone; | static uma_zone_t domainset_zone; | ||||
static struct mtx cpuset_lock; | static struct mtx cpuset_lock; | ||||
static struct setlist cpuset_ids; | static struct setlist cpuset_ids; | ||||
static struct domainlist cpuset_domains; | static struct domainlist cpuset_domains; | ||||
static struct unrhdr *cpuset_unr; | static struct unrhdr *cpuset_unr; | ||||
static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; | static struct cpuset *cpuset_zero, *cpuset_default, *cpuset_kernel; | ||||
static struct domainset domainset0, domainset2; | static struct domainset *domainset0, *domainset2; | ||||
/* Return the size of cpuset_t at the kernel level */ | /* Return the size of cpuset_t at the kernel level */ | ||||
SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, | SYSCTL_INT(_kern_sched, OID_AUTO, cpusetsize, CTLFLAG_RD | CTLFLAG_CAPRD, | ||||
SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); | SYSCTL_NULL_INT_PTR, sizeof(cpuset_t), "sizeof(cpuset_t)"); | ||||
cpuset_t *cpuset_root; | cpuset_t *cpuset_root; | ||||
cpuset_t cpuset_domain[MAXMEMDOM]; | cpuset_t cpuset_domain[MAXMEMDOM]; | ||||
▲ Show 20 Lines • Show All 421 Lines • ▼ Show 20 Lines | domainset_create(const struct domainset *domain) | ||||
* in the mask. | * in the mask. | ||||
*/ | */ | ||||
if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || | if (domain->ds_policy <= DOMAINSET_POLICY_INVALID || | ||||
domain->ds_policy > DOMAINSET_POLICY_MAX) | domain->ds_policy > DOMAINSET_POLICY_MAX) | ||||
return (NULL); | return (NULL); | ||||
if (domain->ds_policy == DOMAINSET_POLICY_PREFER && | if (domain->ds_policy == DOMAINSET_POLICY_PREFER && | ||||
!DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) | !DOMAINSET_ISSET(domain->ds_prefer, &domain->ds_mask)) | ||||
return (NULL); | return (NULL); | ||||
if (!DOMAINSET_SUBSET(&domainset0.ds_mask, &domain->ds_mask)) | if (!DOMAINSET_SUBSET(&domainset0->ds_mask, &domain->ds_mask)) | ||||
return (NULL); | return (NULL); | ||||
ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); | ndomain = uma_zalloc(domainset_zone, M_WAITOK | M_ZERO); | ||||
domainset_copy(domain, ndomain); | domainset_copy(domain, ndomain); | ||||
return _domainset_create(ndomain, NULL); | return _domainset_create(ndomain, NULL); | ||||
} | } | ||||
/* | /* | ||||
* Update thread domainset pointers. | * Update thread domainset pointers. | ||||
▲ Show 20 Lines • Show All 947 Lines • ▼ Show 20 Lines | |||||
* called before memory allocators are initialized. | * called before memory allocators are initialized. | ||||
*/ | */ | ||||
void | void | ||||
domainset_init(void) | domainset_init(void) | ||||
{ | { | ||||
struct domainset *dset; | struct domainset *dset; | ||||
int i; | int i; | ||||
dset = &domainset_firsttouch; | |||||
DOMAINSET_COPY(&all_domains, &dset->ds_mask); | |||||
dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; | |||||
dset->ds_prefer = -1; | |||||
_domainset_create(dset, NULL); | |||||
dset = &domainset_interleave; | |||||
DOMAINSET_COPY(&all_domains, &dset->ds_mask); | |||||
dset->ds_policy = DOMAINSET_POLICY_INTERLEAVE; | |||||
dset->ds_prefer = -1; | |||||
_domainset_create(dset, NULL); | |||||
dset = &domainset_roundrobin; | dset = &domainset_roundrobin; | ||||
DOMAINSET_COPY(&all_domains, &dset->ds_mask); | DOMAINSET_COPY(&all_domains, &dset->ds_mask); | ||||
dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; | dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; | ||||
dset->ds_prefer = -1; | dset->ds_prefer = -1; | ||||
_domainset_create(dset, NULL); | _domainset_create(dset, NULL); | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
dset = &domainset_fixed[i]; | dset = &domainset_fixed[i]; | ||||
DOMAINSET_ZERO(&dset->ds_mask); | DOMAINSET_ZERO(&dset->ds_mask); | ||||
DOMAINSET_SET(i, &dset->ds_mask); | DOMAINSET_SET(i, &dset->ds_mask); | ||||
dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; | dset->ds_policy = DOMAINSET_POLICY_ROUNDROBIN; | ||||
_domainset_create(dset, NULL); | _domainset_create(dset, NULL); | ||||
dset = &domainset_prefer[i]; | dset = &domainset_prefer[i]; | ||||
DOMAINSET_COPY(&all_domains, &dset->ds_mask); | DOMAINSET_COPY(&all_domains, &dset->ds_mask); | ||||
dset->ds_policy = DOMAINSET_POLICY_PREFER; | dset->ds_policy = DOMAINSET_POLICY_PREFER; | ||||
dset->ds_prefer = i; | dset->ds_prefer = i; | ||||
_domainset_create(dset, NULL); | _domainset_create(dset, NULL); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Create the domainset for cpuset 0, 1 and cpuset 2. | * Define the domainsets for cpuset 0, 1 and cpuset 2. | ||||
*/ | */ | ||||
void | void | ||||
domainset_zero(void) | domainset_zero(void) | ||||
{ | { | ||||
struct domainset *dset, *tmp; | struct domainset *dset, *tmp; | ||||
mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); | mtx_init(&cpuset_lock, "cpuset", NULL, MTX_SPIN | MTX_RECURSE); | ||||
dset = &domainset0; | domainset0 = &domainset_firsttouch; | ||||
DOMAINSET_COPY(&all_domains, &dset->ds_mask); | curthread->td_domain.dr_policy = domainset0; | ||||
dset->ds_policy = DOMAINSET_POLICY_FIRSTTOUCH; | |||||
dset->ds_prefer = -1; | |||||
curthread->td_domain.dr_policy = _domainset_create(dset, NULL); | |||||
domainset_copy(dset, &domainset2); | domainset2 = &domainset_interleave; | ||||
domainset2.ds_policy = DOMAINSET_POLICY_INTERLEAVE; | kernel_object->domain.dr_policy = domainset2; | ||||
kernel_object->domain.dr_policy = _domainset_create(&domainset2, NULL); | |||||
/* Remove empty domains from the global policies. */ | /* Remove empty domains from the global policies. */ | ||||
LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) | LIST_FOREACH_SAFE(dset, &cpuset_domains, ds_link, tmp) | ||||
if (domainset_empty_vm(dset)) | if (domainset_empty_vm(dset)) | ||||
LIST_REMOVE(dset, ds_link); | LIST_REMOVE(dset, ds_link); | ||||
} | } | ||||
/* | /* | ||||
Show All 25 Lines | cpuset_thread0(void) | ||||
* cpuset_create() due to NULL parent. | * cpuset_create() due to NULL parent. | ||||
*/ | */ | ||||
set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | ||||
CPU_COPY(&all_cpus, &set->cs_mask); | CPU_COPY(&all_cpus, &set->cs_mask); | ||||
LIST_INIT(&set->cs_children); | LIST_INIT(&set->cs_children); | ||||
LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); | LIST_INSERT_HEAD(&cpuset_ids, set, cs_link); | ||||
refcount_init(&set->cs_ref, 1); | refcount_init(&set->cs_ref, 1); | ||||
set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; | set->cs_flags = CPU_SET_ROOT | CPU_SET_RDONLY; | ||||
set->cs_domain = &domainset0; | set->cs_domain = DOMAINSET_FT(); | ||||
mav: Shouldn't cpuset_zero always match domainset0 without having custom knowledge that DOMAINSET_FT… | |||||
markjAuthorUnsubmitted Done Inline ActionsThanks, yes, it was an oversight. markj: Thanks, yes, it was an oversight. | |||||
cpuset_zero = set; | cpuset_zero = set; | ||||
cpuset_root = &set->cs_mask; | cpuset_root = &set->cs_mask; | ||||
/* | /* | ||||
* Now derive a default (1), modifiable set from that to give out. | * Now derive a default (1), modifiable set from that to give out. | ||||
*/ | */ | ||||
set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | ||||
error = cpuset_init(set, cpuset_zero, NULL, NULL, 1); | error = cpuset_init(set, cpuset_zero, NULL, NULL, 1); | ||||
KASSERT(error == 0, ("Error creating default set: %d\n", error)); | KASSERT(error == 0, ("Error creating default set: %d\n", error)); | ||||
cpuset_default = set; | cpuset_default = set; | ||||
/* | /* | ||||
* Create the kernel set (2). | * Create the kernel set (2). | ||||
*/ | */ | ||||
set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | set = uma_zalloc(cpuset_zone, M_WAITOK | M_ZERO); | ||||
error = cpuset_init(set, cpuset_zero, NULL, NULL, 2); | error = cpuset_init(set, cpuset_zero, NULL, NULL, 2); | ||||
KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); | KASSERT(error == 0, ("Error creating kernel set: %d\n", error)); | ||||
set->cs_domain = &domainset2; | set->cs_domain = domainset2; | ||||
cpuset_kernel = set; | cpuset_kernel = set; | ||||
/* | /* | ||||
* Initialize the unit allocator. 0 and 1 are allocated above. | * Initialize the unit allocator. 0 and 1 are allocated above. | ||||
*/ | */ | ||||
cpuset_unr = new_unrhdr(3, INT_MAX, NULL); | cpuset_unr = new_unrhdr(3, INT_MAX, NULL); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 668 Lines • ▼ Show 20 Lines | if (policy == DOMAINSET_POLICY_PREFER) { | ||||
DOMAINSET_COPY(&all_domains, &domain.ds_mask); | DOMAINSET_COPY(&all_domains, &domain.ds_mask); | ||||
} | } | ||||
/* | /* | ||||
* When given an impossible policy, fall back to interleaving | * When given an impossible policy, fall back to interleaving | ||||
* across all domains. | * across all domains. | ||||
*/ | */ | ||||
if (domainset_empty_vm(&domain)) | if (domainset_empty_vm(&domain)) | ||||
domainset_copy(&domainset2, &domain); | domainset_copy(domainset2, &domain); | ||||
switch (level) { | switch (level) { | ||||
case CPU_LEVEL_ROOT: | case CPU_LEVEL_ROOT: | ||||
case CPU_LEVEL_CPUSET: | case CPU_LEVEL_CPUSET: | ||||
error = cpuset_which(which, id, &p, &ttd, &set); | error = cpuset_which(which, id, &p, &ttd, &set); | ||||
if (error) | if (error) | ||||
break; | break; | ||||
switch (which) { | switch (which) { | ||||
▲ Show 20 Lines • Show All 123 Lines • Show Last 20 Lines |
Shouldn't cpuset_zero always match domainset0 without having custom knowledge that DOMAINSET_FT() is the default now?