Changeset View
Changeset View
Standalone View
Standalone View
head/sys/kern/subr_gtaskqueue.c
Show All 27 Lines | |||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/cpuset.h> | #include <sys/cpuset.h> | ||||
#include <sys/interrupt.h> | |||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/kthread.h> | #include <sys/kthread.h> | ||||
#include <sys/libkern.h> | #include <sys/libkern.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
Show All 14 Lines | |||||
struct gtaskqueue_busy { | struct gtaskqueue_busy { | ||||
struct gtask *tb_running; | struct gtask *tb_running; | ||||
TAILQ_ENTRY(gtaskqueue_busy) tb_link; | TAILQ_ENTRY(gtaskqueue_busy) tb_link; | ||||
}; | }; | ||||
static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; | static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; | ||||
typedef void (*gtaskqueue_enqueue_fn)(void *context); | |||||
struct gtaskqueue { | struct gtaskqueue { | ||||
STAILQ_HEAD(, gtask) tq_queue; | STAILQ_HEAD(, gtask) tq_queue; | ||||
gtaskqueue_enqueue_fn tq_enqueue; | gtaskqueue_enqueue_fn tq_enqueue; | ||||
void *tq_context; | void *tq_context; | ||||
char *tq_name; | char *tq_name; | ||||
TAILQ_HEAD(, gtaskqueue_busy) tq_active; | TAILQ_HEAD(, gtaskqueue_busy) tq_active; | ||||
struct mtx tq_mutex; | struct mtx tq_mutex; | ||||
struct thread **tq_threads; | struct thread **tq_threads; | ||||
▲ Show 20 Lines • Show All 601 Lines • ▼ Show 20 Lines | for (i = 0; i < qgroup->tqg_cnt; i++) { | ||||
if (n != NULL) | if (n != NULL) | ||||
continue; | continue; | ||||
} | } | ||||
mincnt = qgroup->tqg_queue[i].tgc_cnt; | mincnt = qgroup->tqg_queue[i].tgc_cnt; | ||||
idx = i; | idx = i; | ||||
} | } | ||||
} | } | ||||
if (idx == -1) | if (idx == -1) | ||||
panic("taskqgroup_find: Failed to pick a qid."); | panic("%s: failed to pick a qid.", __func__); | ||||
return (idx); | return (idx); | ||||
} | } | ||||
/* | /* | ||||
* smp_started is unusable since it is not set for UP kernels or even for | * smp_started is unusable since it is not set for UP kernels or even for | ||||
* SMP kernels when there is 1 CPU. This is usually handled by adding a | * SMP kernels when there is 1 CPU. This is usually handled by adding a | ||||
* (mp_ncpus == 1) test, but that would be broken here since we need to | * (mp_ncpus == 1) test, but that would be broken here since we need to | ||||
Show All 15 Lines | tqg_record_smp_started(void *arg) | ||||
tqg_smp_started = 1; | tqg_smp_started = 1; | ||||
} | } | ||||
SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | ||||
tqg_record_smp_started, NULL); | tqg_record_smp_started, NULL); | ||||
void | void | ||||
taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int irq, const char *name) | void *uniq, device_t dev, struct resource *irq, const char *name) | ||||
{ | { | ||||
cpuset_t mask; | int cpu, qid, error; | ||||
int qid, error; | |||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | ||||
gtask->gt_dev = dev; | |||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = -1; | gtask->gt_cpu = -1; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qid = taskqgroup_find(qgroup, uniq); | qid = taskqgroup_find(qgroup, uniq); | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
if (irq != -1 && tqg_smp_started) { | if (dev != NULL && irq != NULL && tqg_smp_started) { | ||||
gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
CPU_ZERO(&mask); | gtask->gt_cpu = cpu; | ||||
CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); | |||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | error = bus_bind_intr(dev, irq, cpu); | ||||
if (error) | if (error) | ||||
printf("%s: setaffinity failed for %s: %d\n", __func__, gtask->gt_name, error); | printf("%s: binding interrupt failed for %s: %d\n", | ||||
__func__, gtask->gt_name, error); | |||||
} else | } else | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | |||||
int qid, cpu, error; | int qid, cpu, error; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qid = taskqgroup_find(qgroup, gtask->gt_uniq); | qid = taskqgroup_find(qgroup, gtask->gt_uniq); | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
if (gtask->gt_irq != -1) { | if (gtask->gt_dev != NULL && gtask->gt_irq != NULL) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
error = bus_bind_intr(gtask->gt_dev, gtask->gt_irq, cpu); | |||||
CPU_ZERO(&mask); | |||||
CPU_SET(cpu, &mask); | |||||
error = intr_setaffinity(gtask->gt_irq, CPU_WHICH_IRQ, &mask); | |||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
if (error) | if (error) | ||||
printf("%s: %s setaffinity failed: %d\n", __func__, gtask->gt_name, error); | printf("%s: binding interrupt failed for %s: %d\n", | ||||
__func__, gtask->gt_name, error); | |||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | |||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, | |||||
gt_list); | |||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
int | int | ||||
taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int cpu, int irq, const char *name) | void *uniq, int cpu, device_t dev, struct resource *irq, const char *name) | ||||
{ | { | ||||
cpuset_t mask; | |||||
int i, qid, error; | int i, qid, error; | ||||
qid = -1; | qid = -1; | ||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | ||||
gtask->gt_dev = dev; | |||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = cpu; | gtask->gt_cpu = cpu; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
if (tqg_smp_started) { | if (tqg_smp_started) { | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
} else | } else | ||||
qid = 0; | qid = 0; | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | if (dev != NULL && irq != NULL && tqg_smp_started) { | ||||
CPU_SET(cpu, &mask); | error = bus_bind_intr(dev, irq, cpu); | ||||
if (irq != -1 && tqg_smp_started) { | |||||
error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | |||||
if (error) | if (error) | ||||
printf("%s: setaffinity failed: %d\n", __func__, error); | printf("%s: binding interrupt failed for %s: %d\n", | ||||
__func__, gtask->gt_name, error); | |||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | device_t dev; | ||||
int i, qid, irq, cpu, error; | struct resource *irq; | ||||
int cpu, error, i, qid; | |||||
qid = -1; | qid = -1; | ||||
dev = gtask->gt_dev; | |||||
irq = gtask->gt_irq; | irq = gtask->gt_irq; | ||||
cpu = gtask->gt_cpu; | cpu = gtask->gt_cpu; | ||||
MPASS(tqg_smp_started); | MPASS(tqg_smp_started); | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | if (dev != NULL && irq != NULL) { | ||||
CPU_SET(cpu, &mask); | error = bus_bind_intr(dev, irq, cpu); | ||||
if (irq != -1) { | |||||
error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | |||||
if (error) | if (error) | ||||
printf("%s: setaffinity failed: %d\n", __func__, error); | printf("%s: binding interrupt failed for %s: %d\n", | ||||
__func__, gtask->gt_name, error); | |||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
int i; | int i; | ||||
grouptask_block(gtask); | grouptask_block(gtask); | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) | if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) | ||||
break; | break; | ||||
if (i == qgroup->tqg_cnt) | if (i == qgroup->tqg_cnt) | ||||
panic("taskqgroup_detach: task %s not in group\n", gtask->gt_name); | panic("%s: task %s not in group", __func__, gtask->gt_name); | ||||
qgroup->tqg_queue[i].tgc_cnt--; | qgroup->tqg_queue[i].tgc_cnt--; | ||||
LIST_REMOVE(gtask, gt_list); | LIST_REMOVE(gtask, gt_list); | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
gtask->gt_taskqueue = NULL; | gtask->gt_taskqueue = NULL; | ||||
gtask->gt_task.ta_flags &= ~TASK_NOENQUEUE; | gtask->gt_task.ta_flags &= ~TASK_NOENQUEUE; | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_binder(void *ctx) | taskqgroup_binder(void *ctx) | ||||
{ | { | ||||
struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; | struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int error; | int error; | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(gtask->bt_cpuid, &mask); | CPU_SET(gtask->bt_cpuid, &mask); | ||||
error = cpuset_setthread(curthread->td_tid, &mask); | error = cpuset_setthread(curthread->td_tid, &mask); | ||||
thread_lock(curthread); | thread_lock(curthread); | ||||
sched_bind(curthread, gtask->bt_cpuid); | sched_bind(curthread, gtask->bt_cpuid); | ||||
thread_unlock(curthread); | thread_unlock(curthread); | ||||
if (error) | if (error) | ||||
printf("%s: setaffinity failed: %d\n", __func__, | printf("%s: binding curthread failed: %d\n", __func__, error); | ||||
error); | |||||
free(gtask, M_DEVBUF); | free(gtask, M_DEVBUF); | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_bind(struct taskqgroup *qgroup) | taskqgroup_bind(struct taskqgroup *qgroup) | ||||
{ | { | ||||
struct taskq_bind_task *gtask; | struct taskq_bind_task *gtask; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
taskqgroup_destroy(struct taskqgroup *qgroup) | taskqgroup_destroy(struct taskqgroup *qgroup) | ||||
{ | { | ||||
} | } | ||||
void | void | ||||
taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, | taskqgroup_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn, | ||||
const char *name) | const char *name) | ||||
{ | { | ||||
GROUPTASK_INIT(gtask, 0, fn, ctx); | GROUPTASK_INIT(gtask, 0, fn, ctx); | ||||
taskqgroup_attach(qgroup_config, gtask, gtask, -1, name); | taskqgroup_attach(qgroup_config, gtask, gtask, NULL, NULL, name); | ||||
} | } | ||||
void | void | ||||
taskqgroup_config_gtask_deinit(struct grouptask *gtask) | taskqgroup_config_gtask_deinit(struct grouptask *gtask) | ||||
{ | { | ||||
taskqgroup_detach(qgroup_config, gtask); | taskqgroup_detach(qgroup_config, gtask); | ||||
} | } |