Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/subr_gtaskqueue.c
Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/gtaskqueue.h> | #include <sys/gtaskqueue.h> | ||||
#include <sys/unistd.h> | #include <sys/unistd.h> | ||||
#include <machine/stdarg.h> | #include <machine/stdarg.h> | ||||
static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues"); | static MALLOC_DEFINE(M_GTASKQUEUE, "gtaskqueue", "Group Task Queues"); | ||||
static void gtaskqueue_thread_enqueue(void *); | static void gtaskqueue_thread_enqueue(void *); | ||||
static void gtaskqueue_thread_loop(void *arg); | static void gtaskqueue_thread_loop(void *arg); | ||||
TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); | TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); | ||||
struct gtaskqueue_busy { | struct gtaskqueue_busy { | ||||
struct gtask *tb_running; | struct gtask *tb_running; | ||||
TAILQ_ENTRY(gtaskqueue_busy) tb_link; | TAILQ_ENTRY(gtaskqueue_busy) tb_link; | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | _gtaskqueue_create(const char *name, int mflags, | ||||
tq_name = malloc(TASKQUEUE_NAMELEN, M_GTASKQUEUE, mflags | M_ZERO); | tq_name = malloc(TASKQUEUE_NAMELEN, M_GTASKQUEUE, mflags | M_ZERO); | ||||
if (!tq_name) | if (!tq_name) | ||||
return (NULL); | return (NULL); | ||||
snprintf(tq_name, TASKQUEUE_NAMELEN, "%s", (name) ? name : "taskqueue"); | snprintf(tq_name, TASKQUEUE_NAMELEN, "%s", (name) ? name : "taskqueue"); | ||||
queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO); | queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags | M_ZERO); | ||||
if (!queue) | if (!queue) { | ||||
free(tq_name, M_GTASKQUEUE); | |||||
return (NULL); | return (NULL); | ||||
} | |||||
STAILQ_INIT(&queue->tq_queue); | STAILQ_INIT(&queue->tq_queue); | ||||
TAILQ_INIT(&queue->tq_active); | TAILQ_INIT(&queue->tq_active); | ||||
queue->tq_enqueue = enqueue; | queue->tq_enqueue = enqueue; | ||||
queue->tq_context = context; | queue->tq_context = context; | ||||
queue->tq_name = tq_name; | queue->tq_name = tq_name; | ||||
queue->tq_spin = (mtxflags & MTX_SPIN) != 0; | queue->tq_spin = (mtxflags & MTX_SPIN) != 0; | ||||
queue->tq_flags |= TQ_FLAGS_ACTIVE; | queue->tq_flags |= TQ_FLAGS_ACTIVE; | ||||
▲ Show 20 Lines • Show All 511 Lines • ▼ Show 20 Lines | |||||
SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | ||||
tqg_record_smp_started, NULL); | tqg_record_smp_started, NULL); | ||||
void | void | ||||
taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int irq, char *name) | void *uniq, int irq, char *name) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int qid; | int qid, error; | ||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
gtask->gt_name = name; | snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | ||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = -1; | gtask->gt_cpu = -1; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qid = taskqgroup_find(qgroup, uniq); | qid = taskqgroup_find(qgroup, uniq); | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
if (irq != -1 && tqg_smp_started) { | if (irq != -1 && tqg_smp_started) { | ||||
gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; | gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); | CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
intr_setaffinity(irq, &mask); | error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | ||||
if (error) | |||||
printf("%s: setaffinity failed for %s: %d\n", __func__, gtask->gt_name, error); | |||||
} else | } else | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int qid, cpu; | int qid, cpu, error; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qid = taskqgroup_find(qgroup, gtask->gt_uniq); | qid = taskqgroup_find(qgroup, gtask->gt_uniq); | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
if (gtask->gt_irq != -1) { | if (gtask->gt_irq != -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
intr_setaffinity(gtask->gt_irq, &mask); | error = intr_setaffinity(gtask->gt_irq, CPU_WHICH_IRQ, &mask); | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
if (error) | |||||
printf("%s: %s setaffinity failed: %d\n", __func__, gtask->gt_name, error); | |||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, | ||||
gt_list); | gt_list); | ||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
int | int | ||||
taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int cpu, int irq, char *name) | void *uniq, int cpu, int irq, char *name) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int i, qid; | int i, qid, error; | ||||
qid = -1; | qid = -1; | ||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
gtask->gt_name = name; | snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask"); | ||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = cpu; | gtask->gt_cpu = cpu; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
if (tqg_smp_started) { | if (tqg_smp_started) { | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | |||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
} else | } else | ||||
qid = 0; | qid = 0; | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
if (irq != -1 && tqg_smp_started) | if (irq != -1 && tqg_smp_started) { | ||||
intr_setaffinity(irq, &mask); | error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | ||||
if (error) | |||||
printf("%s: setaffinity failed: %d\n", __func__, error); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int i, qid, irq, cpu; | int i, qid, irq, cpu, error; | ||||
qid = -1; | qid = -1; | ||||
irq = gtask->gt_irq; | irq = gtask->gt_irq; | ||||
cpu = gtask->gt_cpu; | cpu = gtask->gt_cpu; | ||||
MPASS(tqg_smp_started); | MPASS(tqg_smp_started); | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu); | |||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
if (irq != -1) | if (irq != -1) { | ||||
intr_setaffinity(irq, &mask); | error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | ||||
if (error) | |||||
printf("%s: setaffinity failed: %d\n", __func__, error); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
int i; | int i; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) | if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue) | ||||
break; | break; | ||||
if (i == qgroup->tqg_cnt) | if (i == qgroup->tqg_cnt) | ||||
panic("taskqgroup_detach: task not in group\n"); | panic("taskqgroup_detach: task %s not in group\n", gtask->gt_name); | ||||
qgroup->tqg_queue[i].tgc_cnt--; | qgroup->tqg_queue[i].tgc_cnt--; | ||||
LIST_REMOVE(gtask, gt_list); | LIST_REMOVE(gtask, gt_list); | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
gtask->gt_taskqueue = NULL; | gtask->gt_taskqueue = NULL; | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_binder(void *ctx) | taskqgroup_binder(void *ctx) | ||||
{ | { | ||||
struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; | struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int error; | int error; | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(gtask->bt_cpuid, &mask); | CPU_SET(gtask->bt_cpuid, &mask); | ||||
error = cpuset_setthread(curthread->td_tid, &mask); | error = cpuset_setthread(curthread->td_tid, &mask); | ||||
thread_lock(curthread); | thread_lock(curthread); | ||||
sched_bind(curthread, gtask->bt_cpuid); | sched_bind(curthread, gtask->bt_cpuid); | ||||
thread_unlock(curthread); | thread_unlock(curthread); | ||||
if (error) | if (error) | ||||
printf("taskqgroup_binder: setaffinity failed: %d\n", | printf("%s: setaffinity failed: %d\n", __func__, | ||||
error); | error); | ||||
free(gtask, M_DEVBUF); | free(gtask, M_DEVBUF); | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_bind(struct taskqgroup *qgroup) | taskqgroup_bind(struct taskqgroup *qgroup) | ||||
{ | { | ||||
struct taskq_bind_task *gtask; | struct taskq_bind_task *gtask; | ||||
Show All 26 Lines | _taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) | ||||
if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { | if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { | ||||
printf("%s: failed cnt: %d stride: %d " | printf("%s: failed cnt: %d stride: %d " | ||||
"mp_ncpus: %d tqg_smp_started: %d\n", | "mp_ncpus: %d tqg_smp_started: %d\n", | ||||
__func__, cnt, stride, mp_ncpus, tqg_smp_started); | __func__, cnt, stride, mp_ncpus, tqg_smp_started); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
if (qgroup->tqg_adjusting) { | if (qgroup->tqg_adjusting) { | ||||
printf("taskqgroup_adjust failed: adjusting\n"); | printf("%s failed: adjusting\n", __func__); | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
qgroup->tqg_adjusting = 1; | qgroup->tqg_adjusting = 1; | ||||
old_cnt = qgroup->tqg_cnt; | old_cnt = qgroup->tqg_cnt; | ||||
old_cpu = 0; | old_cpu = 0; | ||||
if (old_cnt < cnt) | if (old_cnt < cnt) | ||||
old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; | old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
▲ Show 20 Lines • Show All 96 Lines • Show Last 20 Lines |