Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/subr_gtaskqueue.c
Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/gtaskqueue.h> | #include <sys/gtaskqueue.h> | ||||
#include <sys/unistd.h> | #include <sys/unistd.h> | ||||
#include <machine/stdarg.h> | #include <machine/stdarg.h> | ||||
static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues"); | static MALLOC_DEFINE(M_GTASKQUEUE, "gtaskqueue", "Group Task Queues"); | ||||
static void gtaskqueue_thread_enqueue(void *); | static void gtaskqueue_thread_enqueue(void *); | ||||
static void gtaskqueue_thread_loop(void *arg); | static void gtaskqueue_thread_loop(void *arg); | ||||
static int _taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri); | |||||
TASKQGROUP_DEFINE(softirq, mp_ncpus, 1, false, PI_SOFT); | |||||
TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); | |||||
struct gtaskqueue_busy { | struct gtaskqueue_busy { | ||||
struct gtask *tb_running; | struct gtask *tb_running; | ||||
TAILQ_ENTRY(gtaskqueue_busy) tb_link; | TAILQ_ENTRY(gtaskqueue_busy) tb_link; | ||||
}; | }; | ||||
struct gt_intr_thread { | |||||
int git_flags; /* (j) IT_* flags. */ | |||||
int git_need; /* Needs service. */ | |||||
}; | |||||
/* Interrupt thread flags kept in it_flags */ | |||||
#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ | |||||
#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ | |||||
static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; | static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; | ||||
struct gtaskqueue { | struct gtaskqueue { | ||||
STAILQ_HEAD(, gtask) tq_queue; | STAILQ_HEAD(, gtask) tq_queue; | ||||
gtaskqueue_enqueue_fn tq_enqueue; | gtaskqueue_enqueue_fn tq_enqueue; | ||||
void *tq_context; | void *tq_context; | ||||
char *tq_name; | char *tq_name; | ||||
TAILQ_HEAD(, gtaskqueue_busy) tq_active; | TAILQ_HEAD(, gtaskqueue_busy) tq_active; | ||||
struct mtx tq_mutex; | struct mtx tq_mutex; | ||||
struct thread **tq_threads; | struct thread **tq_threads; | ||||
struct gt_intr_thread *tq_gt_intrs; | |||||
int tq_tcount; | int tq_tcount; | ||||
int tq_spin; | int tq_spin; | ||||
int tq_flags; | int tq_flags; | ||||
int tq_callouts; | int tq_callouts; | ||||
taskqueue_callback_fn tq_callbacks[TASKQUEUE_NUM_CALLBACKS]; | taskqueue_callback_fn tq_callbacks[TASKQUEUE_NUM_CALLBACKS]; | ||||
void *tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS]; | void *tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS]; | ||||
}; | }; | ||||
#define TQ_FLAGS_ACTIVE (1 << 0) | #define TQ_FLAGS_ACTIVE (1 << 0) | ||||
#define TQ_FLAGS_BLOCKED (1 << 1) | #define TQ_FLAGS_BLOCKED (1 << 1) | ||||
#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) | #define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) | ||||
#define TQ_FLAGS_INTR (1 << 3) | |||||
#define DT_CALLOUT_ARMED (1 << 0) | #define DT_CALLOUT_ARMED (1 << 0) | ||||
#define TQ_LOCK(tq) \ | #define TQ_LOCK(tq) \ | ||||
do { \ | do { \ | ||||
if ((tq)->tq_spin) \ | if ((tq)->tq_spin) \ | ||||
mtx_lock_spin(&(tq)->tq_mutex); \ | mtx_lock_spin(&(tq)->tq_mutex); \ | ||||
else \ | else \ | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | gtaskqueue_free(struct gtaskqueue *queue) | ||||
KASSERT(TAILQ_EMPTY(&queue->tq_active), ("Tasks still running?")); | KASSERT(TAILQ_EMPTY(&queue->tq_active), ("Tasks still running?")); | ||||
KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks")); | KASSERT(queue->tq_callouts == 0, ("Armed timeout tasks")); | ||||
mtx_destroy(&queue->tq_mutex); | mtx_destroy(&queue->tq_mutex); | ||||
free(queue->tq_threads, M_GTASKQUEUE); | free(queue->tq_threads, M_GTASKQUEUE); | ||||
free(queue->tq_name, M_GTASKQUEUE); | free(queue->tq_name, M_GTASKQUEUE); | ||||
free(queue, M_GTASKQUEUE); | free(queue, M_GTASKQUEUE); | ||||
} | } | ||||
static void | |||||
schedule_ithread(struct gtaskqueue *queue) | |||||
{ | |||||
struct proc *p; | |||||
struct thread *td; | |||||
struct gt_intr_thread *git; | |||||
MPASS(queue->tq_tcount == 1); | |||||
td = queue->tq_threads[0]; | |||||
git = &queue->tq_gt_intrs[0]; | |||||
p = td->td_proc; | |||||
atomic_store_rel_int(&git->git_need, 1); | |||||
thread_lock(td); | |||||
if (TD_AWAITING_INTR(td)) { | |||||
CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, | |||||
td->td_name); | |||||
TD_CLR_IWAIT(td); | |||||
sched_add(td, SRQ_INTR); | |||||
} else { | |||||
CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", | |||||
__func__, p->p_pid, td->td_name, git->git_need, td->td_state); | |||||
} | |||||
thread_unlock(td); | |||||
} | |||||
int | int | ||||
grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask) | grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (queue == NULL) { | if (queue == NULL) { | ||||
gtask_dump(gtask); | gtask_dump(gtask); | ||||
panic("queue == NULL"); | panic("queue == NULL"); | ||||
} | } | ||||
#endif | #endif | ||||
TQ_LOCK(queue); | TQ_LOCK(queue); | ||||
if (gtask->ta_flags & TASK_ENQUEUED) { | if (gtask->ta_flags & TASK_ENQUEUED) { | ||||
TQ_UNLOCK(queue); | TQ_UNLOCK(queue); | ||||
return (0); | return (0); | ||||
} | } | ||||
STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link); | STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link); | ||||
gtask->ta_flags |= TASK_ENQUEUED; | gtask->ta_flags |= TASK_ENQUEUED; | ||||
TQ_UNLOCK(queue); | TQ_UNLOCK(queue); | ||||
if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) | if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) { | ||||
if (queue->tq_flags & TQ_FLAGS_INTR) { | |||||
schedule_ithread(queue); | |||||
} else { | |||||
queue->tq_enqueue(queue->tq_context); | queue->tq_enqueue(queue->tq_context); | ||||
} | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
gtaskqueue_task_nop_fn(void *context) | gtaskqueue_task_nop_fn(void *context) | ||||
{ | { | ||||
} | } | ||||
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines | gtaskqueue_drain_all(struct gtaskqueue *queue) | ||||
TQ_LOCK(queue); | TQ_LOCK(queue); | ||||
gtaskqueue_drain_tq_queue(queue); | gtaskqueue_drain_tq_queue(queue); | ||||
gtaskqueue_drain_tq_active(queue); | gtaskqueue_drain_tq_active(queue); | ||||
TQ_UNLOCK(queue); | TQ_UNLOCK(queue); | ||||
} | } | ||||
static int | static int | ||||
_gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, | _gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, | ||||
cpuset_t *mask, const char *name, va_list ap) | cpuset_t *mask, bool intr, const char *name, va_list ap) | ||||
{ | { | ||||
char ktname[MAXCOMLEN + 1]; | char ktname[MAXCOMLEN + 1]; | ||||
struct thread *td; | struct thread *td; | ||||
struct gtaskqueue *tq; | struct gtaskqueue *tq; | ||||
int i, error; | int i, error; | ||||
if (count <= 0) | if (count <= 0) | ||||
return (EINVAL); | return (EINVAL); | ||||
vsnprintf(ktname, sizeof(ktname), name, ap); | vsnprintf(ktname, sizeof(ktname), name, ap); | ||||
tq = *tqp; | tq = *tqp; | ||||
tq->tq_threads = malloc(sizeof(struct thread *) * count, M_GTASKQUEUE, | tq->tq_threads = malloc(sizeof(struct thread *) * count, M_GTASKQUEUE, | ||||
M_NOWAIT | M_ZERO); | M_NOWAIT | M_ZERO); | ||||
if (tq->tq_threads == NULL) { | if (tq->tq_threads == NULL) { | ||||
printf("%s: no memory for %s threads\n", __func__, ktname); | printf("%s: no memory for %s threads\n", __func__, ktname); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
tq->tq_gt_intrs = malloc(sizeof(struct gt_intr_thread) * count, M_GTASKQUEUE, | |||||
M_NOWAIT | M_ZERO); | |||||
if (tq->tq_gt_intrs == NULL) { | |||||
printf("%s: no memory for %s intr info\n", __func__, ktname); | |||||
return (ENOMEM); | |||||
} | |||||
for (i = 0; i < count; i++) { | for (i = 0; i < count; i++) { | ||||
if (count == 1) | if (count == 1) | ||||
error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, | error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, | ||||
&tq->tq_threads[i], RFSTOPPED, 0, "%s", ktname); | &tq->tq_threads[i], RFSTOPPED, 0, "%s", ktname); | ||||
else | else | ||||
error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, | error = kthread_add(gtaskqueue_thread_loop, tqp, NULL, | ||||
&tq->tq_threads[i], RFSTOPPED, 0, | &tq->tq_threads[i], RFSTOPPED, 0, | ||||
"%s_%d", ktname, i); | "%s_%d", ktname, i); | ||||
if (error) { | if (error) { | ||||
/* should be ok to continue, taskqueue_free will dtrt */ | /* should be ok to continue, taskqueue_free will dtrt */ | ||||
printf("%s: kthread_add(%s): error %d", __func__, | printf("%s: kthread_add(%s): error %d", __func__, | ||||
ktname, error); | ktname, error); | ||||
tq->tq_threads[i] = NULL; /* paranoid */ | tq->tq_threads[i] = NULL; /* paranoid */ | ||||
} else | } else | ||||
tq->tq_tcount++; | tq->tq_tcount++; | ||||
} | } | ||||
if (intr) | |||||
tq->tq_flags |= TQ_FLAGS_INTR; | |||||
for (i = 0; i < count; i++) { | for (i = 0; i < count; i++) { | ||||
if (tq->tq_threads[i] == NULL) | if (tq->tq_threads[i] == NULL) | ||||
continue; | continue; | ||||
td = tq->tq_threads[i]; | td = tq->tq_threads[i]; | ||||
if (mask) { | if (mask) { | ||||
error = cpuset_setthread(td->td_tid, mask); | error = cpuset_setthread(td->td_tid, mask); | ||||
/* | /* | ||||
* Failing to pin is rarely an actual fatal error; | * Failing to pin is rarely an actual fatal error; | ||||
* it'll just affect performance. | * it'll just affect performance. | ||||
*/ | */ | ||||
if (error) | if (error) | ||||
printf("%s: curthread=%llu: can't pin; " | printf("%s: curthread=%llu: can't pin; " | ||||
"error=%d\n", | "error=%d\n", | ||||
__func__, | __func__, | ||||
(unsigned long long) td->td_tid, | (unsigned long long) td->td_tid, | ||||
error); | error); | ||||
} | } | ||||
thread_lock(td); | thread_lock(td); | ||||
sched_prio(td, pri); | sched_prio(td, pri); | ||||
if (intr) { | |||||
/* we need to schedule the thread from the interrupt handler for this to work */ | |||||
TD_SET_IWAIT(td); | |||||
sched_class(td, PRI_ITHD); | |||||
td->td_pflags |= TDP_ITHREAD; | |||||
} else { | |||||
sched_add(td, SRQ_BORING); | sched_add(td, SRQ_BORING); | ||||
} | |||||
thread_unlock(td); | thread_unlock(td); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, | gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, | ||||
const char *name, ...) | bool intr, const char *name, ...) | ||||
{ | { | ||||
va_list ap; | va_list ap; | ||||
int error; | int error; | ||||
va_start(ap, name); | va_start(ap, name); | ||||
error = _gtaskqueue_start_threads(tqp, count, pri, NULL, name, ap); | error = _gtaskqueue_start_threads(tqp, count, pri, NULL, intr, name, ap); | ||||
va_end(ap); | va_end(ap); | ||||
return (error); | return (error); | ||||
} | } | ||||
static inline void | static inline void | ||||
gtaskqueue_run_callback(struct gtaskqueue *tq, | gtaskqueue_run_callback(struct gtaskqueue *tq, | ||||
enum taskqueue_callback_type cb_type) | enum taskqueue_callback_type cb_type) | ||||
{ | { | ||||
taskqueue_callback_fn tq_callback; | taskqueue_callback_fn tq_callback; | ||||
TQ_ASSERT_UNLOCKED(tq); | TQ_ASSERT_UNLOCKED(tq); | ||||
tq_callback = tq->tq_callbacks[cb_type]; | tq_callback = tq->tq_callbacks[cb_type]; | ||||
if (tq_callback != NULL) | if (tq_callback != NULL) | ||||
tq_callback(tq->tq_cb_contexts[cb_type]); | tq_callback(tq->tq_cb_contexts[cb_type]); | ||||
} | } | ||||
static void | static void | ||||
gtaskqueue_thread_loop(void *arg) | intr_thread_loop(struct gtaskqueue *tq) | ||||
{ | { | ||||
struct gtaskqueue **tqp, *tq; | struct gt_intr_thread *git; | ||||
struct thread *td; | |||||
tqp = arg; | git = &tq->tq_gt_intrs[0]; | ||||
tq = *tqp; | td = tq->tq_threads[0]; | ||||
gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); | MPASS(tq->tq_tcount == 1); | ||||
while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { | |||||
THREAD_NO_SLEEPING(); | |||||
while (atomic_cmpset_acq_int(&git->git_need, 1, 0) != 0) { | |||||
gtaskqueue_run_locked(tq); | |||||
} | |||||
THREAD_SLEEPING_OK(); | |||||
/* | |||||
* Because taskqueue_run() can drop tq_mutex, we need to | |||||
* check if the TQ_FLAGS_ACTIVE flag wasn't removed in the | |||||
* meantime, which means we missed a wakeup. | |||||
*/ | |||||
if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) | |||||
break; | |||||
TQ_UNLOCK(tq); | |||||
WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); | |||||
mtx_assert(&Giant, MA_NOTOWNED); | |||||
thread_lock(td); | |||||
if (atomic_load_acq_int(&git->git_need) == 0 && | |||||
(git->git_flags & (IT_DEAD | IT_WAIT)) == 0) { | |||||
TD_SET_IWAIT(td); | |||||
mi_switch(SW_VOL | SWT_IWAIT, NULL); | |||||
} | |||||
#if 0 | |||||
/* XXX is this something we want? */ | |||||
if (git->git_flags & IT_WAIT) { | |||||
wake = 1; | |||||
git->git_flags &= ~IT_WAIT; | |||||
} | |||||
#endif | |||||
thread_unlock(td); | |||||
TQ_LOCK(tq); | TQ_LOCK(tq); | ||||
} | |||||
THREAD_NO_SLEEPING(); | |||||
gtaskqueue_run_locked(tq); | |||||
THREAD_SLEEPING_OK(); | |||||
} | |||||
static void | |||||
timeshare_thread_loop(struct gtaskqueue *tq) | |||||
{ | |||||
while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { | while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { | ||||
/* XXX ? */ | |||||
gtaskqueue_run_locked(tq); | gtaskqueue_run_locked(tq); | ||||
/* | /* | ||||
* Because taskqueue_run() can drop tq_mutex, we need to | * Because taskqueue_run() can drop tq_mutex, we need to | ||||
* check if the TQ_FLAGS_ACTIVE flag wasn't removed in the | * check if the TQ_FLAGS_ACTIVE flag wasn't removed in the | ||||
* meantime, which means we missed a wakeup. | * meantime, which means we missed a wakeup. | ||||
*/ | */ | ||||
if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) | if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) | ||||
break; | break; | ||||
TQ_SLEEP(tq, tq, &tq->tq_mutex, 0, "-", 0); | TQ_SLEEP(tq, tq, &tq->tq_mutex, 0, "-", 0); | ||||
} | } | ||||
gtaskqueue_run_locked(tq); | gtaskqueue_run_locked(tq); | ||||
} | |||||
static void | |||||
gtaskqueue_thread_loop(void *arg) | |||||
{ | |||||
struct gtaskqueue **tqp, *tq; | |||||
tqp = arg; | |||||
tq = *tqp; | |||||
gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); | |||||
TQ_LOCK(tq); | |||||
if (curthread->td_pflags & TDP_ITHREAD) { | |||||
intr_thread_loop(tq); | |||||
} else { | |||||
timeshare_thread_loop(tq); | |||||
} | |||||
/* | /* | ||||
* This thread is on its way out, so just drop the lock temporarily | * This thread is on its way out, so just drop the lock temporarily | ||||
* in order to call the shutdown callback. This allows the callback | * in order to call the shutdown callback. This allows the callback | ||||
* to look at the taskqueue, even just before it dies. | * to look at the taskqueue, even just before it dies. | ||||
*/ | */ | ||||
TQ_UNLOCK(tq); | TQ_UNLOCK(tq); | ||||
gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN); | gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN); | ||||
TQ_LOCK(tq); | TQ_LOCK(tq); | ||||
Show All 30 Lines | struct taskqgroup_cpu { | ||||
struct gtaskqueue *tgc_taskq; | struct gtaskqueue *tgc_taskq; | ||||
int tgc_cnt; | int tgc_cnt; | ||||
int tgc_cpu; | int tgc_cpu; | ||||
}; | }; | ||||
struct taskqgroup { | struct taskqgroup { | ||||
struct taskqgroup_cpu tqg_queue[MAXCPU]; | struct taskqgroup_cpu tqg_queue[MAXCPU]; | ||||
struct mtx tqg_lock; | struct mtx tqg_lock; | ||||
void (*adjust_func)(void*); | |||||
char * tqg_name; | char * tqg_name; | ||||
int tqg_adjusting; | int tqg_adjusting; | ||||
int tqg_stride; | int tqg_stride; | ||||
int tqg_cnt; | int tqg_cnt; | ||||
int tqg_pri; | |||||
int tqg_flags; | |||||
bool tqg_intr; | |||||
}; | }; | ||||
#define TQG_NEED_ADJUST 0x1 | |||||
#define TQG_ADJUSTED 0x2 | |||||
struct taskq_bind_task { | struct taskq_bind_task { | ||||
struct gtask bt_task; | struct gtask bt_task; | ||||
int bt_cpuid; | int bt_cpuid; | ||||
}; | }; | ||||
static void | static void | ||||
taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu) | taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu, bool intr, int pri) | ||||
{ | { | ||||
struct taskqgroup_cpu *qcpu; | struct taskqgroup_cpu *qcpu; | ||||
qcpu = &qgroup->tqg_queue[idx]; | qcpu = &qgroup->tqg_queue[idx]; | ||||
LIST_INIT(&qcpu->tgc_tasks); | LIST_INIT(&qcpu->tgc_tasks); | ||||
qcpu->tgc_taskq = gtaskqueue_create_fast(NULL, M_WAITOK, | qcpu->tgc_taskq = gtaskqueue_create_fast(NULL, M_WAITOK | M_ZERO, | ||||
taskqueue_thread_enqueue, &qcpu->tgc_taskq); | taskqueue_thread_enqueue, &qcpu->tgc_taskq); | ||||
gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, PI_SOFT, | gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, pri, | ||||
"%s_%d", qgroup->tqg_name, idx); | intr, "%s_%d", qgroup->tqg_name, idx); | ||||
qcpu->tgc_cpu = cpu; | qcpu->tgc_cpu = cpu; | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_cpu_remove(struct taskqgroup *qgroup, int idx) | taskqgroup_cpu_remove(struct taskqgroup *qgroup, int idx) | ||||
{ | { | ||||
gtaskqueue_free(qgroup->tqg_queue[idx].tgc_taskq); | gtaskqueue_free(qgroup->tqg_queue[idx].tgc_taskq); | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | |||||
SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | SYSINIT(tqg_record_smp_started, SI_SUB_SMP, SI_ORDER_FOURTH, | ||||
tqg_record_smp_started, NULL); | tqg_record_smp_started, NULL); | ||||
void | void | ||||
taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int irq, char *name) | void *uniq, int irq, char *name) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int qid; | int qid, error; | ||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
gtask->gt_name = name; | gtask->gt_name = name; | ||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = -1; | gtask->gt_cpu = -1; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qgroup->tqg_flags |= TQG_NEED_ADJUST; | |||||
mtx_unlock(&qgroup->tqg_lock); | |||||
if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED)) | |||||
qgroup->adjust_func(NULL); | |||||
mtx_lock(&qgroup->tqg_lock); | |||||
qid = taskqgroup_find(qgroup, uniq); | qid = taskqgroup_find(qgroup, uniq); | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
if (irq != -1 && tqg_smp_started) { | if (irq != -1 && tqg_smp_started) { | ||||
gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; | gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); | CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask); | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | error = intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &mask); | ||||
if (error) | |||||
printf("taskqgroup_attach: setaffinity failed: %d\n", error); | |||||
} else | } else | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
static void | static void | ||||
taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int qid, cpu; | int qid, cpu, error; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qid = taskqgroup_find(qgroup, gtask->gt_uniq); | qid = taskqgroup_find(qgroup, gtask->gt_uniq); | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
if (gtask->gt_irq != -1) { | if (gtask->gt_irq != -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
intr_setaffinity(gtask->gt_irq, CPU_WHICH_IRQ, &mask); | error = intr_setaffinity(gtask->gt_irq, CPU_WHICH_INTRHANDLER, &mask); | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
if (error) | |||||
printf("taskqgroup_attach_deferred: setaffinity failed: %d\n", error); | |||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, | ||||
gt_list); | gt_list); | ||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
} | } | ||||
static int | |||||
taskqgroup_adjust_deferred(struct taskqgroup *qgroup, int cpu) | |||||
{ | |||||
int i, error = 0, cpu_max = -1; | |||||
mtx_lock(&qgroup->tqg_lock); | |||||
for (i = 0; i < qgroup->tqg_cnt; i++) | |||||
if (qgroup->tqg_queue[i].tgc_cpu > cpu_max) | |||||
cpu_max = qgroup->tqg_queue[i].tgc_cpu; | |||||
if (cpu_max >= cpu) { | |||||
mtx_unlock(&qgroup->tqg_lock); | |||||
return (0); | |||||
} | |||||
MPASS(cpu <= mp_maxid); | |||||
error = _taskqgroup_adjust(qgroup, cpu + 1, qgroup->tqg_stride, | |||||
qgroup->tqg_intr, qgroup->tqg_pri); | |||||
if (error) { | |||||
printf("%s: _taskqgroup_adjust(%p, %d, %d, %d, %d) => %d\n\n", | |||||
__func__, qgroup, cpu + 1, qgroup->tqg_stride, qgroup->tqg_intr, | |||||
qgroup->tqg_pri, error); | |||||
goto out; | |||||
} | |||||
for (i = 0; i < qgroup->tqg_cnt; i++) | |||||
if (qgroup->tqg_queue[i].tgc_cpu > cpu_max) | |||||
cpu_max = qgroup->tqg_queue[i].tgc_cpu; | |||||
MPASS(cpu_max >= cpu); | |||||
out: | |||||
mtx_unlock(&qgroup->tqg_lock); | |||||
return (error); | |||||
} | |||||
int | int | ||||
taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | taskqgroup_attach_cpu(struct taskqgroup *qgroup, struct grouptask *gtask, | ||||
void *uniq, int cpu, int irq, char *name) | void *uniq, int cpu, int irq, char *name) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int i, qid; | int i, error, qid; | ||||
qid = -1; | qid = -1; | ||||
gtask->gt_uniq = uniq; | gtask->gt_uniq = uniq; | ||||
gtask->gt_name = name; | gtask->gt_name = name; | ||||
gtask->gt_irq = irq; | gtask->gt_irq = irq; | ||||
gtask->gt_cpu = cpu; | gtask->gt_cpu = cpu; | ||||
MPASS(cpu >= 0); | |||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qgroup->tqg_flags |= TQG_NEED_ADJUST; | |||||
mtx_unlock(&qgroup->tqg_lock); | |||||
if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED)) { | |||||
uintptr_t cpuid = cpu + 1; | |||||
qgroup->adjust_func((void *)cpuid); | |||||
} | |||||
if ((error = taskqgroup_adjust_deferred(qgroup, cpu))) | |||||
return (error); | |||||
mtx_lock(&qgroup->tqg_lock); | |||||
if (tqg_smp_started) { | if (tqg_smp_started) { | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) { | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
#ifdef INVARIANTS | |||||
else | |||||
printf("qgroup->tqg_queue[%d].tgc_cpu=0x%x tgc_cnt=0x%x\n", | |||||
i, qgroup->tqg_queue[i].tgc_cpu, qgroup->tqg_queue[i].tgc_cnt); | |||||
#endif | |||||
} | |||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for cpu=%d\n", __func__, cpu); | |||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
} else | } else | ||||
qid = 0; | qid = 0; | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
cpu = qgroup->tqg_queue[qid].tgc_cpu; | cpu = qgroup->tqg_queue[qid].tgc_cpu; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
if (irq != -1 && tqg_smp_started) | if (irq != -1 && tqg_smp_started) { | ||||
intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | error = intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &mask); | ||||
if (error) | |||||
printf("taskqgroup_attach_cpu: setaffinity failed: %d\n", error); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_attach_cpu_deferred(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
cpuset_t mask; | cpuset_t mask; | ||||
int i, qid, irq, cpu; | int i, qid, irq, cpu, error; | ||||
qid = -1; | qid = -1; | ||||
irq = gtask->gt_irq; | irq = gtask->gt_irq; | ||||
cpu = gtask->gt_cpu; | cpu = gtask->gt_cpu; | ||||
MPASS(tqg_smp_started); | MPASS(tqg_smp_started); | ||||
if ((error = taskqgroup_adjust_deferred(qgroup, cpu))) | |||||
return (error); | |||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
/* adjust as needed */ | |||||
MPASS(cpu <= mp_maxid); | |||||
for (i = 0; i < qgroup->tqg_cnt; i++) | for (i = 0; i < qgroup->tqg_cnt; i++) | ||||
if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | if (qgroup->tqg_queue[i].tgc_cpu == cpu) { | ||||
qid = i; | qid = i; | ||||
break; | break; | ||||
} | } | ||||
if (qid == -1) { | if (qid == -1) { | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
printf("%s: qid not found for cpu=%d\n", __func__, cpu); | |||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
qgroup->tqg_queue[qid].tgc_cnt++; | qgroup->tqg_queue[qid].tgc_cnt++; | ||||
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list); | ||||
MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | MPASS(qgroup->tqg_queue[qid].tgc_taskq != NULL); | ||||
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq; | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
CPU_ZERO(&mask); | CPU_ZERO(&mask); | ||||
CPU_SET(cpu, &mask); | CPU_SET(cpu, &mask); | ||||
if (irq != -1) | if (irq != -1) { | ||||
intr_setaffinity(irq, CPU_WHICH_IRQ, &mask); | error = intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &mask); | ||||
if (error) | |||||
printf("taskqgroup_attach_cpu: setaffinity failed: %d\n", error); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask) | ||||
{ | { | ||||
int i; | int i; | ||||
Show All 22 Lines | taskqgroup_binder(void *ctx) | ||||
thread_lock(curthread); | thread_lock(curthread); | ||||
sched_bind(curthread, gtask->bt_cpuid); | sched_bind(curthread, gtask->bt_cpuid); | ||||
thread_unlock(curthread); | thread_unlock(curthread); | ||||
if (error) | if (error) | ||||
printf("taskqgroup_binder: setaffinity failed: %d\n", | printf("taskqgroup_binder: setaffinity failed: %d\n", | ||||
error); | error); | ||||
free(gtask, M_DEVBUF); | free(gtask, M_DEVBUF); | ||||
} | } | ||||
static void | |||||
taskqgroup_ithread_binder(void *ctx) | |||||
{ | |||||
struct taskq_bind_task *gtask = (struct taskq_bind_task *)ctx; | |||||
cpuset_t mask; | |||||
int error; | |||||
CPU_ZERO(&mask); | |||||
CPU_SET(gtask->bt_cpuid, &mask); | |||||
error = cpuset_setthread(curthread->td_tid, &mask); | |||||
if (error) | |||||
printf("taskqgroup_binder: setaffinity failed: %d\n", | |||||
error); | |||||
free(gtask, M_DEVBUF); | |||||
} | |||||
static void | static void | ||||
taskqgroup_bind(struct taskqgroup *qgroup) | taskqgroup_bind(struct taskqgroup *qgroup) | ||||
{ | { | ||||
struct taskq_bind_task *gtask; | struct taskq_bind_task *gtask; | ||||
int i; | int i; | ||||
/* | /* | ||||
* Bind taskqueue threads to specific CPUs, if they have been assigned | * Bind taskqueue threads to specific CPUs, if they have been assigned | ||||
* one. | * one. | ||||
*/ | */ | ||||
if (qgroup->tqg_cnt == 1) | if (qgroup->tqg_cnt == 1) | ||||
return; | return; | ||||
for (i = 0; i < qgroup->tqg_cnt; i++) { | for (i = 0; i < qgroup->tqg_cnt; i++) { | ||||
gtask = malloc(sizeof (*gtask), M_DEVBUF, M_WAITOK); | gtask = malloc(sizeof (*gtask), M_DEVBUF, M_WAITOK); | ||||
if (qgroup->tqg_intr) | |||||
GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_ithread_binder, gtask); | |||||
else | |||||
GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_binder, gtask); | GTASK_INIT(>ask->bt_task, 0, 0, taskqgroup_binder, gtask); | ||||
gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu; | gtask->bt_cpuid = qgroup->tqg_queue[i].tgc_cpu; | ||||
grouptaskqueue_enqueue(qgroup->tqg_queue[i].tgc_taskq, | grouptaskqueue_enqueue(qgroup->tqg_queue[i].tgc_taskq, | ||||
>ask->bt_task); | >ask->bt_task); | ||||
} | } | ||||
} | } | ||||
static int | static int | ||||
_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) | _taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) | ||||
{ | { | ||||
LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); | LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
int i, k, old_cnt, old_cpu, cpu; | int i, k, old_cnt, old_cpu, cpu; | ||||
mtx_assert(&qgroup->tqg_lock, MA_OWNED); | mtx_assert(&qgroup->tqg_lock, MA_OWNED); | ||||
if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { | if (cnt < 1 || cnt * stride > mp_ncpus || !tqg_smp_started) { | ||||
printf("%s: failed cnt: %d stride: %d " | printf("%s: failed cnt: %d stride: %d " | ||||
"mp_ncpus: %d tqg_smp_started: %d\n", | "mp_ncpus: %d tqg_smp_started: %d\n", | ||||
__func__, cnt, stride, mp_ncpus, tqg_smp_started); | __func__, cnt, stride, mp_ncpus, tqg_smp_started); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
if (qgroup->tqg_adjusting) { | if (qgroup->tqg_adjusting) { | ||||
printf("taskqgroup_adjust failed: adjusting\n"); | printf("%s: failed: adjusting\n", __func__); | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
/* No work to be done */ | |||||
if (qgroup->tqg_cnt == cnt) | |||||
return (0); | |||||
qgroup->tqg_adjusting = 1; | qgroup->tqg_adjusting = 1; | ||||
old_cnt = qgroup->tqg_cnt; | old_cnt = qgroup->tqg_cnt; | ||||
old_cpu = 0; | old_cpu = 0; | ||||
if (old_cnt < cnt) | if (old_cnt < cnt) { | ||||
old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; | int old_max_idx = max(0, old_cnt-1); | ||||
old_cpu = qgroup->tqg_queue[old_max_idx].tgc_cpu; | |||||
if (old_cnt > 0) | |||||
for (k = 0; k < stride; k++) | |||||
old_cpu = CPU_NEXT(old_cpu); | |||||
} | |||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
/* | /* | ||||
* Set up queue for tasks added before boot. | * Set up queue for tasks added before boot. | ||||
*/ | */ | ||||
if (old_cnt == 0) { | if (old_cnt == 0) { | ||||
LIST_SWAP(>ask_head, &qgroup->tqg_queue[0].tgc_tasks, | LIST_SWAP(>ask_head, &qgroup->tqg_queue[0].tgc_tasks, | ||||
grouptask, gt_list); | grouptask, gt_list); | ||||
qgroup->tqg_queue[0].tgc_cnt = 0; | qgroup->tqg_queue[0].tgc_cnt = 0; | ||||
} | } | ||||
/* | /* | ||||
* If new taskq threads have been added. | * If new taskq threads have been added. | ||||
*/ | */ | ||||
cpu = old_cpu; | cpu = old_cpu; | ||||
for (i = old_cnt; i < cnt; i++) { | for (i = old_cnt; i < cnt; i++) { | ||||
taskqgroup_cpu_create(qgroup, i, cpu); | taskqgroup_cpu_create(qgroup, i, cpu, ithread, pri); | ||||
for (k = 0; k < stride; k++) | for (k = 0; k < stride; k++) | ||||
cpu = CPU_NEXT(cpu); | cpu = CPU_NEXT(cpu); | ||||
} | } | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qgroup->tqg_cnt = cnt; | qgroup->tqg_cnt = cnt; | ||||
qgroup->tqg_stride = stride; | qgroup->tqg_stride = stride; | ||||
qgroup->tqg_intr = ithread; | |||||
qgroup->tqg_pri = pri; | |||||
/* | /* | ||||
* Adjust drivers to use new taskqs. | * Adjust drivers to use new taskqs. | ||||
*/ | */ | ||||
for (i = 0; i < old_cnt; i++) { | for (i = 0; i < old_cnt; i++) { | ||||
while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) { | while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) { | ||||
LIST_REMOVE(gtask, gt_list); | LIST_REMOVE(gtask, gt_list); | ||||
qgroup->tqg_queue[i].tgc_cnt--; | qgroup->tqg_queue[i].tgc_cnt--; | ||||
Show All 29 Lines | #endif | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
qgroup->tqg_adjusting = 0; | qgroup->tqg_adjusting = 0; | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) | taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) | ||||
{ | { | ||||
int error; | int error; | ||||
mtx_lock(&qgroup->tqg_lock); | mtx_lock(&qgroup->tqg_lock); | ||||
error = _taskqgroup_adjust(qgroup, cnt, stride); | error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri); | ||||
mtx_unlock(&qgroup->tqg_lock); | mtx_unlock(&qgroup->tqg_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
void | |||||
taskqgroup_set_adjust(struct taskqgroup *qgroup, void (*adjust_func)(void*)) | |||||
{ | |||||
qgroup-> adjust_func = adjust_func; | |||||
} | |||||
int | |||||
taskqgroup_adjust_once(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) | |||||
{ | |||||
int error = 0; | |||||
mtx_lock(&qgroup->tqg_lock); | |||||
if ((qgroup->tqg_flags & (TQG_ADJUSTED|TQG_NEED_ADJUST)) == TQG_NEED_ADJUST) { | |||||
qgroup->tqg_flags |= TQG_ADJUSTED; | |||||
error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri); | |||||
MPASS(error == 0); | |||||
} | |||||
mtx_unlock(&qgroup->tqg_lock); | |||||
return (error); | |||||
} | |||||
struct taskqgroup * | struct taskqgroup * | ||||
taskqgroup_create(char *name) | taskqgroup_create(char *name) | ||||
{ | { | ||||
struct taskqgroup *qgroup; | struct taskqgroup *qgroup; | ||||
qgroup = malloc(sizeof(*qgroup), M_GTASKQUEUE, M_WAITOK | M_ZERO); | qgroup = malloc(sizeof(*qgroup), M_GTASKQUEUE, M_WAITOK | M_ZERO); | ||||
mtx_init(&qgroup->tqg_lock, "taskqgroup", NULL, MTX_DEF); | mtx_init(&qgroup->tqg_lock, "taskqgroup", NULL, MTX_DEF); | ||||
qgroup->tqg_name = name; | qgroup->tqg_name = name; | ||||
LIST_INIT(&qgroup->tqg_queue[0].tgc_tasks); | LIST_INIT(&qgroup->tqg_queue[0].tgc_tasks); | ||||
MPASS(qgroup->tqg_queue[0].tgc_cnt == 0); | |||||
MPASS(qgroup->tqg_queue[0].tgc_cpu == 0); | |||||
MPASS(qgroup->tqg_queue[0].tgc_taskq == 0); | |||||
return (qgroup); | return (qgroup); | ||||
} | } | ||||
void | void | ||||
taskqgroup_destroy(struct taskqgroup *qgroup) | taskqgroup_destroy(struct taskqgroup *qgroup) | ||||
{ | { | ||||
} | } |