Index: sys/kern/subr_gtaskqueue.c =================================================================== --- sys/kern/subr_gtaskqueue.c +++ sys/kern/subr_gtaskqueue.c @@ -52,13 +52,22 @@ static void gtaskqueue_thread_enqueue(void *); static void gtaskqueue_thread_loop(void *arg); -TASKQGROUP_DEFINE(softirq, mp_ncpus, 1); +TASKQGROUP_DEFINE(softirq, mp_ncpus, 1, false, PI_SOFT); struct gtaskqueue_busy { struct gtask *tb_running; TAILQ_ENTRY(gtaskqueue_busy) tb_link; }; +struct gt_intr_thread { + int git_flags; /* (j) IT_* flags. */ + int git_need; /* Needs service. */ +}; + +/* Interrupt thread flags kept in it_flags */ +#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ +#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ + static struct gtask * const TB_DRAIN_WAITER = (struct gtask *)0x1; struct gtaskqueue { @@ -69,6 +78,7 @@ TAILQ_HEAD(, gtaskqueue_busy) tq_active; struct mtx tq_mutex; struct thread **tq_threads; + struct gt_intr_thread *tq_gt_intrs; int tq_tcount; int tq_spin; int tq_flags; @@ -80,6 +90,7 @@ #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) #define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) +#define TQ_FLAGS_INTR (1 << 3) #define DT_CALLOUT_ARMED (1 << 0) @@ -180,6 +191,32 @@ free(queue, M_GTASKQUEUE); } +static void +schedule_ithread(struct gtaskqueue *queue) +{ + struct proc *p; + struct thread *td; + struct gt_intr_thread *git; + + MPASS(queue->tq_tcount == 1); + td = queue->tq_threads[0]; + git = &queue->tq_gt_intrs[0]; + p = td->td_proc; + + atomic_store_rel_int(&git->git_need, 1); + thread_lock(td); + if (TD_AWAITING_INTR(td)) { + CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, + td->td_name); + TD_CLR_IWAIT(td); + sched_add(td, SRQ_INTR); + } else { + CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", + __func__, p->p_pid, td->td_name, git->git_need, td->td_state); + } + thread_unlock(td); +} + int grouptaskqueue_enqueue(struct gtaskqueue *queue, struct gtask *gtask) { @@ -197,8 +234,13 @@ STAILQ_INSERT_TAIL(&queue->tq_queue, gtask, ta_link); gtask->ta_flags |= TASK_ENQUEUED; TQ_UNLOCK(queue); - if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) - queue->tq_enqueue(queue->tq_context); + if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) { + if (queue->tq_flags & TQ_FLAGS_INTR) { + schedule_ithread(queue); + } else { + queue->tq_enqueue(queue->tq_context); + } + } return (0); } @@ -403,7 +445,7 @@ static int _gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, - cpuset_t *mask, const char *name, va_list ap) + cpuset_t *mask, bool intr, const char *name, va_list ap) { char ktname[MAXCOMLEN + 1]; struct thread *td; @@ -422,6 +464,12 @@ printf("%s: no memory for %s threads\n", __func__, ktname); return (ENOMEM); } + tq->tq_gt_intrs = malloc(sizeof(struct gt_intr_thread) * count, M_GTASKQUEUE, + M_NOWAIT | M_ZERO); + if (tq->tq_gt_intrs == NULL) { + printf("%s: no memory for %s intr info\n", __func__, ktname); + return (ENOMEM); + } for (i = 0; i < count; i++) { if (count == 1) @@ -439,6 +487,9 @@ } else tq->tq_tcount++; } + if (intr) + tq->tq_flags |= TQ_FLAGS_INTR; + for (i = 0; i < count; i++) { if (tq->tq_threads[i] == NULL) continue; @@ -458,7 +509,14 @@ } thread_lock(td); sched_prio(td, pri); - sched_add(td, SRQ_BORING); + if (intr) { + /* we need to schedule the thread from the interrupt handler for this to work */ + TD_SET_IWAIT(td); + sched_class(td, PRI_ITHD); + td->td_pflags |= TDP_ITHREAD; + } else { + sched_add(td, SRQ_BORING); + } thread_unlock(td); } @@ -467,13 +525,13 @@ static int gtaskqueue_start_threads(struct gtaskqueue **tqp, int count, int pri, - const char *name, ...) + bool intr, const char *name, ...) { va_list ap; int error; va_start(ap, name); - error = _gtaskqueue_start_threads(tqp, count, pri, NULL, name, ap); + error = _gtaskqueue_start_threads(tqp, count, pri, NULL, intr, name, ap); va_end(ap); return (error); } @@ -491,16 +549,58 @@ } static void -gtaskqueue_thread_loop(void *arg) +intr_thread_loop(struct gtaskqueue *tq) { - struct gtaskqueue **tqp, *tq; + struct gt_intr_thread *git; + struct thread *td; - tqp = arg; - tq = *tqp; - gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); - TQ_LOCK(tq); + git = &tq->tq_gt_intrs[0]; + td = tq->tq_threads[0]; + MPASS(tq->tq_tcount == 1); + + while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { + THREAD_NO_SLEEPING(); + while (atomic_cmpset_acq_int(&git->git_need, 1, 0) != 0) { + gtaskqueue_run_locked(tq); + } + THREAD_SLEEPING_OK(); + + /* + * Because taskqueue_run() can drop tq_mutex, we need to + * check if the TQ_FLAGS_ACTIVE flag wasn't removed in the + * meantime, which means we missed a wakeup. + */ + if ((tq->tq_flags & TQ_FLAGS_ACTIVE) == 0) + break; + + TQ_UNLOCK(tq); + WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); + mtx_assert(&Giant, MA_NOTOWNED); + thread_lock(td); + if (atomic_load_acq_int(&git->git_need) == 0 && + (git->git_flags & (IT_DEAD | IT_WAIT)) == 0) { + TD_SET_IWAIT(td); + mi_switch(SW_VOL | SWT_IWAIT, NULL); + } +#if 0 + /* XXX is this something we want? */ + if (git->git_flags & IT_WAIT) { + wake = 1; + git->git_flags &= ~IT_WAIT; + } +#endif + thread_unlock(td); + TQ_LOCK(tq); + } + THREAD_NO_SLEEPING(); + gtaskqueue_run_locked(tq); + THREAD_SLEEPING_OK(); +} + +static void +timeshare_thread_loop(struct gtaskqueue *tq) +{ while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0) { - /* XXX ? */ gtaskqueue_run_locked(tq); /* * Because taskqueue_run() can drop tq_mutex, we need to @@ -512,6 +612,23 @@ TQ_SLEEP(tq, tq, &tq->tq_mutex, 0, "-", 0); } gtaskqueue_run_locked(tq); +} + +static void +gtaskqueue_thread_loop(void *arg) +{ + struct gtaskqueue **tqp, *tq; + + tqp = arg; + tq = *tqp; + gtaskqueue_run_callback(tq, TASKQUEUE_CALLBACK_TYPE_INIT); + TQ_LOCK(tq); + if (curthread->td_pflags & TDP_ITHREAD) { + intr_thread_loop(tq); + } else { + timeshare_thread_loop(tq); + } + /* * This thread is on its way out, so just drop the lock temporarily * in order to call the shutdown callback. This allows the callback @@ -570,7 +687,7 @@ }; static void -taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu) +taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu, bool intr, int pri) { struct taskqgroup_cpu *qcpu; @@ -578,8 +695,8 @@ LIST_INIT(&qcpu->tgc_tasks); qcpu->tgc_taskq = gtaskqueue_create_fast(NULL, M_WAITOK, taskqueue_thread_enqueue, &qcpu->tgc_taskq); - gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, PI_SOFT, - "%s_%d", qgroup->tqg_name, idx); + gtaskqueue_start_threads(&qcpu->tgc_taskq, 1, pri, + intr, "%s_%d", qgroup->tqg_name, idx); qcpu->tgc_cpu = cpu; } @@ -843,7 +960,7 @@ } static int -_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) +_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) { LIST_HEAD(, grouptask) gtask_head = LIST_HEAD_INITIALIZER(NULL); struct grouptask *gtask; @@ -881,7 +998,7 @@ */ cpu = old_cpu; for (i = old_cnt; i < cnt; i++) { - taskqgroup_cpu_create(qgroup, i, cpu); + taskqgroup_cpu_create(qgroup, i, cpu, ithread, pri); for (k = 0; k < stride; k++) cpu = CPU_NEXT(cpu); @@ -934,12 +1051,12 @@ } int -taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride) +taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) { int error; mtx_lock(&qgroup->tqg_lock); - error = _taskqgroup_adjust(qgroup, cnt, stride); + error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri); mtx_unlock(&qgroup->tqg_lock); return (error); Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -553,8 +553,8 @@ MODULE_DEPEND(iflib, pci, 1, 1, 1); MODULE_DEPEND(iflib, ether, 1, 1, 1); -TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1); -TASKQGROUP_DEFINE(if_config_tqg, 1, 1); +TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); +TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); #ifndef IFLIB_DEBUG_COUNTERS #ifdef INVARIANTS @@ -4173,7 +4173,7 @@ GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx); /* XXX format name */ - taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin"); + taskqgroup_attach(qgroup_if_config, &ctx->ifc_admin_task, ctx, -1, "admin"); /* ** Now setup MSI or MSI/X, should ** return us the number of supported @@ -4307,7 +4307,7 @@ if (ctx->ifc_led_dev != NULL) led_destroy(ctx->ifc_led_dev); /* XXX drain any dependent tasks */ - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { callout_drain(&txq->ift_timer); if (txq->ift_task.gt_uniq != NULL) @@ -4321,7 +4321,7 @@ free(fl->ifl_rx_bitmap, M_IFLIB); } - tqg = qgroup_if_config_tqg; + tqg = qgroup_if_config; if (ctx->ifc_admin_task.gt_uniq != NULL) taskqgroup_detach(tqg, &ctx->ifc_admin_task); if (ctx->ifc_vflr_task.gt_uniq != NULL) @@ -4935,7 +4935,7 @@ q = &ctx->ifc_txqs[qid]; info = &ctx->ifc_txqs[qid].ift_filter_info; gtask = &ctx->ifc_txqs[qid].ift_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; fn = _task_fn_tx; intr_fast = iflib_fast_intr; GROUPTASK_INIT(gtask, 0, fn, q); @@ -4944,7 +4944,7 @@ q = &ctx->ifc_rxqs[qid]; info = &ctx->ifc_rxqs[qid].ifr_filter_info; gtask = &ctx->ifc_rxqs[qid].ifr_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; fn = _task_fn_rx; intr_fast = iflib_fast_intr; GROUPTASK_INIT(gtask, 0, fn, q); @@ -4953,7 +4953,7 @@ q = &ctx->ifc_rxqs[qid]; info = &ctx->ifc_rxqs[qid].ifr_filter_info; gtask = &ctx->ifc_rxqs[qid].ifr_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; fn = _task_fn_rx; intr_fast = iflib_fast_intr_rxtx; GROUPTASK_INIT(gtask, 0, fn, q); @@ -4963,7 +4963,7 @@ tqrid = -1; info = &ctx->ifc_filter_info; gtask = &ctx->ifc_admin_task; - tqg = qgroup_if_config_tqg; + tqg = qgroup_if_config; fn = _task_fn_admin; intr_fast = iflib_fast_intr_ctx; break; @@ -5006,19 +5006,19 @@ case IFLIB_INTR_TX: q = &ctx->ifc_txqs[qid]; gtask = &ctx->ifc_txqs[qid].ift_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; fn = _task_fn_tx; break; case IFLIB_INTR_RX: q = &ctx->ifc_rxqs[qid]; gtask = &ctx->ifc_rxqs[qid].ifr_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; fn = _task_fn_rx; break; case IFLIB_INTR_IOV: q = ctx; gtask = &ctx->ifc_vflr_task; - tqg = qgroup_if_config_tqg; + tqg = qgroup_if_config; rid = -1; fn = _task_fn_iov; break; @@ -5056,7 +5056,7 @@ q = &ctx->ifc_rxqs[0]; info = &rxq[0].ifr_filter_info; gtask = &rxq[0].ifr_task; - tqg = qgroup_if_io_tqg; + tqg = qgroup_if_io; tqrid = irq->ii_rid = *rid; fn = _task_fn_rx; @@ -5073,7 +5073,7 @@ taskqgroup_attach(tqg, gtask, q, tqrid, name); GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); - taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, tqrid, "tx"); + taskqgroup_attach(qgroup_if_io, &txq->ift_task, txq, tqrid, "tx"); return (0); } @@ -5123,7 +5123,7 @@ iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name) { - taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name); + taskqgroup_attach_cpu(qgroup_if_io, gt, uniq, cpu, -1, name); } void @@ -5132,14 +5132,14 @@ { GROUPTASK_INIT(gtask, 0, fn, ctx); - taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name); + taskqgroup_attach(qgroup_if_config, gtask, gtask, -1, name); } void iflib_config_gtask_deinit(struct grouptask *gtask) { - taskqgroup_detach(qgroup_if_config_tqg, gtask); + taskqgroup_detach(qgroup_if_config, gtask); } void Index: sys/sys/gtaskqueue.h =================================================================== --- sys/sys/gtaskqueue.h +++ sys/sys/gtaskqueue.h @@ -58,7 +58,7 @@ void taskqgroup_detach(struct taskqgroup *qgroup, struct grouptask *gtask); struct taskqgroup *taskqgroup_create(char *name); void taskqgroup_destroy(struct taskqgroup *qgroup); -int taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride); +int taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri); #define TASK_ENQUEUED 0x1 #define TASK_SKIP_WAKEUP 0x2 @@ -80,7 +80,8 @@ #define TASKQGROUP_DECLARE(name) \ extern struct taskqgroup *qgroup_##name -#define TASKQGROUP_DEFINE(name, cnt, stride) \ + +#define TASKQGROUP_DEFINE(name, cnt, stride, intr, pri) \ \ struct taskqgroup *qgroup_##name; \ \ @@ -96,12 +97,17 @@ static void \ taskqgroup_adjust_##name(void *arg) \ { \ - taskqgroup_adjust(qgroup_##name, (cnt), (stride)); \ + taskqgroup_adjust(qgroup_##name, (cnt), (stride), (intr), (pri)); \ } \ \ SYSINIT(taskqgroup_adj_##name, SI_SUB_SMP, SI_ORDER_ANY, \ taskqgroup_adjust_##name, NULL) + + + + + TASKQGROUP_DECLARE(net); TASKQGROUP_DECLARE(softirq);