Index: sys/kern/subr_gtaskqueue.c =================================================================== --- sys/kern/subr_gtaskqueue.c +++ sys/kern/subr_gtaskqueue.c @@ -51,7 +51,7 @@ static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues"); static void gtaskqueue_thread_enqueue(void *); static void gtaskqueue_thread_loop(void *arg); - +static int _taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri); TASKQGROUP_DEFINE(softirq, mp_ncpus, 1, false, PI_SOFT); struct gtaskqueue_busy { @@ -675,11 +675,17 @@ struct taskqgroup { struct taskqgroup_cpu tqg_queue[MAXCPU]; struct mtx tqg_lock; + void (*adjust_func)(void*); char * tqg_name; int tqg_adjusting; int tqg_stride; int tqg_cnt; + int tqg_pri; + int tqg_flags; + bool tqg_intr; }; +#define TQG_NEED_ADJUST 0x1 +#define TQG_ADJUSTED 0x2 struct taskq_bind_task { struct gtask bt_task; @@ -786,6 +792,14 @@ gtask->gt_name = name; gtask->gt_irq = irq; gtask->gt_cpu = -1; + + mtx_lock(&qgroup->tqg_lock); + qgroup->tqg_flags |= TQG_NEED_ADJUST; + mtx_unlock(&qgroup->tqg_lock); + + if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED)) + qgroup->adjust_func(NULL); + mtx_lock(&qgroup->tqg_lock); qid = taskqgroup_find(qgroup, uniq); qgroup->tqg_queue[qid].tgc_cnt++; @@ -833,15 +847,45 @@ void *uniq, int cpu, int irq, char *name) { cpuset_t mask; - int i, qid; + int i, err, qid, cpu_max; qid = -1; gtask->gt_uniq = uniq; gtask->gt_name = name; gtask->gt_irq = irq; gtask->gt_cpu = cpu; + cpu_max = -1; + MPASS(cpu >= 0); + + mtx_lock(&qgroup->tqg_lock); + qgroup->tqg_flags |= TQG_NEED_ADJUST; + mtx_unlock(&qgroup->tqg_lock); + + if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED)) { + uintptr_t cpuid = cpu + 1; + qgroup->adjust_func((void *)cpuid); + } mtx_lock(&qgroup->tqg_lock); if (tqg_smp_started) { + /* adjust as needed */ + for (i = 0; i < qgroup->tqg_cnt; i++) + if (qgroup->tqg_queue[i].tgc_cpu > cpu_max) + cpu_max = qgroup->tqg_queue[i].tgc_cpu; + MPASS(cpu <= mp_maxid); + if (cpu > cpu_max) { + err = _taskqgroup_adjust(qgroup, cpu + 1, qgroup->tqg_stride, + qgroup->tqg_intr, qgroup->tqg_pri); + if (err) { + printf("_taskqgroup_adjust(%p, %d, %d, %d, %d) => %d\n\n", + qgroup, cpu + 1, qgroup->tqg_stride, qgroup->tqg_intr, qgroup->tqg_pri, + err); + mtx_unlock(&qgroup->tqg_lock); + return (err); + } + } + for (i = 0; i < qgroup->tqg_cnt; i++) + if (qgroup->tqg_queue[i].tgc_cpu > cpu_max) + cpu_max = qgroup->tqg_queue[i].tgc_cpu; for (i = 0; i < qgroup->tqg_cnt; i++) if (qgroup->tqg_queue[i].tgc_cpu == cpu) { qid = i; @@ -849,6 +893,7 @@ } if (qid == -1) { mtx_unlock(&qgroup->tqg_lock); + printf("qid not found for cpu=%d\n", cpu); return (EINVAL); } } else @@ -975,14 +1020,22 @@ return (EINVAL); } if (qgroup->tqg_adjusting) { - printf("taskqgroup_adjust failed: adjusting\n"); + printf("%s: failed: adjusting\n", __func__); return (EBUSY); } + /* No work to be done */ + if (qgroup->tqg_cnt == cnt) + return (0); qgroup->tqg_adjusting = 1; old_cnt = qgroup->tqg_cnt; old_cpu = 0; - if (old_cnt < cnt) - old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu; + if (old_cnt < cnt) { + int old_max_idx = max(0, old_cnt-1); + old_cpu = qgroup->tqg_queue[old_max_idx].tgc_cpu; + if (old_cnt > 0) + for (k = 0; k < stride; k++) + old_cpu = CPU_NEXT(old_cpu); + } mtx_unlock(&qgroup->tqg_lock); /* * Set up queue for tasks added before boot. @@ -1006,6 +1059,8 @@ mtx_lock(&qgroup->tqg_lock); qgroup->tqg_cnt = cnt; qgroup->tqg_stride = stride; + qgroup->tqg_intr = ithread; + qgroup->tqg_pri = pri; /* * Adjust drivers to use new taskqs. @@ -1062,6 +1117,28 @@ return (error); } +void +taskqgroup_set_adjust(struct taskqgroup *qgroup, void (*adjust_func)(void*)) +{ + qgroup-> adjust_func = adjust_func; +} + +int +taskqgroup_adjust_once(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri) +{ + int error = 0; + + mtx_lock(&qgroup->tqg_lock); + if ((qgroup->tqg_flags & (TQG_ADJUSTED|TQG_NEED_ADJUST)) == TQG_NEED_ADJUST) { + qgroup->tqg_flags |= TQG_ADJUSTED; + error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri); + MPASS(error == 0); + } + mtx_unlock(&qgroup->tqg_lock); + + return (error); +} + struct taskqgroup * taskqgroup_create(char *name) { Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -185,6 +185,7 @@ uint16_t ifc_sysctl_ntxqs; uint16_t ifc_sysctl_nrxqs; uint16_t ifc_sysctl_qs_eq_override; + uint16_t ifc_cpuid_highest; qidx_t ifc_sysctl_ntxds[8]; qidx_t ifc_sysctl_nrxds[8]; @@ -203,8 +204,66 @@ eventhandler_tag ifc_vlan_detach_event; uint8_t ifc_mac[ETHER_ADDR_LEN]; char ifc_mtx_name[16]; + LIST_ENTRY(iflib_ctx) ifc_next; }; +static LIST_HEAD(ctx_head, iflib_ctx) ctx_list; +static struct mtx ctx_list_lock; + +TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); +TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); + +static void +iflib_ctx_apply(void (*fn)(if_ctx_t ctx, void *arg), void *arg) +{ + if_ctx_t ctx; + + mtx_lock(&ctx_list_lock); + LIST_FOREACH(ctx, &ctx_list, ifc_next) { + (fn)(ctx, arg); + } + mtx_unlock(&ctx_list_lock); +} + +static void +_iflib_cpuid_highest(if_ctx_t ctx, void *arg) { + int *cpuid = arg; + + if (*cpuid < ctx->ifc_cpuid_highest) + *cpuid = ctx->ifc_cpuid_highest; +} + +static int +iflib_cpuid_highest(void) +{ + int cpuid = 0; + + iflib_ctx_apply(_iflib_cpuid_highest, &cpuid); + return (cpuid); +} + +static void +iflib_ctx_insert(if_ctx_t ctx) +{ + mtx_lock(&ctx_list_lock); + LIST_INSERT_HEAD(&ctx_list, ctx, ifc_next); + mtx_unlock(&ctx_list_lock); +} + +static void +iflib_ctx_remove(if_ctx_t ctx) +{ + int max_cpuid_prev, max_cpuid_new; + + max_cpuid_prev = iflib_cpuid_highest(); + mtx_lock(&ctx_list_lock); + LIST_REMOVE(ctx, ifc_next); + mtx_unlock(&ctx_list_lock); + max_cpuid_new = max(1, iflib_cpuid_highest()); + if (max_cpuid_new < max_cpuid_prev) { + taskqgroup_adjust(qgroup_if_io, max_cpuid_new, 1, true, PI_NET); + } +} void * iflib_get_softc(if_ctx_t ctx) @@ -553,9 +612,6 @@ MODULE_DEPEND(iflib, pci, 1, 1, 1); MODULE_DEPEND(iflib, ether, 1, 1, 1); -TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); -TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); - #ifndef IFLIB_DEBUG_COUNTERS #ifdef INVARIANTS #define IFLIB_DEBUG_COUNTERS 1 @@ -4244,6 +4300,7 @@ if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); iflib_add_device_sysctl_post(ctx); + iflib_ctx_insert(ctx); ctx->ifc_flags |= IFC_INIT_DONE; return (0); fail_detach: @@ -4348,6 +4405,7 @@ iflib_rx_structures_free(ctx); if (ctx->ifc_flags & IFC_SC_ALLOCATED) free(ctx->ifc_softc, M_IFLIB); + iflib_ctx_remove(ctx); free(ctx, M_IFLIB); return (0); } @@ -4443,13 +4501,11 @@ * **********************************************************************/ -/* - * - Start a fast taskqueue thread for each core - * - Start a taskqueue for control operations - */ static int iflib_module_init(void) { + LIST_INIT(&ctx_list); + mtx_init(&ctx_list_lock, "ctx list", NULL, MTX_DEF); return (0); } @@ -4986,7 +5042,14 @@ if (tqrid != -1) { cpuid = find_nth(ctx, &cpus, qid); - taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name); + err = taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name); + if (err) { + device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err); + return (err); + } + if (cpuid > ctx->ifc_cpuid_highest) + ctx->ifc_cpuid_highest = cpuid; + MPASS(gtask->gt_taskqueue != NULL); } else { taskqgroup_attach(tqg, gtask, q, tqrid, name); } Index: sys/sys/gtaskqueue.h =================================================================== --- sys/sys/gtaskqueue.h +++ sys/sys/gtaskqueue.h @@ -59,6 +59,8 @@ struct taskqgroup *taskqgroup_create(char *name); void taskqgroup_destroy(struct taskqgroup *qgroup); int taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri); +int taskqgroup_adjust_once(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri); +void taskqgroup_set_adjust(struct taskqgroup *qgroup, void (*adjust_func)(void*)); #define TASK_ENQUEUED 0x1 #define TASK_SKIP_WAKEUP 0x2 @@ -86,22 +88,29 @@ struct taskqgroup *qgroup_##name; \ \ static void \ -taskqgroup_define_##name(void *arg) \ +taskqgroup_adjust_##name(void *arg) \ { \ - qgroup_##name = taskqgroup_create(#name); \ + int max = (intr) ? 1 : (cnt); \ + if (arg != NULL) { \ + uintptr_t maxcpu = (uintptr_t) arg; \ + max = maxcpu; \ + } \ + \ + taskqgroup_adjust_once(qgroup_##name, max, (stride), (intr), (pri)); \ } \ \ -SYSINIT(taskqgroup_##name, SI_SUB_TASKQ, SI_ORDER_FIRST, \ - taskqgroup_define_##name, NULL); \ +SYSINIT(taskqgroup_adj_##name, SI_SUB_SMP, SI_ORDER_ANY, \ + taskqgroup_adjust_##name, NULL); \ \ static void \ -taskqgroup_adjust_##name(void *arg) \ +taskqgroup_define_##name(void *arg) \ { \ - taskqgroup_adjust(qgroup_##name, (cnt), (stride), (intr), (pri)); \ + qgroup_##name = taskqgroup_create(#name); \ + taskqgroup_set_adjust(qgroup_##name, taskqgroup_adjust_##name); \ } \ - \ -SYSINIT(taskqgroup_adj_##name, SI_SUB_SMP, SI_ORDER_ANY, \ - taskqgroup_adjust_##name, NULL) +SYSINIT(taskqgroup_##name, SI_SUB_TASKQ, SI_ORDER_FIRST, \ + taskqgroup_define_##name, NULL) +