Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 4,980 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, | iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name) | driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name) | ||||
{ | { | ||||
return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name)); | return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name)); | ||||
} | } | ||||
#ifdef SMP | |||||
static int | static int | ||||
find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid) | find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid) | ||||
{ | { | ||||
int i, cpuid, eqid, count; | int i, cpuid, eqid, count; | ||||
CPU_COPY(&ctx->ifc_cpus, cpus); | CPU_COPY(&ctx->ifc_cpus, cpus); | ||||
count = CPU_COUNT(cpus); | count = CPU_COUNT(cpus); | ||||
eqid = qid % count; | eqid = qid % count; | ||||
/* clear up to the qid'th bit */ | /* clear up to the qid'th bit */ | ||||
for (i = 0; i < eqid; i++) { | for (i = 0; i < eqid; i++) { | ||||
cpuid = CPU_FFS(cpus); | cpuid = CPU_FFS(cpus); | ||||
MPASS(cpuid != 0); | MPASS(cpuid != 0); | ||||
CPU_CLR(cpuid-1, cpus); | CPU_CLR(cpuid-1, cpus); | ||||
} | } | ||||
cpuid = CPU_FFS(cpus); | cpuid = CPU_FFS(cpus); | ||||
MPASS(cpuid != 0); | MPASS(cpuid != 0); | ||||
return (cpuid-1); | return (cpuid-1); | ||||
} | } | ||||
static int | |||||
find_child_with_core(int cpu, struct cpu_group *grp) | |||||
{ | |||||
int i; | |||||
if (grp->cg_children == 0) | |||||
return -1; | |||||
MPASS(grp->cg_child); | |||||
for (i = 0; i < grp->cg_children; i++) { | |||||
if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) | |||||
return i; | |||||
} | |||||
return -1; | |||||
} | |||||
/* | |||||
* Find the nth thread on the specified core | |||||
*/ | |||||
static int | |||||
find_thread(int cpu, int thread_num) | |||||
{ | |||||
struct cpu_group *grp; | |||||
int i; | |||||
cpuset_t cs; | |||||
grp = smp_topo(); | |||||
if (grp == NULL) | |||||
return cpu; | |||||
i = 0; | |||||
while ((i = find_child_with_core(cpu, grp)) != -1) { | |||||
/* If the child only has one cpu, don't descend */ | |||||
if (grp->cg_child[i].cg_count <= 1) | |||||
break; | |||||
grp = &grp->cg_child[i]; | |||||
} | |||||
/* If they don't share at least an L2 cache, use the same CPU */ | |||||
if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) | |||||
return cpu; | |||||
/* Now pick one */ | |||||
CPU_COPY(&grp->cg_mask, &cs); | |||||
for (i = thread_num % grp->cg_count; i > 0; i--) { | |||||
MPASS(CPU_FFS(&cs)); | |||||
CPU_CLR(CPU_FFS(&cs) - 1, &cs); | |||||
} | |||||
MPASS(CPU_FFS(&cs)); | |||||
return CPU_FFS(&cs) - 1; | |||||
} | |||||
static int | |||||
get_thread_num(if_ctx_t ctx, iflib_intr_type_t type, int qid) | |||||
{ | |||||
switch (type) { | |||||
case IFLIB_INTR_TX: | |||||
/* TX queues get threads on the same core as the corresponding RX queue */ | |||||
/* XXX handle multiple RX threads per core and more than two threads per core */ | |||||
return qid / CPU_COUNT(&ctx->ifc_cpus) + 1; | |||||
case IFLIB_INTR_RX: | |||||
case IFLIB_INTR_RXTX: | |||||
/* RX queues get the first thread on their core */ | |||||
return qid / CPU_COUNT(&ctx->ifc_cpus); | |||||
default: | |||||
return -1; | |||||
} | |||||
} | |||||
#else | |||||
#define get_thread_num(ctx, type, qid) 0 | |||||
#define find_thread(cpuid, tid) 0 | |||||
#define find_nth(ctx, cpus, gid) 0 | |||||
#endif | |||||
/* Just to avoid copy/paste */ | |||||
static inline int | |||||
iflib_irq_set_affinity(if_ctx_t ctx, int irq, iflib_intr_type_t type, int qid, | |||||
struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, char *name) | |||||
{ | |||||
cpuset_t cpus; | |||||
int cpuid; | |||||
int err, tid; | |||||
cpuid = find_nth(ctx, &cpus, qid); | |||||
tid = get_thread_num(ctx, type, qid); | |||||
MPASS(tid >= 0); | |||||
cpuid = find_thread(cpuid, tid); | |||||
err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, irq, name); | |||||
if (err) { | |||||
device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err); | |||||
return (err); | |||||
} | |||||
#ifdef notyet | |||||
if (cpuid > ctx->ifc_cpuid_highest) | |||||
ctx->ifc_cpuid_highest = cpuid; | |||||
#endif | |||||
MPASS(gtask->gt_taskqueue != NULL); | |||||
return 0; | |||||
} | |||||
int | int | ||||
iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
iflib_intr_type_t type, driver_filter_t *filter, | iflib_intr_type_t type, driver_filter_t *filter, | ||||
void *filter_arg, int qid, char *name) | void *filter_arg, int qid, char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
iflib_filter_info_t info; | iflib_filter_info_t info; | ||||
cpuset_t cpus; | |||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
int tqrid, err, cpuid; | int tqrid, err; | ||||
driver_filter_t *intr_fast; | driver_filter_t *intr_fast; | ||||
void *q; | void *q; | ||||
info = &ctx->ifc_filter_info; | info = &ctx->ifc_filter_info; | ||||
tqrid = rid; | tqrid = rid; | ||||
switch (type) { | switch (type) { | ||||
/* XXX merge tx/rx for netmap? */ | /* XXX merge tx/rx for netmap? */ | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | ||||
return (err); | return (err); | ||||
} | } | ||||
if (type == IFLIB_INTR_ADMIN) | if (type == IFLIB_INTR_ADMIN) | ||||
return (0); | return (0); | ||||
if (tqrid != -1) { | if (tqrid != -1) { | ||||
cpuid = find_nth(ctx, &cpus, qid); | err = iflib_irq_set_affinity(ctx, rman_get_start(irq->ii_res), type, qid, gtask, tqg, q, name); | ||||
taskqgroup_attach_cpu(tqg, gtask, q, cpuid, rman_get_start(irq->ii_res), name); | if (err) | ||||
return (err); | |||||
} else { | } else { | ||||
taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, char *name) | iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
void *q; | void *q; | ||||
int irq_num = -1; | int irq_num = -1; | ||||
int err; | |||||
switch (type) { | switch (type) { | ||||
case IFLIB_INTR_TX: | case IFLIB_INTR_TX: | ||||
q = &ctx->ifc_txqs[qid]; | q = &ctx->ifc_txqs[qid]; | ||||
gtask = &ctx->ifc_txqs[qid].ift_task; | gtask = &ctx->ifc_txqs[qid].ift_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io_tqg; | ||||
fn = _task_fn_tx; | fn = _task_fn_tx; | ||||
if (irq != NULL) | if (irq != NULL) | ||||
Show All 12 Lines | case IFLIB_INTR_IOV: | ||||
gtask = &ctx->ifc_vflr_task; | gtask = &ctx->ifc_vflr_task; | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config_tqg; | ||||
fn = _task_fn_iov; | fn = _task_fn_iov; | ||||
break; | break; | ||||
default: | default: | ||||
panic("unknown net intr type"); | panic("unknown net intr type"); | ||||
} | } | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
if (irq_num != -1) { | |||||
err = iflib_irq_set_affinity(ctx, irq_num, type, qid, gtask, tqg, q, name); | |||||
if (err) | |||||
taskqgroup_attach(tqg, gtask, q, irq_num, name); | taskqgroup_attach(tqg, gtask, q, irq_num, name); | ||||
} | |||||
else { | |||||
taskqgroup_attach(tqg, gtask, q, irq_num, name); | |||||
} | |||||
} | } | ||||
void | void | ||||
iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | ||||
{ | { | ||||
if (irq->ii_tag) | if (irq->ii_tag) | ||||
bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | ||||
▲ Show 20 Lines • Show All 650 Lines • Show Last 20 Lines |