Differential D12142 Diff 32467 sys/kern/subr_gtaskqueue.c

Changeset View

Standalone View

sys/kern/subr_gtaskqueue.c

Show First 20 Lines • Show All 45 Lines • ▼ Show 20 Lines
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/gtaskqueue.h>		#include <sys/gtaskqueue.h>
#include <sys/unistd.h>		#include <sys/unistd.h>
#include <machine/stdarg.h>		#include <machine/stdarg.h>

static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues");		static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues");
static void gtaskqueue_thread_enqueue(void *);		static void gtaskqueue_thread_enqueue(void *);
static void gtaskqueue_thread_loop(void *arg);		static void gtaskqueue_thread_loop(void *arg);
		static int _taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri);
TASKQGROUP_DEFINE(softirq, mp_ncpus, 1, false, PI_SOFT);		TASKQGROUP_DEFINE(softirq, mp_ncpus, 1, false, PI_SOFT);

struct gtaskqueue_busy {		struct gtaskqueue_busy {
struct gtask *tb_running;		struct gtask *tb_running;
TAILQ_ENTRY(gtaskqueue_busy) tb_link;		TAILQ_ENTRY(gtaskqueue_busy) tb_link;
};		};

struct gt_intr_thread {		struct gt_intr_thread {
▲ Show 20 Lines • Show All 607 Lines • ▼ Show 20 Lines	struct taskqgroup_cpu {
struct gtaskqueue *tgc_taskq;		struct gtaskqueue *tgc_taskq;
int tgc_cnt;		int tgc_cnt;
int tgc_cpu;		int tgc_cpu;
};		};

struct taskqgroup {		struct taskqgroup {
struct taskqgroup_cpu tqg_queue[MAXCPU];		struct taskqgroup_cpu tqg_queue[MAXCPU];
struct mtx tqg_lock;		struct mtx tqg_lock;
		void (adjust_func)(void);
char * tqg_name;		char * tqg_name;
int tqg_adjusting;		int tqg_adjusting;
int tqg_stride;		int tqg_stride;
int tqg_cnt;		int tqg_cnt;
		int tqg_pri;
		int tqg_flags;
		bool tqg_intr;
};		};
		#define TQG_NEED_ADJUST 0x1
		#define TQG_ADJUSTED 0x2

struct taskq_bind_task {		struct taskq_bind_task {
struct gtask bt_task;		struct gtask bt_task;
int bt_cpuid;		int bt_cpuid;
};		};

static void		static void
taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu, bool intr, int pri)		taskqgroup_cpu_create(struct taskqgroup *qgroup, int idx, int cpu, bool intr, int pri)
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines
{		{
cpuset_t mask;		cpuset_t mask;
int qid;		int qid;

gtask->gt_uniq = uniq;		gtask->gt_uniq = uniq;
gtask->gt_name = name;		gtask->gt_name = name;
gtask->gt_irq = irq;		gtask->gt_irq = irq;
gtask->gt_cpu = -1;		gtask->gt_cpu = -1;

mtx_lock(&qgroup->tqg_lock);		mtx_lock(&qgroup->tqg_lock);
		qgroup->tqg_flags \|= TQG_NEED_ADJUST;
		mtx_unlock(&qgroup->tqg_lock);

		if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED))
		qgroup->adjust_func(NULL);

		mtx_lock(&qgroup->tqg_lock);
qid = taskqgroup_find(qgroup, uniq);		qid = taskqgroup_find(qgroup, uniq);
qgroup->tqg_queue[qid].tgc_cnt++;		qgroup->tqg_queue[qid].tgc_cnt++;
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);		LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;		gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;
if (irq != -1 && tqg_smp_started) {		if (irq != -1 && tqg_smp_started) {
gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu;		gtask->gt_cpu = qgroup->tqg_queue[qid].tgc_cpu;
CPU_ZERO(&mask);		CPU_ZERO(&mask);
CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask);		CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask);
Show All 30 Lines	taskqgroup_attach_deferred(struct taskqgroup qgroup, struct grouptask gtask)
mtx_unlock(&qgroup->tqg_lock);		mtx_unlock(&qgroup->tqg_lock);
}		}

int		int
taskqgroup_attach_cpu(struct taskqgroup qgroup, struct grouptask gtask,		taskqgroup_attach_cpu(struct taskqgroup qgroup, struct grouptask gtask,
void uniq, int cpu, int irq, char name)		void uniq, int cpu, int irq, char name)
{		{
cpuset_t mask;		cpuset_t mask;
int i, qid;		int i, err, qid, cpu_max;

qid = -1;		qid = -1;
gtask->gt_uniq = uniq;		gtask->gt_uniq = uniq;
gtask->gt_name = name;		gtask->gt_name = name;
gtask->gt_irq = irq;		gtask->gt_irq = irq;
gtask->gt_cpu = cpu;		gtask->gt_cpu = cpu;
		cpu_max = -1;
		MPASS(cpu >= 0);

mtx_lock(&qgroup->tqg_lock);		mtx_lock(&qgroup->tqg_lock);
		qgroup->tqg_flags \|= TQG_NEED_ADJUST;
		mtx_unlock(&qgroup->tqg_lock);

		if (tqg_smp_started && !(qgroup->tqg_flags & TQG_ADJUSTED)) {
		uintptr_t cpuid = cpu + 1;
		qgroup->adjust_func((void *)cpuid);
		}
		mtx_lock(&qgroup->tqg_lock);
if (tqg_smp_started) {		if (tqg_smp_started) {
		/* adjust as needed */
for (i = 0; i < qgroup->tqg_cnt; i++)		for (i = 0; i < qgroup->tqg_cnt; i++)
		if (qgroup->tqg_queue[i].tgc_cpu > cpu_max)
		cpu_max = qgroup->tqg_queue[i].tgc_cpu;
		MPASS(cpu <= mp_maxid);
		if (cpu > cpu_max) {
		err = _taskqgroup_adjust(qgroup, cpu + 1, qgroup->tqg_stride,
		qgroup->tqg_intr, qgroup->tqg_pri);
		if (err) {
		printf("_taskqgroup_adjust(%p, %d, %d, %d, %d) => %d\n\n",
		qgroup, cpu + 1, qgroup->tqg_stride, qgroup->tqg_intr, qgroup->tqg_pri,
		err);
		mtx_unlock(&qgroup->tqg_lock);
		return (err);
		}
		}
		for (i = 0; i < qgroup->tqg_cnt; i++)
		if (qgroup->tqg_queue[i].tgc_cpu > cpu_max)
		cpu_max = qgroup->tqg_queue[i].tgc_cpu;
		for (i = 0; i < qgroup->tqg_cnt; i++)
if (qgroup->tqg_queue[i].tgc_cpu == cpu) {		if (qgroup->tqg_queue[i].tgc_cpu == cpu) {
qid = i;		qid = i;
break;		break;
}		}
if (qid == -1) {		if (qid == -1) {
mtx_unlock(&qgroup->tqg_lock);		mtx_unlock(&qgroup->tqg_lock);
		printf("qid not found for cpu=%d\n", cpu);
return (EINVAL);		return (EINVAL);
}		}
} else		} else
qid = 0;		qid = 0;
qgroup->tqg_queue[qid].tgc_cnt++;		qgroup->tqg_queue[qid].tgc_cnt++;
LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);		LIST_INSERT_HEAD(&qgroup->tqg_queue[qid].tgc_tasks, gtask, gt_list);
gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;		gtask->gt_taskqueue = qgroup->tqg_queue[qid].tgc_taskq;
cpu = qgroup->tqg_queue[qid].tgc_cpu;		cpu = qgroup->tqg_queue[qid].tgc_cpu;
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	_taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri)

if (cnt < 1 \|\| cnt * stride > mp_ncpus \|\| !tqg_smp_started) {		if (cnt < 1 \|\| cnt * stride > mp_ncpus \|\| !tqg_smp_started) {
printf("%s: failed cnt: %d stride: %d "		printf("%s: failed cnt: %d stride: %d "
"mp_ncpus: %d tqg_smp_started: %d\n",		"mp_ncpus: %d tqg_smp_started: %d\n",
__func__, cnt, stride, mp_ncpus, tqg_smp_started);		__func__, cnt, stride, mp_ncpus, tqg_smp_started);
return (EINVAL);		return (EINVAL);
}		}
if (qgroup->tqg_adjusting) {		if (qgroup->tqg_adjusting) {
printf("taskqgroup_adjust failed: adjusting\n");		printf("%s: failed: adjusting\n", __func__);
return (EBUSY);		return (EBUSY);
}		}
		/* No work to be done */
		if (qgroup->tqg_cnt == cnt)
		return (0);
qgroup->tqg_adjusting = 1;		qgroup->tqg_adjusting = 1;
old_cnt = qgroup->tqg_cnt;		old_cnt = qgroup->tqg_cnt;
old_cpu = 0;		old_cpu = 0;
if (old_cnt < cnt)		if (old_cnt < cnt) {
old_cpu = qgroup->tqg_queue[old_cnt].tgc_cpu;		int old_max_idx = max(0, old_cnt-1);
		old_cpu = qgroup->tqg_queue[old_max_idx].tgc_cpu;
		if (old_cnt > 0)
		for (k = 0; k < stride; k++)
		old_cpu = CPU_NEXT(old_cpu);
		}
mtx_unlock(&qgroup->tqg_lock);		mtx_unlock(&qgroup->tqg_lock);
/*		/*
* Set up queue for tasks added before boot.		* Set up queue for tasks added before boot.
*/		*/
if (old_cnt == 0) {		if (old_cnt == 0) {
LIST_SWAP(&gtask_head, &qgroup->tqg_queue[0].tgc_tasks,		LIST_SWAP(&gtask_head, &qgroup->tqg_queue[0].tgc_tasks,
grouptask, gt_list);		grouptask, gt_list);
qgroup->tqg_queue[0].tgc_cnt = 0;		qgroup->tqg_queue[0].tgc_cnt = 0;
}		}

/*		/*
* If new taskq threads have been added.		* If new taskq threads have been added.
*/		*/
cpu = old_cpu;		cpu = old_cpu;
for (i = old_cnt; i < cnt; i++) {		for (i = old_cnt; i < cnt; i++) {
taskqgroup_cpu_create(qgroup, i, cpu, ithread, pri);		taskqgroup_cpu_create(qgroup, i, cpu, ithread, pri);

for (k = 0; k < stride; k++)		for (k = 0; k < stride; k++)
cpu = CPU_NEXT(cpu);		cpu = CPU_NEXT(cpu);
}		}
mtx_lock(&qgroup->tqg_lock);		mtx_lock(&qgroup->tqg_lock);
qgroup->tqg_cnt = cnt;		qgroup->tqg_cnt = cnt;
qgroup->tqg_stride = stride;		qgroup->tqg_stride = stride;
		qgroup->tqg_intr = ithread;
		qgroup->tqg_pri = pri;

/*		/*
* Adjust drivers to use new taskqs.		* Adjust drivers to use new taskqs.
*/		*/
for (i = 0; i < old_cnt; i++) {		for (i = 0; i < old_cnt; i++) {
while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) {		while ((gtask = LIST_FIRST(&qgroup->tqg_queue[i].tgc_tasks))) {
LIST_REMOVE(gtask, gt_list);		LIST_REMOVE(gtask, gt_list);
qgroup->tqg_queue[i].tgc_cnt--;		qgroup->tqg_queue[i].tgc_cnt--;
Show All 35 Lines

int		int
taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri)		taskqgroup_adjust(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri)
{		{
int error;		int error;

mtx_lock(&qgroup->tqg_lock);		mtx_lock(&qgroup->tqg_lock);
error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri);		error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri);
		mtx_unlock(&qgroup->tqg_lock);

		return (error);
		}

		void
		taskqgroup_set_adjust(struct taskqgroup qgroup, void (adjust_func)(void*))
		{
		qgroup-> adjust_func = adjust_func;
		}

		int
		taskqgroup_adjust_once(struct taskqgroup *qgroup, int cnt, int stride, bool ithread, int pri)
		{
		int error = 0;

		mtx_lock(&qgroup->tqg_lock);
		if ((qgroup->tqg_flags & (TQG_ADJUSTED\|TQG_NEED_ADJUST)) == TQG_NEED_ADJUST) {
		qgroup->tqg_flags \|= TQG_ADJUSTED;
		error = _taskqgroup_adjust(qgroup, cnt, stride, ithread, pri);
		MPASS(error == 0);
		}
mtx_unlock(&qgroup->tqg_lock);		mtx_unlock(&qgroup->tqg_lock);

return (error);		return (error);
}		}

struct taskqgroup *		struct taskqgroup *
taskqgroup_create(char *name)		taskqgroup_create(char *name)
{		{
Show All 15 Lines