Index: head/sys/kern/subr_gtaskqueue.c =================================================================== --- head/sys/kern/subr_gtaskqueue.c +++ head/sys/kern/subr_gtaskqueue.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -342,13 +343,16 @@ static void gtaskqueue_run_locked(struct gtaskqueue *queue) { + struct epoch_tracker et; struct gtaskqueue_busy tb; struct gtask *gtask; + bool in_net_epoch; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -358,11 +362,20 @@ TQ_UNLOCK(queue); KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(gtask)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(gtask)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } gtask->ta_func(gtask->ta_context); TQ_LOCK(queue); wakeup(gtask); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } Index: head/sys/kern/subr_taskqueue.c =================================================================== --- head/sys/kern/subr_taskqueue.c +++ head/sys/kern/subr_taskqueue.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -371,7 +372,7 @@ * anyway) so just insert it at tail while we have the * queue lock. */ - TASK_INIT(&t_barrier, USHRT_MAX, taskqueue_task_nop_fn, &t_barrier); + TASK_INIT(&t_barrier, UCHAR_MAX, taskqueue_task_nop_fn, &t_barrier); STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link); queue->tq_hint = &t_barrier; t_barrier.ta_pending = 1; @@ -442,14 +443,17 @@ static void taskqueue_run_locked(struct taskqueue *queue) { + struct epoch_tracker et; struct taskqueue_busy tb; struct task *task; + bool in_net_epoch; int pending; KASSERT(queue != NULL, ("tq is NULL")); TQ_ASSERT_LOCKED(queue); tb.tb_running = NULL; LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link); + in_net_epoch = false; while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) { STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link); @@ -462,11 +466,20 @@ TQ_UNLOCK(queue); KASSERT(task->ta_func != NULL, ("task->ta_func is NULL")); + if (!in_net_epoch && TASK_IS_NET(task)) { + in_net_epoch = true; + NET_EPOCH_ENTER(et); + } else if (in_net_epoch && !TASK_IS_NET(task)) { + NET_EPOCH_EXIT(et); + in_net_epoch = false; + } task->ta_func(task->ta_context, pending); TQ_LOCK(queue); wakeup(task); } + if (in_net_epoch) + NET_EPOCH_EXIT(et); LIST_REMOVE(&tb, tb_link); } Index: head/sys/sys/_task.h =================================================================== --- head/sys/sys/_task.h +++ head/sys/sys/_task.h @@ -48,10 +48,17 @@ struct task { STAILQ_ENTRY(task) ta_link; /* (q) link for queue */ uint16_t ta_pending; /* (q) count times queued */ - u_short ta_priority; /* (c) Priority */ + uint8_t ta_priority; /* (c) Priority */ + uint8_t ta_flags; /* (c) Flags */ task_fn_t *ta_func; /* (c) task handler */ void *ta_context; /* (c) argument for handler */ }; + +#define TASK_ENQUEUED 0x1 +#define TASK_NOENQUEUE 0x2 +#define TASK_NETWORK 0x4 + +#define TASK_IS_NET(ta) ((ta)->ta_flags & TASK_NETWORK) #ifdef _KERNEL Index: head/sys/sys/epoch.h =================================================================== --- head/sys/sys/epoch.h +++ head/sys/sys/epoch.h @@ -104,6 +104,9 @@ #define NET_EPOCH_WAIT() epoch_wait_preempt(net_epoch_preempt) #define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (f), (c)) #define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt)) +#define NET_TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, TASK_NETWORK) +#define NET_GROUPTASK_INIT(gtask, prio, func, ctx) \ + GTASK_INIT(&(gtask)->gt_task, TASK_NETWORK, (prio), (func), (ctx)) #endif /* _KERNEL */ #endif /* _SYS_EPOCH_H_ */ Index: head/sys/sys/gtaskqueue.h =================================================================== --- head/sys/sys/gtaskqueue.h +++ head/sys/sys/gtaskqueue.h @@ -84,10 +84,6 @@ gtask_fn_t *fn, const char *name); void taskqgroup_config_gtask_deinit(struct grouptask *gtask); -#define TASK_ENQUEUED 0x1 -#define TASK_SKIP_WAKEUP 0x2 -#define TASK_NOENQUEUE 0x4 - #define GTASK_INIT(gtask, flags, priority, func, context) do { \ (gtask)->ta_flags = flags; \ (gtask)->ta_priority = (priority); \ @@ -96,7 +92,7 @@ } while (0) #define GROUPTASK_INIT(gtask, priority, func, context) \ - GTASK_INIT(&(gtask)->gt_task, TASK_SKIP_WAKEUP, priority, func, context) + GTASK_INIT(&(gtask)->gt_task, 0, priority, func, context) #define GROUPTASK_ENQUEUE(gtask) \ grouptaskqueue_enqueue((gtask)->gt_taskqueue, &(gtask)->gt_task) Index: head/sys/sys/taskqueue.h =================================================================== --- head/sys/sys/taskqueue.h +++ head/sys/sys/taskqueue.h @@ -107,8 +107,7 @@ taskqueue_callback_fn callback, void *context); #define TASK_INITIALIZER(priority, func, context) \ - { .ta_pending = 0, \ - .ta_priority = (priority), \ + { .ta_priority = (priority), \ .ta_func = (func), \ .ta_context = (context) } @@ -121,18 +120,25 @@ /* * Initialise a task structure. */ -#define TASK_INIT(task, priority, func, context) do { \ - (task)->ta_pending = 0; \ - (task)->ta_priority = (priority); \ - (task)->ta_func = (func); \ - (task)->ta_context = (context); \ +#define TASK_INIT_FLAGS(task, priority, func, context, flags) do { \ + MPASS((priority) >= 0 && (priority) <= 255); \ + (task)->ta_pending = 0; \ + (task)->ta_priority = (priority); \ + (task)->ta_flags = (flags); \ + (task)->ta_func = (func); \ + (task)->ta_context = (context); \ } while (0) +#define TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, 0) + void _timeout_task_init(struct taskqueue *queue, struct timeout_task *timeout_task, int priority, task_fn_t func, void *context); -#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) \ - _timeout_task_init(queue, timeout_task, priority, func, context); +#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) do { \ + _Static_assert((priority) >= 0 && (priority) <= 255, \ + "struct task priority is 8 bit in size"); \ + _timeout_task_init(queue, timeout_task, priority, func, context); \ +} while (0) /* * Declare a reference to a taskqueue.