diff --git a/include/os/freebsd/spl/sys/taskq.h b/include/os/freebsd/spl/sys/taskq.h index aaa435f2f372..200f9e838b8d 100644 --- a/include/os/freebsd/spl/sys/taskq.h +++ b/include/os/freebsd/spl/sys/taskq.h @@ -1,114 +1,115 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_TASKQ_H #define _SYS_TASKQ_H #include #include #include #include +#include #ifdef __cplusplus extern "C" { #endif #define TASKQ_NAMELEN 31 -struct taskqueue; -struct taskq { +typedef struct taskq { struct taskqueue *tq_queue; -}; +} taskq_t; -typedef struct taskq taskq_t; typedef uintptr_t taskqid_t; typedef void (task_func_t)(void *); typedef struct taskq_ent { struct task tqent_task; + struct timeout_task tqent_timeout_task; task_func_t *tqent_func; void *tqent_arg; - struct timeout_task tqent_timeout_task; - int tqent_type; - int tqent_gen; + taskqid_t tqent_id; + CK_LIST_ENTRY(taskq_ent) tqent_hash; + uint8_t tqent_type; + uint8_t tqent_registered; + uint8_t tqent_cancelled; + volatile uint32_t tqent_rc; } taskq_ent_t; -struct proc; - /* * Public flags for taskq_create(): bit range 0-15 */ #define TASKQ_PREPOPULATE 0x0001 /* Prepopulate with threads and data */ #define TASKQ_CPR_SAFE 0x0002 /* Use CPR safe protocol */ #define TASKQ_DYNAMIC 0x0004 /* Use dynamic thread scheduling */ #define TASKQ_THREADS_CPU_PCT 0x0008 /* number of threads as % of ncpu */ #define TASKQ_DC_BATCH 0x0010 /* Taskq uses SDC in batch mode */ /* * Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as * KM_SLEEP/KM_NOSLEEP. */ #define TQ_SLEEP 0x00 /* Can block for memory */ #define TQ_NOSLEEP 0x01 /* cannot block for memory; may fail */ #define TQ_NOQUEUE 0x02 /* Do not enqueue if can't dispatch */ #define TQ_NOALLOC 0x04 /* cannot allocate memory; may fail */ #define TQ_FRONT 0x08 /* Put task at the front of the queue */ #define TASKQID_INVALID ((taskqid_t)0) #define taskq_init_ent(x) extern taskq_t *system_taskq; /* Global dynamic task queue for long delay */ extern taskq_t *system_delay_taskq; extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t); extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *, uint_t, clock_t); extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t, taskq_ent_t *); extern int taskq_empty_ent(taskq_ent_t *); taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t); taskq_t *taskq_create_instance(const char *, int, int, pri_t, int, int, uint_t); taskq_t *taskq_create_proc(const char *, int, pri_t, int, int, struct proc *, uint_t); taskq_t *taskq_create_sysdc(const char *, int, int, int, struct proc *, uint_t, uint_t); void nulltask(void *); extern void taskq_destroy(taskq_t *); extern void taskq_wait_id(taskq_t *, taskqid_t); extern void taskq_wait_outstanding(taskq_t *, taskqid_t); extern void taskq_wait(taskq_t *); extern int taskq_cancel_id(taskq_t *, taskqid_t); extern int taskq_member(taskq_t *, kthread_t *); extern taskq_t *taskq_of_curthread(void); void taskq_suspend(taskq_t *); int taskq_suspended(taskq_t *); void taskq_resume(taskq_t *); #ifdef __cplusplus } #endif #endif /* _SYS_TASKQ_H */ diff --git a/module/os/freebsd/spl/spl_taskq.c b/module/os/freebsd/spl/spl_taskq.c index 28cebc5dcbde..f32dad2a2a89 100644 --- a/module/os/freebsd/spl/spl_taskq.c +++ b/module/os/freebsd/spl/spl_taskq.c @@ -1,330 +1,405 @@ /* * Copyright (c) 2009 Pawel Jakub Dawidek * All rights reserved. * * Copyright (c) 2012 Spectra Logic Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include +#include +#include #include static uint_t taskq_tsd; static uma_zone_t taskq_zone; taskq_t *system_taskq = NULL; taskq_t *system_delay_taskq = NULL; taskq_t *dynamic_taskq = NULL; extern int uma_align_cache; -#define TQ_MASK uma_align_cache -#define TQ_PTR_MASK ~uma_align_cache +static MALLOC_DEFINE(M_TASKQ, "taskq", "taskq structures"); -#define TIMEOUT_TASK 1 -#define NORMAL_TASK 2 +static CK_LIST_HEAD(tqenthashhead, taskq_ent) *tqenthashtbl; +static unsigned long tqenthash; +static unsigned long tqenthashlock; +static struct sx *tqenthashtbl_lock; -static int -taskqent_init(void *mem, int size, int flags) -{ - bzero(mem, sizeof (taskq_ent_t)); - return (0); -} +static uint32_t tqidnext = 1; -static int -taskqent_ctor(void *mem, int size, void *arg, int flags) -{ - return (0); -} - -static void -taskqent_dtor(void *mem, int size, void *arg) -{ - taskq_ent_t *ent = mem; +#define TQIDHASH(tqid) (&tqenthashtbl[(tqid) & tqenthash]) +#define TQIDHASHLOCK(tqid) (&tqenthashtbl_lock[((tqid) & tqenthashlock)]) - ent->tqent_gen = (ent->tqent_gen + 1) & TQ_MASK; -} +#define TIMEOUT_TASK 1 +#define NORMAL_TASK 2 static void system_taskq_init(void *arg) { + int i; tsd_create(&taskq_tsd, NULL); + tqenthashtbl = hashinit(mp_ncpus * 8, M_TASKQ, &tqenthash); + tqenthashlock = (tqenthash + 1) / 8; + if (tqenthashlock > 0) + tqenthashlock--; + tqenthashtbl_lock = + malloc(sizeof (*tqenthashtbl_lock) * (tqenthashlock + 1), + M_TASKQ, M_WAITOK | M_ZERO); + for (i = 0; i < tqenthashlock + 1; i++) + sx_init_flags(&tqenthashtbl_lock[i], "tqenthash", SX_DUPOK); + tqidnext = 1; taskq_zone = uma_zcreate("taskq_zone", sizeof (taskq_ent_t), - taskqent_ctor, taskqent_dtor, taskqent_init, NULL, - UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); + NULL, NULL, NULL, NULL, + UMA_ALIGN_CACHE, 0); system_taskq = taskq_create("system_taskq", mp_ncpus, minclsyspri, 0, 0, 0); system_delay_taskq = taskq_create("system_delay_taskq", mp_ncpus, minclsyspri, 0, 0, 0); } SYSINIT(system_taskq_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_init, NULL); static void system_taskq_fini(void *arg) { + int i; taskq_destroy(system_delay_taskq); taskq_destroy(system_taskq); uma_zdestroy(taskq_zone); tsd_destroy(&taskq_tsd); + for (i = 0; i < tqenthashlock + 1; i++) + sx_destroy(&tqenthashtbl_lock[i]); + for (i = 0; i < tqenthash + 1; i++) + VERIFY(CK_LIST_EMPTY(&tqenthashtbl[i])); + free(tqenthashtbl_lock, M_TASKQ); + free(tqenthashtbl, M_TASKQ); } SYSUNINIT(system_taskq_fini, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_fini, NULL); +static taskq_ent_t * +taskq_lookup(taskqid_t tqid) +{ + taskq_ent_t *ent = NULL; + + sx_xlock(TQIDHASHLOCK(tqid)); + CK_LIST_FOREACH(ent, TQIDHASH(tqid), tqent_hash) { + if (ent->tqent_id == tqid) + break; + } + if (ent != NULL) + refcount_acquire(&ent->tqent_rc); + sx_xunlock(TQIDHASHLOCK(tqid)); + return (ent); +} + +static taskqid_t +taskq_insert(taskq_ent_t *ent) +{ + taskqid_t tqid = atomic_fetchadd_int(&tqidnext, 1); + + ent->tqent_id = tqid; + ent->tqent_registered = B_TRUE; + sx_xlock(TQIDHASHLOCK(tqid)); + CK_LIST_INSERT_HEAD(TQIDHASH(tqid), ent, tqent_hash); + sx_xunlock(TQIDHASHLOCK(tqid)); + return (tqid); +} + +static void +taskq_remove(taskq_ent_t *ent) +{ + taskqid_t tqid = ent->tqent_id; + + if (!ent->tqent_registered) + return; + + sx_xlock(TQIDHASHLOCK(tqid)); + CK_LIST_REMOVE(ent, tqent_hash); + sx_xunlock(TQIDHASHLOCK(tqid)); + ent->tqent_registered = B_FALSE; +} + static void taskq_tsd_set(void *context) { taskq_t *tq = context; tsd_set(taskq_tsd, tq); } static taskq_t * taskq_create_with_init(const char *name, int nthreads, pri_t pri, int minalloc __unused, int maxalloc __unused, uint_t flags) { taskq_t *tq; if ((flags & TASKQ_THREADS_CPU_PCT) != 0) nthreads = MAX((mp_ncpus * nthreads) / 100, 1); tq = kmem_alloc(sizeof (*tq), KM_SLEEP); tq->tq_queue = taskqueue_create(name, M_WAITOK, taskqueue_thread_enqueue, &tq->tq_queue); taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_INIT, taskq_tsd_set, tq); taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN, taskq_tsd_set, NULL); (void) taskqueue_start_threads(&tq->tq_queue, nthreads, pri, "%s", name); return ((taskq_t *)tq); } taskq_t * taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused, int maxalloc __unused, uint_t flags) { return (taskq_create_with_init(name, nthreads, pri, minalloc, maxalloc, flags)); } taskq_t * taskq_create_proc(const char *name, int nthreads, pri_t pri, int minalloc, int maxalloc, proc_t *proc __unused, uint_t flags) { return (taskq_create_with_init(name, nthreads, pri, minalloc, maxalloc, flags)); } void taskq_destroy(taskq_t *tq) { taskqueue_free(tq->tq_queue); kmem_free(tq, sizeof (*tq)); } int taskq_member(taskq_t *tq, kthread_t *thread) { return (taskqueue_member(tq->tq_queue, thread)); } taskq_t * taskq_of_curthread(void) { return (tsd_get(taskq_tsd)); } +static void +taskq_free(taskq_ent_t *task) +{ + taskq_remove(task); + if (refcount_release(&task->tqent_rc)) + uma_zfree(taskq_zone, task); +} + int taskq_cancel_id(taskq_t *tq, taskqid_t tid) { uint32_t pend; int rc; - taskq_ent_t *ent = (void*)(tid & TQ_PTR_MASK); + taskq_ent_t *ent; - if (ent == NULL) + if (tid == 0) return (0); - if ((tid & TQ_MASK) != ent->tqent_gen) + + if ((ent = taskq_lookup(tid)) == NULL) return (0); + + ent->tqent_cancelled = B_TRUE; if (ent->tqent_type == TIMEOUT_TASK) { rc = taskqueue_cancel_timeout(tq->tq_queue, &ent->tqent_timeout_task, &pend); } else rc = taskqueue_cancel(tq->tq_queue, &ent->tqent_task, &pend); - if (rc == EBUSY) - taskq_wait_id(tq, tid); - else - uma_zfree(taskq_zone, ent); + if (rc == EBUSY) { + taskqueue_drain(tq->tq_queue, &ent->tqent_task); + } else if (pend) { + /* + * Tasks normally free themselves when run, but here the task + * was cancelled so it did not free itself. + */ + taskq_free(ent); + } + /* Free the extra reference we added with taskq_lookup. */ + taskq_free(ent); return (rc); } static void taskq_run(void *arg, int pending __unused) { taskq_ent_t *task = arg; - task->tqent_func(task->tqent_arg); - uma_zfree(taskq_zone, task); + if (!task->tqent_cancelled) + task->tqent_func(task->tqent_arg); + taskq_free(task); } taskqid_t taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg, uint_t flags, clock_t expire_time) { taskq_ent_t *task; taskqid_t tid; clock_t timo; int mflag; timo = expire_time - ddi_get_lbolt(); if (timo <= 0) return (taskq_dispatch(tq, func, arg, flags)); if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) mflag = M_WAITOK; else mflag = M_NOWAIT; task = uma_zalloc(taskq_zone, mflag); if (task == NULL) return (0); - tid = (uintptr_t)task; - MPASS((tid & TQ_MASK) == 0); task->tqent_func = func; task->tqent_arg = arg; task->tqent_type = TIMEOUT_TASK; - tid |= task->tqent_gen; + task->tqent_cancelled = B_FALSE; + refcount_init(&task->tqent_rc, 1); + tid = taskq_insert(task); TIMEOUT_TASK_INIT(tq->tq_queue, &task->tqent_timeout_task, 0, taskq_run, task); taskqueue_enqueue_timeout(tq->tq_queue, &task->tqent_timeout_task, timo); return (tid); } taskqid_t taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) { taskq_ent_t *task; int mflag, prio; taskqid_t tid; if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) mflag = M_WAITOK; else mflag = M_NOWAIT; /* * If TQ_FRONT is given, we want higher priority for this task, so it * can go at the front of the queue. */ prio = !!(flags & TQ_FRONT); task = uma_zalloc(taskq_zone, mflag); if (task == NULL) return (0); - - tid = (uintptr_t)task; - MPASS((tid & TQ_MASK) == 0); + refcount_init(&task->tqent_rc, 1); task->tqent_func = func; task->tqent_arg = arg; + task->tqent_cancelled = B_FALSE; task->tqent_type = NORMAL_TASK; + tid = taskq_insert(task); TASK_INIT(&task->tqent_task, prio, taskq_run, task); - tid |= task->tqent_gen; taskqueue_enqueue(tq->tq_queue, &task->tqent_task); + VERIFY(tid); return (tid); } static void taskq_run_ent(void *arg, int pending __unused) { taskq_ent_t *task = arg; task->tqent_func(task->tqent_arg); } void taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint32_t flags, taskq_ent_t *task) { int prio; /* * If TQ_FRONT is given, we want higher priority for this task, so it * can go at the front of the queue. */ prio = !!(flags & TQ_FRONT); - + task->tqent_cancelled = B_FALSE; + task->tqent_registered = B_FALSE; + task->tqent_id = 0; task->tqent_func = func; task->tqent_arg = arg; TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task); taskqueue_enqueue(tq->tq_queue, &task->tqent_task); } void taskq_wait(taskq_t *tq) { taskqueue_quiesce(tq->tq_queue); } void taskq_wait_id(taskq_t *tq, taskqid_t tid) { - taskq_ent_t *ent = (void*)(tid & TQ_PTR_MASK); + taskq_ent_t *ent; - if ((tid & TQ_MASK) != ent->tqent_gen) + if (tid == 0) + return; + if ((ent = taskq_lookup(tid)) == NULL) return; taskqueue_drain(tq->tq_queue, &ent->tqent_task); + taskq_free(ent); } void taskq_wait_outstanding(taskq_t *tq, taskqid_t id __unused) { taskqueue_drain_all(tq->tq_queue); } int taskq_empty_ent(taskq_ent_t *t) { return (t->tqent_task.ta_pending == 0); }