diff --git a/sys/compat/linuxkpi/common/include/linux/wait.h b/sys/compat/linuxkpi/common/include/linux/wait.h index bd496793e27e..03ddce2c06f5 100644 --- a/sys/compat/linuxkpi/common/include/linux/wait.h +++ b/sys/compat/linuxkpi/common/include/linux/wait.h @@ -1,311 +1,319 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. * Copyright (c) 2017 Mark Johnston * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _LINUXKPI_LINUX_WAIT_H_ #define _LINUXKPI_LINUX_WAIT_H_ #include #include #include #include #include #include #include #define SKIP_SLEEP() (SCHEDULER_STOPPED() || kdb_active) #define might_sleep() \ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "might_sleep()") #define might_sleep_if(cond) do { \ if (cond) { might_sleep(); } \ } while (0) struct wait_queue; struct wait_queue_head; #define wait_queue_entry wait_queue typedef struct wait_queue wait_queue_t; typedef struct wait_queue_entry wait_queue_entry_t; typedef struct wait_queue_head wait_queue_head_t; typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *); +#define WQ_FLAG_WOKEN 0x02 + /* * Many API consumers directly reference these fields and those of * wait_queue_head. */ struct wait_queue { - unsigned int flags; /* always 0 */ + unsigned int flags; void *private; wait_queue_func_t *func; union { struct list_head task_list; /* < v4.13 */ struct list_head entry; /* >= v4.13 */ }; }; struct wait_queue_head { spinlock_t lock; union { struct list_head task_list; /* < v4.13 */ struct list_head head; /* >= v4.13 */ }; }; /* * This function is referenced by at least one DRM driver, so it may not be * renamed and furthermore must be the default wait queue callback. */ -extern wait_queue_func_t autoremove_wake_function; -extern wait_queue_func_t default_wake_function; +wait_queue_func_t autoremove_wake_function; +wait_queue_func_t default_wake_function; +wait_queue_func_t woken_wake_function; + +long linux_wait_woken(wait_queue_t *wq, unsigned state, long timeout); + +#define wait_woken(wq, state, timeout) \ + linux_wait_woken((wq), (state), (timeout)) #define DEFINE_WAIT_FUNC(name, function) \ wait_queue_t name = { \ .private = current, \ .func = function, \ .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ } #define DEFINE_WAIT(name) \ DEFINE_WAIT_FUNC(name, autoremove_wake_function) #define DECLARE_WAITQUEUE(name, task) \ wait_queue_t name = { \ .private = task, \ .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ } #define DECLARE_WAIT_QUEUE_HEAD(name) \ wait_queue_head_t name = { \ .task_list = LINUX_LIST_HEAD_INIT(name.task_list), \ }; \ MTX_SYSINIT(name, &(name).lock, spin_lock_name("wqhead"), MTX_DEF) #define init_waitqueue_head(wqh) do { \ mtx_init(&(wqh)->lock, spin_lock_name("wqhead"), \ NULL, MTX_DEF | MTX_NEW | MTX_NOWITNESS); \ INIT_LIST_HEAD(&(wqh)->task_list); \ } while (0) #define __init_waitqueue_head(wqh, name, lk) init_waitqueue_head(wqh) void linux_init_wait_entry(wait_queue_t *, int); void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool); #define init_wait_entry(wq, flags) \ linux_init_wait_entry(wq, flags) #define wake_up(wqh) \ linux_wake_up(wqh, TASK_NORMAL, 1, false) #define wake_up_all(wqh) \ linux_wake_up(wqh, TASK_NORMAL, 0, false) #define wake_up_locked(wqh) \ linux_wake_up(wqh, TASK_NORMAL, 1, true) #define wake_up_all_locked(wqh) \ linux_wake_up(wqh, TASK_NORMAL, 0, true) #define wake_up_interruptible(wqh) \ linux_wake_up(wqh, TASK_INTERRUPTIBLE, 1, false) #define wake_up_interruptible_all(wqh) \ linux_wake_up(wqh, TASK_INTERRUPTIBLE, 0, false) int linux_wait_event_common(wait_queue_head_t *, wait_queue_t *, long, unsigned int, spinlock_t *); /* * Returns -ERESTARTSYS for a signal, 0 if cond is false after timeout, 1 if * cond is true after timeout, remaining jiffies (> 0) if cond is true before * timeout. */ #define __wait_event_common(wqh, cond, timeout, state, lock) ({ \ DEFINE_WAIT(__wq); \ const long __timeout = ((long)(timeout)) < 1 ? 1 : (timeout); \ long __start = jiffies; \ long __ret = 0; \ \ for (;;) { \ linux_prepare_to_wait(&(wqh), &__wq, state); \ if (cond) \ break; \ __ret = linux_wait_event_common(&(wqh), &__wq, \ __timeout, state, lock); \ if (__ret != 0) \ break; \ } \ linux_finish_wait(&(wqh), &__wq); \ if (__timeout != MAX_SCHEDULE_TIMEOUT) { \ if (__ret == -EWOULDBLOCK) \ __ret = !!(cond); \ else if (__ret != -ERESTARTSYS) { \ __ret = __timeout + __start - jiffies; \ /* range check return value */ \ if (__ret < 1) \ __ret = 1; \ else if (__ret > __timeout) \ __ret = __timeout; \ } \ } \ __ret; \ }) #define wait_event(wqh, cond) do { \ (void) __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_UNINTERRUPTIBLE, NULL); \ } while (0) #define wait_event_timeout(wqh, cond, timeout) ({ \ __wait_event_common(wqh, cond, timeout, TASK_UNINTERRUPTIBLE, \ NULL); \ }) #define wait_event_killable(wqh, cond) ({ \ __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_INTERRUPTIBLE, NULL); \ }) #define wait_event_interruptible(wqh, cond) ({ \ __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_INTERRUPTIBLE, NULL); \ }) #define wait_event_interruptible_timeout(wqh, cond, timeout) ({ \ __wait_event_common(wqh, cond, timeout, TASK_INTERRUPTIBLE, \ NULL); \ }) /* * Wait queue is already locked. */ #define wait_event_interruptible_locked(wqh, cond) ({ \ int __ret; \ \ spin_unlock(&(wqh).lock); \ __ret = __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_INTERRUPTIBLE, NULL); \ spin_lock(&(wqh).lock); \ __ret; \ }) /* * The passed spinlock is held when testing the condition. */ #define wait_event_interruptible_lock_irq(wqh, cond, lock) ({ \ __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_INTERRUPTIBLE, &(lock)); \ }) /* * The passed spinlock is held when testing the condition. */ #define wait_event_lock_irq(wqh, cond, lock) ({ \ __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ TASK_UNINTERRUPTIBLE, &(lock)); \ }) static inline void __add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) { list_add(&wq->task_list, &wqh->task_list); } static inline void add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) { spin_lock(&wqh->lock); __add_wait_queue(wqh, wq); spin_unlock(&wqh->lock); } static inline void __add_wait_queue_tail(wait_queue_head_t *wqh, wait_queue_t *wq) { list_add_tail(&wq->task_list, &wqh->task_list); } static inline void __add_wait_queue_entry_tail(wait_queue_head_t *wqh, wait_queue_entry_t *wq) { list_add_tail(&wq->entry, &wqh->head); } static inline void __remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) { list_del(&wq->task_list); } static inline void remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) { spin_lock(&wqh->lock); __remove_wait_queue(wqh, wq); spin_unlock(&wqh->lock); } bool linux_waitqueue_active(wait_queue_head_t *); #define waitqueue_active(wqh) linux_waitqueue_active(wqh) void linux_prepare_to_wait(wait_queue_head_t *, wait_queue_t *, int); void linux_finish_wait(wait_queue_head_t *, wait_queue_t *); #define prepare_to_wait(wqh, wq, state) linux_prepare_to_wait(wqh, wq, state) #define finish_wait(wqh, wq) linux_finish_wait(wqh, wq) void linux_wake_up_bit(void *, int); int linux_wait_on_bit_timeout(unsigned long *, int, unsigned int, long); void linux_wake_up_atomic_t(atomic_t *); int linux_wait_on_atomic_t(atomic_t *, unsigned int); #define wake_up_bit(word, bit) linux_wake_up_bit(word, bit) #define wait_on_bit(word, bit, state) \ linux_wait_on_bit_timeout(word, bit, state, MAX_SCHEDULE_TIMEOUT) #define wait_on_bit_timeout(word, bit, state, timeout) \ linux_wait_on_bit_timeout(word, bit, state, timeout) #define wake_up_atomic_t(a) linux_wake_up_atomic_t(a) /* * All existing callers have a cb that just schedule()s. To avoid adding * complexity, just emulate that internally. The prototype is different so that * callers must be manually modified; a cb that does something other than call * schedule() will require special treatment. */ #define wait_on_atomic_t(a, state) linux_wait_on_atomic_t(a, state) struct task_struct; bool linux_wake_up_state(struct task_struct *, unsigned int); #define wake_up_process(task) linux_wake_up_state(task, TASK_NORMAL) #define wake_up_state(task, state) linux_wake_up_state(task, state) #endif /* _LINUXKPI_LINUX_WAIT_H_ */ diff --git a/sys/compat/linuxkpi/common/src/linux_schedule.c b/sys/compat/linuxkpi/common/src/linux_schedule.c index 3f3605096d62..c6b7a2ebbd66 100644 --- a/sys/compat/linuxkpi/common/src/linux_schedule.c +++ b/sys/compat/linuxkpi/common/src/linux_schedule.c @@ -1,416 +1,475 @@ /*- * Copyright (c) 2017 Mark Johnston * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conds * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conds, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conds and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include /* * Convert a relative time in jiffies to a tick count, suitable for use with * native FreeBSD interfaces (callouts, sleepqueues, etc.). */ static int linux_jiffies_timeout_to_ticks(long timeout) { if (timeout < 1) return (1); else if (timeout == MAX_SCHEDULE_TIMEOUT) return (0); else if (timeout > INT_MAX) return (INT_MAX); else return (timeout); } static int linux_add_to_sleepqueue(void *wchan, struct task_struct *task, const char *wmesg, long timeout, int state) { int flags, ret, stimeout; MPASS((state & ~(TASK_PARKED | TASK_NORMAL)) == 0); flags = SLEEPQ_SLEEP | ((state & TASK_INTERRUPTIBLE) != 0 ? SLEEPQ_INTERRUPTIBLE : 0); stimeout = linux_jiffies_timeout_to_ticks(timeout); sleepq_add(wchan, NULL, wmesg, flags, 0); if (stimeout != 0) sleepq_set_timeout(wchan, stimeout); DROP_GIANT(); if ((state & TASK_INTERRUPTIBLE) != 0) { if (stimeout == 0) ret = -sleepq_wait_sig(wchan, 0); else ret = -sleepq_timedwait_sig(wchan, 0); } else { if (stimeout == 0) { sleepq_wait(wchan, 0); ret = 0; } else ret = -sleepq_timedwait(wchan, 0); } PICKUP_GIANT(); /* filter return value */ if (ret != 0 && ret != -EWOULDBLOCK) { linux_schedule_save_interrupt_value(task, ret); ret = -ERESTARTSYS; } return (ret); } unsigned int linux_msleep_interruptible(unsigned int ms) { int ret; /* guard against invalid values */ if (ms == 0) ms = 1; ret = -pause_sbt("lnxsleep", mstosbt(ms), 0, C_HARDCLOCK | C_CATCH); switch (ret) { case -EWOULDBLOCK: return (0); default: linux_schedule_save_interrupt_value(current, ret); return (ms); } } static int wake_up_task(struct task_struct *task, unsigned int state) { int ret; ret = 0; sleepq_lock(task); if ((atomic_read(&task->state) & state) != 0) { set_task_state(task, TASK_WAKING); sleepq_signal(task, SLEEPQ_SLEEP, 0, 0); ret = 1; } sleepq_release(task); return (ret); } bool linux_signal_pending(struct task_struct *task) { struct thread *td; sigset_t pending; td = task->task_thread; PROC_LOCK(td->td_proc); pending = td->td_siglist; SIGSETOR(pending, td->td_proc->p_siglist); SIGSETNAND(pending, td->td_sigmask); PROC_UNLOCK(td->td_proc); return (!SIGISEMPTY(pending)); } bool linux_fatal_signal_pending(struct task_struct *task) { struct thread *td; bool ret; td = task->task_thread; PROC_LOCK(td->td_proc); ret = SIGISMEMBER(td->td_siglist, SIGKILL) || SIGISMEMBER(td->td_proc->p_siglist, SIGKILL); PROC_UNLOCK(td->td_proc); return (ret); } bool linux_signal_pending_state(long state, struct task_struct *task) { MPASS((state & ~TASK_NORMAL) == 0); if ((state & TASK_INTERRUPTIBLE) == 0) return (false); return (linux_signal_pending(task)); } void linux_send_sig(int signo, struct task_struct *task) { struct thread *td; td = task->task_thread; PROC_LOCK(td->td_proc); tdsignal(td, signo); PROC_UNLOCK(td->td_proc); } int autoremove_wake_function(wait_queue_t *wq, unsigned int state, int flags, void *key __unused) { struct task_struct *task; int ret; task = wq->private; if ((ret = wake_up_task(task, state)) != 0) list_del_init(&wq->task_list); return (ret); } int default_wake_function(wait_queue_t *wq, unsigned int state, int flags, void *key __unused) { return (wake_up_task(wq->private, state)); } +long +linux_wait_woken(wait_queue_t *wq, unsigned state, long timeout) +{ + void *wchan; + struct task_struct *task; + int ret; + int remainder; + + task = current; + wchan = wq->private; + + remainder = jiffies + timeout; + + set_task_state(task, state); + + sleepq_lock(wchan); + if (!(wq->flags & WQ_FLAG_WOKEN)) { + ret = linux_add_to_sleepqueue(wchan, task, "woken", + timeout, state); + } else { + sleepq_release(wchan); + ret = 0; + } + + set_task_state(task, TASK_RUNNING); + wq->flags &= ~WQ_FLAG_WOKEN; + + if (timeout == MAX_SCHEDULE_TIMEOUT) + return (MAX_SCHEDULE_TIMEOUT); + + /* range check return value */ + remainder -= jiffies; + + /* range check return value */ + if (ret == -ERESTARTSYS && remainder < 1) + remainder = 1; + else if (remainder < 0) + remainder = 0; + else if (remainder > timeout) + remainder = timeout; + return (remainder); +} + +int +woken_wake_function(wait_queue_t *wq, unsigned int state, + int flags __unused, void *key __unused) +{ + void *wchan; + + wchan = wq->private; + + sleepq_lock(wchan); + wq->flags |= WQ_FLAG_WOKEN; + sleepq_signal(wchan, SLEEPQ_SLEEP, 0, 0); + sleepq_release(wchan); + + return (1); +} + void linux_init_wait_entry(wait_queue_t *wq, int flags) { memset(wq, 0, sizeof(*wq)); wq->flags = flags; wq->private = current; wq->func = autoremove_wake_function; INIT_LIST_HEAD(&wq->task_list); } void linux_wake_up(wait_queue_head_t *wqh, unsigned int state, int nr, bool locked) { wait_queue_t *pos, *next; if (!locked) spin_lock(&wqh->lock); list_for_each_entry_safe(pos, next, &wqh->task_list, task_list) { if (pos->func == NULL) { if (wake_up_task(pos->private, state) != 0 && --nr == 0) break; } else { if (pos->func(pos, state, 0, NULL) != 0 && --nr == 0) break; } } if (!locked) spin_unlock(&wqh->lock); } void linux_prepare_to_wait(wait_queue_head_t *wqh, wait_queue_t *wq, int state) { spin_lock(&wqh->lock); if (list_empty(&wq->task_list)) __add_wait_queue(wqh, wq); set_task_state(current, state); spin_unlock(&wqh->lock); } void linux_finish_wait(wait_queue_head_t *wqh, wait_queue_t *wq) { spin_lock(&wqh->lock); set_task_state(current, TASK_RUNNING); if (!list_empty(&wq->task_list)) { __remove_wait_queue(wqh, wq); INIT_LIST_HEAD(&wq->task_list); } spin_unlock(&wqh->lock); } bool linux_waitqueue_active(wait_queue_head_t *wqh) { bool ret; spin_lock(&wqh->lock); ret = !list_empty(&wqh->task_list); spin_unlock(&wqh->lock); return (ret); } int linux_wait_event_common(wait_queue_head_t *wqh, wait_queue_t *wq, long timeout, unsigned int state, spinlock_t *lock) { struct task_struct *task; int ret; if (lock != NULL) spin_unlock_irq(lock); task = current; sleepq_lock(task); if (atomic_read(&task->state) != TASK_WAKING) { ret = linux_add_to_sleepqueue(task, task, "wevent", timeout, state); } else { sleepq_release(task); ret = 0; } if (lock != NULL) spin_lock_irq(lock); return (ret); } long linux_schedule_timeout(long timeout) { struct task_struct *task; long remainder; int ret, state; task = current; remainder = jiffies + timeout; sleepq_lock(task); state = atomic_read(&task->state); if (state != TASK_WAKING) { ret = linux_add_to_sleepqueue(task, task, "sched", timeout, state); } else { sleepq_release(task); ret = 0; } set_task_state(task, TASK_RUNNING); if (timeout == MAX_SCHEDULE_TIMEOUT) return (MAX_SCHEDULE_TIMEOUT); /* range check return value */ remainder -= jiffies; /* range check return value */ if (ret == -ERESTARTSYS && remainder < 1) remainder = 1; else if (remainder < 0) remainder = 0; else if (remainder > timeout) remainder = timeout; return (remainder); } static void wake_up_sleepers(void *wchan) { sleepq_lock(wchan); sleepq_signal(wchan, SLEEPQ_SLEEP, 0, 0); sleepq_release(wchan); } #define bit_to_wchan(word, bit) ((void *)(((uintptr_t)(word) << 6) | (bit))) void linux_wake_up_bit(void *word, int bit) { wake_up_sleepers(bit_to_wchan(word, bit)); } int linux_wait_on_bit_timeout(unsigned long *word, int bit, unsigned int state, long timeout) { struct task_struct *task; void *wchan; int ret; task = current; wchan = bit_to_wchan(word, bit); for (;;) { sleepq_lock(wchan); if ((*word & (1 << bit)) == 0) { sleepq_release(wchan); ret = 0; break; } set_task_state(task, state); ret = linux_add_to_sleepqueue(wchan, task, "wbit", timeout, state); if (ret != 0) break; } set_task_state(task, TASK_RUNNING); return (ret); } void linux_wake_up_atomic_t(atomic_t *a) { wake_up_sleepers(a); } int linux_wait_on_atomic_t(atomic_t *a, unsigned int state) { struct task_struct *task; void *wchan; int ret; task = current; wchan = a; for (;;) { sleepq_lock(wchan); if (atomic_read(a) == 0) { sleepq_release(wchan); ret = 0; break; } set_task_state(task, state); ret = linux_add_to_sleepqueue(wchan, task, "watomic", 0, state); if (ret != 0) break; } set_task_state(task, TASK_RUNNING); return (ret); } bool linux_wake_up_state(struct task_struct *task, unsigned int state) { return (wake_up_task(task, state) != 0); }