Index: head/sys/compat/linuxkpi/common/include/linux/sched.h =================================================================== --- head/sys/compat/linuxkpi/common/include/linux/sched.h (revision 325707) +++ head/sys/compat/linuxkpi/common/include/linux/sched.h (revision 325708) @@ -1,164 +1,163 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2010 iX Systems, Inc. * Copyright (c) 2010 Panasas, Inc. * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LINUX_SCHED_H_ #define _LINUX_SCHED_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX_SCHEDULE_TIMEOUT INT_MAX #define TASK_RUNNING 0x0000 #define TASK_INTERRUPTIBLE 0x0001 #define TASK_UNINTERRUPTIBLE 0x0002 #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) #define TASK_WAKING 0x0100 #define TASK_PARKED 0x0200 #define TASK_COMM_LEN (MAXCOMLEN + 1) struct task_struct { struct thread *task_thread; struct mm_struct *mm; linux_task_fn_t *task_fn; void *task_data; int task_ret; atomic_t usage; - int state; + atomic_t state; atomic_t kthread_flags; pid_t pid; /* BSD thread ID */ const char *comm; void *bsd_ioctl_data; unsigned bsd_ioctl_len; struct completion parked; struct completion exited; TAILQ_ENTRY(task_struct) rcu_entry; int rcu_recurse; }; #define current ({ \ struct thread *__td = curthread; \ linux_set_current(__td); \ ((struct task_struct *)__td->td_lkpi_task); \ }) #define task_pid_group_leader(task) (task)->task_thread->td_proc->p_pid #define task_pid(task) ((task)->pid) #define task_pid_nr(task) ((task)->pid) #define get_pid(x) (x) #define put_pid(x) do { } while (0) #define current_euid() (curthread->td_ucred->cr_uid) -#define set_task_state(task, x) \ - atomic_store_rel_int((volatile int *)&task->state, (x)) -#define __set_task_state(task, x) (task->state = (x)) +#define set_task_state(task, x) atomic_set(&(task)->state, (x)) +#define __set_task_state(task, x) ((task)->state.counter = (x)) #define set_current_state(x) set_task_state(current, x) #define __set_current_state(x) __set_task_state(current, x) static inline void get_task_struct(struct task_struct *task) { atomic_inc(&task->usage); } static inline void put_task_struct(struct task_struct *task) { if (atomic_dec_and_test(&task->usage)) linux_free_current(task); } #define cond_resched() if (!cold) sched_relinquish(curthread) #define yield() kern_yield(PRI_UNCHANGED) #define sched_yield() sched_relinquish(curthread) #define need_resched() (curthread->td_flags & TDF_NEEDRESCHED) bool linux_signal_pending(struct task_struct *task); bool linux_fatal_signal_pending(struct task_struct *task); bool linux_signal_pending_state(long state, struct task_struct *task); void linux_send_sig(int signo, struct task_struct *task); #define signal_pending(task) linux_signal_pending(task) #define fatal_signal_pending(task) linux_fatal_signal_pending(task) #define signal_pending_state(state, task) \ linux_signal_pending_state(state, task) #define send_sig(signo, task, priv) do { \ CTASSERT(priv == 0); \ linux_send_sig(signo, task); \ } while (0) int linux_schedule_timeout(int timeout); #define schedule() \ (void)linux_schedule_timeout(MAX_SCHEDULE_TIMEOUT) #define schedule_timeout(timeout) \ linux_schedule_timeout(timeout) #define schedule_timeout_killable(timeout) \ schedule_timeout_uninterruptible(timeout) #define schedule_timeout_interruptible(timeout) ({ \ set_current_state(TASK_INTERRUPTIBLE); \ schedule_timeout(timeout); \ }) #define schedule_timeout_uninterruptible(timeout) ({ \ set_current_state(TASK_UNINTERRUPTIBLE); \ schedule_timeout(timeout); \ }) #define io_schedule() schedule() #define io_schedule_timeout(timeout) schedule_timeout(timeout) static inline uint64_t local_clock(void) { struct timespec ts; nanotime(&ts); return ((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec); } #endif /* _LINUX_SCHED_H_ */ Index: head/sys/compat/linuxkpi/common/src/linux_current.c =================================================================== --- head/sys/compat/linuxkpi/common/src/linux_current.c (revision 325707) +++ head/sys/compat/linuxkpi/common/src/linux_current.c (revision 325708) @@ -1,248 +1,248 @@ /*- * Copyright (c) 2017 Hans Petter Selasky * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include static eventhandler_tag linuxkpi_thread_dtor_tag; static MALLOC_DEFINE(M_LINUX_CURRENT, "linuxcurrent", "LinuxKPI task structure"); int linux_alloc_current(struct thread *td, int flags) { struct proc *proc; struct thread *td_other; struct task_struct *ts; struct task_struct *ts_other; struct mm_struct *mm; struct mm_struct *mm_other; MPASS(td->td_lkpi_task == NULL); ts = malloc(sizeof(*ts), M_LINUX_CURRENT, flags | M_ZERO); if (ts == NULL) return (ENOMEM); mm = malloc(sizeof(*mm), M_LINUX_CURRENT, flags | M_ZERO); if (mm == NULL) { free(ts, M_LINUX_CURRENT); return (ENOMEM); } /* setup new task structure */ atomic_set(&ts->kthread_flags, 0); ts->task_thread = td; ts->comm = td->td_name; ts->pid = td->td_tid; atomic_set(&ts->usage, 1); - ts->state = TASK_RUNNING; + atomic_set(&ts->state, TASK_RUNNING); init_completion(&ts->parked); init_completion(&ts->exited); proc = td->td_proc; /* check if another thread already has a mm_struct */ PROC_LOCK(proc); FOREACH_THREAD_IN_PROC(proc, td_other) { ts_other = td_other->td_lkpi_task; if (ts_other == NULL) continue; mm_other = ts_other->mm; if (mm_other == NULL) continue; /* try to share other mm_struct */ if (atomic_inc_not_zero(&mm_other->mm_users)) { /* set mm_struct pointer */ ts->mm = mm_other; break; } } /* use allocated mm_struct as a fallback */ if (ts->mm == NULL) { /* setup new mm_struct */ init_rwsem(&mm->mmap_sem); atomic_set(&mm->mm_count, 1); atomic_set(&mm->mm_users, 1); /* set mm_struct pointer */ ts->mm = mm; /* clear pointer to not free memory */ mm = NULL; } /* store pointer to task struct */ td->td_lkpi_task = ts; PROC_UNLOCK(proc); /* free mm_struct pointer, if any */ free(mm, M_LINUX_CURRENT); return (0); } struct mm_struct * linux_get_task_mm(struct task_struct *task) { struct mm_struct *mm; mm = task->mm; if (mm != NULL) { atomic_inc(&mm->mm_users); return (mm); } return (NULL); } void linux_mm_dtor(struct mm_struct *mm) { free(mm, M_LINUX_CURRENT); } void linux_free_current(struct task_struct *ts) { mmput(ts->mm); free(ts, M_LINUX_CURRENT); } static void linuxkpi_thread_dtor(void *arg __unused, struct thread *td) { struct task_struct *ts; ts = td->td_lkpi_task; if (ts == NULL) return; td->td_lkpi_task = NULL; put_task_struct(ts); } struct task_struct * linux_pid_task(pid_t pid) { struct thread *td; struct proc *p; /* try to find corresponding thread */ td = tdfind(pid, -1); if (td != NULL) { struct task_struct *ts = td->td_lkpi_task; PROC_UNLOCK(td->td_proc); return (ts); } /* try to find corresponding procedure */ p = pfind(pid); if (p != NULL) { FOREACH_THREAD_IN_PROC(p, td) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) { PROC_UNLOCK(p); return (ts); } } PROC_UNLOCK(p); } return (NULL); } struct task_struct * linux_get_pid_task(pid_t pid) { struct thread *td; struct proc *p; /* try to find corresponding thread */ td = tdfind(pid, -1); if (td != NULL) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) get_task_struct(ts); PROC_UNLOCK(td->td_proc); return (ts); } /* try to find corresponding procedure */ p = pfind(pid); if (p != NULL) { FOREACH_THREAD_IN_PROC(p, td) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) { get_task_struct(ts); PROC_UNLOCK(p); return (ts); } } PROC_UNLOCK(p); } return (NULL); } static void linux_current_init(void *arg __unused) { linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); } SYSINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_init, NULL); static void linux_current_uninit(void *arg __unused) { struct proc *p; struct task_struct *ts; struct thread *td; sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { PROC_LOCK(p); FOREACH_THREAD_IN_PROC(p, td) { if ((ts = td->td_lkpi_task) != NULL) { td->td_lkpi_task = NULL; put_task_struct(ts); } } PROC_UNLOCK(p); } sx_sunlock(&allproc_lock); EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); } SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_uninit, NULL); Index: head/sys/compat/linuxkpi/common/src/linux_schedule.c =================================================================== --- head/sys/compat/linuxkpi/common/src/linux_schedule.c (revision 325707) +++ head/sys/compat/linuxkpi/common/src/linux_schedule.c (revision 325708) @@ -1,393 +1,393 @@ /*- * Copyright (c) 2017 Mark Johnston * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conds * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conds, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conds and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include static int linux_add_to_sleepqueue(void *wchan, const char *wmesg, int timeout, int state) { int flags, ret; MPASS((state & ~TASK_NORMAL) == 0); flags = SLEEPQ_SLEEP | ((state & TASK_INTERRUPTIBLE) != 0 ? SLEEPQ_INTERRUPTIBLE : 0); sleepq_add(wchan, NULL, wmesg, flags, 0); if (timeout != 0) sleepq_set_timeout(wchan, timeout); if ((state & TASK_INTERRUPTIBLE) != 0) { if (timeout == 0) ret = -sleepq_wait_sig(wchan, 0); else ret = -sleepq_timedwait_sig(wchan, 0); } else { if (timeout == 0) { sleepq_wait(wchan, 0); ret = 0; } else ret = -sleepq_timedwait(wchan, 0); } /* filter return value */ if (ret != 0 && ret != -EWOULDBLOCK) ret = -ERESTARTSYS; return (ret); } static int wake_up_task(struct task_struct *task, unsigned int state) { int ret, wakeup_swapper; ret = wakeup_swapper = 0; sleepq_lock(task); - if ((atomic_load_acq_int(&task->state) & state) != 0) { + if ((atomic_read(&task->state) & state) != 0) { set_task_state(task, TASK_WAKING); wakeup_swapper = sleepq_signal(task, SLEEPQ_SLEEP, 0, 0); ret = 1; } sleepq_release(task); if (wakeup_swapper) kick_proc0(); return (ret); } bool linux_signal_pending(struct task_struct *task) { struct thread *td; sigset_t pending; td = task->task_thread; PROC_LOCK(td->td_proc); pending = td->td_siglist; SIGSETOR(pending, td->td_proc->p_siglist); SIGSETNAND(pending, td->td_sigmask); PROC_UNLOCK(td->td_proc); return (!SIGISEMPTY(pending)); } bool linux_fatal_signal_pending(struct task_struct *task) { struct thread *td; bool ret; td = task->task_thread; PROC_LOCK(td->td_proc); ret = SIGISMEMBER(td->td_siglist, SIGKILL) || SIGISMEMBER(td->td_proc->p_siglist, SIGKILL); PROC_UNLOCK(td->td_proc); return (ret); } bool linux_signal_pending_state(long state, struct task_struct *task) { MPASS((state & ~TASK_NORMAL) == 0); if ((state & TASK_INTERRUPTIBLE) == 0) return (false); return (linux_signal_pending(task)); } void linux_send_sig(int signo, struct task_struct *task) { struct thread *td; td = task->task_thread; PROC_LOCK(td->td_proc); tdsignal(td, signo); PROC_UNLOCK(td->td_proc); } int autoremove_wake_function(wait_queue_t *wq, unsigned int state, int flags, void *key __unused) { struct task_struct *task; int ret; task = wq->private; if ((ret = wake_up_task(task, state)) != 0) list_del_init(&wq->task_list); return (ret); } void linux_wake_up(wait_queue_head_t *wqh, unsigned int state, int nr, bool locked) { wait_queue_t *pos, *next; if (!locked) spin_lock(&wqh->lock); list_for_each_entry_safe(pos, next, &wqh->task_list, task_list) { if (pos->func == NULL) { if (wake_up_task(pos->private, state) != 0 && --nr == 0) break; } else { if (pos->func(pos, state, 0, NULL) != 0 && --nr == 0) break; } } if (!locked) spin_unlock(&wqh->lock); } void linux_prepare_to_wait(wait_queue_head_t *wqh, wait_queue_t *wq, int state) { spin_lock(&wqh->lock); if (list_empty(&wq->task_list)) __add_wait_queue(wqh, wq); set_task_state(current, state); spin_unlock(&wqh->lock); } void linux_finish_wait(wait_queue_head_t *wqh, wait_queue_t *wq) { spin_lock(&wqh->lock); set_task_state(current, TASK_RUNNING); if (!list_empty(&wq->task_list)) { __remove_wait_queue(wqh, wq); INIT_LIST_HEAD(&wq->task_list); } spin_unlock(&wqh->lock); } bool linux_waitqueue_active(wait_queue_head_t *wqh) { bool ret; spin_lock(&wqh->lock); ret = !list_empty(&wqh->task_list); spin_unlock(&wqh->lock); return (ret); } int linux_wait_event_common(wait_queue_head_t *wqh, wait_queue_t *wq, int timeout, unsigned int state, spinlock_t *lock) { struct task_struct *task; int ret; if (lock != NULL) spin_unlock_irq(lock); DROP_GIANT(); /* range check timeout */ if (timeout < 1) timeout = 1; else if (timeout == MAX_SCHEDULE_TIMEOUT) timeout = 0; task = current; /* * Our wait queue entry is on the stack - make sure it doesn't * get swapped out while we sleep. */ PHOLD(task->task_thread->td_proc); sleepq_lock(task); - if (atomic_load_acq_int(&task->state) != TASK_WAKING) { + if (atomic_read(&task->state) != TASK_WAKING) { ret = linux_add_to_sleepqueue(task, "wevent", timeout, state); } else { sleepq_release(task); ret = linux_signal_pending_state(state, task) ? -ERESTARTSYS : 0; } PRELE(task->task_thread->td_proc); PICKUP_GIANT(); if (lock != NULL) spin_lock_irq(lock); return (ret); } int linux_schedule_timeout(int timeout) { struct task_struct *task; int state; int remainder; task = current; /* range check timeout */ if (timeout < 1) timeout = 1; else if (timeout == MAX_SCHEDULE_TIMEOUT) timeout = 0; remainder = ticks + timeout; DROP_GIANT(); sleepq_lock(task); - state = atomic_load_acq_int(&task->state); + state = atomic_read(&task->state); if (state != TASK_WAKING) (void)linux_add_to_sleepqueue(task, "sched", timeout, state); else sleepq_release(task); set_task_state(task, TASK_RUNNING); PICKUP_GIANT(); if (timeout == 0) return (MAX_SCHEDULE_TIMEOUT); /* range check return value */ remainder -= ticks; if (remainder < 0) remainder = 0; else if (remainder > timeout) remainder = timeout; return (remainder); } static void wake_up_sleepers(void *wchan) { int wakeup_swapper; sleepq_lock(wchan); wakeup_swapper = sleepq_signal(wchan, SLEEPQ_SLEEP, 0, 0); sleepq_release(wchan); if (wakeup_swapper) kick_proc0(); } #define bit_to_wchan(word, bit) ((void *)(((uintptr_t)(word) << 6) | (bit))) void linux_wake_up_bit(void *word, int bit) { wake_up_sleepers(bit_to_wchan(word, bit)); } int linux_wait_on_bit_timeout(unsigned long *word, int bit, unsigned int state, int timeout) { struct task_struct *task; void *wchan; int ret; DROP_GIANT(); /* range check timeout */ if (timeout < 1) timeout = 1; else if (timeout == MAX_SCHEDULE_TIMEOUT) timeout = 0; task = current; wchan = bit_to_wchan(word, bit); for (;;) { sleepq_lock(wchan); if ((*word & (1 << bit)) == 0) { sleepq_release(wchan); ret = 0; break; } set_task_state(task, state); ret = linux_add_to_sleepqueue(wchan, "wbit", timeout, state); if (ret != 0) break; } set_task_state(task, TASK_RUNNING); PICKUP_GIANT(); return (ret); } void linux_wake_up_atomic_t(atomic_t *a) { wake_up_sleepers(a); } int linux_wait_on_atomic_t(atomic_t *a, unsigned int state) { struct task_struct *task; void *wchan; int ret; DROP_GIANT(); task = current; wchan = a; for (;;) { sleepq_lock(wchan); if (atomic_read(a) == 0) { sleepq_release(wchan); ret = 0; break; } set_task_state(task, state); ret = linux_add_to_sleepqueue(wchan, "watomic", 0, state); if (ret != 0) break; } set_task_state(task, TASK_RUNNING); PICKUP_GIANT(); return (ret); } bool linux_wake_up_state(struct task_struct *task, unsigned int state) { return (wake_up_task(task, state) != 0); }