Index: sys/compat/linuxkpi/common/include/asm/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/asm/uaccess.h +++ sys/compat/linuxkpi/common/include/asm/uaccess.h @@ -52,7 +52,9 @@ #define __copy_from_user(...) copy_from_user(__VA_ARGS__) #define __copy_in_user(...) copy_from_user(__VA_ARGS__) -#define user_access_begin() do { } while (0) +#ifndef user_access_begin /* For backwards compatibility */ +#define user_access_begin(ptr, len) access_ok(ptr, len) +#endif #define user_access_end() do { } while (0) #define unsafe_get_user(x, ptr, err) do { \ Index: sys/compat/linuxkpi/common/include/linux/atomic.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/atomic.h +++ sys/compat/linuxkpi/common/include/linux/atomic.h @@ -31,5 +31,6 @@ #include #include +#include #endif /* _LINUX_ATOMIC_H_ */ Index: sys/compat/linuxkpi/common/include/linux/device.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/device.h +++ sys/compat/linuxkpi/common/include/linux/device.h @@ -61,6 +61,7 @@ }; struct dev_pm_ops { + int (*prepare)(struct device *dev); int (*suspend)(struct device *dev); int (*suspend_late)(struct device *dev); int (*resume)(struct device *dev); Index: sys/compat/linuxkpi/common/include/linux/dma-mapping.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -176,7 +176,7 @@ enum dma_data_direction dir, struct dma_attrs *attrs) { - return vtophys(ptr); + return (vtophys(ptr)); } static inline void @@ -217,7 +217,7 @@ unsigned long offset, size_t size, enum dma_data_direction direction) { - return VM_PAGE_TO_PHYS(page) + offset; + return (VM_PAGE_TO_PHYS(page) + offset); } static inline void @@ -236,6 +236,7 @@ dma_sync_single(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { + dma_sync_single_for_cpu(dev, addr, size, dir); } Index: sys/compat/linuxkpi/common/include/linux/fs.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/fs.h +++ sys/compat/linuxkpi/common/include/linux/fs.h @@ -323,4 +323,13 @@ #define shmem_truncate_range(...) \ linux_shmem_truncate_range(__VA_ARGS__) +static inline int +simple_open(struct inode *inode, struct file *file) +{ + + if (inode->i_private) + file->private_data = inode->i_private; + return (0); +} + #endif /* _LINUX_FS_H_ */ Index: sys/compat/linuxkpi/common/include/linux/interrupt.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/interrupt.h +++ sys/compat/linuxkpi/common/include/linux/interrupt.h @@ -189,11 +189,14 @@ struct tasklet_struct { TAILQ_ENTRY(tasklet_struct) entry; tasklet_func_t *func; + /* Our state implementation is different. Avoid same name as Linux. */ + volatile u_int _state; + atomic_t count; unsigned long data; }; #define DECLARE_TASKLET(name, func, data) \ -struct tasklet_struct name = { { NULL, NULL }, func, data } +struct tasklet_struct name = { { NULL, NULL }, func, ATOMIC_INIT(0), data } #define tasklet_hi_schedule(t) tasklet_schedule(t) @@ -202,6 +205,10 @@ extern void tasklet_init(struct tasklet_struct *, tasklet_func_t *, unsigned long data); extern void tasklet_enable(struct tasklet_struct *); +extern bool tasklet_is_enabled(struct tasklet_struct *); extern void tasklet_disable(struct tasklet_struct *); +extern int tasklet_trylock(struct tasklet_struct *); +extern void tasklet_unlock(struct tasklet_struct *); +extern void tasklet_unlock_wait(struct tasklet_struct *ts); #endif /* _LINUX_INTERRUPT_H_ */ Index: sys/compat/linuxkpi/common/include/linux/kernel.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/kernel.h +++ sys/compat/linuxkpi/common/include/linux/kernel.h @@ -130,9 +130,11 @@ #define ALIGN(x, y) roundup2((x), (y)) #undef PTR_ALIGN #define PTR_ALIGN(p, a) ((__typeof(p))ALIGN((uintptr_t)(p), (a))) +#define IS_ALIGNED(x, a) (((x) & ((__typeof(x))(a) - 1)) == 0) #define DIV_ROUND_UP(x, n) howmany(x, n) #define __KERNEL_DIV_ROUND_UP(x, n) howmany(x, n) #define DIV_ROUND_UP_ULL(x, n) DIV_ROUND_UP((unsigned long long)(x), (n)) +#define DIV_ROUND_DOWN_ULL(x, n) (((unsigned long long)(x) / (n)) * (n)) #define FIELD_SIZEOF(t, f) sizeof(((t *)0)->f) #define printk(...) printf(__VA_ARGS__) Index: sys/compat/linuxkpi/common/include/linux/ktime.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ktime.h +++ sys/compat/linuxkpi/common/include/linux/ktime.h @@ -35,8 +35,6 @@ #include #include -#define ktime_get_ts(x) getnanouptime(x) - /* time values in nanoseconds */ typedef s64 ktime_t; @@ -92,6 +90,13 @@ return (ktime_add_ns(kt, ms * NSEC_PER_MSEC)); } +static inline ktime_t +ktime_add_us(ktime_t kt, int64_t us) +{ + + return (ktime_add_ns(kt, us * NSEC_PER_USEC)); +} + static inline ktime_t ktime_sub_ns(ktime_t kt, int64_t ns) { @@ -172,18 +177,26 @@ return (ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC)); } +static inline int64_t +timespec64_to_ns(struct timespec64 *ts) +{ + return (timespec_to_ns(ts)); +} + #define ktime_to_timespec(kt) ns_to_timespec(kt) #define ktime_to_timespec64(kt) ns_to_timespec(kt) #define ktime_to_timeval(kt) ns_to_timeval(kt) #define ktime_to_ns(kt) (kt) -#define ktime_get_ts64(ts) ktime_get_ts(ts) +#define ktime_get_ts64(ts) getnanouptime(ts) +#define ktime_get_raw_ts64(ts) getnanouptime(ts) +#define getrawmonotonic64(ts) getnanouptime(ts) static inline int64_t ktime_get_ns(void) { struct timespec ts; - ktime_get_ts(&ts); + ktime_get_ts64(&ts); return (ktime_to_ns(timespec_to_ktime(ts))); } @@ -193,7 +206,7 @@ { struct timespec ts; - ktime_get_ts(&ts); + ktime_get_ts64(&ts); return (timespec_to_ktime(ts)); } @@ -229,7 +242,7 @@ { struct timespec ts; - nanotime(&ts); + nanouptime(&ts); return (timespec_to_ktime(ts)); } Index: sys/compat/linuxkpi/common/include/linux/mm_types.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/mm_types.h +++ sys/compat/linuxkpi/common/include/linux/mm_types.h @@ -35,6 +35,8 @@ #include +typedef int vm_fault_t; + struct vm_area_struct; struct task_struct; Index: sys/compat/linuxkpi/common/include/linux/pci.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/pci.h +++ sys/compat/linuxkpi/common/include/linux/pci.h @@ -535,6 +535,7 @@ int linux_pci_register_driver(struct pci_driver *pdrv); int linux_pci_register_drm_driver(struct pci_driver *pdrv); void linux_pci_unregister_driver(struct pci_driver *pdrv); +void linux_pci_unregister_drm_driver(struct pci_driver *pdrv); #define pci_register_driver(pdrv) linux_pci_register_driver(pdrv) #define pci_unregister_driver(pdrv) linux_pci_unregister_driver(pdrv) Index: sys/compat/linuxkpi/common/include/linux/preempt.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/preempt.h +++ sys/compat/linuxkpi/common/include/linux/preempt.h @@ -34,6 +34,9 @@ #define in_interrupt() \ (curthread->td_intr_nesting_level || curthread->td_critnest) +#define in_task() \ + (curthread->td_intr_nesting_level == 0 && curthread->td_critnest == 0) + #define preempt_disable() critical_enter() #define preempt_enable() critical_exit() Index: sys/compat/linuxkpi/common/include/linux/random.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/random.h +++ sys/compat/linuxkpi/common/include/linux/random.h @@ -35,6 +35,8 @@ #include #include +#define get_random_u32 get_random_int + static inline void get_random_bytes(void *buf, int nbytes) { Index: sys/compat/linuxkpi/common/include/linux/sched.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/sched.h +++ sys/compat/linuxkpi/common/include/linux/sched.h @@ -80,6 +80,7 @@ int rcu_recurse; int bsd_interrupt_value; struct work_struct *work; /* current work struct, if set */ + struct task_struct *group_leader; }; #define current ({ \ Index: sys/compat/linuxkpi/common/include/linux/timer.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/timer.h +++ sys/compat/linuxkpi/common/include/linux/timer.h @@ -81,8 +81,8 @@ extern void mod_timer(struct timer_list *, int); extern void add_timer(struct timer_list *); extern void add_timer_on(struct timer_list *, int cpu); +extern int del_timer(struct timer_list *); -#define del_timer(timer) (void)callout_stop(&(timer)->callout) #define del_timer_sync(timer) (void)callout_drain(&(timer)->callout) #define timer_pending(timer) callout_pending(&(timer)->callout) #define round_jiffies(j) \ Index: sys/compat/linuxkpi/common/include/linux/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/uaccess.h +++ sys/compat/linuxkpi/common/include/linux/uaccess.h @@ -60,12 +60,14 @@ #define get_user(_x, _p) linux_copyin((_p), &(_x), sizeof(*(_p))) #define put_user(_x, _p) __put_user(_x, _p) #define clear_user(...) linux_clear_user(__VA_ARGS__) +#ifndef access_ok /* For backwards compatibility */ #define access_ok(...) linux_access_ok(__VA_ARGS__) +#endif extern int linux_copyin(const void *uaddr, void *kaddr, size_t len); extern int linux_copyout(const void *kaddr, void *uaddr, size_t len); extern size_t linux_clear_user(void *uaddr, size_t len); -extern int linux_access_ok(int rw, const void *uaddr, size_t len); +extern int linux_access_ok(const void *uaddr, size_t len); /* * NOTE: Each pagefault_disable() call must have a corresponding Index: sys/compat/linuxkpi/common/include/linux/ww_mutex.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ww_mutex.h +++ sys/compat/linuxkpi/common/include/linux/ww_mutex.h @@ -45,6 +45,7 @@ struct ww_mutex { struct mutex base; struct cv condvar; + struct ww_acquire_ctx *ctx; }; #define DEFINE_WW_CLASS(name) \ Index: sys/compat/linuxkpi/common/src/linux_compat.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_compat.c +++ sys/compat/linuxkpi/common/src/linux_compat.c @@ -893,7 +893,7 @@ } int -linux_access_ok(int rw, const void *uaddr, size_t len) +linux_access_ok(const void *uaddr, size_t len) { uintptr_t saddr; uintptr_t eaddr; @@ -1902,6 +1902,15 @@ &linux_timer_callback_wrapper, timer, cpu); } +int +del_timer(struct timer_list *timer) +{ + + if (callout_stop(&(timer)->callout) == -1) + return (0); + return (1); +} + static void linux_timer_init(void *arg) { @@ -2328,7 +2337,7 @@ for (i = baseminor; i < baseminor + count; i++) { cdev = cdev_alloc(); - cdev_init(cdev, fops); + cdev->ops = fops; kobject_set_name(&cdev->kobj, name); ret = cdev_add(cdev, makedev(major, i), 1); @@ -2350,7 +2359,7 @@ for (i = baseminor; i < baseminor + count; i++) { cdev = cdev_alloc(); - cdev_init(cdev, fops); + cdev->ops = fops; kobject_set_name(&cdev->kobj, name); ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); Index: sys/compat/linuxkpi/common/src/linux_current.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_current.c +++ sys/compat/linuxkpi/common/src/linux_current.c @@ -67,6 +67,7 @@ ts->task_thread = td; ts->comm = td->td_name; ts->pid = td->td_tid; + ts->group_leader = ts; atomic_set(&ts->usage, 1); atomic_set(&ts->state, TASK_RUNNING); init_completion(&ts->parked); Index: sys/compat/linuxkpi/common/src/linux_pci.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_pci.c +++ sys/compat/linuxkpi/common/src/linux_pci.c @@ -330,3 +330,19 @@ devclass_delete_driver(bus, &pdrv->bsddriver); mtx_unlock(&Giant); } + +void +linux_pci_unregister_drm_driver(struct pci_driver *pdrv) +{ + devclass_t bus; + + bus = devclass_find("vgapci"); + + spin_lock(&pci_lock); + list_del(&pdrv->links); + spin_unlock(&pci_lock); + mtx_lock(&Giant); + if (bus != NULL) + devclass_delete_driver(bus, &pdrv->bsddriver); + mtx_unlock(&Giant); +} Index: sys/compat/linuxkpi/common/src/linux_tasklet.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_tasklet.c +++ sys/compat/linuxkpi/common/src/linux_tasklet.c @@ -41,20 +41,22 @@ #define TASKLET_ST_BUSY 1 #define TASKLET_ST_EXEC 2 #define TASKLET_ST_LOOP 3 -#define TASKLET_ST_PAUSED 4 #define TASKLET_ST_CMPSET(ts, old, new) \ - atomic_cmpset_ptr((volatile uintptr_t *)&(ts)->entry.tqe_prev, old, new) + atomic_cmpset_int((volatile u_int *)&(ts)->_state, old, new) #define TASKLET_ST_SET(ts, new) \ - WRITE_ONCE(*(volatile uintptr_t *)&(ts)->entry.tqe_prev, new) + WRITE_ONCE(*(volatile u_int *)&(ts)->_state, new) #define TASKLET_ST_GET(ts) \ - READ_ONCE(*(volatile uintptr_t *)&(ts)->entry.tqe_prev) + READ_ONCE(*(volatile u_int *)&(ts)->_state) + +#define TASKLET_ST_TESTANDSET(ts, new) \ + atomic_testandset_int((volatile u_int *)&(ts)->_state, new) struct tasklet_worker { struct mtx mtx; - TAILQ_HEAD(, tasklet_struct) head; + TAILQ_HEAD(tasklet_list, tasklet_struct) head; struct grouptask gtask; } __aligned(CACHE_LINE_SIZE); @@ -68,25 +70,34 @@ { struct tasklet_worker *tw = (struct tasklet_worker *)arg; struct tasklet_struct *ts; + struct tasklet_struct *last; linux_set_current(curthread); TASKLET_WORKER_LOCK(tw); + last = TAILQ_LAST(&tw->head, tasklet_list); while (1) { ts = TAILQ_FIRST(&tw->head); if (ts == NULL) break; TAILQ_REMOVE(&tw->head, ts, entry); - TASKLET_WORKER_UNLOCK(tw); - do { - /* reset executing state */ - TASKLET_ST_SET(ts, TASKLET_ST_EXEC); - - ts->func(ts->data); - - } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_IDLE) == 0); - TASKLET_WORKER_LOCK(tw); + if (!atomic_read(&ts->count)) { + TASKLET_WORKER_UNLOCK(tw); + do { + /* reset executing state */ + TASKLET_ST_SET(ts, TASKLET_ST_EXEC); + + ts->func(ts->data); + + } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, + TASKLET_ST_IDLE) == 0); + TASKLET_WORKER_LOCK(tw); + } else { + TAILQ_INSERT_TAIL(&tw->head, ts, entry); + } + if (ts == last) + break; } TASKLET_WORKER_UNLOCK(tw); } @@ -140,6 +151,8 @@ ts->entry.tqe_next = NULL; ts->func = func; ts->data = data; + atomic_set_int(&(ts)->_state, TASKLET_ST_IDLE); + atomic_set(&ts->count, 0); } void @@ -158,6 +171,10 @@ tasklet_schedule(struct tasklet_struct *ts) { + /* tasklet is paused */ + if (atomic_read(&ts->count)) + return; + if (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_LOOP)) { /* tasklet_handler() will loop */ } else if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_BUSY)) { @@ -201,17 +218,47 @@ void tasklet_enable(struct tasklet_struct *ts) { - (void) TASKLET_ST_CMPSET(ts, TASKLET_ST_PAUSED, TASKLET_ST_IDLE); + + atomic_dec(&ts->count); +} + +bool +tasklet_is_enabled(struct tasklet_struct *ts) +{ + + return !atomic_read(&ts->count); } void tasklet_disable(struct tasklet_struct *ts) { - while (1) { - if (TASKLET_ST_GET(ts) == TASKLET_ST_PAUSED) - break; - if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_PAUSED)) - break; + + atomic_inc(&ts->count); + tasklet_unlock_wait(ts); + mb(); +} + +int +tasklet_trylock(struct tasklet_struct *ts) +{ + + return (!TASKLET_ST_TESTANDSET(ts, TASKLET_ST_BUSY)); +} + +void +tasklet_unlock(struct tasklet_struct *ts) +{ + + TASKLET_ST_SET(ts, TASKLET_ST_IDLE); +} + +void +tasklet_unlock_wait(struct tasklet_struct *ts) +{ + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); + + /* wait until tasklet is no longer busy */ + while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) pause("W", 1); - } } Index: sys/dev/pms/freebsd/driver/common/lxencrypt.c =================================================================== --- sys/dev/pms/freebsd/driver/common/lxencrypt.c +++ sys/dev/pms/freebsd/driver/common/lxencrypt.c @@ -838,7 +838,7 @@ printf("%s: Add kek at index 0x%x wrapper 0x%x format 0x%x\n", __FUNCTION__, kek_add->kekIndex, kek_add->wrapperKekIndex, kek_add->blobFormat); /* Copy kek_blob from user pointer to local buffer */ - if(access_ok(VERIFY_READ, kek_add->EncryptKekBlob, sizeof(kek_blob))) { + if(access_ok(kek_add->EncryptKekBlob, sizeof(kek_blob))) { printf("%s: Starting copy from user %p to kernel %p\n", __FUNCTION__, kek_add->EncryptKekBlob, &kek_blob); if((rv = copy_from_user(&kek_blob, kek_add->EncryptKekBlob, sizeof(kek_blob))) != 0) { printf("%s: Copy error, %d left\n", __FUNCTION__, rv); @@ -873,7 +873,7 @@ printf("%s: Add dek at index 0x%x, table %x, kek index %x, blob format %x, entry size %x\n", __FUNCTION__, dek_index, dek_table, kek_index, blob_format, entry_sz); /* Copy dek_blob from user pointer to local buffer */ - if(access_ok(VERIFY_READ, dek_add->dekBlob, sizeof(dek_blob))) { + if(access_ok(dek_add->dekBlob, sizeof(dek_blob))) { printf("%s: Starting copy from user %p to kernel %p\n", __FUNCTION__, dek_add->dekBlob, &dek_blob); if((rv = copy_from_user(&dek_blob, dek_add->dekBlob, sizeof(dek_blob))) != 0) { printf("%s: Copy error, %d left\n", __FUNCTION__, rv); Index: sys/ofed/drivers/infiniband/core/ib_uverbs_main.c =================================================================== --- sys/ofed/drivers/infiniband/core/ib_uverbs_main.c +++ sys/ofed/drivers/infiniband/core/ib_uverbs_main.c @@ -856,8 +856,7 @@ goto out; } - if (!access_ok(VERIFY_WRITE, - (void __user *) (unsigned long) ex_hdr.response, + if (!access_ok((void __user *) (unsigned long) ex_hdr.response, (hdr.out_words + ex_hdr.provider_out_words) * 8)) { ret = -EFAULT; goto out;