Index: sys/compat/linuxkpi/common/include/asm/atomic.h =================================================================== --- sys/compat/linuxkpi/common/include/asm/atomic.h +++ sys/compat/linuxkpi/common/include/asm/atomic.h @@ -159,6 +159,23 @@ return (ret); } +static inline int +atomic_dec_if_positive(atomic_t *v) { + int c, old, dec; + + c = atomic_read(v); + for (;;) { + dec = c - 1; + if (unlikely(dec < 0)) + break; + old = atomic_cmpxchg((v), c, dec); + if (likely(old == c)) + break; + c = old; + } + return dec; +} + #define cmpxchg(ptr, old, new) ({ \ union { \ __typeof(*(ptr)) val; \ Index: sys/compat/linuxkpi/common/include/asm/msr.h =================================================================== --- sys/compat/linuxkpi/common/include/asm/msr.h +++ sys/compat/linuxkpi/common/include/asm/msr.h @@ -32,5 +32,6 @@ #include #define rdmsrl(msr, val) ((val) = rdmsr(msr)) +#define rdmsrl_safe(msr, val) rdmsr_safe((msr), (val)) #endif /* _ASM_MSR_H_ */ Index: sys/compat/linuxkpi/common/include/asm/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/asm/uaccess.h +++ sys/compat/linuxkpi/common/include/asm/uaccess.h @@ -31,25 +31,32 @@ #ifndef _ASM_UACCESS_H_ #define _ASM_UACCESS_H_ -#include +/* -static inline long -copy_to_user(void *to, const void *from, unsigned long n) -{ - if (linux_copyout(from, to, n) != 0) - return n; - return 0; -} -#define __copy_to_user(...) copy_to_user(__VA_ARGS__) +Disable. Need __{get,put}_user_size() functions... +Use defaults in linux/uaccess.h for now. -static inline long -copy_from_user(void *to, const void *from, unsigned long n) -{ - if (linux_copyin(from, to, n) != 0) - return n; - return 0; -} -#define __copy_from_user(...) copy_from_user(__VA_ARGS__) -#define __copy_in_user(...) copy_from_user(__VA_ARGS__) +#define user_access_begin() stac() +#define user_access_end() clac() + +#define unsafe_put_user(x, ptr, err_label) \ + do { \ + int __error; \ + __typeof__(*(ptr)) __x = (x); \ + __put_user_size(__x, (ptr), sizeof(*(ptr)), \ + __error, -EFAULT); \ + if (unlikely(__error)) goto err_label; \ + } while (0) + +#define unsafe_get_user(x, ptr, err_label) \ + do { \ + int __error; \ + __inttype(*(ptr)) __x; \ + __get_user_size(__x, (ptr), sizeof(*(ptr)), \ + __error, -EFAULT); \ + (x) = (__force __typeof__(*(ptr)))__x; \ + if (unlikely(__error)) goto err_label; \ + } while (0) +*/ #endif /* _ASM_UACCESS_H_ */ Index: sys/compat/linuxkpi/common/include/linux/bitmap.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/bitmap.h +++ sys/compat/linuxkpi/common/include/linux/bitmap.h @@ -243,6 +243,21 @@ return (1); } +static inline void +bitmap_complement(unsigned long *dst, const unsigned long *src, + unsigned size) +{ + const unsigned int end = BIT_WORD(size); + const unsigned int tail = size & (BITS_PER_LONG - 1); + unsigned int i; + + for (i = 0; i != end; i++) + dst[i] = ~src[i]; + + if (tail) + dst[i] = ~src[i]; +} + static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, const unsigned int size) Index: sys/compat/linuxkpi/common/include/linux/compiler.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/compiler.h +++ sys/compat/linuxkpi/common/include/linux/compiler.h @@ -34,6 +34,10 @@ #include +#ifndef __LinuxKPI_version +#define __LinuxKPI_version 40009 +#endif + #define __user #define __kernel #define __safe @@ -52,6 +56,7 @@ #define __devinitdata #define __deprecated #define __init +#define __initconst #define __devinit #define __devexit #define __exit Index: sys/compat/linuxkpi/common/include/linux/completion.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/completion.h +++ sys/compat/linuxkpi/common/include/linux/completion.h @@ -49,7 +49,7 @@ linux_complete_common((c), 1) #define wait_for_completion(c) \ linux_wait_for_common((c), 0) -#define wait_for_completion_interuptible(c) \ +#define wait_for_completion_interruptible(c) \ linux_wait_for_common((c), 1) #define wait_for_completion_timeout(c, timeout) \ linux_wait_for_timeout_common((c), (timeout), 0) Index: sys/compat/linuxkpi/common/include/linux/device.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/device.h +++ sys/compat/linuxkpi/common/include/linux/device.h @@ -461,6 +461,12 @@ device_unregister(device_get_softc(bsddev)); } +static inline void +dev_pm_set_driver_flags(struct device *dev, u32 flags) +{ + printf("%s: missing implementation\n", __func__); +} + static inline void linux_class_kfree(struct class *class) { Index: sys/compat/linuxkpi/common/include/linux/export.h =================================================================== --- /dev/null +++ sys/compat/linuxkpi/common/include/linux/export.h @@ -0,0 +1,7 @@ +#ifndef _LINUX_EXPORT_H +#define _LINUX_EXPORT_H + +#define EXPORT_SYMBOL(name) +#define EXPORT_SYMBOL_GPL(name) + +#endif Index: sys/compat/linuxkpi/common/include/linux/file.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/file.h +++ sys/compat/linuxkpi/common/include/linux/file.h @@ -186,5 +186,6 @@ #define file linux_file #define fget(...) linux_fget(__VA_ARGS__) +#define fcheck(...) linux_fget(__VA_ARGS__) #endif /* _LINUX_FILE_H_ */ Index: sys/compat/linuxkpi/common/include/linux/gfp.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/gfp.h +++ sys/compat/linuxkpi/common/include/linux/gfp.h @@ -49,6 +49,9 @@ #define __GFP_NORETRY 0 #define __GFP_RECLAIM 0 #define __GFP_RECLAIMABLE 0 +#define __GFP_RETRY_MAYFAIL 0 +#define __GFP_MOVABLE 0 +#define __GFP_COMP 0 #define __GFP_IO 0 #define __GFP_NO_KSWAPD 0 @@ -56,6 +59,7 @@ #define __GFP_DMA32 (1U << 24) /* LinuxKPI only */ #define __GFP_BITS_SHIFT 25 #define __GFP_BITS_MASK ((1 << __GFP_BITS_SHIFT) - 1) +#define __GFP_NOFAIL M_WAITOK #define GFP_NOWAIT M_NOWAIT #define GFP_ATOMIC (M_NOWAIT | M_USE_RESERVE) @@ -68,6 +72,7 @@ #define GFP_DMA32 __GFP_DMA32 #define GFP_TEMPORARY M_NOWAIT #define GFP_NATIVE_MASK (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_ZERO) +#define GFP_TRANSHUGE 0 CTASSERT((__GFP_DMA32 & GFP_NATIVE_MASK) == 0); CTASSERT((__GFP_BITS_MASK & GFP_NATIVE_MASK) == GFP_NATIVE_MASK); Index: sys/compat/linuxkpi/common/include/linux/hrtimer.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/hrtimer.h +++ sys/compat/linuxkpi/common/include/linux/hrtimer.h @@ -36,6 +36,7 @@ enum hrtimer_mode { HRTIMER_MODE_REL, + HRTIMER_MODE_REL_PINNED, }; enum hrtimer_restart { @@ -47,31 +48,42 @@ enum hrtimer_restart (*function)(struct hrtimer *); struct mtx mtx; struct callout callout; + s64 expires; /* relative time in nanoseconds */ + s64 precision; /* in nanoseconds */ }; #define hrtimer_active(hrtimer) linux_hrtimer_active(hrtimer) #define hrtimer_cancel(hrtimer) linux_hrtimer_cancel(hrtimer) + #define hrtimer_init(hrtimer, clock, mode) do { \ CTASSERT((clock) == CLOCK_MONOTONIC); \ CTASSERT((mode) == HRTIMER_MODE_REL); \ linux_hrtimer_init(hrtimer); \ } while (0) + #define hrtimer_set_expires(hrtimer, time) \ linux_hrtimer_set_expires(hrtimer, time) + #define hrtimer_start(hrtimer, time, mode) do { \ CTASSERT((mode) == HRTIMER_MODE_REL); \ linux_hrtimer_start(hrtimer, time); \ } while (0) + #define hrtimer_start_range_ns(hrtimer, time, prec, mode) do { \ CTASSERT((mode) == HRTIMER_MODE_REL); \ linux_hrtimer_start_range_ns(hrtimer, time, prec); \ } while (0) +#define hrtimer_forward_now(hrtimer, interval) do { \ + linux_hrtimer_forward_now(hrtimer, interval) \ +} while (0) + bool linux_hrtimer_active(struct hrtimer *); int linux_hrtimer_cancel(struct hrtimer *); void linux_hrtimer_init(struct hrtimer *); void linux_hrtimer_set_expires(struct hrtimer *, ktime_t); void linux_hrtimer_start(struct hrtimer *, ktime_t); void linux_hrtimer_start_range_ns(struct hrtimer *, ktime_t, int64_t); +void linux_hrtimer_forward_now(struct hrtimer *, ktime_t); #endif /* _LINUX_HRTIMER_H_ */ Index: sys/compat/linuxkpi/common/include/linux/idr.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/idr.h +++ sys/compat/linuxkpi/common/include/linux/idr.h @@ -79,11 +79,12 @@ void idr_preload_end(void); void *idr_find(struct idr *idp, int id); void *idr_get_next(struct idr *idp, int *nextid); +bool idr_is_empty(struct idr *idp); int idr_pre_get(struct idr *idp, gfp_t gfp_mask); int idr_get_new(struct idr *idp, void *ptr, int *id); int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id); void *idr_replace(struct idr *idp, void *ptr, int id); -void idr_remove(struct idr *idp, int id); +void *idr_remove(struct idr *idp, int id); void idr_remove_all(struct idr *idp); void idr_destroy(struct idr *idp); void idr_init(struct idr *idp); Index: sys/compat/linuxkpi/common/include/linux/kernel.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/kernel.h +++ sys/compat/linuxkpi/common/include/linux/kernel.h @@ -88,7 +88,9 @@ #define BUILD_BUG() do { CTASSERT(0); } while (0) #define BUILD_BUG_ON(x) CTASSERT(!(x)) +#ifndef BUILD_BUG_ON_MSG #define BUILD_BUG_ON_MSG(x, msg) BUILD_BUG_ON(x) +#endif #define BUILD_BUG_ON_NOT_POWER_OF_2(x) BUILD_BUG_ON(!powerof2(x)) #define BUILD_BUG_ON_INVALID(expr) while (0) { (void)(expr); } @@ -372,6 +374,56 @@ return (0); } +static inline int +kstrtobool(const char *s, bool *res) +{ + if (!s) + return -EINVAL; + + switch (s[0]) { + case 'y': + case 'Y': + case '1': + *res = true; + return 0; + case 'n': + case 'N': + case '0': + *res = false; + return 0; + case 'o': + case 'O': + switch (s[1]) { + case 'n': + case 'N': + *res = true; + return 0; + case 'f': + case 'F': + *res = false; + return 0; + default: + break; + } + default: + break; + } + + return -EINVAL; +} + +static inline int +kstrtobool_from_user(const char __user *s, size_t count, bool *res) +{ + char buf[4]; + + count = min(count, sizeof(buf) - 1); + if (copy_from_user(buf, s, count)) + return -EFAULT; + buf[count] = '\0'; + return kstrtobool(buf, res); +} + #define min(x, y) ((x) < (y) ? (x) : (y)) #define max(x, y) ((x) > (y) ? (x) : (y)) Index: sys/compat/linuxkpi/common/include/linux/ktime.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ktime.h +++ sys/compat/linuxkpi/common/include/linux/ktime.h @@ -127,6 +127,22 @@ return (lhs + rhs); } +static inline int +ktime_compare(const ktime_t cmp1, const ktime_t cmp2) +{ + if (cmp1 < cmp2) + return -1; + if (cmp1 > cmp2) + return 1; + return 0; +} + +static inline bool +ktime_after(const ktime_t cmp1, const ktime_t cmp2) +{ + return ktime_compare(cmp1, cmp2) > 0; +} + static inline ktime_t timespec_to_ktime(struct timespec ts) { @@ -140,6 +156,7 @@ } #define ktime_to_timespec(kt) ns_to_timespec(kt) +#define ktime_to_timespec64(kt) ns_to_timespec(kt) #define ktime_to_timeval(kt) ns_to_timeval(kt) #define ktime_to_ns(kt) (kt) Index: sys/compat/linuxkpi/common/include/linux/list.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/list.h +++ sys/compat/linuxkpi/common/include/linux/list.h @@ -337,16 +337,23 @@ hlist_empty(const struct hlist_head *h) { - return !h->first; + return !READ_ONCE(h->first); +} + +static inline void +__hlist_del(struct hlist_node *n) +{ + + WRITE_ONCE(*(n->pprev), n->next); + if (n->next) + n->next->pprev = n->pprev; } static inline void hlist_del(struct hlist_node *n) { - if (n->next) - n->next->pprev = n->pprev; - *n->pprev = n->next; + __hlist_del(n); } static inline void @@ -366,7 +373,7 @@ n->next = h->first; if (h->first) h->first->pprev = &n->next; - h->first = n; + WRITE_ONCE(h->first, n); n->pprev = &h->first; } @@ -377,18 +384,19 @@ n->pprev = next->pprev; n->next = next; next->pprev = &n->next; - *(n->pprev) = n; + WRITE_ONCE(*(n->pprev), n); } static inline void -hlist_add_after(struct hlist_node *n, struct hlist_node *next) +hlist_add_behind(struct hlist_node *n, struct hlist_node *prev) { - next->next = n->next; - n->next = next; - next->pprev = &n->next; - if (next->next) - next->next->pprev = &next->next; + n->next = prev->next; + WRITE_ONCE(prev->next, n); + n->pprev = &prev->next; + + if (n->next) + n->next->pprev = &n->next; } static inline void Index: sys/compat/linuxkpi/common/include/linux/math64.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/math64.h +++ sys/compat/linuxkpi/common/include/linux/math64.h @@ -84,4 +84,10 @@ return (dividend / divisor); } +static inline uint64_t +mul_u32_u32(uint32_t a, uint32_t b) +{ + return (uint64_t)a * b; +} + #endif /* _LINUX_MATH64_H */ Index: sys/compat/linuxkpi/common/include/linux/mm.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/mm.h +++ sys/compat/linuxkpi/common/include/linux/mm.h @@ -120,8 +120,8 @@ pgoff_t pgoff; union { /* user-space address */ - void *virtual_address; - unsigned long address; + void *virtual_address; /* < v4.11 */ + unsigned long address; /* >= v4.11 */ }; struct page *page; struct vm_area_struct *vma; @@ -131,6 +131,9 @@ void (*open) (struct vm_area_struct *); void (*close) (struct vm_area_struct *); int (*fault) (struct vm_area_struct *, struct vm_fault *); + int (*access) (struct vm_area_struct *, unsigned long, void *, + int, int); + }; /* Index: sys/compat/linuxkpi/common/include/linux/mmu_context.h =================================================================== --- /dev/null +++ sys/compat/linuxkpi/common/include/linux/mmu_context.h @@ -0,0 +1,20 @@ +#ifndef _LINUX_MMU_CONTEXT_H +#define _LINUX_MMU_CONTEXT_H + +struct mm_struct; + +static inline void +use_mm(struct mm_struct *mm) +{ + + printf("%s: missing implementation!\n", __func__); +} + +static inline void +unuse_mm(struct mm_struct *mm) +{ + + printf("%s: missing implementation!\n", __func__); +} + +#endif Index: sys/compat/linuxkpi/common/include/linux/module.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/module.h +++ sys/compat/linuxkpi/common/include/linux/module.h @@ -42,6 +42,7 @@ #include #include #include +#include #define MODULE_AUTHOR(name) #define MODULE_DESCRIPTION(name) @@ -51,8 +52,6 @@ #define THIS_MODULE ((struct module *)0) -#define EXPORT_SYMBOL(name) -#define EXPORT_SYMBOL_GPL(name) #define __MODULE_STRING(x) __stringify(x) /* OFED pre-module initialization */ Index: sys/compat/linuxkpi/common/include/linux/pci.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/pci.h +++ sys/compat/linuxkpi/common/include/linux/pci.h @@ -98,6 +98,7 @@ #define PCI_DEVFN(slot, func) ((((slot) & 0x1f) << 3) | ((func) & 0x07)) #define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) #define PCI_FUNC(devfn) ((devfn) & 0x07) +#define PCI_BUS_NUM(x) (((x) >> 8) & 0xff) #define PCI_VDEVICE(_vendor, _device) \ .vendor = PCI_VENDOR_ID_##_vendor, .device = (_device), \ Index: sys/compat/linuxkpi/common/include/linux/radix-tree.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/radix-tree.h +++ sys/compat/linuxkpi/common/include/linux/radix-tree.h @@ -56,6 +56,7 @@ struct radix_tree_iter { unsigned long index; + unsigned long next_index; }; #define RADIX_TREE_INIT(mask) \ @@ -79,5 +80,6 @@ void *radix_tree_delete(struct radix_tree_root *, unsigned long); int radix_tree_insert(struct radix_tree_root *, unsigned long, void *); bool radix_tree_iter_find(struct radix_tree_root *, struct radix_tree_iter *, void ***); +void radix_tree_iter_delete(struct radix_tree_root *, struct radix_tree_iter *, void **); #endif /* _LINUX_RADIX_TREE_H_ */ Index: sys/compat/linuxkpi/common/include/linux/scatterlist.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/scatterlist.h +++ sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -64,6 +64,8 @@ } internal; }; +#define SCATTERLIST_MAX_SEGMENT (UINT_MAX & PAGE_MASK) + #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) #define SG_MAGIC 0x87654321UL @@ -286,18 +288,26 @@ } static inline int -sg_alloc_table_from_pages(struct sg_table *sgt, +__sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int count, unsigned long off, unsigned long size, - gfp_t gfp_mask) + unsigned int max_segment, gfp_t gfp_mask) { - unsigned int i, segs, cur; + unsigned int i, segs, cur, len; int rc; struct scatterlist *s; + if (WARN_ON(!max_segment || offset_in_page(max_segment))) + return -EINVAL; + + len = 0; for (segs = i = 1; i < count; ++i) { - if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) + len += PAGE_SIZE; + if (len >= max_segment || + page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { ++segs; + len = 0; + } } if (__predict_false((rc = sg_alloc_table(sgt, segs, gfp_mask)))) return (rc); @@ -307,10 +317,13 @@ unsigned long seg_size; unsigned int j; - for (j = cur + 1; j < count; ++j) - if (page_to_pfn(pages[j]) != + len = 0; + for (j = cur + 1; j < count; ++j) { + len += PAGE_SIZE; + if (len >= max_segment || page_to_pfn(pages[j]) != page_to_pfn(pages[j - 1]) + 1) break; + } seg_size = ((j - cur) << PAGE_SHIFT) - off; sg_set_page(s, pages[cur], min(size, seg_size), off); @@ -321,6 +334,15 @@ return (0); } +static inline int +sg_alloc_table_from_pages(struct sg_table *sgt, + struct page **pages, unsigned int count, + unsigned long off, unsigned long size, + gfp_t gfp_mask) +{ + return __sg_alloc_table_from_pages(sgt, pages, count, off, size, + SCATTERLIST_MAX_SEGMENT, gfp_mask); +} static inline int sg_nents(struct scatterlist *sg) Index: sys/compat/linuxkpi/common/include/linux/sched.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/sched.h +++ sys/compat/linuxkpi/common/include/linux/sched.h @@ -89,6 +89,7 @@ #define task_pid_group_leader(task) (task)->task_thread->td_proc->p_pid #define task_pid(task) ((task)->pid) #define task_pid_nr(task) ((task)->pid) +#define task_pid_vnr(task) ((task)->pid) #define get_pid(x) (x) #define put_pid(x) do { } while (0) #define current_euid() (curthread->td_ucred->cr_uid) Index: sys/compat/linuxkpi/common/include/linux/slab.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/slab.h +++ sys/compat/linuxkpi/common/include/linux/slab.h @@ -85,6 +85,8 @@ #define SLAB_DESTROY_BY_RCU \ SLAB_TYPESAFE_BY_RCU +#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) + static inline gfp_t linux_check_m_flags(gfp_t flags) { @@ -131,6 +133,12 @@ return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); } +static inline void * +kvmalloc_array(size_t n, size_t size, gfp_t flags) +{ + return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); +} + static inline void * krealloc(void *ptr, size_t size, gfp_t flags) { Index: sys/compat/linuxkpi/common/include/linux/time.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/time.h +++ sys/compat/linuxkpi/common/include/linux/time.h @@ -38,6 +38,8 @@ #include #include +#define timespec64 timespec + static inline struct timeval ns_to_timeval(const int64_t nsec) { Index: sys/compat/linuxkpi/common/include/linux/timer.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/timer.h +++ sys/compat/linuxkpi/common/include/linux/timer.h @@ -39,8 +39,17 @@ struct timer_list { struct callout callout; +#if __LinuxKPI_version >= 40015 + void (*function) (struct timer_list *); + /* + * This field is temporary until we can get all users up to date + * and fix the callback routine in linux_compat.c + */ + struct timer_list *data; +#else void (*function) (unsigned long); unsigned long data; +#endif int expires; }; @@ -48,10 +57,26 @@ #define TIMER_IRQSAFE 0x0001 +#if __LinuxKPI_version >= 40015 + +#define from_timer(var, cb, field) \ + container_of(cb, typeof(*var), field) + +#define __timer_setup(timer, func) do { \ + (timer)->function = (func); \ + (timer)->data = (timer); \ + callout_init(&(timer)->callout, 1); \ +} while (0) + +#define timer_setup(timer, func, flags) \ + __timer_setup((timer), (func)) + +#else + #define setup_timer(timer, func, dat) do { \ (timer)->function = (func); \ (timer)->data = (dat); \ - callout_init(&(timer)->callout, 1); \ + callout_init(&(timer)->callout, 1); \ } while (0) #define __setup_timer(timer, func, dat, flags) do { \ @@ -62,9 +87,11 @@ #define init_timer(timer) do { \ (timer)->function = NULL; \ (timer)->data = 0; \ - callout_init(&(timer)->callout, 1); \ + callout_init(&(timer)->callout, 1); \ } while (0) +#endif + extern void mod_timer(struct timer_list *, int); extern void add_timer(struct timer_list *); extern void add_timer_on(struct timer_list *, int cpu); Index: sys/compat/linuxkpi/common/include/linux/types.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/types.h +++ sys/compat/linuxkpi/common/include/linux/types.h @@ -75,4 +75,6 @@ typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); typedef int linux_task_fn_t(void *data); +typedef size_t __kernel_size_t; + #endif /* _LINUX_TYPES_H_ */ Index: sys/compat/linuxkpi/common/include/linux/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/uaccess.h +++ sys/compat/linuxkpi/common/include/linux/uaccess.h @@ -42,6 +42,8 @@ #include +#include + #define VERIFY_READ VM_PROT_READ #define VERIFY_WRITE VM_PROT_WRITE @@ -67,6 +69,25 @@ extern size_t linux_clear_user(void *uaddr, size_t len); extern int linux_access_ok(int rw, const void *uaddr, size_t len); +static inline long +copy_to_user(void *to, const void *from, unsigned long n) +{ + if (linux_copyout(from, to, n) != 0) + return n; + return 0; +} +#define __copy_to_user(...) copy_to_user(__VA_ARGS__) + +static inline long +copy_from_user(void *to, const void *from, unsigned long n) +{ + if (linux_copyin(from, to, n) != 0) + return n; + return 0; +} +#define __copy_from_user(...) copy_from_user(__VA_ARGS__) +#define __copy_in_user(...) copy_from_user(__VA_ARGS__) + /* * NOTE: Each pagefault_disable() call must have a corresponding * pagefault_enable() call in the same scope. The former creates a new @@ -88,4 +109,20 @@ return ((curthread->td_pflags & TDP_NOFAULTING) != 0); } +#ifndef user_access_begin +#define user_access_begin() do { } while (0) +#define user_access_end() do { } while (0) + +#define unsafe_get_user(x, ptr, err) do { \ + if (unlikely(__get_user(x, ptr))) \ + goto err; \ + } while (0) + +#define unsafe_put_user(x, ptr, err) do { \ + if (unlikely(__put_user(x, ptr))) \ + goto err; \ + } while (0) + +#endif + #endif /* _LINUX_UACCESS_H_ */ Index: sys/compat/linuxkpi/common/include/linux/wait.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/wait.h +++ sys/compat/linuxkpi/common/include/linux/wait.h @@ -47,12 +47,19 @@ #define might_sleep() \ WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "might_sleep()") +#define might_sleep_if(cond) do { if (cond) might_sleep(); } while (0) + +#if __LinuxKPI_version >= 40013 +#define __add_wait_queue_entry_tail(wqh,wq) __add_wait_queue_tail((wqh),(wq)) +#define wait_queue_entry wait_queue +#define wait_queue_entry_t wait_queue_t +#endif + struct wait_queue; struct wait_queue_head; typedef struct wait_queue wait_queue_t; typedef struct wait_queue_head wait_queue_head_t; - typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *); /* @@ -63,12 +70,23 @@ unsigned int flags; /* always 0 */ void *private; wait_queue_func_t *func; - struct list_head task_list; + union { + /* + * Changes in Linux v4.13. + * Remove deprecated code when we + * don't depend on < 4.13 any more. + */ + struct list_head task_list; /* < v.4.13 */ + struct list_head entry; /* >= v4.13 */ + }; }; struct wait_queue_head { spinlock_t lock; - struct list_head task_list; + union { + struct list_head task_list; /* < v.4.13 */ + struct list_head head; /* >= v4.13 */ + }; }; /* @@ -106,8 +124,12 @@ INIT_LIST_HEAD(&(wqh)->task_list); \ } while (0) +void linux_init_wait_entry(wait_queue_t *wait, int flags); void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool); +#define init_wait_entry(w, f) \ + linux_init_wait_entry(w, f) + #define wake_up(wqh) \ linux_wake_up(wqh, TASK_NORMAL, 1, false) #define wake_up_all(wqh) \ Index: sys/compat/linuxkpi/common/include/linux/workqueue.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/workqueue.h +++ sys/compat/linuxkpi/common/include/linux/workqueue.h @@ -117,6 +117,9 @@ #define INIT_DELAYED_WORK(dwork, fn) \ linux_init_delayed_work(dwork, fn) +#define INIT_DELAYED_WORK_ONSTACK(dwork, fn) \ + linux_init_delayed_work(dwork, fn) + #define INIT_DEFERRABLE_WORK(dwork, fn) \ INIT_DELAYED_WORK(dwork, fn) @@ -211,6 +214,7 @@ extern struct workqueue_struct *system_wq; extern struct workqueue_struct *system_long_wq; extern struct workqueue_struct *system_unbound_wq; +extern struct workqueue_struct *system_highpri_wq; extern struct workqueue_struct *system_power_efficient_wq; extern void linux_init_delayed_work(struct delayed_work *, work_func_t); Index: sys/compat/linuxkpi/common/include/linux/ww_mutex.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ww_mutex.h +++ sys/compat/linuxkpi/common/include/linux/ww_mutex.h @@ -35,6 +35,8 @@ #include +struct task_struct; + struct ww_class { const char *mutex_name; }; @@ -44,6 +46,7 @@ struct ww_mutex { struct mutex base; + struct ww_acquire_ctx *ctx; struct cv condvar; }; @@ -111,32 +114,35 @@ } static inline void -ww_mutex_destroy(struct ww_mutex *lock) +ww_mutex_init(struct ww_mutex *lock, struct ww_class *ww_class) { - cv_destroy(&lock->condvar); - mutex_destroy(&lock->base); + linux_mutex_init(&lock->base, ww_class->mutex_name, SX_NOWITNESS); + cv_init(&lock->condvar, "lkpi-ww"); } static inline void -ww_acquire_init(struct ww_acquire_ctx *ctx, struct ww_class *ww_class) +ww_mutex_destroy(struct ww_mutex *lock) { + cv_destroy(&lock->condvar); + mutex_destroy(&lock->base); } static inline void -ww_mutex_init(struct ww_mutex *lock, struct ww_class *ww_class) +ww_acquire_init(struct ww_acquire_ctx *ctx, struct ww_class *ww_class) { - linux_mutex_init(&lock->base, ww_class->mutex_name, SX_NOWITNESS); - cv_init(&lock->condvar, "lkpi-ww"); + // NOP is deliberate } static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) { + // NOP is deliberate } static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) { + // NOP is deliberate } #endif /* _LINUX_WW_MUTEX_H_ */ Index: sys/compat/linuxkpi/common/src/linux_hrtimer.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_hrtimer.c +++ sys/compat/linuxkpi/common/src/linux_hrtimer.c @@ -44,8 +44,13 @@ hrtimer = arg; ret = hrtimer->function(hrtimer); - MPASS(ret == HRTIMER_NORESTART); - callout_deactivate(&hrtimer->callout); + + if (ret == HRTIMER_RESTART) { + callout_schedule_sbt(&hrtimer->callout, + nstosbt(hrtimer->expires), nstosbt(hrtimer->precision), 0); + } else { + callout_deactivate(&hrtimer->callout); + } } bool @@ -75,14 +80,17 @@ { hrtimer->function = NULL; + hrtimer->expires = 0; + hrtimer->precision = 0; mtx_init(&hrtimer->mtx, "hrtimer", NULL, MTX_DEF | MTX_RECURSE); callout_init_mtx(&hrtimer->callout, &hrtimer->mtx, 0); } void -linux_hrtimer_set_expires(struct hrtimer *hrtimer __unused, - ktime_t time __unused) +linux_hrtimer_set_expires(struct hrtimer *hrtimer, ktime_t time) { + + hrtimer->expires = ktime_to_ns(time); } void @@ -93,11 +101,23 @@ } void -linux_hrtimer_start_range_ns(struct hrtimer *hrtimer, ktime_t time, int64_t nsec) +linux_hrtimer_start_range_ns(struct hrtimer *hrtimer, ktime_t time, + int64_t nsec) +{ + + mtx_lock(&hrtimer->mtx); + hrtimer->precision = nsec; + callout_reset_sbt(&hrtimer->callout, nstosbt(ktime_to_ns(time)), + nstosbt(nsec), hrtimer_call_handler, hrtimer, 0); + mtx_unlock(&hrtimer->mtx); +} + +void +linux_hrtimer_forward_now(struct hrtimer *hrtimer, ktime_t interval) { mtx_lock(&hrtimer->mtx); - callout_reset_sbt(&hrtimer->callout, nstosbt(time), nstosbt(nsec), - hrtimer_call_handler, hrtimer, 0); + callout_reset_sbt(&hrtimer->callout, nstosbt(ktime_to_ns(interval)), + nstosbt(hrtimer->precision), hrtimer_call_handler, hrtimer, 0); mtx_unlock(&hrtimer->mtx); } Index: sys/compat/linuxkpi/common/src/linux_idr.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_idr.c +++ sys/compat/linuxkpi/common/src/linux_idr.c @@ -218,10 +218,11 @@ mtx_unlock(&idr->lock); } -static void +static void * idr_remove_locked(struct idr *idr, int id) { struct idr_layer *il; + void *res; int layer; int idx; @@ -229,7 +230,7 @@ il = idr->top; layer = idr->layers - 1; if (il == NULL || id > idr_max(idr)) - return; + return NULL; /* * Walk down the tree to this item setting bitmaps along the way * as we know at least one item will be free along this path. @@ -240,6 +241,7 @@ il = il->ary[idx]; layer--; } + res = il->ary[idx]; idx = id & IDR_MASK; /* * At this point we've set free space bitmaps up the whole tree. @@ -251,14 +253,20 @@ id, idr, il); il->ary[idx] = NULL; il->bitmap |= 1 << idx; + + return (res); } -void +void * idr_remove(struct idr *idr, int id) { + void *res; + mtx_lock(&idr->lock); - idr_remove_locked(idr, id); + res = idr_remove_locked(idr, id); mtx_unlock(&idr->lock); + + return (res); } @@ -712,9 +720,24 @@ int idr_for_each(struct idr *idp, int (*f)(int id, void *p, void *data), void *data) { + return (idr_for_each_layer(idp->top, 0, idp->layers - 1, f, data)); } +static int +idr_has_entry(int id, void *p, void *data) +{ + + return 1; +} + +bool +idr_is_empty(struct idr *idp) +{ + + return !idr_for_each(idp, idr_has_entry, NULL); +} + int ida_pre_get(struct ida *ida, gfp_t flags) { Index: sys/compat/linuxkpi/common/src/linux_radix.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_radix.c +++ sys/compat/linuxkpi/common/src/linux_radix.c @@ -112,6 +112,7 @@ } } while (height != -1); iter->index = index; + iter->next_index = index++; return (true); } @@ -162,6 +163,15 @@ return (item); } +void +radix_tree_iter_delete(struct radix_tree_root *root, + struct radix_tree_iter *iter, void **slot) +{ + if (radix_tree_delete(root, iter->index)) { + iter->index = iter->next_index; + } +} + int radix_tree_insert(struct radix_tree_root *root, unsigned long index, void *item) { Index: sys/compat/linuxkpi/common/src/linux_schedule.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_schedule.c +++ sys/compat/linuxkpi/common/src/linux_schedule.c @@ -187,6 +187,15 @@ return (wake_up_task(wq->private, state)); } +void +linux_init_wait_entry(wait_queue_t *wait, int flags) +{ + wait->flags = flags; + wait->private = current; + wait->func = autoremove_wake_function; + INIT_LIST_HEAD(&wait->task_list); +} + void linux_wake_up(wait_queue_head_t *wqh, unsigned int state, int nr, bool locked) { Index: sys/compat/linuxkpi/common/src/linux_work.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_work.c +++ sys/compat/linuxkpi/common/src/linux_work.c @@ -55,6 +55,7 @@ struct workqueue_struct *system_wq; struct workqueue_struct *system_long_wq; struct workqueue_struct *system_unbound_wq; +struct workqueue_struct *system_highpri_wq; struct workqueue_struct *system_power_efficient_wq; static int linux_default_wq_cpus = 4; @@ -598,6 +599,7 @@ system_wq = linux_system_short_wq; system_power_efficient_wq = linux_system_short_wq; system_unbound_wq = linux_system_short_wq; + system_highpri_wq = linux_system_short_wq; } SYSINIT(linux_work_init, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_init, NULL); @@ -612,5 +614,6 @@ system_wq = NULL; system_power_efficient_wq = NULL; system_unbound_wq = NULL; + system_highpri_wq = NULL; } SYSUNINIT(linux_work_uninit, SI_SUB_TASKQ, SI_ORDER_THIRD, linux_work_uninit, NULL);