Index: sys/compat/linuxkpi/common/include/asm/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/asm/uaccess.h +++ sys/compat/linuxkpi/common/include/asm/uaccess.h @@ -52,7 +52,9 @@ #define __copy_from_user(...) copy_from_user(__VA_ARGS__) #define __copy_in_user(...) copy_from_user(__VA_ARGS__) -#define user_access_begin() do { } while (0) +#ifndef user_access_begin /* For backwards compatibility */ +#define user_access_begin(ptr, len) access_ok(ptr, len) +#endif #define user_access_end() do { } while (0) #define unsafe_get_user(x, ptr, err) do { \ Index: sys/compat/linuxkpi/common/include/linux/atomic.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/atomic.h +++ sys/compat/linuxkpi/common/include/linux/atomic.h @@ -31,5 +31,6 @@ #include #include +#include #endif /* _LINUX_ATOMIC_H_ */ Index: sys/compat/linuxkpi/common/include/linux/bitops.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/bitops.h +++ sys/compat/linuxkpi/common/include/linux/bitops.h @@ -51,10 +51,11 @@ #define BITMAP_LAST_WORD_MASK(n) (~0UL >> (BITS_PER_LONG - (n))) #define BITS_TO_LONGS(n) howmany((n), BITS_PER_LONG) #define BIT_MASK(nr) (1UL << ((nr) & (BITS_PER_LONG - 1))) -#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) #define GENMASK(h, l) (((~0UL) >> (BITS_PER_LONG - (h) - 1)) & ((~0UL) << (l))) #define GENMASK_ULL(h, l) (((~0ULL) >> (BITS_PER_LONG_LONG - (h) - 1)) & ((~0ULL) << (l))) -#define BITS_PER_BYTE 8 +#define BITS_PER_BYTE 8 +#define BITS_PER_TYPE(t) (sizeof(t) * BITS_PER_BYTE) #define hweight8(x) bitcount((uint8_t)(x)) #define hweight16(x) bitcount16(x) Index: sys/compat/linuxkpi/common/include/linux/cdev.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/cdev.h +++ sys/compat/linuxkpi/common/include/linux/cdev.h @@ -56,6 +56,9 @@ u_int siref; }; +void linux_destroy_dev(struct linux_cdev *); +void linux_cdev_deref(struct linux_cdev *); + static inline void cdev_init(struct linux_cdev *cdev, const struct file_operations *ops) { @@ -135,14 +138,13 @@ return (0); } -void linux_destroy_dev(struct linux_cdev *); - static inline void cdev_del(struct linux_cdev *cdev) { linux_destroy_dev(cdev); kobject_put(&cdev->kobj); + linux_cdev_deref(cdev); } struct linux_cdev *linux_find_cdev(const char *name, unsigned major, unsigned minor); Index: sys/compat/linuxkpi/common/include/linux/device.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/device.h +++ sys/compat/linuxkpi/common/include/linux/device.h @@ -61,6 +61,7 @@ }; struct dev_pm_ops { + int (*prepare)(struct device *dev); int (*suspend)(struct device *dev); int (*suspend_late)(struct device *dev); int (*resume)(struct device *dev); @@ -183,6 +184,14 @@ #define dev_printk(lvl, dev, fmt, ...) \ device_printf((dev)->bsddev, fmt, ##__VA_ARGS__) +#define dev_err_once(dev, ...) do { \ + static bool __dev_err_once; \ + if (!__dev_err_once) { \ + __dev_err_once = 1; \ + dev_err(dev, __VA_ARGS__); \ + } \ +} while (0) + #define dev_err_ratelimited(dev, ...) do { \ static linux_ratelimit_t __ratelimited; \ if (linux_ratelimited(&__ratelimited)) \ Index: sys/compat/linuxkpi/common/include/linux/dma-attrs.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/dma-attrs.h +++ sys/compat/linuxkpi/common/include/linux/dma-attrs.h @@ -31,9 +31,16 @@ #ifndef _LINUX_DMA_ATTR_H_ #define _LINUX_DMA_ATTR_H_ -enum dma_attr { DMA_ATTR_WRITE_BARRIER, DMA_ATTR_WEAK_ORDERING, DMA_ATTR_MAX, }; - -#define __DMA_ATTRS_LONGS BITS_TO_LONGS(DMA_ATTR_MAX) +#define DMA_ATTR_WRITE_BARRIER (1 << 0) +#define DMA_ATTR_WEAK_ORDERING (1 << 1) +#define DMA_ATTR_WRITE_COMBINE (1 << 2) +#define DMA_ATTR_NON_CONSISTENT (1 << 3) +#define DMA_ATTR_NO_KERNEL_MAPPING (1 << 4) +#define DMA_ATTR_SKIP_CPU_SYNC (1 << 5) +#define DMA_ATTR_FORCE_CONTIGUOUS (1 << 6) +#define DMA_ATTR_ALLOC_SINGLE_PAGES (1 << 7) +#define DMA_ATTR_NO_WARN (1 << 8) +#define DMA_ATTR_PRIVILEGED (1 << 9) struct dma_attrs { unsigned long flags; Index: sys/compat/linuxkpi/common/include/linux/dma-mapping.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/dma-mapping.h +++ sys/compat/linuxkpi/common/include/linux/dma-mapping.h @@ -119,6 +119,17 @@ return 0; } +static inline int +dma_set_mask_and_coherent(struct device *dev, u64 mask) +{ + int r; + + r = dma_set_mask(dev, mask); + if (r == 0) + dma_set_coherent_mask(dev, mask); + return (r); +} + static inline void * dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag) @@ -165,7 +176,7 @@ enum dma_data_direction dir, struct dma_attrs *attrs) { - return vtophys(ptr); + return (vtophys(ptr)); } static inline void @@ -174,6 +185,14 @@ { } +static inline dma_addr_t +dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + + return (VM_PAGE_TO_PHYS(page) + offset); +} + static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, struct dma_attrs *attrs) @@ -198,7 +217,7 @@ unsigned long offset, size_t size, enum dma_data_direction direction) { - return VM_PAGE_TO_PHYS(page) + offset; + return (VM_PAGE_TO_PHYS(page) + offset); } static inline void @@ -217,6 +236,7 @@ dma_sync_single(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { + dma_sync_single_for_cpu(dev, addr, size, dir); } Index: sys/compat/linuxkpi/common/include/linux/fs.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/fs.h +++ sys/compat/linuxkpi/common/include/linux/fs.h @@ -323,4 +323,13 @@ #define shmem_truncate_range(...) \ linux_shmem_truncate_range(__VA_ARGS__) +static inline int +simple_open(struct inode *inode, struct file *file) +{ + + if (inode->i_private) + file->private_data = inode->i_private; + return (0); +} + #endif /* _LINUX_FS_H_ */ Index: sys/compat/linuxkpi/common/include/linux/idr.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/idr.h +++ sys/compat/linuxkpi/common/include/linux/idr.h @@ -112,6 +112,7 @@ int ida_pre_get(struct ida *ida, gfp_t gfp_mask); int ida_get_new_above(struct ida *ida, int starting_id, int *p_id); void ida_remove(struct ida *ida, int id); +void ida_free(struct ida *ida, int id); void ida_destroy(struct ida *ida); void ida_init(struct ida *ida); @@ -126,6 +127,13 @@ return (ida_get_new_above(ida, 0, p_id)); } +static inline int +ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp) +{ + + return (ida_simple_get(ida, 0, max, gfp)); +} + static inline bool ida_is_empty(struct ida *ida) { Index: sys/compat/linuxkpi/common/include/linux/interrupt.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/interrupt.h +++ sys/compat/linuxkpi/common/include/linux/interrupt.h @@ -189,11 +189,13 @@ struct tasklet_struct { TAILQ_ENTRY(tasklet_struct) entry; tasklet_func_t *func; + volatile u_int _state; /* Our impl differ, avoid same name as Linux */ + atomic_t count; unsigned long data; }; #define DECLARE_TASKLET(name, func, data) \ -struct tasklet_struct name = { { NULL, NULL }, func, data } +struct tasklet_struct name = { { NULL, NULL }, func, ATOMIC_INIT(0), data } #define tasklet_hi_schedule(t) tasklet_schedule(t) @@ -202,6 +204,10 @@ extern void tasklet_init(struct tasklet_struct *, tasklet_func_t *, unsigned long data); extern void tasklet_enable(struct tasklet_struct *); +extern bool tasklet_is_enabled(struct tasklet_struct *); extern void tasklet_disable(struct tasklet_struct *); +extern int tasklet_trylock(struct tasklet_struct *); +extern void tasklet_unlock(struct tasklet_struct *); +extern void tasklet_unlock_wait(struct tasklet_struct *ts); #endif /* _LINUX_INTERRUPT_H_ */ Index: sys/compat/linuxkpi/common/include/linux/kernel.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/kernel.h +++ sys/compat/linuxkpi/common/include/linux/kernel.h @@ -130,9 +130,11 @@ #define ALIGN(x, y) roundup2((x), (y)) #undef PTR_ALIGN #define PTR_ALIGN(p, a) ((__typeof(p))ALIGN((uintptr_t)(p), (a))) +#define IS_ALIGNED(x, a) (((x) & ((__typeof(x))(a) - 1)) == 0) #define DIV_ROUND_UP(x, n) howmany(x, n) #define __KERNEL_DIV_ROUND_UP(x, n) howmany(x, n) #define DIV_ROUND_UP_ULL(x, n) DIV_ROUND_UP((unsigned long long)(x), (n)) +#define DIV_ROUND_DOWN_ULL(x, n) (((unsigned long long)(x) / (n)) * (n)) #define FIELD_SIZEOF(t, f) sizeof(((t *)0)->f) #define printk(...) printf(__VA_ARGS__) Index: sys/compat/linuxkpi/common/include/linux/ktime.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ktime.h +++ sys/compat/linuxkpi/common/include/linux/ktime.h @@ -35,7 +35,8 @@ #include #include -#define ktime_get_ts(x) getnanouptime(x) +#define ktime_get_ts(x) getnanotime(x) +#define ktime_get_raw_ts(x) getnanotime(x) /* time values in nanoseconds */ typedef s64 ktime_t; @@ -92,6 +93,13 @@ return (ktime_add_ns(kt, ms * NSEC_PER_MSEC)); } +static inline ktime_t +ktime_add_us(ktime_t kt, int64_t us) +{ + + return (ktime_add_ns(kt, us * NSEC_PER_USEC)); +} + static inline ktime_t ktime_sub_ns(ktime_t kt, int64_t ns) { @@ -172,11 +180,19 @@ return (ktime_set(tv.tv_sec, tv.tv_usec * NSEC_PER_USEC)); } +static inline int64_t +timespec64_to_ns(struct timespec64 *ts) +{ + return (timespec_to_ns(ts)); +} + #define ktime_to_timespec(kt) ns_to_timespec(kt) #define ktime_to_timespec64(kt) ns_to_timespec(kt) #define ktime_to_timeval(kt) ns_to_timeval(kt) #define ktime_to_ns(kt) (kt) #define ktime_get_ts64(ts) ktime_get_ts(ts) +#define ktime_get_raw_ts64(ts) ktime_get_raw_ts(ts) +#define getrawmonotonic64(ts) ktime_get_raw_ts64(ts) static inline int64_t ktime_get_ns(void) @@ -238,7 +254,7 @@ { struct timespec ts; - nanouptime(&ts); + nanotime(&ts); return (ktime_to_ns(timespec_to_ktime(ts))); } Index: sys/compat/linuxkpi/common/include/linux/list.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/list.h +++ sys/compat/linuxkpi/common/include/linux/list.h @@ -228,6 +228,10 @@ #define list_for_each_prev(p, h) for (p = (h)->prev; p != (h); p = (p)->prev) +#define list_for_each_entry_from_reverse(p, h, field) \ + for (; &p->field != (h); \ + p = list_prev_entry(p, field)) + static inline void list_add(struct list_head *new, struct list_head *head) { @@ -258,6 +262,18 @@ list_add_tail(entry, head); } +static inline void +list_bulk_move_tail(struct list_head *head, struct list_head *first, + struct list_head *last) +{ + first->prev->next = last->next; + last->next->prev = first->prev; + head->prev->next = first; + first->prev = head->prev; + last->next = head; + head->prev = last; +} + static inline void linux_list_splice(const struct list_head *list, struct list_head *prev, struct list_head *next) Index: sys/compat/linuxkpi/common/include/linux/mm.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/mm.h +++ sys/compat/linuxkpi/common/include/linux/mm.h @@ -92,6 +92,7 @@ #define FAULT_FLAG_INSTRUCTION (1 << 8) typedef int (*pte_fn_t)(linux_pte_t *, pgtable_t, unsigned long addr, void *data); +typedef int vm_fault_t; struct vm_area_struct { vm_offset_t vm_start; @@ -134,6 +135,12 @@ int (*access) (struct vm_area_struct *, unsigned long, void *, int, int); }; +struct sysinfo { + uint64_t totalram; + uint64_t totalhigh; + uint32_t mem_unit; +}; + /* * Compute log2 of the power of two rounded up count of pages * needed for size bytes. @@ -268,5 +275,6 @@ } extern int is_vmalloc_addr(const void *addr); +void si_meminfo(struct sysinfo *si); #endif /* _LINUX_MM_H_ */ Index: sys/compat/linuxkpi/common/include/linux/pci.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/pci.h +++ sys/compat/linuxkpi/common/include/linux/pci.h @@ -139,10 +139,13 @@ #define PCI_EXP_TYPE_RC_EC PCIEM_TYPE_ROOT_EC /* Root Complex Event Collector */ #define PCI_EXP_LNKCAP_SLS_2_5GB 0x01 /* Supported Link Speed 2.5GT/s */ #define PCI_EXP_LNKCAP_SLS_5_0GB 0x02 /* Supported Link Speed 5.0GT/s */ +#define PCI_EXP_LNKCAP_SLS_8_0GB 0x04 /* Supported Link Speed 8.0GT/s */ +#define PCI_EXP_LNKCAP_SLS_16_0GB 0x08 /* Supported Link Speed 16.0GT/s */ #define PCI_EXP_LNKCAP_MLW 0x03f0 /* Maximum Link Width */ #define PCI_EXP_LNKCAP2_SLS_2_5GB 0x02 /* Supported Link Speed 2.5GT/s */ #define PCI_EXP_LNKCAP2_SLS_5_0GB 0x04 /* Supported Link Speed 5.0GT/s */ #define PCI_EXP_LNKCAP2_SLS_8_0GB 0x08 /* Supported Link Speed 8.0GT/s */ +#define PCI_EXP_LNKCAP2_SLS_16_0GB 0x10 /* Supported Link Speed 16.0GT/s */ #define PCI_EXP_LNKCTL_HAWD PCIEM_LINK_CTL_HAWD #define PCI_EXP_LNKCAP_CLKPM 0x00040000 @@ -157,10 +160,19 @@ PCIE_SPEED_2_5GT, PCIE_SPEED_5_0GT, PCIE_SPEED_8_0GT, + PCIE_SPEED_16_0GT, }; enum pcie_link_width { - PCIE_LNK_WIDTH_UNKNOWN = 0xFF, + PCIE_LNK_WIDTH_RESRV = 0x00, + PCIE_LNK_X1 = 0x01, + PCIE_LNK_X2 = 0x02, + PCIE_LNK_X4 = 0x04, + PCIE_LNK_X8 = 0x08, + PCIE_LNK_X12 = 0x0c, + PCIE_LNK_X16 = 0x10, + PCIE_LNK_X32 = 0x20, + PCIE_LNK_WIDTH_UNKNOWN = 0xff, }; typedef int pci_power_t; @@ -850,4 +862,59 @@ return (0); } +static inline enum pci_bus_speed +pcie_get_speed_cap(struct pci_dev *dev) +{ + device_t root; + uint32_t lnkcap, lnkcap2; + int error, pos; + + root = device_get_parent(device_get_parent( + device_get_parent(dev->dev.bsddev))); + + /* we've been informed via and serverworks don't make the cut */ + if (pci_get_vendor(root) == PCI_VENDOR_ID_VIA || + pci_get_vendor(root) == PCI_VENDOR_ID_SERVERWORKS) + return (PCI_SPEED_UNKNOWN); + + if ((error = pci_find_cap(root, PCIY_EXPRESS, &pos)) != 0) + return (PCI_SPEED_UNKNOWN); + + lnkcap2 = pci_read_config(root, pos + PCIER_LINK_CAP2, 4); + + if (lnkcap2) { /* PCIe r3.0-compliant */ + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) + return (PCIE_SPEED_2_5GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) + return (PCIE_SPEED_5_0GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) + return (PCIE_SPEED_8_0GT); + if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_16_0GB) + return (PCIE_SPEED_16_0GT); + } else { /* pre-r3.0 */ + lnkcap = pci_read_config(root, pos + PCIER_LINK_CAP, 4); + if (lnkcap & PCI_EXP_LNKCAP_SLS_2_5GB) + return (PCIE_SPEED_2_5GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_5_0GB) + return (PCIE_SPEED_5_0GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_8_0GB) + return (PCIE_SPEED_8_0GT); + if (lnkcap & PCI_EXP_LNKCAP_SLS_16_0GB) + return (PCIE_SPEED_16_0GT); + } + return (PCI_SPEED_UNKNOWN); +} + +static inline enum pcie_link_width +pcie_get_width_cap(struct pci_dev *dev) +{ + uint32_t lnkcap; + + pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); + if (lnkcap) + return ((lnkcap & PCI_EXP_LNKCAP_MLW) >> 4); + + return PCIE_LNK_WIDTH_UNKNOWN; +} + #endif /* _LINUX_PCI_H_ */ Index: sys/compat/linuxkpi/common/include/linux/preempt.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/preempt.h +++ sys/compat/linuxkpi/common/include/linux/preempt.h @@ -34,6 +34,9 @@ #define in_interrupt() \ (curthread->td_intr_nesting_level || curthread->td_critnest) +#define in_task() \ + (curthread->td_intr_nesting_level == 0 && curthread->td_critnest == 0) + #define preempt_disable() critical_enter() #define preempt_enable() critical_exit() Index: sys/compat/linuxkpi/common/include/linux/printk.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/printk.h +++ sys/compat/linuxkpi/common/include/linux/printk.h @@ -121,4 +121,7 @@ #define pr_err_ratelimited(fmt, ...) \ printk_ratelimited(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info_ratelimited(fmt, ...) \ + printk_ratelimited(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) + #endif /* _LINUX_PRINTK_H_ */ Index: sys/compat/linuxkpi/common/include/linux/random.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/random.h +++ sys/compat/linuxkpi/common/include/linux/random.h @@ -35,6 +35,8 @@ #include #include +#define get_random_u32 get_random_int + static inline void get_random_bytes(void *buf, int nbytes) { Index: sys/compat/linuxkpi/common/include/linux/rcupdate.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/rcupdate.h +++ sys/compat/linuxkpi/common/include/linux/rcupdate.h @@ -100,4 +100,10 @@ extern void linux_rcu_read_unlock(void); extern void linux_synchronize_rcu(void); +/* Empty implementation for !DEBUG */ +#define init_rcu_head(...) +#define destroy_rcu_head(...) +#define init_rcu_head_on_stack(...) +#define destroy_rcu_head_on_stack(...) + #endif /* _LINUX_RCUPDATE_H_ */ Index: sys/compat/linuxkpi/common/include/linux/scatterlist.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/scatterlist.h +++ sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -69,6 +69,8 @@ #define SG_MAX_SINGLE_ALLOC (PAGE_SIZE / sizeof(struct scatterlist)) #define SG_MAGIC 0x87654321UL +#define SG_CHAIN 0x01UL +#define SG_END 0x02UL #define sg_is_chain(sg) ((sg)->page_link & SG_PAGE_LINK_CHAIN) #define sg_is_last(sg) ((sg)->page_link & SG_PAGE_LINK_LAST) @@ -135,6 +137,12 @@ return (VM_PAGE_TO_PHYS(sg_page(sg)) + sg->offset); } +static inline void * +sg_virt(struct scatterlist *sg) +{ + return ((void *)((unsigned long)page_address(sg_page(sg)) + sg->offset)); +} + static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents, struct scatterlist *sgl) Index: sys/compat/linuxkpi/common/include/linux/sched.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/sched.h +++ sys/compat/linuxkpi/common/include/linux/sched.h @@ -80,6 +80,7 @@ int rcu_recurse; int bsd_interrupt_value; struct work_struct *work; /* current work struct, if set */ + struct task_struct *group_leader; }; #define current ({ \ @@ -95,7 +96,9 @@ #define get_pid(x) (x) #define put_pid(x) do { } while (0) #define current_euid() (curthread->td_ucred->cr_uid) +#define task_euid(task) ((task)->task_thread->td_ucred->cr_uid) +#define get_task_state(task) atomic_read(&(task)->state) #define set_task_state(task, x) atomic_set(&(task)->state, (x)) #define __set_task_state(task, x) ((task)->state.counter = (x)) #define set_current_state(x) set_task_state(current, x) @@ -143,6 +146,11 @@ task->bsd_interrupt_value = value; } +bool linux_task_exiting(struct task_struct *task); + +#define current_exiting() \ + linux_task_exiting(current) + static inline int linux_schedule_get_interrupt_value(struct task_struct *task) { @@ -178,4 +186,12 @@ return ((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec); } +static inline const char * +get_task_comm(char *buf, struct task_struct *task) +{ + + buf[0] = 0; /* buffer is too small */ + return (task->comm); +} + #endif /* _LINUX_SCHED_H_ */ Index: sys/compat/linuxkpi/common/include/linux/slab.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/slab.h +++ sys/compat/linuxkpi/common/include/linux/slab.h @@ -42,7 +42,9 @@ MALLOC_DECLARE(M_KMALLOC); -#define kvmalloc(size) kmalloc(size, 0) +#define kvmalloc(size, flags) kmalloc(size, flags) +#define kvzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) +#define kvcalloc(n, size, flags) kvmalloc_array(n, size, (flags) | __GFP_ZERO) #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) #define kzalloc_node(size, flags, node) kmalloc(size, (flags) | __GFP_ZERO) #define kfree_const(ptr) kfree(ptr) Index: sys/compat/linuxkpi/common/include/linux/srcu.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/srcu.h +++ sys/compat/linuxkpi/common/include/linux/srcu.h @@ -34,6 +34,9 @@ #define srcu_dereference(ptr,srcu) ((__typeof(*(ptr)) *)(ptr)) +#define DEFINE_STATIC_SRCU(name) \ + static struct srcu_struct name = {}; + /* prototypes */ extern int srcu_read_lock(struct srcu_struct *); Index: sys/compat/linuxkpi/common/include/linux/sysfs.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/sysfs.h +++ sys/compat/linuxkpi/common/include/linux/sysfs.h @@ -132,10 +132,14 @@ static inline int sysfs_create_file(struct kobject *kobj, const struct attribute *attr) { + struct sysctl_oid *oid; - SYSCTL_ADD_OID(NULL, SYSCTL_CHILDREN(kobj->oidp), OID_AUTO, + oid = SYSCTL_ADD_OID(NULL, SYSCTL_CHILDREN(kobj->oidp), OID_AUTO, attr->name, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, kobj, (uintptr_t)attr, sysctl_handle_attr, "A", ""); + if (!oid) { + return (-ENOMEM); + } return (0); } @@ -176,9 +180,14 @@ static inline int sysfs_create_dir(struct kobject *kobj) { + struct sysctl_oid *oid; - kobj->oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->parent->oidp), + oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(kobj->parent->oidp), OID_AUTO, kobj->name, CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, kobj->name); + if (!oid) { + return (-ENOMEM); + } + kobj->oidp = oid; return (0); } Index: sys/compat/linuxkpi/common/include/linux/timer.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/timer.h +++ sys/compat/linuxkpi/common/include/linux/timer.h @@ -81,8 +81,8 @@ extern void mod_timer(struct timer_list *, int); extern void add_timer(struct timer_list *); extern void add_timer_on(struct timer_list *, int cpu); +extern int del_timer(struct timer_list *); -#define del_timer(timer) (void)callout_stop(&(timer)->callout) #define del_timer_sync(timer) (void)callout_drain(&(timer)->callout) #define timer_pending(timer) callout_pending(&(timer)->callout) #define round_jiffies(j) \ Index: sys/compat/linuxkpi/common/include/linux/uaccess.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/uaccess.h +++ sys/compat/linuxkpi/common/include/linux/uaccess.h @@ -60,12 +60,14 @@ #define get_user(_x, _p) linux_copyin((_p), &(_x), sizeof(*(_p))) #define put_user(_x, _p) __put_user(_x, _p) #define clear_user(...) linux_clear_user(__VA_ARGS__) +#ifndef access_ok /* For backwards compatibility */ #define access_ok(...) linux_access_ok(__VA_ARGS__) +#endif extern int linux_copyin(const void *uaddr, void *kaddr, size_t len); extern int linux_copyout(const void *kaddr, void *uaddr, size_t len); extern size_t linux_clear_user(void *uaddr, size_t len); -extern int linux_access_ok(int rw, const void *uaddr, size_t len); +extern int linux_access_ok(const void *uaddr, size_t len); /* * NOTE: Each pagefault_disable() call must have a corresponding Index: sys/compat/linuxkpi/common/include/linux/ww_mutex.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/ww_mutex.h +++ sys/compat/linuxkpi/common/include/linux/ww_mutex.h @@ -45,6 +45,7 @@ struct ww_mutex { struct mutex base; struct cv condvar; + struct ww_acquire_ctx *ctx; }; #define DEFINE_WW_CLASS(name) \ Index: sys/compat/linuxkpi/common/src/linux_compat.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_compat.c +++ sys/compat/linuxkpi/common/src/linux_compat.c @@ -101,7 +101,6 @@ #undef cdev #define RB_ROOT(head) (head)->rbh_root -static void linux_cdev_deref(struct linux_cdev *ldev); static struct vm_area_struct *linux_cdev_handle_find(void *handle); struct kobject linux_class_root; @@ -893,7 +892,7 @@ } int -linux_access_ok(int rw, const void *uaddr, size_t len) +linux_access_ok(const void *uaddr, size_t len) { uintptr_t saddr; uintptr_t eaddr; @@ -1902,6 +1901,15 @@ &linux_timer_callback_wrapper, timer, cpu); } +int +del_timer(struct timer_list *timer) +{ + + if (callout_stop(&(timer)->callout) == -1) + return (0); + return (1); +} + static void linux_timer_init(void *arg) { @@ -2064,7 +2072,7 @@ return (isdone); } -static void +void linux_cdev_deref(struct linux_cdev *ldev) { Index: sys/compat/linuxkpi/common/src/linux_current.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_current.c +++ sys/compat/linuxkpi/common/src/linux_current.c @@ -67,6 +67,7 @@ ts->task_thread = td; ts->comm = td->td_name; ts->pid = td->td_tid; + ts->group_leader = ts; atomic_set(&ts->usage, 1); atomic_set(&ts->state, TASK_RUNNING); init_completion(&ts->parked); @@ -215,6 +216,22 @@ return (NULL); } +bool +linux_task_exiting(struct task_struct *task) +{ + struct proc *p; + bool ret; + + ret = false; + p = pfind(task->pid); + if (p != NULL) { + if ((p->p_flag & P_WEXIT) != 0) + ret = true; + PROC_UNLOCK(p); + } + return (ret); +} + static void linux_current_init(void *arg __unused) { Index: sys/compat/linuxkpi/common/src/linux_idr.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_idr.c +++ sys/compat/linuxkpi/common/src/linux_idr.c @@ -796,6 +796,12 @@ idr_remove(&ida->idr, id); } +void +ida_free(struct ida *ida, int id) +{ + ida_remove(ida, id); +} + void ida_init(struct ida *ida) { Index: sys/compat/linuxkpi/common/src/linux_page.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_page.c +++ sys/compat/linuxkpi/common/src/linux_page.c @@ -63,6 +63,14 @@ #include #include +void +si_meminfo(struct sysinfo *si) +{ + si->totalram = physmem; + si->totalhigh = 0; + si->mem_unit = PAGE_SIZE; +} + void * linux_page_address(struct page *page) { Index: sys/compat/linuxkpi/common/src/linux_tasklet.c =================================================================== --- sys/compat/linuxkpi/common/src/linux_tasklet.c +++ sys/compat/linuxkpi/common/src/linux_tasklet.c @@ -41,16 +41,18 @@ #define TASKLET_ST_BUSY 1 #define TASKLET_ST_EXEC 2 #define TASKLET_ST_LOOP 3 -#define TASKLET_ST_PAUSED 4 #define TASKLET_ST_CMPSET(ts, old, new) \ - atomic_cmpset_ptr((volatile uintptr_t *)&(ts)->entry.tqe_prev, old, new) + atomic_cmpset_int((volatile u_int *)&(ts)->_state, old, new) #define TASKLET_ST_SET(ts, new) \ - WRITE_ONCE(*(volatile uintptr_t *)&(ts)->entry.tqe_prev, new) + WRITE_ONCE(*(volatile u_int *)&(ts)->_state, new) #define TASKLET_ST_GET(ts) \ - READ_ONCE(*(volatile uintptr_t *)&(ts)->entry.tqe_prev) + READ_ONCE(*(volatile u_int *)&(ts)->_state) + +#define TASKLET_ST_TESTANDSET(ts, new) \ + atomic_testandset_int((volatile u_int *)&(ts)->_state, new) struct tasklet_worker { struct mtx mtx; @@ -67,26 +69,34 @@ tasklet_handler(void *arg) { struct tasklet_worker *tw = (struct tasklet_worker *)arg; - struct tasklet_struct *ts; + struct tasklet_struct *ts, *first; linux_set_current(curthread); TASKLET_WORKER_LOCK(tw); + first = TAILQ_FIRST(&tw->head); while (1) { ts = TAILQ_FIRST(&tw->head); if (ts == NULL) break; TAILQ_REMOVE(&tw->head, ts, entry); - TASKLET_WORKER_UNLOCK(tw); - do { - /* reset executing state */ - TASKLET_ST_SET(ts, TASKLET_ST_EXEC); - - ts->func(ts->data); - - } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_IDLE) == 0); - TASKLET_WORKER_LOCK(tw); + if (!atomic_read(&ts->count)) { + TASKLET_WORKER_UNLOCK(tw); + do { + /* reset executing state */ + TASKLET_ST_SET(ts, TASKLET_ST_EXEC); + + ts->func(ts->data); + + } while (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, + TASKLET_ST_IDLE) == 0); + TASKLET_WORKER_LOCK(tw); + } else { + TAILQ_INSERT_TAIL(&tw->head, ts, entry); + } + if (TAILQ_FIRST(&tw->head) == first) + break; } TASKLET_WORKER_UNLOCK(tw); } @@ -140,6 +150,8 @@ ts->entry.tqe_next = NULL; ts->func = func; ts->data = data; + ts->_state = TASKLET_ST_IDLE; + atomic_set(&ts->count, 0); } void @@ -158,6 +170,10 @@ tasklet_schedule(struct tasklet_struct *ts) { + /* tasklet is paused */ + if (atomic_read(&ts->count)) + return; + if (TASKLET_ST_CMPSET(ts, TASKLET_ST_EXEC, TASKLET_ST_LOOP)) { /* tasklet_handler() will loop */ } else if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_BUSY)) { @@ -201,17 +217,49 @@ void tasklet_enable(struct tasklet_struct *ts) { - (void) TASKLET_ST_CMPSET(ts, TASKLET_ST_PAUSED, TASKLET_ST_IDLE); + + barrier(); + atomic_dec(&ts->count); +} + +bool +tasklet_is_enabled(struct tasklet_struct *ts) +{ + + return !atomic_read(&ts->count); } void tasklet_disable(struct tasklet_struct *ts) { - while (1) { - if (TASKLET_ST_GET(ts) == TASKLET_ST_PAUSED) - break; - if (TASKLET_ST_CMPSET(ts, TASKLET_ST_IDLE, TASKLET_ST_PAUSED)) - break; + + atomic_inc(&ts->count); + barrier(); + tasklet_unlock_wait(ts); + mb(); +} + +int +tasklet_trylock(struct tasklet_struct *ts) +{ + + return (!TASKLET_ST_TESTANDSET(ts, TASKLET_ST_BUSY)); +} + +void +tasklet_unlock(struct tasklet_struct *ts) +{ + + TASKLET_ST_SET(ts, TASKLET_ST_IDLE); +} + +void +tasklet_unlock_wait(struct tasklet_struct *ts) +{ + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "tasklet_kill() can sleep"); + + /* wait until tasklet is no longer busy */ + while (TASKLET_ST_GET(ts) != TASKLET_ST_IDLE) pause("W", 1); - } }