diff --git a/sys/compat/linuxkpi/common/include/acpi/actbl.h b/sys/compat/linuxkpi/common/include/acpi/actbl.h new file mode 100644 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/acpi/actbl.h @@ -0,0 +1 @@ +#include diff --git a/sys/compat/linuxkpi/common/include/asm/atomic.h b/sys/compat/linuxkpi/common/include/asm/atomic.h --- a/sys/compat/linuxkpi/common/include/asm/atomic.h +++ b/sys/compat/linuxkpi/common/include/asm/atomic.h @@ -220,6 +220,7 @@ __ret.val; \ }) +#define cmpxchg64(...) cmpxchg(__VA_ARGS__) #define cmpxchg_relaxed(...) cmpxchg(__VA_ARGS__) #define xchg(ptr, new) ({ \ diff --git a/sys/compat/linuxkpi/common/include/asm/processor.h b/sys/compat/linuxkpi/common/include/asm/processor.h --- a/sys/compat/linuxkpi/common/include/asm/processor.h +++ b/sys/compat/linuxkpi/common/include/asm/processor.h @@ -36,6 +36,7 @@ struct cpuinfo_x86 { uint8_t x86; uint16_t x86_clflush_size; + uint16_t x86_max_cores; }; #define cpu_relax() cpu_spinwait() diff --git a/sys/compat/linuxkpi/common/include/linux/device.h b/sys/compat/linuxkpi/common/include/linux/device.h --- a/sys/compat/linuxkpi/common/include/linux/device.h +++ b/sys/compat/linuxkpi/common/include/linux/device.h @@ -204,6 +204,14 @@ } \ } while (0) +#define dev_warn_once(dev, ...) do { \ + static bool __dev_warn_once; \ + if (!__dev_warn_once) { \ + __dev_warn_once = 1; \ + dev_warn(dev, __VA_ARGS__); \ + } \ +} while (0) + #define dev_err_once(dev, ...) do { \ static bool __dev_err_once; \ if (!__dev_err_once) { \ diff --git a/sys/compat/linuxkpi/common/include/linux/dma-buf-map.h b/sys/compat/linuxkpi/common/include/linux/dma-buf-map.h new file mode 100644 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/dma-buf-map.h @@ -0,0 +1,91 @@ +/* Public domain. */ + +#ifndef _LINUX_DMA_BUF_MAP_H +#define _LINUX_DMA_BUF_MAP_H + +#include +#include + +struct dma_buf_map { + union { + void *vaddr_iomem; + void *vaddr; + }; + bool is_iomem; +}; + +static inline void +dma_buf_map_incr(struct dma_buf_map *dbm, size_t n) +{ + if (dbm->is_iomem) + dbm->vaddr_iomem += n; + else + dbm->vaddr += n; +} + +static inline void +dma_buf_map_memcpy_to(struct dma_buf_map *dbm, const void *src, size_t len) +{ + if (dbm->is_iomem) + memcpy_toio(dbm->vaddr_iomem, src, len); + else + memcpy(dbm->vaddr, src, len); +} + +static inline bool +dma_buf_map_is_null(const struct dma_buf_map *dbm) +{ + if (dbm->is_iomem) + return (dbm->vaddr_iomem == NULL); + else + return (dbm->vaddr == NULL); +} + +static inline bool +dma_buf_map_is_set(const struct dma_buf_map *dbm) +{ + if (dbm->is_iomem) + return (dbm->vaddr_iomem != NULL); + else + return (dbm->vaddr != NULL); +} + +static inline bool +dma_buf_map_is_equal( + const struct dma_buf_map *dbm_a, const struct dma_buf_map *dbm_b) +{ + if (dbm_a->is_iomem != dbm_b->is_iomem) + return (false); + + if (dbm_a->is_iomem) + return (dbm_a->vaddr_iomem == dbm_b->vaddr_iomem); + else + return (dbm_a->vaddr == dbm_b->vaddr); +} + +static inline void +dma_buf_map_clear(struct dma_buf_map *dbm) +{ + if (dbm->is_iomem) { + dbm->vaddr_iomem = NULL; + dbm->is_iomem = false; + } else { + dbm->vaddr = NULL; + } +} + +static inline void +dma_buf_map_set_vaddr_iomem(struct dma_buf_map *dbm, void *addr) +{ + dbm->vaddr_iomem = addr; + dbm->is_iomem = true; +} + +static inline void +dma_buf_map_set_vaddr(struct dma_buf_map *dbm, void *addr) +{ + dbm->vaddr = addr; + dbm->is_iomem = false; +} + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/gfp.h b/sys/compat/linuxkpi/common/include/linux/gfp.h --- a/sys/compat/linuxkpi/common/include/linux/gfp.h +++ b/sys/compat/linuxkpi/common/include/linux/gfp.h @@ -47,6 +47,7 @@ #define __GFP_HIGHMEM 0 #define __GFP_ZERO M_ZERO #define __GFP_NORETRY 0 +#define __GFP_NOMEMALLOC 0 #define __GFP_RECLAIM 0 #define __GFP_RECLAIMABLE 0 #define __GFP_RETRY_MAYFAIL 0 diff --git a/sys/compat/linuxkpi/common/include/linux/irq_work.h b/sys/compat/linuxkpi/common/include/linux/irq_work.h --- a/sys/compat/linuxkpi/common/include/linux/irq_work.h +++ b/sys/compat/linuxkpi/common/include/linux/irq_work.h @@ -51,7 +51,9 @@ struct irq_work { struct task irq_task; - struct llist_node llnode; + struct { + struct llist_node llist; + } node; irq_work_func_t func; }; diff --git a/sys/compat/linuxkpi/common/include/linux/kernel.h b/sys/compat/linuxkpi/common/include/linux/kernel.h --- a/sys/compat/linuxkpi/common/include/linux/kernel.h +++ b/sys/compat/linuxkpi/common/include/linux/kernel.h @@ -577,6 +577,8 @@ #define offsetofend(t, m) \ (offsetof(t, m) + sizeof((((t *)0)->m))) +#define typeof_member(s, e) typeof(((s *)0)->e) + #define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) #define clamp(x, lo, hi) min( max(x,lo), hi) #define clamp_val(val, lo, hi) clamp_t(typeof(val), val, lo, hi) diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h --- a/sys/compat/linuxkpi/common/include/linux/mm.h +++ b/sys/compat/linuxkpi/common/include/linux/mm.h @@ -40,6 +40,7 @@ #include #include #include +#include #include @@ -310,4 +311,6 @@ #define PAGE_ALIGNED(p) __is_aligned(p, PAGE_SIZE) +void vma_set_file(struct vm_area_struct *vma, struct linux_file *file); + #endif /* _LINUXKPI_LINUX_MM_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/mman.h b/sys/compat/linuxkpi/common/include/linux/mman.h new file mode 100644 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/mman.h @@ -0,0 +1,10 @@ +#ifndef _LINUX_MMAN_H +#define _LINUX_MMAN_H + +/* + * In Linux, includes , which includes + * . + */ +#include + +#endif /* _LINUX_MMAN_H */ diff --git a/sys/compat/linuxkpi/common/include/linux/mmzone.h b/sys/compat/linuxkpi/common/include/linux/mmzone.h new file mode 100644 --- /dev/null +++ b/sys/compat/linuxkpi/common/include/linux/mmzone.h @@ -0,0 +1,11 @@ +/* Public domain. */ + +#ifndef _LINUX_MMZONE_H +#define _LINUX_MMZONE_H + +#include +#include + +#define MAX_ORDER 11 + +#endif diff --git a/sys/compat/linuxkpi/common/include/linux/moduleparam.h b/sys/compat/linuxkpi/common/include/linux/moduleparam.h --- a/sys/compat/linuxkpi/common/include/linux/moduleparam.h +++ b/sys/compat/linuxkpi/common/include/linux/moduleparam.h @@ -90,6 +90,12 @@ LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ LINUXKPI_PARAM_DESC(name))) +#define LINUXKPI_PARAM_hexint(name, var, perm) \ + extern const char LINUXKPI_PARAM_DESC(name)[]; \ + LINUXKPI_PARAM_PASS(SYSCTL_UINT(LINUXKPI_PARAM_PARENT, OID_AUTO, \ + LINUXKPI_PARAM_NAME(name), LINUXKPI_PARAM_PERM(perm), &(var), 0, \ + LINUXKPI_PARAM_DESC(name))) + #define LINUXKPI_PARAM_long(name, var, perm) \ extern const char LINUXKPI_PARAM_DESC(name)[]; \ LINUXKPI_PARAM_PASS(SYSCTL_LONG(LINUXKPI_PARAM_PARENT, OID_AUTO, \ diff --git a/sys/compat/linuxkpi/common/include/linux/smp.h b/sys/compat/linuxkpi/common/include/linux/page-flags.h copy from sys/compat/linuxkpi/common/include/linux/smp.h copy to sys/compat/linuxkpi/common/include/linux/page-flags.h --- a/sys/compat/linuxkpi/common/include/linux/smp.h +++ b/sys/compat/linuxkpi/common/include/linux/page-flags.h @@ -1,5 +1,10 @@ /*- - * Copyright (c) 2017 Mark Johnston + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2010 Isilon Systems, Inc. + * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) + * Copyright (c) 2017 Mellanox Technologies, Ltd. + * Copyright (c) 2021 Vladimir Kondratyev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -22,30 +27,11 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. - * - * $FreeBSD$ */ -#ifndef _LINUXKPI_LINUX_SMP_H_ -#define _LINUXKPI_LINUX_SMP_H_ - -/* - * Important note about the use of the function provided below: - * - * The callback function passed to on_each_cpu() is called from a - * so-called critical section, and if you need a mutex you will have - * to rewrite the code to use native FreeBSD mtx spinlocks instead of - * the spinlocks provided by the LinuxKPI! Be very careful to not call - * any LinuxKPI functions inside the on_each_cpu()'s callback - * function, because they may sleep, unlike in native Linux. - * - * Enabling witness(4) when testing, can catch such issues. - */ -#define on_each_cpu(cb, data, wait) ({ \ - CTASSERT(wait); \ - linux_on_each_cpu(cb, data); \ -}) +#ifndef _LINUXKPI_LINUX_PAGEFLAGS_H_ +#define _LINUXKPI_LINUX_PAGEFLAGS_H_ -extern int linux_on_each_cpu(void (*)(void *), void *); +#define PageHighMem(p) (0) -#endif /* _LINUXKPI_LINUX_SMP_H_ */ +#endif /* _LINUXKPI_LINUX_PAGEFLAGS_H_ */ diff --git a/sys/compat/linuxkpi/common/include/linux/rculist.h b/sys/compat/linuxkpi/common/include/linux/rculist.h --- a/sys/compat/linuxkpi/common/include/linux/rculist.h +++ b/sys/compat/linuxkpi/common/include/linux/rculist.h @@ -44,6 +44,11 @@ &(pos)->member != (head); \ pos = list_entry_rcu((pos)->member.next, typeof(*(pos)), member)) +#define list_for_each_entry_from_rcu(pos, head, member) \ + for (; \ + &(pos)->member != (head); \ + pos = list_entry_rcu((pos)->member.next, typeof(*(pos)), member)) + #define list_for_each_entry_lockless(pos, head, member) \ list_for_each_entry_rcu(pos, head, member) diff --git a/sys/compat/linuxkpi/common/include/linux/seqlock.h b/sys/compat/linuxkpi/common/include/linux/seqlock.h --- a/sys/compat/linuxkpi/common/include/linux/seqlock.h +++ b/sys/compat/linuxkpi/common/include/linux/seqlock.h @@ -33,8 +33,11 @@ #include #include #include +#include #include +#include + struct lock_class_key; struct seqcount { @@ -48,6 +51,12 @@ }; typedef struct seqlock seqlock_t; +struct seqcount_mutex { + struct mutex *seqm_lock; + struct seqcount seqm_count; +}; +typedef struct seqcount_mutex seqcount_mutex_t; + static inline void __seqcount_init(struct seqcount *seqcount, const char *name __unused, struct lock_class_key *key __unused) @@ -57,37 +66,99 @@ #define seqcount_init(seqcount) __seqcount_init(seqcount, NULL, NULL) static inline void -write_seqcount_begin(struct seqcount *seqcount) +seqcount_mutex_init(struct seqcount_mutex *seqcount, struct mutex *mutex) +{ + seqcount->seqm_lock = mutex; + seqcount_init(&seqcount->seqm_count); +} + +#define write_seqcount_begin(s) \ + _Generic(*(s), \ + struct seqcount: lkpi_write_seqcount_begin, \ + struct seqcount_mutex: lkpi_write_seqcount_mutex_begin \ + )(s) + +static inline void +lkpi_write_seqcount_begin(struct seqcount *seqcount) { seqc_sleepable_write_begin(&seqcount->seqc); } static inline void -write_seqcount_end(struct seqcount *seqcount) +lkpi_write_seqcount_mutex_begin(struct seqcount_mutex *seqcount) +{ + mutex_lock(seqcount->seqm_lock); + lkpi_write_seqcount_begin(&seqcount->seqm_count); +} + +#define write_seqcount_end(s) \ + _Generic(*(s), \ + struct seqcount: lkpi_write_seqcount_end, \ + struct seqcount_mutex: lkpi_write_seqcount_mutex_end \ + )(s) + +static inline void +lkpi_write_seqcount_end(struct seqcount *seqcount) { seqc_sleepable_write_end(&seqcount->seqc); } -/* - * XXX: Are predicts from inline functions still not honored by clang? - */ -#define __read_seqcount_retry(seqcount, gen) \ - (!seqc_consistent_no_fence(&(seqcount)->seqc, gen)) -#define read_seqcount_retry(seqcount, gen) \ - (!seqc_consistent(&(seqcount)->seqc, gen)) +static inline void +lkpi_write_seqcount_mutex_end(struct seqcount_mutex *seqcount) +{ + lkpi_write_seqcount_end(&seqcount->seqm_count); + mutex_unlock(seqcount->seqm_lock); +} + +#define read_seqcount_begin(s) \ + _Generic(*(s), \ + struct seqcount: lkpi_read_seqcount_begin, \ + struct seqcount_mutex: lkpi_read_seqcount_mutex_begin \ + )(s) static inline unsigned -read_seqcount_begin(const struct seqcount *seqcount) +lkpi_read_seqcount_begin(const struct seqcount *seqcount) { return (seqc_read(&seqcount->seqc)); } +static inline unsigned +lkpi_read_seqcount_mutex_begin(const struct seqcount_mutex *seqcount) +{ + return (lkpi_read_seqcount_begin(&seqcount->seqm_count)); +} + static inline unsigned raw_read_seqcount(const struct seqcount *seqcount) { return (seqc_read_any(&seqcount->seqc)); } +/* + * XXX: Are predicts from inline functions still not honored by clang? + */ +#define __read_seqcount_retry(seqcount, gen) \ + (!seqc_consistent_no_fence(&(seqcount)->seqc, gen)) +#define read_seqcount_retry(s, old) \ + _Generic(*(s), \ + struct seqcount: lkpi_read_seqcount_retry, \ + struct seqcount_mutex: lkpi_read_seqcount_mutex_retry \ + )(s, old) + +static inline int +lkpi_read_seqcount_retry( + const struct seqcount *seqcount, unsigned int old) +{ + return (!seqc_consistent(&seqcount->seqc, old)); +} + +static inline int +lkpi_read_seqcount_mutex_retry( + const struct seqcount_mutex *seqcount, unsigned int old) +{ + return (!seqc_consistent(&seqcount->seqm_count.seqc, old)); +} + static inline void seqlock_init(struct seqlock *seqlock) { diff --git a/sys/compat/linuxkpi/common/include/linux/slab.h b/sys/compat/linuxkpi/common/include/linux/slab.h --- a/sys/compat/linuxkpi/common/include/linux/slab.h +++ b/sys/compat/linuxkpi/common/include/linux/slab.h @@ -178,6 +178,12 @@ return (realloc(ptr, size, M_KMALLOC, linux_check_m_flags(flags))); } +static inline void * +krealloc_array(void *ptr, size_t n, size_t size, gfp_t flags) +{ + return (realloc(ptr, n * size, M_KMALLOC, linux_check_m_flags(flags))); +} + extern void linux_kfree_async(void *); static inline void diff --git a/sys/compat/linuxkpi/common/include/linux/smp.h b/sys/compat/linuxkpi/common/include/linux/smp.h --- a/sys/compat/linuxkpi/common/include/linux/smp.h +++ b/sys/compat/linuxkpi/common/include/linux/smp.h @@ -29,6 +29,8 @@ #ifndef _LINUXKPI_LINUX_SMP_H_ #define _LINUXKPI_LINUX_SMP_H_ +#include + /* * Important note about the use of the function provided below: * diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c --- a/sys/compat/linuxkpi/common/src/linux_compat.c +++ b/sys/compat/linuxkpi/common/src/linux_compat.c @@ -833,6 +833,18 @@ return (0); } +void +vma_set_file(struct vm_area_struct *vma, struct linux_file *file) +{ + struct linux_file *tmp; + + /* Changing an anonymous vma with this is illegal */ + get_file(file); + tmp = vma->vm_file; + vma->vm_file = file; + fput(tmp); +} + static struct file_operations dummy_ldev_ops = { /* XXXKIB */ }; @@ -2743,6 +2755,7 @@ #if defined(__i386__) || defined(__amd64__) linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); boot_cpu_data.x86_clflush_size = cpu_clflush_line_size; + boot_cpu_data.x86_max_cores = mp_ncpus; boot_cpu_data.x86 = ((cpu_id & 0xf0000) >> 12) | ((cpu_id & 0xf0) >> 4); #endif rw_init(&linux_vma_lock, "lkpi-vma-lock");