diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4 index c6efebd8cf61..edfde1a02d30 100644 --- a/config/kernel-fpu.m4 +++ b/config/kernel-fpu.m4 @@ -1,113 +1,130 @@ dnl # dnl # Handle differences in kernel FPU code. dnl # dnl # Kernel dnl # 5.19: The asm/fpu/internal.h header was removed, it has been dnl # effectively empty since the 5.16 kernel. dnl # dnl # 5.11: kernel_fpu_begin() is an inlined function now, so don't check dnl # for it inside the kernel symbols. dnl # dnl # 5.0: Wrappers have been introduced to save/restore the FPU state. dnl # This change was made to the 4.19.38 and 4.14.120 LTS kernels. dnl # HAVE_KERNEL_FPU_INTERNAL dnl # dnl # 4.2: Use __kernel_fpu_{begin,end}() dnl # HAVE_UNDERSCORE_KERNEL_FPU & KERNEL_EXPORTS_X86_FPU dnl # dnl # Pre-4.2: Use kernel_fpu_{begin,end}() dnl # HAVE_KERNEL_FPU & KERNEL_EXPORTS_X86_FPU dnl # dnl # N.B. The header check is performed before all other checks since it dnl # depends on HAVE_KERNEL_FPU_API_HEADER being set in confdefs.h. dnl # AC_DEFUN([ZFS_AC_KERNEL_FPU_HEADER], [ AC_MSG_CHECKING([whether fpu headers are available]) ZFS_LINUX_TRY_COMPILE([ #include #include ],[ ],[ AC_DEFINE(HAVE_KERNEL_FPU_API_HEADER, 1, [kernel has asm/fpu/api.h]) ZFS_LINUX_TRY_COMPILE([ #include #include ],[ ],[ AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL_HEADER, 1, [kernel has asm/fpu/internal.h]) AC_MSG_RESULT([asm/fpu/api.h asm/fpu/internal.h]) ],[ AC_MSG_RESULT([asm/fpu/api.h]) ]) ],[ AC_MSG_RESULT([i387.h]) ]) ]) AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [ ZFS_LINUX_TEST_SRC([kernel_fpu], [ #include #ifdef HAVE_KERNEL_FPU_API_HEADER #include #ifdef HAVE_KERNEL_FPU_INTERNAL_HEADER #include #endif #else #include #endif ], [ kernel_fpu_begin(); kernel_fpu_end(); ], [], [ZFS_META_LICENSE]) ZFS_LINUX_TEST_SRC([__kernel_fpu], [ #include #ifdef HAVE_KERNEL_FPU_API_HEADER #include #ifdef HAVE_KERNEL_FPU_INTERNAL_HEADER #include #endif #else #include #endif ], [ __kernel_fpu_begin(); __kernel_fpu_end(); ], [], [ZFS_META_LICENSE]) + ZFS_LINUX_TEST_SRC([kernel_neon], [ + #include + ], [ + kernel_neon_begin(); + kernel_neon_end(); + ], [], [ZFS_META_LICENSE]) ]) AC_DEFUN([ZFS_AC_KERNEL_FPU], [ dnl # dnl # Legacy kernel dnl # AC_MSG_CHECKING([whether kernel fpu is available]) ZFS_LINUX_TEST_RESULT([kernel_fpu_license], [ AC_MSG_RESULT(kernel_fpu_*) AC_DEFINE(HAVE_KERNEL_FPU, 1, [kernel has kernel_fpu_* functions]) AC_DEFINE(KERNEL_EXPORTS_X86_FPU, 1, [kernel exports FPU functions]) ],[ dnl # dnl # Linux 4.2 kernel dnl # ZFS_LINUX_TEST_RESULT_SYMBOL([__kernel_fpu_license], [__kernel_fpu_begin], [arch/x86/kernel/fpu/core.c arch/x86/kernel/i387.c], [ AC_MSG_RESULT(__kernel_fpu_*) AC_DEFINE(HAVE_UNDERSCORE_KERNEL_FPU, 1, [kernel has __kernel_fpu_* functions]) AC_DEFINE(KERNEL_EXPORTS_X86_FPU, 1, [kernel exports FPU functions]) ],[ - AC_MSG_RESULT(internal) - AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1, - [kernel fpu internal]) + dnl # + dnl # ARM neon symbols (only on arm and arm64) + dnl # could be GPL-only on arm64 after Linux 6.2 + dnl # + ZFS_LINUX_TEST_RESULT([kernel_neon_license],[ + AC_MSG_RESULT(kernel_neon_*) + AC_DEFINE(HAVE_KERNEL_NEON, 1, + [kernel has kernel_neon_* functions]) + ],[ + # catch-all + AC_MSG_RESULT(internal) + AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1, + [kernel fpu internal]) + ]) ]) ]) ]) diff --git a/include/os/linux/kernel/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h index 16276b08c759..123a0c72bc6a 100644 --- a/include/os/linux/kernel/linux/simd_aarch64.h +++ b/include/os/linux/kernel/linux/simd_aarch64.h @@ -1,116 +1,122 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or https://opensource.org/licenses/CDDL-1.0. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau . * Copyright (C) 2022 Tino Reichardt * Copyright (C) 2022 Sebastian Gottschall */ /* * USER API: * * Kernel fpu methods: * kfpu_allowed() * kfpu_begin() * kfpu_end() * kfpu_init() * kfpu_fini() * * SIMD support: * * Following functions should be called to determine whether CPU feature * is supported. All functions are usable in kernel and user space. * If a SIMD algorithm is using more than one instruction set * all relevant feature test functions should be called. * * Supported features: * zfs_neon_available() * zfs_sha256_available() * zfs_sha512_available() */ #ifndef _LINUX_SIMD_AARCH64_H #define _LINUX_SIMD_AARCH64_H #include #include #include #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) #include #else #define sys_reg(op0, op1, crn, crm, op2) ( \ ((op0) << Op0_shift) | \ ((op1) << Op1_shift) | \ ((crn) << CRn_shift) | \ ((crm) << CRm_shift) | \ ((op2) << Op2_shift)) #endif #define ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 1, 0) #define ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0) +#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON)) #define kfpu_allowed() 1 #define kfpu_begin() kernel_neon_begin() #define kfpu_end() kernel_neon_end() +#else +#define kfpu_allowed() 0 +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) +#endif #define kfpu_init() (0) #define kfpu_fini() do {} while (0) #define get_ftr(id) { \ unsigned long __val; \ asm("mrs %0, "#id : "=r" (__val)); \ __val; \ } /* * Check if NEON is available */ static inline boolean_t zfs_neon_available(void) { unsigned long ftr = ((get_ftr(ID_AA64PFR0_EL1)) >> 16) & 0xf; return (ftr == 0 || ftr == 1); } /* * Check if SHA256 is available */ static inline boolean_t zfs_sha256_available(void) { unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; return (ftr & 0x1); } /* * Check if SHA512 is available */ static inline boolean_t zfs_sha512_available(void) { unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; return (ftr & 0x2); } #endif /* _LINUX_SIMD_AARCH64_H */ diff --git a/include/os/linux/kernel/linux/simd_arm.h b/include/os/linux/kernel/linux/simd_arm.h index c432a6d4abd1..bc70eaef3073 100644 --- a/include/os/linux/kernel/linux/simd_arm.h +++ b/include/os/linux/kernel/linux/simd_arm.h @@ -1,80 +1,86 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or https://opensource.org/licenses/CDDL-1.0. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2022 Tino Reichardt */ /* * USER API: * * Kernel fpu methods: * kfpu_allowed() * kfpu_begin() * kfpu_end() * kfpu_init() * kfpu_fini() * * SIMD support: * * Following functions should be called to determine whether CPU feature * is supported. All functions are usable in kernel and user space. * If a SIMD algorithm is using more than one instruction set * all relevant feature test functions should be called. * * Supported features: * zfs_neon_available() * zfs_sha256_available() */ #ifndef _LINUX_SIMD_ARM_H #define _LINUX_SIMD_ARM_H #include #include #include #include +#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON)) #define kfpu_allowed() 1 #define kfpu_begin() kernel_neon_begin() #define kfpu_end() kernel_neon_end() +#else +#define kfpu_allowed() 0 +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) +#endif #define kfpu_init() (0) #define kfpu_fini() do {} while (0) /* * Check if NEON is available */ static inline boolean_t zfs_neon_available(void) { return (elf_hwcap & HWCAP_NEON); } /* * Check if SHA256 is available */ static inline boolean_t zfs_sha256_available(void) { return (elf_hwcap2 & HWCAP2_SHA2); } #endif /* _LINUX_SIMD_ARM_H */