diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index c12ab9db030a..a342242ac66e 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -1,453 +1,454 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # # common files stuff between i386 and amd64 include "conf/files.x86" # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # # elf-vdso.so.o standard \ dependency "$S/amd64/amd64/sigtramp.S assym.inc $S/conf/vdso_amd64.ldscript $S/tools/amd64_vdso.sh" \ compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_vdso.sh" \ no-ctfconvert \ no-implicit-rule before-depend \ clean "elf-vdso.so.o elf-vdso.so.1 vdso_offsets.h sigtramp.pico" # elf-vdso32.so.o optional compat_freebsd32 \ dependency "$S/amd64/ia32/ia32_sigtramp.S ia32_assym.h $S/conf/vdso_amd64_ia32.ldscript $S/tools/amd64_ia32_vdso.sh" \ compile-with "env AWK='${AWK}' NM='${NM}' LD='${LD}' CC='${CC}' OBJCOPY='${OBJCOPY}' ELFDUMP='${ELFDUMP}' S='${S}' sh $S/tools/amd64_ia32_vdso.sh" \ no-ctfconvert \ no-implicit-rule before-depend \ clean "elf-vdso32.so.o elf-vdso32.so.1 vdso_ia32_offsets.h ia32_sigtramp.pico" # ia32_genassym.o standard \ dependency "$S/compat/ia32/ia32_genassym.c offset.inc" \ compile-with "${CC} ${CFLAGS:N-flto*:N-fno-common:N-fsanitize*:N-fno-sanitize*} -fcommon -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "ia32_genassym.o" # ia32_assym.h standard \ dependency "$S/kern/genassym.sh ia32_genassym.o" \ compile-with "env NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh ia32_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "ia32_assym.h" # amd64/acpica/acpi_machdep.c optional acpi amd64/acpica/acpi_wakeup.c optional acpi acpi_wakecode.o optional acpi \ dependency "$S/amd64/acpica/acpi_wakecode.S assym.inc" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # #amd64/amd64/apic_vector.S standard amd64/amd64/bios.c standard amd64/amd64/bpf_jit_machdep.c optional bpf_jitter amd64/amd64/copyout.c standard amd64/amd64/cpu_switch.S standard amd64/amd64/db_disasm.c optional ddb amd64/amd64/db_interface.c optional ddb amd64/amd64/db_trace.c optional ddb amd64/amd64/efirt_machdep.c optional efirt amd64/amd64/efirt_support.S optional efirt amd64/amd64/elf_machdep.c standard amd64/amd64/exception.S standard amd64/amd64/exec_machdep.c standard amd64/amd64/fpu.c standard amd64/amd64/gdb_machdep.c optional gdb amd64/amd64/initcpu.c standard amd64/amd64/io.c optional io amd64/amd64/locore.S standard no-obj amd64/amd64/xen-locore.S optional xenhvm \ compile-with "${NORMAL_S} -g0" \ no-ctfconvert amd64/amd64/machdep.c standard amd64/amd64/mem.c optional mem amd64/amd64/minidump_machdep.c standard amd64/amd64/mp_machdep.c standard amd64/amd64/mpboot.S standard amd64/amd64/pmap.c standard amd64/amd64/ptrace_machdep.c standard amd64/amd64/support.S standard amd64/amd64/sys_machdep.c standard amd64/amd64/trap.c standard amd64/amd64/uio_machdep.c standard amd64/amd64/vm_machdep.c standard amd64/pci/pci_cfgreg.c optional pci cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb crypto/openssl/amd64/aes-gcm-avx512.S optional ossl crypto/openssl/amd64/aesni-x86_64.S optional ossl crypto/openssl/amd64/aesni-gcm-x86_64.S optional ossl crypto/openssl/amd64/chacha-x86_64.S optional ossl crypto/openssl/amd64/ghash-x86_64.S optional ossl crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl crypto/openssl/amd64/sha256-x86_64.S optional ossl crypto/openssl/amd64/sha512-x86_64.S optional ossl -crypto/openssl/amd64/ossl_aes_gcm.c optional ossl +crypto/openssl/amd64/ossl_aes_gcm_avx512.c optional ossl +crypto/openssl/ossl_aes_gcm.c optional ossl dev/amdgpio/amdgpio.c optional amdgpio dev/axgbe/if_axgbe_pci.c optional axp dev/axgbe/xgbe-desc.c optional axp dev/axgbe/xgbe-dev.c optional axp dev/axgbe/xgbe-drv.c optional axp dev/axgbe/xgbe-mdio.c optional axp dev/axgbe/xgbe-sysctl.c optional axp dev/axgbe/xgbe-txrx.c optional axp dev/axgbe/xgbe_osdep.c optional axp dev/axgbe/xgbe-i2c.c optional axp dev/axgbe/xgbe-phy-v2.c optional axp dev/enic/enic_res.c optional enic dev/enic/enic_txrx.c optional enic dev/enic/if_enic.c optional enic dev/enic/vnic_cq.c optional enic dev/enic/vnic_dev.c optional enic dev/enic/vnic_intr.c optional enic dev/enic/vnic_rq.c optional enic dev/enic/vnic_wq.c optional enic dev/ftgpio/ftgpio.c optional ftgpio superio dev/hyperv/vmbus/amd64/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/amd64/vmbus_vector.S optional hyperv dev/hyperv/vmbus/hyperv_mmu.c optional hyperv dev/iavf/if_iavf_iflib.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_lib.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_osdep.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_txrx_iflib.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_common.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_adminq.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_vc_common.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/iavf/iavf_vc_iflib.c optional iavf pci \ compile-with "${NORMAL_C} -I$S/dev/iavf" dev/ice/if_ice_iflib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_lib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_osdep.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_resmgr.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_strings.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_recovery_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_common.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_controlq.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_dcb.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flex_pipe.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flow.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_nvm.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_sched.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_switch.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_vlan_mode.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fw_logging.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fwlog.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_rdma.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/irdma_if.m optional ice pci \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/irdma_di_if.m optional ice pci \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/ice_ddp_common.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iov.c optional ice pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_vf_mbx.c optional ice pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ice" ice_ddp.c optional ice_ddp \ compile-with "${AWK} -f $S/tools/fw_stub.awk ice_ddp.fw:ice_ddp:0x01032900 -mice_ddp -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "ice_ddp.c" ice_ddp.fwo optional ice_ddp \ dependency "ice_ddp.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ice_ddp.fwo" ice_ddp.fw optional ice_ddp \ dependency "$S/contrib/dev/ice/ice-1.3.41.0.pkg" \ compile-with "${CP} $S/contrib/dev/ice/ice-1.3.41.0.pkg ice_ddp.fw" \ no-obj no-implicit-rule \ clean "ice_ddp.fw" dev/ioat/ioat.c optional ioat pci dev/ioat/ioat_test.c optional ioat pci dev/ixl/if_ixl.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_main.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iflib.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_qmgr.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iov.c optional ixl pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_i2c.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_txrx.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_osdep.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_lan_hmc.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_hmc.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_common.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_nvm.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_adminq.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_dcb.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ncthwm/ncthwm.c optional ncthwm superio dev/qlxge/qls_dbg.c optional qlxge pci dev/qlxge/qls_dump.c optional qlxge pci dev/qlxge/qls_hw.c optional qlxge pci dev/qlxge/qls_ioctl.c optional qlxge pci dev/qlxge/qls_isr.c optional qlxge pci dev/qlxge/qls_os.c optional qlxge pci dev/qlxgb/qla_dbg.c optional qlxgb pci dev/qlxgb/qla_hw.c optional qlxgb pci dev/qlxgb/qla_ioctl.c optional qlxgb pci dev/qlxgb/qla_isr.c optional qlxgb pci dev/qlxgb/qla_misc.c optional qlxgb pci dev/qlxgb/qla_os.c optional qlxgb pci dev/qlxgbe/ql_dbg.c optional qlxgbe pci dev/qlxgbe/ql_hw.c optional qlxgbe pci dev/qlxgbe/ql_ioctl.c optional qlxgbe pci dev/qlxgbe/ql_isr.c optional qlxgbe pci dev/qlxgbe/ql_misc.c optional qlxgbe pci dev/qlxgbe/ql_os.c optional qlxgbe pci dev/qlxgbe/ql_reset.c optional qlxgbe pci dev/qlxgbe/ql_fw.c optional qlxgbe pci dev/qlxgbe/ql_boot.c optional qlxgbe pci dev/qlxgbe/ql_minidump.c optional qlxgbe pci dev/qlnx/qlnxe/ecore_cxt.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dbg_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dcbx.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dev.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_hw.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_ops.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_int.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_l2.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_mcp.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_sp_commands.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_spq.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_sriov.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_vf.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_ll2.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_iwarp.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_rdma.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_roce.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_ooo.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_rdma.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_ioctl.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_os.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/sfxge/common/ef10_ev.c optional sfxge pci dev/sfxge/common/ef10_filter.c optional sfxge pci dev/sfxge/common/ef10_image.c optional sfxge pci dev/sfxge/common/ef10_intr.c optional sfxge pci dev/sfxge/common/ef10_mac.c optional sfxge pci dev/sfxge/common/ef10_mcdi.c optional sfxge pci dev/sfxge/common/ef10_nic.c optional sfxge pci dev/sfxge/common/ef10_nvram.c optional sfxge pci dev/sfxge/common/ef10_phy.c optional sfxge pci dev/sfxge/common/ef10_rx.c optional sfxge pci dev/sfxge/common/ef10_tx.c optional sfxge pci dev/sfxge/common/ef10_vpd.c optional sfxge pci dev/sfxge/common/efx_bootcfg.c optional sfxge pci dev/sfxge/common/efx_crc32.c optional sfxge pci dev/sfxge/common/efx_ev.c optional sfxge pci dev/sfxge/common/efx_filter.c optional sfxge pci dev/sfxge/common/efx_hash.c optional sfxge pci dev/sfxge/common/efx_intr.c optional sfxge pci dev/sfxge/common/efx_lic.c optional sfxge pci dev/sfxge/common/efx_mac.c optional sfxge pci dev/sfxge/common/efx_mcdi.c optional sfxge pci dev/sfxge/common/efx_mon.c optional sfxge pci dev/sfxge/common/efx_nic.c optional sfxge pci dev/sfxge/common/efx_nvram.c optional sfxge pci dev/sfxge/common/efx_phy.c optional sfxge pci dev/sfxge/common/efx_port.c optional sfxge pci dev/sfxge/common/efx_rx.c optional sfxge pci dev/sfxge/common/efx_sram.c optional sfxge pci dev/sfxge/common/efx_tunnel.c optional sfxge pci dev/sfxge/common/efx_tx.c optional sfxge pci dev/sfxge/common/efx_vpd.c optional sfxge pci dev/sfxge/common/hunt_nic.c optional sfxge pci dev/sfxge/common/mcdi_mon.c optional sfxge pci dev/sfxge/common/medford_nic.c optional sfxge pci dev/sfxge/common/medford2_nic.c optional sfxge pci dev/sfxge/common/siena_mac.c optional sfxge pci dev/sfxge/common/siena_mcdi.c optional sfxge pci dev/sfxge/common/siena_nic.c optional sfxge pci dev/sfxge/common/siena_nvram.c optional sfxge pci dev/sfxge/common/siena_phy.c optional sfxge pci dev/sfxge/common/siena_sram.c optional sfxge pci dev/sfxge/common/siena_vpd.c optional sfxge pci dev/sfxge/sfxge.c optional sfxge pci dev/sfxge/sfxge_dma.c optional sfxge pci dev/sfxge/sfxge_ev.c optional sfxge pci dev/sfxge/sfxge_intr.c optional sfxge pci dev/sfxge/sfxge_mcdi.c optional sfxge pci dev/sfxge/sfxge_nvram.c optional sfxge pci dev/sfxge/sfxge_port.c optional sfxge pci dev/sfxge/sfxge_rx.c optional sfxge pci dev/sfxge/sfxge_tx.c optional sfxge pci dev/smartpqi/smartpqi_cam.c optional smartpqi dev/smartpqi/smartpqi_cmd.c optional smartpqi dev/smartpqi/smartpqi_discovery.c optional smartpqi dev/smartpqi/smartpqi_event.c optional smartpqi dev/smartpqi/smartpqi_features.c optional smartpqi dev/smartpqi/smartpqi_helper.c optional smartpqi dev/smartpqi/smartpqi_init.c optional smartpqi dev/smartpqi/smartpqi_intr.c optional smartpqi dev/smartpqi/smartpqi_ioctl.c optional smartpqi dev/smartpqi/smartpqi_main.c optional smartpqi dev/smartpqi/smartpqi_mem.c optional smartpqi dev/smartpqi/smartpqi_misc.c optional smartpqi dev/smartpqi/smartpqi_queue.c optional smartpqi dev/smartpqi/smartpqi_request.c optional smartpqi dev/smartpqi/smartpqi_response.c optional smartpqi dev/smartpqi/smartpqi_sis.c optional smartpqi dev/smartpqi/smartpqi_tag.c optional smartpqi dev/sume/if_sume.c optional sume dev/syscons/apm/apm_saver.c optional apm_saver apm dev/tpm/tpm_crb.c optional tpm acpi dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/p2sb/p2sb.c optional p2sb pci dev/p2sb/lewisburg_gpiocm.c optional lbggpiocm p2sb dev/p2sb/lewisburg_gpio.c optional lbggpio lbggpiocm kern/link_elf_obj.c standard # # IA32 binary support # #amd64/ia32/ia32_exception.S optional compat_freebsd32 amd64/ia32/ia32_reg.c optional compat_freebsd32 amd64/ia32/ia32_signal.c optional compat_freebsd32 amd64/ia32/ia32_syscall.c optional compat_freebsd32 amd64/ia32/ia32_misc.c optional compat_freebsd32 compat/ia32/ia32_sysvec.c optional compat_freebsd32 # # x86 real mode BIOS emulator, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa contrib/x86emu/x86emu.c optional x86bios | dpms | pci | vesa # Common files where we currently configure the system differently, but perhaps shouldn't # config(8) doesn't have a way to force standard options, so we've been inconsistent # about marking non-optional things 'standard'. x86/acpica/madt.c optional acpi x86/isa/atpic.c optional atpic isa x86/isa/elcr.c optional atpic isa | mptable x86/isa/isa.c standard x86/isa/isa_dma.c standard x86/pci/pci_early_quirks.c optional pci x86/x86/io_apic.c standard x86/x86/local_apic.c standard x86/x86/mptable.c optional mptable x86/x86/mptable_pci.c optional mptable pci x86/x86/msi.c optional pci x86/xen/pv.c optional xenhvm # zfs blake3 hash support contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_avx2.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_avx512.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_sse2.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-x86_64/blake3/blake3_sse41.S optional zfs compile-with "${ZFS_S}" # zfs AVX2 implementation of aes-gcm from BoringSSL contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S optional zfs compile-with "${ZFS_S}" # zfs sha2 hash support zfs-sha256-x86_64.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S" \ no-implicit-rule \ clean "zfs-sha256-x86_64.o" zfs-sha512-x86_64.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S" \ no-implicit-rule \ clean "zfs-sha512-x86_64.o" # zfs checksums / zcommon contrib/openzfs/module/zcommon/zfs_fletcher_avx512.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zcommon/zfs_fletcher_intel.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zcommon/zfs_fletcher_sse.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zfs/vdev_raidz_math_avx2.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zfs/vdev_raidz_math_avx512bw.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zfs/vdev_raidz_math_avx512f.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zfs/vdev_raidz_math_sse2.c optional zfs compile-with "${ZFS_C}" contrib/openzfs/module/zfs/vdev_raidz_math_ssse3.c optional zfs compile-with "${ZFS_C}" # Clock calibration subroutine; uses floating-point arithmetic subr_clockcalib.o standard \ dependency "$S/kern/subr_clockcalib.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} -mmmx -msse -msse2 ${.IMPSRC}" \ no-implicit-rule \ clean "subr_clockcalib.o" diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 91b01845519e..880e804b6c95 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -1,160 +1,160 @@ arm/arm/autoconf.c standard arm/arm/bcopy_page.S standard arm/arm/bcopyinout.S standard arm/arm/blockio.S standard arm/arm/bus_space_asm_generic.S standard arm/arm/bus_space_base.c optional fdt arm/arm/bus_space_generic.c standard arm/arm/busdma_machdep.c standard arm/arm/copystr.S standard arm/arm/cpufunc.c standard arm/arm/cpufunc_asm.S standard arm/arm/cpufunc_asm_armv7.S optional cpu_cortexa | cpu_krait | cpu_mv_pj4b arm/arm/cpufunc_asm_pj4b.S optional cpu_mv_pj4b arm/arm/cpuinfo.c standard arm/arm/cpu_asm-v6.S standard arm/arm/db_disasm.c optional ddb arm/arm/db_interface.c optional ddb arm/arm/db_trace.c optional ddb arm/arm/debug_monitor.c optional ddb | gdb arm/arm/disassem.c optional ddb arm/arm/dump_machdep.c standard arm/arm/elf_machdep.c standard arm/arm/elf_note.S standard arm/arm/exception.S standard arm/arm/exec_machdep.c standard arm/arm/fusu.S standard arm/arm/gdb_machdep.c optional gdb arm/arm/generic_timer.c optional generic_timer arm/arm/gic.c optional gic arm/arm/gic_fdt.c optional gic fdt arm/arm/gic_if.m optional gic arm/arm/identcpu-v6.c standard arm/arm/in_cksum_arm.S optional inet | inet6 arm/arm/in_cksum_machdep.c optional inet | inet6 kern/subr_intr.c standard arm/arm/locore.S standard no-obj arm/arm/hypervisor-stub.S standard arm/arm/machdep.c standard arm/arm/machdep_boot.c standard arm/arm/machdep_kdb.c standard arm/arm/machdep_intr.c standard arm/arm/machdep_ptrace.c standard arm/arm/mem.c optional mem arm/arm/minidump_machdep.c standard arm/arm/mp_machdep.c optional smp arm/arm/mpcore_timer.c optional mpcore_timer arm/arm/nexus.c standard arm/arm/ofw_machdep.c optional fdt arm/arm/pl190.c optional pl190 arm/arm/pl310.c optional pl310 arm/arm/platform.c optional platform arm/arm/platform_if.m optional platform arm/arm/platform_pl310_if.m optional platform pl310 arm/arm/pmap-v6.c standard arm/arm/pmu.c optional pmu | hwpmc arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc arm/arm/ptrace_machdep.c standard arm/arm/sc_machdep.c optional sc arm/arm/sdt_machdep.c optional kdtrace_hooks arm/arm/setcpsr.S standard arm/arm/setstack.S standard arm/arm/stack_machdep.c optional ddb | stack arm/arm/stdatomic.c standard \ compile-with "${NORMAL_C:N-Wmissing-prototypes}" arm/arm/support.S standard arm/arm/swtch.S standard arm/arm/swtch-v6.S standard arm/arm/sys_machdep.c standard arm/arm/syscall.c standard arm/arm/trap-v6.c standard arm/arm/uio_machdep.c standard arm/arm/undefined.c standard arm/arm/unwind.c optional ddb | kdtrace_hooks | stack arm/arm/vm_machdep.c standard arm/arm/vfp.c optional vfp cddl/dev/dtrace/arm/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/arm/dtrace_isa.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/arm/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/arm/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" # zfs sha2 hash support contrib/openzfs/module/icp/asm-arm/sha2/sha256-armv7.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-arm/sha2/sha512-armv7.S optional zfs compile-with "${ZFS_S}" crypto/des/des_enc.c optional netsmb dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/fb/fb.c optional sc dev/fdt/fdt_arm_platform.c optional platform fdt dev/video/crtc_if.m optional vt dev/hwpmc/hwpmc_arm.c optional hwpmc dev/hwpmc/hwpmc_armv7.c optional hwpmc dev/iicbus/controller/twsi/twsi.c optional twsi dev/ofw/ofw_pcib.c optional fdt pci dev/pci/pci_host_generic.c optional pci_host_generic pci dev/pci/pci_host_generic_fdt.c optional pci_host_generic pci fdt dev/psci/psci.c optional psci dev/psci/smccc_arm.S optional psci dev/syscons/scgfbrndr.c optional sc dev/uart/uart_cpu_fdt.c optional uart fdt dev/usb/controller/dwc3/dwc3.c optional fdt dwc3 dev/usb/controller/generic_xhci.c optional xhci dev/usb/controller/generic_xhci_fdt.c optional xhci fdt kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_busdma_bufalloc.c standard kern/subr_devmap.c standard kern/subr_efi_map.c optional efi kern/subr_physmem.c standard kern/subr_sfbuf.c standard libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard libkern/arm/ldivmod.S standard libkern/arm/ldivmod_helper.c standard libkern/arm/memclr.S standard libkern/arm/memcpy.S standard libkern/arm/memset.S standard libkern/arm/muldi3.c standard libkern/ashldi3.c standard libkern/ashrdi3.c standard libkern/divdi3.c standard libkern/lshrdi3.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/strcmp.c standard libkern/strlen.c standard libkern/strncmp.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard crypto/openssl/ossl_arm.c optional ossl -crypto/openssl/arm/ossl_aes_gcm.c optional ossl +crypto/openssl/arm/ossl_aes_gcm_neon.c optional ossl crypto/openssl/arm/aes-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/bsaes-armv7.S optional ossl \ compile-with "${CC} -D__KERNEL__ -c ${CFLAGS:N-mgeneral-regs-only} -I${SRCTOP}/sys/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/arm/chacha-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/ghash-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/poly1305-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/sha1-armv4-large.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/sha256-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" crypto/openssl/arm/sha512-armv4.S optional ossl \ compile-with "${NORMAL_C} -I${SRCTOP}/sys/crypto/openssl" # Annapurna support arm/annapurna/alpine/alpine_ccu.c optional al_ccu fdt arm/annapurna/alpine/alpine_nb_service.c optional al_nb_service fdt arm/annapurna/alpine/alpine_pci.c optional al_pci fdt arm/annapurna/alpine/alpine_pci_msix.c optional al_pci fdt arm/annapurna/alpine/alpine_serdes.c optional al_serdes fdt \ no-depend \ compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${.IMPSRC}" diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index d2c3aa260cd9..0deada385f31 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -1,432 +1,435 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # # # There is only an asm version on ppc64. # zfs blake3 hash support contrib/openzfs/module/icp/asm-ppc64/blake3/b3_ppc64le_sse2.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-ppc64/blake3/b3_ppc64le_sse41.S optional zfs compile-with "${ZFS_S}" # zfs sha2 hash support contrib/openzfs/module/icp/asm-ppc64/sha2/sha256-p8.S optional zfs compile-with "${ZFS_S}" contrib/openzfs/module/icp/asm-ppc64/sha2/sha512-p8.S optional zfs compile-with "${ZFS_S}" zfs-sha256-ppc.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-ppc64/sha2/sha256-ppc.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-ppc64/sha2/sha256-ppc.S" \ no-implicit-rule \ clean "zfs-sha256-ppc.o" zfs-sha512-ppc.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-ppc64/sha2/sha512-ppc.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-ppc64/sha2/sha512-ppc.S" \ no-implicit-rule \ clean "zfs-sha512-ppc.o" # openssl ppc common files crypto/openssl/ossl_ppc.c optional ossl powerpc64 | ossl powerpc64le +crypto/openssl/ossl_aes_gcm.c optional ossl powerpc64 | ossl powerpc64le # openssl assembly files (powerpc64le) crypto/openssl/powerpc64le/aes-ppc.S optional ossl powerpc64le +crypto/openssl/powerpc64le/aes-gcm-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/aesp8-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/chacha-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/ecp_nistz256-ppc64.S optional ossl powerpc64le crypto/openssl/powerpc64le/ghashp8-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/keccak1600-ppc64.S optional ossl powerpc64le crypto/openssl/powerpc64le/poly1305-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/poly1305-ppcfp.S optional ossl powerpc64le crypto/openssl/powerpc64le/ppc-mont.S optional ossl powerpc64le crypto/openssl/powerpc64le/ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/ppccpuid.S optional ossl powerpc64le crypto/openssl/powerpc64le/sha1-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/sha256-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/sha256p8-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/sha512-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/sha512p8-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/vpaes-ppc.S optional ossl powerpc64le crypto/openssl/powerpc64le/x25519-ppc64.S optional ossl powerpc64le # openssl assembly files (powerpc64) crypto/openssl/powerpc64/aes-ppc.S optional ossl powerpc64 +crypto/openssl/powerpc64/aes-gcm-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/aesp8-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/chacha-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/ecp_nistz256-ppc64.S optional ossl powerpc64 crypto/openssl/powerpc64/ghashp8-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/keccak1600-ppc64.S optional ossl powerpc64 crypto/openssl/powerpc64/poly1305-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/poly1305-ppcfp.S optional ossl powerpc64 crypto/openssl/powerpc64/ppc-mont.S optional ossl powerpc64 crypto/openssl/powerpc64/ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/ppccpuid.S optional ossl powerpc64 crypto/openssl/powerpc64/sha1-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/sha256-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/sha256p8-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/sha512-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/sha512p8-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/vpaes-ppc.S optional ossl powerpc64 crypto/openssl/powerpc64/x25519-ppc64.S optional ossl powerpc64 cddl/compat/opensolaris/kern/opensolaris_atomic.c optional zfs powerpc | dtrace powerpc | zfs powerpcspe | dtrace powerpcspe compile-with "${ZFS_C}" cddl/dev/dtrace/powerpc/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/powerpc/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/powerpc/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/des/des_enc.c optional netsmb dev/aacraid/aacraid_endian.c optional aacraid dev/adb/adb_bus.c optional adb dev/adb/adb_kbd.c optional adb dev/adb/adb_mouse.c optional adb dev/adb/adb_hb_if.m optional adb dev/adb/adb_if.m optional adb dev/adb/adb_buttons.c optional adb dev/agp/agp_apple.c optional agp powermac dev/fb/fb.c optional sc dev/gpio/qoriq_gpio.c optional mpc85xx gpio dev/hwpmc/hwpmc_e500.c optional hwpmc dev/hwpmc/hwpmc_mpc7xxx.c optional hwpmc dev/hwpmc/hwpmc_power8.c optional hwpmc dev/hwpmc/hwpmc_powerpc.c optional hwpmc dev/hwpmc/hwpmc_ppc970.c optional hwpmc dev/iicbus/adc/ad7417.c optional ad7417 powermac dev/iicbus/ofw_iicbus.c optional iicbus aim dev/iicbus/ofw_iicbus_if.m optional iicbus aim dev/iicbus/pwm/adm1030.c optional powermac windtunnel | adm1030 powermac dev/iicbus/pwm/adt746x.c optional adt746x powermac dev/iicbus/sensor/ds1631.c optional ds1631 powermac dev/iicbus/sensor/ds1775.c optional ds1775 powermac dev/iicbus/sensor/max6690.c optional max6690 powermac dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_opal.c optional powernv ipmi dev/ice/if_ice_iflib.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_lib.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_osdep.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_resmgr.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_strings.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_recovery_txrx.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_txrx.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_common.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_controlq.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_dcb.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flex_pipe.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flow.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_nvm.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_sched.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_switch.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_vlan_mode.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fw_logging.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fwlog.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_rdma.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/irdma_if.m optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/irdma_di_if.m optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/ice_ddp_common.c optional ice pci powerpc64 | ice pci powerpc64le \ compile-with "${NORMAL_C} -I$S/dev/ice" ice_ddp.c optional ice_ddp powerpc64 | ice pci powerpc64le \ compile-with "${AWK} -f $S/tools/fw_stub.awk ice_ddp.fw:ice_ddp:0x01032900 -mice_ddp -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "ice_ddp.c" ice_ddp.fwo optional ice_ddp powerpc64 | ice pci powerpc64le \ dependency "ice_ddp.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ice_ddp.fwo" ice_ddp.fw optional ice_ddp powerpc64 | ice pci powerpc64le \ dependency "$S/contrib/dev/ice/ice-1.3.41.0.pkg" \ compile-with "${CP} $S/contrib/dev/ice/ice-1.3.41.0.pkg ice_ddp.fw" \ no-obj no-implicit-rule \ clean "ice_ddp.fw" dev/ixl/if_ixl.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_main.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iflib.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_qmgr.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iov.c optional ixl pci pci_iov powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_i2c.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/if_iavf.c optional iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/iavf_vc.c optional iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_txrx.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_osdep.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_lan_hmc.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_hmc.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_common.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_nvm.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_adminq.c optional ixl pci powerpc64 | \ iavf pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_dcb.c optional ixl pci powerpc64 \ compile-with "${NORMAL_C} -I$S/dev/ixl" # Most ofw stuff below is brought in by conf/files for options FDT, but # we always want it, even on non-FDT platforms. dev/fdt/simplebus.c standard dev/ofw/openfirm.c standard dev/ofw/openfirmio.c standard dev/ofw/ofw_bus_if.m standard dev/ofw/ofw_cpu.c standard dev/ofw/ofw_if.m standard dev/ofw/ofw_bus_subr.c standard dev/ofw/ofw_console.c optional aim dev/ofw/ofw_disk.c optional ofwd aim dev/ofw/ofwbus.c standard dev/ofw/ofw_pcib.c optional pci dev/ofw/ofw_standard.c optional aim powerpc dev/ofw/ofw_subr.c standard dev/powermac_nvram/powermac_nvram.c optional powermac_nvram powermac dev/quicc/quicc_bfe_fdt.c optional quicc mpc85xx dev/random/darn.c optional powerpc64 !random_loadable | powerpc64le !random_loadable dev/scc/scc_bfe_macio.c optional scc powermac dev/sdhci/fsl_sdhci.c optional mpc85xx sdhci dev/sec/sec.c optional sec mpc85xx dev/sound/macio/aoa.c optional snd_davbus | snd_ai2s powermac dev/sound/macio/davbus.c optional snd_davbus powermac dev/sound/macio/i2s.c optional snd_ai2s powermac dev/sound/macio/onyx.c optional snd_ai2s iicbus powermac dev/sound/macio/snapper.c optional snd_ai2s iicbus powermac dev/sound/macio/tumbler.c optional snd_ai2s iicbus powermac dev/syscons/scgfbrndr.c optional sc dev/tsec/if_tsec.c optional tsec dev/tsec/if_tsec_fdt.c optional tsec dev/uart/uart_cpu_powerpc.c optional uart dev/usb/controller/ehci_fsl.c optional ehci mpc85xx dev/vt/hw/ofwfb/ofwfb.c optional vt aim kern/subr_atomic64.c optional powerpc | powerpcspe kern/subr_dummy_vdso_tc.c standard kern/subr_sfbuf.c standard libkern/ashldi3.c optional powerpc | powerpcspe libkern/ashrdi3.c optional powerpc | powerpcspe libkern/bcopy.c standard libkern/cmpdi2.c optional powerpc | powerpcspe libkern/divdi3.c optional powerpc | powerpcspe libkern/lshrdi3.c optional powerpc | powerpcspe libkern/memcmp.c standard libkern/memset.c standard libkern/moddi3.c optional powerpc | powerpcspe libkern/qdivrem.c optional powerpc | powerpcspe libkern/strcmp.c standard libkern/strlen.c standard libkern/strncmp.c standard libkern/ucmpdi2.c optional powerpc | powerpcspe libkern/udivdi3.c optional powerpc | powerpcspe libkern/umoddi3.c optional powerpc | powerpcspe powerpc/aim/locore.S optional aim no-obj powerpc/aim/aim_machdep.c optional aim powerpc/aim/mmu_oea.c optional aim powerpc powerpc/aim/mmu_oea64.c optional aim powerpc/aim/mmu_radix.c optional aim powerpc64 | aim powerpc64le powerpc/aim/moea64_native.c optional aim powerpc/aim/mp_cpudep.c optional aim powerpc/aim/slb.c optional aim powerpc64 | aim powerpc64le powerpc/amigaone/platform_amigaone.c optional amigaone powerpc/amigaone/cpld_a1222.c optional powerpc amigaone | powerpcspe amigaone powerpc/amigaone/cpld_x5000.c optional powerpc amigaone | powerpc64 amigaone powerpc/booke/locore.S optional booke no-obj powerpc/booke/booke_machdep.c optional booke powerpc/booke/machdep_e500.c optional booke_e500 powerpc/booke/mp_cpudep.c optional booke smp powerpc/booke/platform_bare.c optional booke powerpc/booke/pmap.c optional booke powerpc/booke/spe.c optional powerpcspe powerpc/cpufreq/dfs.c optional cpufreq powerpc/cpufreq/mpc85xx_jog.c optional cpufreq mpc85xx powerpc/cpufreq/pcr.c optional cpufreq aim powerpc/cpufreq/pmcr.c optional cpufreq aim powerpc64 | cpufreq aim powerpc64le powerpc/cpufreq/pmufreq.c optional cpufreq aim pmu powerpc/fpu/fpu_add.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_compare.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_div.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_emu.c optional fpu_emu powerpc/fpu/fpu_explode.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_implode.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_mul.c optional fpu_emu | powerpcspe powerpc/fpu/fpu_sqrt.c optional fpu_emu powerpc/fpu/fpu_subr.c optional fpu_emu | powerpcspe powerpc/mambo/mambocall.S optional mambo powerpc/mambo/mambo.c optional mambo powerpc/mambo/mambo_console.c optional mambo powerpc/mambo/mambo_disk.c optional mambo powerpc/mikrotik/platform_rb.c optional mikrotik powerpc/mikrotik/rb_led.c optional mikrotik powerpc/mpc85xx/atpic.c optional mpc85xx isa powerpc/mpc85xx/ds1553_bus_fdt.c optional ds1553 powerpc/mpc85xx/ds1553_core.c optional ds1553 powerpc/mpc85xx/fsl_diu.c optional mpc85xx diu powerpc/mpc85xx/fsl_espi.c optional mpc85xx spibus powerpc/mpc85xx/fsl_sata.c optional mpc85xx ata powerpc/mpc85xx/i2c.c optional mpc85xx iicbus powerpc/mpc85xx/isa.c optional mpc85xx isa powerpc/mpc85xx/lbc.c optional mpc85xx powerpc/mpc85xx/mpc85xx.c optional mpc85xx powerpc/mpc85xx/mpc85xx_cache.c optional mpc85xx powerpc/mpc85xx/mpc85xx_gpio.c optional mpc85xx gpio powerpc/mpc85xx/platform_mpc85xx.c optional mpc85xx powerpc/mpc85xx/pci_mpc85xx.c optional pci mpc85xx powerpc/mpc85xx/pci_mpc85xx_pcib.c optional pci mpc85xx powerpc/ofw/ofw_machdep.c standard powerpc/ofw/ofw_pcibus.c optional pci powerpc/ofw/ofw_pcib_pci.c optional pci powerpc/ofw/ofw_real.c optional aim powerpc/ofw/ofw_syscons.c optional sc aim powerpc/ofw/ofwcall32.S optional aim powerpc powerpc/ofw/ofwcall64.S optional aim powerpc64 | aim powerpc64le powerpc/ofw/openpic_ofw.c standard powerpc/ofw/rtas.c optional aim powerpc/ofw/ofw_initrd.c optional md_root_mem powerpc64 | md_root_mem powerpc64le powerpc/powermac/ata_kauai.c optional powermac ata | powermac atamacio powerpc/powermac/ata_macio.c optional powermac ata | powermac atamacio powerpc/powermac/ata_dbdma.c optional powermac ata | powermac atamacio powerpc/powermac/atibl.c optional powermac atibl powerpc/powermac/cuda.c optional powermac cuda powerpc/powermac/cpcht.c optional powermac pci powerpc/powermac/dbdma.c optional powermac pci powerpc/powermac/fcu.c optional powermac fcu powerpc/powermac/grackle.c optional powermac pci powerpc/powermac/hrowpic.c optional powermac pci powerpc/powermac/kiic.c optional powermac kiic powerpc/powermac/macgpio.c optional powermac pci powerpc/powermac/macio.c optional powermac pci powerpc/powermac/nvbl.c optional powermac nvbl powerpc/powermac/platform_powermac.c optional powermac powerpc/powermac/powermac_thermal.c optional powermac powerpc/powermac/pswitch.c optional powermac pswitch powerpc/powermac/pmu.c optional powermac pmu powerpc/powermac/smu.c optional powermac smu powerpc/powermac/smusat.c optional powermac smu powerpc/powermac/tbgpio.c optional powermac pci smp powerpc/powermac/uninorth.c optional powermac powerpc/powermac/uninorthpci.c optional powermac pci powerpc/powermac/vcoregpio.c optional powermac powerpc/powernv/opal.c optional powernv powerpc/powernv/opal_async.c optional powernv powerpc/powernv/opal_console.c optional powernv powerpc/powernv/opal_dbg.c optional powernv gdb powerpc/powernv/opal_dev.c optional powernv powerpc/powernv/opal_flash.c optional powernv opalflash powerpc/powernv/opal_hmi.c optional powernv powerpc/powernv/opal_i2c.c optional iicbus fdt powernv powerpc/powernv/opal_i2cm.c optional iicbus fdt powernv powerpc/powernv/opal_nvram.c optional powernv nvram powerpc/powernv/opal_pci.c optional powernv pci powerpc/powernv/opal_sensor.c optional powernv powerpc/powernv/opalcall.S optional powernv powerpc/powernv/platform_powernv.c optional powernv powerpc/powernv/powernv_centaur.c optional powernv powerpc/powernv/powernv_xscom.c optional powernv powerpc/powernv/xive.c optional powernv powerpc/powerpc/altivec.c optional !powerpcspe powerpc/powerpc/autoconf.c standard powerpc/powerpc/bus_machdep.c standard powerpc/powerpc/busdma_machdep.c standard powerpc/powerpc/clock.c standard powerpc/powerpc/copyinout.c optional aim powerpc/powerpc/cpu.c standard powerpc/powerpc/cpu_subr64.S optional powerpc64 | powerpc64le powerpc/powerpc/db_disasm.c optional ddb powerpc/powerpc/db_hwwatch.c optional ddb powerpc/powerpc/db_interface.c optional ddb powerpc/powerpc/db_trace.c optional ddb powerpc/powerpc/dump_machdep.c standard powerpc/powerpc/elf32_machdep.c optional powerpc | powerpcspe | compat_freebsd32 powerpc/powerpc/elf64_machdep.c optional powerpc64 | powerpc64le powerpc/powerpc/exec_machdep.c standard powerpc/powerpc/fpu.c standard powerpc/powerpc/gdb_machdep.c optional gdb powerpc/powerpc/interrupt.c standard powerpc/powerpc/intr_machdep.c standard powerpc/powerpc/iommu_if.m standard powerpc/powerpc/machdep.c standard powerpc/powerpc/mem.c optional mem powerpc/powerpc/minidump_machdep.c optional powerpc64 | powerpc64le powerpc/powerpc/mp_machdep.c optional smp powerpc/powerpc/nexus.c standard powerpc/powerpc/openpic.c standard powerpc/powerpc/pic_if.m standard powerpc/powerpc/pmap_dispatch.c standard powerpc/powerpc/platform.c standard powerpc/powerpc/platform_if.m standard powerpc/powerpc/ptrace_machdep.c standard powerpc/powerpc/sc_machdep.c optional sc powerpc/powerpc/sdt_machdep.c optional powerpc64 kdtrace_hooks powerpc/powerpc/setjmp.S standard powerpc/powerpc/sigcode32.S optional powerpc | powerpcspe | compat_freebsd32 powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le powerpc/powerpc/swtch32.S optional powerpc | powerpcspe powerpc/powerpc/swtch64.S optional powerpc64 | powerpc64le powerpc/powerpc/stack_machdep.c optional ddb | stack powerpc/powerpc/support.S optional powerpc64 | powerpc64le | booke powerpc/powerpc/syncicache.c standard powerpc/powerpc/sys_machdep.c standard powerpc/powerpc/trap.c standard powerpc/powerpc/uio_machdep.c standard powerpc/powerpc/uma_machdep.c standard powerpc/powerpc/vm_machdep.c standard powerpc/ps3/ehci_ps3.c optional ps3 ehci powerpc/ps3/ohci_ps3.c optional ps3 ohci powerpc/ps3/if_glc.c optional ps3 glc powerpc/ps3/mmu_ps3.c optional ps3 powerpc/ps3/platform_ps3.c optional ps3 powerpc/ps3/ps3bus.c optional ps3 powerpc/ps3/ps3cdrom.c optional ps3 scbus powerpc/ps3/ps3disk.c optional ps3 powerpc/ps3/ps3pic.c optional ps3 powerpc/ps3/ps3_syscons.c optional ps3 vt powerpc/ps3/ps3-hvcall.S optional ps3 powerpc/pseries/phyp-hvcall.S optional pseries powerpc64 | pseries powerpc64le powerpc/pseries/mmu_phyp.c optional pseries powerpc64 | pseries powerpc64le powerpc/pseries/phyp_console.c optional pseries powerpc64 uart | pseries powerpc64le uart powerpc/pseries/phyp_dbg.c optional pseries powerpc64 gdb | pseries powerpc64le gdb powerpc/pseries/phyp_llan.c optional llan powerpc/pseries/phyp_vscsi.c optional pseries powerpc64 scbus | pseries powerpc64le scbus powerpc/pseries/platform_chrp.c optional pseries powerpc/pseries/plpar_iommu.c optional pseries powerpc64 | pseries powerpc64le powerpc/pseries/plpar_pcibus.c optional pseries powerpc64 pci | pseries powerpc64le pci powerpc/pseries/rtas_dev.c optional pseries powerpc/pseries/rtas_pci.c optional pseries pci powerpc/pseries/vdevice.c optional pseries powerpc64 | pseries powerpc64le powerpc/pseries/xics.c optional pseries powerpc64 | pseries powerpc64le powerpc/psim/iobus.c optional psim powerpc/psim/ata_iobus.c optional ata psim powerpc/psim/openpic_iobus.c optional psim powerpc/psim/uart_iobus.c optional uart psim diff --git a/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c b/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c new file mode 100644 index 000000000000..694ed4fc8b32 --- /dev/null +++ b/sys/crypto/openssl/amd64/ossl_aes_gcm_avx512.c @@ -0,0 +1,232 @@ +/* + * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (c) 2021, Intel Corporation. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* + * This file contains an AES-GCM wrapper implementation from OpenSSL, using + * VAES extensions. It was ported from cipher_aes_gcm_hw_vaes_avx512.inc. + */ + +#include +#include + +#include +#include +#include + +#include + +_Static_assert( + sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context), + "ossl_gcm_context too large"); + +void aesni_set_encrypt_key(const void *key, int bits, void *ctx); + +static void +gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +{ + KASSERT(keylen == 128 || keylen == 192 || keylen == 256, + ("%s: invalid key length %zu", __func__, keylen)); + + memset(&ctx->gcm, 0, sizeof(ctx->gcm)); + memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); + aesni_set_encrypt_key(key, keylen, &ctx->aes_ks); + ctx->ops->init(ctx, key, keylen); +} + +static void +gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) +{ + (void)ctx->ops->finish(ctx, NULL, 0); + memcpy(tag, ctx->gcm.Xi.c, len); +} + +void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx); +void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); +void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, + const unsigned char *iv, size_t ivlen); +void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, + size_t len); +void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx, + unsigned int *pblocklen, const unsigned char *in, size_t len, + unsigned char *out); +void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx, + unsigned int *pblocklen, const unsigned char *in, size_t len, + unsigned char *out); +void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); + +static void +gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +{ + ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm); +} + +static void +gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv, + size_t len) +{ + KASSERT(len == AES_GCM_IV_LEN, + ("%s: invalid IV length %zu", __func__, len)); + + ctx->gcm.Yi.u[0] = 0; /* Current counter */ + ctx->gcm.Yi.u[1] = 0; + ctx->gcm.Xi.u[0] = 0; /* AAD hash */ + ctx->gcm.Xi.u[1] = 0; + ctx->gcm.len.u[0] = 0; /* AAD length */ + ctx->gcm.len.u[1] = 0; /* Message length */ + ctx->gcm.ares = 0; + ctx->gcm.mres = 0; + + ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len); +} + +static int +gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad, + size_t len) +{ + uint64_t alen = ctx->gcm.len.u[0]; + size_t lenblks; + unsigned int ares; + + /* Bad sequence: call of AAD update after message processing */ + if (ctx->gcm.len.u[1]) + return -2; + + alen += len; + /* AAD is limited by 2^64 bits, thus 2^61 bytes */ + if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) + return -1; + ctx->gcm.len.u[0] = alen; + + ares = ctx->gcm.ares; + /* Partial AAD block left from previous AAD update calls */ + if (ares > 0) { + /* + * Fill partial block buffer till full block + * (note, the hash is stored reflected) + */ + while (ares > 0 && len > 0) { + ctx->gcm.Xi.c[15 - ares] ^= *(aad++); + --len; + ares = (ares + 1) % AES_BLOCK_LEN; + } + /* Full block gathered */ + if (ares == 0) { + ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); + } else { /* no more AAD */ + ctx->gcm.ares = ares; + return 0; + } + } + + /* Bulk AAD processing */ + lenblks = len & ((size_t)(-AES_BLOCK_LEN)); + if (lenblks > 0) { + ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks); + aad += lenblks; + len -= lenblks; + } + + /* Add remaining AAD to the hash (note, the hash is stored reflected) */ + if (len > 0) { + ares = (unsigned int)len; + for (size_t i = 0; i < len; ++i) + ctx->gcm.Xi.c[15 - i] ^= aad[i]; + } + + ctx->gcm.ares = ares; + + return 0; +} + +static int +_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len, bool encrypt) +{ + uint64_t mlen = ctx->gcm.len.u[1]; + + mlen += len; + if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) + return -1; + + ctx->gcm.len.u[1] = mlen; + + /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ + if (ctx->gcm.ares > 0) { + ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); + ctx->gcm.ares = 0; + } + + if (encrypt) { + ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, + in, len, out); + } else { + ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, + in, len, out); + } + + return 0; +} + +static int +gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len) +{ + return _gcm_encrypt_avx512(ctx, in, out, len, true); +} + +static int +gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, + unsigned char *out, size_t len) +{ + return _gcm_encrypt_avx512(ctx, in, out, len, false); +} + +static int +gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag, + size_t len) +{ + unsigned int *res = &ctx->gcm.mres; + + /* Finalize AAD processing */ + if (ctx->gcm.ares > 0) + res = &ctx->gcm.ares; + + ossl_aes_gcm_finalize_avx512(ctx, *res); + + ctx->gcm.ares = ctx->gcm.mres = 0; + + if (tag != NULL) + return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); + return 0; +} + +static const struct ossl_aes_gcm_ops gcm_ops_avx512 = { + .init = gcm_init_avx512, + .setiv = gcm_setiv_avx512, + .aad = gcm_aad_avx512, + .encrypt = gcm_encrypt_avx512, + .decrypt = gcm_decrypt_avx512, + .finish = gcm_finish_avx512, + .tag = gcm_tag, +}; + +int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx); + +int +ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, + void *_ctx) +{ + struct ossl_gcm_context *ctx; + + ctx = _ctx; + ctx->ops = &gcm_ops_avx512; + gcm_init(ctx, key, klen); + return (0); +} diff --git a/sys/crypto/openssl/arm/ossl_aes_gcm.c b/sys/crypto/openssl/arm/ossl_aes_gcm_neon.c similarity index 100% rename from sys/crypto/openssl/arm/ossl_aes_gcm.c rename to sys/crypto/openssl/arm/ossl_aes_gcm_neon.c diff --git a/sys/crypto/openssl/amd64/ossl_aes_gcm.c b/sys/crypto/openssl/ossl_aes_gcm.c similarity index 52% rename from sys/crypto/openssl/amd64/ossl_aes_gcm.c rename to sys/crypto/openssl/ossl_aes_gcm.c index d08b2ac8a759..e1cdc710c3aa 100644 --- a/sys/crypto/openssl/amd64/ossl_aes_gcm.c +++ b/sys/crypto/openssl/ossl_aes_gcm.c @@ -1,702 +1,615 @@ /* * Copyright 2010-2022 The OpenSSL Project Authors. All Rights Reserved. * Copyright (c) 2021, Intel Corporation. All Rights Reserved. + * Copyright (c) 2023, Raptor Engineering, LLC. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html */ /* - * This file contains 2 AES-GCM wrapper implementations from OpenSSL, using - * AES-NI and VAES extensions respectively. These were ported from - * cipher_aes_gcm_hw_aesni.inc and cipher_aes_gcm_hw_vaes_avx512.inc. The - * AES-NI implementation makes use of a generic C implementation for partial - * blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined. + * This file contains an AES-GCM wrapper implementation from OpenSSL, using + * AES-NI (x86) or POWER8 Crypto Extensions (ppc). It was ported from + * cipher_aes_gcm_hw_aesni.inc and it makes use of a generic C implementation + * for partial blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined. */ #include #include #include #include #include #include _Static_assert( sizeof(struct ossl_gcm_context) <= sizeof(struct ossl_cipher_context), "ossl_gcm_context too large"); -void aesni_set_encrypt_key(const void *key, int bits, void *ctx); - -static void -gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) -{ - KASSERT(keylen == 128 || keylen == 192 || keylen == 256, - ("%s: invalid key length %zu", __func__, keylen)); - - memset(&ctx->gcm, 0, sizeof(ctx->gcm)); - memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); - aesni_set_encrypt_key(key, keylen, &ctx->aes_ks); - ctx->ops->init(ctx, key, keylen); -} - -static void -gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) -{ - (void)ctx->ops->finish(ctx, NULL, 0); - memcpy(tag, ctx->gcm.Xi.c, len); -} +#if defined(__amd64__) || defined(__i386__) +#define AES_set_encrypt_key aesni_set_encrypt_key +#define AES_gcm_encrypt aesni_gcm_encrypt +#define AES_gcm_decrypt aesni_gcm_decrypt +#define AES_encrypt aesni_encrypt +#define AES_ctr32_encrypt_blocks aesni_ctr32_encrypt_blocks +#define GCM_init gcm_init_avx +#define GCM_gmult gcm_gmult_avx +#define GCM_ghash gcm_ghash_avx + +void AES_set_encrypt_key(const void *key, int bits, void *ctx); +size_t AES_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +size_t AES_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks); +void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, void *ks, const unsigned char *iv); -void ossl_gcm_gmult_avx512(uint64_t Xi[2], void *gcm128ctx); -void ossl_aes_gcm_init_avx512(const void *ks, void *gcm128ctx); -void ossl_aes_gcm_setiv_avx512(const void *ks, void *gcm128ctx, - const unsigned char *iv, size_t ivlen); -void ossl_aes_gcm_update_aad_avx512(void *gcm128ctx, const unsigned char *aad, +void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]); +void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]); +void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, size_t len); -void ossl_aes_gcm_encrypt_avx512(const void *ks, void *gcm128ctx, - unsigned int *pblocklen, const unsigned char *in, size_t len, - unsigned char *out); -void ossl_aes_gcm_decrypt_avx512(const void *ks, void *gcm128ctx, - unsigned int *pblocklen, const unsigned char *in, size_t len, - unsigned char *out); -void ossl_aes_gcm_finalize_avx512(void *gcm128ctx, unsigned int pblocklen); - -static void -gcm_init_avx512(struct ossl_gcm_context *ctx, const void *key, size_t keylen) -{ - ossl_aes_gcm_init_avx512(&ctx->aes_ks, &ctx->gcm); -} -static void -gcm_setiv_avx512(struct ossl_gcm_context *ctx, const unsigned char *iv, - size_t len) -{ - KASSERT(len == AES_GCM_IV_LEN, - ("%s: invalid IV length %zu", __func__, len)); +#elif defined(__powerpc64__) +#define AES_set_encrypt_key aes_p8_set_encrypt_key +#define AES_gcm_encrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,1) +#define AES_gcm_decrypt(i,o,l,k,v,x) ppc_aes_gcm_crypt(i,o,l,k,v,x,0) +#define AES_encrypt aes_p8_encrypt +#define AES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks +#define GCM_init gcm_init_p8 +#define GCM_gmult gcm_gmult_p8 +#define GCM_ghash gcm_ghash_p8 + +size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); +size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, + const void *key, unsigned char ivec[16], uint64_t *Xi); - ctx->gcm.Yi.u[0] = 0; /* Current counter */ - ctx->gcm.Yi.u[1] = 0; - ctx->gcm.Xi.u[0] = 0; /* AAD hash */ - ctx->gcm.Xi.u[1] = 0; - ctx->gcm.len.u[0] = 0; /* AAD length */ - ctx->gcm.len.u[1] = 0; /* Message length */ - ctx->gcm.ares = 0; - ctx->gcm.mres = 0; +void AES_set_encrypt_key(const void *key, int bits, void *ctx); +void AES_encrypt(const unsigned char *in, unsigned char *out, void *ks); +void AES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, void *ks, const unsigned char *iv); - ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len); -} +void GCM_init(__uint128_t Htable[16], uint64_t Xi[2]); +void GCM_gmult(uint64_t Xi[2], const __uint128_t Htable[16]); +void GCM_ghash(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, + size_t len); -static int -gcm_aad_avx512(struct ossl_gcm_context *ctx, const unsigned char *aad, - size_t len) +static size_t +ppc_aes_gcm_crypt(const unsigned char *in, unsigned char *out, + size_t len, const void *key, unsigned char ivec_[16], uint64_t *Xi, + int encrypt) { - uint64_t alen = ctx->gcm.len.u[0]; - size_t lenblks; - unsigned int ares; - - /* Bad sequence: call of AAD update after message processing */ - if (ctx->gcm.len.u[1]) - return -2; - - alen += len; - /* AAD is limited by 2^64 bits, thus 2^61 bytes */ - if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) - return -1; - ctx->gcm.len.u[0] = alen; + union { + uint32_t d[4]; + uint8_t c[16]; + } *ivec = (void *)ivec_; + int s = 0; + int ndone = 0; + int ctr_reset = 0; + uint32_t ivec_val; + uint64_t blocks_unused; + uint64_t nb = len / 16; + uint64_t next_ctr = 0; + unsigned char ctr_saved[12]; + + memcpy(ctr_saved, ivec, 12); + + while (nb) { + ivec_val = ivec->d[3]; +#if BYTE_ORDER == LITTLE_ENDIAN + ivec_val = bswap32(ivec_val); +#endif - ares = ctx->gcm.ares; - /* Partial AAD block left from previous AAD update calls */ - if (ares > 0) { - /* - * Fill partial block buffer till full block - * (note, the hash is stored reflected) - */ - while (ares > 0 && len > 0) { - ctx->gcm.Xi.c[15 - ares] ^= *(aad++); - --len; - ares = (ares + 1) % AES_BLOCK_LEN; - } - /* Full block gathered */ - if (ares == 0) { - ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); - } else { /* no more AAD */ - ctx->gcm.ares = ares; - return 0; + blocks_unused = (uint64_t)0xffffffffU + 1 - (uint64_t)ivec_val; + if (nb > blocks_unused) { + len = blocks_unused * 16; + nb -= blocks_unused; + next_ctr = blocks_unused; + ctr_reset = 1; + } else { + len = nb * 16; + next_ctr = nb; + nb = 0; } - } - /* Bulk AAD processing */ - lenblks = len & ((size_t)(-AES_BLOCK_LEN)); - if (lenblks > 0) { - ossl_aes_gcm_update_aad_avx512(ctx, aad, lenblks); - aad += lenblks; - len -= lenblks; - } + s = encrypt ? ppc_aes_gcm_encrypt(in, out, len, key, ivec->c, Xi) : + ppc_aes_gcm_decrypt(in, out, len, key, ivec->c, Xi); - /* Add remaining AAD to the hash (note, the hash is stored reflected) */ - if (len > 0) { - ares = (unsigned int)len; - for (size_t i = 0; i < len; ++i) - ctx->gcm.Xi.c[15 - i] ^= aad[i]; + /* add counter to ivec */ +#if BYTE_ORDER == LITTLE_ENDIAN + ivec->d[3] = bswap32(ivec_val + next_ctr); +#else + ivec->d[3] += next_ctr; +#endif + if (ctr_reset) { + ctr_reset = 0; + in += len; + out += len; + } + memcpy(ivec, ctr_saved, 12); + ndone += s; } - ctx->gcm.ares = ares; - - return 0; + return ndone; } -static int -_gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len, bool encrypt) -{ - uint64_t mlen = ctx->gcm.len.u[1]; - - mlen += len; - if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) - return -1; - - ctx->gcm.len.u[1] = mlen; - - /* Finalize GHASH(AAD) if AAD partial blocks left unprocessed */ - if (ctx->gcm.ares > 0) { - ossl_gcm_gmult_avx512(ctx->gcm.Xi.u, ctx); - ctx->gcm.ares = 0; - } - - if (encrypt) { - ossl_aes_gcm_encrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, - in, len, out); - } else { - ossl_aes_gcm_decrypt_avx512(&ctx->aes_ks, ctx, &ctx->gcm.mres, - in, len, out); - } - - return 0; -} +#else +#error "Unsupported architecture!" +#endif -static int -gcm_encrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len) +static void +gcm_init(struct ossl_gcm_context *ctx, const void *key, size_t keylen) { - return _gcm_encrypt_avx512(ctx, in, out, len, true); -} + KASSERT(keylen == 128 || keylen == 192 || keylen == 256, + ("%s: invalid key length %zu", __func__, keylen)); -static int -gcm_decrypt_avx512(struct ossl_gcm_context *ctx, const unsigned char *in, - unsigned char *out, size_t len) -{ - return _gcm_encrypt_avx512(ctx, in, out, len, false); + memset(&ctx->gcm, 0, sizeof(ctx->gcm)); + memset(&ctx->aes_ks, 0, sizeof(ctx->aes_ks)); + AES_set_encrypt_key(key, keylen, &ctx->aes_ks); + ctx->ops->init(ctx, key, keylen); } -static int -gcm_finish_avx512(struct ossl_gcm_context *ctx, const unsigned char *tag, - size_t len) +static void +gcm_tag_op(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len) { - unsigned int *res = &ctx->gcm.mres; - - /* Finalize AAD processing */ - if (ctx->gcm.ares > 0) - res = &ctx->gcm.ares; - - ossl_aes_gcm_finalize_avx512(ctx, *res); - - ctx->gcm.ares = ctx->gcm.mres = 0; - - if (tag != NULL) - return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); - return 0; + (void)ctx->ops->finish(ctx, NULL, 0); + memcpy(tag, ctx->gcm.Xi.c, len); } -static const struct ossl_aes_gcm_ops gcm_ops_avx512 = { - .init = gcm_init_avx512, - .setiv = gcm_setiv_avx512, - .aad = gcm_aad_avx512, - .encrypt = gcm_encrypt_avx512, - .decrypt = gcm_decrypt_avx512, - .finish = gcm_finish_avx512, - .tag = gcm_tag, -}; - -size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi); -size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, - const void *key, unsigned char ivec[16], uint64_t *Xi); -void aesni_encrypt(const unsigned char *in, unsigned char *out, void *ks); -void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, - size_t blocks, void *ks, const unsigned char *iv); - -void gcm_init_avx(__uint128_t Htable[16], uint64_t Xi[2]); -void gcm_gmult_avx(uint64_t Xi[2], const __uint128_t Htable[16]); -void gcm_ghash_avx(uint64_t Xi[2], const __uint128_t Htable[16], const void *in, - size_t len); - static void -gcm_init_aesni(struct ossl_gcm_context *ctx, const void *key, size_t keylen) +gcm_init_op(struct ossl_gcm_context *ctx, const void *key, size_t keylen) { - aesni_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks); #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.H.u[0] = bswap64(ctx->gcm.H.u[0]); ctx->gcm.H.u[1] = bswap64(ctx->gcm.H.u[1]); #endif - gcm_init_avx(ctx->gcm.Htable, ctx->gcm.H.u); + GCM_init(ctx->gcm.Htable, ctx->gcm.H.u); } static void -gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv, +gcm_setiv_op(struct ossl_gcm_context *ctx, const unsigned char *iv, size_t len) { uint32_t ctr; KASSERT(len == AES_GCM_IV_LEN, ("%s: invalid IV length %zu", __func__, len)); ctx->gcm.len.u[0] = 0; ctx->gcm.len.u[1] = 0; ctx->gcm.ares = ctx->gcm.mres = 0; memcpy(ctx->gcm.Yi.c, iv, len); ctx->gcm.Yi.c[12] = 0; ctx->gcm.Yi.c[13] = 0; ctx->gcm.Yi.c[14] = 0; ctx->gcm.Yi.c[15] = 1; ctr = 1; ctx->gcm.Xi.u[0] = 0; ctx->gcm.Xi.u[1] = 0; - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks); ctr++; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif } static int -gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad, +gcm_aad_op(struct ossl_gcm_context *ctx, const unsigned char *aad, size_t len) { size_t i; unsigned int n; uint64_t alen = ctx->gcm.len.u[0]; if (ctx->gcm.len.u[1]) return -2; alen += len; if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len)) return -1; ctx->gcm.len.u[0] = alen; n = ctx->gcm.ares; if (n) { while (n && len) { ctx->gcm.Xi.c[n] ^= *(aad++); --len; n = (n + 1) % 16; } if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); else { ctx->gcm.ares = n; return 0; } } if ((i = (len & (size_t)-AES_BLOCK_LEN))) { - gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i); + GCM_ghash(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i); aad += i; len -= i; } if (len) { n = (unsigned int)len; for (i = 0; i < len; ++i) ctx->gcm.Xi.c[i] ^= aad[i]; } ctx->gcm.ares = n; return 0; } static int gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { unsigned int n, ctr, mres; size_t i; uint64_t mlen = ctx->gcm.len.u[1]; mlen += len; if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) return -1; ctx->gcm.len.u[1] = mlen; mres = ctx->gcm.mres; if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } #if BYTE_ORDER == LITTLE_ENDIAN ctr = bswap32(ctx->gcm.Yi.d[3]); #else ctr = ctx->gcm.Yi.d[3]; #endif n = mres % 16; for (i = 0; i < len; ++i) { if (n == 0) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif } ctx->gcm.Xi.c[n] ^= out[i] = in[i] ^ ctx->gcm.EKi.c[n]; mres = n = (n + 1) % 16; if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); } ctx->gcm.mres = mres; return 0; } static int gcm_encrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { unsigned int n, ctr, mres; size_t i; uint64_t mlen = ctx->gcm.len.u[1]; mlen += len; if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) return -1; ctx->gcm.len.u[1] = mlen; mres = ctx->gcm.mres; if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } #if BYTE_ORDER == LITTLE_ENDIAN ctr = bswap32(ctx->gcm.Yi.d[3]); #else ctr = ctx->gcm.Yi.d[3]; #endif n = mres % 16; if (n) { while (n && len) { ctx->gcm.Xi.c[n] ^= *(out++) = *(in++) ^ ctx->gcm.EKi.c[n]; --len; n = (n + 1) % 16; } if (n == 0) { - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); mres = 0; } else { ctx->gcm.mres = n; return 0; } } if ((i = (len & (size_t)-16))) { size_t j = i / 16; - aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); + AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); ctr += (unsigned int)j; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif in += i; len -= i; while (j--) { for (i = 0; i < 16; ++i) ctx->gcm.Xi.c[i] ^= out[i]; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); out += 16; } } if (len) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif while (len--) { ctx->gcm.Xi.c[mres++] ^= out[n] = in[n] ^ ctx->gcm.EKi.c[n]; ++n; } } ctx->gcm.mres = mres; return 0; } static int -gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, +gcm_encrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { size_t bulk = 0, res; int error; res = MIN(len, (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN); if ((error = gcm_encrypt(ctx, in, out, res)) != 0) return error; - bulk = aesni_gcm_encrypt(in + res, out + res, len - res, + bulk = AES_gcm_encrypt(in + res, out + res, len - res, &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u); ctx->gcm.len.u[1] += bulk; bulk += res; if ((error = gcm_encrypt_ctr32(ctx, in + bulk, out + bulk, len - bulk)) != 0) return error; return 0; } static int gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { unsigned int n, ctr, mres; size_t i; uint64_t mlen = ctx->gcm.len.u[1]; mlen += len; if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) return -1; ctx->gcm.len.u[1] = mlen; mres = ctx->gcm.mres; if (ctx->gcm.ares) { /* First call to encrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } #if BYTE_ORDER == LITTLE_ENDIAN ctr = bswap32(ctx->gcm.Yi.d[3]); #else ctr = ctx->gcm.Yi.d[3]; #endif n = mres % 16; for (i = 0; i < len; ++i) { uint8_t c; if (n == 0) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif } c = in[i]; out[i] = c ^ ctx->gcm.EKi.c[n]; ctx->gcm.Xi.c[n] ^= c; mres = n = (n + 1) % 16; if (n == 0) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); } ctx->gcm.mres = mres; return 0; } static int gcm_decrypt_ctr32(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { unsigned int n, ctr, mres; size_t i; uint64_t mlen = ctx->gcm.len.u[1]; mlen += len; if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len)) return -1; ctx->gcm.len.u[1] = mlen; mres = ctx->gcm.mres; if (ctx->gcm.ares) { /* First call to decrypt finalizes GHASH(AAD) */ - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.ares = 0; } #if BYTE_ORDER == LITTLE_ENDIAN ctr = bswap32(ctx->gcm.Yi.d[3]); #else ctr = ctx->gcm.Yi.d[3]; #endif n = mres % 16; if (n) { while (n && len) { uint8_t c = *(in++); *(out++) = c ^ ctx->gcm.EKi.c[n]; ctx->gcm.Xi.c[n] ^= c; --len; n = (n + 1) % 16; } if (n == 0) { - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); mres = 0; } else { ctx->gcm.mres = n; return 0; } } if ((i = (len & (size_t)-16))) { size_t j = i / 16; while (j--) { size_t k; for (k = 0; k < 16; ++k) ctx->gcm.Xi.c[k] ^= in[k]; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); in += 16; } j = i / 16; in -= i; - aesni_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); + AES_ctr32_encrypt_blocks(in, out, j, &ctx->aes_ks, ctx->gcm.Yi.c); ctr += (unsigned int)j; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif out += i; in += i; len -= i; } if (len) { - aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); + AES_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c, &ctx->aes_ks); ++ctr; #if BYTE_ORDER == LITTLE_ENDIAN ctx->gcm.Yi.d[3] = bswap32(ctr); #else ctx->gcm.Yi.d[3] = ctr; #endif while (len--) { uint8_t c = in[n]; ctx->gcm.Xi.c[mres++] ^= c; out[n] = c ^ ctx->gcm.EKi.c[n]; ++n; } } ctx->gcm.mres = mres; return 0; } static int -gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in, +gcm_decrypt_op(struct ossl_gcm_context *ctx, const unsigned char *in, unsigned char *out, size_t len) { size_t bulk = 0, res; int error; res = MIN(len, (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN); if ((error = gcm_decrypt(ctx, in, out, res)) != 0) return error; - bulk = aesni_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks, + bulk = AES_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u); ctx->gcm.len.u[1] += bulk; bulk += res; if ((error = gcm_decrypt_ctr32(ctx, in + bulk, out + bulk, len - bulk)) != 0) return error; return 0; } static int -gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag, +gcm_finish_op(struct ossl_gcm_context *ctx, const unsigned char *tag, size_t len) { uint64_t alen = ctx->gcm.len.u[0] << 3; uint64_t clen = ctx->gcm.len.u[1] << 3; if (ctx->gcm.mres || ctx->gcm.ares) - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); #if BYTE_ORDER == LITTLE_ENDIAN alen = bswap64(alen); clen = bswap64(clen); #endif ctx->gcm.Xi.u[0] ^= alen; ctx->gcm.Xi.u[1] ^= clen; - gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable); + GCM_gmult(ctx->gcm.Xi.u, ctx->gcm.Htable); ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0]; ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1]; if (tag != NULL) return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len); return 0; } -static const struct ossl_aes_gcm_ops gcm_ops_aesni = { - .init = gcm_init_aesni, - .setiv = gcm_setiv_aesni, - .aad = gcm_aad_aesni, - .encrypt = gcm_encrypt_aesni, - .decrypt = gcm_decrypt_aesni, - .finish = gcm_finish_aesni, - .tag = gcm_tag, +static const struct ossl_aes_gcm_ops gcm_ops = { + .init = gcm_init_op, + .setiv = gcm_setiv_op, + .aad = gcm_aad_op, + .encrypt = gcm_encrypt_op, + .decrypt = gcm_decrypt_op, + .finish = gcm_finish_op, + .tag = gcm_tag_op, }; -int ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, void *_ctx); - -int -ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, - void *_ctx) -{ - struct ossl_gcm_context *ctx; - - ctx = _ctx; - ctx->ops = &gcm_ops_aesni; - gcm_init(ctx, key, klen); - return (0); -} - -int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx); +int ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx); int -ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, +ossl_aes_gcm_setkey(const unsigned char *key, int klen, void *_ctx) { struct ossl_gcm_context *ctx; ctx = _ctx; - ctx->ops = &gcm_ops_avx512; + ctx->ops = &gcm_ops; gcm_init(ctx, key, klen); return (0); } diff --git a/sys/crypto/openssl/ossl_ppc.c b/sys/crypto/openssl/ossl_ppc.c index 0951745c4b43..980211f46a76 100644 --- a/sys/crypto/openssl/ossl_ppc.c +++ b/sys/crypto/openssl/ossl_ppc.c @@ -1,154 +1,161 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Raptor Engineering, LLC * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include unsigned int OPENSSL_ppccap_P = 0; ossl_cipher_setkey_t aes_p8_set_encrypt_key; ossl_cipher_setkey_t aes_p8_set_decrypt_key; + ossl_cipher_setkey_t vpaes_set_encrypt_key; ossl_cipher_setkey_t vpaes_set_decrypt_key; +ossl_cipher_setkey_t ossl_aes_gcm_setkey; + void ossl_cpuid(struct ossl_softc *sc) { if (cpu_features2 & PPC_FEATURE2_HAS_VEC_CRYPTO) { OPENSSL_ppccap_P |= PPC_CRYPTO207; } if (cpu_features2 & PPC_FEATURE2_ARCH_3_00) { OPENSSL_ppccap_P |= PPC_MADD300; } if (cpu_features & PPC_FEATURE_64) { OPENSSL_ppccap_P |= PPC_MFTB; } else { OPENSSL_ppccap_P |= PPC_MFSPR268; } if (cpu_features & PPC_FEATURE_HAS_FPU) { OPENSSL_ppccap_P |= PPC_FPU; if (cpu_features & PPC_FEATURE_64) { OPENSSL_ppccap_P |= PPC_FPU64; } } if (cpu_features & PPC_FEATURE_HAS_ALTIVEC) { OPENSSL_ppccap_P |= PPC_ALTIVEC; } /* Pick P8 crypto if available, otherwise fall back to altivec */ if (OPENSSL_ppccap_P & PPC_CRYPTO207) { ossl_cipher_aes_cbc.set_encrypt_key = aes_p8_set_encrypt_key; ossl_cipher_aes_cbc.set_decrypt_key = aes_p8_set_decrypt_key; sc->has_aes = true; - } else if (OPENSSL_ppccap_P & PPC_ALTIVEC) { + + ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey; + ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey; + sc->has_aes_gcm = true; + } else if (OPENSSL_ppccap_P & PPC_ALTIVEC) { ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key; ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key; sc->has_aes = true; } } /* * The following trivial wrapper functions were copied from OpenSSL 1.1.1v's * crypto/ppccap.c. */ void sha256_block_p8(void *ctx, const void *inp, size_t len); void sha256_block_ppc(void *ctx, const void *inp, size_t len); void sha256_block_data_order(void *ctx, const void *inp, size_t len); void sha256_block_data_order(void *ctx, const void *inp, size_t len) { OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) : sha256_block_ppc(ctx, inp, len); } void sha512_block_p8(void *ctx, const void *inp, size_t len); void sha512_block_ppc(void *ctx, const void *inp, size_t len); void sha512_block_data_order(void *ctx, const void *inp, size_t len); void sha512_block_data_order(void *ctx, const void *inp, size_t len) { OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) : sha512_block_ppc(ctx, inp, len); } void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]); void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len, const unsigned int key[8], const unsigned int counter[4]) { OPENSSL_ppccap_P & PPC_CRYPTO207 ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) : OPENSSL_ppccap_P & PPC_ALTIVEC ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) : ChaCha20_ctr32_int(out, inp, len, key, counter); } void poly1305_init_int(void *ctx, const unsigned char key[16]); void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, unsigned int padbit); void poly1305_emit(void *ctx, unsigned char mac[16], const unsigned int nonce[4]); void poly1305_init_fpu(void *ctx, const unsigned char key[16]); void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len, unsigned int padbit); void poly1305_emit_fpu(void *ctx, unsigned char mac[16], const unsigned int nonce[4]); int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]); int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]) { if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) { poly1305_init_fpu(ctx, key); func[0] = (void*)(uintptr_t)poly1305_blocks_fpu; func[1] = (void*)(uintptr_t)poly1305_emit_fpu; } else { poly1305_init_int(ctx, key); func[0] = (void*)(uintptr_t)poly1305_blocks; func[1] = (void*)(uintptr_t)poly1305_emit; } return 1; } diff --git a/sys/crypto/openssl/ossl_x86.c b/sys/crypto/openssl/ossl_x86.c index 9161eaf39beb..bf034f05b783 100644 --- a/sys/crypto/openssl/ossl_x86.c +++ b/sys/crypto/openssl/ossl_x86.c @@ -1,153 +1,153 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Netflix, Inc * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. */ #include #include #include #include #include #include #include #include #include /* * See OPENSSL_ia32cap(3). * * [0] = cpu_feature but with a few custom bits * [1] = cpu_feature2 but with AMD XOP in bit 11 * [2] = cpu_stdext_feature * [3] = cpu_stdext_feature2 */ unsigned int OPENSSL_ia32cap_P[4]; #define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) ossl_cipher_setkey_t aesni_set_encrypt_key; ossl_cipher_setkey_t aesni_set_decrypt_key; #ifdef __amd64__ int ossl_vaes_vpclmulqdq_capable(void); -ossl_cipher_setkey_t ossl_aes_gcm_setkey_aesni; +ossl_cipher_setkey_t ossl_aes_gcm_setkey; ossl_cipher_setkey_t ossl_aes_gcm_setkey_avx512; #endif void ossl_cpuid(struct ossl_softc *sc) { uint64_t xcr0; u_int regs[4]; u_int max_cores; /* Derived from OpenSSL_ia32_cpuid. */ OPENSSL_ia32cap_P[0] = cpu_feature & ~(CPUID_B20 | CPUID_IA64); if (cpu_vendor_id == CPU_VENDOR_INTEL) { OPENSSL_ia32cap_P[0] |= CPUID_IA64; if ((cpu_id & 0xf00) != 0xf00) OPENSSL_ia32cap_P[0] |= CPUID_B20; } /* Only leave CPUID_HTT on if HTT is present. */ if (cpu_vendor_id == CPU_VENDOR_AMD && cpu_exthigh >= 0x80000008) { max_cores = (cpu_procinfo2 & AMDID_CMP_CORES) + 1; if (cpu_feature & CPUID_HTT) { if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 <= max_cores) OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; } } else { if (cpu_high >= 4) { cpuid_count(4, 0, regs); max_cores = (regs[0] >> 26) & 0xfff; } else max_cores = -1; } if (max_cores == 0) OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; else if ((cpu_procinfo & CPUID_HTT_CORES) >> 16 == 0) OPENSSL_ia32cap_P[0] &= ~CPUID_HTT; OPENSSL_ia32cap_P[1] = cpu_feature2 & ~AMDID2_XOP; if (cpu_vendor_id == CPU_VENDOR_AMD) OPENSSL_ia32cap_P[1] |= amd_feature2 & AMDID2_XOP; OPENSSL_ia32cap_P[2] = cpu_stdext_feature; if ((OPENSSL_ia32cap_P[1] & CPUID2_XSAVE) == 0) OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F | CPUID_STDEXT_AVX512DQ); /* Disable AVX512F on Skylake-X. */ if ((cpu_id & 0x0fff0ff0) == 0x00050650) OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512F); if (cpu_feature2 & CPUID2_OSXSAVE) xcr0 = rxcr(0); else xcr0 = 0; if ((xcr0 & (XFEATURE_AVX512 | XFEATURE_AVX)) != (XFEATURE_AVX512 | XFEATURE_AVX)) OPENSSL_ia32cap_P[2] &= ~(CPUID_STDEXT_AVX512VL | CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512IFMA | CPUID_STDEXT_AVX512F); if ((xcr0 & XFEATURE_AVX) != XFEATURE_AVX) { OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; } OPENSSL_ia32cap_P[3] = cpu_stdext_feature2; if (!AESNI_CAPABLE) return; sc->has_aes = true; ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key; ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key; #ifdef __amd64__ if (ossl_vaes_vpclmulqdq_capable()) { ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_avx512; ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_avx512; sc->has_aes_gcm = true; } else if ((cpu_feature2 & (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) == (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) { - ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_aesni; - ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_aesni; + ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey; + ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey; sc->has_aes_gcm = true; } else { sc->has_aes_gcm = false; } #else sc->has_aes_gcm = false; #endif } diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile index ac2c752e922e..c516fe0c158d 100644 --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -1,123 +1,128 @@ .PATH: ${SRCTOP}/sys/crypto/openssl .PATH: ${SRCTOP}/sys/crypto/openssl/${MACHINE_CPUARCH} KMOD= ossl OBJS+= ${OBJS.${MACHINE_CPUARCH}} SRCS= bus_if.h \ cryptodev_if.h \ device_if.h \ ossl.c \ ossl_aes.c \ ossl_chacha20.c \ ossl_poly1305.c \ ossl_sha1.c \ ossl_sha256.c \ ossl_sha512.c \ ${SRCS.${MACHINE_CPUARCH}} SRCS.arm= \ aes-armv4.S \ bsaes-armv7.S \ chacha-armv4.S \ ghash-armv4.S \ poly1305-armv4.S \ sha1-armv4-large.S \ sha256-armv4.S \ sha512-armv4.S \ ossl_arm.c \ - ossl_aes_gcm.c + ossl_aes_gcm_neon.c SRCS.aarch64= \ chacha-armv8.S \ chacha-armv8-sve.S \ poly1305-armv8.S \ sha1-armv8.S \ sha256-armv8.S \ sha512-armv8.S \ vpaes-armv8.S \ ossl_aarch64.c SRCS.amd64= \ aes-gcm-avx512.S \ aesni-x86_64.S \ aesni-gcm-x86_64.S \ chacha-x86_64.S \ ghash-x86_64.S \ poly1305-x86_64.S \ sha1-x86_64.S \ sha256-x86_64.S \ sha512-x86_64.S \ ossl_aes_gcm.c \ + ossl_aes_gcm_avx512.c \ ossl_x86.c SRCS.i386= \ aesni-x86.S \ chacha-x86.S \ poly1305-x86.S \ sha1-586.S \ sha256-586.S \ sha512-586.S \ ossl_x86.c SRCS.powerpc64le= \ + aes-gcm-ppc.S \ + ossl_aes_gcm.c \ ossl_ppccap.c \ aes-ppc.S \ aesp8-ppc.S \ chacha-ppc.S \ ecp_nistz256-ppc64.S \ ghashp8-ppc.S \ keccak1600-ppc64.S \ poly1305-ppc.S \ poly1305-ppcfp.S \ ppc-mont.S \ ppc.S \ ppccpuid.S \ sha1-ppc.S \ sha256-ppc.S \ sha256p8-ppc.S \ sha512-ppc.S \ sha512p8-ppc.S \ vpaes-ppc.S \ x25519-ppc64.S SRCS.powerpc64= \ + aes-gcm-ppc.S \ + ossl_aes_gcm.c \ ossl_ppccap.c \ aes-ppc.S \ aesp8-ppc.S \ chacha-ppc.S \ ecp_nistz256-ppc64.S \ ghashp8-ppc.S \ keccak1600-ppc64.S \ poly1305-ppc.S \ poly1305-ppcfp.S \ ppc-mont.S \ ppc.S \ ppccpuid.S \ sha1-ppc.S \ sha256-ppc.S \ sha256p8-ppc.S \ sha512-ppc.S \ sha512p8-ppc.S \ vpaes-ppc.S \ x25519-ppc64.S CFLAGS.bsaes-armv7.S+= -D__KERNEL__ CFLAGS+= -I${SRCTOP}/sys/crypto/openssl # For arm64, we are forced to rewrite the compiler invocation for the assembly # files, to remove -mgeneral-regs-only. ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC} ${CTFCONVERT_CMD} # Based on modules/armv8crypto/Makefile. # Clang doesn't recognize "aes*" instructions without -march set. aesv8-armx.o: aesv8-armx.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} \ -march=armv8-a+crypto ${.IMPSRC} ${CTFCONVERT_CMD} OBJS.aarch64= aesv8-armx.o .include