diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index fb001f95a672..f7003b1048c8 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -1,499 +1,509 @@ # $FreeBSD$ cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_armv6_on_64bit.S" \ compile-with "${CC} -x assembler-with-cpp -m32 -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_armv6_on_64bit.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-littleaarch64 --binary-architecture aarch64 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # cloudabi64_vdso.o optional compat_cloudabi64 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_aarch64.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_aarch64.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi64_vdso.o" # cloudabi64_vdso_blob.o optional compat_cloudabi64 \ dependency "cloudabi64_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-littleaarch64 --binary-architecture aarch64 cloudabi64_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi64_vdso_blob.o" # # Allwinner common files arm/allwinner/a10_timer.c optional a10_timer fdt arm/allwinner/a10_codec.c optional sound a10_codec arm/allwinner/a31_dmac.c optional a31_dmac arm/allwinner/sunxi_dma_if.m optional a31_dmac arm/allwinner/aw_cir.c optional evdev aw_cir fdt arm/allwinner/aw_dwc3.c optional aw_dwc3 fdt arm/allwinner/aw_gpio.c optional gpio aw_gpio fdt arm/allwinner/aw_mmc.c optional mmc aw_mmc fdt | mmccam aw_mmc fdt arm/allwinner/aw_nmi.c optional aw_nmi fdt \ compile-with "${NORMAL_C} -I$S/contrib/device-tree/include" arm/allwinner/aw_pwm.c optional aw_pwm fdt arm/allwinner/aw_rsb.c optional aw_rsb fdt arm/allwinner/aw_rtc.c optional aw_rtc fdt arm/allwinner/aw_sid.c optional aw_sid nvmem fdt arm/allwinner/aw_spi.c optional aw_spi fdt arm/allwinner/aw_syscon.c optional aw_syscon ext_resources syscon fdt arm/allwinner/aw_thermal.c optional aw_thermal nvmem fdt arm/allwinner/aw_usbphy.c optional ehci aw_usbphy fdt arm/allwinner/aw_usb3phy.c optional xhci aw_usbphy fdt arm/allwinner/aw_wdog.c optional aw_wdog fdt arm/allwinner/axp81x.c optional axp81x fdt arm/allwinner/if_awg.c optional awg ext_resources syscon aw_sid nvmem fdt # Allwinner clock driver arm/allwinner/clkng/aw_ccung.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_frac.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_m.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_mipi.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_nkmp.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_nm.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_nmm.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_np.c optional aw_ccu fdt arm/allwinner/clkng/aw_clk_prediv_mux.c optional aw_ccu fdt arm/allwinner/clkng/ccu_a64.c optional soc_allwinner_a64 aw_ccu fdt arm/allwinner/clkng/ccu_h3.c optional soc_allwinner_h5 aw_ccu fdt arm/allwinner/clkng/ccu_h6.c optional soc_allwinner_h6 aw_ccu fdt arm/allwinner/clkng/ccu_h6_r.c optional soc_allwinner_h6 aw_ccu fdt arm/allwinner/clkng/ccu_sun8i_r.c optional aw_ccu fdt arm/allwinner/clkng/ccu_de2.c optional aw_ccu fdt # Allwinner padconf files arm/allwinner/a64/a64_padconf.c optional soc_allwinner_a64 fdt arm/allwinner/a64/a64_r_padconf.c optional soc_allwinner_a64 fdt arm/allwinner/h3/h3_padconf.c optional soc_allwinner_h5 fdt arm/allwinner/h3/h3_r_padconf.c optional soc_allwinner_h5 fdt arm/allwinner/h6/h6_padconf.c optional soc_allwinner_h6 fdt arm/allwinner/h6/h6_r_padconf.c optional soc_allwinner_h6 fdt arm/annapurna/alpine/alpine_ccu.c optional al_ccu fdt arm/annapurna/alpine/alpine_nb_service.c optional al_nb_service fdt arm/annapurna/alpine/alpine_pci.c optional al_pci fdt arm/annapurna/alpine/alpine_pci_msix.c optional al_pci fdt arm/annapurna/alpine/alpine_serdes.c optional al_serdes fdt \ no-depend \ compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${PROF} ${.IMPSRC}" arm/arm/generic_timer.c standard arm/arm/gic.c standard arm/arm/gic_acpi.c optional acpi arm/arm/gic_fdt.c optional fdt arm/arm/pmu.c standard arm/arm/pmu_fdt.c optional fdt arm/broadcom/bcm2835/bcm2835_audio.c optional sound vchiq fdt \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" arm/broadcom/bcm2835/bcm2835_bsc.c optional bcm2835_bsc fdt arm/broadcom/bcm2835/bcm2835_clkman.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_cpufreq.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_dma.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_fbd.c optional vt soc_brcm_bcm2837 fdt | vt soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_firmware.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_ft5406.c optional evdev bcm2835_ft5406 fdt arm/broadcom/bcm2835/bcm2835_gpio.c optional gpio soc_brcm_bcm2837 fdt | gpio soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_intr.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_mbox.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_rng.c optional !random_loadable soc_brcm_bcm2837 fdt | !random_loadable soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_sdhci.c optional sdhci soc_brcm_bcm2837 fdt | sdhci soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_sdhost.c optional sdhci soc_brcm_bcm2837 fdt | sdhci soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_spi.c optional bcm2835_spi fdt arm/broadcom/bcm2835/bcm2835_vcbus.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_vcio.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_wdog.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2836.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm283x_dwc_fdt.c optional dwcotg fdt soc_brcm_bcm2837 | dwcotg fdt soc_brcm_bcm2838 arm/broadcom/bcm2835/bcm2838_pci.c optional soc_brcm_bcm2838 fdt pci arm/broadcom/bcm2835/bcm2838_xhci.c optional soc_brcm_bcm2838 fdt pci xhci arm/broadcom/bcm2835/raspberrypi_gpio.c optional soc_brcm_bcm2837 gpio | soc_brcm_bcm2838 gpio arm/freescale/vybrid/vf_i2c.c optional vf_i2c iicbus SOC_NXP_LS arm/mv/a37x0_gpio.c optional a37x0_gpio gpio fdt arm/mv/a37x0_iic.c optional a37x0_iic iicbus fdt arm/mv/a37x0_spi.c optional a37x0_spi spibus fdt arm/mv/armada38x/armada38x_rtc.c optional mv_rtc fdt arm/mv/gpio.c optional mv_gpio fdt arm/mv/mvebu_gpio.c optional mv_gpio fdt arm/mv/mvebu_pinctrl.c optional mvebu_pinctrl fdt arm/mv/mv_ap806_clock.c optional SOC_MARVELL_8K fdt arm/mv/mv_ap806_gicp.c optional mv_ap806_gicp fdt arm/mv/mv_ap806_sei.c optional mv_ap806_sei fdt arm/mv/mv_cp110_clock.c optional SOC_MARVELL_8K fdt arm/mv/mv_cp110_icu.c optional mv_cp110_icu fdt arm/mv/mv_cp110_icu_bus.c optional mv_cp110_icu fdt arm/mv/mv_thermal.c optional SOC_MARVELL_8K mv_thermal fdt arm/mv/armada38x/armada38x_rtc.c optional mv_rtc fdt arm/nvidia/tegra_abpmisc.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_ahci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_efuse.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_ehci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_gpio.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_i2c.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_lic.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_mc.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_pcie.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_sdhci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_soctherm_if.m optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_soctherm.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_uart.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_usbphy.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_xhci.c optional fdt soc_nvidia_tegra210 arm/xilinx/uart_dev_cdnc.c optional uart soc_xilinx_zynq arm64/acpica/acpi_iort.c optional acpi arm64/acpica/acpi_machdep.c optional acpi arm64/acpica/OsdEnvironment.c optional acpi arm64/acpica/acpi_wakeup.c optional acpi arm64/acpica/pci_cfgreg.c optional acpi pci arm64/arm64/autoconf.c standard arm64/arm64/bus_machdep.c standard arm64/arm64/bus_space_asm.S standard arm64/arm64/busdma_bounce.c standard arm64/arm64/busdma_machdep.c standard arm64/arm64/bzero.S standard arm64/arm64/clock.c standard arm64/arm64/copyinout.S standard arm64/arm64/cpu_errata.c standard arm64/arm64/cpufunc_asm.S standard arm64/arm64/db_disasm.c optional ddb arm64/arm64/db_interface.c optional ddb arm64/arm64/db_trace.c optional ddb arm64/arm64/debug_monitor.c standard arm64/arm64/disassem.c optional ddb arm64/arm64/dump_machdep.c standard arm64/arm64/efirt_machdep.c optional efirt arm64/arm64/elf32_machdep.c optional compat_freebsd32 arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/freebsd32_machdep.c optional compat_freebsd32 arm64/arm64/gdb_machdep.c optional gdb arm64/arm64/gicv3_its.c optional intrng fdt arm64/arm64/gic_v3.c standard arm64/arm64/gic_v3_acpi.c optional acpi arm64/arm64/gic_v3_fdt.c optional fdt arm64/arm64/identcpu.c standard arm64/arm64/in_cksum.c optional inet | inet6 arm64/arm64/locore.S standard no-obj arm64/arm64/machdep.c standard arm64/arm64/machdep_boot.c standard arm64/arm64/mem.c standard arm64/arm64/memcpy.S standard arm64/arm64/memmove.S standard arm64/arm64/minidump_machdep.c standard arm64/arm64/mp_machdep.c optional smp arm64/arm64/nexus.c standard arm64/arm64/ofw_machdep.c optional fdt arm64/arm64/pmap.c standard arm64/arm64/stack_machdep.c optional ddb | stack arm64/arm64/support.S standard arm64/arm64/swtch.S standard arm64/arm64/sys_machdep.c standard arm64/arm64/trap.c standard arm64/arm64/uio_machdep.c standard arm64/arm64/uma_machdep.c standard arm64/arm64/undefined.c standard arm64/arm64/unwind.c optional ddb | kdtrace_hooks | stack arm64/arm64/vfp.c standard arm64/arm64/vm_machdep.c standard arm64/broadcom/brcmmdio/mdio_mux_iproc.c optional fdt arm64/broadcom/brcmmdio/mdio_nexus_iproc.c optional fdt arm64/broadcom/brcmmdio/mdio_ns2_pcie_phy.c optional fdt pci arm64/broadcom/genet/if_genet.c optional SOC_BRCM_BCM2838 fdt genet arm64/cavium/thunder_pcie_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_pem.c optional soc_cavm_thunderx pci arm64/cavium/thunder_pcie_pem_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_common.c optional soc_cavm_thunderx pci arm64/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 arm64/cloudabi64/cloudabi64_sysvec.c optional compat_cloudabi64 arm64/coresight/coresight.c standard arm64/coresight/coresight_acpi.c optional acpi arm64/coresight/coresight_fdt.c optional fdt arm64/coresight/coresight_if.m standard arm64/coresight/coresight_cmd.c standard arm64/coresight/coresight_cpu_debug.c standard arm64/coresight/coresight_etm4x.c standard arm64/coresight/coresight_etm4x_acpi.c optional acpi arm64/coresight/coresight_etm4x_fdt.c optional fdt arm64/coresight/coresight_funnel.c standard arm64/coresight/coresight_funnel_acpi.c optional acpi arm64/coresight/coresight_funnel_fdt.c optional fdt arm64/coresight/coresight_replicator.c standard arm64/coresight/coresight_replicator_acpi.c optional acpi arm64/coresight/coresight_replicator_fdt.c optional fdt arm64/coresight/coresight_tmc.c standard arm64/coresight/coresight_tmc_acpi.c optional acpi arm64/coresight/coresight_tmc_fdt.c optional fdt arm64/intel/firmware.c optional soc_intel_stratix10 arm64/intel/stratix10-soc-fpga-mgr.c optional soc_intel_stratix10 arm64/intel/stratix10-svc.c optional soc_intel_stratix10 arm64/iommu/iommu.c optional iommu arm64/iommu/iommu_if.m optional iommu arm64/iommu/smmu.c optional iommu arm64/iommu/smmu_acpi.c optional acpi iommu arm64/iommu/smmu_quirks.c optional iommu arm64/nvidia/tegra210/max77620.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_gpio.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_regulators.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_rtc.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_car.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_per.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_pll.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_super.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_coretemp.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_cpufreq.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_pinmux.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_pmc.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_xusbpadctl.c optional fdt soc_nvidia_tegra210 arm64/qoriq/ls1046_gpio.c optional ls1046_gpio gpio fdt SOC_NXP_LS arm64/qoriq/qoriq_dw_pci.c optional pci fdt SOC_NXP_LS arm64/qoriq/qoriq_therm.c optional pci fdt SOC_NXP_LS arm64/qoriq/qoriq_therm_if.m optional pci fdt SOC_NXP_LS arm64/qoriq/clk/ls1046a_clkgen.c optional clk SOC_NXP_LS arm64/qoriq/clk/lx2160a_clkgen.c optional clk SOC_NXP_LS arm64/qoriq/clk/qoriq_clk_pll.c optional clk SOC_NXP_LS arm64/qoriq/clk/qoriq_clkgen.c optional clk SOC_NXP_LS arm64/qualcomm/qcom_gcc.c optional qcom_gcc fdt contrib/vchiq/interface/compat/vchi_bsd.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -Wno-unused -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_arm.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -Wno-unused -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_connected.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_core.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_shim.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_util.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" crypto/armv8/armv8_crypto.c optional armv8crypto armv8_crypto_wrap.o optional armv8crypto \ dependency "$S/crypto/armv8/armv8_crypto_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "armv8_crypto_wrap.o" +aesv8-armx.o optional armv8crypto \ + dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ -I$S/crypto/openssl/crypto ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \ + no-implicit-rule \ + clean "aesv8-armx.o" +ghashv8-armx.o optional armv8crypto \ + dependency "$S/crypto/openssl/aarch64/ghashv8-armx.S" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ -I$S/crypto/openssl/crypto ${WERROR} ${NO_WCAST_QUAL} ${PROF} -march=armv8-a+crypto ${.IMPSRC}" \ + no-implicit-rule \ + clean "ghashv8-armx.o" crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl crypto/openssl/aarch64/sha1-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/sha256-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" crypto/openssl/aarch64/sha512-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pxm.c optional acpi dev/ahci/ahci_fsl_fdt.c optional SOC_NXP_LS ahci fdt dev/ahci/ahci_generic.c optional ahci dev/altera/dwc/if_dwc_socfpga.c optional fdt dwc_socfpga dev/axgbe/if_axgbe.c optional axa dev/axgbe/xgbe-desc.c optional axa dev/axgbe/xgbe-dev.c optional axa dev/axgbe/xgbe-drv.c optional axa dev/axgbe/xgbe-mdio.c optional axa dev/axgbe/xgbe-sysctl.c optional axa dev/axgbe/xgbe-txrx.c optional axa dev/axgbe/xgbe_osdep.c optional axa dev/axgbe/xgbe-phy-v1.c optional axa dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/gpio/pl061.c optional pl061 gpio dev/gpio/pl061_acpi.c optional pl061 gpio acpi dev/gpio/pl061_fdt.c optional pl061 gpio fdt dev/hwpmc/hwpmc_arm64.c optional hwpmc dev/hwpmc/hwpmc_arm64_md.c optional hwpmc dev/ice/if_ice_iflib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_lib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_osdep.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_resmgr.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_strings.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_recovery_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_common.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_controlq.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_dcb.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flex_pipe.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flow.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_nvm.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_sched.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_sriov.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_switch.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" ice_ddp.c optional ice_ddp \ compile-with "${AWK} -f $S/tools/fw_stub.awk ice_ddp.fw:ice_ddp:0x01031000 -mice_ddp -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "ice_ddp.c" ice_ddp.fwo optional ice_ddp \ dependency "ice_ddp.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ice_ddp.fwo" ice_ddp.fw optional ice_ddp \ dependency "$S/contrib/dev/ice/ice-1.3.16.0.pkg" \ compile-with "${CP} $S/contrib/dev/ice/ice-1.3.16.0.pkg ice_ddp.fw" \ no-obj no-implicit-rule \ clean "ice_ddp.fw" dev/iicbus/sy8106a.c optional sy8106a fdt dev/iicbus/twsi/mv_twsi.c optional twsi fdt dev/iicbus/twsi/a10_twsi.c optional twsi fdt dev/iicbus/twsi/twsi.c optional twsi fdt dev/iommu/busdma_iommu.c optional iommu dev/iommu/iommu_gas.c optional iommu dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/mbox/mbox_if.m optional soc_brcm_bcm2837 dev/mmc/host/dwmmc.c optional dwmmc fdt dev/mmc/host/dwmmc_altera.c optional dwmmc dwmmc_altera fdt dev/mmc/host/dwmmc_hisi.c optional dwmmc dwmmc_hisi fdt dev/mmc/host/dwmmc_rockchip.c optional dwmmc rk_dwmmc fdt dev/neta/if_mvneta_fdt.c optional neta fdt dev/neta/if_mvneta.c optional neta mdio mii dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofwpci.c optional fdt pci dev/pci/controller/pci_n1sdp.c optional pci_n1sdp acpi dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_acpi.c optional pci acpi dev/pci/pci_host_generic_fdt.c optional pci fdt dev/pci/pci_dw_mv.c optional pci fdt dev/pci/pci_dw.c optional pci fdt dev/pci/pci_dw_if.m optional pci fdt dev/psci/psci.c standard dev/psci/smccc_arm64.S standard dev/psci/smccc.c standard dev/safexcel/safexcel.c optional safexcel fdt dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci fdt dev/uart/uart_cpu_arm64.c optional uart dev/uart/uart_dev_mu.c optional uart uart_mu dev/uart/uart_dev_pl011.c optional uart pl011 dev/usb/controller/dwc_otg_hisi.c optional dwcotg fdt soc_hisi_hi6220 dev/usb/controller/dwc3.c optional fdt dwc3 dev/usb/controller/ehci_mv.c optional ehci_mv fdt dev/usb/controller/generic_ehci.c optional ehci dev/usb/controller/generic_ehci_acpi.c optional ehci acpi dev/usb/controller/generic_ehci_fdt.c optional ehci fdt dev/usb/controller/generic_ohci.c optional ohci fdt dev/usb/controller/generic_usb_if.m optional ohci fdt dev/usb/controller/musb_otg_allwinner.c optional musb fdt soc_allwinner_a64 dev/usb/controller/usb_nop_xceiv.c optional fdt ext_resources dev/usb/controller/generic_xhci.c optional xhci dev/usb/controller/generic_xhci_acpi.c optional xhci acpi dev/usb/controller/generic_xhci_fdt.c optional xhci fdt dev/vnic/mrml_bridge.c optional vnic fdt dev/vnic/nic_main.c optional vnic pci dev/vnic/nicvf_main.c optional vnic pci pci_iov dev/vnic/nicvf_queues.c optional vnic pci pci_iov dev/vnic/thunder_bgx_fdt.c optional vnic fdt dev/vnic/thunder_bgx.c optional vnic pci dev/vnic/thunder_mdio_fdt.c optional vnic fdt dev/vnic/thunder_mdio.c optional vnic dev/vnic/lmac_if.m optional inet | inet6 | vnic kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_intr.c optional intrng kern/subr_physmem.c standard libkern/bcmp.c standard libkern/memcmp.c standard \ compile-with "${NORMAL_C:N-fsanitize*}" libkern/memset.c standard \ compile-with "${NORMAL_C:N-fsanitize*}" libkern/arm64/crc32c_armv8.S standard cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/aarch64/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" # RockChip Drivers arm64/rockchip/rk3399_emmcphy.c optional fdt rk_emmcphy soc_rockchip_rk3399 arm64/rockchip/rk_dwc3.c optional fdt rk_dwc3 soc_rockchip_rk3399 arm64/rockchip/rk_i2c.c optional fdt rk_i2c soc_rockchip_rk3328 | fdt rk_i2c soc_rockchip_rk3399 arm64/rockchip/rk805.c optional fdt rk805 soc_rockchip_rk3328 | fdt rk805 soc_rockchip_rk3399 arm64/rockchip/rk_grf.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/rk_pinctrl.c optional fdt rk_pinctrl soc_rockchip_rk3328 | fdt rk_pinctrl soc_rockchip_rk3399 arm64/rockchip/rk_gpio.c optional fdt rk_gpio soc_rockchip_rk3328 | fdt rk_gpio soc_rockchip_rk3399 arm64/rockchip/rk_iodomain.c optional fdt rk_iodomain arm64/rockchip/rk_spi.c optional fdt rk_spi arm64/rockchip/rk_usb2phy.c optional fdt rk_usb2phy soc_rockchip_rk3328 | soc_rockchip_rk3399 arm64/rockchip/rk_typec_phy.c optional fdt rk_typec_phy soc_rockchip_rk3399 arm64/rockchip/if_dwc_rk.c optional fdt dwc_rk soc_rockchip_rk3328 | fdt dwc_rk soc_rockchip_rk3399 arm64/rockchip/rk_tsadc_if.m optional fdt soc_rockchip_rk3399 arm64/rockchip/rk_tsadc.c optional fdt soc_rockchip_rk3399 arm64/rockchip/rk_pwm.c optional fdt rk_pwm arm64/rockchip/rk_pcie.c optional fdt pci soc_rockchip_rk3399 arm64/rockchip/rk_pcie_phy.c optional fdt pci soc_rockchip_rk3399 dev/dwc/if_dwc.c optional fdt dwc_rk soc_rockchip_rk3328 | fdt dwc_rk soc_rockchip_rk3399 dev/dwc/if_dwc_if.m optional fdt dwc_rk soc_rockchip_rk3328 | fdt dwc_rk soc_rockchip_rk3399 # RockChip Clock support arm64/rockchip/clk/rk_cru.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_armclk.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_composite.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_fract.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_gate.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_mux.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk_clk_pll.c optional fdt soc_rockchip_rk3328 | fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk3328_cru.c optional fdt soc_rockchip_rk3328 arm64/rockchip/clk/rk3399_cru.c optional fdt soc_rockchip_rk3399 arm64/rockchip/clk/rk3399_pmucru.c optional fdt soc_rockchip_rk3399 # i.MX8 Clock support arm64/freescale/imx/imx8mq_ccm.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_gate.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_mux.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_composite.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_sscg_pll.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_frac_pll.c optional fdt soc_freescale_imx8 # iMX drivers arm/freescale/imx/imx_gpio.c optional gpio soc_freescale_imx8 arm/freescale/imx/imx_i2c.c optional fsliic arm/freescale/imx/imx_machdep.c optional fdt soc_freescale_imx8 arm64/freescale/imx/imx7gpc.c optional fdt soc_freescale_imx8 dev/ffec/if_ffec.c optional ffec # Nvidia firmware for Tegra tegra210_xusb_fw.c optional tegra210_xusb_fw \ dependency "$S/conf/files.arm64" \ compile-with "${AWK} -f $S/tools/fw_stub.awk tegra210_xusb.fw:tegra210_xusb_fw -mtegra210_xusb_fw -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "tegra210_xusb_fw.c" tegra210_xusb.fwo optional tegra210_xusb_fw \ dependency "tegra210_xusb.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "tegra210_xusb.fwo" tegra210_xusb.fw optional tegra210_xusb_fw \ dependency "$S/contrib/dev/nvidia/tegra210_xusb.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "tegra210_xusb.fw" compat/linuxkpi/common/src/linux_fpu.c optional compat_linuxkpi \ compile-with "${LINUXKPI_C}" diff --git a/sys/crypto/armv8/armv8_crypto.c b/sys/crypto/armv8/armv8_crypto.c index f296337e6f33..be39168d50f3 100644 --- a/sys/crypto/armv8/armv8_crypto.c +++ b/sys/crypto/armv8/armv8_crypto.c @@ -1,416 +1,544 @@ /*- * Copyright (c) 2005-2008 Pawel Jakub Dawidek * Copyright (c) 2010 Konstantin Belousov * Copyright (c) 2014,2016 The FreeBSD Foundation + * Copyright (c) 2020 Ampere Computing * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This is based on the aesni code. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include struct armv8_crypto_softc { int dieing; int32_t cid; struct rwlock lock; + bool has_pmul; }; static struct mtx *ctx_mtx; static struct fpu_kern_ctx **ctx_vfp; #define AQUIRE_CTX(i, ctx) \ do { \ (i) = PCPU_GET(cpuid); \ mtx_lock(&ctx_mtx[(i)]); \ (ctx) = ctx_vfp[(i)]; \ } while (0) #define RELEASE_CTX(i, ctx) \ do { \ mtx_unlock(&ctx_mtx[(i)]); \ (i) = -1; \ (ctx) = NULL; \ } while (0) static int armv8_crypto_cipher_process(struct armv8_crypto_session *, struct cryptop *); MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data"); static void armv8_crypto_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "armv8crypto", -1) == NULL && BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0) panic("ARMv8 crypto: could not attach"); } static int armv8_crypto_probe(device_t dev) { uint64_t reg; int ret = ENXIO; reg = READ_SPECIALREG(id_aa64isar0_el1); switch (ID_AA64ISAR0_AES_VAL(reg)) { case ID_AA64ISAR0_AES_BASE: + ret = 0; + device_set_desc(dev, "AES-CBC,AES-XTS"); + break; case ID_AA64ISAR0_AES_PMULL: ret = 0; + device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM"); + break; + default: break; case ID_AA64ISAR0_AES_NONE: device_printf(dev, "CPU lacks AES instructions"); break; } - device_set_desc_copy(dev, "AES-CBC,AES-XTS"); - /* TODO: Check more fields as we support more features */ return (ret); } static int armv8_crypto_attach(device_t dev) { struct armv8_crypto_softc *sc; + uint64_t reg; int i; sc = device_get_softc(dev); sc->dieing = 0; + reg = READ_SPECIALREG(id_aa64isar0_el1); + + if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL) + sc->has_pmul = true; + sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); if (sc->cid < 0) { device_printf(dev, "Could not get crypto driver id.\n"); return (ENOMEM); } rw_init(&sc->lock, "armv8crypto"); ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO, M_WAITOK|M_ZERO); ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO, M_WAITOK|M_ZERO); CPU_FOREACH(i) { ctx_vfp[i] = fpu_kern_alloc_ctx(0); mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW); } return (0); } static int armv8_crypto_detach(device_t dev) { struct armv8_crypto_softc *sc; int i; sc = device_get_softc(dev); rw_wlock(&sc->lock); sc->dieing = 1; rw_wunlock(&sc->lock); crypto_unregister_all(sc->cid); rw_destroy(&sc->lock); CPU_FOREACH(i) { if (ctx_vfp[i] != NULL) { mtx_destroy(&ctx_mtx[i]); fpu_kern_free_ctx(ctx_vfp[i]); } ctx_vfp[i] = NULL; } free(ctx_mtx, M_ARMV8_CRYPTO); ctx_mtx = NULL; free(ctx_vfp, M_ARMV8_CRYPTO); ctx_vfp = NULL; return (0); } +#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD) + static int armv8_crypto_probesession(device_t dev, const struct crypto_session_params *csp) { + struct armv8_crypto_softc *sc; + + sc = device_get_softc(dev); - if (csp->csp_flags != 0) + if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (!sc->has_pmul) + return (EINVAL); + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return (EINVAL); + if (csp->csp_auth_mlen != 0 && + csp->csp_auth_mlen != GMAC_DIGEST_LEN) + return (EINVAL); + switch (csp->csp_cipher_klen * 8) { + case 128: + case 192: + case 256: + break; + default: + return (EINVAL); + } + break; + default: + return (EINVAL); + } + break; case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: if (csp->csp_ivlen != AES_BLOCK_LEN) return (EINVAL); switch (csp->csp_cipher_klen * 8) { case 128: case 192: case 256: break; default: return (EINVAL); } break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return (EINVAL); switch (csp->csp_cipher_klen * 8) { case 256: case 512: break; default: return (EINVAL); } break; default: return (EINVAL); } break; default: return (EINVAL); } return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); } -static void +static int armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, const struct crypto_session_params *csp, const uint8_t *key, int keylen) { - int i; + __uint128_val_t H; + struct fpu_kern_ctx *ctx; + int kt, i; if (csp->csp_cipher_alg == CRYPTO_AES_XTS) keylen /= 2; switch (keylen * 8) { case 128: - ses->rounds = AES128_ROUNDS; - break; case 192: - ses->rounds = AES192_ROUNDS; - break; case 256: - ses->rounds = AES256_ROUNDS; break; default: - panic("invalid AES key length"); + return (EINVAL); } - rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8); - rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8); - if (csp->csp_cipher_alg == CRYPTO_AES_XTS) - rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8); + kt = is_fpu_kern_thread(0); + if (!kt) { + AQUIRE_CTX(i, ctx); + fpu_kern_enter(curthread, ctx, + FPU_KERN_NORMAL | FPU_KERN_KTHR); + } + + aes_v8_set_encrypt_key(key, + keylen * 8, &ses->enc_schedule); + + if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) || + (csp->csp_cipher_alg == CRYPTO_AES_CBC)) + aes_v8_set_decrypt_key(key, + keylen * 8, &ses->dec_schedule); + + if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { + memset(H.c, 0, sizeof(H.c)); + aes_v8_encrypt(H.c, H.c, &ses->enc_schedule); + H.u[0] = bswap64(H.u[0]); + H.u[1] = bswap64(H.u[1]); + gcm_init_v8(ses->Htable, H.u); + } - for (i = 0; i < nitems(ses->enc_schedule); i++) { - ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]); - ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]); - if (csp->csp_cipher_alg == CRYPTO_AES_XTS) - ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]); + if (!kt) { + fpu_kern_leave(curthread, ctx); + RELEASE_CTX(i, ctx); } + + return (0); } static int armv8_crypto_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct armv8_crypto_softc *sc; struct armv8_crypto_session *ses; + int error; sc = device_get_softc(dev); rw_wlock(&sc->lock); if (sc->dieing) { rw_wunlock(&sc->lock); return (EINVAL); } ses = crypto_get_driver_session(cses); - armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, + error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, csp->csp_cipher_klen); rw_wunlock(&sc->lock); - return (0); + return (error); } static int armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) { struct armv8_crypto_session *ses; - int error; - - /* We can only handle full blocks for now */ - if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { - error = EINVAL; - goto out; - } ses = crypto_get_driver_session(crp->crp_session); - error = armv8_crypto_cipher_process(ses, crp); - -out: - crp->crp_etype = error; + crp->crp_etype = armv8_crypto_cipher_process(ses, crp); crypto_done(crp); return (0); } static uint8_t * -armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated) +armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated) { uint8_t *addr; - addr = crypto_contiguous_subsegment(crp, crp->crp_payload_start, - crp->crp_payload_length); + addr = crypto_contiguous_subsegment(crp, start, length); if (addr != NULL) { *allocated = 0; return (addr); } addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); if (addr != NULL) { *allocated = 1; - crypto_copydata(crp, crp->crp_payload_start, - crp->crp_payload_length, addr); + crypto_copydata(crp, start, length, addr); } else *allocated = 0; return (addr); } static int armv8_crypto_cipher_process(struct armv8_crypto_session *ses, struct cryptop *crp) { const struct crypto_session_params *csp; struct fpu_kern_ctx *ctx; - uint8_t *buf; - uint8_t iv[AES_BLOCK_LEN]; - int allocated, i; + uint8_t *buf, *authbuf, *outbuf; + uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN]; + int allocated, authallocated, outallocated, i; int encflag; int kt; + int error; + bool outcopy; csp = crypto_get_params(crp->crp_session); encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); - buf = armv8_crypto_cipher_alloc(crp, &allocated); + allocated = 0; + outallocated = 0; + authallocated = 0; + authbuf = NULL; + kt = 1; + + buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start, + crp->crp_payload_length, &allocated); if (buf == NULL) return (ENOMEM); + if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { + if (crp->crp_aad != NULL) + authbuf = crp->crp_aad; + else + authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start, + crp->crp_aad_length, &authallocated); + if (authbuf == NULL) { + error = ENOMEM; + goto out; + } + } + + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf, + crp->crp_payload_output_start, crp->crp_payload_length); + if (outbuf == NULL) { + outcopy = true; + if (allocated) + outbuf = buf; + else { + outbuf = malloc(crp->crp_payload_length, + M_ARMV8_CRYPTO, M_NOWAIT); + if (outbuf == NULL) { + error = ENOMEM; + goto out; + } + outallocated = true; + } + } else + outcopy = false; + } else { + outbuf = buf; + outcopy = allocated; + } + kt = is_fpu_kern_thread(0); if (!kt) { AQUIRE_CTX(i, ctx); fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); } if (crp->crp_cipher_key != NULL) { armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, csp->csp_cipher_klen); } crypto_read_iv(crp, iv); /* Do work */ switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: + if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) { + error = EINVAL; + goto out; + } if (encflag) - armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule, + armv8_aes_encrypt_cbc(&ses->enc_schedule, crp->crp_payload_length, buf, buf, iv); else - armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule, + armv8_aes_decrypt_cbc(&ses->dec_schedule, crp->crp_payload_length, buf, iv); break; case CRYPTO_AES_XTS: if (encflag) - armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule, - ses->xts_schedule, crp->crp_payload_length, buf, + armv8_aes_encrypt_xts(&ses->enc_schedule, + &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, buf, iv); else - armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule, - ses->xts_schedule, crp->crp_payload_length, buf, + armv8_aes_decrypt_xts(&ses->dec_schedule, + &ses->xts_schedule.aes_key, crp->crp_payload_length, buf, buf, iv); break; + case CRYPTO_AES_NIST_GCM_16: + if (encflag) { + memset(tag, 0, sizeof(tag)); + armv8_aes_encrypt_gcm(&ses->enc_schedule, + crp->crp_payload_length, + buf, outbuf, + crp->crp_aad_length, authbuf, + tag, iv, ses->Htable); + crypto_copyback(crp, crp->crp_digest_start, sizeof(tag), + tag); + } else { + crypto_copydata(crp, crp->crp_digest_start, sizeof(tag), + tag); + if (armv8_aes_decrypt_gcm(&ses->enc_schedule, + crp->crp_payload_length, + buf, outbuf, + crp->crp_aad_length, authbuf, + tag, iv, ses->Htable) != 0) { + error = EBADMSG; + goto out; + } + } + break; } - if (allocated) - crypto_copyback(crp, crp->crp_payload_start, - crp->crp_payload_length, buf); + if (outcopy) + crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ? + crp->crp_payload_output_start : crp->crp_payload_start, + crp->crp_payload_length, outbuf); + error = 0; +out: if (!kt) { fpu_kern_leave(curthread, ctx); RELEASE_CTX(i, ctx); } + if (allocated) zfree(buf, M_ARMV8_CRYPTO); - return (0); + if (authallocated) + zfree(authbuf, M_ARMV8_CRYPTO); + if (outallocated) + zfree(outbuf, M_ARMV8_CRYPTO); + explicit_bzero(iv, sizeof(iv)); + explicit_bzero(tag, sizeof(tag)); + + return (error); } static device_method_t armv8_crypto_methods[] = { DEVMETHOD(device_identify, armv8_crypto_identify), DEVMETHOD(device_probe, armv8_crypto_probe), DEVMETHOD(device_attach, armv8_crypto_attach), DEVMETHOD(device_detach, armv8_crypto_detach), DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession), DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession), DEVMETHOD(cryptodev_process, armv8_crypto_process), DEVMETHOD_END, }; static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods, sizeof(struct armv8_crypto_softc)); static devclass_t armv8_crypto_devclass; DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass, 0, 0); diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h index 2d0be163b072..0e4cd91e7062 100644 --- a/sys/crypto/armv8/armv8_crypto.h +++ b/sys/crypto/armv8/armv8_crypto.h @@ -1,58 +1,87 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARMV8_CRYPTO_H_ #define _ARMV8_CRYPTO_H_ -#define AES128_ROUNDS 10 -#define AES192_ROUNDS 12 #define AES256_ROUNDS 14 #define AES_SCHED_LEN ((AES256_ROUNDS + 1) * AES_BLOCK_LEN) +typedef struct { + uint32_t aes_key[AES_SCHED_LEN/4]; + int aes_rounds; +} AES_key_t; + +typedef union { + uint64_t u[2]; + uint32_t d[4]; + uint8_t c[16]; + size_t t[16 / sizeof(size_t)]; +} __uint128_val_t; + struct armv8_crypto_session { - uint32_t enc_schedule[AES_SCHED_LEN/4]; - uint32_t dec_schedule[AES_SCHED_LEN/4]; - uint32_t xts_schedule[AES_SCHED_LEN/4]; - int algo; - int rounds; + AES_key_t enc_schedule; + AES_key_t dec_schedule; + AES_key_t xts_schedule; + __uint128_val_t Htable[16]; }; -void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *, +/* Prototypes for aesv8-armx.S */ +void aes_v8_encrypt(uint8_t *in, uint8_t *out, const AES_key_t *key); +int aes_v8_set_encrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key); +int aes_v8_set_decrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key); + +/* Prototypes for ghashv8-armx.S */ +void gcm_init_v8(__uint128_val_t Htable[16], const uint64_t Xi[2]); +void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]); +void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len); + +void armv8_aes_encrypt_cbc(const AES_key_t *, size_t, const uint8_t *, uint8_t *, const uint8_t[static AES_BLOCK_LEN]); -void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *, +void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, uint8_t *, const uint8_t[static AES_BLOCK_LEN]); +void armv8_aes_encrypt_gcm(AES_key_t *, size_t, const uint8_t *, + uint8_t *, size_t, const uint8_t*, + uint8_t tag[static GMAC_DIGEST_LEN], + const uint8_t[static AES_BLOCK_LEN], + const __uint128_val_t *); +int armv8_aes_decrypt_gcm(AES_key_t *, size_t, const uint8_t *, + uint8_t *, size_t, const uint8_t*, + const uint8_t tag[static GMAC_DIGEST_LEN], + const uint8_t[static AES_BLOCK_LEN], + const __uint128_val_t *); -void armv8_aes_encrypt_xts(int, const void *, const void *, size_t, +void armv8_aes_encrypt_xts(AES_key_t *, const void *, size_t, const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); -void armv8_aes_decrypt_xts(int, const void *, const void *, size_t, +void armv8_aes_decrypt_xts(AES_key_t *, const void *, size_t, const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); #endif /* _ARMV8_CRYPTO_H_ */ diff --git a/sys/crypto/armv8/armv8_crypto_wrap.c b/sys/crypto/armv8/armv8_crypto_wrap.c index 83af3fad40ef..ea93f1b9a176 100644 --- a/sys/crypto/armv8/armv8_crypto_wrap.c +++ b/sys/crypto/armv8/armv8_crypto_wrap.c @@ -1,220 +1,406 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation + * Copyright (c) 2020 Ampere Computing * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This code is built with floating-point enabled. Make sure to have entered * into floating-point context before calling any of these functions. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include +#include +#include #include #include static uint8x16_t armv8_aes_enc(int rounds, const uint8x16_t *keysched, const uint8x16_t from) { uint8x16_t tmp; int i; tmp = from; for (i = 0; i < rounds - 1; i += 2) { tmp = vaeseq_u8(tmp, keysched[i]); tmp = vaesmcq_u8(tmp); tmp = vaeseq_u8(tmp, keysched[i + 1]); tmp = vaesmcq_u8(tmp); } tmp = vaeseq_u8(tmp, keysched[rounds - 1]); tmp = vaesmcq_u8(tmp); tmp = vaeseq_u8(tmp, keysched[rounds]); tmp = veorq_u8(tmp, keysched[rounds + 1]); return (tmp); } static uint8x16_t armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from) { uint8x16_t tmp; int i; tmp = from; for (i = 0; i < rounds - 1; i += 2) { tmp = vaesdq_u8(tmp, keysched[i]); tmp = vaesimcq_u8(tmp); tmp = vaesdq_u8(tmp, keysched[i+1]); tmp = vaesimcq_u8(tmp); } tmp = vaesdq_u8(tmp, keysched[rounds - 1]); tmp = vaesimcq_u8(tmp); tmp = vaesdq_u8(tmp, keysched[rounds]); tmp = veorq_u8(tmp, keysched[rounds + 1]); return (tmp); } void -armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len, +armv8_aes_encrypt_cbc(const AES_key_t *key, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN]) { uint8x16_t tot, ivreg, tmp; size_t i; len /= AES_BLOCK_LEN; ivreg = vld1q_u8(iv); for (i = 0; i < len; i++) { tmp = vld1q_u8(from); - tot = armv8_aes_enc(rounds - 1, key_schedule, - veorq_u8(tmp, ivreg)); + tot = armv8_aes_enc(key->aes_rounds - 1, + (const void*)key->aes_key, veorq_u8(tmp, ivreg)); ivreg = tot; vst1q_u8(to, tot); from += AES_BLOCK_LEN; to += AES_BLOCK_LEN; } } void -armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len, +armv8_aes_decrypt_cbc(const AES_key_t *key, size_t len, uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN]) { uint8x16_t ivreg, nextiv, tmp; size_t i; len /= AES_BLOCK_LEN; ivreg = vld1q_u8(iv); for (i = 0; i < len; i++) { nextiv = vld1q_u8(buf); - tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv); + tmp = armv8_aes_dec(key->aes_rounds - 1, + (const void*)key->aes_key, nextiv); vst1q_u8(buf, veorq_u8(tmp, ivreg)); ivreg = nextiv; buf += AES_BLOCK_LEN; } } #define AES_XTS_BLOCKSIZE 16 #define AES_XTS_IVSIZE 8 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ static inline int32x4_t xts_crank_lfsr(int32x4_t inp) { const int32x4_t alphamask = {AES_XTS_ALPHA, 1, 1, 1}; int32x4_t xtweak, ret; /* set up xor mask */ xtweak = vextq_s32(inp, inp, 3); xtweak = vshrq_n_s32(xtweak, 31); xtweak &= alphamask; /* next term */ ret = vshlq_n_s32(inp, 1); ret ^= xtweak; return ret; } static void armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule, uint8x16_t *tweak, const uint8_t *from, uint8_t *to, int do_encrypt) { uint8x16_t block; block = vld1q_u8(from) ^ *tweak; if (do_encrypt) block = armv8_aes_enc(rounds - 1, key_schedule, block); else block = armv8_aes_dec(rounds - 1, key_schedule, block); vst1q_u8(to, block ^ *tweak); *tweak = vreinterpretq_u8_s32(xts_crank_lfsr(vreinterpretq_s32_u8(*tweak))); } static void armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule, const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt) { uint8x16_t tweakreg; uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16); size_t i, cnt; /* * Prepare tweak as E_k2(IV). IV is specified as LE representation * of a 64-bit block number which we allow to be passed in directly. */ #if BYTE_ORDER == LITTLE_ENDIAN bcopy(iv, tweak, AES_XTS_IVSIZE); /* Last 64 bits of IV are always zero. */ bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); #else #error Only LITTLE_ENDIAN architectures are supported. #endif tweakreg = vld1q_u8(tweak); tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg); cnt = len / AES_XTS_BLOCKSIZE; for (i = 0; i < cnt; i++) { armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg, from, to, do_encrypt); from += AES_XTS_BLOCKSIZE; to += AES_XTS_BLOCKSIZE; } } void -armv8_aes_encrypt_xts(int rounds, const void *data_schedule, +armv8_aes_encrypt_xts(AES_key_t *data_schedule, const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN]) { - armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, - iv, 1); + armv8_aes_crypt_xts(data_schedule->aes_rounds, + (const void *)&data_schedule->aes_key, tweak_schedule, len, from, + to, iv, 1); } void -armv8_aes_decrypt_xts(int rounds, const void *data_schedule, +armv8_aes_decrypt_xts(AES_key_t *data_schedule, const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN]) { - armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to, - iv, 0); + armv8_aes_crypt_xts(data_schedule->aes_rounds, + (const void *)&data_schedule->aes_key, tweak_schedule, len, from, + to,iv, 0); + +} + +#define AES_INC_COUNTER(counter) \ + do { \ + for (int pos = AES_BLOCK_LEN - 1; \ + pos >= 0; pos--) \ + if (++(counter)[pos]) \ + break; \ + } while (0) + +void +armv8_aes_encrypt_gcm(AES_key_t *aes_key, size_t len, + const uint8_t *from, uint8_t *to, + size_t authdatalen, const uint8_t *authdata, + uint8_t tag[static GMAC_DIGEST_LEN], + const uint8_t iv[static AES_GCM_IV_LEN], + const __uint128_val_t *Htable) +{ + size_t i; + const uint64_t *from64; + uint64_t *to64; + uint8_t aes_counter[AES_BLOCK_LEN]; + uint8_t block[AES_BLOCK_LEN]; + size_t trailer; + __uint128_val_t EK0, EKi, Xi, lenblock; + + bzero(&aes_counter, AES_BLOCK_LEN); + memcpy(aes_counter, iv, AES_GCM_IV_LEN); + + /* Setup the counter */ + aes_counter[AES_BLOCK_LEN - 1] = 1; + + /* EK0 for a final GMAC round */ + aes_v8_encrypt(aes_counter, EK0.c, aes_key); + + /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ + aes_counter[AES_BLOCK_LEN - 1] = 2; + + memset(Xi.c, 0, sizeof(Xi.c)); + memset(block, 0, sizeof(block)); + memcpy(block, authdata, min(authdatalen, sizeof(block))); + gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + + from64 = (const uint64_t*)from; + to64 = (uint64_t*)to; + trailer = len % AES_BLOCK_LEN; + + for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { + aes_v8_encrypt(aes_counter, EKi.c, aes_key); + AES_INC_COUNTER(aes_counter); + to64[0] = from64[0] ^ EKi.u[0]; + to64[1] = from64[1] ^ EKi.u[1]; + gcm_ghash_v8(Xi.u, Htable, (uint8_t*)to64, AES_BLOCK_LEN); + + to64 += 2; + from64 += 2; + } + + to += (len - trailer); + from += (len - trailer); + + if (trailer) { + aes_v8_encrypt(aes_counter, EKi.c, aes_key); + AES_INC_COUNTER(aes_counter); + for (i = 0; i < trailer; i++) { + block[i] = to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN]; + } + + for (; i < AES_BLOCK_LEN; i++) + block[i] = 0; + + gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + } + + /* Lengths block */ + lenblock.u[0] = lenblock.u[1] = 0; + lenblock.d[1] = htobe32(authdatalen * 8); + lenblock.d[3] = htobe32(len * 8); + gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); + + Xi.u[0] ^= EK0.u[0]; + Xi.u[1] ^= EK0.u[1]; + memcpy(tag, Xi.c, GMAC_DIGEST_LEN); + + explicit_bzero(aes_counter, sizeof(aes_counter)); + explicit_bzero(Xi.c, sizeof(Xi.c)); + explicit_bzero(EK0.c, sizeof(EK0.c)); + explicit_bzero(EKi.c, sizeof(EKi.c)); + explicit_bzero(lenblock.c, sizeof(lenblock.c)); +} + +int +armv8_aes_decrypt_gcm(AES_key_t *aes_key, size_t len, + const uint8_t *from, uint8_t *to, + size_t authdatalen, const uint8_t *authdata, + const uint8_t tag[static GMAC_DIGEST_LEN], + const uint8_t iv[static AES_GCM_IV_LEN], + const __uint128_val_t *Htable) +{ + size_t i; + const uint64_t *from64; + uint64_t *to64; + uint8_t aes_counter[AES_BLOCK_LEN]; + uint8_t block[AES_BLOCK_LEN]; + size_t trailer; + __uint128_val_t EK0, EKi, Xi, lenblock; + int error; + + error = 0; + bzero(&aes_counter, AES_BLOCK_LEN); + memcpy(aes_counter, iv, AES_GCM_IV_LEN); + + /* Setup the counter */ + aes_counter[AES_BLOCK_LEN - 1] = 1; + + /* EK0 for a final GMAC round */ + aes_v8_encrypt(aes_counter, EK0.c, aes_key); + + memset(Xi.c, 0, sizeof(Xi.c)); + memset(block, 0, sizeof(block)); + memcpy(block, authdata, min(authdatalen, sizeof(block))); + gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + trailer = len % AES_BLOCK_LEN; + gcm_ghash_v8(Xi.u, Htable, from, len - trailer); + + if (trailer) { + for (i = 0; i < trailer; i++) + block[i] = from[len - trailer + i]; + for (; i < AES_BLOCK_LEN; i++) + block[i] = 0; + gcm_ghash_v8(Xi.u, Htable, block, AES_BLOCK_LEN); + } + + /* Lengths block */ + lenblock.u[0] = lenblock.u[1] = 0; + lenblock.d[1] = htobe32(authdatalen * 8); + lenblock.d[3] = htobe32(len * 8); + gcm_ghash_v8(Xi.u, Htable, lenblock.c, AES_BLOCK_LEN); + + Xi.u[0] ^= EK0.u[0]; + Xi.u[1] ^= EK0.u[1]; + if (timingsafe_bcmp(tag, Xi.c, GMAC_DIGEST_LEN) != 0) { + error = EBADMSG; + goto out; + } + + /* GCM starts with 2 as counter, 1 is used for final xor of tag. */ + aes_counter[AES_BLOCK_LEN - 1] = 2; + + from64 = (const uint64_t*)from; + to64 = (uint64_t*)to; + + for (i = 0; i < (len - trailer); i += AES_BLOCK_LEN) { + aes_v8_encrypt(aes_counter, EKi.c, aes_key); + AES_INC_COUNTER(aes_counter); + to64[0] = from64[0] ^ EKi.u[0]; + to64[1] = from64[1] ^ EKi.u[1]; + to64 += 2; + from64 += 2; + } + + to += (len - trailer); + from += (len - trailer); + + if (trailer) { + aes_v8_encrypt(aes_counter, EKi.c, aes_key); + AES_INC_COUNTER(aes_counter); + for (i = 0; i < trailer; i++) + to[i] = from[i] ^ EKi.c[i % AES_BLOCK_LEN]; + } + +out: + explicit_bzero(aes_counter, sizeof(aes_counter)); + explicit_bzero(Xi.c, sizeof(Xi.c)); + explicit_bzero(EK0.c, sizeof(EK0.c)); + explicit_bzero(EKi.c, sizeof(EKi.c)); + explicit_bzero(lenblock.c, sizeof(lenblock.c)); + + return (error); } diff --git a/sys/modules/armv8crypto/Makefile b/sys/modules/armv8crypto/Makefile index 4b9a84c953d2..bd38fe039fc9 100644 --- a/sys/modules/armv8crypto/Makefile +++ b/sys/modules/armv8crypto/Makefile @@ -1,21 +1,38 @@ # $FreeBSD$ .PATH: ${SRCTOP}/sys/crypto/armv8 +.PATH: ${SRCTOP}/sys/crypto/openssl/aarch64 KMOD= armv8crypto SRCS= armv8_crypto.c SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h -OBJS+= armv8_crypto_wrap.o +OBJS+= armv8_crypto_wrap.o aesv8-armx.o ghashv8-armx.o # Remove -nostdinc so we can get the intrinsics. armv8_crypto_wrap.o: armv8_crypto_wrap.c ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ -I${SRCTOP}/sys/crypto/armv8 \ ${WERROR} ${PROF} \ -march=armv8-a+crypto ${.IMPSRC} ${CTFCONVERT_CMD} +aesv8-armx.o: aesv8-armx.S + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ + -I${SRCTOP}/sys/crypto/armv8 \ + -I${SRCTOP}/sys/crypto/openssl/crypto \ + ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + +ghashv8-armx.o: ghashv8-armx.S + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} \ + -I${SRCTOP}/sys/crypto/armv8 \ + -I${SRCTOP}/sys/crypto/openssl/crypto \ + ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + armv8_crypto_wrap.o: armv8_crypto.h .include