Index: head/share/man/man4/aesni.4 =================================================================== --- head/share/man/man4/aesni.4 (revision 324036) +++ head/share/man/man4/aesni.4 (revision 324037) @@ -1,95 +1,111 @@ .\" Copyright (c) 2010 Konstantin Belousov .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd December 14, 2015 +.Dd September 26, 2017 .Dt AESNI 4 .Os .Sh NAME .Nm aesni -.Nd "driver for the AES accelerator on Intel CPUs" +.Nd "driver for the AES and SHA accelerator on x86 CPUs" .Sh SYNOPSIS To compile this driver into the kernel, place the following lines in your kernel configuration file: .Bd -ragged -offset indent .Cd "device crypto" .Cd "device cryptodev" .Cd "device aesni" .Ed .Pp Alternatively, to load the driver as a module at boot time, place the following line in .Xr loader.conf 5 : .Bd -literal -offset indent aesni_load="YES" .Ed .Sh DESCRIPTION -Starting with some models of Core i5/i7, Intel processors implement -a new set of instructions called AESNI. +Starting with Intel Westmere and AMD Bulldozer, some x86 processors implement a +new set of instructions called AESNI. The set of six instructions accelerates the calculation of the key schedule for key lengths of 128, 192, and 256 of the Advanced Encryption Standard (AES) symmetric cipher, and provides a hardware implementation of the regular and the last encryption and decryption rounds. .Pp The processor capability is reported as AESNI in the Features2 line at boot. +.Pp +Starting with the Intel Goldmont and AMD Ryzen microarchitectures, some x86 +processors implement a new set of SHA instructions. +The set of seven instructions accelerates the calculation of SHA1 and SHA256 +hashes. +.Pp +The processor capability is reported as SHA in the Structured Extended Features +line at boot. +.Pp The .Nm -driver does not attach on systems that lack the required CPU capability. +driver does not attach on systems that lack both CPU capabilities. +On systems that support only one of AESNI or SHA extensions, the driver will +attach and support that one function. .Pp The .Nm -driver registers itself to accelerate AES operations for +driver registers itself to accelerate AES and SHA operations for .Xr crypto 4 . Besides speed, the advantage of using the .Nm driver is that the AESNI operation is data-independent, thus eliminating some attack vectors based on measuring cache use and timings typically present in table-driven implementations. .Sh SEE ALSO .Xr crypt 3 , .Xr crypto 4 , .Xr intro 4 , .Xr ipsec 4 , .Xr padlock 4 , .Xr random 4 , .Xr crypto 9 .Sh HISTORY The .Nm driver first appeared in .Fx 9.0 . +SHA support was added in +.Fx 12.0 . .Sh AUTHORS .An -nosplit The .Nm driver was written by -.An Konstantin Belousov Aq Mt kib@FreeBSD.org . +.An Konstantin Belousov Aq Mt kib@FreeBSD.org +and +.An Conrad Meyer Aq Mt cem@FreeBSD.org . The key schedule calculation code was adopted from the sample provided by Intel and used in the analogous .Ox driver. +The hash step intrinsics implementations were supplied by Intel. Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 (revision 324036) +++ head/sys/conf/files.amd64 (revision 324037) @@ -1,717 +1,727 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # # cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S" \ compile-with "${CC} -x assembler-with-cpp -m32 -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # cloudabi64_vdso.o optional compat_cloudabi64 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_x86_64.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_x86_64.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi64_vdso.o" # cloudabi64_vdso_blob.o optional compat_cloudabi64 \ dependency "cloudabi64_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 cloudabi64_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi64_vdso_blob.o" # linux32_genassym.o optional compat_linux32 \ dependency "$S/amd64/linux32/linux32_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "linux32_genassym.o" # linux32_assym.h optional compat_linux32 \ dependency "$S/kern/genassym.sh linux32_genassym.o" \ compile-with "sh $S/kern/genassym.sh linux32_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "linux32_assym.h" # linux32_locore.o optional compat_linux32 \ dependency "linux32_assym.h $S/amd64/linux32/linux32_locore.s" \ compile-with "${CC} -x assembler-with-cpp -DLOCORE -m32 -shared -s -pipe -I. -I$S -Werror -Wall -fPIC -fno-common -nostdinc -nostdlib -Wl,-T$S/amd64/linux32/linux32_vdso.lds.s -Wl,-soname=linux32_vdso.so,--eh-frame-hdr,-warn-common ${.IMPSRC} -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "linux32_locore.o" # linux32_vdso.so optional compat_linux32 \ dependency "linux32_locore.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 linux32_locore.o ${.TARGET}" \ no-implicit-rule \ clean "linux32_vdso.so" # ia32_genassym.o standard \ dependency "$S/compat/ia32/ia32_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "ia32_genassym.o" # ia32_assym.h standard \ dependency "$S/kern/genassym.sh ia32_genassym.o" \ compile-with "env NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh ia32_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "ia32_assym.h" # font.h optional sc_dflt_font \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'static u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'static u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'static u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # ukbdmap.h optional ukbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "ukbdmap.h" # hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/amd64-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/amd64-elf.hpt27xx_lib.o.uu" \ no-implicit-rule # hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/amd64-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/amd64-elf.raid.o.uu" \ no-implicit-rule # hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/amd64-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/amd64-elf.hptnr_lib.o.uu" \ no-implicit-rule # hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/amd64-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/amd64-elf.hptrr_lib.o.uu" \ no-implicit-rule # amd64/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ dependency "$S/amd64/acpica/acpi_wakecode.S assym.s" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # #amd64/amd64/apic_vector.S standard amd64/amd64/atomic.c standard amd64/amd64/bios.c standard amd64/amd64/bpf_jit_machdep.c optional bpf_jitter amd64/amd64/cpu_switch.S standard amd64/amd64/db_disasm.c optional ddb amd64/amd64/db_interface.c optional ddb amd64/amd64/db_trace.c optional ddb amd64/amd64/efirt.c optional efirt amd64/amd64/elf_machdep.c standard amd64/amd64/exception.S standard amd64/amd64/fpu.c standard amd64/amd64/gdb_machdep.c optional gdb amd64/amd64/in_cksum.c optional inet | inet6 amd64/amd64/initcpu.c standard amd64/amd64/io.c optional io amd64/amd64/locore.S standard no-obj amd64/amd64/xen-locore.S optional xenhvm amd64/amd64/machdep.c standard amd64/amd64/mem.c optional mem amd64/amd64/minidump_machdep.c standard amd64/amd64/mp_machdep.c optional smp amd64/amd64/mpboot.S optional smp amd64/amd64/pmap.c standard amd64/amd64/prof_machdep.c optional profiling-routine amd64/amd64/ptrace_machdep.c standard amd64/amd64/sigtramp.S standard amd64/amd64/support.S standard amd64/amd64/sys_machdep.c standard amd64/amd64/trap.c standard amd64/amd64/uio_machdep.c standard amd64/amd64/uma_machdep.c standard amd64/amd64/vm_machdep.c standard amd64/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 amd64/cloudabi64/cloudabi64_sysvec.c optional compat_cloudabi64 amd64/pci/pci_cfgreg.c optional pci cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S optional zfs | dtrace compile-with "${ZFS_S}" cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" crypto/blowfish/bf_enc.c optional crypto | ipsec | ipsec_support crypto/des/des_enc.c optional crypto | ipsec | \ ipsec_support | netsmb +intel_sha1.o optional aesni \ + dependency "$S/crypto/aesni/intel_sha1.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ + no-implicit-rule \ + clean "intel_sha1.o" +intel_sha256.o optional aesni \ + dependency "$S/crypto/aesni/intel_sha256.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ + no-implicit-rule \ + clean "intel_sha256.o" crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_if.m standard dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_timer.c optional acpi dev/acpi_support/acpi_wmi_if.m standard dev/agp/agp_amd64.c optional agp dev/agp/agp_i810.c optional agp dev/agp/agp_via.c optional agp dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/coretemp/coretemp.c optional coretemp dev/cpuctl/cpuctl.c optional cpuctl dev/dpms/dpms.c optional dpms # There are no systems with isa slots, so all ed isa entries should go.. dev/ed/if_ed_3c503.c optional ed isa ed_3c503 dev/ed/if_ed_isa.c optional ed isa dev/ed/if_ed_wd80x3.c optional ed isa dev/ed/if_ed_hpp.c optional ed isa ed_hpp dev/ed/if_ed_sic.c optional ed isa ed_sic dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ioat/ioat.c optional ioat pci dev/ioat/ioat_test.c optional ioat pci dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux32 dev/ixl/if_ixl.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_main.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_qmgr.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iov.c optional ixl pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_i2c.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_iw.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/if_ixlv.c optional ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixlvc.c optional ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_txrx.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_osdep.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_lan_hmc.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_hmc.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_common.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_nvm.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_adminq.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/fdc/fdc_pccard.c optional fdc pccard dev/gpio/bytgpio.c optional bytgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/input/hv_kbd.c optional hyperv dev/hyperv/input/hv_kbdc.c optional hyperv dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/hyperv/vmbus/amd64/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/amd64/vmbus_vector.S optional hyperv dev/nctgpio/nctgpio.c optional nctgpio dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme dev/nvme/nvme_ns_cmd.c optional nvme dev/nvme/nvme_qpair.c optional nvme dev/nvme/nvme_sim.c optional nvme scbus dev/nvme/nvme_sysctl.c optional nvme dev/nvme/nvme_test.c optional nvme dev/nvme/nvme_util.c optional nvme dev/nvram/nvram.c optional nvram isa dev/random/ivy.c optional rdrand_rng dev/random/nehemiah.c optional padlock_rng dev/qlxge/qls_dbg.c optional qlxge pci dev/qlxge/qls_dump.c optional qlxge pci dev/qlxge/qls_hw.c optional qlxge pci dev/qlxge/qls_ioctl.c optional qlxge pci dev/qlxge/qls_isr.c optional qlxge pci dev/qlxge/qls_os.c optional qlxge pci dev/qlxgb/qla_dbg.c optional qlxgb pci dev/qlxgb/qla_hw.c optional qlxgb pci dev/qlxgb/qla_ioctl.c optional qlxgb pci dev/qlxgb/qla_isr.c optional qlxgb pci dev/qlxgb/qla_misc.c optional qlxgb pci dev/qlxgb/qla_os.c optional qlxgb pci dev/qlxgbe/ql_dbg.c optional qlxgbe pci dev/qlxgbe/ql_hw.c optional qlxgbe pci dev/qlxgbe/ql_ioctl.c optional qlxgbe pci dev/qlxgbe/ql_isr.c optional qlxgbe pci dev/qlxgbe/ql_misc.c optional qlxgbe pci dev/qlxgbe/ql_os.c optional qlxgbe pci dev/qlxgbe/ql_reset.c optional qlxgbe pci dev/qlnx/qlnxe/ecore_cxt.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dbg_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dcbx.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dev.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_hw.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_ops.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_int.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_l2.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_mcp.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_sp_commands.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_spq.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_ioctl.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_os.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/sfxge/common/ef10_ev.c optional sfxge pci dev/sfxge/common/ef10_filter.c optional sfxge pci dev/sfxge/common/ef10_intr.c optional sfxge pci dev/sfxge/common/ef10_mac.c optional sfxge pci dev/sfxge/common/ef10_mcdi.c optional sfxge pci dev/sfxge/common/ef10_nic.c optional sfxge pci dev/sfxge/common/ef10_nvram.c optional sfxge pci dev/sfxge/common/ef10_phy.c optional sfxge pci dev/sfxge/common/ef10_rx.c optional sfxge pci dev/sfxge/common/ef10_tx.c optional sfxge pci dev/sfxge/common/ef10_vpd.c optional sfxge pci dev/sfxge/common/efx_bootcfg.c optional sfxge pci dev/sfxge/common/efx_crc32.c optional sfxge pci dev/sfxge/common/efx_ev.c optional sfxge pci dev/sfxge/common/efx_filter.c optional sfxge pci dev/sfxge/common/efx_hash.c optional sfxge pci dev/sfxge/common/efx_intr.c optional sfxge pci dev/sfxge/common/efx_lic.c optional sfxge pci dev/sfxge/common/efx_mac.c optional sfxge pci dev/sfxge/common/efx_mcdi.c optional sfxge pci dev/sfxge/common/efx_mon.c optional sfxge pci dev/sfxge/common/efx_nic.c optional sfxge pci dev/sfxge/common/efx_nvram.c optional sfxge pci dev/sfxge/common/efx_phy.c optional sfxge pci dev/sfxge/common/efx_port.c optional sfxge pci dev/sfxge/common/efx_rx.c optional sfxge pci dev/sfxge/common/efx_sram.c optional sfxge pci dev/sfxge/common/efx_tx.c optional sfxge pci dev/sfxge/common/efx_vpd.c optional sfxge pci dev/sfxge/common/hunt_nic.c optional sfxge pci dev/sfxge/common/mcdi_mon.c optional sfxge pci dev/sfxge/common/medford_nic.c optional sfxge pci dev/sfxge/common/siena_mac.c optional sfxge pci dev/sfxge/common/siena_mcdi.c optional sfxge pci dev/sfxge/common/siena_nic.c optional sfxge pci dev/sfxge/common/siena_nvram.c optional sfxge pci dev/sfxge/common/siena_phy.c optional sfxge pci dev/sfxge/common/siena_sram.c optional sfxge pci dev/sfxge/common/siena_vpd.c optional sfxge pci dev/sfxge/sfxge.c optional sfxge pci dev/sfxge/sfxge_dma.c optional sfxge pci dev/sfxge/sfxge_ev.c optional sfxge pci dev/sfxge/sfxge_intr.c optional sfxge pci dev/sfxge/sfxge_mcdi.c optional sfxge pci dev/sfxge/sfxge_nvram.c optional sfxge pci dev/sfxge/sfxge_port.c optional sfxge pci dev/sfxge/sfxge_rx.c optional sfxge pci dev/sfxge/sfxge_tx.c optional sfxge pci dev/sio/sio.c optional sio dev/sio/sio_isa.c optional sio isa dev/sio/sio_pccard.c optional sio pccard dev/sio/sio_pci.c optional sio pci dev/sio/sio_puc.c optional sio puc dev/speaker/spkr.c optional speaker dev/syscons/apm/apm_saver.c optional apm_saver apm dev/syscons/scterm-teken.c optional sc dev/syscons/scvesactl.c optional sc vga vesa dev/syscons/scvgarndr.c optional sc vga dev/syscons/scvtb.c optional sc dev/tpm/tpm.c optional tpm dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd dev/vmware/vmxnet3/if_vmx.c optional vmx dev/wbwd/wbwd.c optional wbwd dev/wpi/if_wpi.c optional wpi dev/xen/pci/xen_acpi_pci.c optional xenhvm dev/xen/pci/xen_pci.c optional xenhvm dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard kern/link_elf_obj.c standard libkern/x86/crc32_sse42.c standard libkern/memmove.c standard libkern/memset.c standard # # IA32 binary support # #amd64/ia32/ia32_exception.S optional compat_freebsd32 amd64/ia32/ia32_reg.c optional compat_freebsd32 amd64/ia32/ia32_signal.c optional compat_freebsd32 amd64/ia32/ia32_sigtramp.S optional compat_freebsd32 amd64/ia32/ia32_syscall.c optional compat_freebsd32 amd64/ia32/ia32_misc.c optional compat_freebsd32 compat/ia32/ia32_sysvec.c optional compat_freebsd32 compat/linprocfs/linprocfs.c optional linprocfs compat/linsysfs/linsysfs.c optional linsysfs # # Linux/i386 binary support # amd64/linux32/linux32_dummy.c optional compat_linux32 amd64/linux32/linux32_machdep.c optional compat_linux32 amd64/linux32/linux32_support.s optional compat_linux32 \ dependency "linux32_assym.h" amd64/linux32/linux32_sysent.c optional compat_linux32 amd64/linux32/linux32_sysvec.c optional compat_linux32 compat/linux/linux_emul.c optional compat_linux32 compat/linux/linux_file.c optional compat_linux32 compat/linux/linux_fork.c optional compat_linux32 compat/linux/linux_futex.c optional compat_linux32 compat/linux/linux_getcwd.c optional compat_linux32 compat/linux/linux_ioctl.c optional compat_linux32 compat/linux/linux_ipc.c optional compat_linux32 compat/linux/linux_mib.c optional compat_linux32 compat/linux/linux_misc.c optional compat_linux32 compat/linux/linux_mmap.c optional compat_linux32 compat/linux/linux_signal.c optional compat_linux32 compat/linux/linux_socket.c optional compat_linux32 compat/linux/linux_stats.c optional compat_linux32 compat/linux/linux_sysctl.c optional compat_linux32 compat/linux/linux_time.c optional compat_linux32 compat/linux/linux_timer.c optional compat_linux32 compat/linux/linux_uid16.c optional compat_linux32 compat/linux/linux_util.c optional compat_linux32 compat/linux/linux_vdso.c optional compat_linux32 compat/linux/linux_common.c optional compat_linux32 compat/linux/linux_event.c optional compat_linux32 compat/linux/linux.c optional compat_linux32 dev/amr/amr_linux.c optional compat_linux32 amr dev/mfi/mfi_linux.c optional compat_linux32 mfi # # Windows NDIS driver support # compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx64_wrap.S optional ndisapi pci # # x86 real mode BIOS emulator, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa contrib/x86emu/x86emu.c optional x86bios | dpms | pci | vesa # # bvm console # dev/bvm/bvm_console.c optional bvmconsole dev/bvm/bvm_dbg.c optional bvmdebug # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/madt.c optional acpi x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci x86/iommu/intel_drv.c optional acpi acpi_dmar pci x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci x86/isa/atpic.c optional atpic isa x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/elcr.c optional atpic isa | mptable x86/isa/isa.c standard x86/isa/isa_dma.c standard x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/io_apic.c standard x86/x86/legacy.c standard x86/x86/local_apic.c standard x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mptable.c optional mptable x86/x86/mptable_pci.c optional mptable pci x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/msi.c optional pci x86/x86/nexus.c standard x86/x86/pvclock.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/pv.c optional xenhvm x86/xen/pvcpu_enum.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm x86/xen/xen_msi.c optional xenhvm x86/xen/xen_pci_bus.c optional xenhvm Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 (revision 324036) +++ head/sys/conf/files.i386 (revision 324037) @@ -1,619 +1,629 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_i686.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_i686.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # linux_genassym.o optional compat_linux \ dependency "$S/i386/linux/linux_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "linux_genassym.o" # linux_assym.h optional compat_linux \ dependency "$S/kern/genassym.sh linux_genassym.o" \ compile-with "sh $S/kern/genassym.sh linux_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "linux_assym.h" # linux_locore.o optional compat_linux \ dependency "linux_assym.h $S/i386/linux/linux_locore.s" \ compile-with "${CC} -x assembler-with-cpp -DLOCORE -shared -s -pipe -I. -I$S -Werror -Wall -fPIC -fno-common -nostdinc -nostdlib -Wl,-T$S/i386/linux/linux_vdso.lds.s -Wl,-soname=linux_vdso.so,--eh-frame-hdr,-warn-common ${.IMPSRC} -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "linux_locore.o" # linux_vdso.so optional compat_linux \ dependency "linux_locore.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 linux_locore.o ${.TARGET}" \ no-implicit-rule \ clean "linux_vdso.so" # font.h optional sc_dflt_font \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'static u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'static u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'static u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # ukbdmap.h optional ukbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "ukbdmap.h" # hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ no-implicit-rule # hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/i386-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/i386-elf.raid.o.uu" \ no-implicit-rule # hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ no-implicit-rule # hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ no-implicit-rule # cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S optional zfs | dtrace compile-with "${ZFS_S}" cddl/dev/dtrace/i386/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" compat/linprocfs/linprocfs.c optional linprocfs compat/linsysfs/linsysfs.c optional linsysfs compat/linux/linux_event.c optional compat_linux compat/linux/linux_emul.c optional compat_linux compat/linux/linux_file.c optional compat_linux compat/linux/linux_fork.c optional compat_linux compat/linux/linux_futex.c optional compat_linux compat/linux/linux_getcwd.c optional compat_linux compat/linux/linux_ioctl.c optional compat_linux compat/linux/linux_ipc.c optional compat_linux compat/linux/linux_mib.c optional compat_linux compat/linux/linux_misc.c optional compat_linux compat/linux/linux_mmap.c optional compat_linux compat/linux/linux_signal.c optional compat_linux compat/linux/linux_socket.c optional compat_linux compat/linux/linux_stats.c optional compat_linux compat/linux/linux_sysctl.c optional compat_linux compat/linux/linux_time.c optional compat_linux compat/linux/linux_timer.c optional compat_linux compat/linux/linux_uid16.c optional compat_linux compat/linux/linux_util.c optional compat_linux compat/linux/linux_vdso.c optional compat_linux compat/linux/linux.c optional compat_linux compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx32_wrap.S optional ndisapi pci bf_enc.o optional crypto | ipsec | ipsec_support \ dependency "$S/crypto/blowfish/arch/i386/bf_enc.S $S/crypto/blowfish/arch/i386/bf_enc_586.S $S/crypto/blowfish/arch/i386/bf_enc_686.S" \ compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS} ${WERROR} ${.IMPSRC}" \ no-implicit-rule crypto/aesni/aeskeys_i386.S optional aesni crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" crypto/des/arch/i386/des_enc.S optional crypto | ipsec | ipsec_support | netsmb +intel_sha1.o optional aesni \ + dependency "$S/crypto/aesni/intel_sha1.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ + no-implicit-rule \ + clean "intel_sha1.o" +intel_sha256.o optional aesni \ + dependency "$S/crypto/aesni/intel_sha256.c" \ + compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ + no-implicit-rule \ + clean "intel_sha256.o" crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/advansys/adv_isa.c optional adv isa dev/agp/agp_ali.c optional agp dev/agp/agp_amd.c optional agp dev/agp/agp_amd64.c optional agp dev/agp/agp_ati.c optional agp dev/agp/agp_i810.c optional agp dev/agp/agp_intel.c optional agp dev/agp/agp_nvidia.c optional agp dev/agp/agp_sis.c optional agp dev/agp/agp_via.c optional agp dev/aic/aic_isa.c optional aic isa dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/ce/ceddk.c optional ce dev/ce/if_ce.c optional ce dev/ce/tau32-ddk.c optional ce \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/cm/if_cm_isa.c optional cm isa dev/coretemp/coretemp.c optional coretemp dev/cp/cpddk.c optional cp dev/cp/if_cp.c optional cp dev/cpuctl/cpuctl.c optional cpuctl dev/ctau/ctau.c optional ctau dev/ctau/ctddk.c optional ctau dev/ctau/if_ct.c optional ctau dev/cx/csigma.c optional cx dev/cx/cxddk.c optional cx dev/cx/if_cx.c optional cx dev/dpms/dpms.c optional dpms dev/ed/if_ed_3c503.c optional ed isa ed_3c503 dev/ed/if_ed_isa.c optional ed isa dev/ed/if_ed_wd80x3.c optional ed isa dev/ed/if_ed_hpp.c optional ed isa ed_hpp dev/ed/if_ed_sic.c optional ed isa ed_sic dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/fdc/fdc_pccard.c optional fdc pccard dev/fe/if_fe_isa.c optional fe isa dev/glxiic/glxiic.c optional glxiic dev/glxsb/glxsb.c optional glxsb dev/glxsb/glxsb_hash.c optional glxsb dev/gpio/bytgpio.c optional bytgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_pentium.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_ppro.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/hyperv/vmbus/i386/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/i386/vmbus_vector.S optional hyperv dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux dev/le/if_le_isa.c optional le isa dev/mse/mse.c optional mse dev/mse/mse_isa.c optional mse isa dev/nctgpio/nctgpio.c optional nctgpio dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme dev/nvme/nvme_ns_cmd.c optional nvme dev/nvme/nvme_qpair.c optional nvme dev/nvme/nvme_sysctl.c optional nvme dev/nvme/nvme_test.c optional nvme dev/nvme/nvme_util.c optional nvme dev/nvram/nvram.c optional nvram isa dev/ofw/ofwpci.c optional fdt pci dev/pcf/pcf_isa.c optional pcf dev/random/ivy.c optional rdrand_rng dev/random/nehemiah.c optional padlock_rng dev/sbni/if_sbni.c optional sbni dev/sbni/if_sbni_isa.c optional sbni isa dev/sbni/if_sbni_pci.c optional sbni pci dev/sio/sio.c optional sio dev/sio/sio_isa.c optional sio isa dev/sio/sio_pccard.c optional sio pccard dev/sio/sio_pci.c optional sio pci dev/sio/sio_puc.c optional sio puc dev/speaker/spkr.c optional speaker dev/syscons/apm/apm_saver.c optional apm_saver apm dev/syscons/scterm-teken.c optional sc dev/syscons/scvesactl.c optional sc vga vesa dev/syscons/scvgarndr.c optional sc vga dev/syscons/scvtb.c optional sc dev/tpm/tpm.c optional tpm dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd dev/vmware/vmxnet3/if_vmx.c optional vmx dev/acpica/acpi_if.m standard dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpi_support/acpi_wmi_if.m standard dev/wbwd/wbwd.c optional wbwd dev/wpi/if_wpi.c optional wpi dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci i386/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ dependency "$S/i386/acpica/acpi_wakecode.S assym.s" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # i386/bios/apm.c optional apm i386/bios/smapi.c optional smapi i386/bios/smapi_bios.S optional smapi i386/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 #i386/i386/apic_vector.s optional apic i386/i386/atomic.c standard \ compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}" i386/i386/bios.c standard i386/i386/bioscall.s standard i386/i386/bpf_jit_machdep.c optional bpf_jitter i386/i386/db_disasm.c optional ddb i386/i386/db_interface.c optional ddb i386/i386/db_trace.c optional ddb i386/i386/elan-mmcr.c optional cpu_elan | cpu_soekris i386/i386/elf_machdep.c standard i386/i386/exception.s standard i386/i386/gdb_machdep.c optional gdb i386/i386/geode.c optional cpu_geode i386/i386/in_cksum.c optional inet | inet6 i386/i386/initcpu.c standard i386/i386/io.c optional io i386/i386/k6_mem.c optional mem i386/i386/locore.s standard no-obj i386/i386/longrun.c optional cpu_enable_longrun i386/i386/machdep.c standard i386/i386/mem.c optional mem i386/i386/minidump_machdep.c standard i386/i386/mp_clock.c optional smp i386/i386/mp_machdep.c optional smp i386/i386/mpboot.s optional smp i386/i386/perfmon.c optional perfmon i386/i386/pmap.c standard i386/i386/ptrace_machdep.c standard i386/i386/support.s standard i386/i386/swtch.s standard i386/i386/sys_machdep.c standard i386/i386/trap.c standard i386/i386/uio_machdep.c standard i386/i386/vm86.c standard i386/i386/vm_machdep.c standard i386/ibcs2/ibcs2_errno.c optional ibcs2 i386/ibcs2/ibcs2_fcntl.c optional ibcs2 i386/ibcs2/ibcs2_ioctl.c optional ibcs2 i386/ibcs2/ibcs2_ipc.c optional ibcs2 i386/ibcs2/ibcs2_isc.c optional ibcs2 i386/ibcs2/ibcs2_isc_sysent.c optional ibcs2 i386/ibcs2/ibcs2_misc.c optional ibcs2 i386/ibcs2/ibcs2_msg.c optional ibcs2 i386/ibcs2/ibcs2_other.c optional ibcs2 i386/ibcs2/ibcs2_signal.c optional ibcs2 i386/ibcs2/ibcs2_socksys.c optional ibcs2 i386/ibcs2/ibcs2_stat.c optional ibcs2 i386/ibcs2/ibcs2_sysent.c optional ibcs2 i386/ibcs2/ibcs2_sysi86.c optional ibcs2 i386/ibcs2/ibcs2_sysvec.c optional ibcs2 i386/ibcs2/ibcs2_util.c optional ibcs2 i386/ibcs2/ibcs2_xenix.c optional ibcs2 i386/ibcs2/ibcs2_xenix_sysent.c optional ibcs2 i386/ibcs2/imgact_coff.c optional ibcs2 i386/isa/elink.c optional ep | ie i386/isa/npx.c standard i386/isa/pmtimer.c optional pmtimer i386/isa/prof_machdep.c optional profiling-routine i386/linux/imgact_linux.c optional compat_linux i386/linux/linux_dummy.c optional compat_linux i386/linux/linux_machdep.c optional compat_linux i386/linux/linux_ptrace.c optional compat_linux i386/linux/linux_support.s optional compat_linux \ dependency "linux_assym.h" i386/linux/linux_sysent.c optional compat_linux i386/linux/linux_sysvec.c optional compat_linux i386/pci/pci_cfgreg.c optional pci i386/pci/pci_pir.c optional pci isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard kern/imgact_aout.c optional compat_aout kern/imgact_gzip.c optional gzip kern/subr_sfbuf.c standard libkern/divdi3.c standard libkern/ffsll.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard libkern/x86/crc32_sse42.c standard i386/xbox/xbox.c optional xbox i386/xbox/xboxfb.c optional xboxfb dev/fb/boot_font.c optional xboxfb i386/xbox/pic16l.s optional xbox # # x86 real mode BIOS support, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa # # bvm console # dev/bvm/bvm_console.c optional bvmconsole dev/bvm/bvm_dbg.c optional bvmdebug # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/madt.c optional acpi apic x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/smist.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci x86/iommu/intel_drv.c optional acpi acpi_dmar pci x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci x86/isa/atpic.c optional atpic x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/elcr.c optional atpic | apic x86/isa/isa.c optional isa x86/isa/isa_dma.c optional isa x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/io_apic.c optional apic x86/x86/legacy.c standard x86/x86/local_apic.c optional apic x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mptable.c optional apic x86/x86/mptable_pci.c optional apic pci x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/msi.c optional apic pci x86/x86/nexus.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/pvclock.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm x86/xen/xen_msi.c optional xenhvm Index: head/sys/crypto/aesni/aesni.c =================================================================== --- head/sys/crypto/aesni/aesni.c (revision 324036) +++ head/sys/crypto/aesni/aesni.c (revision 324037) @@ -1,665 +1,951 @@ /*- * Copyright (c) 2005-2008 Pawel Jakub Dawidek * Copyright (c) 2010 Konstantin Belousov * Copyright (c) 2014 The FreeBSD Foundation + * Copyright (c) 2017 Conrad Meyer * All rights reserved. * * Portions of this software were developed by John-Mark Gurney * under sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include + #include -#include +#include +#include +#include + +#include #include +#include +#include +#include +#if defined(__i386__) +#include +#elif defined(__amd64__) +#include +#endif + static struct mtx_padalign *ctx_mtx; static struct fpu_kern_ctx **ctx_fpu; struct aesni_softc { int dieing; int32_t cid; uint32_t sid; + bool has_aes; + bool has_sha; TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions; struct rwlock lock; }; #define ACQUIRE_CTX(i, ctx) \ do { \ (i) = PCPU_GET(cpuid); \ mtx_lock(&ctx_mtx[(i)]); \ (ctx) = ctx_fpu[(i)]; \ } while (0) #define RELEASE_CTX(i, ctx) \ do { \ mtx_unlock(&ctx_mtx[(i)]); \ (i) = -1; \ (ctx) = NULL; \ } while (0) static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri); static int aesni_freesession(device_t, uint64_t tid); static void aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses); static int aesni_cipher_setup(struct aesni_session *ses, - struct cryptoini *encini); + struct cryptoini *encini, struct cryptoini *authini); static int aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp); +static int aesni_cipher_crypt(struct aesni_session *ses, + struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp); +static int aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd, + struct cryptop *crp); MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data"); static void aesni_identify(driver_t *drv, device_t parent) { /* NB: order 10 is so we get attached after h/w devices */ if (device_find_child(parent, "aesni", -1) == NULL && BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0) panic("aesni: could not attach"); } +static void +detect_cpu_features(bool *has_aes, bool *has_sha) +{ + + *has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 && + (cpu_feature2 & CPUID2_SSE41) != 0); + *has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 && + (cpu_feature2 & CPUID2_SSSE3) != 0); +} + static int aesni_probe(device_t dev) { + bool has_aes, has_sha; - if ((cpu_feature2 & CPUID2_AESNI) == 0) { - device_printf(dev, "No AESNI support.\n"); + detect_cpu_features(&has_aes, &has_sha); + if (!has_aes && !has_sha) { + device_printf(dev, "No AES or SHA support.\n"); return (EINVAL); - } + } else if (has_aes && has_sha) + device_set_desc(dev, + "AES-CBC,AES-XTS,AES-GCM,AES-ICM,SHA1,SHA256"); + else if (has_aes) + device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM"); + else + device_set_desc(dev, "SHA1,SHA256"); - if ((cpu_feature2 & CPUID2_SSE41) == 0) { - device_printf(dev, "No SSE4.1 support.\n"); - return (EINVAL); - } - - device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM"); return (0); } static void aesni_cleanctx(void) { int i; /* XXX - no way to return driverid */ CPU_FOREACH(i) { if (ctx_fpu[i] != NULL) { mtx_destroy(&ctx_mtx[i]); fpu_kern_free_ctx(ctx_fpu[i]); } ctx_fpu[i] = NULL; } free(ctx_mtx, M_AESNI); ctx_mtx = NULL; free(ctx_fpu, M_AESNI); ctx_fpu = NULL; } static int aesni_attach(device_t dev) { struct aesni_softc *sc; int i; sc = device_get_softc(dev); sc->dieing = 0; TAILQ_INIT(&sc->sessions); sc->sid = 1; sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC); if (sc->cid < 0) { device_printf(dev, "Could not get crypto driver id.\n"); return (ENOMEM); } ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI, M_WAITOK|M_ZERO); ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI, M_WAITOK|M_ZERO); CPU_FOREACH(i) { ctx_fpu[i] = fpu_kern_alloc_ctx(0); mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW); } rw_init(&sc->lock, "aesni_lock"); - crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); - crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + + detect_cpu_features(&sc->has_aes, &sc->has_sha); + if (sc->has_aes) { + crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + } + if (sc->has_sha) { + crypto_register(sc->cid, CRYPTO_SHA1, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); + } return (0); } static int aesni_detach(device_t dev) { struct aesni_softc *sc; struct aesni_session *ses; sc = device_get_softc(dev); rw_wlock(&sc->lock); TAILQ_FOREACH(ses, &sc->sessions, next) { if (ses->used) { rw_wunlock(&sc->lock); device_printf(dev, "Cannot detach, sessions still active.\n"); return (EBUSY); } } sc->dieing = 1; while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) { TAILQ_REMOVE(&sc->sessions, ses, next); free(ses, M_AESNI); } rw_wunlock(&sc->lock); crypto_unregister_all(sc->cid); rw_destroy(&sc->lock); aesni_cleanctx(); return (0); } static int aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) { struct aesni_softc *sc; struct aesni_session *ses; - struct cryptoini *encini; + struct cryptoini *encini, *authini; + bool gcm_hash, gcm; int error; if (sidp == NULL || cri == NULL) { CRYPTDEB("no sidp or cri"); return (EINVAL); } sc = device_get_softc(dev); if (sc->dieing) return (EINVAL); ses = NULL; + authini = NULL; encini = NULL; + gcm = false; + gcm_hash = false; for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { + case CRYPTO_AES_NIST_GCM_16: + gcm = true; + /* FALLTHROUGH */ case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: - case CRYPTO_AES_NIST_GCM_16: + if (!sc->has_aes) + goto unhandled; if (encini != NULL) { CRYPTDEB("encini already set"); return (EINVAL); } encini = cri; break; case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: /* * nothing to do here, maybe in the future cache some * values for GHASH */ + gcm_hash = true; break; + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + if (!sc->has_sha) + goto unhandled; + if (authini != NULL) { + CRYPTDEB("authini already set"); + return (EINVAL); + } + authini = cri; + break; default: +unhandled: CRYPTDEB("unhandled algorithm"); return (EINVAL); } } - if (encini == NULL) { + if (encini == NULL && authini == NULL) { CRYPTDEB("no cipher"); return (EINVAL); } + /* + * GMAC algorithms are only supported with simultaneous GCM. Likewise + * GCM is not supported without GMAC. + */ + if (gcm_hash != gcm) + return (EINVAL); rw_wlock(&sc->lock); if (sc->dieing) { rw_wunlock(&sc->lock); return (EINVAL); } /* * Free sessions goes first, so if first session is used, we need to * allocate one. */ ses = TAILQ_FIRST(&sc->sessions); if (ses == NULL || ses->used) { ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO); if (ses == NULL) { rw_wunlock(&sc->lock); return (ENOMEM); } ses->id = sc->sid++; } else { TAILQ_REMOVE(&sc->sessions, ses, next); } ses->used = 1; TAILQ_INSERT_TAIL(&sc->sessions, ses, next); rw_wunlock(&sc->lock); - ses->algo = encini->cri_alg; - error = aesni_cipher_setup(ses, encini); + if (encini != NULL) + ses->algo = encini->cri_alg; + if (authini != NULL) + ses->auth_algo = authini->cri_alg; + + error = aesni_cipher_setup(ses, encini, authini); if (error != 0) { CRYPTDEB("setup failed"); rw_wlock(&sc->lock); aesni_freesession_locked(sc, ses); rw_wunlock(&sc->lock); return (error); } *sidp = ses->id; return (0); } static void aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses) { uint32_t sid; rw_assert(&sc->lock, RA_WLOCKED); sid = ses->id; TAILQ_REMOVE(&sc->sessions, ses, next); - *ses = (struct aesni_session){}; + explicit_bzero(ses, sizeof(*ses)); ses->id = sid; TAILQ_INSERT_HEAD(&sc->sessions, ses, next); } static int aesni_freesession(device_t dev, uint64_t tid) { struct aesni_softc *sc; struct aesni_session *ses; uint32_t sid; sc = device_get_softc(dev); sid = ((uint32_t)tid) & 0xffffffff; rw_wlock(&sc->lock); TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { if (ses->id == sid) break; } if (ses == NULL) { rw_wunlock(&sc->lock); return (EINVAL); } aesni_freesession_locked(sc, ses); rw_wunlock(&sc->lock); return (0); } static int aesni_process(device_t dev, struct cryptop *crp, int hint __unused) { struct aesni_softc *sc = device_get_softc(dev); struct aesni_session *ses = NULL; struct cryptodesc *crd, *enccrd, *authcrd; int error, needauth; error = 0; enccrd = NULL; authcrd = NULL; needauth = 0; /* Sanity check. */ if (crp == NULL) return (EINVAL); if (crp->crp_callback == NULL || crp->crp_desc == NULL) { error = EINVAL; goto out; } for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { + case CRYPTO_AES_NIST_GCM_16: + needauth = 1; + /* FALLTHROUGH */ case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: case CRYPTO_AES_XTS: if (enccrd != NULL) { error = EINVAL; goto out; } enccrd = crd; break; - case CRYPTO_AES_NIST_GCM_16: - if (enccrd != NULL) { - error = EINVAL; - goto out; - } - enccrd = crd; - needauth = 1; - break; - case CRYPTO_AES_128_NIST_GMAC: case CRYPTO_AES_192_NIST_GMAC: case CRYPTO_AES_256_NIST_GMAC: + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: if (authcrd != NULL) { error = EINVAL; goto out; } authcrd = crd; - needauth = 1; break; default: error = EINVAL; goto out; } } - if (enccrd == NULL || (needauth && authcrd == NULL)) { + if ((enccrd == NULL && authcrd == NULL) || + (needauth && authcrd == NULL)) { error = EINVAL; goto out; } /* CBC & XTS can only handle full blocks for now */ - if ((enccrd->crd_alg == CRYPTO_AES_CBC || enccrd->crd_alg == - CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) { + if (enccrd != NULL && (enccrd->crd_alg == CRYPTO_AES_CBC || + enccrd->crd_alg == CRYPTO_AES_XTS) && + (enccrd->crd_len % AES_BLOCK_LEN) != 0) { error = EINVAL; goto out; } rw_rlock(&sc->lock); TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { if (ses->id == (crp->crp_sid & 0xffffffff)) break; } rw_runlock(&sc->lock); if (ses == NULL) { error = EINVAL; goto out; } error = aesni_cipher_process(ses, enccrd, authcrd, crp); if (error != 0) goto out; out: crp->crp_etype = error; crypto_done(crp); return (error); } -uint8_t * +static uint8_t * aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, - int *allocated) + bool *allocated) { struct mbuf *m; struct uio *uio; struct iovec *iov; uint8_t *addr; if (crp->crp_flags & CRYPTO_F_IMBUF) { m = (struct mbuf *)crp->crp_buf; if (m->m_next != NULL) goto alloc; addr = mtod(m, uint8_t *); } else if (crp->crp_flags & CRYPTO_F_IOV) { uio = (struct uio *)crp->crp_buf; if (uio->uio_iovcnt != 1) goto alloc; iov = uio->uio_iov; addr = (uint8_t *)iov->iov_base; } else addr = (uint8_t *)crp->crp_buf; - *allocated = 0; + *allocated = false; addr += enccrd->crd_skip; return (addr); alloc: addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT); if (addr != NULL) { - *allocated = 1; + *allocated = true; crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, addr); } else - *allocated = 0; + *allocated = false; return (addr); } static device_method_t aesni_methods[] = { DEVMETHOD(device_identify, aesni_identify), DEVMETHOD(device_probe, aesni_probe), DEVMETHOD(device_attach, aesni_attach), DEVMETHOD(device_detach, aesni_detach), DEVMETHOD(cryptodev_newsession, aesni_newsession), DEVMETHOD(cryptodev_freesession, aesni_freesession), DEVMETHOD(cryptodev_process, aesni_process), {0, 0}, }; static driver_t aesni_driver = { "aesni", aesni_methods, sizeof(struct aesni_softc), }; static devclass_t aesni_devclass; DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0); MODULE_VERSION(aesni, 1); MODULE_DEPEND(aesni, crypto, 1, 1, 1); static int -aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) +aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini, + struct cryptoini *authini) { struct fpu_kern_ctx *ctx; - int error; - int kt, ctxidx; + int kt, ctxidx, keylen, error; - kt = is_fpu_kern_thread(0); + switch (ses->auth_algo) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + if (authini->cri_klen % 8 != 0) + return (EINVAL); + keylen = authini->cri_klen / 8; + if (keylen > sizeof(ses->hmac_key)) + return (EINVAL); + if (ses->auth_algo == CRYPTO_SHA1 && keylen > 0) + return (EINVAL); + memcpy(ses->hmac_key, authini->cri_key, keylen); + ses->mlen = authini->cri_mlen; + } + + kt = is_fpu_kern_thread(0) || (encini == NULL); if (!kt) { ACQUIRE_CTX(ctxidx, ctx); error = fpu_kern_enter(curthread, ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); if (error != 0) goto out; } - error = aesni_cipher_setup_common(ses, encini->cri_key, - encini->cri_klen); + error = 0; + if (encini != NULL) + error = aesni_cipher_setup_common(ses, encini->cri_key, + encini->cri_klen); if (!kt) { fpu_kern_leave(curthread, ctx); out: RELEASE_CTX(ctxidx, ctx); } return (error); } +static int +intel_sha1_update(void *vctx, const void *vdata, u_int datalen) +{ + struct sha1_ctxt *ctx = vctx; + const char *data = vdata; + size_t gaplen; + size_t gapstart; + size_t off; + size_t copysiz; + u_int blocks; + + off = 0; + /* Do any aligned blocks without redundant copying. */ + if (datalen >= 64 && ctx->count % 64 == 0) { + blocks = datalen / 64; + ctx->c.b64[0] += blocks * 64 * 8; + intel_sha1_step(ctx->h.b32, data + off, blocks); + off += blocks * 64; + } + + while (off < datalen) { + gapstart = ctx->count % 64; + gaplen = 64 - gapstart; + + copysiz = (gaplen < datalen - off) ? gaplen : datalen - off; + bcopy(&data[off], &ctx->m.b8[gapstart], copysiz); + ctx->count += copysiz; + ctx->count %= 64; + ctx->c.b64[0] += copysiz * 8; + if (ctx->count % 64 == 0) + intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1); + off += copysiz; + } + return (0); +} + +static void +SHA1_Finalize_fn(void *digest, void *ctx) +{ + sha1_result(ctx, digest); +} + +static int +intel_sha256_update(void *vctx, const void *vdata, u_int len) +{ + SHA256_CTX *ctx = vctx; + uint64_t bitlen; + uint32_t r; + u_int blocks; + const unsigned char *src = vdata; + + /* Number of bytes left in the buffer from previous updates */ + r = (ctx->count >> 3) & 0x3f; + + /* Convert the length into a number of bits */ + bitlen = len << 3; + + /* Update number of bits */ + ctx->count += bitlen; + + /* Handle the case where we don't need to perform any transforms */ + if (len < 64 - r) { + memcpy(&ctx->buf[r], src, len); + return (0); + } + + /* Finish the current block */ + memcpy(&ctx->buf[r], src, 64 - r); + intel_sha256_step(ctx->state, ctx->buf, 1); + src += 64 - r; + len -= 64 - r; + + /* Perform complete blocks */ + if (len >= 64) { + blocks = len / 64; + intel_sha256_step(ctx->state, src, blocks); + src += blocks * 64; + len -= blocks * 64; + } + + /* Copy left over data into buffer */ + memcpy(ctx->buf, src, len); + return (0); +} + +static void +SHA256_Finalize_fn(void *digest, void *ctx) +{ + SHA256_Final(digest, ctx); +} + /* - * authcrd contains the associated date. + * Compute the HASH( (key ^ xorbyte) || buf ) */ +static void +hmac_internal(void *ctx, uint32_t *res, + int (*update)(void *, const void *, u_int), + void (*finalize)(void *, void *), uint8_t *key, uint8_t xorbyte, + const void *buf, size_t off, size_t buflen, int crpflags) +{ + size_t i; + + for (i = 0; i < 64; i++) + key[i] ^= xorbyte; + update(ctx, key, 64); + for (i = 0; i < 64; i++) + key[i] ^= xorbyte; + + crypto_apply(crpflags, __DECONST(void *, buf), off, buflen, + __DECONST(int (*)(void *, void *, u_int), update), ctx); + finalize(res, ctx); +} + static int aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp) { struct fpu_kern_ctx *ctx; - uint8_t iv[AES_BLOCK_LEN]; - uint8_t tag[GMAC_DIGEST_LEN]; - uint8_t *buf, *authbuf; - int error, allocated, authallocated; - int ivlen, encflag; - int kt, ctxidx; + int error, ctxidx; + bool kt; - encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; + if (enccrd != NULL) { + if ((enccrd->crd_alg == CRYPTO_AES_ICM || + enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && + (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) + return (EINVAL); + } - if ((enccrd->crd_alg == CRYPTO_AES_ICM || - enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) && - (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) - return (EINVAL); + error = 0; + kt = is_fpu_kern_thread(0); + if (!kt) { + ACQUIRE_CTX(ctxidx, ctx); + error = fpu_kern_enter(curthread, ctx, + FPU_KERN_NORMAL | FPU_KERN_KTHR); + if (error != 0) + goto out2; + } + /* Do work */ + if (enccrd != NULL && authcrd != NULL) { + /* Perform the first operation */ + if (crp->crp_desc == enccrd) + error = aesni_cipher_crypt(ses, enccrd, authcrd, crp); + else + error = aesni_cipher_mac(ses, authcrd, crp); + if (error != 0) + goto out; + /* Perform the second operation */ + if (crp->crp_desc == enccrd) + error = aesni_cipher_mac(ses, authcrd, crp); + else + error = aesni_cipher_crypt(ses, enccrd, authcrd, crp); + } else if (enccrd != NULL) + error = aesni_cipher_crypt(ses, enccrd, authcrd, crp); + else + error = aesni_cipher_mac(ses, authcrd, crp); + + if (error != 0) + goto out; + +out: + if (!kt) { + fpu_kern_leave(curthread, ctx); +out2: + RELEASE_CTX(ctxidx, ctx); + } + return (error); +} + +static int +aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd, + struct cryptodesc *authcrd, struct cryptop *crp) +{ + uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf; + int error, ivlen; + bool encflag, allocated, authallocated; + buf = aesni_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) return (ENOMEM); - error = 0; - authbuf = NULL; - authallocated = 0; - if (authcrd != NULL) { + authallocated = false; + if (ses->algo == CRYPTO_AES_NIST_GCM_16 && authcrd != NULL) { authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); if (authbuf == NULL) { error = ENOMEM; - goto out1; + goto out; } } - kt = is_fpu_kern_thread(0); - if (!kt) { - ACQUIRE_CTX(ctxidx, ctx); - error = fpu_kern_enter(curthread, ctx, - FPU_KERN_NORMAL|FPU_KERN_KTHR); - if (error != 0) - goto out2; - } - + error = 0; + encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { error = aesni_cipher_setup_common(ses, enccrd->crd_key, enccrd->crd_klen); if (error != 0) goto out; } - /* XXX - validate that enccrd and authcrd have/use same key? */ switch (enccrd->crd_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: ivlen = AES_BLOCK_LEN; break; case CRYPTO_AES_XTS: ivlen = 8; break; case CRYPTO_AES_NIST_GCM_16: ivlen = 12; /* should support arbitarily larger */ break; } /* Setup iv */ if (encflag) { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, iv, ivlen); else arc4rand(iv, ivlen, 0); if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, ivlen, iv); } else { if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) bcopy(enccrd->crd_iv, iv, ivlen); else crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_inject, ivlen, iv); } - if (authcrd != NULL && !encflag) - crypto_copydata(crp->crp_flags, crp->crp_buf, - authcrd->crd_inject, GMAC_DIGEST_LEN, tag); - else - bzero(tag, sizeof tag); - - /* Do work */ switch (ses->algo) { case CRYPTO_AES_CBC: if (encflag) aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, enccrd->crd_len, buf, buf, iv); else aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, enccrd->crd_len, buf, iv); break; case CRYPTO_AES_ICM: /* encryption & decryption are the same */ aesni_encrypt_icm(ses->rounds, ses->enc_schedule, enccrd->crd_len, buf, buf, iv); break; case CRYPTO_AES_XTS: if (encflag) aesni_encrypt_xts(ses->rounds, ses->enc_schedule, ses->xts_schedule, enccrd->crd_len, buf, buf, iv); else aesni_decrypt_xts(ses->rounds, ses->dec_schedule, ses->xts_schedule, enccrd->crd_len, buf, buf, iv); break; case CRYPTO_AES_NIST_GCM_16: - if (encflag) + if (authcrd != NULL && !encflag) + crypto_copydata(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + else + bzero(tag, sizeof tag); + + if (encflag) { AES_GCM_encrypt(buf, buf, authbuf, iv, tag, enccrd->crd_len, authcrd->crd_len, ivlen, ses->enc_schedule, ses->rounds); - else { + + if (authcrd != NULL) + crypto_copyback(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + } else { if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, enccrd->crd_len, authcrd->crd_len, ivlen, ses->enc_schedule, ses->rounds)) error = EBADMSG; } break; } - if (allocated) - crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, - enccrd->crd_len, buf); - - if (!error && authcrd != NULL) { - crypto_copyback(crp->crp_flags, crp->crp_buf, - authcrd->crd_inject, GMAC_DIGEST_LEN, tag); - } - out: - if (!kt) { - fpu_kern_leave(curthread, ctx); -out2: - RELEASE_CTX(ctxidx, ctx); - } - -out1: if (allocated) { - bzero(buf, enccrd->crd_len); + explicit_bzero(buf, enccrd->crd_len); free(buf, M_AESNI); } - if (authallocated) + if (authallocated) { + explicit_bzero(authbuf, authcrd->crd_len); free(authbuf, M_AESNI); + } return (error); +} + +static int +aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd, + struct cryptop *crp) +{ + union { + struct SHA256Context sha2 __aligned(16); + struct sha1_ctxt sha1 __aligned(16); + } sctx; + uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)]; + int hashlen; + + if (crd->crd_flags != 0) + return (EINVAL); + + switch (ses->auth_algo) { + case CRYPTO_SHA1_HMAC: + hashlen = SHA1_HASH_LEN; + /* Inner hash: (K ^ IPAD) || data */ + sha1_init(&sctx.sha1); + hmac_internal(&sctx.sha1, res, intel_sha1_update, + SHA1_Finalize_fn, ses->hmac_key, 0x36, crp->crp_buf, + crd->crd_skip, crd->crd_len, crp->crp_flags); + /* Outer hash: (K ^ OPAD) || inner hash */ + sha1_init(&sctx.sha1); + hmac_internal(&sctx.sha1, res, intel_sha1_update, + SHA1_Finalize_fn, ses->hmac_key, 0x5C, res, 0, hashlen, 0); + break; + case CRYPTO_SHA1: + hashlen = SHA1_HASH_LEN; + sha1_init(&sctx.sha1); + crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip, + crd->crd_len, __DECONST(int (*)(void *, void *, u_int), + intel_sha1_update), &sctx.sha1); + sha1_result(&sctx.sha1, (void *)res); + break; + case CRYPTO_SHA2_256_HMAC: + hashlen = SHA2_256_HASH_LEN; + /* Inner hash: (K ^ IPAD) || data */ + SHA256_Init(&sctx.sha2); + hmac_internal(&sctx.sha2, res, intel_sha256_update, + SHA256_Finalize_fn, ses->hmac_key, 0x36, crp->crp_buf, + crd->crd_skip, crd->crd_len, crp->crp_flags); + /* Outer hash: (K ^ OPAD) || inner hash */ + SHA256_Init(&sctx.sha2); + hmac_internal(&sctx.sha2, res, intel_sha256_update, + SHA256_Finalize_fn, ses->hmac_key, 0x5C, res, 0, hashlen, + 0); + break; + default: + /* + * AES-GMAC authentication is verified while processing the + * enccrd + */ + return (0); + } + + if (ses->mlen != 0 && ses->mlen < hashlen) + hashlen = ses->mlen; + + crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen, + (void *)res); + return (0); } Index: head/sys/crypto/aesni/aesni.h =================================================================== --- head/sys/crypto/aesni/aesni.h (revision 324036) +++ head/sys/crypto/aesni/aesni.h (revision 324037) @@ -1,117 +1,119 @@ /*- * Copyright (c) 2010 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _AESNI_H_ #define _AESNI_H_ #include #include #include #include #if defined(__amd64__) || defined(__i386__) #include #include #include #include #endif #if defined(__i386__) #include #elif defined(__amd64__) #include #endif #define AES128_ROUNDS 10 #define AES192_ROUNDS 12 #define AES256_ROUNDS 14 #define AES_SCHED_LEN ((AES256_ROUNDS + 1) * AES_BLOCK_LEN) struct aesni_session { uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16); uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16); uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16); + /* Same as the SHA256 Blocksize. */ + uint8_t hmac_key[SHA1_HMAC_BLOCK_LEN] __aligned(16); int algo; int rounds; /* uint8_t *ses_ictx; */ /* uint8_t *ses_octx; */ /* int ses_mlen; */ int used; + int auth_algo; + int mlen; uint32_t id; TAILQ_ENTRY(aesni_session) next; }; /* * Internal functions, implemented in assembler. */ void aesni_set_enckey(const uint8_t *userkey, uint8_t *encrypt_schedule /*__aligned(16)*/, int number_of_rounds); void aesni_set_deckey(const uint8_t *encrypt_schedule /*__aligned(16)*/, uint8_t *decrypt_schedule /*__aligned(16)*/, int number_of_rounds); /* * Slightly more public interfaces. */ void aesni_encrypt_cbc(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[__min_size(AES_BLOCK_LEN)]); void aesni_decrypt_cbc(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, uint8_t *buf, const uint8_t iv[__min_size(AES_BLOCK_LEN)]); void aesni_encrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to); void aesni_decrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to); void aesni_encrypt_icm(int rounds, const void *key_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[__min_size(AES_BLOCK_LEN)]); void aesni_encrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/, const void *tweak_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[__min_size(AES_BLOCK_LEN)]); void aesni_decrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/, const void *tweak_schedule /*__aligned(16)*/, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[__min_size(AES_BLOCK_LEN)]); /* GCM & GHASH functions */ void AES_GCM_encrypt(const unsigned char *in, unsigned char *out, const unsigned char *addt, const unsigned char *ivec, unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, const unsigned char *key, int nr); int AES_GCM_decrypt(const unsigned char *in, unsigned char *out, const unsigned char *addt, const unsigned char *ivec, const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes, const unsigned char *key, int nr); int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key, int keylen); -uint8_t *aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, - int *allocated); #endif /* _AESNI_H_ */ Index: head/sys/crypto/aesni/intel_sha1.c =================================================================== --- head/sys/crypto/aesni/intel_sha1.c (nonexistent) +++ head/sys/crypto/aesni/intel_sha1.c (revision 324037) @@ -0,0 +1,261 @@ +/******************************************************************************* +* Copyright (c) 2013, Intel Corporation +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the +* distribution. +* +* * Neither the name of the Intel Corporation nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* +* THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY +* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR +* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +******************************************************************************** +* +* Intel SHA Extensions optimized implementation of a SHA-1 update function +* +* The function takes a pointer to the current hash values, a pointer to the +* input data, and a number of 64 byte blocks to process. Once all blocks have +* been processed, the digest pointer is updated with the resulting hash value. +* The function only processes complete blocks, there is no functionality to +* store partial blocks. All message padding and hash value initialization must +* be done outside the update function. +* +* The indented lines in the loop are instructions related to rounds processing. +* The non-indented lines are instructions related to the message schedule. +* +* Author: Sean Gulley +* Date: July 2013 +* +******************************************************************************** +* +* Example complier command line: +* icc intel_sha_extensions_sha1_intrinsic.c +* gcc -msha -msse4 intel_sha_extensions_sha1_intrinsic.c +* +*******************************************************************************/ +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +void intel_sha1_step(uint32_t *digest, const char *data, uint32_t num_blks) { + __m128i abcd, e0, e1; + __m128i abcd_save, e_save; + __m128i msg0, msg1, msg2, msg3; + __m128i shuf_mask, e_mask; + +#if 0 + e_mask = _mm_set_epi64x(0xFFFFFFFF00000000ull, 0x0000000000000000ull); +#else + (void)e_mask; + e0 = _mm_set_epi64x(0, 0); +#endif + shuf_mask = _mm_set_epi64x(0x0001020304050607ull, 0x08090a0b0c0d0e0full); + + // Load initial hash values + abcd = _mm_loadu_si128((__m128i*) digest); + e0 = _mm_insert_epi32(e0, *(digest+4), 3); + abcd = _mm_shuffle_epi32(abcd, 0x1B); +#if 0 + e0 = _mm_and_si128(e0, e_mask); +#endif + + while (num_blks > 0) { + // Save hash values for addition after rounds + abcd_save = abcd; + e_save = e0; + + // Rounds 0-3 + msg0 = _mm_loadu_si128((const __m128i*) data); + msg0 = _mm_shuffle_epi8(msg0, shuf_mask); + e0 = _mm_add_epi32(e0, msg0); + e1 = abcd; + abcd = _mm_sha1rnds4_epu32(abcd, e0, 0); + + // Rounds 4-7 + msg1 = _mm_loadu_si128((const __m128i*) (data+16)); + msg1 = _mm_shuffle_epi8(msg1, shuf_mask); + e1 = _mm_sha1nexte_epu32(e1, msg1); + e0 = abcd; + abcd = _mm_sha1rnds4_epu32(abcd, e1, 0); + msg0 = _mm_sha1msg1_epu32(msg0, msg1); + + // Rounds 8-11 + msg2 = _mm_loadu_si128((const __m128i*) (data+32)); + msg2 = _mm_shuffle_epi8(msg2, shuf_mask); + e0 = _mm_sha1nexte_epu32(e0, msg2); + e1 = abcd; + abcd = _mm_sha1rnds4_epu32(abcd, e0, 0); + msg1 = _mm_sha1msg1_epu32(msg1, msg2); + msg0 = _mm_xor_si128(msg0, msg2); + + // Rounds 12-15 + msg3 = _mm_loadu_si128((const __m128i*) (data+48)); + msg3 = _mm_shuffle_epi8(msg3, shuf_mask); + e1 = _mm_sha1nexte_epu32(e1, msg3); + e0 = abcd; + msg0 = _mm_sha1msg2_epu32(msg0, msg3); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 0); + msg2 = _mm_sha1msg1_epu32(msg2, msg3); + msg1 = _mm_xor_si128(msg1, msg3); + + // Rounds 16-19 + e0 = _mm_sha1nexte_epu32(e0, msg0); + e1 = abcd; + msg1 = _mm_sha1msg2_epu32(msg1, msg0); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 0); + msg3 = _mm_sha1msg1_epu32(msg3, msg0); + msg2 = _mm_xor_si128(msg2, msg0); + + // Rounds 20-23 + e1 = _mm_sha1nexte_epu32(e1, msg1); + e0 = abcd; + msg2 = _mm_sha1msg2_epu32(msg2, msg1); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 1); + msg0 = _mm_sha1msg1_epu32(msg0, msg1); + msg3 = _mm_xor_si128(msg3, msg1); + + // Rounds 24-27 + e0 = _mm_sha1nexte_epu32(e0, msg2); + e1 = abcd; + msg3 = _mm_sha1msg2_epu32(msg3, msg2); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 1); + msg1 = _mm_sha1msg1_epu32(msg1, msg2); + msg0 = _mm_xor_si128(msg0, msg2); + + // Rounds 28-31 + e1 = _mm_sha1nexte_epu32(e1, msg3); + e0 = abcd; + msg0 = _mm_sha1msg2_epu32(msg0, msg3); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 1); + msg2 = _mm_sha1msg1_epu32(msg2, msg3); + msg1 = _mm_xor_si128(msg1, msg3); + + // Rounds 32-35 + e0 = _mm_sha1nexte_epu32(e0, msg0); + e1 = abcd; + msg1 = _mm_sha1msg2_epu32(msg1, msg0); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 1); + msg3 = _mm_sha1msg1_epu32(msg3, msg0); + msg2 = _mm_xor_si128(msg2, msg0); + + // Rounds 36-39 + e1 = _mm_sha1nexte_epu32(e1, msg1); + e0 = abcd; + msg2 = _mm_sha1msg2_epu32(msg2, msg1); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 1); + msg0 = _mm_sha1msg1_epu32(msg0, msg1); + msg3 = _mm_xor_si128(msg3, msg1); + + // Rounds 40-43 + e0 = _mm_sha1nexte_epu32(e0, msg2); + e1 = abcd; + msg3 = _mm_sha1msg2_epu32(msg3, msg2); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 2); + msg1 = _mm_sha1msg1_epu32(msg1, msg2); + msg0 = _mm_xor_si128(msg0, msg2); + + // Rounds 44-47 + e1 = _mm_sha1nexte_epu32(e1, msg3); + e0 = abcd; + msg0 = _mm_sha1msg2_epu32(msg0, msg3); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 2); + msg2 = _mm_sha1msg1_epu32(msg2, msg3); + msg1 = _mm_xor_si128(msg1, msg3); + + // Rounds 48-51 + e0 = _mm_sha1nexte_epu32(e0, msg0); + e1 = abcd; + msg1 = _mm_sha1msg2_epu32(msg1, msg0); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 2); + msg3 = _mm_sha1msg1_epu32(msg3, msg0); + msg2 = _mm_xor_si128(msg2, msg0); + + // Rounds 52-55 + e1 = _mm_sha1nexte_epu32(e1, msg1); + e0 = abcd; + msg2 = _mm_sha1msg2_epu32(msg2, msg1); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 2); + msg0 = _mm_sha1msg1_epu32(msg0, msg1); + msg3 = _mm_xor_si128(msg3, msg1); + + // Rounds 56-59 + e0 = _mm_sha1nexte_epu32(e0, msg2); + e1 = abcd; + msg3 = _mm_sha1msg2_epu32(msg3, msg2); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 2); + msg1 = _mm_sha1msg1_epu32(msg1, msg2); + msg0 = _mm_xor_si128(msg0, msg2); + + // Rounds 60-63 + e1 = _mm_sha1nexte_epu32(e1, msg3); + e0 = abcd; + msg0 = _mm_sha1msg2_epu32(msg0, msg3); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 3); + msg2 = _mm_sha1msg1_epu32(msg2, msg3); + msg1 = _mm_xor_si128(msg1, msg3); + + // Rounds 64-67 + e0 = _mm_sha1nexte_epu32(e0, msg0); + e1 = abcd; + msg1 = _mm_sha1msg2_epu32(msg1, msg0); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 3); + msg3 = _mm_sha1msg1_epu32(msg3, msg0); + msg2 = _mm_xor_si128(msg2, msg0); + + // Rounds 68-71 + e1 = _mm_sha1nexte_epu32(e1, msg1); + e0 = abcd; + msg2 = _mm_sha1msg2_epu32(msg2, msg1); + abcd = _mm_sha1rnds4_epu32(abcd, e1, 3); + msg3 = _mm_xor_si128(msg3, msg1); + + // Rounds 72-75 + e0 = _mm_sha1nexte_epu32(e0, msg2); + e1 = abcd; + msg3 = _mm_sha1msg2_epu32(msg3, msg2); + abcd = _mm_sha1rnds4_epu32(abcd, e0, 3); + + // Rounds 76-79 + e1 = _mm_sha1nexte_epu32(e1, msg3); + e0 = abcd; + abcd = _mm_sha1rnds4_epu32(abcd, e1, 3); + + // Add current hash values with previously saved + e0 = _mm_sha1nexte_epu32(e0, e_save); + abcd = _mm_add_epi32(abcd, abcd_save); + + data += 64; + num_blks--; + } + + abcd = _mm_shuffle_epi32(abcd, 0x1B); + _mm_store_si128((__m128i*) digest, abcd); + *(digest+4) = _mm_extract_epi32(e0, 3); +} + Property changes on: head/sys/crypto/aesni/intel_sha1.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/crypto/aesni/intel_sha256.c =================================================================== --- head/sys/crypto/aesni/intel_sha256.c (nonexistent) +++ head/sys/crypto/aesni/intel_sha256.c (revision 324037) @@ -0,0 +1,277 @@ +/******************************************************************************* +* Copyright (c) 2013, Intel Corporation +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the +* distribution. +* +* * Neither the name of the Intel Corporation nor the names of its +* contributors may be used to endorse or promote products derived from +* this software without specific prior written permission. +* +* +* THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY +* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR +* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +******************************************************************************** +* +* Intel SHA Extensions optimized implementation of a SHA-256 update function +* +* The function takes a pointer to the current hash values, a pointer to the +* input data, and a number of 64 byte blocks to process. Once all blocks have +* been processed, the digest pointer is updated with the resulting hash value. +* The function only processes complete blocks, there is no functionality to +* store partial blocks. All message padding and hash value initialization must +* be done outside the update function. +* +* The indented lines in the loop are instructions related to rounds processing. +* The non-indented lines are instructions related to the message schedule. +* +* Author: Sean Gulley +* Date: July 2013 +* +******************************************************************************** +* +* Example complier command line: +* icc intel_sha_extensions_sha256_intrinsic.c +* gcc -msha -msse4 intel_sha_extensions_sha256_intrinsic.c +* +*******************************************************************************/ +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks) { + __m128i state0, state1; + __m128i msg; + __m128i msgtmp0, msgtmp1, msgtmp2, msgtmp3; + __m128i tmp; + __m128i shuf_mask; + __m128i abef_save, cdgh_save; + + // Load initial hash values + // Need to reorder these appropriately + // DCBA, HGFE -> ABEF, CDGH + tmp = _mm_loadu_si128((__m128i*) digest); + state1 = _mm_loadu_si128((__m128i*) (digest+4)); + + tmp = _mm_shuffle_epi32(tmp, 0xB1); // CDAB + state1 = _mm_shuffle_epi32(state1, 0x1B); // EFGH + state0 = _mm_alignr_epi8(tmp, state1, 8); // ABEF + state1 = _mm_blend_epi16(state1, tmp, 0xF0); // CDGH + + shuf_mask = _mm_set_epi64x(0x0c0d0e0f08090a0bull, 0x0405060700010203ull); + + while (num_blks > 0) { + // Save hash values for addition after rounds + abef_save = state0; + cdgh_save = state1; + + // Rounds 0-3 + msg = _mm_loadu_si128((const __m128i*) data); + msgtmp0 = _mm_shuffle_epi8(msg, shuf_mask); + msg = _mm_add_epi32(msgtmp0, + _mm_set_epi64x(0xE9B5DBA5B5C0FBCFull, 0x71374491428A2F98ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + + // Rounds 4-7 + msgtmp1 = _mm_loadu_si128((const __m128i*) (data+16)); + msgtmp1 = _mm_shuffle_epi8(msgtmp1, shuf_mask); + msg = _mm_add_epi32(msgtmp1, + _mm_set_epi64x(0xAB1C5ED5923F82A4ull, 0x59F111F13956C25Bull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); + + // Rounds 8-11 + msgtmp2 = _mm_loadu_si128((const __m128i*) (data+32)); + msgtmp2 = _mm_shuffle_epi8(msgtmp2, shuf_mask); + msg = _mm_add_epi32(msgtmp2, + _mm_set_epi64x(0x550C7DC3243185BEull, 0x12835B01D807AA98ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); + + // Rounds 12-15 + msgtmp3 = _mm_loadu_si128((const __m128i*) (data+48)); + msgtmp3 = _mm_shuffle_epi8(msgtmp3, shuf_mask); + msg = _mm_add_epi32(msgtmp3, + _mm_set_epi64x(0xC19BF1749BDC06A7ull, 0x80DEB1FE72BE5D74ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); + msgtmp0 = _mm_add_epi32(msgtmp0, tmp); + msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); + + // Rounds 16-19 + msg = _mm_add_epi32(msgtmp0, + _mm_set_epi64x(0x240CA1CC0FC19DC6ull, 0xEFBE4786E49B69C1ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); + msgtmp1 = _mm_add_epi32(msgtmp1, tmp); + msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); + + // Rounds 20-23 + msg = _mm_add_epi32(msgtmp1, + _mm_set_epi64x(0x76F988DA5CB0A9DCull, 0x4A7484AA2DE92C6Full)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); + msgtmp2 = _mm_add_epi32(msgtmp2, tmp); + msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); + + // Rounds 24-27 + msg = _mm_add_epi32(msgtmp2, + _mm_set_epi64x(0xBF597FC7B00327C8ull, 0xA831C66D983E5152ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); + msgtmp3 = _mm_add_epi32(msgtmp3, tmp); + msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); + + // Rounds 28-31 + msg = _mm_add_epi32(msgtmp3, + _mm_set_epi64x(0x1429296706CA6351ull, 0xD5A79147C6E00BF3ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); + msgtmp0 = _mm_add_epi32(msgtmp0, tmp); + msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); + + // Rounds 32-35 + msg = _mm_add_epi32(msgtmp0, + _mm_set_epi64x(0x53380D134D2C6DFCull, 0x2E1B213827B70A85ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); + msgtmp1 = _mm_add_epi32(msgtmp1, tmp); + msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); + + // Rounds 36-39 + msg = _mm_add_epi32(msgtmp1, + _mm_set_epi64x(0x92722C8581C2C92Eull, 0x766A0ABB650A7354ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); + msgtmp2 = _mm_add_epi32(msgtmp2, tmp); + msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1); + + // Rounds 40-43 + msg = _mm_add_epi32(msgtmp2, + _mm_set_epi64x(0xC76C51A3C24B8B70ull, 0xA81A664BA2BFE8A1ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); + msgtmp3 = _mm_add_epi32(msgtmp3, tmp); + msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2); + + // Rounds 44-47 + msg = _mm_add_epi32(msgtmp3, + _mm_set_epi64x(0x106AA070F40E3585ull, 0xD6990624D192E819ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4); + msgtmp0 = _mm_add_epi32(msgtmp0, tmp); + msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3); + + // Rounds 48-51 + msg = _mm_add_epi32(msgtmp0, + _mm_set_epi64x(0x34B0BCB52748774Cull, 0x1E376C0819A4C116ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4); + msgtmp1 = _mm_add_epi32(msgtmp1, tmp); + msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0); + + // Rounds 52-55 + msg = _mm_add_epi32(msgtmp1, + _mm_set_epi64x(0x682E6FF35B9CCA4Full, 0x4ED8AA4A391C0CB3ull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4); + msgtmp2 = _mm_add_epi32(msgtmp2, tmp); + msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + + // Rounds 56-59 + msg = _mm_add_epi32(msgtmp2, + _mm_set_epi64x(0x8CC7020884C87814ull, 0x78A5636F748F82EEull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4); + msgtmp3 = _mm_add_epi32(msgtmp3, tmp); + msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + + // Rounds 60-63 + msg = _mm_add_epi32(msgtmp3, + _mm_set_epi64x(0xC67178F2BEF9A3F7ull, 0xA4506CEB90BEFFFAull)); + state1 = _mm_sha256rnds2_epu32(state1, state0, msg); + msg = _mm_shuffle_epi32(msg, 0x0E); + state0 = _mm_sha256rnds2_epu32(state0, state1, msg); + + // Add current hash values with previously saved + state0 = _mm_add_epi32(state0, abef_save); + state1 = _mm_add_epi32(state1, cdgh_save); + + data += 64; + num_blks--; + } + + // Write hash values back in the correct order + tmp = _mm_shuffle_epi32(state0, 0x1B); // FEBA + state1 = _mm_shuffle_epi32(state1, 0xB1); // DCHG + state0 = _mm_blend_epi16(tmp, state1, 0xF0); // DCBA + state1 = _mm_alignr_epi8(state1, tmp, 8); // ABEF + + _mm_store_si128((__m128i*) digest, state0); + _mm_store_si128((__m128i*) (digest+4), state1); +} + Property changes on: head/sys/crypto/aesni/intel_sha256.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/crypto/aesni/sha_sse.h =================================================================== --- head/sys/crypto/aesni/sha_sse.h (nonexistent) +++ head/sys/crypto/aesni/sha_sse.h (revision 324037) @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2017 Conrad Meyer + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _CRYPTO__SHA_SSE_H_ +#define _CRYPTO__SHA_SSE_H_ + +/* + * Internal functions, implemented in intrinsics. + */ +void intel_sha1_step(uint32_t *digest, const char *data, uint32_t num_blks); +void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks); + +#endif /* _CRYPTO__SHA_SSE_H_ */ Property changes on: head/sys/crypto/aesni/sha_sse.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/modules/aesni/Makefile =================================================================== --- head/sys/modules/aesni/Makefile (revision 324036) +++ head/sys/modules/aesni/Makefile (revision 324037) @@ -1,30 +1,44 @@ # $FreeBSD$ .PATH: ${SRCTOP}/sys/crypto/aesni +.PATH: ${SRCTOP}/contrib/llvm/tools/clang/lib/Headers KMOD= aesni SRCS= aesni.c SRCS+= aeskeys_${MACHINE_CPUARCH}.S SRCS+= device_if.h bus_if.h opt_bus.h cryptodev_if.h OBJS+= aesni_ghash.o aesni_wrap.o +OBJS+= intel_sha1.o intel_sha256.o # Remove -nostdinc so we can get the intrinsics. aesni_ghash.o: aesni_ghash.c # XXX - gcc won't understand -mpclmul ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC} ${CTFCONVERT_CMD} aesni_wrap.o: aesni_wrap.c ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ -mmmx -msse -msse4 -maes ${.IMPSRC} ${CTFCONVERT_CMD} +intel_sha1.o: intel_sha1.c + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ + -mmmx -msse -msse4 -msha ${.IMPSRC} + ${CTFCONVERT_CMD} + +intel_sha256.o: intel_sha256.c + ${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} \ + -mmmx -msse -msse4 -msha ${.IMPSRC} + ${CTFCONVERT_CMD} + aesni_ghash.o: aesni.h aesni_wrap.o: aesni.h +intel_sha1.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h +intel_sha256.o: sha_sse.h immintrin.h shaintrin.h tmmintrin.h xmmintrin.h .include CWARNFLAGS.aesni_ghash.c= ${NO_WCAST_QUAL} CWARNFLAGS.aesni_wrap.c= ${NO_WCAST_QUAL} Index: head/tests/sys/opencrypto/cryptotest.py =================================================================== --- head/tests/sys/opencrypto/cryptotest.py (revision 324036) +++ head/tests/sys/opencrypto/cryptotest.py (revision 324037) @@ -1,314 +1,314 @@ #!/usr/bin/env python # # Copyright (c) 2014 The FreeBSD Foundation # All rights reserved. # # This software was developed by John-Mark Gurney under # the sponsorship from the FreeBSD Foundation. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # from __future__ import print_function import errno import cryptodev import itertools import os import struct import unittest from cryptodev import * from glob import iglob katdir = '/usr/local/share/nist-kat' def katg(base, glob): assert os.path.exists(os.path.join(katdir, base)), "Please 'pkg install nist-kat'" return iglob(os.path.join(katdir, base, glob)) aesmodules = [ 'cryptosoft0', 'aesni0', 'ccr0' ] desmodules = [ 'cryptosoft0', ] -shamodules = [ 'cryptosoft0', 'ccr0' ] +shamodules = [ 'cryptosoft0', 'aesni0', 'ccr0' ] def GenTestCase(cname): try: crid = cryptodev.Crypto.findcrid(cname) except IOError: return None class GendCryptoTestCase(unittest.TestCase): ############### ##### AES ##### ############### @unittest.skipIf(cname not in aesmodules, 'skipping AES on %s' % (cname)) def test_xts(self): for i in katg('XTSTestVectors/format tweak value input - data unit seq no', '*.rsp'): self.runXTS(i, cryptodev.CRYPTO_AES_XTS) @unittest.skipIf(cname not in aesmodules, 'skipping AES on %s' % (cname)) def test_cbc(self): for i in katg('KAT_AES', 'CBC[GKV]*.rsp'): self.runCBC(i) @unittest.skipIf(cname not in aesmodules, 'skipping AES on %s' % (cname)) def test_gcm(self): for i in katg('gcmtestvectors', 'gcmEncrypt*'): self.runGCM(i, 'ENCRYPT') for i in katg('gcmtestvectors', 'gcmDecrypt*'): self.runGCM(i, 'DECRYPT') _gmacsizes = { 32: cryptodev.CRYPTO_AES_256_NIST_GMAC, 24: cryptodev.CRYPTO_AES_192_NIST_GMAC, 16: cryptodev.CRYPTO_AES_128_NIST_GMAC, } def runGCM(self, fname, mode): curfun = None if mode == 'ENCRYPT': swapptct = False curfun = Crypto.encrypt elif mode == 'DECRYPT': swapptct = True curfun = Crypto.decrypt else: raise RuntimeError('unknown mode: %r' % repr(mode)) for bogusmode, lines in cryptodev.KATParser(fname, [ 'Count', 'Key', 'IV', 'CT', 'AAD', 'Tag', 'PT', ]): for data in lines: curcnt = int(data['Count']) cipherkey = data['Key'].decode('hex') iv = data['IV'].decode('hex') aad = data['AAD'].decode('hex') tag = data['Tag'].decode('hex') if 'FAIL' not in data: pt = data['PT'].decode('hex') ct = data['CT'].decode('hex') if len(iv) != 12: # XXX - isn't supported continue c = Crypto(cryptodev.CRYPTO_AES_NIST_GCM_16, cipherkey, mac=self._gmacsizes[len(cipherkey)], mackey=cipherkey, crid=crid) if mode == 'ENCRYPT': rct, rtag = c.encrypt(pt, iv, aad) rtag = rtag[:len(tag)] data['rct'] = rct.encode('hex') data['rtag'] = rtag.encode('hex') self.assertEqual(rct, ct, repr(data)) self.assertEqual(rtag, tag, repr(data)) else: if len(tag) != 16: continue args = (ct, iv, aad, tag) if 'FAIL' in data: self.assertRaises(IOError, c.decrypt, *args) else: rpt, rtag = c.decrypt(*args) data['rpt'] = rpt.encode('hex') data['rtag'] = rtag.encode('hex') self.assertEqual(rpt, pt, repr(data)) def runCBC(self, fname): curfun = None for mode, lines in cryptodev.KATParser(fname, [ 'COUNT', 'KEY', 'IV', 'PLAINTEXT', 'CIPHERTEXT', ]): if mode == 'ENCRYPT': swapptct = False curfun = Crypto.encrypt elif mode == 'DECRYPT': swapptct = True curfun = Crypto.decrypt else: raise RuntimeError('unknown mode: %r' % repr(mode)) for data in lines: curcnt = int(data['COUNT']) cipherkey = data['KEY'].decode('hex') iv = data['IV'].decode('hex') pt = data['PLAINTEXT'].decode('hex') ct = data['CIPHERTEXT'].decode('hex') if swapptct: pt, ct = ct, pt # run the fun c = Crypto(cryptodev.CRYPTO_AES_CBC, cipherkey, crid=crid) r = curfun(c, pt, iv) self.assertEqual(r, ct) def runXTS(self, fname, meth): curfun = None for mode, lines in cryptodev.KATParser(fname, [ 'COUNT', 'DataUnitLen', 'Key', 'DataUnitSeqNumber', 'PT', 'CT' ]): if mode == 'ENCRYPT': swapptct = False curfun = Crypto.encrypt elif mode == 'DECRYPT': swapptct = True curfun = Crypto.decrypt else: raise RuntimeError('unknown mode: %r' % repr(mode)) for data in lines: curcnt = int(data['COUNT']) nbits = int(data['DataUnitLen']) cipherkey = data['Key'].decode('hex') iv = struct.pack('QQ', int(data['DataUnitSeqNumber']), 0) pt = data['PT'].decode('hex') ct = data['CT'].decode('hex') if nbits % 128 != 0: # XXX - mark as skipped continue if swapptct: pt, ct = ct, pt # run the fun c = Crypto(meth, cipherkey, crid=crid) r = curfun(c, pt, iv) self.assertEqual(r, ct) ############### ##### DES ##### ############### @unittest.skipIf(cname not in desmodules, 'skipping DES on %s' % (cname)) def test_tdes(self): for i in katg('KAT_TDES', 'TCBC[a-z]*.rsp'): self.runTDES(i) def runTDES(self, fname): curfun = None for mode, lines in cryptodev.KATParser(fname, [ 'COUNT', 'KEYs', 'IV', 'PLAINTEXT', 'CIPHERTEXT', ]): if mode == 'ENCRYPT': swapptct = False curfun = Crypto.encrypt elif mode == 'DECRYPT': swapptct = True curfun = Crypto.decrypt else: raise RuntimeError('unknown mode: %r' % repr(mode)) for data in lines: curcnt = int(data['COUNT']) key = data['KEYs'] * 3 cipherkey = key.decode('hex') iv = data['IV'].decode('hex') pt = data['PLAINTEXT'].decode('hex') ct = data['CIPHERTEXT'].decode('hex') if swapptct: pt, ct = ct, pt # run the fun c = Crypto(cryptodev.CRYPTO_3DES_CBC, cipherkey, crid=crid) r = curfun(c, pt, iv) self.assertEqual(r, ct) ############### ##### SHA ##### ############### @unittest.skipIf(cname not in shamodules, 'skipping SHA on %s' % str(cname)) def test_sha(self): # SHA not available in software pass #for i in iglob('SHA1*'): # self.runSHA(i) @unittest.skipIf(cname not in shamodules, 'skipping SHA on %s' % str(cname)) def test_sha1hmac(self): for i in katg('hmactestvectors', 'HMAC.rsp'): self.runSHA1HMAC(i) def runSHA1HMAC(self, fname): for hashlength, lines in cryptodev.KATParser(fname, [ 'Count', 'Klen', 'Tlen', 'Key', 'Msg', 'Mac' ]): # E.g., hashlength will be "L=20" (bytes) hashlen = int(hashlength.split("=")[1]) blocksize = None if hashlen == 20: alg = cryptodev.CRYPTO_SHA1_HMAC blocksize = 64 elif hashlen == 28: # Cryptodev doesn't support SHA-224 # Slurp remaining input in section for data in lines: continue continue elif hashlen == 32: alg = cryptodev.CRYPTO_SHA2_256_HMAC blocksize = 64 elif hashlen == 48: alg = cryptodev.CRYPTO_SHA2_384_HMAC blocksize = 128 elif hashlen == 64: alg = cryptodev.CRYPTO_SHA2_512_HMAC blocksize = 128 else: # Skip unsupported hashes # Slurp remaining input in section for data in lines: continue continue for data in lines: key = data['Key'].decode('hex') msg = data['Msg'].decode('hex') mac = data['Mac'].decode('hex') tlen = int(data['Tlen']) if len(key) > blocksize: continue try: c = Crypto(mac=alg, mackey=key, crid=crid) except EnvironmentError, e: # Can't test hashes the driver does not support. if e.errno != errno.EOPNOTSUPP: raise continue _, r = c.encrypt(msg, iv="") # A limitation in cryptodev.py means we # can only store MACs up to 16 bytes. # That's good enough to validate the # correct behavior, more or less. maclen = min(tlen, 16) self.assertEqual(r[:maclen], mac[:maclen], "Actual: " + \ repr(r[:maclen].encode("hex")) + " Expected: " + repr(data)) return GendCryptoTestCase cryptosoft = GenTestCase('cryptosoft0') aesni = GenTestCase('aesni0') ccr = GenTestCase('ccr0') if __name__ == '__main__': unittest.main()