Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 (revision 327379) +++ head/sys/conf/files.i386 (revision 327380) @@ -1,625 +1,625 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_i686.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_i686.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # linux_genassym.o optional compat_linux \ dependency "$S/i386/linux/linux_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "linux_genassym.o" # linux_assym.h optional compat_linux \ dependency "$S/kern/genassym.sh linux_genassym.o" \ compile-with "sh $S/kern/genassym.sh linux_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "linux_assym.h" # linux_locore.o optional compat_linux \ dependency "linux_assym.h $S/i386/linux/linux_locore.s" \ compile-with "${CC} -x assembler-with-cpp -DLOCORE -shared -s -pipe -I. -I$S -Werror -Wall -fPIC -fno-common -nostdinc -nostdlib -Wl,-T$S/i386/linux/linux_vdso.lds.s -Wl,-soname=linux_vdso.so,--eh-frame-hdr,-warn-common ${.IMPSRC} -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "linux_locore.o" # linux_vdso.so optional compat_linux \ dependency "linux_locore.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 linux_locore.o ${.TARGET}" \ no-implicit-rule \ clean "linux_vdso.so" # font.h optional sc_dflt_font \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'static u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'static u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'static u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # ukbdmap.h optional ukbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "ukbdmap.h" # hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ no-implicit-rule # hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/i386-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/i386-elf.raid.o.uu" \ no-implicit-rule # hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ no-implicit-rule # hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ no-implicit-rule # cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S optional zfs | dtrace compile-with "${ZFS_S}" cddl/dev/dtrace/i386/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" compat/linprocfs/linprocfs.c optional linprocfs compat/linsysfs/linsysfs.c optional linsysfs compat/linux/linux_event.c optional compat_linux compat/linux/linux_emul.c optional compat_linux compat/linux/linux_file.c optional compat_linux compat/linux/linux_fork.c optional compat_linux compat/linux/linux_futex.c optional compat_linux compat/linux/linux_getcwd.c optional compat_linux compat/linux/linux_ioctl.c optional compat_linux compat/linux/linux_ipc.c optional compat_linux compat/linux/linux_mib.c optional compat_linux compat/linux/linux_misc.c optional compat_linux compat/linux/linux_mmap.c optional compat_linux compat/linux/linux_signal.c optional compat_linux compat/linux/linux_socket.c optional compat_linux compat/linux/linux_stats.c optional compat_linux compat/linux/linux_sysctl.c optional compat_linux compat/linux/linux_time.c optional compat_linux compat/linux/linux_timer.c optional compat_linux compat/linux/linux_uid16.c optional compat_linux compat/linux/linux_util.c optional compat_linux compat/linux/linux_vdso.c optional compat_linux compat/linux/linux.c optional compat_linux compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx32_wrap.S optional ndisapi pci bf_enc.o optional crypto | ipsec | ipsec_support \ dependency "$S/crypto/blowfish/arch/i386/bf_enc.S $S/crypto/blowfish/arch/i386/bf_enc_586.S $S/crypto/blowfish/arch/i386/bf_enc_686.S" \ compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS} ${WERROR} ${.IMPSRC}" \ no-implicit-rule crypto/aesni/aeskeys_i386.S optional aesni crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" crypto/des/arch/i386/des_enc.S optional crypto | ipsec | ipsec_support | netsmb intel_sha1.o optional aesni \ dependency "$S/crypto/aesni/intel_sha1.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha1.o" intel_sha256.o optional aesni \ dependency "$S/crypto/aesni/intel_sha256.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/advansys/adv_isa.c optional adv isa dev/agp/agp_ali.c optional agp dev/agp/agp_amd.c optional agp dev/agp/agp_amd64.c optional agp dev/agp/agp_ati.c optional agp dev/agp/agp_i810.c optional agp dev/agp/agp_intel.c optional agp dev/agp/agp_nvidia.c optional agp dev/agp/agp_sis.c optional agp dev/agp/agp_via.c optional agp dev/aic/aic_isa.c optional aic isa dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/ce/ceddk.c optional ce dev/ce/if_ce.c optional ce dev/ce/tau32-ddk.c optional ce \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/cm/if_cm_isa.c optional cm isa dev/coretemp/coretemp.c optional coretemp dev/cp/cpddk.c optional cp dev/cp/if_cp.c optional cp dev/cpuctl/cpuctl.c optional cpuctl dev/ctau/ctau.c optional ctau dev/ctau/ctddk.c optional ctau dev/ctau/if_ct.c optional ctau dev/cx/csigma.c optional cx dev/cx/cxddk.c optional cx dev/cx/if_cx.c optional cx dev/dpms/dpms.c optional dpms dev/ed/if_ed_3c503.c optional ed isa ed_3c503 dev/ed/if_ed_isa.c optional ed isa dev/ed/if_ed_wd80x3.c optional ed isa dev/ed/if_ed_hpp.c optional ed isa ed_hpp dev/ed/if_ed_sic.c optional ed isa ed_sic dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/fdc/fdc_pccard.c optional fdc pccard dev/fe/if_fe_isa.c optional fe isa dev/glxiic/glxiic.c optional glxiic dev/glxsb/glxsb.c optional glxsb dev/glxsb/glxsb_hash.c optional glxsb dev/gpio/bytgpio.c optional bytgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_pentium.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_ppro.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/hyperv/vmbus/i386/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/i386/vmbus_vector.S optional hyperv dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux dev/le/if_le_isa.c optional le isa dev/mse/mse.c optional mse dev/mse/mse_isa.c optional mse isa dev/nctgpio/nctgpio.c optional nctgpio dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme dev/nvme/nvme_ns_cmd.c optional nvme dev/nvme/nvme_qpair.c optional nvme dev/nvme/nvme_sysctl.c optional nvme dev/nvme/nvme_test.c optional nvme dev/nvme/nvme_util.c optional nvme dev/nvram/nvram.c optional nvram isa dev/ofw/ofwpci.c optional fdt pci dev/pcf/pcf_isa.c optional pcf dev/random/ivy.c optional rdrand_rng dev/random/nehemiah.c optional padlock_rng dev/sbni/if_sbni.c optional sbni dev/sbni/if_sbni_isa.c optional sbni isa dev/sbni/if_sbni_pci.c optional sbni pci dev/sio/sio.c optional sio dev/sio/sio_isa.c optional sio isa dev/sio/sio_pccard.c optional sio pccard dev/sio/sio_pci.c optional sio pci dev/sio/sio_puc.c optional sio puc dev/speaker/spkr.c optional speaker dev/syscons/apm/apm_saver.c optional apm_saver apm dev/syscons/scterm-teken.c optional sc dev/syscons/scvesactl.c optional sc vga vesa dev/syscons/scvgarndr.c optional sc vga dev/syscons/scvtb.c optional sc dev/tpm/tpm.c optional tpm dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd dev/vmware/vmxnet3/if_vmx.c optional vmx dev/acpica/acpi_if.m standard dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpi_support/acpi_wmi_if.m standard dev/wbwd/wbwd.c optional wbwd dev/wpi/if_wpi.c optional wpi dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci i386/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ dependency "$S/i386/acpica/acpi_wakecode.S assym.s" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # i386/bios/apm.c optional apm i386/bios/smapi.c optional smapi i386/bios/smapi_bios.S optional smapi i386/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 #i386/i386/apic_vector.s optional apic i386/i386/atomic.c standard \ compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}" i386/i386/bios.c standard i386/i386/bioscall.s standard i386/i386/bpf_jit_machdep.c optional bpf_jitter i386/i386/db_disasm.c optional ddb i386/i386/db_interface.c optional ddb i386/i386/db_trace.c optional ddb i386/i386/elan-mmcr.c optional cpu_elan | cpu_soekris i386/i386/elf_machdep.c standard i386/i386/exception.s standard i386/i386/gdb_machdep.c optional gdb i386/i386/geode.c optional cpu_geode i386/i386/in_cksum.c optional inet | inet6 i386/i386/initcpu.c standard i386/i386/io.c optional io i386/i386/k6_mem.c optional mem i386/i386/locore.s standard no-obj i386/i386/longrun.c optional cpu_enable_longrun i386/i386/machdep.c standard i386/i386/mem.c optional mem i386/i386/minidump_machdep.c standard i386/i386/mp_clock.c optional smp i386/i386/mp_machdep.c optional smp i386/i386/mpboot.s optional smp +i386/i386/npx.c standard i386/i386/perfmon.c optional perfmon i386/i386/pmap.c standard i386/i386/ptrace_machdep.c standard i386/i386/support.s standard i386/i386/swtch.s standard i386/i386/sys_machdep.c standard i386/i386/trap.c standard i386/i386/uio_machdep.c standard i386/i386/vm86.c standard i386/i386/vm_machdep.c standard i386/ibcs2/ibcs2_errno.c optional ibcs2 i386/ibcs2/ibcs2_fcntl.c optional ibcs2 i386/ibcs2/ibcs2_ioctl.c optional ibcs2 i386/ibcs2/ibcs2_ipc.c optional ibcs2 i386/ibcs2/ibcs2_isc.c optional ibcs2 i386/ibcs2/ibcs2_isc_sysent.c optional ibcs2 i386/ibcs2/ibcs2_misc.c optional ibcs2 i386/ibcs2/ibcs2_msg.c optional ibcs2 i386/ibcs2/ibcs2_other.c optional ibcs2 i386/ibcs2/ibcs2_signal.c optional ibcs2 i386/ibcs2/ibcs2_socksys.c optional ibcs2 i386/ibcs2/ibcs2_stat.c optional ibcs2 i386/ibcs2/ibcs2_sysent.c optional ibcs2 i386/ibcs2/ibcs2_sysi86.c optional ibcs2 i386/ibcs2/ibcs2_sysvec.c optional ibcs2 i386/ibcs2/ibcs2_util.c optional ibcs2 i386/ibcs2/ibcs2_xenix.c optional ibcs2 i386/ibcs2/ibcs2_xenix_sysent.c optional ibcs2 i386/ibcs2/imgact_coff.c optional ibcs2 i386/isa/elink.c optional ep -i386/isa/npx.c standard i386/isa/pmtimer.c optional pmtimer i386/isa/prof_machdep.c optional profiling-routine i386/linux/imgact_linux.c optional compat_linux i386/linux/linux_dummy.c optional compat_linux i386/linux/linux_machdep.c optional compat_linux i386/linux/linux_ptrace.c optional compat_linux i386/linux/linux_support.s optional compat_linux \ dependency "linux_assym.h" i386/linux/linux_sysent.c optional compat_linux i386/linux/linux_sysvec.c optional compat_linux i386/pci/pci_cfgreg.c optional pci i386/pci/pci_pir.c optional pci isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard kern/imgact_aout.c optional compat_aout kern/imgact_gzip.c optional gzip kern/subr_sfbuf.c standard libkern/divdi3.c standard libkern/ffsll.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard libkern/x86/crc32_sse42.c standard # # x86 real mode BIOS support, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa # # bvm console # dev/bvm/bvm_console.c optional bvmconsole dev/bvm/bvm_dbg.c optional bvmdebug # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/madt.c optional acpi apic x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/smist.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci x86/iommu/intel_drv.c optional acpi acpi_dmar pci x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci x86/isa/atpic.c optional atpic x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/elcr.c optional atpic | apic x86/isa/isa.c optional isa x86/isa/isa_dma.c optional isa x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/io_apic.c optional apic x86/x86/legacy.c standard x86/x86/local_apic.c optional apic x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mptable.c optional apic x86/x86/mptable_pci.c optional apic pci x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/msi.c optional apic pci x86/x86/nexus.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/pvclock.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm x86/xen/xen_msi.c optional xenhvm Index: head/sys/i386/i386/npx.c =================================================================== --- head/sys/i386/i386/npx.c (nonexistent) +++ head/sys/i386/i386/npx.c (revision 327380) @@ -0,0 +1,1430 @@ +/*- + * Copyright (c) 1990 William Jolitz. + * Copyright (c) 1991 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_cpu.h" +#include "opt_isa.h" +#include "opt_npx.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef NPX_DEBUG +#include +#endif +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef DEV_ISA +#include +#endif + +/* + * 387 and 287 Numeric Coprocessor Extension (NPX) Driver. + */ + +#if defined(__GNUCLIKE_ASM) && !defined(lint) + +#define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) +#define fnclex() __asm __volatile("fnclex") +#define fninit() __asm __volatile("fninit") +#define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr))) +#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) +#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) +#define fp_divide_by_0() __asm __volatile( \ + "fldz; fld1; fdiv %st,%st(1); fnop") +#define frstor(addr) __asm __volatile("frstor %0" : : "m" (*(addr))) +#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) +#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) +#define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) +#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) + +static __inline void +xrstor(char *addr, uint64_t mask) +{ + uint32_t low, hi; + + low = mask; + hi = mask >> 32; + __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); +} + +static __inline void +xsave(char *addr, uint64_t mask) +{ + uint32_t low, hi; + + low = mask; + hi = mask >> 32; + __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : + "memory"); +} + +static __inline void +xsaveopt(char *addr, uint64_t mask) +{ + uint32_t low, hi; + + low = mask; + hi = mask >> 32; + __asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) : + "memory"); +} +#else /* !(__GNUCLIKE_ASM && !lint) */ + +void fldcw(u_short cw); +void fnclex(void); +void fninit(void); +void fnsave(caddr_t addr); +void fnstcw(caddr_t addr); +void fnstsw(caddr_t addr); +void fp_divide_by_0(void); +void frstor(caddr_t addr); +void fxsave(caddr_t addr); +void fxrstor(caddr_t addr); +void ldmxcsr(u_int csr); +void stmxcsr(u_int *csr); +void xrstor(char *addr, uint64_t mask); +void xsave(char *addr, uint64_t mask); +void xsaveopt(char *addr, uint64_t mask); + +#endif /* __GNUCLIKE_ASM && !lint */ + +#define start_emulating() load_cr0(rcr0() | CR0_TS) +#define stop_emulating() clts() + +#define GET_FPU_CW(thread) \ + (cpu_fxsr ? \ + (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \ + (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw) +#define GET_FPU_SW(thread) \ + (cpu_fxsr ? \ + (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \ + (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw) +#define SET_FPU_CW(savefpu, value) do { \ + if (cpu_fxsr) \ + (savefpu)->sv_xmm.sv_env.en_cw = (value); \ + else \ + (savefpu)->sv_87.sv_env.en_cw = (value); \ +} while (0) + +CTASSERT(sizeof(union savefpu) == 512); +CTASSERT(sizeof(struct xstate_hdr) == 64); +CTASSERT(sizeof(struct savefpu_ymm) == 832); + +/* + * This requirement is to make it easier for asm code to calculate + * offset of the fpu save area from the pcb address. FPU save area + * must be 64-byte aligned. + */ +CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0); + +/* + * Ensure the copy of XCR0 saved in a core is contained in the padding + * area. + */ +CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) && + X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm)); + +static void fpu_clean_state(void); + +static void fpusave(union savefpu *); +static void fpurstor(union savefpu *); + +int hw_float; + +SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, + &hw_float, 0, "Floating point instructions executed in hardware"); + +int use_xsave; +uint64_t xsave_mask; +static uma_zone_t fpu_save_area_zone; +static union savefpu *npx_initialstate; + +struct xsave_area_elm_descr { + u_int offset; + u_int size; +} *xsave_area_desc; + +static int use_xsaveopt; + +static volatile u_int npx_traps_while_probing; + +alias_for_inthand_t probetrap; +__asm(" \n\ + .text \n\ + .p2align 2,0x90 \n\ + .type " __XSTRING(CNAME(probetrap)) ",@function \n\ +" __XSTRING(CNAME(probetrap)) ": \n\ + ss \n\ + incl " __XSTRING(CNAME(npx_traps_while_probing)) " \n\ + fnclex \n\ + iret \n\ +"); + +/* + * Determine if an FPU is present and how to use it. + */ +static int +npx_probe(void) +{ + struct gate_descriptor save_idt_npxtrap; + u_short control, status; + + /* + * Modern CPUs all have an FPU that uses the INT16 interface + * and provide a simple way to verify that, so handle the + * common case right away. + */ + if (cpu_feature & CPUID_FPU) { + hw_float = 1; + return (1); + } + + save_idt_npxtrap = idt[IDT_MF]; + setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL, + GSEL(GCODE_SEL, SEL_KPL)); + + /* + * Don't trap while we're probing. + */ + stop_emulating(); + + /* + * Finish resetting the coprocessor, if any. If there is an error + * pending, then we may get a bogus IRQ13, but npx_intr() will handle + * it OK. Bogus halts have never been observed, but we enabled + * IRQ13 and cleared the BUSY# latch early to handle them anyway. + */ + fninit(); + + /* + * Don't use fwait here because it might hang. + * Don't use fnop here because it usually hangs if there is no FPU. + */ + DELAY(1000); /* wait for any IRQ13 */ +#ifdef DIAGNOSTIC + if (npx_traps_while_probing != 0) + printf("fninit caused %u bogus npx trap(s)\n", + npx_traps_while_probing); +#endif + /* + * Check for a status of mostly zero. + */ + status = 0x5a5a; + fnstsw(&status); + if ((status & 0xb8ff) == 0) { + /* + * Good, now check for a proper control word. + */ + control = 0x5a5a; + fnstcw(&control); + if ((control & 0x1f3f) == 0x033f) { + /* + * We have an npx, now divide by 0 to see if exception + * 16 works. + */ + control &= ~(1 << 2); /* enable divide by 0 trap */ + fldcw(control); + npx_traps_while_probing = 0; + fp_divide_by_0(); + if (npx_traps_while_probing != 0) { + /* + * Good, exception 16 works. + */ + hw_float = 1; + goto cleanup; + } + printf( + "FPU does not use exception 16 for error reporting\n"); + goto cleanup; + } + } + + /* + * Probe failed. Floating point simply won't work. + * Notify user and disable FPU/MMX/SSE instruction execution. + */ + printf("WARNING: no FPU!\n"); + __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : + "n" (CR0_EM | CR0_MP) : "ax"); + +cleanup: + idt[IDT_MF] = save_idt_npxtrap; + return (hw_float); +} + +/* + * Enable XSAVE if supported and allowed by user. + * Calculate the xsave_mask. + */ +static void +npxinit_bsp1(void) +{ + u_int cp[4]; + uint64_t xsave_mask_user; + + if (cpu_fxsr && (cpu_feature2 & CPUID2_XSAVE) != 0) { + use_xsave = 1; + TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); + } + if (!use_xsave) + return; + + cpuid_count(0xd, 0x0, cp); + xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; + if ((cp[0] & xsave_mask) != xsave_mask) + panic("CPU0 does not support X87 or SSE: %x", cp[0]); + xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; + xsave_mask_user = xsave_mask; + TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user); + xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; + xsave_mask &= xsave_mask_user; + if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512) + xsave_mask &= ~XFEATURE_AVX512; + if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX) + xsave_mask &= ~XFEATURE_MPX; + + cpuid_count(0xd, 0x1, cp); + if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) + use_xsaveopt = 1; +} + +/* + + * Calculate the fpu save area size. + */ +static void +npxinit_bsp2(void) +{ + u_int cp[4]; + + if (use_xsave) { + cpuid_count(0xd, 0x0, cp); + cpu_max_ext_state_size = cp[1]; + + /* + * Reload the cpu_feature2, since we enabled OSXSAVE. + */ + do_cpuid(1, cp); + cpu_feature2 = cp[2]; + } else + cpu_max_ext_state_size = sizeof(union savefpu); +} + +/* + * Initialize floating point unit. + */ +void +npxinit(bool bsp) +{ + static union savefpu dummy; + register_t saveintr; + u_int mxcsr; + u_short control; + + if (bsp) { + if (!npx_probe()) + return; + npxinit_bsp1(); + } + + if (use_xsave) { + load_cr4(rcr4() | CR4_XSAVE); + load_xcr(XCR0, xsave_mask); + } + + /* + * XCR0 shall be set up before CPU can report the save area size. + */ + if (bsp) + npxinit_bsp2(); + + /* + * fninit has the same h/w bugs as fnsave. Use the detoxified + * fnsave to throw away any junk in the fpu. fpusave() initializes + * the fpu. + * + * It is too early for critical_enter() to work on AP. + */ + saveintr = intr_disable(); + stop_emulating(); + if (cpu_fxsr) + fninit(); + else + fnsave(&dummy); + control = __INITIAL_NPXCW__; + fldcw(control); + if (cpu_fxsr) { + mxcsr = __INITIAL_MXCSR__; + ldmxcsr(mxcsr); + } + start_emulating(); + intr_restore(saveintr); +} + +/* + * On the boot CPU we generate a clean state that is used to + * initialize the floating point unit when it is first used by a + * process. + */ +static void +npxinitstate(void *arg __unused) +{ + register_t saveintr; + int cp[4], i, max_ext_n; + + if (!hw_float) + return; + + npx_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF, + M_WAITOK | M_ZERO); + saveintr = intr_disable(); + stop_emulating(); + + fpusave(npx_initialstate); + if (cpu_fxsr) { + if (npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask) + cpu_mxcsr_mask = + npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask; + else + cpu_mxcsr_mask = 0xFFBF; + + /* + * The fninit instruction does not modify XMM + * registers or x87 registers (MM/ST). The fpusave + * call dumped the garbage contained in the registers + * after reset to the initial state saved. Clear XMM + * and x87 registers file image to make the startup + * program state and signal handler XMM/x87 register + * content predictable. + */ + bzero(npx_initialstate->sv_xmm.sv_fp, + sizeof(npx_initialstate->sv_xmm.sv_fp)); + bzero(npx_initialstate->sv_xmm.sv_xmm, + sizeof(npx_initialstate->sv_xmm.sv_xmm)); + } else + bzero(npx_initialstate->sv_87.sv_ac, + sizeof(npx_initialstate->sv_87.sv_ac)); + + /* + * Create a table describing the layout of the CPU Extended + * Save Area. + */ + if (use_xsave) { + if (xsave_mask >> 32 != 0) + max_ext_n = fls(xsave_mask >> 32) + 32; + else + max_ext_n = fls(xsave_mask); + xsave_area_desc = malloc(max_ext_n * sizeof(struct + xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO); + /* x87 state */ + xsave_area_desc[0].offset = 0; + xsave_area_desc[0].size = 160; + /* XMM */ + xsave_area_desc[1].offset = 160; + xsave_area_desc[1].size = 288 - 160; + + for (i = 2; i < max_ext_n; i++) { + cpuid_count(0xd, i, cp); + xsave_area_desc[i].offset = cp[1]; + xsave_area_desc[i].size = cp[0]; + } + } + + fpu_save_area_zone = uma_zcreate("FPU_save_area", + cpu_max_ext_state_size, NULL, NULL, NULL, NULL, + XSAVE_AREA_ALIGN - 1, 0); + + start_emulating(); + intr_restore(saveintr); +} +SYSINIT(npxinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, npxinitstate, NULL); + +/* + * Free coprocessor (if we have it). + */ +void +npxexit(struct thread *td) +{ + + critical_enter(); + if (curthread == PCPU_GET(fpcurthread)) { + stop_emulating(); + fpusave(curpcb->pcb_save); + start_emulating(); + PCPU_SET(fpcurthread, NULL); + } + critical_exit(); +#ifdef NPX_DEBUG + if (hw_float) { + u_int masked_exceptions; + + masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f; + /* + * Log exceptions that would have trapped with the old + * control word (overflow, divide by 0, and invalid operand). + */ + if (masked_exceptions & 0x0d) + log(LOG_ERR, + "pid %d (%s) exited with masked floating point exceptions 0x%02x\n", + td->td_proc->p_pid, td->td_proc->p_comm, + masked_exceptions); + } +#endif +} + +int +npxformat(void) +{ + + if (!hw_float) + return (_MC_FPFMT_NODEV); + if (cpu_fxsr) + return (_MC_FPFMT_XMM); + return (_MC_FPFMT_387); +} + +/* + * The following mechanism is used to ensure that the FPE_... value + * that is passed as a trapcode to the signal handler of the user + * process does not have more than one bit set. + * + * Multiple bits may be set if the user process modifies the control + * word while a status word bit is already set. While this is a sign + * of bad coding, we have no choise than to narrow them down to one + * bit, since we must not send a trapcode that is not exactly one of + * the FPE_ macros. + * + * The mechanism has a static table with 127 entries. Each combination + * of the 7 FPU status word exception bits directly translates to a + * position in this table, where a single FPE_... value is stored. + * This FPE_... value stored there is considered the "most important" + * of the exception bits and will be sent as the signal code. The + * precedence of the bits is based upon Intel Document "Numerical + * Applications", Chapter "Special Computational Situations". + * + * The macro to choose one of these values does these steps: 1) Throw + * away status word bits that cannot be masked. 2) Throw away the bits + * currently masked in the control word, assuming the user isn't + * interested in them anymore. 3) Reinsert status word bit 7 (stack + * fault) if it is set, which cannot be masked but must be presered. + * 4) Use the remaining bits to point into the trapcode table. + * + * The 6 maskable bits in order of their preference, as stated in the + * above referenced Intel manual: + * 1 Invalid operation (FP_X_INV) + * 1a Stack underflow + * 1b Stack overflow + * 1c Operand of unsupported format + * 1d SNaN operand. + * 2 QNaN operand (not an exception, irrelavant here) + * 3 Any other invalid-operation not mentioned above or zero divide + * (FP_X_INV, FP_X_DZ) + * 4 Denormal operand (FP_X_DNML) + * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) + * 6 Inexact result (FP_X_IMP) + */ +static char fpetable[128] = { + 0, + FPE_FLTINV, /* 1 - INV */ + FPE_FLTUND, /* 2 - DNML */ + FPE_FLTINV, /* 3 - INV | DNML */ + FPE_FLTDIV, /* 4 - DZ */ + FPE_FLTINV, /* 5 - INV | DZ */ + FPE_FLTDIV, /* 6 - DNML | DZ */ + FPE_FLTINV, /* 7 - INV | DNML | DZ */ + FPE_FLTOVF, /* 8 - OFL */ + FPE_FLTINV, /* 9 - INV | OFL */ + FPE_FLTUND, /* A - DNML | OFL */ + FPE_FLTINV, /* B - INV | DNML | OFL */ + FPE_FLTDIV, /* C - DZ | OFL */ + FPE_FLTINV, /* D - INV | DZ | OFL */ + FPE_FLTDIV, /* E - DNML | DZ | OFL */ + FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ + FPE_FLTUND, /* 10 - UFL */ + FPE_FLTINV, /* 11 - INV | UFL */ + FPE_FLTUND, /* 12 - DNML | UFL */ + FPE_FLTINV, /* 13 - INV | DNML | UFL */ + FPE_FLTDIV, /* 14 - DZ | UFL */ + FPE_FLTINV, /* 15 - INV | DZ | UFL */ + FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ + FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ + FPE_FLTOVF, /* 18 - OFL | UFL */ + FPE_FLTINV, /* 19 - INV | OFL | UFL */ + FPE_FLTUND, /* 1A - DNML | OFL | UFL */ + FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ + FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ + FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ + FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ + FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ + FPE_FLTRES, /* 20 - IMP */ + FPE_FLTINV, /* 21 - INV | IMP */ + FPE_FLTUND, /* 22 - DNML | IMP */ + FPE_FLTINV, /* 23 - INV | DNML | IMP */ + FPE_FLTDIV, /* 24 - DZ | IMP */ + FPE_FLTINV, /* 25 - INV | DZ | IMP */ + FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ + FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ + FPE_FLTOVF, /* 28 - OFL | IMP */ + FPE_FLTINV, /* 29 - INV | OFL | IMP */ + FPE_FLTUND, /* 2A - DNML | OFL | IMP */ + FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ + FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ + FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ + FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ + FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ + FPE_FLTUND, /* 30 - UFL | IMP */ + FPE_FLTINV, /* 31 - INV | UFL | IMP */ + FPE_FLTUND, /* 32 - DNML | UFL | IMP */ + FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ + FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ + FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ + FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ + FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ + FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ + FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ + FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ + FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ + FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ + FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ + FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ + FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ + FPE_FLTSUB, /* 40 - STK */ + FPE_FLTSUB, /* 41 - INV | STK */ + FPE_FLTUND, /* 42 - DNML | STK */ + FPE_FLTSUB, /* 43 - INV | DNML | STK */ + FPE_FLTDIV, /* 44 - DZ | STK */ + FPE_FLTSUB, /* 45 - INV | DZ | STK */ + FPE_FLTDIV, /* 46 - DNML | DZ | STK */ + FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ + FPE_FLTOVF, /* 48 - OFL | STK */ + FPE_FLTSUB, /* 49 - INV | OFL | STK */ + FPE_FLTUND, /* 4A - DNML | OFL | STK */ + FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ + FPE_FLTDIV, /* 4C - DZ | OFL | STK */ + FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ + FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ + FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ + FPE_FLTUND, /* 50 - UFL | STK */ + FPE_FLTSUB, /* 51 - INV | UFL | STK */ + FPE_FLTUND, /* 52 - DNML | UFL | STK */ + FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ + FPE_FLTDIV, /* 54 - DZ | UFL | STK */ + FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ + FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ + FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ + FPE_FLTOVF, /* 58 - OFL | UFL | STK */ + FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ + FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ + FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ + FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ + FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ + FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ + FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ + FPE_FLTRES, /* 60 - IMP | STK */ + FPE_FLTSUB, /* 61 - INV | IMP | STK */ + FPE_FLTUND, /* 62 - DNML | IMP | STK */ + FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ + FPE_FLTDIV, /* 64 - DZ | IMP | STK */ + FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ + FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ + FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ + FPE_FLTOVF, /* 68 - OFL | IMP | STK */ + FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ + FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ + FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ + FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ + FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ + FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ + FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ + FPE_FLTUND, /* 70 - UFL | IMP | STK */ + FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ + FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ + FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ + FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ + FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ + FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ + FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ + FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ + FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ + FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ + FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ + FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ + FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ + FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ + FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ +}; + +/* + * Read the FP status and control words, then generate si_code value + * for SIGFPE. The error code chosen will be one of the + * FPE_... macros. It will be sent as the second argument to old + * BSD-style signal handlers and as "siginfo_t->si_code" (second + * argument) to SA_SIGINFO signal handlers. + * + * Some time ago, we cleared the x87 exceptions with FNCLEX there. + * Clearing exceptions was necessary mainly to avoid IRQ13 bugs. The + * usermode code which understands the FPU hardware enough to enable + * the exceptions, can also handle clearing the exception state in the + * handler. The only consequence of not clearing the exception is the + * rethrow of the SIGFPE on return from the signal handler and + * reexecution of the corresponding instruction. + * + * For XMM traps, the exceptions were never cleared. + */ +int +npxtrap_x87(void) +{ + u_short control, status; + + if (!hw_float) { + printf( + "npxtrap_x87: fpcurthread = %p, curthread = %p, hw_float = %d\n", + PCPU_GET(fpcurthread), curthread, hw_float); + panic("npxtrap from nowhere"); + } + critical_enter(); + + /* + * Interrupt handling (for another interrupt) may have pushed the + * state to memory. Fetch the relevant parts of the state from + * wherever they are. + */ + if (PCPU_GET(fpcurthread) != curthread) { + control = GET_FPU_CW(curthread); + status = GET_FPU_SW(curthread); + } else { + fnstcw(&control); + fnstsw(&status); + } + critical_exit(); + return (fpetable[status & ((~control & 0x3f) | 0x40)]); +} + +int +npxtrap_sse(void) +{ + u_int mxcsr; + + if (!hw_float) { + printf( + "npxtrap_sse: fpcurthread = %p, curthread = %p, hw_float = %d\n", + PCPU_GET(fpcurthread), curthread, hw_float); + panic("npxtrap from nowhere"); + } + critical_enter(); + if (PCPU_GET(fpcurthread) != curthread) + mxcsr = curthread->td_pcb->pcb_save->sv_xmm.sv_env.en_mxcsr; + else + stmxcsr(&mxcsr); + critical_exit(); + return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]); +} + +/* + * Implement device not available (DNA) exception + * + * It would be better to switch FP context here (if curthread != fpcurthread) + * and not necessarily for every context switch, but it is too hard to + * access foreign pcb's. + */ + +static int err_count = 0; + +int +npxdna(void) +{ + + if (!hw_float) + return (0); + critical_enter(); + if (PCPU_GET(fpcurthread) == curthread) { + printf("npxdna: fpcurthread == curthread %d times\n", + ++err_count); + stop_emulating(); + critical_exit(); + return (1); + } + if (PCPU_GET(fpcurthread) != NULL) { + printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n", + PCPU_GET(fpcurthread), + PCPU_GET(fpcurthread)->td_proc->p_pid, + curthread, curthread->td_proc->p_pid); + panic("npxdna"); + } + stop_emulating(); + /* + * Record new context early in case frstor causes a trap. + */ + PCPU_SET(fpcurthread, curthread); + + if (cpu_fxsr) + fpu_clean_state(); + + if ((curpcb->pcb_flags & PCB_NPXINITDONE) == 0) { + /* + * This is the first time this thread has used the FPU or + * the PCB doesn't contain a clean FPU state. Explicitly + * load an initial state. + * + * We prefer to restore the state from the actual save + * area in PCB instead of directly loading from + * npx_initialstate, to ignite the XSAVEOPT + * tracking engine. + */ + bcopy(npx_initialstate, curpcb->pcb_save, cpu_max_ext_state_size); + fpurstor(curpcb->pcb_save); + if (curpcb->pcb_initial_npxcw != __INITIAL_NPXCW__) + fldcw(curpcb->pcb_initial_npxcw); + curpcb->pcb_flags |= PCB_NPXINITDONE; + if (PCB_USER_FPU(curpcb)) + curpcb->pcb_flags |= PCB_NPXUSERINITDONE; + } else { + fpurstor(curpcb->pcb_save); + } + critical_exit(); + + return (1); +} + +/* + * Wrapper for fpusave() called from context switch routines. + * + * npxsave() must be called with interrupts disabled, so that it clears + * fpcurthread atomically with saving the state. We require callers to do the + * disabling, since most callers need to disable interrupts anyway to call + * npxsave() atomically with checking fpcurthread. + */ +void +npxsave(addr) + union savefpu *addr; +{ + + stop_emulating(); + if (use_xsaveopt) + xsaveopt((char *)addr, xsave_mask); + else + fpusave(addr); + start_emulating(); + PCPU_SET(fpcurthread, NULL); +} + +/* + * Unconditionally save the current co-processor state across suspend and + * resume. + */ +void +npxsuspend(union savefpu *addr) +{ + register_t cr0; + + if (!hw_float) + return; + if (PCPU_GET(fpcurthread) == NULL) { + bcopy(npx_initialstate, addr, cpu_max_ext_state_size); + return; + } + cr0 = rcr0(); + stop_emulating(); + fpusave(addr); + load_cr0(cr0); +} + +void +npxresume(union savefpu *addr) +{ + register_t cr0; + + if (!hw_float) + return; + + cr0 = rcr0(); + npxinit(false); + stop_emulating(); + fpurstor(addr); + load_cr0(cr0); +} + +void +npxdrop(void) +{ + struct thread *td; + + /* + * Discard pending exceptions in the !cpu_fxsr case so that unmasked + * ones don't cause a panic on the next frstor. + */ + if (!cpu_fxsr) + fnclex(); + + td = PCPU_GET(fpcurthread); + KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); + CRITICAL_ASSERT(td); + PCPU_SET(fpcurthread, NULL); + td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; + start_emulating(); +} + +/* + * Get the user state of the FPU into pcb->pcb_user_save without + * dropping ownership (if possible). It returns the FPU ownership + * status. + */ +int +npxgetregs(struct thread *td) +{ + struct pcb *pcb; + uint64_t *xstate_bv, bit; + char *sa; + int max_ext_n, i; + int owned; + + if (!hw_float) + return (_MC_FPOWNED_NONE); + + pcb = td->td_pcb; + if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) { + bcopy(npx_initialstate, get_pcb_user_save_pcb(pcb), + cpu_max_ext_state_size); + SET_FPU_CW(get_pcb_user_save_pcb(pcb), pcb->pcb_initial_npxcw); + npxuserinited(td); + return (_MC_FPOWNED_PCB); + } + critical_enter(); + if (td == PCPU_GET(fpcurthread)) { + fpusave(get_pcb_user_save_pcb(pcb)); + if (!cpu_fxsr) + /* + * fnsave initializes the FPU and destroys whatever + * context it contains. Make sure the FPU owner + * starts with a clean state next time. + */ + npxdrop(); + owned = _MC_FPOWNED_FPU; + } else { + owned = _MC_FPOWNED_PCB; + } + critical_exit(); + if (use_xsave) { + /* + * Handle partially saved state. + */ + sa = (char *)get_pcb_user_save_pcb(pcb); + xstate_bv = (uint64_t *)(sa + sizeof(union savefpu) + + offsetof(struct xstate_hdr, xstate_bv)); + if (xsave_mask >> 32 != 0) + max_ext_n = fls(xsave_mask >> 32) + 32; + else + max_ext_n = fls(xsave_mask); + for (i = 0; i < max_ext_n; i++) { + bit = 1ULL << i; + if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0) + continue; + bcopy((char *)npx_initialstate + + xsave_area_desc[i].offset, + sa + xsave_area_desc[i].offset, + xsave_area_desc[i].size); + *xstate_bv |= bit; + } + } + return (owned); +} + +void +npxuserinited(struct thread *td) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_NPXINITDONE; + pcb->pcb_flags |= PCB_NPXUSERINITDONE; +} + +int +npxsetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size) +{ + struct xstate_hdr *hdr, *ehdr; + size_t len, max_len; + uint64_t bv; + + /* XXXKIB should we clear all extended state in xstate_bv instead ? */ + if (xfpustate == NULL) + return (0); + if (!use_xsave) + return (EOPNOTSUPP); + + len = xfpustate_size; + if (len < sizeof(struct xstate_hdr)) + return (EINVAL); + max_len = cpu_max_ext_state_size - sizeof(union savefpu); + if (len > max_len) + return (EINVAL); + + ehdr = (struct xstate_hdr *)xfpustate; + bv = ehdr->xstate_bv; + + /* + * Avoid #gp. + */ + if (bv & ~xsave_mask) + return (EINVAL); + + hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1); + + hdr->xstate_bv = bv; + bcopy(xfpustate + sizeof(struct xstate_hdr), + (char *)(hdr + 1), len - sizeof(struct xstate_hdr)); + + return (0); +} + +int +npxsetregs(struct thread *td, union savefpu *addr, char *xfpustate, + size_t xfpustate_size) +{ + struct pcb *pcb; + int error; + + if (!hw_float) + return (ENXIO); + + if (cpu_fxsr) + addr->sv_xmm.sv_env.en_mxcsr &= cpu_mxcsr_mask; + pcb = td->td_pcb; + critical_enter(); + if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { + error = npxsetxstate(td, xfpustate, xfpustate_size); + if (error != 0) { + critical_exit(); + return (error); + } + if (!cpu_fxsr) + fnclex(); /* As in npxdrop(). */ + bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); + fpurstor(get_pcb_user_save_td(td)); + critical_exit(); + pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE; + } else { + critical_exit(); + error = npxsetxstate(td, xfpustate, xfpustate_size); + if (error != 0) + return (error); + bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); + npxuserinited(td); + } + return (0); +} + +static void +fpusave(addr) + union savefpu *addr; +{ + + if (use_xsave) + xsave((char *)addr, xsave_mask); + else if (cpu_fxsr) + fxsave(addr); + else + fnsave(addr); +} + +static void +npx_fill_fpregs_xmm1(struct savexmm *sv_xmm, struct save87 *sv_87) +{ + struct env87 *penv_87; + struct envxmm *penv_xmm; + int i; + + penv_87 = &sv_87->sv_env; + penv_xmm = &sv_xmm->sv_env; + + /* FPU control/status */ + penv_87->en_cw = penv_xmm->en_cw; + penv_87->en_sw = penv_xmm->en_sw; + penv_87->en_fip = penv_xmm->en_fip; + penv_87->en_fcs = penv_xmm->en_fcs; + penv_87->en_opcode = penv_xmm->en_opcode; + penv_87->en_foo = penv_xmm->en_foo; + penv_87->en_fos = penv_xmm->en_fos; + + /* FPU registers and tags */ + penv_87->en_tw = 0xffff; + for (i = 0; i < 8; ++i) { + sv_87->sv_ac[i] = sv_xmm->sv_fp[i].fp_acc; + if ((penv_xmm->en_tw & (1 << i)) != 0) + /* zero and special are set as valid */ + penv_87->en_tw &= ~(3 << i * 2); + } +} + +void +npx_fill_fpregs_xmm(struct savexmm *sv_xmm, struct save87 *sv_87) +{ + + bzero(sv_87, sizeof(*sv_87)); + npx_fill_fpregs_xmm1(sv_xmm, sv_87); +} + +void +npx_set_fpregs_xmm(struct save87 *sv_87, struct savexmm *sv_xmm) +{ + struct env87 *penv_87; + struct envxmm *penv_xmm; + int i; + + penv_87 = &sv_87->sv_env; + penv_xmm = &sv_xmm->sv_env; + + /* FPU control/status */ + penv_xmm->en_cw = penv_87->en_cw; + penv_xmm->en_sw = penv_87->en_sw; + penv_xmm->en_fip = penv_87->en_fip; + penv_xmm->en_fcs = penv_87->en_fcs; + penv_xmm->en_opcode = penv_87->en_opcode; + penv_xmm->en_foo = penv_87->en_foo; + penv_xmm->en_fos = penv_87->en_fos; + + /* + * FPU registers and tags. + * Abridged / Full translation (values in binary), see FXSAVE spec. + * 0 11 + * 1 00, 01, 10 + */ + penv_xmm->en_tw = 0; + for (i = 0; i < 8; ++i) { + sv_xmm->sv_fp[i].fp_acc = sv_87->sv_ac[i]; + if ((penv_87->en_tw & (3 << i * 2)) != (3 << i * 2)) + penv_xmm->en_tw |= 1 << i; + } +} + +void +npx_get_fsave(void *addr) +{ + struct thread *td; + union savefpu *sv; + + td = curthread; + npxgetregs(td); + sv = get_pcb_user_save_td(td); + if (cpu_fxsr) + npx_fill_fpregs_xmm1(&sv->sv_xmm, addr); + else + bcopy(sv, addr, sizeof(struct env87) + + sizeof(struct fpacc87[8])); +} + +int +npx_set_fsave(void *addr) +{ + union savefpu sv; + int error; + + bzero(&sv, sizeof(sv)); + if (cpu_fxsr) + npx_set_fpregs_xmm(addr, &sv.sv_xmm); + else + bcopy(addr, &sv, sizeof(struct env87) + + sizeof(struct fpacc87[8])); + error = npxsetregs(curthread, &sv, NULL, 0); + return (error); +} + +/* + * On AuthenticAMD processors, the fxrstor instruction does not restore + * the x87's stored last instruction pointer, last data pointer, and last + * opcode values, except in the rare case in which the exception summary + * (ES) bit in the x87 status word is set to 1. + * + * In order to avoid leaking this information across processes, we clean + * these values by performing a dummy load before executing fxrstor(). + */ +static void +fpu_clean_state(void) +{ + static float dummy_variable = 0.0; + u_short status; + + /* + * Clear the ES bit in the x87 status word if it is currently + * set, in order to avoid causing a fault in the upcoming load. + */ + fnstsw(&status); + if (status & 0x80) + fnclex(); + + /* + * Load the dummy variable into the x87 stack. This mangles + * the x87 stack, but we don't care since we're about to call + * fxrstor() anyway. + */ + __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable)); +} + +static void +fpurstor(union savefpu *addr) +{ + + if (use_xsave) + xrstor((char *)addr, xsave_mask); + else if (cpu_fxsr) + fxrstor(addr); + else + frstor(addr); +} + +#ifdef DEV_ISA +/* + * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI. + */ +static struct isa_pnp_id npxisa_ids[] = { + { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */ + { 0 } +}; + +static int +npxisa_probe(device_t dev) +{ + int result; + if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) { + device_quiet(dev); + } + return(result); +} + +static int +npxisa_attach(device_t dev) +{ + return (0); +} + +static device_method_t npxisa_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, npxisa_probe), + DEVMETHOD(device_attach, npxisa_attach), + DEVMETHOD(device_detach, bus_generic_detach), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + DEVMETHOD(device_suspend, bus_generic_suspend), + DEVMETHOD(device_resume, bus_generic_resume), + + { 0, 0 } +}; + +static driver_t npxisa_driver = { + "npxisa", + npxisa_methods, + 1, /* no softc */ +}; + +static devclass_t npxisa_devclass; + +DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0); +DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0); +#endif /* DEV_ISA */ + +static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", + "Kernel contexts for FPU state"); + +#define FPU_KERN_CTX_NPXINITDONE 0x01 +#define FPU_KERN_CTX_DUMMY 0x02 +#define FPU_KERN_CTX_INUSE 0x04 + +struct fpu_kern_ctx { + union savefpu *prev; + uint32_t flags; + char hwstate1[]; +}; + +struct fpu_kern_ctx * +fpu_kern_alloc_ctx(u_int flags) +{ + struct fpu_kern_ctx *res; + size_t sz; + + sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN + + cpu_max_ext_state_size; + res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? + M_NOWAIT : M_WAITOK) | M_ZERO); + return (res); +} + +void +fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) +{ + + KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx")); + /* XXXKIB clear the memory ? */ + free(ctx, M_FPUKERN_CTX); +} + +static union savefpu * +fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx) +{ + vm_offset_t p; + + p = (vm_offset_t)&ctx->hwstate1; + p = roundup2(p, XSAVE_AREA_ALIGN); + return ((union savefpu *)p); +} + +int +fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) +{ + struct pcb *pcb; + + KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("using inuse ctx")); + + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; + return (0); + } + pcb = td->td_pcb; + KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == + get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); + ctx->flags = FPU_KERN_CTX_INUSE; + if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0) + ctx->flags |= FPU_KERN_CTX_NPXINITDONE; + npxexit(td); + ctx->prev = pcb->pcb_save; + pcb->pcb_save = fpu_kern_ctx_savefpu(ctx); + pcb->pcb_flags |= PCB_KERNNPX; + pcb->pcb_flags &= ~PCB_NPXINITDONE; + return (0); +} + +int +fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) +{ + struct pcb *pcb; + + KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, + ("leaving not inuse ctx")); + ctx->flags &= ~FPU_KERN_CTX_INUSE; + + if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); + pcb = td->td_pcb; + critical_enter(); + if (curthread == PCPU_GET(fpcurthread)) + npxdrop(); + critical_exit(); + pcb->pcb_save = ctx->prev; + if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) { + if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0) + pcb->pcb_flags |= PCB_NPXINITDONE; + else + pcb->pcb_flags &= ~PCB_NPXINITDONE; + pcb->pcb_flags &= ~PCB_KERNNPX; + } else { + if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0) + pcb->pcb_flags |= PCB_NPXINITDONE; + else + pcb->pcb_flags &= ~PCB_NPXINITDONE; + KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); + } + return (0); +} + +int +fpu_kern_thread(u_int flags) +{ + + KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, + ("Only kthread may use fpu_kern_thread")); + KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb), + ("mangled pcb_save")); + KASSERT(PCB_USER_FPU(curpcb), ("recursive call")); + + curpcb->pcb_flags |= PCB_KERNNPX; + return (0); +} + +int +is_fpu_kern_thread(u_int flags) +{ + + if ((curthread->td_pflags & TDP_KTHREAD) == 0) + return (0); + return ((curpcb->pcb_flags & PCB_KERNNPX) != 0); +} + +/* + * FPU save area alloc/free/init utility routines + */ +union savefpu * +fpu_save_area_alloc(void) +{ + + return (uma_zalloc(fpu_save_area_zone, 0)); +} + +void +fpu_save_area_free(union savefpu *fsa) +{ + + uma_zfree(fpu_save_area_zone, fsa); +} + +void +fpu_save_area_reset(union savefpu *fsa) +{ + + bcopy(npx_initialstate, fsa, cpu_max_ext_state_size); +} Property changes on: head/sys/i386/i386/npx.c ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/i386/isa/npx.c =================================================================== --- head/sys/i386/isa/npx.c (revision 327379) +++ head/sys/i386/isa/npx.c (nonexistent) @@ -1,1430 +0,0 @@ -/*- - * Copyright (c) 1990 William Jolitz. - * Copyright (c) 1991 The Regents of the University of California. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 - */ - -#include -__FBSDID("$FreeBSD$"); - -#include "opt_cpu.h" -#include "opt_isa.h" -#include "opt_npx.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef NPX_DEBUG -#include -#endif -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifdef DEV_ISA -#include -#endif - -/* - * 387 and 287 Numeric Coprocessor Extension (NPX) Driver. - */ - -#if defined(__GNUCLIKE_ASM) && !defined(lint) - -#define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) -#define fnclex() __asm __volatile("fnclex") -#define fninit() __asm __volatile("fninit") -#define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr))) -#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) -#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) -#define fp_divide_by_0() __asm __volatile( \ - "fldz; fld1; fdiv %st,%st(1); fnop") -#define frstor(addr) __asm __volatile("frstor %0" : : "m" (*(addr))) -#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) -#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) -#define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) -#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) - -static __inline void -xrstor(char *addr, uint64_t mask) -{ - uint32_t low, hi; - - low = mask; - hi = mask >> 32; - __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); -} - -static __inline void -xsave(char *addr, uint64_t mask) -{ - uint32_t low, hi; - - low = mask; - hi = mask >> 32; - __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : - "memory"); -} - -static __inline void -xsaveopt(char *addr, uint64_t mask) -{ - uint32_t low, hi; - - low = mask; - hi = mask >> 32; - __asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) : - "memory"); -} -#else /* !(__GNUCLIKE_ASM && !lint) */ - -void fldcw(u_short cw); -void fnclex(void); -void fninit(void); -void fnsave(caddr_t addr); -void fnstcw(caddr_t addr); -void fnstsw(caddr_t addr); -void fp_divide_by_0(void); -void frstor(caddr_t addr); -void fxsave(caddr_t addr); -void fxrstor(caddr_t addr); -void ldmxcsr(u_int csr); -void stmxcsr(u_int *csr); -void xrstor(char *addr, uint64_t mask); -void xsave(char *addr, uint64_t mask); -void xsaveopt(char *addr, uint64_t mask); - -#endif /* __GNUCLIKE_ASM && !lint */ - -#define start_emulating() load_cr0(rcr0() | CR0_TS) -#define stop_emulating() clts() - -#define GET_FPU_CW(thread) \ - (cpu_fxsr ? \ - (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \ - (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw) -#define GET_FPU_SW(thread) \ - (cpu_fxsr ? \ - (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \ - (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw) -#define SET_FPU_CW(savefpu, value) do { \ - if (cpu_fxsr) \ - (savefpu)->sv_xmm.sv_env.en_cw = (value); \ - else \ - (savefpu)->sv_87.sv_env.en_cw = (value); \ -} while (0) - -CTASSERT(sizeof(union savefpu) == 512); -CTASSERT(sizeof(struct xstate_hdr) == 64); -CTASSERT(sizeof(struct savefpu_ymm) == 832); - -/* - * This requirement is to make it easier for asm code to calculate - * offset of the fpu save area from the pcb address. FPU save area - * must be 64-byte aligned. - */ -CTASSERT(sizeof(struct pcb) % XSAVE_AREA_ALIGN == 0); - -/* - * Ensure the copy of XCR0 saved in a core is contained in the padding - * area. - */ -CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) && - X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm)); - -static void fpu_clean_state(void); - -static void fpusave(union savefpu *); -static void fpurstor(union savefpu *); - -int hw_float; - -SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, - &hw_float, 0, "Floating point instructions executed in hardware"); - -int use_xsave; -uint64_t xsave_mask; -static uma_zone_t fpu_save_area_zone; -static union savefpu *npx_initialstate; - -struct xsave_area_elm_descr { - u_int offset; - u_int size; -} *xsave_area_desc; - -static int use_xsaveopt; - -static volatile u_int npx_traps_while_probing; - -alias_for_inthand_t probetrap; -__asm(" \n\ - .text \n\ - .p2align 2,0x90 \n\ - .type " __XSTRING(CNAME(probetrap)) ",@function \n\ -" __XSTRING(CNAME(probetrap)) ": \n\ - ss \n\ - incl " __XSTRING(CNAME(npx_traps_while_probing)) " \n\ - fnclex \n\ - iret \n\ -"); - -/* - * Determine if an FPU is present and how to use it. - */ -static int -npx_probe(void) -{ - struct gate_descriptor save_idt_npxtrap; - u_short control, status; - - /* - * Modern CPUs all have an FPU that uses the INT16 interface - * and provide a simple way to verify that, so handle the - * common case right away. - */ - if (cpu_feature & CPUID_FPU) { - hw_float = 1; - return (1); - } - - save_idt_npxtrap = idt[IDT_MF]; - setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL, - GSEL(GCODE_SEL, SEL_KPL)); - - /* - * Don't trap while we're probing. - */ - stop_emulating(); - - /* - * Finish resetting the coprocessor, if any. If there is an error - * pending, then we may get a bogus IRQ13, but npx_intr() will handle - * it OK. Bogus halts have never been observed, but we enabled - * IRQ13 and cleared the BUSY# latch early to handle them anyway. - */ - fninit(); - - /* - * Don't use fwait here because it might hang. - * Don't use fnop here because it usually hangs if there is no FPU. - */ - DELAY(1000); /* wait for any IRQ13 */ -#ifdef DIAGNOSTIC - if (npx_traps_while_probing != 0) - printf("fninit caused %u bogus npx trap(s)\n", - npx_traps_while_probing); -#endif - /* - * Check for a status of mostly zero. - */ - status = 0x5a5a; - fnstsw(&status); - if ((status & 0xb8ff) == 0) { - /* - * Good, now check for a proper control word. - */ - control = 0x5a5a; - fnstcw(&control); - if ((control & 0x1f3f) == 0x033f) { - /* - * We have an npx, now divide by 0 to see if exception - * 16 works. - */ - control &= ~(1 << 2); /* enable divide by 0 trap */ - fldcw(control); - npx_traps_while_probing = 0; - fp_divide_by_0(); - if (npx_traps_while_probing != 0) { - /* - * Good, exception 16 works. - */ - hw_float = 1; - goto cleanup; - } - printf( - "FPU does not use exception 16 for error reporting\n"); - goto cleanup; - } - } - - /* - * Probe failed. Floating point simply won't work. - * Notify user and disable FPU/MMX/SSE instruction execution. - */ - printf("WARNING: no FPU!\n"); - __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : - "n" (CR0_EM | CR0_MP) : "ax"); - -cleanup: - idt[IDT_MF] = save_idt_npxtrap; - return (hw_float); -} - -/* - * Enable XSAVE if supported and allowed by user. - * Calculate the xsave_mask. - */ -static void -npxinit_bsp1(void) -{ - u_int cp[4]; - uint64_t xsave_mask_user; - - if (cpu_fxsr && (cpu_feature2 & CPUID2_XSAVE) != 0) { - use_xsave = 1; - TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); - } - if (!use_xsave) - return; - - cpuid_count(0xd, 0x0, cp); - xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; - if ((cp[0] & xsave_mask) != xsave_mask) - panic("CPU0 does not support X87 or SSE: %x", cp[0]); - xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; - xsave_mask_user = xsave_mask; - TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user); - xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; - xsave_mask &= xsave_mask_user; - if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512) - xsave_mask &= ~XFEATURE_AVX512; - if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX) - xsave_mask &= ~XFEATURE_MPX; - - cpuid_count(0xd, 0x1, cp); - if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) - use_xsaveopt = 1; -} - -/* - - * Calculate the fpu save area size. - */ -static void -npxinit_bsp2(void) -{ - u_int cp[4]; - - if (use_xsave) { - cpuid_count(0xd, 0x0, cp); - cpu_max_ext_state_size = cp[1]; - - /* - * Reload the cpu_feature2, since we enabled OSXSAVE. - */ - do_cpuid(1, cp); - cpu_feature2 = cp[2]; - } else - cpu_max_ext_state_size = sizeof(union savefpu); -} - -/* - * Initialize floating point unit. - */ -void -npxinit(bool bsp) -{ - static union savefpu dummy; - register_t saveintr; - u_int mxcsr; - u_short control; - - if (bsp) { - if (!npx_probe()) - return; - npxinit_bsp1(); - } - - if (use_xsave) { - load_cr4(rcr4() | CR4_XSAVE); - load_xcr(XCR0, xsave_mask); - } - - /* - * XCR0 shall be set up before CPU can report the save area size. - */ - if (bsp) - npxinit_bsp2(); - - /* - * fninit has the same h/w bugs as fnsave. Use the detoxified - * fnsave to throw away any junk in the fpu. fpusave() initializes - * the fpu. - * - * It is too early for critical_enter() to work on AP. - */ - saveintr = intr_disable(); - stop_emulating(); - if (cpu_fxsr) - fninit(); - else - fnsave(&dummy); - control = __INITIAL_NPXCW__; - fldcw(control); - if (cpu_fxsr) { - mxcsr = __INITIAL_MXCSR__; - ldmxcsr(mxcsr); - } - start_emulating(); - intr_restore(saveintr); -} - -/* - * On the boot CPU we generate a clean state that is used to - * initialize the floating point unit when it is first used by a - * process. - */ -static void -npxinitstate(void *arg __unused) -{ - register_t saveintr; - int cp[4], i, max_ext_n; - - if (!hw_float) - return; - - npx_initialstate = malloc(cpu_max_ext_state_size, M_DEVBUF, - M_WAITOK | M_ZERO); - saveintr = intr_disable(); - stop_emulating(); - - fpusave(npx_initialstate); - if (cpu_fxsr) { - if (npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask) - cpu_mxcsr_mask = - npx_initialstate->sv_xmm.sv_env.en_mxcsr_mask; - else - cpu_mxcsr_mask = 0xFFBF; - - /* - * The fninit instruction does not modify XMM - * registers or x87 registers (MM/ST). The fpusave - * call dumped the garbage contained in the registers - * after reset to the initial state saved. Clear XMM - * and x87 registers file image to make the startup - * program state and signal handler XMM/x87 register - * content predictable. - */ - bzero(npx_initialstate->sv_xmm.sv_fp, - sizeof(npx_initialstate->sv_xmm.sv_fp)); - bzero(npx_initialstate->sv_xmm.sv_xmm, - sizeof(npx_initialstate->sv_xmm.sv_xmm)); - } else - bzero(npx_initialstate->sv_87.sv_ac, - sizeof(npx_initialstate->sv_87.sv_ac)); - - /* - * Create a table describing the layout of the CPU Extended - * Save Area. - */ - if (use_xsave) { - if (xsave_mask >> 32 != 0) - max_ext_n = fls(xsave_mask >> 32) + 32; - else - max_ext_n = fls(xsave_mask); - xsave_area_desc = malloc(max_ext_n * sizeof(struct - xsave_area_elm_descr), M_DEVBUF, M_WAITOK | M_ZERO); - /* x87 state */ - xsave_area_desc[0].offset = 0; - xsave_area_desc[0].size = 160; - /* XMM */ - xsave_area_desc[1].offset = 160; - xsave_area_desc[1].size = 288 - 160; - - for (i = 2; i < max_ext_n; i++) { - cpuid_count(0xd, i, cp); - xsave_area_desc[i].offset = cp[1]; - xsave_area_desc[i].size = cp[0]; - } - } - - fpu_save_area_zone = uma_zcreate("FPU_save_area", - cpu_max_ext_state_size, NULL, NULL, NULL, NULL, - XSAVE_AREA_ALIGN - 1, 0); - - start_emulating(); - intr_restore(saveintr); -} -SYSINIT(npxinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, npxinitstate, NULL); - -/* - * Free coprocessor (if we have it). - */ -void -npxexit(struct thread *td) -{ - - critical_enter(); - if (curthread == PCPU_GET(fpcurthread)) { - stop_emulating(); - fpusave(curpcb->pcb_save); - start_emulating(); - PCPU_SET(fpcurthread, NULL); - } - critical_exit(); -#ifdef NPX_DEBUG - if (hw_float) { - u_int masked_exceptions; - - masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f; - /* - * Log exceptions that would have trapped with the old - * control word (overflow, divide by 0, and invalid operand). - */ - if (masked_exceptions & 0x0d) - log(LOG_ERR, - "pid %d (%s) exited with masked floating point exceptions 0x%02x\n", - td->td_proc->p_pid, td->td_proc->p_comm, - masked_exceptions); - } -#endif -} - -int -npxformat(void) -{ - - if (!hw_float) - return (_MC_FPFMT_NODEV); - if (cpu_fxsr) - return (_MC_FPFMT_XMM); - return (_MC_FPFMT_387); -} - -/* - * The following mechanism is used to ensure that the FPE_... value - * that is passed as a trapcode to the signal handler of the user - * process does not have more than one bit set. - * - * Multiple bits may be set if the user process modifies the control - * word while a status word bit is already set. While this is a sign - * of bad coding, we have no choise than to narrow them down to one - * bit, since we must not send a trapcode that is not exactly one of - * the FPE_ macros. - * - * The mechanism has a static table with 127 entries. Each combination - * of the 7 FPU status word exception bits directly translates to a - * position in this table, where a single FPE_... value is stored. - * This FPE_... value stored there is considered the "most important" - * of the exception bits and will be sent as the signal code. The - * precedence of the bits is based upon Intel Document "Numerical - * Applications", Chapter "Special Computational Situations". - * - * The macro to choose one of these values does these steps: 1) Throw - * away status word bits that cannot be masked. 2) Throw away the bits - * currently masked in the control word, assuming the user isn't - * interested in them anymore. 3) Reinsert status word bit 7 (stack - * fault) if it is set, which cannot be masked but must be presered. - * 4) Use the remaining bits to point into the trapcode table. - * - * The 6 maskable bits in order of their preference, as stated in the - * above referenced Intel manual: - * 1 Invalid operation (FP_X_INV) - * 1a Stack underflow - * 1b Stack overflow - * 1c Operand of unsupported format - * 1d SNaN operand. - * 2 QNaN operand (not an exception, irrelavant here) - * 3 Any other invalid-operation not mentioned above or zero divide - * (FP_X_INV, FP_X_DZ) - * 4 Denormal operand (FP_X_DNML) - * 5 Numeric over/underflow (FP_X_OFL, FP_X_UFL) - * 6 Inexact result (FP_X_IMP) - */ -static char fpetable[128] = { - 0, - FPE_FLTINV, /* 1 - INV */ - FPE_FLTUND, /* 2 - DNML */ - FPE_FLTINV, /* 3 - INV | DNML */ - FPE_FLTDIV, /* 4 - DZ */ - FPE_FLTINV, /* 5 - INV | DZ */ - FPE_FLTDIV, /* 6 - DNML | DZ */ - FPE_FLTINV, /* 7 - INV | DNML | DZ */ - FPE_FLTOVF, /* 8 - OFL */ - FPE_FLTINV, /* 9 - INV | OFL */ - FPE_FLTUND, /* A - DNML | OFL */ - FPE_FLTINV, /* B - INV | DNML | OFL */ - FPE_FLTDIV, /* C - DZ | OFL */ - FPE_FLTINV, /* D - INV | DZ | OFL */ - FPE_FLTDIV, /* E - DNML | DZ | OFL */ - FPE_FLTINV, /* F - INV | DNML | DZ | OFL */ - FPE_FLTUND, /* 10 - UFL */ - FPE_FLTINV, /* 11 - INV | UFL */ - FPE_FLTUND, /* 12 - DNML | UFL */ - FPE_FLTINV, /* 13 - INV | DNML | UFL */ - FPE_FLTDIV, /* 14 - DZ | UFL */ - FPE_FLTINV, /* 15 - INV | DZ | UFL */ - FPE_FLTDIV, /* 16 - DNML | DZ | UFL */ - FPE_FLTINV, /* 17 - INV | DNML | DZ | UFL */ - FPE_FLTOVF, /* 18 - OFL | UFL */ - FPE_FLTINV, /* 19 - INV | OFL | UFL */ - FPE_FLTUND, /* 1A - DNML | OFL | UFL */ - FPE_FLTINV, /* 1B - INV | DNML | OFL | UFL */ - FPE_FLTDIV, /* 1C - DZ | OFL | UFL */ - FPE_FLTINV, /* 1D - INV | DZ | OFL | UFL */ - FPE_FLTDIV, /* 1E - DNML | DZ | OFL | UFL */ - FPE_FLTINV, /* 1F - INV | DNML | DZ | OFL | UFL */ - FPE_FLTRES, /* 20 - IMP */ - FPE_FLTINV, /* 21 - INV | IMP */ - FPE_FLTUND, /* 22 - DNML | IMP */ - FPE_FLTINV, /* 23 - INV | DNML | IMP */ - FPE_FLTDIV, /* 24 - DZ | IMP */ - FPE_FLTINV, /* 25 - INV | DZ | IMP */ - FPE_FLTDIV, /* 26 - DNML | DZ | IMP */ - FPE_FLTINV, /* 27 - INV | DNML | DZ | IMP */ - FPE_FLTOVF, /* 28 - OFL | IMP */ - FPE_FLTINV, /* 29 - INV | OFL | IMP */ - FPE_FLTUND, /* 2A - DNML | OFL | IMP */ - FPE_FLTINV, /* 2B - INV | DNML | OFL | IMP */ - FPE_FLTDIV, /* 2C - DZ | OFL | IMP */ - FPE_FLTINV, /* 2D - INV | DZ | OFL | IMP */ - FPE_FLTDIV, /* 2E - DNML | DZ | OFL | IMP */ - FPE_FLTINV, /* 2F - INV | DNML | DZ | OFL | IMP */ - FPE_FLTUND, /* 30 - UFL | IMP */ - FPE_FLTINV, /* 31 - INV | UFL | IMP */ - FPE_FLTUND, /* 32 - DNML | UFL | IMP */ - FPE_FLTINV, /* 33 - INV | DNML | UFL | IMP */ - FPE_FLTDIV, /* 34 - DZ | UFL | IMP */ - FPE_FLTINV, /* 35 - INV | DZ | UFL | IMP */ - FPE_FLTDIV, /* 36 - DNML | DZ | UFL | IMP */ - FPE_FLTINV, /* 37 - INV | DNML | DZ | UFL | IMP */ - FPE_FLTOVF, /* 38 - OFL | UFL | IMP */ - FPE_FLTINV, /* 39 - INV | OFL | UFL | IMP */ - FPE_FLTUND, /* 3A - DNML | OFL | UFL | IMP */ - FPE_FLTINV, /* 3B - INV | DNML | OFL | UFL | IMP */ - FPE_FLTDIV, /* 3C - DZ | OFL | UFL | IMP */ - FPE_FLTINV, /* 3D - INV | DZ | OFL | UFL | IMP */ - FPE_FLTDIV, /* 3E - DNML | DZ | OFL | UFL | IMP */ - FPE_FLTINV, /* 3F - INV | DNML | DZ | OFL | UFL | IMP */ - FPE_FLTSUB, /* 40 - STK */ - FPE_FLTSUB, /* 41 - INV | STK */ - FPE_FLTUND, /* 42 - DNML | STK */ - FPE_FLTSUB, /* 43 - INV | DNML | STK */ - FPE_FLTDIV, /* 44 - DZ | STK */ - FPE_FLTSUB, /* 45 - INV | DZ | STK */ - FPE_FLTDIV, /* 46 - DNML | DZ | STK */ - FPE_FLTSUB, /* 47 - INV | DNML | DZ | STK */ - FPE_FLTOVF, /* 48 - OFL | STK */ - FPE_FLTSUB, /* 49 - INV | OFL | STK */ - FPE_FLTUND, /* 4A - DNML | OFL | STK */ - FPE_FLTSUB, /* 4B - INV | DNML | OFL | STK */ - FPE_FLTDIV, /* 4C - DZ | OFL | STK */ - FPE_FLTSUB, /* 4D - INV | DZ | OFL | STK */ - FPE_FLTDIV, /* 4E - DNML | DZ | OFL | STK */ - FPE_FLTSUB, /* 4F - INV | DNML | DZ | OFL | STK */ - FPE_FLTUND, /* 50 - UFL | STK */ - FPE_FLTSUB, /* 51 - INV | UFL | STK */ - FPE_FLTUND, /* 52 - DNML | UFL | STK */ - FPE_FLTSUB, /* 53 - INV | DNML | UFL | STK */ - FPE_FLTDIV, /* 54 - DZ | UFL | STK */ - FPE_FLTSUB, /* 55 - INV | DZ | UFL | STK */ - FPE_FLTDIV, /* 56 - DNML | DZ | UFL | STK */ - FPE_FLTSUB, /* 57 - INV | DNML | DZ | UFL | STK */ - FPE_FLTOVF, /* 58 - OFL | UFL | STK */ - FPE_FLTSUB, /* 59 - INV | OFL | UFL | STK */ - FPE_FLTUND, /* 5A - DNML | OFL | UFL | STK */ - FPE_FLTSUB, /* 5B - INV | DNML | OFL | UFL | STK */ - FPE_FLTDIV, /* 5C - DZ | OFL | UFL | STK */ - FPE_FLTSUB, /* 5D - INV | DZ | OFL | UFL | STK */ - FPE_FLTDIV, /* 5E - DNML | DZ | OFL | UFL | STK */ - FPE_FLTSUB, /* 5F - INV | DNML | DZ | OFL | UFL | STK */ - FPE_FLTRES, /* 60 - IMP | STK */ - FPE_FLTSUB, /* 61 - INV | IMP | STK */ - FPE_FLTUND, /* 62 - DNML | IMP | STK */ - FPE_FLTSUB, /* 63 - INV | DNML | IMP | STK */ - FPE_FLTDIV, /* 64 - DZ | IMP | STK */ - FPE_FLTSUB, /* 65 - INV | DZ | IMP | STK */ - FPE_FLTDIV, /* 66 - DNML | DZ | IMP | STK */ - FPE_FLTSUB, /* 67 - INV | DNML | DZ | IMP | STK */ - FPE_FLTOVF, /* 68 - OFL | IMP | STK */ - FPE_FLTSUB, /* 69 - INV | OFL | IMP | STK */ - FPE_FLTUND, /* 6A - DNML | OFL | IMP | STK */ - FPE_FLTSUB, /* 6B - INV | DNML | OFL | IMP | STK */ - FPE_FLTDIV, /* 6C - DZ | OFL | IMP | STK */ - FPE_FLTSUB, /* 6D - INV | DZ | OFL | IMP | STK */ - FPE_FLTDIV, /* 6E - DNML | DZ | OFL | IMP | STK */ - FPE_FLTSUB, /* 6F - INV | DNML | DZ | OFL | IMP | STK */ - FPE_FLTUND, /* 70 - UFL | IMP | STK */ - FPE_FLTSUB, /* 71 - INV | UFL | IMP | STK */ - FPE_FLTUND, /* 72 - DNML | UFL | IMP | STK */ - FPE_FLTSUB, /* 73 - INV | DNML | UFL | IMP | STK */ - FPE_FLTDIV, /* 74 - DZ | UFL | IMP | STK */ - FPE_FLTSUB, /* 75 - INV | DZ | UFL | IMP | STK */ - FPE_FLTDIV, /* 76 - DNML | DZ | UFL | IMP | STK */ - FPE_FLTSUB, /* 77 - INV | DNML | DZ | UFL | IMP | STK */ - FPE_FLTOVF, /* 78 - OFL | UFL | IMP | STK */ - FPE_FLTSUB, /* 79 - INV | OFL | UFL | IMP | STK */ - FPE_FLTUND, /* 7A - DNML | OFL | UFL | IMP | STK */ - FPE_FLTSUB, /* 7B - INV | DNML | OFL | UFL | IMP | STK */ - FPE_FLTDIV, /* 7C - DZ | OFL | UFL | IMP | STK */ - FPE_FLTSUB, /* 7D - INV | DZ | OFL | UFL | IMP | STK */ - FPE_FLTDIV, /* 7E - DNML | DZ | OFL | UFL | IMP | STK */ - FPE_FLTSUB, /* 7F - INV | DNML | DZ | OFL | UFL | IMP | STK */ -}; - -/* - * Read the FP status and control words, then generate si_code value - * for SIGFPE. The error code chosen will be one of the - * FPE_... macros. It will be sent as the second argument to old - * BSD-style signal handlers and as "siginfo_t->si_code" (second - * argument) to SA_SIGINFO signal handlers. - * - * Some time ago, we cleared the x87 exceptions with FNCLEX there. - * Clearing exceptions was necessary mainly to avoid IRQ13 bugs. The - * usermode code which understands the FPU hardware enough to enable - * the exceptions, can also handle clearing the exception state in the - * handler. The only consequence of not clearing the exception is the - * rethrow of the SIGFPE on return from the signal handler and - * reexecution of the corresponding instruction. - * - * For XMM traps, the exceptions were never cleared. - */ -int -npxtrap_x87(void) -{ - u_short control, status; - - if (!hw_float) { - printf( - "npxtrap_x87: fpcurthread = %p, curthread = %p, hw_float = %d\n", - PCPU_GET(fpcurthread), curthread, hw_float); - panic("npxtrap from nowhere"); - } - critical_enter(); - - /* - * Interrupt handling (for another interrupt) may have pushed the - * state to memory. Fetch the relevant parts of the state from - * wherever they are. - */ - if (PCPU_GET(fpcurthread) != curthread) { - control = GET_FPU_CW(curthread); - status = GET_FPU_SW(curthread); - } else { - fnstcw(&control); - fnstsw(&status); - } - critical_exit(); - return (fpetable[status & ((~control & 0x3f) | 0x40)]); -} - -int -npxtrap_sse(void) -{ - u_int mxcsr; - - if (!hw_float) { - printf( - "npxtrap_sse: fpcurthread = %p, curthread = %p, hw_float = %d\n", - PCPU_GET(fpcurthread), curthread, hw_float); - panic("npxtrap from nowhere"); - } - critical_enter(); - if (PCPU_GET(fpcurthread) != curthread) - mxcsr = curthread->td_pcb->pcb_save->sv_xmm.sv_env.en_mxcsr; - else - stmxcsr(&mxcsr); - critical_exit(); - return (fpetable[(mxcsr & (~mxcsr >> 7)) & 0x3f]); -} - -/* - * Implement device not available (DNA) exception - * - * It would be better to switch FP context here (if curthread != fpcurthread) - * and not necessarily for every context switch, but it is too hard to - * access foreign pcb's. - */ - -static int err_count = 0; - -int -npxdna(void) -{ - - if (!hw_float) - return (0); - critical_enter(); - if (PCPU_GET(fpcurthread) == curthread) { - printf("npxdna: fpcurthread == curthread %d times\n", - ++err_count); - stop_emulating(); - critical_exit(); - return (1); - } - if (PCPU_GET(fpcurthread) != NULL) { - printf("npxdna: fpcurthread = %p (%d), curthread = %p (%d)\n", - PCPU_GET(fpcurthread), - PCPU_GET(fpcurthread)->td_proc->p_pid, - curthread, curthread->td_proc->p_pid); - panic("npxdna"); - } - stop_emulating(); - /* - * Record new context early in case frstor causes a trap. - */ - PCPU_SET(fpcurthread, curthread); - - if (cpu_fxsr) - fpu_clean_state(); - - if ((curpcb->pcb_flags & PCB_NPXINITDONE) == 0) { - /* - * This is the first time this thread has used the FPU or - * the PCB doesn't contain a clean FPU state. Explicitly - * load an initial state. - * - * We prefer to restore the state from the actual save - * area in PCB instead of directly loading from - * npx_initialstate, to ignite the XSAVEOPT - * tracking engine. - */ - bcopy(npx_initialstate, curpcb->pcb_save, cpu_max_ext_state_size); - fpurstor(curpcb->pcb_save); - if (curpcb->pcb_initial_npxcw != __INITIAL_NPXCW__) - fldcw(curpcb->pcb_initial_npxcw); - curpcb->pcb_flags |= PCB_NPXINITDONE; - if (PCB_USER_FPU(curpcb)) - curpcb->pcb_flags |= PCB_NPXUSERINITDONE; - } else { - fpurstor(curpcb->pcb_save); - } - critical_exit(); - - return (1); -} - -/* - * Wrapper for fpusave() called from context switch routines. - * - * npxsave() must be called with interrupts disabled, so that it clears - * fpcurthread atomically with saving the state. We require callers to do the - * disabling, since most callers need to disable interrupts anyway to call - * npxsave() atomically with checking fpcurthread. - */ -void -npxsave(addr) - union savefpu *addr; -{ - - stop_emulating(); - if (use_xsaveopt) - xsaveopt((char *)addr, xsave_mask); - else - fpusave(addr); - start_emulating(); - PCPU_SET(fpcurthread, NULL); -} - -/* - * Unconditionally save the current co-processor state across suspend and - * resume. - */ -void -npxsuspend(union savefpu *addr) -{ - register_t cr0; - - if (!hw_float) - return; - if (PCPU_GET(fpcurthread) == NULL) { - bcopy(npx_initialstate, addr, cpu_max_ext_state_size); - return; - } - cr0 = rcr0(); - stop_emulating(); - fpusave(addr); - load_cr0(cr0); -} - -void -npxresume(union savefpu *addr) -{ - register_t cr0; - - if (!hw_float) - return; - - cr0 = rcr0(); - npxinit(false); - stop_emulating(); - fpurstor(addr); - load_cr0(cr0); -} - -void -npxdrop(void) -{ - struct thread *td; - - /* - * Discard pending exceptions in the !cpu_fxsr case so that unmasked - * ones don't cause a panic on the next frstor. - */ - if (!cpu_fxsr) - fnclex(); - - td = PCPU_GET(fpcurthread); - KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); - CRITICAL_ASSERT(td); - PCPU_SET(fpcurthread, NULL); - td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; - start_emulating(); -} - -/* - * Get the user state of the FPU into pcb->pcb_user_save without - * dropping ownership (if possible). It returns the FPU ownership - * status. - */ -int -npxgetregs(struct thread *td) -{ - struct pcb *pcb; - uint64_t *xstate_bv, bit; - char *sa; - int max_ext_n, i; - int owned; - - if (!hw_float) - return (_MC_FPOWNED_NONE); - - pcb = td->td_pcb; - if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) { - bcopy(npx_initialstate, get_pcb_user_save_pcb(pcb), - cpu_max_ext_state_size); - SET_FPU_CW(get_pcb_user_save_pcb(pcb), pcb->pcb_initial_npxcw); - npxuserinited(td); - return (_MC_FPOWNED_PCB); - } - critical_enter(); - if (td == PCPU_GET(fpcurthread)) { - fpusave(get_pcb_user_save_pcb(pcb)); - if (!cpu_fxsr) - /* - * fnsave initializes the FPU and destroys whatever - * context it contains. Make sure the FPU owner - * starts with a clean state next time. - */ - npxdrop(); - owned = _MC_FPOWNED_FPU; - } else { - owned = _MC_FPOWNED_PCB; - } - critical_exit(); - if (use_xsave) { - /* - * Handle partially saved state. - */ - sa = (char *)get_pcb_user_save_pcb(pcb); - xstate_bv = (uint64_t *)(sa + sizeof(union savefpu) + - offsetof(struct xstate_hdr, xstate_bv)); - if (xsave_mask >> 32 != 0) - max_ext_n = fls(xsave_mask >> 32) + 32; - else - max_ext_n = fls(xsave_mask); - for (i = 0; i < max_ext_n; i++) { - bit = 1ULL << i; - if ((xsave_mask & bit) == 0 || (*xstate_bv & bit) != 0) - continue; - bcopy((char *)npx_initialstate + - xsave_area_desc[i].offset, - sa + xsave_area_desc[i].offset, - xsave_area_desc[i].size); - *xstate_bv |= bit; - } - } - return (owned); -} - -void -npxuserinited(struct thread *td) -{ - struct pcb *pcb; - - pcb = td->td_pcb; - if (PCB_USER_FPU(pcb)) - pcb->pcb_flags |= PCB_NPXINITDONE; - pcb->pcb_flags |= PCB_NPXUSERINITDONE; -} - -int -npxsetxstate(struct thread *td, char *xfpustate, size_t xfpustate_size) -{ - struct xstate_hdr *hdr, *ehdr; - size_t len, max_len; - uint64_t bv; - - /* XXXKIB should we clear all extended state in xstate_bv instead ? */ - if (xfpustate == NULL) - return (0); - if (!use_xsave) - return (EOPNOTSUPP); - - len = xfpustate_size; - if (len < sizeof(struct xstate_hdr)) - return (EINVAL); - max_len = cpu_max_ext_state_size - sizeof(union savefpu); - if (len > max_len) - return (EINVAL); - - ehdr = (struct xstate_hdr *)xfpustate; - bv = ehdr->xstate_bv; - - /* - * Avoid #gp. - */ - if (bv & ~xsave_mask) - return (EINVAL); - - hdr = (struct xstate_hdr *)(get_pcb_user_save_td(td) + 1); - - hdr->xstate_bv = bv; - bcopy(xfpustate + sizeof(struct xstate_hdr), - (char *)(hdr + 1), len - sizeof(struct xstate_hdr)); - - return (0); -} - -int -npxsetregs(struct thread *td, union savefpu *addr, char *xfpustate, - size_t xfpustate_size) -{ - struct pcb *pcb; - int error; - - if (!hw_float) - return (ENXIO); - - if (cpu_fxsr) - addr->sv_xmm.sv_env.en_mxcsr &= cpu_mxcsr_mask; - pcb = td->td_pcb; - critical_enter(); - if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { - error = npxsetxstate(td, xfpustate, xfpustate_size); - if (error != 0) { - critical_exit(); - return (error); - } - if (!cpu_fxsr) - fnclex(); /* As in npxdrop(). */ - bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); - fpurstor(get_pcb_user_save_td(td)); - critical_exit(); - pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE; - } else { - critical_exit(); - error = npxsetxstate(td, xfpustate, xfpustate_size); - if (error != 0) - return (error); - bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); - npxuserinited(td); - } - return (0); -} - -static void -fpusave(addr) - union savefpu *addr; -{ - - if (use_xsave) - xsave((char *)addr, xsave_mask); - else if (cpu_fxsr) - fxsave(addr); - else - fnsave(addr); -} - -static void -npx_fill_fpregs_xmm1(struct savexmm *sv_xmm, struct save87 *sv_87) -{ - struct env87 *penv_87; - struct envxmm *penv_xmm; - int i; - - penv_87 = &sv_87->sv_env; - penv_xmm = &sv_xmm->sv_env; - - /* FPU control/status */ - penv_87->en_cw = penv_xmm->en_cw; - penv_87->en_sw = penv_xmm->en_sw; - penv_87->en_fip = penv_xmm->en_fip; - penv_87->en_fcs = penv_xmm->en_fcs; - penv_87->en_opcode = penv_xmm->en_opcode; - penv_87->en_foo = penv_xmm->en_foo; - penv_87->en_fos = penv_xmm->en_fos; - - /* FPU registers and tags */ - penv_87->en_tw = 0xffff; - for (i = 0; i < 8; ++i) { - sv_87->sv_ac[i] = sv_xmm->sv_fp[i].fp_acc; - if ((penv_xmm->en_tw & (1 << i)) != 0) - /* zero and special are set as valid */ - penv_87->en_tw &= ~(3 << i * 2); - } -} - -void -npx_fill_fpregs_xmm(struct savexmm *sv_xmm, struct save87 *sv_87) -{ - - bzero(sv_87, sizeof(*sv_87)); - npx_fill_fpregs_xmm1(sv_xmm, sv_87); -} - -void -npx_set_fpregs_xmm(struct save87 *sv_87, struct savexmm *sv_xmm) -{ - struct env87 *penv_87; - struct envxmm *penv_xmm; - int i; - - penv_87 = &sv_87->sv_env; - penv_xmm = &sv_xmm->sv_env; - - /* FPU control/status */ - penv_xmm->en_cw = penv_87->en_cw; - penv_xmm->en_sw = penv_87->en_sw; - penv_xmm->en_fip = penv_87->en_fip; - penv_xmm->en_fcs = penv_87->en_fcs; - penv_xmm->en_opcode = penv_87->en_opcode; - penv_xmm->en_foo = penv_87->en_foo; - penv_xmm->en_fos = penv_87->en_fos; - - /* - * FPU registers and tags. - * Abridged / Full translation (values in binary), see FXSAVE spec. - * 0 11 - * 1 00, 01, 10 - */ - penv_xmm->en_tw = 0; - for (i = 0; i < 8; ++i) { - sv_xmm->sv_fp[i].fp_acc = sv_87->sv_ac[i]; - if ((penv_87->en_tw & (3 << i * 2)) != (3 << i * 2)) - penv_xmm->en_tw |= 1 << i; - } -} - -void -npx_get_fsave(void *addr) -{ - struct thread *td; - union savefpu *sv; - - td = curthread; - npxgetregs(td); - sv = get_pcb_user_save_td(td); - if (cpu_fxsr) - npx_fill_fpregs_xmm1(&sv->sv_xmm, addr); - else - bcopy(sv, addr, sizeof(struct env87) + - sizeof(struct fpacc87[8])); -} - -int -npx_set_fsave(void *addr) -{ - union savefpu sv; - int error; - - bzero(&sv, sizeof(sv)); - if (cpu_fxsr) - npx_set_fpregs_xmm(addr, &sv.sv_xmm); - else - bcopy(addr, &sv, sizeof(struct env87) + - sizeof(struct fpacc87[8])); - error = npxsetregs(curthread, &sv, NULL, 0); - return (error); -} - -/* - * On AuthenticAMD processors, the fxrstor instruction does not restore - * the x87's stored last instruction pointer, last data pointer, and last - * opcode values, except in the rare case in which the exception summary - * (ES) bit in the x87 status word is set to 1. - * - * In order to avoid leaking this information across processes, we clean - * these values by performing a dummy load before executing fxrstor(). - */ -static void -fpu_clean_state(void) -{ - static float dummy_variable = 0.0; - u_short status; - - /* - * Clear the ES bit in the x87 status word if it is currently - * set, in order to avoid causing a fault in the upcoming load. - */ - fnstsw(&status); - if (status & 0x80) - fnclex(); - - /* - * Load the dummy variable into the x87 stack. This mangles - * the x87 stack, but we don't care since we're about to call - * fxrstor() anyway. - */ - __asm __volatile("ffree %%st(7); flds %0" : : "m" (dummy_variable)); -} - -static void -fpurstor(union savefpu *addr) -{ - - if (use_xsave) - xrstor((char *)addr, xsave_mask); - else if (cpu_fxsr) - fxrstor(addr); - else - frstor(addr); -} - -#ifdef DEV_ISA -/* - * This sucks up the legacy ISA support assignments from PNPBIOS/ACPI. - */ -static struct isa_pnp_id npxisa_ids[] = { - { 0x040cd041, "Legacy ISA coprocessor support" }, /* PNP0C04 */ - { 0 } -}; - -static int -npxisa_probe(device_t dev) -{ - int result; - if ((result = ISA_PNP_PROBE(device_get_parent(dev), dev, npxisa_ids)) <= 0) { - device_quiet(dev); - } - return(result); -} - -static int -npxisa_attach(device_t dev) -{ - return (0); -} - -static device_method_t npxisa_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, npxisa_probe), - DEVMETHOD(device_attach, npxisa_attach), - DEVMETHOD(device_detach, bus_generic_detach), - DEVMETHOD(device_shutdown, bus_generic_shutdown), - DEVMETHOD(device_suspend, bus_generic_suspend), - DEVMETHOD(device_resume, bus_generic_resume), - - { 0, 0 } -}; - -static driver_t npxisa_driver = { - "npxisa", - npxisa_methods, - 1, /* no softc */ -}; - -static devclass_t npxisa_devclass; - -DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0); -DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0); -#endif /* DEV_ISA */ - -static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", - "Kernel contexts for FPU state"); - -#define FPU_KERN_CTX_NPXINITDONE 0x01 -#define FPU_KERN_CTX_DUMMY 0x02 -#define FPU_KERN_CTX_INUSE 0x04 - -struct fpu_kern_ctx { - union savefpu *prev; - uint32_t flags; - char hwstate1[]; -}; - -struct fpu_kern_ctx * -fpu_kern_alloc_ctx(u_int flags) -{ - struct fpu_kern_ctx *res; - size_t sz; - - sz = sizeof(struct fpu_kern_ctx) + XSAVE_AREA_ALIGN + - cpu_max_ext_state_size; - res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? - M_NOWAIT : M_WAITOK) | M_ZERO); - return (res); -} - -void -fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) -{ - - KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx")); - /* XXXKIB clear the memory ? */ - free(ctx, M_FPUKERN_CTX); -} - -static union savefpu * -fpu_kern_ctx_savefpu(struct fpu_kern_ctx *ctx) -{ - vm_offset_t p; - - p = (vm_offset_t)&ctx->hwstate1; - p = roundup2(p, XSAVE_AREA_ALIGN); - return ((union savefpu *)p); -} - -int -fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) -{ - struct pcb *pcb; - - KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("using inuse ctx")); - - if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { - ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; - return (0); - } - pcb = td->td_pcb; - KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == - get_pcb_user_save_pcb(pcb), ("mangled pcb_save")); - ctx->flags = FPU_KERN_CTX_INUSE; - if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0) - ctx->flags |= FPU_KERN_CTX_NPXINITDONE; - npxexit(td); - ctx->prev = pcb->pcb_save; - pcb->pcb_save = fpu_kern_ctx_savefpu(ctx); - pcb->pcb_flags |= PCB_KERNNPX; - pcb->pcb_flags &= ~PCB_NPXINITDONE; - return (0); -} - -int -fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) -{ - struct pcb *pcb; - - KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, - ("leaving not inuse ctx")); - ctx->flags &= ~FPU_KERN_CTX_INUSE; - - if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) - return (0); - pcb = td->td_pcb; - critical_enter(); - if (curthread == PCPU_GET(fpcurthread)) - npxdrop(); - critical_exit(); - pcb->pcb_save = ctx->prev; - if (pcb->pcb_save == get_pcb_user_save_pcb(pcb)) { - if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0) - pcb->pcb_flags |= PCB_NPXINITDONE; - else - pcb->pcb_flags &= ~PCB_NPXINITDONE; - pcb->pcb_flags &= ~PCB_KERNNPX; - } else { - if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0) - pcb->pcb_flags |= PCB_NPXINITDONE; - else - pcb->pcb_flags &= ~PCB_NPXINITDONE; - KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); - } - return (0); -} - -int -fpu_kern_thread(u_int flags) -{ - - KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, - ("Only kthread may use fpu_kern_thread")); - KASSERT(curpcb->pcb_save == get_pcb_user_save_pcb(curpcb), - ("mangled pcb_save")); - KASSERT(PCB_USER_FPU(curpcb), ("recursive call")); - - curpcb->pcb_flags |= PCB_KERNNPX; - return (0); -} - -int -is_fpu_kern_thread(u_int flags) -{ - - if ((curthread->td_pflags & TDP_KTHREAD) == 0) - return (0); - return ((curpcb->pcb_flags & PCB_KERNNPX) != 0); -} - -/* - * FPU save area alloc/free/init utility routines - */ -union savefpu * -fpu_save_area_alloc(void) -{ - - return (uma_zalloc(fpu_save_area_zone, 0)); -} - -void -fpu_save_area_free(union savefpu *fsa) -{ - - uma_zfree(fpu_save_area_zone, fsa); -} - -void -fpu_save_area_reset(union savefpu *fsa) -{ - - bcopy(npx_initialstate, fsa, cpu_max_ext_state_size); -} Property changes on: head/sys/i386/isa/npx.c ___________________________________________________________________ Deleted: svn:keywords ## -1 +0,0 ## -FreeBSD=%H \ No newline at end of property