Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -700,7 +700,8 @@ x86/bios/vpd.c optional vpd x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/est.c optional cpufreq -x86/cpufreq/hwpstate.c optional cpufreq +x86/cpufreq/hwpstate_amd.c optional cpufreq +x86/cpufreq/hwpstate_intel.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci Index: sys/conf/files.i386 =================================================================== --- sys/conf/files.i386 +++ sys/conf/files.i386 @@ -572,7 +572,8 @@ x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq -x86/cpufreq/hwpstate.c optional cpufreq +x86/cpufreq/hwpstate_amd.c optional cpufreq +x86/cpufreq/hwpstate_intel.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/smist.c optional cpufreq Index: sys/modules/cpufreq/Makefile =================================================================== --- sys/modules/cpufreq/Makefile +++ sys/modules/cpufreq/Makefile @@ -11,7 +11,7 @@ .PATH: ${SRCTOP}/sys/x86/cpufreq SRCS+= acpi_if.h opt_acpi.h -SRCS+= est.c hwpstate.c p4tcc.c powernow.c +SRCS+= est.c hwpstate_amd.c p4tcc.c powernow.c hwpstate_intel.c .endif .if ${MACHINE} == "i386" Index: sys/x86/cpufreq/est.c =================================================================== --- sys/x86/cpufreq/est.c +++ sys/x86/cpufreq/est.c @@ -50,6 +50,8 @@ #include #include "acpi_if.h" +extern uint32_t intel_speed_shift; + /* Status/control registers (from the IA-32 System Programming Guide). */ #define MSR_PERF_STATUS 0x198 #define MSR_PERF_CTL 0x199 @@ -916,6 +918,10 @@ { device_t child; + /* If the Intel driver is handling this */ + if (intel_speed_shift != 0) + return; + /* Make sure we're not being doubly invoked. */ if (device_find_child(parent, "est", -1) != NULL) return; Index: sys/x86/cpufreq/hwpstate_amd.c =================================================================== --- sys/x86/cpufreq/hwpstate_amd.c +++ sys/x86/cpufreq/hwpstate_amd.c @@ -124,11 +124,11 @@ static int hwpstate_goto_pstate(device_t dev, int pstate_id); static int hwpstate_verbose; -SYSCTL_INT(_debug, OID_AUTO, hwpstate_verbose, CTLFLAG_RWTUN, +SYSCTL_INT(_debug, OID_AUTO, hwpstate_amd_verbose, CTLFLAG_RWTUN, &hwpstate_verbose, 0, "Debug hwpstate"); static int hwpstate_verify; -SYSCTL_INT(_debug, OID_AUTO, hwpstate_verify, CTLFLAG_RWTUN, +SYSCTL_INT(_debug, OID_AUTO, hwpstate_amd_verify, CTLFLAG_RWTUN, &hwpstate_verify, 0, "Verify P-state after setting"); static device_method_t hwpstate_methods[] = { @@ -151,14 +151,14 @@ {0, 0} }; -static devclass_t hwpstate_devclass; -static driver_t hwpstate_driver = { - "hwpstate", +static devclass_t hwpstate_amd_devclass; +static driver_t hwpstate_amd_driver = { + "hwpstate_amd", hwpstate_methods, sizeof(struct hwpstate_softc), }; -DRIVER_MODULE(hwpstate, cpu, hwpstate_driver, hwpstate_devclass, 0, 0); +DRIVER_MODULE(hwpstate_amd, cpu, hwpstate_amd_driver, hwpstate_amd_devclass, 0, 0); /* * Go to Px-state on all cpus considering the limit. @@ -312,7 +312,7 @@ hwpstate_identify(driver_t *driver, device_t parent) { - if (device_find_child(parent, "hwpstate", -1) != NULL) + if (device_find_child(parent, "hwpstate_amd", -1) != NULL) return; if (cpu_vendor_id != CPU_VENDOR_AMD || CPUID_TO_FAMILY(cpu_id) < 0x10) @@ -326,10 +326,10 @@ return; } - if (resource_disabled("hwpstate", 0)) + if (resource_disabled("hwpstate_amd", 0)) return; - if (BUS_ADD_CHILD(parent, 10, "hwpstate", -1) == NULL) + if (BUS_ADD_CHILD(parent, 10, "hwpstate_amd", -1) == NULL) device_printf(parent, "hwpstate: add child failed\n"); } Index: sys/x86/cpufreq/hwpstate_intel.c =================================================================== --- /dev/null +++ sys/x86/cpufreq/hwpstate_intel.c @@ -0,0 +1,244 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018 Intel Corporation + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpufreq_if.h" +#include +#include +#include +#include + +#include + +#include + +#include "acpi_if.h" +#include "cpufreq_if.h" + +uint32_t intel_speed_shift=1; +SYSCTL_UINT(_machdep, OID_AUTO, intel_speed_shift, CTLFLAG_RDTUN, &intel_speed_shift, + 0, "Enable Intel Speed Shift (HWP)"); + +static void intel_hwpstate_identify(driver_t *driver, device_t parent); +static int intel_hwpstate_probe(device_t dev); +static int intel_hwpstate_attach(device_t dev); +static int intel_hwpstate_detach(device_t dev); + +#if 0 +static int intel_hwpstate_get(device_t dev, struct cf_setting *cf); +static int intel_hwpstate_set(device_t dev, const struct cf_setting *cf); +static int intel_hwpstate_settings(device_t dev, struct cf_setting *sets, int *count); +static int intel_hwpstate_type(device_t dev, int *type); +#endif + +static device_method_t intel_hwpstate_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, intel_hwpstate_identify), + DEVMETHOD(device_probe, intel_hwpstate_probe), + DEVMETHOD(device_attach, intel_hwpstate_attach), + DEVMETHOD(device_detach, intel_hwpstate_detach), + +#if 0 + /* cpufreq interface */ + DEVMETHOD(cpufreq_drv_get, intel_hwpstate_get), + DEVMETHOD(cpufreq_drv_set, intel_hwpstate_set), + DEVMETHOD(cpufreq_drv_settings, intel_hwpstate_settings), + DEVMETHOD(cpufreq_drv_type, intel_hwpstate_type), +#endif + {0, 0} +}; + +struct hwp_softc { + device_t dev; + bool hwp_notifications; + bool hwp_activity_window; + bool hwp_pref_ctrl; + bool hwp_pkg_ctrl; +}; + +static devclass_t hwpstate_intel_devclass; +static driver_t hwpstate_intel_driver = { + "hwpstate_intel", + intel_hwpstate_methods, + sizeof(struct hwp_softc), +}; + +/* NB: This must run before the est module!!!! */ +DRIVER_MODULE_ORDERED(hwpstate_intel, cpu, hwpstate_intel_driver, + hwpstate_intel_devclass, 0, 0, SI_ORDER_FIRST); + +static void +intel_hwpstate_identify(driver_t *driver, device_t parent) +{ + uint32_t regs[4]; + + if (intel_speed_shift == 0) + return; + + if (device_find_child(parent, "hwpstate_intel", -1) != NULL) + return; + + if (cpu_vendor_id != CPU_VENDOR_INTEL) + return; + + if (resource_disabled("hwpstate_intel", 0)) + return; + /* + * Intel SDM 14.4.1 (HWP Programming Interfaces): + * The CPUID instruction allows software to discover the presence of + * HWP support in an Intel processor. Specifically, execute CPUID + * instruction with EAX=06H as input will return 5 bit flags covering + * the following aspects in bits 7 through 11 of CPUID.06H:EAX. + */ + if (cpu_high < 6) + goto err; + /* + * Intel SDM 14.4.1 (HWP Programming Interfaces): + * Availability of HWP baseline resource and capability, + * CPUID.06H:EAX[bit 7]: If this bit is set, HWP provides several new + * architectural MSRs: IA32_PM_ENABLE, IA32_HWP_CAPABILITIES, + * IA32_HWP_REQUEST, IA32_HWP_STATUS. + */ + do_cpuid(6, regs); + if ((regs[0] & (1 << 7)) == 0) + goto err; + + if (BUS_ADD_CHILD(parent, 10, "hwpstate_intel", -1) == NULL) + goto err; + + device_printf(parent, "hwpstate registered (0x%x)\n", (regs[0] >> 7) & 0x1f); + return; + +err: + device_printf(parent, "Speed Shift unavailable. Falling back to est\n"); + intel_speed_shift = 0; +} + +static int +intel_hwpstate_probe(device_t dev) +{ + device_t perf_dev; + + perf_dev = device_find_child(device_get_parent(dev), "acpi_perf", -1); + if (perf_dev && device_is_attached(perf_dev)) { + int error, type; + error = CPUFREQ_DRV_TYPE(perf_dev, &type); + if (error == 0) { + if ((type & CPUFREQ_FLAG_INFO_ONLY) != 0) { + /* We don't handle collision with acpi_perf */ + device_printf(dev, + "Passive-aggressively avoiding acpi_perf\n"); + return (ENXIO); + } + } + } + + device_set_desc(dev, "Intel Speed Shift"); + return (BUS_PROBE_DEFAULT); +} + + +static void +set_autonomous_hwp(void) +{ + uint64_t caps, req; + + wrmsr_safe(MSR_IA32_PM_ENABLE, 1); + + /* Set up the defaults from SDM 14.4.10 */ + rdmsr_safe(MSR_IA32_HWP_REQUEST, &req); + rdmsr_safe(MSR_IA32_HWP_CAPABILITIES, &caps); + + + /* hardware autonomous selection determines the performance target */ + req &= ~IA32_HWP_DESIRED_PERFORMANCE; + + /* enable HW dynamic selection of window size */ + req &= ~IA32_HWP_ACTIVITY_WINDOW; + + /* IA32_HWP_REQUEST.Minimum_Performance = IA32_HWP_CAPABILITIES.Lowest_Performance */ + req |= IA32_HWP_CAPABILITIES_LOWEST_PERFORMANCE(caps); + + /* IA32_HWP_REQUEST.Maximum_Performance = IA32_HWP_CAPABILITIES.Highest_Performance. */ + req |= IA32_HWP_CAPABILITIES_HIGHEST_PERFORMANCE(caps) << 8; + + wrmsr_safe(MSR_IA32_HWP_REQUEST, req); +} + +static int +intel_hwpstate_attach(device_t dev) +{ + struct hwp_softc *sc; + uint32_t regs[4]; + int i; + + KASSERT(device_find_child(device_get_parent(dev), "est", -1) == NULL, + ("EST driver already loaded")); + + sc = device_get_softc(dev); + sc->dev = dev; + + do_cpuid(6, regs); + if (regs[0] & (1 << 8)) + sc->hwp_notifications = true; + if (regs[0] & (1 << 9)) + sc->hwp_activity_window = true; + if (regs[0] & (1 << 10)) + sc->hwp_pref_ctrl = true; + if (regs[0] & (1 << 11)) + sc->hwp_pkg_ctrl = true; + + CPU_FOREACH(i) { + thread_lock(curthread); + sched_bind(curthread, i); + thread_unlock(curthread); + + set_autonomous_hwp(); + } + + return (cpufreq_register(dev)); +} + +static int +intel_hwpstate_detach(device_t dev) +{ + return (cpufreq_unregister(dev)); +} Index: sys/x86/include/specialreg.h =================================================================== --- sys/x86/include/specialreg.h +++ sys/x86/include/specialreg.h @@ -539,7 +539,15 @@ #define MSR_DRAM_ENERGY_STATUS 0x619 #define MSR_PP0_ENERGY_STATUS 0x639 #define MSR_PP1_ENERGY_STATUS 0x641 +#define MSR_PPERF 0x64e #define MSR_TSC_DEADLINE 0x6e0 /* Writes are not serializing */ +#define MSR_IA32_PM_ENABLE 0x770 +#define MSR_IA32_HWP_CAPABILITIES 0x771 +#define MSR_IA32_HWP_REQUEST_PKG 0x772 +#define MSR_IA32_HWP_INTERRUPT 0x773 +#define MSR_IA32_HWP_REQUEST 0x774 +#define MSR_IA32_HWP_PECI_REQUEST_INFO 0x775 +#define MSR_IA32_HWP_STATUS 0x777 /* * VMX MSRs @@ -716,6 +724,25 @@ /* MSR IA32_FLUSH_CMD */ #define IA32_FLUSH_CMD_L1D 0x00000001 +/* MSR IA32_HWP_CAPABILITIES */ +#define IA32_HWP_CAPABILITIES_HIGHEST_PERFORMANCE(x) (((x) >> 0) & 0xff) +#define IA32_HWP_CAPABILITIES_GUARANTEED_PERFORMANCE(x) (((x) >> 8) & 0xff) +#define IA32_HWP_CAPABILITIES_EFFICIENT_PERFORMANCE(x) (((x) >> 16) & 0xff) +#define IA32_HWP_CAPABILITIES_LOWEST_PERFORMANCE(x) (((x) >> 24) & 0xff) + +/* MSR IA32_HWP_REQUEST */ +#define IA32_HWP_REQUEST_MINIMUM_VALID (1ULL << 63) +#define IA32_HWP_REQUEST_MAXIMUM_VALID (1ULL << 62) +#define IA32_HWP_REQUEST_DESIRED_VALID (1ULL << 61) +#define IA32_HWP_REQUEST_EPP_VALID (1ULL << 60) +#define IA32_HWP_REQUEST_ACTIVITY_WINDOW_VALID (1ULL << 59) +#define IA32_HWP_REQUEST_PACKAGE_CONTROL (1ULL << 42) +#define IA32_HWP_ACTIVITY_WINDOW (0x3ffULL << 32) +#define IA32_HWP_REQUEST_ENERGY_PERFORMANCE_PREFERENCE (0xffULL << 24) +#define IA32_HWP_DESIRED_PERFORMANCE (0xffULL << 16) +#define IA32_HWP_REQUEST_MAXIMUM_PERFORMANCE (0xffULL << 8) +#define IA32_HWP_MINIMUM_PERFORMANCE (0xffULL << 0) + /* * PAT modes. */