Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm64/arm64/identcpu.c
Show All 40 Lines | |||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <machine/atomic.h> | #include <machine/atomic.h> | ||||
#include <machine/cpu.h> | #include <machine/cpu.h> | ||||
#include <machine/cpufunc.h> | #include <machine/cpufunc.h> | ||||
#include <machine/undefined.h> | #include <machine/undefined.h> | ||||
#include <machine/elf.h> | #include <machine/elf.h> | ||||
static int ident_lock; | |||||
static void print_cpu_features(u_int cpu); | static void print_cpu_features(u_int cpu); | ||||
static u_long parse_cpu_features_hwcap(u_int cpu); | static u_long parse_cpu_features_hwcap(u_int cpu); | ||||
char machine[] = "arm64"; | char machine[] = "arm64"; | ||||
#ifdef SCTL_MASK32 | #ifdef SCTL_MASK32 | ||||
extern int adaptive_machine_arch; | extern int adaptive_machine_arch; | ||||
#endif | #endif | ||||
static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | ||||
"Cache management tuning"); | "Cache management tuning"); | ||||
static int allow_dic = 1; | static int allow_dic = 1; | ||||
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, | SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, | ||||
"Allow optimizations based on the DIC cache bit"); | "Allow optimizations based on the DIC cache bit"); | ||||
static int allow_idc = 1; | static int allow_idc = 1; | ||||
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, | SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, | ||||
"Allow optimizations based on the IDC cache bit"); | "Allow optimizations based on the IDC cache bit"); | ||||
static void check_cpu_regs(u_int cpu); | |||||
/* | /* | ||||
* The default implementation of I-cache sync assumes we have an | * The default implementation of I-cache sync assumes we have an | ||||
* aliasing cache until we know otherwise. | * aliasing cache until we know otherwise. | ||||
*/ | */ | ||||
void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) = | void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) = | ||||
&arm64_aliasing_icache_sync_range; | &arm64_aliasing_icache_sync_range; | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 980 Lines • ▼ Show 20 Lines | |||||
identify_cpu_sysinit(void *dummy __unused) | identify_cpu_sysinit(void *dummy __unused) | ||||
{ | { | ||||
int cpu; | int cpu; | ||||
u_long hwcap; | u_long hwcap; | ||||
bool dic, idc; | bool dic, idc; | ||||
dic = (allow_dic != 0); | dic = (allow_dic != 0); | ||||
idc = (allow_idc != 0); | idc = (allow_idc != 0); | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
print_cpu_features(cpu); | check_cpu_regs(cpu); | ||||
hwcap = parse_cpu_features_hwcap(cpu); | hwcap = parse_cpu_features_hwcap(cpu); | ||||
if (elf_hwcap == 0) | if (elf_hwcap == 0) | ||||
elf_hwcap = hwcap; | elf_hwcap = hwcap; | ||||
else | else | ||||
elf_hwcap &= hwcap; | elf_hwcap &= hwcap; | ||||
update_special_regs(cpu); | update_special_regs(cpu); | ||||
if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0) | if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0) | ||||
Show All 15 Lines | identify_cpu_sysinit(void *dummy __unused) | ||||
} | } | ||||
#ifdef LSE_ATOMICS | #ifdef LSE_ATOMICS | ||||
if (!lse_supported) | if (!lse_supported) | ||||
panic("CPU does not support LSE atomic instructions"); | panic("CPU does not support LSE atomic instructions"); | ||||
#endif | #endif | ||||
install_undef_handler(true, user_mrs_handler); | install_undef_handler(true, user_mrs_handler); | ||||
} | } | ||||
SYSINIT(identify_cpu, SI_SUB_SMP, SI_ORDER_ANY, identify_cpu_sysinit, NULL); | SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_ANY, identify_cpu_sysinit, NULL); | ||||
static void | |||||
cpu_features_sysinit(void *dummy __unused) | |||||
{ | |||||
u_int cpu; | |||||
CPU_FOREACH(cpu) | |||||
print_cpu_features(cpu); | |||||
} | |||||
SYSINIT(cpu_features, SI_SUB_SMP, SI_ORDER_ANY, cpu_features_sysinit, NULL); | |||||
static u_long | static u_long | ||||
parse_cpu_features_hwcap(u_int cpu) | parse_cpu_features_hwcap(u_int cpu) | ||||
{ | { | ||||
u_long hwcap = 0; | u_long hwcap = 0; | ||||
if (ID_AA64ISAR0_DP_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_DP_IMPL) | if (ID_AA64ISAR0_DP_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_DP_IMPL) | ||||
hwcap |= HWCAP_ASIMDDP; | hwcap |= HWCAP_ASIMDDP; | ||||
▲ Show 20 Lines • Show All 354 Lines • ▼ Show 20 Lines | identify_cpu(void) | ||||
*/ | */ | ||||
PCPU_SET(midr, midr); | PCPU_SET(midr, midr); | ||||
impl_id = CPU_IMPL(midr); | impl_id = CPU_IMPL(midr); | ||||
for (i = 0; i < nitems(cpu_implementers); i++) { | for (i = 0; i < nitems(cpu_implementers); i++) { | ||||
if (impl_id == cpu_implementers[i].impl_id || | if (impl_id == cpu_implementers[i].impl_id || | ||||
cpu_implementers[i].impl_id == 0) { | cpu_implementers[i].impl_id == 0) { | ||||
cpu_desc[cpu].cpu_impl = impl_id; | cpu_desc[cpu].cpu_impl = impl_id; | ||||
cpu_desc[cpu].cpu_impl_name = cpu_implementers[i].impl_name; | cpu_desc[cpu].cpu_impl_name = | ||||
cpu_implementers[i].impl_name; | |||||
cpu_partsp = cpu_implementers[i].cpu_parts; | cpu_partsp = cpu_implementers[i].cpu_parts; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
part_id = CPU_PART(midr); | part_id = CPU_PART(midr); | ||||
for (i = 0; &cpu_partsp[i] != NULL; i++) { | for (i = 0; &cpu_partsp[i] != NULL; i++) { | ||||
if (part_id == cpu_partsp[i].part_id || | if (part_id == cpu_partsp[i].part_id || | ||||
Show All 20 Lines | identify_cpu(void) | ||||
cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1); | cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1); | ||||
cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1); | cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1); | ||||
cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1); | cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1); | ||||
cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1); | cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1); | ||||
cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); | cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); | ||||
cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1); | cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1); | ||||
cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1); | cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1); | ||||
cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1); | cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1); | ||||
} | |||||
if (cpu != 0) { | static void | ||||
/* | check_cpu_regs(u_int cpu) | ||||
* This code must run on one cpu at a time, but we are | { | ||||
* not scheduling on the current core so implement a | |||||
* simple spinlock. | |||||
*/ | |||||
while (atomic_cmpset_acq_int(&ident_lock, 0, 1) == 0) | |||||
__asm __volatile("wfe" ::: "memory"); | |||||
switch (cpu_aff_levels) { | switch (cpu_aff_levels) { | ||||
case 0: | case 0: | ||||
if (CPU_AFF0(cpu_desc[cpu].mpidr) != | if (CPU_AFF0(cpu_desc[cpu].mpidr) != | ||||
CPU_AFF0(cpu_desc[0].mpidr)) | CPU_AFF0(cpu_desc[0].mpidr)) | ||||
cpu_aff_levels = 1; | cpu_aff_levels = 1; | ||||
/* FALLTHROUGH */ | /* FALLTHROUGH */ | ||||
case 1: | case 1: | ||||
if (CPU_AFF1(cpu_desc[cpu].mpidr) != | if (CPU_AFF1(cpu_desc[cpu].mpidr) != | ||||
CPU_AFF1(cpu_desc[0].mpidr)) | CPU_AFF1(cpu_desc[0].mpidr)) | ||||
cpu_aff_levels = 2; | cpu_aff_levels = 2; | ||||
/* FALLTHROUGH */ | /* FALLTHROUGH */ | ||||
case 2: | case 2: | ||||
if (CPU_AFF2(cpu_desc[cpu].mpidr) != | if (CPU_AFF2(cpu_desc[cpu].mpidr) != | ||||
CPU_AFF2(cpu_desc[0].mpidr)) | CPU_AFF2(cpu_desc[0].mpidr)) | ||||
cpu_aff_levels = 3; | cpu_aff_levels = 3; | ||||
/* FALLTHROUGH */ | /* FALLTHROUGH */ | ||||
case 3: | case 3: | ||||
if (CPU_AFF3(cpu_desc[cpu].mpidr) != | if (CPU_AFF3(cpu_desc[cpu].mpidr) != | ||||
CPU_AFF3(cpu_desc[0].mpidr)) | CPU_AFF3(cpu_desc[0].mpidr)) | ||||
cpu_aff_levels = 4; | cpu_aff_levels = 4; | ||||
break; | break; | ||||
} | } | ||||
if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0) | if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0) | ||||
cpu_print_regs |= PRINT_ID_AA64_AFR0; | cpu_print_regs |= PRINT_ID_AA64_AFR0; | ||||
if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1) | if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1) | ||||
cpu_print_regs |= PRINT_ID_AA64_AFR1; | cpu_print_regs |= PRINT_ID_AA64_AFR1; | ||||
if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0) | if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0) | ||||
cpu_print_regs |= PRINT_ID_AA64_DFR0; | cpu_print_regs |= PRINT_ID_AA64_DFR0; | ||||
if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1) | if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1) | ||||
cpu_print_regs |= PRINT_ID_AA64_DFR1; | cpu_print_regs |= PRINT_ID_AA64_DFR1; | ||||
if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0) | if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0) | ||||
cpu_print_regs |= PRINT_ID_AA64_ISAR0; | cpu_print_regs |= PRINT_ID_AA64_ISAR0; | ||||
if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1) | if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1) | ||||
cpu_print_regs |= PRINT_ID_AA64_ISAR1; | cpu_print_regs |= PRINT_ID_AA64_ISAR1; | ||||
if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0) | if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0) | ||||
cpu_print_regs |= PRINT_ID_AA64_MMFR0; | cpu_print_regs |= PRINT_ID_AA64_MMFR0; | ||||
if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1) | if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1) | ||||
cpu_print_regs |= PRINT_ID_AA64_MMFR1; | cpu_print_regs |= PRINT_ID_AA64_MMFR1; | ||||
if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2) | if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2) | ||||
cpu_print_regs |= PRINT_ID_AA64_MMFR2; | cpu_print_regs |= PRINT_ID_AA64_MMFR2; | ||||
if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0) | if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0) | ||||
cpu_print_regs |= PRINT_ID_AA64_PFR0; | cpu_print_regs |= PRINT_ID_AA64_PFR0; | ||||
if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1) | if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1) | ||||
cpu_print_regs |= PRINT_ID_AA64_PFR1; | cpu_print_regs |= PRINT_ID_AA64_PFR1; | ||||
if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) { | if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) { | ||||
/* | /* | ||||
* If the cache type register is different we may | * If the cache type register is different we may | ||||
* have a different l1 cache type. | * have a different l1 cache type. | ||||
*/ | */ | ||||
identify_cache(cpu_desc[cpu].ctr); | identify_cache(cpu_desc[cpu].ctr); | ||||
cpu_print_regs |= PRINT_CTR_EL0; | cpu_print_regs |= PRINT_CTR_EL0; | ||||
} | |||||
/* Wake up the other CPUs */ | |||||
atomic_store_rel_int(&ident_lock, 0); | |||||
__asm __volatile("sev" ::: "memory"); | |||||
} | } | ||||
} | } |