Changeset View
Changeset View
Standalone View
Standalone View
head/sys/riscv/riscv/mp_machdep.c
Show First 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | |||||
static device_probe_t riscv64_cpu_probe; | static device_probe_t riscv64_cpu_probe; | ||||
static device_attach_t riscv64_cpu_attach; | static device_attach_t riscv64_cpu_attach; | ||||
static int ipi_handler(void *); | static int ipi_handler(void *); | ||||
struct mtx ap_boot_mtx; | struct mtx ap_boot_mtx; | ||||
struct pcb stoppcbs[MAXCPU]; | struct pcb stoppcbs[MAXCPU]; | ||||
extern uint32_t boot_hart; | |||||
extern cpuset_t all_harts; | |||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
static uint32_t cpu_reg[MAXCPU][2]; | static uint32_t cpu_reg[MAXCPU][2]; | ||||
#endif | #endif | ||||
static device_t cpu_list[MAXCPU]; | static device_t cpu_list[MAXCPU]; | ||||
void mpentry(unsigned long cpuid); | void mpentry(unsigned long cpuid); | ||||
void init_secondary(uint64_t); | void init_secondary(uint64_t); | ||||
uint8_t secondary_stacks[MAXCPU - 1][PAGE_SIZE * KSTACK_PAGES] __aligned(16); | uint8_t secondary_stacks[MAXCPU][PAGE_SIZE * KSTACK_PAGES] __aligned(16); | ||||
/* Set to 1 once we're ready to let the APs out of the pen. */ | /* Set to 1 once we're ready to let the APs out of the pen. */ | ||||
volatile int aps_ready = 0; | volatile int aps_ready = 0; | ||||
/* Temporary variables for init_secondary() */ | /* Temporary variables for init_secondary() */ | ||||
void *dpcpu[MAXCPU - 1]; | void *dpcpu[MAXCPU - 1]; | ||||
static device_method_t riscv64_cpu_methods[] = { | static device_method_t riscv64_cpu_methods[] = { | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | riscv64_cpu_attach(device_t dev) | ||||
cpu_list[cpuid] = dev; | cpu_list[cpuid] = dev; | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
release_aps(void *dummy __unused) | release_aps(void *dummy __unused) | ||||
{ | { | ||||
u_long mask; | cpuset_t mask; | ||||
int cpu, i; | int cpu, i; | ||||
if (mp_ncpus == 1) | if (mp_ncpus == 1) | ||||
return; | return; | ||||
/* Setup the IPI handler */ | /* Setup the IPI handler */ | ||||
riscv_setup_ipihandler(ipi_handler); | riscv_setup_ipihandler(ipi_handler); | ||||
atomic_store_rel_int(&aps_ready, 1); | atomic_store_rel_int(&aps_ready, 1); | ||||
/* Wake up the other CPUs */ | /* Wake up the other CPUs */ | ||||
mask = 0; | mask = all_harts; | ||||
CPU_CLR(boot_hart, &mask); | |||||
for (i = 1; i < mp_ncpus; i++) | |||||
mask |= (1 << i); | |||||
sbi_send_ipi(&mask); | |||||
printf("Release APs\n"); | printf("Release APs\n"); | ||||
sbi_send_ipi(mask.__bits); | |||||
for (i = 0; i < 2000; i++) { | for (i = 0; i < 2000; i++) { | ||||
if (smp_started) { | if (smp_started) { | ||||
for (cpu = 0; cpu <= mp_maxid; cpu++) { | for (cpu = 0; cpu <= mp_maxid; cpu++) { | ||||
if (CPU_ABSENT(cpu)) | if (CPU_ABSENT(cpu)) | ||||
continue; | continue; | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
DELAY(1000); | DELAY(1000); | ||||
} | } | ||||
printf("APs not started\n"); | printf("APs not started\n"); | ||||
} | } | ||||
SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); | SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); | ||||
void | void | ||||
init_secondary(uint64_t cpu) | init_secondary(uint64_t hart) | ||||
{ | { | ||||
struct pcpu *pcpup; | struct pcpu *pcpup; | ||||
u_int cpuid; | |||||
/* Renumber this cpu */ | |||||
cpuid = hart; | |||||
if (cpuid < boot_hart) | |||||
cpuid += mp_maxid + 1; | |||||
cpuid -= boot_hart; | |||||
/* Setup the pcpu pointer */ | /* Setup the pcpu pointer */ | ||||
pcpup = &__pcpu[cpu]; | pcpup = &__pcpu[cpuid]; | ||||
__asm __volatile("mv gp, %0" :: "r"(pcpup)); | __asm __volatile("mv gp, %0" :: "r"(pcpup)); | ||||
/* Workaround: make sure wfi doesn't halt the hart */ | /* Workaround: make sure wfi doesn't halt the hart */ | ||||
csr_set(sie, SIE_SSIE); | csr_set(sie, SIE_SSIE); | ||||
csr_set(sip, SIE_SSIE); | csr_set(sip, SIE_SSIE); | ||||
/* Spin until the BSP releases the APs */ | /* Spin until the BSP releases the APs */ | ||||
while (!aps_ready) | while (!aps_ready) | ||||
▲ Show 20 Lines • Show All 125 Lines • ▼ Show 20 Lines | cpu_mp_probe(void) | ||||
return (mp_ncpus > 1); | return (mp_ncpus > 1); | ||||
} | } | ||||
#ifdef FDT | #ifdef FDT | ||||
static boolean_t | static boolean_t | ||||
cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) | cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) | ||||
{ | { | ||||
uint64_t target_cpu; | |||||
struct pcpu *pcpup; | struct pcpu *pcpup; | ||||
uint64_t hart; | |||||
u_int cpuid; | |||||
/* Check we are able to start this cpu */ | /* Check if this hart supports MMU. */ | ||||
if (id > mp_maxid) | if (OF_getproplen(node, "mmu-type") < 0) | ||||
return (0); | return (0); | ||||
KASSERT(id < MAXCPU, ("Too many CPUs")); | KASSERT(id < MAXCPU, ("Too many CPUs")); | ||||
KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); | KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
cpu_reg[id][0] = reg[0]; | cpu_reg[id][0] = reg[0]; | ||||
if (addr_size == 2) | if (addr_size == 2) | ||||
cpu_reg[id][1] = reg[1]; | cpu_reg[id][1] = reg[1]; | ||||
#endif | #endif | ||||
target_cpu = reg[0]; | hart = reg[0]; | ||||
if (addr_size == 2) { | if (addr_size == 2) { | ||||
target_cpu <<= 32; | hart <<= 32; | ||||
target_cpu |= reg[1]; | hart |= reg[1]; | ||||
} | } | ||||
pcpup = &__pcpu[id]; | KASSERT(hart < MAXCPU, ("Too many harts.")); | ||||
/* We are already running on cpu 0 */ | /* We are already running on this cpu */ | ||||
if (id == 0) { | if (hart == boot_hart) | ||||
return (1); | return (1); | ||||
} | |||||
pcpu_init(pcpup, id, sizeof(struct pcpu)); | /* | ||||
* Rotate the CPU IDs to put the boot CPU as CPU 0. | |||||
* We keep the other CPUs ordered. | |||||
*/ | |||||
cpuid = hart; | |||||
if (cpuid < boot_hart) | |||||
cpuid += mp_maxid + 1; | |||||
cpuid -= boot_hart; | |||||
dpcpu[id - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); | /* Check if we are able to start this cpu */ | ||||
dpcpu_init(dpcpu[id - 1], id); | if (cpuid > mp_maxid) | ||||
return (0); | |||||
printf("Starting CPU %u (%lx)\n", id, target_cpu); | pcpup = &__pcpu[cpuid]; | ||||
__riscv_boot_ap[id] = 1; | pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); | ||||
pcpup->pc_hart = hart; | |||||
CPU_SET(id, &all_cpus); | dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); | ||||
dpcpu_init(dpcpu[cpuid - 1], cpuid); | |||||
printf("Starting CPU %u (hart %lx)\n", cpuid, hart); | |||||
__riscv_boot_ap[hart] = 1; | |||||
CPU_SET(cpuid, &all_cpus); | |||||
CPU_SET(hart, &all_harts); | |||||
return (1); | return (1); | ||||
} | } | ||||
#endif | #endif | ||||
/* Initialize and fire up non-boot processors */ | /* Initialize and fire up non-boot processors */ | ||||
void | void | ||||
cpu_mp_start(void) | cpu_mp_start(void) | ||||
{ | { | ||||
mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); | mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); | ||||
CPU_SET(0, &all_cpus); | CPU_SET(0, &all_cpus); | ||||
CPU_SET(boot_hart, &all_harts); | |||||
switch(cpu_enum_method) { | switch(cpu_enum_method) { | ||||
#ifdef FDT | #ifdef FDT | ||||
case CPUS_FDT: | case CPUS_FDT: | ||||
ofw_cpu_early_foreach(cpu_init_fdt, true); | ofw_cpu_early_foreach(cpu_init_fdt, true); | ||||
break; | break; | ||||
#endif | #endif | ||||
case CPUS_UNKNOWN: | case CPUS_UNKNOWN: | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
/* Introduce rest of cores to the world */ | /* Introduce rest of cores to the world */ | ||||
void | void | ||||
cpu_mp_announce(void) | cpu_mp_announce(void) | ||||
{ | { | ||||
} | } | ||||
static boolean_t | |||||
cpu_check_mmu(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) | |||||
{ | |||||
/* Check if this hart supports MMU. */ | |||||
if (OF_getproplen(node, "mmu-type") < 0) | |||||
return (0); | |||||
return (1); | |||||
} | |||||
void | void | ||||
cpu_mp_setmaxid(void) | cpu_mp_setmaxid(void) | ||||
{ | { | ||||
#ifdef FDT | #ifdef FDT | ||||
int cores; | int cores; | ||||
cores = ofw_cpu_early_foreach(NULL, false); | cores = ofw_cpu_early_foreach(cpu_check_mmu, true); | ||||
if (cores > 0) { | if (cores > 0) { | ||||
cores = MIN(cores, MAXCPU); | cores = MIN(cores, MAXCPU); | ||||
if (bootverbose) | if (bootverbose) | ||||
printf("Found %d CPUs in the device tree\n", cores); | printf("Found %d CPUs in the device tree\n", cores); | ||||
mp_ncpus = cores; | mp_ncpus = cores; | ||||
mp_maxid = cores - 1; | mp_maxid = cores - 1; | ||||
cpu_enum_method = CPUS_FDT; | cpu_enum_method = CPUS_FDT; | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
if (bootverbose) | if (bootverbose) | ||||
printf("No CPU data, limiting to 1 core\n"); | printf("No CPU data, limiting to 1 core\n"); | ||||
mp_ncpus = 1; | mp_ncpus = 1; | ||||
mp_maxid = 0; | mp_maxid = 0; | ||||
} | } |