Differential D24158 Diff 69784 sys/riscv/riscv/mp_machdep.c

Changeset View

Standalone View

sys/riscv/riscv/mp_machdep.c

Show First 20 Lines • Show All 81 Lines • ▼ Show 20 Lines
} cpu_enum_method;		} cpu_enum_method;

static device_identify_t riscv64_cpu_identify;		static device_identify_t riscv64_cpu_identify;
static device_probe_t riscv64_cpu_probe;		static device_probe_t riscv64_cpu_probe;
static device_attach_t riscv64_cpu_attach;		static device_attach_t riscv64_cpu_attach;

static int ipi_handler(void *);		static int ipi_handler(void *);

struct mtx ap_boot_mtx;
struct pcb stoppcbs[MAXCPU];		struct pcb stoppcbs[MAXCPU];

extern uint32_t boot_hart;		extern uint32_t boot_hart;
extern cpuset_t all_harts;		extern cpuset_t all_harts;

#ifdef INVARIANTS		#ifdef INVARIANTS
static uint32_t cpu_reg[MAXCPU][2];		static uint32_t cpu_reg[MAXCPU][2];
#endif		#endif
static device_t cpu_list[MAXCPU];		static device_t cpu_list[MAXCPU];

void mpentry(unsigned long cpuid);
void init_secondary(uint64_t);		void init_secondary(uint64_t);

uint8_t secondary_stacks[MAXCPU][PAGE_SIZE * KSTACK_PAGES] __aligned(16);		static struct mtx ap_boot_mtx;

		/* Stacks for AP initialization, discarded once idle threads are started. */
		void *bootstack;
		static void *bootstacks[MAXCPU];

		/* Count of started APs, used to synchronize access to bootstack. */
		static volatile int aps_started;

/* Set to 1 once we're ready to let the APs out of the pen. */		/* Set to 1 once we're ready to let the APs out of the pen. */
volatile int aps_ready = 0;		static volatile int aps_ready;

/* Temporary variables for init_secondary() */		/* Temporary variables for init_secondary() */
void *dpcpu[MAXCPU - 1];		void *dpcpu[MAXCPU - 1];

static device_method_t riscv64_cpu_methods[] = {		static device_method_t riscv64_cpu_methods[] = {
/* Device interface */		/* Device interface */
DEVMETHOD(device_identify, riscv64_cpu_identify),		DEVMETHOD(device_identify, riscv64_cpu_identify),
DEVMETHOD(device_probe, riscv64_cpu_probe),		DEVMETHOD(device_probe, riscv64_cpu_probe),
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines	init_secondary(uint64_t hart)
/* Setup the pcpu pointer */		/* Setup the pcpu pointer */
pcpup = &__pcpu[cpuid];		pcpup = &__pcpu[cpuid];
__asm __volatile("mv tp, %0" :: "r"(pcpup));		__asm __volatile("mv tp, %0" :: "r"(pcpup));

/* Workaround: make sure wfi doesn't halt the hart */		/* Workaround: make sure wfi doesn't halt the hart */
csr_set(sie, SIE_SSIE);		csr_set(sie, SIE_SSIE);
csr_set(sip, SIE_SSIE);		csr_set(sip, SIE_SSIE);

/* Spin until the BSP releases the APs */		/* Signal the BSP and spin until it has released all APs. */
while (!aps_ready)		atomic_add_int(&aps_started, 1);
		while (!atomic_load_int(&aps_ready))
__asm __volatile("wfi");		__asm __volatile("wfi");

/* Initialize curthread */		/* Initialize curthread */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));		KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
pcpup->pc_curthread = pcpup->pc_idlethread;		pcpup->pc_curthread = pcpup->pc_idlethread;
pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb;

/*		/*
* Identify current CPU. This is necessary to setup		* Identify current CPU. This is necessary to setup
* affinity registers and to provide support for		* affinity registers and to provide support for
* runtime chip identification.		* runtime chip identification.
*/		*/
identify_cpu();		identify_cpu();

Show All 17 Lines	#endif

if (smp_cpus == mp_ncpus) {		if (smp_cpus == mp_ncpus) {
/* enable IPI's, tlb shootdown, freezes etc */		/* enable IPI's, tlb shootdown, freezes etc */
atomic_store_rel_int(&smp_started, 1);		atomic_store_rel_int(&smp_started, 1);
}		}

mtx_unlock_spin(&ap_boot_mtx);		mtx_unlock_spin(&ap_boot_mtx);

		/*
		* Assert that smp_after_idle_runnable condition is reasonable.
		*/
		MPASS(PCPU_GET(curpcb) == NULL);

/* Enter the scheduler */		/* Enter the scheduler */
sched_throw(NULL);		sched_throw(NULL);

panic("scheduler returned us to init_secondary");		panic("scheduler returned us to init_secondary");
/* NOTREACHED */		/* NOTREACHED */
}		}

		static void
		smp_after_idle_runnable(void *arg __unused)
		{
		struct pcpu *pc;
		int cpu;

		for (cpu = 1; cpu < mp_ncpus; cpu++) {
		if (bootstacks[cpu] != NULL) {
		pc = pcpu_find(cpu);
		while (atomic_load_ptr(&pc->pc_curpcb) == NULL)
		cpu_spinwait();
		kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE);
		}
		}
		}
		SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
		smp_after_idle_runnable, NULL);

static int		static int
ipi_handler(void *arg)		ipi_handler(void *arg)
{		{
u_int ipi_bitmap;		u_int ipi_bitmap;
u_int cpu, ipi;		u_int cpu, ipi;
int bit;		int bit;

sbi_clear_ipi();		sbi_clear_ipi();
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines

#ifdef FDT		#ifdef FDT
static boolean_t		static boolean_t
cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)		cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
{		{
struct pcpu *pcpup;		struct pcpu *pcpup;
uint64_t hart;		uint64_t hart;
u_int cpuid;		u_int cpuid;
		int naps;

/* Check if this hart supports MMU. */		/* Check if this hart supports MMU. */
if (OF_getproplen(node, "mmu-type") < 0)		if (OF_getproplen(node, "mmu-type") < 0)
return (0);		return (0);

KASSERT(id < MAXCPU, ("Too many CPUs"));		KASSERT(id < MAXCPU, ("Too many CPUs"));

KASSERT(addr_size == 1 \|\| addr_size == 2, ("Invalid register size"));		KASSERT(addr_size == 1 \|\| addr_size == 2, ("Invalid register size"));
Show All 30 Lines	#endif

pcpup = &__pcpu[cpuid];		pcpup = &__pcpu[cpuid];
pcpu_init(pcpup, cpuid, sizeof(struct pcpu));		pcpu_init(pcpup, cpuid, sizeof(struct pcpu));
pcpup->pc_hart = hart;		pcpup->pc_hart = hart;

dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK \| M_ZERO);		dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK \| M_ZERO);
dpcpu_init(dpcpu[cpuid - 1], cpuid);		dpcpu_init(dpcpu[cpuid - 1], cpuid);

		bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK \| M_ZERO);

		naps = atomic_load_int(&aps_started);
		bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE;

printf("Starting CPU %u (hart %lx)\n", cpuid, hart);		printf("Starting CPU %u (hart %lx)\n", cpuid, hart);
__riscv_boot_ap[hart] = 1;		atomic_store_32(&__riscv_boot_ap[hart], 1);

		/* Wait for the AP to switch to its boot stack. */
		while (atomic_load_int(&aps_started) < naps + 1)
		cpu_spinwait();

CPU_SET(cpuid, &all_cpus);		CPU_SET(cpuid, &all_cpus);
CPU_SET(hart, &all_harts);		CPU_SET(hart, &all_harts);

return (1);		return (1);
}		}
#endif		#endif

▲ Show 20 Lines • Show All 61 Lines • Show Last 20 Lines