Differential D20225 Diff 57339 head/sys/riscv/riscv/pmap.c

Changeset View

Standalone View

head/sys/riscv/riscv/pmap.c

Show First 20 Lines • Show All 267 Lines • ▼ Show 20 Lines
* Data for the pv entry allocation mechanism		* Data for the pv entry allocation mechanism
*/		*/
static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);		static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
static struct mtx pv_chunks_mutex;		static struct mtx pv_chunks_mutex;
static struct rwlock pv_list_locks[NPV_LIST_LOCKS];		static struct rwlock pv_list_locks[NPV_LIST_LOCKS];
static struct md_page *pv_table;		static struct md_page *pv_table;
static struct md_page pv_dummy;		static struct md_page pv_dummy;

		extern cpuset_t all_harts;

/*		/*
* Internal flags for pmap_enter()'s helper functions.		* Internal flags for pmap_enter()'s helper functions.
*/		*/
#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */		#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */		#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */

static void free_pv_chunk(struct pv_chunk *pc);		static void free_pv_chunk(struct pv_chunk *pc);
static void free_pv_entry(pmap_t pmap, pv_entry_t pv);		static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
▲ Show 20 Lines • Show All 448 Lines • ▼ Show 20 Lines
*/		*/
static void		static void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)		pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{		{
cpuset_t mask;		cpuset_t mask;

sched_pin();		sched_pin();
mask = pmap->pm_active;		mask = pmap->pm_active;
CPU_CLR(PCPU_GET(cpuid), &mask);		CPU_CLR(PCPU_GET(hart), &mask);
fence();		fence();
if (!CPU_EMPTY(&mask) && smp_started)		if (!CPU_EMPTY(&mask) && smp_started)
sbi_remote_sfence_vma(mask.__bits, va, 1);		sbi_remote_sfence_vma(mask.__bits, va, 1);
sfence_vma_page(va);		sfence_vma_page(va);
sched_unpin();		sched_unpin();
}		}

static void		static void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)		pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{		{
cpuset_t mask;		cpuset_t mask;

sched_pin();		sched_pin();
mask = pmap->pm_active;		mask = pmap->pm_active;
CPU_CLR(PCPU_GET(cpuid), &mask);		CPU_CLR(PCPU_GET(hart), &mask);
fence();		fence();
if (!CPU_EMPTY(&mask) && smp_started)		if (!CPU_EMPTY(&mask) && smp_started)
sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1);		sbi_remote_sfence_vma(mask.__bits, sva, eva - sva + 1);

/*		/*
* Might consider a loop of sfence_vma_page() for a small		* Might consider a loop of sfence_vma_page() for a small
* number of pages in the future.		* number of pages in the future.
*/		*/
sfence_vma();		sfence_vma();
sched_unpin();		sched_unpin();
}		}

static void		static void
pmap_invalidate_all(pmap_t pmap)		pmap_invalidate_all(pmap_t pmap)
{		{
cpuset_t mask;		cpuset_t mask;

sched_pin();		sched_pin();
mask = pmap->pm_active;		mask = pmap->pm_active;
CPU_CLR(PCPU_GET(cpuid), &mask);		CPU_CLR(PCPU_GET(hart), &mask);

/*		/*
* XXX: The SBI doc doesn't detail how to specify x0 as the		* XXX: The SBI doc doesn't detail how to specify x0 as the
* address to perform a global fence. BBL currently treats		* address to perform a global fence. BBL currently treats
* all sfence_vma requests as global however.		* all sfence_vma requests as global however.
*/		*/
fence();		fence();
if (!CPU_EMPTY(&mask) && smp_started)		if (!CPU_EMPTY(&mask) && smp_started)
▲ Show 20 Lines • Show All 3,466 Lines • ▼ Show 20 Lines	done:
PMAP_UNLOCK(pmap);		PMAP_UNLOCK(pmap);
return (val);		return (val);
}		}

void		void
pmap_activate_sw(struct thread *td)		pmap_activate_sw(struct thread *td)
{		{
pmap_t oldpmap, pmap;		pmap_t oldpmap, pmap;
u_int cpu;		u_int hart;

oldpmap = PCPU_GET(curpmap);		oldpmap = PCPU_GET(curpmap);
pmap = vmspace_pmap(td->td_proc->p_vmspace);		pmap = vmspace_pmap(td->td_proc->p_vmspace);
if (pmap == oldpmap)		if (pmap == oldpmap)
return;		return;
load_satp(pmap->pm_satp);		load_satp(pmap->pm_satp);

cpu = PCPU_GET(cpuid);		hart = PCPU_GET(hart);
#ifdef SMP		#ifdef SMP
CPU_SET_ATOMIC(cpu, &pmap->pm_active);		CPU_SET_ATOMIC(hart, &pmap->pm_active);
CPU_CLR_ATOMIC(cpu, &oldpmap->pm_active);		CPU_CLR_ATOMIC(hart, &oldpmap->pm_active);
#else		#else
CPU_SET(cpu, &pmap->pm_active);		CPU_SET(hart, &pmap->pm_active);
CPU_CLR(cpu, &oldpmap->pm_active);		CPU_CLR(hart, &oldpmap->pm_active);
#endif		#endif
PCPU_SET(curpmap, pmap);		PCPU_SET(curpmap, pmap);

sfence_vma();		sfence_vma();
}		}

void		void
pmap_activate(struct thread *td)		pmap_activate(struct thread *td)
{		{

critical_enter();		critical_enter();
pmap_activate_sw(td);		pmap_activate_sw(td);
critical_exit();		critical_exit();
}		}

void		void
pmap_activate_boot(pmap_t pmap)		pmap_activate_boot(pmap_t pmap)
{		{
u_int cpu;		u_int hart;

cpu = PCPU_GET(cpuid);		hart = PCPU_GET(hart);
#ifdef SMP		#ifdef SMP
CPU_SET_ATOMIC(cpu, &pmap->pm_active);		CPU_SET_ATOMIC(hart, &pmap->pm_active);
#else		#else
CPU_SET(cpu, &pmap->pm_active);		CPU_SET(hart, &pmap->pm_active);
#endif		#endif
PCPU_SET(curpmap, pmap);		PCPU_SET(curpmap, pmap);
}		}

void		void
pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)		pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz)
{		{
cpuset_t mask;		cpuset_t mask;

/*		/*
* From the RISC-V User-Level ISA V2.2:		* From the RISC-V User-Level ISA V2.2:
*		*
* "To make a store to instruction memory visible to all		* "To make a store to instruction memory visible to all
* RISC-V harts, the writing hart has to execute a data FENCE		* RISC-V harts, the writing hart has to execute a data FENCE
* before requesting that all remote RISC-V harts execute a		* before requesting that all remote RISC-V harts execute a
* FENCE.I."		* FENCE.I."
*/		*/
sched_pin();		sched_pin();
mask = all_cpus;		mask = all_harts;
CPU_CLR(PCPU_GET(cpuid), &mask);		CPU_CLR(PCPU_GET(hart), &mask);
fence();		fence();
if (!CPU_EMPTY(&mask) && smp_started)		if (!CPU_EMPTY(&mask) && smp_started)
sbi_remote_fence_i(mask.__bits);		sbi_remote_fence_i(mask.__bits);
sched_unpin();		sched_unpin();
}		}

/*		/*
* Increase the starting virtual address of the given mapping if a		* Increase the starting virtual address of the given mapping if a
▲ Show 20 Lines • Show All 141 Lines • Show Last 20 Lines