Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm/include/cpu-v6.h
Show All 39 Lines | |||||
#include <machine/sysreg.h> | #include <machine/sysreg.h> | ||||
#if __ARM_ARCH < 6 | #if __ARM_ARCH < 6 | ||||
#error Only include this file for ARMv6 | #error Only include this file for ARMv6 | ||||
#else | #else | ||||
#define CPU_ASID_KERNEL 0 | #define CPU_ASID_KERNEL 0 | ||||
#if __ARM_ARCH >= 7 | |||||
#if !defined(SMP) | |||||
/* No SMP so no need to use the MP extensions */ | |||||
#define ARM_USE_MP_EXTENSIONS 0 | |||||
#elif defined(CPU_CORTEXA8) && \ | |||||
(defined(CPU_CORTEXA_MP) || defined(CPU_KRAIT) || defined(CPU_MV_PJ4B)) | |||||
#define ARM_USE_MP_EXTENSIONS (cpuinfo.mp_ext != 0) | |||||
#elif defined(CPU_CORTEXA8) | |||||
#define ARM_USE_MP_EXTENSIONS 0 | |||||
#else | |||||
#define ARM_USE_MP_EXTENSIONS 1 | |||||
#endif | |||||
#endif /* __ARM_ARCH >= 7 */ | |||||
void dcache_wbinv_poc_all(void); /* !!! NOT SMP coherent function !!! */ | void dcache_wbinv_poc_all(void); /* !!! NOT SMP coherent function !!! */ | ||||
vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t); | vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t); | ||||
vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t); | vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t); | ||||
#ifdef DEV_PMU | #ifdef DEV_PMU | ||||
#include <sys/pcpu.h> | #include <sys/pcpu.h> | ||||
#define PMU_OVSR_C 0x80000000 /* Cycle Counter */ | #define PMU_OVSR_C 0x80000000 /* Cycle Counter */ | ||||
extern uint32_t ccnt_hi[MAXCPU]; | extern uint32_t ccnt_hi[MAXCPU]; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Raw CP15 maintenance operations | * Raw CP15 maintenance operations | ||||
* !!! not for external use !!! | * !!! not for external use !!! | ||||
*/ | */ | ||||
/* TLB */ | /* TLB */ | ||||
_WF0(_CP15_TLBIALL, CP15_TLBIALL) /* Invalidate entire unified TLB */ | _WF0(_CP15_TLBIALL, CP15_TLBIALL) /* Invalidate entire unified TLB */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
_WF0(_CP15_TLBIALLIS, CP15_TLBIALLIS) /* Invalidate entire unified TLB IS */ | _WF0(_CP15_TLBIALLIS, CP15_TLBIALLIS) /* Invalidate entire unified TLB IS */ | ||||
#endif | #endif | ||||
_WF1(_CP15_TLBIASID, CP15_TLBIASID(%0)) /* Invalidate unified TLB by ASID */ | _WF1(_CP15_TLBIASID, CP15_TLBIASID(%0)) /* Invalidate unified TLB by ASID */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
_WF1(_CP15_TLBIASIDIS, CP15_TLBIASIDIS(%0)) /* Invalidate unified TLB by ASID IS */ | _WF1(_CP15_TLBIASIDIS, CP15_TLBIASIDIS(%0)) /* Invalidate unified TLB by ASID IS */ | ||||
#endif | #endif | ||||
_WF1(_CP15_TLBIMVAA, CP15_TLBIMVAA(%0)) /* Invalidate unified TLB by MVA, all ASID */ | _WF1(_CP15_TLBIMVAA, CP15_TLBIMVAA(%0)) /* Invalidate unified TLB by MVA, all ASID */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
_WF1(_CP15_TLBIMVAAIS, CP15_TLBIMVAAIS(%0)) /* Invalidate unified TLB by MVA, all ASID IS */ | _WF1(_CP15_TLBIMVAAIS, CP15_TLBIMVAAIS(%0)) /* Invalidate unified TLB by MVA, all ASID IS */ | ||||
#endif | #endif | ||||
_WF1(_CP15_TLBIMVA, CP15_TLBIMVA(%0)) /* Invalidate unified TLB by MVA */ | _WF1(_CP15_TLBIMVA, CP15_TLBIMVA(%0)) /* Invalidate unified TLB by MVA */ | ||||
_WF1(_CP15_TTB_SET, CP15_TTBR0(%0)) | _WF1(_CP15_TTB_SET, CP15_TTBR0(%0)) | ||||
/* Cache and Branch predictor */ | /* Cache and Branch predictor */ | ||||
_WF0(_CP15_BPIALL, CP15_BPIALL) /* Branch predictor invalidate all */ | _WF0(_CP15_BPIALL, CP15_BPIALL) /* Branch predictor invalidate all */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
_WF0(_CP15_BPIALLIS, CP15_BPIALLIS) /* Branch predictor invalidate all IS */ | _WF0(_CP15_BPIALLIS, CP15_BPIALLIS) /* Branch predictor invalidate all IS */ | ||||
#endif | #endif | ||||
_WF1(_CP15_BPIMVA, CP15_BPIMVA(%0)) /* Branch predictor invalidate by MVA */ | _WF1(_CP15_BPIMVA, CP15_BPIMVA(%0)) /* Branch predictor invalidate by MVA */ | ||||
_WF1(_CP15_DCCIMVAC, CP15_DCCIMVAC(%0)) /* Data cache clean and invalidate by MVA PoC */ | _WF1(_CP15_DCCIMVAC, CP15_DCCIMVAC(%0)) /* Data cache clean and invalidate by MVA PoC */ | ||||
_WF1(_CP15_DCCISW, CP15_DCCISW(%0)) /* Data cache clean and invalidate by set/way */ | _WF1(_CP15_DCCISW, CP15_DCCISW(%0)) /* Data cache clean and invalidate by set/way */ | ||||
_WF1(_CP15_DCCMVAC, CP15_DCCMVAC(%0)) /* Data cache clean by MVA PoC */ | _WF1(_CP15_DCCMVAC, CP15_DCCMVAC(%0)) /* Data cache clean by MVA PoC */ | ||||
#if __ARM_ARCH >= 7 | #if __ARM_ARCH >= 7 | ||||
_WF1(_CP15_DCCMVAU, CP15_DCCMVAU(%0)) /* Data cache clean by MVA PoU */ | _WF1(_CP15_DCCMVAU, CP15_DCCMVAU(%0)) /* Data cache clean by MVA PoU */ | ||||
#endif | #endif | ||||
_WF1(_CP15_DCCSW, CP15_DCCSW(%0)) /* Data cache clean by set/way */ | _WF1(_CP15_DCCSW, CP15_DCCSW(%0)) /* Data cache clean by set/way */ | ||||
_WF1(_CP15_DCIMVAC, CP15_DCIMVAC(%0)) /* Data cache invalidate by MVA PoC */ | _WF1(_CP15_DCIMVAC, CP15_DCIMVAC(%0)) /* Data cache invalidate by MVA PoC */ | ||||
_WF1(_CP15_DCISW, CP15_DCISW(%0)) /* Data cache invalidate by set/way */ | _WF1(_CP15_DCISW, CP15_DCISW(%0)) /* Data cache invalidate by set/way */ | ||||
_WF0(_CP15_ICIALLU, CP15_ICIALLU) /* Instruction cache invalidate all PoU */ | _WF0(_CP15_ICIALLU, CP15_ICIALLU) /* Instruction cache invalidate all PoU */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
_WF0(_CP15_ICIALLUIS, CP15_ICIALLUIS) /* Instruction cache invalidate all PoU IS */ | _WF0(_CP15_ICIALLUIS, CP15_ICIALLUIS) /* Instruction cache invalidate all PoU IS */ | ||||
#endif | #endif | ||||
_WF1(_CP15_ICIMVAU, CP15_ICIMVAU(%0)) /* Instruction cache invalidate */ | _WF1(_CP15_ICIMVAU, CP15_ICIMVAU(%0)) /* Instruction cache invalidate */ | ||||
/* | /* | ||||
* Publicly accessible functions | * Publicly accessible functions | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 185 Lines • ▼ Show 20 Lines | tlb_flush_range_local(vm_offset_t va, vm_size_t size) | ||||
dsb(); | dsb(); | ||||
for (; va < eva; va += PAGE_SIZE) | for (; va < eva; va += PAGE_SIZE) | ||||
_CP15_TLBIMVA(va | CPU_ASID_KERNEL); | _CP15_TLBIMVA(va | CPU_ASID_KERNEL); | ||||
dsb(); | dsb(); | ||||
} | } | ||||
/* Broadcasting operations. */ | /* Broadcasting operations. */ | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
#if defined(CPU_CORTEXA8) | |||||
#define ARM_HAVE_MP_EXTENSIONS (cpuinfo.mp_ext != 0) | |||||
#else | |||||
#define ARM_HAVE_MP_EXTENSIONS 1 | |||||
#endif | |||||
static __inline void | static __inline void | ||||
tlb_flush_all(void) | tlb_flush_all(void) | ||||
{ | { | ||||
dsb(); | dsb(); | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_TLBIALLIS(); | _CP15_TLBIALLIS(); | ||||
else | else | ||||
_CP15_TLBIALL(); | _CP15_TLBIALL(); | ||||
dsb(); | dsb(); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
tlb_flush_all_ng(void) | tlb_flush_all_ng(void) | ||||
{ | { | ||||
dsb(); | dsb(); | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_TLBIASIDIS(CPU_ASID_KERNEL); | _CP15_TLBIASIDIS(CPU_ASID_KERNEL); | ||||
else | else | ||||
_CP15_TLBIASID(CPU_ASID_KERNEL); | _CP15_TLBIASID(CPU_ASID_KERNEL); | ||||
dsb(); | dsb(); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
tlb_flush(vm_offset_t va) | tlb_flush(vm_offset_t va) | ||||
{ | { | ||||
KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); | KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); | ||||
dsb(); | dsb(); | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_TLBIMVAAIS(va); | _CP15_TLBIMVAAIS(va); | ||||
else | else | ||||
_CP15_TLBIMVA(va | CPU_ASID_KERNEL); | _CP15_TLBIMVA(va | CPU_ASID_KERNEL); | ||||
dsb(); | dsb(); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
tlb_flush_range(vm_offset_t va, vm_size_t size) | tlb_flush_range(vm_offset_t va, vm_size_t size) | ||||
{ | { | ||||
vm_offset_t eva = va + size; | vm_offset_t eva = va + size; | ||||
KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); | KASSERT((va & PAGE_MASK) == 0, ("%s: va %#x not aligned", __func__, va)); | ||||
KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, | KASSERT((size & PAGE_MASK) == 0, ("%s: size %#x not aligned", __func__, | ||||
size)); | size)); | ||||
dsb(); | dsb(); | ||||
if (ARM_HAVE_MP_EXTENSIONS) { | if (ARM_USE_MP_EXTENSIONS) { | ||||
for (; va < eva; va += PAGE_SIZE) | for (; va < eva; va += PAGE_SIZE) | ||||
_CP15_TLBIMVAAIS(va); | _CP15_TLBIMVAAIS(va); | ||||
} else { | } else { | ||||
for (; va < eva; va += PAGE_SIZE) | for (; va < eva; va += PAGE_SIZE) | ||||
_CP15_TLBIMVA(va | CPU_ASID_KERNEL); | _CP15_TLBIMVA(va | CPU_ASID_KERNEL); | ||||
} | } | ||||
dsb(); | dsb(); | ||||
} | } | ||||
#else /* SMP */ | #else /* __ARM_ARCH < 7 */ | ||||
#define tlb_flush_all() tlb_flush_all_local() | #define tlb_flush_all() tlb_flush_all_local() | ||||
#define tlb_flush_all_ng() tlb_flush_all_ng_local() | #define tlb_flush_all_ng() tlb_flush_all_ng_local() | ||||
#define tlb_flush(va) tlb_flush_local(va) | #define tlb_flush(va) tlb_flush_local(va) | ||||
#define tlb_flush_range(va, size) tlb_flush_range_local(va, size) | #define tlb_flush_range(va, size) tlb_flush_range_local(va, size) | ||||
#endif /* SMP */ | #endif /* __ARM_ARCH < 7 */ | ||||
/* | /* | ||||
* Cache maintenance operations. | * Cache maintenance operations. | ||||
*/ | */ | ||||
/* Sync I and D caches to PoU */ | /* Sync I and D caches to PoU */ | ||||
static __inline void | static __inline void | ||||
icache_sync(vm_offset_t va, vm_size_t size) | icache_sync(vm_offset_t va, vm_size_t size) | ||||
{ | { | ||||
vm_offset_t eva = va + size; | vm_offset_t eva = va + size; | ||||
dsb(); | dsb(); | ||||
va &= ~cpuinfo.dcache_line_mask; | va &= ~cpuinfo.dcache_line_mask; | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
if (ARM_HAVE_MP_EXTENSIONS) { | if (ARM_USE_MP_EXTENSIONS) { | ||||
for ( ; va < eva; va += cpuinfo.dcache_line_size) | for ( ; va < eva; va += cpuinfo.dcache_line_size) | ||||
_CP15_DCCMVAU(va); | _CP15_DCCMVAU(va); | ||||
} else | } else | ||||
#endif | #endif | ||||
{ | { | ||||
for ( ; va < eva; va += cpuinfo.dcache_line_size) | for ( ; va < eva; va += cpuinfo.dcache_line_size) | ||||
_CP15_DCCMVAC(va); | _CP15_DCCMVAC(va); | ||||
} | } | ||||
dsb(); | dsb(); | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_ICIALLUIS(); | _CP15_ICIALLUIS(); | ||||
else | else | ||||
#endif | #endif | ||||
_CP15_ICIALLU(); | _CP15_ICIALLU(); | ||||
dsb(); | dsb(); | ||||
isb(); | isb(); | ||||
} | } | ||||
/* Invalidate I cache */ | /* Invalidate I cache */ | ||||
static __inline void | static __inline void | ||||
icache_inv_all(void) | icache_inv_all(void) | ||||
{ | { | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_ICIALLUIS(); | _CP15_ICIALLUIS(); | ||||
else | else | ||||
#endif | #endif | ||||
_CP15_ICIALLU(); | _CP15_ICIALLU(); | ||||
dsb(); | dsb(); | ||||
isb(); | isb(); | ||||
} | } | ||||
/* Invalidate branch predictor buffer */ | /* Invalidate branch predictor buffer */ | ||||
static __inline void | static __inline void | ||||
bpb_inv_all(void) | bpb_inv_all(void) | ||||
{ | { | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
if (ARM_HAVE_MP_EXTENSIONS) | if (ARM_USE_MP_EXTENSIONS) | ||||
_CP15_BPIALLIS(); | _CP15_BPIALLIS(); | ||||
else | else | ||||
#endif | #endif | ||||
_CP15_BPIALL(); | _CP15_BPIALL(); | ||||
dsb(); | dsb(); | ||||
isb(); | isb(); | ||||
} | } | ||||
/* Write back D-cache to PoU */ | /* Write back D-cache to PoU */ | ||||
static __inline void | static __inline void | ||||
dcache_wb_pou(vm_offset_t va, vm_size_t size) | dcache_wb_pou(vm_offset_t va, vm_size_t size) | ||||
{ | { | ||||
vm_offset_t eva = va + size; | vm_offset_t eva = va + size; | ||||
dsb(); | dsb(); | ||||
va &= ~cpuinfo.dcache_line_mask; | va &= ~cpuinfo.dcache_line_mask; | ||||
#if __ARM_ARCH >= 7 && defined SMP | #if __ARM_ARCH >= 7 | ||||
if (ARM_HAVE_MP_EXTENSIONS) { | if (ARM_USE_MP_EXTENSIONS) { | ||||
for ( ; va < eva; va += cpuinfo.dcache_line_size) | for ( ; va < eva; va += cpuinfo.dcache_line_size) | ||||
_CP15_DCCMVAU(va); | _CP15_DCCMVAU(va); | ||||
} else | } else | ||||
#endif | #endif | ||||
{ | { | ||||
for ( ; va < eva; va += cpuinfo.dcache_line_size) | for ( ; va < eva; va += cpuinfo.dcache_line_size) | ||||
_CP15_DCCMVAC(va); | _CP15_DCCMVAC(va); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 164 Lines • Show Last 20 Lines |