Index: sys/arm64/arm64/locore.S =================================================================== --- sys/arm64/arm64/locore.S +++ sys/arm64/arm64/locore.S @@ -27,6 +27,8 @@ */ #include "assym.s" +#include "opt_kstack_pages.h" + #include #include #include @@ -43,6 +45,12 @@ #define NORMAL_UNCACHED 1 #define NORMAL_MEM 2 +#ifdef SMP +#define SHAREABILITY ATTR_SH(ATTR_SH_IS) +#else +#define SHAREABILITY 0 +#endif + /* * We assume: * MMU on with an identity map, or off @@ -180,8 +188,7 @@ mp_virtdone: ldr x4, =secondary_stacks mov x5, #(PAGE_SIZE * KSTACK_PAGES) - sub x1, x0, #1 - mul x5, x1, x5 + mul x5, x0, x5 add sp, x4, x5 b init_secondary @@ -318,18 +325,18 @@ */ create_pagetables: /* Save the Link register */ - mov x5, x30 + mov x4, x30 /* Clean the page table */ - adr x6, pagetable - mov x26, x6 + adr x5, pagetable + mov x26, x5 adr x27, pagetable_end 1: - stp xzr, xzr, [x6], #16 - stp xzr, xzr, [x6], #16 - stp xzr, xzr, [x6], #16 - stp xzr, xzr, [x6], #16 - cmp x6, x27 + stp xzr, xzr, [x5], #16 + stp xzr, xzr, [x5], #16 + stp xzr, xzr, [x5], #16 + stp xzr, xzr, [x5], #16 + cmp x5, x27 b.lo 1b /* @@ -337,17 +344,18 @@ */ /* Find the size of the kernel */ - mov x6, #(KERNBASE) - ldr x7, .Lend + mov x5, #(KERNBASE) + ldr x6, .Lend /* Find the end - begin */ - sub x8, x7, x6 + sub x7, x6, x5 /* Get the number of l2 pages to allocate, rounded down */ - lsr x10, x8, #(L2_SHIFT) + lsr x10, x7, #(L2_SHIFT) /* Add 4 MiB for any rounding above and the module data */ add x10, x10, #2 /* Create the kernel space L2 table */ - mov x6, x26 + mov x5, x26 + mov x6, #SHAREABILITY mov x7, #NORMAL_MEM mov x8, #(KERNBASE & L2_BLOCK_MASK) mov x9, x28 @@ -357,8 +365,8 @@ add x26, x26, #PAGE_SIZE /* Link the l1 -> l2 table */ - mov x9, x6 - mov x6, x26 + mov x9, x5 + mov x5, x26 bl link_l1_pagetable @@ -377,19 +385,21 @@ #endif /* Create the VA = PA map */ - mov x6, x27 /* The initial page table */ + mov x5, x27 /* The initial page table */ + mov x6, #0 mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ mov x9, x27 mov x8, x9 /* VA start (== PA start) */ bl build_section_pagetable /* Restore the Link register */ - mov x30, x5 + mov x30, x4 ret /* * Builds a 1 GiB page table entry - * x6 = L1 table + * x5 = L1 table + * x6 = Shareability attrubute * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) @@ -407,13 +417,14 @@ lsl x12, x7, #2 orr x12, x12, #L1_BLOCK orr x12, x12, #(ATTR_AF) + orr x12, x12, x6 /* Only use the output address bits */ lsr x9, x9, #L1_SHIFT orr x12, x12, x9, lsl #L1_SHIFT /* Store the entry */ - str x12, [x6, x11, lsl #3] + str x12, [x5, x11, lsl #3] ret @@ -423,7 +434,7 @@ * This is a link for a 1GiB block of memory with up to 2MiB regions mapped * within it by build_block_pagetable. * - * x6 = L1 table + * x5 = L1 table * x8 = Virtual Address * x9 = L2 PA (trashed) * x11, x12 and x13 are trashed @@ -444,13 +455,14 @@ orr x12, x12, x9, lsl #12 /* Store the entry */ - str x12, [x6, x11, lsl #3] + str x12, [x5, x11, lsl #3] ret /* * Builds count 2 MiB page table entry - * x6 = L2 table + * x5 = L2 table + * x6 = Shareability attribute * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) @@ -469,9 +481,7 @@ lsl x12, x7, #2 orr x12, x12, #L2_BLOCK orr x12, x12, #(ATTR_AF) -#ifdef SMP - orr x12, x12, ATTR_SH(ATTR_SH_IS) -#endif + orr x12, x12, x6 /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT @@ -480,7 +490,7 @@ 1: orr x12, x12, x9, lsl #L2_SHIFT /* Store the entry */ - str x12, [x6, x11, lsl #3] + str x12, [x5, x11, lsl #3] /* Clear the address bits */ and x12, x12, #ATTR_MASK_L @@ -535,7 +545,8 @@ /* Device Normal, no cache Normal, write-back */ .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) tcr: - .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K) + .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ + TCR_CACHE_ATTRS | TCR_SMP_ATTRS) sctlr_set: /* Bits to set */ .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ Index: sys/arm64/include/armreg.h =================================================================== --- sys/arm64/include/armreg.h +++ sys/arm64/include/armreg.h @@ -200,6 +200,28 @@ #define TCR_TG1_4K (2 << TCR_TG1_SHIFT) #define TCR_TG1_64K (3 << TCR_TG1_SHIFT) +#define TCR_SH1_SHIFT 28 +#define TCR_SH1_IS (0x3UL << TCR_SH1_SHIFT) +#define TCR_ORGN1_SHIFT 26 +#define TCR_ORGN1_WBWA (0x1UL << TCR_ORGN1_SHIFT) +#define TCR_IRGN1_SHIFT 24 +#define TCR_IRGN1_WBWA (0x1UL << TCR_IRGN1_SHIFT) +#define TCR_SH0_SHIFT 12 +#define TCR_SH0_IS (0x3UL << TCR_SH0_SHIFT) +#define TCR_ORGN0_SHIFT 10 +#define TCR_ORGN0_WBWA (0x1UL << TCR_ORGN0_SHIFT) +#define TCR_IRGN0_SHIFT 8 +#define TCR_IRGN0_WBWA (0x1UL << TCR_IRGN0_SHIFT) + +#define TCR_CACHE_ATTRS ((TCR_IRGN0_WBWA | TCR_IRGN1_WBWA) |\ + (TCR_ORGN0_WBWA | TCR_ORGN1_WBWA)) + +#ifdef SMP +#define TCR_SMP_ATTRS (TCR_SH0_IS | TCR_SH1_IS) +#else +#define TCR_SMP_ATTRS 0 +#endif + #define TCR_T1SZ_SHIFT 16 #define TCR_T0SZ_SHIFT 0 #define TCR_TxSZ(x) (((x) << TCR_T1SZ_SHIFT) | ((x) << TCR_T0SZ_SHIFT))