diff --git a/sys/arm64/arm64/kexec_support.c b/sys/arm64/arm64/kexec_support.c new file mode 100644 index 000000000000..8b9719c05b67 --- /dev/null +++ b/sys/arm64/arm64/kexec_support.c @@ -0,0 +1,188 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2025 Juniper Networks, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * Idea behind this: + * + * kexec_load_md(): + * - Update boot page tables (identity map) to include all pages needed before + * disabling MMU. + * + * kexec_reboot_md(): + * - Copy pages into target(s) + * - Do "other stuff" + * - Does not return + */ + +extern pt_entry_t pagetable_l0_ttbr0_bootstrap[]; +extern unsigned long initstack_end[]; +void switch_stack(void *, void (*)(void *, void *, struct kexec_image *), void *); + +#define SCTLR_EL1_NO_MMU (SCTLR_RES1 | SCTLR_LSMAOE | SCTLR_nTLSMD | \ + SCTLR_EIS | SCTLR_TSCXT | SCTLR_EOS) +#define vm_page_offset(m) ((vm_offset_t)(m) - vm_page_base) +static inline vm_page_t +phys_vm_page(vm_page_t m, vm_offset_t vm_page_v, vm_paddr_t vm_page_p) +{ + return ((vm_page_t)((vm_offset_t)m - vm_page_v + vm_page_p)); +} + +/* First 2 args are filler for switch_stack() */ +static void __aligned(16) __dead2 +kexec_reboot_bottom( void *arg1 __unused, void *arg2 __unused, + struct kexec_image *image) +{ + void (*e)(void) = (void *)image->entry; + vm_offset_t vm_page_base = (vm_offset_t)vm_page_array; + vm_paddr_t vm_page_phys = pmap_kextract((vm_offset_t)vm_page_array); + struct kexec_segment_stage *phys_segs = + (void *)pmap_kextract((vm_offset_t)&image->segments); + vm_paddr_t from_pa, to_pa; + vm_size_t size; + vm_page_t first, m, mp; + struct pctrie_iter pct_i; + + /* + * Create a linked list of all pages in the object before we disable the + * MMU. Once the MMU is disabled we can't use the vm_radix iterators, + * as they rely on virtual address pointers. + */ + first = NULL; + vm_radix_iter_init(&pct_i, &image->map_obj->rtree); + VM_RADIX_FORALL(m, &pct_i) { + if (first == NULL) + first = m; + else + SLIST_INSERT_AFTER(mp, m, plinks.s.ss); + mp = m; + } + + /* + * We're running out of the identity map now, disable the MMU before we + * continue. It's possible page tables can be overwritten, which would + * be very bad if we were running with the MMU enabled. + */ + WRITE_SPECIALREG(sctlr_el1, SCTLR_EL1_NO_MMU); + isb(); + for (int i = 0; i < KEXEC_SEGMENT_MAX; i++) { + if (phys_segs[i].size == 0) + break; + to_pa = phys_segs[i].target; + /* Copy the segment here... */ + for (vm_page_t p = phys_segs[i].first_page; + p != NULL && to_pa - phys_segs[i].target < phys_segs[i].size; + p = SLIST_NEXT(p, plinks.s.ss)) { + p = phys_vm_page(p, vm_page_base, vm_page_phys); + from_pa = p->phys_addr; + if (p->phys_addr == to_pa) { + to_pa += PAGE_SIZE; + continue; + } + for (size = PAGE_SIZE / sizeof(register_t); + size > 0; --size) { + *(register_t *)to_pa = *(register_t *)from_pa; + to_pa += sizeof(register_t); + from_pa += sizeof(register_t); + } + } + } + invalidate_icache(); + e(); + while (1) + ; +} + +void +kexec_reboot_md(struct kexec_image *image) +{ + uintptr_t ptr; + register_t reg; + + for (int i = 0; i < KEXEC_SEGMENT_MAX; i++) { + if (image->segments[i].size > 0) + cpu_dcache_inv_range((void *)PHYS_TO_DMAP(image->segments[i].target), + image->segments[i].size); + } + ptr = pmap_kextract((vm_offset_t)kexec_reboot_bottom); + serror_disable(); + + reg = pmap_kextract((vm_offset_t)pagetable_l0_ttbr0_bootstrap); + set_ttbr0(reg); + cpu_tlb_flushID(); + + typeof(kexec_reboot_bottom) *p = (void *)ptr; + switch_stack((void *)pmap_kextract((vm_offset_t)initstack_end), + p, image); + while (1) + ; +} + +int +kexec_load_md(struct kexec_image *image) +{ + vm_paddr_t tmp; + pt_entry_t *pte; + + /* Create L2 page blocks for the trampoline. L0/L1 are from the startup. */ + + /* + * There are exactly 2 pages before the pagetable_l0_ttbr0_bootstrap, so + * move to there. + */ + pte = pagetable_l0_ttbr0_bootstrap; + pte -= (Ln_ENTRIES * 2); /* move to start of L2 pages */ + + /* + * Populate the identity map with symbols we know we'll need before we + * turn off the MMU. + */ + tmp = pmap_kextract((vm_offset_t)kexec_reboot_bottom); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + tmp = pmap_kextract((vm_offset_t)initstack_end); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + /* We'll need vm_page_array for doing offset calculations. */ + tmp = pmap_kextract((vm_offset_t)&vm_page_array); + pte[pmap_l2_index(tmp)] = (tmp | L2_BLOCK | ATTR_AF | ATTR_S1_UXN); + + return (0); +} diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S index d35e334905a7..3ec12140f139 100644 --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -1,1162 +1,1206 @@ /*- * Copyright (c) 2012-2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "assym.inc" #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #define VIRT_BITS 48 /* * Loads a 64-bit value into reg using 1 to 4 mov/movk instructions. * This can be used early on when we don't know the CPUs endianness. */ .macro mov_q reg, val mov \reg, :abs_g0_nc:\val .if (\val >> 16) & 0xffff != 0 movk \reg, :abs_g1_nc:\val .endif .if (\val >> 32) & 0xffff != 0 movk \reg, :abs_g2_nc:\val .endif .if (\val >> 48) & 0xffff != 0 movk \reg, :abs_g3:\val .endif .endm #if PAGE_SIZE == PAGE_SIZE_16K /* * The number of level 3 tables to create. 32 will allow for 1G of address * space, the same as a single level 2 page with 4k pages. */ #define L3_PAGE_COUNT 32 #elif PAGE_SIZE == PAGE_SIZE_4K /* * Space for a level 3 table holding the end of the executable memory and * the start of the non-executable data. */ #define L3_PAGE_COUNT 1 #endif /* * The size of our bootstrap stack. */ #define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) .globl kernbase .set kernbase, KERNBASE /* * We assume: * MMU on with an identity map, or off * D-Cache: off * I-Cache: on or off * We are loaded at a 2MiB aligned address */ ENTRY(_start) /* Enter the kernel exception level */ bl enter_kernel_el /* Set the context id */ msr contextidr_el1, xzr /* Get the virt -> phys offset */ bl get_load_phys_addr /* * At this point: * x28 = Our physical load address */ /* Create the page tables */ bl create_pagetables /* * At this point: * x27 = TTBR0 table * x24 = TTBR1 table * x22 = PTE shareability attributes * x21 = BTI guarded page attribute if supported */ /* Enable the mmu */ bl start_mmu /* Load the new ttbr0 pagetable */ adrp x27, pagetable_l0_ttbr0 add x27, x27, :lo12:pagetable_l0_ttbr0 /* Jump to the virtual address space */ ldr x15, .Lvirtdone br x15 virtdone: BTI_J /* Set up the stack */ adrp x25, initstack_end add x25, x25, :lo12:initstack_end sub sp, x25, #PCB_SIZE /* Zero the BSS */ ldr x15, .Lbss ldr x14, .Lend 1: stp xzr, xzr, [x15], #16 cmp x15, x14 b.lo 1b #if defined(PERTHREAD_SSP) /* Set sp_el0 to the boot canary for early per-thread SSP to work */ adrp x15, boot_canary add x15, x15, :lo12:boot_canary msr sp_el0, x15 #endif /* Backup the module pointer */ mov x1, x0 sub sp, sp, #BOOTPARAMS_SIZE mov x0, sp str x1, [x0, #BP_MODULEP] adrp x25, initstack add x25, x25, :lo12:initstack str x25, [x0, #BP_KERN_STACK] str x27, [x0, #BP_KERN_TTBR0] str x23, [x0, #BP_BOOT_EL] /* Set these before they are used in kasan_init_early */ adrp x1, pmap_sh_attr str x22, [x1, :lo12:pmap_sh_attr] #ifdef __ARM_FEATURE_BTI_DEFAULT adrp x1, pmap_gp_attr str x21, [x1, :lo12:pmap_gp_attr] #endif #ifdef KASAN /* Save bootparams */ mov x19, x0 /* Bootstrap an early shadow map for the boot stack. */ ldr x0, [x0, #BP_KERN_STACK] ldr x1, =BOOT_STACK_SIZE bl kasan_init_early /* Restore bootparams */ mov x0, x19 #endif /* trace back starts here */ mov fp, #0 /* Branch to C code */ bl initarm /* We are done with the boot params */ add sp, sp, #BOOTPARAMS_SIZE /* * Enable pointer authentication in the kernel. We set the keys for * thread0 in initarm so have to wait until it returns to enable it. * If we were to enable it in initarm then any authentication when * returning would fail as it was called with pointer authentication * disabled. */ bl ptrauth_start bl mi_startup /* We should not get here */ brk 0 .align 3 .Lvirtdone: .quad virtdone .Lbss: .quad __bss_start .Lend: .quad __bss_end END(_start) #ifdef SMP /* * void * mpentry_psci(unsigned long) * * Called by a core when it is being brought online with psci. * The data in x0 is passed straight to init_secondary. */ ENTRY(mpentry_psci) mov x26, xzr b mpentry_common END(mpentry_psci) /* * void * mpentry_spintable(void) * * Called by a core when it is being brought online with a spin-table. * Reads the new CPU ID and passes this to init_secondary. */ ENTRY(mpentry_spintable) ldr x26, =spintable_wait b mpentry_common END(mpentry_spintable) /* Wait for the current CPU to be released */ LENTRY(spintable_wait) /* Read the affinity bits from mpidr_el1 */ mrs x1, mpidr_el1 ldr x2, =CPU_AFF_MASK and x1, x1, x2 adrp x2, ap_cpuid 1: ldr x0, [x2, :lo12:ap_cpuid] cmp x0, x1 b.ne 1b str xzr, [x2, :lo12:ap_cpuid] dsb sy sev ret LEND(mpentry_spintable) LENTRY(mpentry_common) /* Disable interrupts */ msr daifset, #DAIF_INTR /* Enter the kernel exception level */ bl enter_kernel_el /* Set the context id */ msr contextidr_el1, xzr /* Load the kernel page table */ adrp x24, pagetable_l0_ttbr1 add x24, x24, :lo12:pagetable_l0_ttbr1 /* Load the identity page table */ adrp x27, pagetable_l0_ttbr0_bootstrap add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap /* Enable the mmu */ bl start_mmu /* Load the new ttbr0 pagetable */ adrp x27, pagetable_l0_ttbr0 add x27, x27, :lo12:pagetable_l0_ttbr0 /* Jump to the virtual address space */ ldr x15, =mp_virtdone br x15 mp_virtdone: BTI_J /* * Allow this CPU to wait until the kernel is ready for it, * e.g. with spin-table but each CPU uses the same release address */ cbz x26, 1f blr x26 1: /* Start using the AP boot stack */ adrp x4, bootstack ldr x4, [x4, :lo12:bootstack] mov sp, x4 #if defined(PERTHREAD_SSP) /* Set sp_el0 to the boot canary for early per-thread SSP to work */ adrp x15, boot_canary add x15, x15, :lo12:boot_canary msr sp_el0, x15 #endif /* Load the kernel ttbr0 pagetable */ msr ttbr0_el1, x27 isb /* Invalidate the TLB */ tlbi vmalle1 dsb sy isb /* * Initialize the per-CPU pointer before calling into C code, for the * benefit of kernel sanitizers. */ adrp x18, bootpcpu ldr x18, [x18, :lo12:bootpcpu] msr tpidr_el1, x18 b init_secondary LEND(mpentry_common) + +ENTRY(mp_cpu_spinloop) +0: + wfe + ldr x0, mp_cpu_spin_table_release_addr + cbz x0, 0b + blr x0 + .globl mp_cpu_spin_table_release_addr +mp_cpu_spin_table_release_addr: + .quad 0 + .globl mp_cpu_spinloop_end +mp_cpu_spinloop_end: +END(mp_cpu_spinloop) #endif /* * Enter the exception level the kernel will use: * * - If in EL1 continue in EL1 * - If the CPU supports FEAT_VHE then set HCR_E2H and HCR_TGE and continue * in EL2 * - Configure EL2 to support running the kernel at EL1 and exit to that */ LENTRY(enter_kernel_el) mrs x23, CurrentEL and x23, x23, #(CURRENTEL_EL_MASK) cmp x23, #(CURRENTEL_EL_EL2) b.eq 1f /* * Ensure there are no memory operations here. If the boot loader * enters the kernel in big-endian mode then loading sctlr will * be incorrect. As instructions are the same in both endians it is * safe to use mov instructions. */ mov_q x2, SCTLR_MMU_OFF msr sctlr_el1, x2 /* * SCTLR_EOS is set to make eret a context synchronizing event. We * need an isb here to ensure it's observed by later instructions, * but don't need it in the eret below. */ isb /* * Ensure SPSR_EL1 and pstate are in sync. The only way to set the * latter is to set the former and return from an exception with eret. */ mov x2, #(PSR_DAIF | PSR_M_EL1h) msr spsr_el1, x2 msr elr_el1, lr eret 1: dsb sy /* * Set just the reserved bits in sctlr_el2. This will disable the * MMU which may have broken the kernel if we enter the kernel in * EL2, e.g. when using VHE. * * As with sctlr_el1 above use mov instructions to ensure there are * no memory operations. */ mov_q x2, (SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) msr sctlr_el2, x2 isb /* * The hardware is now in little-endian mode so memory operations * are safe. */ /* Configure the Hypervisor */ ldr x2, =(HCR_RW | HCR_APK | HCR_API) msr hcr_el2, x2 /* Stash value of HCR_EL2 for later */ isb mrs x4, hcr_el2 /* Load the Virtualization Process ID Register */ mrs x2, midr_el1 msr vpidr_el2, x2 /* Load the Virtualization Multiprocess ID Register */ mrs x2, mpidr_el1 msr vmpidr_el2, x2 /* Set the initial sctlr_el1 */ ldr x2, =SCTLR_MMU_OFF msr sctlr_el1, x2 /* Check for VHE */ CHECK_CPU_FEAT(x2, ID_AA64MMFR1, VH, IMPL, .Lno_vhe) /* * The kernel will be running in EL2, route exceptions here rather * than EL1. */ orr x4, x4, #HCR_E2H orr x4, x4, #HCR_TGE msr hcr_el2, x4 isb msr SCTLR_EL12_REG, x2 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ ldr x3, =(CNTHCTL_E2H_EL1PCTEN_NOTRAP | CNTHCTL_E2H_EL1PTEN_NOTRAP) ldr x5, =(PSR_DAIF | PSR_M_EL2h) b .Ldone_vhe .Lno_vhe: /* Hypervisor trap functions */ adrp x2, hyp_stub_vectors add x2, x2, :lo12:hyp_stub_vectors msr vbar_el2, x2 ldr x2, =(CPTR_RES1) ldr x3, =(CNTHCTL_EL1PCTEN_NOTRAP | CNTHCTL_EL1PCEN_NOTRAP) ldr x5, =(PSR_DAIF | PSR_M_EL1h) .Ldone_vhe: msr cptr_el2, x2 /* Enable access to the physical timers at EL1 */ msr cnthctl_el2, x3 /* Set the return PSTATE */ msr spsr_el2, x5 /* * Configure the Extended Hypervisor register. This is only valid if * FEAT_HCX is enabled. */ CHECK_CPU_FEAT(x2, ID_AA64MMFR1, HCX, IMPL, 2f) /* Extended Hypervisor Configuration */ msr HCRX_EL2_REG, xzr isb 2: /* Don't trap to EL2 for CP15 traps */ msr hstr_el2, xzr /* Set the counter offset to a known value */ msr cntvoff_el2, xzr /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ msr vttbr_el2, xzr /* Check the CPU supports GIC, and configure the CPU interface */ CHECK_CPU_FEAT(x2, ID_AA64PFR0, GIC, CPUIF_EN, 3f) mrs x2, icc_sre_el2 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ msr icc_sre_el2, x2 3: /* Set the address to return to our return address */ msr elr_el2, x30 isb eret LEND(enter_kernel_el) +/* Turn off the MMU. Install ttbr0 from the bootstrap page table, and go there. + * Does not return. + * - x0 - target address to jump to after stopping the MMU. + * - x1 - kernel load address + */ +ENTRY(stop_mmu) + mov x16, x0 /* Save target. */ + ldr x2, =(1f - KERNBASE) + add x17, x1, x2 + ldr x3, =(pagetable_l0_ttbr0_bootstrap - KERNBASE) + add x1, x1, x3 + msr ttbr0_el1, x1 + isb + br x17 +1: + BTI_J + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_M + bic x0, x0, SCTLR_C + msr sctlr_el1, x0 + isb + br x16 +END(stop_mmu) /* * Get the physical address the kernel was loaded at. */ LENTRY(get_load_phys_addr) /* Load the offset of get_load_phys_addr from KERNBASE */ ldr x28, =(get_load_phys_addr - KERNBASE) /* Load the physical address of get_load_phys_addr */ adr x29, get_load_phys_addr /* Find the physical address of KERNBASE, i.e. our load address */ sub x28, x29, x28 ret LEND(get_load_phys_addr) /* * This builds the page tables containing the identity map, and the kernel * virtual map. * * It relys on: * We were loaded to an address that is on a 2MiB boundary * All the memory must not cross a 1GiB boundaty * x28 contains the physical address we were loaded from * * There are 7 or 8 pages before that address for the page tables * The pages used are: * - The Kernel L3 tables (only for 16k kernel) * - The Kernel L2 table * - The Kernel L1 table * - The Kernel L0 table (TTBR1) * - The identity (PA = VA) L2 table * - The identity (PA = VA) L1 table * - The identity (PA = VA) L0 table (Early TTBR0) * - The Kernel empty L0 table (Late TTBR0) */ LENTRY(create_pagetables) /* Save the Link register */ mov x5, x30 /* Clean the page table */ adrp x6, pagetable add x6, x6, :lo12:pagetable adrp x27, pagetable_end add x27, x27, :lo12:pagetable_end 1: stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 cmp x6, x27 b.lo 1b #ifdef __ARM_FEATURE_BTI_DEFAULT /* * Check if the CPU supports BTI */ mrs x6, id_aa64pfr1_el1 /* Read the ID register */ and x6, x6, ID_AA64PFR1_BT_MASK /* Mask the field we need */ cmp x6, xzr /* Check it's non-zero */ cset x6, ne /* x6 is set if non-zero */ lsl x21, x6, ATTR_S1_GP_SHIFT /* Shift to the correct bit */ #endif /* * Find the shareability attribute we should use. If FEAT_LPA2 is * enabled then the shareability field is moved from the page table * to tcr_el1 and the bits in the page table are reused by the * address field. */ #if PAGE_SIZE == PAGE_SIZE_4K #define LPA2_MASK ID_AA64MMFR0_TGran4_MASK #define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 #elif PAGE_SIZE == PAGE_SIZE_16K #define LPA2_MASK ID_AA64MMFR0_TGran16_MASK #define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 #else #error Unsupported page size #endif mrs x6, id_aa64mmfr0_el1 mov x7, LPA2_VAL and x6, x6, LPA2_MASK cmp x6, x7 ldr x22, =(ATTR_SH(ATTR_SH_IS)) csel x22, xzr, x22, eq #undef LPA2_MASK #undef LPA2_VAL /* * Build the TTBR1 maps. */ /* Find the size of the kernel */ mov x6, #(KERNBASE) #if defined(LINUX_BOOT_ABI) /* X19 is used as 'map FDT data' flag */ mov x19, xzr /* No modules or FDT pointer ? */ cbz x0, booti_no_fdt /* * Test if x0 points to modules descriptor(virtual address) or * to FDT (physical address) */ cmp x0, x6 /* x6 is #(KERNBASE) */ b.lo booti_fdt #endif /* Booted with modules pointer */ /* Find modulep - begin */ sub x8, x0, x6 /* * Add space for the module data. When PAGE_SIZE is 4k this will * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is * larger it will be at least as large as we use smaller level 3 * pages. */ ldr x7, =((6 * 1024 * 1024) - 1) add x8, x8, x7 b common #if defined(LINUX_BOOT_ABI) booti_fdt: /* Booted by U-Boot booti with FDT data */ /* Set 'map FDT data' flag */ mov x19, #1 booti_no_fdt: /* Booted by U-Boot booti without FTD data */ /* Find the end - begin */ ldr x7, .Lend sub x8, x7, x6 /* * Add one 2MiB page for copy of FDT data (maximum FDT size), * one for metadata and round up */ ldr x7, =(3 * L2_SIZE - 1) add x8, x8, x7 #endif common: #if PAGE_SIZE != PAGE_SIZE_4K /* * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned * address, enabling the creation of L3C pages. However, when the page * size is larger than 4k, L2 blocks are too large to map the kernel * with 2M alignment. */ #define PTE_SHIFT L3_SHIFT #define LL_PAGE_TABLE pagetable_l3_ttbr1 #define BUILD_PTE_FUNC build_l3_page_pagetable #else #define PTE_SHIFT L2_SHIFT #define LL_PAGE_TABLE pagetable_l2_ttbr1 #define BUILD_PTE_FUNC build_l2_block_pagetable #endif /* Get the number of blocks/pages to allocate, rounded down */ lsr x14, x8, #(PTE_SHIFT) ldr x26, =etext #if PAGE_SIZE != PAGE_SIZE_4K ldr x8, =((1 << PTE_SHIFT) - 1) add x26, x26, x8 #endif mov x8, #(KERNBASE) sub x25, x26, x8 lsr x25, x25, #(PTE_SHIFT) #if PAGE_SIZE == PAGE_SIZE_4K /* Calculate the number of executable level 3 pages to create */ lsr x26, x26, #(L3_SHIFT) bfc x26, #(Ln_ENTRIES_SHIFT), #(64 - Ln_ENTRIES_SHIFT) /* Build the L3 table holding the end of the exectuable code */ lsl x15, x25, #(PTE_SHIFT) adrp x6, pagetable_l3_ttbr1 add x6, x6, :lo12:pagetable_l3_ttbr1 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ ATTR_S1_AP(ATTR_S1_AP_RO)) ldr x8, =(KERNBASE) add x8, x8, x15 add x9, x28, x15 mov x10, x26 bl build_l3_page_pagetable /* Build the remaining level 3 pages */ ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) lsl x27, x26, #(L3_SHIFT) add x8, x8, x27 add x9, x28, x15 add x9, x9, x27 ldr x10, =(Ln_ENTRIES) sub x10, x10, x26 bl build_l3_page_pagetable /* Link the l2 -> l3 table */ mov x9, x6 adrp x6, pagetable_l2_ttbr1 add x6, x6, :lo12:pagetable_l2_ttbr1 bl link_l2_pagetable #endif /* Create the kernel space PTE table */ adrp x6, LL_PAGE_TABLE add x6, x6, :lo12:LL_PAGE_TABLE ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ ATTR_S1_AP(ATTR_S1_AP_RO)) mov x8, #(KERNBASE) mov x9, x28 mov x10, x25 bl BUILD_PTE_FUNC #if PAGE_SIZE == PAGE_SIZE_4K /* Skip memory mapped through the L2 table */ add x25, x25, #1 #endif /* Create the kernel space XN PTE table */ lsl x10, x25, #(PTE_SHIFT) ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) ldr x8, =(KERNBASE) add x8, x8, x10 add x9, x28, x10 sub x10, x14, x25 bl BUILD_PTE_FUNC #undef PTE_SHIFT #undef LL_PAGE_TABLE #undef BUILD_PTE_FUNC #if PAGE_SIZE != PAGE_SIZE_4K /* Link the l2 -> l3 table */ mov x9, x6 adrp x6, pagetable_l2_ttbr1 add x6, x6, :lo12:pagetable_l2_ttbr1 bl link_l2_pagetable #endif /* Link the l1 -> l2 table */ mov x9, x6 adrp x6, pagetable_l1_ttbr1 add x6, x6, :lo12:pagetable_l1_ttbr1 bl link_l1_pagetable /* Link the l0 -> l1 table */ mov x9, x6 adrp x6, pagetable_l0_ttbr1 add x6, x6, :lo12:pagetable_l0_ttbr1 mov x10, #1 bl link_l0_pagetable /* Save the TTBR1 table physical address */ mov x24, x6 /* * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. * They are only needed early on, so the VA = PA map is uncached. */ adrp x6, pagetable_l2_ttbr0_bootstrap add x6, x6, :lo12:pagetable_l2_ttbr0_bootstrap /* Create the VA = PA map */ mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) adrp x16, _start and x16, x16, #(~L2_OFFSET) mov x9, x16 /* PA start */ mov x8, x16 /* VA start (== PA start) */ mov x10, #1 bl build_l2_block_pagetable #if defined(SOCDEV_PA) /* Create a table for the UART */ mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) ldr x9, =(L2_SIZE) add x16, x16, x9 /* VA start */ mov x8, x16 /* Store the socdev virtual address */ add x17, x8, #(SOCDEV_PA & L2_OFFSET) adrp x9, socdev_va str x17, [x9, :lo12:socdev_va] mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ mov x10, #1 bl build_l2_block_pagetable #endif #if defined(LINUX_BOOT_ABI) /* Map FDT data ? */ cbz x19, 1f /* Create the mapping for FDT data (2 MiB max) */ mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) ldr x9, =(L2_SIZE) add x16, x16, x9 /* VA start */ mov x8, x16 mov x9, x0 /* PA start */ /* Update the module pointer to point at the allocated memory */ and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ add x0, x0, x8 /* Add the aligned virtual address */ mov x10, #1 bl build_l2_block_pagetable 1: #endif /* Link the l1 -> l2 table */ mov x9, x6 adrp x6, pagetable_l1_ttbr0_bootstrap add x6, x6, :lo12:pagetable_l1_ttbr0_bootstrap bl link_l1_pagetable /* Link the l0 -> l1 table */ mov x9, x6 adrp x6, pagetable_l0_ttbr0_bootstrap add x6, x6, :lo12:pagetable_l0_ttbr0_bootstrap mov x10, #1 bl link_l0_pagetable /* Save the TTBR0 table physical address */ mov x27, x6 /* Restore the Link register */ mov x30, x5 ret LEND(create_pagetables) /* * Builds an L0 -> L1 table descriptor * * x6 = L0 table * x8 = Virtual Address * x9 = L1 PA (trashed) * x10 = Entry count (trashed) * x11, x12 and x13 are trashed */ LENTRY(link_l0_pagetable) /* * Link an L0 -> L1 table entry. */ /* Find the table index */ lsr x11, x8, #L0_SHIFT and x11, x11, #L0_ADDR_MASK /* Build the L0 block entry */ mov x12, #L0_TABLE orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) /* Only use the output address bits */ lsr x9, x9, #PAGE_SHIFT 1: orr x13, x12, x9, lsl #PAGE_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b ret LEND(link_l0_pagetable) /* * Builds an L1 -> L2 table descriptor * * x6 = L1 table * x8 = Virtual Address * x9 = L2 PA (trashed) * x11, x12 and x13 are trashed */ LENTRY(link_l1_pagetable) /* * Link an L1 -> L2 table entry. */ /* Find the table index */ lsr x11, x8, #L1_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ mov x12, #L1_TABLE /* Only use the output address bits */ lsr x9, x9, #PAGE_SHIFT orr x13, x12, x9, lsl #PAGE_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] ret LEND(link_l1_pagetable) /* * Builds count 2 MiB page table entry * x6 = L2 table * x7 = Block attributes * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count (trashed) * x11, x12 and x13 are trashed */ LENTRY(build_l2_block_pagetable) /* * Build the L2 table entry. */ /* Find the table index */ lsr x11, x8, #L2_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L2 block entry */ orr x12, x7, #L2_BLOCK orr x12, x12, #(ATTR_AF) orr x12, x12, #(ATTR_S1_UXN) #ifdef __ARM_FEATURE_BTI_DEFAULT orr x12, x12, x21 #endif /* Set the shareability attribute */ orr x12, x12, x22 /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT /* Set the physical address for this virtual address */ 1: orr x13, x12, x9, lsl #L2_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b ret LEND(build_l2_block_pagetable) /* * Builds an L2 -> L3 table descriptor * * x6 = L2 table * x8 = Virtual Address * x9 = L3 PA (trashed) * x11, x12 and x13 are trashed */ LENTRY(link_l2_pagetable) /* * Link an L2 -> L3 table entry. */ /* Find the table index */ lsr x11, x8, #L2_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ mov x12, #L2_TABLE /* Only use the output address bits */ lsr x9, x9, #PAGE_SHIFT orr x13, x12, x9, lsl #PAGE_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] ret LEND(link_l2_pagetable) /* * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create * large page (L3C) mappings when the current VA and remaining count allow * it. * x6 = L3 table * x7 = Block attributes * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count (trashed) * x11, x12 and x13 are trashed * * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. */ LENTRY(build_l3_page_pagetable) cbz x10, 2f /* * Build the L3 table entry. */ /* Find the table index */ lsr x11, x8, #L3_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L3 page entry */ orr x12, x7, #L3_PAGE orr x12, x12, #(ATTR_AF) orr x12, x12, #(ATTR_S1_UXN) #ifdef __ARM_FEATURE_BTI_DEFAULT orr x12, x12, x21 #endif /* Set the shareability attribute */ orr x12, x12, x22 /* Only use the output address bits */ lsr x9, x9, #L3_SHIFT /* Check if an ATTR_CONTIGUOUS mapping is possible */ 1: tst x11, #(L3C_ENTRIES - 1) b.ne 2f cmp x10, #L3C_ENTRIES b.lo 3f orr x12, x12, #(ATTR_CONTIGUOUS) b 2f 3: and x12, x12, #(~ATTR_CONTIGUOUS) /* Set the physical address for this virtual address */ 2: orr x13, x12, x9, lsl #L3_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b 2: ret LEND(build_l3_page_pagetable) LENTRY(start_mmu) dsb sy /* Load the exception vectors */ ldr x2, =exception_vectors msr vbar_el1, x2 /* Load ttbr0 and ttbr1 */ msr ttbr0_el1, x27 msr ttbr1_el1, x24 isb /* Clear the Monitor Debug System control register */ msr mdscr_el1, xzr /* Invalidate the TLB */ tlbi vmalle1is dsb ish isb ldr x2, mair msr mair_el1, x2 /* * Setup TCR according to the PARange and ASIDBits fields * from ID_AA64MMFR0_EL1 and the HAFDBS field from the * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS * to 1 only if the ASIDBits field equals 0b0010. */ ldr x2, tcr /* If x22 contains a non-zero value then LPA2 is not implemented */ cbnz x22, .Lno_lpa2 ldr x3, =(TCR_DS) orr x2, x2, x3 .Lno_lpa2: mrs x3, id_aa64mmfr0_el1 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) /* Check if the HW supports 16 bit ASIDS */ cmp x3, #(ID_AA64MMFR0_ASIDBits_16) /* If so x3 == 1, else x3 == 0 */ cset x3, eq /* Set TCR.AS with x3 */ bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) /* * Check if the HW supports access flag updates, and set * TCR_EL1.HA accordingly. The TCR_EL1.HD flag to enable * HW management of dirty state is set in C code as it may * need to be disabled because of CPU errata. */ CHECK_CPU_FEAT(x3, ID_AA64MMFR1, HAFDBS, AF, 1f) orr x2, x2, #(TCR_HA) 1: msr tcr_el1, x2 /* * Setup SCTLR. */ ldr x1, =SCTLR_MMU_ON msr sctlr_el1, x1 isb ret .align 3 mair: .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) tcr: #if PAGE_SIZE == PAGE_SIZE_4K #define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) #elif PAGE_SIZE == PAGE_SIZE_16K #define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) #else #error Unsupported page size #endif .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ TCR_SH1_IS | TCR_ORGN1_WBWA | TCR_IRGN1_WBWA | \ TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) LEND(start_mmu) +ENTRY(switch_stack) + mov sp, x0 + mov x16, x1 + br x16 +END(switch_stack) + ENTRY(abort) b abort END(abort) .bss .align PAGE_SHIFT + .globl initstack_end initstack: .space BOOT_STACK_SIZE initstack_end: .section .init_pagetable, "aw", %nobits .align PAGE_SHIFT /* * 6 initial tables (in the following order): * L2 for kernel (High addresses) * L1 for kernel * L0 for kernel * L1 bootstrap for user (Low addresses) * L0 bootstrap for user * L0 for user */ .globl pagetable_l0_ttbr1 + .globl pagetable_l0_ttbr0_bootstrap pagetable: pagetable_l3_ttbr1: .space (PAGE_SIZE * L3_PAGE_COUNT) pagetable_l2_ttbr1: .space PAGE_SIZE pagetable_l1_ttbr1: .space PAGE_SIZE pagetable_l0_ttbr1: .space PAGE_SIZE pagetable_l2_ttbr0_bootstrap: .space PAGE_SIZE pagetable_l1_ttbr0_bootstrap: .space PAGE_SIZE pagetable_l0_ttbr0_bootstrap: .space PAGE_SIZE pagetable_l0_ttbr0: .space PAGE_SIZE pagetable_end: el2_pagetable: .space PAGE_SIZE .section .rodata, "a", %progbits .globl aarch32_sigcode .align 2 aarch32_sigcode: .word 0xe1a0000d // mov r0, sp .word 0xe2800040 // add r0, r0, #SIGF_UC .word 0xe59f700c // ldr r7, [pc, #12] .word 0xef000000 // swi #0 .word 0xe59f7008 // ldr r7, [pc, #8] .word 0xef000000 // swi #0 .word 0xeafffffa // b . - 16 .word SYS_sigreturn .word SYS_exit .align 3 .size aarch32_sigcode, . - aarch32_sigcode aarch32_esigcode: .data .global sz_aarch32_sigcode sz_aarch32_sigcode: .quad aarch32_esigcode - aarch32_sigcode GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index e4d011df3a06..0bdd2ecfd8a7 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -1,882 +1,960 @@ /*- * Copyright (c) 2015-2016 The FreeBSD Foundation * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_acpi.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #ifdef VFP #include #endif #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #include #include #endif #include #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) #define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */ /* don't panic if one fails to start */ static uint32_t mp_quirks; #ifdef FDT static struct { const char *compat; uint32_t quirks; } fdt_quirks[] = { { "arm,foundation-aarch64", MP_QUIRK_CPULIST }, { "arm,fvp-base", MP_QUIRK_CPULIST }, { "arm,fvp-base-revc", MP_QUIRK_CPULIST }, /* This is incorrect in some DTS files */ { "arm,vfp-base", MP_QUIRK_CPULIST }, { NULL, 0 }, }; #endif static void ipi_ast(void *); static void ipi_hardclock(void *); static void ipi_preempt(void *); static void ipi_rendezvous(void *); static void ipi_stop(void *); +static void ipi_off(void *); #ifdef FDT static u_int fdt_cpuid; #endif void mpentry_psci(unsigned long cpuid); void mpentry_spintable(void); void init_secondary(uint64_t); /* Used to initialize the PCPU ahead of calling init_secondary(). */ void *bootpcpu; uint64_t ap_cpuid; /* Stacks for AP initialization, discarded once idle threads are started. */ void *bootstack; static void *bootstacks[MAXCPU]; /* Count of started APs, used to synchronize access to bootstack. */ static volatile int aps_started; /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_after_dev, aps_ready; /* Temporary variables for init_secondary() */ static void *dpcpu[MAXCPU - 1]; static bool is_boot_cpu(uint64_t target_cpu) { return (PCPU_GET_MPIDR(cpuid_to_pcpu[0]) == (target_cpu & CPU_AFF_MASK)); } static bool wait_for_aps(void) { for (int i = 0, started = 0; i < 2000; i++) { int32_t nstarted; nstarted = atomic_load_32(&aps_started); if (nstarted == mp_ncpus - 1) return (true); /* * Don't time out while we are making progress. Some large * systems can take a while to start all CPUs. */ if (nstarted > started) { i = 0; started = nstarted; } DELAY(1000); } return (false); } static void release_aps_after_dev(void *dummy __unused) { /* Only release CPUs if they exist */ if (mp_ncpus == 1) return; atomic_store_int(&aps_started, 0); atomic_store_rel_int(&aps_after_dev, 1); /* Wake up the other CPUs */ __asm __volatile( "dsb ishst \n" "sev \n" ::: "memory"); wait_for_aps(); } SYSINIT(aps_after_dev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE + 1, release_aps_after_dev, NULL); static void release_aps(void *dummy __unused) { /* Only release CPUs if they exist */ if (mp_ncpus == 1) return; intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); + intr_ipi_setup(IPI_OFF, "off", ipi_off, NULL); atomic_store_int(&aps_started, 0); atomic_store_rel_int(&aps_ready, 1); /* Wake up the other CPUs */ __asm __volatile( "dsb ishst \n" "sev \n" ::: "memory"); printf("Release APs..."); if (wait_for_aps()) printf("done\n"); else printf("APs not started\n"); smp_cpus = atomic_load_int(&aps_started) + 1; atomic_store_rel_int(&smp_started, 1); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); void init_secondary(uint64_t cpu) { struct pcpu *pcpup; pmap_t pmap0; uint64_t mpidr; ptrauth_mp_start(cpu); /* * Verify that the value passed in 'cpu' argument (aka context_id) is * valid. Some older U-Boot based PSCI implementations are buggy, * they can pass random value in it. */ mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; if (cpu >= MAXCPU || cpuid_to_pcpu[cpu] == NULL || PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) != mpidr) { for (cpu = 0; cpu < mp_maxid; cpu++) if (cpuid_to_pcpu[cpu] != NULL && PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) == mpidr) break; if ( cpu >= MAXCPU) panic("MPIDR for this CPU is not in pcpu table"); } /* * Identify current CPU. This is necessary to setup * affinity registers and to provide support for * runtime chip identification. * * We need this before signalling the CPU is ready to * let the boot CPU use the results. */ pcpup = cpuid_to_pcpu[cpu]; pcpup->pc_midr = get_midr(); identify_cpu(cpu); /* Ensure the stores in identify_cpu have completed */ atomic_thread_fence_acq_rel(); /* Detect early CPU feature support */ enable_cpu_feat(CPU_FEAT_EARLY_BOOT); /* Signal we are waiting for aps_after_dev */ atomic_add_int(&aps_started, 1); /* Wait for devices to be ready */ while (!atomic_load_int(&aps_after_dev)) __asm __volatile("wfe"); install_cpu_errata(); enable_cpu_feat(CPU_FEAT_AFTER_DEV); /* Signal we are done */ atomic_add_int(&aps_started, 1); /* Wait until we can run the scheduler */ while (!atomic_load_int(&aps_ready)) __asm __volatile("wfe"); /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pcpup->pc_curthread = pcpup->pc_idlethread; schedinit_ap(); /* Initialize curpmap to match TTBR0's current setting. */ pmap0 = vmspace_pmap(&vmspace0); KASSERT(pmap_to_ttbr0(pmap0) == READ_SPECIALREG(ttbr0_el1), ("pmap0 doesn't match cpu %ld's ttbr0", cpu)); pcpup->pc_curpmap = pmap0; intr_pic_init_secondary(); /* Start per-CPU event timers. */ cpu_initclocks_ap(); #ifdef VFP vfp_init_secondary(); #endif dbg_init(); /* Signal the CPU is ready */ atomic_add_int(&aps_started, 1); kcsan_cpu_init(cpu); /* Enter the scheduler */ sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ } static void smp_after_idle_runnable(void *arg __unused) { int cpu; if (mp_ncpus == 1) return; KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); /* * Wait for all APs to handle an interrupt. After that, we know that * the APs have entered the scheduler at least once, so the boot stacks * are safe to free. */ smp_rendezvous(smp_no_rendezvous_barrier, NULL, smp_no_rendezvous_barrier, NULL); for (cpu = 1; cpu < mp_ncpus; cpu++) { if (bootstacks[cpu] != NULL) kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); } } SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); static void ipi_ast(void *dummy __unused) { CTR0(KTR_SMP, "IPI_AST"); } static void ipi_hardclock(void *dummy __unused) { CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); } static void ipi_preempt(void *dummy __unused) { CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(curthread); } static void ipi_rendezvous(void *dummy __unused) { CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); } static void ipi_stop(void *dummy __unused) { u_int cpu; CTR0(KTR_SMP, "IPI_STOP"); cpu = PCPU_GET(cpuid); savectx(&stoppcbs[cpu]); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); #ifdef DDB dbg_register_sync(NULL); #endif CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); CTR0(KTR_SMP, "IPI_STOP (restart)"); } +void stop_mmu(vm_paddr_t, vm_paddr_t) __dead2; +extern uint32_t mp_cpu_spinloop[]; +extern uint32_t mp_cpu_spinloop_end[]; +extern uint64_t mp_cpu_spin_table_release_addr; +static void +ipi_off(void *dummy __unused) +{ + CTR0(KTR_SMP, "IPI_OFF"); + if (psci_present) + psci_cpu_off(); + else { + uint64_t release_addr; + vm_size_t size; + + size = (vm_offset_t)&mp_cpu_spin_table_release_addr - + (vm_offset_t)mp_cpu_spinloop; + release_addr = PCPU_GET(release_addr) - size; + isb(); + invalidate_icache(); + /* Go catatonic, don't take any interrupts. */ + intr_disable(); + stop_mmu(release_addr, pmap_kextract(KERNBASE)); + + + } + CTR0(KTR_SMP, "IPI_OFF failed"); +} + struct cpu_group * cpu_topo(void) { struct cpu_group *dom, *root; int i; root = smp_topo_alloc(1); dom = smp_topo_alloc(vm_ndomains); root->cg_parent = NULL; root->cg_child = dom; CPU_COPY(&all_cpus, &root->cg_mask); root->cg_count = mp_ncpus; root->cg_children = vm_ndomains; root->cg_level = CG_SHARE_NONE; root->cg_flags = 0; /* * Redundant layers will be collapsed by the caller so we don't need a * special case for a single domain. */ for (i = 0; i < vm_ndomains; i++, dom++) { dom->cg_parent = root; dom->cg_child = NULL; CPU_COPY(&cpuset_domain[i], &dom->cg_mask); dom->cg_count = CPU_COUNT(&dom->cg_mask); dom->cg_children = 0; dom->cg_level = CG_SHARE_L3; dom->cg_flags = 0; } return (root); } /* Determine if we running MP machine */ int cpu_mp_probe(void) { /* ARM64TODO: Read the u bit of mpidr_el1 to determine this */ return (1); } static int enable_cpu_psci(uint64_t target_cpu, vm_paddr_t entry, u_int cpuid) { int err; err = psci_cpu_on(target_cpu, entry, cpuid); if (err != PSCI_RETVAL_SUCCESS) { /* * Panic here if INVARIANTS are enabled and PSCI failed to * start the requested CPU. psci_cpu_on() returns PSCI_MISSING * to indicate we are unable to use it to start the given CPU. */ KASSERT(err == PSCI_MISSING || (mp_quirks & MP_QUIRK_CPULIST) == MP_QUIRK_CPULIST, ("Failed to start CPU %u (%lx), error %d\n", cpuid, target_cpu, err)); return (EINVAL); } return (0); } static int enable_cpu_spin(uint64_t cpu, vm_paddr_t entry, vm_paddr_t release_paddr) { vm_paddr_t *release_addr; ap_cpuid = cpu & CPU_AFF_MASK; release_addr = pmap_mapdev_attr(release_paddr, sizeof(*release_addr), VM_MEMATTR_DEFAULT); if (release_addr == NULL) return (ENOMEM); *release_addr = entry; cpu_dcache_wbinv_range(release_addr, sizeof(*release_addr)); pmap_unmapdev(release_addr, sizeof(*release_addr)); __asm __volatile( "sev \n" ::: "memory"); /* Wait for the target CPU to start */ while (atomic_load_64(&ap_cpuid) != 0) __asm __volatile("wfe"); return (0); } /* * Starts a given CPU. If the CPU is already running, i.e. it is the boot CPU, * do nothing. Returns true if the CPU is present and running. */ static bool start_cpu(u_int cpuid, uint64_t target_cpu, int domain, vm_paddr_t release_addr) { struct pcpu *pcpup; vm_size_t size; vm_paddr_t pa; int err, naps; /* Check we are able to start this cpu */ if (cpuid > mp_maxid) return (false); /* Skip boot CPU */ if (is_boot_cpu(target_cpu)) return (true); KASSERT(cpuid < MAXCPU, ("Too many CPUs")); size = round_page(sizeof(*pcpup) + DPCPU_SIZE); pcpup = kmem_malloc_domainset(DOMAINSET_PREF(domain), size, M_WAITOK | M_ZERO); pmap_disable_promotion((vm_offset_t)pcpup, size); pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); pcpup->pc_mpidr = target_cpu & CPU_AFF_MASK; bootpcpu = pcpup; + pcpup->pc_release_addr = release_addr; dpcpu[cpuid - 1] = (void *)(pcpup + 1); dpcpu_init(dpcpu[cpuid - 1], cpuid); bootstacks[cpuid] = kmem_malloc_domainset(DOMAINSET_PREF(domain), MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); naps = atomic_load_int(&aps_started); bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; printf("Starting CPU %u (%lx)\n", cpuid, target_cpu); /* * A limited set of hardware we support can only do spintables and * remain useful, due to lack of EL3. Thus, we'll usually fall into the * PSCI branch here. */ MPASS(release_addr == 0 || !psci_present); if (release_addr != 0) { pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry_spintable); err = enable_cpu_spin(target_cpu, pa, release_addr); } else { pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry_psci); err = enable_cpu_psci(target_cpu, pa, cpuid); } if (err != 0) { pcpu_destroy(pcpup); dpcpu[cpuid - 1] = NULL; kmem_free(bootstacks[cpuid], MP_BOOTSTACK_SIZE); kmem_free(pcpup, size); bootstacks[cpuid] = NULL; mp_ncpus--; return (false); } /* Wait for the AP to switch to its boot stack. */ while (atomic_load_int(&aps_started) < naps + 1) cpu_spinwait(); CPU_SET(cpuid, &all_cpus); return (true); } #ifdef DEV_ACPI static void madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_INTERRUPT *intr; u_int *cpuid; u_int id; int domain; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_INTERRUPT: intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; cpuid = arg; if (is_boot_cpu(intr->ArmMpidr)) id = 0; else id = *cpuid; domain = 0; #ifdef NUMA if (vm_ndomains > 1) domain = acpi_pxm_get_cpu_locality(intr->Uid); #endif if (start_cpu(id, intr->ArmMpidr, domain, 0)) { MPASS(cpuid_to_pcpu[id] != NULL); cpuid_to_pcpu[id]->pc_acpi_id = intr->Uid; /* * Don't increment for the boot CPU, its CPU ID is * reserved. */ if (!is_boot_cpu(intr->ArmMpidr)) (*cpuid)++; } break; default: break; } } static void cpu_init_acpi(void) { ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; u_int cpuid; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { printf("Unable to map the MADT, not starting APs\n"); return; } /* Boot CPU is always 0 */ cpuid = 1; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_handler, &cpuid); acpi_unmap_table(madt); #if MAXMEMDOM > 1 acpi_pxm_set_cpu_locality(); #endif } #endif #ifdef FDT /* * Failure is indicated by failing to populate *release_addr. */ static void populate_release_addr(phandle_t node, vm_paddr_t *release_addr) { pcell_t buf[2]; if (OF_getencprop(node, "cpu-release-addr", buf, sizeof(buf)) != sizeof(buf)) return; *release_addr = (((uintptr_t)buf[0] << 32) | buf[1]); } static bool start_cpu_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) { uint64_t target_cpu; vm_paddr_t release_addr; char *enable_method; int domain; int cpuid; target_cpu = reg[0]; if (addr_size == 2) { target_cpu <<= 32; target_cpu |= reg[1]; } if (is_boot_cpu(target_cpu)) cpuid = 0; else cpuid = fdt_cpuid; /* * If PSCI is present, we'll always use that -- the cpu_on method is * mandated in both v0.1 and v0.2. We'll check the enable-method if * we don't have PSCI and use spin table if it's provided. */ release_addr = 0; if (!psci_present && cpuid != 0) { if (OF_getprop_alloc(node, "enable-method", (void **)&enable_method) <= 0) return (false); if (strcmp(enable_method, "spin-table") != 0) { OF_prop_free(enable_method); return (false); } OF_prop_free(enable_method); populate_release_addr(node, &release_addr); if (release_addr == 0) { printf("Failed to fetch release address for CPU %u", cpuid); return (false); } } if (!start_cpu(cpuid, target_cpu, 0, release_addr)) return (false); /* * Don't increment for the boot CPU, its CPU ID is reserved. */ if (!is_boot_cpu(target_cpu)) fdt_cpuid++; /* Try to read the numa node of this cpu */ if (vm_ndomains == 1 || OF_getencprop(node, "numa-node-id", &domain, sizeof(domain)) <= 0) domain = 0; cpuid_to_pcpu[cpuid]->pc_domain = domain; if (domain < MAXMEMDOM) CPU_SET(cpuid, &cpuset_domain[domain]); return (true); } static void cpu_init_fdt(void) { phandle_t node; int i; node = OF_peer(0); for (i = 0; fdt_quirks[i].compat != NULL; i++) { if (ofw_bus_node_is_compatible(node, fdt_quirks[i].compat) != 0) { mp_quirks = fdt_quirks[i].quirks; } } fdt_cpuid = 1; ofw_cpu_early_foreach(start_cpu_fdt, true); } #endif /* Initialize and fire up non-boot processors */ void cpu_mp_start(void) { uint64_t mpidr; /* CPU 0 is always boot CPU. */ CPU_SET(0, &all_cpus); mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; cpuid_to_pcpu[0]->pc_mpidr = mpidr; cpu_desc_init(); switch(arm64_bus_method) { #ifdef DEV_ACPI case ARM64_BUS_ACPI: mp_quirks = MP_QUIRK_CPULIST; cpu_init_acpi(); break; #endif #ifdef FDT case ARM64_BUS_FDT: cpu_init_fdt(); break; #endif default: break; } } +void +cpu_mp_stop(void) +{ + + /* Short-circuit for single-CPU */ + if (CPU_COUNT(&all_cpus) == 1) + return; + + KASSERT(PCPU_GET(cpuid) == CPU_FIRST(), ("Not on the first CPU!\n")); + + /* + * If we use spin-table, assume U-boot method for now (single address + * shared by all CPUs). + */ + if (!psci_present) { + int cpu; + vm_paddr_t release_addr; + void *release_vaddr; + vm_size_t size; + + /* Find the shared release address. */ + CPU_FOREACH(cpu) { + release_addr = pcpu_find(cpu)->pc_release_addr; + if (release_addr != 0) + break; + } + /* No release address? No way of notifying other CPUs. */ + if (release_addr == 0) + return; + + size = (vm_offset_t)&mp_cpu_spinloop_end - + (vm_offset_t)&mp_cpu_spinloop; + + release_addr -= (vm_offset_t)&mp_cpu_spin_table_release_addr - + (vm_offset_t)mp_cpu_spinloop; + + release_vaddr = pmap_mapdev(release_addr, size); + bcopy(mp_cpu_spinloop, release_vaddr, size); + cpu_dcache_wbinv_range(release_vaddr, size); + pmap_unmapdev(release_vaddr, size); + invalidate_icache(); + } + ipi_all_but_self(IPI_OFF); + DELAY(1000000); +} + /* Introduce rest of cores to the world */ void cpu_mp_announce(void) { } #ifdef DEV_ACPI static void cpu_count_acpi_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { u_int *cores = arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_INTERRUPT: (*cores)++; break; default: break; } } static u_int cpu_count_acpi(void) { ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; u_int cores; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return (0); madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { printf("Unable to map the MADT, not starting APs\n"); return (0); } cores = 0; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, cpu_count_acpi_handler, &cores); acpi_unmap_table(madt); return (cores); } #endif void cpu_mp_setmaxid(void) { int cores; mp_ncpus = 1; mp_maxid = 0; switch(arm64_bus_method) { #ifdef DEV_ACPI case ARM64_BUS_ACPI: cores = cpu_count_acpi(); if (cores > 0) { cores = MIN(cores, MAXCPU); if (bootverbose) printf("Found %d CPUs in the ACPI tables\n", cores); mp_ncpus = cores; mp_maxid = cores - 1; } break; #endif #ifdef FDT case ARM64_BUS_FDT: cores = ofw_cpu_early_foreach(NULL, false); if (cores > 0) { cores = MIN(cores, MAXCPU); if (bootverbose) printf("Found %d CPUs in the device tree\n", cores); mp_ncpus = cores; mp_maxid = cores - 1; } break; #endif default: if (bootverbose) printf("No CPU data, limiting to 1 core\n"); break; } if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { if (cores > 0 && cores < mp_ncpus) { mp_ncpus = cores; mp_maxid = cores - 1; } } } /* Sending IPI */ void ipi_all_but_self(u_int ipi) { cpuset_t cpus; cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); intr_ipi_send(cpus, ipi); } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); intr_ipi_send(cpus, ipi); } void ipi_selected(cpuset_t cpus, u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); intr_ipi_send(cpus, ipi); } diff --git a/sys/arm64/include/cpufunc.h b/sys/arm64/include/cpufunc.h index e6e1f682794e..e9eee643216b 100644 --- a/sys/arm64/include/cpufunc.h +++ b/sys/arm64/include/cpufunc.h @@ -1,217 +1,224 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef __arm__ #include #else /* !__arm__ */ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ static __inline void breakpoint(void) { __asm("brk #0"); } #ifdef _KERNEL #include static __inline register_t dbg_disable(void) { uint32_t ret; __asm __volatile( "mrs %x0, daif \n" "msr daifset, #(" __XSTRING(DAIF_D) ") \n" : "=&r" (ret)); return (ret); } static __inline void dbg_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_D) ")"); } static __inline register_t intr_disable(void) { /* DAIF is a 32-bit register */ uint32_t ret; __asm __volatile( "mrs %x0, daif \n" "msr daifset, #(" __XSTRING(DAIF_INTR) ") \n" : "=&r" (ret)); return (ret); } static __inline void intr_restore(register_t s) { WRITE_SPECIALREG(daif, s); } static __inline void intr_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_INTR) ")"); } static __inline void serror_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_A) ")"); } +static __inline void +serror_disable(void) +{ + + __asm __volatile("msr daifset, #(" __XSTRING(DAIF_A) ")"); +} + static __inline register_t get_midr(void) { uint64_t midr; midr = READ_SPECIALREG(midr_el1); return (midr); } static __inline register_t get_mpidr(void) { uint64_t mpidr; mpidr = READ_SPECIALREG(mpidr_el1); return (mpidr); } static __inline void clrex(void) { /* * Ensure compiler barrier, otherwise the monitor clear might * occur too late for us ? */ __asm __volatile("clrex" : : : "memory"); } static __inline void set_ttbr0(uint64_t ttbr0) { __asm __volatile( "msr ttbr0_el1, %0 \n" "isb \n" : : "r" (ttbr0)); } static __inline void invalidate_icache(void) { __asm __volatile( "ic ialluis \n" "dsb ish \n" "isb \n"); } static __inline void invalidate_local_icache(void) { __asm __volatile( "ic iallu \n" "dsb nsh \n" "isb \n"); } static __inline void wfet(uint64_t val) { __asm __volatile( "msr s0_3_c1_c0_0, %0\n" : : "r" ((val)) : "memory"); } static __inline void wfit(uint64_t val) { __asm __volatile( "msr s0_3_c1_c0_1, %0\n" : : "r" ((val)) : "memory"); } extern bool icache_aliasing; extern bool icache_vmid; extern int64_t dcache_line_size; extern int64_t icache_line_size; extern int64_t idcache_line_size; extern int64_t dczva_line_size; #define cpu_nullop() arm64_nullop() #define cpufunc_nullop() arm64_nullop() #define cpu_tlb_flushID() arm64_tlb_flushID() #define cpu_dcache_wbinv_range(a, s) arm64_dcache_wbinv_range((a), (s)) #define cpu_dcache_inv_range(a, s) arm64_dcache_inv_range((a), (s)) #define cpu_dcache_wb_range(a, s) arm64_dcache_wb_range((a), (s)) extern void (*arm64_icache_sync_range)(void *, vm_size_t); #define cpu_icache_sync_range(a, s) arm64_icache_sync_range((a), (s)) #define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s)) void arm64_nullop(void); void arm64_tlb_flushID(void); void arm64_dic_idc_icache_sync_range(void *, vm_size_t); void arm64_idc_aliasing_icache_sync_range(void *, vm_size_t); void arm64_aliasing_icache_sync_range(void *, vm_size_t); int arm64_icache_sync_range_checked(void *, vm_size_t); void arm64_dcache_wbinv_range(void *, vm_size_t); void arm64_dcache_inv_range(void *, vm_size_t); void arm64_dcache_wb_range(void *, vm_size_t); bool arm64_get_writable_addr(void *, void **); #endif /* _KERNEL */ #endif /* _MACHINE_CPUFUNC_H_ */ #endif /* !__arm__ */ diff --git a/sys/arm64/include/smp.h b/sys/arm64/include/kexec.h similarity index 70% copy from sys/arm64/include/smp.h copy to sys/arm64/include/kexec.h index 500cd1ef4f02..0a8c7a053331 100644 --- a/sys/arm64/include/smp.h +++ b/sys/arm64/include/kexec.h @@ -1,50 +1,33 @@ /*- - * Copyright (c) 2014 Andrew Turner - * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause * - * This software was developed by Andrew Turner under sponsorship from - * the FreeBSD Foundation. + * Copyright (c) 2025 Juniper Networks, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#ifndef _MACHINE_SMP_H_ -#define _MACHINE_SMP_H_ +#ifndef _ARM64_KEXEC_H_ +#define _ARM64_KEXEC_H_ -#include +#define KEXEC_MD_PAGES(x) 0 -enum { - IPI_AST, - IPI_PREEMPT, - IPI_RENDEZVOUS, - IPI_STOP, - IPI_STOP_HARD, - IPI_HARDCLOCK, - INTR_IPI_COUNT, -}; - -void ipi_all_but_self(u_int ipi); -void ipi_cpu(int cpu, u_int ipi); -void ipi_selected(cpuset_t cpus, u_int ipi); - -#endif /* !_MACHINE_SMP_H_ */ +#endif /* _ARM64_KEXEC_H_ */ diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h index 09bd8fa8a966..73399d2c3f8c 100644 --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -1,93 +1,94 @@ /*- * Copyright (c) 1999 Luoqi Chen * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 */ #ifdef __arm__ #include #else /* !__arm__ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include typedef int (*pcpu_bp_harden)(void); typedef int (*pcpu_ssbd)(int); struct debug_monitor_state; #define PCPU_MD_FIELDS \ u_int pc_acpi_id; /* ACPI CPU id */ \ u_int pc_midr; /* stored MIDR value */ \ uint64_t pc_clock; \ pcpu_bp_harden pc_bp_harden; \ pcpu_ssbd pc_ssbd; \ struct pmap *pc_curpmap; \ struct pmap *pc_curvmpmap; \ uint64_t pc_mpidr; \ u_int pc_bcast_tlbi_workaround; \ - char __pad[197] + uint64_t pc_release_addr; \ + char __pad[189] #ifdef _KERNEL struct pcb; struct pcpu; register struct pcpu *pcpup __asm ("x18"); static inline struct pcpu * get_pcpu(void) { struct pcpu *pcpu; __asm __volatile("mov %0, x18" : "=&r"(pcpu)); return (pcpu); } static inline struct thread * get_curthread(void) { struct thread *td; __asm __volatile("ldr %0, [x18]" : "=&r"(td)); return (td); } #define curthread get_curthread() #define PCPU_GET(member) (pcpup->pc_ ## member) #define PCPU_ADD(member, value) (pcpup->pc_ ## member += (value)) #define PCPU_PTR(member) (&pcpup->pc_ ## member) #define PCPU_SET(member,value) (pcpup->pc_ ## member = (value)) #define PCPU_GET_MPIDR(pc) ((pc)->pc_mpidr) #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ #endif /* !__arm__ */ diff --git a/sys/arm64/include/smp.h b/sys/arm64/include/smp.h index 500cd1ef4f02..4a5bfda3ac1c 100644 --- a/sys/arm64/include/smp.h +++ b/sys/arm64/include/smp.h @@ -1,50 +1,51 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ #include enum { IPI_AST, IPI_PREEMPT, IPI_RENDEZVOUS, IPI_STOP, IPI_STOP_HARD, IPI_HARDCLOCK, + IPI_OFF, INTR_IPI_COUNT, }; void ipi_all_but_self(u_int ipi); void ipi_cpu(int cpu, u_int ipi); void ipi_selected(cpuset_t cpus, u_int ipi); #endif /* !_MACHINE_SMP_H_ */ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 2f412fa3cb1b..882aca705336 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -1,787 +1,788 @@ ## ## Kernel ## kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_efi_map.c standard kern/subr_intr.c optional intrng kern/subr_physmem.c standard libkern/strlen.c standard libkern/arm64/crc32c_armv8.S standard arm/arm/generic_timer.c standard arm/arm/gic.c standard arm/arm/gic_acpi.c optional acpi arm/arm/gic_fdt.c optional fdt arm/arm/gic_if.m standard arm/arm/pmu.c standard arm/arm/pmu_acpi.c optional acpi arm/arm/pmu_fdt.c optional fdt arm64/acpica/acpi_iort.c optional acpi arm64/acpica/acpi_machdep.c optional acpi arm64/acpica/OsdEnvironment.c optional acpi arm64/acpica/acpi_wakeup.c optional acpi arm64/acpica/pci_cfgreg.c optional acpi pci arm64/arm64/autoconf.c standard arm64/arm64/bus_machdep.c standard arm64/arm64/bus_space_asm.S standard arm64/arm64/busdma_bounce.c standard arm64/arm64/busdma_machdep.c standard arm64/arm64/clock.c standard arm64/arm64/copyinout.S standard arm64/arm64/cpu_errata.c standard arm64/arm64/cpu_feat.c standard arm64/arm64/cpufunc_asm.S standard arm64/arm64/db_disasm.c optional ddb arm64/arm64/db_interface.c optional ddb arm64/arm64/db_trace.c optional ddb arm64/arm64/debug_monitor.c standard arm64/arm64/disassem.c optional ddb arm64/arm64/dump_machdep.c standard arm64/arm64/efirt_machdep.c optional efirt arm64/arm64/efirt_support.S optional efirt arm64/arm64/elf32_machdep.c optional compat_freebsd32 arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard arm64/arm64/exec_machdep.c standard arm64/arm64/freebsd32_machdep.c optional compat_freebsd32 arm64/arm64/gdb_machdep.c optional gdb arm64/arm64/gicv3_its.c optional intrng fdt arm64/arm64/gic_v3.c standard arm64/arm64/gic_v3_acpi.c optional acpi arm64/arm64/gic_v3_fdt.c optional fdt arm64/arm64/hyp_stub.S standard arm64/arm64/identcpu.c standard +arm64/arm64/kexec_support.c standard arm64/arm64/locore.S standard no-obj arm64/arm64/machdep.c standard arm64/arm64/machdep_boot.c standard arm64/arm64/mem.c standard arm64/arm64/memcmp.S standard arm64/arm64/memcpy.S standard arm64/arm64/memset.S standard arm64/arm64/minidump_machdep.c standard arm64/arm64/mp_machdep.c optional smp arm64/arm64/nexus.c standard arm64/arm64/ofw_machdep.c optional fdt arm64/arm64/pl031_rtc.c optional fdt pl031 arm64/arm64/ptrauth.c standard \ compile-with "${NORMAL_C:N-mbranch-protection*} -mbranch-protection=bti" arm64/arm64/pmap.c standard arm64/arm64/ptrace_machdep.c standard arm64/arm64/sdt_machdep.c optional kdtrace_hooks arm64/arm64/sigtramp.S standard arm64/arm64/spec_workaround.c standard arm64/arm64/stack_machdep.c optional ddb | stack arm64/arm64/strcmp.S standard arm64/arm64/strncmp.S standard arm64/arm64/support_ifunc.c standard arm64/arm64/support.S standard arm64/arm64/swtch.S standard arm64/arm64/sys_machdep.c standard arm64/arm64/trap.c standard arm64/arm64/uio_machdep.c standard arm64/arm64/undefined.c standard arm64/arm64/unwind.c optional ddb | kdtrace_hooks | stack \ compile-with "${NOSAN_C}" arm64/arm64/vfp.c standard arm64/arm64/vm_machdep.c standard arm64/coresight/coresight.c standard arm64/coresight/coresight_acpi.c optional acpi arm64/coresight/coresight_fdt.c optional fdt arm64/coresight/coresight_if.m standard arm64/coresight/coresight_cmd.c standard arm64/coresight/coresight_cpu_debug.c optional fdt arm64/coresight/coresight_etm4x.c standard arm64/coresight/coresight_etm4x_acpi.c optional acpi arm64/coresight/coresight_etm4x_fdt.c optional fdt arm64/coresight/coresight_funnel.c standard arm64/coresight/coresight_funnel_acpi.c optional acpi arm64/coresight/coresight_funnel_fdt.c optional fdt arm64/coresight/coresight_replicator.c standard arm64/coresight/coresight_replicator_acpi.c optional acpi arm64/coresight/coresight_replicator_fdt.c optional fdt arm64/coresight/coresight_tmc.c standard arm64/coresight/coresight_tmc_acpi.c optional acpi arm64/coresight/coresight_tmc_fdt.c optional fdt dev/smbios/smbios_subr.c standard arm64/iommu/iommu.c optional iommu arm64/iommu/iommu_if.m optional iommu arm64/iommu/iommu_pmap.c optional iommu arm64/iommu/smmu.c optional iommu arm64/iommu/smmu_acpi.c optional iommu acpi arm64/iommu/smmu_fdt.c optional iommu fdt arm64/iommu/smmu_quirks.c optional iommu dev/iommu/busdma_iommu.c optional iommu dev/iommu/iommu_gas.c optional iommu arm64/vmm/vmm.c optional vmm arm64/vmm/vmm_dev_machdep.c optional vmm arm64/vmm/vmm_instruction_emul.c optional vmm arm64/vmm/vmm_arm64.c optional vmm arm64/vmm/vmm_reset.c optional vmm arm64/vmm/vmm_handlers.c optional vmm arm64/vmm/vmm_call.S optional vmm arm64/vmm/vmm_nvhe_exception.S optional vmm \ dependency "$S/arm64/vmm/vmm_hyp_exception.S" \ compile-with "${NOSAN_C} -fpie" \ no-obj arm64/vmm/vmm_nvhe.c optional vmm \ dependency "$S/arm64/vmm/vmm_hyp.c" \ compile-with "${NOSAN_C} -fpie" \ no-obj vmm_hyp_blob.elf.full optional vmm \ dependency "vmm_nvhe.o vmm_nvhe_exception.o" \ compile-with "${SYSTEM_LD_BASECMD} -o ${.TARGET} ${.ALLSRC} --defsym=_start='0x0' --defsym=text_start='0x0'" \ no-obj no-implicit-rule vmm_hyp_blob.elf optional vmm \ dependency "vmm_hyp_blob.elf.full" \ compile-with "${OBJCOPY} --strip-debug ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule vmm_hyp_blob.bin optional vmm \ dependency vmm_hyp_blob.elf \ compile-with "${OBJCOPY} --output-target=binary ${.ALLSRC} ${.TARGET}" \ no-obj no-implicit-rule arm64/vmm/vmm_hyp_el2.S optional vmm \ dependency vmm_hyp_blob.bin arm64/vmm/vmm_mmu.c optional vmm arm64/vmm/vmm_vhe.c optional vmm arm64/vmm/vmm_vhe_exception.S optional vmm arm64/vmm/io/vgic.c optional vmm arm64/vmm/io/vgic_v3.c optional vmm arm64/vmm/io/vgic_if.m optional vmm arm64/vmm/io/vtimer.c optional vmm dev/vmm/vmm_dev.c optional vmm dev/vmm/vmm_mem.c optional vmm dev/vmm/vmm_stat.c optional vmm crypto/armv8/armv8_crypto.c optional armv8crypto armv8_crypto_wrap.o optional armv8crypto \ dependency "$S/crypto/armv8/armv8_crypto_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "armv8_crypto_wrap.o" aesv8-armx.o optional armv8crypto | ossl \ dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "aesv8-armx.o" ghashv8-armx.o optional armv8crypto \ dependency "$S/crypto/openssl/aarch64/ghashv8-armx.S" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8 -I$S/crypto/openssl ${WERROR} ${NO_WCAST_QUAL} ${CFLAGS:M-march=*:S/^$/-march=armv8-a/}+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "ghashv8-armx.o" crypto/des/des_enc.c optional netsmb crypto/openssl/ossl_aarch64.c optional ossl crypto/openssl/aarch64/chacha-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/chacha-armv8-sve.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/poly1305-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha1-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha256-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha512-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/vpaes-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} -I$S/crypto/openssl ${WERROR} ${.IMPSRC}" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pxm.c optional acpi dev/ahci/ahci_generic.c optional ahci cddl/dev/dtrace/aarch64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/aarch64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/aarch64/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" # zfs blake3 hash support contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S optional zfs compile-with "${ZFS_S:N-mgeneral-regs-only}" contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S optional zfs compile-with "${ZFS_S:N-mgeneral-regs-only}" # zfs sha2 hash support zfs-sha256-armv8.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS:N-mgeneral-regs-only} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-aarch64/sha2/sha256-armv8.S" \ no-implicit-rule \ clean "zfs-sha256-armv8.o" zfs-sha512-armv8.o optional zfs \ dependency "$S/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S" \ compile-with "${CC} -c ${ZFS_ASM_CFLAGS:N-mgeneral-regs-only} -o ${.TARGET} ${WERROR} $S/contrib/openzfs/module/icp/asm-aarch64/sha2/sha512-armv8.S" \ no-implicit-rule \ clean "zfs-sha512-armv8.o" ## ## ASoC support ## dev/sound/fdt/audio_dai_if.m optional sound fdt dev/sound/fdt/audio_soc.c optional sound fdt dev/sound/fdt/dummy_codec.c optional sound fdt dev/sound/fdt/simple_amplifier.c optional sound fdt ## ## Device drivers ## dev/axgbe/if_axgbe.c optional axa fdt dev/axgbe/xgbe-desc.c optional axa fdt dev/axgbe/xgbe-dev.c optional axa fdt dev/axgbe/xgbe-drv.c optional axa fdt dev/axgbe/xgbe-mdio.c optional axa fdt dev/axgbe/xgbe-sysctl.c optional axa fdt dev/axgbe/xgbe-txrx.c optional axa fdt dev/axgbe/xgbe_osdep.c optional axa fdt dev/axgbe/xgbe-phy-v1.c optional axa fdt dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/dpaa2/dpaa2_bp.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_buf.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_channel.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_cmd_if.m optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_con.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_console.c optional soc_nxp_ls dpaa2 fdt dev/dpaa2/dpaa2_io.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_mac.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_mc.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_mc_acpi.c optional soc_nxp_ls dpaa2 acpi dev/dpaa2/dpaa2_mc_fdt.c optional soc_nxp_ls dpaa2 fdt dev/dpaa2/dpaa2_mc_if.m optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_mcp.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_ni.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_rc.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_swp.c optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_swp_if.m optional soc_nxp_ls dpaa2 dev/dpaa2/dpaa2_types.c optional soc_nxp_ls dpaa2 dev/dpaa2/memac_mdio_acpi.c optional soc_nxp_ls dpaa2 acpi dev/dpaa2/memac_mdio_common.c optional soc_nxp_ls dpaa2 acpi | soc_nxp_ls dpaa2 fdt dev/dpaa2/memac_mdio_fdt.c optional soc_nxp_ls dpaa2 fdt dev/dpaa2/memac_mdio_if.m optional soc_nxp_ls dpaa2 acpi | soc_nxp_ls dpaa2 fdt # Synopsys DesignWare Ethernet Controller dev/dwc/if_dwc_rk.c optional fdt dwc_rk soc_rockchip_rk3328 | fdt dwc_rk soc_rockchip_rk3399 dev/dwc/if_dwc_socfpga.c optional fdt dwc_socfpga dev/enetc/enetc_mdio.c optional enetc soc_nxp_ls dev/enetc/if_enetc.c optional enetc iflib pci fdt soc_nxp_ls dev/eqos/if_eqos.c optional eqos dev/eqos/if_eqos_if.m optional eqos dev/eqos/if_eqos_fdt.c optional eqos fdt dev/etherswitch/felix/felix.c optional enetc etherswitch fdt felix pci soc_nxp_ls dev/firmware/arm/scmi.c optional fdt scmi dev/firmware/arm/scmi_clk.c optional fdt scmi dev/firmware/arm/scmi_if.m optional fdt scmi dev/firmware/arm/scmi_mailbox.c optional fdt scmi dev/firmware/arm/scmi_smc.c optional fdt scmi dev/firmware/arm/scmi_virtio.c optional fdt scmi virtio dev/firmware/arm/scmi_shmem.c optional fdt scmi dev/gpio/pl061.c optional pl061 gpio dev/gpio/pl061_acpi.c optional pl061 gpio acpi dev/gpio/pl061_fdt.c optional pl061 gpio fdt dev/gpio/qoriq_gpio.c optional soc_nxp_ls gpio fdt dev/hwpmc/hwpmc_arm64.c optional hwpmc dev/hwpmc/hwpmc_arm64_md.c optional hwpmc dev/hwpmc/hwpmc_cmn600.c optional hwpmc acpi arm64/arm64/cmn600.c optional hwpmc acpi dev/hwpmc/hwpmc_dmc620.c optional hwpmc acpi dev/hwpmc/pmu_dmc620.c optional hwpmc acpi # Microsoft Hyper-V dev/hyperv/vmbus/hyperv.c optional hyperv acpi dev/hyperv/vmbus/aarch64/hyperv_aarch64.c optional hyperv acpi dev/hyperv/vmbus/vmbus.c optional hyperv acpi pci dev/hyperv/vmbus/aarch64/vmbus_aarch64.c optional hyperv acpi dev/hyperv/vmbus/vmbus_if.m optional hyperv acpi dev/hyperv/vmbus/vmbus_res.c optional hyperv acpi dev/hyperv/vmbus/vmbus_xact.c optional hyperv acpi dev/hyperv/vmbus/aarch64/hyperv_machdep.c optional hyperv acpi dev/hyperv/vmbus/vmbus_chan.c optional hyperv acpi dev/hyperv/vmbus/hyperv_busdma.c optional hyperv acpi dev/hyperv/vmbus/vmbus_br.c optional hyperv acpi dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv acpi dev/hyperv/utilities/vmbus_timesync.c optional hyperv acpi dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv acpi dev/hyperv/utilities/vmbus_ic.c optional hyperv acpi dev/hyperv/utilities/vmbus_shutdown.c optional hyperv acpi dev/hyperv/utilities/hv_kvp.c optional hyperv acpi dev/hyperv/input/hv_kbd.c optional hyperv acpi dev/hyperv/input/hv_kbdc.c optional hyperv acpi dev/hyperv/netvsc/hn_nvs.c optional hyperv acpi dev/hyperv/netvsc/hn_rndis.c optional hyperv acpi dev/hyperv/netvsc/if_hn.c optional hyperv acpi dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci acpi dev/ice/if_ice_iflib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_lib.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_osdep.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_resmgr.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_strings.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_recovery_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iflib_txrx.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_common.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_controlq.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_dcb.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flex_pipe.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_flow.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_nvm.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_sched.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_switch.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_vlan_mode.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fw_logging.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_fwlog.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_rdma.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/irdma_if.m optional ice pci \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/irdma_di_if.m optional ice pci \ compile-with "${NORMAL_M} -I$S/dev/ice" dev/ice/ice_ddp_common.c optional ice pci \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_iov.c optional ice pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ice" dev/ice/ice_vf_mbx.c optional ice pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ice" ice_ddp.c optional ice_ddp \ compile-with "${AWK} -f $S/tools/fw_stub.awk ice_ddp.fw:ice_ddp:0x01032900 -mice_ddp -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "ice_ddp.c" ice_ddp.fwo optional ice_ddp \ dependency "ice_ddp.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "ice_ddp.fwo" ice_ddp.fw optional ice_ddp \ dependency "$S/contrib/dev/ice/ice-1.3.41.0.pkg" \ compile-with "${CP} $S/contrib/dev/ice/ice-1.3.41.0.pkg ice_ddp.fw" \ no-obj no-implicit-rule \ clean "ice_ddp.fw" dev/iicbus/controller/twsi/mv_twsi.c optional twsi fdt dev/iicbus/controller/twsi/a10_twsi.c optional twsi fdt dev/iicbus/controller/twsi/twsi.c optional twsi fdt dev/iicbus/controller/rockchip/rk_i2c.c optional rk_i2c fdt dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_bt.c optional ipmi dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/mailbox/arm/arm_doorbell.c optional fdt arm_doorbell dev/mbox/mbox_if.m optional soc_brcm_bcm2837 dev/mmc/host/dwmmc_altera.c optional dwmmc dwmmc_altera fdt dev/mmc/host/dwmmc_hisi.c optional dwmmc dwmmc_hisi fdt dev/mmc/host/dwmmc_rockchip.c optional dwmmc rk_dwmmc fdt dev/neta/if_mvneta_fdt.c optional neta fdt dev/neta/if_mvneta.c optional neta mdio mii fdt dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofw_pci.c optional fdt pci dev/ofw/ofw_pcib.c optional fdt pci dev/pci/controller/pci_n1sdp.c optional pci_n1sdp acpi dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_acpi.c optional pci acpi dev/pci/pci_host_generic_den0115.c optional pci acpi dev/pci/pci_host_generic_fdt.c optional pci fdt dev/pci/pci_dw_mv.c optional pci fdt dev/pci/pci_dw.c optional pci fdt dev/pci/pci_dw_if.m optional pci fdt dev/psci/psci.c standard dev/psci/smccc_arm64.S standard dev/psci/smccc_errata.c standard dev/psci/smccc_trng.c standard dev/psci/smccc.c standard dev/pwm/controller/allwinner/aw_pwm.c optional fdt aw_pwm dev/pwm/controller//rockchip/rk_pwm.c optional fdt rk_pwm dev/random/armv8rng.c optional armv8_rng !random_loadable dev/safexcel/safexcel.c optional safexcel fdt dev/sdhci/sdhci_xenon.c optional sdhci_xenon sdhci dev/sdhci/sdhci_xenon_acpi.c optional sdhci_xenon sdhci acpi dev/sdhci/sdhci_xenon_fdt.c optional sdhci_xenon sdhci fdt dev/sdhci/sdhci_fdt_xilinx.c optional sdhci_xilinx sdhci fdt dev/sdhci/sdhci_fdt_rockchip.c optional sdhci fdt soc_rockchip dev/sram/mmio_sram.c optional fdt mmio_sram dev/sram/mmio_sram_if.m optional fdt mmio_sram dev/spibus/controller/allwinner/aw_spi.c optional fdt aw_spi dev/spibus/controller/rockchip/rk_spi.c optional fdt rk_spi dev/uart/uart_cpu_arm64.c optional uart dev/uart/uart_dev_mu.c optional uart uart_mu fdt dev/uart/uart_dev_pl011.c optional uart pl011 dev/usb/controller/dwc_otg_hisi.c optional dwcotg fdt soc_hisi_hi6220 dev/usb/controller/ehci_mv.c optional ehci_mv fdt dev/usb/controller/generic_ehci.c optional ehci dev/usb/controller/generic_ehci_acpi.c optional ehci acpi dev/usb/controller/generic_ehci_fdt.c optional ehci fdt dev/usb/controller/generic_ohci.c optional ohci fdt dev/usb/controller/generic_usb_if.m optional ohci fdt dev/usb/controller/musb_otg_allwinner.c optional musb fdt soc_allwinner_a64 dev/usb/controller/usb_nop_xceiv.c optional fdt dev/usb/controller/generic_xhci.c optional xhci dev/usb/controller/generic_xhci_acpi.c optional xhci acpi dev/usb/controller/generic_xhci_fdt.c optional xhci fdt dev/usb/controller/dwc3/dwc3.c optional xhci acpi dwc3 | xhci fdt dwc3 dev/usb/controller/dwc3/aw_dwc3.c optional xhci fdt dwc3 aw_dwc3 dev/usb/controller/dwc3/rk_dwc3.c optional xhci fdt dwc3 rk_dwc3 dev/vnic/mrml_bridge.c optional vnic fdt dev/vnic/nic_main.c optional vnic pci dev/vnic/nicvf_main.c optional vnic pci pci_iov dev/vnic/nicvf_queues.c optional vnic pci pci_iov dev/vnic/thunder_bgx_fdt.c optional soc_cavm_thunderx pci vnic fdt dev/vnic/thunder_bgx.c optional soc_cavm_thunderx pci vnic pci dev/vnic/thunder_mdio_fdt.c optional soc_cavm_thunderx pci vnic fdt dev/vnic/thunder_mdio.c optional soc_cavm_thunderx pci vnic dev/vnic/lmac_if.m optional inet | inet6 | vnic ## ## SoC Support ## # Allwinner common files arm/allwinner/a10_codec.c optional sound a10_codec fdt arm/allwinner/a31_dmac.c optional a31_dmac fdt arm/allwinner/a33_codec.c optional fdt sound a33_codec arm/allwinner/a64/sun50i_a64_acodec.c optional fdt sound a64_codec arm/allwinner/sunxi_dma_if.m optional a31_dmac arm/allwinner/aw_cir.c optional evdev aw_cir fdt arm/allwinner/aw_gpio.c optional gpio aw_gpio fdt arm/allwinner/aw_i2s.c optional fdt sound aw_i2s arm/allwinner/aw_mmc.c optional mmc aw_mmc fdt | mmccam aw_mmc fdt arm/allwinner/aw_nmi.c optional aw_nmi fdt \ compile-with "${NORMAL_C} -I$S/contrib/device-tree/include" arm/allwinner/aw_r_intc.c optional aw_r_intc fdt arm/allwinner/aw_rsb.c optional aw_rsb fdt arm/allwinner/aw_rtc.c optional aw_rtc fdt arm/allwinner/aw_sid.c optional aw_sid nvmem fdt arm/allwinner/aw_syscon.c optional aw_syscon syscon fdt arm/allwinner/aw_timer.c optional aw_timer fdt arm/allwinner/aw_thermal.c optional aw_thermal nvmem fdt arm/allwinner/aw_usbphy.c optional ehci aw_usbphy fdt arm/allwinner/aw_usb3phy.c optional xhci aw_usbphy fdt arm/allwinner/aw_wdog.c optional aw_wdog fdt arm/allwinner/axp81x.c optional axp81x fdt arm/allwinner/if_awg.c optional awg syscon aw_sid nvmem fdt # Allwinner clock driver dev/clk/allwinner/aw_ccung.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_frac.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_m.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_mipi.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_nkmp.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_nm.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_nmm.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_np.c optional aw_ccu fdt dev/clk/allwinner/aw_clk_prediv_mux.c optional aw_ccu fdt dev/clk/allwinner/ccu_a64.c optional soc_allwinner_a64 aw_ccu fdt dev/clk/allwinner/ccu_h3.c optional soc_allwinner_h5 aw_ccu fdt dev/clk/allwinner/ccu_h6.c optional soc_allwinner_h6 aw_ccu fdt dev/clk/allwinner/ccu_h6_r.c optional soc_allwinner_h6 aw_ccu fdt dev/clk/allwinner/ccu_sun8i_r.c optional aw_ccu fdt dev/clk/allwinner/ccu_de2.c optional aw_ccu fdt # Allwinner padconf files arm/allwinner/a64/a64_padconf.c optional soc_allwinner_a64 fdt arm/allwinner/a64/a64_r_padconf.c optional soc_allwinner_a64 fdt arm/allwinner/h3/h3_padconf.c optional soc_allwinner_h5 fdt arm/allwinner/h3/h3_r_padconf.c optional soc_allwinner_h5 fdt arm/allwinner/h6/h6_padconf.c optional soc_allwinner_h6 fdt arm/allwinner/h6/h6_r_padconf.c optional soc_allwinner_h6 fdt # Altera/Intel arm64/intel/stratix10-soc-fpga-mgr.c optional soc_intel_stratix10 fdt arm64/intel/stratix10-svc.c optional soc_intel_stratix10 fdt # Annapurna arm/annapurna/alpine/alpine_ccu.c optional al_ccu fdt arm/annapurna/alpine/alpine_nb_service.c optional al_nb_service fdt arm/annapurna/alpine/alpine_pci.c optional al_pci fdt arm/annapurna/alpine/alpine_pci_msix.c optional al_pci fdt arm/annapurna/alpine/alpine_serdes.c optional al_serdes fdt \ no-depend \ compile-with "${CC} -c -o ${.TARGET} ${CFLAGS} -I$S/contrib/alpine-hal -I$S/contrib/alpine-hal/eth ${.IMPSRC}" # Apple arm64/apple/apple_aic.c optional soc_apple_t8103 fdt arm64/apple/apple_pinctrl.c optional soc_apple_t8103 fdt arm64/apple/apple_wdog.c optional soc_apple_t8103 fdt arm64/apple/exynos_uart.c optional soc_apple_t8103 fdt # Broadcom arm64/broadcom/brcmmdio/mdio_mux_iproc.c optional soc_brcm_ns2 fdt arm64/broadcom/brcmmdio/mdio_nexus_iproc.c optional soc_brcm_ns2 fdt arm64/broadcom/brcmmdio/mdio_ns2_pcie_phy.c optional soc_brcm_ns2 fdt pci arm64/broadcom/genet/if_genet.c optional soc_brcm_bcm2838 fdt genet arm/broadcom/bcm2835/bcm2835_audio.c optional sound vchiq fdt \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" arm/broadcom/bcm2835/bcm2835_bsc.c optional bcm2835_bsc fdt arm/broadcom/bcm2835/bcm2835_clkman.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_cpufreq.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_dma.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_fbd.c optional vt soc_brcm_bcm2837 fdt | vt soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_firmware.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_ft5406.c optional evdev bcm2835_ft5406 fdt arm/broadcom/bcm2835/bcm2835_gpio.c optional gpio soc_brcm_bcm2837 fdt | gpio soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_intr.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_mbox.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_rng.c optional !random_loadable soc_brcm_bcm2837 fdt | !random_loadable soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_sdhci.c optional sdhci soc_brcm_bcm2837 fdt | sdhci soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_sdhost.c optional sdhci soc_brcm_bcm2837 fdt | sdhci soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_spi.c optional bcm2835_spi fdt arm/broadcom/bcm2835/bcm2835_vcbus.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_vcio.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2835_wdog.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm2836.c optional soc_brcm_bcm2837 fdt | soc_brcm_bcm2838 fdt arm/broadcom/bcm2835/bcm283x_dwc_fdt.c optional dwcotg fdt soc_brcm_bcm2837 | dwcotg fdt soc_brcm_bcm2838 arm/broadcom/bcm2835/bcm2838_pci.c optional soc_brcm_bcm2838 fdt pci arm/broadcom/bcm2835/bcm2838_xhci.c optional soc_brcm_bcm2838 fdt pci xhci arm/broadcom/bcm2835/raspberrypi_gpio.c optional soc_brcm_bcm2837 gpio fdt | soc_brcm_bcm2838 gpio fdt contrib/vchiq/interface/compat/vchi_bsd.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -Wno-unused -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_arm.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -Wno-unused -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_connected.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_core.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_shim.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" contrib/vchiq/interface/vchiq_arm/vchiq_util.c optional vchiq soc_brcm_bcm2837 \ compile-with "${NORMAL_C} -DUSE_VCHIQ_ARM -D__VCCOREVER__=0x04000000 -I$S/contrib/vchiq" # Cavium arm64/cavium/thunder_pcie_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_pem.c optional soc_cavm_thunderx pci arm64/cavium/thunder_pcie_pem_fdt.c optional soc_cavm_thunderx pci fdt arm64/cavium/thunder_pcie_common.c optional soc_cavm_thunderx pci # i.MX8 Clock support arm64/freescale/imx/imx_ccm.c optional fdt soc_freescale_imx8 arm64/freescale/imx/imx8mp_ccm.c optional fdt soc_freescale_imx8 arm64/freescale/imx/imx8mq_ccm.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_gate.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_mux.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_composite.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_sscg_pll.c optional fdt soc_freescale_imx8 arm64/freescale/imx/clk/imx_clk_frac_pll.c optional fdt soc_freescale_imx8 # iMX drivers arm/freescale/imx/imx_gpio.c optional gpio soc_freescale_imx8 fdt arm/freescale/imx/imx_i2c.c optional fdt fsliic arm/freescale/imx/imx_machdep.c optional fdt soc_freescale_imx8 arm64/freescale/imx/imx7gpc.c optional fdt soc_freescale_imx8 dev/ffec/if_ffec.c optional fdt ffec # Marvell arm/mv/a37x0_gpio.c optional a37x0_gpio gpio fdt arm/mv/a37x0_iic.c optional a37x0_iic iicbus fdt arm/mv/a37x0_spi.c optional a37x0_spi spibus fdt arm/mv/clk/a37x0_tbg.c optional a37x0_tbg clk fdt syscon arm/mv/clk/a37x0_xtal.c optional a37x0_xtal clk fdt syscon arm/mv/armada38x/armada38x_rtc.c optional mv_rtc fdt arm/mv/gpio.c optional mv_gpio fdt arm/mv/mvebu_gpio.c optional mv_gpio fdt arm/mv/mvebu_pinctrl.c optional mvebu_pinctrl fdt arm/mv/mv_ap806_clock.c optional soc_marvell_8k fdt arm/mv/mv_ap806_gicp.c optional mv_ap806_gicp fdt arm/mv/mv_ap806_sei.c optional mv_ap806_sei fdt arm/mv/mv_cp110_clock.c optional soc_marvell_8k fdt arm/mv/mv_cp110_icu.c optional mv_cp110_icu fdt arm/mv/mv_cp110_icu_bus.c optional mv_cp110_icu fdt arm/mv/mv_thermal.c optional soc_marvell_8k mv_thermal fdt arm/mv/clk/a37x0_tbg_pll.c optional a37x0_tbg clk fdt syscon arm/mv/clk/a37x0_periph_clk_driver.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon arm/mv/clk/a37x0_nb_periph_clk_driver.c optional a37x0_nb_periph clk fdt syscon arm/mv/clk/a37x0_sb_periph_clk_driver.c optional a37x0_sb_periph clk fdt syscon arm/mv/clk/periph.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon arm/mv/clk/periph_clk_d.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon arm/mv/clk/periph_clk_fixed.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon arm/mv/clk/periph_clk_gate.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon arm/mv/clk/periph_clk_mux_gate.c optional a37x0_nb_periph a37x0_sb_periph clk fdt syscon # NVidia arm/nvidia/tegra_abpmisc.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_ahci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_efuse.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_ehci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_gpio.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_i2c.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_lic.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_mc.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_pcie.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_sdhci.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_soctherm_if.m optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_soctherm.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_uart.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_usbphy.c optional fdt soc_nvidia_tegra210 arm/nvidia/tegra_xhci.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_gpio.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_regulators.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/max77620_rtc.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_car.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_per.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_pll.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_clk_super.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_coretemp.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_cpufreq.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_pinmux.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_pmc.c optional fdt soc_nvidia_tegra210 arm64/nvidia/tegra210/tegra210_xusbpadctl.c optional fdt soc_nvidia_tegra210 # Nvidia firmware for Tegra tegra210_xusb_fw.c optional tegra210_xusb_fw \ dependency "$S/conf/files.arm64" \ compile-with "${AWK} -f $S/tools/fw_stub.awk tegra210_xusb.fw:tegra210_xusb_fw -mtegra210_xusb_fw -c${.TARGET}" \ no-ctfconvert no-implicit-rule before-depend local \ clean "tegra210_xusb_fw.c" tegra210_xusb.fwo optional tegra210_xusb_fw \ dependency "tegra210_xusb.fw" \ compile-with "${NORMAL_FWO}" \ no-implicit-rule \ clean "tegra210_xusb.fwo" tegra210_xusb.fw optional tegra210_xusb_fw \ dependency "$S/contrib/dev/nvidia/tegra210_xusb.bin.uu" \ compile-with "${NORMAL_FW}" \ no-obj no-implicit-rule \ clean "tegra210_xusb.fw" # NXP dev/iicbus/controller/vybrid/vf_i2c.c optional vf_i2c iicbus soc_nxp_ls dev/iicbus/controller/vybrid/vf_i2c_acpi.c optional vf_i2c iicbus acpi soc_nxp_ls dev/iicbus/controller/vybrid/vf_i2c_fdt.c optional vf_i2c iicbus fdt soc_nxp_ls arm64/qoriq/qoriq_dw_pci.c optional pci fdt soc_nxp_ls arm64/qoriq/qoriq_gpio_pic.c optional gpio fdt soc_nxp_ls arm64/qoriq/qoriq_therm.c optional pci fdt soc_nxp_ls arm64/qoriq/qoriq_therm_if.m optional pci fdt soc_nxp_ls arm64/qoriq/clk/ls1028a_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/ls1028a_flexspi_clk.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/ls1046a_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/ls1088a_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/lx2160a_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/qoriq_clk_pll.c optional clk soc_nxp_ls arm64/qoriq/clk/qoriq_clkgen.c optional clk soc_nxp_ls fdt dev/ahci/ahci_fsl_fdt.c optional soc_nxp_ls ahci fdt dev/flash/flexspi/flex_spi.c optional clk flex_spi soc_nxp_ls fdt # Qualcomm arm64/qualcomm/qcom_gcc.c optional qcom_gcc fdt dev/qcom_mdio/qcom_mdio_ipq4018.c optional qcom_mdio fdt mdio mii # RockChip Drivers arm64/rockchip/rk3328_codec.c optional fdt rk3328codec soc_rockchip_rk3328 arm64/rockchip/rk3399_emmcphy.c optional fdt rk_emmcphy soc_rockchip_rk3399 arm64/rockchip/rk3568_combphy.c optional fdt rk_combphy soc_rockchip_rk3568 arm64/rockchip/rk3568_pcie.c optional fdt pci soc_rockchip_rk3568 arm64/rockchip/rk3568_pciephy.c optional fdt pci soc_rockchip_rk3568 arm64/rockchip/rk_i2s.c optional fdt sound soc_rockchip_rk3328 | fdt sound soc_rockchip_rk3399 arm64/rockchip/rk_otp.c optional fdt soc_rockchip_rk3568 arm64/rockchip/rk_otp_if.m optional fdt soc_rockchip_rk3568 dev/iicbus/pmic/rockchip/rk8xx.c optional fdt rk8xx soc_rockchip dev/iicbus/pmic/rockchip/rk8xx_clocks.c optional fdt rk8xx soc_rockchip dev/iicbus/pmic/rockchip/rk8xx_regulators.c optional fdt rk8xx soc_rockchip dev/iicbus/pmic/rockchip/rk8xx_rtc.c optional fdt rk8xx soc_rockchip dev/iicbus/pmic/rockchip/rk805.c optional fdt rk805 soc_rockchip_rk3328 dev/iicbus/pmic/rockchip/rk808.c optional fdt rk808 soc_rockchip_rk3399 dev/iicbus/pmic/rockchip/rk817.c optional fdt rk817 soc_rockchip_rk3568 arm64/rockchip/rk_grf.c optional fdt soc_rockchip arm64/rockchip/rk_grf_gpio.c optional fdt soc_rockchip arm64/rockchip/rk_pinctrl.c optional fdt rk_pinctrl soc_rockchip arm64/rockchip/rk_gpio.c optional fdt rk_gpio soc_rockchip arm64/rockchip/rk_iodomain.c optional fdt rk_iodomain arm64/rockchip/rk_usb2phy.c optional fdt rk_usb2phy soc_rockchip arm64/rockchip/rk_typec_phy.c optional fdt rk_typec_phy soc_rockchip_rk3399 arm64/rockchip/rk_tsadc_if.m optional fdt soc_rockchip_rk3399 | fdt soc_rockchip_rk3568 arm64/rockchip/rk_tsadc.c optional fdt soc_rockchip_rk3399 | fdt soc_rockchip_rk3568 arm64/rockchip/rk_pcie.c optional fdt pci soc_rockchip_rk3399 arm64/rockchip/rk_pcie_phy.c optional fdt pci soc_rockchip_rk3399 # RockChip Clock support dev/clk/rockchip/rk_cru.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_armclk.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_composite.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_fract.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_gate.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_mux.c optional fdt soc_rockchip dev/clk/rockchip/rk_clk_pll.c optional fdt soc_rockchip dev/clk/rockchip/rk3328_cru.c optional fdt soc_rockchip_rk3328 dev/clk/rockchip/rk3399_cru.c optional fdt soc_rockchip_rk3399 dev/clk/rockchip/rk3399_pmucru.c optional fdt soc_rockchip_rk3399 dev/clk/rockchip/rk3568_cru.c optional fdt soc_rockchip_rk3568 dev/clk/rockchip/rk3568_pmucru.c optional fdt soc_rockchip_rk3568 # Xilinx arm/xilinx/uart_dev_cdnc.c optional uart soc_xilinx_zynq fdt arm/xilinx/zy7_gpio.c optional gpio soc_xilinx_zynq fdt dev/iicbus/controller/cadence/cdnc_i2c.c optional cdnc_i2c iicbus soc_xilinx_zynq fdt dev/usb/controller/xlnx_dwc3.c optional xhci soc_xilinx_zynq fdt dev/firmware/xilinx/zynqmp_firmware.c optional fdt soc_xilinx_zynq dev/firmware/xilinx/zynqmp_firmware_if.m optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clock.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clk_div.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clk_fixed.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clk_gate.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clk_mux.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_clk_pll.c optional fdt soc_xilinx_zynq dev/clk/xilinx/zynqmp_reset.c optional fdt soc_xilinx_zynq diff --git a/sys/dev/psci/psci.c b/sys/dev/psci/psci.c index 497b23d2d4c3..2b250401ae83 100644 --- a/sys/dev/psci/psci.c +++ b/sys/dev/psci/psci.c @@ -1,620 +1,633 @@ /*- * Copyright (c) 2014 Robin Randhawa * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * This implements support for ARM's Power State Co-ordination Interface * [PSCI]. The implementation adheres to version 0.2 of the PSCI specification * but also supports v0.1. PSCI standardizes operations such as system reset, CPU * on/off/suspend. PSCI requires a compliant firmware implementation. * * The PSCI specification used for this implementation is available at: * * . * * TODO: * - Add support for remaining PSCI calls [this implementation only * supports get_version, system_reset and cpu_on]. */ #include #include "opt_acpi.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #include #include #endif #include struct psci_softc { device_t dev; device_t smccc_dev; uint32_t psci_version; uint32_t psci_fnids[PSCI_FN_MAX]; }; #ifdef FDT static int psci_v0_1_init(device_t dev, int default_version); #endif static int psci_v0_2_init(device_t dev, int default_version); struct psci_softc *psci_softc = NULL; bool psci_present; #ifdef __arm__ #define USE_ACPI 0 #define USE_FDT 1 #elif defined(__aarch64__) #define USE_ACPI (arm64_bus_method == ARM64_BUS_ACPI) #define USE_FDT (arm64_bus_method == ARM64_BUS_FDT) #else #error Unknown architecture #endif #ifdef FDT struct psci_init_def { int default_version; psci_initfn_t psci_init; }; static struct psci_init_def psci_v1_0_init_def = { .default_version = (1 << 16) | 0, .psci_init = psci_v0_2_init }; static struct psci_init_def psci_v0_2_init_def = { .default_version = (0 << 16) | 2, .psci_init = psci_v0_2_init }; static struct psci_init_def psci_v0_1_init_def = { .default_version = (0 << 16) | 1, .psci_init = psci_v0_1_init }; static struct ofw_compat_data compat_data[] = { {"arm,psci-1.0", (uintptr_t)&psci_v1_0_init_def}, {"arm,psci-0.2", (uintptr_t)&psci_v0_2_init_def}, {"arm,psci", (uintptr_t)&psci_v0_1_init_def}, {NULL, 0} }; #endif static int psci_attach(device_t, psci_initfn_t, int); static int psci_find_callfn(psci_callfn_t *); static int psci_def_callfn(register_t, register_t, register_t, register_t, register_t, register_t, register_t, register_t, struct arm_smccc_res *res); psci_callfn_t psci_callfn = psci_def_callfn; static void psci_init(void *dummy) { psci_callfn_t new_callfn; if (psci_find_callfn(&new_callfn) != PSCI_RETVAL_SUCCESS) { printf("No PSCI/SMCCC call function found\n"); return; } psci_callfn = new_callfn; psci_present = true; } /* This needs to be before cpu_mp at SI_SUB_CPU, SI_ORDER_THIRD */ SYSINIT(psci_start, SI_SUB_CPU, SI_ORDER_FIRST, psci_init, NULL); static int psci_def_callfn(register_t a __unused, register_t b __unused, register_t c __unused, register_t d __unused, register_t e __unused, register_t f __unused, register_t g __unused, register_t h __unused, struct arm_smccc_res *res __unused) { panic("No PSCI/SMCCC call function set"); } #ifdef FDT static int psci_fdt_probe(device_t dev); static int psci_fdt_attach(device_t dev); static device_method_t psci_fdt_methods[] = { DEVMETHOD(device_probe, psci_fdt_probe), DEVMETHOD(device_attach, psci_fdt_attach), DEVMETHOD_END }; static driver_t psci_fdt_driver = { "psci", psci_fdt_methods, sizeof(struct psci_softc), }; EARLY_DRIVER_MODULE(psci, simplebus, psci_fdt_driver, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_FIRST); EARLY_DRIVER_MODULE(psci, ofwbus, psci_fdt_driver, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_FIRST); static psci_callfn_t psci_fdt_get_callfn(phandle_t node) { char method[16]; if ((OF_getprop(node, "method", method, sizeof(method))) > 0) { if (strcmp(method, "hvc") == 0) return (arm_smccc_hvc); else if (strcmp(method, "smc") == 0) return (arm_smccc_smc); else printf("psci: PSCI conduit \"%s\" invalid\n", method); } else printf("psci: PSCI conduit not supplied in the device tree\n"); return (NULL); } static int psci_fdt_probe(device_t dev) { const struct ofw_compat_data *ocd; if (!ofw_bus_status_okay(dev)) return (ENXIO); ocd = ofw_bus_search_compatible(dev, compat_data); if (ocd->ocd_str == NULL) return (ENXIO); device_set_desc(dev, "ARM Power State Co-ordination Interface Driver"); return (BUS_PROBE_SPECIFIC); } static int psci_fdt_attach(device_t dev) { const struct ofw_compat_data *ocd; struct psci_init_def *psci_init_def; ocd = ofw_bus_search_compatible(dev, compat_data); psci_init_def = (struct psci_init_def *)ocd->ocd_data; return (psci_attach(dev, psci_init_def->psci_init, psci_init_def->default_version)); } #endif #ifdef DEV_ACPI static void psci_acpi_identify(driver_t *, device_t); static int psci_acpi_probe(device_t); static int psci_acpi_attach(device_t); static device_method_t psci_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, psci_acpi_identify), DEVMETHOD(device_probe, psci_acpi_probe), DEVMETHOD(device_attach, psci_acpi_attach), DEVMETHOD_END }; static driver_t psci_acpi_driver = { "psci", psci_acpi_methods, sizeof(struct psci_softc), }; EARLY_DRIVER_MODULE(psci, acpi, psci_acpi_driver, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_FIRST); static int psci_acpi_bootflags(void) { ACPI_TABLE_FADT *fadt; vm_paddr_t physaddr; int flags; physaddr = acpi_find_table(ACPI_SIG_FADT); if (physaddr == 0) return (0); fadt = acpi_map_table(physaddr, ACPI_SIG_FADT); if (fadt == NULL) { printf("psci: Unable to map the FADT\n"); return (0); } flags = fadt->ArmBootFlags; acpi_unmap_table(fadt); return (flags); } static psci_callfn_t psci_acpi_get_callfn(int flags) { if ((flags & ACPI_FADT_PSCI_COMPLIANT) != 0) { if ((flags & ACPI_FADT_PSCI_USE_HVC) != 0) return (arm_smccc_hvc); else return (arm_smccc_smc); } else { printf("psci: PSCI conduit not supplied in the device tree\n"); } return (NULL); } static void psci_acpi_identify(driver_t *driver, device_t parent) { device_t dev; int flags; flags = psci_acpi_bootflags(); if ((flags & ACPI_FADT_PSCI_COMPLIANT) != 0) { dev = BUS_ADD_CHILD(parent, BUS_PASS_CPU + BUS_PASS_ORDER_FIRST, "psci", -1); if (dev != NULL) acpi_set_private(dev, (void *)(uintptr_t)flags); } } static int psci_acpi_probe(device_t dev) { uintptr_t flags; flags = (uintptr_t)acpi_get_private(dev); if ((flags & ACPI_FADT_PSCI_COMPLIANT) == 0) return (ENXIO); device_set_desc(dev, "ARM Power State Co-ordination Interface Driver"); return (BUS_PROBE_SPECIFIC); } static int psci_acpi_attach(device_t dev) { return (psci_attach(dev, psci_v0_2_init, PSCI_RETVAL_NOT_SUPPORTED)); } #endif static int psci_attach(device_t dev, psci_initfn_t psci_init, int default_version) { struct psci_softc *sc = device_get_softc(dev); if (psci_softc != NULL) return (ENXIO); KASSERT(psci_init != NULL, ("PSCI init function cannot be NULL")); if (psci_init(dev, default_version)) return (ENXIO); psci_softc = sc; #ifdef __aarch64__ smccc_init(); sc->smccc_dev = device_add_child(dev, "smccc", DEVICE_UNIT_ANY); if (sc->smccc_dev == NULL) device_printf(dev, "Unable to add SMCCC device\n"); bus_attach_children(dev); #endif return (0); } static int _psci_get_version(struct psci_softc *sc) { uint32_t fnid; /* PSCI version wasn't supported in v0.1. */ fnid = sc->psci_fnids[PSCI_FN_VERSION]; if (fnid) return (psci_call(fnid, 0, 0, 0)); return (PSCI_RETVAL_NOT_SUPPORTED); } int psci_get_version(void) { if (psci_softc == NULL) return (PSCI_RETVAL_NOT_SUPPORTED); return (_psci_get_version(psci_softc)); } #ifdef FDT static int psci_fdt_callfn(psci_callfn_t *callfn) { phandle_t node; /* XXX: This is suboptimal, we should walk the tree & check each * node against compat_data, but we only have a few entries so * it's ok for now. */ for (int i = 0; compat_data[i].ocd_str != NULL; i++) { node = ofw_bus_find_compatible(OF_peer(0), compat_data[i].ocd_str); if (node != 0) break; } if (node == 0) return (PSCI_MISSING); if (!ofw_bus_node_status_okay(node)) return (PSCI_MISSING); *callfn = psci_fdt_get_callfn(node); return (0); } #endif #ifdef DEV_ACPI static int psci_acpi_callfn(psci_callfn_t *callfn) { int flags; flags = psci_acpi_bootflags(); if ((flags & ACPI_FADT_PSCI_COMPLIANT) == 0) return (PSCI_MISSING); *callfn = psci_acpi_get_callfn(flags); return (0); } #endif static int psci_find_callfn(psci_callfn_t *callfn) { int error; *callfn = NULL; #ifdef FDT if (USE_FDT) { error = psci_fdt_callfn(callfn); if (error != 0) return (error); } #endif #ifdef DEV_ACPI if (*callfn == NULL && USE_ACPI) { error = psci_acpi_callfn(callfn); if (error != 0) return (error); } #endif if (*callfn == NULL) return (PSCI_MISSING); return (PSCI_RETVAL_SUCCESS); } int32_t psci_features(uint32_t psci_func_id) { if (psci_softc == NULL) return (PSCI_RETVAL_NOT_SUPPORTED); /* The feature flags were added to PSCI 1.0 */ if (PSCI_VER_MAJOR(psci_softc->psci_version) < 1) return (PSCI_RETVAL_NOT_SUPPORTED); return (psci_call(PSCI_FNID_FEATURES, psci_func_id, 0, 0)); } int psci_cpu_on(unsigned long cpu, unsigned long entry, unsigned long context_id) { uint32_t fnid; fnid = PSCI_FNID_CPU_ON; if (psci_softc != NULL) fnid = psci_softc->psci_fnids[PSCI_FN_CPU_ON]; /* PSCI v0.1 and v0.2 both support cpu_on. */ return (psci_call(fnid, cpu, entry, context_id)); } +int +psci_cpu_off(void) +{ + uint32_t fnid; + + fnid = PSCI_FNID_CPU_OFF; + if (psci_softc != NULL) + fnid = psci_softc->psci_fnids[PSCI_FN_CPU_OFF]; + + /* Returns PSCI_RETVAL_DENIED on error. */ + return (psci_call(fnid, 0, 0, 0)); +} + static void psci_shutdown(void *xsc, int howto) { uint32_t fn = 0; if (psci_softc == NULL) return; if ((howto & RB_POWEROFF) != 0) fn = psci_softc->psci_fnids[PSCI_FN_SYSTEM_OFF]; if (fn) psci_call(fn, 0, 0, 0); /* System reset and off do not return. */ } static void psci_reboot(void *xsc, int howto) { uint32_t fn = 0; if (psci_softc == NULL) return; if ((howto & RB_HALT) == 0) fn = psci_softc->psci_fnids[PSCI_FN_SYSTEM_RESET]; if (fn) psci_call(fn, 0, 0, 0); /* System reset and off do not return. */ } void psci_reset(void) { psci_reboot(NULL, 0); } #ifdef FDT /* Only support PSCI 0.1 on FDT */ static int psci_v0_1_init(device_t dev, int default_version __unused) { struct psci_softc *sc = device_get_softc(dev); int psci_fn; uint32_t psci_fnid; phandle_t node; int len; /* Zero out the function ID table - Is this needed ? */ for (psci_fn = PSCI_FN_VERSION, psci_fnid = PSCI_FNID_VERSION; psci_fn < PSCI_FN_MAX; psci_fn++, psci_fnid++) sc->psci_fnids[psci_fn] = 0; /* PSCI v0.1 doesn't specify function IDs. Get them from DT */ node = ofw_bus_get_node(dev); if ((len = OF_getproplen(node, "cpu_suspend")) > 0) { OF_getencprop(node, "cpu_suspend", &psci_fnid, len); sc->psci_fnids[PSCI_FN_CPU_SUSPEND] = psci_fnid; } if ((len = OF_getproplen(node, "cpu_on")) > 0) { OF_getencprop(node, "cpu_on", &psci_fnid, len); sc->psci_fnids[PSCI_FN_CPU_ON] = psci_fnid; } if ((len = OF_getproplen(node, "cpu_off")) > 0) { OF_getencprop(node, "cpu_off", &psci_fnid, len); sc->psci_fnids[PSCI_FN_CPU_OFF] = psci_fnid; } if ((len = OF_getproplen(node, "migrate")) > 0) { OF_getencprop(node, "migrate", &psci_fnid, len); sc->psci_fnids[PSCI_FN_MIGRATE] = psci_fnid; } sc->psci_version = (0 << 16) | 1; if (bootverbose) device_printf(dev, "PSCI version 0.1 available\n"); return(0); } #endif static int psci_v0_2_init(device_t dev, int default_version) { struct psci_softc *sc = device_get_softc(dev); int version; /* PSCI v0.2 specifies explicit function IDs. */ sc->psci_fnids[PSCI_FN_VERSION] = PSCI_FNID_VERSION; sc->psci_fnids[PSCI_FN_CPU_SUSPEND] = PSCI_FNID_CPU_SUSPEND; sc->psci_fnids[PSCI_FN_CPU_OFF] = PSCI_FNID_CPU_OFF; sc->psci_fnids[PSCI_FN_CPU_ON] = PSCI_FNID_CPU_ON; sc->psci_fnids[PSCI_FN_AFFINITY_INFO] = PSCI_FNID_AFFINITY_INFO; sc->psci_fnids[PSCI_FN_MIGRATE] = PSCI_FNID_MIGRATE; sc->psci_fnids[PSCI_FN_MIGRATE_INFO_TYPE] = PSCI_FNID_MIGRATE_INFO_TYPE; sc->psci_fnids[PSCI_FN_MIGRATE_INFO_UP_CPU] = PSCI_FNID_MIGRATE_INFO_UP_CPU; sc->psci_fnids[PSCI_FN_SYSTEM_OFF] = PSCI_FNID_SYSTEM_OFF; sc->psci_fnids[PSCI_FN_SYSTEM_RESET] = PSCI_FNID_SYSTEM_RESET; version = _psci_get_version(sc); /* * U-Boot PSCI implementation doesn't have psci_get_version() * method implemented for many boards. In this case, use the version * readed from FDT as fallback. No fallback method for ACPI. */ if (version == PSCI_RETVAL_NOT_SUPPORTED) { if (default_version == PSCI_RETVAL_NOT_SUPPORTED) return (1); version = default_version; printf("PSCI get_version() function is not implemented, " " assuming v%d.%d\n", PSCI_VER_MAJOR(version), PSCI_VER_MINOR(version)); } sc->psci_version = version; if ((PSCI_VER_MAJOR(version) == 0 && PSCI_VER_MINOR(version) == 2) || PSCI_VER_MAJOR(version) == 1) { if (bootverbose) device_printf(dev, "PSCI version 0.2 compatible\n"); /* * We only register this for v0.2 since v0.1 doesn't support * system_reset. */ EVENTHANDLER_REGISTER(shutdown_final, psci_shutdown, sc, SHUTDOWN_PRI_LAST); /* Handle reboot after shutdown_panic. */ EVENTHANDLER_REGISTER(shutdown_final, psci_reboot, sc, SHUTDOWN_PRI_LAST + 150); return (0); } device_printf(dev, "PSCI version number mismatched with DT\n"); return (1); } diff --git a/sys/dev/psci/psci.h b/sys/dev/psci/psci.h index 451d40c0178d..6704eaf26c71 100644 --- a/sys/dev/psci/psci.h +++ b/sys/dev/psci/psci.h @@ -1,126 +1,127 @@ /*- * Copyright (c) 2013, 2014 Robin Randhawa * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_PSCI_H_ #define _MACHINE_PSCI_H_ #include #include #ifdef _KERNEL typedef int (*psci_initfn_t)(device_t dev, int default_version); typedef int (*psci_callfn_t)(register_t, register_t, register_t, register_t, register_t, register_t, register_t, register_t, struct arm_smccc_res *res); extern bool psci_present; int psci_cpu_on(unsigned long, unsigned long, unsigned long); +int psci_cpu_off(void); /* Operates on caller. */ void psci_reset(void); int32_t psci_features(uint32_t); int psci_get_version(void); /* Handler to let us call into the PSCI/SMCCC firmware */ extern psci_callfn_t psci_callfn; static inline int psci_call(register_t a, register_t b, register_t c, register_t d) { return (psci_callfn(a, b, c, d, 0, 0, 0, 0, NULL)); } #endif /* * PSCI return codes. */ #define PSCI_RETVAL_SUCCESS 0 #define PSCI_RETVAL_NOT_SUPPORTED -1 #define PSCI_RETVAL_INVALID_PARAMS -2 #define PSCI_RETVAL_DENIED -3 #define PSCI_RETVAL_ALREADY_ON -4 #define PSCI_RETVAL_ON_PENDING -5 #define PSCI_RETVAL_INTERNAL_FAILURE -6 #define PSCI_RETVAL_NOT_PRESENT -7 #define PSCI_RETVAL_DISABLED -8 /* * Used to signal PSCI is not available, e.g. to start a CPU. */ #define PSCI_MISSING 1 /* * PSCI function codes (as per PSCI v0.2). */ #ifdef __aarch64__ #define PSCI_FNID_VERSION 0x84000000 #define PSCI_FNID_CPU_SUSPEND 0xc4000001 #define PSCI_FNID_CPU_OFF 0x84000002 #define PSCI_FNID_CPU_ON 0xc4000003 #define PSCI_FNID_AFFINITY_INFO 0xc4000004 #define PSCI_FNID_MIGRATE 0xc4000005 #define PSCI_FNID_MIGRATE_INFO_TYPE 0x84000006 #define PSCI_FNID_MIGRATE_INFO_UP_CPU 0xc4000007 #define PSCI_FNID_SYSTEM_OFF 0x84000008 #define PSCI_FNID_SYSTEM_RESET 0x84000009 #define PSCI_FNID_FEATURES 0x8400000a #else #define PSCI_FNID_VERSION 0x84000000 #define PSCI_FNID_CPU_SUSPEND 0x84000001 #define PSCI_FNID_CPU_OFF 0x84000002 #define PSCI_FNID_CPU_ON 0x84000003 #define PSCI_FNID_AFFINITY_INFO 0x84000004 #define PSCI_FNID_MIGRATE 0x84000005 #define PSCI_FNID_MIGRATE_INFO_TYPE 0x84000006 #define PSCI_FNID_MIGRATE_INFO_UP_CPU 0x84000007 #define PSCI_FNID_SYSTEM_OFF 0x84000008 #define PSCI_FNID_SYSTEM_RESET 0x84000009 #define PSCI_FNID_FEATURES 0x8400000a #endif #define PSCI_VER_MAJOR(v) (((v) >> 16) & 0xFF) #define PSCI_VER_MINOR(v) ((v) & 0xFF) #define PSCI_VER(maj, min) (((maj) << 16) | (min)) #define PSCI_AFFINITY_INFO_ON 0 #define PSCI_AFFINITY_INFO_OFF 1 #define PSCI_AFFINITY_INFO_ON_PENDING 2 #ifdef _KERNEL enum psci_fn { PSCI_FN_VERSION, PSCI_FN_CPU_SUSPEND, PSCI_FN_CPU_OFF, PSCI_FN_CPU_ON, PSCI_FN_AFFINITY_INFO, PSCI_FN_MIGRATE, PSCI_FN_MIGRATE_INFO_TYPE, PSCI_FN_MIGRATE_INFO_UP_CPU, PSCI_FN_SYSTEM_OFF, PSCI_FN_SYSTEM_RESET, PSCI_FN_MAX }; #endif #endif /* _MACHINE_PSCI_H_ */