diff --git a/sys/arm64/iommu/iommu_pmap.c b/sys/arm64/iommu/iommu_pmap.c index 5f3873044f02..80a0515a0c2c 100644 --- a/sys/arm64/iommu/iommu_pmap.c +++ b/sys/arm64/iommu/iommu_pmap.c @@ -1,892 +1,899 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020-2021 Ruslan Bukin * Copyright (c) 2014-2021 Andrew Turner * Copyright (c) 2014-2016 The FreeBSD Foundation * All rights reserved. * * This work was supported by Innovate UK project 105694, "Digital Security * by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps for ARM SMMUv3 and ARM Mali GPU. */ #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define IOMMU_PAGE_SIZE 4096 #define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) #define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t))) #define NUL0E IOMMU_L0_ENTRIES #define NUL1E (NUL0E * NL1PG) #define NUL2E (NUL1E * NL2PG) -#define iommu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) -#define iommu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) -#define iommu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) +#define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) +#define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) +#define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) + +#define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK) +#define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK) +#define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK) +#define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK) /* This code assumes all L1 DMAP entries will be used */ CTASSERT((DMAP_MIN_ADDRESS & ~IOMMU_L0_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~IOMMU_L0_OFFSET) == DMAP_MAX_ADDRESS); static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex); -static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, +static void _smmu_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at * the same time as the CPU. */ -#define pmap_load(table) (*table) -#define pmap_clear(table) atomic_store_64(table, 0) -#define pmap_store(table, entry) atomic_store_64(table, entry) +#define smmu_pmap_load(table) (*table) +#define smmu_pmap_clear(table) atomic_store_64(table, 0) +#define smmu_pmap_store(table, entry) atomic_store_64(table, entry) /********************/ /* Inline functions */ /********************/ static __inline pd_entry_t * -pmap_l0(pmap_t pmap, vm_offset_t va) +smmu_pmap_l0(pmap_t pmap, vm_offset_t va) { - return (&pmap->pm_l0[iommu_l0_index(va)]); + return (&pmap->pm_l0[smmu_l0_index(va)]); } static __inline pd_entry_t * -pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) +smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) { pd_entry_t *l1; - l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); - return (&l1[iommu_l1_index(va)]); + l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK); + return (&l1[smmu_l1_index(va)]); } static __inline pd_entry_t * -pmap_l1(pmap_t pmap, vm_offset_t va) +smmu_pmap_l1(pmap_t pmap, vm_offset_t va) { pd_entry_t *l0; - l0 = pmap_l0(pmap, va); - if ((pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) + l0 = smmu_pmap_l0(pmap, va); + if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) return (NULL); - return (pmap_l0_to_l1(l0, va)); + return (smmu_pmap_l0_to_l1(l0, va)); } static __inline pd_entry_t * -pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) +smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) { pd_entry_t l1, *l2p; - l1 = pmap_load(l1p); + l1 = smmu_pmap_load(l1p); /* * The valid bit may be clear if pmap_update_entry() is concurrently * modifying the entry, so for KVA only the entry type may be checked. */ KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); - return (&l2p[iommu_l2_index(va)]); + return (&l2p[smmu_l2_index(va)]); } static __inline pd_entry_t * -pmap_l2(pmap_t pmap, vm_offset_t va) +smmu_pmap_l2(pmap_t pmap, vm_offset_t va) { pd_entry_t *l1; - l1 = pmap_l1(pmap, va); - if ((pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) + l1 = smmu_pmap_l1(pmap, va); + if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) return (NULL); - return (pmap_l1_to_l2(l1, va)); + return (smmu_pmap_l1_to_l2(l1, va)); } static __inline pt_entry_t * -pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) +smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) { pd_entry_t l2; pt_entry_t *l3p; - l2 = pmap_load(l2p); + l2 = smmu_pmap_load(l2p); /* * The valid bit may be clear if pmap_update_entry() is concurrently * modifying the entry, so for KVA only the entry type may be checked. */ KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); - return (&l3p[iommu_l3_index(va)]); + return (&l3p[smmu_l3_index(va)]); } /* * Returns the lowest valid pde for a given virtual address. * The next level may or may not point to a valid page or block. */ static __inline pd_entry_t * -pmap_pde(pmap_t pmap, vm_offset_t va, int *level) +smmu_pmap_pde(pmap_t pmap, vm_offset_t va, int *level) { pd_entry_t *l0, *l1, *l2, desc; - l0 = pmap_l0(pmap, va); - desc = pmap_load(l0) & ATTR_DESCR_MASK; + l0 = smmu_pmap_l0(pmap, va); + desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK; if (desc != IOMMU_L0_TABLE) { *level = -1; return (NULL); } - l1 = pmap_l0_to_l1(l0, va); - desc = pmap_load(l1) & ATTR_DESCR_MASK; + l1 = smmu_pmap_l0_to_l1(l0, va); + desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; if (desc != IOMMU_L1_TABLE) { *level = 0; return (l0); } - l2 = pmap_l1_to_l2(l1, va); - desc = pmap_load(l2) & ATTR_DESCR_MASK; + l2 = smmu_pmap_l1_to_l2(l1, va); + desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; if (desc != IOMMU_L2_TABLE) { *level = 1; return (l1); } *level = 2; return (l2); } /* * Returns the lowest valid pte block or table entry for a given virtual * address. If there are no valid entries return NULL and set the level to * the first invalid level. */ static __inline pt_entry_t * -pmap_pte(pmap_t pmap, vm_offset_t va, int *level) +smmu_pmap_pte(pmap_t pmap, vm_offset_t va, int *level) { pd_entry_t *l1, *l2, desc; pt_entry_t *l3; - l1 = pmap_l1(pmap, va); + l1 = smmu_pmap_l1(pmap, va); if (l1 == NULL) { *level = 0; return (NULL); } - desc = pmap_load(l1) & ATTR_DESCR_MASK; + desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; if (desc == IOMMU_L1_BLOCK) { *level = 1; return (l1); } if (desc != IOMMU_L1_TABLE) { *level = 1; return (NULL); } - l2 = pmap_l1_to_l2(l1, va); - desc = pmap_load(l2) & ATTR_DESCR_MASK; + l2 = smmu_pmap_l1_to_l2(l1, va); + desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; if (desc == IOMMU_L2_BLOCK) { *level = 2; return (l2); } if (desc != IOMMU_L2_TABLE) { *level = 2; return (NULL); } *level = 3; - l3 = pmap_l2_to_l3(l2, va); - if ((pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) + l3 = smmu_pmap_l2_to_l3(l2, va); + if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) return (NULL); return (l3); } static __inline int -pmap_l3_valid(pt_entry_t l3) +smmu_pmap_l3_valid(pt_entry_t l3) { return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE); } CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK); static __inline void -pmap_resident_count_inc(pmap_t pmap, int count) +smmu_pmap_resident_count_inc(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); pmap->pm_stats.resident_count += count; } static __inline void -pmap_resident_count_dec(pmap_t pmap, int count) +smmu_pmap_resident_count_dec(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(pmap->pm_stats.resident_count >= count, ("pmap %p resident count underflow %ld %d", pmap, pmap->pm_stats.resident_count, count)); pmap->pm_stats.resident_count -= count; } /*************************************************** * Page table page management routines..... ***************************************************/ /* * Schedule the specified unused page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void -pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, +smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, boolean_t set_PG_ZERO) { if (set_PG_ZERO) m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /*************************************************** * Low level mapping routines..... ***************************************************/ /* * Decrements a page table page's reference count, which is used to record the * number of valid page table entries within the page. If the reference count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static inline boolean_t -pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) +smmu_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, + struct spglist *free) { --m->ref_count; if (m->ref_count == 0) { - _pmap_unwire_l3(pmap, va, m, free); + _smmu_pmap_unwire_l3(pmap, va, m, free); return (TRUE); } else return (FALSE); } static void -_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) +_smmu_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, + struct spglist *free) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ if (m->pindex >= (NUL2E + NUL1E)) { /* l1 page */ pd_entry_t *l0; - l0 = pmap_l0(pmap, va); - pmap_clear(l0); + l0 = smmu_pmap_l0(pmap, va); + smmu_pmap_clear(l0); } else if (m->pindex >= NUL2E) { /* l2 page */ pd_entry_t *l1; - l1 = pmap_l1(pmap, va); - pmap_clear(l1); + l1 = smmu_pmap_l1(pmap, va); + smmu_pmap_clear(l1); } else { /* l3 page */ pd_entry_t *l2; - l2 = pmap_l2(pmap, va); - pmap_clear(l2); + l2 = smmu_pmap_l2(pmap, va); + smmu_pmap_clear(l2); } - pmap_resident_count_dec(pmap, 1); + smmu_pmap_resident_count_dec(pmap, 1); if (m->pindex < NUL2E) { /* We just released an l3, unhold the matching l2 */ pd_entry_t *l1, tl1; vm_page_t l2pg; - l1 = pmap_l1(pmap, va); - tl1 = pmap_load(l1); + l1 = smmu_pmap_l1(pmap, va); + tl1 = smmu_pmap_load(l1); l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); - pmap_unwire_l3(pmap, va, l2pg, free); + smmu_pmap_unwire_l3(pmap, va, l2pg, free); } else if (m->pindex < (NUL2E + NUL1E)) { /* We just released an l2, unhold the matching l1 */ pd_entry_t *l0, tl0; vm_page_t l1pg; - l0 = pmap_l0(pmap, va); - tl0 = pmap_load(l0); + l0 = smmu_pmap_l0(pmap, va); + tl0 = smmu_pmap_load(l0); l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); - pmap_unwire_l3(pmap, va, l1pg, free); + smmu_pmap_unwire_l3(pmap, va, l1pg, free); } /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ - pmap_add_delayed_free_list(m, free, TRUE); + smmu_pmap_add_delayed_free_list(m, free, TRUE); } static int -iommu_pmap_pinit_levels(pmap_t pmap, int levels) +smmu_pmap_pinit_levels(pmap_t pmap, int levels) { vm_page_t m; /* * allocate the l0 page */ m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED | VM_ALLOC_ZERO); pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); vm_radix_init(&pmap->pm_root); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); MPASS(levels == 3 || levels == 4); pmap->pm_levels = levels; /* * Allocate the level 1 entry to use as the root. This will increase * the refcount on the level 1 page so it won't be removed until * pmap_release() is called. */ if (pmap->pm_levels == 3) { PMAP_LOCK(pmap); m = _pmap_alloc_l3(pmap, NUL2E + NUL1E); PMAP_UNLOCK(pmap); } pmap->pm_ttbr = VM_PAGE_TO_PHYS(m); return (1); } int -iommu_pmap_pinit(pmap_t pmap) +smmu_pmap_pinit(pmap_t pmap) { - return (iommu_pmap_pinit_levels(pmap, 4)); + return (smmu_pmap_pinit_levels(pmap, 4)); } /* * This routine is called if the desired page table page does not exist. * * If page table page allocation fails, this routine may sleep before * returning NULL. It sleeps only if a lock pointer was given. * * Note: If a page allocation fails at page table level two or three, * one or two pages may be held during the wait, only to be released * afterwards. This conservative approach is easily argued to avoid * race conditions. */ static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex) { vm_page_t m, l1pg, l2pg; PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Allocate a page table page. */ if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { /* * Indicate the need to retry. While waiting, the page table * page may have been allocated. */ return (NULL); } m->pindex = ptepindex; /* * Because of AArch64's weak memory consistency model, we must have a * barrier here to ensure that the stores for zeroing "m", whether by * pmap_zero_page() or an earlier function, are visible before adding * "m" to the page table. Otherwise, a page table walk by another * processor's MMU could see the mapping to "m" and a stale, non-zero * PTE within "m". */ dmb(ishst); /* * Map the pagetable page into the process address space, if * it isn't already there. */ if (ptepindex >= (NUL2E + NUL1E)) { pd_entry_t *l0; vm_pindex_t l0index; l0index = ptepindex - (NUL2E + NUL1E); l0 = &pmap->pm_l0[l0index]; - pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); + smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); } else if (ptepindex >= NUL2E) { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1; pd_entry_t tl0; l1index = ptepindex - NUL2E; l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; l0 = &pmap->pm_l0[l0index]; - tl0 = pmap_load(l0); + tl0 = smmu_pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } } else { l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); l1pg->ref_count++; } - l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); + l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK); l1 = &l1[ptepindex & Ln_ADDR_MASK]; - pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); + smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); } else { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1, *l2; pd_entry_t tl0, tl1; l1index = ptepindex >> Ln_ENTRIES_SHIFT; l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; l0 = &pmap->pm_l0[l0index]; - tl0 = pmap_load(l0); + tl0 = smmu_pmap_load(l0); if (tl0 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } - tl0 = pmap_load(l0); + tl0 = smmu_pmap_load(l0); l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; } else { l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); l1 = &l1[l1index & Ln_ADDR_MASK]; - tl1 = pmap_load(l1); + tl1 = smmu_pmap_load(l1); if (tl1 == 0) { /* recurse for allocating page dir */ if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { vm_page_unwire_noq(m); vm_page_free_zero(m); return (NULL); } } else { l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); l2pg->ref_count++; } } - l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); + l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK); l2 = &l2[ptepindex & Ln_ADDR_MASK]; - pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); + smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); } - pmap_resident_count_inc(pmap, 1); + smmu_pmap_resident_count_inc(pmap, 1); return (m); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void -iommu_pmap_release(pmap_t pmap) +smmu_pmap_release(pmap_t pmap) { boolean_t rv __diagused; struct spglist free; vm_page_t m; if (pmap->pm_levels != 4) { KASSERT(pmap->pm_stats.resident_count == 1, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); KASSERT((pmap->pm_l0[0] & ATTR_DESCR_VALID) == ATTR_DESCR_VALID, ("pmap_release: Invalid l0 entry: %lx", pmap->pm_l0[0])); SLIST_INIT(&free); m = PHYS_TO_VM_PAGE(pmap->pm_ttbr); PMAP_LOCK(pmap); - rv = pmap_unwire_l3(pmap, 0, m, &free); + rv = smmu_pmap_unwire_l3(pmap, 0, m, &free); PMAP_UNLOCK(pmap); MPASS(rv == TRUE); vm_page_free_pages_toq(&free, true); } KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); KASSERT(vm_radix_is_empty(&pmap->pm_root), ("pmap_release: pmap has reserved page table page(s)")); m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr); vm_page_unwire_noq(m); vm_page_free_zero(m); } /*************************************************** * page management routines. ***************************************************/ /* * Add a single Mali GPU entry. This function does not sleep. */ int pmap_gpu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags) { pd_entry_t *pde; pt_entry_t new_l3; pt_entry_t orig_l3 __diagused; pt_entry_t *l3; vm_page_t mpte; pd_entry_t *l1p; pd_entry_t *l2p; int lvl; int rv; KASSERT(pmap != kernel_pmap, ("kernel pmap used for GPU")); KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK); if ((prot & VM_PROT_WRITE) != 0) new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); if ((prot & VM_PROT_READ) != 0) new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); if ((prot & VM_PROT_EXECUTE) == 0) new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL); CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa); PMAP_LOCK(pmap); /* * In the case that a page table page is not * resident, we are creating it here. */ retry: - pde = pmap_pde(pmap, va, &lvl); + pde = smmu_pmap_pde(pmap, va, &lvl); if (pde != NULL && lvl == 2) { - l3 = pmap_l2_to_l3(pde, va); + l3 = smmu_pmap_l2_to_l3(pde, va); } else { - mpte = _pmap_alloc_l3(pmap, iommu_l2_pindex(va)); + mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); if (mpte == NULL) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); rv = KERN_RESOURCE_SHORTAGE; goto out; } /* * Ensure newly created l1, l2 are visible to GPU. * l0 is already visible by similar call in panfrost driver. * The cache entry for l3 handled below. */ - l1p = pmap_l1(pmap, va); - l2p = pmap_l2(pmap, va); + l1p = smmu_pmap_l1(pmap, va); + l2p = smmu_pmap_l2(pmap, va); cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t)); cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t)); goto retry; } - orig_l3 = pmap_load(l3); - KASSERT(!pmap_l3_valid(orig_l3), ("l3 is valid")); + orig_l3 = smmu_pmap_load(l3); + KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); /* New mapping */ - pmap_store(l3, new_l3); + smmu_pmap_store(l3, new_l3); cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t)); - pmap_resident_count_inc(pmap, 1); + smmu_pmap_resident_count_inc(pmap, 1); dsb(ishst); rv = KERN_SUCCESS; out: PMAP_UNLOCK(pmap); return (rv); } /* * Remove a single Mali GPU entry. */ int pmap_gpu_remove(pmap_t pmap, vm_offset_t va) { pd_entry_t *pde; pt_entry_t *pte; int lvl; int rc; KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); KASSERT(pmap != kernel_pmap, ("kernel pmap used for GPU")); PMAP_LOCK(pmap); - pde = pmap_pde(pmap, va, &lvl); + pde = smmu_pmap_pde(pmap, va, &lvl); if (pde == NULL || lvl != 2) { rc = KERN_FAILURE; goto out; } - pte = pmap_l2_to_l3(pde, va); + pte = smmu_pmap_l2_to_l3(pde, va); - pmap_resident_count_dec(pmap, 1); - pmap_clear(pte); + smmu_pmap_resident_count_dec(pmap, 1); + smmu_pmap_clear(pte); cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t)); rc = KERN_SUCCESS; out: PMAP_UNLOCK(pmap); return (rc); } /* * Add a single SMMU entry. This function does not sleep. */ int -pmap_smmu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, +smmu_pmap_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags) { pd_entry_t *pde; pt_entry_t new_l3; pt_entry_t orig_l3 __diagused; pt_entry_t *l3; vm_page_t mpte; int lvl; int rv; KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); va = trunc_page(va); new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE); if ((prot & VM_PROT_WRITE) == 0) new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); new_l3 |= ATTR_S1_XN; /* Execute never. */ new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER); new_l3 |= ATTR_S1_nG; /* Non global. */ CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa); PMAP_LOCK(pmap); /* * In the case that a page table page is not * resident, we are creating it here. */ retry: - pde = pmap_pde(pmap, va, &lvl); + pde = smmu_pmap_pde(pmap, va, &lvl); if (pde != NULL && lvl == 2) { - l3 = pmap_l2_to_l3(pde, va); + l3 = smmu_pmap_l2_to_l3(pde, va); } else { - mpte = _pmap_alloc_l3(pmap, iommu_l2_pindex(va)); + mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); if (mpte == NULL) { CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); rv = KERN_RESOURCE_SHORTAGE; goto out; } goto retry; } - orig_l3 = pmap_load(l3); - KASSERT(!pmap_l3_valid(orig_l3), ("l3 is valid")); + orig_l3 = smmu_pmap_load(l3); + KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); /* New mapping */ - pmap_store(l3, new_l3); - pmap_resident_count_inc(pmap, 1); + smmu_pmap_store(l3, new_l3); + smmu_pmap_resident_count_inc(pmap, 1); dsb(ishst); rv = KERN_SUCCESS; out: PMAP_UNLOCK(pmap); return (rv); } /* * Remove a single SMMU entry. */ int -pmap_smmu_remove(pmap_t pmap, vm_offset_t va) +smmu_pmap_remove(pmap_t pmap, vm_offset_t va) { pt_entry_t *pte; int lvl; int rc; PMAP_LOCK(pmap); - pte = pmap_pte(pmap, va, &lvl); + pte = smmu_pmap_pte(pmap, va, &lvl); KASSERT(lvl == 3, ("Invalid SMMU pagetable level: %d != 3", lvl)); if (pte != NULL) { - pmap_resident_count_dec(pmap, 1); - pmap_clear(pte); + smmu_pmap_resident_count_dec(pmap, 1); + smmu_pmap_clear(pte); rc = KERN_SUCCESS; } else rc = KERN_FAILURE; PMAP_UNLOCK(pmap); return (rc); } /* * Remove all the allocated L1, L2 pages from SMMU pmap. * All the L3 entires must be cleared in advance, otherwise * this function panics. */ void -iommu_pmap_remove_pages(pmap_t pmap) +smmu_pmap_remove_pages(pmap_t pmap) { pd_entry_t l0e, *l1, l1e, *l2, l2e; pt_entry_t *l3, l3e; vm_page_t m, m0, m1; vm_offset_t sva; vm_paddr_t pa; vm_paddr_t pa0; vm_paddr_t pa1; int i, j, k, l; PMAP_LOCK(pmap); - for (sva = VM_MINUSER_ADDRESS, i = iommu_l0_index(sva); + for (sva = VM_MINUSER_ADDRESS, i = smmu_l0_index(sva); (i < Ln_ENTRIES && sva < VM_MAXUSER_ADDRESS); i++) { l0e = pmap->pm_l0[i]; if ((l0e & ATTR_DESCR_VALID) == 0) { sva += IOMMU_L0_SIZE; continue; } pa0 = l0e & ~ATTR_MASK; m0 = PHYS_TO_VM_PAGE(pa0); l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0); - for (j = iommu_l1_index(sva); j < Ln_ENTRIES; j++) { + for (j = smmu_l1_index(sva); j < Ln_ENTRIES; j++) { l1e = l1[j]; if ((l1e & ATTR_DESCR_VALID) == 0) { sva += IOMMU_L1_SIZE; continue; } if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) { sva += IOMMU_L1_SIZE; continue; } pa1 = l1e & ~ATTR_MASK; m1 = PHYS_TO_VM_PAGE(pa1); l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1); - for (k = iommu_l2_index(sva); k < Ln_ENTRIES; k++) { + for (k = smmu_l2_index(sva); k < Ln_ENTRIES; k++) { l2e = l2[k]; if ((l2e & ATTR_DESCR_VALID) == 0) { sva += IOMMU_L2_SIZE; continue; } pa = l2e & ~ATTR_MASK; m = PHYS_TO_VM_PAGE(pa); l3 = (pt_entry_t *)PHYS_TO_DMAP(pa); - for (l = iommu_l3_index(sva); l < Ln_ENTRIES; + for (l = smmu_l3_index(sva); l < Ln_ENTRIES; l++, sva += IOMMU_L3_SIZE) { l3e = l3[l]; if ((l3e & ATTR_DESCR_VALID) == 0) continue; panic("%s: l3e found for va %jx\n", __func__, sva); } vm_page_unwire_noq(m1); vm_page_unwire_noq(m); - pmap_resident_count_dec(pmap, 1); + smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m); - pmap_clear(&l2[k]); + smmu_pmap_clear(&l2[k]); } vm_page_unwire_noq(m0); - pmap_resident_count_dec(pmap, 1); + smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m1); - pmap_clear(&l1[j]); + smmu_pmap_clear(&l1[j]); } - pmap_resident_count_dec(pmap, 1); + smmu_pmap_resident_count_dec(pmap, 1); vm_page_free(m0); - pmap_clear(&pmap->pm_l0[i]); + smmu_pmap_clear(&pmap->pm_l0[i]); } KASSERT(pmap->pm_stats.resident_count == 0, ("Invalid resident count %jd", pmap->pm_stats.resident_count)); PMAP_UNLOCK(pmap); } diff --git a/sys/arm64/iommu/iommu_pmap.h b/sys/arm64/iommu/iommu_pmap.h index 98fb04971787..1b7027337242 100644 --- a/sys/arm64/iommu/iommu_pmap.h +++ b/sys/arm64/iommu/iommu_pmap.h @@ -1,51 +1,51 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Ruslan Bukin * * This work was supported by Innovate UK project 105694, "Digital Security * by Design (DSbD) Technology Platform Prototype". * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM64_IOMMU_IOMMU_PMAP_H_ #define _ARM64_IOMMU_IOMMU_PMAP_H_ /* System MMU (SMMU). */ -int pmap_smmu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, +int smmu_pmap_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags); -int pmap_smmu_remove(pmap_t pmap, vm_offset_t va); +int smmu_pmap_remove(pmap_t pmap, vm_offset_t va); /* Mali GPU */ int pmap_gpu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, vm_prot_t prot, u_int flags); int pmap_gpu_remove(pmap_t pmap, vm_offset_t va); /* Common */ -void iommu_pmap_remove_pages(pmap_t pmap); -void iommu_pmap_release(pmap_t pmap); -int iommu_pmap_pinit(pmap_t); +void smmu_pmap_remove_pages(pmap_t pmap); +void smmu_pmap_release(pmap_t pmap); +int smmu_pmap_pinit(pmap_t); #endif /* !_ARM64_IOMMU_IOMMU_PMAP_H_ */ diff --git a/sys/arm64/iommu/iommu_pte.h b/sys/arm64/iommu/iommu_pte.h index 814fb23df507..99f1ca30ec55 100644 --- a/sys/arm64/iommu/iommu_pte.h +++ b/sys/arm64/iommu/iommu_pte.h @@ -1,88 +1,83 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014-2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM64_IOMMU_IOMMU_PTE_H_ #define _ARM64_IOMMU_IOMMU_PTE_H_ /* Level 0 table, 512GiB per entry */ #define IOMMU_L0_SHIFT 39 #define IOMMU_L0_SIZE (1ul << IOMMU_L0_SHIFT) #define IOMMU_L0_OFFSET (IOMMU_L0_SIZE - 1ul) #define IOMMU_L0_INVAL 0x0 /* An invalid address */ /* 0x1 Level 0 doesn't support block translation */ /* 0x2 also marks an invalid address */ #define IOMMU_L0_TABLE 0x3 /* A next-level table */ /* Level 1 table, 1GiB per entry */ #define IOMMU_L1_SHIFT 30 #define IOMMU_L1_SIZE (1 << IOMMU_L1_SHIFT) #define IOMMU_L1_OFFSET (IOMMU_L1_SIZE - 1) #define IOMMU_L1_INVAL IOMMU_L0_INVAL #define IOMMU_L1_BLOCK 0x1 #define IOMMU_L1_TABLE IOMMU_L0_TABLE /* Level 2 table, 2MiB per entry */ #define IOMMU_L2_SHIFT 21 #define IOMMU_L2_SIZE (1 << IOMMU_L2_SHIFT) #define IOMMU_L2_OFFSET (IOMMU_L2_SIZE - 1) #define IOMMU_L2_INVAL IOMMU_L1_INVAL #define IOMMU_L2_BLOCK IOMMU_L1_BLOCK #define IOMMU_L2_TABLE IOMMU_L1_TABLE #define IOMMU_L2_BLOCK_MASK UINT64_C(0xffffffe00000) /* Level 3 table, 4KiB per entry */ #define IOMMU_L3_SHIFT 12 #define IOMMU_L3_SIZE (1 << IOMMU_L3_SHIFT) #define IOMMU_L3_OFFSET (IOMMU_L3_SIZE - 1) #define IOMMU_L3_SHIFT 12 #define IOMMU_L3_INVAL 0x0 /* 0x1 is reserved */ /* 0x2 also marks an invalid address */ #define IOMMU_L3_PAGE 0x3 #define IOMMU_L3_BLOCK IOMMU_L2_BLOCK /* Mali GPU only. */ #define IOMMU_L0_ENTRIES_SHIFT 9 #define IOMMU_L0_ENTRIES (1 << IOMMU_L0_ENTRIES_SHIFT) #define IOMMU_L0_ADDR_MASK (IOMMU_L0_ENTRIES - 1) #define IOMMU_Ln_ENTRIES_SHIFT 9 #define IOMMU_Ln_ENTRIES (1 << IOMMU_Ln_ENTRIES_SHIFT) #define IOMMU_Ln_ADDR_MASK (IOMMU_Ln_ENTRIES - 1) #define IOMMU_Ln_TABLE_MASK ((1 << 12) - 1) -#define iommu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK) -#define iommu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK) -#define iommu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK) -#define iommu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK) - #endif /* !_ARM64_IOMMU_IOMMU_PTE_H_ */ diff --git a/sys/arm64/iommu/smmu.c b/sys/arm64/iommu/smmu.c index e307440e725e..35c529ef5de6 100644 --- a/sys/arm64/iommu/smmu.c +++ b/sys/arm64/iommu/smmu.c @@ -1,2069 +1,2069 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019-2020 Ruslan Bukin * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Hardware overview. * * An incoming transaction from a peripheral device has an address, size, * attributes and StreamID. * * In case of PCI-based devices, StreamID is a PCI rid. * * The StreamID is used to select a Stream Table Entry (STE) in a Stream table, * which contains per-device configuration. * * Stream table is a linear or 2-level walk table (this driver supports both). * Note that a linear table could occupy 1GB or more of memory depending on * sid_bits value. * * STE is used to locate a Context Descriptor, which is a struct in memory * that describes stages of translation, translation table type, pointer to * level 0 of page tables, ASID, etc. * * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2): * o S1 is used for the host machine traffic translation * o S2 is for a hypervisor * * This driver enables S1 stage with standard AArch64 page tables. * * Note that SMMU does not share TLB with a main CPU. * Command queue is used by this driver to Invalidate SMMU TLB, STE cache. * * An arm64 SoC could have more than one SMMU instance. * ACPI IORT table describes which SMMU unit is assigned for a particular * peripheral device. * * Queues. * * Register interface and Memory-based circular buffer queues are used * to inferface SMMU. * * These are a Command queue for commands to send to the SMMU and an Event * queue for event/fault reports from the SMMU. Optionally PRI queue is * designed for PCIe page requests reception. * * Note that not every hardware supports PRI services. For instance they were * not found in Neoverse N1 SDP machine. * (This drivers does not implement PRI queue.) * * All SMMU queues are arranged as circular buffers in memory. They are used * in a producer-consumer fashion so that an output queue contains data * produced by the SMMU and consumed by software. * An input queue contains data produced by software, consumed by the SMMU. * * Interrupts. * * Interrupts are not required by this driver for normal operation. * The standard wired interrupt is only triggered when an event comes from * the SMMU, which is only in a case of errors (e.g. translation fault). */ #include "opt_platform.h" #include "opt_acpi.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_ACPI #include #include #endif #include #include #include #include #include #ifdef FDT #include #include #include #endif #include "iommu.h" #include "iommu_if.h" #include "smmureg.h" #include "smmuvar.h" #define STRTAB_L1_SZ_SHIFT 20 #define STRTAB_SPLIT 8 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6) #define STRTAB_L1_DESC_DWORDS 1 #define STRTAB_STE_DWORDS 8 #define CMDQ_ENTRY_DWORDS 2 #define EVTQ_ENTRY_DWORDS 4 #define PRIQ_ENTRY_DWORDS 2 #define CD_DWORDS 8 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2)) #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1)) #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */ #define SMMU_Q_ALIGN (64 * 1024) #define MAXADDR_48BIT 0xFFFFFFFFFFFFUL #define MAXADDR_52BIT 0xFFFFFFFFFFFFFUL static struct resource_spec smmu_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 2, RF_ACTIVE }, { SYS_RES_IRQ, 3, RF_ACTIVE }, RESOURCE_SPEC_END }; MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR); #define dprintf(fmt, ...) struct smmu_event { int ident; char *str; char *msg; }; static struct smmu_event events[] = { { 0x01, "F_UUT", "Unsupported Upstream Transaction."}, { 0x02, "C_BAD_STREAMID", "Transaction StreamID out of range."}, { 0x03, "F_STE_FETCH", "Fetch of STE caused external abort."}, { 0x04, "C_BAD_STE", "Used STE invalid."}, { 0x05, "F_BAD_ATS_TREQ", "Address Translation Request disallowed for a StreamID " "and a PCIe ATS Translation Request received."}, { 0x06, "F_STREAM_DISABLED", "The STE of a transaction marks non-substream transactions " "disabled."}, { 0x07, "F_TRANSL_FORBIDDEN", "An incoming PCIe transaction is marked Translated but " "SMMU bypass is disallowed for this StreamID."}, { 0x08, "C_BAD_SUBSTREAMID", "Incoming SubstreamID present, but configuration is invalid."}, { 0x09, "F_CD_FETCH", "Fetch of CD caused external abort."}, { 0x0a, "C_BAD_CD", "Fetched CD invalid."}, { 0x0b, "F_WALK_EABT", "An external abort occurred fetching (or updating) " "a translation table descriptor."}, { 0x10, "F_TRANSLATION", "Translation fault."}, { 0x11, "F_ADDR_SIZE", "Address Size fault."}, { 0x12, "F_ACCESS", "Access flag fault due to AF == 0 in a page or block TTD."}, { 0x13, "F_PERMISSION", "Permission fault occurred on page access."}, { 0x20, "F_TLB_CONFLICT", "A TLB conflict occurred because of the transaction."}, { 0x21, "F_CFG_CONFLICT", "A configuration cache conflict occurred due to " "the transaction."}, { 0x24, "E_PAGE_REQUEST", "Speculative page request hint."}, { 0x25, "F_VMS_FETCH", "Fetch of VMS caused external abort."}, { 0, NULL, NULL }, }; static int smmu_q_has_space(struct smmu_queue *q) { /* * See 6.3.27 SMMU_CMDQ_PROD * * There is space in the queue for additional commands if: * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR || * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP */ if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) || Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) return (1); return (0); } static int smmu_q_empty(struct smmu_queue *q) { if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) && Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod)) return (1); return (0); } static int __unused smmu_q_consumed(struct smmu_queue *q, uint32_t prod) { if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) && (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod))) return (1); if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) && (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod))) return (1); return (0); } static uint32_t smmu_q_inc_cons(struct smmu_queue *q) { uint32_t cons; uint32_t val; cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1; val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons)); return (val); } static uint32_t smmu_q_inc_prod(struct smmu_queue *q) { uint32_t prod; uint32_t val; prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1; val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod)); return (val); } static int smmu_write_ack(struct smmu_softc *sc, uint32_t reg, uint32_t reg_ack, uint32_t val) { uint32_t v; int timeout; timeout = 100000; bus_write_4(sc->res[0], reg, val); do { v = bus_read_4(sc->res[0], reg_ack); if (v == val) break; } while (timeout--); if (timeout <= 0) { device_printf(sc->dev, "Failed to write reg.\n"); return (-1); } return (0); } static inline int ilog2(long x) { KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x)); return (flsl(x) - 1); } static int smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q, uint32_t prod_off, uint32_t cons_off, uint32_t dwords) { int sz; sz = (1 << q->size_log2) * dwords * 8; /* Set up the command circular buffer */ q->vaddr = contigmalloc(sz, M_SMMU, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0); if (q->vaddr == NULL) { device_printf(sc->dev, "failed to allocate %d bytes\n", sz); return (-1); } q->prod_off = prod_off; q->cons_off = cons_off; q->paddr = vtophys(q->vaddr); q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA; q->base |= q->paddr & Q_BASE_ADDR_M; q->base |= q->size_log2 << Q_LOG2SIZE_S; return (0); } static int smmu_init_queues(struct smmu_softc *sc) { int err; /* Command queue. */ err = smmu_init_queue(sc, &sc->cmdq, SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS); if (err) return (ENXIO); /* Event queue. */ err = smmu_init_queue(sc, &sc->evtq, SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS); if (err) return (ENXIO); if (!(sc->features & SMMU_FEATURE_PRI)) return (0); /* PRI queue. */ err = smmu_init_queue(sc, &sc->priq, SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS); if (err) return (ENXIO); return (0); } /* * Dump 2LVL or linear STE. */ static void smmu_dump_ste(struct smmu_softc *sc, int sid) { struct smmu_strtab *strtab; struct l1_desc *l1_desc; uint64_t *ste, *l1; int i; strtab = &sc->strtab; if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { i = sid >> STRTAB_SPLIT; l1 = (void *)((uint64_t)strtab->vaddr + STRTAB_L1_DESC_DWORDS * 8 * i); device_printf(sc->dev, "L1 ste == %lx\n", l1[0]); l1_desc = &strtab->l1[i]; ste = l1_desc->va; if (ste == NULL) /* L2 is not initialized */ return; } else { ste = (void *)((uint64_t)strtab->vaddr + sid * (STRTAB_STE_DWORDS << 3)); } /* Dump L2 or linear STE. */ for (i = 0; i < STRTAB_STE_DWORDS; i++) device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]); } static void __unused smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd) { uint64_t *vaddr; int i; device_printf(sc->dev, "%s\n", __func__); vaddr = cd->vaddr; for (i = 0; i < CD_DWORDS; i++) device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]); } static void smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt) { struct smmu_queue *evtq; void *entry_addr; evtq = &sc->evtq; evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off); entry_addr = (void *)((uint64_t)evtq->vaddr + evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8); memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8); evtq->lc.cons = smmu_q_inc_cons(evtq); bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons); } static void smmu_print_event(struct smmu_softc *sc, uint32_t *evt) { struct smmu_event *ev; uintptr_t input_addr; uint8_t event_id; device_t dev; int sid; int i; dev = sc->dev; ev = NULL; event_id = evt[0] & 0xff; for (i = 0; events[i].ident != 0; i++) { if (events[i].ident == event_id) { ev = &events[i]; break; } } sid = evt[1]; input_addr = evt[5]; input_addr <<= 32; input_addr |= evt[4]; if (smmu_quirks_check(dev, sid, event_id, input_addr)) { /* The event is known. Don't print anything. */ return; } if (ev) { device_printf(sc->dev, "Event %s (%s) received.\n", ev->str, ev->msg); } else device_printf(sc->dev, "Event 0x%x received\n", event_id); device_printf(sc->dev, "SID %x, Input Address: %jx\n", sid, input_addr); for (i = 0; i < 8; i++) device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]); smmu_dump_ste(sc, sid); } static void make_cmd(struct smmu_softc *sc, uint64_t *cmd, struct smmu_cmdq_entry *entry) { memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8); cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S; switch (entry->opcode) { case CMD_TLBI_NH_VA: cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M; if (entry->tlbi.leaf) { /* * Leaf flag means that only cached entries * for the last level of translation table walk * are required to be invalidated. */ cmd[1] |= TLBI_1_LEAF; } break; case CMD_TLBI_NH_ASID: cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S; break; case CMD_TLBI_NSNH_ALL: case CMD_TLBI_NH_ALL: case CMD_TLBI_EL2_ALL: break; case CMD_CFGI_CD: cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S); /* FALLTROUGH */ case CMD_CFGI_STE: cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S); cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S); break; case CMD_CFGI_STE_RANGE: cmd[1] = (31 << CFGI_1_STE_RANGE_S); break; case CMD_SYNC: cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB; if (entry->sync.msiaddr) { cmd[0] |= SYNC_0_CS_SIG_IRQ; cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M); } else cmd[0] |= SYNC_0_CS_SIG_SEV; break; case CMD_PREFETCH_CONFIG: cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S); break; }; } static void smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry) { uint64_t cmd[CMDQ_ENTRY_DWORDS]; struct smmu_queue *cmdq; void *entry_addr; cmdq = &sc->cmdq; make_cmd(sc, cmd, entry); SMMU_LOCK(sc); /* Ensure that a space is available. */ do { cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off); } while (smmu_q_has_space(cmdq) == 0); /* Write the command to the current prod entry. */ entry_addr = (void *)((uint64_t)cmdq->vaddr + Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8); memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8); /* Increment prod index. */ cmdq->lc.prod = smmu_q_inc_prod(cmdq); bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod); SMMU_UNLOCK(sc); } static void __unused smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q) { while (1) { q->lc.val = bus_read_8(sc->res[0], q->prod_off); if (smmu_q_empty(q)) break; cpu_spinwait(); } } static int smmu_sync(struct smmu_softc *sc) { struct smmu_cmdq_entry cmd; struct smmu_queue *q; uint32_t *base; int timeout; int prod; q = &sc->cmdq; prod = q->lc.prod; /* Enqueue sync command. */ cmd.opcode = CMD_SYNC; cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8; smmu_cmdq_enqueue_cmd(sc, &cmd); /* Wait for the sync completion. */ base = (void *)((uint64_t)q->vaddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8); /* * It takes around 200 loops (6 instructions each) * on Neoverse N1 to complete the sync. */ timeout = 10000; do { if (*base == 0) { /* MSI write completed. */ break; } cpu_spinwait(); } while (timeout--); if (timeout < 0) device_printf(sc->dev, "Failed to sync\n"); return (0); } static int smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf) { struct smmu_cmdq_entry cmd; cmd.opcode = CMD_CFGI_CD; cmd.cfgi.sid = sid; cmd.cfgi.ssid = ssid; cmd.cfgi.leaf = leaf; smmu_cmdq_enqueue_cmd(sc, &cmd); return (0); } static void smmu_invalidate_all_sid(struct smmu_softc *sc) { struct smmu_cmdq_entry cmd; /* Invalidate cached config */ cmd.opcode = CMD_CFGI_STE_RANGE; smmu_cmdq_enqueue_cmd(sc, &cmd); smmu_sync(sc); } static void smmu_tlbi_all(struct smmu_softc *sc) { struct smmu_cmdq_entry cmd; /* Invalidate entire TLB */ cmd.opcode = CMD_TLBI_NSNH_ALL; smmu_cmdq_enqueue_cmd(sc, &cmd); smmu_sync(sc); } static void smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid) { struct smmu_cmdq_entry cmd; /* Invalidate TLB for an ASID. */ cmd.opcode = CMD_TLBI_NH_ASID; cmd.tlbi.asid = asid; smmu_cmdq_enqueue_cmd(sc, &cmd); smmu_sync(sc); } static void smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid) { struct smmu_cmdq_entry cmd; /* Invalidate specific range */ cmd.opcode = CMD_TLBI_NH_VA; cmd.tlbi.asid = asid; cmd.tlbi.vmid = 0; cmd.tlbi.leaf = true; /* We change only L3. */ cmd.tlbi.addr = va; smmu_cmdq_enqueue_cmd(sc, &cmd); } static void smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid) { struct smmu_cmdq_entry cmd; /* Invalidate cached config */ cmd.opcode = CMD_CFGI_STE; cmd.cfgi.sid = sid; smmu_cmdq_enqueue_cmd(sc, &cmd); smmu_sync(sc); } static void smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid) { struct smmu_cmdq_entry cmd; cmd.opcode = CMD_PREFETCH_CONFIG; cmd.prefetch.sid = sid; smmu_cmdq_enqueue_cmd(sc, &cmd); smmu_sync(sc); } /* * Init STE in bypass mode. Traffic is not translated for the sid. */ static void smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste) { uint64_t val; val = STE0_VALID | STE0_CONFIG_BYPASS; ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS; ste[2] = 0; ste[3] = 0; ste[4] = 0; ste[5] = 0; ste[6] = 0; ste[7] = 0; smmu_invalidate_sid(sc, sid); ste[0] = val; dsb(sy); smmu_invalidate_sid(sc, sid); smmu_prefetch_sid(sc, sid); } /* * Enable Stage1 (S1) translation for the sid. */ static int smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd, uint32_t sid, uint64_t *ste) { uint64_t val; val = STE0_VALID; /* S1 */ ste[1] = STE1_EATS_FULLATS | STE1_S1CSH_IS | STE1_S1CIR_WBRA | STE1_S1COR_WBRA | STE1_STRW_NS_EL1; ste[2] = 0; ste[3] = 0; ste[4] = 0; ste[5] = 0; ste[6] = 0; ste[7] = 0; if (sc->features & SMMU_FEATURE_STALL && ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0)) ste[1] |= STE1_S1STALLD; /* Configure STE */ val |= (cd->paddr & STE0_S1CONTEXTPTR_M); val |= STE0_CONFIG_S1_TRANS; smmu_invalidate_sid(sc, sid); /* The STE[0] has to be written in a single blast, last of all. */ ste[0] = val; dsb(sy); smmu_invalidate_sid(sc, sid); smmu_sync_cd(sc, sid, 0, true); smmu_invalidate_sid(sc, sid); /* The sid will be used soon most likely. */ smmu_prefetch_sid(sc, sid); return (0); } static uint64_t * smmu_get_ste_addr(struct smmu_softc *sc, int sid) { struct smmu_strtab *strtab; struct l1_desc *l1_desc; uint64_t *addr; strtab = &sc->strtab; if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; addr = l1_desc->va; addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; } else { addr = (void *)((uint64_t)strtab->vaddr + STRTAB_STE_DWORDS * 8 * sid); }; return (addr); } static int smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass) { uint64_t *addr; addr = smmu_get_ste_addr(sc, sid); if (bypass) smmu_init_ste_bypass(sc, sid, addr); else smmu_init_ste_s1(sc, cd, sid, addr); smmu_sync(sc); return (0); } static void smmu_deinit_ste(struct smmu_softc *sc, int sid) { uint64_t *ste; ste = smmu_get_ste_addr(sc, sid); ste[0] = 0; smmu_invalidate_sid(sc, sid); smmu_sync_cd(sc, sid, 0, true); smmu_invalidate_sid(sc, sid); smmu_sync(sc); } static int smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain) { vm_paddr_t paddr; uint64_t *ptr; uint64_t val; vm_size_t size; struct smmu_cd *cd; pmap_t p; size = 1 * (CD_DWORDS << 3); p = &domain->p; cd = domain->cd = malloc(sizeof(struct smmu_cd), M_SMMU, M_WAITOK | M_ZERO); cd->vaddr = contigmalloc(size, M_SMMU, M_WAITOK | M_ZERO, /* flags */ 0, /* low */ (1ul << 40) - 1, /* high */ size, /* alignment */ 0); /* boundary */ if (cd->vaddr == NULL) { device_printf(sc->dev, "Failed to allocate CD\n"); return (ENXIO); } cd->size = size; cd->paddr = vtophys(cd->vaddr); ptr = cd->vaddr; val = CD0_VALID; val |= CD0_AA64; val |= CD0_R; val |= CD0_A; val |= CD0_ASET; val |= (uint64_t)domain->asid << CD0_ASID_S; val |= CD0_TG0_4KB; val |= CD0_EPD1; /* Disable TT1 */ val |= ((64 - sc->ias) << CD0_T0SZ_S); val |= CD0_IPS_48BITS; paddr = p->pm_l0_paddr & CD1_TTB0_M; KASSERT(paddr == p->pm_l0_paddr, ("bad allocation 1")); ptr[1] = paddr; ptr[2] = 0; ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH); /* Install the CD. */ ptr[0] = val; return (0); } static int smmu_init_strtab_linear(struct smmu_softc *sc) { struct smmu_strtab *strtab; vm_paddr_t base; uint32_t size; uint64_t reg; strtab = &sc->strtab; strtab->num_l1_entries = (1 << sc->sid_bits); size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3); if (bootverbose) device_printf(sc->dev, "%s: linear strtab size %d, num_l1_entries %d\n", __func__, size, strtab->num_l1_entries); strtab->vaddr = contigmalloc(size, M_SMMU, M_WAITOK | M_ZERO, /* flags */ 0, /* low */ (1ul << 48) - 1, /* high */ size, /* alignment */ 0); /* boundary */ if (strtab->vaddr == NULL) { device_printf(sc->dev, "failed to allocate strtab\n"); return (ENXIO); } reg = STRTAB_BASE_CFG_FMT_LINEAR; reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S; strtab->base_cfg = (uint32_t)reg; base = vtophys(strtab->vaddr); reg = base & STRTAB_BASE_ADDR_M; KASSERT(reg == base, ("bad allocation 2")); reg |= STRTAB_BASE_RA; strtab->base = reg; return (0); } static int smmu_init_strtab_2lvl(struct smmu_softc *sc) { struct smmu_strtab *strtab; vm_paddr_t base; uint64_t reg_base; uint32_t l1size; uint32_t size; uint32_t reg; int sz; strtab = &sc->strtab; size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); size = min(size, sc->sid_bits - STRTAB_SPLIT); strtab->num_l1_entries = (1 << size); size += STRTAB_SPLIT; l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3); if (bootverbose) device_printf(sc->dev, "%s: size %d, l1 entries %d, l1size %d\n", __func__, size, strtab->num_l1_entries, l1size); strtab->vaddr = contigmalloc(l1size, M_SMMU, M_WAITOK | M_ZERO, /* flags */ 0, /* low */ (1ul << 48) - 1, /* high */ l1size, /* alignment */ 0); /* boundary */ if (strtab->vaddr == NULL) { device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n"); return (ENOMEM); } sz = strtab->num_l1_entries * sizeof(struct l1_desc); strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO); if (strtab->l1 == NULL) { contigfree(strtab->vaddr, l1size, M_SMMU); return (ENOMEM); } reg = STRTAB_BASE_CFG_FMT_2LVL; reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S; reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S; strtab->base_cfg = (uint32_t)reg; base = vtophys(strtab->vaddr); reg_base = base & STRTAB_BASE_ADDR_M; KASSERT(reg_base == base, ("bad allocation 3")); reg_base |= STRTAB_BASE_RA; strtab->base = reg_base; return (0); } static int smmu_init_strtab(struct smmu_softc *sc) { int error; if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) error = smmu_init_strtab_2lvl(sc); else error = smmu_init_strtab_linear(sc); return (error); } static int smmu_init_l1_entry(struct smmu_softc *sc, int sid) { struct smmu_strtab *strtab; struct l1_desc *l1_desc; uint64_t *addr; uint64_t val; size_t size; int i; strtab = &sc->strtab; l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; if (l1_desc->va) { /* Already allocated. */ return (0); } size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); l1_desc->span = STRTAB_SPLIT + 1; l1_desc->size = size; l1_desc->va = contigmalloc(size, M_SMMU, M_WAITOK | M_ZERO, /* flags */ 0, /* low */ (1ul << 48) - 1, /* high */ size, /* alignment */ 0); /* boundary */ if (l1_desc->va == NULL) { device_printf(sc->dev, "failed to allocate l2 entry\n"); return (ENXIO); } l1_desc->pa = vtophys(l1_desc->va); i = sid >> STRTAB_SPLIT; addr = (void *)((uint64_t)strtab->vaddr + STRTAB_L1_DESC_DWORDS * 8 * i); /* Install the L1 entry. */ val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M; KASSERT(val == l1_desc->pa, ("bad allocation 4")); val |= l1_desc->span; *addr = val; return (0); } static void __unused smmu_deinit_l1_entry(struct smmu_softc *sc, int sid) { struct smmu_strtab *strtab; struct l1_desc *l1_desc; uint64_t *addr; int i; strtab = &sc->strtab; i = sid >> STRTAB_SPLIT; addr = (void *)((uint64_t)strtab->vaddr + STRTAB_L1_DESC_DWORDS * 8 * i); *addr = 0; l1_desc = &strtab->l1[sid >> STRTAB_SPLIT]; contigfree(l1_desc->va, l1_desc->size, M_SMMU); } static int smmu_disable(struct smmu_softc *sc) { uint32_t reg; int error; /* Disable SMMU */ reg = bus_read_4(sc->res[0], SMMU_CR0); reg &= ~CR0_SMMUEN; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) device_printf(sc->dev, "Could not disable SMMU.\n"); return (0); } static int smmu_event_intr(void *arg) { uint32_t evt[EVTQ_ENTRY_DWORDS * 2]; struct smmu_softc *sc; sc = arg; do { smmu_evtq_dequeue(sc, evt); smmu_print_event(sc, evt); } while (!smmu_q_empty(&sc->evtq)); return (FILTER_HANDLED); } static int __unused smmu_sync_intr(void *arg) { struct smmu_softc *sc; sc = arg; device_printf(sc->dev, "%s\n", __func__); return (FILTER_HANDLED); } static int smmu_gerr_intr(void *arg) { struct smmu_softc *sc; sc = arg; device_printf(sc->dev, "SMMU Global Error\n"); return (FILTER_HANDLED); } static int smmu_enable_interrupts(struct smmu_softc *sc) { uint32_t reg; int error; /* Disable MSI. */ bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0); bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0); bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0); bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0); bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0); bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0); if (sc->features & CR0_PRIQEN) { bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0); bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0); bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0); } /* Disable any interrupts. */ error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0); if (error) { device_printf(sc->dev, "Could not disable interrupts.\n"); return (ENXIO); } /* Enable interrupts. */ reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; if (sc->features & SMMU_FEATURE_PRI) reg |= IRQ_CTRL_PRIQ_IRQEN; error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg); if (error) { device_printf(sc->dev, "Could not enable interrupts.\n"); return (ENXIO); } return (0); } #ifdef DEV_ACPI static void smmu_configure_intr(struct smmu_softc *sc, struct resource *res) { struct intr_map_data_acpi *ad; struct intr_map_data *data; data = rman_get_virtual(res); KASSERT(data != NULL, ("data is NULL")); if (data->type == INTR_MAP_DATA_ACPI) { ad = (struct intr_map_data_acpi *)data; ad->trig = INTR_TRIGGER_EDGE; ad->pol = INTR_POLARITY_HIGH; } } #endif static int smmu_setup_interrupts(struct smmu_softc *sc) { device_t dev; int error; dev = sc->dev; #ifdef DEV_ACPI /* * Configure SMMU interrupts as EDGE triggered manually * as ACPI tables carries no information for that. */ smmu_configure_intr(sc, sc->res[1]); /* PRIQ is not in use. */ smmu_configure_intr(sc, sc->res[3]); smmu_configure_intr(sc, sc->res[4]); #endif error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC, smmu_event_intr, NULL, sc, &sc->intr_cookie[0]); if (error) { device_printf(dev, "Couldn't setup Event interrupt handler\n"); return (ENXIO); } error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC, smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]); if (error) { device_printf(dev, "Couldn't setup Gerr interrupt handler\n"); return (ENXIO); } return (0); } static int smmu_reset(struct smmu_softc *sc) { struct smmu_cmdq_entry cmd; struct smmu_strtab *strtab; int error; int reg; reg = bus_read_4(sc->res[0], SMMU_CR0); if (reg & CR0_SMMUEN) device_printf(sc->dev, "%s: Warning: SMMU is enabled\n", __func__); error = smmu_disable(sc); if (error) device_printf(sc->dev, "%s: Could not disable SMMU.\n", __func__); if (smmu_enable_interrupts(sc) != 0) { device_printf(sc->dev, "Could not enable interrupts.\n"); return (ENXIO); } reg = CR1_TABLE_SH_IS | CR1_TABLE_OC_WBC | CR1_TABLE_IC_WBC | CR1_QUEUE_SH_IS | CR1_QUEUE_OC_WBC | CR1_QUEUE_IC_WBC; bus_write_4(sc->res[0], SMMU_CR1, reg); reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; bus_write_4(sc->res[0], SMMU_CR2, reg); /* Stream table. */ strtab = &sc->strtab; bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base); bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg); /* Command queue. */ bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base); bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod); bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons); reg = CR0_CMDQEN; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) { device_printf(sc->dev, "Could not enable command queue\n"); return (ENXIO); } /* Invalidate cached configuration. */ smmu_invalidate_all_sid(sc); if (sc->features & SMMU_FEATURE_HYP) { cmd.opcode = CMD_TLBI_EL2_ALL; smmu_cmdq_enqueue_cmd(sc, &cmd); }; /* Invalidate TLB. */ smmu_tlbi_all(sc); /* Event queue */ bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base); bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod); bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons); reg |= CR0_EVENTQEN; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) { device_printf(sc->dev, "Could not enable event queue\n"); return (ENXIO); } if (sc->features & SMMU_FEATURE_PRI) { /* PRI queue */ bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base); bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod); bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons); reg |= CR0_PRIQEN; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) { device_printf(sc->dev, "Could not enable PRI queue\n"); return (ENXIO); } } if (sc->features & SMMU_FEATURE_ATS) { reg |= CR0_ATSCHK; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) { device_printf(sc->dev, "Could not enable ATS check.\n"); return (ENXIO); } } reg |= CR0_SMMUEN; error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg); if (error) { device_printf(sc->dev, "Could not enable SMMU.\n"); return (ENXIO); } return (0); } static int smmu_check_features(struct smmu_softc *sc) { uint32_t reg; uint32_t val; sc->features = 0; reg = bus_read_4(sc->res[0], SMMU_IDR0); if (reg & IDR0_ST_LVL_2) { if (bootverbose) device_printf(sc->dev, "2-level stream table supported.\n"); sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE; } if (reg & IDR0_CD2L) { if (bootverbose) device_printf(sc->dev, "2-level CD table supported.\n"); sc->features |= SMMU_FEATURE_2_LVL_CD; } switch (reg & IDR0_TTENDIAN_M) { case IDR0_TTENDIAN_MIXED: if (bootverbose) device_printf(sc->dev, "Mixed endianess supported.\n"); sc->features |= SMMU_FEATURE_TT_LE; sc->features |= SMMU_FEATURE_TT_BE; break; case IDR0_TTENDIAN_LITTLE: if (bootverbose) device_printf(sc->dev, "Little endian supported only.\n"); sc->features |= SMMU_FEATURE_TT_LE; break; case IDR0_TTENDIAN_BIG: if (bootverbose) device_printf(sc->dev, "Big endian supported only.\n"); sc->features |= SMMU_FEATURE_TT_BE; break; default: device_printf(sc->dev, "Unsupported endianness.\n"); return (ENXIO); } if (reg & IDR0_SEV) sc->features |= SMMU_FEATURE_SEV; if (reg & IDR0_MSI) { if (bootverbose) device_printf(sc->dev, "MSI feature present.\n"); sc->features |= SMMU_FEATURE_MSI; } if (reg & IDR0_HYP) { if (bootverbose) device_printf(sc->dev, "HYP feature present.\n"); sc->features |= SMMU_FEATURE_HYP; } if (reg & IDR0_ATS) sc->features |= SMMU_FEATURE_ATS; if (reg & IDR0_PRI) sc->features |= SMMU_FEATURE_PRI; switch (reg & IDR0_STALL_MODEL_M) { case IDR0_STALL_MODEL_FORCE: /* Stall is forced. */ sc->features |= SMMU_FEATURE_STALL_FORCE; /* FALLTHROUGH */ case IDR0_STALL_MODEL_STALL: sc->features |= SMMU_FEATURE_STALL; break; } /* Grab translation stages supported. */ if (reg & IDR0_S1P) { if (bootverbose) device_printf(sc->dev, "Stage 1 translation supported.\n"); sc->features |= SMMU_FEATURE_S1P; } if (reg & IDR0_S2P) { if (bootverbose) device_printf(sc->dev, "Stage 2 translation supported.\n"); sc->features |= SMMU_FEATURE_S2P; } switch (reg & IDR0_TTF_M) { case IDR0_TTF_ALL: case IDR0_TTF_AA64: sc->ias = 40; break; default: device_printf(sc->dev, "No AArch64 table format support.\n"); return (ENXIO); } if (reg & IDR0_ASID16) sc->asid_bits = 16; else sc->asid_bits = 8; if (bootverbose) device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits); if (reg & IDR0_VMID16) sc->vmid_bits = 16; else sc->vmid_bits = 8; reg = bus_read_4(sc->res[0], SMMU_IDR1); if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { device_printf(sc->dev, "Embedded implementations not supported by this driver.\n"); return (ENXIO); } val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S; sc->cmdq.size_log2 = val; if (bootverbose) device_printf(sc->dev, "CMD queue bits %d\n", val); val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S; sc->evtq.size_log2 = val; if (bootverbose) device_printf(sc->dev, "EVENT queue bits %d\n", val); if (sc->features & SMMU_FEATURE_PRI) { val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S; sc->priq.size_log2 = val; if (bootverbose) device_printf(sc->dev, "PRI queue bits %d\n", val); } sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S; sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S; if (sc->sid_bits <= STRTAB_SPLIT) sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE; if (bootverbose) { device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits); device_printf(sc->dev, "SID bits %d\n", sc->sid_bits); } /* IDR3 */ reg = bus_read_4(sc->res[0], SMMU_IDR3); if (reg & IDR3_RIL) sc->features |= SMMU_FEATURE_RANGE_INV; /* IDR5 */ reg = bus_read_4(sc->res[0], SMMU_IDR5); switch (reg & IDR5_OAS_M) { case IDR5_OAS_32: sc->oas = 32; break; case IDR5_OAS_36: sc->oas = 36; break; case IDR5_OAS_40: sc->oas = 40; break; case IDR5_OAS_42: sc->oas = 42; break; case IDR5_OAS_44: sc->oas = 44; break; case IDR5_OAS_48: sc->oas = 48; break; case IDR5_OAS_52: sc->oas = 52; break; } sc->pgsizes = 0; if (reg & IDR5_GRAN64K) sc->pgsizes |= 64 * 1024; if (reg & IDR5_GRAN16K) sc->pgsizes |= 16 * 1024; if (reg & IDR5_GRAN4K) sc->pgsizes |= 4 * 1024; if ((reg & IDR5_VAX_M) == IDR5_VAX_52) sc->features |= SMMU_FEATURE_VAX; return (0); } static void smmu_init_asids(struct smmu_softc *sc) { sc->asid_set_size = (1 << sc->asid_bits); sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK); mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN); } static int smmu_asid_alloc(struct smmu_softc *sc, int *new_asid) { mtx_lock_spin(&sc->asid_set_mutex); bit_ffc(sc->asid_set, sc->asid_set_size, new_asid); if (*new_asid == -1) { mtx_unlock_spin(&sc->asid_set_mutex); return (ENOMEM); } bit_set(sc->asid_set, *new_asid); mtx_unlock_spin(&sc->asid_set_mutex); return (0); } static void smmu_asid_free(struct smmu_softc *sc, int asid) { mtx_lock_spin(&sc->asid_set_mutex); bit_clear(sc->asid_set, asid); mtx_unlock_spin(&sc->asid_set_mutex); } /* * Device interface. */ int smmu_attach(device_t dev) { struct smmu_softc *sc; int error; sc = device_get_softc(dev); sc->dev = dev; mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF); error = smmu_setup_interrupts(sc); if (error) { bus_release_resources(dev, smmu_spec, sc->res); return (ENXIO); } error = smmu_check_features(sc); if (error) { device_printf(dev, "Some features are required " "but not supported by hardware.\n"); return (ENXIO); } smmu_init_asids(sc); error = smmu_init_queues(sc); if (error) { device_printf(dev, "Couldn't allocate queues.\n"); return (ENXIO); } error = smmu_init_strtab(sc); if (error) { device_printf(dev, "Couldn't allocate strtab.\n"); return (ENXIO); } error = smmu_reset(sc); if (error) { device_printf(dev, "Couldn't reset SMMU.\n"); return (ENXIO); } return (0); } int smmu_detach(device_t dev) { struct smmu_softc *sc; sc = device_get_softc(dev); bus_release_resources(dev, smmu_spec, sc->res); return (0); } static int smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct smmu_softc *sc; sc = device_get_softc(dev); device_printf(sc->dev, "%s\n", __func__); return (ENOENT); } static int smmu_unmap(device_t dev, struct iommu_domain *iodom, vm_offset_t va, bus_size_t size) { struct smmu_domain *domain; struct smmu_softc *sc; int err; int i; sc = device_get_softc(dev); domain = (struct smmu_domain *)iodom; err = 0; dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid); for (i = 0; i < size; i += PAGE_SIZE) { - if (pmap_smmu_remove(&domain->p, va) == 0) { + if (smmu_pmap_remove(&domain->p, va) == 0) { /* pmap entry removed, invalidate TLB. */ smmu_tlbi_va(sc, va, domain->asid); } else { err = ENOENT; break; } va += PAGE_SIZE; } smmu_sync(sc); return (err); } static int smmu_map(device_t dev, struct iommu_domain *iodom, vm_offset_t va, vm_page_t *ma, vm_size_t size, vm_prot_t prot) { struct smmu_domain *domain; struct smmu_softc *sc; vm_paddr_t pa; int error; int i; sc = device_get_softc(dev); domain = (struct smmu_domain *)iodom; dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size, domain->asid); for (i = 0; size > 0; size -= PAGE_SIZE) { pa = VM_PAGE_TO_PHYS(ma[i++]); - error = pmap_smmu_enter(&domain->p, va, pa, prot, 0); + error = smmu_pmap_enter(&domain->p, va, pa, prot, 0); if (error) return (error); smmu_tlbi_va(sc, va, domain->asid); va += PAGE_SIZE; } smmu_sync(sc); return (0); } static struct iommu_domain * smmu_domain_alloc(device_t dev, struct iommu_unit *iommu) { struct iommu_domain *iodom; struct smmu_domain *domain; struct smmu_unit *unit; struct smmu_softc *sc; int error; int new_asid; sc = device_get_softc(dev); unit = (struct smmu_unit *)iommu; domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO); error = smmu_asid_alloc(sc, &new_asid); if (error) { free(domain, M_SMMU); device_printf(sc->dev, "Could not allocate ASID for a new domain.\n"); return (NULL); } domain->asid = (uint16_t)new_asid; - iommu_pmap_pinit(&domain->p); + smmu_pmap_pinit(&domain->p); PMAP_LOCK_INIT(&domain->p); error = smmu_init_cd(sc, domain); if (error) { free(domain, M_SMMU); device_printf(sc->dev, "Could not initialize CD\n"); return (NULL); } smmu_tlbi_asid(sc, domain->asid); LIST_INIT(&domain->ctx_list); IOMMU_LOCK(iommu); LIST_INSERT_HEAD(&unit->domain_list, domain, next); IOMMU_UNLOCK(iommu); iodom = &domain->iodom; /* * Use 48-bit address space regardless of VAX bit * as we need 64k IOMMU_PAGE_SIZE for 52-bit space. */ iodom->end = MAXADDR_48BIT; return (iodom); } static void smmu_domain_free(device_t dev, struct iommu_domain *iodom) { struct smmu_domain *domain; struct smmu_softc *sc; struct smmu_cd *cd; sc = device_get_softc(dev); domain = (struct smmu_domain *)iodom; LIST_REMOVE(domain, next); cd = domain->cd; - iommu_pmap_remove_pages(&domain->p); - iommu_pmap_release(&domain->p); + smmu_pmap_remove_pages(&domain->p); + smmu_pmap_release(&domain->p); smmu_tlbi_asid(sc, domain->asid); smmu_asid_free(sc, domain->asid); contigfree(cd->vaddr, cd->size, M_SMMU); free(cd, M_SMMU); free(domain, M_SMMU); } static int smmu_set_buswide(device_t dev, struct smmu_domain *domain, struct smmu_ctx *ctx) { struct smmu_softc *sc; int i; sc = device_get_softc(dev); for (i = 0; i < PCI_SLOTMAX; i++) smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass); return (0); } #ifdef DEV_ACPI static int smmu_pci_get_sid_acpi(device_t child, u_int *xref0, u_int *sid0) { uint16_t rid; u_int xref; int seg; int err; int sid; seg = pci_get_domain(child); rid = pci_get_rid(child); err = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid); if (err == 0) { if (sid0) *sid0 = sid; if (xref0) *xref0 = xref; } return (err); } #endif #ifdef FDT static int smmu_pci_get_sid_fdt(device_t child, u_int *xref0, u_int *sid0) { struct pci_id_ofw_iommu pi; uint64_t base, size; phandle_t node; u_int xref; int err; err = pci_get_id(child, PCI_ID_OFW_IOMMU, (uintptr_t *)&pi); if (err == 0) { /* Our xref is memory base address. */ node = OF_node_from_xref(pi.xref); fdt_regsize(node, &base, &size); xref = base; if (sid0) *sid0 = pi.id; if (xref0) *xref0 = xref; } return (err); } #endif static struct iommu_ctx * smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child, bool disabled) { struct smmu_domain *domain; struct smmu_ctx *ctx; domain = (struct smmu_domain *)iodom; ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO); ctx->dev = child; ctx->domain = domain; if (disabled) ctx->bypass = true; IOMMU_DOMAIN_LOCK(iodom); LIST_INSERT_HEAD(&domain->ctx_list, ctx, next); IOMMU_DOMAIN_UNLOCK(iodom); return (&ctx->ioctx); } static int smmu_ctx_init(device_t dev, struct iommu_ctx *ioctx) { struct smmu_domain *domain; struct iommu_domain *iodom; struct smmu_softc *sc; struct smmu_ctx *ctx; devclass_t pci_class; u_int sid; int err; ctx = (struct smmu_ctx *)ioctx; sc = device_get_softc(dev); domain = ctx->domain; iodom = (struct iommu_domain *)domain; pci_class = devclass_find("pci"); if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) { #ifdef DEV_ACPI err = smmu_pci_get_sid_acpi(ctx->dev, NULL, &sid); #else err = smmu_pci_get_sid_fdt(ctx->dev, NULL, &sid); #endif if (err) return (err); ioctx->rid = pci_get_rid(dev); ctx->sid = sid; ctx->vendor = pci_get_vendor(ctx->dev); ctx->device = pci_get_device(ctx->dev); } if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) { err = smmu_init_l1_entry(sc, ctx->sid); if (err) return (err); } /* * Neoverse N1 SDP: * 0x800 xhci * 0x700 re * 0x600 sata */ smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass); if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev))) smmu_set_buswide(dev, domain, ctx); return (0); } static void smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx) { struct smmu_softc *sc; struct smmu_ctx *ctx; IOMMU_ASSERT_LOCKED(ioctx->domain->iommu); sc = device_get_softc(dev); ctx = (struct smmu_ctx *)ioctx; smmu_deinit_ste(sc, ctx->sid); LIST_REMOVE(ctx, next); free(ctx, M_SMMU); } struct smmu_ctx * smmu_ctx_lookup_by_sid(device_t dev, u_int sid) { struct smmu_softc *sc; struct smmu_domain *domain; struct smmu_unit *unit; struct smmu_ctx *ctx; sc = device_get_softc(dev); unit = &sc->unit; LIST_FOREACH(domain, &unit->domain_list, next) { LIST_FOREACH(ctx, &domain->ctx_list, next) { if (ctx->sid == sid) return (ctx); } } return (NULL); } static struct iommu_ctx * smmu_ctx_lookup(device_t dev, device_t child) { struct iommu_unit *iommu __diagused; struct smmu_softc *sc; struct smmu_domain *domain; struct smmu_unit *unit; struct smmu_ctx *ctx; sc = device_get_softc(dev); unit = &sc->unit; iommu = &unit->iommu; IOMMU_ASSERT_LOCKED(iommu); LIST_FOREACH(domain, &unit->domain_list, next) { IOMMU_DOMAIN_LOCK(&domain->iodom); LIST_FOREACH(ctx, &domain->ctx_list, next) { if (ctx->dev == child) { IOMMU_DOMAIN_UNLOCK(&domain->iodom); return (&ctx->ioctx); } } IOMMU_DOMAIN_UNLOCK(&domain->iodom); } return (NULL); } static int smmu_find(device_t dev, device_t child) { struct smmu_softc *sc; u_int xref; int err; sc = device_get_softc(dev); #ifdef DEV_ACPI err = smmu_pci_get_sid_acpi(child, &xref, NULL); #else err = smmu_pci_get_sid_fdt(child, &xref, NULL); #endif if (err) return (ENOENT); /* Check if xref is ours. */ if (xref != sc->xref) return (EFAULT); return (0); } #ifdef FDT static int smmu_ofw_md_data(device_t dev, struct iommu_ctx *ioctx, pcell_t *cells, int ncells) { struct smmu_ctx *ctx; ctx = (struct smmu_ctx *)ioctx; if (ncells != 1) return (-1); ctx->sid = cells[0]; return (0); } #endif static device_method_t smmu_methods[] = { /* Device interface */ DEVMETHOD(device_detach, smmu_detach), /* SMMU interface */ DEVMETHOD(iommu_find, smmu_find), DEVMETHOD(iommu_map, smmu_map), DEVMETHOD(iommu_unmap, smmu_unmap), DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc), DEVMETHOD(iommu_domain_free, smmu_domain_free), DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc), DEVMETHOD(iommu_ctx_init, smmu_ctx_init), DEVMETHOD(iommu_ctx_free, smmu_ctx_free), DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup), #ifdef FDT DEVMETHOD(iommu_ofw_md_data, smmu_ofw_md_data), #endif /* Bus interface */ DEVMETHOD(bus_read_ivar, smmu_read_ivar), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc));