Changeset View
Changeset View
Standalone View
Standalone View
head/sys/x86/iommu/intel_idpgtbl.c
Show First 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | |||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include <x86/include/busdma_impl.h> | #include <x86/include/busdma_impl.h> | ||||
#include <x86/iommu/intel_reg.h> | #include <x86/iommu/intel_reg.h> | ||||
#include <x86/iommu/busdma_dmar.h> | #include <x86/iommu/busdma_dmar.h> | ||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <x86/iommu/intel_dmar.h> | #include <x86/iommu/intel_dmar.h> | ||||
static int domain_unmap_buf_locked(struct dmar_domain *domain, | static int domain_unmap_buf_locked(struct dmar_domain *domain, | ||||
dmar_gaddr_t base, dmar_gaddr_t size, int flags); | iommu_gaddr_t base, iommu_gaddr_t size, int flags); | ||||
/* | /* | ||||
* The cache of the identity mapping page tables for the DMARs. Using | * The cache of the identity mapping page tables for the DMARs. Using | ||||
* the cache saves significant amount of memory for page tables by | * the cache saves significant amount of memory for page tables by | ||||
* reusing the page tables, since usually DMARs are identical and have | * reusing the page tables, since usually DMARs are identical and have | ||||
* the same capabilities. Still, cache records the information needed | * the same capabilities. Still, cache records the information needed | ||||
* to match DMAR capabilities and page table format, to correctly | * to match DMAR capabilities and page table format, to correctly | ||||
* handle different DMARs. | * handle different DMARs. | ||||
*/ | */ | ||||
struct idpgtbl { | struct idpgtbl { | ||||
dmar_gaddr_t maxaddr; /* Page table covers the guest address | iommu_gaddr_t maxaddr; /* Page table covers the guest address | ||||
range [0..maxaddr) */ | range [0..maxaddr) */ | ||||
int pglvl; /* Total page table levels ignoring | int pglvl; /* Total page table levels ignoring | ||||
superpages */ | superpages */ | ||||
int leaf; /* The last materialized page table | int leaf; /* The last materialized page table | ||||
level, it is non-zero if superpages | level, it is non-zero if superpages | ||||
are supported */ | are supported */ | ||||
vm_object_t pgtbl_obj; /* The page table pages */ | vm_object_t pgtbl_obj; /* The page table pages */ | ||||
LIST_ENTRY(idpgtbl) link; | LIST_ENTRY(idpgtbl) link; | ||||
Show All 10 Lines | |||||
* - lvl is the level to build; | * - lvl is the level to build; | ||||
* - idx is the index of the page table page in the pgtbl_obj, which is | * - idx is the index of the page table page in the pgtbl_obj, which is | ||||
* being allocated filled now; | * being allocated filled now; | ||||
* - addr is the starting address in the bus address space which is | * - addr is the starting address in the bus address space which is | ||||
* mapped by the page table page. | * mapped by the page table page. | ||||
*/ | */ | ||||
static void | static void | ||||
domain_idmap_nextlvl(struct idpgtbl *tbl, int lvl, vm_pindex_t idx, | domain_idmap_nextlvl(struct idpgtbl *tbl, int lvl, vm_pindex_t idx, | ||||
dmar_gaddr_t addr) | iommu_gaddr_t addr) | ||||
{ | { | ||||
vm_page_t m1; | vm_page_t m1; | ||||
dmar_pte_t *pte; | dmar_pte_t *pte; | ||||
struct sf_buf *sf; | struct sf_buf *sf; | ||||
dmar_gaddr_t f, pg_sz; | iommu_gaddr_t f, pg_sz; | ||||
vm_pindex_t base; | vm_pindex_t base; | ||||
int i; | int i; | ||||
VM_OBJECT_ASSERT_LOCKED(tbl->pgtbl_obj); | VM_OBJECT_ASSERT_LOCKED(tbl->pgtbl_obj); | ||||
if (addr >= tbl->maxaddr) | if (addr >= tbl->maxaddr) | ||||
return; | return; | ||||
(void)dmar_pgalloc(tbl->pgtbl_obj, idx, DMAR_PGF_OBJL | DMAR_PGF_WAITOK | | (void)dmar_pgalloc(tbl->pgtbl_obj, idx, DMAR_PGF_OBJL | DMAR_PGF_WAITOK | | ||||
DMAR_PGF_ZERO); | DMAR_PGF_ZERO); | ||||
Show All 32 Lines | |||||
* Find a ready and compatible identity-mapping page table in the | * Find a ready and compatible identity-mapping page table in the | ||||
* cache. If not found, populate the identity-mapping page table for | * cache. If not found, populate the identity-mapping page table for | ||||
* the context, up to the maxaddr. The maxaddr byte is allowed to be | * the context, up to the maxaddr. The maxaddr byte is allowed to be | ||||
* not mapped, which is aligned with the definition of Maxmem as the | * not mapped, which is aligned with the definition of Maxmem as the | ||||
* highest usable physical address + 1. If superpages are used, the | * highest usable physical address + 1. If superpages are used, the | ||||
* maxaddr is typically mapped. | * maxaddr is typically mapped. | ||||
*/ | */ | ||||
vm_object_t | vm_object_t | ||||
domain_get_idmap_pgtbl(struct dmar_domain *domain, dmar_gaddr_t maxaddr) | domain_get_idmap_pgtbl(struct dmar_domain *domain, iommu_gaddr_t maxaddr) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
struct idpgtbl *tbl; | struct idpgtbl *tbl; | ||||
vm_object_t res; | vm_object_t res; | ||||
vm_page_t m; | vm_page_t m; | ||||
int leaf, i; | int leaf, i; | ||||
leaf = 0; /* silence gcc */ | leaf = 0; /* silence gcc */ | ||||
▲ Show 20 Lines • Show All 143 Lines • ▼ Show 20 Lines | |||||
* address. Support superpages. | * address. Support superpages. | ||||
*/ | */ | ||||
/* | /* | ||||
* Index of the pte for the guest address base in the page table at | * Index of the pte for the guest address base in the page table at | ||||
* the level lvl. | * the level lvl. | ||||
*/ | */ | ||||
static int | static int | ||||
domain_pgtbl_pte_off(struct dmar_domain *domain, dmar_gaddr_t base, int lvl) | domain_pgtbl_pte_off(struct dmar_domain *domain, iommu_gaddr_t base, int lvl) | ||||
{ | { | ||||
base >>= DMAR_PAGE_SHIFT + (domain->pglvl - lvl - 1) * | base >>= DMAR_PAGE_SHIFT + (domain->pglvl - lvl - 1) * | ||||
DMAR_NPTEPGSHIFT; | DMAR_NPTEPGSHIFT; | ||||
return (base & DMAR_PTEMASK); | return (base & DMAR_PTEMASK); | ||||
} | } | ||||
/* | /* | ||||
* Returns the page index of the page table page in the page table | * Returns the page index of the page table page in the page table | ||||
* object, which maps the given address base at the page table level | * object, which maps the given address base at the page table level | ||||
* lvl. | * lvl. | ||||
*/ | */ | ||||
static vm_pindex_t | static vm_pindex_t | ||||
domain_pgtbl_get_pindex(struct dmar_domain *domain, dmar_gaddr_t base, int lvl) | domain_pgtbl_get_pindex(struct dmar_domain *domain, iommu_gaddr_t base, int lvl) | ||||
{ | { | ||||
vm_pindex_t idx, pidx; | vm_pindex_t idx, pidx; | ||||
int i; | int i; | ||||
KASSERT(lvl >= 0 && lvl < domain->pglvl, | KASSERT(lvl >= 0 && lvl < domain->pglvl, | ||||
("wrong lvl %p %d", domain, lvl)); | ("wrong lvl %p %d", domain, lvl)); | ||||
for (pidx = idx = 0, i = 0; i < lvl; i++, pidx = idx) { | for (pidx = idx = 0, i = 0; i < lvl; i++, pidx = idx) { | ||||
idx = domain_pgtbl_pte_off(domain, base, i) + | idx = domain_pgtbl_pte_off(domain, base, i) + | ||||
pidx * DMAR_NPTEPG + 1; | pidx * DMAR_NPTEPG + 1; | ||||
} | } | ||||
return (idx); | return (idx); | ||||
} | } | ||||
static dmar_pte_t * | static dmar_pte_t * | ||||
domain_pgtbl_map_pte(struct dmar_domain *domain, dmar_gaddr_t base, int lvl, | domain_pgtbl_map_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, | ||||
int flags, vm_pindex_t *idxp, struct sf_buf **sf) | int flags, vm_pindex_t *idxp, struct sf_buf **sf) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
struct sf_buf *sfp; | struct sf_buf *sfp; | ||||
dmar_pte_t *pte, *ptep; | dmar_pte_t *pte, *ptep; | ||||
vm_pindex_t idx, idx1; | vm_pindex_t idx, idx1; | ||||
DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | if (pte == NULL) { | ||||
goto retry; | goto retry; | ||||
} | } | ||||
} | } | ||||
pte += domain_pgtbl_pte_off(domain, base, lvl); | pte += domain_pgtbl_pte_off(domain, base, lvl); | ||||
return (pte); | return (pte); | ||||
} | } | ||||
static int | static int | ||||
domain_map_buf_locked(struct dmar_domain *domain, dmar_gaddr_t base, | domain_map_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
dmar_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags) | iommu_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags) | ||||
{ | { | ||||
dmar_pte_t *pte; | dmar_pte_t *pte; | ||||
struct sf_buf *sf; | struct sf_buf *sf; | ||||
dmar_gaddr_t pg_sz, base1, size1; | iommu_gaddr_t pg_sz, base1, size1; | ||||
vm_pindex_t pi, c, idx, run_sz; | vm_pindex_t pi, c, idx, run_sz; | ||||
int lvl; | int lvl; | ||||
bool superpage; | bool superpage; | ||||
DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | ||||
base1 = base; | base1 = base; | ||||
size1 = size; | size1 = size; | ||||
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | domain_map_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
} | } | ||||
if (sf != NULL) | if (sf != NULL) | ||||
dmar_unmap_pgtbl(sf); | dmar_unmap_pgtbl(sf); | ||||
TD_PINNED_ASSERT; | TD_PINNED_ASSERT; | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
domain_map_buf(struct dmar_domain *domain, dmar_gaddr_t base, dmar_gaddr_t size, | domain_map_buf(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, | ||||
vm_page_t *ma, uint64_t pflags, int flags) | vm_page_t *ma, uint64_t pflags, int flags) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
int error; | int error; | ||||
unit = domain->dmar; | unit = domain->dmar; | ||||
KASSERT((domain->flags & DMAR_DOMAIN_IDMAP) == 0, | KASSERT((domain->flags & DMAR_DOMAIN_IDMAP) == 0, | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) { | ||||
DMAR_LOCK(unit); | DMAR_LOCK(unit); | ||||
dmar_flush_write_bufs(unit); | dmar_flush_write_bufs(unit); | ||||
DMAR_UNLOCK(unit); | DMAR_UNLOCK(unit); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static void domain_unmap_clear_pte(struct dmar_domain *domain, | static void domain_unmap_clear_pte(struct dmar_domain *domain, | ||||
dmar_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, | iommu_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, | ||||
struct sf_buf **sf, bool free_fs); | struct sf_buf **sf, bool free_fs); | ||||
static void | static void | ||||
domain_free_pgtbl_pde(struct dmar_domain *domain, dmar_gaddr_t base, | domain_free_pgtbl_pde(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
int lvl, int flags) | int lvl, int flags) | ||||
{ | { | ||||
struct sf_buf *sf; | struct sf_buf *sf; | ||||
dmar_pte_t *pde; | dmar_pte_t *pde; | ||||
vm_pindex_t idx; | vm_pindex_t idx; | ||||
sf = NULL; | sf = NULL; | ||||
pde = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf); | pde = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf); | ||||
domain_unmap_clear_pte(domain, base, lvl, flags, pde, &sf, true); | domain_unmap_clear_pte(domain, base, lvl, flags, pde, &sf, true); | ||||
} | } | ||||
static void | static void | ||||
domain_unmap_clear_pte(struct dmar_domain *domain, dmar_gaddr_t base, int lvl, | domain_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, | ||||
int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) | int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
dmar_pte_clear(&pte->pte); | dmar_pte_clear(&pte->pte); | ||||
dmar_flush_pte_to_ram(domain->dmar, pte); | dmar_flush_pte_to_ram(domain->dmar, pte); | ||||
m = sf_buf_page(*sf); | m = sf_buf_page(*sf); | ||||
if (free_sf) { | if (free_sf) { | ||||
Show All 12 Lines | domain_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, | ||||
dmar_pgfree(domain->pgtbl_obj, m->pindex, flags); | dmar_pgfree(domain->pgtbl_obj, m->pindex, flags); | ||||
domain_free_pgtbl_pde(domain, base, lvl - 1, flags); | domain_free_pgtbl_pde(domain, base, lvl - 1, flags); | ||||
} | } | ||||
/* | /* | ||||
* Assumes that the unmap is never partial. | * Assumes that the unmap is never partial. | ||||
*/ | */ | ||||
static int | static int | ||||
domain_unmap_buf_locked(struct dmar_domain *domain, dmar_gaddr_t base, | domain_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
dmar_gaddr_t size, int flags) | iommu_gaddr_t size, int flags) | ||||
{ | { | ||||
dmar_pte_t *pte; | dmar_pte_t *pte; | ||||
struct sf_buf *sf; | struct sf_buf *sf; | ||||
vm_pindex_t idx; | vm_pindex_t idx; | ||||
dmar_gaddr_t pg_sz; | iommu_gaddr_t pg_sz; | ||||
int lvl; | int lvl; | ||||
DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | DMAR_DOMAIN_ASSERT_PGLOCKED(domain); | ||||
if (size == 0) | if (size == 0) | ||||
return (0); | return (0); | ||||
KASSERT((domain->flags & DMAR_DOMAIN_IDMAP) == 0, | KASSERT((domain->flags & DMAR_DOMAIN_IDMAP) == 0, | ||||
("modifying idmap pagetable domain %p", domain)); | ("modifying idmap pagetable domain %p", domain)); | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | domain_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
* can be ignored there. | * can be ignored there. | ||||
*/ | */ | ||||
TD_PINNED_ASSERT; | TD_PINNED_ASSERT; | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
domain_unmap_buf(struct dmar_domain *domain, dmar_gaddr_t base, | domain_unmap_buf(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
dmar_gaddr_t size, int flags) | iommu_gaddr_t size, int flags) | ||||
{ | { | ||||
int error; | int error; | ||||
DMAR_DOMAIN_PGLOCK(domain); | DMAR_DOMAIN_PGLOCK(domain); | ||||
error = domain_unmap_buf_locked(domain, base, size, flags); | error = domain_unmap_buf_locked(domain, base, size, flags); | ||||
DMAR_DOMAIN_PGUNLOCK(domain); | DMAR_DOMAIN_PGUNLOCK(domain); | ||||
return (error); | return (error); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | for (;;) { | ||||
if ((iotlbr & DMAR_IOTLB_IVT) == 0) | if ((iotlbr & DMAR_IOTLB_IVT) == 0) | ||||
break; | break; | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
} | } | ||||
return (iotlbr); | return (iotlbr); | ||||
} | } | ||||
void | void | ||||
domain_flush_iotlb_sync(struct dmar_domain *domain, dmar_gaddr_t base, | domain_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base, | ||||
dmar_gaddr_t size) | iommu_gaddr_t size) | ||||
{ | { | ||||
struct dmar_unit *unit; | struct dmar_unit *unit; | ||||
dmar_gaddr_t isize; | iommu_gaddr_t isize; | ||||
uint64_t iotlbr; | uint64_t iotlbr; | ||||
int am, iro; | int am, iro; | ||||
unit = domain->dmar; | unit = domain->dmar; | ||||
KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call", | KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call", | ||||
unit->unit)); | unit->iommu.unit)); | ||||
iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16; | iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16; | ||||
DMAR_LOCK(unit); | DMAR_LOCK(unit); | ||||
if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) { | if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) { | ||||
iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM | | iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM | | ||||
DMAR_IOTLB_DID(domain->domain), iro); | DMAR_IOTLB_DID(domain->domain), iro); | ||||
KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != | KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != | ||||
DMAR_IOTLB_IAIG_INVLD, | DMAR_IOTLB_IAIG_INVLD, | ||||
("dmar%d: invalidation failed %jx", unit->unit, | ("dmar%d: invalidation failed %jx", unit->iommu.unit, | ||||
(uintmax_t)iotlbr)); | (uintmax_t)iotlbr)); | ||||
} else { | } else { | ||||
for (; size > 0; base += isize, size -= isize) { | for (; size > 0; base += isize, size -= isize) { | ||||
am = calc_am(unit, base, size, &isize); | am = calc_am(unit, base, size, &isize); | ||||
dmar_write8(unit, iro, base | am); | dmar_write8(unit, iro, base | am); | ||||
iotlbr = domain_wait_iotlb_flush(unit, | iotlbr = domain_wait_iotlb_flush(unit, | ||||
DMAR_IOTLB_IIRG_PAGE | | DMAR_IOTLB_IIRG_PAGE | | ||||
DMAR_IOTLB_DID(domain->domain), iro); | DMAR_IOTLB_DID(domain->domain), iro); | ||||
KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != | KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != | ||||
DMAR_IOTLB_IAIG_INVLD, | DMAR_IOTLB_IAIG_INVLD, | ||||
("dmar%d: PSI invalidation failed " | ("dmar%d: PSI invalidation failed " | ||||
"iotlbr 0x%jx base 0x%jx size 0x%jx am %d", | "iotlbr 0x%jx base 0x%jx size 0x%jx am %d", | ||||
unit->unit, (uintmax_t)iotlbr, | unit->iommu.unit, (uintmax_t)iotlbr, | ||||
(uintmax_t)base, (uintmax_t)size, am)); | (uintmax_t)base, (uintmax_t)size, am)); | ||||
/* | /* | ||||
* Any non-page granularity covers whole guest | * Any non-page granularity covers whole guest | ||||
* address space for the domain. | * address space for the domain. | ||||
*/ | */ | ||||
if ((iotlbr & DMAR_IOTLB_IAIG_MASK) != | if ((iotlbr & DMAR_IOTLB_IAIG_MASK) != | ||||
DMAR_IOTLB_IAIG_PAGE) | DMAR_IOTLB_IAIG_PAGE) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
DMAR_UNLOCK(unit); | DMAR_UNLOCK(unit); | ||||
} | } |