Changeset View
Changeset View
Standalone View
Standalone View
head/sys/x86/iommu/intel_ctx.c
Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
device_tag_init(struct dmar_ctx *ctx, device_t dev) | device_tag_init(struct dmar_ctx *ctx, device_t dev) | ||||
{ | { | ||||
struct dmar_domain *domain; | struct dmar_domain *domain; | ||||
bus_addr_t maxaddr; | bus_addr_t maxaddr; | ||||
domain = (struct dmar_domain *)ctx->context.domain; | domain = (struct dmar_domain *)ctx->context.domain; | ||||
maxaddr = MIN(domain->end, BUS_SPACE_MAXADDR); | maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR); | ||||
ctx->context.tag->common.ref_count = 1; /* Prevent free */ | ctx->context.tag->common.ref_count = 1; /* Prevent free */ | ||||
ctx->context.tag->common.impl = &bus_dma_iommu_impl; | ctx->context.tag->common.impl = &bus_dma_iommu_impl; | ||||
ctx->context.tag->common.boundary = 0; | ctx->context.tag->common.boundary = 0; | ||||
ctx->context.tag->common.lowaddr = maxaddr; | ctx->context.tag->common.lowaddr = maxaddr; | ||||
ctx->context.tag->common.highaddr = maxaddr; | ctx->context.tag->common.highaddr = maxaddr; | ||||
ctx->context.tag->common.maxsize = maxaddr; | ctx->context.tag->common.maxsize = maxaddr; | ||||
ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; | ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; | ||||
ctx->context.tag->common.maxsegsz = maxaddr; | ctx->context.tag->common.maxsegsz = maxaddr; | ||||
Show All 37 Lines | ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, | ||||
domain = (struct dmar_domain *)ctx->context.domain; | domain = (struct dmar_domain *)ctx->context.domain; | ||||
unit = (struct dmar_unit *)domain->iodom.iommu; | unit = (struct dmar_unit *)domain->iodom.iommu; | ||||
KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), | KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), | ||||
("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", | ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", | ||||
unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), | unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), | ||||
pci_get_function(ctx->context.tag->owner), | pci_get_function(ctx->context.tag->owner), | ||||
ctxp->ctx1, ctxp->ctx2)); | ctxp->ctx1, ctxp->ctx2)); | ||||
if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 && | if ((domain->iodom.flags & DMAR_DOMAIN_IDMAP) != 0 && | ||||
(unit->hw_ecap & DMAR_ECAP_PT) != 0) { | (unit->hw_ecap & DMAR_ECAP_PT) != 0) { | ||||
KASSERT(domain->pgtbl_obj == NULL, | KASSERT(domain->pgtbl_obj == NULL, | ||||
("ctx %p non-null pgtbl_obj", ctx)); | ("ctx %p non-null pgtbl_obj", ctx)); | ||||
ctx_root = NULL; | ctx_root = NULL; | ||||
} else { | } else { | ||||
ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC); | ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, | ||||
TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { | TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { | ||||
/* | /* | ||||
* VT-d specification requires that the start of an | * VT-d specification requires that the start of an | ||||
* RMRR entry is 4k-aligned. Buggy BIOSes put | * RMRR entry is 4k-aligned. Buggy BIOSes put | ||||
* anything into the start and end fields. Truncate | * anything into the start and end fields. Truncate | ||||
* and round as neccesary. | * and round as neccesary. | ||||
* | * | ||||
* We also allow the overlapping RMRR entries, see | * We also allow the overlapping RMRR entries, see | ||||
* dmar_gas_alloc_region(). | * iommu_gas_alloc_region(). | ||||
*/ | */ | ||||
start = entry->start; | start = entry->start; | ||||
end = entry->end; | end = entry->end; | ||||
if (bootverbose) | if (bootverbose) | ||||
printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", | printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", | ||||
domain->iodom.iommu->unit, bus, slot, func, | domain->iodom.iommu->unit, bus, slot, func, | ||||
(uintmax_t)start, (uintmax_t)end); | (uintmax_t)start, (uintmax_t)end); | ||||
entry->start = trunc_page(start); | entry->start = trunc_page(start); | ||||
Show All 11 Lines | if (entry->start == entry->end) { | ||||
entry->end += DMAR_PAGE_SIZE * 0x20; | entry->end += DMAR_PAGE_SIZE * 0x20; | ||||
} | } | ||||
size = OFF_TO_IDX(entry->end - entry->start); | size = OFF_TO_IDX(entry->end - entry->start); | ||||
ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); | ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); | ||||
for (i = 0; i < size; i++) { | for (i = 0; i < size; i++) { | ||||
ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, | ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, | ||||
VM_MEMATTR_DEFAULT); | VM_MEMATTR_DEFAULT); | ||||
} | } | ||||
error1 = dmar_gas_map_region(domain, entry, | error1 = iommu_gas_map_region((struct iommu_domain *)domain, | ||||
entry, | |||||
IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, | IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, | ||||
IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); | IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); | ||||
/* | /* | ||||
* Non-failed RMRR entries are owned by context rb | * Non-failed RMRR entries are owned by context rb | ||||
* tree. Get rid of the failed entry, but do not stop | * tree. Get rid of the failed entry, but do not stop | ||||
* the loop. Rest of the parsed RMRR entries are | * the loop. Rest of the parsed RMRR entries are | ||||
* loaded and removed on the context destruction. | * loaded and removed on the context destruction. | ||||
*/ | */ | ||||
if (error1 == 0 && entry->end != entry->start) { | if (error1 == 0 && entry->end != entry->start) { | ||||
IOMMU_LOCK(domain->iodom.iommu); | IOMMU_LOCK(domain->iodom.iommu); | ||||
domain->refs++; /* XXXKIB prevent free */ | domain->refs++; /* XXXKIB prevent free */ | ||||
domain->flags |= DMAR_DOMAIN_RMRR; | domain->iodom.flags |= DMAR_DOMAIN_RMRR; | ||||
IOMMU_UNLOCK(domain->iodom.iommu); | IOMMU_UNLOCK(domain->iodom.iommu); | ||||
} else { | } else { | ||||
if (error1 != 0) { | if (error1 != 0) { | ||||
if (dev != NULL) | if (dev != NULL) | ||||
device_printf(dev, ""); | device_printf(dev, ""); | ||||
printf("pci%d:%d:%d ", bus, slot, func); | printf("pci%d:%d:%d ", bus, slot, func); | ||||
printf( | printf( | ||||
"dmar%d failed to map RMRR region (%jx, %jx) %d\n", | "dmar%d failed to map RMRR region (%jx, %jx) %d\n", | ||||
domain->iodom.iommu->unit, start, end, | domain->iodom.iommu->unit, start, end, | ||||
error1); | error1); | ||||
error = error1; | error = error1; | ||||
} | } | ||||
TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); | TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); | ||||
dmar_gas_free_entry(domain, entry); | iommu_gas_free_entry((struct iommu_domain *)domain, | ||||
entry); | |||||
} | } | ||||
for (i = 0; i < size; i++) | for (i = 0; i < size; i++) | ||||
vm_page_putfake(ma[i]); | vm_page_putfake(ma[i]); | ||||
free(ma, M_TEMP); | free(ma, M_TEMP); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
static struct dmar_domain * | static struct dmar_domain * | ||||
dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) | dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) | ||||
{ | { | ||||
struct iommu_domain *iodom; | |||||
struct dmar_domain *domain; | struct dmar_domain *domain; | ||||
int error, id, mgaw; | int error, id, mgaw; | ||||
id = alloc_unr(dmar->domids); | id = alloc_unr(dmar->domids); | ||||
if (id == -1) | if (id == -1) | ||||
return (NULL); | return (NULL); | ||||
domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); | domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); | ||||
iodom = (struct iommu_domain *)domain; | |||||
domain->domain = id; | domain->domain = id; | ||||
LIST_INIT(&domain->contexts); | LIST_INIT(&domain->contexts); | ||||
RB_INIT(&domain->rb_root); | RB_INIT(&domain->iodom.rb_root); | ||||
TAILQ_INIT(&domain->iodom.unload_entries); | TAILQ_INIT(&domain->iodom.unload_entries); | ||||
TASK_INIT(&domain->iodom.unload_task, 0, dmar_domain_unload_task, | TASK_INIT(&domain->iodom.unload_task, 0, dmar_domain_unload_task, | ||||
domain); | domain); | ||||
mtx_init(&domain->iodom.lock, "dmardom", NULL, MTX_DEF); | mtx_init(&domain->iodom.lock, "dmardom", NULL, MTX_DEF); | ||||
domain->dmar = dmar; | domain->dmar = dmar; | ||||
domain->iodom.iommu = &dmar->iommu; | domain->iodom.iommu = &dmar->iommu; | ||||
/* | /* | ||||
* For now, use the maximal usable physical address of the | * For now, use the maximal usable physical address of the | ||||
* installed memory to calculate the mgaw on id_mapped domain. | * installed memory to calculate the mgaw on id_mapped domain. | ||||
* It is useful for the identity mapping, and less so for the | * It is useful for the identity mapping, and less so for the | ||||
* virtualized bus address space. | * virtualized bus address space. | ||||
*/ | */ | ||||
domain->end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; | domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; | ||||
mgaw = dmar_maxaddr2mgaw(dmar, domain->end, !id_mapped); | mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped); | ||||
error = domain_set_agaw(domain, mgaw); | error = domain_set_agaw(domain, mgaw); | ||||
if (error != 0) | if (error != 0) | ||||
goto fail; | goto fail; | ||||
if (!id_mapped) | if (!id_mapped) | ||||
/* Use all supported address space for remapping. */ | /* Use all supported address space for remapping. */ | ||||
domain->end = 1ULL << (domain->agaw - 1); | domain->iodom.end = 1ULL << (domain->agaw - 1); | ||||
dmar_gas_init_domain(domain); | iommu_gas_init_domain((struct iommu_domain *)domain); | ||||
if (id_mapped) { | if (id_mapped) { | ||||
if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { | if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { | ||||
domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, | domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, | ||||
domain->end); | domain->iodom.end); | ||||
} | } | ||||
domain->flags |= DMAR_DOMAIN_IDMAP; | domain->iodom.flags |= DMAR_DOMAIN_IDMAP; | ||||
} else { | } else { | ||||
error = domain_alloc_pgtbl(domain); | error = domain_alloc_pgtbl(domain); | ||||
if (error != 0) | if (error != 0) | ||||
goto fail; | goto fail; | ||||
/* Disable local apic region access */ | /* Disable local apic region access */ | ||||
error = dmar_gas_reserve_region(domain, 0xfee00000, | error = iommu_gas_reserve_region(iodom, 0xfee00000, | ||||
0xfeefffff + 1); | 0xfeefffff + 1); | ||||
if (error != 0) | if (error != 0) | ||||
goto fail; | goto fail; | ||||
} | } | ||||
return (domain); | return (domain); | ||||
fail: | fail: | ||||
dmar_domain_destroy(domain); | dmar_domain_destroy(domain); | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | dmar_domain_destroy(struct dmar_domain *domain) | ||||
KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), | KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), | ||||
("unfinished unloads %p", domain)); | ("unfinished unloads %p", domain)); | ||||
KASSERT(LIST_EMPTY(&domain->contexts), | KASSERT(LIST_EMPTY(&domain->contexts), | ||||
("destroying dom %p with contexts", domain)); | ("destroying dom %p with contexts", domain)); | ||||
KASSERT(domain->ctx_cnt == 0, | KASSERT(domain->ctx_cnt == 0, | ||||
("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); | ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); | ||||
KASSERT(domain->refs == 0, | KASSERT(domain->refs == 0, | ||||
("destroying dom %p with refs %d", domain, domain->refs)); | ("destroying dom %p with refs %d", domain, domain->refs)); | ||||
if ((domain->flags & DMAR_DOMAIN_GAS_INITED) != 0) { | if ((domain->iodom.flags & DMAR_DOMAIN_GAS_INITED) != 0) { | ||||
DMAR_DOMAIN_LOCK(domain); | DMAR_DOMAIN_LOCK(domain); | ||||
dmar_gas_fini_domain(domain); | iommu_gas_fini_domain((struct iommu_domain *)domain); | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
} | } | ||||
if ((domain->flags & DMAR_DOMAIN_PGTBL_INITED) != 0) { | if ((domain->iodom.flags & DMAR_DOMAIN_PGTBL_INITED) != 0) { | ||||
if (domain->pgtbl_obj != NULL) | if (domain->pgtbl_obj != NULL) | ||||
DMAR_DOMAIN_PGLOCK(domain); | DMAR_DOMAIN_PGLOCK(domain); | ||||
domain_free_pgtbl(domain); | domain_free_pgtbl(domain); | ||||
} | } | ||||
mtx_destroy(&domain->iodom.lock); | mtx_destroy(&domain->iodom.lock); | ||||
dmar = (struct dmar_unit *)domain->iodom.iommu; | dmar = (struct dmar_unit *)domain->iodom.iommu; | ||||
free_unr(dmar->domids, domain->domain); | free_unr(dmar->domids, domain->domain); | ||||
free(domain, M_DMAR_DOMAIN); | free(domain, M_DMAR_DOMAIN); | ||||
▲ Show 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) | ||||
ctx->context.domain = &domain->iodom; | ctx->context.domain = &domain->iodom; | ||||
dmar_ctx_link(ctx); | dmar_ctx_link(ctx); | ||||
ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); | ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); | ||||
dmar_unmap_pgtbl(sf); | dmar_unmap_pgtbl(sf); | ||||
error = dmar_flush_for_ctx_entry(dmar, true); | error = dmar_flush_for_ctx_entry(dmar, true); | ||||
/* If flush failed, rolling back would not work as well. */ | /* If flush failed, rolling back would not work as well. */ | ||||
printf("dmar%d rid %x domain %d->%d %s-mapped\n", | printf("dmar%d rid %x domain %d->%d %s-mapped\n", | ||||
dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain, | dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain, | ||||
(domain->flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re"); | (domain->iodom.flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re"); | ||||
dmar_unref_domain_locked(dmar, old_domain); | dmar_unref_domain_locked(dmar, old_domain); | ||||
TD_PINNED_ASSERT; | TD_PINNED_ASSERT; | ||||
return (error); | return (error); | ||||
} | } | ||||
static void | static void | ||||
dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) | dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) | ||||
{ | { | ||||
DMAR_ASSERT_LOCKED(dmar); | DMAR_ASSERT_LOCKED(dmar); | ||||
KASSERT(domain->refs >= 1, | KASSERT(domain->refs >= 1, | ||||
("dmar %d domain %p refs %u", dmar->iommu.unit, domain, | ("dmar %d domain %p refs %u", dmar->iommu.unit, domain, | ||||
domain->refs)); | domain->refs)); | ||||
KASSERT(domain->refs > domain->ctx_cnt, | KASSERT(domain->refs > domain->ctx_cnt, | ||||
("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, | ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, | ||||
domain->refs, domain->ctx_cnt)); | domain->refs, domain->ctx_cnt)); | ||||
if (domain->refs > 1) { | if (domain->refs > 1) { | ||||
domain->refs--; | domain->refs--; | ||||
DMAR_UNLOCK(dmar); | DMAR_UNLOCK(dmar); | ||||
return; | return; | ||||
} | } | ||||
KASSERT((domain->flags & DMAR_DOMAIN_RMRR) == 0, | KASSERT((domain->iodom.flags & DMAR_DOMAIN_RMRR) == 0, | ||||
("lost ref on RMRR domain %p", domain)); | ("lost ref on RMRR domain %p", domain)); | ||||
LIST_REMOVE(domain, link); | LIST_REMOVE(domain, link); | ||||
DMAR_UNLOCK(dmar); | DMAR_UNLOCK(dmar); | ||||
taskqueue_drain(dmar->iommu.delayed_taskqueue, | taskqueue_drain(dmar->iommu.delayed_taskqueue, | ||||
&domain->iodom.unload_task); | &domain->iodom.unload_task); | ||||
dmar_domain_destroy(domain); | dmar_domain_destroy(domain); | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | LIST_FOREACH(domain, &dmar->domains, link) { | ||||
} | } | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
void | void | ||||
dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) | dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) | ||||
{ | { | ||||
struct dmar_domain *domain; | struct iommu_domain *domain; | ||||
domain = (struct dmar_domain *)entry->domain; | domain = entry->domain; | ||||
DMAR_DOMAIN_LOCK(domain); | IOMMU_DOMAIN_LOCK(domain); | ||||
if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) | if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) | ||||
dmar_gas_free_region(domain, entry); | iommu_gas_free_region(domain, entry); | ||||
else | else | ||||
dmar_gas_free_space(domain, entry); | iommu_gas_free_space(domain, entry); | ||||
DMAR_DOMAIN_UNLOCK(domain); | IOMMU_DOMAIN_UNLOCK(domain); | ||||
if (free) | if (free) | ||||
dmar_gas_free_entry(domain, entry); | iommu_gas_free_entry(domain, entry); | ||||
else | else | ||||
entry->flags = 0; | entry->flags = 0; | ||||
} | } | ||||
void | void | ||||
dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free) | dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free) | ||||
{ | { | ||||
struct dmar_domain *domain; | struct dmar_domain *domain; | ||||
▲ Show 20 Lines • Show All 142 Lines • Show Last 20 Lines |