Differential D25743 Diff 74794 sys/x86/iommu/intel_ctx.c

Changeset View

Standalone View

sys/x86/iommu/intel_ctx.c

Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines

static void		static void
device_tag_init(struct dmar_ctx *ctx, device_t dev)		device_tag_init(struct dmar_ctx *ctx, device_t dev)
{		{
struct dmar_domain *domain;		struct dmar_domain *domain;
bus_addr_t maxaddr;		bus_addr_t maxaddr;

domain = (struct dmar_domain *)ctx->context.domain;		domain = (struct dmar_domain *)ctx->context.domain;
maxaddr = MIN(domain->end, BUS_SPACE_MAXADDR);		maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR);
ctx->context.tag->common.ref_count = 1; /* Prevent free */		ctx->context.tag->common.ref_count = 1; /* Prevent free */
ctx->context.tag->common.impl = &bus_dma_iommu_impl;		ctx->context.tag->common.impl = &bus_dma_iommu_impl;
ctx->context.tag->common.boundary = 0;		ctx->context.tag->common.boundary = 0;
ctx->context.tag->common.lowaddr = maxaddr;		ctx->context.tag->common.lowaddr = maxaddr;
ctx->context.tag->common.highaddr = maxaddr;		ctx->context.tag->common.highaddr = maxaddr;
ctx->context.tag->common.maxsize = maxaddr;		ctx->context.tag->common.maxsize = maxaddr;
ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED;		ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED;
ctx->context.tag->common.maxsegsz = maxaddr;		ctx->context.tag->common.maxsegsz = maxaddr;
Show All 37 Lines	ctx_id_entry_init(struct dmar_ctx ctx, dmar_ctx_entry_t ctxp, bool move,
domain = (struct dmar_domain *)ctx->context.domain;		domain = (struct dmar_domain *)ctx->context.domain;
unit = (struct dmar_unit *)domain->iodom.iommu;		unit = (struct dmar_unit *)domain->iodom.iommu;
KASSERT(move \|\| (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),		KASSERT(move \|\| (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",		("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),		unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),
pci_get_function(ctx->context.tag->owner),		pci_get_function(ctx->context.tag->owner),
ctxp->ctx1, ctxp->ctx2));		ctxp->ctx1, ctxp->ctx2));

if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 &&		if ((domain->iodom.flags & DMAR_DOMAIN_IDMAP) != 0 &&
(unit->hw_ecap & DMAR_ECAP_PT) != 0) {		(unit->hw_ecap & DMAR_ECAP_PT) != 0) {
KASSERT(domain->pgtbl_obj == NULL,		KASSERT(domain->pgtbl_obj == NULL,
("ctx %p non-null pgtbl_obj", ctx));		("ctx %p non-null pgtbl_obj", ctx));
ctx_root = NULL;		ctx_root = NULL;
} else {		} else {
ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC);		ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC);
}		}

▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus,
TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {		TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) {
/*		/*
* VT-d specification requires that the start of an		* VT-d specification requires that the start of an
* RMRR entry is 4k-aligned. Buggy BIOSes put		* RMRR entry is 4k-aligned. Buggy BIOSes put
* anything into the start and end fields. Truncate		* anything into the start and end fields. Truncate
* and round as neccesary.		* and round as neccesary.
*		*
* We also allow the overlapping RMRR entries, see		* We also allow the overlapping RMRR entries, see
* dmar_gas_alloc_region().		* iommu_gas_alloc_region().
*/		*/
start = entry->start;		start = entry->start;
end = entry->end;		end = entry->end;
if (bootverbose)		if (bootverbose)
printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",		printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",
domain->iodom.iommu->unit, bus, slot, func,		domain->iodom.iommu->unit, bus, slot, func,
(uintmax_t)start, (uintmax_t)end);		(uintmax_t)start, (uintmax_t)end);
entry->start = trunc_page(start);		entry->start = trunc_page(start);
Show All 11 Lines	if (entry->start == entry->end) {
entry->end += DMAR_PAGE_SIZE * 0x20;		entry->end += DMAR_PAGE_SIZE * 0x20;
}		}
size = OFF_TO_IDX(entry->end - entry->start);		size = OFF_TO_IDX(entry->end - entry->start);
ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);		ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
for (i = 0; i < size; i++) {		for (i = 0; i < size; i++) {
ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,		ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
VM_MEMATTR_DEFAULT);		VM_MEMATTR_DEFAULT);
}		}
error1 = dmar_gas_map_region(domain, entry,		error1 = iommu_gas_map_region((struct iommu_domain *)domain,
		entry,
IOMMU_MAP_ENTRY_READ \| IOMMU_MAP_ENTRY_WRITE,		IOMMU_MAP_ENTRY_READ \| IOMMU_MAP_ENTRY_WRITE,
IOMMU_MF_CANWAIT \| IOMMU_MF_RMRR, ma);		IOMMU_MF_CANWAIT \| IOMMU_MF_RMRR, ma);
/*		/*
* Non-failed RMRR entries are owned by context rb		* Non-failed RMRR entries are owned by context rb
* tree. Get rid of the failed entry, but do not stop		* tree. Get rid of the failed entry, but do not stop
* the loop. Rest of the parsed RMRR entries are		* the loop. Rest of the parsed RMRR entries are
* loaded and removed on the context destruction.		* loaded and removed on the context destruction.
*/		*/
if (error1 == 0 && entry->end != entry->start) {		if (error1 == 0 && entry->end != entry->start) {
IOMMU_LOCK(domain->iodom.iommu);		IOMMU_LOCK(domain->iodom.iommu);
domain->refs++; /* XXXKIB prevent free */		domain->refs++; /* XXXKIB prevent free */
domain->flags \|= DMAR_DOMAIN_RMRR;		domain->iodom.flags \|= DMAR_DOMAIN_RMRR;
IOMMU_UNLOCK(domain->iodom.iommu);		IOMMU_UNLOCK(domain->iodom.iommu);
} else {		} else {
if (error1 != 0) {		if (error1 != 0) {
if (dev != NULL)		if (dev != NULL)
device_printf(dev, "");		device_printf(dev, "");
printf("pci%d:%d:%d ", bus, slot, func);		printf("pci%d:%d:%d ", bus, slot, func);
printf(		printf(
"dmar%d failed to map RMRR region (%jx, %jx) %d\n",		"dmar%d failed to map RMRR region (%jx, %jx) %d\n",
domain->iodom.iommu->unit, start, end,		domain->iodom.iommu->unit, start, end,
error1);		error1);
error = error1;		error = error1;
}		}
TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);		TAILQ_REMOVE(&rmrr_entries, entry, unroll_link);
dmar_gas_free_entry(domain, entry);		iommu_gas_free_entry((struct iommu_domain *)domain,
		entry);
}		}
for (i = 0; i < size; i++)		for (i = 0; i < size; i++)
vm_page_putfake(ma[i]);		vm_page_putfake(ma[i]);
free(ma, M_TEMP);		free(ma, M_TEMP);
}		}
return (error);		return (error);
}		}

static struct dmar_domain *		static struct dmar_domain *
dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)		dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
{		{
		struct iommu_domain *iodom;
struct dmar_domain *domain;		struct dmar_domain *domain;
int error, id, mgaw;		int error, id, mgaw;

id = alloc_unr(dmar->domids);		id = alloc_unr(dmar->domids);
if (id == -1)		if (id == -1)
return (NULL);		return (NULL);
domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK \| M_ZERO);		domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK \| M_ZERO);
		iodom = (struct iommu_domain *)domain;
domain->domain = id;		domain->domain = id;
LIST_INIT(&domain->contexts);		LIST_INIT(&domain->contexts);
RB_INIT(&domain->rb_root);		RB_INIT(&domain->iodom.rb_root);
TAILQ_INIT(&domain->iodom.unload_entries);		TAILQ_INIT(&domain->iodom.unload_entries);
TASK_INIT(&domain->iodom.unload_task, 0, dmar_domain_unload_task,		TASK_INIT(&domain->iodom.unload_task, 0, dmar_domain_unload_task,
domain);		domain);
mtx_init(&domain->iodom.lock, "dmardom", NULL, MTX_DEF);		mtx_init(&domain->iodom.lock, "dmardom", NULL, MTX_DEF);
domain->dmar = dmar;		domain->dmar = dmar;
domain->iodom.iommu = &dmar->iommu;		domain->iodom.iommu = &dmar->iommu;

/*		/*
* For now, use the maximal usable physical address of the		* For now, use the maximal usable physical address of the
* installed memory to calculate the mgaw on id_mapped domain.		* installed memory to calculate the mgaw on id_mapped domain.
* It is useful for the identity mapping, and less so for the		* It is useful for the identity mapping, and less so for the
* virtualized bus address space.		* virtualized bus address space.
*/		*/
domain->end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;		domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
mgaw = dmar_maxaddr2mgaw(dmar, domain->end, !id_mapped);		mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped);
error = domain_set_agaw(domain, mgaw);		error = domain_set_agaw(domain, mgaw);
if (error != 0)		if (error != 0)
goto fail;		goto fail;
if (!id_mapped)		if (!id_mapped)
/* Use all supported address space for remapping. */		/* Use all supported address space for remapping. */
domain->end = 1ULL << (domain->agaw - 1);		domain->iodom.end = 1ULL << (domain->agaw - 1);

dmar_gas_init_domain(domain);		iommu_gas_init_domain((struct iommu_domain *)domain);

if (id_mapped) {		if (id_mapped) {
if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {		if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,		domain->pgtbl_obj = domain_get_idmap_pgtbl(domain,
domain->end);		domain->iodom.end);
}		}
domain->flags \|= DMAR_DOMAIN_IDMAP;		domain->iodom.flags \|= DMAR_DOMAIN_IDMAP;
} else {		} else {
error = domain_alloc_pgtbl(domain);		error = domain_alloc_pgtbl(domain);
if (error != 0)		if (error != 0)
goto fail;		goto fail;
/* Disable local apic region access */		/* Disable local apic region access */
error = dmar_gas_reserve_region(domain, 0xfee00000,		error = iommu_gas_reserve_region(iodom, 0xfee00000,
0xfeefffff + 1);		0xfeefffff + 1);
if (error != 0)		if (error != 0)
goto fail;		goto fail;
}		}
return (domain);		return (domain);

fail:		fail:
dmar_domain_destroy(domain);		dmar_domain_destroy(domain);
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines	dmar_domain_destroy(struct dmar_domain *domain)
KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),		KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),
("unfinished unloads %p", domain));		("unfinished unloads %p", domain));
KASSERT(LIST_EMPTY(&domain->contexts),		KASSERT(LIST_EMPTY(&domain->contexts),
("destroying dom %p with contexts", domain));		("destroying dom %p with contexts", domain));
KASSERT(domain->ctx_cnt == 0,		KASSERT(domain->ctx_cnt == 0,
("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));		("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
KASSERT(domain->refs == 0,		KASSERT(domain->refs == 0,
("destroying dom %p with refs %d", domain, domain->refs));		("destroying dom %p with refs %d", domain, domain->refs));
if ((domain->flags & DMAR_DOMAIN_GAS_INITED) != 0) {		if ((domain->iodom.flags & DMAR_DOMAIN_GAS_INITED) != 0) {
DMAR_DOMAIN_LOCK(domain);		DMAR_DOMAIN_LOCK(domain);
dmar_gas_fini_domain(domain);		iommu_gas_fini_domain((struct iommu_domain *)domain);
DMAR_DOMAIN_UNLOCK(domain);		DMAR_DOMAIN_UNLOCK(domain);
}		}
if ((domain->flags & DMAR_DOMAIN_PGTBL_INITED) != 0) {		if ((domain->iodom.flags & DMAR_DOMAIN_PGTBL_INITED) != 0) {
if (domain->pgtbl_obj != NULL)		if (domain->pgtbl_obj != NULL)
DMAR_DOMAIN_PGLOCK(domain);		DMAR_DOMAIN_PGLOCK(domain);
domain_free_pgtbl(domain);		domain_free_pgtbl(domain);
}		}
mtx_destroy(&domain->iodom.lock);		mtx_destroy(&domain->iodom.lock);
dmar = (struct dmar_unit *)domain->iodom.iommu;		dmar = (struct dmar_unit *)domain->iodom.iommu;
free_unr(dmar->domids, domain->domain);		free_unr(dmar->domids, domain->domain);
free(domain, M_DMAR_DOMAIN);		free(domain, M_DMAR_DOMAIN);
▲ Show 20 Lines • Show All 181 Lines • ▼ Show 20 Lines	dmar_move_ctx_to_domain(struct dmar_domain domain, struct dmar_ctx ctx)
ctx->context.domain = &domain->iodom;		ctx->context.domain = &domain->iodom;
dmar_ctx_link(ctx);		dmar_ctx_link(ctx);
ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);		ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);
dmar_unmap_pgtbl(sf);		dmar_unmap_pgtbl(sf);
error = dmar_flush_for_ctx_entry(dmar, true);		error = dmar_flush_for_ctx_entry(dmar, true);
/* If flush failed, rolling back would not work as well. */		/* If flush failed, rolling back would not work as well. */
printf("dmar%d rid %x domain %d->%d %s-mapped\n",		printf("dmar%d rid %x domain %d->%d %s-mapped\n",
dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain,		dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain,
(domain->flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re");		(domain->iodom.flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re");
dmar_unref_domain_locked(dmar, old_domain);		dmar_unref_domain_locked(dmar, old_domain);
TD_PINNED_ASSERT;		TD_PINNED_ASSERT;
return (error);		return (error);
}		}

static void		static void
dmar_unref_domain_locked(struct dmar_unit dmar, struct dmar_domain domain)		dmar_unref_domain_locked(struct dmar_unit dmar, struct dmar_domain domain)
{		{

DMAR_ASSERT_LOCKED(dmar);		DMAR_ASSERT_LOCKED(dmar);
KASSERT(domain->refs >= 1,		KASSERT(domain->refs >= 1,
("dmar %d domain %p refs %u", dmar->iommu.unit, domain,		("dmar %d domain %p refs %u", dmar->iommu.unit, domain,
domain->refs));		domain->refs));
KASSERT(domain->refs > domain->ctx_cnt,		KASSERT(domain->refs > domain->ctx_cnt,
("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,		("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,
domain->refs, domain->ctx_cnt));		domain->refs, domain->ctx_cnt));

if (domain->refs > 1) {		if (domain->refs > 1) {
domain->refs--;		domain->refs--;
DMAR_UNLOCK(dmar);		DMAR_UNLOCK(dmar);
return;		return;
}		}

KASSERT((domain->flags & DMAR_DOMAIN_RMRR) == 0,		KASSERT((domain->iodom.flags & DMAR_DOMAIN_RMRR) == 0,
("lost ref on RMRR domain %p", domain));		("lost ref on RMRR domain %p", domain));

LIST_REMOVE(domain, link);		LIST_REMOVE(domain, link);
DMAR_UNLOCK(dmar);		DMAR_UNLOCK(dmar);

taskqueue_drain(dmar->iommu.delayed_taskqueue,		taskqueue_drain(dmar->iommu.delayed_taskqueue,
&domain->iodom.unload_task);		&domain->iodom.unload_task);
dmar_domain_destroy(domain);		dmar_domain_destroy(domain);
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines	LIST_FOREACH(domain, &dmar->domains, link) {
}		}
}		}
return (NULL);		return (NULL);
}		}

void		void
dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)		dmar_domain_free_entry(struct iommu_map_entry *entry, bool free)
{		{
struct dmar_domain *domain;		struct iommu_domain *domain;

domain = (struct dmar_domain *)entry->domain;		domain = entry->domain;
DMAR_DOMAIN_LOCK(domain);		IOMMU_DOMAIN_LOCK(domain);
if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)		if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
dmar_gas_free_region(domain, entry);		iommu_gas_free_region(domain, entry);
else		else
dmar_gas_free_space(domain, entry);		iommu_gas_free_space(domain, entry);
DMAR_DOMAIN_UNLOCK(domain);		IOMMU_DOMAIN_UNLOCK(domain);
if (free)		if (free)
dmar_gas_free_entry(domain, entry);		iommu_gas_free_entry(domain, entry);
else		else
entry->flags = 0;		entry->flags = 0;
}		}

void		void
dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free)		dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free)
{		{
struct dmar_domain *domain;		struct dmar_domain *domain;
▲ Show 20 Lines • Show All 142 Lines • Show Last 20 Lines