Differential D25665 Diff 74622 head/sys/x86/iommu/busdma_dmar.c

Changeset View

Standalone View

head/sys/x86/iommu/busdma_dmar.c

Show All 33 Lines

#include <sys/param.h>		#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/domainset.h>		#include <sys/domainset.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/bus.h>		#include <sys/bus.h>
#include <sys/conf.h>		#include <sys/conf.h>
#include <sys/interrupt.h>		#include <sys/interrupt.h>
		#include <sys/iommu.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/ktr.h>		#include <sys/ktr.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/proc.h>		#include <sys/proc.h>
#include <sys/memdesc.h>		#include <sys/memdesc.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/rman.h>		#include <sys/rman.h>
#include <sys/taskqueue.h>		#include <sys/taskqueue.h>
#include <sys/tree.h>		#include <sys/tree.h>
#include <sys/uio.h>		#include <sys/uio.h>
#include <sys/vmem.h>		#include <sys/vmem.h>
#include <dev/pci/pcireg.h>		#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>		#include <dev/pci/pcivar.h>
#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/vm_kern.h>		#include <vm/vm_kern.h>
#include <vm/vm_object.h>		#include <vm/vm_object.h>
#include <vm/vm_page.h>		#include <vm/vm_page.h>
#include <vm/vm_map.h>		#include <vm/vm_map.h>
#include <machine/atomic.h>		#include <machine/atomic.h>
#include <machine/bus.h>		#include <machine/bus.h>
#include <machine/md_var.h>		#include <machine/md_var.h>
		#if defined(__amd64__) \|\| defined(__i386__)
#include <machine/specialreg.h>		#include <machine/specialreg.h>
#include <x86/include/busdma_impl.h>		#include <x86/include/busdma_impl.h>
#include <x86/iommu/intel_reg.h>		#include <x86/iommu/intel_reg.h>
#include <x86/iommu/busdma_dmar.h>		#include <x86/iommu/busdma_dmar.h>
#include <x86/iommu/intel_dmar.h>		#include <x86/iommu/intel_dmar.h>
		#endif

/*		/*
* busdma_dmar.c, the implementation of the busdma(9) interface using		* busdma_dmar.c, the implementation of the busdma(9) interface using
* DMAR units from Intel VT-d.		* DMAR units from Intel VT-d.
*/		*/

static bool		static bool
iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)		iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
Show All 29 Lines	iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
else		else
ret = default_bounce != 0;		ret = default_bounce != 0;
freeenv(env);		freeenv(env);
return (ret);		return (ret);
}		}

/*		/*
* Given original device, find the requester ID that will be seen by		* Given original device, find the requester ID that will be seen by
* the DMAR unit and used for page table lookup. PCI bridges may take		* the IOMMU unit and used for page table lookup. PCI bridges may take
* ownership of transactions from downstream devices, so it may not be		* ownership of transactions from downstream devices, so it may not be
* the same as the BSF of the target device. In those cases, all		* the same as the BSF of the target device. In those cases, all
* devices downstream of the bridge must share a single mapping		* devices downstream of the bridge must share a single mapping
* domain, and must collectively be assigned to use either DMAR or		* domain, and must collectively be assigned to use either IOMMU or
* bounce mapping.		* bounce mapping.
*/		*/
device_t		device_t
iommu_get_requester(device_t dev, uint16_t *rid)		iommu_get_requester(device_t dev, uint16_t *rid)
{		{
devclass_t pci_class;		devclass_t pci_class;
device_t l, pci, pcib, pcip, pcibp, requester;		device_t l, pci, pcib, pcip, pcibp, requester;
int cap_offset;		int cap_offset;
uint16_t pcie_flags;		uint16_t pcie_flags;
bool bridge_is_pcie;		bool bridge_is_pcie;

pci_class = devclass_find("pci");		pci_class = devclass_find("pci");
l = requester = dev;		l = requester = dev;

*rid = pci_get_rid(dev);		*rid = pci_get_rid(dev);

/*		/*
* Walk the bridge hierarchy from the target device to the		* Walk the bridge hierarchy from the target device to the
* host port to find the translating bridge nearest the DMAR		* host port to find the translating bridge nearest the IOMMU
* unit.		* unit.
*/		*/
for (;;) {		for (;;) {
pci = device_get_parent(l);		pci = device_get_parent(l);
KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent "		KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent "
"for %s", device_get_name(dev), device_get_name(l)));		"for %s", device_get_name(dev), device_get_name(l)));
KASSERT(device_get_devclass(pci) == pci_class,		KASSERT(device_get_devclass(pci) == pci_class,
("iommu_get_requester(%s): non-pci parent %s for %s",		("iommu_get_requester(%s): non-pci parent %s for %s",
Show All 21 Lines	if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
* device is PCIe, because it is possible (but		* device is PCIe, because it is possible (but
* unlikely) to have a PCI->PCIe bridge		* unlikely) to have a PCI->PCIe bridge
* somewhere in the hierarchy.		* somewhere in the hierarchy.
*/		*/
l = pcib;		l = pcib;
} else {		} else {
/*		/*
* Device is not PCIe, it cannot be seen as a		* Device is not PCIe, it cannot be seen as a
* requester by DMAR unit. Check whether the		* requester by IOMMU unit. Check whether the
* bridge is PCIe.		* bridge is PCIe.
*/		*/
bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,		bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
&cap_offset) == 0;		&cap_offset) == 0;
requester = pcib;		requester = pcib;

/*		/*
* Check for a buggy PCIe/PCI bridge that		* Check for a buggy PCIe/PCI bridge that
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines	iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
struct iommu_ctx *ctx;		struct iommu_ctx *ctx;
bool disabled;		bool disabled;
uint16_t rid;		uint16_t rid;

requester = iommu_get_requester(dev, &rid);		requester = iommu_get_requester(dev, &rid);

/*		/*
* If the user requested the IOMMU disabled for the device, we		* If the user requested the IOMMU disabled for the device, we
* cannot disable the DMAR, due to possibility of other		* cannot disable the IOMMU unit, due to possibility of other
* devices on the same DMAR still requiring translation.		* devices on the same IOMMU unit still requiring translation.
* Instead provide the identity mapping for the device		* Instead provide the identity mapping for the device
* context.		* context.
*/		*/
disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),		disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),
pci_get_bus(requester), pci_get_slot(requester),		pci_get_bus(requester), pci_get_slot(requester),
pci_get_function(requester));		pci_get_function(requester));
ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);		ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);
if (ctx == NULL)		if (ctx == NULL)
Show All 18 Lines
bus_dma_tag_t		bus_dma_tag_t
acpi_iommu_get_dma_tag(device_t dev, device_t child)		acpi_iommu_get_dma_tag(device_t dev, device_t child)
{		{
struct iommu_unit *unit;		struct iommu_unit *unit;
struct iommu_ctx *ctx;		struct iommu_ctx *ctx;
bus_dma_tag_t res;		bus_dma_tag_t res;

unit = iommu_find(child, bootverbose);		unit = iommu_find(child, bootverbose);
/* Not in scope of any DMAR ? */		/* Not in scope of any IOMMU ? */
if (unit == NULL)		if (unit == NULL)
return (NULL);		return (NULL);
if (!unit->dma_enabled)		if (!unit->dma_enabled)
return (NULL);		return (NULL);

		#if defined(__amd64__) \|\| defined(__i386__)
dmar_quirks_pre_use(unit);		dmar_quirks_pre_use(unit);
dmar_instantiate_rmrr_ctxs(unit);		dmar_instantiate_rmrr_ctxs(unit);
		#endif

ctx = iommu_instantiate_ctx(unit, child, false);		ctx = iommu_instantiate_ctx(unit, child, false);
res = ctx == NULL ? NULL : (bus_dma_tag_t)ctx->tag;		res = ctx == NULL ? NULL : (bus_dma_tag_t)ctx->tag;
return (res);		return (res);
}		}

bool		bool
bus_dma_dmar_set_buswide(device_t dev)		bus_dma_dmar_set_buswide(device_t dev)
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines
{		{
struct iommu_ctx *ctx;		struct iommu_ctx *ctx;
struct iommu_domain *domain;		struct iommu_domain *domain;
struct iommu_map_entry *entry;		struct iommu_map_entry *entry;
iommu_gaddr_t size;		iommu_gaddr_t size;
bus_size_t buflen1;		bus_size_t buflen1;
int error, idx, gas_flags, seg;		int error, idx, gas_flags, seg;

KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset));		KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset));
if (segs == NULL)		if (segs == NULL)
segs = tag->segments;		segs = tag->segments;
ctx = tag->ctx;		ctx = tag->ctx;
domain = ctx->domain;		domain = ctx->domain;
seg = *segp;		seg = *segp;
error = 0;		error = 0;
idx = 0;		idx = 0;
while (buflen > 0) {		while (buflen > 0) {
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	while (buflen > 0) {
IOMMU_DOMAIN_UNLOCK(domain);		IOMMU_DOMAIN_UNLOCK(domain);
TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);		TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);

segs[seg].ds_addr = entry->start + offset;		segs[seg].ds_addr = entry->start + offset;
segs[seg].ds_len = buflen1;		segs[seg].ds_len = buflen1;

idx += OFF_TO_IDX(trunc_page(offset + buflen1));		idx += OFF_TO_IDX(trunc_page(offset + buflen1));
offset += buflen1;		offset += buflen1;
offset &= DMAR_PAGE_MASK;		offset &= IOMMU_PAGE_MASK;
buflen -= buflen1;		buflen -= buflen1;
}		}
if (error == 0)		if (error == 0)
*segp = seg;		*segp = seg;
return (error);		return (error);
}		}

static int		static int
▲ Show 20 Lines • Show All 203 Lines • ▼ Show 20 Lines	iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
}		}

if (segs == NULL)		if (segs == NULL)
segs = tag->segments;		segs = tag->segments;
return (segs);		return (segs);
}		}

/*		/*
* The limitations of busdma KPI forces the dmar to perform the actual		* The limitations of busdma KPI forces the iommu to perform the actual
* unload, consisting of the unmapping of the map entries page tables,		* unload, consisting of the unmapping of the map entries page tables,
* from the delayed context on i386, since page table page mapping		* from the delayed context on i386, since page table page mapping
* might require a sleep to be successfull. The unfortunate		* might require a sleep to be successfull. The unfortunate
* consequence is that the DMA requests can be served some time after		* consequence is that the DMA requests can be served some time after
* the bus_dmamap_unload() call returned.		* the bus_dmamap_unload() call returned.
*		*
* On amd64, we assume that sf allocation cannot fail.		* On amd64, we assume that sf allocation cannot fail.
*/		*/
▲ Show 20 Lines • Show All 192 Lines • Show Last 20 Lines