Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/vmm/io/iommu.c
Show First 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | |||||
static __inline void | static __inline void | ||||
IOMMU_CLEANUP(void) | IOMMU_CLEANUP(void) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->cleanup)(); | (*ops->cleanup)(); | ||||
} | } | ||||
static __inline void * | static __inline void * | ||||
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) | IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr, bool host_domain) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
return ((*ops->create_domain)(maxaddr)); | return ((*ops->create_domain)(maxaddr, host_domain)); | ||||
else | else | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
IOMMU_DESTROY_DOMAIN(void *dom) | IOMMU_DESTROY_DOMAIN(void *dom) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->destroy_domain)(dom); | (*ops->destroy_domain)(dom); | ||||
} | } | ||||
static __inline uint64_t | static __inline int | ||||
IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) | IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, | ||||
uint64_t len, uint64_t *res_len) | |||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
return ((*ops->create_mapping)(domain, gpa, hpa, len)); | return ((*ops->create_mapping)(domain, gpa, hpa, len, res_len)); | ||||
else | return (EOPNOTSUPP); | ||||
return (len); /* XXX */ | |||||
} | } | ||||
static __inline uint64_t | static __inline int | ||||
IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) | IOMMU_CREATE_MAPPING_BULK(void *domain, vm_paddr_t gpa, struct vm_page **ma, | ||||
size_t len) | |||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
return ((*ops->remove_mapping)(domain, gpa, len)); | return ((*ops->create_mapping_bulk)(domain, gpa, ma, len)); | ||||
else | return (EOPNOTSUPP); | ||||
return (len); /* XXX */ | |||||
} | } | ||||
static __inline void | static __inline int | ||||
IOMMU_ADD_DEVICE(void *domain, uint16_t rid) | IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len, | ||||
uint64_t *res_len) | |||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->add_device)(domain, rid); | return ((*ops->remove_mapping)(domain, gpa, len, res_len)); | ||||
return (EOPNOTSUPP); | |||||
} | } | ||||
static __inline void | static __inline int | ||||
IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) | IOMMU_ADD_DEVICE(void *domain, device_t dev, uint16_t rid) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->remove_device)(domain, rid); | return ((*ops->add_device)(domain, dev, rid)); | ||||
return (EOPNOTSUPP); | |||||
} | } | ||||
static __inline void | static __inline int | ||||
IOMMU_REMOVE_DEVICE(void *domain, device_t dev, uint16_t rid) | |||||
{ | |||||
if (ops != NULL && iommu_avail) | |||||
return ((*ops->remove_device)(domain, dev, rid)); | |||||
return (0); /* To allow ppt_attach() to succeed. */ | |||||
} | |||||
static __inline int | |||||
IOMMU_INVALIDATE_TLB(void *domain) | IOMMU_INVALIDATE_TLB(void *domain) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->invalidate_tlb)(domain); | return ((*ops->invalidate_tlb)(domain)); | ||||
return (0); | |||||
} | } | ||||
static __inline void | static __inline void | ||||
IOMMU_ENABLE(void) | IOMMU_ENABLE(void) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->enable)(); | (*ops->enable)(); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
IOMMU_DISABLE(void) | IOMMU_DISABLE(void) | ||||
{ | { | ||||
if (ops != NULL && iommu_avail) | if (ops != NULL && iommu_avail) | ||||
(*ops->disable)(); | (*ops->disable)(); | ||||
} | } | ||||
static __inline bool | |||||
IOMMU_GET_SWCAPS(enum iommu_swcaps cap) | |||||
{ | |||||
if (ops != NULL && ops->get_swcaps != NULL && iommu_avail) | |||||
return (*ops->get_swcaps)(cap); | |||||
return (false); | |||||
} | |||||
static void | static void | ||||
iommu_pci_add(void *arg, device_t dev) | iommu_pci_add(void *arg, device_t dev) | ||||
{ | { | ||||
/* Add new devices to the host domain. */ | /* Add new devices to the host domain. */ | ||||
iommu_add_device(host_domain, pci_get_rid(dev)); | iommu_add_device(host_domain, dev, pci_get_rid(dev)); | ||||
} | } | ||||
static void | static void | ||||
iommu_pci_delete(void *arg, device_t dev) | iommu_pci_delete(void *arg, device_t dev) | ||||
{ | { | ||||
iommu_remove_device(host_domain, pci_get_rid(dev)); | iommu_remove_device(host_domain, dev, pci_get_rid(dev)); | ||||
} | } | ||||
static void | static void | ||||
iommu_init(void) | iommu_init(void) | ||||
{ | { | ||||
int error, bus, slot, func; | int error, bus, slot, func; | ||||
vm_paddr_t maxaddr; | vm_paddr_t maxaddr; | ||||
devclass_t dc; | devclass_t dc; | ||||
device_t dev; | device_t dev; | ||||
if (!iommu_enable) | if (!iommu_enable) | ||||
return; | return; | ||||
if (vmm_is_intel()) | if (vmm_is_intel()) { | ||||
ops = &iommu_ops_intel; | /* XXXKIB ops = &iommu_ops_intel; */ | ||||
else if (vmm_is_svm()) | ops = &iommu_ops_dmar; | ||||
} else if (vmm_is_svm()) | |||||
ops = &iommu_ops_amd; | ops = &iommu_ops_amd; | ||||
else | else | ||||
ops = NULL; | ops = NULL; | ||||
error = IOMMU_INIT(); | error = IOMMU_INIT(); | ||||
if (error) | if (error) { | ||||
printf("iommu_init: error %d\n", error); | |||||
return; | return; | ||||
} | |||||
iommu_avail = 1; | iommu_avail = 1; | ||||
/* | /* | ||||
* Create a domain for the devices owned by the host | * Create a domain for the devices owned by the host | ||||
*/ | */ | ||||
maxaddr = vmm_mem_maxaddr(); | maxaddr = vmm_mem_maxaddr(); | ||||
host_domain = IOMMU_CREATE_DOMAIN(maxaddr); | host_domain = IOMMU_CREATE_DOMAIN(maxaddr, true); | ||||
if (host_domain == NULL) { | if (host_domain == NULL) { | ||||
printf("iommu_init: unable to create a host domain"); | printf("iommu_init: unable to create a host domain"); | ||||
IOMMU_CLEANUP(); | IOMMU_CLEANUP(); | ||||
ops = NULL; | ops = NULL; | ||||
iommu_avail = 0; | iommu_avail = 0; | ||||
return; | return; | ||||
} | } | ||||
Show All 18 Lines | for (slot = 0; slot <= PCI_SLOTMAX; slot++) { | ||||
if (dc != NULL && | if (dc != NULL && | ||||
device_get_devclass(dev) == dc) | device_get_devclass(dev) == dc) | ||||
continue; | continue; | ||||
/* | /* | ||||
* Everything else belongs to the host | * Everything else belongs to the host | ||||
* domain. | * domain. | ||||
*/ | */ | ||||
iommu_add_device(host_domain, | error = iommu_add_device(host_domain, dev, | ||||
pci_get_rid(dev)); | pci_get_rid(dev)); | ||||
if (error != 0) { | |||||
/* XXXKIB cleanup */ | |||||
return; | |||||
} | } | ||||
} | } | ||||
} | } | ||||
} | |||||
IOMMU_ENABLE(); | IOMMU_ENABLE(); | ||||
} | } | ||||
void | void | ||||
iommu_cleanup(void) | iommu_cleanup(void) | ||||
{ | { | ||||
Show All 18 Lines | iommu_create_domain(vm_paddr_t maxaddr) | ||||
if (iommu_initted < 2) { | if (iommu_initted < 2) { | ||||
if (atomic_cmpset_int(&iommu_initted, 0, 1)) { | if (atomic_cmpset_int(&iommu_initted, 0, 1)) { | ||||
iommu_init(); | iommu_init(); | ||||
atomic_store_rel_int(&iommu_initted, 2); | atomic_store_rel_int(&iommu_initted, 2); | ||||
} else | } else | ||||
while (iommu_initted == 1) | while (iommu_initted == 1) | ||||
cpu_spinwait(); | cpu_spinwait(); | ||||
} | } | ||||
return (IOMMU_CREATE_DOMAIN(maxaddr)); | return (IOMMU_CREATE_DOMAIN(maxaddr, false)); | ||||
} | } | ||||
void | void | ||||
iommu_destroy_domain(void *dom) | iommu_destroy_domain(void *dom) | ||||
{ | { | ||||
IOMMU_DESTROY_DOMAIN(dom); | IOMMU_DESTROY_DOMAIN(dom); | ||||
} | } | ||||
void | int | ||||
iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) | iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) | ||||
{ | { | ||||
uint64_t mapped, remaining; | uint64_t mapped, remaining; | ||||
int error; | |||||
remaining = len; | for (remaining = len; remaining > 0; gpa += mapped, hpa += mapped, | ||||
remaining -= mapped) { | |||||
while (remaining > 0) { | error = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining, | ||||
mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); | &mapped); | ||||
gpa += mapped; | if (error != 0) | ||||
hpa += mapped; | return (error); | ||||
remaining -= mapped; | |||||
} | } | ||||
return (0); | |||||
} | } | ||||
void | int | ||||
iommu_create_mapping_bulk(void *dom, vm_paddr_t gpa, struct vm_page **ma, | |||||
size_t len) | |||||
{ | |||||
return (IOMMU_CREATE_MAPPING_BULK(dom, gpa, ma, len)); | |||||
} | |||||
int | |||||
iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) | iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) | ||||
{ | { | ||||
uint64_t unmapped, remaining; | uint64_t unmapped, remaining; | ||||
int error; | |||||
remaining = len; | for (remaining = len; remaining > 0; gpa += unmapped, | ||||
remaining -= unmapped) { | |||||
while (remaining > 0) { | error = IOMMU_REMOVE_MAPPING(dom, gpa, remaining, &unmapped); | ||||
unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); | if (error != 0) | ||||
gpa += unmapped; | return (error); | ||||
remaining -= unmapped; | |||||
} | } | ||||
return (0); | |||||
} | } | ||||
void * | void * | ||||
iommu_host_domain(void) | iommu_host_domain(void) | ||||
{ | { | ||||
return (host_domain); | return (host_domain); | ||||
} | } | ||||
void | int | ||||
iommu_add_device(void *dom, uint16_t rid) | iommu_add_device(void *dom, device_t dev, uint16_t rid) | ||||
{ | { | ||||
IOMMU_ADD_DEVICE(dom, rid); | return (IOMMU_ADD_DEVICE(dom, dev, rid)); | ||||
} | } | ||||
void | int | ||||
iommu_remove_device(void *dom, uint16_t rid) | iommu_remove_device(void *dom, device_t dev, uint16_t rid) | ||||
{ | { | ||||
IOMMU_REMOVE_DEVICE(dom, rid); | return (IOMMU_REMOVE_DEVICE(dom, dev, rid)); | ||||
} | } | ||||
void | int | ||||
iommu_invalidate_tlb(void *domain) | iommu_invalidate_tlb(void *domain) | ||||
{ | { | ||||
IOMMU_INVALIDATE_TLB(domain); | return (IOMMU_INVALIDATE_TLB(domain)); | ||||
} | |||||
bool | |||||
iommu_get_swcaps(enum iommu_swcaps cap) | |||||
{ | |||||
return (IOMMU_GET_SWCAPS(cap)); | |||||
} | } |