Changeset View
Changeset View
Standalone View
Standalone View
sys/x86/iommu/intel_gas.c
Show First 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | |||||
#include <x86/iommu/busdma_dmar.h> | #include <x86/iommu/busdma_dmar.h> | ||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <x86/iommu/intel_dmar.h> | #include <x86/iommu/intel_dmar.h> | ||||
/* | /* | ||||
* Guest Address Space management. | * Guest Address Space management. | ||||
*/ | */ | ||||
static uma_zone_t dmar_map_entry_zone; | static uma_zone_t iommu_map_entry_zone; | ||||
static void | static void | ||||
intel_gas_init(void) | intel_gas_init(void) | ||||
{ | { | ||||
dmar_map_entry_zone = uma_zcreate("DMAR_MAP_ENTRY", | iommu_map_entry_zone = uma_zcreate("IOMMU_MAP_ENTRY", | ||||
sizeof(struct dmar_map_entry), NULL, NULL, | sizeof(struct iommu_map_entry), NULL, NULL, | ||||
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); | NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); | ||||
} | } | ||||
SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); | SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); | ||||
struct dmar_map_entry * | struct iommu_map_entry * | ||||
dmar_gas_alloc_entry(struct dmar_domain *domain, u_int flags) | dmar_gas_alloc_entry(struct iommu_domain *iodom, u_int flags) | ||||
{ | { | ||||
struct dmar_map_entry *res; | struct dmar_domain *domain; | ||||
struct iommu_map_entry *res; | |||||
domain = (struct dmar_domain *)iodom; | |||||
KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, | KASSERT((flags & ~(DMAR_PGF_WAITOK)) == 0, | ||||
("unsupported flags %x", flags)); | ("unsupported flags %x", flags)); | ||||
res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != | res = uma_zalloc(iommu_map_entry_zone, ((flags & DMAR_PGF_WAITOK) != | ||||
0 ? M_WAITOK : M_NOWAIT) | M_ZERO); | 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); | ||||
if (res != NULL) { | if (res != NULL) { | ||||
res->domain = domain; | res->domain = domain; | ||||
atomic_add_int(&domain->entries_cnt, 1); | atomic_add_int(&domain->entries_cnt, 1); | ||||
} | } | ||||
return (res); | return (res); | ||||
} | } | ||||
void | void | ||||
dmar_gas_free_entry(struct dmar_domain *domain, struct dmar_map_entry *entry) | dmar_gas_free_entry(struct iommu_domain *iodom, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_domain *domain; | |||||
domain = (struct dmar_domain *)iodom; | |||||
KASSERT(domain == entry->domain, | KASSERT(domain == entry->domain, | ||||
("mismatched free domain %p entry %p entry->domain %p", domain, | ("mismatched free domain %p entry %p entry->domain %p", domain, | ||||
entry, entry->domain)); | entry, entry->domain)); | ||||
atomic_subtract_int(&domain->entries_cnt, 1); | atomic_subtract_int(&domain->entries_cnt, 1); | ||||
uma_zfree(dmar_map_entry_zone, entry); | uma_zfree(iommu_map_entry_zone, entry); | ||||
} | } | ||||
static int | static int | ||||
dmar_gas_cmp_entries(struct dmar_map_entry *a, struct dmar_map_entry *b) | dmar_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b) | ||||
{ | { | ||||
/* Last entry have zero size, so <= */ | /* Last entry have zero size, so <= */ | ||||
KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", | KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", | ||||
a, (uintmax_t)a->start, (uintmax_t)a->end)); | a, (uintmax_t)a->start, (uintmax_t)a->end)); | ||||
KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", | KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", | ||||
b, (uintmax_t)b->start, (uintmax_t)b->end)); | b, (uintmax_t)b->start, (uintmax_t)b->end)); | ||||
KASSERT(a->end <= b->start || b->end <= a->start || | KASSERT(a->end <= b->start || b->end <= a->start || | ||||
a->end == a->start || b->end == b->start, | a->end == a->start || b->end == b->start, | ||||
("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", | ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", | ||||
a, (uintmax_t)a->start, (uintmax_t)a->end, | a, (uintmax_t)a->start, (uintmax_t)a->end, | ||||
b, (uintmax_t)b->start, (uintmax_t)b->end)); | b, (uintmax_t)b->start, (uintmax_t)b->end)); | ||||
if (a->end < b->end) | if (a->end < b->end) | ||||
return (-1); | return (-1); | ||||
else if (b->end < a->end) | else if (b->end < a->end) | ||||
return (1); | return (1); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
dmar_gas_augment_entry(struct dmar_map_entry *entry) | dmar_gas_augment_entry(struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_map_entry *child; | struct iommu_map_entry *child; | ||||
dmar_gaddr_t free_down; | dmar_gaddr_t free_down; | ||||
free_down = 0; | free_down = 0; | ||||
if ((child = RB_LEFT(entry, rb_entry)) != NULL) { | if ((child = RB_LEFT(entry, rb_entry)) != NULL) { | ||||
free_down = MAX(free_down, child->free_down); | free_down = MAX(free_down, child->free_down); | ||||
free_down = MAX(free_down, entry->start - child->last); | free_down = MAX(free_down, entry->start - child->last); | ||||
entry->first = child->first; | entry->first = child->first; | ||||
} else | } else | ||||
entry->first = entry->start; | entry->first = entry->start; | ||||
if ((child = RB_RIGHT(entry, rb_entry)) != NULL) { | if ((child = RB_RIGHT(entry, rb_entry)) != NULL) { | ||||
free_down = MAX(free_down, child->free_down); | free_down = MAX(free_down, child->free_down); | ||||
free_down = MAX(free_down, child->first - entry->end); | free_down = MAX(free_down, child->first - entry->end); | ||||
entry->last = child->last; | entry->last = child->last; | ||||
} else | } else | ||||
entry->last = entry->end; | entry->last = entry->end; | ||||
entry->free_down = free_down; | entry->free_down = free_down; | ||||
} | } | ||||
RB_GENERATE(dmar_gas_entries_tree, dmar_map_entry, rb_entry, | RB_GENERATE(dmar_gas_entries_tree, iommu_map_entry, rb_entry, | ||||
dmar_gas_cmp_entries); | dmar_gas_cmp_entries); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
static void | static void | ||||
dmar_gas_check_free(struct dmar_domain *domain) | dmar_gas_check_free(struct dmar_domain *domain) | ||||
{ | { | ||||
struct dmar_map_entry *entry, *l, *r; | struct iommu_map_entry *entry, *l, *r; | ||||
dmar_gaddr_t v; | dmar_gaddr_t v; | ||||
RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { | RB_FOREACH(entry, dmar_gas_entries_tree, &domain->rb_root) { | ||||
KASSERT(domain == entry->domain, | KASSERT(domain == entry->domain, | ||||
("mismatched free domain %p entry %p entry->domain %p", | ("mismatched free domain %p entry %p entry->domain %p", | ||||
domain, entry, entry->domain)); | domain, entry, entry->domain)); | ||||
l = RB_LEFT(entry, rb_entry); | l = RB_LEFT(entry, rb_entry); | ||||
r = RB_RIGHT(entry, rb_entry); | r = RB_RIGHT(entry, rb_entry); | ||||
v = 0; | v = 0; | ||||
if (l != NULL) { | if (l != NULL) { | ||||
v = MAX(v, l->free_down); | v = MAX(v, l->free_down); | ||||
v = MAX(v, entry->start - l->last); | v = MAX(v, entry->start - l->last); | ||||
} | } | ||||
if (r != NULL) { | if (r != NULL) { | ||||
v = MAX(v, r->free_down); | v = MAX(v, r->free_down); | ||||
v = MAX(v, r->first - entry->end); | v = MAX(v, r->first - entry->end); | ||||
} | } | ||||
MPASS(entry->free_down == v); | MPASS(entry->free_down == v); | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
static bool | static bool | ||||
dmar_gas_rb_insert(struct dmar_domain *domain, struct dmar_map_entry *entry) | dmar_gas_rb_insert(struct dmar_domain *domain, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_map_entry *found; | struct iommu_map_entry *found; | ||||
found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); | found = RB_INSERT(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
return (found == NULL); | return (found == NULL); | ||||
} | } | ||||
static void | static void | ||||
dmar_gas_rb_remove(struct dmar_domain *domain, struct dmar_map_entry *entry) | dmar_gas_rb_remove(struct dmar_domain *domain, struct iommu_map_entry *entry) | ||||
{ | { | ||||
RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
} | } | ||||
void | void | ||||
dmar_gas_init_domain(struct dmar_domain *domain) | dmar_gas_init_domain(struct dmar_domain *domain) | ||||
{ | { | ||||
struct dmar_map_entry *begin, *end; | struct iommu_map_entry *begin, *end; | ||||
begin = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); | begin = dmar_gas_alloc_entry(&domain->iodom, DMAR_PGF_WAITOK); | ||||
end = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); | end = dmar_gas_alloc_entry(&domain->iodom, DMAR_PGF_WAITOK); | ||||
DMAR_DOMAIN_LOCK(domain); | DMAR_DOMAIN_LOCK(domain); | ||||
KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); | KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); | ||||
KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); | KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); | ||||
begin->start = 0; | begin->start = 0; | ||||
begin->end = DMAR_PAGE_SIZE; | begin->end = DMAR_PAGE_SIZE; | ||||
begin->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; | begin->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; | ||||
dmar_gas_rb_insert(domain, begin); | dmar_gas_rb_insert(domain, begin); | ||||
end->start = domain->end; | end->start = domain->end; | ||||
end->end = domain->end; | end->end = domain->end; | ||||
end->flags = DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_UNMAPPED; | end->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; | ||||
dmar_gas_rb_insert(domain, end); | dmar_gas_rb_insert(domain, end); | ||||
domain->first_place = begin; | domain->first_place = begin; | ||||
domain->last_place = end; | domain->last_place = end; | ||||
domain->flags |= DMAR_DOMAIN_GAS_INITED; | domain->flags |= DMAR_DOMAIN_GAS_INITED; | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
} | } | ||||
void | void | ||||
dmar_gas_fini_domain(struct dmar_domain *domain) | dmar_gas_fini_domain(struct dmar_domain *domain) | ||||
{ | { | ||||
struct dmar_map_entry *entry, *entry1; | struct iommu_map_entry *entry, *entry1; | ||||
DMAR_DOMAIN_ASSERT_LOCKED(domain); | DMAR_DOMAIN_ASSERT_LOCKED(domain); | ||||
KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); | KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); | ||||
entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); | entry = RB_MIN(dmar_gas_entries_tree, &domain->rb_root); | ||||
KASSERT(entry->start == 0, ("start entry start %p", domain)); | KASSERT(entry->start == 0, ("start entry start %p", domain)); | ||||
KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); | KASSERT(entry->end == DMAR_PAGE_SIZE, ("start entry end %p", domain)); | ||||
KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, | KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE, | ||||
("start entry flags %p", domain)); | ("start entry flags %p", domain)); | ||||
RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
dmar_gas_free_entry(domain, entry); | dmar_gas_free_entry(&domain->iodom, entry); | ||||
entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); | entry = RB_MAX(dmar_gas_entries_tree, &domain->rb_root); | ||||
KASSERT(entry->start == domain->end, ("end entry start %p", domain)); | KASSERT(entry->start == domain->end, ("end entry start %p", domain)); | ||||
KASSERT(entry->end == domain->end, ("end entry end %p", domain)); | KASSERT(entry->end == domain->end, ("end entry end %p", domain)); | ||||
KASSERT(entry->flags == DMAR_MAP_ENTRY_PLACE, | KASSERT(entry->flags == IOMMU_MAP_ENTRY_PLACE, | ||||
("end entry flags %p", domain)); | ("end entry flags %p", domain)); | ||||
RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
dmar_gas_free_entry(domain, entry); | dmar_gas_free_entry(&domain->iodom, entry); | ||||
RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, | RB_FOREACH_SAFE(entry, dmar_gas_entries_tree, &domain->rb_root, | ||||
entry1) { | entry1) { | ||||
KASSERT((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0, | KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0, | ||||
("non-RMRR entry left %p", domain)); | ("non-RMRR entry left %p", domain)); | ||||
RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | RB_REMOVE(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
dmar_gas_free_entry(domain, entry); | dmar_gas_free_entry(&domain->iodom, entry); | ||||
} | } | ||||
} | } | ||||
struct dmar_gas_match_args { | struct dmar_gas_match_args { | ||||
struct dmar_domain *domain; | struct dmar_domain *domain; | ||||
dmar_gaddr_t size; | dmar_gaddr_t size; | ||||
int offset; | int offset; | ||||
const struct bus_dma_tag_common *common; | const struct bus_dma_tag_common *common; | ||||
u_int gas_flags; | u_int gas_flags; | ||||
struct dmar_map_entry *entry; | struct iommu_map_entry *entry; | ||||
}; | }; | ||||
/* | /* | ||||
* The interval [beg, end) is a free interval between two dmar_map_entries. | * The interval [beg, end) is a free interval between two dmar_map_entries. | ||||
* maxaddr is an upper bound on addresses that can be allocated. Try to | * maxaddr is an upper bound on addresses that can be allocated. Try to | ||||
* allocate space in the free interval, subject to the conditions expressed | * allocate space in the free interval, subject to the conditions expressed | ||||
* by a, and return 'true' if and only if the allocation attempt succeeds. | * by a, and return 'true' if and only if the allocation attempt succeeds. | ||||
*/ | */ | ||||
Show All 9 Lines | if (a->entry->start + a->size > maxaddr) | ||||
return (false); | return (false); | ||||
/* DMAR_PAGE_SIZE to create gap after new entry. */ | /* DMAR_PAGE_SIZE to create gap after new entry. */ | ||||
if (a->entry->start < beg + DMAR_PAGE_SIZE || | if (a->entry->start < beg + DMAR_PAGE_SIZE || | ||||
a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > end) | a->entry->start + a->size + a->offset + DMAR_PAGE_SIZE > end) | ||||
return (false); | return (false); | ||||
/* No boundary crossing. */ | /* No boundary crossing. */ | ||||
if (dmar_test_boundary(a->entry->start + a->offset, a->size, | if (iommu_test_boundary(a->entry->start + a->offset, a->size, | ||||
a->common->boundary)) | a->common->boundary)) | ||||
return (true); | return (true); | ||||
/* | /* | ||||
* The start + offset to start + offset + size region crosses | * The start + offset to start + offset + size region crosses | ||||
* the boundary. Check if there is enough space after the | * the boundary. Check if there is enough space after the | ||||
* next boundary after the beg. | * next boundary after the beg. | ||||
*/ | */ | ||||
bs = rounddown2(a->entry->start + a->offset + a->common->boundary, | bs = rounddown2(a->entry->start + a->offset + a->common->boundary, | ||||
a->common->boundary); | a->common->boundary); | ||||
start = roundup2(bs, a->common->alignment); | start = roundup2(bs, a->common->alignment); | ||||
/* DMAR_PAGE_SIZE to create gap after new entry. */ | /* DMAR_PAGE_SIZE to create gap after new entry. */ | ||||
if (start + a->offset + a->size + DMAR_PAGE_SIZE <= end && | if (start + a->offset + a->size + DMAR_PAGE_SIZE <= end && | ||||
start + a->offset + a->size <= maxaddr && | start + a->offset + a->size <= maxaddr && | ||||
dmar_test_boundary(start + a->offset, a->size, | iommu_test_boundary(start + a->offset, a->size, | ||||
a->common->boundary)) { | a->common->boundary)) { | ||||
a->entry->start = start; | a->entry->start = start; | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* Not enough space to align at the requested boundary, or | * Not enough space to align at the requested boundary, or | ||||
* boundary is smaller than the size, but allowed to split. | * boundary is smaller than the size, but allowed to split. | ||||
* We already checked that start + size does not overlap maxaddr. | * We already checked that start + size does not overlap maxaddr. | ||||
* | * | ||||
* XXXKIB. It is possible that bs is exactly at the start of | * XXXKIB. It is possible that bs is exactly at the start of | ||||
* the next entry, then we do not have gap. Ignore for now. | * the next entry, then we do not have gap. Ignore for now. | ||||
*/ | */ | ||||
if ((a->gas_flags & DMAR_GM_CANSPLIT) != 0) { | if ((a->gas_flags & IOMMU_MF_CANSPLIT) != 0) { | ||||
a->size = bs - a->entry->start; | a->size = bs - a->entry->start; | ||||
return (true); | return (true); | ||||
} | } | ||||
return (false); | return (false); | ||||
} | } | ||||
static void | static void | ||||
Show All 9 Lines | dmar_gas_match_insert(struct dmar_gas_match_args *a) | ||||
* The page sized gap is created between consequent | * The page sized gap is created between consequent | ||||
* allocations to ensure that out-of-bounds accesses fault. | * allocations to ensure that out-of-bounds accesses fault. | ||||
*/ | */ | ||||
a->entry->end = a->entry->start + a->size; | a->entry->end = a->entry->start + a->size; | ||||
found = dmar_gas_rb_insert(a->domain, a->entry); | found = dmar_gas_rb_insert(a->domain, a->entry); | ||||
KASSERT(found, ("found dup %p start %jx size %jx", | KASSERT(found, ("found dup %p start %jx size %jx", | ||||
a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); | a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); | ||||
a->entry->flags = DMAR_MAP_ENTRY_MAP; | a->entry->flags = IOMMU_MAP_ENTRY_MAP; | ||||
} | } | ||||
static int | static int | ||||
dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *entry) | dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_map_entry *child; | struct iommu_map_entry *child; | ||||
child = RB_RIGHT(entry, rb_entry); | child = RB_RIGHT(entry, rb_entry); | ||||
if (child != NULL && entry->end < a->common->lowaddr && | if (child != NULL && entry->end < a->common->lowaddr && | ||||
dmar_gas_match_one(a, entry->end, child->first, | dmar_gas_match_one(a, entry->end, child->first, | ||||
a->common->lowaddr)) { | a->common->lowaddr)) { | ||||
dmar_gas_match_insert(a); | dmar_gas_match_insert(a); | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 12 Lines | dmar_gas_lowermatch(struct dmar_gas_match_args *a, struct iommu_map_entry *entry) | ||||
} | } | ||||
child = RB_RIGHT(entry, rb_entry); | child = RB_RIGHT(entry, rb_entry); | ||||
if (child != NULL && 0 == dmar_gas_lowermatch(a, child)) | if (child != NULL && 0 == dmar_gas_lowermatch(a, child)) | ||||
return (0); | return (0); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
static int | static int | ||||
dmar_gas_uppermatch(struct dmar_gas_match_args *a, struct dmar_map_entry *entry) | dmar_gas_uppermatch(struct dmar_gas_match_args *a, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_map_entry *child; | struct iommu_map_entry *child; | ||||
if (entry->free_down < a->size + a->offset + DMAR_PAGE_SIZE) | if (entry->free_down < a->size + a->offset + DMAR_PAGE_SIZE) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
if (entry->last < a->common->highaddr) | if (entry->last < a->common->highaddr) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
child = RB_LEFT(entry, rb_entry); | child = RB_LEFT(entry, rb_entry); | ||||
if (child != NULL && 0 == dmar_gas_uppermatch(a, child)) | if (child != NULL && 0 == dmar_gas_uppermatch(a, child)) | ||||
return (0); | return (0); | ||||
Show All 13 Lines | dmar_gas_uppermatch(struct dmar_gas_match_args *a, struct iommu_map_entry *entry) | ||||
if (child != NULL && 0 == dmar_gas_uppermatch(a, child)) | if (child != NULL && 0 == dmar_gas_uppermatch(a, child)) | ||||
return (0); | return (0); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
static int | static int | ||||
dmar_gas_find_space(struct dmar_domain *domain, | dmar_gas_find_space(struct dmar_domain *domain, | ||||
const struct bus_dma_tag_common *common, dmar_gaddr_t size, | const struct bus_dma_tag_common *common, dmar_gaddr_t size, | ||||
int offset, u_int flags, struct dmar_map_entry *entry) | int offset, u_int flags, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_gas_match_args a; | struct dmar_gas_match_args a; | ||||
int error; | int error; | ||||
DMAR_DOMAIN_ASSERT_LOCKED(domain); | DMAR_DOMAIN_ASSERT_LOCKED(domain); | ||||
KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); | KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); | ||||
KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); | KASSERT((size & DMAR_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); | ||||
Show All 17 Lines | if (common->highaddr >= domain->end) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
error = dmar_gas_uppermatch(&a, RB_ROOT(&domain->rb_root)); | error = dmar_gas_uppermatch(&a, RB_ROOT(&domain->rb_root)); | ||||
KASSERT(error == ENOMEM, | KASSERT(error == ENOMEM, | ||||
("error %d from dmar_gas_uppermatch", error)); | ("error %d from dmar_gas_uppermatch", error)); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
dmar_gas_alloc_region(struct dmar_domain *domain, struct dmar_map_entry *entry, | dmar_gas_alloc_region(struct dmar_domain *domain, struct iommu_map_entry *entry, | ||||
u_int flags) | u_int flags) | ||||
{ | { | ||||
struct dmar_map_entry *next, *prev; | struct iommu_map_entry *next, *prev; | ||||
bool found; | bool found; | ||||
DMAR_DOMAIN_ASSERT_LOCKED(domain); | DMAR_DOMAIN_ASSERT_LOCKED(domain); | ||||
if ((entry->start & DMAR_PAGE_MASK) != 0 || | if ((entry->start & DMAR_PAGE_MASK) != 0 || | ||||
(entry->end & DMAR_PAGE_MASK) != 0) | (entry->end & DMAR_PAGE_MASK) != 0) | ||||
return (EINVAL); | return (EINVAL); | ||||
if (entry->start >= entry->end) | if (entry->start >= entry->end) | ||||
Show All 11 Lines | dmar_gas_alloc_region(struct dmar_domain *domain, struct iommu_map_entry *entry, | ||||
* Adapt to broken BIOSes which specify overlapping RMRR | * Adapt to broken BIOSes which specify overlapping RMRR | ||||
* entries. | * entries. | ||||
* | * | ||||
* XXXKIB: this does not handle a case when prev or next | * XXXKIB: this does not handle a case when prev or next | ||||
* entries are completely covered by the current one, which | * entries are completely covered by the current one, which | ||||
* extends both ways. | * extends both ways. | ||||
*/ | */ | ||||
if (prev != NULL && prev->end > entry->start && | if (prev != NULL && prev->end > entry->start && | ||||
(prev->flags & DMAR_MAP_ENTRY_PLACE) == 0) { | (prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { | ||||
if ((flags & DMAR_GM_RMRR) == 0 || | if ((flags & IOMMU_MF_RMRR) == 0 || | ||||
(prev->flags & DMAR_MAP_ENTRY_RMRR) == 0) | (prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0) | ||||
return (EBUSY); | return (EBUSY); | ||||
entry->start = prev->end; | entry->start = prev->end; | ||||
} | } | ||||
if (next->start < entry->end && | if (next->start < entry->end && | ||||
(next->flags & DMAR_MAP_ENTRY_PLACE) == 0) { | (next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { | ||||
if ((flags & DMAR_GM_RMRR) == 0 || | if ((flags & IOMMU_MF_RMRR) == 0 || | ||||
(next->flags & DMAR_MAP_ENTRY_RMRR) == 0) | (next->flags & IOMMU_MAP_ENTRY_RMRR) == 0) | ||||
return (EBUSY); | return (EBUSY); | ||||
entry->end = next->start; | entry->end = next->start; | ||||
} | } | ||||
if (entry->end == entry->start) | if (entry->end == entry->start) | ||||
return (0); | return (0); | ||||
if (prev != NULL && prev->end > entry->start) { | if (prev != NULL && prev->end > entry->start) { | ||||
/* This assumes that prev is the placeholder entry. */ | /* This assumes that prev is the placeholder entry. */ | ||||
dmar_gas_rb_remove(domain, prev); | dmar_gas_rb_remove(domain, prev); | ||||
prev = NULL; | prev = NULL; | ||||
} | } | ||||
if (next->start < entry->end) { | if (next->start < entry->end) { | ||||
dmar_gas_rb_remove(domain, next); | dmar_gas_rb_remove(domain, next); | ||||
next = NULL; | next = NULL; | ||||
} | } | ||||
found = dmar_gas_rb_insert(domain, entry); | found = dmar_gas_rb_insert(domain, entry); | ||||
KASSERT(found, ("found RMRR dup %p start %jx end %jx", | KASSERT(found, ("found RMRR dup %p start %jx end %jx", | ||||
domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); | domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); | ||||
if ((flags & DMAR_GM_RMRR) != 0) | if ((flags & IOMMU_MF_RMRR) != 0) | ||||
entry->flags = DMAR_MAP_ENTRY_RMRR; | entry->flags = IOMMU_MAP_ENTRY_RMRR; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
struct dmar_map_entry *ip, *in; | struct iommu_map_entry *ip, *in; | ||||
ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); | ip = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); | in = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
KASSERT(prev == NULL || ip == prev, | KASSERT(prev == NULL || ip == prev, | ||||
("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", | ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", | ||||
entry, entry->start, entry->end, prev, | entry, entry->start, entry->end, prev, | ||||
prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, | prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, | ||||
ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); | ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); | ||||
KASSERT(next == NULL || in == next, | KASSERT(next == NULL || in == next, | ||||
("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", | ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", | ||||
entry, entry->start, entry->end, next, | entry, entry->start, entry->end, next, | ||||
next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, | next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, | ||||
in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); | in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); | ||||
#endif | #endif | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
dmar_gas_free_space(struct dmar_domain *domain, struct dmar_map_entry *entry) | dmar_gas_free_space(struct dmar_domain *domain, struct iommu_map_entry *entry) | ||||
{ | { | ||||
DMAR_DOMAIN_ASSERT_LOCKED(domain); | DMAR_DOMAIN_ASSERT_LOCKED(domain); | ||||
KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | | KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | | ||||
DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_MAP, | IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP, | ||||
("permanent entry %p %p", domain, entry)); | ("permanent entry %p %p", domain, entry)); | ||||
dmar_gas_rb_remove(domain, entry); | dmar_gas_rb_remove(domain, entry); | ||||
entry->flags &= ~DMAR_MAP_ENTRY_MAP; | entry->flags &= ~IOMMU_MAP_ENTRY_MAP; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (dmar_check_free) | if (dmar_check_free) | ||||
dmar_gas_check_free(domain); | dmar_gas_check_free(domain); | ||||
#endif | #endif | ||||
} | } | ||||
void | void | ||||
dmar_gas_free_region(struct dmar_domain *domain, struct dmar_map_entry *entry) | dmar_gas_free_region(struct dmar_domain *domain, struct iommu_map_entry *entry) | ||||
{ | { | ||||
struct dmar_map_entry *next, *prev; | struct iommu_map_entry *next, *prev; | ||||
DMAR_DOMAIN_ASSERT_LOCKED(domain); | DMAR_DOMAIN_ASSERT_LOCKED(domain); | ||||
KASSERT((entry->flags & (DMAR_MAP_ENTRY_PLACE | DMAR_MAP_ENTRY_RMRR | | KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | | ||||
DMAR_MAP_ENTRY_MAP)) == DMAR_MAP_ENTRY_RMRR, | IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR, | ||||
("non-RMRR entry %p %p", domain, entry)); | ("non-RMRR entry %p %p", domain, entry)); | ||||
prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); | prev = RB_PREV(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); | next = RB_NEXT(dmar_gas_entries_tree, &domain->rb_root, entry); | ||||
dmar_gas_rb_remove(domain, entry); | dmar_gas_rb_remove(domain, entry); | ||||
entry->flags &= ~DMAR_MAP_ENTRY_RMRR; | entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; | ||||
if (prev == NULL) | if (prev == NULL) | ||||
dmar_gas_rb_insert(domain, domain->first_place); | dmar_gas_rb_insert(domain, domain->first_place); | ||||
if (next == NULL) | if (next == NULL) | ||||
dmar_gas_rb_insert(domain, domain->last_place); | dmar_gas_rb_insert(domain, domain->last_place); | ||||
} | } | ||||
int | int | ||||
dmar_gas_map(struct dmar_domain *domain, | dmar_gas_map(struct iommu_domain *iodom, | ||||
const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, | const struct bus_dma_tag_common *common, dmar_gaddr_t size, int offset, | ||||
u_int eflags, u_int flags, vm_page_t *ma, struct dmar_map_entry **res) | u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) | ||||
{ | { | ||||
struct dmar_map_entry *entry; | struct dmar_domain *domain; | ||||
struct iommu_map_entry *entry; | |||||
int error; | int error; | ||||
KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_CANSPLIT)) == 0, | domain = (struct dmar_domain *)iodom; | ||||
KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_CANSPLIT)) == 0, | |||||
("invalid flags 0x%x", flags)); | ("invalid flags 0x%x", flags)); | ||||
entry = dmar_gas_alloc_entry(domain, (flags & DMAR_GM_CANWAIT) != 0 ? | entry = dmar_gas_alloc_entry(&domain->iodom, | ||||
DMAR_PGF_WAITOK : 0); | (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); | ||||
if (entry == NULL) | if (entry == NULL) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
DMAR_DOMAIN_LOCK(domain); | DMAR_DOMAIN_LOCK(domain); | ||||
error = dmar_gas_find_space(domain, common, size, offset, flags, | error = dmar_gas_find_space(domain, common, size, offset, flags, | ||||
entry); | entry); | ||||
if (error == ENOMEM) { | if (error == ENOMEM) { | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
dmar_gas_free_entry(domain, entry); | dmar_gas_free_entry(&domain->iodom, entry); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (dmar_check_free) | if (dmar_check_free) | ||||
dmar_gas_check_free(domain); | dmar_gas_check_free(domain); | ||||
#endif | #endif | ||||
KASSERT(error == 0, | KASSERT(error == 0, | ||||
("unexpected error %d from dmar_gas_find_entry", error)); | ("unexpected error %d from dmar_gas_find_entry", error)); | ||||
KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", | KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", | ||||
(uintmax_t)entry->end, (uintmax_t)domain->end)); | (uintmax_t)entry->end, (uintmax_t)domain->end)); | ||||
entry->flags |= eflags; | entry->flags |= eflags; | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
error = domain_map_buf(domain, entry->start, entry->end - entry->start, | error = domain_map_buf(domain, entry->start, entry->end - entry->start, | ||||
ma, | ma, | ||||
((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | | ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | | ((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | | ((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), | ((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), | ||||
(flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); | (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); | ||||
if (error == ENOMEM) { | if (error == ENOMEM) { | ||||
dmar_domain_unload_entry(entry, true); | dmar_domain_unload_entry(entry, true); | ||||
return (error); | return (error); | ||||
} | } | ||||
KASSERT(error == 0, | KASSERT(error == 0, | ||||
("unexpected error %d from domain_map_buf", error)); | ("unexpected error %d from domain_map_buf", error)); | ||||
*res = entry; | *res = entry; | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
dmar_gas_map_region(struct dmar_domain *domain, struct dmar_map_entry *entry, | dmar_gas_map_region(struct iommu_domain *iodom, struct iommu_map_entry *entry, | ||||
u_int eflags, u_int flags, vm_page_t *ma) | u_int eflags, u_int flags, vm_page_t *ma) | ||||
{ | { | ||||
struct dmar_domain *domain; | |||||
dmar_gaddr_t start; | dmar_gaddr_t start; | ||||
int error; | int error; | ||||
domain = (struct dmar_domain *)iodom; | |||||
KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, | KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, | ||||
entry, entry->flags)); | entry, entry->flags)); | ||||
KASSERT((flags & ~(DMAR_GM_CANWAIT | DMAR_GM_RMRR)) == 0, | KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0, | ||||
("invalid flags 0x%x", flags)); | ("invalid flags 0x%x", flags)); | ||||
start = entry->start; | start = entry->start; | ||||
DMAR_DOMAIN_LOCK(domain); | DMAR_DOMAIN_LOCK(domain); | ||||
error = dmar_gas_alloc_region(domain, entry, flags); | error = dmar_gas_alloc_region(domain, entry, flags); | ||||
if (error != 0) { | if (error != 0) { | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
return (error); | return (error); | ||||
} | } | ||||
entry->flags |= eflags; | entry->flags |= eflags; | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
if (entry->end == entry->start) | if (entry->end == entry->start) | ||||
return (0); | return (0); | ||||
error = domain_map_buf(domain, entry->start, entry->end - entry->start, | error = domain_map_buf(domain, entry->start, entry->end - entry->start, | ||||
ma + OFF_TO_IDX(start - entry->start), | ma + OFF_TO_IDX(start - entry->start), | ||||
((eflags & DMAR_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | | ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | | ((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | | ((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | | ||||
((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), | ((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0), | ||||
(flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); | (flags & IOMMU_MF_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0); | ||||
if (error == ENOMEM) { | if (error == ENOMEM) { | ||||
dmar_domain_unload_entry(entry, false); | dmar_domain_unload_entry(entry, false); | ||||
return (error); | return (error); | ||||
} | } | ||||
KASSERT(error == 0, | KASSERT(error == 0, | ||||
("unexpected error %d from domain_map_buf", error)); | ("unexpected error %d from domain_map_buf", error)); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, | dmar_gas_reserve_region(struct dmar_domain *domain, dmar_gaddr_t start, | ||||
dmar_gaddr_t end) | dmar_gaddr_t end) | ||||
{ | { | ||||
struct dmar_map_entry *entry; | struct iommu_map_entry *entry; | ||||
int error; | int error; | ||||
entry = dmar_gas_alloc_entry(domain, DMAR_PGF_WAITOK); | entry = dmar_gas_alloc_entry(&domain->iodom, DMAR_PGF_WAITOK); | ||||
entry->start = start; | entry->start = start; | ||||
entry->end = end; | entry->end = end; | ||||
DMAR_DOMAIN_LOCK(domain); | DMAR_DOMAIN_LOCK(domain); | ||||
error = dmar_gas_alloc_region(domain, entry, DMAR_GM_CANWAIT); | error = dmar_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT); | ||||
if (error == 0) | if (error == 0) | ||||
entry->flags |= DMAR_MAP_ENTRY_UNMAPPED; | entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; | ||||
DMAR_DOMAIN_UNLOCK(domain); | DMAR_DOMAIN_UNLOCK(domain); | ||||
if (error != 0) | if (error != 0) | ||||
dmar_gas_free_entry(domain, entry); | dmar_gas_free_entry(&domain->iodom, entry); | ||||
return (error); | return (error); | ||||
} | } |