Page MenuHomeFreeBSD
Differential D35995 Diff 108740 sys/dev/iommu/iommu_gas.c

Changeset View

Standalone View

View Options

sys/dev/iommu/iommu_gas.c

Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines
if (res != NULL && domain != NULL) { if (res != NULL && domain != NULL) {
res->domain = domain; res->domain = domain;
atomic_add_int(&domain->entries_cnt, 1); atomic_add_int(&domain->entries_cnt, 1);
} }
return (res); return (res);
} }
void void
iommu_gas_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) iommu_gas_free_entry(struct iommu_map_entry *entry)
{ {
struct iommu_domain *domain;
KASSERT(domain == entry->domain, domain = entry->domain;
("mismatched free domain %p entry %p entry->domain %p", domain,
entry, entry->domain));
if (domain != NULL) if (domain != NULL)
atomic_subtract_int(&domain->entries_cnt, 1); atomic_subtract_int(&domain->entries_cnt, 1);
uma_zfree(iommu_map_entry_zone, entry); uma_zfree(iommu_map_entry_zone, entry);
} }
static int static int
iommu_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b) iommu_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b)
{ {
▲ Show 20 Lines • Show All 132 Lines • ▼ Show 20 Lines
entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root); entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root);
KASSERT(entry->start == 0, ("start entry start %p", domain)); KASSERT(entry->start == 0, ("start entry start %p", domain));
KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain)); KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain));
KASSERT(entry->flags == KASSERT(entry->flags ==
(IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED),
("start entry flags %p", domain)); ("start entry flags %p", domain));
RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry);
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root); entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root);
KASSERT(entry->start == domain->end, ("end entry start %p", domain)); KASSERT(entry->start == domain->end, ("end entry start %p", domain));
KASSERT(entry->end == domain->end, ("end entry end %p", domain)); KASSERT(entry->end == domain->end, ("end entry end %p", domain));
KASSERT(entry->flags == KASSERT(entry->flags ==
(IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED),
("end entry flags %p", domain)); ("end entry flags %p", domain));
RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry);
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
RB_FOREACH_SAFE(entry, iommu_gas_entries_tree, &domain->rb_root, RB_FOREACH_SAFE(entry, iommu_gas_entries_tree, &domain->rb_root,
entry1) { entry1) {
KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0, KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0,
("non-RMRR entry left %p", domain)); ("non-RMRR entry left %p", domain));
RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root,
entry); entry);
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
} }
} }
struct iommu_gas_match_args { struct iommu_gas_match_args {
struct iommu_domain *domain; struct iommu_domain *domain;
iommu_gaddr_t size; iommu_gaddr_t size;
int offset; int offset;
const struct bus_dma_tag_common *common; const struct bus_dma_tag_common *common;
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines
next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, next == NULL ? 0 : next->start, next == NULL ? 0 : next->end,
in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end));
#endif #endif
return (0); return (0);
} }
void void
iommu_gas_free_space(struct iommu_domain *domain, struct iommu_map_entry *entry) iommu_gas_free_space(struct iommu_map_entry *entry)
{ {
struct iommu_domain *domain;
IOMMU_DOMAIN_ASSERT_LOCKED(domain); domain = entry->domain;
KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR |
IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP, IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP,
("permanent entry %p %p", domain, entry)); ("permanent entry %p %p", domain, entry));
IOMMU_DOMAIN_LOCK(domain);
iommu_gas_rb_remove(domain, entry); iommu_gas_rb_remove(domain, entry);
entry->flags &= ~IOMMU_MAP_ENTRY_MAP; entry->flags &= ~IOMMU_MAP_ENTRY_MAP;
#ifdef INVARIANTS #ifdef INVARIANTS
if (iommu_check_free) if (iommu_check_free)
iommu_gas_check_free(domain); iommu_gas_check_free(domain);
#endif #endif
IOMMU_DOMAIN_UNLOCK(domain);
} }
void void
iommu_gas_free_region(struct iommu_domain *domain, struct iommu_map_entry *entry) iommu_gas_free_region(struct iommu_map_entry *entry)
{ {
struct iommu_domain *domain;
struct iommu_map_entry *next, *prev; struct iommu_map_entry *next, *prev;
IOMMU_DOMAIN_ASSERT_LOCKED(domain); domain = entry->domain;
KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR |
IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR, IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR,
("non-RMRR entry %p %p", domain, entry)); ("non-RMRR entry %p %p", domain, entry));
IOMMU_DOMAIN_LOCK(domain);
prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry);
next = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); next = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry);
iommu_gas_rb_remove(domain, entry); iommu_gas_rb_remove(domain, entry);
entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; entry->flags &= ~IOMMU_MAP_ENTRY_RMRR;
if (prev == NULL) if (prev == NULL)
iommu_gas_rb_insert(domain, domain->first_place); iommu_gas_rb_insert(domain, domain->first_place);
if (next == NULL) if (next == NULL)
iommu_gas_rb_insert(domain, domain->last_place); iommu_gas_rb_insert(domain, domain->last_place);
IOMMU_DOMAIN_UNLOCK(domain);
} }
int int
iommu_gas_map(struct iommu_domain *domain, iommu_gas_map(struct iommu_domain *domain,
const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset,
u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res)
{ {
struct iommu_gas_match_args a; struct iommu_gas_match_args a;
Show All 12 Lines
(flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0); (flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0);
if (entry == NULL) if (entry == NULL)
return (ENOMEM); return (ENOMEM);
a.entry = entry; a.entry = entry;
IOMMU_DOMAIN_LOCK(domain); IOMMU_DOMAIN_LOCK(domain);
error = iommu_gas_find_space(&a); error = iommu_gas_find_space(&a);
if (error == ENOMEM) { if (error == ENOMEM) {
IOMMU_DOMAIN_UNLOCK(domain); IOMMU_DOMAIN_UNLOCK(domain);
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
return (error); return (error);
} }
#ifdef INVARIANTS #ifdef INVARIANTS
if (iommu_check_free) if (iommu_check_free)
iommu_gas_check_free(domain); iommu_gas_check_free(domain);
#endif #endif
KASSERT(error == 0, KASSERT(error == 0,
("unexpected error %d from iommu_gas_find_entry", error)); ("unexpected error %d from iommu_gas_find_entry", error));
Show All 19 Lines
int int
iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry,
u_int eflags, u_int flags, vm_page_t *ma) u_int eflags, u_int flags, vm_page_t *ma)
{ {
iommu_gaddr_t start; iommu_gaddr_t start;
int error; int error;
KASSERT(entry->domain == domain,
("mismatched domain %p entry %p entry->domain %p", domain,
entry, entry->domain));
KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain,
entry, entry->flags)); entry, entry->flags));
KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0, KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0,
("invalid flags 0x%x", flags)); ("invalid flags 0x%x", flags));
start = entry->start; start = entry->start;
IOMMU_DOMAIN_LOCK(domain); IOMMU_DOMAIN_LOCK(domain);
error = iommu_gas_alloc_region(domain, entry, flags); error = iommu_gas_alloc_region(domain, entry, flags);
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines
struct iommu_map_entry *entry; struct iommu_map_entry *entry;
int error; int error;
entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK);
IOMMU_DOMAIN_LOCK(domain); IOMMU_DOMAIN_LOCK(domain);
error = iommu_gas_reserve_region_locked(domain, start, end, entry); error = iommu_gas_reserve_region_locked(domain, start, end, entry);
IOMMU_DOMAIN_UNLOCK(domain); IOMMU_DOMAIN_UNLOCK(domain);
if (error != 0) if (error != 0)
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
else if (entry0 != NULL) else if (entry0 != NULL)
*entry0 = entry; *entry0 = entry;
return (error); return (error);
} }
/* /*
* As in iommu_gas_reserve_region, reserve [start, end), but allow for existing * As in iommu_gas_reserve_region, reserve [start, end), but allow for existing
* entries. * entries.
Show All 37 Lines
break; break;
} }
entry = NULL; entry = NULL;
} }
IOMMU_DOMAIN_UNLOCK(domain); IOMMU_DOMAIN_UNLOCK(domain);
} }
/* Release a preallocated entry if it was not used. */ /* Release a preallocated entry if it was not used. */
if (entry != NULL) if (entry != NULL)
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
return (error); return (error);
} }
void void
iommu_unmap_msi(struct iommu_ctx *ctx) iommu_unmap_msi(struct iommu_ctx *ctx)
{ {
struct iommu_map_entry *entry; struct iommu_map_entry *entry;
struct iommu_domain *domain; struct iommu_domain *domain;
domain = ctx->domain; domain = ctx->domain;
entry = domain->msi_entry; entry = domain->msi_entry;
if (entry == NULL) if (entry == NULL)
return; return;
domain->ops->unmap(domain, entry->start, entry->end - domain->ops->unmap(domain, entry->start, entry->end -
entry->start, IOMMU_PGF_WAITOK); entry->start, IOMMU_PGF_WAITOK);
IOMMU_DOMAIN_LOCK(domain); iommu_gas_free_space(entry);
iommu_gas_free_space(domain, entry);
IOMMU_DOMAIN_UNLOCK(domain);
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
domain->msi_entry = NULL; domain->msi_entry = NULL;
domain->msi_base = 0; domain->msi_base = 0;
domain->msi_phys = 0; domain->msi_phys = 0;
} }
int int
iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset,
Show All 23 Lines
domain->msi_entry = entry; domain->msi_entry = entry;
domain->msi_base = entry->start; domain->msi_base = entry->start;
domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]); domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]);
} else { } else {
/* /*
* We lost the race and already have an * We lost the race and already have an
* MSI page allocated. Free the unneeded entry. * MSI page allocated. Free the unneeded entry.
*/ */
iommu_gas_free_entry(domain, entry); iommu_gas_free_entry(entry);
} }
} else if (domain->msi_entry != NULL) { } else if (domain->msi_entry != NULL) {
/* /*
* The allocation failed, but another succeeded. * The allocation failed, but another succeeded.
* Return success as there is a valid MSI page. * Return success as there is a valid MSI page.
*/ */
error = 0; error = 0;
} }
Show All 28 Lines