diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -947,6 +947,23 @@ return (queue == PQ_LAUNDRY || queue == PQ_UNSWAPPABLE); } +static inline void +vm_page_ref(vm_page_t m, int val) +{ + atomic_add_int(&m->ref_count, val); +} + +static inline void +vm_page_clearref(vm_page_t m) +{ + u_int r; + + r = m->ref_count; + while (atomic_fcmpset_int(&m->ref_count, &r, r & (VPRC_BLOCKED | + VPRC_OBJREF)) == 0) + ; +} + /* * vm_page_drop: * diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c --- a/sys/x86/iommu/intel_idpgtbl.c +++ b/sys/x86/iommu/intel_idpgtbl.c @@ -390,7 +390,7 @@ * pte write and clean while the lock is * dropped. */ - m->ref_count++; + vm_page_ref(m, 1); sfp = NULL; ptep = domain_pgtbl_map_pte(domain, base, lvl - 1, @@ -398,7 +398,7 @@ if (ptep == NULL) { KASSERT(m->pindex != 0, ("loosing root page %p", domain)); - m->ref_count--; + vm_page_ref(m, -1); iommu_pgfree(domain->pgtbl_obj, m->pindex, flags); return (NULL); @@ -406,8 +406,8 @@ dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | VM_PAGE_TO_PHYS(m)); dmar_flush_pte_to_ram(domain->dmar, ptep); - sf_buf_page(sfp)->ref_count += 1; - m->ref_count--; + vm_page_ref(sf_buf_page(sfp), 1); + vm_page_ref(m, -1); iommu_unmap_pgtbl(sfp); /* Only executed once. */ goto retry; @@ -486,7 +486,7 @@ dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | (superpage ? DMAR_PTE_SP : 0)); dmar_flush_pte_to_ram(domain->dmar, pte); - sf_buf_page(sf)->ref_count += 1; + vm_page_ref(sf_buf_page(sf), 1); } if (sf != NULL) iommu_unmap_pgtbl(sf); @@ -592,8 +592,8 @@ iommu_unmap_pgtbl(*sf); *sf = NULL; } - m->ref_count--; - if (m->ref_count != 0) + vm_page_ref(m, -1); + if (VPRC_WIRE_COUNT(m->ref_count) != 0) return; KASSERT(lvl != 0, ("lost reference (lvl) on root pg domain %p base %jx lvl %d", @@ -709,7 +709,7 @@ m = iommu_pgalloc(domain->pgtbl_obj, 0, IOMMU_PGF_WAITOK | IOMMU_PGF_ZERO | IOMMU_PGF_OBJL); /* No implicit free of the top level page table page. */ - m->ref_count = 1; + vm_page_ref(m, 1); DMAR_DOMAIN_PGUNLOCK(domain); DMAR_LOCK(domain->dmar); domain->iodom.flags |= IOMMU_DOMAIN_PGTBL_INITED; @@ -742,7 +742,7 @@ /* Obliterate ref_counts */ VM_OBJECT_ASSERT_WLOCKED(obj); for (m = vm_page_lookup(obj, 0); m != NULL; m = vm_page_next(m)) - m->ref_count = 0; + vm_page_clearref(m); VM_OBJECT_WUNLOCK(obj); vm_object_deallocate(obj); }