Page MenuHomeFreeBSD
Differential D24993 Diff 72280 sys/powerpc/booke/pmap_64.c

Changeset View

Standalone View

View Options

sys/powerpc/booke/pmap_64.c

Show First 20 Lines • Show All 93 Lines • ▼ Show 20 Lines
#include <machine/spr.h> #include <machine/spr.h>
#include <machine/md_var.h> #include <machine/md_var.h>
#include <machine/mmuvar.h> #include <machine/mmuvar.h>
#include <machine/pmap.h> #include <machine/pmap.h>
#include <machine/pte.h> #include <machine/pte.h>
#include <ddb/ddb.h> #include <ddb/ddb.h>
#include "mmu_if.h"
#ifdef DEBUG #ifdef DEBUG
#define debugf(fmt, args...) printf(fmt, ##args) #define debugf(fmt, args...) printf(fmt, ##args)
#else #else
#define debugf(fmt, args...) #define debugf(fmt, args...)
#endif #endif
#define PRI0ptrX "016lx" #define PRI0ptrX "016lx"
Show All 19 Lines
static void tid_flush(tlbtid_t tid); static void tid_flush(tlbtid_t tid);
static unsigned long ilog2(unsigned long); static unsigned long ilog2(unsigned long);
/**************************************************************************/ /**************************************************************************/
/* Page table management */ /* Page table management */
/**************************************************************************/ /**************************************************************************/
#define PMAP_ROOT_SIZE (sizeof(pte_t****) * PG_ROOT_NENTRIES) #define PMAP_ROOT_SIZE (sizeof(pte_t****) * PG_ROOT_NENTRIES)
static pte_t *ptbl_alloc(mmu_t mmu, pmap_t pmap, vm_offset_t va, static pte_t *ptbl_alloc(pmap_t pmap, vm_offset_t va,
bool nosleep, bool *is_new); bool nosleep, bool *is_new);
static void ptbl_hold(mmu_t, pmap_t, pte_t *); static void ptbl_hold(pmap_t, pte_t *);
static int ptbl_unhold(mmu_t, pmap_t, vm_offset_t); static int ptbl_unhold(pmap_t, vm_offset_t);
static vm_paddr_t pte_vatopa(mmu_t, pmap_t, vm_offset_t); static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
static int pte_enter(mmu_t, pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t); static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
static int pte_remove(mmu_t, pmap_t, vm_offset_t, uint8_t); static int pte_remove(pmap_t, vm_offset_t, uint8_t);
static pte_t *pte_find(mmu_t, pmap_t, vm_offset_t); static pte_t *pte_find(pmap_t, vm_offset_t);
static pte_t *pte_find_next(mmu_t, pmap_t, vm_offset_t *); static pte_t *pte_find_next(pmap_t, vm_offset_t *);
static void kernel_pte_alloc(vm_offset_t, vm_offset_t); static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
/**************************************************************************/ /**************************************************************************/
/* Page table related */ /* Page table related */
/**************************************************************************/ /**************************************************************************/
/* Allocate a page, to be used in a page table. */ /* Allocate a page, to be used in a page table. */
static vm_offset_t static vm_offset_t
mmu_booke_alloc_page(mmu_t mmu, pmap_t pmap, unsigned int idx, bool nosleep) mmu_booke_alloc_page(pmap_t pmap, unsigned int idx, bool nosleep)
{ {
vm_page_t m; vm_page_t m;
int req; int req;
req = VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO; req = VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO;
while ((m = vm_page_alloc(NULL, idx, req)) == NULL) { while ((m = vm_page_alloc(NULL, idx, req)) == NULL) {
if (nosleep) if (nosleep)
return (0); return (0);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
rw_wunlock(&pvh_global_lock); rw_wunlock(&pvh_global_lock);
vm_wait(NULL); vm_wait(NULL);
rw_wlock(&pvh_global_lock); rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
} }
if (!(m->flags & PG_ZERO)) if (!(m->flags & PG_ZERO))
/* Zero whole ptbl. */ /* Zero whole ptbl. */
mmu_booke_zero_page(mmu, m); mmu_booke_zero_page(m);
return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
} }
/* Initialize pool of kva ptbl buffers. */ /* Initialize pool of kva ptbl buffers. */
static void static void
ptbl_init(void) ptbl_init(void)
{ {
} }
/* Get a pointer to a PTE in a page table. */ /* Get a pointer to a PTE in a page table. */
static __inline pte_t * static __inline pte_t *
pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va) pte_find(pmap_t pmap, vm_offset_t va)
{ {
pte_t ***pdir_l1; pte_t ***pdir_l1;
pte_t **pdir; pte_t **pdir;
pte_t *ptbl; pte_t *ptbl;
KASSERT((pmap != NULL), ("pte_find: invalid pmap")); KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)]; pdir_l1 = pmap->pm_root[PG_ROOT_IDX(va)];
if (pdir_l1 == NULL) if (pdir_l1 == NULL)
return (NULL); return (NULL);
pdir = pdir_l1[PDIR_L1_IDX(va)]; pdir = pdir_l1[PDIR_L1_IDX(va)];
if (pdir == NULL) if (pdir == NULL)
return (NULL); return (NULL);
ptbl = pdir[PDIR_IDX(va)]; ptbl = pdir[PDIR_IDX(va)];
return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL); return ((ptbl != NULL) ? &ptbl[PTBL_IDX(va)] : NULL);
} }
/* Get a pointer to a PTE in a page table, or the next closest (greater) one. */ /* Get a pointer to a PTE in a page table, or the next closest (greater) one. */
static __inline pte_t * static __inline pte_t *
pte_find_next(mmu_t mmu, pmap_t pmap, vm_offset_t *pva) pte_find_next(pmap_t pmap, vm_offset_t *pva)
{ {
vm_offset_t va; vm_offset_t va;
pte_t ****pm_root; pte_t ****pm_root;
pte_t *pte; pte_t *pte;
unsigned long i, j, k, l; unsigned long i, j, k, l;
KASSERT((pmap != NULL), ("pte_find: invalid pmap")); KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
Show All 26 Lines for (; j < PDIR_L1_NENTRIES; j++, k = 0) {
} }
} }
} }
} }
return (NULL); return (NULL);
} }
static bool static bool
unhold_free_page(mmu_t mmu, pmap_t pmap, vm_page_t m) unhold_free_page(pmap_t pmap, vm_page_t m)
{ {
m->ref_count--; m->ref_count--;
if (m->ref_count == 0) { if (m->ref_count == 0) {
vm_wire_sub(1); vm_wire_sub(1);
vm_page_free_zero(m); vm_page_free_zero(m);
return (true); return (true);
} }
return (false); return (false);
} }
static vm_offset_t static vm_offset_t
alloc_or_hold_page(mmu_t mmu, pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index, alloc_or_hold_page(pmap_t pmap, vm_offset_t *ptr_tbl, uint32_t index,
bool nosleep, bool hold, bool *isnew) bool nosleep, bool hold, bool *isnew)
{ {
vm_offset_t page; vm_offset_t page;
vm_page_t m; vm_page_t m;
page = ptr_tbl[index]; page = ptr_tbl[index];
KASSERT(page != 0 || pmap != kernel_pmap, KASSERT(page != 0 || pmap != kernel_pmap,
("NULL page table page found in kernel pmap!")); ("NULL page table page found in kernel pmap!"));
if (page == 0) { if (page == 0) {
page = mmu_booke_alloc_page(mmu, pmap, index, nosleep); page = mmu_booke_alloc_page(pmap, index, nosleep);
if (ptr_tbl[index] == 0) { if (ptr_tbl[index] == 0) {
*isnew = true; *isnew = true;
ptr_tbl[index] = page; ptr_tbl[index] = page;
return (page); return (page);
} }
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS(page)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS(page));
page = ptr_tbl[index]; page = ptr_tbl[index];
vm_wire_sub(1); vm_wire_sub(1);
vm_page_free_zero(m); vm_page_free_zero(m);
} }
if (hold) { if (hold) {
m = PHYS_TO_VM_PAGE(pmap_kextract(page)); m = PHYS_TO_VM_PAGE(pmap_kextract(page));
m->ref_count++; m->ref_count++;
} }
*isnew = false; *isnew = false;
return (page); return (page);
} }
/* Allocate page table. */ /* Allocate page table. */
static pte_t* static pte_t*
ptbl_alloc(mmu_t mmu, pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new) ptbl_alloc(pmap_t pmap, vm_offset_t va, bool nosleep, bool *is_new)
{ {
unsigned int pg_root_idx = PG_ROOT_IDX(va); unsigned int pg_root_idx = PG_ROOT_IDX(va);
unsigned int pdir_l1_idx = PDIR_L1_IDX(va); unsigned int pdir_l1_idx = PDIR_L1_IDX(va);
unsigned int pdir_idx = PDIR_IDX(va); unsigned int pdir_idx = PDIR_IDX(va);
vm_offset_t pdir_l1, pdir, ptbl; vm_offset_t pdir_l1, pdir, ptbl;
bool hold_page; bool hold_page;
hold_page = (pmap != kernel_pmap); hold_page = (pmap != kernel_pmap);
pdir_l1 = alloc_or_hold_page(mmu, pmap, (vm_offset_t *)pmap->pm_root, pdir_l1 = alloc_or_hold_page(pmap, (vm_offset_t *)pmap->pm_root,
pg_root_idx, nosleep, hold_page, is_new); pg_root_idx, nosleep, hold_page, is_new);
if (pdir_l1 == 0) if (pdir_l1 == 0)
return (NULL); return (NULL);
pdir = alloc_or_hold_page(mmu, pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx, pdir = alloc_or_hold_page(pmap, (vm_offset_t *)pdir_l1, pdir_l1_idx,
nosleep, hold_page, is_new); nosleep, hold_page, is_new);
if (pdir == 0) if (pdir == 0)
return (NULL); return (NULL);
ptbl = alloc_or_hold_page(mmu, pmap, (vm_offset_t *)pdir, pdir_idx, ptbl = alloc_or_hold_page(pmap, (vm_offset_t *)pdir, pdir_idx,
nosleep, false, is_new); nosleep, false, is_new);
return ((pte_t *)ptbl); return ((pte_t *)ptbl);
} }
/* /*
* Decrement ptbl pages hold count and attempt to free ptbl pages. Called * Decrement ptbl pages hold count and attempt to free ptbl pages. Called
* when removing pte entry from ptbl. * when removing pte entry from ptbl.
* *
* Return 1 if ptbl pages were freed. * Return 1 if ptbl pages were freed.
*/ */
static int static int
ptbl_unhold(mmu_t mmu, pmap_t pmap, vm_offset_t va) ptbl_unhold(pmap_t pmap, vm_offset_t va)
{ {
pte_t *ptbl; pte_t *ptbl;
vm_page_t m; vm_page_t m;
u_int pg_root_idx; u_int pg_root_idx;
pte_t ***pdir_l1; pte_t ***pdir_l1;
u_int pdir_l1_idx; u_int pdir_l1_idx;
pte_t **pdir; pte_t **pdir;
u_int pdir_idx; u_int pdir_idx;
pg_root_idx = PG_ROOT_IDX(va); pg_root_idx = PG_ROOT_IDX(va);
pdir_l1_idx = PDIR_L1_IDX(va); pdir_l1_idx = PDIR_L1_IDX(va);
pdir_idx = PDIR_IDX(va); pdir_idx = PDIR_IDX(va);
KASSERT((pmap != kernel_pmap), KASSERT((pmap != kernel_pmap),
("ptbl_unhold: unholding kernel ptbl!")); ("ptbl_unhold: unholding kernel ptbl!"));
pdir_l1 = pmap->pm_root[pg_root_idx]; pdir_l1 = pmap->pm_root[pg_root_idx];
pdir = pdir_l1[pdir_l1_idx]; pdir = pdir_l1[pdir_l1_idx];
ptbl = pdir[pdir_idx]; ptbl = pdir[pdir_idx];
/* decrement hold count */ /* decrement hold count */
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
if (!unhold_free_page(mmu, pmap, m)) if (!unhold_free_page(pmap, m))
return (0); return (0);
pdir[pdir_idx] = NULL; pdir[pdir_idx] = NULL;
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir));
if (!unhold_free_page(mmu, pmap, m)) if (!unhold_free_page(pmap, m))
return (1); return (1);
pdir_l1[pdir_l1_idx] = NULL; pdir_l1[pdir_l1_idx] = NULL;
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir_l1)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) pdir_l1));
if (!unhold_free_page(mmu, pmap, m)) if (!unhold_free_page(pmap, m))
return (1); return (1);
pmap->pm_root[pg_root_idx] = NULL; pmap->pm_root[pg_root_idx] = NULL;
return (1); return (1);
} }
/* /*
* Increment hold count for ptbl pages. This routine is used when new pte * Increment hold count for ptbl pages. This routine is used when new pte
* entry is being inserted into ptbl. * entry is being inserted into ptbl.
*/ */
static void static void
ptbl_hold(mmu_t mmu, pmap_t pmap, pte_t *ptbl) ptbl_hold(pmap_t pmap, pte_t *ptbl)
{ {
vm_page_t m; vm_page_t m;
KASSERT((pmap != kernel_pmap), KASSERT((pmap != kernel_pmap),
("ptbl_hold: holding kernel ptbl!")); ("ptbl_hold: holding kernel ptbl!"));
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t) ptbl));
m->ref_count++; m->ref_count++;
} }
/* /*
* Clean pte entry, try to free page table page if requested. * Clean pte entry, try to free page table page if requested.
* *
* Return 1 if ptbl pages were freed, otherwise return 0. * Return 1 if ptbl pages were freed, otherwise return 0.
*/ */
static int static int
pte_remove(mmu_t mmu, pmap_t pmap, vm_offset_t va, u_int8_t flags) pte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)
{ {
vm_page_t m; vm_page_t m;
pte_t *pte; pte_t *pte;
pte = pte_find(mmu, pmap, va); pte = pte_find(pmap, va);
KASSERT(pte != NULL, ("%s: NULL pte for va %#jx, pmap %p", KASSERT(pte != NULL, ("%s: NULL pte for va %#jx, pmap %p",
__func__, (uintmax_t)va, pmap)); __func__, (uintmax_t)va, pmap));
if (!PTE_ISVALID(pte)) if (!PTE_ISVALID(pte))
return (0); return (0);
/* Get vm_page_t for mapped pte. */ /* Get vm_page_t for mapped pte. */
m = PHYS_TO_VM_PAGE(PTE_PA(pte)); m = PHYS_TO_VM_PAGE(PTE_PA(pte));
Show All 26 Linespte_remove(pmap_t pmap, vm_offset_t va, u_int8_t flags)
*pte = 0; *pte = 0;
tlb_miss_unlock(); tlb_miss_unlock();
mtx_unlock_spin(&tlbivax_mutex); mtx_unlock_spin(&tlbivax_mutex);
pmap->pm_stats.resident_count--; pmap->pm_stats.resident_count--;
if (flags & PTBL_UNHOLD) { if (flags & PTBL_UNHOLD) {
return (ptbl_unhold(mmu, pmap, va)); return (ptbl_unhold(pmap, va));
} }
return (0); return (0);
} }
/* /*
* Insert PTE for a given page and virtual address. * Insert PTE for a given page and virtual address.
*/ */
static int static int
pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags, pte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
boolean_t nosleep) boolean_t nosleep)
{ {
unsigned int ptbl_idx = PTBL_IDX(va); unsigned int ptbl_idx = PTBL_IDX(va);
pte_t *ptbl, *pte, pte_tmp; pte_t *ptbl, *pte, pte_tmp;
bool is_new; bool is_new;
/* Get the page directory pointer. */ /* Get the page directory pointer. */
ptbl = ptbl_alloc(mmu, pmap, va, nosleep, &is_new); ptbl = ptbl_alloc(pmap, va, nosleep, &is_new);
if (ptbl == NULL) { if (ptbl == NULL) {
KASSERT(nosleep, ("nosleep and NULL ptbl")); KASSERT(nosleep, ("nosleep and NULL ptbl"));
return (ENOMEM); return (ENOMEM);
} }
if (is_new) { if (is_new) {
pte = &ptbl[ptbl_idx]; pte = &ptbl[ptbl_idx];
} else { } else {
/* /*
* Check if there is valid mapping for requested va, if there * Check if there is valid mapping for requested va, if there
* is, remove it. * is, remove it.
*/ */
pte = &ptbl[ptbl_idx]; pte = &ptbl[ptbl_idx];
if (PTE_ISVALID(pte)) { if (PTE_ISVALID(pte)) {
pte_remove(mmu, pmap, va, PTBL_HOLD); pte_remove(pmap, va, PTBL_HOLD);
} else { } else {
/* /*
* pte is not used, increment hold count for ptbl * pte is not used, increment hold count for ptbl
* pages. * pages.
*/ */
if (pmap != kernel_pmap) if (pmap != kernel_pmap)
ptbl_hold(mmu, pmap, ptbl); ptbl_hold(pmap, ptbl);
} }
} }
/* /*
* Insert pv_entry into pv_list for mapped page if part of managed * Insert pv_entry into pv_list for mapped page if part of managed
* memory. * memory.
*/ */
if ((m->oflags & VPO_UNMANAGED) == 0) { if ((m->oflags & VPO_UNMANAGED) == 0) {
Show All 17 Linespte_enter(pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
tlb_miss_unlock(); tlb_miss_unlock();
mtx_unlock_spin(&tlbivax_mutex); mtx_unlock_spin(&tlbivax_mutex);
return (0); return (0);
} }
/* Return the pa for the given pmap/va. */ /* Return the pa for the given pmap/va. */
static vm_paddr_t static vm_paddr_t
pte_vatopa(mmu_t mmu, pmap_t pmap, vm_offset_t va) pte_vatopa(pmap_t pmap, vm_offset_t va)
{ {
vm_paddr_t pa = 0; vm_paddr_t pa = 0;
pte_t *pte; pte_t *pte;
pte = pte_find(mmu, pmap, va); pte = pte_find(pmap, va);
if ((pte != NULL) && PTE_ISVALID(pte)) if ((pte != NULL) && PTE_ISVALID(pte))
pa = (PTE_PA(pte) | (va & PTE_PA_MASK)); pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
return (pa); return (pa);
} }
/* allocate pte entries to manage (addr & mask) to (addr & mask) + size */ /* allocate pte entries to manage (addr & mask) to (addr & mask) + size */
static void static void
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Linesmmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
return (data_end); return (data_end);
} }
/* /*
* Initialize a preallocated and zeroed pmap structure, * Initialize a preallocated and zeroed pmap structure,
* such as one in a vmspace structure. * such as one in a vmspace structure.
*/ */
static void static int
mmu_booke_pinit(mmu_t mmu, pmap_t pmap) mmu_booke_pinit(pmap_t pmap)
{ {
int i; int i;
CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap, CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
curthread->td_proc->p_pid, curthread->td_proc->p_comm); curthread->td_proc->p_pid, curthread->td_proc->p_comm);
KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap")); KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
for (i = 0; i < MAXCPU; i++) for (i = 0; i < MAXCPU; i++)
pmap->pm_tid[i] = TID_NONE; pmap->pm_tid[i] = TID_NONE;
CPU_ZERO(&kernel_pmap->pm_active); CPU_ZERO(&kernel_pmap->pm_active);
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK); pmap->pm_root = uma_zalloc(ptbl_root_zone, M_WAITOK);
bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES); bzero(pmap->pm_root, sizeof(pte_t **) * PG_ROOT_NENTRIES);
return (1);
} }
/* /*
* Release any resources held by the given physical map. * Release any resources held by the given physical map.
* Called when a pmap initialized by mmu_booke_pinit is being released. * Called when a pmap initialized by mmu_booke_pinit is being released.
* Should only be called if the map contains no valid mappings. * Should only be called if the map contains no valid mappings.
*/ */
static void static void
mmu_booke_release(mmu_t mmu, pmap_t pmap) mmu_booke_release(pmap_t pmap)
{ {
KASSERT(pmap->pm_stats.resident_count == 0, KASSERT(pmap->pm_stats.resident_count == 0,
("pmap_release: pmap resident count %ld != 0", ("pmap_release: pmap resident count %ld != 0",
pmap->pm_stats.resident_count)); pmap->pm_stats.resident_count));
uma_zfree(ptbl_root_zone, pmap->pm_root); uma_zfree(ptbl_root_zone, pmap->pm_root);
} }
static void static void
mmu_booke_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) mmu_booke_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
{ {
pte_t *pte; pte_t *pte;
vm_paddr_t pa = 0; vm_paddr_t pa = 0;
int sync_sz, valid; int sync_sz, valid;
while (sz > 0) { while (sz > 0) {
PMAP_LOCK(pm); PMAP_LOCK(pm);
pte = pte_find(mmu, pm, va); pte = pte_find(pm, va);
valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0; valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
if (valid) if (valid)
pa = PTE_PA(pte); pa = PTE_PA(pte);
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
sync_sz = PAGE_SIZE - (va & PAGE_MASK); sync_sz = PAGE_SIZE - (va & PAGE_MASK);
sync_sz = min(sync_sz, sz); sync_sz = min(sync_sz, sz);
if (valid) { if (valid) {
pa += (va & PAGE_MASK); pa += (va & PAGE_MASK);
__syncicache((void *)PHYS_TO_DMAP(pa), sync_sz); __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz);
} }
va += sync_sz; va += sync_sz;
sz -= sync_sz; sz -= sync_sz;
} }
} }
/* /*
* mmu_booke_zero_page_area zeros the specified hardware page by * mmu_booke_zero_page_area zeros the specified hardware page by
* mapping it into virtual memory and using bzero to clear * mapping it into virtual memory and using bzero to clear
* its contents. * its contents.
* *
* off and size must reside within a single page. * off and size must reside within a single page.
*/ */
static void static void
mmu_booke_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) mmu_booke_zero_page_area(vm_page_t m, int off, int size)
{ {
vm_offset_t va; vm_offset_t va;
/* XXX KASSERT off and size are within a single page? */ /* XXX KASSERT off and size are within a single page? */
va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
bzero((caddr_t)va + off, size); bzero((caddr_t)va + off, size);
} }
/* /*
* mmu_booke_zero_page zeros the specified hardware page. * mmu_booke_zero_page zeros the specified hardware page.
*/ */
static void static void
mmu_booke_zero_page(mmu_t mmu, vm_page_t m) mmu_booke_zero_page(vm_page_t m)
{ {
vm_offset_t off, va; vm_offset_t off, va;
va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
for (off = 0; off < PAGE_SIZE; off += cacheline_size) for (off = 0; off < PAGE_SIZE; off += cacheline_size)
__asm __volatile("dcbz 0,%0" :: "r"(va + off)); __asm __volatile("dcbz 0,%0" :: "r"(va + off));
} }
/* /*
* mmu_booke_copy_page copies the specified (machine independent) page by * mmu_booke_copy_page copies the specified (machine independent) page by
* mapping the page into virtual memory and using memcopy to copy the page, * mapping the page into virtual memory and using memcopy to copy the page,
* one machine dependent page at a time. * one machine dependent page at a time.
*/ */
static void static void
mmu_booke_copy_page(mmu_t mmu, vm_page_t sm, vm_page_t dm) mmu_booke_copy_page(vm_page_t sm, vm_page_t dm)
{ {
vm_offset_t sva, dva; vm_offset_t sva, dva;
sva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(sm)); sva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(sm));
dva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dm)); dva = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dm));
memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE); memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
} }
static inline void static inline void
mmu_booke_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset, mmu_booke_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize) vm_page_t *mb, vm_offset_t b_offset, int xfersize)
{ {
void *a_cp, *b_cp; void *a_cp, *b_cp;
vm_offset_t a_pg_offset, b_pg_offset; vm_offset_t a_pg_offset, b_pg_offset;
int cnt; int cnt;
vm_page_t pa, pb; vm_page_t pa, pb;
Show All 11 Lines while (xfersize > 0) {
bcopy(a_cp, b_cp, cnt); bcopy(a_cp, b_cp, cnt);
a_offset += cnt; a_offset += cnt;
b_offset += cnt; b_offset += cnt;
xfersize -= cnt; xfersize -= cnt;
} }
} }
static vm_offset_t static vm_offset_t
mmu_booke_quick_enter_page(mmu_t mmu, vm_page_t m) mmu_booke_quick_enter_page(vm_page_t m)
{ {
return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m))); return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
} }
static void static void
mmu_booke_quick_remove_page(mmu_t mmu, vm_offset_t addr) mmu_booke_quick_remove_page(vm_offset_t addr)
{ {
} }
/**************************************************************************/ /**************************************************************************/
/* TID handling */ /* TID handling */
/**************************************************************************/ /**************************************************************************/
/* /*
Show All 37 Lines