Index: sys/mips/include/cpuregs.h =================================================================== --- sys/mips/include/cpuregs.h +++ sys/mips/include/cpuregs.h @@ -171,6 +171,10 @@ #define MIPS_CCA_CACHED MIPS_CCA_CCS #endif +#if defined(CPU_XBURST) +#define MIPS_CCA_UA 0x01 +#endif + #ifndef MIPS_CCA_UNCACHED #define MIPS_CCA_UNCACHED MIPS_CCA_UC #endif @@ -188,6 +192,18 @@ #endif #endif +/* + * Use uncached-accelerated mode for write-combining maps, if one is defined, + * otherwise fall back to uncached + */ +#ifndef MIPS_CCA_WC +#ifdef MIPS_CCA_UA +#define MIPS_CCA_WC MIPS_CCA_UA +#else +#define MIPS_CCA_WC MIPS_CCA_UNCACHED +#endif +#endif + #define MIPS_PHYS_TO_XKPHYS(cca,x) \ ((0x2ULL << 62) | ((unsigned long long)(cca) << 59) | (x)) #define MIPS_PHYS_TO_XKPHYS_CACHED(x) \ Index: sys/mips/include/pmap.h =================================================================== --- sys/mips/include/pmap.h +++ sys/mips/include/pmap.h @@ -74,7 +74,8 @@ }; #define PV_TABLE_REF 0x02 /* referenced */ -#define PV_MEMATTR_UNCACHEABLE 0x04 +#define PV_MEMATTR_MASK 0xf0 /* store vm_memattr_t here */ +#define PV_MEMATTR_SHIF 0x04 #define ASID_BITS 8 #define ASIDGEN_BITS (32 - ASID_BITS) @@ -163,22 +164,24 @@ extern vm_paddr_t dump_avail[PHYS_AVAIL_ENTRIES + 2]; -#define pmap_page_get_memattr(m) VM_MEMATTR_DEFAULT +#define pmap_page_get_memattr(m) (((m)->md.pv_flags & PV_MEMATTR_MASK) >> PV_MEMATTR_SHIFT) #define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list)) #define pmap_page_is_write_mapped(m) (((m)->aflags & PGA_WRITEABLE) != 0) void pmap_bootstrap(void); void *pmap_mapdev(vm_paddr_t, vm_size_t); +void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); void pmap_unmapdev(vm_offset_t, vm_size_t); vm_offset_t pmap_steal_memory(vm_size_t size); void pmap_kenter(vm_offset_t va, vm_paddr_t pa); -void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr); +void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t attr); void pmap_kremove(vm_offset_t va); void *pmap_kenter_temporary(vm_paddr_t pa, int i); void pmap_kenter_temporary_free(vm_paddr_t pa); void pmap_flush_pvcache(vm_page_t m); int pmap_emulate_modified(pmap_t pmap, vm_offset_t va); void pmap_page_set_memattr(vm_page_t, vm_memattr_t); +int pmap_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); #endif /* _KERNEL */ Index: sys/mips/include/pte.h =================================================================== --- sys/mips/include/pte.h +++ sys/mips/include/pte.h @@ -132,8 +132,10 @@ * it is matched. */ #define PTE_C(attr) ((attr & 0x07) << 3) +#define PTE_C_MASK (PTE_C(0x07)) #define PTE_C_UNCACHED (PTE_C(MIPS_CCA_UNCACHED)) #define PTE_C_CACHE (PTE_C(MIPS_CCA_CACHED)) +#define PTE_C_WC (PTE_C(MIPS_CCA_WC)) #define PTE_D 0x04 #define PTE_V 0x02 #define PTE_G 0x01 @@ -158,6 +160,7 @@ #define pte_clear(pte, bit) (*(pte) &= ~(bit)) #define pte_set(pte, bit) (*(pte) |= (bit)) #define pte_test(pte, bit) ((*(pte) & (bit)) == (bit)) +#define pte_cache_bits(pte) ((*(pte) >> 3) & 0x07) /* Assembly support for PTE access*/ #ifdef LOCORE Index: sys/mips/include/vm.h =================================================================== --- sys/mips/include/vm.h +++ sys/mips/include/vm.h @@ -32,7 +32,9 @@ #include /* Memory attributes. */ -#define VM_MEMATTR_UNCACHEABLE ((vm_memattr_t)PTE_C_UNCACHED) -#define VM_MEMATTR_DEFAULT ((vm_memattr_t)PTE_C_CACHE) +#define VM_MEMATTR_UNCACHEABLE ((vm_memattr_t)MIPS_CCA_UNCACHED) +#define VM_MEMATTR_WRITE_COMBINING ((vm_memattr_t)MIPS_CCA_WC) +#define VM_MEMATTR_WRITE_BACK ((vm_memattr_t)MIPS_CCA_CACHED) +#define VM_MEMATTR_DEFAULT VM_MEMATTR_WRITE_BACK #endif /* !_MACHINE_VM_H_ */ Index: sys/mips/mips/pmap.c =================================================================== --- sys/mips/mips/pmap.c +++ sys/mips/mips/pmap.c @@ -189,10 +189,10 @@ * The highmem area does not have a KSEG0 mapping, and we need a mechanism to * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc. * - * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To + * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To * access a highmem physical address on a CPU, we map the physical address to - * the reserved virtual address for the CPU in the kernel pagetable. This is - * done with interrupts disabled(although a spinlock and sched_pin would be + * the reserved virtual address for the CPU in the kernel pagetable. This is + * done with interrupts disabled(although a spinlock and sched_pin would be * sufficient). */ struct local_sysmaps { @@ -303,7 +303,7 @@ return (0); } -static __inline vm_offset_t +static __inline vm_offset_t pmap_lmem_unmap(void) { @@ -312,12 +312,18 @@ #endif /* !__mips_n64 */ static __inline int -is_cacheable_page(vm_paddr_t pa, vm_page_t m) +pmap_pte_cache_bits(vm_paddr_t pa, vm_page_t m) { + vm_memattr_t ma; - return ((m->md.pv_flags & PV_MEMATTR_UNCACHEABLE) == 0 && - is_cacheable_mem(pa)); - + ma = pmap_page_get_memattr(m); + if (ma == VM_MEMATTR_WRITE_BACK && !is_cacheable_mem(pa)) + ma = VM_MEMATTR_UNCACHEABLE; + return PTE_C(ma); +} +#define PMAP_PTE_SET_CACHE_BITS(pte, ps, m) { \ + pte &= ~PTE_C_MASK; \ + pte |= pmap_pte_cache_bits(pa, m); \ } /* @@ -359,7 +365,7 @@ return (pdpe); } -static __inline +static __inline pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va) { @@ -423,7 +429,7 @@ * Bootstrap the system enough to run with virtual memory. This * assumes that the phys_avail array has been initialized. */ -static void +static void pmap_create_kernel_pagetable(void) { int i, j; @@ -486,7 +492,7 @@ pmap_bootstrap(void) { int i; - int need_local_mappings = 0; + int need_local_mappings = 0; /* Sort. */ again: @@ -600,7 +606,7 @@ { TAILQ_INIT(&m->md.pv_list); - m->md.pv_flags = 0; + m->md.pv_flags = VM_MEMATTR_DEFAULT << PV_MEMATTR_SHIFT; } /* @@ -635,8 +641,8 @@ pmap->pm_asid[cpu].gen = 0; } cpuid = PCPU_GET(cpuid); - /* - * XXX: barrier/locking for active? + /* + * XXX: barrier/locking for active? * * Take a snapshot of active here, any further changes are ignored. * tlb update/invalidate should be harmless on inactive CPUs @@ -819,7 +825,7 @@ * add a wired page to the kva */ void -pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr) +pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) { pt_entry_t *pte; pt_entry_t opte, npte; @@ -830,7 +836,7 @@ pte = pmap_pte(kernel_pmap, va); opte = *pte; - npte = TLBLO_PA_TO_PFN(pa) | attr | PTE_D | PTE_V | PTE_G; + npte = TLBLO_PA_TO_PFN(pa) | PTE_C(ma) | PTE_D | PTE_V | PTE_G; *pte = npte; if (pte_test(&opte, PTE_V) && opte != npte) pmap_update_page(kernel_pmap, va, npte); @@ -843,7 +849,7 @@ KASSERT(is_cacheable_mem(pa), ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa)); - pmap_kenter_attr(va, pa, PTE_C_CACHE); + pmap_kenter_attr(va, pa, VM_MEMATTR_DEFAULT); } /* @@ -1144,11 +1150,11 @@ int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT); int pdeindex = ptepindex & (NPDEPG - 1); vm_page_t pg; - + pdep = &pmap->pm_segtab[segindex]; - if (*pdep == NULL) { + if (*pdep == NULL) { /* recurse for allocating page dir */ - if (_pmap_allocpte(pmap, NUPDE + segindex, + if (_pmap_allocpte(pmap, NUPDE + segindex, flags) == NULL) { /* alloc failed, release current */ --m->wire_count; @@ -1677,7 +1683,7 @@ * pmap_remove_pte: do the things to unmap a page in a process */ static int -pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va, +pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t pde) { pt_entry_t oldpte; @@ -1861,7 +1867,7 @@ PMAP_LOCK(pmap); /* - * If it's last mapping writeback all caches from + * If it's last mapping writeback all caches from * the page being destroyed */ if (TAILQ_NEXT(pv, pv_list) == NULL) @@ -2027,10 +2033,7 @@ newpte |= PTE_W; if (is_kernel_pmap(pmap)) newpte |= PTE_G; - if (is_cacheable_page(pa, m)) - newpte |= PTE_C_CACHE; - else - newpte |= PTE_C_UNCACHED; + PMAP_PTE_SET_CACHE_BITS(newpte, pa, m); mpte = NULL; @@ -2215,7 +2218,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { - pt_entry_t *pte; + pt_entry_t *pte, npte; vm_paddr_t pa; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || @@ -2294,18 +2297,16 @@ /* * Now validate mapping with RO protection */ - *pte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V; + npte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V; if ((m->oflags & VPO_UNMANAGED) == 0) - *pte |= PTE_MANAGED; + npte |= PTE_MANAGED; - if (is_cacheable_page(pa, m)) - *pte |= PTE_C_CACHE; - else - *pte |= PTE_C_UNCACHED; + PMAP_PTE_SET_CACHE_BITS(npte, pa, m); if (is_kernel_pmap(pmap)) - *pte |= PTE_G; + *pte = npte | PTE_G; else { + *pte = npte; /* * Sync I & D caches. Do this only if the target pmap * belongs to the current process. Otherwise, an @@ -2646,12 +2647,12 @@ #else vm_paddr_t pa; struct local_sysmaps *sysm; - pt_entry_t *pte; + pt_entry_t *pte, npte; pa = VM_PAGE_TO_PHYS(m); if (MIPS_DIRECT_MAPPABLE(pa)) { - if (m->md.pv_flags & PV_MEMATTR_UNCACHEABLE) + if (pmap_page_get_memattr(m) != VM_MEMATTR_WRITE_BACK) return (MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); else return (MIPS_PHYS_TO_DIRECT(pa)); @@ -2662,8 +2663,9 @@ KASSERT(sysm->valid1 == 0, ("pmap_quick_enter_page: PTE busy")); pte = pmap_pte(kernel_pmap, sysm->base); - *pte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G | - (is_cacheable_page(pa, m) ? PTE_C_CACHE : PTE_C_UNCACHED); + npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G; + PMAP_PTE_SET_CACHE_BITS(npte, pa, m); + *pte = npte; sysm->valid1 = 1; return (sysm->base); @@ -3143,26 +3145,26 @@ * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit. */ void * -pmap_mapdev(vm_paddr_t pa, vm_size_t size) +pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) { vm_offset_t va, tmpva, offset; - /* - * KSEG1 maps only first 512M of phys address space. For + /* + * KSEG1 maps only first 512M of phys address space. For * pa > 0x20000000 we should make proper mapping * using pmap_kenter. */ - if (MIPS_DIRECT_MAPPABLE(pa + size - 1)) + if (MIPS_DIRECT_MAPPABLE(pa + size - 1) && ma == VM_MEMATTR_UNCACHEABLE) return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa)); else { offset = pa & PAGE_MASK; size = roundup(size + offset, PAGE_SIZE); - + va = kva_alloc(size); if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); pa = trunc_page(pa); for (tmpva = va; size > 0;) { - pmap_kenter_attr(tmpva, pa, PTE_C_UNCACHED); + pmap_kenter_attr(tmpva, pa, ma); size -= PAGE_SIZE; tmpva += PAGE_SIZE; pa += PAGE_SIZE; @@ -3172,6 +3174,12 @@ return ((void *)(va + offset)); } +void * +pmap_mapdev(vm_paddr_t pa, vm_size_t size) +{ + return pmap_mapdev_attr(pa, size, VM_MEMATTR_UNCACHEABLE); +} + void pmap_unmapdev(vm_offset_t va, vm_size_t size) { @@ -3217,7 +3225,7 @@ * This may falsely report the given address as * MINCORE_REFERENCED. Unfortunately, due to the lack of * per-PTE reference information, it is impossible to - * determine if the address is MINCORE_REFERENCED. + * determine if the address is MINCORE_REFERENCED. */ m = PHYS_TO_VM_PAGE(pa); if ((m->aflags & PGA_REFERENCED) != 0) @@ -3497,7 +3505,7 @@ mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END); #if defined(__mips_n64) mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END); -#endif +#endif /* * Kernel virtual. */ @@ -3521,7 +3529,7 @@ } -void +void pmap_flush_pvcache(vm_page_t m) { pv_entry_t pv; @@ -3548,12 +3556,85 @@ if (TAILQ_FIRST(&m->md.pv_list) != NULL) panic("Can't change memattr on page with existing mappings"); - /* - * The only memattr we support is UNCACHEABLE, translate the (semi-)MI - * representation of that into our internal flag in the page MD struct. - */ - if (ma == VM_MEMATTR_UNCACHEABLE) - m->md.pv_flags |= PV_MEMATTR_UNCACHEABLE; - else - m->md.pv_flags &= ~PV_MEMATTR_UNCACHEABLE; + /* Clean memattr portion of pv_flags */ + m->md.pv_flags &= ~PV_MEMATTR_MASK; + m->md.pv_flags |= (ma << PV_MEMATTR_SHIFT) & PV_MEMATTR_MASK; +} + +static __inline void +pmap_pte_attr(pt_entry_t *pte, vm_memattr_t ma) +{ + u_int npte; + + npte = *(u_int *)pte; + npte &= ~PTE_C_MASK; + npte |= PTE_C(ma); + *pte = npte; +} + +int +pmap_change_attr(vm_offset_t sva, vm_size_t size, vm_memattr_t ma) +{ + pd_entry_t *pde, *pdpe; + pt_entry_t *pte; + vm_offset_t ova, eva, va, va_next; + pmap_t pmap; + + ova = sva; + eva = sva + size; + if (eva < sva) + return (EINVAL); + + pmap = kernel_pmap; + PMAP_LOCK(pmap); + + for (; sva < eva; sva = va_next) { + pdpe = pmap_segmap(pmap, sva); +#ifdef __mips_n64 + if (*pdpe == 0) { + va_next = (sva + NBSEG) & ~SEGMASK; + if (va_next < sva) + va_next = eva; + continue; + } +#endif + va_next = (sva + NBPDR) & ~PDRMASK; + if (va_next < sva) + va_next = eva; + + pde = pmap_pdpe_to_pde(pdpe, sva); + if (*pde == NULL) + continue; + + /* + * Limit our scan to either the end of the va represented + * by the current page table page, or to the end of the + * range being removed. + */ + if (va_next > eva) + va_next = eva; + + va = va_next; + for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, + sva += PAGE_SIZE) { + if (!pte_test(pte, PTE_V) || pte_cache_bits(pte) == ma) { + if (va != va_next) { + pmap_invalidate_range(pmap, va, sva); + va = va_next; + } + continue; + } + if (va == va_next) + va = sva; + + pmap_pte_attr(pte, ma); + } + if (va != va_next) + pmap_invalidate_range(pmap, va, sva); + } + PMAP_UNLOCK(pmap); + + /* Flush caches to be in the safe side */ + mips_dcache_wbinv_range(ova, size); + return 0; }