Index: sys/i386/i386/pmap.c =================================================================== --- sys/i386/i386/pmap.c +++ sys/i386/i386/pmap.c @@ -257,14 +257,6 @@ /* * All those kernel PT submaps that BSD is so fond of */ -struct sysmaps { - struct mtx lock; - pt_entry_t *CMAP1; - pt_entry_t *CMAP2; - caddr_t CADDR1; - caddr_t CADDR2; -}; -static struct sysmaps sysmaps_pcpu[MAXCPU]; pt_entry_t *CMAP3; static pd_entry_t *KPTD; caddr_t ptvmmap = 0; @@ -377,7 +369,7 @@ void pmap_bootstrap(vm_paddr_t firstaddr) { - vm_offset_t va; + vm_offset_t va, qmap_va; pt_entry_t *pte, *unused; struct sysmaps *sysmaps; int i; @@ -441,16 +433,20 @@ va = virtual_avail; pte = vtopte(va); + /* + * Initialize sysmaps on the current CPU for use + * during early boot. * CMAP1/CMAP2 are used for zeroing and copying pages. * CMAP3 is used for the boot-time memory test. */ - for (i = 0; i < MAXCPU; i++) { - sysmaps = &sysmaps_pcpu[i]; - mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); - SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1) - SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1) - } + sysmaps = PCPU_PTR(sysmaps); + mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); + SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1) + SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1) + SYSMAP(vm_offset_t, pte, qmap_va, 1) + PCPU_SET(qmap_addr, qmap_va); + SYSMAP(caddr_t, CMAP3, CADDR3, 1); /* @@ -520,20 +516,31 @@ } static void -pmap_init_qpages(void) +pmap_init_reserved_pages(void) { struct pcpu *pc; + struct sysmaps *sysmaps; + vm_offset_t pages; int i; CPU_FOREACH(i) { pc = pcpu_find(i); - pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); - if (pc->pc_qmap_addr == 0) - panic("pmap_init_qpages: unable to allocate KVA"); + sysmaps = &pc->pc_sysmaps; + if (sysmaps->CADDR1 != 0) + continue; + mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); + pages = kva_alloc(PAGE_SIZE * 3); + if (pages == 0) + panic("%s: unable to allocate KVA", __func__); + sysmaps->CMAP1 = vtopte(pages); + sysmaps->CMAP2 = vtopte(pages + PAGE_SIZE); + sysmaps->CADDR1 = (caddr_t)pages; + sysmaps->CADDR2 = (caddr_t)(pages + PAGE_SIZE); + pc->pc_qmap_addr = pages + (PAGE_SIZE * 2); } } -SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL); +SYSINIT(rpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_reserved_pages, NULL); /* * Setup the PAT MSR. @@ -4194,20 +4201,22 @@ void pmap_zero_page(vm_page_t m) { + pt_entry_t *CMAP2; struct sysmaps *sysmaps; - sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + sched_pin(); + sysmaps = PCPU_PTR(sysmaps); + CMAP2 = sysmaps->CMAP2; mtx_lock(&sysmaps->lock); - if (*sysmaps->CMAP2) + if (*CMAP2) panic("pmap_zero_page: CMAP2 busy"); - sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | + *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0); invlcaddr(sysmaps->CADDR2); pagezero(sysmaps->CADDR2); - *sysmaps->CMAP2 = 0; - sched_unpin(); + *CMAP2 = 0; mtx_unlock(&sysmaps->lock); + sched_unpin(); } /* @@ -4217,23 +4226,25 @@ void pmap_zero_page_area(vm_page_t m, int off, int size) { + pt_entry_t *CMAP2; struct sysmaps *sysmaps; - sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + sched_pin(); + sysmaps = PCPU_PTR(sysmaps); + CMAP2 = sysmaps->CMAP2; mtx_lock(&sysmaps->lock); - if (*sysmaps->CMAP2) + if (*CMAP2) panic("pmap_zero_page_area: CMAP2 busy"); - sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | + *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0); invlcaddr(sysmaps->CADDR2); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else - bzero((char *)sysmaps->CADDR2 + off, size); - *sysmaps->CMAP2 = 0; - sched_unpin(); + bzero(sysmaps->CADDR2 + off, size); + *CMAP2 = 0; mtx_unlock(&sysmaps->lock); + sched_unpin(); } /* @@ -4242,26 +4253,29 @@ void pmap_copy_page(vm_page_t src, vm_page_t dst) { + pt_entry_t *CMAP1, *CMAP2; struct sysmaps *sysmaps; - sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + sched_pin(); + sysmaps = PCPU_PTR(sysmaps); + CMAP1 = sysmaps->CMAP1; + CMAP2 = sysmaps->CMAP2; mtx_lock(&sysmaps->lock); - if (*sysmaps->CMAP1) + if (*CMAP1) panic("pmap_copy_page: CMAP1 busy"); - if (*sysmaps->CMAP2) + if (*CMAP2) panic("pmap_copy_page: CMAP2 busy"); - sched_pin(); - *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A | + *CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A | pmap_cache_bits(src->md.pat_mode, 0); invlcaddr(sysmaps->CADDR1); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M | + *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M | pmap_cache_bits(dst->md.pat_mode, 0); invlcaddr(sysmaps->CADDR2); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); - *sysmaps->CMAP1 = 0; - *sysmaps->CMAP2 = 0; - sched_unpin(); + *CMAP1 = 0; + *CMAP2 = 0; mtx_unlock(&sysmaps->lock); + sched_unpin(); } int unmapped_buf_allowed = 1; @@ -4270,19 +4284,22 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { + pt_entry_t *CMAP1, *CMAP2; struct sysmaps *sysmaps; vm_page_t a_pg, b_pg; char *a_cp, *b_cp; vm_offset_t a_pg_offset, b_pg_offset; int cnt; - sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + sched_pin(); + sysmaps = PCPU_PTR(sysmaps); + CMAP1 = sysmaps->CMAP1; + CMAP2 = sysmaps->CMAP2; mtx_lock(&sysmaps->lock); - if (*sysmaps->CMAP1 != 0) + if (*CMAP1 != 0) panic("pmap_copy_pages: CMAP1 busy"); - if (*sysmaps->CMAP2 != 0) + if (*CMAP2 != 0) panic("pmap_copy_pages: CMAP2 busy"); - sched_pin(); while (xfersize > 0) { a_pg = ma[a_offset >> PAGE_SHIFT]; a_pg_offset = a_offset & PAGE_MASK; @@ -4290,10 +4307,10 @@ b_pg = mb[b_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); - *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A | + *CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A | pmap_cache_bits(a_pg->md.pat_mode, 0); invlcaddr(sysmaps->CADDR1); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A | + *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A | PG_M | pmap_cache_bits(b_pg->md.pat_mode, 0); invlcaddr(sysmaps->CADDR2); a_cp = sysmaps->CADDR1 + a_pg_offset; @@ -4303,10 +4320,10 @@ b_offset += cnt; xfersize -= cnt; } - *sysmaps->CMAP1 = 0; - *sysmaps->CMAP2 = 0; - sched_unpin(); + *CMAP1 = 0; + *CMAP2 = 0; mtx_unlock(&sysmaps->lock); + sched_unpin(); } /* @@ -5247,18 +5264,20 @@ static void pmap_flush_page(vm_page_t m) { + pt_entry_t *CMAP2; struct sysmaps *sysmaps; vm_offset_t sva, eva; bool useclflushopt; useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0; if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) { - sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + sched_pin(); + sysmaps = PCPU_PTR(sysmaps); + CMAP2 = sysmaps->CMAP2; mtx_lock(&sysmaps->lock); - if (*sysmaps->CMAP2) + if (*CMAP2) panic("pmap_flush_page: CMAP2 busy"); - sched_pin(); - *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | + *CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0); invlcaddr(sysmaps->CADDR2); sva = (vm_offset_t)sysmaps->CADDR2; @@ -5280,9 +5299,9 @@ } if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); - *sysmaps->CMAP2 = 0; - sched_unpin(); + *CMAP2 = 0; mtx_unlock(&sysmaps->lock); + sched_unpin(); } else pmap_invalidate_cache(); } Index: sys/i386/include/pcpu.h =================================================================== --- sys/i386/include/pcpu.h +++ sys/i386/include/pcpu.h @@ -36,6 +36,17 @@ #include #include +#include +#include + +struct sysmaps { + struct mtx lock; + void *CMAP1; + void *CMAP2; + caddr_t CADDR1; + caddr_t CADDR2; +}; + /* * The SMP parts are setup in pmap.c and locore.s for the BSP, and * mp_machdep.c sets up the data for the AP's to "see" when they awake. @@ -58,9 +69,10 @@ int pc_private_tss; /* Flag indicating private tss*/\ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ + struct sysmaps pc_sysmaps; \ vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ - char __pad[225] + char __pad[225 - sizeof(struct sysmaps)] #ifdef _KERNEL