Index: sys/kern/subr_physmem.c =================================================================== --- sys/kern/subr_physmem.c +++ sys/kern/subr_physmem.c @@ -79,6 +79,12 @@ uint32_t flags; }; +/* + * We assume in early early boot where this is used that we don't need to worry + * about multithreaded bits. + */ +static struct region *itregions; + static struct region hwregions[MAX_HWCNT]; static struct region exregions[MAX_EXCNT]; @@ -144,6 +150,63 @@ physmem_dump_tables(printf); } +#ifdef DIAGNOSTIC +static void +physmem_sanity_check(struct region *regions, size_t cnt, bool need_flags) +{ + vm_paddr_t end, pstart, pend, start; + int flags, pflags; + struct region *preg, *reg; + size_t idx, pidx; + + for (idx = 0; idx < cnt; idx++) { + reg = ®ions[idx]; + + start = reg->addr; + end = start + reg->size; + flags = reg->flags; + + /* Overflow and zero-length regions. */ + MPASS(start < end); + /* No confusing hwregions/exregions.*/ + MPASS((need_flags && flags != 0) || + (!need_flags && flags == 0)); + + /* + * Check extents of all regions that come before it. We can't + * naively check just the last one, because it may have been a + * flag mismatch and we know nothing about the potentially + * coalescable region before it. + */ + for (pidx = 0; pidx < idx; pidx++) { + preg = ®ions[pidx]; + + pstart = preg->addr; + pend = preg->addr + preg->size; + pflags = preg->flags; + + if (pflags == flags) { + /* + * Matching flags: must have coalesced if it's + * adjacent. The previous extent must appear + * strictly before this one. + */ + MPASS(pend < start); + } else { + /* + * Non-matching flags: we cannot coalesce. The + * previous extent may start at the same addr, + * but not any later. + */ + MPASS(pstart <= start); + } + } + } +} +#else +#define physmem_sanity_check(r, c, f) +#endif + /* * Walk the list of hardware regions, processing it against the list of * exclusions that contain the given exflags, and generating an "avail list". @@ -367,6 +430,8 @@ rcnt = merge_upper_regions(regions, rcnt, i); } + + physmem_sanity_check(regions, rcnt, flags != 0); return (rcnt); } else if (addr <= rend && nend > rp->addr) { /* @@ -378,6 +443,8 @@ rcnt = merge_upper_regions(regions, rcnt, i); } + + physmem_sanity_check(regions, rcnt, flags != 0); return (rcnt); } } @@ -391,6 +458,7 @@ rp->flags = flags; rcnt++; + physmem_sanity_check(regions, rcnt, flags != 0); return (rcnt); } @@ -401,6 +469,8 @@ physmem_hardware_region(uint64_t pa, uint64_t sz) { + MPASS(itregions == NULL || itregions != hwregions); + /* * Filter out the page at PA 0x00000000. The VM can't handle it, as * pmap_extract() == 0 means failure. @@ -448,12 +518,32 @@ physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t exflags) { + MPASS(itregions == NULL || itregions != exregions); + if (excnt == nitems(exregions)) return (E2BIG); excnt = insert_region(exregions, excnt, pa, sz, exflags); return (0); } +/* + * physmem_max_page() calculates one past the highest usable page in the system. + * Note that this is technically inaccurate because an excluded range may leave + * us with even less usable memory; the final phys_avail should be considered + * more authoritative. + */ +size_t +physmem_max_page(void) +{ + struct region *rp; + + if (hwcnt == 0) + return (0); + + rp = &hwregions[hwcnt - 1]; + return (atop(rp->addr + rp->size)); +} + size_t physmem_avail(vm_paddr_t *avail, size_t maxavail) { @@ -461,6 +551,58 @@ return (regions_to_avail(avail, EXFLAG_NOALLOC, maxavail, 0, NULL, NULL)); } +/* + * Execute a callback on each page. The callback may call back into physmem_* + * bits, but it must avoid modifying the same region table that we're executing + * on. + */ +int +physmem_foreach_page(physmem_page_func pfunc, void *data, uint32_t flags) +{ + struct region *rp; + vm_paddr_t page, end, rend; + size_t idx, regcnt; + int error; + + /* No recursive iteration. */ + MPASS(itregions == NULL); + + if (flags == 0) { + itregions = hwregions; + regcnt = hwcnt; + } else { + itregions = exregions; + regcnt = excnt; + } + + error = 0; + for (idx = 0; idx < regcnt; idx++) { + rp = &itregions[idx]; + if (rp->flags != flags) + continue; + + page = round_page(rp->addr); + rend = rp->addr + rp->size; + end = trunc_page(rend); + /* Tap off any leading partial page. */ + if (page != rp->addr) { + error = pfunc(rp->addr, page, data); + } + for (; error == 0 && page < end; page += PAGE_SIZE) { + error = pfunc(page, page + PAGE_SIZE, data); + } + /* Tap off any trailing partial page. */ + if (error == 0 && end != rend) { + error = pfunc(end, rend, data); + } + if (error != 0) + break; + } + + itregions = NULL; + return (error); +} + /* * Process all the regions added earlier into the global avail lists. * @@ -479,8 +621,14 @@ size_t nextidx; u_long hwphyssz; - hwphyssz = 0; - TUNABLE_ULONG_FETCH("hw.physmem", &hwphyssz); + if (Maxmem != 0) { + hwphyssz = ptoa(Maxmem); + } else if (!TUNABLE_ULONG_FETCH("hw.physmem", &hwphyssz)) { + /* XXX If MAXMEM's undefined? hw.physmem is typically here... */ +#ifdef MAXMEM + hwphyssz = MAXMEM; +#endif + } nextidx = regions_to_avail(dump_avail, EXFLAG_NODUMP, PHYS_AVAIL_ENTRIES, hwphyssz, NULL, NULL); @@ -492,7 +640,8 @@ panic("No memory entries in phys_avail"); if (pa_idx != NULL) *pa_idx = nextidx; - Maxmem = atop(phys_avail[nextidx - 1]); + if (Maxmem == 0) + Maxmem = atop(phys_avail[nextidx - 1]); } #ifdef DDB Index: sys/sys/physmem.h =================================================================== --- sys/sys/physmem.h +++ sys/sys/physmem.h @@ -49,11 +49,15 @@ #define EXFLAG_NODUMP 0x01 #define EXFLAG_NOALLOC 0x02 +typedef int (*physmem_page_func)(vm_paddr_t start, vm_paddr_t end, void *data); + int physmem_hardware_region(uint64_t pa, uint64_t sz); int physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t flags); +size_t physmem_max_page(void); size_t physmem_avail(vm_paddr_t *avail, size_t maxavail); void physmem_init_kernel_globals(size_t *pa_idx, size_t *da_idx); void physmem_print_tables(void); +int physmem_foreach_page(physmem_page_func pfunc, void *data, uint32_t flags); /* * Convenience routines for FDT.