Index: head/.clang-format =================================================================== --- head/.clang-format +++ head/.clang-format @@ -76,6 +76,7 @@ - TAILQ_FOREACH_REVERSE_SAFE - TAILQ_FOREACH_SAFE - VM_MAP_ENTRY_FOREACH + - VM_PAGE_DUMP_FOREACH IndentCaseLabels: false IndentPPDirectives: None Language: Cpp Index: head/sys/amd64/amd64/minidump_machdep.c =================================================================== --- head/sys/amd64/amd64/minidump_machdep.c +++ head/sys/amd64/amd64/minidump_machdep.c @@ -54,9 +54,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -64,7 +61,6 @@ static void *dump_va; static size_t counter, progress, dumpsize, wdog_next; -CTASSERT(sizeof(*vm_page_dump) == 8); static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, "Number of times dump has to retry before bailing out"); @@ -223,9 +219,8 @@ uint32_t pmapsize; vm_offset_t va; int error; - uint64_t bits; uint64_t *pml4, *pdp, *pd, *pt, pa; - int i, ii, j, k, n, bit; + int i, ii, j, k, n; int retry_count; struct minidumphdr mdhdr; @@ -304,19 +299,13 @@ /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -328,7 +317,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapbase = DMAP_MIN_ADDRESS; @@ -357,7 +346,8 @@ goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -409,16 +399,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -453,26 +437,4 @@ } else printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); -} - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); } Index: head/sys/amd64/include/md_var.h =================================================================== --- head/sys/amd64/include/md_var.h +++ head/sys/amd64/include/md_var.h @@ -36,7 +36,6 @@ #include -extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; Index: head/sys/amd64/include/vmparam.h =================================================================== --- head/sys/amd64/include/vmparam.h +++ head/sys/amd64/include/vmparam.h @@ -253,4 +253,9 @@ */ #define VM_BATCHQUEUE_SIZE 31 +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ Index: head/sys/arm/arm/minidump_machdep.c =================================================================== --- head/sys/arm/arm/minidump_machdep.c +++ head/sys/arm/arm/minidump_machdep.c @@ -57,9 +57,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -67,8 +64,6 @@ static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -182,10 +177,9 @@ struct minidumphdr mdhdr; uint64_t dumpsize; uint32_t ptesize; - uint32_t bits; uint32_t pa, prev_pa = 0, count = 0; vm_offset_t va; - int i, bit, error; + int error; char *addr; /* @@ -212,20 +206,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -236,7 +223,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.arch = __ARM_ARCH; @@ -270,7 +257,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -293,28 +280,21 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - if (!count) { - prev_pa = pa; + VM_PAGE_DUMP_FOREACH(pa) { + if (!count) { + prev_pa = pa; + count++; + } else { + if (pa == (prev_pa + count * PAGE_SIZE)) count++; - } else { - if (pa == (prev_pa + count * PAGE_SIZE)) - count++; - else { - error = blk_write(di, NULL, prev_pa, - count * PAGE_SIZE); - if (error) - goto fail; - count = 1; - prev_pa = pa; - } + else { + error = blk_write(di, NULL, prev_pa, + count * PAGE_SIZE); + if (error) + goto fail; + count = 1; + prev_pa = pa; } - bits &= ~(1ul << bit); } } if (count) { @@ -348,26 +328,4 @@ } else printf("\n** DUMP FAILED (ERROR %d) **\n", error); return (error); -} - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); } Index: head/sys/arm/include/md_var.h =================================================================== --- head/sys/arm/include/md_var.h +++ head/sys/arm/include/md_var.h @@ -38,8 +38,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; extern vm_paddr_t arm_physmem_kernaddr; @@ -72,8 +70,6 @@ struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); extern uint32_t initial_fpscr; Index: head/sys/arm/include/vmparam.h =================================================================== --- head/sys/arm/include/vmparam.h +++ head/sys/arm/include/vmparam.h @@ -193,4 +193,9 @@ #define DEVMAP_MAX_VADDR ARM_VECTORS_HIGH +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ Index: head/sys/arm64/arm64/minidump_machdep.c =================================================================== --- head/sys/arm64/arm64/minidump_machdep.c +++ head/sys/arm64/arm64/minidump_machdep.c @@ -56,9 +56,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -68,8 +65,6 @@ static uint64_t tmpbuffer[Ln_ENTRIES]; -CTASSERT(sizeof(*vm_page_dump) == 8); - static int is_dumpable(vm_paddr_t pa) { @@ -213,9 +208,8 @@ pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; - uint64_t bits; uint32_t pmapsize; - int bit, error, i, j, retry_count; + int error, i, j, retry_count; retry_count = 0; retry: @@ -255,20 +249,12 @@ /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -279,7 +265,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -311,7 +297,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -378,17 +364,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -423,26 +402,4 @@ } else printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); -} - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); } Index: head/sys/arm64/include/md_var.h =================================================================== --- head/sys/arm64/include/md_var.h +++ head/sys/arm64/include/md_var.h @@ -36,8 +36,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; @@ -45,8 +43,6 @@ extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); void generic_bs_fault(void) __asm(__STRING(generic_bs_fault)); void generic_bs_peek_1(void) __asm(__STRING(generic_bs_peek_1)); Index: head/sys/arm64/include/vmparam.h =================================================================== --- head/sys/arm64/include/vmparam.h +++ head/sys/arm64/include/vmparam.h @@ -243,4 +243,9 @@ #define DEVMAP_MAX_VADDR VM_MAX_KERNEL_ADDRESS +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ Index: head/sys/i386/i386/minidump_machdep.c =================================================================== --- head/sys/i386/i386/minidump_machdep.c +++ head/sys/i386/i386/minidump_machdep.c @@ -49,33 +49,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - -CTASSERT(sizeof(*vm_page_dump) == 4); - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - int minidumpsys(struct dumperinfo *di) { Index: head/sys/i386/i386/minidump_machdep_base.c =================================================================== --- head/sys/i386/i386/minidump_machdep_base.c +++ head/sys/i386/i386/minidump_machdep_base.c @@ -62,8 +62,6 @@ static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -181,11 +179,10 @@ uint32_t ptesize; vm_offset_t va; int error; - uint32_t bits; uint64_t pa; pd_entry_t *pd; pt_entry_t *pt; - int i, j, k, bit; + int j, k; struct minidumphdr mdhdr; counter = 0; @@ -227,19 +224,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -251,7 +242,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.paemode = pae_mode; @@ -279,7 +270,8 @@ goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -321,16 +313,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); Index: head/sys/i386/include/md_var.h =================================================================== --- head/sys/i386/include/md_var.h +++ head/sys/i386/include/md_var.h @@ -47,7 +47,6 @@ extern int szosigcode; extern int sz_lcall_tramp; #endif -extern uint32_t *vm_page_dump; extern vm_offset_t proc0kstack; extern uintptr_t setidt_disp; Index: head/sys/i386/include/vmparam.h =================================================================== --- head/sys/i386/include/vmparam.h +++ head/sys/i386/include/vmparam.h @@ -240,4 +240,9 @@ #define PHYS_TO_DMAP(x) ({ panic("No direct map exists"); 0; }) #define DMAP_TO_PHYS(x) ({ panic("No direct map exists"); 0; }) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* _MACHINE_VMPARAM_H_ */ Index: head/sys/mips/include/md_var.h =================================================================== --- head/sys/mips/include/md_var.h +++ head/sys/mips/include/md_var.h @@ -50,8 +50,6 @@ extern char sigcode32[]; extern int szsigcode32; #endif -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern vm_offset_t kstack0; extern vm_offset_t kernel_kseg0_end; @@ -84,8 +82,6 @@ void busdma_swi(void); struct dumperinfo; -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ Index: head/sys/mips/include/vmparam.h =================================================================== --- head/sys/mips/include/vmparam.h +++ head/sys/mips/include/vmparam.h @@ -197,4 +197,9 @@ #define PHYS_TO_DMAP(x) MIPS_PHYS_TO_DIRECT(x) #define DMAP_TO_PHYS(x) MIPS_DIRECT_TO_PHYS(x) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 + #endif /* !_MACHINE_VMPARAM_H_ */ Index: head/sys/mips/mips/minidump_machdep.c =================================================================== --- head/sys/mips/mips/minidump_machdep.c +++ head/sys/mips/mips/minidump_machdep.c @@ -54,9 +54,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -66,8 +63,6 @@ extern pd_entry_t *kernel_segmap; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -83,28 +78,6 @@ return (0); } -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - static struct { int min_per; int max_per; @@ -192,13 +165,12 @@ { struct minidumphdr mdhdr; uint32_t ptesize; - uint32_t bits; vm_paddr_t pa; vm_offset_t prev_pte = 0; uint32_t count = 0; vm_offset_t va; pt_entry_t *pte; - int i, bit, error; + int i, error; void *dump_va; /* Flush cache */ @@ -233,20 +205,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -257,7 +222,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.ptesize = ptesize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; @@ -286,7 +251,7 @@ /* Dump bitmap */ error = write_buffer(di, (char *)vm_page_dump, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -320,19 +285,12 @@ } /* Dump memory chunks page by page*/ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - dump_va = pmap_kenter_temporary(pa, 0); - error = write_buffer(di, dump_va, PAGE_SIZE); - if (error) - goto fail; - pmap_kenter_temporary_free(pa); - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + dump_va = pmap_kenter_temporary(pa, 0); + error = write_buffer(di, dump_va, PAGE_SIZE); + if (error) + goto fail; + pmap_kenter_temporary_free(pa); } error = dump_finish(di, &kdh); Index: head/sys/powerpc/include/md_var.h =================================================================== --- head/sys/powerpc/include/md_var.h +++ head/sys/powerpc/include/md_var.h @@ -42,14 +42,9 @@ extern char sigcode64[], sigcode64_elfv2[]; extern int szsigcode64, szsigcode64_elfv2; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; - struct dumperinfo; int minidumpsys(struct dumperinfo *); int is_dumpable(vm_paddr_t); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); #endif extern long Maxmem; Index: head/sys/powerpc/include/vmparam.h =================================================================== --- head/sys/powerpc/include/vmparam.h +++ head/sys/powerpc/include/vmparam.h @@ -307,6 +307,18 @@ #define PMAP_HAS_PAGE_ARRAY 1 #endif +#if defined(__powerpc64__) +/* + * Need a page dump array for minidump. + */ +#define MINIDUMP_PAGE_TRACKING 1 +#else +/* + * No minidump with 32-bit powerpc. + */ +#define MINIDUMP_PAGE_TRACKING 0 +#endif + #define PMAP_HAS_DMAP (hw_direct_map) #define PHYS_TO_DMAP(x) ({ \ KASSERT(hw_direct_map, ("Direct map not provided by PMAP")); \ Index: head/sys/powerpc/powerpc/minidump_machdep.c =================================================================== --- head/sys/powerpc/powerpc/minidump_machdep.c +++ head/sys/powerpc/powerpc/minidump_machdep.c @@ -45,16 +45,6 @@ #include #include -/* - * bit to physical address - * - * bm - bitmap - * i - bitmap entry index - * bit - bit number - */ -#define BTOP(bm, i, bit) \ - (((uint64_t)(i) * sizeof(*(bm)) * NBBY + (bit)) * PAGE_SIZE) - /* Debugging stuff */ #define MINIDUMP_DEBUG 0 #if MINIDUMP_DEBUG @@ -70,9 +60,6 @@ extern vm_offset_t __startkernel, __endkernel; -int vm_page_dump_size; -uint64_t *vm_page_dump; - static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, @@ -103,28 +90,6 @@ /* Handle chunked writes. */ static size_t fragsz; -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} - int is_dumpable(vm_paddr_t pa) { @@ -280,9 +245,8 @@ minidumpsys(struct dumperinfo *di) { vm_paddr_t pa; - int bit, error, i, retry_count; + int error, i, retry_count; uint32_t pmapsize; - uint64_t bits; struct minidumphdr mdhdr; retry_count = 0; @@ -306,24 +270,14 @@ /* Calculate dump size */ dumpsize = PAGE_SIZE; /* header */ dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); dumpsize += pmapsize; - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul<msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.kernend = VM_MAX_SAFE_KERNEL_ADDRESS; @@ -368,10 +322,10 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; - dump_total("bitmap", round_page(vm_page_dump_size)); + dump_total("bitmap", round_page(BITSET_SIZE(vm_page_dump_pages))); /* Dump kernel page directory pages */ error = dump_pmap(di); @@ -380,20 +334,10 @@ dump_total("pmap", pmapsize); /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul< CTASSERT(sizeof(struct kerneldumpheader) == 512); -CTASSERT(sizeof(*vm_page_dump) == 8); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -221,8 +217,7 @@ vm_offset_t va; vm_paddr_t pa; int error; - uint64_t bits; - int i, bit; + int i; int retry_count; retry_count = 0; @@ -262,20 +257,13 @@ /* Calculate dump size */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -286,7 +274,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = round_page(BITSET_SIZE(vm_page_dump_pages)); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = KERNBASE; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -318,7 +306,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -360,17 +348,10 @@ /* Dump memory chunks */ /* XXX cluster it up and use blk_dump() */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -405,32 +386,4 @@ } else printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); -} - -/* - * Add a page to the minidump bitmap. - */ -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -/* - * Remove page from the minidump bitmap. - */ -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); } Index: head/sys/vm/vm_page.h =================================================================== --- head/sys/vm/vm_page.h +++ head/sys/vm/vm_page.h @@ -69,6 +69,8 @@ #ifndef _VM_PAGE_ #define _VM_PAGE_ +#include +#include #include /* @@ -584,6 +586,26 @@ #define PS_ALL_DIRTY 0x1 #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 + +extern struct bitset *vm_page_dump; +extern long vm_page_dump_pages; + +static inline void +dump_add_page(vm_paddr_t pa) +{ + BIT_SET_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +static inline void +dump_drop_page(vm_paddr_t pa) +{ + BIT_CLR_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +#define VM_PAGE_DUMP_FOREACH(pa) \ + for (vm_pindex_t __b = BIT_FFS(vm_page_dump_pages, vm_page_dump); \ + (pa) = (__b - 1) * PAGE_SIZE, __b != 0; \ + __b = BIT_FFS_AT(vm_page_dump_pages, vm_page_dump, __b)) bool vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); Index: head/sys/vm/vm_page.c =================================================================== --- head/sys/vm/vm_page.c +++ head/sys/vm/vm_page.c @@ -155,6 +155,9 @@ long vm_page_array_size; long first_page; +struct bitset *vm_page_dump; +long vm_page_dump_pages; + static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | @@ -554,6 +557,9 @@ vm_paddr_t page_range __unused; vm_paddr_t last_pa, pa; u_long pagecount; +#if MINIDUMP_PAGE_TRACKING + u_long vm_page_dump_size; +#endif int biggestone, i, segind; #ifdef WITNESS vm_offset_t mapped; @@ -588,9 +594,7 @@ witness_startup((void *)mapped); #endif -#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ - defined(__i386__) || defined(__mips__) || defined(__riscv) || \ - defined(__powerpc64__) +#if MINIDUMP_PAGE_TRACKING /* * Allocate a bitmap to indicate that a random physical page * needs to be included in a minidump. @@ -606,8 +610,8 @@ for (i = 0; dump_avail[i + 1] != 0; i += 2) if (dump_avail[i + 1] > last_pa) last_pa = dump_avail[i + 1]; - page_range = last_pa / PAGE_SIZE; - vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); + vm_page_dump_pages = last_pa / PAGE_SIZE; + vm_page_dump_size = round_page(BITSET_SIZE(vm_page_dump_pages)); new_end -= vm_page_dump_size; vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); Index: head/sys/x86/include/x86_var.h =================================================================== --- head/sys/x86/include/x86_var.h +++ head/sys/x86/include/x86_var.h @@ -78,7 +78,6 @@ extern char kstack[]; extern char sigcode[]; extern int szsigcode; -extern int vm_page_dump_size; extern int workaround_erratum383; extern int _udatasel; extern int _ucodesel; @@ -122,8 +121,6 @@ void cpu_setregs(void); bool disable_wp(void); void restore_wp(bool old_wp); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); void finishidentcpu(void); void identify_cpu1(void); void identify_cpu2(void);