Index: sys/amd64/amd64/minidump_machdep.c =================================================================== --- sys/amd64/amd64/minidump_machdep.c +++ sys/amd64/amd64/minidump_machdep.c @@ -54,9 +54,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -64,7 +61,6 @@ static void *dump_va; static size_t counter, progress, dumpsize, wdog_next; -CTASSERT(sizeof(*vm_page_dump) == 8); static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, "Number of times dump has to retry before bailing out"); @@ -223,7 +219,6 @@ uint32_t pmapsize; vm_offset_t va; int error; - uint64_t bits; uint64_t *pml4, *pdp, *pd, *pt, pa; int i, ii, j, k, n, bit; int retry_count; @@ -304,19 +299,13 @@ /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -328,7 +317,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapbase = DMAP_MIN_ADDRESS; @@ -357,7 +346,8 @@ goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -409,16 +399,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfq(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(bit, pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -454,25 +438,3 @@ printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} Index: sys/amd64/include/md_var.h =================================================================== --- sys/amd64/include/md_var.h +++ sys/amd64/include/md_var.h @@ -36,7 +36,6 @@ #include -extern uint64_t *vm_page_dump; extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; Index: sys/arm/arm/minidump_machdep.c =================================================================== --- sys/arm/arm/minidump_machdep.c +++ sys/arm/arm/minidump_machdep.c @@ -57,9 +57,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -67,8 +64,6 @@ static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -182,7 +177,6 @@ struct minidumphdr mdhdr; uint64_t dumpsize; uint32_t ptesize; - uint32_t bits; uint32_t pa, prev_pa = 0, count = 0; vm_offset_t va; int i, bit, error; @@ -212,20 +206,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -236,7 +223,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.arch = __ARM_ARCH; @@ -270,7 +257,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -293,28 +280,21 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - if (!count) { - prev_pa = pa; + VM_PAGE_DUMP_FOREACH(bit, pa) { + if (!count) { + prev_pa = pa; + count++; + } else { + if (pa == (prev_pa + count * PAGE_SIZE)) count++; - } else { - if (pa == (prev_pa + count * PAGE_SIZE)) - count++; - else { - error = blk_write(di, NULL, prev_pa, - count * PAGE_SIZE); - if (error) - goto fail; - count = 1; - prev_pa = pa; - } + else { + error = blk_write(di, NULL, prev_pa, + count * PAGE_SIZE); + if (error) + goto fail; + count = 1; + prev_pa = pa; } - bits &= ~(1ul << bit); } } if (count) { @@ -349,25 +329,3 @@ printf("\n** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} Index: sys/arm/include/md_var.h =================================================================== --- sys/arm/include/md_var.h +++ sys/arm/include/md_var.h @@ -38,8 +38,6 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern u_long elf_hwcap; extern u_long elf_hwcap2; extern vm_paddr_t arm_physmem_kernaddr; @@ -72,8 +70,6 @@ struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); extern uint32_t initial_fpscr; Index: sys/arm64/arm64/minidump_machdep.c =================================================================== --- sys/arm64/arm64/minidump_machdep.c +++ sys/arm64/arm64/minidump_machdep.c @@ -56,9 +56,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint64_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -68,8 +65,6 @@ static uint64_t tmpbuffer[Ln_ENTRIES]; -CTASSERT(sizeof(*vm_page_dump) == 8); - static int is_dumpable(vm_paddr_t pa) { @@ -213,7 +208,6 @@ pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; - uint64_t bits; uint32_t pmapsize; int bit, error, i, j, retry_count; @@ -255,20 +249,12 @@ /* Calculate dump size. */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -279,7 +265,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -311,7 +297,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -378,17 +364,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(bit, pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -424,25 +403,3 @@ printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} Index: sys/arm64/include/md_var.h =================================================================== --- sys/arm64/include/md_var.h +++ sys/arm64/include/md_var.h @@ -36,15 +36,11 @@ extern long Maxmem; extern char sigcode[]; extern int szsigcode; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; struct dumperinfo; extern int busdma_swi_pending; void busdma_swi(void); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ Index: sys/i386/i386/minidump_machdep.c =================================================================== --- sys/i386/i386/minidump_machdep.c +++ sys/i386/i386/minidump_machdep.c @@ -49,33 +49,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - -CTASSERT(sizeof(*vm_page_dump) == 4); - -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - int minidumpsys(struct dumperinfo *di) { Index: sys/i386/i386/minidump_machdep_base.c =================================================================== --- sys/i386/i386/minidump_machdep_base.c +++ sys/i386/i386/minidump_machdep_base.c @@ -62,8 +62,6 @@ static void *dump_va; static uint64_t counter, progress; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -181,7 +179,6 @@ uint32_t ptesize; vm_offset_t va; int error; - uint32_t bits; uint64_t pa; pd_entry_t *pd; pt_entry_t *pt; @@ -227,19 +224,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) { - dumpsize += PAGE_SIZE; - } else { - dump_drop_page(pa); - } - bits &= ~(1ul << bit); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) { + dumpsize += PAGE_SIZE; + } else { + dump_drop_page(pa); } } dumpsize += PAGE_SIZE; @@ -251,7 +242,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.ptesize = ptesize; mdhdr.kernbase = KERNBASE; mdhdr.paemode = pae_mode; @@ -279,7 +270,8 @@ goto fail; /* Dump bitmap */ - error = blk_write(di, (char *)vm_page_dump, 0, round_page(vm_page_dump_size)); + error = blk_write(di, (char *)vm_page_dump, 0, + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -321,16 +313,10 @@ } /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = bsfl(bits); - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(bit, pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); Index: sys/i386/include/md_var.h =================================================================== --- sys/i386/include/md_var.h +++ sys/i386/include/md_var.h @@ -47,7 +47,6 @@ extern int szosigcode; extern int sz_lcall_tramp; #endif -extern uint32_t *vm_page_dump; extern vm_offset_t proc0kstack; extern uintptr_t setidt_disp; Index: sys/mips/include/md_var.h =================================================================== --- sys/mips/include/md_var.h +++ sys/mips/include/md_var.h @@ -50,8 +50,6 @@ extern char sigcode32[]; extern int szsigcode32; #endif -extern uint32_t *vm_page_dump; -extern int vm_page_dump_size; extern vm_offset_t kstack0; extern vm_offset_t kernel_kseg0_end; @@ -84,8 +82,6 @@ void busdma_swi(void); struct dumperinfo; -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); #endif /* !_MACHINE_MD_VAR_H_ */ Index: sys/mips/mips/minidump_machdep.c =================================================================== --- sys/mips/mips/minidump_machdep.c +++ sys/mips/mips/minidump_machdep.c @@ -54,9 +54,6 @@ CTASSERT(sizeof(struct kerneldumpheader) == 512); -uint32_t *vm_page_dump; -int vm_page_dump_size; - static struct kerneldumpheader kdh; /* Handle chunked writes. */ @@ -66,8 +63,6 @@ extern pd_entry_t *kernel_segmap; -CTASSERT(sizeof(*vm_page_dump) == 4); - static int is_dumpable(vm_paddr_t pa) { @@ -83,28 +78,6 @@ return (0); } -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_set_int(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 5; /* 2^5 = 32 */ - bit = pa & 31; - atomic_clear_int(&vm_page_dump[idx], 1ul << bit); -} - static struct { int min_per; int max_per; @@ -192,7 +165,6 @@ { struct minidumphdr mdhdr; uint32_t ptesize; - uint32_t bits; vm_paddr_t pa; vm_offset_t prev_pte = 0; uint32_t count = 0; @@ -233,20 +205,13 @@ /* Calculate dump size. */ dumpsize = ptesize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -257,7 +222,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.ptesize = ptesize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; @@ -286,7 +251,7 @@ /* Dump bitmap */ error = write_buffer(di, (char *)vm_page_dump, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -320,19 +285,12 @@ } /* Dump memory chunks page by page*/ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffs(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - dump_va = pmap_kenter_temporary(pa, 0); - error = write_buffer(di, dump_va, PAGE_SIZE); - if (error) - goto fail; - pmap_kenter_temporary_free(pa); - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(bit, pa) { + dump_va = pmap_kenter_temporary(pa, 0); + error = write_buffer(di, dump_va, PAGE_SIZE); + if (error) + goto fail; + pmap_kenter_temporary_free(pa); } error = dump_finish(di, &kdh); Index: sys/powerpc/include/md_var.h =================================================================== --- sys/powerpc/include/md_var.h +++ sys/powerpc/include/md_var.h @@ -42,14 +42,9 @@ extern char sigcode64[], sigcode64_elfv2[]; extern int szsigcode64, szsigcode64_elfv2; -extern uint64_t *vm_page_dump; -extern int vm_page_dump_size; - struct dumperinfo; int minidumpsys(struct dumperinfo *); int is_dumpable(vm_paddr_t); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); #endif extern long Maxmem; Index: sys/powerpc/powerpc/minidump_machdep.c =================================================================== --- sys/powerpc/powerpc/minidump_machdep.c +++ sys/powerpc/powerpc/minidump_machdep.c @@ -45,16 +45,6 @@ #include #include -/* - * bit to physical address - * - * bm - bitmap - * i - bitmap entry index - * bit - bit number - */ -#define BTOP(bm, i, bit) \ - (((uint64_t)(i) * sizeof(*(bm)) * NBBY + (bit)) * PAGE_SIZE) - /* Debugging stuff */ #define MINIDUMP_DEBUG 0 #if MINIDUMP_DEBUG @@ -71,9 +61,6 @@ extern vm_offset_t __startkernel, __endkernel; -int vm_page_dump_size; -uint64_t *vm_page_dump; - static int dump_retry_count = 5; SYSCTL_INT(_machdep, OID_AUTO, dump_retry_count, CTLFLAG_RWTUN, &dump_retry_count, 0, @@ -104,28 +91,6 @@ /* Handle chunked writes. */ static size_t fragsz; -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} - int is_dumpable(vm_paddr_t pa) { @@ -283,7 +248,6 @@ vm_paddr_t pa; int bit, error, i, retry_count; uint32_t pmapsize; - uint64_t bits; struct minidumphdr mdhdr; retry_count = 0; @@ -307,24 +271,14 @@ /* Calculate dump size */ dumpsize = PAGE_SIZE; /* header */ dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); dumpsize += pmapsize; - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul<msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = VM_MIN_KERNEL_ADDRESS; mdhdr.kernend = VM_MAX_SAFE_KERNEL_ADDRESS; @@ -369,10 +323,10 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; - dump_total("bitmap", round_page(vm_page_dump_size)); + dump_total("bitmap", round_page(BITSET_SIZE(vm_page_dump_pages))); /* Dump kernel page directory pages */ error = dump_pmap(di); @@ -381,20 +335,10 @@ dump_total("pmap", pmapsize); /* Dump memory chunks */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - /* TODO optimize with bit manipulation instructions */ - if (bits == 0) - continue; - for (bit = 0; bit < 64; bit++) { - if ((bits & (1ul< CTASSERT(sizeof(struct kerneldumpheader) == 512); -CTASSERT(sizeof(*vm_page_dump) == 8); - -uint64_t *vm_page_dump; -int vm_page_dump_size; static struct kerneldumpheader kdh; @@ -221,7 +217,6 @@ vm_offset_t va; vm_paddr_t pa; int error; - uint64_t bits; int i, bit; int retry_count; @@ -262,20 +257,13 @@ /* Calculate dump size */ dumpsize = pmapsize; dumpsize += round_page(msgbufp->msg_size); - dumpsize += round_page(vm_page_dump_size); - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - /* Clear out undumpable pages now if needed */ - if (is_dumpable(pa)) - dumpsize += PAGE_SIZE; - else - dump_drop_page(pa); - bits &= ~(1ul << bit); - } + dumpsize += round_page(BITSET_SIZE(vm_page_dump_pages)); + VM_PAGE_DUMP_FOREACH(bit, pa) { + /* Clear out undumpable pages now if needed */ + if (is_dumpable(pa)) + dumpsize += PAGE_SIZE; + else + dump_drop_page(pa); } dumpsize += PAGE_SIZE; @@ -286,7 +274,7 @@ strcpy(mdhdr.magic, MINIDUMP_MAGIC); mdhdr.version = MINIDUMP_VERSION; mdhdr.msgbufsize = msgbufp->msg_size; - mdhdr.bitmapsize = vm_page_dump_size; + mdhdr.bitmapsize = BITSET_SIZE(vm_page_dump_pages); mdhdr.pmapsize = pmapsize; mdhdr.kernbase = KERNBASE; mdhdr.dmapphys = DMAP_MIN_PHYSADDR; @@ -318,7 +306,7 @@ /* Dump bitmap */ error = blk_write(di, (char *)vm_page_dump, 0, - round_page(vm_page_dump_size)); + round_page(BITSET_SIZE(vm_page_dump_pages))); if (error) goto fail; @@ -360,17 +348,10 @@ /* Dump memory chunks */ /* XXX cluster it up and use blk_dump() */ - for (i = 0; i < vm_page_dump_size / sizeof(*vm_page_dump); i++) { - bits = vm_page_dump[i]; - while (bits) { - bit = ffsl(bits) - 1; - pa = (((uint64_t)i * sizeof(*vm_page_dump) * NBBY) + - bit) * PAGE_SIZE; - error = blk_write(di, 0, pa, PAGE_SIZE); - if (error) - goto fail; - bits &= ~(1ul << bit); - } + VM_PAGE_DUMP_FOREACH(bit, pa) { + error = blk_write(di, 0, pa, PAGE_SIZE); + if (error) + goto fail; } error = blk_flush(di); @@ -406,31 +387,3 @@ printf("** DUMP FAILED (ERROR %d) **\n", error); return (error); } - -/* - * Add a page to the minidump bitmap. - */ -void -dump_add_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_set_long(&vm_page_dump[idx], 1ul << bit); -} - -/* - * Remove page from the minidump bitmap. - */ -void -dump_drop_page(vm_paddr_t pa) -{ - int idx, bit; - - pa >>= PAGE_SHIFT; - idx = pa >> 6; /* 2^6 = 64 */ - bit = pa & 63; - atomic_clear_long(&vm_page_dump[idx], 1ul << bit); -} Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -70,6 +70,8 @@ #define _VM_PAGE_ #include +#include +#include /* * Management of resident (logical) pages. @@ -585,6 +587,26 @@ #define PS_ALL_VALID 0x2 #define PS_NONE_BUSY 0x4 +extern struct bitset *vm_page_dump; +extern long vm_page_dump_pages; + +static inline void +dump_add_page(vm_paddr_t pa) +{ + BIT_SET_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +static inline void +dump_drop_page(vm_paddr_t pa) +{ + BIT_CLR_ATOMIC(vm_page_dump_pages, pa >> PAGE_SHIFT, vm_page_dump); +} + +#define VM_PAGE_DUMP_FOREACH(bit, pa) \ + for ((bit) = BIT_FFS(vm_page_dump_pages, vm_page_dump); \ + pa = ((bit) - 1) * PAGE_SIZE, (bit) != 0; \ + (bit) = BIT_FFS_AT(vm_page_dump_pages, vm_page_dump, bit)) \ + bool vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); int vm_page_busy_tryupgrade(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -155,6 +155,9 @@ long vm_page_array_size; long first_page; +struct bitset *vm_page_dump; +long vm_page_dump_pages; + static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | @@ -553,7 +556,7 @@ vm_paddr_t end, high_avail, low_avail, new_end, size; vm_paddr_t page_range __unused; vm_paddr_t last_pa, pa; - u_long pagecount; + u_long pagecount, vm_page_dump_size; int biggestone, i, segind; #ifdef WITNESS vm_offset_t mapped; @@ -588,9 +591,6 @@ witness_startup((void *)mapped); #endif -#if defined(__aarch64__) || defined(__amd64__) || defined(__arm__) || \ - defined(__i386__) || defined(__mips__) || defined(__riscv) || \ - defined(__powerpc64__) /* * Allocate a bitmap to indicate that a random physical page * needs to be included in a minidump. @@ -606,15 +606,13 @@ for (i = 0; dump_avail[i + 1] != 0; i += 2) if (dump_avail[i + 1] > last_pa) last_pa = dump_avail[i + 1]; - page_range = last_pa / PAGE_SIZE; - vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); + vm_page_dump_pages = last_pa / PAGE_SIZE; + vm_page_dump_size = round_page(BITSET_SIZE(vm_page_dump_pages)); new_end -= vm_page_dump_size; vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); bzero((void *)vm_page_dump, vm_page_dump_size); -#else - (void)last_pa; -#endif + #if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) || \ defined(__riscv) || defined(__powerpc64__) /* Index: sys/x86/include/x86_var.h =================================================================== --- sys/x86/include/x86_var.h +++ sys/x86/include/x86_var.h @@ -78,7 +78,6 @@ extern char kstack[]; extern char sigcode[]; extern int szsigcode; -extern int vm_page_dump_size; extern int workaround_erratum383; extern int _udatasel; extern int _ucodesel; @@ -122,8 +121,6 @@ void cpu_setregs(void); bool disable_wp(void); void restore_wp(bool old_wp); -void dump_add_page(vm_paddr_t); -void dump_drop_page(vm_paddr_t); void finishidentcpu(void); void identify_cpu1(void); void identify_cpu2(void);