Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c +++ sys/vm/vm_reserv.c @@ -87,6 +87,13 @@ #define VM_LEVEL_0_NPAGES (1 << VM_LEVEL_0_ORDER) #define VM_LEVEL_0_NPAGES_MAX (1 << VM_LEVEL_0_ORDER_MAX) +/* + * The number of small pages that are contained in a level 1 reservation + */ +#if VM_NRESERVLEVEL > 1 +#define VM_LEVEL_1_NPAGES (1 << VM_LEVEL_1_ORDER) +#endif + /* * The number of bits by which a physical address is shifted to obtain the * reservation number @@ -195,6 +202,10 @@ vm_pindex_t pindex; /* (o, r) offset in object */ vm_page_t pages; /* (c) first page */ uint16_t popcnt; /* (r) # of pages in use */ +#if VM_NRESERVLEVEL > 1 + uint8_t blkcnt[VM_LEVEL_0_NPAGES / + VM_LEVEL_1_NPAGES]/* (r) # of pages in use per block */ +#endif uint8_t domain; /* (c) NUMA domain. */ char inpartpopq; /* (d, r) */ int lasttick; /* (r) last pop update tick. */ @@ -462,14 +473,23 @@ KASSERT(rv->domain < vm_ndomains, ("vm_reserv_depopulate: reserv %p's domain is corrupted %d", rv, rv->domain)); - if (rv->popcnt == VM_LEVEL_0_NPAGES) { - KASSERT(rv->pages->psind == 1, + if (rv->popcnt-- == VM_LEVEL_0_NPAGES) { + KASSERT(rv->pages->psind == VM_NRESERVLEVEL, ("vm_reserv_depopulate: reserv %p is already demoted", rv)); - rv->pages->psind = 0; + --rv->pages->psind; } popmap_clear(rv->popmap, index); - rv->popcnt--; +#if VM_NRESERVLEVEL > 1 + index /= VM_LEVEL_1_NPAGES; + if (rv->blkcnt[index]-- == VM_LEVEL_1_NPAGES) { + index *= VM_LEVEL_1_PAGES; + KASSERT(rv->pages[index].psind == VM_NRESERVLEVEL - 1, + ("vm_reserv_depopulate: reserv %p[%d] is already demoted", + rv, index)); + --rv->pages[index].psind; + } +#endif if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || rv->popcnt == 0) { vm_reserv_domain_lock(rv->domain); @@ -585,6 +605,16 @@ ("vm_reserv_populate: reserv %p's domain is corrupted %d", rv, rv->domain)); popmap_set(rv->popmap, index); +#if VM_NRESERVLEVEL > 1 + index /= VM_LEVEL_1_NPAGES; + if (++rv->blkcnt[index] == VM_LEVEL_1_NPAGES) { + index *= VM_LEVEL_1_PAGES; + KASSERT(rv->pages[index].psind == VM_NRESERVLEVEL - 2, + ("vm_reserv_populate: reserv %p[%d] is already promoted", + rv, index)); + ++rv->pages[index].psind; + } +#endif rv->popcnt++; if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) @@ -593,16 +623,16 @@ vm_reserv_domain_lock(rv->domain); if (rv->inpartpopq) { TAILQ_REMOVE(&vm_rvd[rv->domain].partpop, rv, partpopq); - rv->inpartpopq = FALSE; } - if (rv->popcnt < VM_LEVEL_0_NPAGES) { - rv->inpartpopq = TRUE; - TAILQ_INSERT_TAIL(&vm_rvd[rv->domain].partpop, rv, partpopq); - } else { - KASSERT(rv->pages->psind == 0, + if (rv->popcnt == VM_LEVEL_0_NPAGES) { + rv->inpartpopq = FALSE; + KASSERT(rv->pages->psind == VM_NRESERVLEVEL - 1, ("vm_reserv_populate: reserv %p is already promoted", rv)); - rv->pages->psind = 1; + ++rv->pages->psind; + } else { + rv->inpartpopq = TRUE; + TAILQ_INSERT_TAIL(&vm_rvd[rv->domain].partpop, rv, partpopq); } vm_reserv_domain_unlock(rv->domain); } @@ -993,6 +1023,10 @@ } } rv->popcnt = 0; +#if VM_NRESERVLEVEL > 1 + for (i = 0; i < nitems(rv->blkcnt); i++) + rv->blkcnt[i] = 0; +#endif counter_u64_add(vm_reserv_broken, 1); } @@ -1114,6 +1148,10 @@ rvd->marker.popcnt = VM_LEVEL_0_NPAGES; for (j = 0; j < VM_LEVEL_0_NPAGES; j++) popmap_set(rvd->marker.popmap, j); +#if VM_NRESERVLEVEL > 1 + for (j = 0; j < nitems(rv->blkcnt); j++) + rv->blkcnt[j] = VM_LEVEL_1_PAGES; +#endif } for (i = 0; i < VM_RESERV_OBJ_LOCK_COUNT; i++) @@ -1162,6 +1200,7 @@ return (rv->popcnt == VM_LEVEL_0_NPAGES ? 0 : -1); } +#if VM_NRESERVLEVEL > 1 /* * XXX */ @@ -1172,8 +1211,10 @@ rv = vm_reserv_from_page(m); return (rv->object != NULL && - ((uint16_t *)rv->popmap)[(m - rv->pages) / 16] == 65535); + rv->blkcnt[(m - rv->pages) / VM_LEVEL_1_NPAGES] == + VM_LEVEL_1_NPAGES); } +#endif /* * Remove a partially populated reservation from the queue.