Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/swap_pager.c
Context not available. | |||||
/* | /* | ||||
* Metadata functions | * Metadata functions | ||||
*/ | */ | ||||
static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); | ||||
static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); | static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); | ||||
static void swp_pager_meta_free_all(vm_object_t); | static void swp_pager_meta_free_all(vm_object_t); | ||||
static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); | static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); | ||||
static void | |||||
swp_pager_init_freerange(daddr_t *start, daddr_t *num) | |||||
{ | |||||
kib: Blank line is needed after '{'. | |||||
*start = SWAPBLK_NONE; | |||||
*num = 0; | |||||
} | |||||
static void | |||||
swp_pager_update_freerange(daddr_t *start, daddr_t *num, daddr_t addr) | |||||
{ | |||||
Done Inline ActionsSame. Also please put {} around then case. kib: Same. Also please put {} around then case. | |||||
if (*start + *num == addr) { | |||||
(*num)++; | |||||
} else { | |||||
swp_pager_freeswapspace(*start, *num); | |||||
*start = addr; | |||||
*num = 1; | |||||
} | |||||
} | |||||
static void * | static void * | ||||
swblk_trie_alloc(struct pctrie *ptree) | swblk_trie_alloc(struct pctrie *ptree) | ||||
{ | { | ||||
Context not available. | |||||
int n = 0; | int n = 0; | ||||
daddr_t blk = SWAPBLK_NONE; | daddr_t blk = SWAPBLK_NONE; | ||||
vm_pindex_t beg = start; /* save start index */ | vm_pindex_t beg = start; /* save start index */ | ||||
daddr_t addr, s_free, n_free; | |||||
swp_pager_init_freerange(&s_free, &n_free); | |||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
while (size) { | while (size) { | ||||
if (n == 0) { | if (n == 0) { | ||||
Context not available. | |||||
} | } | ||||
} | } | ||||
} | } | ||||
swp_pager_meta_build(object, start, blk); | addr = swp_pager_meta_build(object, start, blk); | ||||
if (addr != SWAPBLK_NONE) | |||||
swp_pager_update_freerange(&s_free, &n_free, addr); | |||||
--size; | --size; | ||||
++start; | ++start; | ||||
++blk; | ++blk; | ||||
--n; | --n; | ||||
} | } | ||||
swp_pager_freeswapspace(s_free, n_free); | |||||
swp_pager_meta_free(object, start, n); | swp_pager_meta_free(object, start, n); | ||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
return (0); | return (0); | ||||
Context not available. | |||||
vm_pindex_t offset, int destroysource) | vm_pindex_t offset, int destroysource) | ||||
{ | { | ||||
vm_pindex_t i; | vm_pindex_t i; | ||||
daddr_t dstaddr, first_free, num_free, srcaddr; | daddr_t dstaddr, s_free, n_free, srcaddr; | ||||
VM_OBJECT_ASSERT_WLOCKED(srcobject); | VM_OBJECT_ASSERT_WLOCKED(srcobject); | ||||
VM_OBJECT_ASSERT_WLOCKED(dstobject); | VM_OBJECT_ASSERT_WLOCKED(dstobject); | ||||
Context not available. | |||||
/* | /* | ||||
* Transfer source to destination. | * Transfer source to destination. | ||||
*/ | */ | ||||
first_free = SWAPBLK_NONE; | swp_pager_init_freerange(&s_free, &n_free); | ||||
num_free = 0; | |||||
for (i = 0; i < dstobject->size; ++i) { | for (i = 0; i < dstobject->size; ++i) { | ||||
srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); | srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); | ||||
if (srcaddr == SWAPBLK_NONE) | if (srcaddr == SWAPBLK_NONE) | ||||
continue; | continue; | ||||
dstaddr = swp_pager_meta_ctl(dstobject, i, 0); | dstaddr = swp_pager_meta_ctl(dstobject, i, 0); | ||||
if (dstaddr == SWAPBLK_NONE) { | if (dstaddr != SWAPBLK_NONE) { | ||||
/* | |||||
* Destination has no swapblk and is not resident, | |||||
* copy source. | |||||
* | |||||
* swp_pager_meta_build() can sleep. | |||||
*/ | |||||
vm_object_pip_add(srcobject, 1); | |||||
VM_OBJECT_WUNLOCK(srcobject); | |||||
vm_object_pip_add(dstobject, 1); | |||||
swp_pager_meta_build(dstobject, i, srcaddr); | |||||
vm_object_pip_wakeup(dstobject); | |||||
VM_OBJECT_WLOCK(srcobject); | |||||
vm_object_pip_wakeup(srcobject); | |||||
} else { | |||||
/* | /* | ||||
* Destination has valid swapblk or it is represented | * Destination has valid swapblk or it is represented | ||||
* by a resident page. We destroy the sourceblock. | * by a resident page. We destroy the source block. | ||||
*/ | */ | ||||
if (first_free + num_free == srcaddr) | swp_pager_update_freerange(&s_free, &n_free, srcaddr); | ||||
num_free++; | continue; | ||||
else { | |||||
swp_pager_freeswapspace(first_free, num_free); | |||||
first_free = srcaddr; | |||||
num_free = 1; | |||||
} | |||||
} | } | ||||
/* | |||||
* Destination has no swapblk and is not resident, | |||||
* copy source. | |||||
Not Done Inline ActionsThere is no point in this assignment, unless it is followed by a KASSERT(srcaddr == SWAPBLK_NONE, ...); That said, given that this is the destination object, it might be clearer if dstaddr were used here instead. alc: There is no point in this assignment, unless it is followed by a KASSERT(srcaddr ==… | |||||
* | |||||
* swp_pager_meta_build() can sleep. | |||||
*/ | |||||
vm_object_pip_add(srcobject, 1); | |||||
VM_OBJECT_WUNLOCK(srcobject); | |||||
Not Done Inline ActionsDon't we expect srcaddr to be SWAPBLK_NONE here, i.e., we already determined that there is no existing allocated block? So, this should actually be KASSERT(srcaddr == SWAPBLK_NONE, ... continue; or drop the continue and restore the else. alc: Don't we expect srcaddr to be SWAPBLK_NONE here, i.e., we already determined that there is no… | |||||
Not Done Inline ActionsWe do. I'll take the latter option. dougm: We do. I'll take the latter option. | |||||
vm_object_pip_add(dstobject, 1); | |||||
dstaddr = swp_pager_meta_build(dstobject, i, srcaddr); | |||||
KASSERT(dstaddr == SWAPBLK_NONE, | |||||
("Unexpected destination swapblk")); | |||||
Not Done Inline Actions"source block"? markj: "source block"? | |||||
vm_object_pip_wakeup(dstobject); | |||||
VM_OBJECT_WLOCK(srcobject); | |||||
vm_object_pip_wakeup(srcobject); | |||||
} | } | ||||
swp_pager_freeswapspace(first_free, num_free); | swp_pager_freeswapspace(s_free, n_free); | ||||
/* | /* | ||||
* Free left over swap blocks in source. | * Free left over swap blocks in source. | ||||
Context not available. | |||||
{ | { | ||||
int i, n; | int i, n; | ||||
boolean_t sync; | boolean_t sync; | ||||
daddr_t addr, s_free, n_free; | |||||
swp_pager_init_freerange(&s_free, &n_free); | |||||
if (count && ma[0]->object != object) { | if (count && ma[0]->object != object) { | ||||
panic("swap_pager_putpages: object mismatch %p/%p", | panic("swap_pager_putpages: object mismatch %p/%p", | ||||
object, | object, | ||||
Context not available. | |||||
* check for bogus sysops | * check for bogus sysops | ||||
* force sync if not pageout process | * force sync if not pageout process | ||||
*/ | */ | ||||
if (object->type != OBJT_SWAP) | if (object->type != OBJT_SWAP) { | ||||
swp_pager_meta_build(object, 0, SWAPBLK_NONE); | addr = swp_pager_meta_build(object, 0, SWAPBLK_NONE); | ||||
if (addr != SWAPBLK_NONE) | |||||
swp_pager_update_freerange(&s_free, &n_free, addr); | |||||
alcUnsubmitted Not Done Inline ActionsThis should also be a KASSERT(addr == SWAPBLK_NONE, ... alc: This should also be a KASSERT(addr == SWAPBLK_NONE, ... | |||||
} | |||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
n = 0; | n = 0; | ||||
Done Inline ActionsDe-split this line, please. There is a lot of free space left, the call does not need to take 5 lines. kib: De-split this line, please. There is a lot of free space left, the call does not need to take… | |||||
Context not available. | |||||
for (j = 0; j < n; ++j) { | for (j = 0; j < n; ++j) { | ||||
vm_page_t mreq = ma[i+j]; | vm_page_t mreq = ma[i+j]; | ||||
swp_pager_meta_build( | addr = swp_pager_meta_build(mreq->object, mreq->pindex, | ||||
mreq->object, | blk + j); | ||||
mreq->pindex, | if (addr != SWAPBLK_NONE) | ||||
blk + j | swp_pager_update_freerange(&s_free, &n_free, | ||||
); | addr); | ||||
MPASS(mreq->dirty == VM_PAGE_BITS_ALL); | MPASS(mreq->dirty == VM_PAGE_BITS_ALL); | ||||
mreq->oflags |= VPO_SWAPINPROG; | mreq->oflags |= VPO_SWAPINPROG; | ||||
bp->b_pages[j] = mreq; | bp->b_pages[j] = mreq; | ||||
Context not available. | |||||
swp_pager_async_iodone(bp); | swp_pager_async_iodone(bp); | ||||
} | } | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
swp_pager_freeswapspace(s_free, n_free); | |||||
} | } | ||||
/* | /* | ||||
Context not available. | |||||
* | * | ||||
* The specified swapblk is added to the object's swap metadata. If | * The specified swapblk is added to the object's swap metadata. If | ||||
* the swapblk is not valid, it is freed instead. Any previously | * the swapblk is not valid, it is freed instead. Any previously | ||||
* assigned swapblk is freed. | * assigned swapblk is returned. | ||||
*/ | */ | ||||
static void | static daddr_t | ||||
swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) | swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) | ||||
{ | { | ||||
static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; | static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; | ||||
struct swblk *sb, *sb1; | struct swblk *sb, *sb1; | ||||
vm_pindex_t modpi, rdpi; | vm_pindex_t modpi, rdpi; | ||||
daddr_t prev_swapblk; | |||||
int error, i; | int error, i; | ||||
VM_OBJECT_ASSERT_WLOCKED(object); | VM_OBJECT_ASSERT_WLOCKED(object); | ||||
Context not available. | |||||
sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); | sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); | ||||
if (sb == NULL) { | if (sb == NULL) { | ||||
if (swapblk == SWAPBLK_NONE) | if (swapblk == SWAPBLK_NONE) | ||||
return; | return (SWAPBLK_NONE); | ||||
for (;;) { | for (;;) { | ||||
sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == | sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == | ||||
pageproc ? M_USE_RESERVE : 0)); | pageproc ? M_USE_RESERVE : 0)); | ||||
Context not available. | |||||
MPASS(sb->p == rdpi); | MPASS(sb->p == rdpi); | ||||
modpi = pindex % SWAP_META_PAGES; | modpi = pindex % SWAP_META_PAGES; | ||||
/* Delete prior contents of metadata. */ | /* Return prior contents of metadata. */ | ||||
if (sb->d[modpi] != SWAPBLK_NONE) | prev_swapblk = sb->d[modpi]; | ||||
swp_pager_freeswapspace(sb->d[modpi], 1); | |||||
/* Enter block into metadata. */ | /* Enter block into metadata. */ | ||||
sb->d[modpi] = swapblk; | sb->d[modpi] = swapblk; | ||||
Context not available. | |||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); | SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi); | ||||
uma_zfree(swblk_zone, sb); | uma_zfree(swblk_zone, sb); | ||||
} | } | ||||
return (prev_swapblk); | |||||
} | } | ||||
/* | /* | ||||
Context not available. | |||||
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) | swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) | ||||
{ | { | ||||
struct swblk *sb; | struct swblk *sb; | ||||
daddr_t first_free, num_free; | daddr_t s_free, n_free; | ||||
vm_pindex_t last; | vm_pindex_t last; | ||||
int i, limit, start; | int i, limit, start; | ||||
Context not available. | |||||
if (object->type != OBJT_SWAP || count == 0) | if (object->type != OBJT_SWAP || count == 0) | ||||
return; | return; | ||||
first_free = SWAPBLK_NONE; | swp_pager_init_freerange(&s_free, &n_free); | ||||
num_free = 0; | |||||
last = pindex + count; | last = pindex + count; | ||||
for (;;) { | for (;;) { | ||||
sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, | sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, | ||||
Context not available. | |||||
for (i = start; i < limit; i++) { | for (i = start; i < limit; i++) { | ||||
if (sb->d[i] == SWAPBLK_NONE) | if (sb->d[i] == SWAPBLK_NONE) | ||||
continue; | continue; | ||||
if (first_free + num_free == sb->d[i]) | swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); | ||||
num_free++; | |||||
else { | |||||
swp_pager_freeswapspace(first_free, num_free); | |||||
first_free = sb->d[i]; | |||||
num_free = 1; | |||||
} | |||||
sb->d[i] = SWAPBLK_NONE; | sb->d[i] = SWAPBLK_NONE; | ||||
} | } | ||||
if (swp_pager_swblk_empty(sb, 0, start) && | if (swp_pager_swblk_empty(sb, 0, start) && | ||||
Context not available. | |||||
} | } | ||||
pindex = sb->p + SWAP_META_PAGES; | pindex = sb->p + SWAP_META_PAGES; | ||||
} | } | ||||
swp_pager_freeswapspace(first_free, num_free); | swp_pager_freeswapspace(s_free, n_free); | ||||
} | } | ||||
/* | /* | ||||
Context not available. | |||||
swp_pager_meta_free_all(vm_object_t object) | swp_pager_meta_free_all(vm_object_t object) | ||||
{ | { | ||||
struct swblk *sb; | struct swblk *sb; | ||||
daddr_t first_free, num_free; | daddr_t s_free, n_free; | ||||
vm_pindex_t pindex; | vm_pindex_t pindex; | ||||
int i; | int i; | ||||
Context not available. | |||||
if (object->type != OBJT_SWAP) | if (object->type != OBJT_SWAP) | ||||
return; | return; | ||||
first_free = SWAPBLK_NONE; | swp_pager_init_freerange(&s_free, &n_free); | ||||
num_free = 0; | |||||
for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( | for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE( | ||||
&object->un_pager.swp.swp_blks, pindex)) != NULL;) { | &object->un_pager.swp.swp_blks, pindex)) != NULL;) { | ||||
pindex = sb->p + SWAP_META_PAGES; | pindex = sb->p + SWAP_META_PAGES; | ||||
for (i = 0; i < SWAP_META_PAGES; i++) { | for (i = 0; i < SWAP_META_PAGES; i++) { | ||||
if (sb->d[i] == SWAPBLK_NONE) | if (sb->d[i] == SWAPBLK_NONE) | ||||
continue; | continue; | ||||
if (first_free + num_free == sb->d[i]) | swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); | ||||
num_free++; | |||||
else { | |||||
swp_pager_freeswapspace(first_free, num_free); | |||||
first_free = sb->d[i]; | |||||
num_free = 1; | |||||
} | |||||
} | } | ||||
SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); | SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p); | ||||
uma_zfree(swblk_zone, sb); | uma_zfree(swblk_zone, sb); | ||||
} | } | ||||
swp_pager_freeswapspace(first_free, num_free); | swp_pager_freeswapspace(s_free, n_free); | ||||
} | } | ||||
/* | /* | ||||
Context not available. |
Blank line is needed after '{'.