Changeset View
Standalone View
sys/vm/swap_pager.c
Show First 20 Lines • Show All 394 Lines • ▼ Show 20 Lines | |||||
static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | static int nswap_hiwat = 512; /* in pages, swap_pager_almost_full warn */ | ||||
SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, | SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0, | ||||
"Maximum size of a swap block in pages"); | "Maximum size of a swap block in pages"); | ||||
static void swp_sizecheck(void); | static void swp_sizecheck(void); | ||||
static void swp_pager_async_iodone(struct buf *bp); | static void swp_pager_async_iodone(struct buf *bp); | ||||
static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit); | static bool swp_pager_swblk_empty(struct swblk *sb, int start, int limit); | ||||
static int swapongeom(struct vnode *); | static int swapongeom(struct vnode *, int flags, int priority); | ||||
static int swaponvp(struct thread *, struct vnode *, u_long); | static int swaponvp(struct thread *, struct vnode *, u_long nblks, | ||||
int flags, int priority); | |||||
static int swapoff_one(struct swdevt *sp, struct ucred *cred); | static int swapoff_one(struct swdevt *sp, struct ucred *cred); | ||||
/* | /* | ||||
* Swap bitmap functions | * Swap bitmap functions | ||||
*/ | */ | ||||
static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); | static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); | ||||
static daddr_t swp_pager_getswapspace(int *npages, int limit); | static daddr_t swp_pager_getswapspace(int *npages, int limit); | ||||
▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Lines | swp_pager_getswapspace(int *io_npages, int limit) | ||||
blk = SWAPBLK_NONE; | blk = SWAPBLK_NONE; | ||||
mpages = *io_npages; | mpages = *io_npages; | ||||
npages = imin(BLIST_MAX_ALLOC, mpages); | npages = imin(BLIST_MAX_ALLOC, mpages); | ||||
mtx_lock(&sw_dev_mtx); | mtx_lock(&sw_dev_mtx); | ||||
sp = swdevhd; | sp = swdevhd; | ||||
while (!TAILQ_EMPTY(&swtailq)) { | while (!TAILQ_EMPTY(&swtailq)) { | ||||
if (sp == NULL) | if (sp == NULL) | ||||
sp = TAILQ_FIRST(&swtailq); | sp = TAILQ_FIRST(&swtailq); | ||||
if ((sp->sw_flags & SW_CLOSING) == 0) | if ((sp->sw_flags & SW_CLOSING) == 0) | ||||
alc: This is problematic because the cursors are 64-bit variables while npages is a 32-bit variable. | |||||
blk = blist_alloc(sp->sw_blist, &npages, mpages); | blk = blist_alloc(sp->sw_blist, &sp->sw_cursor, | ||||
&npages, mpages); | |||||
if (blk != SWAPBLK_NONE) | if (blk != SWAPBLK_NONE) | ||||
break; | break; | ||||
sp = TAILQ_NEXT(sp, sw_list); | sp = TAILQ_NEXT(sp, sw_list); | ||||
if (swdevhd == sp) { | if (swdevhd == sp) { | ||||
if (npages <= limit) | if (npages <= limit) | ||||
break; | break; | ||||
Not Done Inline ActionsTake a look at ffs_blkfree_sendtrim() in ufs/ffs/ffs_alloc.c alc: Take a look at ffs_blkfree_sendtrim() in ufs/ffs/ffs_alloc.c | |||||
mpages = npages - 1; | mpages = npages - 1; | ||||
npages >>= 1; | npages >>= 1; | ||||
} | } | ||||
Not Done Inline ActionsI think that you are familiar with a pthread_cond_wait(). This is similar, in particular, the mutex must be held by the caller. And that mutex is being used to synchronize access to nsw_wcount_async. alc: I think that you are familiar with a pthread_cond_wait(). This is similar, in particular, the… | |||||
} | } | ||||
if (blk != SWAPBLK_NONE) { | if (blk != SWAPBLK_NONE) { | ||||
*io_npages = npages; | *io_npages = npages; | ||||
blk += sp->sw_first; | blk += sp->sw_first; | ||||
sp->sw_used += npages; | sp->sw_used += npages; | ||||
swap_pager_avail -= npages; | swap_pager_avail -= npages; | ||||
swp_sizecheck(); | swp_sizecheck(); | ||||
swdevhd = TAILQ_NEXT(sp, sw_list); | swdevhd = TAILQ_NEXT(sp, sw_list); | ||||
} else { | } else { | ||||
if (swap_pager_full != 2) { | if (swap_pager_full != 2) { | ||||
printf("swp_pager_getswapspace(%d): failed\n", | printf("swp_pager_getswapspace(%d): failed\n", | ||||
*io_npages); | *io_npages); | ||||
swap_pager_full = 2; | swap_pager_full = 2; | ||||
swap_pager_almost_full = 1; | swap_pager_almost_full = 1; | ||||
} | } | ||||
swdevhd = NULL; | swdevhd = NULL; | ||||
} | } | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
return (blk); | return (blk); | ||||
Not Done Inline ActionsYou can't perform an M_WAITOK allocation when a mutex is held. A mutex is held when this function is called by swap_pager_reserve(). alc: You can't perform an M_WAITOK allocation when a mutex is held. A mutex is held when this… | |||||
} | } | ||||
static bool | static bool | ||||
swp_pager_isondev(daddr_t blk, struct swdevt *sp) | swp_pager_isondev(daddr_t blk, struct swdevt *sp) | ||||
{ | { | ||||
return (blk >= sp->sw_first && blk < sp->sw_end); | return (blk >= sp->sw_first && blk < sp->sw_end); | ||||
} | } | ||||
Show All 18 Lines | if (swp_pager_isondev(bp->b_blkno, sp)) { | ||||
sp->sw_strategy(bp, sp); | sp->sw_strategy(bp, sp); | ||||
return; | return; | ||||
} | } | ||||
} | } | ||||
panic("Swapdev not found"); | panic("Swapdev not found"); | ||||
} | } | ||||
static void | |||||
swp_pager_async_trimdone(struct buf *bp) | |||||
{ | |||||
struct swdevt *sp; | |||||
daddr_t blk; | |||||
int npages; | |||||
sp = (struct swdevt *)bp->b_fsprivate1; | |||||
blk = bp->b_blkno; | |||||
npages = bp->b_npages; | |||||
uma_zfree(swwbuf_zone, bp); | |||||
mtx_lock(&swbuf_mtx); | |||||
if (++nsw_wcount_async == 1) | |||||
wakeup(&nsw_wcount_async); | |||||
mtx_unlock(&swbuf_mtx); | |||||
mtx_lock(&sw_dev_mtx); | |||||
blk -= sp->sw_first; | |||||
blist_free(sp->sw_blist, blk, npages); | |||||
mtx_unlock(&sw_dev_mtx); | |||||
} | |||||
static int trimzone_numer = 1; | |||||
static int trimzone_denom = 8; | |||||
static int min_trim_alloc = 1; | |||||
static int max_trim_alloc = 32768; | |||||
/* | /* | ||||
* The 'trim zone' is the small range of addresses just ahead of the cursor | |||||
* likely to be allocated soon. When the trimmer falls into the trim zone, | |||||
* we allocate blocks with the trimmer. | |||||
*/ | |||||
static bool | |||||
swapdev_in_trim_zone(daddr_t start, daddr_t end, daddr_t nblks) | |||||
{ | |||||
return ((end + nblks - start) % nblks / trimzone_numer < | |||||
nblks / trimzone_denom); | |||||
} | |||||
/* sw_dev_mtx lock is held. */ | |||||
static struct buf * | |||||
swapdev_trim(struct swdevt *sp) | |||||
{ | |||||
struct buf *bp; | |||||
daddr_t blk; | |||||
u_long nblks; | |||||
int npages; | |||||
/* Quit if trimming is disabled. */ | |||||
if ((sp->sw_flags & SW_TRIM) == 0) | |||||
return (NULL); | |||||
/* Quit if the cursor is too far behind the trimmer. */ | |||||
nblks = sp->sw_nblks; | |||||
if (!swapdev_in_trim_zone(sp->sw_cursor, sp->sw_trimmer, nblks)) | |||||
return (NULL); | |||||
/* Grab a (hopefully) lot of free space allocated long ago. */ | |||||
npages = min_trim_alloc; | |||||
blk = blist_alloc(sp->sw_blist, &sp->sw_trimmer, | |||||
&npages, max_trim_alloc); | |||||
/* Quit if there's nothing free. */ | |||||
if (blk == SWAPBLK_NONE) | |||||
return (NULL); | |||||
if (swapdev_in_trim_zone(sp->sw_cursor, sp->sw_trimmer, nblks)) { | |||||
/* | |||||
* Trim allocation is too close to the cursor. Either we've | |||||
* wrapped around and jumped the cursor, or we've found too | |||||
* little free space close to the cursor to move the trimmer | |||||
* out of the trim zone. Bail out. | |||||
* | |||||
*/ | |||||
CTR5(KTR_SPARE5, "%s: cursor %p blk %p size %5d trimmer %p, too close", | |||||
__func__, | |||||
(void*)sp->sw_cursor, (void*)blk, npages, (void*)sp->sw_trimmer); | |||||
blist_free(sp->sw_blist, blk, npages); | |||||
return (NULL); | |||||
} | |||||
mtx_unlock(&sw_dev_mtx); | |||||
mtx_lock(&swbuf_mtx); | |||||
if (nsw_wcount_async == 0) { | |||||
mtx_unlock(&swbuf_mtx); | |||||
CTR5(KTR_SPARE5, "%s: cursor %p blk %p size %5d trimmer %p, low count", | |||||
__func__, | |||||
(void*)sp->sw_cursor, (void*)blk, npages, (void*)sp->sw_trimmer); | |||||
mtx_lock(&sw_dev_mtx); | |||||
blist_free(sp->sw_blist, blk, npages); | |||||
sp->sw_trimmer = blk; | |||||
return (NULL); | |||||
} | |||||
nsw_wcount_async--; | |||||
mtx_unlock(&swbuf_mtx); | |||||
bp = uma_zalloc(swwbuf_zone, M_NOWAIT); | |||||
if (bp == NULL) { | |||||
CTR5(KTR_SPARE5, "%s: cursor %p blk %p size %5d trimmer %p, no buf", | |||||
__func__, | |||||
(void*)sp->sw_cursor, (void*)blk, npages, (void*)sp->sw_trimmer); | |||||
mtx_lock(&sw_dev_mtx); | |||||
blist_free(sp->sw_blist, blk, npages); | |||||
sp->sw_trimmer = blk; | |||||
return (NULL); | |||||
} | |||||
CTR5(KTR_SPARE5, "%s: cursor %p blk %p size %5d trimmer %p, start trim", | |||||
__func__, | |||||
(void*)sp->sw_cursor, (void*)blk, npages, (void*)sp->sw_trimmer); | |||||
bp->b_flags = B_ASYNC; | |||||
bp->b_data = NULL; | |||||
bp->b_iocmd = BIO_DELETE; | |||||
bp->b_rcred = crhold(thread0.td_ucred); | |||||
bp->b_wcred = crhold(thread0.td_ucred); | |||||
bp->b_bcount = PAGE_SIZE * npages; | |||||
Not Done Inline Actionssw_dev_mtx isn't being released. alc: sw_dev_mtx isn't being released. | |||||
Done Inline Actionsswapdev_trim releases it if it returns a non-NULL bp. sw_strategy -> swapgeom_strategy acquires and releases it. dougm: swapdev_trim releases it if it returns a non-NULL bp.
sw_strategy -> swapgeom_strategy… | |||||
bp->b_bufsize = PAGE_SIZE * npages; | |||||
bp->b_blkno = blk + sp->sw_first; | |||||
bp->b_npages = npages; | |||||
bp->b_iodone = swp_pager_async_trimdone; | |||||
bp->b_fsprivate1 = sp; | |||||
BUF_KERNPROC(bp); | |||||
return (bp); | |||||
} | |||||
/* find a swap device that needs trimming, and start trimming */ | |||||
static void | |||||
swp_pager_trimswapspace(void) | |||||
{ | |||||
struct swdevt *sp; | |||||
struct buf *bp; | |||||
mtx_lock(&sw_dev_mtx); | |||||
TAILQ_FOREACH(sp, &swtailq, sw_list) { | |||||
bp = swapdev_trim(sp); | |||||
if (bp == NULL) | |||||
continue; | |||||
/* sw_dev_mtx released in swapdev_trim */ | |||||
sp->sw_strategy(bp,sp); | |||||
return; | |||||
} | |||||
mtx_unlock(&sw_dev_mtx); | |||||
} | |||||
/* | |||||
* SWP_PAGER_FREESWAPSPACE() - free raw swap space | * SWP_PAGER_FREESWAPSPACE() - free raw swap space | ||||
* | * | ||||
* This routine returns the specified swap blocks back to the bitmap. | * This routine returns the specified swap blocks back to the bitmap. | ||||
* | * | ||||
* This routine may not sleep. | * This routine may not sleep. | ||||
*/ | */ | ||||
static void | static void | ||||
swp_pager_freeswapspace(daddr_t blk, daddr_t npages) | swp_pager_freeswapspace(daddr_t blk, daddr_t npages) | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) | swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) | ||||
{ | { | ||||
daddr_t addr, blk, n_free, s_free; | daddr_t addr, blk, n_free, s_free; | ||||
int i, j, n; | int i, j, n; | ||||
swp_pager_init_freerange(&s_free, &n_free); | swp_pager_init_freerange(&s_free, &n_free); | ||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
for (i = 0; i < size; i += n) { | for (i = 0; i < size; i += n) { | ||||
Not Done Inline ActionsIf swap space is fragmented, a call that allocates a moderate amount of swap space will overflow the stack and crash the kernel. alc: If swap space is fragmented, a call that allocates a moderate amount of swap space will… | |||||
n = size - i; | n = size - i; | ||||
blk = swp_pager_getswapspace(&n, 1); | blk = swp_pager_getswapspace(&n, 1); | ||||
if (blk == SWAPBLK_NONE) { | if (blk == SWAPBLK_NONE) { | ||||
swp_pager_meta_free(object, start, i); | swp_pager_meta_free(object, start, i); | ||||
VM_OBJECT_WUNLOCK(object); | break; | ||||
Not Done Inline ActionsThis actually looks like a bug fix. Specifically, it addresses an unlikely, but hypothetically possible, swap space leak. Please create a separate change for this fix. alc: This actually looks like a bug fix. Specifically, it addresses an unlikely, but hypothetically… | |||||
return (-1); | |||||
} | } | ||||
for (j = 0; j < n; ++j) { | for (j = 0; j < n; ++j) { | ||||
addr = swp_pager_meta_build(object, | addr = swp_pager_meta_build(object, | ||||
start + i + j, blk + j); | start + i + j, blk + j); | ||||
if (addr != SWAPBLK_NONE) | if (addr != SWAPBLK_NONE) | ||||
swp_pager_update_freerange(&s_free, &n_free, | swp_pager_update_freerange(&s_free, &n_free, | ||||
addr); | addr); | ||||
} | } | ||||
} | } | ||||
swp_pager_freeswapspace(s_free, n_free); | |||||
VM_OBJECT_WUNLOCK(object); | VM_OBJECT_WUNLOCK(object); | ||||
return (0); | swp_pager_freeswapspace(s_free, n_free); | ||||
swp_pager_trimswapspace(); | |||||
return (i == size ? 0 : -1); | |||||
} | } | ||||
/* | /* | ||||
* SWAP_PAGER_COPY() - copy blocks from source pager to destination pager | * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager | ||||
* and destroy the source. | * and destroy the source. | ||||
* | * | ||||
* Copy any valid swapblks from the source to the destination. In | * Copy any valid swapblks from the source to the destination. In | ||||
* cases where both the source and destination have a valid swapblk, | * cases where both the source and destination have a valid swapblk, | ||||
▲ Show 20 Lines • Show All 533 Lines • ▼ Show 20 Lines | for (i = 0; i < count; i += n) { | ||||
/* | /* | ||||
* Now that we are through with the bp, we can call the | * Now that we are through with the bp, we can call the | ||||
* normal async completion, which frees everything up. | * normal async completion, which frees everything up. | ||||
*/ | */ | ||||
swp_pager_async_iodone(bp); | swp_pager_async_iodone(bp); | ||||
} | } | ||||
swp_pager_freeswapspace(s_free, n_free); | swp_pager_freeswapspace(s_free, n_free); | ||||
swp_pager_trimswapspace(); | |||||
VM_OBJECT_WLOCK(object); | VM_OBJECT_WLOCK(object); | ||||
} | } | ||||
/* | /* | ||||
* swp_pager_async_iodone: | * swp_pager_async_iodone: | ||||
* | * | ||||
* Completion routine for asynchronous reads and writes from/to swap. | * Completion routine for asynchronous reads and writes from/to swap. | ||||
* Also called manually by synchronous code to finish up a bp. | * Also called manually by synchronous code to finish up a bp. | ||||
▲ Show 20 Lines • Show All 691 Lines • ▼ Show 20 Lines | |||||
* if already swapping on this device. | * if already swapping on this device. | ||||
*/ | */ | ||||
#ifndef _SYS_SYSPROTO_H_ | #ifndef _SYS_SYSPROTO_H_ | ||||
struct swapon_args { | struct swapon_args { | ||||
char *name; | char *name; | ||||
}; | }; | ||||
#endif | #endif | ||||
/* | static int | ||||
* MPSAFE | kern_swapon(struct thread *td, const char *name, int flags, int priority) | ||||
*/ | |||||
/* ARGSUSED */ | |||||
int | |||||
sys_swapon(struct thread *td, struct swapon_args *uap) | |||||
{ | { | ||||
struct vattr attr; | struct vattr attr; | ||||
struct vnode *vp; | struct vnode *vp; | ||||
struct nameidata nd; | struct nameidata nd; | ||||
int error; | int error; | ||||
error = priv_check(td, PRIV_SWAPON); | error = priv_check(td, PRIV_SWAPON); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
sx_xlock(&swdev_syscall_lock); | sx_xlock(&swdev_syscall_lock); | ||||
/* | /* | ||||
* Swap metadata may not fit in the KVM if we have physical | * Swap metadata may not fit in the KVM if we have physical | ||||
* memory of >1GB. | * memory of >1GB. | ||||
*/ | */ | ||||
if (swblk_zone == NULL) { | if (swblk_zone == NULL) { | ||||
error = ENOMEM; | error = ENOMEM; | ||||
goto done; | goto done; | ||||
} | } | ||||
NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE, | NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE, | ||||
uap->name, td); | name, td); | ||||
error = namei(&nd); | error = namei(&nd); | ||||
if (error) | if (error) | ||||
goto done; | goto done; | ||||
NDFREE(&nd, NDF_ONLY_PNBUF); | NDFREE(&nd, NDF_ONLY_PNBUF); | ||||
vp = nd.ni_vp; | vp = nd.ni_vp; | ||||
if (vn_isdisk(vp, &error)) { | if (vn_isdisk(vp, &error)) { | ||||
error = swapongeom(vp); | error = swapongeom(vp, flags, priority); | ||||
} else if (vp->v_type == VREG && | } else if (vp->v_type == VREG && | ||||
(vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 && | (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 && | ||||
(error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) { | (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) { | ||||
/* | /* | ||||
* Allow direct swapping to NFS regular files in the same | * Allow direct swapping to NFS regular files in the same | ||||
* way that nfs_mountroot() sets up diskless swapping. | * way that nfs_mountroot() sets up diskless swapping. | ||||
*/ | */ | ||||
error = swaponvp(td, vp, attr.va_size / DEV_BSIZE); | error = swaponvp(td, vp, attr.va_size / DEV_BSIZE, | ||||
flags, priority); | |||||
} | } | ||||
if (error) | if (error) | ||||
vrele(vp); | vrele(vp); | ||||
done: | done: | ||||
sx_xunlock(&swdev_syscall_lock); | sx_xunlock(&swdev_syscall_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* ARGSUSED */ | |||||
int | |||||
sys_swapon(struct thread *td, struct swapon_args *uap) | |||||
{ | |||||
return kern_swapon(td, uap->name, SW_TRIM, 0); | |||||
} | |||||
/* | /* | ||||
* Check that the total amount of swap currently configured does not | * Check that the total amount of swap currently configured does not | ||||
* exceed half the theoretical maximum. If it does, print a warning | * exceed half the theoretical maximum. If it does, print a warning | ||||
* message. | * message. | ||||
*/ | */ | ||||
static void | static void | ||||
swapon_check_swzone(void) | swapon_check_swzone(void) | ||||
{ | { | ||||
Show All 10 Lines | printf("warning: total configured swap (%lu pages) " | ||||
npages, maxpages / 2); | npages, maxpages / 2); | ||||
printf("warning: increase kern.maxswzone " | printf("warning: increase kern.maxswzone " | ||||
"or reduce amount of swap.\n"); | "or reduce amount of swap.\n"); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
swaponsomething(struct vnode *vp, void *id, u_long nblks, | swaponsomething(struct vnode *vp, void *id, u_long nblks, | ||||
sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags) | sw_strategy_t *strategy, sw_close_t *close, dev_t dev, | ||||
int flags, int priority) | |||||
{ | { | ||||
struct swdevt *sp, *tsp; | struct swdevt *sp, *tsp; | ||||
swblk_t dvbase; | swblk_t dvbase; | ||||
u_long mblocks; | u_long mblocks; | ||||
/* | /* | ||||
* nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. | * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks. | ||||
* First chop nblks off to page-align it, then convert. | * First chop nblks off to page-align it, then convert. | ||||
Show All 19 Lines | swaponsomething(struct vnode *vp, void *id, u_long nblks, | ||||
sp->sw_vp = vp; | sp->sw_vp = vp; | ||||
sp->sw_id = id; | sp->sw_id = id; | ||||
sp->sw_dev = dev; | sp->sw_dev = dev; | ||||
sp->sw_nblks = nblks; | sp->sw_nblks = nblks; | ||||
sp->sw_used = 0; | sp->sw_used = 0; | ||||
sp->sw_strategy = strategy; | sp->sw_strategy = strategy; | ||||
sp->sw_close = close; | sp->sw_close = close; | ||||
sp->sw_flags = flags; | sp->sw_flags = flags; | ||||
sp->sw_priority = priority; | |||||
sp->sw_blist = blist_create(nblks, M_WAITOK); | sp->sw_blist = blist_create(nblks, M_WAITOK); | ||||
sp->sw_cursor = 0; | |||||
sp->sw_trimmer = 0; | |||||
/* | /* | ||||
* Do not free the first two block in order to avoid overwriting | * Do not free the first two block in order to avoid overwriting | ||||
* any bsd label at the front of the partition | * any bsd label at the front of the partition | ||||
*/ | */ | ||||
blist_free(sp->sw_blist, 2, nblks - 2); | blist_free(sp->sw_blist, 2, nblks - 2); | ||||
dvbase = 0; | dvbase = 0; | ||||
mtx_lock(&sw_dev_mtx); | mtx_lock(&sw_dev_mtx); | ||||
▲ Show 20 Lines • Show All 423 Lines • ▼ Show 20 Lines | if (cp == NULL) { | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
bp->b_error = ENXIO; | bp->b_error = ENXIO; | ||||
bp->b_ioflags |= BIO_ERROR; | bp->b_ioflags |= BIO_ERROR; | ||||
bufdone(bp); | bufdone(bp); | ||||
return; | return; | ||||
} | } | ||||
swapgeom_acquire(cp); | swapgeom_acquire(cp); | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
if (bp->b_iocmd == BIO_WRITE) | if (bp->b_iocmd == BIO_WRITE || bp->b_iocmd == BIO_DELETE) | ||||
bio = g_new_bio(); | bio = g_new_bio(); | ||||
else | else | ||||
bio = g_alloc_bio(); | bio = g_alloc_bio(); | ||||
if (bio == NULL) { | if (bio == NULL) { | ||||
mtx_lock(&sw_dev_mtx); | mtx_lock(&sw_dev_mtx); | ||||
swapgeom_release(cp, sp); | swapgeom_release(cp, sp); | ||||
mtx_unlock(&sw_dev_mtx); | mtx_unlock(&sw_dev_mtx); | ||||
bp->b_error = ENOMEM; | bp->b_error = ENOMEM; | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | swapgeom_close(struct thread *td, struct swdevt *sw) | ||||
* where we cannot perform topology changes. Delegate the | * where we cannot perform topology changes. Delegate the | ||||
* work to the events thread. | * work to the events thread. | ||||
*/ | */ | ||||
if (cp != NULL) | if (cp != NULL) | ||||
g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL); | g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL); | ||||
} | } | ||||
static int | static int | ||||
swapongeom_locked(struct cdev *dev, struct vnode *vp) | swapongeom_locked(struct cdev *dev, struct vnode *vp, int flags, int priority) | ||||
{ | { | ||||
struct g_provider *pp; | struct g_provider *pp; | ||||
struct g_consumer *cp; | struct g_consumer *cp; | ||||
static struct g_geom *gp; | static struct g_geom *gp; | ||||
struct swdevt *sp; | struct swdevt *sp; | ||||
u_long nblks; | u_long nblks; | ||||
int error; | int error; | ||||
Show All 23 Lines | swapongeom_locked(struct cdev *dev, struct vnode *vp, int flags, int priority) | ||||
*/ | */ | ||||
error = g_access(cp, 1, 1, 0); | error = g_access(cp, 1, 1, 0); | ||||
if (error != 0) { | if (error != 0) { | ||||
g_detach(cp); | g_detach(cp); | ||||
g_destroy_consumer(cp); | g_destroy_consumer(cp); | ||||
return (error); | return (error); | ||||
} | } | ||||
nblks = pp->mediasize / DEV_BSIZE; | nblks = pp->mediasize / DEV_BSIZE; | ||||
if ((pp->flags & G_PF_ACCEPT_UNMAPPED) != 0) | |||||
flags |= SW_UNMAPPED; | |||||
swaponsomething(vp, cp, nblks, swapgeom_strategy, | swaponsomething(vp, cp, nblks, swapgeom_strategy, | ||||
swapgeom_close, dev2udev(dev), | swapgeom_close, dev2udev(dev), flags, priority); | ||||
(pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
swapongeom(struct vnode *vp) | swapongeom(struct vnode *vp, int flags, int priority) | ||||
{ | { | ||||
int error; | int error; | ||||
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); | vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); | ||||
if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) { | if (vp->v_type != VCHR || (vp->v_iflag & VI_DOOMED) != 0) { | ||||
error = ENOENT; | error = ENOENT; | ||||
} else { | } else { | ||||
g_topology_lock(); | g_topology_lock(); | ||||
error = swapongeom_locked(vp->v_rdev, vp); | error = swapongeom_locked(vp->v_rdev, vp, flags, priority); | ||||
g_topology_unlock(); | g_topology_unlock(); | ||||
} | } | ||||
VOP_UNLOCK(vp, 0); | VOP_UNLOCK(vp, 0); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* VNODE backend | * VNODE backend | ||||
Show All 30 Lines | |||||
{ | { | ||||
VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td); | VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td); | ||||
vrele(sp->sw_vp); | vrele(sp->sw_vp); | ||||
} | } | ||||
static int | static int | ||||
swaponvp(struct thread *td, struct vnode *vp, u_long nblks) | swaponvp(struct thread *td, struct vnode *vp, u_long nblks, int flags, | ||||
int priority) | |||||
{ | { | ||||
struct swdevt *sp; | struct swdevt *sp; | ||||
int error; | int error; | ||||
if (nblks == 0) | if (nblks == 0) | ||||
return (ENXIO); | return (ENXIO); | ||||
mtx_lock(&sw_dev_mtx); | mtx_lock(&sw_dev_mtx); | ||||
TAILQ_FOREACH(sp, &swtailq, sw_list) { | TAILQ_FOREACH(sp, &swtailq, sw_list) { | ||||
Show All 10 Lines | #ifdef MAC | ||||
if (error == 0) | if (error == 0) | ||||
#endif | #endif | ||||
error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL); | error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL); | ||||
(void) VOP_UNLOCK(vp, 0); | (void) VOP_UNLOCK(vp, 0); | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close, | swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close, | ||||
NODEV, 0); | NODEV, flags, priority); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) | sysctl_swap_async_max(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
int error, new, n; | int error, new, n; | ||||
Show All 31 Lines |
This is problematic because the cursors are 64-bit variables while npages is a 32-bit variable.