Index: head/sys/geom/geom_vfs.c
===================================================================
--- head/sys/geom/geom_vfs.c (revision 361490)
+++ head/sys/geom/geom_vfs.c (revision 361491)
@@ -1,291 +1,297 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2004 Poul-Henning Kamp
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/sbuf.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <geom/geom.h>
#include <geom/geom_vfs.h>
/*
* subroutines for use by filesystems.
*
* XXX: should maybe live somewhere else ?
*/
#include <sys/buf.h>
struct g_vfs_softc {
struct mtx sc_mtx;
struct bufobj *sc_bo;
int sc_active;
int sc_orphaned;
+ int sc_enxio_active;
};
static struct buf_ops __g_vfs_bufops = {
.bop_name = "GEOM_VFS",
.bop_write = bufwrite,
.bop_strategy = g_vfs_strategy,
.bop_sync = bufsync,
.bop_bdflush = bufbdflush
};
struct buf_ops *g_vfs_bufops = &__g_vfs_bufops;
static g_orphan_t g_vfs_orphan;
static struct g_class g_vfs_class = {
.name = "VFS",
.version = G_VERSION,
.orphan = g_vfs_orphan,
};
DECLARE_GEOM_CLASS(g_vfs_class, g_vfs);
static void
g_vfs_destroy(void *arg, int flags __unused)
{
struct g_consumer *cp;
g_topology_assert();
cp = arg;
if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
g_access(cp, -cp->acr, -cp->acw, -cp->ace);
g_detach(cp);
if (cp->geom->softc == NULL)
g_wither_geom(cp->geom, ENXIO);
}
static void
g_vfs_done(struct bio *bip)
{
struct g_consumer *cp;
struct g_vfs_softc *sc;
struct buf *bp;
int destroy;
struct mount *mp;
struct vnode *vp;
struct cdev *cdevp;
/*
* Collect statistics on synchronous and asynchronous read
* and write counts for disks that have associated filesystems.
*/
bp = bip->bio_caller2;
vp = bp->b_vp;
if (vp != NULL) {
/*
* If not a disk vnode, use its associated mount point
* otherwise use the mountpoint associated with the disk.
*/
VI_LOCK(vp);
if (vp->v_type != VCHR ||
(cdevp = vp->v_rdev) == NULL ||
cdevp->si_devsw == NULL ||
(cdevp->si_devsw->d_flags & D_DISK) == 0)
mp = vp->v_mount;
else
mp = cdevp->si_mountpt;
if (mp != NULL) {
if (bp->b_iocmd == BIO_READ) {
if (LK_HOLDER(bp->b_lock.lk_lock) == LK_KERNPROC)
mp->mnt_stat.f_asyncreads++;
else
mp->mnt_stat.f_syncreads++;
} else if (bp->b_iocmd == BIO_WRITE) {
if (LK_HOLDER(bp->b_lock.lk_lock) == LK_KERNPROC)
mp->mnt_stat.f_asyncwrites++;
else
mp->mnt_stat.f_syncwrites++;
}
}
VI_UNLOCK(vp);
}
cp = bip->bio_from;
sc = cp->geom->softc;
- if (bip->bio_error && bip->bio_error != EOPNOTSUPP)
+ if (bip->bio_error != 0 && bip->bio_error != EOPNOTSUPP) {
+ if ((bp->b_xflags & BX_CVTENXIO) != 0)
+ sc->sc_enxio_active = 1;
+ if (sc->sc_enxio_active)
+ bip->bio_error = ENXIO;
g_print_bio("g_vfs_done():", bip, "error = %d",
bip->bio_error);
+ }
bp->b_error = bip->bio_error;
bp->b_ioflags = bip->bio_flags;
if (bip->bio_error)
bp->b_ioflags |= BIO_ERROR;
bp->b_resid = bp->b_bcount - bip->bio_completed;
g_destroy_bio(bip);
mtx_lock(&sc->sc_mtx);
destroy = ((--sc->sc_active) == 0 && sc->sc_orphaned);
mtx_unlock(&sc->sc_mtx);
if (destroy)
g_post_event(g_vfs_destroy, cp, M_WAITOK, NULL);
bufdone(bp);
}
void
g_vfs_strategy(struct bufobj *bo, struct buf *bp)
{
struct g_vfs_softc *sc;
struct g_consumer *cp;
struct bio *bip;
cp = bo->bo_private;
sc = cp->geom->softc;
/*
* If the provider has orphaned us, just return ENXIO.
*/
mtx_lock(&sc->sc_mtx);
- if (sc->sc_orphaned) {
+ if (sc->sc_orphaned || sc->sc_enxio_active) {
mtx_unlock(&sc->sc_mtx);
bp->b_error = ENXIO;
bp->b_ioflags |= BIO_ERROR;
bufdone(bp);
return;
}
sc->sc_active++;
mtx_unlock(&sc->sc_mtx);
bip = g_alloc_bio();
bip->bio_cmd = bp->b_iocmd;
bip->bio_offset = bp->b_iooffset;
bip->bio_length = bp->b_bcount;
bdata2bio(bp, bip);
if ((bp->b_flags & B_BARRIER) != 0) {
bip->bio_flags |= BIO_ORDERED;
bp->b_flags &= ~B_BARRIER;
}
if (bp->b_iocmd == BIO_SPEEDUP)
bip->bio_flags |= bp->b_ioflags;
bip->bio_done = g_vfs_done;
bip->bio_caller2 = bp;
#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
buf_track(bp, __func__);
bip->bio_track_bp = bp;
#endif
g_io_request(bip, cp);
}
static void
g_vfs_orphan(struct g_consumer *cp)
{
struct g_geom *gp;
struct g_vfs_softc *sc;
int destroy;
g_topology_assert();
gp = cp->geom;
g_trace(G_T_TOPOLOGY, "g_vfs_orphan(%p(%s))", cp, gp->name);
sc = gp->softc;
if (sc == NULL)
return;
mtx_lock(&sc->sc_mtx);
sc->sc_orphaned = 1;
destroy = (sc->sc_active == 0);
mtx_unlock(&sc->sc_mtx);
if (destroy)
g_vfs_destroy(cp, 0);
/*
* Do not destroy the geom. Filesystem will do that during unmount.
*/
}
int
g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr)
{
struct g_geom *gp;
struct g_provider *pp;
struct g_consumer *cp;
struct g_vfs_softc *sc;
struct bufobj *bo;
int error;
g_topology_assert();
*cpp = NULL;
bo = &vp->v_bufobj;
if (bo->bo_private != vp)
return (EBUSY);
pp = g_dev_getprovider(vp->v_rdev);
if (pp == NULL)
return (ENOENT);
gp = g_new_geomf(&g_vfs_class, "%s.%s", fsname, pp->name);
sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
mtx_init(&sc->sc_mtx, "g_vfs", NULL, MTX_DEF);
sc->sc_bo = bo;
gp->softc = sc;
cp = g_new_consumer(gp);
g_attach(cp, pp);
error = g_access(cp, 1, wr, wr);
if (error) {
g_wither_geom(gp, ENXIO);
return (error);
}
vnode_create_vobject(vp, pp->mediasize, curthread);
*cpp = cp;
cp->private = vp;
cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
bo->bo_ops = g_vfs_bufops;
bo->bo_private = cp;
bo->bo_bsize = pp->sectorsize;
return (error);
}
void
g_vfs_close(struct g_consumer *cp)
{
struct g_geom *gp;
struct g_vfs_softc *sc;
g_topology_assert();
gp = cp->geom;
sc = gp->softc;
bufobj_invalbuf(sc->sc_bo, V_SAVE, 0, 0);
sc->sc_bo->bo_private = cp->private;
gp->softc = NULL;
mtx_destroy(&sc->sc_mtx);
if (!sc->sc_orphaned || cp->provider == NULL)
g_wither_geom_close(gp, ENXIO);
g_free(sc);
}
Index: head/sys/kern/vfs_bio.c
===================================================================
--- head/sys/kern/vfs_bio.c (revision 361490)
+++ head/sys/kern/vfs_bio.c (revision 361491)
@@ -1,5454 +1,5458 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2004 Poul-Henning Kamp
* Copyright (c) 1994,1997 John S. Dyson
* Copyright (c) 2013 The FreeBSD Foundation
* All rights reserved.
*
* Portions of this software were developed by Konstantin Belousov
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* this file contains a new buffer I/O scheme implementing a coherent
* VM object and buffer cache scheme. Pains have been taken to make
* sure that the performance degradation associated with schemes such
* as this is not realized.
*
* Author: John S. Dyson
* Significant help during the development and debugging phases
* had been provided by David Greenman, also of the FreeBSD core team.
*
* see man buf(9) for more info.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/bitset.h>
#include <sys/conf.h>
#include <sys/counter.h>
#include <sys/buf.h>
#include <sys/devicestat.h>
#include <sys/eventhandler.h>
#include <sys/fail.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/refcount.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syscallsubr.h>
#include <sys/vmem.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <sys/watchdog.h>
#include <geom/geom.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_pager.h>
#include <vm/vm_extern.h>
#include <vm/vm_map.h>
#include <vm/swap_pager.h>
static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");
struct bio_ops bioops; /* I/O operation notification */
struct buf_ops buf_ops_bio = {
.bop_name = "buf_ops_bio",
.bop_write = bufwrite,
.bop_strategy = bufstrategy,
.bop_sync = bufsync,
.bop_bdflush = bufbdflush,
};
struct bufqueue {
struct mtx_padalign bq_lock;
TAILQ_HEAD(, buf) bq_queue;
uint8_t bq_index;
uint16_t bq_subqueue;
int bq_len;
} __aligned(CACHE_LINE_SIZE);
#define BQ_LOCKPTR(bq) (&(bq)->bq_lock)
#define BQ_LOCK(bq) mtx_lock(BQ_LOCKPTR((bq)))
#define BQ_UNLOCK(bq) mtx_unlock(BQ_LOCKPTR((bq)))
#define BQ_ASSERT_LOCKED(bq) mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)
struct bufdomain {
struct bufqueue bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */
struct bufqueue bd_dirtyq;
struct bufqueue *bd_cleanq;
struct mtx_padalign bd_run_lock;
/* Constants */
long bd_maxbufspace;
long bd_hibufspace;
long bd_lobufspace;
long bd_bufspacethresh;
int bd_hifreebuffers;
int bd_lofreebuffers;
int bd_hidirtybuffers;
int bd_lodirtybuffers;
int bd_dirtybufthresh;
int bd_lim;
/* atomics */
int bd_wanted;
int __aligned(CACHE_LINE_SIZE) bd_numdirtybuffers;
int __aligned(CACHE_LINE_SIZE) bd_running;
long __aligned(CACHE_LINE_SIZE) bd_bufspace;
int __aligned(CACHE_LINE_SIZE) bd_freebuffers;
} __aligned(CACHE_LINE_SIZE);
#define BD_LOCKPTR(bd) (&(bd)->bd_cleanq->bq_lock)
#define BD_LOCK(bd) mtx_lock(BD_LOCKPTR((bd)))
#define BD_UNLOCK(bd) mtx_unlock(BD_LOCKPTR((bd)))
#define BD_ASSERT_LOCKED(bd) mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
#define BD_RUN_LOCKPTR(bd) (&(bd)->bd_run_lock)
#define BD_RUN_LOCK(bd) mtx_lock(BD_RUN_LOCKPTR((bd)))
#define BD_RUN_UNLOCK(bd) mtx_unlock(BD_RUN_LOCKPTR((bd)))
#define BD_DOMAIN(bd) (bd - bdomain)
static struct buf *buf; /* buffer header pool */
extern struct buf *swbuf; /* Swap buffer header pool. */
caddr_t __read_mostly unmapped_buf;
/* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
struct proc *bufdaemonproc;
static int inmem(struct vnode *vp, daddr_t blkno);
static void vm_hold_free_pages(struct buf *bp, int newbsize);
static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
vm_offset_t to);
static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
vm_page_t m);
static void vfs_clean_pages_dirty_buf(struct buf *bp);
static void vfs_setdirty_range(struct buf *bp);
static void vfs_vmio_invalidate(struct buf *bp);
static void vfs_vmio_truncate(struct buf *bp, int npages);
static void vfs_vmio_extend(struct buf *bp, int npages, int size);
static int vfs_bio_clcheck(struct vnode *vp, int size,
daddr_t lblkno, daddr_t blkno);
static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int,
void (*)(struct buf *));
static int buf_flush(struct vnode *vp, struct bufdomain *, int);
static int flushbufqueues(struct vnode *, struct bufdomain *, int, int);
static void buf_daemon(void);
static __inline void bd_wakeup(void);
static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
static void bufkva_reclaim(vmem_t *, int);
static void bufkva_free(struct buf *);
static int buf_import(void *, void **, int, int, int);
static void buf_release(void *, void **, int);
static void maxbcachebuf_adjust(void);
static inline struct bufdomain *bufdomain(struct buf *);
static void bq_remove(struct bufqueue *bq, struct buf *bp);
static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
static int buf_recycle(struct bufdomain *, bool kva);
static void bq_init(struct bufqueue *bq, int qindex, int cpu,
const char *lockname);
static void bd_init(struct bufdomain *bd);
static int bd_flushall(struct bufdomain *bd);
static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);
static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
int vmiodirenable = TRUE;
SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
"Use the VM system for directory writes");
long runningbufspace;
SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
"Amount of presently outstanding async buffer io");
SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
static counter_u64_t bufkvaspace;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
"Kernel virtual memory used for buffers");
static long maxbufspace;
SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &maxbufspace,
__offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
"Maximum allowed value of bufspace (including metadata)");
static long bufmallocspace;
SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
"Amount of malloced memory for buffers");
static long maxbufmallocspace;
SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
0, "Maximum amount of malloced memory for buffers");
static long lobufspace;
SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &lobufspace,
__offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
"Minimum amount of buffers we want to have");
long hibufspace;
SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &hibufspace,
__offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
"Maximum allowed value of bufspace (excluding metadata)");
long bufspacethresh;
SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &bufspacethresh,
__offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
"Bufspace consumed before waking the daemon to free some");
static counter_u64_t buffreekvacnt;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
"Number of times we have freed the KVA space from some buffer");
static counter_u64_t bufdefragcnt;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
"Number of times we have had to repeat buffer allocation to defragment");
static long lorunningspace;
SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
"Minimum preferred space used for in-progress I/O");
static long hirunningspace;
SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
"Maximum amount of space to use for in-progress I/O");
int dirtybufferflushes;
SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
int bdwriteskip;
SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
int altbufferflushes;
SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW | CTLFLAG_STATS,
&altbufferflushes, 0, "Number of fsync flushes to limit dirty buffers");
static int recursiveflushes;
SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW | CTLFLAG_STATS,
&recursiveflushes, 0, "Number of flushes skipped due to being recursive");
static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
"Number of buffers that are dirty (has unwritten changes) at the moment");
static int lodirtybuffers;
SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lodirtybuffers,
__offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
"How many buffers we want to have free before bufdaemon can sleep");
static int hidirtybuffers;
SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hidirtybuffers,
__offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
"When the number of dirty buffers is considered severe");
int dirtybufthresh;
SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &dirtybufthresh,
__offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
"Number of bdwrite to bawrite conversions to clear dirty buffers");
static int numfreebuffers;
SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
"Number of free buffers");
static int lofreebuffers;
SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lofreebuffers,
__offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
"Target number of free buffers");
static int hifreebuffers;
SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hifreebuffers,
__offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
"Threshold for clean buffer recycling");
static counter_u64_t getnewbufcalls;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
&getnewbufcalls, "Number of calls to getnewbuf");
static counter_u64_t getnewbufrestarts;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
&getnewbufrestarts,
"Number of times getnewbuf has had to restart a buffer acquisition");
static counter_u64_t mappingrestarts;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
&mappingrestarts,
"Number of times getblk has had to restart a buffer mapping for "
"unmapped buffer");
static counter_u64_t numbufallocfails;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
&numbufallocfails, "Number of times buffer allocations failed");
static int flushbufqtarget = 100;
SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
"Amount of work to do in flushbufqueues when helping bufdaemon");
static counter_u64_t notbufdflushes;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, ¬bufdflushes,
"Number of dirty buffer flushes done by the bufdaemon helpers");
static long barrierwrites;
SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW | CTLFLAG_STATS,
&barrierwrites, 0, "Number of barrier writes");
SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
&unmapped_buf_allowed, 0,
"Permit the use of the unmapped i/o");
int maxbcachebuf = MAXBCACHEBUF;
SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
"Maximum size of a buffer cache block");
/*
* This lock synchronizes access to bd_request.
*/
static struct mtx_padalign __exclusive_cache_line bdlock;
/*
* This lock protects the runningbufreq and synchronizes runningbufwakeup and
* waitrunningbufspace().
*/
static struct mtx_padalign __exclusive_cache_line rbreqlock;
/*
* Lock that protects bdirtywait.
*/
static struct mtx_padalign __exclusive_cache_line bdirtylock;
/*
* Wakeup point for bufdaemon, as well as indicator of whether it is already
* active. Set to 1 when the bufdaemon is already "on" the queue, 0 when it
* is idling.
*/
static int bd_request;
/*
* Request for the buf daemon to write more buffers than is indicated by
* lodirtybuf. This may be necessary to push out excess dependencies or
* defragment the address space where a simple count of the number of dirty
* buffers is insufficient to characterize the demand for flushing them.
*/
static int bd_speedupreq;
/*
* Synchronization (sleep/wakeup) variable for active buffer space requests.
* Set when wait starts, cleared prior to wakeup().
* Used in runningbufwakeup() and waitrunningbufspace().
*/
static int runningbufreq;
/*
* Synchronization for bwillwrite() waiters.
*/
static int bdirtywait;
/*
* Definitions for the buffer free lists.
*/
#define QUEUE_NONE 0 /* on no queue */
#define QUEUE_EMPTY 1 /* empty buffer headers */
#define QUEUE_DIRTY 2 /* B_DELWRI buffers */
#define QUEUE_CLEAN 3 /* non-B_DELWRI buffers */
#define QUEUE_SENTINEL 4 /* not an queue index, but mark for sentinel */
/* Maximum number of buffer domains. */
#define BUF_DOMAINS 8
struct bufdomainset bdlodirty; /* Domains > lodirty */
struct bufdomainset bdhidirty; /* Domains > hidirty */
/* Configured number of clean queues. */
static int __read_mostly buf_domains;
BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
struct bufqueue __exclusive_cache_line bqempty;
/*
* per-cpu empty buffer cache.
*/
uma_zone_t buf_zone;
/*
* Single global constant for BUF_WMESG, to avoid getting multiple references.
* buf_wmesg is referred from macros.
*/
const char *buf_wmesg = BUF_WMESG;
static int
sysctl_runningspace(SYSCTL_HANDLER_ARGS)
{
long value;
int error;
value = *(long *)arg1;
error = sysctl_handle_long(oidp, &value, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
mtx_lock(&rbreqlock);
if (arg1 == &hirunningspace) {
if (value < lorunningspace)
error = EINVAL;
else
hirunningspace = value;
} else {
KASSERT(arg1 == &lorunningspace,
("%s: unknown arg1", __func__));
if (value > hirunningspace)
error = EINVAL;
else
lorunningspace = value;
}
mtx_unlock(&rbreqlock);
return (error);
}
static int
sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
{
int error;
int value;
int i;
value = *(int *)arg1;
error = sysctl_handle_int(oidp, &value, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
*(int *)arg1 = value;
for (i = 0; i < buf_domains; i++)
*(int *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
value / buf_domains;
return (error);
}
static int
sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
{
long value;
int error;
int i;
value = *(long *)arg1;
error = sysctl_handle_long(oidp, &value, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
*(long *)arg1 = value;
for (i = 0; i < buf_domains; i++)
*(long *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
value / buf_domains;
return (error);
}
#if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)
{
long lvalue;
int ivalue;
int i;
lvalue = 0;
for (i = 0; i < buf_domains; i++)
lvalue += bdomain[i].bd_bufspace;
if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
return (sysctl_handle_long(oidp, &lvalue, 0, req));
if (lvalue > INT_MAX)
/* On overflow, still write out a long to trigger ENOMEM. */
return (sysctl_handle_long(oidp, &lvalue, 0, req));
ivalue = lvalue;
return (sysctl_handle_int(oidp, &ivalue, 0, req));
}
#else
static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)
{
long lvalue;
int i;
lvalue = 0;
for (i = 0; i < buf_domains; i++)
lvalue += bdomain[i].bd_bufspace;
return (sysctl_handle_long(oidp, &lvalue, 0, req));
}
#endif
static int
sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
{
int value;
int i;
value = 0;
for (i = 0; i < buf_domains; i++)
value += bdomain[i].bd_numdirtybuffers;
return (sysctl_handle_int(oidp, &value, 0, req));
}
/*
* bdirtywakeup:
*
* Wakeup any bwillwrite() waiters.
*/
static void
bdirtywakeup(void)
{
mtx_lock(&bdirtylock);
if (bdirtywait) {
bdirtywait = 0;
wakeup(&bdirtywait);
}
mtx_unlock(&bdirtylock);
}
/*
* bd_clear:
*
* Clear a domain from the appropriate bitsets when dirtybuffers
* is decremented.
*/
static void
bd_clear(struct bufdomain *bd)
{
mtx_lock(&bdirtylock);
if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
mtx_unlock(&bdirtylock);
}
/*
* bd_set:
*
* Set a domain in the appropriate bitsets when dirtybuffers
* is incremented.
*/
static void
bd_set(struct bufdomain *bd)
{
mtx_lock(&bdirtylock);
if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
mtx_unlock(&bdirtylock);
}
/*
* bdirtysub:
*
* Decrement the numdirtybuffers count by one and wakeup any
* threads blocked in bwillwrite().
*/
static void
bdirtysub(struct buf *bp)
{
struct bufdomain *bd;
int num;
bd = bufdomain(bp);
num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
bdirtywakeup();
if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
bd_clear(bd);
}
/*
* bdirtyadd:
*
* Increment the numdirtybuffers count by one and wakeup the buf
* daemon if needed.
*/
static void
bdirtyadd(struct buf *bp)
{
struct bufdomain *bd;
int num;
/*
* Only do the wakeup once as we cross the boundary. The
* buf daemon will keep running until the condition clears.
*/
bd = bufdomain(bp);
num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
bd_wakeup();
if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
bd_set(bd);
}
/*
* bufspace_daemon_wakeup:
*
* Wakeup the daemons responsible for freeing clean bufs.
*/
static void
bufspace_daemon_wakeup(struct bufdomain *bd)
{
/*
* avoid the lock if the daemon is running.
*/
if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
BD_RUN_LOCK(bd);
atomic_store_int(&bd->bd_running, 1);
wakeup(&bd->bd_running);
BD_RUN_UNLOCK(bd);
}
}
/*
* bufspace_daemon_wait:
*
* Sleep until the domain falls below a limit or one second passes.
*/
static void
bufspace_daemon_wait(struct bufdomain *bd)
{
/*
* Re-check our limits and sleep. bd_running must be
* cleared prior to checking the limits to avoid missed
* wakeups. The waker will adjust one of bufspace or
* freebuffers prior to checking bd_running.
*/
BD_RUN_LOCK(bd);
atomic_store_int(&bd->bd_running, 0);
if (bd->bd_bufspace < bd->bd_bufspacethresh &&
bd->bd_freebuffers > bd->bd_lofreebuffers) {
msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd), PRIBIO|PDROP,
"-", hz);
} else {
/* Avoid spurious wakeups while running. */
atomic_store_int(&bd->bd_running, 1);
BD_RUN_UNLOCK(bd);
}
}
/*
* bufspace_adjust:
*
* Adjust the reported bufspace for a KVA managed buffer, possibly
* waking any waiters.
*/
static void
bufspace_adjust(struct buf *bp, int bufsize)
{
struct bufdomain *bd;
long space;
int diff;
KASSERT((bp->b_flags & B_MALLOC) == 0,
("bufspace_adjust: malloc buf %p", bp));
bd = bufdomain(bp);
diff = bufsize - bp->b_bufsize;
if (diff < 0) {
atomic_subtract_long(&bd->bd_bufspace, -diff);
} else if (diff > 0) {
space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
/* Wake up the daemon on the transition. */
if (space < bd->bd_bufspacethresh &&
space + diff >= bd->bd_bufspacethresh)
bufspace_daemon_wakeup(bd);
}
bp->b_bufsize = bufsize;
}
/*
* bufspace_reserve:
*
* Reserve bufspace before calling allocbuf(). metadata has a
* different space limit than data.
*/
static int
bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
{
long limit, new;
long space;
if (metadata)
limit = bd->bd_maxbufspace;
else
limit = bd->bd_hibufspace;
space = atomic_fetchadd_long(&bd->bd_bufspace, size);
new = space + size;
if (new > limit) {
atomic_subtract_long(&bd->bd_bufspace, size);
return (ENOSPC);
}
/* Wake up the daemon on the transition. */
if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
bufspace_daemon_wakeup(bd);
return (0);
}
/*
* bufspace_release:
*
* Release reserved bufspace after bufspace_adjust() has consumed it.
*/
static void
bufspace_release(struct bufdomain *bd, int size)
{
atomic_subtract_long(&bd->bd_bufspace, size);
}
/*
* bufspace_wait:
*
* Wait for bufspace, acting as the buf daemon if a locked vnode is
* supplied. bd_wanted must be set prior to polling for space. The
* operation must be re-tried on return.
*/
static void
bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,
int slpflag, int slptimeo)
{
struct thread *td;
int error, fl, norunbuf;
if ((gbflags & GB_NOWAIT_BD) != 0)
return;
td = curthread;
BD_LOCK(bd);
while (bd->bd_wanted) {
if (vp != NULL && vp->v_type != VCHR &&
(td->td_pflags & TDP_BUFNEED) == 0) {
BD_UNLOCK(bd);
/*
* getblk() is called with a vnode locked, and
* some majority of the dirty buffers may as
* well belong to the vnode. Flushing the
* buffers there would make a progress that
* cannot be achieved by the buf_daemon, that
* cannot lock the vnode.
*/
norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
(td->td_pflags & TDP_NORUNNINGBUF);
/*
* Play bufdaemon. The getnewbuf() function
* may be called while the thread owns lock
* for another dirty buffer for the same
* vnode, which makes it impossible to use
* VOP_FSYNC() there, due to the buffer lock
* recursion.
*/
td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
fl = buf_flush(vp, bd, flushbufqtarget);
td->td_pflags &= norunbuf;
BD_LOCK(bd);
if (fl != 0)
continue;
if (bd->bd_wanted == 0)
break;
}
error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
(PRIBIO + 4) | slpflag, "newbuf", slptimeo);
if (error != 0)
break;
}
BD_UNLOCK(bd);
}
/*
* bufspace_daemon:
*
* buffer space management daemon. Tries to maintain some marginal
* amount of free buffer space so that requesting processes neither
* block nor work to reclaim buffers.
*/
static void
bufspace_daemon(void *arg)
{
struct bufdomain *bd;
EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
SHUTDOWN_PRI_LAST + 100);
bd = arg;
for (;;) {
kthread_suspend_check();
/*
* Free buffers from the clean queue until we meet our
* targets.
*
* Theory of operation: The buffer cache is most efficient
* when some free buffer headers and space are always
* available to getnewbuf(). This daemon attempts to prevent
* the excessive blocking and synchronization associated
* with shortfall. It goes through three phases according
* demand:
*
* 1) The daemon wakes up voluntarily once per-second
* during idle periods when the counters are below
* the wakeup thresholds (bufspacethresh, lofreebuffers).
*
* 2) The daemon wakes up as we cross the thresholds
* ahead of any potential blocking. This may bounce
* slightly according to the rate of consumption and
* release.
*
* 3) The daemon and consumers are starved for working
* clean buffers. This is the 'bufspace' sleep below
* which will inefficiently trade bufs with bqrelse
* until we return to condition 2.
*/
while (bd->bd_bufspace > bd->bd_lobufspace ||
bd->bd_freebuffers < bd->bd_hifreebuffers) {
if (buf_recycle(bd, false) != 0) {
if (bd_flushall(bd))
continue;
/*
* Speedup dirty if we've run out of clean
* buffers. This is possible in particular
* because softdep may held many bufs locked
* pending writes to other bufs which are
* marked for delayed write, exhausting
* clean space until they are written.
*/
bd_speedup();
BD_LOCK(bd);
if (bd->bd_wanted) {
msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
PRIBIO|PDROP, "bufspace", hz/10);
} else
BD_UNLOCK(bd);
}
maybe_yield();
}
bufspace_daemon_wait(bd);
}
}
/*
* bufmallocadjust:
*
* Adjust the reported bufspace for a malloc managed buffer, possibly
* waking any waiters.
*/
static void
bufmallocadjust(struct buf *bp, int bufsize)
{
int diff;
KASSERT((bp->b_flags & B_MALLOC) != 0,
("bufmallocadjust: non-malloc buf %p", bp));
diff = bufsize - bp->b_bufsize;
if (diff < 0)
atomic_subtract_long(&bufmallocspace, -diff);
else
atomic_add_long(&bufmallocspace, diff);
bp->b_bufsize = bufsize;
}
/*
* runningwakeup:
*
* Wake up processes that are waiting on asynchronous writes to fall
* below lorunningspace.
*/
static void
runningwakeup(void)
{
mtx_lock(&rbreqlock);
if (runningbufreq) {
runningbufreq = 0;
wakeup(&runningbufreq);
}
mtx_unlock(&rbreqlock);
}
/*
* runningbufwakeup:
*
* Decrement the outstanding write count according.
*/
void
runningbufwakeup(struct buf *bp)
{
long space, bspace;
bspace = bp->b_runningbufspace;
if (bspace == 0)
return;
space = atomic_fetchadd_long(&runningbufspace, -bspace);
KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
space, bspace));
bp->b_runningbufspace = 0;
/*
* Only acquire the lock and wakeup on the transition from exceeding
* the threshold to falling below it.
*/
if (space < lorunningspace)
return;
if (space - bspace > lorunningspace)
return;
runningwakeup();
}
/*
* waitrunningbufspace()
*
* runningbufspace is a measure of the amount of I/O currently
* running. This routine is used in async-write situations to
* prevent creating huge backups of pending writes to a device.
* Only asynchronous writes are governed by this function.
*
* This does NOT turn an async write into a sync write. It waits
* for earlier writes to complete and generally returns before the
* caller's write has reached the device.
*/
void
waitrunningbufspace(void)
{
mtx_lock(&rbreqlock);
while (runningbufspace > hirunningspace) {
runningbufreq = 1;
msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
}
mtx_unlock(&rbreqlock);
}
/*
* vfs_buf_test_cache:
*
* Called when a buffer is extended. This function clears the B_CACHE
* bit if the newly extended portion of the buffer does not contain
* valid data.
*/
static __inline void
vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
vm_offset_t size, vm_page_t m)
{
/*
* This function and its results are protected by higher level
* synchronization requiring vnode and buf locks to page in and
* validate pages.
*/
if (bp->b_flags & B_CACHE) {
int base = (foff + off) & PAGE_MASK;
if (vm_page_is_valid(m, base, size) == 0)
bp->b_flags &= ~B_CACHE;
}
}
/* Wake up the buffer daemon if necessary */
static void
bd_wakeup(void)
{
mtx_lock(&bdlock);
if (bd_request == 0) {
bd_request = 1;
wakeup(&bd_request);
}
mtx_unlock(&bdlock);
}
/*
* Adjust the maxbcachbuf tunable.
*/
static void
maxbcachebuf_adjust(void)
{
int i;
/*
* maxbcachebuf must be a power of 2 >= MAXBSIZE.
*/
i = 2;
while (i * 2 <= maxbcachebuf)
i *= 2;
maxbcachebuf = i;
if (maxbcachebuf < MAXBSIZE)
maxbcachebuf = MAXBSIZE;
if (maxbcachebuf > MAXPHYS)
maxbcachebuf = MAXPHYS;
if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
printf("maxbcachebuf=%d\n", maxbcachebuf);
}
/*
* bd_speedup - speedup the buffer cache flushing code
*/
void
bd_speedup(void)
{
int needwake;
mtx_lock(&bdlock);
needwake = 0;
if (bd_speedupreq == 0 || bd_request == 0)
needwake = 1;
bd_speedupreq = 1;
bd_request = 1;
if (needwake)
wakeup(&bd_request);
mtx_unlock(&bdlock);
}
#ifdef __i386__
#define TRANSIENT_DENOM 5
#else
#define TRANSIENT_DENOM 10
#endif
/*
* Calculating buffer cache scaling values and reserve space for buffer
* headers. This is called during low level kernel initialization and
* may be called more then once. We CANNOT write to the memory area
* being reserved at this time.
*/
caddr_t
kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
{
int tuned_nbuf;
long maxbuf, maxbuf_sz, buf_sz, biotmap_sz;
/*
* physmem_est is in pages. Convert it to kilobytes (assumes
* PAGE_SIZE is >= 1K)
*/
physmem_est = physmem_est * (PAGE_SIZE / 1024);
maxbcachebuf_adjust();
/*
* The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
* For the first 64MB of ram nominally allocate sufficient buffers to
* cover 1/4 of our ram. Beyond the first 64MB allocate additional
* buffers to cover 1/10 of our ram over 64MB. When auto-sizing
* the buffer cache we limit the eventual kva reservation to
* maxbcache bytes.
*
* factor represents the 1/4 x ram conversion.
*/
if (nbuf == 0) {
int factor = 4 * BKVASIZE / 1024;
nbuf = 50;
if (physmem_est > 4096)
nbuf += min((physmem_est - 4096) / factor,
65536 / factor);
if (physmem_est > 65536)
nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
32 * 1024 * 1024 / (factor * 5));
if (maxbcache && nbuf > maxbcache / BKVASIZE)
nbuf = maxbcache / BKVASIZE;
tuned_nbuf = 1;
} else
tuned_nbuf = 0;
/* XXX Avoid unsigned long overflows later on with maxbufspace. */
maxbuf = (LONG_MAX / 3) / BKVASIZE;
if (nbuf > maxbuf) {
if (!tuned_nbuf)
printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
maxbuf);
nbuf = maxbuf;
}
/*
* Ideal allocation size for the transient bio submap is 10%
* of the maximal space buffer map. This roughly corresponds
* to the amount of the buffer mapped for typical UFS load.
*
* Clip the buffer map to reserve space for the transient
* BIOs, if its extent is bigger than 90% (80% on i386) of the
* maximum buffer map extent on the platform.
*
* The fall-back to the maxbuf in case of maxbcache unset,
* allows to not trim the buffer KVA for the architectures
* with ample KVA space.
*/
if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
buf_sz = (long)nbuf * BKVASIZE;
if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
(TRANSIENT_DENOM - 1)) {
/*
* There is more KVA than memory. Do not
* adjust buffer map size, and assign the rest
* of maxbuf to transient map.
*/
biotmap_sz = maxbuf_sz - buf_sz;
} else {
/*
* Buffer map spans all KVA we could afford on
* this platform. Give 10% (20% on i386) of
* the buffer map to the transient bio map.
*/
biotmap_sz = buf_sz / TRANSIENT_DENOM;
buf_sz -= biotmap_sz;
}
if (biotmap_sz / INT_MAX > MAXPHYS)
bio_transient_maxcnt = INT_MAX;
else
bio_transient_maxcnt = biotmap_sz / MAXPHYS;
/*
* Artificially limit to 1024 simultaneous in-flight I/Os
* using the transient mapping.
*/
if (bio_transient_maxcnt > 1024)
bio_transient_maxcnt = 1024;
if (tuned_nbuf)
nbuf = buf_sz / BKVASIZE;
}
if (nswbuf == 0) {
nswbuf = min(nbuf / 4, 256);
if (nswbuf < NSWBUF_MIN)
nswbuf = NSWBUF_MIN;
}
/*
* Reserve space for the buffer cache buffers
*/
buf = (void *)v;
v = (caddr_t)(buf + nbuf);
return(v);
}
/* Initialize the buffer subsystem. Called before use of any buffers. */
void
bufinit(void)
{
struct buf *bp;
int i;
KASSERT(maxbcachebuf >= MAXBSIZE,
("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf,
MAXBSIZE));
bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock");
mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);
unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);
/* finally, initialize each buffer header and stick on empty q */
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
bzero(bp, sizeof *bp);
bp->b_flags = B_INVAL;
bp->b_rcred = NOCRED;
bp->b_wcred = NOCRED;
bp->b_qindex = QUEUE_NONE;
bp->b_domain = -1;
bp->b_subqueue = mp_maxid + 1;
bp->b_xflags = 0;
bp->b_data = bp->b_kvabase = unmapped_buf;
LIST_INIT(&bp->b_dep);
BUF_LOCKINIT(bp);
bq_insert(&bqempty, bp, false);
}
/*
* maxbufspace is the absolute maximum amount of buffer space we are
* allowed to reserve in KVM and in real terms. The absolute maximum
* is nominally used by metadata. hibufspace is the nominal maximum
* used by most other requests. The differential is required to
* ensure that metadata deadlocks don't occur.
*
* maxbufspace is based on BKVASIZE. Allocating buffers larger then
* this may result in KVM fragmentation which is not handled optimally
* by the system. XXX This is less true with vmem. We could use
* PAGE_SIZE.
*/
maxbufspace = (long)nbuf * BKVASIZE;
hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - maxbcachebuf * 10);
lobufspace = (hibufspace / 20) * 19; /* 95% */
bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
/*
* Note: The 16 MiB upper limit for hirunningspace was chosen
* arbitrarily and may need further tuning. It corresponds to
* 128 outstanding write IO requests (if IO size is 128 KiB),
* which fits with many RAID controllers' tagged queuing limits.
* The lower 1 MiB limit is the historical upper limit for
* hirunningspace.
*/
hirunningspace = lmax(lmin(roundup(hibufspace / 64, maxbcachebuf),
16 * 1024 * 1024), 1024 * 1024);
lorunningspace = roundup((hirunningspace * 2) / 3, maxbcachebuf);
/*
* Limit the amount of malloc memory since it is wired permanently into
* the kernel space. Even though this is accounted for in the buffer
* allocation, we don't want the malloced region to grow uncontrolled.
* The malloc scheme improves memory utilization significantly on
* average (small) directories.
*/
maxbufmallocspace = hibufspace / 20;
/*
* Reduce the chance of a deadlock occurring by limiting the number
* of delayed-write dirty buffers we allow to stack up.
*/
hidirtybuffers = nbuf / 4 + 20;
dirtybufthresh = hidirtybuffers * 9 / 10;
/*
* To support extreme low-memory systems, make sure hidirtybuffers
* cannot eat up all available buffer space. This occurs when our
* minimum cannot be met. We try to size hidirtybuffers to 3/4 our
* buffer space assuming BKVASIZE'd buffers.
*/
while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
hidirtybuffers >>= 1;
}
lodirtybuffers = hidirtybuffers / 2;
/*
* lofreebuffers should be sufficient to avoid stalling waiting on
* buf headers under heavy utilization. The bufs in per-cpu caches
* are counted as free but will be unavailable to threads executing
* on other cpus.
*
* hifreebuffers is the free target for the bufspace daemon. This
* should be set appropriately to limit work per-iteration.
*/
lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
hifreebuffers = (3 * lofreebuffers) / 2;
numfreebuffers = nbuf;
/* Setup the kva and free list allocators. */
vmem_set_reclaim(buffer_arena, bufkva_reclaim);
buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf),
NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
/*
* Size the clean queue according to the amount of buffer space.
* One queue per-256mb up to the max. More queues gives better
* concurrency but less accurate LRU.
*/
buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS);
for (i = 0 ; i < buf_domains; i++) {
struct bufdomain *bd;
bd = &bdomain[i];
bd_init(bd);
bd->bd_freebuffers = nbuf / buf_domains;
bd->bd_hifreebuffers = hifreebuffers / buf_domains;
bd->bd_lofreebuffers = lofreebuffers / buf_domains;
bd->bd_bufspace = 0;
bd->bd_maxbufspace = maxbufspace / buf_domains;
bd->bd_hibufspace = hibufspace / buf_domains;
bd->bd_lobufspace = lobufspace / buf_domains;
bd->bd_bufspacethresh = bufspacethresh / buf_domains;
bd->bd_numdirtybuffers = 0;
bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;
bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;
bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;
/* Don't allow more than 2% of bufs in the per-cpu caches. */
bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;
}
getnewbufcalls = counter_u64_alloc(M_WAITOK);
getnewbufrestarts = counter_u64_alloc(M_WAITOK);
mappingrestarts = counter_u64_alloc(M_WAITOK);
numbufallocfails = counter_u64_alloc(M_WAITOK);
notbufdflushes = counter_u64_alloc(M_WAITOK);
buffreekvacnt = counter_u64_alloc(M_WAITOK);
bufdefragcnt = counter_u64_alloc(M_WAITOK);
bufkvaspace = counter_u64_alloc(M_WAITOK);
}
#ifdef INVARIANTS
static inline void
vfs_buf_check_mapped(struct buf *bp)
{
KASSERT(bp->b_kvabase != unmapped_buf,
("mapped buf: b_kvabase was not updated %p", bp));
KASSERT(bp->b_data != unmapped_buf,
("mapped buf: b_data was not updated %p", bp));
KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
MAXPHYS, ("b_data + b_offset unmapped %p", bp));
}
static inline void
vfs_buf_check_unmapped(struct buf *bp)
{
KASSERT(bp->b_data == unmapped_buf,
("unmapped buf: corrupted b_data %p", bp));
}
#define BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
#define BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
#else
#define BUF_CHECK_MAPPED(bp) do {} while (0)
#define BUF_CHECK_UNMAPPED(bp) do {} while (0)
#endif
static int
isbufbusy(struct buf *bp)
{
if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) ||
((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
return (1);
return (0);
}
/*
* Shutdown the system cleanly to prepare for reboot, halt, or power off.
*/
void
bufshutdown(int show_busybufs)
{
static int first_buf_printf = 1;
struct buf *bp;
int iter, nbusy, pbusy;
#ifndef PREEMPTION
int subiter;
#endif
/*
* Sync filesystems for shutdown
*/
wdog_kern_pat(WD_LASTVAL);
kern_sync(curthread);
/*
* With soft updates, some buffers that are
* written will be remarked as dirty until other
* buffers are written.
*/
for (iter = pbusy = 0; iter < 20; iter++) {
nbusy = 0;
for (bp = &buf[nbuf]; --bp >= buf; )
if (isbufbusy(bp))
nbusy++;
if (nbusy == 0) {
if (first_buf_printf)
printf("All buffers synced.");
break;
}
if (first_buf_printf) {
printf("Syncing disks, buffers remaining... ");
first_buf_printf = 0;
}
printf("%d ", nbusy);
if (nbusy < pbusy)
iter = 0;
pbusy = nbusy;
wdog_kern_pat(WD_LASTVAL);
kern_sync(curthread);
#ifdef PREEMPTION
/*
* Spin for a while to allow interrupt threads to run.
*/
DELAY(50000 * iter);
#else
/*
* Context switch several times to allow interrupt
* threads to run.
*/
for (subiter = 0; subiter < 50 * iter; subiter++) {
thread_lock(curthread);
mi_switch(SW_VOL);
DELAY(1000);
}
#endif
}
printf("\n");
/*
* Count only busy local buffers to prevent forcing
* a fsck if we're just a client of a wedged NFS server
*/
nbusy = 0;
for (bp = &buf[nbuf]; --bp >= buf; ) {
if (isbufbusy(bp)) {
#if 0
/* XXX: This is bogus. We should probably have a BO_REMOTE flag instead */
if (bp->b_dev == NULL) {
TAILQ_REMOVE(&mountlist,
bp->b_vp->v_mount, mnt_list);
continue;
}
#endif
nbusy++;
if (show_busybufs > 0) {
printf(
"%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
nbusy, bp, bp->b_vp, bp->b_flags,
(intmax_t)bp->b_blkno,
(intmax_t)bp->b_lblkno);
BUF_LOCKPRINTINFO(bp);
if (show_busybufs > 1)
vn_printf(bp->b_vp,
"vnode content: ");
}
}
}
if (nbusy) {
/*
* Failed to sync all blocks. Indicate this and don't
* unmount filesystems (thus forcing an fsck on reboot).
*/
printf("Giving up on %d buffers\n", nbusy);
DELAY(5000000); /* 5 seconds */
} else {
if (!first_buf_printf)
printf("Final sync complete\n");
/*
* Unmount filesystems
*/
if (!KERNEL_PANICKED())
vfs_unmountall();
}
swapoff_all();
DELAY(100000); /* wait for console output to finish */
}
static void
bpmap_qenter(struct buf *bp)
{
BUF_CHECK_MAPPED(bp);
/*
* bp->b_data is relative to bp->b_offset, but
* bp->b_offset may be offset into the first page.
*/
bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
(vm_offset_t)(bp->b_offset & PAGE_MASK));
}
static inline struct bufdomain *
bufdomain(struct buf *bp)
{
return (&bdomain[bp->b_domain]);
}
static struct bufqueue *
bufqueue(struct buf *bp)
{
switch (bp->b_qindex) {
case QUEUE_NONE:
/* FALLTHROUGH */
case QUEUE_SENTINEL:
return (NULL);
case QUEUE_EMPTY:
return (&bqempty);
case QUEUE_DIRTY:
return (&bufdomain(bp)->bd_dirtyq);
case QUEUE_CLEAN:
return (&bufdomain(bp)->bd_subq[bp->b_subqueue]);
default:
break;
}
panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex);
}
/*
* Return the locked bufqueue that bp is a member of.
*/
static struct bufqueue *
bufqueue_acquire(struct buf *bp)
{
struct bufqueue *bq, *nbq;
/*
* bp can be pushed from a per-cpu queue to the
* cleanq while we're waiting on the lock. Retry
* if the queues don't match.
*/
bq = bufqueue(bp);
BQ_LOCK(bq);
for (;;) {
nbq = bufqueue(bp);
if (bq == nbq)
break;
BQ_UNLOCK(bq);
BQ_LOCK(nbq);
bq = nbq;
}
return (bq);
}
/*
* binsfree:
*
* Insert the buffer into the appropriate free list. Requires a
* locked buffer on entry and buffer is unlocked before return.
*/
static void
binsfree(struct buf *bp, int qindex)
{
struct bufdomain *bd;
struct bufqueue *bq;
KASSERT(qindex == QUEUE_CLEAN || qindex == QUEUE_DIRTY,
("binsfree: Invalid qindex %d", qindex));
BUF_ASSERT_XLOCKED(bp);
/*
* Handle delayed bremfree() processing.
*/
if (bp->b_flags & B_REMFREE) {
if (bp->b_qindex == qindex) {
bp->b_flags |= B_REUSE;
bp->b_flags &= ~B_REMFREE;
BUF_UNLOCK(bp);
return;
}
bq = bufqueue_acquire(bp);
bq_remove(bq, bp);
BQ_UNLOCK(bq);
}
bd = bufdomain(bp);
if (qindex == QUEUE_CLEAN) {
if (bd->bd_lim != 0)
bq = &bd->bd_subq[PCPU_GET(cpuid)];
else
bq = bd->bd_cleanq;
} else
bq = &bd->bd_dirtyq;
bq_insert(bq, bp, true);
}
/*
* buf_free:
*
* Free a buffer to the buf zone once it no longer has valid contents.
*/
static void
buf_free(struct buf *bp)
{
if (bp->b_flags & B_REMFREE)
bremfreef(bp);
if (bp->b_vflags & BV_BKGRDINPROG)
panic("losing buffer 1");
if (bp->b_rcred != NOCRED) {
crfree(bp->b_rcred);
bp->b_rcred = NOCRED;
}
if (bp->b_wcred != NOCRED) {
crfree(bp->b_wcred);
bp->b_wcred = NOCRED;
}
if (!LIST_EMPTY(&bp->b_dep))
buf_deallocate(bp);
bufkva_free(bp);
atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1);
BUF_UNLOCK(bp);
uma_zfree(buf_zone, bp);
}
/*
* buf_import:
*
* Import bufs into the uma cache from the buf list. The system still
* expects a static array of bufs and much of the synchronization
* around bufs assumes type stable storage. As a result, UMA is used
* only as a per-cpu cache of bufs still maintained on a global list.
*/
static int
buf_import(void *arg, void **store, int cnt, int domain, int flags)
{
struct buf *bp;
int i;
BQ_LOCK(&bqempty);
for (i = 0; i < cnt; i++) {
bp = TAILQ_FIRST(&bqempty.bq_queue);
if (bp == NULL)
break;
bq_remove(&bqempty, bp);
store[i] = bp;
}
BQ_UNLOCK(&bqempty);
return (i);
}
/*
* buf_release:
*
* Release bufs from the uma cache back to the buffer queues.
*/
static void
buf_release(void *arg, void **store, int cnt)
{
struct bufqueue *bq;
struct buf *bp;
int i;
bq = &bqempty;
BQ_LOCK(bq);
for (i = 0; i < cnt; i++) {
bp = store[i];
/* Inline bq_insert() to batch locking. */
TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
bp->b_flags &= ~(B_AGE | B_REUSE);
bq->bq_len++;
bp->b_qindex = bq->bq_index;
}
BQ_UNLOCK(bq);
}
/*
* buf_alloc:
*
* Allocate an empty buffer header.
*/
static struct buf *
buf_alloc(struct bufdomain *bd)
{
struct buf *bp;
int freebufs;
/*
* We can only run out of bufs in the buf zone if the average buf
* is less than BKVASIZE. In this case the actual wait/block will
* come from buf_reycle() failing to flush one of these small bufs.
*/
bp = NULL;
freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);
if (freebufs > 0)
bp = uma_zalloc(buf_zone, M_NOWAIT);
if (bp == NULL) {
atomic_add_int(&bd->bd_freebuffers, 1);
bufspace_daemon_wakeup(bd);
counter_u64_add(numbufallocfails, 1);
return (NULL);
}
/*
* Wake-up the bufspace daemon on transition below threshold.
*/
if (freebufs == bd->bd_lofreebuffers)
bufspace_daemon_wakeup(bd);
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
panic("getnewbuf_empty: Locked buf %p on free queue.", bp);
KASSERT(bp->b_vp == NULL,
("bp: %p still has vnode %p.", bp, bp->b_vp));
KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
("invalid buffer %p flags %#x", bp, bp->b_flags));
KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
KASSERT(bp->b_npages == 0,
("bp: %p still has %d vm pages\n", bp, bp->b_npages));
KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
bp->b_domain = BD_DOMAIN(bd);
bp->b_flags = 0;
bp->b_ioflags = 0;
bp->b_xflags = 0;
bp->b_vflags = 0;
bp->b_vp = NULL;
bp->b_blkno = bp->b_lblkno = 0;
bp->b_offset = NOOFFSET;
bp->b_iodone = 0;
bp->b_error = 0;
bp->b_resid = 0;
bp->b_bcount = 0;
bp->b_npages = 0;
bp->b_dirtyoff = bp->b_dirtyend = 0;
bp->b_bufobj = NULL;
bp->b_data = bp->b_kvabase = unmapped_buf;
bp->b_fsprivate1 = NULL;
bp->b_fsprivate2 = NULL;
bp->b_fsprivate3 = NULL;
LIST_INIT(&bp->b_dep);
return (bp);
}
/*
* buf_recycle:
*
* Free a buffer from the given bufqueue. kva controls whether the
* freed buf must own some kva resources. This is used for
* defragmenting.
*/
static int
buf_recycle(struct bufdomain *bd, bool kva)
{
struct bufqueue *bq;
struct buf *bp, *nbp;
if (kva)
counter_u64_add(bufdefragcnt, 1);
nbp = NULL;
bq = bd->bd_cleanq;
BQ_LOCK(bq);
KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),
("buf_recycle: Locks don't match"));
nbp = TAILQ_FIRST(&bq->bq_queue);
/*
* Run scan, possibly freeing data and/or kva mappings on the fly
* depending.
*/
while ((bp = nbp) != NULL) {
/*
* Calculate next bp (we can only use it if we do not
* release the bqlock).
*/
nbp = TAILQ_NEXT(bp, b_freelist);
/*
* If we are defragging then we need a buffer with
* some kva to reclaim.
*/
if (kva && bp->b_kvasize == 0)
continue;
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
continue;
/*
* Implement a second chance algorithm for frequently
* accessed buffers.
*/
if ((bp->b_flags & B_REUSE) != 0) {
TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
bp->b_flags &= ~B_REUSE;
BUF_UNLOCK(bp);
continue;
}
/*
* Skip buffers with background writes in progress.
*/
if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
BUF_UNLOCK(bp);
continue;
}
KASSERT(bp->b_qindex == QUEUE_CLEAN,
("buf_recycle: inconsistent queue %d bp %p",
bp->b_qindex, bp));
KASSERT(bp->b_domain == BD_DOMAIN(bd),
("getnewbuf: queue domain %d doesn't match request %d",
bp->b_domain, (int)BD_DOMAIN(bd)));
/*
* NOTE: nbp is now entirely invalid. We can only restart
* the scan from this point on.
*/
bq_remove(bq, bp);
BQ_UNLOCK(bq);
/*
* Requeue the background write buffer with error and
* restart the scan.
*/
if ((bp->b_vflags & BV_BKGRDERR) != 0) {
bqrelse(bp);
BQ_LOCK(bq);
nbp = TAILQ_FIRST(&bq->bq_queue);
continue;
}
bp->b_flags |= B_INVAL;
brelse(bp);
return (0);
}
bd->bd_wanted = 1;
BQ_UNLOCK(bq);
return (ENOBUFS);
}
/*
* bremfree:
*
* Mark the buffer for removal from the appropriate free list.
*
*/
void
bremfree(struct buf *bp)
{
CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
KASSERT((bp->b_flags & B_REMFREE) == 0,
("bremfree: buffer %p already marked for delayed removal.", bp));
KASSERT(bp->b_qindex != QUEUE_NONE,
("bremfree: buffer %p not on a queue.", bp));
BUF_ASSERT_XLOCKED(bp);
bp->b_flags |= B_REMFREE;
}
/*
* bremfreef:
*
* Force an immediate removal from a free list. Used only in nfs when
* it abuses the b_freelist pointer.
*/
void
bremfreef(struct buf *bp)
{
struct bufqueue *bq;
bq = bufqueue_acquire(bp);
bq_remove(bq, bp);
BQ_UNLOCK(bq);
}
static void
bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname)
{
mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);
TAILQ_INIT(&bq->bq_queue);
bq->bq_len = 0;
bq->bq_index = qindex;
bq->bq_subqueue = subqueue;
}
static void
bd_init(struct bufdomain *bd)
{
int i;
bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];
bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");
bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");
for (i = 0; i <= mp_maxid; i++)
bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,
"bufq clean subqueue lock");
mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);
}
/*
* bq_remove:
*
* Removes a buffer from the free list, must be called with the
* correct qlock held.
*/
static void
bq_remove(struct bufqueue *bq, struct buf *bp)
{
CTR3(KTR_BUF, "bq_remove(%p) vp %p flags %X",
bp, bp->b_vp, bp->b_flags);
KASSERT(bp->b_qindex != QUEUE_NONE,
("bq_remove: buffer %p not on a queue.", bp));
KASSERT(bufqueue(bp) == bq,
("bq_remove: Remove buffer %p from wrong queue.", bp));
BQ_ASSERT_LOCKED(bq);
if (bp->b_qindex != QUEUE_EMPTY) {
BUF_ASSERT_XLOCKED(bp);
}
KASSERT(bq->bq_len >= 1,
("queue %d underflow", bp->b_qindex));
TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
bq->bq_len--;
bp->b_qindex = QUEUE_NONE;
bp->b_flags &= ~(B_REMFREE | B_REUSE);
}
static void
bd_flush(struct bufdomain *bd, struct bufqueue *bq)
{
struct buf *bp;
BQ_ASSERT_LOCKED(bq);
if (bq != bd->bd_cleanq) {
BD_LOCK(bd);
while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {
TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,
b_freelist);
bp->b_subqueue = bd->bd_cleanq->bq_subqueue;
}
bd->bd_cleanq->bq_len += bq->bq_len;
bq->bq_len = 0;
}
if (bd->bd_wanted) {
bd->bd_wanted = 0;
wakeup(&bd->bd_wanted);
}
if (bq != bd->bd_cleanq)
BD_UNLOCK(bd);
}
static int
bd_flushall(struct bufdomain *bd)
{
struct bufqueue *bq;
int flushed;
int i;
if (bd->bd_lim == 0)
return (0);
flushed = 0;
for (i = 0; i <= mp_maxid; i++) {
bq = &bd->bd_subq[i];
if (bq->bq_len == 0)
continue;
BQ_LOCK(bq);
bd_flush(bd, bq);
BQ_UNLOCK(bq);
flushed++;
}
return (flushed);
}
static void
bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)
{
struct bufdomain *bd;
if (bp->b_qindex != QUEUE_NONE)
panic("bq_insert: free buffer %p onto another queue?", bp);
bd = bufdomain(bp);
if (bp->b_flags & B_AGE) {
/* Place this buf directly on the real queue. */
if (bq->bq_index == QUEUE_CLEAN)
bq = bd->bd_cleanq;
BQ_LOCK(bq);
TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);
} else {
BQ_LOCK(bq);
TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
}
bp->b_flags &= ~(B_AGE | B_REUSE);
bq->bq_len++;
bp->b_qindex = bq->bq_index;
bp->b_subqueue = bq->bq_subqueue;
/*
* Unlock before we notify so that we don't wakeup a waiter that
* fails a trylock on the buf and sleeps again.
*/
if (unlock)
BUF_UNLOCK(bp);
if (bp->b_qindex == QUEUE_CLEAN) {
/*
* Flush the per-cpu queue and notify any waiters.
*/
if (bd->bd_wanted || (bq != bd->bd_cleanq &&
bq->bq_len >= bd->bd_lim))
bd_flush(bd, bq);
}
BQ_UNLOCK(bq);
}
/*
* bufkva_free:
*
* Free the kva allocation for a buffer.
*
*/
static void
bufkva_free(struct buf *bp)
{
#ifdef INVARIANTS
if (bp->b_kvasize == 0) {
KASSERT(bp->b_kvabase == unmapped_buf &&
bp->b_data == unmapped_buf,
("Leaked KVA space on %p", bp));
} else if (buf_mapped(bp))
BUF_CHECK_MAPPED(bp);
else
BUF_CHECK_UNMAPPED(bp);
#endif
if (bp->b_kvasize == 0)
return;
vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
counter_u64_add(bufkvaspace, -bp->b_kvasize);
counter_u64_add(buffreekvacnt, 1);
bp->b_data = bp->b_kvabase = unmapped_buf;
bp->b_kvasize = 0;
}
/*
* bufkva_alloc:
*
* Allocate the buffer KVA and set b_kvasize and b_kvabase.
*/
static int
bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
{
vm_offset_t addr;
int error;
KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
("Invalid gbflags 0x%x in %s", gbflags, __func__));
bufkva_free(bp);
addr = 0;
error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
if (error != 0) {
/*
* Buffer map is too fragmented. Request the caller
* to defragment the map.
*/
return (error);
}
bp->b_kvabase = (caddr_t)addr;
bp->b_kvasize = maxsize;
counter_u64_add(bufkvaspace, bp->b_kvasize);
if ((gbflags & GB_UNMAPPED) != 0) {
bp->b_data = unmapped_buf;
BUF_CHECK_UNMAPPED(bp);
} else {
bp->b_data = bp->b_kvabase;
BUF_CHECK_MAPPED(bp);
}
return (0);
}
/*
* bufkva_reclaim:
*
* Reclaim buffer kva by freeing buffers holding kva. This is a vmem
* callback that fires to avoid returning failure.
*/
static void
bufkva_reclaim(vmem_t *vmem, int flags)
{
bool done;
int q;
int i;
done = false;
for (i = 0; i < 5; i++) {
for (q = 0; q < buf_domains; q++)
if (buf_recycle(&bdomain[q], true) != 0)
done = true;
if (done)
break;
}
return;
}
/*
* Attempt to initiate asynchronous I/O on read-ahead blocks. We must
* clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
* the buffer is valid and we do not have to do anything.
*/
static void
breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, int cnt,
struct ucred * cred, int flags, void (*ckhashfunc)(struct buf *))
{
struct buf *rabp;
struct thread *td;
int i;
td = curthread;
for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
if (inmem(vp, *rablkno))
continue;
rabp = getblk(vp, *rablkno, *rabsize, 0, 0, 0);
if ((rabp->b_flags & B_CACHE) != 0) {
brelse(rabp);
continue;
}
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(curproc);
racct_add_buf(curproc, rabp, 0);
PROC_UNLOCK(curproc);
}
#endif /* RACCT */
td->td_ru.ru_inblock++;
rabp->b_flags |= B_ASYNC;
rabp->b_flags &= ~B_INVAL;
if ((flags & GB_CKHASH) != 0) {
rabp->b_flags |= B_CKHASH;
rabp->b_ckhashcalc = ckhashfunc;
}
rabp->b_ioflags &= ~BIO_ERROR;
rabp->b_iocmd = BIO_READ;
if (rabp->b_rcred == NOCRED && cred != NOCRED)
rabp->b_rcred = crhold(cred);
vfs_busy_pages(rabp, 0);
BUF_KERNPROC(rabp);
rabp->b_iooffset = dbtob(rabp->b_blkno);
bstrategy(rabp);
}
}
/*
* Entry point for bread() and breadn() via #defines in sys/buf.h.
*
* Get a buffer with the specified data. Look in the cache first. We
* must clear BIO_ERROR and B_INVAL prior to initiating I/O. If B_CACHE
* is set, the buffer is valid and we do not have to do anything, see
* getblk(). Also starts asynchronous I/O on read-ahead blocks.
*
* Always return a NULL buffer pointer (in bpp) when returning an error.
*
* The blkno parameter is the logical block being requested. Normally
* the mapping of logical block number to disk block address is done
* by calling VOP_BMAP(). However, if the mapping is already known, the
* disk block address can be passed using the dblkno parameter. If the
* disk block address is not known, then the same value should be passed
* for blkno and dblkno.
*/
int
breadn_flags(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
daddr_t *rablkno, int *rabsize, int cnt, struct ucred *cred, int flags,
void (*ckhashfunc)(struct buf *), struct buf **bpp)
{
struct buf *bp;
struct thread *td;
int error, readwait, rv;
CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
td = curthread;
/*
* Can only return NULL if GB_LOCK_NOWAIT or GB_SPARSE flags
* are specified.
*/
error = getblkx(vp, blkno, dblkno, size, 0, 0, flags, &bp);
if (error != 0) {
*bpp = NULL;
return (error);
}
KASSERT(blkno == bp->b_lblkno,
("getblkx returned buffer for blkno %jd instead of blkno %jd",
(intmax_t)bp->b_lblkno, (intmax_t)blkno));
flags &= ~GB_NOSPARSE;
*bpp = bp;
/*
* If not found in cache, do some I/O
*/
readwait = 0;
if ((bp->b_flags & B_CACHE) == 0) {
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(td->td_proc);
racct_add_buf(td->td_proc, bp, 0);
PROC_UNLOCK(td->td_proc);
}
#endif /* RACCT */
td->td_ru.ru_inblock++;
bp->b_iocmd = BIO_READ;
bp->b_flags &= ~B_INVAL;
if ((flags & GB_CKHASH) != 0) {
bp->b_flags |= B_CKHASH;
bp->b_ckhashcalc = ckhashfunc;
}
+ if ((flags & GB_CVTENXIO) != 0)
+ bp->b_xflags |= BX_CVTENXIO;
bp->b_ioflags &= ~BIO_ERROR;
if (bp->b_rcred == NOCRED && cred != NOCRED)
bp->b_rcred = crhold(cred);
vfs_busy_pages(bp, 0);
bp->b_iooffset = dbtob(bp->b_blkno);
bstrategy(bp);
++readwait;
}
/*
* Attempt to initiate asynchronous I/O on read-ahead blocks.
*/
breada(vp, rablkno, rabsize, cnt, cred, flags, ckhashfunc);
rv = 0;
if (readwait) {
rv = bufwait(bp);
if (rv != 0) {
brelse(bp);
*bpp = NULL;
}
}
return (rv);
}
/*
* Write, release buffer on completion. (Done by iodone
* if async). Do not bother writing anything if the buffer
* is invalid.
*
* Note that we set B_CACHE here, indicating that buffer is
* fully valid and thus cacheable. This is true even of NFS
* now so we set it generally. This could be set either here
* or in biodone() since the I/O is synchronous. We put it
* here.
*/
int
bufwrite(struct buf *bp)
{
int oldflags;
struct vnode *vp;
long space;
int vp_md;
CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
bp->b_flags |= B_INVAL | B_RELBUF;
bp->b_flags &= ~B_CACHE;
brelse(bp);
return (ENXIO);
}
if (bp->b_flags & B_INVAL) {
brelse(bp);
return (0);
}
if (bp->b_flags & B_BARRIER)
atomic_add_long(&barrierwrites, 1);
oldflags = bp->b_flags;
KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
("FFS background buffer should not get here %p", bp));
vp = bp->b_vp;
if (vp)
vp_md = vp->v_vflag & VV_MD;
else
vp_md = 0;
/*
* Mark the buffer clean. Increment the bufobj write count
* before bundirty() call, to prevent other thread from seeing
* empty dirty list and zero counter for writes in progress,
* falsely indicating that the bufobj is clean.
*/
bufobj_wref(bp->b_bufobj);
bundirty(bp);
bp->b_flags &= ~B_DONE;
bp->b_ioflags &= ~BIO_ERROR;
bp->b_flags |= B_CACHE;
bp->b_iocmd = BIO_WRITE;
vfs_busy_pages(bp, 1);
/*
* Normal bwrites pipeline writes
*/
bp->b_runningbufspace = bp->b_bufsize;
space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(curproc);
racct_add_buf(curproc, bp, 1);
PROC_UNLOCK(curproc);
}
#endif /* RACCT */
curthread->td_ru.ru_oublock++;
if (oldflags & B_ASYNC)
BUF_KERNPROC(bp);
bp->b_iooffset = dbtob(bp->b_blkno);
buf_track(bp, __func__);
bstrategy(bp);
if ((oldflags & B_ASYNC) == 0) {
int rtval = bufwait(bp);
brelse(bp);
return (rtval);
} else if (space > hirunningspace) {
/*
* don't allow the async write to saturate the I/O
* system. We will not deadlock here because
* we are blocking waiting for I/O that is already in-progress
* to complete. We do not block here if it is the update
* or syncer daemon trying to clean up as that can lead
* to deadlock.
*/
if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
waitrunningbufspace();
}
return (0);
}
void
bufbdflush(struct bufobj *bo, struct buf *bp)
{
struct buf *nbp;
if (bo->bo_dirty.bv_cnt > dirtybufthresh + 10) {
(void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
altbufferflushes++;
} else if (bo->bo_dirty.bv_cnt > dirtybufthresh) {
BO_LOCK(bo);
/*
* Try to find a buffer to flush.
*/
TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
if ((nbp->b_vflags & BV_BKGRDINPROG) ||
BUF_LOCK(nbp,
LK_EXCLUSIVE | LK_NOWAIT, NULL))
continue;
if (bp == nbp)
panic("bdwrite: found ourselves");
BO_UNLOCK(bo);
/* Don't countdeps with the bo lock held. */
if (buf_countdeps(nbp, 0)) {
BO_LOCK(bo);
BUF_UNLOCK(nbp);
continue;
}
if (nbp->b_flags & B_CLUSTEROK) {
vfs_bio_awrite(nbp);
} else {
bremfree(nbp);
bawrite(nbp);
}
dirtybufferflushes++;
break;
}
if (nbp == NULL)
BO_UNLOCK(bo);
}
}
/*
* Delayed write. (Buffer is marked dirty). Do not bother writing
* anything if the buffer is marked invalid.
*
* Note that since the buffer must be completely valid, we can safely
* set B_CACHE. In fact, we have to set B_CACHE here rather then in
* biodone() in order to prevent getblk from writing the buffer
* out synchronously.
*/
void
bdwrite(struct buf *bp)
{
struct thread *td = curthread;
struct vnode *vp;
struct bufobj *bo;
CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
KASSERT((bp->b_flags & B_BARRIER) == 0,
("Barrier request in delayed write %p", bp));
if (bp->b_flags & B_INVAL) {
brelse(bp);
return;
}
/*
* If we have too many dirty buffers, don't create any more.
* If we are wildly over our limit, then force a complete
* cleanup. Otherwise, just keep the situation from getting
* out of control. Note that we have to avoid a recursive
* disaster and not try to clean up after our own cleanup!
*/
vp = bp->b_vp;
bo = bp->b_bufobj;
if ((td->td_pflags & (TDP_COWINPROGRESS|TDP_INBDFLUSH)) == 0) {
td->td_pflags |= TDP_INBDFLUSH;
BO_BDFLUSH(bo, bp);
td->td_pflags &= ~TDP_INBDFLUSH;
} else
recursiveflushes++;
bdirty(bp);
/*
* Set B_CACHE, indicating that the buffer is fully valid. This is
* true even of NFS now.
*/
bp->b_flags |= B_CACHE;
/*
* This bmap keeps the system from needing to do the bmap later,
* perhaps when the system is attempting to do a sync. Since it
* is likely that the indirect block -- or whatever other datastructure
* that the filesystem needs is still in memory now, it is a good
* thing to do this. Note also, that if the pageout daemon is
* requesting a sync -- there might not be enough memory to do
* the bmap then... So, this is important to do.
*/
if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
}
buf_track(bp, __func__);
/*
* Set the *dirty* buffer range based upon the VM system dirty
* pages.
*
* Mark the buffer pages as clean. We need to do this here to
* satisfy the vnode_pager and the pageout daemon, so that it
* thinks that the pages have been "cleaned". Note that since
* the pages are in a delayed write buffer -- the VFS layer
* "will" see that the pages get written out on the next sync,
* or perhaps the cluster will be completed.
*/
vfs_clean_pages_dirty_buf(bp);
bqrelse(bp);
/*
* note: we cannot initiate I/O from a bdwrite even if we wanted to,
* due to the softdep code.
*/
}
/*
* bdirty:
*
* Turn buffer into delayed write request. We must clear BIO_READ and
* B_RELBUF, and we must set B_DELWRI. We reassign the buffer to
* itself to properly update it in the dirty/clean lists. We mark it
* B_DONE to ensure that any asynchronization of the buffer properly
* clears B_DONE ( else a panic will occur later ).
*
* bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
* might have been set pre-getblk(). Unlike bwrite/bdwrite, bdirty()
* should only be called if the buffer is known-good.
*
* Since the buffer is not on a queue, we do not update the numfreebuffers
* count.
*
* The buffer must be on QUEUE_NONE.
*/
void
bdirty(struct buf *bp)
{
CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
bp, bp->b_vp, bp->b_flags);
KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
bp->b_flags &= ~(B_RELBUF);
bp->b_iocmd = BIO_WRITE;
if ((bp->b_flags & B_DELWRI) == 0) {
bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
reassignbuf(bp);
bdirtyadd(bp);
}
}
/*
* bundirty:
*
* Clear B_DELWRI for buffer.
*
* Since the buffer is not on a queue, we do not update the numfreebuffers
* count.
*
* The buffer must be on QUEUE_NONE.
*/
void
bundirty(struct buf *bp)
{
CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
if (bp->b_flags & B_DELWRI) {
bp->b_flags &= ~B_DELWRI;
reassignbuf(bp);
bdirtysub(bp);
}
/*
* Since it is now being written, we can clear its deferred write flag.
*/
bp->b_flags &= ~B_DEFERRED;
}
/*
* bawrite:
*
* Asynchronous write. Start output on a buffer, but do not wait for
* it to complete. The buffer is released when the output completes.
*
* bwrite() ( or the VOP routine anyway ) is responsible for handling
* B_INVAL buffers. Not us.
*/
void
bawrite(struct buf *bp)
{
bp->b_flags |= B_ASYNC;
(void) bwrite(bp);
}
/*
* babarrierwrite:
*
* Asynchronous barrier write. Start output on a buffer, but do not
* wait for it to complete. Place a write barrier after this write so
* that this buffer and all buffers written before it are committed to
* the disk before any buffers written after this write are committed
* to the disk. The buffer is released when the output completes.
*/
void
babarrierwrite(struct buf *bp)
{
bp->b_flags |= B_ASYNC | B_BARRIER;
(void) bwrite(bp);
}
/*
* bbarrierwrite:
*
* Synchronous barrier write. Start output on a buffer and wait for
* it to complete. Place a write barrier after this write so that
* this buffer and all buffers written before it are committed to
* the disk before any buffers written after this write are committed
* to the disk. The buffer is released when the output completes.
*/
int
bbarrierwrite(struct buf *bp)
{
bp->b_flags |= B_BARRIER;
return (bwrite(bp));
}
/*
* bwillwrite:
*
* Called prior to the locking of any vnodes when we are expecting to
* write. We do not want to starve the buffer cache with too many
* dirty buffers so we block here. By blocking prior to the locking
* of any vnodes we attempt to avoid the situation where a locked vnode
* prevents the various system daemons from flushing related buffers.
*/
void
bwillwrite(void)
{
if (buf_dirty_count_severe()) {
mtx_lock(&bdirtylock);
while (buf_dirty_count_severe()) {
bdirtywait = 1;
msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
"flswai", 0);
}
mtx_unlock(&bdirtylock);
}
}
/*
* Return true if we have too many dirty buffers.
*/
int
buf_dirty_count_severe(void)
{
return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty));
}
/*
* brelse:
*
* Release a busy buffer and, if requested, free its resources. The
* buffer will be stashed in the appropriate bufqueue[] allowing it
* to be accessed later as a cache entity or reused for other purposes.
*/
void
brelse(struct buf *bp)
{
struct mount *v_mnt;
int qindex;
/*
* Many functions erroneously call brelse with a NULL bp under rare
* error conditions. Simply return when called with a NULL bp.
*/
if (bp == NULL)
return;
CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
bp, bp->b_vp, bp->b_flags);
KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
("brelse: non-VMIO buffer marked NOREUSE"));
if (BUF_LOCKRECURSED(bp)) {
/*
* Do not process, in particular, do not handle the
* B_INVAL/B_RELBUF and do not release to free list.
*/
BUF_UNLOCK(bp);
return;
}
if (bp->b_flags & B_MANAGED) {
bqrelse(bp);
return;
}
if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
BO_LOCK(bp->b_bufobj);
bp->b_vflags &= ~BV_BKGRDERR;
BO_UNLOCK(bp->b_bufobj);
bdirty(bp);
}
if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
(bp->b_flags & B_INVALONERR)) {
/*
* Forced invalidation of dirty buffer contents, to be used
* after a failed write in the rare case that the loss of the
* contents is acceptable. The buffer is invalidated and
* freed.
*/
bp->b_flags |= B_INVAL | B_RELBUF | B_NOCACHE;
bp->b_flags &= ~(B_ASYNC | B_CACHE);
}
if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
(bp->b_error != ENXIO || !LIST_EMPTY(&bp->b_dep)) &&
!(bp->b_flags & B_INVAL)) {
/*
* Failed write, redirty. All errors except ENXIO (which
* means the device is gone) are treated as being
* transient.
*
* XXX Treating EIO as transient is not correct; the
* contract with the local storage device drivers is that
* they will only return EIO once the I/O is no longer
* retriable. Network I/O also respects this through the
* guarantees of TCP and/or the internal retries of NFS.
* ENOMEM might be transient, but we also have no way of
* knowing when its ok to retry/reschedule. In general,
* this entire case should be made obsolete through better
* error handling/recovery and resource scheduling.
*
* Do this also for buffers that failed with ENXIO, but have
* non-empty dependencies - the soft updates code might need
* to access the buffer to untangle them.
*
* Must clear BIO_ERROR to prevent pages from being scrapped.
*/
bp->b_ioflags &= ~BIO_ERROR;
bdirty(bp);
} else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
(bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
/*
* Either a failed read I/O, or we were asked to free or not
* cache the buffer, or we failed to write to a device that's
* no longer present.
*/
bp->b_flags |= B_INVAL;
if (!LIST_EMPTY(&bp->b_dep))
buf_deallocate(bp);
if (bp->b_flags & B_DELWRI)
bdirtysub(bp);
bp->b_flags &= ~(B_DELWRI | B_CACHE);
if ((bp->b_flags & B_VMIO) == 0) {
allocbuf(bp, 0);
if (bp->b_vp)
brelvp(bp);
}
}
/*
* We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_truncate()
* is called with B_DELWRI set, the underlying pages may wind up
* getting freed causing a previous write (bdwrite()) to get 'lost'
* because pages associated with a B_DELWRI bp are marked clean.
*
* We still allow the B_INVAL case to call vfs_vmio_truncate(), even
* if B_DELWRI is set.
*/
if (bp->b_flags & B_DELWRI)
bp->b_flags &= ~B_RELBUF;
/*
* VMIO buffer rundown. It is not very necessary to keep a VMIO buffer
* constituted, not even NFS buffers now. Two flags effect this. If
* B_INVAL, the struct buf is invalidated but the VM object is kept
* around ( i.e. so it is trivial to reconstitute the buffer later ).
*
* If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
* invalidated. BIO_ERROR cannot be set for a failed write unless the
* buffer is also B_INVAL because it hits the re-dirtying code above.
*
* Normally we can do this whether a buffer is B_DELWRI or not. If
* the buffer is an NFS buffer, it is tracking piecemeal writes or
* the commit state and we cannot afford to lose the buffer. If the
* buffer has a background write in progress, we need to keep it
* around to prevent it from being reconstituted and starting a second
* background write.
*/
v_mnt = bp->b_vp != NULL ? bp->b_vp->v_mount : NULL;
if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
(bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
(v_mnt == NULL || (v_mnt->mnt_vfc->vfc_flags & VFCF_NETWORK) == 0 ||
vn_isdisk(bp->b_vp, NULL) || (bp->b_flags & B_DELWRI) == 0)) {
vfs_vmio_invalidate(bp);
allocbuf(bp, 0);
}
if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
(bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
allocbuf(bp, 0);
bp->b_flags &= ~B_NOREUSE;
if (bp->b_vp != NULL)
brelvp(bp);
}
/*
* If the buffer has junk contents signal it and eventually
* clean up B_DELWRI and diassociate the vnode so that gbincore()
* doesn't find it.
*/
if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
(bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
bp->b_flags |= B_INVAL;
if (bp->b_flags & B_INVAL) {
if (bp->b_flags & B_DELWRI)
bundirty(bp);
if (bp->b_vp)
brelvp(bp);
}
buf_track(bp, __func__);
/* buffers with no memory */
if (bp->b_bufsize == 0) {
buf_free(bp);
return;
}
/* buffers with junk contents */
if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
(bp->b_ioflags & BIO_ERROR)) {
bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
if (bp->b_vflags & BV_BKGRDINPROG)
panic("losing buffer 2");
qindex = QUEUE_CLEAN;
bp->b_flags |= B_AGE;
/* remaining buffers */
} else if (bp->b_flags & B_DELWRI)
qindex = QUEUE_DIRTY;
else
qindex = QUEUE_CLEAN;
if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
panic("brelse: not dirty");
bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_RELBUF | B_DIRECT);
+ bp->b_xflags &= ~(BX_CVTENXIO);
/* binsfree unlocks bp. */
binsfree(bp, qindex);
}
/*
* Release a buffer back to the appropriate queue but do not try to free
* it. The buffer is expected to be used again soon.
*
* bqrelse() is used by bdwrite() to requeue a delayed write, and used by
* biodone() to requeue an async I/O on completion. It is also used when
* known good buffers need to be requeued but we think we may need the data
* again soon.
*
* XXX we should be able to leave the B_RELBUF hint set on completion.
*/
void
bqrelse(struct buf *bp)
{
int qindex;
CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
qindex = QUEUE_NONE;
if (BUF_LOCKRECURSED(bp)) {
/* do not release to free list */
BUF_UNLOCK(bp);
return;
}
bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
+ bp->b_xflags &= ~(BX_CVTENXIO);
if (bp->b_flags & B_MANAGED) {
if (bp->b_flags & B_REMFREE)
bremfreef(bp);
goto out;
}
/* buffers with stale but valid contents */
if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
BV_BKGRDERR)) == BV_BKGRDERR) {
BO_LOCK(bp->b_bufobj);
bp->b_vflags &= ~BV_BKGRDERR;
BO_UNLOCK(bp->b_bufobj);
qindex = QUEUE_DIRTY;
} else {
if ((bp->b_flags & B_DELWRI) == 0 &&
(bp->b_xflags & BX_VNDIRTY))
panic("bqrelse: not dirty");
if ((bp->b_flags & B_NOREUSE) != 0) {
brelse(bp);
return;
}
qindex = QUEUE_CLEAN;
}
buf_track(bp, __func__);
/* binsfree unlocks bp. */
binsfree(bp, qindex);
return;
out:
buf_track(bp, __func__);
/* unlock */
BUF_UNLOCK(bp);
}
/*
* Complete I/O to a VMIO backed page. Validate the pages as appropriate,
* restore bogus pages.
*/
static void
vfs_vmio_iodone(struct buf *bp)
{
vm_ooffset_t foff;
vm_page_t m;
vm_object_t obj;
struct vnode *vp __unused;
int i, iosize, resid;
bool bogus;
obj = bp->b_bufobj->bo_object;
KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages,
("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
blockcount_read(&obj->paging_in_progress), bp->b_npages));
vp = bp->b_vp;
VNPASS(vp->v_holdcnt > 0, vp);
VNPASS(vp->v_object != NULL, vp);
foff = bp->b_offset;
KASSERT(bp->b_offset != NOOFFSET,
("vfs_vmio_iodone: bp %p has no buffer offset", bp));
bogus = false;
iosize = bp->b_bcount - bp->b_resid;
for (i = 0; i < bp->b_npages; i++) {
resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
if (resid > iosize)
resid = iosize;
/*
* cleanup bogus pages, restoring the originals
*/
m = bp->b_pages[i];
if (m == bogus_page) {
bogus = true;
m = vm_page_relookup(obj, OFF_TO_IDX(foff));
if (m == NULL)
panic("biodone: page disappeared!");
bp->b_pages[i] = m;
} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
/*
* In the write case, the valid and clean bits are
* already changed correctly ( see bdwrite() ), so we
* only need to do this here in the read case.
*/
KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
resid)) == 0, ("vfs_vmio_iodone: page %p "
"has unexpected dirty bits", m));
vfs_page_set_valid(bp, foff, m);
}
KASSERT(OFF_TO_IDX(foff) == m->pindex,
("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
(intmax_t)foff, (uintmax_t)m->pindex));
vm_page_sunbusy(m);
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
iosize -= resid;
}
vm_object_pip_wakeupn(obj, bp->b_npages);
if (bogus && buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
}
}
/*
* Perform page invalidation when a buffer is released. The fully invalid
* pages will be reclaimed later in vfs_vmio_truncate().
*/
static void
vfs_vmio_invalidate(struct buf *bp)
{
vm_object_t obj;
vm_page_t m;
int flags, i, resid, poffset, presid;
if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
} else
BUF_CHECK_UNMAPPED(bp);
/*
* Get the base offset and length of the buffer. Note that
* in the VMIO case if the buffer block size is not
* page-aligned then b_data pointer may not be page-aligned.
* But our b_pages[] array *IS* page aligned.
*
* block sizes less then DEV_BSIZE (usually 512) are not
* supported due to the page granularity bits (m->valid,
* m->dirty, etc...).
*
* See man buf(9) for more information
*/
flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
obj = bp->b_bufobj->bo_object;
resid = bp->b_bufsize;
poffset = bp->b_offset & PAGE_MASK;
VM_OBJECT_WLOCK(obj);
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
if (m == bogus_page)
panic("vfs_vmio_invalidate: Unexpected bogus page.");
bp->b_pages[i] = NULL;
presid = resid > (PAGE_SIZE - poffset) ?
(PAGE_SIZE - poffset) : resid;
KASSERT(presid >= 0, ("brelse: extra page"));
vm_page_busy_acquire(m, VM_ALLOC_SBUSY);
if (pmap_page_wired_mappings(m) == 0)
vm_page_set_invalid(m, poffset, presid);
vm_page_sunbusy(m);
vm_page_release_locked(m, flags);
resid -= presid;
poffset = 0;
}
VM_OBJECT_WUNLOCK(obj);
bp->b_npages = 0;
}
/*
* Page-granular truncation of an existing VMIO buffer.
*/
static void
vfs_vmio_truncate(struct buf *bp, int desiredpages)
{
vm_object_t obj;
vm_page_t m;
int flags, i;
if (bp->b_npages == desiredpages)
return;
if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
(desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
} else
BUF_CHECK_UNMAPPED(bp);
/*
* The object lock is needed only if we will attempt to free pages.
*/
flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
if ((bp->b_flags & B_DIRECT) != 0) {
flags |= VPR_TRYFREE;
obj = bp->b_bufobj->bo_object;
VM_OBJECT_WLOCK(obj);
} else {
obj = NULL;
}
for (i = desiredpages; i < bp->b_npages; i++) {
m = bp->b_pages[i];
KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
bp->b_pages[i] = NULL;
if (obj != NULL)
vm_page_release_locked(m, flags);
else
vm_page_release(m, flags);
}
if (obj != NULL)
VM_OBJECT_WUNLOCK(obj);
bp->b_npages = desiredpages;
}
/*
* Byte granular extension of VMIO buffers.
*/
static void
vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
{
/*
* We are growing the buffer, possibly in a
* byte-granular fashion.
*/
vm_object_t obj;
vm_offset_t toff;
vm_offset_t tinc;
vm_page_t m;
/*
* Step 1, bring in the VM pages from the object, allocating
* them if necessary. We must clear B_CACHE if these pages
* are not valid for the range covered by the buffer.
*/
obj = bp->b_bufobj->bo_object;
if (bp->b_npages < desiredpages) {
/*
* We must allocate system pages since blocking
* here could interfere with paging I/O, no
* matter which process we are.
*
* Only exclusive busy can be tested here.
* Blocking on shared busy might lead to
* deadlocks once allocbuf() is called after
* pages are vfs_busy_pages().
*/
(void)vm_page_grab_pages_unlocked(obj,
OFF_TO_IDX(bp->b_offset) + bp->b_npages,
VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY |
VM_ALLOC_NOBUSY | VM_ALLOC_WIRED,
&bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
bp->b_npages = desiredpages;
}
/*
* Step 2. We've loaded the pages into the buffer,
* we have to figure out if we can still have B_CACHE
* set. Note that B_CACHE is set according to the
* byte-granular range ( bcount and size ), not the
* aligned range ( newbsize ).
*
* The VM test is against m->valid, which is DEV_BSIZE
* aligned. Needless to say, the validity of the data
* needs to also be DEV_BSIZE aligned. Note that this
* fails with NFS if the server or some other client
* extends the file's EOF. If our buffer is resized,
* B_CACHE may remain set! XXX
*/
toff = bp->b_bcount;
tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
while ((bp->b_flags & B_CACHE) && toff < size) {
vm_pindex_t pi;
if (tinc > (size - toff))
tinc = size - toff;
pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
m = bp->b_pages[pi];
vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
toff += tinc;
tinc = PAGE_SIZE;
}
/*
* Step 3, fixup the KVA pmap.
*/
if (buf_mapped(bp))
bpmap_qenter(bp);
else
BUF_CHECK_UNMAPPED(bp);
}
/*
* Check to see if a block at a particular lbn is available for a clustered
* write.
*/
static int
vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
{
struct buf *bpa;
int match;
match = 0;
/* If the buf isn't in core skip it */
if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
return (0);
/* If the buf is busy we don't want to wait for it */
if (BUF_LOCK(bpa, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
return (0);
/* Only cluster with valid clusterable delayed write buffers */
if ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) !=
(B_DELWRI | B_CLUSTEROK))
goto done;
if (bpa->b_bufsize != size)
goto done;
/*
* Check to see if it is in the expected place on disk and that the
* block has been mapped.
*/
if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
match = 1;
done:
BUF_UNLOCK(bpa);
return (match);
}
/*
* vfs_bio_awrite:
*
* Implement clustered async writes for clearing out B_DELWRI buffers.
* This is much better then the old way of writing only one buffer at
* a time. Note that we may not be presented with the buffers in the
* correct order, so we search for the cluster in both directions.
*/
int
vfs_bio_awrite(struct buf *bp)
{
struct bufobj *bo;
int i;
int j;
daddr_t lblkno = bp->b_lblkno;
struct vnode *vp = bp->b_vp;
int ncl;
int nwritten;
int size;
int maxcl;
int gbflags;
bo = &vp->v_bufobj;
gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
/*
* right now we support clustered writing only to regular files. If
* we find a clusterable block we could be in the middle of a cluster
* rather then at the beginning.
*/
if ((vp->v_type == VREG) &&
(vp->v_mount != 0) && /* Only on nodes that have the size info */
(bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
size = vp->v_mount->mnt_stat.f_iosize;
maxcl = MAXPHYS / size;
BO_RLOCK(bo);
for (i = 1; i < maxcl; i++)
if (vfs_bio_clcheck(vp, size, lblkno + i,
bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
break;
for (j = 1; i + j <= maxcl && j <= lblkno; j++)
if (vfs_bio_clcheck(vp, size, lblkno - j,
bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
break;
BO_RUNLOCK(bo);
--j;
ncl = i + j;
/*
* this is a possible cluster write
*/
if (ncl != 1) {
BUF_UNLOCK(bp);
nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
gbflags);
return (nwritten);
}
}
bremfree(bp);
bp->b_flags |= B_ASYNC;
/*
* default (old) behavior, writing out only one block
*
* XXX returns b_bufsize instead of b_bcount for nwritten?
*/
nwritten = bp->b_bufsize;
(void) bwrite(bp);
return (nwritten);
}
/*
* getnewbuf_kva:
*
* Allocate KVA for an empty buf header according to gbflags.
*/
static int
getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
{
if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_UNMAPPED) {
/*
* In order to keep fragmentation sane we only allocate kva
* in BKVASIZE chunks. XXX with vmem we can do page size.
*/
maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
if (maxsize != bp->b_kvasize &&
bufkva_alloc(bp, maxsize, gbflags))
return (ENOSPC);
}
return (0);
}
/*
* getnewbuf:
*
* Find and initialize a new buffer header, freeing up existing buffers
* in the bufqueues as necessary. The new buffer is returned locked.
*
* We block if:
* We have insufficient buffer headers
* We have insufficient buffer space
* buffer_arena is too fragmented ( space reservation fails )
* If we have to flush dirty buffers ( but we try to avoid this )
*
* The caller is responsible for releasing the reserved bufspace after
* allocbuf() is called.
*/
static struct buf *
getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
{
struct bufdomain *bd;
struct buf *bp;
bool metadata, reserved;
bp = NULL;
KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
if (!unmapped_buf_allowed)
gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
vp->v_type == VCHR)
metadata = true;
else
metadata = false;
if (vp == NULL)
bd = &bdomain[0];
else
bd = &bdomain[vp->v_bufobj.bo_domain];
counter_u64_add(getnewbufcalls, 1);
reserved = false;
do {
if (reserved == false &&
bufspace_reserve(bd, maxsize, metadata) != 0) {
counter_u64_add(getnewbufrestarts, 1);
continue;
}
reserved = true;
if ((bp = buf_alloc(bd)) == NULL) {
counter_u64_add(getnewbufrestarts, 1);
continue;
}
if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
return (bp);
break;
} while (buf_recycle(bd, false) == 0);
if (reserved)
bufspace_release(bd, maxsize);
if (bp != NULL) {
bp->b_flags |= B_INVAL;
brelse(bp);
}
bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);
return (NULL);
}
/*
* buf_daemon:
*
* buffer flushing daemon. Buffers are normally flushed by the
* update daemon but if it cannot keep up this process starts to
* take the load in an attempt to prevent getnewbuf() from blocking.
*/
static struct kproc_desc buf_kp = {
"bufdaemon",
buf_daemon,
&bufdaemonproc
};
SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
static int
buf_flush(struct vnode *vp, struct bufdomain *bd, int target)
{
int flushed;
flushed = flushbufqueues(vp, bd, target, 0);
if (flushed == 0) {
/*
* Could not find any buffers without rollback
* dependencies, so just write the first one
* in the hopes of eventually making progress.
*/
if (vp != NULL && target > 2)
target /= 2;
flushbufqueues(vp, bd, target, 1);
}
return (flushed);
}
static void
buf_daemon()
{
struct bufdomain *bd;
int speedupreq;
int lodirty;
int i;
/*
* This process needs to be suspended prior to shutdown sync.
*/
EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
SHUTDOWN_PRI_LAST + 100);
/*
* Start the buf clean daemons as children threads.
*/
for (i = 0 ; i < buf_domains; i++) {
int error;
error = kthread_add((void (*)(void *))bufspace_daemon,
&bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i);
if (error)
panic("error %d spawning bufspace daemon", error);
}
/*
* This process is allowed to take the buffer cache to the limit
*/
curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
mtx_lock(&bdlock);
for (;;) {
bd_request = 0;
mtx_unlock(&bdlock);
kthread_suspend_check();
/*
* Save speedupreq for this pass and reset to capture new
* requests.
*/
speedupreq = bd_speedupreq;
bd_speedupreq = 0;
/*
* Flush each domain sequentially according to its level and
* the speedup request.
*/
for (i = 0; i < buf_domains; i++) {
bd = &bdomain[i];
if (speedupreq)
lodirty = bd->bd_numdirtybuffers / 2;
else
lodirty = bd->bd_lodirtybuffers;
while (bd->bd_numdirtybuffers > lodirty) {
if (buf_flush(NULL, bd,
bd->bd_numdirtybuffers - lodirty) == 0)
break;
kern_yield(PRI_USER);
}
}
/*
* Only clear bd_request if we have reached our low water
* mark. The buf_daemon normally waits 1 second and
* then incrementally flushes any dirty buffers that have
* built up, within reason.
*
* If we were unable to hit our low water mark and couldn't
* find any flushable buffers, we sleep for a short period
* to avoid endless loops on unlockable buffers.
*/
mtx_lock(&bdlock);
if (!BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) {
/*
* We reached our low water mark, reset the
* request and sleep until we are needed again.
* The sleep is just so the suspend code works.
*/
bd_request = 0;
/*
* Do an extra wakeup in case dirty threshold
* changed via sysctl and the explicit transition
* out of shortfall was missed.
*/
bdirtywakeup();
if (runningbufspace <= lorunningspace)
runningwakeup();
msleep(&bd_request, &bdlock, PVM, "psleep", hz);
} else {
/*
* We couldn't find any flushable dirty buffers but
* still have too many dirty buffers, we
* have to sleep and try again. (rare)
*/
msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
}
}
}
/*
* flushbufqueues:
*
* Try to flush a buffer in the dirty queue. We must be careful to
* free up B_INVAL buffers instead of write them, which NFS is
* particularly sensitive to.
*/
static int flushwithdeps = 0;
SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW | CTLFLAG_STATS,
&flushwithdeps, 0,
"Number of buffers flushed with dependecies that require rollbacks");
static int
flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target,
int flushdeps)
{
struct bufqueue *bq;
struct buf *sentinel;
struct vnode *vp;
struct mount *mp;
struct buf *bp;
int hasdeps;
int flushed;
int error;
bool unlock;
flushed = 0;
bq = &bd->bd_dirtyq;
bp = NULL;
sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
sentinel->b_qindex = QUEUE_SENTINEL;
BQ_LOCK(bq);
TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);
BQ_UNLOCK(bq);
while (flushed != target) {
maybe_yield();
BQ_LOCK(bq);
bp = TAILQ_NEXT(sentinel, b_freelist);
if (bp != NULL) {
TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,
b_freelist);
} else {
BQ_UNLOCK(bq);
break;
}
/*
* Skip sentinels inserted by other invocations of the
* flushbufqueues(), taking care to not reorder them.
*
* Only flush the buffers that belong to the
* vnode locked by the curthread.
*/
if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
bp->b_vp != lvp)) {
BQ_UNLOCK(bq);
continue;
}
error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
BQ_UNLOCK(bq);
if (error != 0)
continue;
/*
* BKGRDINPROG can only be set with the buf and bufobj
* locks both held. We tolerate a race to clear it here.
*/
if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
(bp->b_flags & B_DELWRI) == 0) {
BUF_UNLOCK(bp);
continue;
}
if (bp->b_flags & B_INVAL) {
bremfreef(bp);
brelse(bp);
flushed++;
continue;
}
if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
if (flushdeps == 0) {
BUF_UNLOCK(bp);
continue;
}
hasdeps = 1;
} else
hasdeps = 0;
/*
* We must hold the lock on a vnode before writing
* one of its buffers. Otherwise we may confuse, or
* in the case of a snapshot vnode, deadlock the
* system.
*
* The lock order here is the reverse of the normal
* of vnode followed by buf lock. This is ok because
* the NOWAIT will prevent deadlock.
*/
vp = bp->b_vp;
if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
BUF_UNLOCK(bp);
continue;
}
if (lvp == NULL) {
unlock = true;
error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT);
} else {
ASSERT_VOP_LOCKED(vp, "getbuf");
unlock = false;
error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
vn_lock(vp, LK_TRYUPGRADE);
}
if (error == 0) {
CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
bp, bp->b_vp, bp->b_flags);
if (curproc == bufdaemonproc) {
vfs_bio_awrite(bp);
} else {
bremfree(bp);
bwrite(bp);
counter_u64_add(notbufdflushes, 1);
}
vn_finished_write(mp);
if (unlock)
VOP_UNLOCK(vp);
flushwithdeps += hasdeps;
flushed++;
/*
* Sleeping on runningbufspace while holding
* vnode lock leads to deadlock.
*/
if (curproc == bufdaemonproc &&
runningbufspace > hirunningspace)
waitrunningbufspace();
continue;
}
vn_finished_write(mp);
BUF_UNLOCK(bp);
}
BQ_LOCK(bq);
TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
BQ_UNLOCK(bq);
free(sentinel, M_TEMP);
return (flushed);
}
/*
* Check to see if a block is currently memory resident.
*/
struct buf *
incore(struct bufobj *bo, daddr_t blkno)
{
struct buf *bp;
BO_RLOCK(bo);
bp = gbincore(bo, blkno);
BO_RUNLOCK(bo);
return (bp);
}
/*
* Returns true if no I/O is needed to access the
* associated VM object. This is like incore except
* it also hunts around in the VM system for the data.
*/
static int
inmem(struct vnode * vp, daddr_t blkno)
{
vm_object_t obj;
vm_offset_t toff, tinc, size;
vm_page_t m;
vm_ooffset_t off;
ASSERT_VOP_LOCKED(vp, "inmem");
if (incore(&vp->v_bufobj, blkno))
return 1;
if (vp->v_mount == NULL)
return 0;
obj = vp->v_object;
if (obj == NULL)
return (0);
size = PAGE_SIZE;
if (size > vp->v_mount->mnt_stat.f_iosize)
size = vp->v_mount->mnt_stat.f_iosize;
off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
VM_OBJECT_RLOCK(obj);
for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
if (!m)
goto notinmem;
tinc = size;
if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
if (vm_page_is_valid(m,
(vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
goto notinmem;
}
VM_OBJECT_RUNLOCK(obj);
return 1;
notinmem:
VM_OBJECT_RUNLOCK(obj);
return (0);
}
/*
* Set the dirty range for a buffer based on the status of the dirty
* bits in the pages comprising the buffer. The range is limited
* to the size of the buffer.
*
* Tell the VM system that the pages associated with this buffer
* are clean. This is used for delayed writes where the data is
* going to go to disk eventually without additional VM intevention.
*
* Note that while we only really need to clean through to b_bcount, we
* just go ahead and clean through to b_bufsize.
*/
static void
vfs_clean_pages_dirty_buf(struct buf *bp)
{
vm_ooffset_t foff, noff, eoff;
vm_page_t m;
int i;
if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
return;
foff = bp->b_offset;
KASSERT(bp->b_offset != NOOFFSET,
("vfs_clean_pages_dirty_buf: no buffer offset"));
vfs_busy_pages_acquire(bp);
vfs_setdirty_range(bp);
for (i = 0; i < bp->b_npages; i++) {
noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
eoff = noff;
if (eoff > bp->b_offset + bp->b_bufsize)
eoff = bp->b_offset + bp->b_bufsize;
m = bp->b_pages[i];
vfs_page_set_validclean(bp, foff, m);
/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
foff = noff;
}
vfs_busy_pages_release(bp);
}
static void
vfs_setdirty_range(struct buf *bp)
{
vm_offset_t boffset;
vm_offset_t eoffset;
int i;
/*
* test the pages to see if they have been modified directly
* by users through the VM system.
*/
for (i = 0; i < bp->b_npages; i++)
vm_page_test_dirty(bp->b_pages[i]);
/*
* Calculate the encompassing dirty range, boffset and eoffset,
* (eoffset - boffset) bytes.
*/
for (i = 0; i < bp->b_npages; i++) {
if (bp->b_pages[i]->dirty)
break;
}
boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
for (i = bp->b_npages - 1; i >= 0; --i) {
if (bp->b_pages[i]->dirty) {
break;
}
}
eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
/*
* Fit it to the buffer.
*/
if (eoffset > bp->b_bcount)
eoffset = bp->b_bcount;
/*
* If we have a good dirty range, merge with the existing
* dirty range.
*/
if (boffset < eoffset) {
if (bp->b_dirtyoff > boffset)
bp->b_dirtyoff = boffset;
if (bp->b_dirtyend < eoffset)
bp->b_dirtyend = eoffset;
}
}
/*
* Allocate the KVA mapping for an existing buffer.
* If an unmapped buffer is provided but a mapped buffer is requested, take
* also care to properly setup mappings between pages and KVA.
*/
static void
bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
{
int bsize, maxsize, need_mapping, need_kva;
off_t offset;
need_mapping = bp->b_data == unmapped_buf &&
(gbflags & GB_UNMAPPED) == 0;
need_kva = bp->b_kvabase == unmapped_buf &&
bp->b_data == unmapped_buf &&
(gbflags & GB_KVAALLOC) != 0;
if (!need_mapping && !need_kva)
return;
BUF_CHECK_UNMAPPED(bp);
if (need_mapping && bp->b_kvabase != unmapped_buf) {
/*
* Buffer is not mapped, but the KVA was already
* reserved at the time of the instantiation. Use the
* allocated space.
*/
goto has_addr;
}
/*
* Calculate the amount of the address space we would reserve
* if the buffer was mapped.
*/
bsize = vn_isdisk(bp->b_vp, NULL) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
offset = blkno * bsize;
maxsize = size + (offset & PAGE_MASK);
maxsize = imax(maxsize, bsize);
while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
if ((gbflags & GB_NOWAIT_BD) != 0) {
/*
* XXXKIB: defragmentation cannot
* succeed, not sure what else to do.
*/
panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
}
counter_u64_add(mappingrestarts, 1);
bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0);
}
has_addr:
if (need_mapping) {
/* b_offset is handled by bpmap_qenter. */
bp->b_data = bp->b_kvabase;
BUF_CHECK_MAPPED(bp);
bpmap_qenter(bp);
}
}
struct buf *
getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
int flags)
{
struct buf *bp;
int error;
error = getblkx(vp, blkno, blkno, size, slpflag, slptimeo, flags, &bp);
if (error != 0)
return (NULL);
return (bp);
}
/*
* getblkx:
*
* Get a block given a specified block and offset into a file/device.
* The buffers B_DONE bit will be cleared on return, making it almost
* ready for an I/O initiation. B_INVAL may or may not be set on
* return. The caller should clear B_INVAL prior to initiating a
* READ.
*
* For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
* an existing buffer.
*
* For a VMIO buffer, B_CACHE is modified according to the backing VM.
* If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
* and then cleared based on the backing VM. If the previous buffer is
* non-0-sized but invalid, B_CACHE will be cleared.
*
* If getblk() must create a new buffer, the new buffer is returned with
* both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
* case it is returned with B_INVAL clear and B_CACHE set based on the
* backing VM.
*
* getblk() also forces a bwrite() for any B_DELWRI buffer whose
* B_CACHE bit is clear.
*
* What this means, basically, is that the caller should use B_CACHE to
* determine whether the buffer is fully valid or not and should clear
* B_INVAL prior to issuing a read. If the caller intends to validate
* the buffer by loading its data area with something, the caller needs
* to clear B_INVAL. If the caller does this without issuing an I/O,
* the caller should set B_CACHE ( as an optimization ), else the caller
* should issue the I/O and biodone() will set B_CACHE if the I/O was
* a write attempt or if it was a successful read. If the caller
* intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
* prior to issuing the READ. biodone() will *not* clear B_INVAL.
*
* The blkno parameter is the logical block being requested. Normally
* the mapping of logical block number to disk block address is done
* by calling VOP_BMAP(). However, if the mapping is already known, the
* disk block address can be passed using the dblkno parameter. If the
* disk block address is not known, then the same value should be passed
* for blkno and dblkno.
*/
int
getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
int slptimeo, int flags, struct buf **bpp)
{
struct buf *bp;
struct bufobj *bo;
daddr_t d_blkno;
int bsize, error, maxsize, vmio;
off_t offset;
CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
KASSERT((flags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
ASSERT_VOP_LOCKED(vp, "getblk");
if (size > maxbcachebuf)
panic("getblk: size(%d) > maxbcachebuf(%d)\n", size,
maxbcachebuf);
if (!unmapped_buf_allowed)
flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
bo = &vp->v_bufobj;
d_blkno = dblkno;
loop:
BO_RLOCK(bo);
bp = gbincore(bo, blkno);
if (bp != NULL) {
int lockflags;
/*
* Buffer is in-core. If the buffer is not busy nor managed,
* it must be on a queue.
*/
lockflags = LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK;
if ((flags & GB_LOCK_NOWAIT) != 0)
lockflags |= LK_NOWAIT;
error = BUF_TIMELOCK(bp, lockflags,
BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);
/*
* If we slept and got the lock we have to restart in case
* the buffer changed identities.
*/
if (error == ENOLCK)
goto loop;
/* We timed out or were interrupted. */
else if (error != 0)
return (error);
/* If recursed, assume caller knows the rules. */
else if (BUF_LOCKRECURSED(bp))
goto end;
/*
* The buffer is locked. B_CACHE is cleared if the buffer is
* invalid. Otherwise, for a non-VMIO buffer, B_CACHE is set
* and for a VMIO buffer B_CACHE is adjusted according to the
* backing VM cache.
*/
if (bp->b_flags & B_INVAL)
bp->b_flags &= ~B_CACHE;
else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
bp->b_flags |= B_CACHE;
if (bp->b_flags & B_MANAGED)
MPASS(bp->b_qindex == QUEUE_NONE);
else
bremfree(bp);
/*
* check for size inconsistencies for non-VMIO case.
*/
if (bp->b_bcount != size) {
if ((bp->b_flags & B_VMIO) == 0 ||
(size > bp->b_kvasize)) {
if (bp->b_flags & B_DELWRI) {
bp->b_flags |= B_NOCACHE;
bwrite(bp);
} else {
if (LIST_EMPTY(&bp->b_dep)) {
bp->b_flags |= B_RELBUF;
brelse(bp);
} else {
bp->b_flags |= B_NOCACHE;
bwrite(bp);
}
}
goto loop;
}
}
/*
* Handle the case of unmapped buffer which should
* become mapped, or the buffer for which KVA
* reservation is requested.
*/
bp_unmapped_get_kva(bp, blkno, size, flags);
/*
* If the size is inconsistent in the VMIO case, we can resize
* the buffer. This might lead to B_CACHE getting set or
* cleared. If the size has not changed, B_CACHE remains
* unchanged from its previous state.
*/
allocbuf(bp, size);
KASSERT(bp->b_offset != NOOFFSET,
("getblk: no buffer offset"));
/*
* A buffer with B_DELWRI set and B_CACHE clear must
* be committed before we can return the buffer in
* order to prevent the caller from issuing a read
* ( due to B_CACHE not being set ) and overwriting
* it.
*
* Most callers, including NFS and FFS, need this to
* operate properly either because they assume they
* can issue a read if B_CACHE is not set, or because
* ( for example ) an uncached B_DELWRI might loop due
* to softupdates re-dirtying the buffer. In the latter
* case, B_CACHE is set after the first write completes,
* preventing further loops.
* NOTE! b*write() sets B_CACHE. If we cleared B_CACHE
* above while extending the buffer, we cannot allow the
* buffer to remain with B_CACHE set after the write
* completes or it will represent a corrupt state. To
* deal with this we set B_NOCACHE to scrap the buffer
* after the write.
*
* We might be able to do something fancy, like setting
* B_CACHE in bwrite() except if B_DELWRI is already set,
* so the below call doesn't set B_CACHE, but that gets real
* confusing. This is much easier.
*/
if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
bp->b_flags |= B_NOCACHE;
bwrite(bp);
goto loop;
}
bp->b_flags &= ~B_DONE;
} else {
/*
* Buffer is not in-core, create new buffer. The buffer
* returned by getnewbuf() is locked. Note that the returned
* buffer is also considered valid (not marked B_INVAL).
*/
BO_RUNLOCK(bo);
/*
* If the user does not want us to create the buffer, bail out
* here.
*/
if (flags & GB_NOCREAT)
return (EEXIST);
bsize = vn_isdisk(vp, NULL) ? DEV_BSIZE : bo->bo_bsize;
KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
offset = blkno * bsize;
vmio = vp->v_object != NULL;
if (vmio) {
maxsize = size + (offset & PAGE_MASK);
} else {
maxsize = size;
/* Do not allow non-VMIO notmapped buffers. */
flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
}
maxsize = imax(maxsize, bsize);
if ((flags & GB_NOSPARSE) != 0 && vmio &&
!vn_isdisk(vp, NULL)) {
error = VOP_BMAP(vp, blkno, NULL, &d_blkno, 0, 0);
KASSERT(error != EOPNOTSUPP,
("GB_NOSPARSE from fs not supporting bmap, vp %p",
vp));
if (error != 0)
return (error);
if (d_blkno == -1)
return (EJUSTRETURN);
}
bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
if (bp == NULL) {
if (slpflag || slptimeo)
return (ETIMEDOUT);
/*
* XXX This is here until the sleep path is diagnosed
* enough to work under very low memory conditions.
*
* There's an issue on low memory, 4BSD+non-preempt
* systems (eg MIPS routers with 32MB RAM) where buffer
* exhaustion occurs without sleeping for buffer
* reclaimation. This just sticks in a loop and
* constantly attempts to allocate a buffer, which
* hits exhaustion and tries to wakeup bufdaemon.
* This never happens because we never yield.
*
* The real solution is to identify and fix these cases
* so we aren't effectively busy-waiting in a loop
* until the reclaimation path has cycles to run.
*/
kern_yield(PRI_USER);
goto loop;
}
/*
* This code is used to make sure that a buffer is not
* created while the getnewbuf routine is blocked.
* This can be a problem whether the vnode is locked or not.
* If the buffer is created out from under us, we have to
* throw away the one we just created.
*
* Note: this must occur before we associate the buffer
* with the vp especially considering limitations in
* the splay tree implementation when dealing with duplicate
* lblkno's.
*/
BO_LOCK(bo);
if (gbincore(bo, blkno)) {
BO_UNLOCK(bo);
bp->b_flags |= B_INVAL;
bufspace_release(bufdomain(bp), maxsize);
brelse(bp);
goto loop;
}
/*
* Insert the buffer into the hash, so that it can
* be found by incore.
*/
bp->b_lblkno = blkno;
bp->b_blkno = d_blkno;
bp->b_offset = offset;
bgetvp(vp, bp);
BO_UNLOCK(bo);
/*
* set B_VMIO bit. allocbuf() the buffer bigger. Since the
* buffer size starts out as 0, B_CACHE will be set by
* allocbuf() for the VMIO case prior to it testing the
* backing store for validity.
*/
if (vmio) {
bp->b_flags |= B_VMIO;
KASSERT(vp->v_object == bp->b_bufobj->bo_object,
("ARGH! different b_bufobj->bo_object %p %p %p\n",
bp, vp->v_object, bp->b_bufobj->bo_object));
} else {
bp->b_flags &= ~B_VMIO;
KASSERT(bp->b_bufobj->bo_object == NULL,
("ARGH! has b_bufobj->bo_object %p %p\n",
bp, bp->b_bufobj->bo_object));
BUF_CHECK_MAPPED(bp);
}
allocbuf(bp, size);
bufspace_release(bufdomain(bp), maxsize);
bp->b_flags &= ~B_DONE;
}
CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
end:
buf_track(bp, __func__);
KASSERT(bp->b_bufobj == bo,
("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
*bpp = bp;
return (0);
}
/*
* Get an empty, disassociated buffer of given size. The buffer is initially
* set to B_INVAL.
*/
struct buf *
geteblk(int size, int flags)
{
struct buf *bp;
int maxsize;
maxsize = (size + BKVAMASK) & ~BKVAMASK;
while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
if ((flags & GB_NOWAIT_BD) &&
(curthread->td_pflags & TDP_BUFNEED) != 0)
return (NULL);
}
allocbuf(bp, size);
bufspace_release(bufdomain(bp), maxsize);
bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
return (bp);
}
/*
* Truncate the backing store for a non-vmio buffer.
*/
static void
vfs_nonvmio_truncate(struct buf *bp, int newbsize)
{
if (bp->b_flags & B_MALLOC) {
/*
* malloced buffers are not shrunk
*/
if (newbsize == 0) {
bufmallocadjust(bp, 0);
free(bp->b_data, M_BIOBUF);
bp->b_data = bp->b_kvabase;
bp->b_flags &= ~B_MALLOC;
}
return;
}
vm_hold_free_pages(bp, newbsize);
bufspace_adjust(bp, newbsize);
}
/*
* Extend the backing for a non-VMIO buffer.
*/
static void
vfs_nonvmio_extend(struct buf *bp, int newbsize)
{
caddr_t origbuf;
int origbufsize;
/*
* We only use malloced memory on the first allocation.
* and revert to page-allocated memory when the buffer
* grows.
*
* There is a potential smp race here that could lead
* to bufmallocspace slightly passing the max. It
* is probably extremely rare and not worth worrying
* over.
*/
if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
bufmallocspace < maxbufmallocspace) {
bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
bp->b_flags |= B_MALLOC;
bufmallocadjust(bp, newbsize);
return;
}
/*
* If the buffer is growing on its other-than-first
* allocation then we revert to the page-allocation
* scheme.
*/
origbuf = NULL;
origbufsize = 0;
if (bp->b_flags & B_MALLOC) {
origbuf = bp->b_data;
origbufsize = bp->b_bufsize;
bp->b_data = bp->b_kvabase;
bufmallocadjust(bp, 0);
bp->b_flags &= ~B_MALLOC;
newbsize = round_page(newbsize);
}
vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
(vm_offset_t) bp->b_data + newbsize);
if (origbuf != NULL) {
bcopy(origbuf, bp->b_data, origbufsize);
free(origbuf, M_BIOBUF);
}
bufspace_adjust(bp, newbsize);
}
/*
* This code constitutes the buffer memory from either anonymous system
* memory (in the case of non-VMIO operations) or from an associated
* VM object (in the case of VMIO operations). This code is able to
* resize a buffer up or down.
*
* Note that this code is tricky, and has many complications to resolve
* deadlock or inconsistent data situations. Tread lightly!!!
* There are B_CACHE and B_DELWRI interactions that must be dealt with by
* the caller. Calling this code willy nilly can result in the loss of data.
*
* allocbuf() only adjusts B_CACHE for VMIO buffers. getblk() deals with
* B_CACHE for the non-VMIO case.
*/
int
allocbuf(struct buf *bp, int size)
{
int newbsize;
if (bp->b_bcount == size)
return (1);
if (bp->b_kvasize != 0 && bp->b_kvasize < size)
panic("allocbuf: buffer too small");
newbsize = roundup2(size, DEV_BSIZE);
if ((bp->b_flags & B_VMIO) == 0) {
if ((bp->b_flags & B_MALLOC) == 0)
newbsize = round_page(newbsize);
/*
* Just get anonymous memory from the kernel. Don't
* mess with B_CACHE.
*/
if (newbsize < bp->b_bufsize)
vfs_nonvmio_truncate(bp, newbsize);
else if (newbsize > bp->b_bufsize)
vfs_nonvmio_extend(bp, newbsize);
} else {
int desiredpages;
desiredpages = (size == 0) ? 0 :
num_pages((bp->b_offset & PAGE_MASK) + newbsize);
if (bp->b_flags & B_MALLOC)
panic("allocbuf: VMIO buffer can't be malloced");
/*
* Set B_CACHE initially if buffer is 0 length or will become
* 0-length.
*/
if (size == 0 || bp->b_bufsize == 0)
bp->b_flags |= B_CACHE;
if (newbsize < bp->b_bufsize)
vfs_vmio_truncate(bp, desiredpages);
/* XXX This looks as if it should be newbsize > b_bufsize */
else if (size > bp->b_bcount)
vfs_vmio_extend(bp, desiredpages, size);
bufspace_adjust(bp, newbsize);
}
bp->b_bcount = size; /* requested buffer size. */
return (1);
}
extern int inflight_transient_maps;
static struct bio_queue nondump_bios;
void
biodone(struct bio *bp)
{
struct mtx *mtxp;
void (*done)(struct bio *);
vm_offset_t start, end;
biotrack(bp, __func__);
/*
* Avoid completing I/O when dumping after a panic since that may
* result in a deadlock in the filesystem or pager code. Note that
* this doesn't affect dumps that were started manually since we aim
* to keep the system usable after it has been resumed.
*/
if (__predict_false(dumping && SCHEDULER_STOPPED())) {
TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
return;
}
if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
bp->bio_flags |= BIO_UNMAPPED;
start = trunc_page((vm_offset_t)bp->bio_data);
end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
bp->bio_data = unmapped_buf;
pmap_qremove(start, atop(end - start));
vmem_free(transient_arena, start, end - start);
atomic_add_int(&inflight_transient_maps, -1);
}
done = bp->bio_done;
if (done == NULL) {
mtxp = mtx_pool_find(mtxpool_sleep, bp);
mtx_lock(mtxp);
bp->bio_flags |= BIO_DONE;
wakeup(bp);
mtx_unlock(mtxp);
} else
done(bp);
}
/*
* Wait for a BIO to finish.
*/
int
biowait(struct bio *bp, const char *wchan)
{
struct mtx *mtxp;
mtxp = mtx_pool_find(mtxpool_sleep, bp);
mtx_lock(mtxp);
while ((bp->bio_flags & BIO_DONE) == 0)
msleep(bp, mtxp, PRIBIO, wchan, 0);
mtx_unlock(mtxp);
if (bp->bio_error != 0)
return (bp->bio_error);
if (!(bp->bio_flags & BIO_ERROR))
return (0);
return (EIO);
}
void
biofinish(struct bio *bp, struct devstat *stat, int error)
{
if (error) {
bp->bio_error = error;
bp->bio_flags |= BIO_ERROR;
}
if (stat != NULL)
devstat_end_transaction_bio(stat, bp);
biodone(bp);
}
#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
void
biotrack_buf(struct bio *bp, const char *location)
{
buf_track(bp->bio_track_bp, location);
}
#endif
/*
* bufwait:
*
* Wait for buffer I/O completion, returning error status. The buffer
* is left locked and B_DONE on return. B_EINTR is converted into an EINTR
* error and cleared.
*/
int
bufwait(struct buf *bp)
{
if (bp->b_iocmd == BIO_READ)
bwait(bp, PRIBIO, "biord");
else
bwait(bp, PRIBIO, "biowr");
if (bp->b_flags & B_EINTR) {
bp->b_flags &= ~B_EINTR;
return (EINTR);
}
if (bp->b_ioflags & BIO_ERROR) {
return (bp->b_error ? bp->b_error : EIO);
} else {
return (0);
}
}
/*
* bufdone:
*
* Finish I/O on a buffer, optionally calling a completion function.
* This is usually called from an interrupt so process blocking is
* not allowed.
*
* biodone is also responsible for setting B_CACHE in a B_VMIO bp.
* In a non-VMIO bp, B_CACHE will be set on the next getblk()
* assuming B_INVAL is clear.
*
* For the VMIO case, we set B_CACHE if the op was a read and no
* read error occurred, or if the op was a write. B_CACHE is never
* set if the buffer is invalid or otherwise uncacheable.
*
* bufdone does not mess with B_INVAL, allowing the I/O routine or the
* initiator to leave B_INVAL set to brelse the buffer out of existence
* in the biodone routine.
*/
void
bufdone(struct buf *bp)
{
struct bufobj *dropobj;
void (*biodone)(struct buf *);
buf_track(bp, __func__);
CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
dropobj = NULL;
KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
runningbufwakeup(bp);
if (bp->b_iocmd == BIO_WRITE)
dropobj = bp->b_bufobj;
/* call optional completion function if requested */
if (bp->b_iodone != NULL) {
biodone = bp->b_iodone;
bp->b_iodone = NULL;
(*biodone) (bp);
if (dropobj)
bufobj_wdrop(dropobj);
return;
}
if (bp->b_flags & B_VMIO) {
/*
* Set B_CACHE if the op was a normal read and no error
* occurred. B_CACHE is set for writes in the b*write()
* routines.
*/
if (bp->b_iocmd == BIO_READ &&
!(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
!(bp->b_ioflags & BIO_ERROR))
bp->b_flags |= B_CACHE;
vfs_vmio_iodone(bp);
}
if (!LIST_EMPTY(&bp->b_dep))
buf_complete(bp);
if ((bp->b_flags & B_CKHASH) != 0) {
KASSERT(bp->b_iocmd == BIO_READ,
("bufdone: b_iocmd %d not BIO_READ", bp->b_iocmd));
KASSERT(buf_mapped(bp), ("bufdone: bp %p not mapped", bp));
(*bp->b_ckhashcalc)(bp);
}
/*
* For asynchronous completions, release the buffer now. The brelse
* will do a wakeup there if necessary - so no need to do a wakeup
* here in the async case. The sync case always needs to do a wakeup.
*/
if (bp->b_flags & B_ASYNC) {
if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
(bp->b_ioflags & BIO_ERROR))
brelse(bp);
else
bqrelse(bp);
} else
bdone(bp);
if (dropobj)
bufobj_wdrop(dropobj);
}
/*
* This routine is called in lieu of iodone in the case of
* incomplete I/O. This keeps the busy status for pages
* consistent.
*/
void
vfs_unbusy_pages(struct buf *bp)
{
int i;
vm_object_t obj;
vm_page_t m;
runningbufwakeup(bp);
if (!(bp->b_flags & B_VMIO))
return;
obj = bp->b_bufobj->bo_object;
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
if (m == bogus_page) {
m = vm_page_relookup(obj, OFF_TO_IDX(bp->b_offset) + i);
if (!m)
panic("vfs_unbusy_pages: page missing\n");
bp->b_pages[i] = m;
if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
} else
BUF_CHECK_UNMAPPED(bp);
}
vm_page_sunbusy(m);
}
vm_object_pip_wakeupn(obj, bp->b_npages);
}
/*
* vfs_page_set_valid:
*
* Set the valid bits in a page based on the supplied offset. The
* range is restricted to the buffer's size.
*
* This routine is typically called after a read completes.
*/
static void
vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
{
vm_ooffset_t eoff;
/*
* Compute the end offset, eoff, such that [off, eoff) does not span a
* page boundary and eoff is not greater than the end of the buffer.
* The end of the buffer, in this case, is our file EOF, not the
* allocation size of the buffer.
*/
eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
if (eoff > bp->b_offset + bp->b_bcount)
eoff = bp->b_offset + bp->b_bcount;
/*
* Set valid range. This is typically the entire buffer and thus the
* entire page.
*/
if (eoff > off)
vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
}
/*
* vfs_page_set_validclean:
*
* Set the valid bits and clear the dirty bits in a page based on the
* supplied offset. The range is restricted to the buffer's size.
*/
static void
vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
{
vm_ooffset_t soff, eoff;
/*
* Start and end offsets in buffer. eoff - soff may not cross a
* page boundary or cross the end of the buffer. The end of the
* buffer, in this case, is our file EOF, not the allocation size
* of the buffer.
*/
soff = off;
eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
if (eoff > bp->b_offset + bp->b_bcount)
eoff = bp->b_offset + bp->b_bcount;
/*
* Set valid range. This is typically the entire buffer and thus the
* entire page.
*/
if (eoff > soff) {
vm_page_set_validclean(
m,
(vm_offset_t) (soff & PAGE_MASK),
(vm_offset_t) (eoff - soff)
);
}
}
/*
* Acquire a shared busy on all pages in the buf.
*/
void
vfs_busy_pages_acquire(struct buf *bp)
{
int i;
for (i = 0; i < bp->b_npages; i++)
vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY);
}
void
vfs_busy_pages_release(struct buf *bp)
{
int i;
for (i = 0; i < bp->b_npages; i++)
vm_page_sunbusy(bp->b_pages[i]);
}
/*
* This routine is called before a device strategy routine.
* It is used to tell the VM system that paging I/O is in
* progress, and treat the pages associated with the buffer
* almost as being exclusive busy. Also the object paging_in_progress
* flag is handled to make sure that the object doesn't become
* inconsistent.
*
* Since I/O has not been initiated yet, certain buffer flags
* such as BIO_ERROR or B_INVAL may be in an inconsistent state
* and should be ignored.
*/
void
vfs_busy_pages(struct buf *bp, int clear_modify)
{
vm_object_t obj;
vm_ooffset_t foff;
vm_page_t m;
int i;
bool bogus;
if (!(bp->b_flags & B_VMIO))
return;
obj = bp->b_bufobj->bo_object;
foff = bp->b_offset;
KASSERT(bp->b_offset != NOOFFSET,
("vfs_busy_pages: no buffer offset"));
if ((bp->b_flags & B_CLUSTER) == 0) {
vm_object_pip_add(obj, bp->b_npages);
vfs_busy_pages_acquire(bp);
}
if (bp->b_bufsize != 0)
vfs_setdirty_range(bp);
bogus = false;
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
vm_page_assert_sbusied(m);
/*
* When readying a buffer for a read ( i.e
* clear_modify == 0 ), it is important to do
* bogus_page replacement for valid pages in
* partially instantiated buffers. Partially
* instantiated buffers can, in turn, occur when
* reconstituting a buffer from its VM backing store
* base. We only have to do this if B_CACHE is
* clear ( which causes the I/O to occur in the
* first place ). The replacement prevents the read
* I/O from overwriting potentially dirty VM-backed
* pages. XXX bogus page replacement is, uh, bogus.
* It may not work properly with small-block devices.
* We need to find a better way.
*/
if (clear_modify) {
pmap_remove_write(m);
vfs_page_set_validclean(bp, foff, m);
} else if (vm_page_all_valid(m) &&
(bp->b_flags & B_CACHE) == 0) {
bp->b_pages[i] = bogus_page;
bogus = true;
}
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
}
if (bogus && buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
}
}
/*
* vfs_bio_set_valid:
*
* Set the range within the buffer to valid. The range is
* relative to the beginning of the buffer, b_offset. Note that
* b_offset itself may be offset from the beginning of the first
* page.
*/
void
vfs_bio_set_valid(struct buf *bp, int base, int size)
{
int i, n;
vm_page_t m;
if (!(bp->b_flags & B_VMIO))
return;
/*
* Fixup base to be relative to beginning of first page.
* Set initial n to be the maximum number of bytes in the
* first page that can be validated.
*/
base += (bp->b_offset & PAGE_MASK);
n = PAGE_SIZE - (base & PAGE_MASK);
/*
* Busy may not be strictly necessary here because the pages are
* unlikely to be fully valid and the vnode lock will synchronize
* their access via getpages. It is grabbed for consistency with
* other page validation.
*/
vfs_busy_pages_acquire(bp);
for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
m = bp->b_pages[i];
if (n > size)
n = size;
vm_page_set_valid_range(m, base & PAGE_MASK, n);
base += n;
size -= n;
n = PAGE_SIZE;
}
vfs_busy_pages_release(bp);
}
/*
* vfs_bio_clrbuf:
*
* If the specified buffer is a non-VMIO buffer, clear the entire
* buffer. If the specified buffer is a VMIO buffer, clear and
* validate only the previously invalid portions of the buffer.
* This routine essentially fakes an I/O, so we need to clear
* BIO_ERROR and B_INVAL.
*
* Note that while we only theoretically need to clear through b_bcount,
* we go ahead and clear through b_bufsize.
*/
void
vfs_bio_clrbuf(struct buf *bp)
{
int i, j, mask, sa, ea, slide;
if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
clrbuf(bp);
return;
}
bp->b_flags &= ~B_INVAL;
bp->b_ioflags &= ~BIO_ERROR;
vfs_busy_pages_acquire(bp);
sa = bp->b_offset & PAGE_MASK;
slide = 0;
for (i = 0; i < bp->b_npages; i++, sa = 0) {
slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
ea = slide & PAGE_MASK;
if (ea == 0)
ea = PAGE_SIZE;
if (bp->b_pages[i] == bogus_page)
continue;
j = sa / DEV_BSIZE;
mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
if ((bp->b_pages[i]->valid & mask) == mask)
continue;
if ((bp->b_pages[i]->valid & mask) == 0)
pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
else {
for (; sa < ea; sa += DEV_BSIZE, j++) {
if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
pmap_zero_page_area(bp->b_pages[i],
sa, DEV_BSIZE);
}
}
}
vm_page_set_valid_range(bp->b_pages[i], j * DEV_BSIZE,
roundup2(ea - sa, DEV_BSIZE));
}
vfs_busy_pages_release(bp);
bp->b_resid = 0;
}
void
vfs_bio_bzero_buf(struct buf *bp, int base, int size)
{
vm_page_t m;
int i, n;
if (buf_mapped(bp)) {
BUF_CHECK_MAPPED(bp);
bzero(bp->b_data + base, size);
} else {
BUF_CHECK_UNMAPPED(bp);
n = PAGE_SIZE - (base & PAGE_MASK);
for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
m = bp->b_pages[i];
if (n > size)
n = size;
pmap_zero_page_area(m, base & PAGE_MASK, n);
base += n;
size -= n;
n = PAGE_SIZE;
}
}
}
/*
* Update buffer flags based on I/O request parameters, optionally releasing the
* buffer. If it's VMIO or direct I/O, the buffer pages are released to the VM,
* where they may be placed on a page queue (VMIO) or freed immediately (direct
* I/O). Otherwise the buffer is released to the cache.
*/
static void
b_io_dismiss(struct buf *bp, int ioflag, bool release)
{
KASSERT((ioflag & IO_NOREUSE) == 0 || (ioflag & IO_VMIO) != 0,
("buf %p non-VMIO noreuse", bp));
if ((ioflag & IO_DIRECT) != 0)
bp->b_flags |= B_DIRECT;
if ((ioflag & IO_EXT) != 0)
bp->b_xflags |= BX_ALTDATA;
if ((ioflag & (IO_VMIO | IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
bp->b_flags |= B_RELBUF;
if ((ioflag & IO_NOREUSE) != 0)
bp->b_flags |= B_NOREUSE;
if (release)
brelse(bp);
} else if (release)
bqrelse(bp);
}
void
vfs_bio_brelse(struct buf *bp, int ioflag)
{
b_io_dismiss(bp, ioflag, true);
}
void
vfs_bio_set_flags(struct buf *bp, int ioflag)
{
b_io_dismiss(bp, ioflag, false);
}
/*
* vm_hold_load_pages and vm_hold_free_pages get pages into
* a buffers address space. The pages are anonymous and are
* not associated with a file object.
*/
static void
vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
{
vm_offset_t pg;
vm_page_t p;
int index;
BUF_CHECK_MAPPED(bp);
to = round_page(to);
from = round_page(from);
index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
/*
* note: must allocate system pages since blocking here
* could interfere with paging I/O, no matter which
* process we are.
*/
p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) |
VM_ALLOC_WAITOK);
pmap_qenter(pg, &p, 1);
bp->b_pages[index] = p;
}
bp->b_npages = index;
}
/* Return pages associated with this buf to the vm system */
static void
vm_hold_free_pages(struct buf *bp, int newbsize)
{
vm_offset_t from;
vm_page_t p;
int index, newnpages;
BUF_CHECK_MAPPED(bp);
from = round_page((vm_offset_t)bp->b_data + newbsize);
newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
if (bp->b_npages > newnpages)
pmap_qremove(from, bp->b_npages - newnpages);
for (index = newnpages; index < bp->b_npages; index++) {
p = bp->b_pages[index];
bp->b_pages[index] = NULL;
vm_page_unwire_noq(p);
vm_page_free(p);
}
bp->b_npages = newnpages;
}
/*
* Map an IO request into kernel virtual address space.
*
* All requests are (re)mapped into kernel VA space.
* Notice that we use b_bufsize for the size of the buffer
* to be mapped. b_bcount might be modified by the driver.
*
* Note that even if the caller determines that the address space should
* be valid, a race or a smaller-file mapped into a larger space may
* actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
* check the return value.
*
* This function only works with pager buffers.
*/
int
vmapbuf(struct buf *bp, int mapbuf)
{
vm_prot_t prot;
int pidx;
if (bp->b_bufsize < 0)
return (-1);
prot = VM_PROT_READ;
if (bp->b_iocmd == BIO_READ)
prot |= VM_PROT_WRITE; /* Less backwards than it looks */
if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
(vm_offset_t)bp->b_data, bp->b_bufsize, prot, bp->b_pages,
btoc(MAXPHYS))) < 0)
return (-1);
bp->b_npages = pidx;
bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
if (mapbuf || !unmapped_buf_allowed) {
pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
bp->b_data = bp->b_kvabase + bp->b_offset;
} else
bp->b_data = unmapped_buf;
return(0);
}
/*
* Free the io map PTEs associated with this IO operation.
* We also invalidate the TLB entries and restore the original b_addr.
*
* This function only works with pager buffers.
*/
void
vunmapbuf(struct buf *bp)
{
int npages;
npages = bp->b_npages;
if (buf_mapped(bp))
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
vm_page_unhold_pages(bp->b_pages, npages);
bp->b_data = unmapped_buf;
}
void
bdone(struct buf *bp)
{
struct mtx *mtxp;
mtxp = mtx_pool_find(mtxpool_sleep, bp);
mtx_lock(mtxp);
bp->b_flags |= B_DONE;
wakeup(bp);
mtx_unlock(mtxp);
}
void
bwait(struct buf *bp, u_char pri, const char *wchan)
{
struct mtx *mtxp;
mtxp = mtx_pool_find(mtxpool_sleep, bp);
mtx_lock(mtxp);
while ((bp->b_flags & B_DONE) == 0)
msleep(bp, mtxp, pri, wchan, 0);
mtx_unlock(mtxp);
}
int
bufsync(struct bufobj *bo, int waitfor)
{
return (VOP_FSYNC(bo2vnode(bo), waitfor, curthread));
}
void
bufstrategy(struct bufobj *bo, struct buf *bp)
{
int i __unused;
struct vnode *vp;
vp = bp->b_vp;
KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
i = VOP_STRATEGY(vp, bp);
KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
}
/*
* Initialize a struct bufobj before use. Memory is assumed zero filled.
*/
void
bufobj_init(struct bufobj *bo, void *private)
{
static volatile int bufobj_cleanq;
bo->bo_domain =
atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains;
rw_init(BO_LOCKPTR(bo), "bufobj interlock");
bo->bo_private = private;
TAILQ_INIT(&bo->bo_clean.bv_hd);
TAILQ_INIT(&bo->bo_dirty.bv_hd);
}
void
bufobj_wrefl(struct bufobj *bo)
{
KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
ASSERT_BO_WLOCKED(bo);
bo->bo_numoutput++;
}
void
bufobj_wref(struct bufobj *bo)
{
KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
BO_LOCK(bo);
bo->bo_numoutput++;
BO_UNLOCK(bo);
}
void
bufobj_wdrop(struct bufobj *bo)
{
KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
BO_LOCK(bo);
KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
bo->bo_flag &= ~BO_WWAIT;
wakeup(&bo->bo_numoutput);
}
BO_UNLOCK(bo);
}
int
bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
{
int error;
KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
ASSERT_BO_WLOCKED(bo);
error = 0;
while (bo->bo_numoutput) {
bo->bo_flag |= BO_WWAIT;
error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
slpflag | (PRIBIO + 1), "bo_wwait", timeo);
if (error)
break;
}
return (error);
}
/*
* Set bio_data or bio_ma for struct bio from the struct buf.
*/
void
bdata2bio(struct buf *bp, struct bio *bip)
{
if (!buf_mapped(bp)) {
KASSERT(unmapped_buf_allowed, ("unmapped"));
bip->bio_ma = bp->b_pages;
bip->bio_ma_n = bp->b_npages;
bip->bio_data = unmapped_buf;
bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
bip->bio_flags |= BIO_UNMAPPED;
KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
PAGE_SIZE == bp->b_npages,
("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
(long long)bip->bio_length, bip->bio_ma_n));
} else {
bip->bio_data = bp->b_data;
bip->bio_ma = NULL;
}
}
/*
* The MIPS pmap code currently doesn't handle aliased pages.
* The VIPT caches may not handle page aliasing themselves, leading
* to data corruption.
*
* As such, this code makes a system extremely unhappy if said
* system doesn't support unaliasing the above situation in hardware.
* Some "recent" systems (eg some mips24k/mips74k cores) don't enable
* this feature at build time, so it has to be handled in software.
*
* Once the MIPS pmap/cache code grows to support this function on
* earlier chips, it should be flipped back off.
*/
#ifdef __mips__
static int buf_pager_relbuf = 1;
#else
static int buf_pager_relbuf = 0;
#endif
SYSCTL_INT(_vfs, OID_AUTO, buf_pager_relbuf, CTLFLAG_RWTUN,
&buf_pager_relbuf, 0,
"Make buffer pager release buffers after reading");
/*
* The buffer pager. It uses buffer reads to validate pages.
*
* In contrast to the generic local pager from vm/vnode_pager.c, this
* pager correctly and easily handles volumes where the underlying
* device block size is greater than the machine page size. The
* buffer cache transparently extends the requested page run to be
* aligned at the block boundary, and does the necessary bogus page
* replacements in the addends to avoid obliterating already valid
* pages.
*
* The only non-trivial issue is that the exclusive busy state for
* pages, which is assumed by the vm_pager_getpages() interface, is
* incompatible with the VMIO buffer cache's desire to share-busy the
* pages. This function performs a trivial downgrade of the pages'
* state before reading buffers, and a less trivial upgrade from the
* shared-busy to excl-busy state after the read.
*/
int
vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count,
int *rbehind, int *rahead, vbg_get_lblkno_t get_lblkno,
vbg_get_blksize_t get_blksize)
{
vm_page_t m;
vm_object_t object;
struct buf *bp;
struct mount *mp;
daddr_t lbn, lbnp;
vm_ooffset_t la, lb, poff, poffe;
long bsize;
int bo_bs, br_flags, error, i, pgsin, pgsin_a, pgsin_b;
bool redo, lpart;
object = vp->v_object;
mp = vp->v_mount;
error = 0;
la = IDX_TO_OFF(ma[count - 1]->pindex);
if (la >= object->un_pager.vnp.vnp_size)
return (VM_PAGER_BAD);
/*
* Change the meaning of la from where the last requested page starts
* to where it ends, because that's the end of the requested region
* and the start of the potential read-ahead region.
*/
la += PAGE_SIZE;
lpart = la > object->un_pager.vnp.vnp_size;
bo_bs = get_blksize(vp, get_lblkno(vp, IDX_TO_OFF(ma[0]->pindex)));
/*
* Calculate read-ahead, behind and total pages.
*/
pgsin = count;
lb = IDX_TO_OFF(ma[0]->pindex);
pgsin_b = OFF_TO_IDX(lb - rounddown2(lb, bo_bs));
pgsin += pgsin_b;
if (rbehind != NULL)
*rbehind = pgsin_b;
pgsin_a = OFF_TO_IDX(roundup2(la, bo_bs) - la);
if (la + IDX_TO_OFF(pgsin_a) >= object->un_pager.vnp.vnp_size)
pgsin_a = OFF_TO_IDX(roundup2(object->un_pager.vnp.vnp_size,
PAGE_SIZE) - la);
pgsin += pgsin_a;
if (rahead != NULL)
*rahead = pgsin_a;
VM_CNT_INC(v_vnodein);
VM_CNT_ADD(v_vnodepgsin, pgsin);
br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
!= 0) ? GB_UNMAPPED : 0;
again:
for (i = 0; i < count; i++) {
if (ma[i] != bogus_page)
vm_page_busy_downgrade(ma[i]);
}
lbnp = -1;
for (i = 0; i < count; i++) {
m = ma[i];
if (m == bogus_page)
continue;
/*
* Pages are shared busy and the object lock is not
* owned, which together allow for the pages'
* invalidation. The racy test for validity avoids
* useless creation of the buffer for the most typical
* case when invalidation is not used in redo or for
* parallel read. The shared->excl upgrade loop at
* the end of the function catches the race in a
* reliable way (protected by the object lock).
*/
if (vm_page_all_valid(m))
continue;
poff = IDX_TO_OFF(m->pindex);
poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size);
for (; poff < poffe; poff += bsize) {
lbn = get_lblkno(vp, poff);
if (lbn == lbnp)
goto next_page;
lbnp = lbn;
bsize = get_blksize(vp, lbn);
error = bread_gb(vp, lbn, bsize, curthread->td_ucred,
br_flags, &bp);
if (error != 0)
goto end_pages;
if (bp->b_rcred == curthread->td_ucred) {
crfree(bp->b_rcred);
bp->b_rcred = NOCRED;
}
if (LIST_EMPTY(&bp->b_dep)) {
/*
* Invalidation clears m->valid, but
* may leave B_CACHE flag if the
* buffer existed at the invalidation
* time. In this case, recycle the
* buffer to do real read on next
* bread() after redo.
*
* Otherwise B_RELBUF is not strictly
* necessary, enable to reduce buf
* cache pressure.
*/
if (buf_pager_relbuf ||
!vm_page_all_valid(m))
bp->b_flags |= B_RELBUF;
bp->b_flags &= ~B_NOCACHE;
brelse(bp);
} else {
bqrelse(bp);
}
}
KASSERT(1 /* racy, enable for debugging */ ||
vm_page_all_valid(m) || i == count - 1,
("buf %d %p invalid", i, m));
if (i == count - 1 && lpart) {
if (!vm_page_none_valid(m) &&
!vm_page_all_valid(m))
vm_page_zero_invalid(m, TRUE);
}
next_page:;
}
end_pages:
redo = false;
for (i = 0; i < count; i++) {
if (ma[i] == bogus_page)
continue;
if (vm_page_busy_tryupgrade(ma[i]) == 0) {
vm_page_sunbusy(ma[i]);
ma[i] = vm_page_grab_unlocked(object, ma[i]->pindex,
VM_ALLOC_NORMAL);
}
/*
* Since the pages were only sbusy while neither the
* buffer nor the object lock was held by us, or
* reallocated while vm_page_grab() slept for busy
* relinguish, they could have been invalidated.
* Recheck the valid bits and re-read as needed.
*
* Note that the last page is made fully valid in the
* read loop, and partial validity for the page at
* index count - 1 could mean that the page was
* invalidated or removed, so we must restart for
* safety as well.
*/
if (!vm_page_all_valid(ma[i]))
redo = true;
}
if (redo && error == 0)
goto again;
return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
}
#include "opt_ddb.h"
#ifdef DDB
#include <ddb/ddb.h>
/* DDB command to show buffer data */
DB_SHOW_COMMAND(buffer, db_show_buffer)
{
/* get args */
struct buf *bp = (struct buf *)addr;
#ifdef FULL_BUF_TRACKING
uint32_t i, j;
#endif
if (!have_addr) {
db_printf("usage: show buffer <addr>\n");
return;
}
db_printf("buf at %p\n", bp);
db_printf("b_flags = 0x%b, b_xflags=0x%b\n",
(u_int)bp->b_flags, PRINT_BUF_FLAGS,
(u_int)bp->b_xflags, PRINT_BUF_XFLAGS);
db_printf("b_vflags=0x%b b_ioflags0x%b\n",
(u_int)bp->b_vflags, PRINT_BUF_VFLAGS,
(u_int)bp->b_ioflags, PRINT_BIO_FLAGS);
db_printf(
"b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
"b_bufobj = (%p), b_data = %p\n, b_blkno = %jd, b_lblkno = %jd, "
"b_vp = %p, b_dep = %p\n",
bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
(intmax_t)bp->b_lblkno, bp->b_vp, bp->b_dep.lh_first);
db_printf("b_kvabase = %p, b_kvasize = %d\n",
bp->b_kvabase, bp->b_kvasize);
if (bp->b_npages) {
int i;
db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
for (i = 0; i < bp->b_npages; i++) {
vm_page_t m;
m = bp->b_pages[i];
if (m != NULL)
db_printf("(%p, 0x%lx, 0x%lx)", m->object,
(u_long)m->pindex,
(u_long)VM_PAGE_TO_PHYS(m));
else
db_printf("( ??? )");
if ((i + 1) < bp->b_npages)
db_printf(",");
}
db_printf("\n");
}
BUF_LOCKPRINTINFO(bp);
#if defined(FULL_BUF_TRACKING)
db_printf("b_io_tracking: b_io_tcnt = %u\n", bp->b_io_tcnt);
i = bp->b_io_tcnt % BUF_TRACKING_SIZE;
for (j = 1; j <= BUF_TRACKING_SIZE; j++) {
if (bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)] == NULL)
continue;
db_printf(" %2u: %s\n", j,
bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)]);
}
#elif defined(BUF_TRACKING)
db_printf("b_io_tracking: %s\n", bp->b_io_tracking);
#endif
db_printf(" ");
}
DB_SHOW_COMMAND(bufqueues, bufqueues)
{
struct bufdomain *bd;
struct buf *bp;
long total;
int i, j, cnt;
db_printf("bqempty: %d\n", bqempty.bq_len);
for (i = 0; i < buf_domains; i++) {
bd = &bdomain[i];
db_printf("Buf domain %d\n", i);
db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);
db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);
db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);
db_printf("\n");
db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);
db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);
db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);
db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);
db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);
db_printf("\n");
db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);
db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);
db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);
db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);
db_printf("\n");
total = 0;
TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)
total += bp->b_bufsize;
db_printf("\tcleanq count\t%d (%ld)\n",
bd->bd_cleanq->bq_len, total);
total = 0;
TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)
total += bp->b_bufsize;
db_printf("\tdirtyq count\t%d (%ld)\n",
bd->bd_dirtyq.bq_len, total);
db_printf("\twakeup\t\t%d\n", bd->bd_wanted);
db_printf("\tlim\t\t%d\n", bd->bd_lim);
db_printf("\tCPU ");
for (j = 0; j <= mp_maxid; j++)
db_printf("%d, ", bd->bd_subq[j].bq_len);
db_printf("\n");
cnt = 0;
total = 0;
for (j = 0; j < nbuf; j++)
if (buf[j].b_domain == i && BUF_ISLOCKED(&buf[j])) {
cnt++;
total += buf[j].b_bufsize;
}
db_printf("\tLocked buffers: %d space %ld\n", cnt, total);
cnt = 0;
total = 0;
for (j = 0; j < nbuf; j++)
if (buf[j].b_domain == i) {
cnt++;
total += buf[j].b_bufsize;
}
db_printf("\tTotal buffers: %d space %ld\n", cnt, total);
}
}
DB_SHOW_COMMAND(lockedbufs, lockedbufs)
{
struct buf *bp;
int i;
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
if (BUF_ISLOCKED(bp)) {
db_show_buffer((uintptr_t)bp, 1, 0, NULL);
db_printf("\n");
if (db_pager_quit)
break;
}
}
}
DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
{
struct vnode *vp;
struct buf *bp;
if (!have_addr) {
db_printf("usage: show vnodebufs <addr>\n");
return;
}
vp = (struct vnode *)addr;
db_printf("Clean buffers:\n");
TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
db_show_buffer((uintptr_t)bp, 1, 0, NULL);
db_printf("\n");
}
db_printf("Dirty buffers:\n");
TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
db_show_buffer((uintptr_t)bp, 1, 0, NULL);
db_printf("\n");
}
}
DB_COMMAND(countfreebufs, db_coundfreebufs)
{
struct buf *bp;
int i, used = 0, nfree = 0;
if (have_addr) {
db_printf("usage: countfreebufs\n");
return;
}
for (i = 0; i < nbuf; i++) {
bp = &buf[i];
if (bp->b_qindex == QUEUE_EMPTY)
nfree++;
else
used++;
}
db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
nfree + used);
db_printf("numfreebuffers is %d\n", numfreebuffers);
}
#endif /* DDB */
Index: head/sys/sys/buf.h
===================================================================
--- head/sys/sys/buf.h (revision 361490)
+++ head/sys/sys/buf.h (revision 361491)
@@ -1,591 +1,594 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)buf.h 8.9 (Berkeley) 3/30/95
* $FreeBSD$
*/
#ifndef _SYS_BUF_H_
#define _SYS_BUF_H_
#include <sys/bufobj.h>
#include <sys/queue.h>
#include <sys/lock.h>
#include <sys/lockmgr.h>
#include <vm/uma.h>
struct bio;
struct buf;
struct bufobj;
struct mount;
struct vnode;
struct uio;
/*
* To avoid including <ufs/ffs/softdep.h>
*/
LIST_HEAD(workhead, worklist);
/*
* These are currently used only by the soft dependency code, hence
* are stored once in a global variable. If other subsystems wanted
* to use these hooks, a pointer to a set of bio_ops could be added
* to each buffer.
*/
extern struct bio_ops {
void (*io_start)(struct buf *);
void (*io_complete)(struct buf *);
void (*io_deallocate)(struct buf *);
int (*io_countdeps)(struct buf *, int);
} bioops;
struct vm_object;
struct vm_page;
typedef uint32_t b_xflags_t;
/*
* The buffer header describes an I/O operation in the kernel.
*
* NOTES:
* b_bufsize, b_bcount. b_bufsize is the allocation size of the
* buffer, either DEV_BSIZE or PAGE_SIZE aligned. b_bcount is the
* originally requested buffer size and can serve as a bounds check
* against EOF. For most, but not all uses, b_bcount == b_bufsize.
*
* b_dirtyoff, b_dirtyend. Buffers support piecemeal, unaligned
* ranges of dirty data that need to be written to backing store.
* The range is typically clipped at b_bcount ( not b_bufsize ).
*
* b_resid. Number of bytes remaining in I/O. After an I/O operation
* completes, b_resid is usually 0 indicating 100% success.
*
* All fields are protected by the buffer lock except those marked:
* V - Protected by owning bufobj lock
* Q - Protected by the buf queue lock
* D - Protected by an dependency implementation specific lock
*/
struct buf {
struct bufobj *b_bufobj;
long b_bcount;
void *b_caller1;
caddr_t b_data;
int b_error;
uint16_t b_iocmd; /* BIO_* bio_cmd from bio.h */
uint16_t b_ioflags; /* BIO_* bio_flags from bio.h */
off_t b_iooffset;
long b_resid;
void (*b_iodone)(struct buf *);
void (*b_ckhashcalc)(struct buf *);
uint64_t b_ckhash; /* B_CKHASH requested check-hash */
daddr_t b_blkno; /* Underlying physical block number. */
off_t b_offset; /* Offset into file. */
TAILQ_ENTRY(buf) b_bobufs; /* (V) Buffer's associated vnode. */
uint32_t b_vflags; /* (V) BV_* flags */
uint8_t b_qindex; /* (Q) buffer queue index */
uint8_t b_domain; /* (Q) buf domain this resides in */
uint16_t b_subqueue; /* (Q) per-cpu q if any */
uint32_t b_flags; /* B_* flags. */
b_xflags_t b_xflags; /* extra flags */
struct lock b_lock; /* Buffer lock */
long b_bufsize; /* Allocated buffer size. */
int b_runningbufspace; /* when I/O is running, pipelining */
int b_kvasize; /* size of kva for buffer */
int b_dirtyoff; /* Offset in buffer of dirty region. */
int b_dirtyend; /* Offset of end of dirty region. */
caddr_t b_kvabase; /* base kva for buffer */
daddr_t b_lblkno; /* Logical block number. */
struct vnode *b_vp; /* Device vnode. */
struct ucred *b_rcred; /* Read credentials reference. */
struct ucred *b_wcred; /* Write credentials reference. */
union {
TAILQ_ENTRY(buf) b_freelist; /* (Q) */
struct {
void (*b_pgiodone)(void *, vm_page_t *, int, int);
int b_pgbefore;
int b_pgafter;
};
};
union cluster_info {
TAILQ_HEAD(cluster_list_head, buf) cluster_head;
TAILQ_ENTRY(buf) cluster_entry;
} b_cluster;
struct vm_page *b_pages[btoc(MAXPHYS)];
int b_npages;
struct workhead b_dep; /* (D) List of filesystem dependencies. */
void *b_fsprivate1;
void *b_fsprivate2;
void *b_fsprivate3;
#if defined(FULL_BUF_TRACKING)
#define BUF_TRACKING_SIZE 32
#define BUF_TRACKING_ENTRY(x) ((x) & (BUF_TRACKING_SIZE - 1))
const char *b_io_tracking[BUF_TRACKING_SIZE];
uint32_t b_io_tcnt;
#elif defined(BUF_TRACKING)
const char *b_io_tracking;
#endif
};
#define b_object b_bufobj->bo_object
/*
* These flags are kept in b_flags.
*
* Notes:
*
* B_ASYNC VOP calls on bp's are usually async whether or not
* B_ASYNC is set, but some subsystems, such as NFS, like
* to know what is best for the caller so they can
* optimize the I/O.
*
* B_PAGING Indicates that bp is being used by the paging system or
* some paging system and that the bp is not linked into
* the b_vp's clean/dirty linked lists or ref counts.
* Buffer vp reassignments are illegal in this case.
*
* B_CACHE This may only be set if the buffer is entirely valid.
* The situation where B_DELWRI is set and B_CACHE is
* clear MUST be committed to disk by getblk() so
* B_DELWRI can also be cleared. See the comments for
* getblk() in kern/vfs_bio.c. If B_CACHE is clear,
* the caller is expected to clear BIO_ERROR and B_INVAL,
* set BIO_READ, and initiate an I/O.
*
* The 'entire buffer' is defined to be the range from
* 0 through b_bcount.
*
* B_MALLOC Request that the buffer be allocated from the malloc
* pool, DEV_BSIZE aligned instead of PAGE_SIZE aligned.
*
* B_CLUSTEROK This flag is typically set for B_DELWRI buffers
* by filesystems that allow clustering when the buffer
* is fully dirty and indicates that it may be clustered
* with other adjacent dirty buffers. Note the clustering
* may not be used with the stage 1 data write under NFS
* but may be used for the commit rpc portion.
*
* B_INVALONERR This flag is set on dirty buffers. It specifies that a
* write error should forcibly invalidate the buffer
* contents. This flag should be used with caution, as it
* discards data. It is incompatible with B_ASYNC.
*
* B_VMIO Indicates that the buffer is tied into an VM object.
* The buffer's data is always PAGE_SIZE aligned even
* if b_bufsize and b_bcount are not. ( b_bufsize is
* always at least DEV_BSIZE aligned, though ).
*
* B_DIRECT Hint that we should attempt to completely free
* the pages underlying the buffer. B_DIRECT is
* sticky until the buffer is released and typically
* only has an effect when B_RELBUF is also set.
*
*/
#define B_AGE 0x00000001 /* Move to age queue when I/O done. */
#define B_NEEDCOMMIT 0x00000002 /* Append-write in progress. */
#define B_ASYNC 0x00000004 /* Start I/O, do not wait. */
#define B_DIRECT 0x00000008 /* direct I/O flag (pls free vmio) */
#define B_DEFERRED 0x00000010 /* Skipped over for cleaning */
#define B_CACHE 0x00000020 /* Bread found us in the cache. */
#define B_VALIDSUSPWRT 0x00000040 /* Valid write during suspension. */
#define B_DELWRI 0x00000080 /* Delay I/O until buffer reused. */
#define B_CKHASH 0x00000100 /* checksum hash calculated on read */
#define B_DONE 0x00000200 /* I/O completed. */
#define B_EINTR 0x00000400 /* I/O was interrupted */
#define B_NOREUSE 0x00000800 /* Contents not reused once released. */
#define B_REUSE 0x00001000 /* Contents reused, second chance. */
#define B_INVAL 0x00002000 /* Does not contain valid info. */
#define B_BARRIER 0x00004000 /* Write this and all preceding first. */
#define B_NOCACHE 0x00008000 /* Do not cache block after use. */
#define B_MALLOC 0x00010000 /* malloced b_data */
#define B_CLUSTEROK 0x00020000 /* Pagein op, so swap() can count it. */
#define B_INVALONERR 0x00040000 /* Invalidate on write error. */
#define B_00080000 0x00080000 /* Available flag. */
#define B_00100000 0x00100000 /* Available flag. */
#define B_00200000 0x00200000 /* Available flag. */
#define B_RELBUF 0x00400000 /* Release VMIO buffer. */
#define B_FS_FLAG1 0x00800000 /* Available flag for FS use. */
#define B_NOCOPY 0x01000000 /* Don't copy-on-write this buf. */
#define B_INFREECNT 0x02000000 /* buf is counted in numfreebufs */
#define B_PAGING 0x04000000 /* volatile paging I/O -- bypass VMIO */
#define B_MANAGED 0x08000000 /* Managed by FS. */
#define B_RAM 0x10000000 /* Read ahead mark (flag) */
#define B_VMIO 0x20000000 /* VMIO flag */
#define B_CLUSTER 0x40000000 /* pagein op, so swap() can count it */
#define B_REMFREE 0x80000000 /* Delayed bremfree */
#define PRINT_BUF_FLAGS "\20\40remfree\37cluster\36vmio\35ram\34managed" \
"\33paging\32infreecnt\31nocopy\30b23\27relbuf\26b21\25b20" \
"\24b19\23invalonerr\22clusterok\21malloc\20nocache\17b14\16inval" \
"\15reuse\14noreuse\13eintr\12done\11b8\10delwri" \
"\7validsuspwrt\6cache\5deferred\4direct\3async\2needcommit\1age"
/*
* These flags are kept in b_xflags.
*
* BX_FSPRIV reserves a set of eight flags that may be used by individual
* filesystems for their own purpose. Their specific definitions are
* found in the header files for each filesystem that uses them.
*/
#define BX_VNDIRTY 0x00000001 /* On vnode dirty list */
#define BX_VNCLEAN 0x00000002 /* On vnode clean list */
+#define BX_CVTENXIO 0x00000004 /* Convert errors to ENXIO */
#define BX_BKGRDWRITE 0x00000010 /* Do writes in background */
#define BX_BKGRDMARKER 0x00000020 /* Mark buffer for splay tree */
#define BX_ALTDATA 0x00000040 /* Holds extended data */
#define BX_FSPRIV 0x00FF0000 /* Filesystem-specific flags mask */
-#define PRINT_BUF_XFLAGS "\20\7altdata\6bkgrdmarker\5bkgrdwrite\2clean\1dirty"
+#define PRINT_BUF_XFLAGS "\20\7altdata\6bkgrdmarker\5bkgrdwrite\3cvtenxio" \
+ "\2clean\1dirty"
#define NOOFFSET (-1LL) /* No buffer offset calculated yet */
/*
* These flags are kept in b_vflags.
*/
#define BV_SCANNED 0x00000001 /* VOP_FSYNC funcs mark written bufs */
#define BV_BKGRDINPROG 0x00000002 /* Background write in progress */
#define BV_BKGRDWAIT 0x00000004 /* Background write waiting */
#define BV_BKGRDERR 0x00000008 /* Error from background write */
#define PRINT_BUF_VFLAGS "\20\4bkgrderr\3bkgrdwait\2bkgrdinprog\1scanned"
#ifdef _KERNEL
#ifndef NSWBUF_MIN
#define NSWBUF_MIN 16
#endif
/*
* Buffer locking
*/
extern const char *buf_wmesg; /* Default buffer lock message */
#define BUF_WMESG "bufwait"
#include <sys/proc.h> /* XXX for curthread */
#include <sys/mutex.h>
/*
* Initialize a lock.
*/
#define BUF_LOCKINIT(bp) \
lockinit(&(bp)->b_lock, PRIBIO + 4, buf_wmesg, 0, LK_NEW)
/*
*
* Get a lock sleeping non-interruptably until it becomes available.
*/
#define BUF_LOCK(bp, locktype, interlock) \
_lockmgr_args_rw(&(bp)->b_lock, (locktype), (interlock), \
LK_WMESG_DEFAULT, LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, \
LOCK_FILE, LOCK_LINE)
/*
* Get a lock sleeping with specified interruptably and timeout.
*/
#define BUF_TIMELOCK(bp, locktype, interlock, wmesg, catch, timo) \
_lockmgr_args_rw(&(bp)->b_lock, (locktype) | LK_TIMELOCK, \
(interlock), (wmesg), (PRIBIO + 4) | (catch), (timo), \
LOCK_FILE, LOCK_LINE)
/*
* Release a lock. Only the acquiring process may free the lock unless
* it has been handed off to biodone.
*/
#define BUF_UNLOCK(bp) do { \
KASSERT(((bp)->b_flags & B_REMFREE) == 0, \
("BUF_UNLOCK %p while B_REMFREE is still set.", (bp))); \
\
(void)_lockmgr_args(&(bp)->b_lock, LK_RELEASE, NULL, \
LK_WMESG_DEFAULT, LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, \
LOCK_FILE, LOCK_LINE); \
} while (0)
/*
* Check if a buffer lock is recursed.
*/
#define BUF_LOCKRECURSED(bp) \
lockmgr_recursed(&(bp)->b_lock)
/*
* Check if a buffer lock is currently held.
*/
#define BUF_ISLOCKED(bp) \
lockstatus(&(bp)->b_lock)
/*
* Free a buffer lock.
*/
#define BUF_LOCKFREE(bp) \
lockdestroy(&(bp)->b_lock)
/*
* Print informations on a buffer lock.
*/
#define BUF_LOCKPRINTINFO(bp) \
lockmgr_printinfo(&(bp)->b_lock)
/*
* Buffer lock assertions.
*/
#if defined(INVARIANTS) && defined(INVARIANT_SUPPORT)
#define BUF_ASSERT_LOCKED(bp) \
_lockmgr_assert(&(bp)->b_lock, KA_LOCKED, LOCK_FILE, LOCK_LINE)
#define BUF_ASSERT_SLOCKED(bp) \
_lockmgr_assert(&(bp)->b_lock, KA_SLOCKED, LOCK_FILE, LOCK_LINE)
#define BUF_ASSERT_XLOCKED(bp) \
_lockmgr_assert(&(bp)->b_lock, KA_XLOCKED, LOCK_FILE, LOCK_LINE)
#define BUF_ASSERT_UNLOCKED(bp) \
_lockmgr_assert(&(bp)->b_lock, KA_UNLOCKED, LOCK_FILE, LOCK_LINE)
#else
#define BUF_ASSERT_LOCKED(bp)
#define BUF_ASSERT_SLOCKED(bp)
#define BUF_ASSERT_XLOCKED(bp)
#define BUF_ASSERT_UNLOCKED(bp)
#endif
#ifdef _SYS_PROC_H_ /* Avoid #include <sys/proc.h> pollution */
/*
* When initiating asynchronous I/O, change ownership of the lock to the
* kernel. Once done, the lock may legally released by biodone. The
* original owning process can no longer acquire it recursively, but must
* wait until the I/O is completed and the lock has been freed by biodone.
*/
#define BUF_KERNPROC(bp) \
_lockmgr_disown(&(bp)->b_lock, LOCK_FILE, LOCK_LINE)
#endif
#endif /* _KERNEL */
struct buf_queue_head {
TAILQ_HEAD(buf_queue, buf) queue;
daddr_t last_pblkno;
struct buf *insert_point;
struct buf *switch_point;
};
/*
* This structure describes a clustered I/O.
*/
struct cluster_save {
long bs_bcount; /* Saved b_bcount. */
long bs_bufsize; /* Saved b_bufsize. */
int bs_nchildren; /* Number of associated buffers. */
struct buf **bs_children; /* List of associated buffers. */
};
#ifdef _KERNEL
static __inline int
bwrite(struct buf *bp)
{
KASSERT(bp->b_bufobj != NULL, ("bwrite: no bufobj bp=%p", bp));
KASSERT(bp->b_bufobj->bo_ops != NULL, ("bwrite: no bo_ops bp=%p", bp));
KASSERT(bp->b_bufobj->bo_ops->bop_write != NULL,
("bwrite: no bop_write bp=%p", bp));
return (BO_WRITE(bp->b_bufobj, bp));
}
static __inline void
bstrategy(struct buf *bp)
{
KASSERT(bp->b_bufobj != NULL, ("bstrategy: no bufobj bp=%p", bp));
KASSERT(bp->b_bufobj->bo_ops != NULL,
("bstrategy: no bo_ops bp=%p", bp));
KASSERT(bp->b_bufobj->bo_ops->bop_strategy != NULL,
("bstrategy: no bop_strategy bp=%p", bp));
BO_STRATEGY(bp->b_bufobj, bp);
}
static __inline void
buf_start(struct buf *bp)
{
if (bioops.io_start)
(*bioops.io_start)(bp);
}
static __inline void
buf_complete(struct buf *bp)
{
if (bioops.io_complete)
(*bioops.io_complete)(bp);
}
static __inline void
buf_deallocate(struct buf *bp)
{
if (bioops.io_deallocate)
(*bioops.io_deallocate)(bp);
}
static __inline int
buf_countdeps(struct buf *bp, int i)
{
if (bioops.io_countdeps)
return ((*bioops.io_countdeps)(bp, i));
else
return (0);
}
static __inline void
buf_track(struct buf *bp __unused, const char *location __unused)
{
#if defined(FULL_BUF_TRACKING)
bp->b_io_tracking[BUF_TRACKING_ENTRY(bp->b_io_tcnt++)] = location;
#elif defined(BUF_TRACKING)
bp->b_io_tracking = location;
#endif
}
#endif /* _KERNEL */
/*
* Zero out the buffer's data area.
*/
#define clrbuf(bp) { \
bzero((bp)->b_data, (u_int)(bp)->b_bcount); \
(bp)->b_resid = 0; \
}
/*
* Flags for getblk's last parameter.
*/
#define GB_LOCK_NOWAIT 0x0001 /* Fail if we block on a buf lock. */
#define GB_NOCREAT 0x0002 /* Don't create a buf if not found. */
#define GB_NOWAIT_BD 0x0004 /* Do not wait for bufdaemon. */
#define GB_UNMAPPED 0x0008 /* Do not mmap buffer pages. */
#define GB_KVAALLOC 0x0010 /* But allocate KVA. */
#define GB_CKHASH 0x0020 /* If reading, calc checksum hash */
#define GB_NOSPARSE 0x0040 /* Do not instantiate holes */
+#define GB_CVTENXIO 0x0080 /* Convert errors to ENXIO */
#ifdef _KERNEL
extern int nbuf; /* The number of buffer headers */
extern long maxswzone; /* Max KVA for swap structures */
extern long maxbcache; /* Max KVA for buffer cache */
extern int maxbcachebuf; /* Max buffer cache block size */
extern long runningbufspace;
extern long hibufspace;
extern int dirtybufthresh;
extern int bdwriteskip;
extern int dirtybufferflushes;
extern int altbufferflushes;
extern int nswbuf; /* Number of swap I/O buffer headers. */
extern caddr_t unmapped_buf; /* Data address for unmapped buffers. */
static inline int
buf_mapped(struct buf *bp)
{
return (bp->b_data != unmapped_buf);
}
void runningbufwakeup(struct buf *);
void waitrunningbufspace(void);
caddr_t kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est);
void bufinit(void);
void bufshutdown(int);
void bdata2bio(struct buf *bp, struct bio *bip);
void bwillwrite(void);
int buf_dirty_count_severe(void);
void bremfree(struct buf *);
void bremfreef(struct buf *); /* XXX Force bremfree, only for nfs. */
#define bread(vp, blkno, size, cred, bpp) \
breadn_flags(vp, blkno, blkno, size, NULL, NULL, 0, cred, 0, \
NULL, bpp)
#define bread_gb(vp, blkno, size, cred, gbflags, bpp) \
breadn_flags(vp, blkno, blkno, size, NULL, NULL, 0, cred, \
gbflags, NULL, bpp)
#define breadn(vp, blkno, size, rablkno, rabsize, cnt, cred, bpp) \
breadn_flags(vp, blkno, blkno, size, rablkno, rabsize, cnt, cred, \
0, NULL, bpp)
int breadn_flags(struct vnode *, daddr_t, daddr_t, int, daddr_t *, int *,
int, struct ucred *, int, void (*)(struct buf *), struct buf **);
void bdwrite(struct buf *);
void bawrite(struct buf *);
void babarrierwrite(struct buf *);
int bbarrierwrite(struct buf *);
void bdirty(struct buf *);
void bundirty(struct buf *);
void bufstrategy(struct bufobj *, struct buf *);
void brelse(struct buf *);
void bqrelse(struct buf *);
int vfs_bio_awrite(struct buf *);
void vfs_busy_pages_acquire(struct buf *bp);
void vfs_busy_pages_release(struct buf *bp);
struct buf *incore(struct bufobj *, daddr_t);
struct buf *gbincore(struct bufobj *, daddr_t);
struct buf *getblk(struct vnode *, daddr_t, int, int, int, int);
int getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
int slpflag, int slptimeo, int flags, struct buf **bpp);
struct buf *geteblk(int, int);
int bufwait(struct buf *);
int bufwrite(struct buf *);
void bufdone(struct buf *);
void bd_speedup(void);
extern uma_zone_t pbuf_zone;
uma_zone_t pbuf_zsecond_create(const char *name, int max);
int cluster_read(struct vnode *, u_quad_t, daddr_t, long,
struct ucred *, long, int, int, struct buf **);
int cluster_wbuild(struct vnode *, long, daddr_t, int, int);
void cluster_write(struct vnode *, struct buf *, u_quad_t, int, int);
void vfs_bio_brelse(struct buf *bp, int ioflags);
void vfs_bio_bzero_buf(struct buf *bp, int base, int size);
void vfs_bio_clrbuf(struct buf *);
void vfs_bio_set_flags(struct buf *bp, int ioflags);
void vfs_bio_set_valid(struct buf *, int base, int size);
void vfs_busy_pages(struct buf *, int clear_modify);
void vfs_unbusy_pages(struct buf *);
int vmapbuf(struct buf *, int);
void vunmapbuf(struct buf *);
void brelvp(struct buf *);
void bgetvp(struct vnode *, struct buf *);
void pbgetbo(struct bufobj *bo, struct buf *bp);
void pbgetvp(struct vnode *, struct buf *);
void pbrelbo(struct buf *);
void pbrelvp(struct buf *);
int allocbuf(struct buf *bp, int size);
void reassignbuf(struct buf *);
void bwait(struct buf *, u_char, const char *);
void bdone(struct buf *);
typedef daddr_t (vbg_get_lblkno_t)(struct vnode *, vm_ooffset_t);
typedef int (vbg_get_blksize_t)(struct vnode *, daddr_t);
int vfs_bio_getpages(struct vnode *vp, struct vm_page **ma, int count,
int *rbehind, int *rahead, vbg_get_lblkno_t get_lblkno,
vbg_get_blksize_t get_blksize);
#endif /* _KERNEL */
#endif /* !_SYS_BUF_H_ */
Index: head/sys/ufs/ffs/ffs_alloc.c
===================================================================
--- head/sys/ufs/ffs/ffs_alloc.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_alloc.c (revision 361491)
@@ -1,3483 +1,3519 @@
/*-
* SPDX-License-Identifier: (BSD-2-Clause-FreeBSD AND BSD-3-Clause)
*
* Copyright (c) 2002 Networks Associates Technology, Inc.
* All rights reserved.
*
* This software was developed for the FreeBSD Project by Marshall
* Kirk McKusick and Network Associates Laboratories, the Security
* Research Division of Network Associates, Inc. under DARPA/SPAWAR
* contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS
* research program
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_alloc.c 8.18 (Berkeley) 5/26/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_quota.h"
#include <sys/param.h>
#include <sys/capsicum.h>
#include <sys/gsb_crc32.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/kernel.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/taskqueue.h>
#include <security/audit/audit.h>
#include <geom/geom.h>
#include <geom/geom_vfs.h>
#include <ufs/ufs/dir.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
#include <ufs/ffs/softdep.h>
typedef ufs2_daddr_t allocfcn_t(struct inode *ip, u_int cg, ufs2_daddr_t bpref,
int size, int rsize);
static ufs2_daddr_t ffs_alloccg(struct inode *, u_int, ufs2_daddr_t, int, int);
static ufs2_daddr_t
ffs_alloccgblk(struct inode *, struct buf *, ufs2_daddr_t, int);
static void ffs_blkfree_cg(struct ufsmount *, struct fs *,
struct vnode *, ufs2_daddr_t, long, ino_t,
struct workhead *);
#ifdef INVARIANTS
static int ffs_checkblk(struct inode *, ufs2_daddr_t, long);
#endif
static ufs2_daddr_t ffs_clusteralloc(struct inode *, u_int, ufs2_daddr_t, int);
static ino_t ffs_dirpref(struct inode *);
static ufs2_daddr_t ffs_fragextend(struct inode *, u_int, ufs2_daddr_t,
int, int);
static ufs2_daddr_t ffs_hashalloc
(struct inode *, u_int, ufs2_daddr_t, int, int, allocfcn_t *);
static ufs2_daddr_t ffs_nodealloccg(struct inode *, u_int, ufs2_daddr_t, int,
int);
static ufs1_daddr_t ffs_mapsearch(struct fs *, struct cg *, ufs2_daddr_t, int);
static int ffs_reallocblks_ufs1(struct vop_reallocblks_args *);
static int ffs_reallocblks_ufs2(struct vop_reallocblks_args *);
static void ffs_ckhash_cg(struct buf *);
/*
* Allocate a block in the filesystem.
*
* The size of the requested block is given, which must be some
* multiple of fs_fsize and <= fs_bsize.
* A preference may be optionally specified. If a preference is given
* the following hierarchy is used to allocate a block:
* 1) allocate the requested block.
* 2) allocate a rotationally optimal block in the same cylinder.
* 3) allocate a block in the same cylinder group.
* 4) quadradically rehash into other cylinder groups, until an
* available block is located.
* If no block preference is given the following hierarchy is used
* to allocate a block:
* 1) allocate a block in the cylinder group that contains the
* inode for the file.
* 2) quadradically rehash into other cylinder groups, until an
* available block is located.
*/
int
ffs_alloc(ip, lbn, bpref, size, flags, cred, bnp)
struct inode *ip;
ufs2_daddr_t lbn, bpref;
int size, flags;
struct ucred *cred;
ufs2_daddr_t *bnp;
{
struct fs *fs;
struct ufsmount *ump;
ufs2_daddr_t bno;
u_int cg, reclaimed;
int64_t delta;
#ifdef QUOTA
int error;
#endif
*bnp = 0;
ump = ITOUMP(ip);
fs = ump->um_fs;
mtx_assert(UFS_MTX(ump), MA_OWNED);
#ifdef INVARIANTS
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) {
printf("dev = %s, bsize = %ld, size = %d, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, size,
fs->fs_fsmnt);
panic("ffs_alloc: bad size");
}
if (cred == NOCRED)
panic("ffs_alloc: missing credential");
#endif /* INVARIANTS */
reclaimed = 0;
retry:
#ifdef QUOTA
UFS_UNLOCK(ump);
error = chkdq(ip, btodb(size), cred, 0);
if (error)
return (error);
UFS_LOCK(ump);
#endif
if (size == fs->fs_bsize && fs->fs_cstotal.cs_nbfree == 0)
goto nospace;
if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE) &&
freespace(fs, fs->fs_minfree) - numfrags(fs, size) < 0)
goto nospace;
if (bpref >= fs->fs_size)
bpref = 0;
if (bpref == 0)
cg = ino_to_cg(fs, ip->i_number);
else
cg = dtog(fs, bpref);
bno = ffs_hashalloc(ip, cg, bpref, size, size, ffs_alloccg);
if (bno > 0) {
delta = btodb(size);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
*bnp = bno;
return (0);
}
nospace:
#ifdef QUOTA
UFS_UNLOCK(ump);
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(size), cred, FORCE);
UFS_LOCK(ump);
#endif
if (reclaimed == 0 && (flags & IO_BUFLOCKED) == 0) {
reclaimed = 1;
softdep_request_cleanup(fs, ITOV(ip), cred, FLUSH_BLOCKS_WAIT);
goto retry;
}
+ if (ffs_fsfail_cleanup_locked(ump, 0)) {
+ UFS_UNLOCK(ump);
+ return (ENXIO);
+ }
if (reclaimed > 0 &&
ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem is full\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
return (ENOSPC);
}
/*
* Reallocate a fragment to a bigger size
*
* The number and size of the old block is given, and a preference
* and new size is also specified. The allocator attempts to extend
* the original block. Failing that, the regular block allocator is
* invoked to get an appropriate block.
*/
int
ffs_realloccg(ip, lbprev, bprev, bpref, osize, nsize, flags, cred, bpp)
struct inode *ip;
ufs2_daddr_t lbprev;
ufs2_daddr_t bprev;
ufs2_daddr_t bpref;
int osize, nsize, flags;
struct ucred *cred;
struct buf **bpp;
{
struct vnode *vp;
struct fs *fs;
struct buf *bp;
struct ufsmount *ump;
u_int cg, request, reclaimed;
int error, gbflags;
ufs2_daddr_t bno;
int64_t delta;
vp = ITOV(ip);
ump = ITOUMP(ip);
fs = ump->um_fs;
bp = NULL;
gbflags = (flags & BA_UNMAPPED) != 0 ? GB_UNMAPPED : 0;
mtx_assert(UFS_MTX(ump), MA_OWNED);
#ifdef INVARIANTS
if (vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED)
panic("ffs_realloccg: allocation on suspended filesystem");
if ((u_int)osize > fs->fs_bsize || fragoff(fs, osize) != 0 ||
(u_int)nsize > fs->fs_bsize || fragoff(fs, nsize) != 0) {
printf(
"dev = %s, bsize = %ld, osize = %d, nsize = %d, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, osize,
nsize, fs->fs_fsmnt);
panic("ffs_realloccg: bad size");
}
if (cred == NOCRED)
panic("ffs_realloccg: missing credential");
#endif /* INVARIANTS */
reclaimed = 0;
retry:
if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE) &&
freespace(fs, fs->fs_minfree) - numfrags(fs, nsize - osize) < 0) {
goto nospace;
}
if (bprev == 0) {
printf("dev = %s, bsize = %ld, bprev = %jd, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, (intmax_t)bprev,
fs->fs_fsmnt);
panic("ffs_realloccg: bad bprev");
}
UFS_UNLOCK(ump);
/*
* Allocate the extra space in the buffer.
*/
error = bread_gb(vp, lbprev, osize, NOCRED, gbflags, &bp);
if (error) {
return (error);
}
if (bp->b_blkno == bp->b_lblkno) {
if (lbprev >= UFS_NDADDR)
panic("ffs_realloccg: lbprev out of range");
bp->b_blkno = fsbtodb(fs, bprev);
}
#ifdef QUOTA
error = chkdq(ip, btodb(nsize - osize), cred, 0);
if (error) {
brelse(bp);
return (error);
}
#endif
/*
* Check for extension in the existing location.
*/
*bpp = NULL;
cg = dtog(fs, bprev);
UFS_LOCK(ump);
bno = ffs_fragextend(ip, cg, bprev, osize, nsize);
if (bno) {
if (bp->b_blkno != fsbtodb(fs, bno))
panic("ffs_realloccg: bad blockno");
delta = btodb(nsize - osize);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allocbuf(bp, nsize);
bp->b_flags |= B_DONE;
vfs_bio_bzero_buf(bp, osize, nsize - osize);
if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO)
vfs_bio_set_valid(bp, osize, nsize - osize);
*bpp = bp;
return (0);
}
/*
* Allocate a new disk location.
*/
if (bpref >= fs->fs_size)
bpref = 0;
switch ((int)fs->fs_optim) {
case FS_OPTSPACE:
/*
* Allocate an exact sized fragment. Although this makes
* best use of space, we will waste time relocating it if
* the file continues to grow. If the fragmentation is
* less than half of the minimum free reserve, we choose
* to begin optimizing for time.
*/
request = nsize;
if (fs->fs_minfree <= 5 ||
fs->fs_cstotal.cs_nffree >
(off_t)fs->fs_dsize * fs->fs_minfree / (2 * 100))
break;
log(LOG_NOTICE, "%s: optimization changed from SPACE to TIME\n",
fs->fs_fsmnt);
fs->fs_optim = FS_OPTTIME;
break;
case FS_OPTTIME:
/*
* At this point we have discovered a file that is trying to
* grow a small fragment to a larger fragment. To save time,
* we allocate a full sized block, then free the unused portion.
* If the file continues to grow, the `ffs_fragextend' call
* above will be able to grow it in place without further
* copying. If aberrant programs cause disk fragmentation to
* grow within 2% of the free reserve, we choose to begin
* optimizing for space.
*/
request = fs->fs_bsize;
if (fs->fs_cstotal.cs_nffree <
(off_t)fs->fs_dsize * (fs->fs_minfree - 2) / 100)
break;
log(LOG_NOTICE, "%s: optimization changed from TIME to SPACE\n",
fs->fs_fsmnt);
fs->fs_optim = FS_OPTSPACE;
break;
default:
printf("dev = %s, optim = %ld, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_optim, fs->fs_fsmnt);
panic("ffs_realloccg: bad optim");
/* NOTREACHED */
}
bno = ffs_hashalloc(ip, cg, bpref, request, nsize, ffs_alloccg);
if (bno > 0) {
bp->b_blkno = fsbtodb(fs, bno);
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a smaller fragment that
* was just allocated has been replaced with a bigger
* fragment or a full-size block. If it is marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the block was written
* earlier, but very uncommon. If the block has never
* been written, there is no need to send a BIO_DELETE
* for it when it is freed. The gain from avoiding the
* TRIMs for the common case of unwritten blocks far
* exceeds the cost of the write amplification for the
* uncommon case of failing to send a TRIM for a block
* that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp, bprev, (long)osize,
ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
delta = btodb(nsize - osize);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allocbuf(bp, nsize);
bp->b_flags |= B_DONE;
vfs_bio_bzero_buf(bp, osize, nsize - osize);
if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO)
vfs_bio_set_valid(bp, osize, nsize - osize);
*bpp = bp;
return (0);
}
#ifdef QUOTA
UFS_UNLOCK(ump);
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(nsize - osize), cred, FORCE);
UFS_LOCK(ump);
#endif
nospace:
/*
* no space available
*/
if (reclaimed == 0 && (flags & IO_BUFLOCKED) == 0) {
reclaimed = 1;
UFS_UNLOCK(ump);
if (bp) {
brelse(bp);
bp = NULL;
}
UFS_LOCK(ump);
softdep_request_cleanup(fs, vp, cred, FLUSH_BLOCKS_WAIT);
goto retry;
}
+ if (bp)
+ brelse(bp);
+ if (ffs_fsfail_cleanup_locked(ump, 0)) {
+ UFS_UNLOCK(ump);
+ return (ENXIO);
+ }
if (reclaimed > 0 &&
ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem is full\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
- if (bp)
- brelse(bp);
return (ENOSPC);
}
/*
* Reallocate a sequence of blocks into a contiguous sequence of blocks.
*
* The vnode and an array of buffer pointers for a range of sequential
* logical blocks to be made contiguous is given. The allocator attempts
* to find a range of sequential blocks starting as close as possible
* from the end of the allocation for the logical block immediately
* preceding the current range. If successful, the physical block numbers
* in the buffer pointers and in the inode are changed to reflect the new
* allocation. If unsuccessful, the allocation is left unchanged. The
* success in doing the reallocation is returned. Note that the error
* return is not reflected back to the user. Rather the previous block
* allocation will be used.
*/
SYSCTL_NODE(_vfs, OID_AUTO, ffs, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"FFS filesystem");
static int doasyncfree = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doasyncfree, CTLFLAG_RW, &doasyncfree, 0,
"do not force synchronous writes when blocks are reallocated");
static int doreallocblks = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doreallocblks, CTLFLAG_RW, &doreallocblks, 0,
"enable block reallocation");
static int dotrimcons = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, dotrimcons, CTLFLAG_RWTUN, &dotrimcons, 0,
"enable BIO_DELETE / TRIM consolidation");
static int maxclustersearch = 10;
SYSCTL_INT(_vfs_ffs, OID_AUTO, maxclustersearch, CTLFLAG_RW, &maxclustersearch,
0, "max number of cylinder group to search for contigous blocks");
#ifdef DIAGNOSTIC
static int prtrealloc = 0;
SYSCTL_INT(_debug, OID_AUTO, ffs_prtrealloc, CTLFLAG_RW, &prtrealloc, 0,
"print out FFS filesystem block reallocation operations");
#endif
int
ffs_reallocblks(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct ufsmount *ump;
/*
* We used to skip reallocating the blocks of a file into a
* contiguous sequence if the underlying flash device requested
* BIO_DELETE notifications, because devices that benefit from
* BIO_DELETE also benefit from not moving the data. However,
* the destination for the data is usually moved before the data
* is written to the initially allocated location, so we rarely
* suffer the penalty of extra writes. With the addition of the
* consolidation of contiguous blocks into single BIO_DELETE
* operations, having fewer but larger contiguous blocks reduces
* the number of (slow and expensive) BIO_DELETE operations. So
* when doing BIO_DELETE consolidation, we do block reallocation.
*
* Skip if reallocblks has been disabled globally.
*/
ump = ap->a_vp->v_mount->mnt_data;
if ((((ump->um_flags) & UM_CANDELETE) != 0 && dotrimcons == 0) ||
doreallocblks == 0)
return (ENOSPC);
/*
* We can't wait in softdep prealloc as it may fsync and recurse
* here. Instead we simply fail to reallocate blocks if this
* rare condition arises.
*/
if (DOINGSOFTDEP(ap->a_vp))
if (softdep_prealloc(ap->a_vp, MNT_NOWAIT) != 0)
return (ENOSPC);
if (ump->um_fstype == UFS1)
return (ffs_reallocblks_ufs1(ap));
return (ffs_reallocblks_ufs2(ap));
}
static int
ffs_reallocblks_ufs1(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct fs *fs;
struct inode *ip;
struct vnode *vp;
struct buf *sbp, *ebp, *bp;
ufs1_daddr_t *bap, *sbap, *ebap;
struct cluster_save *buflist;
struct ufsmount *ump;
ufs_lbn_t start_lbn, end_lbn;
ufs1_daddr_t soff, newblk, blkno;
ufs2_daddr_t pref;
struct indir start_ap[UFS_NIADDR + 1], end_ap[UFS_NIADDR + 1], *idp;
int i, cg, len, start_lvl, end_lvl, ssize;
vp = ap->a_vp;
ip = VTOI(vp);
ump = ITOUMP(ip);
fs = ump->um_fs;
/*
* If we are not tracking block clusters or if we have less than 4%
* free blocks left, then do not attempt to cluster. Running with
* less than 5% free block reserve is not recommended and those that
* choose to do so do not expect to have good file layout.
*/
if (fs->fs_contigsumsize <= 0 || freespace(fs, 4) < 0)
return (ENOSPC);
buflist = ap->a_buflist;
len = buflist->bs_nchildren;
start_lbn = buflist->bs_children[0]->b_lblkno;
end_lbn = start_lbn + len - 1;
#ifdef INVARIANTS
for (i = 0; i < len; i++)
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 1");
for (i = 1; i < len; i++)
if (buflist->bs_children[i]->b_lblkno != start_lbn + i)
panic("ffs_reallocblks: non-logical cluster");
blkno = buflist->bs_children[0]->b_blkno;
ssize = fsbtodb(fs, fs->fs_frag);
for (i = 1; i < len - 1; i++)
if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize))
panic("ffs_reallocblks: non-physical cluster %d", i);
#endif
/*
* If the cluster crosses the boundary for the first indirect
* block, leave space for the indirect block. Indirect blocks
* are initially laid out in a position after the last direct
* block. Block reallocation would usually destroy locality by
* moving the indirect block out of the way to make room for
* data blocks if we didn't compensate here. We should also do
* this for other indirect block boundaries, but it is only
* important for the first one.
*/
if (start_lbn < UFS_NDADDR && end_lbn >= UFS_NDADDR)
return (ENOSPC);
/*
* If the latest allocation is in a new cylinder group, assume that
* the filesystem has decided to move and do not force it back to
* the previous cylinder group.
*/
if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) !=
dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno)))
return (ENOSPC);
if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) ||
ufs_getlbns(vp, end_lbn, end_ap, &end_lvl))
return (ENOSPC);
/*
* Get the starting offset and block map for the first block.
*/
if (start_lvl == 0) {
sbap = &ip->i_din1->di_db[0];
soff = start_lbn;
} else {
idp = &start_ap[start_lvl - 1];
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) {
brelse(sbp);
return (ENOSPC);
}
sbap = (ufs1_daddr_t *)sbp->b_data;
soff = idp->in_off;
}
/*
* If the block range spans two block maps, get the second map.
*/
ebap = NULL;
if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) {
ssize = len;
} else {
#ifdef INVARIANTS
if (start_lvl > 0 &&
start_ap[start_lvl - 1].in_lbn == idp->in_lbn)
panic("ffs_reallocblk: start == end");
#endif
ssize = len - (idp->in_off + 1);
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp))
goto fail;
ebap = (ufs1_daddr_t *)ebp->b_data;
}
/*
* Find the preferred location for the cluster. If we have not
* previously failed at this endeavor, then follow our standard
* preference calculation. If we have failed at it, then pick up
* where we last ended our search.
*/
UFS_LOCK(ump);
if (ip->i_nextclustercg == -1)
pref = ffs_blkpref_ufs1(ip, start_lbn, soff, sbap);
else
pref = cgdata(fs, ip->i_nextclustercg);
/*
* Search the block map looking for an allocation of the desired size.
* To avoid wasting too much time, we limit the number of cylinder
* groups that we will search.
*/
cg = dtog(fs, pref);
for (i = min(maxclustersearch, fs->fs_ncg); i > 0; i--) {
if ((newblk = ffs_clusteralloc(ip, cg, pref, len)) != 0)
break;
cg += 1;
if (cg >= fs->fs_ncg)
cg = 0;
}
/*
* If we have failed in our search, record where we gave up for
* next time. Otherwise, fall back to our usual search citerion.
*/
if (newblk == 0) {
ip->i_nextclustercg = cg;
UFS_UNLOCK(ump);
goto fail;
}
ip->i_nextclustercg = -1;
/*
* We have found a new contiguous block.
*
* First we have to replace the old block pointers with the new
* block pointers in the inode and indirect blocks associated
* with the file.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("realloc: ino %ju, lbns %jd-%jd\n\told:",
(uintmax_t)ip->i_number,
(intmax_t)start_lbn, (intmax_t)end_lbn);
#endif
blkno = newblk;
for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) {
if (i == ssize) {
bap = ebap;
soff = -i;
}
#ifdef INVARIANTS
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 2");
if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap)
panic("ffs_reallocblks: alloc mismatch");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %d,", *bap);
#endif
if (DOINGSOFTDEP(vp)) {
if (sbap == &ip->i_din1->di_db[0] && i < ssize)
softdep_setup_allocdirect(ip, start_lbn + i,
blkno, *bap, fs->fs_bsize, fs->fs_bsize,
buflist->bs_children[i]);
else
softdep_setup_allocindir_page(ip, start_lbn + i,
i < ssize ? sbp : ebp, soff + i, blkno,
*bap, buflist->bs_children[i]);
}
*bap++ = blkno;
}
/*
* Next we must write out the modified inode and indirect blocks.
* For strict correctness, the writes should be synchronous since
* the old block values may have been written to disk. In practise
* they are almost never written, but if we are concerned about
* strict correctness, the `doasyncfree' flag should be set to zero.
*
* The test on `doasyncfree' should be changed to test a flag
* that shows whether the associated buffers and inodes have
* been written. The flag should be set when the cluster is
* started and cleared whenever the buffer or inode is flushed.
* We can then check below to see if it is set, and do the
* synchronous write only when it has been cleared.
*/
if (sbap != &ip->i_din1->di_db[0]) {
if (doasyncfree)
bdwrite(sbp);
else
bwrite(sbp);
} else {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (!doasyncfree)
ffs_update(vp, 1);
}
if (ssize < len) {
if (doasyncfree)
bdwrite(ebp);
else
bwrite(ebp);
}
/*
* Last, free the old blocks and assign the new blocks to the buffers.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("\n\tnew:");
#endif
for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) {
bp = buflist->bs_children[i];
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a set of N-contiguous blocks
* that was just allocated has been replaced with a
* set of N+1-contiguous blocks. If they are marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the blocks were written
* earlier, but very uncommon. If the blocks have never
* been written, there is no need to send a BIO_DELETE
* for them when they are freed. The gain from avoiding
* the TRIMs for the common case of unwritten blocks
* far exceeds the cost of the write amplification for
* the uncommon case of failing to send a TRIM for the
* blocks that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp,
dbtofsb(fs, bp->b_blkno),
fs->fs_bsize, ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
bp->b_blkno = fsbtodb(fs, blkno);
#ifdef INVARIANTS
if (!ffs_checkblk(ip, dbtofsb(fs, bp->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 3");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %d,", blkno);
#endif
}
#ifdef DIAGNOSTIC
if (prtrealloc) {
prtrealloc--;
printf("\n");
}
#endif
return (0);
fail:
if (ssize < len)
brelse(ebp);
if (sbap != &ip->i_din1->di_db[0])
brelse(sbp);
return (ENOSPC);
}
static int
ffs_reallocblks_ufs2(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct fs *fs;
struct inode *ip;
struct vnode *vp;
struct buf *sbp, *ebp, *bp;
ufs2_daddr_t *bap, *sbap, *ebap;
struct cluster_save *buflist;
struct ufsmount *ump;
ufs_lbn_t start_lbn, end_lbn;
ufs2_daddr_t soff, newblk, blkno, pref;
struct indir start_ap[UFS_NIADDR + 1], end_ap[UFS_NIADDR + 1], *idp;
int i, cg, len, start_lvl, end_lvl, ssize;
vp = ap->a_vp;
ip = VTOI(vp);
ump = ITOUMP(ip);
fs = ump->um_fs;
/*
* If we are not tracking block clusters or if we have less than 4%
* free blocks left, then do not attempt to cluster. Running with
* less than 5% free block reserve is not recommended and those that
* choose to do so do not expect to have good file layout.
*/
if (fs->fs_contigsumsize <= 0 || freespace(fs, 4) < 0)
return (ENOSPC);
buflist = ap->a_buflist;
len = buflist->bs_nchildren;
start_lbn = buflist->bs_children[0]->b_lblkno;
end_lbn = start_lbn + len - 1;
#ifdef INVARIANTS
for (i = 0; i < len; i++)
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 1");
for (i = 1; i < len; i++)
if (buflist->bs_children[i]->b_lblkno != start_lbn + i)
panic("ffs_reallocblks: non-logical cluster");
blkno = buflist->bs_children[0]->b_blkno;
ssize = fsbtodb(fs, fs->fs_frag);
for (i = 1; i < len - 1; i++)
if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize))
panic("ffs_reallocblks: non-physical cluster %d", i);
#endif
/*
* If the cluster crosses the boundary for the first indirect
* block, do not move anything in it. Indirect blocks are
* usually initially laid out in a position between the data
* blocks. Block reallocation would usually destroy locality by
* moving the indirect block out of the way to make room for
* data blocks if we didn't compensate here. We should also do
* this for other indirect block boundaries, but it is only
* important for the first one.
*/
if (start_lbn < UFS_NDADDR && end_lbn >= UFS_NDADDR)
return (ENOSPC);
/*
* If the latest allocation is in a new cylinder group, assume that
* the filesystem has decided to move and do not force it back to
* the previous cylinder group.
*/
if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) !=
dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno)))
return (ENOSPC);
if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) ||
ufs_getlbns(vp, end_lbn, end_ap, &end_lvl))
return (ENOSPC);
/*
* Get the starting offset and block map for the first block.
*/
if (start_lvl == 0) {
sbap = &ip->i_din2->di_db[0];
soff = start_lbn;
} else {
idp = &start_ap[start_lvl - 1];
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) {
brelse(sbp);
return (ENOSPC);
}
sbap = (ufs2_daddr_t *)sbp->b_data;
soff = idp->in_off;
}
/*
* If the block range spans two block maps, get the second map.
*/
ebap = NULL;
if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) {
ssize = len;
} else {
#ifdef INVARIANTS
if (start_lvl > 0 &&
start_ap[start_lvl - 1].in_lbn == idp->in_lbn)
panic("ffs_reallocblk: start == end");
#endif
ssize = len - (idp->in_off + 1);
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp))
goto fail;
ebap = (ufs2_daddr_t *)ebp->b_data;
}
/*
* Find the preferred location for the cluster. If we have not
* previously failed at this endeavor, then follow our standard
* preference calculation. If we have failed at it, then pick up
* where we last ended our search.
*/
UFS_LOCK(ump);
if (ip->i_nextclustercg == -1)
pref = ffs_blkpref_ufs2(ip, start_lbn, soff, sbap);
else
pref = cgdata(fs, ip->i_nextclustercg);
/*
* Search the block map looking for an allocation of the desired size.
* To avoid wasting too much time, we limit the number of cylinder
* groups that we will search.
*/
cg = dtog(fs, pref);
for (i = min(maxclustersearch, fs->fs_ncg); i > 0; i--) {
if ((newblk = ffs_clusteralloc(ip, cg, pref, len)) != 0)
break;
cg += 1;
if (cg >= fs->fs_ncg)
cg = 0;
}
/*
* If we have failed in our search, record where we gave up for
* next time. Otherwise, fall back to our usual search citerion.
*/
if (newblk == 0) {
ip->i_nextclustercg = cg;
UFS_UNLOCK(ump);
goto fail;
}
ip->i_nextclustercg = -1;
/*
* We have found a new contiguous block.
*
* First we have to replace the old block pointers with the new
* block pointers in the inode and indirect blocks associated
* with the file.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("realloc: ino %ju, lbns %jd-%jd\n\told:", (uintmax_t)ip->i_number,
(intmax_t)start_lbn, (intmax_t)end_lbn);
#endif
blkno = newblk;
for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) {
if (i == ssize) {
bap = ebap;
soff = -i;
}
#ifdef INVARIANTS
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 2");
if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap)
panic("ffs_reallocblks: alloc mismatch");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %jd,", (intmax_t)*bap);
#endif
if (DOINGSOFTDEP(vp)) {
if (sbap == &ip->i_din2->di_db[0] && i < ssize)
softdep_setup_allocdirect(ip, start_lbn + i,
blkno, *bap, fs->fs_bsize, fs->fs_bsize,
buflist->bs_children[i]);
else
softdep_setup_allocindir_page(ip, start_lbn + i,
i < ssize ? sbp : ebp, soff + i, blkno,
*bap, buflist->bs_children[i]);
}
*bap++ = blkno;
}
/*
* Next we must write out the modified inode and indirect blocks.
* For strict correctness, the writes should be synchronous since
* the old block values may have been written to disk. In practise
* they are almost never written, but if we are concerned about
* strict correctness, the `doasyncfree' flag should be set to zero.
*
* The test on `doasyncfree' should be changed to test a flag
* that shows whether the associated buffers and inodes have
* been written. The flag should be set when the cluster is
* started and cleared whenever the buffer or inode is flushed.
* We can then check below to see if it is set, and do the
* synchronous write only when it has been cleared.
*/
if (sbap != &ip->i_din2->di_db[0]) {
if (doasyncfree)
bdwrite(sbp);
else
bwrite(sbp);
} else {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (!doasyncfree)
ffs_update(vp, 1);
}
if (ssize < len) {
if (doasyncfree)
bdwrite(ebp);
else
bwrite(ebp);
}
/*
* Last, free the old blocks and assign the new blocks to the buffers.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("\n\tnew:");
#endif
for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) {
bp = buflist->bs_children[i];
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a set of N-contiguous blocks
* that was just allocated has been replaced with a
* set of N+1-contiguous blocks. If they are marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the blocks were written
* earlier, but very uncommon. If the blocks have never
* been written, there is no need to send a BIO_DELETE
* for them when they are freed. The gain from avoiding
* the TRIMs for the common case of unwritten blocks
* far exceeds the cost of the write amplification for
* the uncommon case of failing to send a TRIM for the
* blocks that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp,
dbtofsb(fs, bp->b_blkno),
fs->fs_bsize, ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
bp->b_blkno = fsbtodb(fs, blkno);
#ifdef INVARIANTS
if (!ffs_checkblk(ip, dbtofsb(fs, bp->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 3");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %jd,", (intmax_t)blkno);
#endif
}
#ifdef DIAGNOSTIC
if (prtrealloc) {
prtrealloc--;
printf("\n");
}
#endif
return (0);
fail:
if (ssize < len)
brelse(ebp);
if (sbap != &ip->i_din2->di_db[0])
brelse(sbp);
return (ENOSPC);
}
/*
* Allocate an inode in the filesystem.
*
* If allocating a directory, use ffs_dirpref to select the inode.
* If allocating in a directory, the following hierarchy is followed:
* 1) allocate the preferred inode.
* 2) allocate an inode in the same cylinder group.
* 3) quadradically rehash into other cylinder groups, until an
* available inode is located.
* If no inode preference is given the following hierarchy is used
* to allocate an inode:
* 1) allocate an inode in cylinder group 0.
* 2) quadradically rehash into other cylinder groups, until an
* available inode is located.
*/
int
ffs_valloc(pvp, mode, cred, vpp)
struct vnode *pvp;
int mode;
struct ucred *cred;
struct vnode **vpp;
{
struct inode *pip;
struct fs *fs;
struct inode *ip;
struct timespec ts;
struct ufsmount *ump;
ino_t ino, ipref;
u_int cg;
- int error, error1, reclaimed;
+ int error, reclaimed;
*vpp = NULL;
pip = VTOI(pvp);
ump = ITOUMP(pip);
fs = ump->um_fs;
UFS_LOCK(ump);
reclaimed = 0;
retry:
if (fs->fs_cstotal.cs_nifree == 0)
goto noinodes;
if ((mode & IFMT) == IFDIR)
ipref = ffs_dirpref(pip);
else
ipref = pip->i_number;
if (ipref >= fs->fs_ncg * fs->fs_ipg)
ipref = 0;
cg = ino_to_cg(fs, ipref);
/*
* Track number of dirs created one after another
* in a same cg without intervening by files.
*/
if ((mode & IFMT) == IFDIR) {
if (fs->fs_contigdirs[cg] < 255)
fs->fs_contigdirs[cg]++;
} else {
if (fs->fs_contigdirs[cg] > 0)
fs->fs_contigdirs[cg]--;
}
ino = (ino_t)ffs_hashalloc(pip, cg, ipref, mode, 0,
(allocfcn_t *)ffs_nodealloccg);
if (ino == 0)
goto noinodes;
-
/*
* Get rid of the cached old vnode, force allocation of a new vnode
- * for this inode.
+ * for this inode. If this fails, release the allocated ino and
+ * return the error.
*/
- error = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp, FFSV_REPLACE);
- if (error) {
- error1 = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp,
- FFSV_FORCEINSMQ | FFSV_REPLACE);
+ if ((error = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp,
+ FFSV_FORCEINSMQ | FFSV_REPLACE)) != 0) {
ffs_vfree(pvp, ino, mode);
- if (error1 == 0) {
- ip = VTOI(*vpp);
- if (ip->i_mode)
- goto dup_alloc;
- UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
- vput(*vpp);
- }
return (error);
}
+ /*
+ * We got an inode, so check mode and panic if it is already allocated.
+ */
ip = VTOI(*vpp);
if (ip->i_mode) {
-dup_alloc:
printf("mode = 0%o, inum = %ju, fs = %s\n",
ip->i_mode, (uintmax_t)ip->i_number, fs->fs_fsmnt);
panic("ffs_valloc: dup alloc");
}
if (DIP(ip, i_blocks) && (fs->fs_flags & FS_UNCLEAN) == 0) { /* XXX */
printf("free inode %s/%lu had %ld blocks\n",
fs->fs_fsmnt, (u_long)ino, (long)DIP(ip, i_blocks));
DIP_SET(ip, i_blocks, 0);
}
ip->i_flags = 0;
DIP_SET(ip, i_flags, 0);
/*
* Set up a new generation number for this inode.
*/
while (ip->i_gen == 0 || ++ip->i_gen == 0)
ip->i_gen = arc4random();
DIP_SET(ip, i_gen, ip->i_gen);
if (fs->fs_magic == FS_UFS2_MAGIC) {
vfs_timestamp(&ts);
ip->i_din2->di_birthtime = ts.tv_sec;
ip->i_din2->di_birthnsec = ts.tv_nsec;
}
ip->i_flag = 0;
(*vpp)->v_vflag = 0;
(*vpp)->v_type = VNON;
if (fs->fs_magic == FS_UFS2_MAGIC) {
(*vpp)->v_op = &ffs_vnodeops2;
UFS_INODE_SET_FLAG(ip, IN_UFS2);
} else {
(*vpp)->v_op = &ffs_vnodeops1;
}
return (0);
noinodes:
if (reclaimed == 0) {
reclaimed = 1;
softdep_request_cleanup(fs, pvp, cred, FLUSH_INODES_WAIT);
goto retry;
}
+ if (ffs_fsfail_cleanup_locked(ump, 0)) {
+ UFS_UNLOCK(ump);
+ return (ENXIO);
+ }
if (ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, pip->i_number, "out of inodes");
uprintf("\n%s: create/symlink failed, no inodes free\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
return (ENOSPC);
}
/*
* Find a cylinder group to place a directory.
*
* The policy implemented by this algorithm is to allocate a
* directory inode in the same cylinder group as its parent
* directory, but also to reserve space for its files inodes
* and data. Restrict the number of directories which may be
* allocated one after another in the same cylinder group
* without intervening allocation of files.
*
* If we allocate a first level directory then force allocation
* in another cylinder group.
*/
static ino_t
ffs_dirpref(pip)
struct inode *pip;
{
struct fs *fs;
int cg, prefcg, dirsize, cgsize;
u_int avgifree, avgbfree, avgndir, curdirsize;
u_int minifree, minbfree, maxndir;
u_int mincg, minndir;
u_int maxcontigdirs;
mtx_assert(UFS_MTX(ITOUMP(pip)), MA_OWNED);
fs = ITOFS(pip);
avgifree = fs->fs_cstotal.cs_nifree / fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
avgndir = fs->fs_cstotal.cs_ndir / fs->fs_ncg;
/*
* Force allocation in another cg if creating a first level dir.
*/
ASSERT_VOP_LOCKED(ITOV(pip), "ffs_dirpref");
if (ITOV(pip)->v_vflag & VV_ROOT) {
prefcg = arc4random() % fs->fs_ncg;
mincg = prefcg;
minndir = fs->fs_ipg;
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
mincg = cg;
minndir = fs->fs_cs(fs, cg).cs_ndir;
}
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
mincg = cg;
minndir = fs->fs_cs(fs, cg).cs_ndir;
}
return ((ino_t)(fs->fs_ipg * mincg));
}
/*
* Count various limits which used for
* optimal allocation of a directory inode.
*/
maxndir = min(avgndir + fs->fs_ipg / 16, fs->fs_ipg);
minifree = avgifree - avgifree / 4;
if (minifree < 1)
minifree = 1;
minbfree = avgbfree - avgbfree / 4;
if (minbfree < 1)
minbfree = 1;
cgsize = fs->fs_fsize * fs->fs_fpg;
dirsize = fs->fs_avgfilesize * fs->fs_avgfpdir;
curdirsize = avgndir ? (cgsize - avgbfree * fs->fs_bsize) / avgndir : 0;
if (dirsize < curdirsize)
dirsize = curdirsize;
if (dirsize <= 0)
maxcontigdirs = 0; /* dirsize overflowed */
else
maxcontigdirs = min((avgbfree * fs->fs_bsize) / dirsize, 255);
if (fs->fs_avgfpdir > 0)
maxcontigdirs = min(maxcontigdirs,
fs->fs_ipg / fs->fs_avgfpdir);
if (maxcontigdirs == 0)
maxcontigdirs = 1;
/*
* Limit number of dirs in one cg and reserve space for
* regular files, but only if we have no deficit in
* inodes or space.
*
* We are trying to find a suitable cylinder group nearby
* our preferred cylinder group to place a new directory.
* We scan from our preferred cylinder group forward looking
* for a cylinder group that meets our criterion. If we get
* to the final cylinder group and do not find anything,
* we start scanning forwards from the beginning of the
* filesystem. While it might seem sensible to start scanning
* backwards or even to alternate looking forward and backward,
* this approach fails badly when the filesystem is nearly full.
* Specifically, we first search all the areas that have no space
* and finally try the one preceding that. We repeat this on
* every request and in the case of the final block end up
* searching the entire filesystem. By jumping to the front
* of the filesystem, our future forward searches always look
* in new cylinder groups so finds every possible block after
* one pass over the filesystem.
*/
prefcg = ino_to_cg(fs, pip->i_number);
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
fs->fs_cs(fs, cg).cs_nifree >= minifree &&
fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
if (fs->fs_contigdirs[cg] < maxcontigdirs)
return ((ino_t)(fs->fs_ipg * cg));
}
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
fs->fs_cs(fs, cg).cs_nifree >= minifree &&
fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
if (fs->fs_contigdirs[cg] < maxcontigdirs)
return ((ino_t)(fs->fs_ipg * cg));
}
/*
* This is a backstop when we have deficit in space.
*/
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
return ((ino_t)(fs->fs_ipg * cg));
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
break;
return ((ino_t)(fs->fs_ipg * cg));
}
/*
* Select the desired position for the next block in a file. The file is
* logically divided into sections. The first section is composed of the
* direct blocks and the next fs_maxbpg blocks. Each additional section
* contains fs_maxbpg blocks.
*
* If no blocks have been allocated in the first section, the policy is to
* request a block in the same cylinder group as the inode that describes
* the file. The first indirect is allocated immediately following the last
* direct block and the data blocks for the first indirect immediately
* follow it.
*
* If no blocks have been allocated in any other section, the indirect
* block(s) are allocated in the same cylinder group as its inode in an
* area reserved immediately following the inode blocks. The policy for
* the data blocks is to place them in a cylinder group with a greater than
* average number of free blocks. An appropriate cylinder group is found
* by using a rotor that sweeps the cylinder groups. When a new group of
* blocks is needed, the sweep begins in the cylinder group following the
* cylinder group from which the previous allocation was made. The sweep
* continues until a cylinder group with greater than the average number
* of free blocks is found. If the allocation is for the first block in an
* indirect block or the previous block is a hole, then the information on
* the previous allocation is unavailable; here a best guess is made based
* on the logical block number being allocated.
*
* If a section is already partially allocated, the policy is to
* allocate blocks contiguously within the section if possible.
*/
ufs2_daddr_t
ffs_blkpref_ufs1(ip, lbn, indx, bap)
struct inode *ip;
ufs_lbn_t lbn;
int indx;
ufs1_daddr_t *bap;
{
struct fs *fs;
u_int cg, inocg;
u_int avgbfree, startcg;
ufs2_daddr_t pref, prevbn;
KASSERT(indx <= 0 || bap != NULL, ("need non-NULL bap"));
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
fs = ITOFS(ip);
/*
* Allocation of indirect blocks is indicated by passing negative
* values in indx: -1 for single indirect, -2 for double indirect,
* -3 for triple indirect. As noted below, we attempt to allocate
* the first indirect inline with the file data. For all later
* indirect blocks, the data is often allocated in other cylinder
* groups. However to speed random file access and to speed up
* fsck, the filesystem reserves the first fs_metaspace blocks
* (typically half of fs_minfree) of the data area of each cylinder
* group to hold these later indirect blocks.
*/
inocg = ino_to_cg(fs, ip->i_number);
if (indx < 0) {
/*
* Our preference for indirect blocks is the zone at the
* beginning of the inode's cylinder group data area that
* we try to reserve for indirect blocks.
*/
pref = cgmeta(fs, inocg);
/*
* If we are allocating the first indirect block, try to
* place it immediately following the last direct block.
*/
if (indx == -1 && lbn < UFS_NDADDR + NINDIR(fs) &&
ip->i_din1->di_db[UFS_NDADDR - 1] != 0)
pref = ip->i_din1->di_db[UFS_NDADDR - 1] + fs->fs_frag;
return (pref);
}
/*
* If we are allocating the first data block in the first indirect
* block and the indirect has been allocated in the data block area,
* try to place it immediately following the indirect block.
*/
if (lbn == UFS_NDADDR) {
pref = ip->i_din1->di_ib[0];
if (pref != 0 && pref >= cgdata(fs, inocg) &&
pref < cgbase(fs, inocg + 1))
return (pref + fs->fs_frag);
}
/*
* If we are at the beginning of a file, or we have already allocated
* the maximum number of blocks per cylinder group, or we do not
* have a block allocated immediately preceding us, then we need
* to decide where to start allocating new blocks.
*/
if (indx == 0) {
prevbn = 0;
} else {
prevbn = bap[indx - 1];
if (UFS_CHECK_BLKNO(ITOVFS(ip), ip->i_number, prevbn,
fs->fs_bsize) != 0)
prevbn = 0;
}
if (indx % fs->fs_maxbpg == 0 || prevbn == 0) {
/*
* If we are allocating a directory data block, we want
* to place it in the metadata area.
*/
if ((ip->i_mode & IFMT) == IFDIR)
return (cgmeta(fs, inocg));
/*
* Until we fill all the direct and all the first indirect's
* blocks, we try to allocate in the data area of the inode's
* cylinder group.
*/
if (lbn < UFS_NDADDR + NINDIR(fs))
return (cgdata(fs, inocg));
/*
* Find a cylinder with greater than average number of
* unused data blocks.
*/
if (indx == 0 || prevbn == 0)
startcg = inocg + lbn / fs->fs_maxbpg;
else
startcg = dtog(fs, prevbn) + 1;
startcg %= fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
for (cg = startcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
for (cg = 0; cg <= startcg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
return (0);
}
/*
* Otherwise, we just always try to lay things out contiguously.
*/
return (prevbn + fs->fs_frag);
}
/*
* Same as above, but for UFS2
*/
ufs2_daddr_t
ffs_blkpref_ufs2(ip, lbn, indx, bap)
struct inode *ip;
ufs_lbn_t lbn;
int indx;
ufs2_daddr_t *bap;
{
struct fs *fs;
u_int cg, inocg;
u_int avgbfree, startcg;
ufs2_daddr_t pref, prevbn;
KASSERT(indx <= 0 || bap != NULL, ("need non-NULL bap"));
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
fs = ITOFS(ip);
/*
* Allocation of indirect blocks is indicated by passing negative
* values in indx: -1 for single indirect, -2 for double indirect,
* -3 for triple indirect. As noted below, we attempt to allocate
* the first indirect inline with the file data. For all later
* indirect blocks, the data is often allocated in other cylinder
* groups. However to speed random file access and to speed up
* fsck, the filesystem reserves the first fs_metaspace blocks
* (typically half of fs_minfree) of the data area of each cylinder
* group to hold these later indirect blocks.
*/
inocg = ino_to_cg(fs, ip->i_number);
if (indx < 0) {
/*
* Our preference for indirect blocks is the zone at the
* beginning of the inode's cylinder group data area that
* we try to reserve for indirect blocks.
*/
pref = cgmeta(fs, inocg);
/*
* If we are allocating the first indirect block, try to
* place it immediately following the last direct block.
*/
if (indx == -1 && lbn < UFS_NDADDR + NINDIR(fs) &&
ip->i_din2->di_db[UFS_NDADDR - 1] != 0)
pref = ip->i_din2->di_db[UFS_NDADDR - 1] + fs->fs_frag;
return (pref);
}
/*
* If we are allocating the first data block in the first indirect
* block and the indirect has been allocated in the data block area,
* try to place it immediately following the indirect block.
*/
if (lbn == UFS_NDADDR) {
pref = ip->i_din2->di_ib[0];
if (pref != 0 && pref >= cgdata(fs, inocg) &&
pref < cgbase(fs, inocg + 1))
return (pref + fs->fs_frag);
}
/*
* If we are at the beginning of a file, or we have already allocated
* the maximum number of blocks per cylinder group, or we do not
* have a block allocated immediately preceding us, then we need
* to decide where to start allocating new blocks.
*/
if (indx == 0) {
prevbn = 0;
} else {
prevbn = bap[indx - 1];
if (UFS_CHECK_BLKNO(ITOVFS(ip), ip->i_number, prevbn,
fs->fs_bsize) != 0)
prevbn = 0;
}
if (indx % fs->fs_maxbpg == 0 || prevbn == 0) {
/*
* If we are allocating a directory data block, we want
* to place it in the metadata area.
*/
if ((ip->i_mode & IFMT) == IFDIR)
return (cgmeta(fs, inocg));
/*
* Until we fill all the direct and all the first indirect's
* blocks, we try to allocate in the data area of the inode's
* cylinder group.
*/
if (lbn < UFS_NDADDR + NINDIR(fs))
return (cgdata(fs, inocg));
/*
* Find a cylinder with greater than average number of
* unused data blocks.
*/
if (indx == 0 || prevbn == 0)
startcg = inocg + lbn / fs->fs_maxbpg;
else
startcg = dtog(fs, prevbn) + 1;
startcg %= fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
for (cg = startcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
for (cg = 0; cg <= startcg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
return (0);
}
/*
* Otherwise, we just always try to lay things out contiguously.
*/
return (prevbn + fs->fs_frag);
}
/*
* Implement the cylinder overflow algorithm.
*
* The policy implemented by this algorithm is:
* 1) allocate the block in its requested cylinder group.
* 2) quadradically rehash on the cylinder group number.
* 3) brute force search for a free block.
*
* Must be called with the UFS lock held. Will release the lock on success
* and return with it held on failure.
*/
/*VARARGS5*/
static ufs2_daddr_t
ffs_hashalloc(ip, cg, pref, size, rsize, allocator)
struct inode *ip;
u_int cg;
ufs2_daddr_t pref;
int size; /* Search size for data blocks, mode for inodes */
int rsize; /* Real allocated size. */
allocfcn_t *allocator;
{
struct fs *fs;
ufs2_daddr_t result;
u_int i, icg = cg;
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
#ifdef INVARIANTS
if (ITOV(ip)->v_mount->mnt_kern_flag & MNTK_SUSPENDED)
panic("ffs_hashalloc: allocation on suspended filesystem");
#endif
fs = ITOFS(ip);
/*
* 1: preferred cylinder group
*/
result = (*allocator)(ip, cg, pref, size, rsize);
if (result)
return (result);
/*
* 2: quadratic rehash
*/
for (i = 1; i < fs->fs_ncg; i *= 2) {
cg += i;
if (cg >= fs->fs_ncg)
cg -= fs->fs_ncg;
result = (*allocator)(ip, cg, 0, size, rsize);
if (result)
return (result);
}
/*
* 3: brute force search
* Note that we start at i == 2, since 0 was checked initially,
* and 1 is always checked in the quadratic rehash.
*/
cg = (icg + 2) % fs->fs_ncg;
for (i = 2; i < fs->fs_ncg; i++) {
result = (*allocator)(ip, cg, 0, size, rsize);
if (result)
return (result);
cg++;
if (cg == fs->fs_ncg)
cg = 0;
}
return (0);
}
/*
* Determine whether a fragment can be extended.
*
* Check to see if the necessary fragments are available, and
* if they are, allocate them.
*/
static ufs2_daddr_t
ffs_fragextend(ip, cg, bprev, osize, nsize)
struct inode *ip;
u_int cg;
ufs2_daddr_t bprev;
int osize, nsize;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
int nffree;
long bno;
int frags, bbase;
int i, error;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_cs(fs, cg).cs_nffree < numfrags(fs, nsize - osize))
return (0);
frags = numfrags(fs, nsize);
bbase = fragnum(fs, bprev);
if (bbase > fragnum(fs, (bprev + frags - 1))) {
/* cannot extend across a block boundary */
return (0);
}
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0)
goto fail;
bno = dtogd(fs, bprev);
blksfree = cg_blksfree(cgp);
for (i = numfrags(fs, osize); i < frags; i++)
if (isclr(blksfree, bno + i))
goto fail;
/*
* the current fragment can be extended
* deduct the count on fragment being extended into
* increase the count on the remaining fragment (if any)
* allocate the extended piece
*/
for (i = frags; i < fs->fs_frag - bbase; i++)
if (isclr(blksfree, bno + i))
break;
cgp->cg_frsum[i - numfrags(fs, osize)]--;
if (i != frags)
cgp->cg_frsum[i - frags]++;
for (i = numfrags(fs, osize), nffree = 0; i < frags; i++) {
clrbit(blksfree, bno + i);
cgp->cg_cs.cs_nffree--;
nffree++;
}
UFS_LOCK(ump);
fs->fs_cstotal.cs_nffree -= nffree;
fs->fs_cs(fs, cg).cs_nffree -= nffree;
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), bprev,
frags, numfrags(fs, osize));
bdwrite(bp);
return (bprev);
fail:
brelse(bp);
UFS_LOCK(ump);
return (0);
}
/*
* Determine whether a block can be allocated.
*
* Check to see if a block of the appropriate size is available,
* and if it is, allocate it.
*/
static ufs2_daddr_t
ffs_alloccg(ip, cg, bpref, size, rsize)
struct inode *ip;
u_int cg;
ufs2_daddr_t bpref;
int size;
int rsize;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
ufs1_daddr_t bno;
ufs2_daddr_t blkno;
int i, allocsiz, error, frags;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_cs(fs, cg).cs_nbfree == 0 && size == fs->fs_bsize)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0 ||
(cgp->cg_cs.cs_nbfree == 0 && size == fs->fs_bsize))
goto fail;
if (size == fs->fs_bsize) {
UFS_LOCK(ump);
blkno = ffs_alloccgblk(ip, bp, bpref, rsize);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (blkno);
}
/*
* check to see if any fragments are already available
* allocsiz is the size which will be allocated, hacking
* it down to a smaller size if necessary
*/
blksfree = cg_blksfree(cgp);
frags = numfrags(fs, size);
for (allocsiz = frags; allocsiz < fs->fs_frag; allocsiz++)
if (cgp->cg_frsum[allocsiz] != 0)
break;
if (allocsiz == fs->fs_frag) {
/*
* no fragments were available, so a block will be
* allocated, and hacked up
*/
if (cgp->cg_cs.cs_nbfree == 0)
goto fail;
UFS_LOCK(ump);
blkno = ffs_alloccgblk(ip, bp, bpref, rsize);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (blkno);
}
KASSERT(size == rsize,
("ffs_alloccg: size(%d) != rsize(%d)", size, rsize));
bno = ffs_mapsearch(fs, cgp, bpref, allocsiz);
if (bno < 0)
goto fail;
for (i = 0; i < frags; i++)
clrbit(blksfree, bno + i);
cgp->cg_cs.cs_nffree -= frags;
cgp->cg_frsum[allocsiz]--;
if (frags != allocsiz)
cgp->cg_frsum[allocsiz - frags]++;
UFS_LOCK(ump);
fs->fs_cstotal.cs_nffree -= frags;
fs->fs_cs(fs, cg).cs_nffree -= frags;
fs->fs_fmod = 1;
blkno = cgbase(fs, cg) + bno;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno, frags, 0);
bdwrite(bp);
return (blkno);
fail:
brelse(bp);
UFS_LOCK(ump);
return (0);
}
/*
* Allocate a block in a cylinder group.
*
* This algorithm implements the following policy:
* 1) allocate the requested block.
* 2) allocate a rotationally optimal block in the same cylinder.
* 3) allocate the next available block on the block rotor for the
* specified cylinder group.
* Note that this routine only allocates fs_bsize blocks; these
* blocks may be fragmented by the routine that allocates them.
*/
static ufs2_daddr_t
ffs_alloccgblk(ip, bp, bpref, size)
struct inode *ip;
struct buf *bp;
ufs2_daddr_t bpref;
int size;
{
struct fs *fs;
struct cg *cgp;
struct ufsmount *ump;
ufs1_daddr_t bno;
ufs2_daddr_t blkno;
u_int8_t *blksfree;
int i, cgbpref;
ump = ITOUMP(ip);
fs = ump->um_fs;
mtx_assert(UFS_MTX(ump), MA_OWNED);
cgp = (struct cg *)bp->b_data;
blksfree = cg_blksfree(cgp);
if (bpref == 0) {
bpref = cgbase(fs, cgp->cg_cgx) + cgp->cg_rotor + fs->fs_frag;
} else if ((cgbpref = dtog(fs, bpref)) != cgp->cg_cgx) {
/* map bpref to correct zone in this cg */
if (bpref < cgdata(fs, cgbpref))
bpref = cgmeta(fs, cgp->cg_cgx);
else
bpref = cgdata(fs, cgp->cg_cgx);
}
/*
* if the requested block is available, use it
*/
bno = dtogd(fs, blknum(fs, bpref));
if (ffs_isblock(fs, blksfree, fragstoblks(fs, bno)))
goto gotit;
/*
* Take the next available block in this cylinder group.
*/
bno = ffs_mapsearch(fs, cgp, bpref, (int)fs->fs_frag);
if (bno < 0)
return (0);
/* Update cg_rotor only if allocated from the data zone */
if (bno >= dtogd(fs, cgdata(fs, cgp->cg_cgx)))
cgp->cg_rotor = bno;
gotit:
blkno = fragstoblks(fs, bno);
ffs_clrblock(fs, blksfree, (long)blkno);
ffs_clusteracct(fs, cgp, blkno, -1);
cgp->cg_cs.cs_nbfree--;
fs->fs_cstotal.cs_nbfree--;
fs->fs_cs(fs, cgp->cg_cgx).cs_nbfree--;
fs->fs_fmod = 1;
blkno = cgbase(fs, cgp->cg_cgx) + bno;
/*
* If the caller didn't want the whole block free the frags here.
*/
size = numfrags(fs, size);
if (size != fs->fs_frag) {
bno = dtogd(fs, blkno);
for (i = size; i < fs->fs_frag; i++)
setbit(blksfree, bno + i);
i = fs->fs_frag - size;
cgp->cg_cs.cs_nffree += i;
fs->fs_cstotal.cs_nffree += i;
fs->fs_cs(fs, cgp->cg_cgx).cs_nffree += i;
fs->fs_fmod = 1;
cgp->cg_frsum[i]++;
}
/* XXX Fixme. */
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno, size, 0);
UFS_LOCK(ump);
return (blkno);
}
/*
* Determine whether a cluster can be allocated.
*
* We do not currently check for optimal rotational layout if there
* are multiple choices in the same cylinder group. Instead we just
* take the first one that we find following bpref.
*/
static ufs2_daddr_t
ffs_clusteralloc(ip, cg, bpref, len)
struct inode *ip;
u_int cg;
ufs2_daddr_t bpref;
int len;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
int i, run, bit, map, got, error;
ufs2_daddr_t bno;
u_char *mapp;
int32_t *lp;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_maxcluster[cg] < len)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0) {
UFS_LOCK(ump);
return (0);
}
/*
* Check to see if a cluster of the needed size (or bigger) is
* available in this cylinder group.
*/
lp = &cg_clustersum(cgp)[len];
for (i = len; i <= fs->fs_contigsumsize; i++)
if (*lp++ > 0)
break;
if (i > fs->fs_contigsumsize) {
/*
* This is the first time looking for a cluster in this
* cylinder group. Update the cluster summary information
* to reflect the true maximum sized cluster so that
* future cluster allocation requests can avoid reading
* the cylinder group map only to find no clusters.
*/
lp = &cg_clustersum(cgp)[len - 1];
for (i = len - 1; i > 0; i--)
if (*lp-- > 0)
break;
UFS_LOCK(ump);
fs->fs_maxcluster[cg] = i;
brelse(bp);
return (0);
}
/*
* Search the cluster map to find a big enough cluster.
* We take the first one that we find, even if it is larger
* than we need as we prefer to get one close to the previous
* block allocation. We do not search before the current
* preference point as we do not want to allocate a block
* that is allocated before the previous one (as we will
* then have to wait for another pass of the elevator
* algorithm before it will be read). We prefer to fail and
* be recalled to try an allocation in the next cylinder group.
*/
if (dtog(fs, bpref) != cg)
bpref = cgdata(fs, cg);
else
bpref = blknum(fs, bpref);
bpref = fragstoblks(fs, dtogd(fs, bpref));
mapp = &cg_clustersfree(cgp)[bpref / NBBY];
map = *mapp++;
bit = 1 << (bpref % NBBY);
for (run = 0, got = bpref; got < cgp->cg_nclusterblks; got++) {
if ((map & bit) == 0) {
run = 0;
} else {
run++;
if (run == len)
break;
}
if ((got & (NBBY - 1)) != (NBBY - 1)) {
bit <<= 1;
} else {
map = *mapp++;
bit = 1;
}
}
if (got >= cgp->cg_nclusterblks) {
UFS_LOCK(ump);
brelse(bp);
return (0);
}
/*
* Allocate the cluster that we have found.
*/
blksfree = cg_blksfree(cgp);
for (i = 1; i <= len; i++)
if (!ffs_isblock(fs, blksfree, got - run + i))
panic("ffs_clusteralloc: map mismatch");
bno = cgbase(fs, cg) + blkstofrags(fs, got - run + 1);
if (dtog(fs, bno) != cg)
panic("ffs_clusteralloc: allocated out of group");
len = blkstofrags(fs, len);
UFS_LOCK(ump);
for (i = 0; i < len; i += fs->fs_frag)
if (ffs_alloccgblk(ip, bp, bno + i, fs->fs_bsize) != bno + i)
panic("ffs_clusteralloc: lost block");
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (bno);
}
static inline struct buf *
getinobuf(struct inode *ip, u_int cg, u_int32_t cginoblk, int gbflags)
{
struct fs *fs;
fs = ITOFS(ip);
return (getblk(ITODEVVP(ip), fsbtodb(fs, ino_to_fsba(fs,
cg * fs->fs_ipg + cginoblk)), (int)fs->fs_bsize, 0, 0,
gbflags));
}
/*
* Synchronous inode initialization is needed only when barrier writes do not
* work as advertised, and will impose a heavy cost on file creation in a newly
* created filesystem.
*/
static int doasyncinodeinit = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doasyncinodeinit, CTLFLAG_RWTUN,
&doasyncinodeinit, 0,
"Perform inode block initialization using asynchronous writes");
/*
* Determine whether an inode can be allocated.
*
* Check to see if an inode is available, and if it is,
* allocate it using the following policy:
* 1) allocate the requested inode.
* 2) allocate the next available inode after the requested
* inode in the specified cylinder group.
*/
static ufs2_daddr_t
ffs_nodealloccg(ip, cg, ipref, mode, unused)
struct inode *ip;
u_int cg;
ufs2_daddr_t ipref;
int mode;
int unused;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp, *ibp;
struct ufsmount *ump;
u_int8_t *inosused, *loc;
struct ufs2_dinode *dp2;
int error, start, len, i;
u_int32_t old_initediblk;
ump = ITOUMP(ip);
fs = ump->um_fs;
check_nifree:
if (fs->fs_cs(fs, cg).cs_nifree == 0)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0) {
UFS_LOCK(ump);
return (0);
}
restart:
if (cgp->cg_cs.cs_nifree == 0) {
brelse(bp);
UFS_LOCK(ump);
return (0);
}
inosused = cg_inosused(cgp);
if (ipref) {
ipref %= fs->fs_ipg;
if (isclr(inosused, ipref))
goto gotit;
}
start = cgp->cg_irotor / NBBY;
len = howmany(fs->fs_ipg - cgp->cg_irotor, NBBY);
loc = memcchr(&inosused[start], 0xff, len);
if (loc == NULL) {
len = start + 1;
start = 0;
loc = memcchr(&inosused[start], 0xff, len);
if (loc == NULL) {
printf("cg = %d, irotor = %ld, fs = %s\n",
cg, (long)cgp->cg_irotor, fs->fs_fsmnt);
panic("ffs_nodealloccg: map corrupted");
/* NOTREACHED */
}
}
ipref = (loc - inosused) * NBBY + ffs(~*loc) - 1;
gotit:
/*
* Check to see if we need to initialize more inodes.
*/
if (fs->fs_magic == FS_UFS2_MAGIC &&
ipref + INOPB(fs) > cgp->cg_initediblk &&
cgp->cg_initediblk < cgp->cg_niblk) {
old_initediblk = cgp->cg_initediblk;
/*
* Free the cylinder group lock before writing the
* initialized inode block. Entering the
* babarrierwrite() with the cylinder group lock
* causes lock order violation between the lock and
* snaplk.
*
* Another thread can decide to initialize the same
* inode block, but whichever thread first gets the
* cylinder group lock after writing the newly
* allocated inode block will update it and the other
* will realize that it has lost and leave the
* cylinder group unchanged.
*/
ibp = getinobuf(ip, cg, old_initediblk, GB_LOCK_NOWAIT);
brelse(bp);
if (ibp == NULL) {
/*
* The inode block buffer is already owned by
* another thread, which must initialize it.
* Wait on the buffer to allow another thread
* to finish the updates, with dropped cg
* buffer lock, then retry.
*/
ibp = getinobuf(ip, cg, old_initediblk, 0);
brelse(ibp);
UFS_LOCK(ump);
goto check_nifree;
}
bzero(ibp->b_data, (int)fs->fs_bsize);
dp2 = (struct ufs2_dinode *)(ibp->b_data);
for (i = 0; i < INOPB(fs); i++) {
while (dp2->di_gen == 0)
dp2->di_gen = arc4random();
dp2++;
}
/*
* Rather than adding a soft updates dependency to ensure
* that the new inode block is written before it is claimed
* by the cylinder group map, we just do a barrier write
* here. The barrier write will ensure that the inode block
* gets written before the updated cylinder group map can be
* written. The barrier write should only slow down bulk
* loading of newly created filesystems.
*/
if (doasyncinodeinit)
babarrierwrite(ibp);
else
bwrite(ibp);
/*
* After the inode block is written, try to update the
* cg initediblk pointer. If another thread beat us
* to it, then leave it unchanged as the other thread
* has already set it correctly.
*/
error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp);
UFS_LOCK(ump);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (error != 0)
return (error);
if (cgp->cg_initediblk == old_initediblk)
cgp->cg_initediblk += INOPB(fs);
goto restart;
}
cgp->cg_irotor = ipref;
UFS_LOCK(ump);
ACTIVECLEAR(fs, cg);
setbit(inosused, ipref);
cgp->cg_cs.cs_nifree--;
fs->fs_cstotal.cs_nifree--;
fs->fs_cs(fs, cg).cs_nifree--;
fs->fs_fmod = 1;
if ((mode & IFMT) == IFDIR) {
cgp->cg_cs.cs_ndir++;
fs->fs_cstotal.cs_ndir++;
fs->fs_cs(fs, cg).cs_ndir++;
}
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_inomapdep(bp, ip, cg * fs->fs_ipg + ipref, mode);
bdwrite(bp);
return ((ino_t)(cg * fs->fs_ipg + ipref));
}
/*
* Free a block or fragment.
*
* The specified block or fragment is placed back in the
* free map. If a fragment is deallocated, a possible
* block reassembly is checked.
*/
static void
ffs_blkfree_cg(ump, fs, devvp, bno, size, inum, dephd)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
struct workhead *dephd;
{
struct mount *mp;
struct cg *cgp;
struct buf *bp;
+ daddr_t dbn;
ufs1_daddr_t fragno, cgbno;
int i, blk, frags, bbase, error;
u_int cg;
u_int8_t *blksfree;
struct cdev *dev;
cg = dtog(fs, bno);
if (devvp->v_type == VREG) {
/* devvp is a snapshot */
MPASS(devvp->v_mount->mnt_data == ump);
dev = ump->um_devvp->v_rdev;
} else if (devvp->v_type == VCHR) {
/* devvp is a normal disk device */
dev = devvp->v_rdev;
ASSERT_VOP_LOCKED(devvp, "ffs_blkfree_cg");
} else
return;
#ifdef INVARIANTS
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0 ||
fragnum(fs, bno) + numfrags(fs, size) > fs->fs_frag) {
printf("dev=%s, bno = %jd, bsize = %ld, size = %ld, fs = %s\n",
devtoname(dev), (intmax_t)bno, (long)fs->fs_bsize,
size, fs->fs_fsmnt);
panic("ffs_blkfree_cg: bad size");
}
#endif
if ((u_int)bno >= fs->fs_size) {
printf("bad block %jd, ino %lu\n", (intmax_t)bno,
(u_long)inum);
ffs_fserr(fs, inum, "bad block");
return;
}
- if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
+ if ((error = ffs_getcg(fs, devvp, cg, GB_CVTENXIO, &bp, &cgp)) != 0) {
+ if (!ffs_fsfail_cleanup(ump, error) ||
+ !MOUNTEDSOFTDEP(UFSTOVFS(ump)) || devvp->v_type != VCHR)
+ return;
+ if (devvp->v_type == VREG)
+ dbn = fragstoblks(fs, cgtod(fs, cg));
+ else
+ dbn = fsbtodb(fs, cgtod(fs, cg));
+ error = getblkx(devvp, dbn, dbn, fs->fs_cgsize, 0, 0, 0, &bp);
+ KASSERT(error == 0, ("getblkx failed"));
+ softdep_setup_blkfree(UFSTOVFS(ump), bp, bno,
+ numfrags(fs, size), dephd);
+ bp->b_flags |= B_RELBUF | B_NOCACHE;
+ bp->b_flags &= ~B_CACHE;
+ bawrite(bp);
return;
+ }
cgbno = dtogd(fs, bno);
blksfree = cg_blksfree(cgp);
UFS_LOCK(ump);
if (size == fs->fs_bsize) {
fragno = fragstoblks(fs, cgbno);
if (!ffs_isfreeblock(fs, blksfree, fragno)) {
if (devvp->v_type == VREG) {
UFS_UNLOCK(ump);
/* devvp is a snapshot */
brelse(bp);
return;
}
printf("dev = %s, block = %jd, fs = %s\n",
devtoname(dev), (intmax_t)bno, fs->fs_fsmnt);
panic("ffs_blkfree_cg: freeing free block");
}
ffs_setblock(fs, blksfree, fragno);
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
fs->fs_cstotal.cs_nbfree++;
fs->fs_cs(fs, cg).cs_nbfree++;
} else {
bbase = cgbno - fragnum(fs, cgbno);
/*
* decrement the counts associated with the old frags
*/
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
/*
* deallocate the fragment
*/
frags = numfrags(fs, size);
for (i = 0; i < frags; i++) {
if (isset(blksfree, cgbno + i)) {
printf("dev = %s, block = %jd, fs = %s\n",
devtoname(dev), (intmax_t)(bno + i),
fs->fs_fsmnt);
panic("ffs_blkfree_cg: freeing free frag");
}
setbit(blksfree, cgbno + i);
}
cgp->cg_cs.cs_nffree += i;
fs->fs_cstotal.cs_nffree += i;
fs->fs_cs(fs, cg).cs_nffree += i;
/*
* add back in counts associated with the new frags
*/
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
/*
* if a complete block has been reassembled, account for it
*/
fragno = fragstoblks(fs, bbase);
if (ffs_isblock(fs, blksfree, fragno)) {
cgp->cg_cs.cs_nffree -= fs->fs_frag;
fs->fs_cstotal.cs_nffree -= fs->fs_frag;
fs->fs_cs(fs, cg).cs_nffree -= fs->fs_frag;
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
fs->fs_cstotal.cs_nbfree++;
fs->fs_cs(fs, cg).cs_nbfree++;
}
}
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
mp = UFSTOVFS(ump);
if (MOUNTEDSOFTDEP(mp) && devvp->v_type == VCHR)
softdep_setup_blkfree(UFSTOVFS(ump), bp, bno,
numfrags(fs, size), dephd);
bdwrite(bp);
}
/*
* Structures and routines associated with trim management.
*
* The following requests are passed to trim_lookup to indicate
* the actions that should be taken.
*/
#define NEW 1 /* if found, error else allocate and hash it */
#define OLD 2 /* if not found, error, else return it */
#define REPLACE 3 /* if not found, error else unhash and reallocate it */
#define DONE 4 /* if not found, error else unhash and return it */
#define SINGLE 5 /* don't look up, just allocate it and don't hash it */
MALLOC_DEFINE(M_TRIM, "ufs_trim", "UFS trim structures");
#define TRIMLIST_HASH(ump, key) \
(&(ump)->um_trimhash[(key) & (ump)->um_trimlisthashsize])
/*
* These structures describe each of the block free requests aggregated
* together to make up a trim request.
*/
struct trim_blkreq {
TAILQ_ENTRY(trim_blkreq) blkreqlist;
ufs2_daddr_t bno;
long size;
struct workhead *pdephd;
struct workhead dephd;
};
/*
* Description of a trim request.
*/
struct ffs_blkfree_trim_params {
TAILQ_HEAD(, trim_blkreq) blklist;
LIST_ENTRY(ffs_blkfree_trim_params) hashlist;
struct task task;
struct ufsmount *ump;
struct vnode *devvp;
ino_t inum;
ufs2_daddr_t bno;
long size;
long key;
};
static void ffs_blkfree_trim_completed(struct buf *);
static void ffs_blkfree_trim_task(void *ctx, int pending __unused);
static struct ffs_blkfree_trim_params *trim_lookup(struct ufsmount *,
struct vnode *, ufs2_daddr_t, long, ino_t, u_long, int);
static void ffs_blkfree_sendtrim(struct ffs_blkfree_trim_params *);
/*
* Called on trim completion to start a task to free the associated block(s).
*/
static void
ffs_blkfree_trim_completed(bp)
struct buf *bp;
{
struct ffs_blkfree_trim_params *tp;
tp = bp->b_fsprivate1;
free(bp, M_TRIM);
TASK_INIT(&tp->task, 0, ffs_blkfree_trim_task, tp);
taskqueue_enqueue(tp->ump->um_trim_tq, &tp->task);
}
/*
* Trim completion task that free associated block(s).
*/
static void
ffs_blkfree_trim_task(ctx, pending)
void *ctx;
int pending;
{
struct ffs_blkfree_trim_params *tp;
struct trim_blkreq *blkelm;
struct ufsmount *ump;
tp = ctx;
ump = tp->ump;
while ((blkelm = TAILQ_FIRST(&tp->blklist)) != NULL) {
ffs_blkfree_cg(ump, ump->um_fs, tp->devvp, blkelm->bno,
blkelm->size, tp->inum, blkelm->pdephd);
TAILQ_REMOVE(&tp->blklist, blkelm, blkreqlist);
free(blkelm, M_TRIM);
}
vn_finished_secondary_write(UFSTOVFS(ump));
UFS_LOCK(ump);
ump->um_trim_inflight -= 1;
ump->um_trim_inflight_blks -= numfrags(ump->um_fs, tp->size);
UFS_UNLOCK(ump);
free(tp, M_TRIM);
}
/*
* Lookup a trim request by inode number.
* Allocate if requested (NEW, REPLACE, SINGLE).
*/
static struct ffs_blkfree_trim_params *
trim_lookup(ump, devvp, bno, size, inum, key, alloctype)
struct ufsmount *ump;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
u_long key;
int alloctype;
{
struct trimlist_hashhead *tphashhead;
struct ffs_blkfree_trim_params *tp, *ntp;
ntp = malloc(sizeof(struct ffs_blkfree_trim_params), M_TRIM, M_WAITOK);
if (alloctype != SINGLE) {
KASSERT(key >= FIRST_VALID_KEY, ("trim_lookup: invalid key"));
UFS_LOCK(ump);
tphashhead = TRIMLIST_HASH(ump, key);
LIST_FOREACH(tp, tphashhead, hashlist)
if (key == tp->key)
break;
}
switch (alloctype) {
case NEW:
KASSERT(tp == NULL, ("trim_lookup: found trim"));
break;
case OLD:
KASSERT(tp != NULL,
("trim_lookup: missing call to ffs_blkrelease_start()"));
UFS_UNLOCK(ump);
free(ntp, M_TRIM);
return (tp);
case REPLACE:
KASSERT(tp != NULL, ("trim_lookup: missing REPLACE trim"));
LIST_REMOVE(tp, hashlist);
/* tp will be freed by caller */
break;
case DONE:
KASSERT(tp != NULL, ("trim_lookup: missing DONE trim"));
LIST_REMOVE(tp, hashlist);
UFS_UNLOCK(ump);
free(ntp, M_TRIM);
return (tp);
}
TAILQ_INIT(&ntp->blklist);
ntp->ump = ump;
ntp->devvp = devvp;
ntp->bno = bno;
ntp->size = size;
ntp->inum = inum;
ntp->key = key;
if (alloctype != SINGLE) {
LIST_INSERT_HEAD(tphashhead, ntp, hashlist);
UFS_UNLOCK(ump);
}
return (ntp);
}
/*
* Dispatch a trim request.
*/
static void
ffs_blkfree_sendtrim(tp)
struct ffs_blkfree_trim_params *tp;
{
struct ufsmount *ump;
struct mount *mp;
struct buf *bp;
/*
* Postpone the set of the free bit in the cg bitmap until the
* BIO_DELETE is completed. Otherwise, due to disk queue
* reordering, TRIM might be issued after we reuse the block
* and write some new data into it.
*/
ump = tp->ump;
bp = malloc(sizeof(*bp), M_TRIM, M_WAITOK | M_ZERO);
bp->b_iocmd = BIO_DELETE;
bp->b_iooffset = dbtob(fsbtodb(ump->um_fs, tp->bno));
bp->b_iodone = ffs_blkfree_trim_completed;
bp->b_bcount = tp->size;
bp->b_fsprivate1 = tp;
UFS_LOCK(ump);
ump->um_trim_total += 1;
ump->um_trim_inflight += 1;
ump->um_trim_inflight_blks += numfrags(ump->um_fs, tp->size);
ump->um_trim_total_blks += numfrags(ump->um_fs, tp->size);
UFS_UNLOCK(ump);
mp = UFSTOVFS(ump);
vn_start_secondary_write(NULL, &mp, 0);
g_vfs_strategy(ump->um_bo, bp);
}
/*
* Allocate a new key to use to identify a range of blocks.
*/
u_long
ffs_blkrelease_start(ump, devvp, inum)
struct ufsmount *ump;
struct vnode *devvp;
ino_t inum;
{
static u_long masterkey;
u_long key;
if (((ump->um_flags & UM_CANDELETE) == 0) || dotrimcons == 0)
return (SINGLETON_KEY);
do {
key = atomic_fetchadd_long(&masterkey, 1);
} while (key < FIRST_VALID_KEY);
(void) trim_lookup(ump, devvp, 0, 0, inum, key, NEW);
return (key);
}
/*
* Deallocate a key that has been used to identify a range of blocks.
*/
void
ffs_blkrelease_finish(ump, key)
struct ufsmount *ump;
u_long key;
{
struct ffs_blkfree_trim_params *tp;
if (((ump->um_flags & UM_CANDELETE) == 0) || dotrimcons == 0)
return;
/*
* If the vfs.ffs.dotrimcons sysctl option is enabled while
* a file deletion is active, specifically after a call
* to ffs_blkrelease_start() but before the call to
* ffs_blkrelease_finish(), ffs_blkrelease_start() will
* have handed out SINGLETON_KEY rather than starting a
* collection sequence. Thus if we get a SINGLETON_KEY
* passed to ffs_blkrelease_finish(), we just return rather
* than trying to finish the nonexistent sequence.
*/
if (key == SINGLETON_KEY) {
#ifdef INVARIANTS
printf("%s: vfs.ffs.dotrimcons enabled on active filesystem\n",
ump->um_mountp->mnt_stat.f_mntonname);
#endif
return;
}
/*
* We are done with sending blocks using this key. Look up the key
* using the DONE alloctype (in tp) to request that it be unhashed
* as we will not be adding to it. If the key has never been used,
* tp->size will be zero, so we can just free tp. Otherwise the call
* to ffs_blkfree_sendtrim(tp) causes the block range described by
* tp to be issued (and then tp to be freed).
*/
tp = trim_lookup(ump, NULL, 0, 0, 0, key, DONE);
if (tp->size == 0)
free(tp, M_TRIM);
else
ffs_blkfree_sendtrim(tp);
}
/*
* Setup to free a block or fragment.
*
* Check for snapshots that might want to claim the block.
* If trims are requested, prepare a trim request. Attempt to
* aggregate consecutive blocks into a single trim request.
*/
void
ffs_blkfree(ump, fs, devvp, bno, size, inum, vtype, dephd, key)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
enum vtype vtype;
struct workhead *dephd;
u_long key;
{
struct ffs_blkfree_trim_params *tp, *ntp;
struct trim_blkreq *blkelm;
/*
* Check to see if a snapshot wants to claim the block.
* Check that devvp is a normal disk device, not a snapshot,
* it has a snapshot(s) associated with it, and one of the
* snapshots wants to claim the block.
*/
if (devvp->v_type == VCHR &&
(devvp->v_vflag & VV_COPYONWRITE) &&
ffs_snapblkfree(fs, devvp, bno, size, inum, vtype, dephd)) {
return;
}
/*
* Nothing to delay if TRIM is not required for this block or TRIM
* is disabled or the operation is performed on a snapshot.
*/
if (key == NOTRIM_KEY || ((ump->um_flags & UM_CANDELETE) == 0) ||
devvp->v_type == VREG) {
ffs_blkfree_cg(ump, fs, devvp, bno, size, inum, dephd);
return;
}
blkelm = malloc(sizeof(struct trim_blkreq), M_TRIM, M_WAITOK);
blkelm->bno = bno;
blkelm->size = size;
if (dephd == NULL) {
blkelm->pdephd = NULL;
} else {
LIST_INIT(&blkelm->dephd);
LIST_SWAP(dephd, &blkelm->dephd, worklist, wk_list);
blkelm->pdephd = &blkelm->dephd;
}
if (key == SINGLETON_KEY) {
/*
* Just a single non-contiguous piece. Use the SINGLE
* alloctype to return a trim request that will not be
* hashed for future lookup.
*/
tp = trim_lookup(ump, devvp, bno, size, inum, key, SINGLE);
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
ffs_blkfree_sendtrim(tp);
return;
}
/*
* The callers of this function are not tracking whether or not
* the blocks are contiguous. They are just saying that they
* are freeing a set of blocks. It is this code that determines
* the pieces of that range that are actually contiguous.
*
* Calling ffs_blkrelease_start() will have created an entry
* that we will use.
*/
tp = trim_lookup(ump, devvp, bno, size, inum, key, OLD);
if (tp->size == 0) {
/*
* First block of a potential range, set block and size
* for the trim block.
*/
tp->bno = bno;
tp->size = size;
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
return;
}
/*
* If this block is a continuation of the range (either
* follows at the end or preceeds in the front) then we
* add it to the front or back of the list and return.
*
* If it is not a continuation of the trim that we were
* building, using the REPLACE alloctype, we request that
* the old trim request (still in tp) be unhashed and a
* new range started (in ntp). The ffs_blkfree_sendtrim(tp)
* call causes the block range described by tp to be issued
* (and then tp to be freed).
*/
if (bno + numfrags(fs, size) == tp->bno) {
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
tp->bno = bno;
tp->size += size;
return;
} else if (bno == tp->bno + numfrags(fs, tp->size)) {
TAILQ_INSERT_TAIL(&tp->blklist, blkelm, blkreqlist);
tp->size += size;
return;
}
ntp = trim_lookup(ump, devvp, bno, size, inum, key, REPLACE);
TAILQ_INSERT_HEAD(&ntp->blklist, blkelm, blkreqlist);
ffs_blkfree_sendtrim(tp);
}
#ifdef INVARIANTS
/*
* Verify allocation of a block or fragment. Returns true if block or
* fragment is allocated, false if it is free.
*/
static int
ffs_checkblk(ip, bno, size)
struct inode *ip;
ufs2_daddr_t bno;
long size;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
ufs1_daddr_t cgbno;
int i, error, frags, free;
u_int8_t *blksfree;
fs = ITOFS(ip);
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) {
printf("bsize = %ld, size = %ld, fs = %s\n",
(long)fs->fs_bsize, size, fs->fs_fsmnt);
panic("ffs_checkblk: bad size");
}
if ((u_int)bno >= fs->fs_size)
panic("ffs_checkblk: bad block %jd", (intmax_t)bno);
error = ffs_getcg(fs, ITODEVVP(ip), dtog(fs, bno), 0, &bp, &cgp);
if (error)
panic("ffs_checkblk: cylinder group read failed");
blksfree = cg_blksfree(cgp);
cgbno = dtogd(fs, bno);
if (size == fs->fs_bsize) {
free = ffs_isblock(fs, blksfree, fragstoblks(fs, cgbno));
} else {
frags = numfrags(fs, size);
for (free = 0, i = 0; i < frags; i++)
if (isset(blksfree, cgbno + i))
free++;
if (free != 0 && free != frags)
panic("ffs_checkblk: partially free fragment");
}
brelse(bp);
return (!free);
}
#endif /* INVARIANTS */
/*
* Free an inode.
*/
int
ffs_vfree(pvp, ino, mode)
struct vnode *pvp;
ino_t ino;
int mode;
{
struct ufsmount *ump;
if (DOINGSOFTDEP(pvp)) {
softdep_freefile(pvp, ino, mode);
return (0);
}
ump = VFSTOUFS(pvp->v_mount);
return (ffs_freefile(ump, ump->um_fs, ump->um_devvp, ino, mode, NULL));
}
/*
* Do the actual free operation.
* The specified inode is placed back in the free map.
*/
int
ffs_freefile(ump, fs, devvp, ino, mode, wkhd)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ino_t ino;
int mode;
struct workhead *wkhd;
{
struct cg *cgp;
struct buf *bp;
+ daddr_t dbn;
int error;
u_int cg;
u_int8_t *inosused;
struct cdev *dev;
ino_t cgino;
cg = ino_to_cg(fs, ino);
if (devvp->v_type == VREG) {
/* devvp is a snapshot */
MPASS(devvp->v_mount->mnt_data == ump);
dev = ump->um_devvp->v_rdev;
} else if (devvp->v_type == VCHR) {
/* devvp is a normal disk device */
dev = devvp->v_rdev;
} else {
bp = NULL;
return (0);
}
if (ino >= fs->fs_ipg * fs->fs_ncg)
panic("ffs_freefile: range: dev = %s, ino = %ju, fs = %s",
devtoname(dev), (uintmax_t)ino, fs->fs_fsmnt);
- if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
+ if ((error = ffs_getcg(fs, devvp, cg, GB_CVTENXIO, &bp, &cgp)) != 0) {
+ if (!ffs_fsfail_cleanup(ump, error) ||
+ !MOUNTEDSOFTDEP(UFSTOVFS(ump)) || devvp->v_type != VCHR)
+ return (error);
+ if (devvp->v_type == VREG)
+ dbn = fragstoblks(fs, cgtod(fs, cg));
+ else
+ dbn = fsbtodb(fs, cgtod(fs, cg));
+ error = getblkx(devvp, dbn, dbn, fs->fs_cgsize, 0, 0, 0, &bp);
+ KASSERT(error == 0, ("getblkx failed"));
+ softdep_setup_inofree(UFSTOVFS(ump), bp, ino, wkhd);
+ bp->b_flags |= B_RELBUF | B_NOCACHE;
+ bp->b_flags &= ~B_CACHE;
+ bawrite(bp);
return (error);
+ }
inosused = cg_inosused(cgp);
cgino = ino % fs->fs_ipg;
if (isclr(inosused, cgino)) {
printf("dev = %s, ino = %ju, fs = %s\n", devtoname(dev),
(uintmax_t)ino, fs->fs_fsmnt);
if (fs->fs_ronly == 0)
panic("ffs_freefile: freeing free inode");
}
clrbit(inosused, cgino);
if (cgino < cgp->cg_irotor)
cgp->cg_irotor = cgino;
cgp->cg_cs.cs_nifree++;
UFS_LOCK(ump);
fs->fs_cstotal.cs_nifree++;
fs->fs_cs(fs, cg).cs_nifree++;
if ((mode & IFMT) == IFDIR) {
cgp->cg_cs.cs_ndir--;
fs->fs_cstotal.cs_ndir--;
fs->fs_cs(fs, cg).cs_ndir--;
}
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (MOUNTEDSOFTDEP(UFSTOVFS(ump)) && devvp->v_type == VCHR)
softdep_setup_inofree(UFSTOVFS(ump), bp, ino, wkhd);
bdwrite(bp);
return (0);
}
/*
* Check to see if a file is free.
* Used to check for allocated files in snapshots.
*/
int
ffs_checkfreefile(fs, devvp, ino)
struct fs *fs;
struct vnode *devvp;
ino_t ino;
{
struct cg *cgp;
struct buf *bp;
int ret, error;
u_int cg;
u_int8_t *inosused;
cg = ino_to_cg(fs, ino);
if ((devvp->v_type != VREG) && (devvp->v_type != VCHR))
return (1);
if (ino >= fs->fs_ipg * fs->fs_ncg)
return (1);
if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
return (1);
inosused = cg_inosused(cgp);
ino %= fs->fs_ipg;
ret = isclr(inosused, ino);
brelse(bp);
return (ret);
}
/*
* Find a block of the specified size in the specified cylinder group.
*
* It is a panic if a request is made to find a block if none are
* available.
*/
static ufs1_daddr_t
ffs_mapsearch(fs, cgp, bpref, allocsiz)
struct fs *fs;
struct cg *cgp;
ufs2_daddr_t bpref;
int allocsiz;
{
ufs1_daddr_t bno;
int start, len, loc, i;
int blk, field, subfield, pos;
u_int8_t *blksfree;
/*
* find the fragment by searching through the free block
* map for an appropriate bit pattern
*/
if (bpref)
start = dtogd(fs, bpref) / NBBY;
else
start = cgp->cg_frotor / NBBY;
blksfree = cg_blksfree(cgp);
len = howmany(fs->fs_fpg, NBBY) - start;
loc = scanc((u_int)len, (u_char *)&blksfree[start],
fragtbl[fs->fs_frag],
(u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY))));
if (loc == 0) {
len = start + 1;
start = 0;
loc = scanc((u_int)len, (u_char *)&blksfree[0],
fragtbl[fs->fs_frag],
(u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY))));
if (loc == 0) {
printf("start = %d, len = %d, fs = %s\n",
start, len, fs->fs_fsmnt);
panic("ffs_alloccg: map corrupted");
/* NOTREACHED */
}
}
bno = (start + len - loc) * NBBY;
cgp->cg_frotor = bno;
/*
* found the byte in the map
* sift through the bits to find the selected frag
*/
for (i = bno + NBBY; bno < i; bno += fs->fs_frag) {
blk = blkmap(fs, blksfree, bno);
blk <<= 1;
field = around[allocsiz];
subfield = inside[allocsiz];
for (pos = 0; pos <= fs->fs_frag - allocsiz; pos++) {
if ((blk & field) == subfield)
return (bno + pos);
field <<= 1;
subfield <<= 1;
}
}
printf("bno = %lu, fs = %s\n", (u_long)bno, fs->fs_fsmnt);
panic("ffs_alloccg: block not in map");
return (-1);
}
static const struct statfs *
ffs_getmntstat(struct vnode *devvp)
{
if (devvp->v_type == VCHR)
return (&devvp->v_rdev->si_mountpt->mnt_stat);
return (ffs_getmntstat(VFSTOUFS(devvp->v_mount)->um_devvp));
}
/*
* Fetch and verify a cylinder group.
*/
int
ffs_getcg(fs, devvp, cg, flags, bpp, cgpp)
struct fs *fs;
struct vnode *devvp;
u_int cg;
int flags;
struct buf **bpp;
struct cg **cgpp;
{
struct buf *bp;
struct cg *cgp;
const struct statfs *sfs;
daddr_t blkno;
int error;
*bpp = NULL;
*cgpp = NULL;
if ((fs->fs_metackhash & CK_CYLGRP) != 0)
flags |= GB_CKHASH;
if (devvp->v_type == VREG)
blkno = fragstoblks(fs, cgtod(fs, cg));
else
blkno = fsbtodb(fs, cgtod(fs, cg));
error = breadn_flags(devvp, blkno, blkno, (int)fs->fs_cgsize, NULL,
NULL, 0, NOCRED, flags, ffs_ckhash_cg, &bp);
if (error != 0)
return (error);
cgp = (struct cg *)bp->b_data;
if ((fs->fs_metackhash & CK_CYLGRP) != 0 &&
(bp->b_flags & B_CKHASH) != 0 &&
cgp->cg_ckhash != bp->b_ckhash) {
sfs = ffs_getmntstat(devvp);
printf("UFS %s%s (%s) cylinder checksum failed: cg %u, cgp: "
"0x%x != bp: 0x%jx\n",
devvp->v_type == VCHR ? "" : "snapshot of ",
sfs->f_mntfromname, sfs->f_mntonname,
cg, cgp->cg_ckhash, (uintmax_t)bp->b_ckhash);
bp->b_flags &= ~B_CKHASH;
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
return (EIO);
}
if (!cg_chkmagic(cgp) || cgp->cg_cgx != cg) {
sfs = ffs_getmntstat(devvp);
printf("UFS %s%s (%s)",
devvp->v_type == VCHR ? "" : "snapshot of ",
sfs->f_mntfromname, sfs->f_mntonname);
if (!cg_chkmagic(cgp))
printf(" cg %u: bad magic number 0x%x should be 0x%x\n",
cg, cgp->cg_magic, CG_MAGIC);
else
printf(": wrong cylinder group cg %u != cgx %u\n", cg,
cgp->cg_cgx);
bp->b_flags &= ~B_CKHASH;
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
return (EIO);
}
bp->b_flags &= ~B_CKHASH;
bp->b_xflags |= BX_BKGRDWRITE;
/*
* If we are using check hashes on the cylinder group then we want
* to limit changing the cylinder group time to when we are actually
* going to write it to disk so that its check hash remains correct
* in memory. If the CK_CYLGRP flag is set the time is updated in
* ffs_bufwrite() as the buffer is queued for writing. Otherwise we
* update the time here as we have done historically.
*/
if ((fs->fs_metackhash & CK_CYLGRP) != 0)
bp->b_xflags |= BX_CYLGRP;
else
cgp->cg_old_time = cgp->cg_time = time_second;
*bpp = bp;
*cgpp = cgp;
return (0);
}
static void
ffs_ckhash_cg(bp)
struct buf *bp;
{
uint32_t ckhash;
struct cg *cgp;
cgp = (struct cg *)bp->b_data;
ckhash = cgp->cg_ckhash;
cgp->cg_ckhash = 0;
bp->b_ckhash = calculate_crc32c(~0L, bp->b_data, bp->b_bcount);
cgp->cg_ckhash = ckhash;
}
/*
* Fserr prints the name of a filesystem with an error diagnostic.
*
* The form of the error message is:
* fs: error message
*/
void
ffs_fserr(fs, inum, cp)
struct fs *fs;
ino_t inum;
char *cp;
{
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
log(LOG_ERR, "pid %d (%s), uid %d inumber %ju on %s: %s\n",
p->p_pid, p->p_comm, td->td_ucred->cr_uid, (uintmax_t)inum,
fs->fs_fsmnt, cp);
}
/*
* This function provides the capability for the fsck program to
* update an active filesystem. Fourteen operations are provided:
*
* adjrefcnt(inode, amt) - adjusts the reference count on the
* specified inode by the specified amount. Under normal
* operation the count should always go down. Decrementing
* the count to zero will cause the inode to be freed.
* adjblkcnt(inode, amt) - adjust the number of blocks used by the
* inode by the specified amount.
* setsize(inode, size) - set the size of the inode to the
* specified size.
* adjndir, adjbfree, adjifree, adjffree, adjnumclusters(amt) -
* adjust the superblock summary.
* freedirs(inode, count) - directory inodes [inode..inode + count - 1]
* are marked as free. Inodes should never have to be marked
* as in use.
* freefiles(inode, count) - file inodes [inode..inode + count - 1]
* are marked as free. Inodes should never have to be marked
* as in use.
* freeblks(blockno, size) - blocks [blockno..blockno + size - 1]
* are marked as free. Blocks should never have to be marked
* as in use.
* setflags(flags, set/clear) - the fs_flags field has the specified
* flags set (second parameter +1) or cleared (second parameter -1).
* setcwd(dirinode) - set the current directory to dirinode in the
* filesystem associated with the snapshot.
* setdotdot(oldvalue, newvalue) - Verify that the inode number for ".."
* in the current directory is oldvalue then change it to newvalue.
* unlink(nameptr, oldvalue) - Verify that the inode number associated
* with nameptr in the current directory is oldvalue then unlink it.
*/
static int sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vfs_ffs, FFS_ADJ_REFCNT, adjrefcnt,
CTLFLAG_WR | CTLTYPE_STRUCT | CTLFLAG_NEEDGIANT,
0, 0, sysctl_ffs_fsck, "S,fsck",
"Adjust Inode Reference Count");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_BLKCNT, adjblkcnt,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust Inode Used Blocks Count");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_SIZE, setsize,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Set the inode size");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NDIR, adjndir,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of directories");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NBFREE, adjnbfree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free blocks");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NIFREE, adjnifree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NFFREE, adjnffree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free frags");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NUMCLUSTERS, adjnumclusters,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free clusters");
static SYSCTL_NODE(_vfs_ffs, FFS_DIR_FREE, freedirs,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of Directory Inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_FILE_FREE, freefiles,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of File Inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_BLK_FREE, freeblks,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of Blocks");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_FLAGS, setflags,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Change Filesystem Flags");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_CWD, setcwd,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Set Current Working Directory");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_DOTDOT, setdotdot,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Change Value of .. Entry");
static SYSCTL_NODE(_vfs_ffs, FFS_UNLINK, unlink,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Unlink a Duplicate Name");
#ifdef DIAGNOSTIC
static int fsckcmds = 0;
SYSCTL_INT(_debug, OID_AUTO, ffs_fsckcmds, CTLFLAG_RW, &fsckcmds, 0,
"print out fsck_ffs-based filesystem update commands");
#endif /* DIAGNOSTIC */
static int
sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS)
{
struct thread *td = curthread;
struct fsck_cmd cmd;
struct ufsmount *ump;
struct vnode *vp, *dvp, *fdvp;
struct inode *ip, *dp;
struct mount *mp;
struct fs *fs;
struct pwd *pwd;
ufs2_daddr_t blkno;
long blkcnt, blksize;
u_long key;
struct file *fp;
cap_rights_t rights;
int filetype, error;
if (req->newlen > sizeof cmd)
return (EBADRPC);
if ((error = SYSCTL_IN(req, &cmd, sizeof cmd)) != 0)
return (error);
if (cmd.version != FFS_CMD_VERSION)
return (ERPCMISMATCH);
if ((error = getvnode(td, cmd.handle,
cap_rights_init(&rights, CAP_FSCK), &fp)) != 0)
return (error);
vp = fp->f_data;
if (vp->v_type != VREG && vp->v_type != VDIR) {
fdrop(fp, td);
return (EINVAL);
}
vn_start_write(vp, &mp, V_WAIT);
if (mp == NULL ||
strncmp(mp->mnt_stat.f_fstypename, "ufs", MFSNAMELEN)) {
vn_finished_write(mp);
fdrop(fp, td);
return (EINVAL);
}
ump = VFSTOUFS(mp);
if ((mp->mnt_flag & MNT_RDONLY) &&
ump->um_fsckpid != td->td_proc->p_pid) {
vn_finished_write(mp);
fdrop(fp, td);
return (EROFS);
}
fs = ump->um_fs;
filetype = IFREG;
switch (oidp->oid_number) {
case FFS_SET_FLAGS:
#ifdef DIAGNOSTIC
if (fsckcmds)
printf("%s: %s flags\n", mp->mnt_stat.f_mntonname,
cmd.size > 0 ? "set" : "clear");
#endif /* DIAGNOSTIC */
if (cmd.size > 0)
fs->fs_flags |= (long)cmd.value;
else
fs->fs_flags &= ~(long)cmd.value;
break;
case FFS_ADJ_REFCNT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust inode %jd link count by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
ip->i_nlink += cmd.size;
DIP_SET(ip, i_nlink, ip->i_nlink);
ip->i_effnlink += cmd.size;
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
if (DOINGSOFTDEP(vp))
softdep_change_linkcnt(ip);
vput(vp);
break;
case FFS_ADJ_BLKCNT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust inode %jd block count by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + cmd.size);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
vput(vp);
break;
case FFS_SET_SIZE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: set inode %jd size to %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
DIP_SET(ip, i_size, cmd.size);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
vput(vp);
break;
case FFS_DIR_FREE:
filetype = IFDIR;
/* fall through */
case FFS_FILE_FREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
if (cmd.size == 1)
printf("%s: free %s inode %ju\n",
mp->mnt_stat.f_mntonname,
filetype == IFDIR ? "directory" : "file",
(uintmax_t)cmd.value);
else
printf("%s: free %s inodes %ju-%ju\n",
mp->mnt_stat.f_mntonname,
filetype == IFDIR ? "directory" : "file",
(uintmax_t)cmd.value,
(uintmax_t)(cmd.value + cmd.size - 1));
}
#endif /* DIAGNOSTIC */
while (cmd.size > 0) {
if ((error = ffs_freefile(ump, fs, ump->um_devvp,
cmd.value, filetype, NULL)))
break;
cmd.size -= 1;
cmd.value += 1;
}
break;
case FFS_BLK_FREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
if (cmd.size == 1)
printf("%s: free block %jd\n",
mp->mnt_stat.f_mntonname,
(intmax_t)cmd.value);
else
printf("%s: free blocks %jd-%jd\n",
mp->mnt_stat.f_mntonname,
(intmax_t)cmd.value,
(intmax_t)cmd.value + cmd.size - 1);
}
#endif /* DIAGNOSTIC */
blkno = cmd.value;
blkcnt = cmd.size;
blksize = fs->fs_frag - (blkno % fs->fs_frag);
key = ffs_blkrelease_start(ump, ump->um_devvp, UFS_ROOTINO);
while (blkcnt > 0) {
if (blkcnt < blksize)
blksize = blkcnt;
ffs_blkfree(ump, fs, ump->um_devvp, blkno,
blksize * fs->fs_fsize, UFS_ROOTINO,
VDIR, NULL, key);
blkno += blksize;
blkcnt -= blksize;
blksize = fs->fs_frag;
}
ffs_blkrelease_finish(ump, key);
break;
/*
* Adjust superblock summaries. fsck(8) is expected to
* submit deltas when necessary.
*/
case FFS_ADJ_NDIR:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of directories by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_ndir += cmd.value;
break;
case FFS_ADJ_NBFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free blocks by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nbfree += cmd.value;
break;
case FFS_ADJ_NIFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free inodes by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nifree += cmd.value;
break;
case FFS_ADJ_NFFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free frags by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nffree += cmd.value;
break;
case FFS_ADJ_NUMCLUSTERS:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free clusters by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_numclusters += cmd.value;
break;
case FFS_SET_CWD:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: set current directory to inode %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_SHARED, &vp)))
break;
AUDIT_ARG_VNODE1(vp);
if ((error = change_dir(vp, td)) != 0) {
vput(vp);
break;
}
VOP_UNLOCK(vp);
pwd_chdir(td, vp);
break;
case FFS_SET_DOTDOT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: change .. in cwd from %jd to %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
/*
* First we have to get and lock the parent directory
* to which ".." points.
*/
error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &fdvp);
if (error)
break;
/*
* Now we get and lock the child directory containing "..".
*/
pwd = pwd_hold(td);
dvp = pwd->pwd_cdir;
if ((error = vget(dvp, LK_EXCLUSIVE, td)) != 0) {
vput(fdvp);
pwd_drop(pwd);
break;
}
dp = VTOI(dvp);
dp->i_offset = 12; /* XXX mastertemplate.dot_reclen */
error = ufs_dirrewrite(dp, VTOI(fdvp), (ino_t)cmd.size,
DT_DIR, 0);
cache_purge(fdvp);
cache_purge(dvp);
vput(dvp);
vput(fdvp);
pwd_drop(pwd);
break;
case FFS_UNLINK:
#ifdef DIAGNOSTIC
if (fsckcmds) {
char buf[32];
if (copyinstr((char *)(intptr_t)cmd.value, buf,32,NULL))
strncpy(buf, "Name_too_long", 32);
printf("%s: unlink %s (inode %jd)\n",
mp->mnt_stat.f_mntonname, buf, (intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
/*
* kern_funlinkat will do its own start/finish writes and
* they do not nest, so drop ours here. Setting mp == NULL
* indicates that vn_finished_write is not needed down below.
*/
vn_finished_write(mp);
mp = NULL;
error = kern_funlinkat(td, AT_FDCWD,
(char *)(intptr_t)cmd.value, FD_NONE, UIO_USERSPACE,
0, (ino_t)cmd.size);
break;
default:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("Invalid request %d from fsck\n",
oidp->oid_number);
}
#endif /* DIAGNOSTIC */
error = EINVAL;
break;
}
fdrop(fp, td);
vn_finished_write(mp);
return (error);
}
Index: head/sys/ufs/ffs/ffs_balloc.c
===================================================================
--- head/sys/ufs/ffs/ffs_balloc.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_balloc.c (revision 361491)
@@ -1,1171 +1,1175 @@
/*-
* SPDX-License-Identifier: (BSD-2-Clause-FreeBSD AND BSD-3-Clause)
*
* Copyright (c) 2002 Networks Associates Technology, Inc.
* All rights reserved.
*
* This software was developed for the FreeBSD Project by Marshall
* Kirk McKusick and Network Associates Laboratories, the Security
* Research Division of Network Associates, Inc. under DARPA/SPAWAR
* contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS
* research program
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_balloc.c 8.8 (Berkeley) 6/16/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/lock.h>
#include <sys/mount.h>
#include <sys/vnode.h>
#include <sys/vmmeter.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
/*
* Balloc defines the structure of filesystem storage
* by allocating the physical blocks on a device given
* the inode and the logical block number in a file.
* This is the allocation strategy for UFS1. Below is
* the allocation strategy for UFS2.
*/
int
ffs_balloc_ufs1(struct vnode *vp, off_t startoffset, int size,
struct ucred *cred, int flags, struct buf **bpp)
{
struct inode *ip;
struct ufs1_dinode *dp;
ufs_lbn_t lbn, lastlbn;
struct fs *fs;
ufs1_daddr_t nb;
struct buf *bp, *nbp;
struct mount *mp;
struct ufsmount *ump;
struct indir indirs[UFS_NIADDR + 2];
int deallocated, osize, nsize, num, i, error;
ufs2_daddr_t newb;
ufs1_daddr_t *bap, pref;
ufs1_daddr_t *allocib, *blkp, *allocblk, allociblk[UFS_NIADDR + 1];
ufs2_daddr_t *lbns_remfree, lbns[UFS_NIADDR + 1];
int unwindidx = -1;
int saved_inbdflush;
int gbflags, reclaimed;
ip = VTOI(vp);
dp = ip->i_din1;
fs = ITOFS(ip);
mp = ITOVFS(ip);
ump = ITOUMP(ip);
lbn = lblkno(fs, startoffset);
size = blkoff(fs, startoffset) + size;
reclaimed = 0;
if (size > fs->fs_bsize)
panic("ffs_balloc_ufs1: blk too big");
*bpp = NULL;
if (flags & IO_EXT)
return (EOPNOTSUPP);
if (lbn < 0)
return (EFBIG);
gbflags = (flags & BA_UNMAPPED) != 0 ? GB_UNMAPPED : 0;
if (DOINGSOFTDEP(vp))
softdep_prealloc(vp, MNT_WAIT);
/*
* If the next write will extend the file into a new block,
* and the file is currently composed of a fragment
* this fragment has to be extended to be a full block.
*/
lastlbn = lblkno(fs, ip->i_size);
if (lastlbn < UFS_NDADDR && lastlbn < lbn) {
nb = lastlbn;
osize = blksize(fs, ip, nb);
if (osize < fs->fs_bsize && osize > 0) {
UFS_LOCK(ump);
error = ffs_realloccg(ip, nb, dp->di_db[nb],
ffs_blkpref_ufs1(ip, lastlbn, (int)nb,
&dp->di_db[0]), osize, (int)fs->fs_bsize, flags,
cred, &bp);
if (error)
return (error);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, nb,
dbtofsb(fs, bp->b_blkno), dp->di_db[nb],
fs->fs_bsize, osize, bp);
ip->i_size = smalllblktosize(fs, nb + 1);
dp->di_size = ip->i_size;
dp->di_db[nb] = dbtofsb(fs, bp->b_blkno);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (flags & IO_SYNC)
bwrite(bp);
else if (DOINGASYNC(vp))
bdwrite(bp);
else
bawrite(bp);
}
}
/*
* The first UFS_NDADDR blocks are direct blocks
*/
if (lbn < UFS_NDADDR) {
if (flags & BA_METAONLY)
panic("ffs_balloc_ufs1: BA_METAONLY for direct block");
nb = dp->di_db[lbn];
if (nb != 0 && ip->i_size >= smalllblktosize(fs, lbn + 1)) {
error = bread(vp, lbn, fs->fs_bsize, NOCRED, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
*bpp = bp;
return (0);
}
if (nb != 0) {
/*
* Consider need to reallocate a fragment.
*/
osize = fragroundup(fs, blkoff(fs, ip->i_size));
nsize = fragroundup(fs, size);
if (nsize <= osize) {
error = bread(vp, lbn, osize, NOCRED, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
} else {
UFS_LOCK(ump);
error = ffs_realloccg(ip, lbn, dp->di_db[lbn],
ffs_blkpref_ufs1(ip, lbn, (int)lbn,
&dp->di_db[0]), osize, nsize, flags,
cred, &bp);
if (error)
return (error);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, lbn,
dbtofsb(fs, bp->b_blkno), nb,
nsize, osize, bp);
}
} else {
if (ip->i_size < smalllblktosize(fs, lbn + 1))
nsize = fragroundup(fs, size);
else
nsize = fs->fs_bsize;
UFS_LOCK(ump);
error = ffs_alloc(ip, lbn,
ffs_blkpref_ufs1(ip, lbn, (int)lbn, &dp->di_db[0]),
nsize, flags, cred, &newb);
if (error)
return (error);
bp = getblk(vp, lbn, nsize, 0, 0, gbflags);
bp->b_blkno = fsbtodb(fs, newb);
if (flags & BA_CLRBUF)
vfs_bio_clrbuf(bp);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, lbn, newb, 0,
nsize, 0, bp);
}
dp->di_db[lbn] = dbtofsb(fs, bp->b_blkno);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
*bpp = bp;
return (0);
}
/*
* Determine the number of levels of indirection.
*/
pref = 0;
if ((error = ufs_getlbns(vp, lbn, indirs, &num)) != 0)
return(error);
#ifdef INVARIANTS
if (num < 1)
panic ("ffs_balloc_ufs1: ufs_getlbns returned indirect block");
#endif
saved_inbdflush = curthread_pflags_set(TDP_INBDFLUSH);
/*
* Fetch the first indirect block allocating if necessary.
*/
--num;
nb = dp->di_ib[indirs[0].in_off];
allocib = NULL;
allocblk = allociblk;
lbns_remfree = lbns;
if (nb == 0) {
UFS_LOCK(ump);
pref = ffs_blkpref_ufs1(ip, lbn, -indirs[0].in_off - 1,
(ufs1_daddr_t *)0);
if ((error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags, cred, &newb)) != 0) {
curthread_pflags_restore(saved_inbdflush);
return (error);
}
pref = newb + fs->fs_frag;
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = indirs[1].in_lbn;
bp = getblk(vp, indirs[1].in_lbn, fs->fs_bsize, 0, 0, gbflags);
bp->b_blkno = fsbtodb(fs, nb);
vfs_bio_clrbuf(bp);
if (DOINGSOFTDEP(vp)) {
softdep_setup_allocdirect(ip,
UFS_NDADDR + indirs[0].in_off, newb, 0,
fs->fs_bsize, 0, bp);
bdwrite(bp);
} else if ((flags & IO_SYNC) == 0 && DOINGASYNC(vp)) {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
} else {
if ((error = bwrite(bp)) != 0)
goto fail;
}
allocib = &dp->di_ib[indirs[0].in_off];
*allocib = nb;
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
}
/*
* Fetch through the indirect blocks, allocating as necessary.
*/
retry:
for (i = 1;;) {
error = bread(vp,
indirs[i].in_lbn, (int)fs->fs_bsize, NOCRED, &bp);
if (error) {
goto fail;
}
bap = (ufs1_daddr_t *)bp->b_data;
nb = bap[indirs[i].in_off];
if ((error = UFS_CHECK_BLKNO(mp, ip->i_number, nb,
fs->fs_bsize)) != 0) {
brelse(bp);
goto fail;
}
if (i == num)
break;
i += 1;
if (nb != 0) {
bqrelse(bp);
continue;
}
UFS_LOCK(ump);
/*
* If parent indirect has just been allocated, try to cluster
* immediately following it.
*/
if (pref == 0)
pref = ffs_blkpref_ufs1(ip, lbn, i - num - 1,
(ufs1_daddr_t *)0);
if ((error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags | IO_BUFLOCKED, cred, &newb)) != 0) {
brelse(bp);
UFS_LOCK(ump);
if (DOINGSOFTDEP(vp) && ++reclaimed == 1) {
softdep_request_cleanup(fs, vp, cred,
FLUSH_BLOCKS_WAIT);
UFS_UNLOCK(ump);
goto retry;
}
- if (ppsratecheck(&ump->um_last_fullmsg,
+ if (!ffs_fsfail_cleanup_locked(ump, error) &&
+ ppsratecheck(&ump->um_last_fullmsg,
&ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem "
"is full\n", fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
goto fail;
}
pref = newb + fs->fs_frag;
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = indirs[i].in_lbn;
nbp = getblk(vp, indirs[i].in_lbn, fs->fs_bsize, 0, 0, 0);
nbp->b_blkno = fsbtodb(fs, nb);
vfs_bio_clrbuf(nbp);
if (DOINGSOFTDEP(vp)) {
softdep_setup_allocindir_meta(nbp, ip, bp,
indirs[i - 1].in_off, nb);
bdwrite(nbp);
} else if ((flags & IO_SYNC) == 0 && DOINGASYNC(vp)) {
if (nbp->b_bufsize == fs->fs_bsize)
nbp->b_flags |= B_CLUSTEROK;
bdwrite(nbp);
} else {
if ((error = bwrite(nbp)) != 0) {
brelse(bp);
goto fail;
}
}
bap[indirs[i - 1].in_off] = nb;
if (allocib == NULL && unwindidx < 0)
unwindidx = i - 1;
/*
* If required, write synchronously, otherwise use
* delayed write.
*/
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
}
/*
* If asked only for the indirect block, then return it.
*/
if (flags & BA_METAONLY) {
curthread_pflags_restore(saved_inbdflush);
*bpp = bp;
return (0);
}
/*
* Get the data block, allocating if necessary.
*/
if (nb == 0) {
UFS_LOCK(ump);
/*
* If allocating metadata at the front of the cylinder
* group and parent indirect block has just been allocated,
* then cluster next to it if it is the first indirect in
* the file. Otherwise it has been allocated in the metadata
* area, so we want to find our own place out in the data area.
*/
if (pref == 0 || (lbn > UFS_NDADDR && fs->fs_metaspace != 0))
pref = ffs_blkpref_ufs1(ip, lbn, indirs[i].in_off,
&bap[0]);
error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags | IO_BUFLOCKED, cred, &newb);
if (error) {
brelse(bp);
UFS_LOCK(ump);
if (DOINGSOFTDEP(vp) && ++reclaimed == 1) {
softdep_request_cleanup(fs, vp, cred,
FLUSH_BLOCKS_WAIT);
UFS_UNLOCK(ump);
goto retry;
}
- if (ppsratecheck(&ump->um_last_fullmsg,
+ if (!ffs_fsfail_cleanup_locked(ump, error) &&
+ ppsratecheck(&ump->um_last_fullmsg,
&ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem "
"is full\n", fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
goto fail;
}
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = lbn;
nbp = getblk(vp, lbn, fs->fs_bsize, 0, 0, gbflags);
nbp->b_blkno = fsbtodb(fs, nb);
if (flags & BA_CLRBUF)
vfs_bio_clrbuf(nbp);
if (DOINGSOFTDEP(vp))
softdep_setup_allocindir_page(ip, lbn, bp,
indirs[i].in_off, nb, 0, nbp);
bap[indirs[i].in_off] = nb;
/*
* If required, write synchronously, otherwise use
* delayed write.
*/
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
curthread_pflags_restore(saved_inbdflush);
*bpp = nbp;
return (0);
}
brelse(bp);
if (flags & BA_CLRBUF) {
int seqcount = (flags & BA_SEQMASK) >> BA_SEQSHIFT;
if (seqcount != 0 &&
(vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0 &&
!(vm_page_count_severe() || buf_dirty_count_severe())) {
error = cluster_read(vp, ip->i_size, lbn,
(int)fs->fs_bsize, NOCRED,
MAXBSIZE, seqcount, gbflags, &nbp);
} else {
error = bread_gb(vp, lbn, (int)fs->fs_bsize, NOCRED,
gbflags, &nbp);
}
if (error) {
brelse(nbp);
goto fail;
}
} else {
nbp = getblk(vp, lbn, fs->fs_bsize, 0, 0, gbflags);
nbp->b_blkno = fsbtodb(fs, nb);
}
curthread_pflags_restore(saved_inbdflush);
*bpp = nbp;
return (0);
fail:
curthread_pflags_restore(saved_inbdflush);
/*
* If we have failed to allocate any blocks, simply return the error.
* This is the usual case and avoids the need to fsync the file.
*/
if (allocblk == allociblk && allocib == NULL && unwindidx == -1)
return (error);
/*
* If we have failed part way through block allocation, we
* have to deallocate any indirect blocks that we have allocated.
* We have to fsync the file before we start to get rid of all
* of its dependencies so that we do not leave them dangling.
* We have to sync it at the end so that the soft updates code
* does not find any untracked changes. Although this is really
* slow, running out of disk space is not expected to be a common
* occurrence. The error return from fsync is ignored as we already
* have an error to return to the user.
*
* XXX Still have to journal the free below
*/
(void) ffs_syncvnode(vp, MNT_WAIT, 0);
for (deallocated = 0, blkp = allociblk, lbns_remfree = lbns;
blkp < allocblk; blkp++, lbns_remfree++) {
/*
* We shall not leave the freed blocks on the vnode
* buffer object lists.
*/
bp = getblk(vp, *lbns_remfree, fs->fs_bsize, 0, 0,
GB_NOCREAT | GB_UNMAPPED);
if (bp != NULL) {
KASSERT(bp->b_blkno == fsbtodb(fs, *blkp),
("mismatch1 l %jd %jd b %ju %ju",
(intmax_t)bp->b_lblkno, (uintmax_t)*lbns_remfree,
(uintmax_t)bp->b_blkno,
(uintmax_t)fsbtodb(fs, *blkp)));
bp->b_flags |= B_INVAL | B_RELBUF | B_NOCACHE;
bp->b_flags &= ~(B_ASYNC | B_CACHE);
brelse(bp);
}
deallocated += fs->fs_bsize;
}
if (allocib != NULL) {
*allocib = 0;
} else if (unwindidx >= 0) {
int r;
r = bread(vp, indirs[unwindidx].in_lbn,
(int)fs->fs_bsize, NOCRED, &bp);
if (r) {
panic("Could not unwind indirect block, error %d", r);
brelse(bp);
} else {
bap = (ufs1_daddr_t *)bp->b_data;
bap[indirs[unwindidx].in_off] = 0;
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
}
}
if (deallocated) {
#ifdef QUOTA
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(deallocated), cred, FORCE);
#endif
dp->di_blocks -= btodb(deallocated);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
}
(void) ffs_syncvnode(vp, MNT_WAIT, 0);
/*
* After the buffers are invalidated and on-disk pointers are
* cleared, free the blocks.
*/
for (blkp = allociblk; blkp < allocblk; blkp++) {
#ifdef INVARIANTS
if (blkp == allociblk)
lbns_remfree = lbns;
bp = getblk(vp, *lbns_remfree, fs->fs_bsize, 0, 0,
GB_NOCREAT | GB_UNMAPPED);
if (bp != NULL) {
panic("zombie1 %jd %ju %ju",
(intmax_t)bp->b_lblkno, (uintmax_t)bp->b_blkno,
(uintmax_t)fsbtodb(fs, *blkp));
}
lbns_remfree++;
#endif
ffs_blkfree(ump, fs, ump->um_devvp, *blkp, fs->fs_bsize,
ip->i_number, vp->v_type, NULL, SINGLETON_KEY);
}
return (error);
}
/*
* Balloc defines the structure of file system storage
* by allocating the physical blocks on a device given
* the inode and the logical block number in a file.
* This is the allocation strategy for UFS2. Above is
* the allocation strategy for UFS1.
*/
int
ffs_balloc_ufs2(struct vnode *vp, off_t startoffset, int size,
struct ucred *cred, int flags, struct buf **bpp)
{
struct inode *ip;
struct ufs2_dinode *dp;
ufs_lbn_t lbn, lastlbn;
struct fs *fs;
struct buf *bp, *nbp;
struct mount *mp;
struct ufsmount *ump;
struct indir indirs[UFS_NIADDR + 2];
ufs2_daddr_t nb, newb, *bap, pref;
ufs2_daddr_t *allocib, *blkp, *allocblk, allociblk[UFS_NIADDR + 1];
ufs2_daddr_t *lbns_remfree, lbns[UFS_NIADDR + 1];
int deallocated, osize, nsize, num, i, error;
int unwindidx = -1;
int saved_inbdflush;
int gbflags, reclaimed;
ip = VTOI(vp);
dp = ip->i_din2;
fs = ITOFS(ip);
mp = ITOVFS(ip);
ump = ITOUMP(ip);
lbn = lblkno(fs, startoffset);
size = blkoff(fs, startoffset) + size;
reclaimed = 0;
if (size > fs->fs_bsize)
panic("ffs_balloc_ufs2: blk too big");
*bpp = NULL;
if (lbn < 0)
return (EFBIG);
gbflags = (flags & BA_UNMAPPED) != 0 ? GB_UNMAPPED : 0;
if (DOINGSOFTDEP(vp))
softdep_prealloc(vp, MNT_WAIT);
/*
* Check for allocating external data.
*/
if (flags & IO_EXT) {
if (lbn >= UFS_NXADDR)
return (EFBIG);
/*
* If the next write will extend the data into a new block,
* and the data is currently composed of a fragment
* this fragment has to be extended to be a full block.
*/
lastlbn = lblkno(fs, dp->di_extsize);
if (lastlbn < lbn) {
nb = lastlbn;
osize = sblksize(fs, dp->di_extsize, nb);
if (osize < fs->fs_bsize && osize > 0) {
UFS_LOCK(ump);
error = ffs_realloccg(ip, -1 - nb,
dp->di_extb[nb],
ffs_blkpref_ufs2(ip, lastlbn, (int)nb,
&dp->di_extb[0]), osize,
(int)fs->fs_bsize, flags, cred, &bp);
if (error)
return (error);
if (DOINGSOFTDEP(vp))
softdep_setup_allocext(ip, nb,
dbtofsb(fs, bp->b_blkno),
dp->di_extb[nb],
fs->fs_bsize, osize, bp);
dp->di_extsize = smalllblktosize(fs, nb + 1);
dp->di_extb[nb] = dbtofsb(fs, bp->b_blkno);
bp->b_xflags |= BX_ALTDATA;
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (flags & IO_SYNC)
bwrite(bp);
else
bawrite(bp);
}
}
/*
* All blocks are direct blocks
*/
if (flags & BA_METAONLY)
panic("ffs_balloc_ufs2: BA_METAONLY for ext block");
nb = dp->di_extb[lbn];
if (nb != 0 && dp->di_extsize >= smalllblktosize(fs, lbn + 1)) {
error = bread_gb(vp, -1 - lbn, fs->fs_bsize, NOCRED,
gbflags, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
bp->b_xflags |= BX_ALTDATA;
*bpp = bp;
return (0);
}
if (nb != 0) {
/*
* Consider need to reallocate a fragment.
*/
osize = fragroundup(fs, blkoff(fs, dp->di_extsize));
nsize = fragroundup(fs, size);
if (nsize <= osize) {
error = bread_gb(vp, -1 - lbn, osize, NOCRED,
gbflags, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
bp->b_xflags |= BX_ALTDATA;
} else {
UFS_LOCK(ump);
error = ffs_realloccg(ip, -1 - lbn,
dp->di_extb[lbn],
ffs_blkpref_ufs2(ip, lbn, (int)lbn,
&dp->di_extb[0]), osize, nsize, flags,
cred, &bp);
if (error)
return (error);
bp->b_xflags |= BX_ALTDATA;
if (DOINGSOFTDEP(vp))
softdep_setup_allocext(ip, lbn,
dbtofsb(fs, bp->b_blkno), nb,
nsize, osize, bp);
}
} else {
if (dp->di_extsize < smalllblktosize(fs, lbn + 1))
nsize = fragroundup(fs, size);
else
nsize = fs->fs_bsize;
UFS_LOCK(ump);
error = ffs_alloc(ip, lbn,
ffs_blkpref_ufs2(ip, lbn, (int)lbn, &dp->di_extb[0]),
nsize, flags, cred, &newb);
if (error)
return (error);
bp = getblk(vp, -1 - lbn, nsize, 0, 0, gbflags);
bp->b_blkno = fsbtodb(fs, newb);
bp->b_xflags |= BX_ALTDATA;
if (flags & BA_CLRBUF)
vfs_bio_clrbuf(bp);
if (DOINGSOFTDEP(vp))
softdep_setup_allocext(ip, lbn, newb, 0,
nsize, 0, bp);
}
dp->di_extb[lbn] = dbtofsb(fs, bp->b_blkno);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
*bpp = bp;
return (0);
}
/*
* If the next write will extend the file into a new block,
* and the file is currently composed of a fragment
* this fragment has to be extended to be a full block.
*/
lastlbn = lblkno(fs, ip->i_size);
if (lastlbn < UFS_NDADDR && lastlbn < lbn) {
nb = lastlbn;
osize = blksize(fs, ip, nb);
if (osize < fs->fs_bsize && osize > 0) {
UFS_LOCK(ump);
error = ffs_realloccg(ip, nb, dp->di_db[nb],
ffs_blkpref_ufs2(ip, lastlbn, (int)nb,
&dp->di_db[0]), osize, (int)fs->fs_bsize,
flags, cred, &bp);
if (error)
return (error);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, nb,
dbtofsb(fs, bp->b_blkno),
dp->di_db[nb],
fs->fs_bsize, osize, bp);
ip->i_size = smalllblktosize(fs, nb + 1);
dp->di_size = ip->i_size;
dp->di_db[nb] = dbtofsb(fs, bp->b_blkno);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (flags & IO_SYNC)
bwrite(bp);
else
bawrite(bp);
}
}
/*
* The first UFS_NDADDR blocks are direct blocks
*/
if (lbn < UFS_NDADDR) {
if (flags & BA_METAONLY)
panic("ffs_balloc_ufs2: BA_METAONLY for direct block");
nb = dp->di_db[lbn];
if (nb != 0 && ip->i_size >= smalllblktosize(fs, lbn + 1)) {
error = bread_gb(vp, lbn, fs->fs_bsize, NOCRED,
gbflags, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
*bpp = bp;
return (0);
}
if (nb != 0) {
/*
* Consider need to reallocate a fragment.
*/
osize = fragroundup(fs, blkoff(fs, ip->i_size));
nsize = fragroundup(fs, size);
if (nsize <= osize) {
error = bread_gb(vp, lbn, osize, NOCRED,
gbflags, &bp);
if (error) {
return (error);
}
bp->b_blkno = fsbtodb(fs, nb);
} else {
UFS_LOCK(ump);
error = ffs_realloccg(ip, lbn, dp->di_db[lbn],
ffs_blkpref_ufs2(ip, lbn, (int)lbn,
&dp->di_db[0]), osize, nsize, flags,
cred, &bp);
if (error)
return (error);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, lbn,
dbtofsb(fs, bp->b_blkno), nb,
nsize, osize, bp);
}
} else {
if (ip->i_size < smalllblktosize(fs, lbn + 1))
nsize = fragroundup(fs, size);
else
nsize = fs->fs_bsize;
UFS_LOCK(ump);
error = ffs_alloc(ip, lbn,
ffs_blkpref_ufs2(ip, lbn, (int)lbn,
&dp->di_db[0]), nsize, flags, cred, &newb);
if (error)
return (error);
bp = getblk(vp, lbn, nsize, 0, 0, gbflags);
bp->b_blkno = fsbtodb(fs, newb);
if (flags & BA_CLRBUF)
vfs_bio_clrbuf(bp);
if (DOINGSOFTDEP(vp))
softdep_setup_allocdirect(ip, lbn, newb, 0,
nsize, 0, bp);
}
dp->di_db[lbn] = dbtofsb(fs, bp->b_blkno);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
*bpp = bp;
return (0);
}
/*
* Determine the number of levels of indirection.
*/
pref = 0;
if ((error = ufs_getlbns(vp, lbn, indirs, &num)) != 0)
return(error);
#ifdef INVARIANTS
if (num < 1)
panic ("ffs_balloc_ufs2: ufs_getlbns returned indirect block");
#endif
saved_inbdflush = curthread_pflags_set(TDP_INBDFLUSH);
/*
* Fetch the first indirect block allocating if necessary.
*/
--num;
nb = dp->di_ib[indirs[0].in_off];
allocib = NULL;
allocblk = allociblk;
lbns_remfree = lbns;
if (nb == 0) {
UFS_LOCK(ump);
pref = ffs_blkpref_ufs2(ip, lbn, -indirs[0].in_off - 1,
(ufs2_daddr_t *)0);
if ((error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags, cred, &newb)) != 0) {
curthread_pflags_restore(saved_inbdflush);
return (error);
}
pref = newb + fs->fs_frag;
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = indirs[1].in_lbn;
bp = getblk(vp, indirs[1].in_lbn, fs->fs_bsize, 0, 0,
GB_UNMAPPED);
bp->b_blkno = fsbtodb(fs, nb);
vfs_bio_clrbuf(bp);
if (DOINGSOFTDEP(vp)) {
softdep_setup_allocdirect(ip,
UFS_NDADDR + indirs[0].in_off, newb, 0,
fs->fs_bsize, 0, bp);
bdwrite(bp);
} else if ((flags & IO_SYNC) == 0 && DOINGASYNC(vp)) {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
} else {
if ((error = bwrite(bp)) != 0)
goto fail;
}
allocib = &dp->di_ib[indirs[0].in_off];
*allocib = nb;
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
}
/*
* Fetch through the indirect blocks, allocating as necessary.
*/
retry:
for (i = 1;;) {
error = bread(vp,
indirs[i].in_lbn, (int)fs->fs_bsize, NOCRED, &bp);
if (error) {
goto fail;
}
bap = (ufs2_daddr_t *)bp->b_data;
nb = bap[indirs[i].in_off];
if ((error = UFS_CHECK_BLKNO(mp, ip->i_number, nb,
fs->fs_bsize)) != 0) {
brelse(bp);
goto fail;
}
if (i == num)
break;
i += 1;
if (nb != 0) {
bqrelse(bp);
continue;
}
UFS_LOCK(ump);
/*
* If parent indirect has just been allocated, try to cluster
* immediately following it.
*/
if (pref == 0)
pref = ffs_blkpref_ufs2(ip, lbn, i - num - 1,
(ufs2_daddr_t *)0);
if ((error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags | IO_BUFLOCKED, cred, &newb)) != 0) {
brelse(bp);
UFS_LOCK(ump);
if (DOINGSOFTDEP(vp) && ++reclaimed == 1) {
softdep_request_cleanup(fs, vp, cred,
FLUSH_BLOCKS_WAIT);
UFS_UNLOCK(ump);
goto retry;
}
- if (ppsratecheck(&ump->um_last_fullmsg,
+ if (!ffs_fsfail_cleanup_locked(ump, error) &&
+ ppsratecheck(&ump->um_last_fullmsg,
&ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem "
"is full\n", fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
goto fail;
}
pref = newb + fs->fs_frag;
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = indirs[i].in_lbn;
nbp = getblk(vp, indirs[i].in_lbn, fs->fs_bsize, 0, 0,
GB_UNMAPPED);
nbp->b_blkno = fsbtodb(fs, nb);
vfs_bio_clrbuf(nbp);
if (DOINGSOFTDEP(vp)) {
softdep_setup_allocindir_meta(nbp, ip, bp,
indirs[i - 1].in_off, nb);
bdwrite(nbp);
} else if ((flags & IO_SYNC) == 0 && DOINGASYNC(vp)) {
if (nbp->b_bufsize == fs->fs_bsize)
nbp->b_flags |= B_CLUSTEROK;
bdwrite(nbp);
} else {
if ((error = bwrite(nbp)) != 0) {
brelse(bp);
goto fail;
}
}
bap[indirs[i - 1].in_off] = nb;
if (allocib == NULL && unwindidx < 0)
unwindidx = i - 1;
/*
* If required, write synchronously, otherwise use
* delayed write.
*/
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
}
/*
* If asked only for the indirect block, then return it.
*/
if (flags & BA_METAONLY) {
curthread_pflags_restore(saved_inbdflush);
*bpp = bp;
return (0);
}
/*
* Get the data block, allocating if necessary.
*/
if (nb == 0) {
UFS_LOCK(ump);
/*
* If allocating metadata at the front of the cylinder
* group and parent indirect block has just been allocated,
* then cluster next to it if it is the first indirect in
* the file. Otherwise it has been allocated in the metadata
* area, so we want to find our own place out in the data area.
*/
if (pref == 0 || (lbn > UFS_NDADDR && fs->fs_metaspace != 0))
pref = ffs_blkpref_ufs2(ip, lbn, indirs[i].in_off,
&bap[0]);
error = ffs_alloc(ip, lbn, pref, (int)fs->fs_bsize,
flags | IO_BUFLOCKED, cred, &newb);
if (error) {
brelse(bp);
UFS_LOCK(ump);
if (DOINGSOFTDEP(vp) && ++reclaimed == 1) {
softdep_request_cleanup(fs, vp, cred,
FLUSH_BLOCKS_WAIT);
UFS_UNLOCK(ump);
goto retry;
}
- if (ppsratecheck(&ump->um_last_fullmsg,
+ if (!ffs_fsfail_cleanup_locked(ump, error) &&
+ ppsratecheck(&ump->um_last_fullmsg,
&ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem "
"is full\n", fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
goto fail;
}
nb = newb;
MPASS(allocblk < allociblk + nitems(allociblk));
MPASS(lbns_remfree < lbns + nitems(lbns));
*allocblk++ = nb;
*lbns_remfree++ = lbn;
nbp = getblk(vp, lbn, fs->fs_bsize, 0, 0, gbflags);
nbp->b_blkno = fsbtodb(fs, nb);
if (flags & BA_CLRBUF)
vfs_bio_clrbuf(nbp);
if (DOINGSOFTDEP(vp))
softdep_setup_allocindir_page(ip, lbn, bp,
indirs[i].in_off, nb, 0, nbp);
bap[indirs[i].in_off] = nb;
/*
* If required, write synchronously, otherwise use
* delayed write.
*/
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
curthread_pflags_restore(saved_inbdflush);
*bpp = nbp;
return (0);
}
brelse(bp);
/*
* If requested clear invalid portions of the buffer. If we
* have to do a read-before-write (typical if BA_CLRBUF is set),
* try to do some read-ahead in the sequential case to reduce
* the number of I/O transactions.
*/
if (flags & BA_CLRBUF) {
int seqcount = (flags & BA_SEQMASK) >> BA_SEQSHIFT;
if (seqcount != 0 &&
(vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0 &&
!(vm_page_count_severe() || buf_dirty_count_severe())) {
error = cluster_read(vp, ip->i_size, lbn,
(int)fs->fs_bsize, NOCRED,
MAXBSIZE, seqcount, gbflags, &nbp);
} else {
error = bread_gb(vp, lbn, (int)fs->fs_bsize,
NOCRED, gbflags, &nbp);
}
if (error) {
brelse(nbp);
goto fail;
}
} else {
nbp = getblk(vp, lbn, fs->fs_bsize, 0, 0, gbflags);
nbp->b_blkno = fsbtodb(fs, nb);
}
curthread_pflags_restore(saved_inbdflush);
*bpp = nbp;
return (0);
fail:
curthread_pflags_restore(saved_inbdflush);
/*
* If we have failed to allocate any blocks, simply return the error.
* This is the usual case and avoids the need to fsync the file.
*/
if (allocblk == allociblk && allocib == NULL && unwindidx == -1)
return (error);
/*
* If we have failed part way through block allocation, we
* have to deallocate any indirect blocks that we have allocated.
* We have to fsync the file before we start to get rid of all
* of its dependencies so that we do not leave them dangling.
* We have to sync it at the end so that the soft updates code
* does not find any untracked changes. Although this is really
* slow, running out of disk space is not expected to be a common
* occurrence. The error return from fsync is ignored as we already
* have an error to return to the user.
*
* XXX Still have to journal the free below
*/
(void) ffs_syncvnode(vp, MNT_WAIT, 0);
for (deallocated = 0, blkp = allociblk, lbns_remfree = lbns;
blkp < allocblk; blkp++, lbns_remfree++) {
/*
* We shall not leave the freed blocks on the vnode
* buffer object lists.
*/
bp = getblk(vp, *lbns_remfree, fs->fs_bsize, 0, 0,
GB_NOCREAT | GB_UNMAPPED);
if (bp != NULL) {
KASSERT(bp->b_blkno == fsbtodb(fs, *blkp),
("mismatch2 l %jd %jd b %ju %ju",
(intmax_t)bp->b_lblkno, (uintmax_t)*lbns_remfree,
(uintmax_t)bp->b_blkno,
(uintmax_t)fsbtodb(fs, *blkp)));
bp->b_flags |= B_INVAL | B_RELBUF | B_NOCACHE;
bp->b_flags &= ~(B_ASYNC | B_CACHE);
brelse(bp);
}
deallocated += fs->fs_bsize;
}
if (allocib != NULL) {
*allocib = 0;
} else if (unwindidx >= 0) {
int r;
r = bread(vp, indirs[unwindidx].in_lbn,
(int)fs->fs_bsize, NOCRED, &bp);
if (r) {
panic("Could not unwind indirect block, error %d", r);
brelse(bp);
} else {
bap = (ufs2_daddr_t *)bp->b_data;
bap[indirs[unwindidx].in_off] = 0;
if (flags & IO_SYNC) {
bwrite(bp);
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
}
}
if (deallocated) {
#ifdef QUOTA
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(deallocated), cred, FORCE);
#endif
dp->di_blocks -= btodb(deallocated);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
}
(void) ffs_syncvnode(vp, MNT_WAIT, 0);
/*
* After the buffers are invalidated and on-disk pointers are
* cleared, free the blocks.
*/
for (blkp = allociblk; blkp < allocblk; blkp++) {
#ifdef INVARIANTS
if (blkp == allociblk)
lbns_remfree = lbns;
bp = getblk(vp, *lbns_remfree, fs->fs_bsize, 0, 0,
GB_NOCREAT | GB_UNMAPPED);
if (bp != NULL) {
panic("zombie2 %jd %ju %ju",
(intmax_t)bp->b_lblkno, (uintmax_t)bp->b_blkno,
(uintmax_t)fsbtodb(fs, *blkp));
}
lbns_remfree++;
#endif
ffs_blkfree(ump, fs, ump->um_devvp, *blkp, fs->fs_bsize,
ip->i_number, vp->v_type, NULL, SINGLETON_KEY);
}
return (error);
}
Index: head/sys/ufs/ffs/ffs_extern.h
===================================================================
--- head/sys/ufs/ffs/ffs_extern.h (revision 361490)
+++ head/sys/ufs/ffs/ffs_extern.h (revision 361491)
@@ -1,236 +1,242 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1991, 1993, 1994
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_extern.h 8.6 (Berkeley) 3/30/95
* $FreeBSD$
*/
#ifndef _UFS_FFS_EXTERN_H
#define _UFS_FFS_EXTERN_H
#ifndef _KERNEL
#error "No user-serving parts inside"
#else
struct buf;
struct cg;
struct fid;
struct fs;
struct inode;
struct malloc_type;
struct mount;
struct thread;
struct sockaddr;
struct statfs;
struct ucred;
struct vnode;
struct vop_fsync_args;
struct vop_reallocblks_args;
struct workhead;
int ffs_alloc(struct inode *, ufs2_daddr_t, ufs2_daddr_t, int, int,
struct ucred *, ufs2_daddr_t *);
int ffs_balloc_ufs1(struct vnode *a_vp, off_t a_startoffset, int a_size,
struct ucred *a_cred, int a_flags, struct buf **a_bpp);
int ffs_balloc_ufs2(struct vnode *a_vp, off_t a_startoffset, int a_size,
struct ucred *a_cred, int a_flags, struct buf **a_bpp);
int ffs_blkatoff(struct vnode *, off_t, char **, struct buf **);
void ffs_blkfree(struct ufsmount *, struct fs *, struct vnode *,
ufs2_daddr_t, long, ino_t, enum vtype, struct workhead *, u_long);
ufs2_daddr_t ffs_blkpref_ufs1(struct inode *, ufs_lbn_t, int, ufs1_daddr_t *);
ufs2_daddr_t ffs_blkpref_ufs2(struct inode *, ufs_lbn_t, int, ufs2_daddr_t *);
void ffs_blkrelease_finish(struct ufsmount *, u_long);
u_long ffs_blkrelease_start(struct ufsmount *, struct vnode *, ino_t);
uint32_t ffs_calc_sbhash(struct fs *);
int ffs_check_blkno(struct mount *, ino_t, ufs2_daddr_t, int);
int ffs_checkfreefile(struct fs *, struct vnode *, ino_t);
void ffs_clrblock(struct fs *, u_char *, ufs1_daddr_t);
void ffs_clusteracct(struct fs *, struct cg *, ufs1_daddr_t, int);
void ffs_bdflush(struct bufobj *, struct buf *);
int ffs_copyonwrite(struct vnode *, struct buf *);
int ffs_flushfiles(struct mount *, int, struct thread *);
void ffs_fragacct(struct fs *, int, int32_t [], int);
int ffs_freefile(struct ufsmount *, struct fs *, struct vnode *, ino_t,
int, struct workhead *);
void ffs_fserr(struct fs *, ino_t, char *);
int ffs_getcg(struct fs *, struct vnode *, u_int, int, struct buf **,
struct cg **);
int ffs_isblock(struct fs *, u_char *, ufs1_daddr_t);
int ffs_isfreeblock(struct fs *, u_char *, ufs1_daddr_t);
int ffs_load_inode(struct buf *, struct inode *, struct fs *, ino_t);
void ffs_oldfscompat_write(struct fs *, struct ufsmount *);
int ffs_own_mount(const struct mount *mp);
int ffs_reallocblks(struct vop_reallocblks_args *);
int ffs_realloccg(struct inode *, ufs2_daddr_t, ufs2_daddr_t,
ufs2_daddr_t, int, int, int, struct ucred *, struct buf **);
int ffs_reload(struct mount *, struct thread *, int);
int ffs_sbget(void *, struct fs **, off_t, struct malloc_type *,
int (*)(void *, off_t, void **, int));
int ffs_sbput(void *, struct fs *, off_t, int (*)(void *, off_t, void *,
int));
int ffs_sbupdate(struct ufsmount *, int, int);
void ffs_setblock(struct fs *, u_char *, ufs1_daddr_t);
int ffs_snapblkfree(struct fs *, struct vnode *, ufs2_daddr_t, long, ino_t,
enum vtype, struct workhead *);
void ffs_snapremove(struct vnode *vp);
int ffs_snapshot(struct mount *mp, char *snapfile);
void ffs_snapshot_mount(struct mount *mp);
void ffs_snapshot_unmount(struct mount *mp);
void ffs_susp_initialize(void);
void ffs_susp_uninitialize(void);
void ffs_sync_snap(struct mount *, int);
int ffs_syncvnode(struct vnode *vp, int waitfor, int flags);
int ffs_truncate(struct vnode *, off_t, int, struct ucred *);
int ffs_update(struct vnode *, int);
void ffs_update_dinode_ckhash(struct fs *, struct ufs2_dinode *);
int ffs_verify_dinode_ckhash(struct fs *, struct ufs2_dinode *);
int ffs_valloc(struct vnode *, int, struct ucred *, struct vnode **);
int ffs_vfree(struct vnode *, ino_t, int);
vfs_vget_t ffs_vget;
int ffs_vgetf(struct mount *, ino_t, int, struct vnode **, int);
void process_deferred_inactive(struct mount *mp);
+int ffs_fsfail_cleanup(struct ufsmount *, int);
+int ffs_fsfail_cleanup_locked(struct ufsmount *, int);
+int ffs_breadz(struct ufsmount *, struct vnode *, daddr_t, daddr_t, int,
+ daddr_t *, int *, int, struct ucred *, int, void (*)(struct buf *),
+ struct buf **);
/*
* Flags to ffs_vgetf
*/
#define FFSV_FORCEINSMQ 0x0001
#define FFSV_REPLACE 0x0002
/*
* Flags to ffs_reload
*/
#define FFSR_FORCE 0x0001
#define FFSR_UNSUSPEND 0x0002
/*
* Request standard superblock location in ffs_sbget
*/
#define STDSB -1 /* Fail if check-hash is bad */
#define STDSB_NOHASHFAIL -2 /* Ignore check-hash failure */
/*
* Definitions for TRIM interface
*
* Special keys and recommended hash table size
*/
#define NOTRIM_KEY 1 /* never written, so don't call trim for it */
#define SINGLETON_KEY 2 /* only block being freed, so trim it now */
#define FIRST_VALID_KEY 3 /* first valid key describing a block range */
#define MAXTRIMIO 1024 /* maximum expected outstanding trim requests */
extern struct vop_vector ffs_vnodeops1;
extern struct vop_vector ffs_fifoops1;
extern struct vop_vector ffs_vnodeops2;
extern struct vop_vector ffs_fifoops2;
/*
* Soft update function prototypes.
*/
int softdep_check_suspend(struct mount *, struct vnode *,
int, int, int, int);
void softdep_get_depcounts(struct mount *, int *, int *);
void softdep_initialize(void);
void softdep_uninitialize(void);
int softdep_mount(struct vnode *, struct mount *, struct fs *,
struct ucred *);
void softdep_unmount(struct mount *);
+void softdep_handle_error(struct buf *);
int softdep_move_dependencies(struct buf *, struct buf *);
int softdep_flushworklist(struct mount *, int *, struct thread *);
int softdep_flushfiles(struct mount *, int, struct thread *);
void softdep_update_inodeblock(struct inode *, struct buf *, int);
void softdep_load_inodeblock(struct inode *);
void softdep_freefile(struct vnode *, ino_t, int);
int softdep_request_cleanup(struct fs *, struct vnode *,
struct ucred *, int);
void softdep_setup_freeblocks(struct inode *, off_t, int);
void softdep_setup_inomapdep(struct buf *, struct inode *, ino_t, int);
void softdep_setup_blkmapdep(struct buf *, struct mount *, ufs2_daddr_t,
int, int);
void softdep_setup_allocdirect(struct inode *, ufs_lbn_t, ufs2_daddr_t,
ufs2_daddr_t, long, long, struct buf *);
void softdep_setup_allocext(struct inode *, ufs_lbn_t, ufs2_daddr_t,
ufs2_daddr_t, long, long, struct buf *);
void softdep_setup_allocindir_meta(struct buf *, struct inode *,
struct buf *, int, ufs2_daddr_t);
void softdep_setup_allocindir_page(struct inode *, ufs_lbn_t,
struct buf *, int, ufs2_daddr_t, ufs2_daddr_t, struct buf *);
void softdep_setup_blkfree(struct mount *, struct buf *, ufs2_daddr_t, int,
struct workhead *);
void softdep_setup_inofree(struct mount *, struct buf *, ino_t,
struct workhead *);
void softdep_setup_sbupdate(struct ufsmount *, struct fs *, struct buf *);
void softdep_fsync_mountdev(struct vnode *);
int softdep_sync_metadata(struct vnode *);
int softdep_sync_buf(struct vnode *, struct buf *, int);
int softdep_fsync(struct vnode *);
int softdep_prealloc(struct vnode *, int);
int softdep_journal_lookup(struct mount *, struct vnode **);
void softdep_journal_freeblocks(struct inode *, struct ucred *, off_t, int);
void softdep_journal_fsync(struct inode *);
void softdep_buf_append(struct buf *, struct workhead *);
void softdep_inode_append(struct inode *, struct ucred *, struct workhead *);
void softdep_freework(struct workhead *);
/*
* Things to request flushing in softdep_request_cleanup()
*/
#define FLUSH_INODES 1
#define FLUSH_INODES_WAIT 2
#define FLUSH_BLOCKS 3
#define FLUSH_BLOCKS_WAIT 4
/*
* Flag to ffs_syncvnode() to request flushing of data only,
* but skip the ffs_update() on the inode itself. Used to avoid
* deadlock when flushing snapshot inodes while holding snaplk.
*/
#define NO_INO_UPDT 0x00000001
/*
* Request data sync only from ffs_syncvnode(), not touching even more
* metadata than NO_INO_UPDT.
*/
#define DATA_ONLY 0x00000002
int ffs_rdonly(struct inode *);
TAILQ_HEAD(snaphead, inode);
struct snapdata {
LIST_ENTRY(snapdata) sn_link;
struct snaphead sn_head;
daddr_t sn_listsize;
daddr_t *sn_blklist;
struct lock sn_lock;
};
#endif /* _KERNEL */
#endif /* !_UFS_FFS_EXTERN_H */
Index: head/sys/ufs/ffs/ffs_inode.c
===================================================================
--- head/sys/ufs/ffs/ffs_inode.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_inode.c (revision 361491)
@@ -1,771 +1,774 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_inode.c 8.13 (Berkeley) 4/21/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_quota.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/random.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/stat.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
static int ffs_indirtrunc(struct inode *, ufs2_daddr_t, ufs2_daddr_t,
ufs2_daddr_t, int, ufs2_daddr_t *);
/*
* Update the access, modified, and inode change times as specified by the
* IN_ACCESS, IN_UPDATE, and IN_CHANGE flags respectively. Write the inode
* to disk if the IN_MODIFIED flag is set (it may be set initially, or by
* the timestamp update). The IN_LAZYMOD flag is set to force a write
* later if not now. The IN_LAZYACCESS is set instead of IN_MODIFIED if the fs
* is currently being suspended (or is suspended) and vnode has been accessed.
* If we write now, then clear IN_MODIFIED, IN_LAZYACCESS and IN_LAZYMOD to
* reflect the presumably successful write, and if waitfor is set, then wait
* for the write to complete.
*/
int
ffs_update(vp, waitfor)
struct vnode *vp;
int waitfor;
{
struct fs *fs;
struct buf *bp;
struct inode *ip;
+ daddr_t bn;
int flags, error;
ASSERT_VOP_ELOCKED(vp, "ffs_update");
ufs_itimes(vp);
ip = VTOI(vp);
if ((ip->i_flag & IN_MODIFIED) == 0 && waitfor == 0)
return (0);
ip->i_flag &= ~(IN_LAZYACCESS | IN_LAZYMOD | IN_MODIFIED);
fs = ITOFS(ip);
if (fs->fs_ronly && ITOUMP(ip)->um_fsckpid == 0)
return (0);
/*
* If we are updating a snapshot and another process is currently
* writing the buffer containing the inode for this snapshot then
* a deadlock can occur when it tries to check the snapshot to see
* if that block needs to be copied. Thus when updating a snapshot
* we check to see if the buffer is already locked, and if it is
* we drop the snapshot lock until the buffer has been written
* and is available to us. We have to grab a reference to the
* snapshot vnode to prevent it from being removed while we are
* waiting for the buffer.
*/
flags = 0;
if (IS_SNAPSHOT(ip))
flags = GB_LOCK_NOWAIT;
loop:
- error = bread_gb(ITODEVVP(ip),
- fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
- (int) fs->fs_bsize, NOCRED, flags, &bp);
+ bn = fsbtodb(fs, ino_to_fsba(fs, ip->i_number));
+ error = ffs_breadz(VFSTOUFS(vp->v_mount), ITODEVVP(ip), bn, bn,
+ (int) fs->fs_bsize, NULL, NULL, 0, NOCRED, flags, NULL, &bp);
if (error != 0) {
if (error != EBUSY)
return (error);
KASSERT((IS_SNAPSHOT(ip)), ("EBUSY from non-snapshot"));
/*
* Wait for our inode block to become available.
*
* Hold a reference to the vnode to protect against
* ffs_snapgone(). Since we hold a reference, it can only
* get reclaimed (VIRF_DOOMED flag) in a forcible downgrade
* or unmount. For an unmount, the entire filesystem will be
* gone, so we cannot attempt to touch anything associated
* with it while the vnode is unlocked; all we can do is
* pause briefly and try again. If when we relock the vnode
* we discover that it has been reclaimed, updating it is no
* longer necessary and we can just return an error.
*/
vref(vp);
VOP_UNLOCK(vp);
pause("ffsupd", 1);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
vrele(vp);
if (VN_IS_DOOMED(vp))
return (ENOENT);
goto loop;
}
if (DOINGSOFTDEP(vp))
softdep_update_inodeblock(ip, bp, waitfor);
else if (ip->i_effnlink != ip->i_nlink)
panic("ffs_update: bad link cnt");
if (I_IS_UFS1(ip)) {
*((struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1;
/*
* XXX: FIX? The entropy here is desirable,
* but the harvesting may be expensive
*/
random_harvest_queue(&(ip->i_din1), sizeof(ip->i_din1), RANDOM_FS_ATIME);
} else {
ffs_update_dinode_ckhash(fs, ip->i_din2);
*((struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2;
/*
* XXX: FIX? The entropy here is desirable,
* but the harvesting may be expensive
*/
random_harvest_queue(&(ip->i_din2), sizeof(ip->i_din2), RANDOM_FS_ATIME);
}
- if (waitfor)
+ if (waitfor) {
error = bwrite(bp);
- else if (vm_page_count_severe() || buf_dirty_count_severe()) {
+ if (ffs_fsfail_cleanup(VFSTOUFS(vp->v_mount), error))
+ error = 0;
+ } else if (vm_page_count_severe() || buf_dirty_count_severe()) {
bawrite(bp);
error = 0;
} else {
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
error = 0;
}
return (error);
}
#define SINGLE 0 /* index of single indirect block */
#define DOUBLE 1 /* index of double indirect block */
#define TRIPLE 2 /* index of triple indirect block */
/*
* Truncate the inode ip to at most length size, freeing the
* disk blocks.
*/
int
ffs_truncate(vp, length, flags, cred)
struct vnode *vp;
off_t length;
int flags;
struct ucred *cred;
{
struct inode *ip;
ufs2_daddr_t bn, lbn, lastblock, lastiblock[UFS_NIADDR];
ufs2_daddr_t indir_lbn[UFS_NIADDR], oldblks[UFS_NDADDR + UFS_NIADDR];
ufs2_daddr_t newblks[UFS_NDADDR + UFS_NIADDR];
ufs2_daddr_t count, blocksreleased = 0, datablocks, blkno;
struct bufobj *bo;
struct fs *fs;
struct buf *bp;
struct ufsmount *ump;
int softdeptrunc, journaltrunc;
int needextclean, extblocks;
int offset, size, level, nblocks;
int i, error, allerror, indiroff, waitforupdate;
u_long key;
off_t osize;
ip = VTOI(vp);
ump = VFSTOUFS(vp->v_mount);
fs = ump->um_fs;
bo = &vp->v_bufobj;
ASSERT_VOP_LOCKED(vp, "ffs_truncate");
if (length < 0)
return (EINVAL);
if (length > fs->fs_maxfilesize)
return (EFBIG);
#ifdef QUOTA
error = getinoquota(ip);
if (error)
return (error);
#endif
/*
* Historically clients did not have to specify which data
* they were truncating. So, if not specified, we assume
* traditional behavior, e.g., just the normal data.
*/
if ((flags & (IO_EXT | IO_NORMAL)) == 0)
flags |= IO_NORMAL;
if (!DOINGSOFTDEP(vp) && !DOINGASYNC(vp))
flags |= IO_SYNC;
waitforupdate = (flags & IO_SYNC) != 0 || !DOINGASYNC(vp);
/*
* If we are truncating the extended-attributes, and cannot
* do it with soft updates, then do it slowly here. If we are
* truncating both the extended attributes and the file contents
* (e.g., the file is being unlinked), then pick it off with
* soft updates below.
*/
allerror = 0;
needextclean = 0;
softdeptrunc = 0;
journaltrunc = DOINGSUJ(vp);
journaltrunc = 0; /* XXX temp patch until bug found */
if (journaltrunc == 0 && DOINGSOFTDEP(vp) && length == 0)
softdeptrunc = !softdep_slowdown(vp);
extblocks = 0;
datablocks = DIP(ip, i_blocks);
if (fs->fs_magic == FS_UFS2_MAGIC && ip->i_din2->di_extsize > 0) {
extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
datablocks -= extblocks;
}
if ((flags & IO_EXT) && extblocks > 0) {
if (length != 0)
panic("ffs_truncate: partial trunc of extdata");
if (softdeptrunc || journaltrunc) {
if ((flags & IO_NORMAL) == 0)
goto extclean;
needextclean = 1;
} else {
if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0)
return (error);
#ifdef QUOTA
(void) chkdq(ip, -extblocks, NOCRED, FORCE);
#endif
vinvalbuf(vp, V_ALT, 0, 0);
vn_pages_remove(vp,
OFF_TO_IDX(lblktosize(fs, -extblocks)), 0);
osize = ip->i_din2->di_extsize;
ip->i_din2->di_blocks -= extblocks;
ip->i_din2->di_extsize = 0;
for (i = 0; i < UFS_NXADDR; i++) {
oldblks[i] = ip->i_din2->di_extb[i];
ip->i_din2->di_extb[i] = 0;
}
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if ((error = ffs_update(vp, waitforupdate)))
return (error);
for (i = 0; i < UFS_NXADDR; i++) {
if (oldblks[i] == 0)
continue;
ffs_blkfree(ump, fs, ITODEVVP(ip), oldblks[i],
sblksize(fs, osize, i), ip->i_number,
vp->v_type, NULL, SINGLETON_KEY);
}
}
}
if ((flags & IO_NORMAL) == 0)
return (0);
if (vp->v_type == VLNK &&
(ip->i_size < vp->v_mount->mnt_maxsymlinklen ||
datablocks == 0)) {
#ifdef INVARIANTS
if (length != 0)
panic("ffs_truncate: partial truncate of symlink");
#endif
bzero(SHORTLINK(ip), (u_int)ip->i_size);
ip->i_size = 0;
DIP_SET(ip, i_size, 0);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (needextclean)
goto extclean;
return (ffs_update(vp, waitforupdate));
}
if (ip->i_size == length) {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (needextclean)
goto extclean;
return (ffs_update(vp, 0));
}
if (fs->fs_ronly)
panic("ffs_truncate: read-only filesystem");
if (IS_SNAPSHOT(ip))
ffs_snapremove(vp);
vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
osize = ip->i_size;
/*
* Lengthen the size of the file. We must ensure that the
* last byte of the file is allocated. Since the smallest
* value of osize is 0, length will be at least 1.
*/
if (osize < length) {
vnode_pager_setsize(vp, length);
flags |= BA_CLRBUF;
error = UFS_BALLOC(vp, length - 1, 1, cred, flags, &bp);
if (error) {
vnode_pager_setsize(vp, osize);
return (error);
}
ip->i_size = length;
DIP_SET(ip, i_size, length);
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
if (flags & IO_SYNC)
bwrite(bp);
else if (DOINGASYNC(vp))
bdwrite(bp);
else
bawrite(bp);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
return (ffs_update(vp, waitforupdate));
}
/*
* Lookup block number for a given offset. Zero length files
* have no blocks, so return a blkno of -1.
*/
lbn = lblkno(fs, length - 1);
if (length == 0) {
blkno = -1;
} else if (lbn < UFS_NDADDR) {
blkno = DIP(ip, i_db[lbn]);
} else {
error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn), fs->fs_bsize,
cred, BA_METAONLY, &bp);
if (error)
return (error);
indiroff = (lbn - UFS_NDADDR) % NINDIR(fs);
if (I_IS_UFS1(ip))
blkno = ((ufs1_daddr_t *)(bp->b_data))[indiroff];
else
blkno = ((ufs2_daddr_t *)(bp->b_data))[indiroff];
/*
* If the block number is non-zero, then the indirect block
* must have been previously allocated and need not be written.
* If the block number is zero, then we may have allocated
* the indirect block and hence need to write it out.
*/
if (blkno != 0)
brelse(bp);
else if (flags & IO_SYNC)
bwrite(bp);
else
bdwrite(bp);
}
/*
* If the block number at the new end of the file is zero,
* then we must allocate it to ensure that the last block of
* the file is allocated. Soft updates does not handle this
* case, so here we have to clean up the soft updates data
* structures describing the allocation past the truncation
* point. Finding and deallocating those structures is a lot of
* work. Since partial truncation with a hole at the end occurs
* rarely, we solve the problem by syncing the file so that it
* will have no soft updates data structures left.
*/
if (blkno == 0 && (error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0)
return (error);
if (blkno != 0 && DOINGSOFTDEP(vp)) {
if (softdeptrunc == 0 && journaltrunc == 0) {
/*
* If soft updates cannot handle this truncation,
* clean up soft dependency data structures and
* fall through to the synchronous truncation.
*/
if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0)
return (error);
} else {
flags = IO_NORMAL | (needextclean ? IO_EXT: 0);
if (journaltrunc)
softdep_journal_freeblocks(ip, cred, length,
flags);
else
softdep_setup_freeblocks(ip, length, flags);
ASSERT_VOP_LOCKED(vp, "ffs_truncate1");
if (journaltrunc == 0) {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
error = ffs_update(vp, 0);
}
return (error);
}
}
/*
* Shorten the size of the file. If the last block of the
* shortened file is unallocated, we must allocate it.
* Additionally, if the file is not being truncated to a
* block boundary, the contents of the partial block
* following the end of the file must be zero'ed in
* case it ever becomes accessible again because of
* subsequent file growth. Directories however are not
* zero'ed as they should grow back initialized to empty.
*/
offset = blkoff(fs, length);
if (blkno != 0 && offset == 0) {
ip->i_size = length;
DIP_SET(ip, i_size, length);
} else {
lbn = lblkno(fs, length);
flags |= BA_CLRBUF;
error = UFS_BALLOC(vp, length - 1, 1, cred, flags, &bp);
if (error)
return (error);
/*
* When we are doing soft updates and the UFS_BALLOC
* above fills in a direct block hole with a full sized
* block that will be truncated down to a fragment below,
* we must flush out the block dependency with an FSYNC
* so that we do not get a soft updates inconsistency
* when we create the fragment below.
*/
if (DOINGSOFTDEP(vp) && lbn < UFS_NDADDR &&
fragroundup(fs, blkoff(fs, length)) < fs->fs_bsize &&
(error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0)
return (error);
ip->i_size = length;
DIP_SET(ip, i_size, length);
size = blksize(fs, ip, lbn);
if (vp->v_type != VDIR && offset != 0)
bzero((char *)bp->b_data + offset,
(u_int)(size - offset));
/* Kirk's code has reallocbuf(bp, size, 1) here */
allocbuf(bp, size);
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
if (flags & IO_SYNC)
bwrite(bp);
else if (DOINGASYNC(vp))
bdwrite(bp);
else
bawrite(bp);
}
/*
* Calculate index into inode's block list of
* last direct and indirect blocks (if any)
* which we want to keep. Lastblock is -1 when
* the file is truncated to 0.
*/
lastblock = lblkno(fs, length + fs->fs_bsize - 1) - 1;
lastiblock[SINGLE] = lastblock - UFS_NDADDR;
lastiblock[DOUBLE] = lastiblock[SINGLE] - NINDIR(fs);
lastiblock[TRIPLE] = lastiblock[DOUBLE] - NINDIR(fs) * NINDIR(fs);
nblocks = btodb(fs->fs_bsize);
/*
* Update file and block pointers on disk before we start freeing
* blocks. If we crash before free'ing blocks below, the blocks
* will be returned to the free list. lastiblock values are also
* normalized to -1 for calls to ffs_indirtrunc below.
*/
for (level = TRIPLE; level >= SINGLE; level--) {
oldblks[UFS_NDADDR + level] = DIP(ip, i_ib[level]);
if (lastiblock[level] < 0) {
DIP_SET(ip, i_ib[level], 0);
lastiblock[level] = -1;
}
}
for (i = 0; i < UFS_NDADDR; i++) {
oldblks[i] = DIP(ip, i_db[i]);
if (i > lastblock)
DIP_SET(ip, i_db[i], 0);
}
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allerror = ffs_update(vp, waitforupdate);
/*
* Having written the new inode to disk, save its new configuration
* and put back the old block pointers long enough to process them.
* Note that we save the new block configuration so we can check it
* when we are done.
*/
for (i = 0; i < UFS_NDADDR; i++) {
newblks[i] = DIP(ip, i_db[i]);
DIP_SET(ip, i_db[i], oldblks[i]);
}
for (i = 0; i < UFS_NIADDR; i++) {
newblks[UFS_NDADDR + i] = DIP(ip, i_ib[i]);
DIP_SET(ip, i_ib[i], oldblks[UFS_NDADDR + i]);
}
ip->i_size = osize;
DIP_SET(ip, i_size, osize);
error = vtruncbuf(vp, length, fs->fs_bsize);
if (error && (allerror == 0))
allerror = error;
/*
* Indirect blocks first.
*/
indir_lbn[SINGLE] = -UFS_NDADDR;
indir_lbn[DOUBLE] = indir_lbn[SINGLE] - NINDIR(fs) - 1;
indir_lbn[TRIPLE] = indir_lbn[DOUBLE] - NINDIR(fs) * NINDIR(fs) - 1;
for (level = TRIPLE; level >= SINGLE; level--) {
bn = DIP(ip, i_ib[level]);
if (bn != 0) {
error = ffs_indirtrunc(ip, indir_lbn[level],
fsbtodb(fs, bn), lastiblock[level], level, &count);
if (error)
allerror = error;
blocksreleased += count;
if (lastiblock[level] < 0) {
DIP_SET(ip, i_ib[level], 0);
ffs_blkfree(ump, fs, ump->um_devvp, bn,
fs->fs_bsize, ip->i_number,
vp->v_type, NULL, SINGLETON_KEY);
blocksreleased += nblocks;
}
}
if (lastiblock[level] >= 0)
goto done;
}
/*
* All whole direct blocks or frags.
*/
key = ffs_blkrelease_start(ump, ump->um_devvp, ip->i_number);
for (i = UFS_NDADDR - 1; i > lastblock; i--) {
long bsize;
bn = DIP(ip, i_db[i]);
if (bn == 0)
continue;
DIP_SET(ip, i_db[i], 0);
bsize = blksize(fs, ip, i);
ffs_blkfree(ump, fs, ump->um_devvp, bn, bsize, ip->i_number,
vp->v_type, NULL, key);
blocksreleased += btodb(bsize);
}
ffs_blkrelease_finish(ump, key);
if (lastblock < 0)
goto done;
/*
* Finally, look for a change in size of the
* last direct block; release any frags.
*/
bn = DIP(ip, i_db[lastblock]);
if (bn != 0) {
long oldspace, newspace;
/*
* Calculate amount of space we're giving
* back as old block size minus new block size.
*/
oldspace = blksize(fs, ip, lastblock);
ip->i_size = length;
DIP_SET(ip, i_size, length);
newspace = blksize(fs, ip, lastblock);
if (newspace == 0)
panic("ffs_truncate: newspace");
if (oldspace - newspace > 0) {
/*
* Block number of space to be free'd is
* the old block # plus the number of frags
* required for the storage we're keeping.
*/
bn += numfrags(fs, newspace);
ffs_blkfree(ump, fs, ump->um_devvp, bn,
oldspace - newspace, ip->i_number, vp->v_type,
NULL, SINGLETON_KEY);
blocksreleased += btodb(oldspace - newspace);
}
}
done:
#ifdef INVARIANTS
for (level = SINGLE; level <= TRIPLE; level++)
if (newblks[UFS_NDADDR + level] != DIP(ip, i_ib[level]))
panic("ffs_truncate1: level %d newblks %jd != i_ib %jd",
level, (intmax_t)newblks[UFS_NDADDR + level],
(intmax_t)DIP(ip, i_ib[level]));
for (i = 0; i < UFS_NDADDR; i++)
if (newblks[i] != DIP(ip, i_db[i]))
panic("ffs_truncate2: blkno %d newblks %jd != i_db %jd",
i, (intmax_t)newblks[UFS_NDADDR + level],
(intmax_t)DIP(ip, i_ib[level]));
BO_LOCK(bo);
if (length == 0 &&
(fs->fs_magic != FS_UFS2_MAGIC || ip->i_din2->di_extsize == 0) &&
(bo->bo_dirty.bv_cnt > 0 || bo->bo_clean.bv_cnt > 0))
panic("ffs_truncate3: vp = %p, buffers: dirty = %d, clean = %d",
vp, bo->bo_dirty.bv_cnt, bo->bo_clean.bv_cnt);
BO_UNLOCK(bo);
#endif /* INVARIANTS */
/*
* Put back the real size.
*/
ip->i_size = length;
DIP_SET(ip, i_size, length);
if (DIP(ip, i_blocks) >= blocksreleased)
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - blocksreleased);
else /* sanity */
DIP_SET(ip, i_blocks, 0);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
#ifdef QUOTA
(void) chkdq(ip, -blocksreleased, NOCRED, FORCE);
#endif
return (allerror);
extclean:
if (journaltrunc)
softdep_journal_freeblocks(ip, cred, length, IO_EXT);
else
softdep_setup_freeblocks(ip, length, IO_EXT);
return (ffs_update(vp, waitforupdate));
}
/*
* Release blocks associated with the inode ip and stored in the indirect
* block bn. Blocks are free'd in LIFO order up to (but not including)
* lastbn. If level is greater than SINGLE, the block is an indirect block
* and recursive calls to indirtrunc must be used to cleanse other indirect
* blocks.
*/
static int
ffs_indirtrunc(ip, lbn, dbn, lastbn, level, countp)
struct inode *ip;
ufs2_daddr_t lbn, lastbn;
ufs2_daddr_t dbn;
int level;
ufs2_daddr_t *countp;
{
struct buf *bp;
struct fs *fs;
struct ufsmount *ump;
struct vnode *vp;
caddr_t copy = NULL;
u_long key;
int i, nblocks, error = 0, allerror = 0;
ufs2_daddr_t nb, nlbn, last;
ufs2_daddr_t blkcount, factor, blocksreleased = 0;
ufs1_daddr_t *bap1 = NULL;
ufs2_daddr_t *bap2 = NULL;
#define BAP(ip, i) (I_IS_UFS1(ip) ? bap1[i] : bap2[i])
fs = ITOFS(ip);
ump = ITOUMP(ip);
/*
* Calculate index in current block of last
* block to be kept. -1 indicates the entire
* block so we need not calculate the index.
*/
factor = lbn_offset(fs, level);
last = lastbn;
if (lastbn > 0)
last /= factor;
nblocks = btodb(fs->fs_bsize);
/*
* Get buffer of block pointers, zero those entries corresponding
* to blocks to be free'd, and update on disk copy first. Since
* double(triple) indirect before single(double) indirect, calls
* to VOP_BMAP() on these blocks will fail. However, we already
* have the on-disk address, so we just pass it to bread() instead
* of having bread() attempt to calculate it using VOP_BMAP().
*/
vp = ITOV(ip);
- error = breadn_flags(vp, lbn, dbn, (int)fs->fs_bsize, NULL, NULL, 0,
+ error = ffs_breadz(ump, vp, lbn, dbn, (int)fs->fs_bsize, NULL, NULL, 0,
NOCRED, 0, NULL, &bp);
if (error) {
*countp = 0;
return (error);
}
if (I_IS_UFS1(ip))
bap1 = (ufs1_daddr_t *)bp->b_data;
else
bap2 = (ufs2_daddr_t *)bp->b_data;
if (lastbn != -1) {
copy = malloc(fs->fs_bsize, M_TEMP, M_WAITOK);
bcopy((caddr_t)bp->b_data, copy, (u_int)fs->fs_bsize);
for (i = last + 1; i < NINDIR(fs); i++)
if (I_IS_UFS1(ip))
bap1[i] = 0;
else
bap2[i] = 0;
if (DOINGASYNC(vp)) {
bdwrite(bp);
} else {
error = bwrite(bp);
if (error)
allerror = error;
}
if (I_IS_UFS1(ip))
bap1 = (ufs1_daddr_t *)copy;
else
bap2 = (ufs2_daddr_t *)copy;
}
/*
* Recursively free totally unused blocks.
*/
key = ffs_blkrelease_start(ump, ITODEVVP(ip), ip->i_number);
for (i = NINDIR(fs) - 1, nlbn = lbn + 1 - i * factor; i > last;
i--, nlbn += factor) {
nb = BAP(ip, i);
if (nb == 0)
continue;
if (level > SINGLE) {
if ((error = ffs_indirtrunc(ip, nlbn, fsbtodb(fs, nb),
(ufs2_daddr_t)-1, level - 1, &blkcount)) != 0)
allerror = error;
blocksreleased += blkcount;
}
ffs_blkfree(ump, fs, ITODEVVP(ip), nb, fs->fs_bsize,
ip->i_number, vp->v_type, NULL, key);
blocksreleased += nblocks;
}
ffs_blkrelease_finish(ump, key);
/*
* Recursively free last partial block.
*/
if (level > SINGLE && lastbn >= 0) {
last = lastbn % factor;
nb = BAP(ip, i);
if (nb != 0) {
error = ffs_indirtrunc(ip, nlbn, fsbtodb(fs, nb),
last, level - 1, &blkcount);
if (error)
allerror = error;
blocksreleased += blkcount;
}
}
if (copy != NULL) {
free(copy, M_TEMP);
} else {
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
}
*countp = blocksreleased;
return (allerror);
}
int
ffs_rdonly(struct inode *ip)
{
return (ITOFS(ip)->fs_ronly != 0);
}
Index: head/sys/ufs/ffs/ffs_softdep.c
===================================================================
--- head/sys/ufs/ffs/ffs_softdep.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_softdep.c (revision 361491)
@@ -1,14771 +1,14823 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright 1998, 2000 Marshall Kirk McKusick.
* Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org>
* All rights reserved.
*
* The soft updates code is derived from the appendix of a University
* of Michigan technical report (Gregory R. Ganger and Yale N. Patt,
* "Soft Updates: A Solution to the Metadata Update Problem in File
* Systems", CSE-TR-254-95, August 1995).
*
* Further information about soft updates can be obtained from:
*
* Marshall Kirk McKusick http://www.mckusick.com/softdep/
* 1614 Oxford Street mckusick@mckusick.com
* Berkeley, CA 94709-1608 +1-510-843-9542
* USA
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
* USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ffs.h"
#include "opt_quota.h"
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/kdb.h>
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/rwlock.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/vnode.h>
#include <sys/conf.h>
#include <ufs/ufs/dir.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/softdep.h>
#include <ufs/ffs/ffs_extern.h>
#include <ufs/ufs/ufs_extern.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <geom/geom.h>
#include <geom/geom_vfs.h>
#include <ddb/ddb.h>
#define KTR_SUJ 0 /* Define to KTR_SPARE. */
#ifndef SOFTUPDATES
int
softdep_flushfiles(oldmnt, flags, td)
struct mount *oldmnt;
int flags;
struct thread *td;
{
panic("softdep_flushfiles called");
}
int
softdep_mount(devvp, mp, fs, cred)
struct vnode *devvp;
struct mount *mp;
struct fs *fs;
struct ucred *cred;
{
return (0);
}
void
softdep_initialize()
{
return;
}
void
softdep_uninitialize()
{
return;
}
void
softdep_unmount(mp)
struct mount *mp;
{
panic("softdep_unmount called");
}
void
softdep_setup_sbupdate(ump, fs, bp)
struct ufsmount *ump;
struct fs *fs;
struct buf *bp;
{
panic("softdep_setup_sbupdate called");
}
void
softdep_setup_inomapdep(bp, ip, newinum, mode)
struct buf *bp;
struct inode *ip;
ino_t newinum;
int mode;
{
panic("softdep_setup_inomapdep called");
}
void
softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags)
struct buf *bp;
struct mount *mp;
ufs2_daddr_t newblkno;
int frags;
int oldfrags;
{
panic("softdep_setup_blkmapdep called");
}
void
softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
struct inode *ip;
ufs_lbn_t lbn;
ufs2_daddr_t newblkno;
ufs2_daddr_t oldblkno;
long newsize;
long oldsize;
struct buf *bp;
{
panic("softdep_setup_allocdirect called");
}
void
softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp)
struct inode *ip;
ufs_lbn_t lbn;
ufs2_daddr_t newblkno;
ufs2_daddr_t oldblkno;
long newsize;
long oldsize;
struct buf *bp;
{
panic("softdep_setup_allocext called");
}
void
softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
struct inode *ip;
ufs_lbn_t lbn;
struct buf *bp;
int ptrno;
ufs2_daddr_t newblkno;
ufs2_daddr_t oldblkno;
struct buf *nbp;
{
panic("softdep_setup_allocindir_page called");
}
void
softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
struct buf *nbp;
struct inode *ip;
struct buf *bp;
int ptrno;
ufs2_daddr_t newblkno;
{
panic("softdep_setup_allocindir_meta called");
}
void
softdep_journal_freeblocks(ip, cred, length, flags)
struct inode *ip;
struct ucred *cred;
off_t length;
int flags;
{
panic("softdep_journal_freeblocks called");
}
void
softdep_journal_fsync(ip)
struct inode *ip;
{
panic("softdep_journal_fsync called");
}
void
softdep_setup_freeblocks(ip, length, flags)
struct inode *ip;
off_t length;
int flags;
{
panic("softdep_setup_freeblocks called");
}
void
softdep_freefile(pvp, ino, mode)
struct vnode *pvp;
ino_t ino;
int mode;
{
panic("softdep_freefile called");
}
int
softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
struct buf *bp;
struct inode *dp;
off_t diroffset;
ino_t newinum;
struct buf *newdirbp;
int isnewblk;
{
panic("softdep_setup_directory_add called");
}
void
softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize)
struct buf *bp;
struct inode *dp;
caddr_t base;
caddr_t oldloc;
caddr_t newloc;
int entrysize;
{
panic("softdep_change_directoryentry_offset called");
}
void
softdep_setup_remove(bp, dp, ip, isrmdir)
struct buf *bp;
struct inode *dp;
struct inode *ip;
int isrmdir;
{
panic("softdep_setup_remove called");
}
void
softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
struct buf *bp;
struct inode *dp;
struct inode *ip;
ino_t newinum;
int isrmdir;
{
panic("softdep_setup_directory_change called");
}
void
softdep_setup_blkfree(mp, bp, blkno, frags, wkhd)
struct mount *mp;
struct buf *bp;
ufs2_daddr_t blkno;
int frags;
struct workhead *wkhd;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_inofree(mp, bp, ino, wkhd)
struct mount *mp;
struct buf *bp;
ino_t ino;
struct workhead *wkhd;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_unlink(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_revert_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_rmdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_revert_rmdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_create(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_revert_create(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_mkdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_revert_mkdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
void
softdep_setup_dotdot_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
panic("%s called", __FUNCTION__);
}
int
softdep_prealloc(vp, waitok)
struct vnode *vp;
int waitok;
{
panic("%s called", __FUNCTION__);
}
int
softdep_journal_lookup(mp, vpp)
struct mount *mp;
struct vnode **vpp;
{
return (ENOENT);
}
void
softdep_change_linkcnt(ip)
struct inode *ip;
{
panic("softdep_change_linkcnt called");
}
void
softdep_load_inodeblock(ip)
struct inode *ip;
{
panic("softdep_load_inodeblock called");
}
void
softdep_update_inodeblock(ip, bp, waitfor)
struct inode *ip;
struct buf *bp;
int waitfor;
{
panic("softdep_update_inodeblock called");
}
int
softdep_fsync(vp)
struct vnode *vp; /* the "in_core" copy of the inode */
{
return (0);
}
void
softdep_fsync_mountdev(vp)
struct vnode *vp;
{
return;
}
int
softdep_flushworklist(oldmnt, countp, td)
struct mount *oldmnt;
int *countp;
struct thread *td;
{
*countp = 0;
return (0);
}
int
softdep_sync_metadata(struct vnode *vp)
{
panic("softdep_sync_metadata called");
}
int
softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor)
{
panic("softdep_sync_buf called");
}
int
softdep_slowdown(vp)
struct vnode *vp;
{
panic("softdep_slowdown called");
}
int
softdep_request_cleanup(fs, vp, cred, resource)
struct fs *fs;
struct vnode *vp;
struct ucred *cred;
int resource;
{
return (0);
}
int
softdep_check_suspend(struct mount *mp,
struct vnode *devvp,
int softdep_depcnt,
int softdep_accdepcnt,
int secondary_writes,
int secondary_accwrites)
{
struct bufobj *bo;
int error;
(void) softdep_depcnt,
(void) softdep_accdepcnt;
bo = &devvp->v_bufobj;
ASSERT_BO_WLOCKED(bo);
MNT_ILOCK(mp);
while (mp->mnt_secondary_writes != 0) {
BO_UNLOCK(bo);
msleep(&mp->mnt_secondary_writes, MNT_MTX(mp),
(PUSER - 1) | PDROP, "secwr", 0);
BO_LOCK(bo);
MNT_ILOCK(mp);
}
/*
* Reasons for needing more work before suspend:
* - Dirty buffers on devvp.
* - Secondary writes occurred after start of vnode sync loop
*/
error = 0;
if (bo->bo_numoutput > 0 ||
bo->bo_dirty.bv_cnt > 0 ||
secondary_writes != 0 ||
mp->mnt_secondary_writes != 0 ||
secondary_accwrites != mp->mnt_secondary_accwrites)
error = EAGAIN;
BO_UNLOCK(bo);
return (error);
}
void
softdep_get_depcounts(struct mount *mp,
int *softdepactivep,
int *softdepactiveaccp)
{
(void) mp;
*softdepactivep = 0;
*softdepactiveaccp = 0;
}
void
softdep_buf_append(bp, wkhd)
struct buf *bp;
struct workhead *wkhd;
{
panic("softdep_buf_appendwork called");
}
void
softdep_inode_append(ip, cred, wkhd)
struct inode *ip;
struct ucred *cred;
struct workhead *wkhd;
{
panic("softdep_inode_appendwork called");
}
void
softdep_freework(wkhd)
struct workhead *wkhd;
{
panic("softdep_freework called");
}
#else
FEATURE(softupdates, "FFS soft-updates support");
static SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"soft updates stats");
static SYSCTL_NODE(_debug_softdep, OID_AUTO, total,
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"total dependencies allocated");
static SYSCTL_NODE(_debug_softdep, OID_AUTO, highuse,
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"high use dependencies allocated");
static SYSCTL_NODE(_debug_softdep, OID_AUTO, current,
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"current dependencies allocated");
static SYSCTL_NODE(_debug_softdep, OID_AUTO, write,
CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"current dependencies written");
unsigned long dep_current[D_LAST + 1];
unsigned long dep_highuse[D_LAST + 1];
unsigned long dep_total[D_LAST + 1];
unsigned long dep_write[D_LAST + 1];
#define SOFTDEP_TYPE(type, str, long) \
static MALLOC_DEFINE(M_ ## type, #str, long); \
SYSCTL_ULONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \
&dep_total[D_ ## type], 0, ""); \
SYSCTL_ULONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \
&dep_current[D_ ## type], 0, ""); \
SYSCTL_ULONG(_debug_softdep_highuse, OID_AUTO, str, CTLFLAG_RD, \
&dep_highuse[D_ ## type], 0, ""); \
SYSCTL_ULONG(_debug_softdep_write, OID_AUTO, str, CTLFLAG_RD, \
&dep_write[D_ ## type], 0, "");
SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies");
SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies");
SOFTDEP_TYPE(BMSAFEMAP, bmsafemap,
"Block or frag allocated from cyl group map");
SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency");
SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode");
SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies");
SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block");
SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode");
SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode");
SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated");
SOFTDEP_TYPE(DIRADD, diradd, "New directory entry");
SOFTDEP_TYPE(MKDIR, mkdir, "New directory");
SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted");
SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block");
SOFTDEP_TYPE(FREEWORK, freework, "free an inode block");
SOFTDEP_TYPE(FREEDEP, freedep, "track a block free");
SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add");
SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove");
SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move");
SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block");
SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block");
SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag");
SOFTDEP_TYPE(JSEG, jseg, "Journal segment");
SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete");
SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency");
SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation");
SOFTDEP_TYPE(JFSYNC, jfsync, "Journal fsync complete");
static MALLOC_DEFINE(M_SENTINEL, "sentinel", "Worklist sentinel");
static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes");
static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations");
static MALLOC_DEFINE(M_MOUNTDATA, "softdep", "Softdep per-mount data");
#define M_SOFTDEP_FLAGS (M_WAITOK)
/*
* translate from workitem type to memory type
* MUST match the defines above, such that memtype[D_XXX] == M_XXX
*/
static struct malloc_type *memtype[] = {
NULL,
M_PAGEDEP,
M_INODEDEP,
M_BMSAFEMAP,
M_NEWBLK,
M_ALLOCDIRECT,
M_INDIRDEP,
M_ALLOCINDIR,
M_FREEFRAG,
M_FREEBLKS,
M_FREEFILE,
M_DIRADD,
M_MKDIR,
M_DIRREM,
M_NEWDIRBLK,
M_FREEWORK,
M_FREEDEP,
M_JADDREF,
M_JREMREF,
M_JMVREF,
M_JNEWBLK,
M_JFREEBLK,
M_JFREEFRAG,
M_JSEG,
M_JSEGDEP,
M_SBDEP,
M_JTRUNC,
M_JFSYNC,
M_SENTINEL
};
#define DtoM(type) (memtype[type])
/*
* Names of malloc types.
*/
#define TYPENAME(type) \
((unsigned)(type) <= D_LAST && (unsigned)(type) >= D_FIRST ? \
memtype[type]->ks_shortdesc : "???")
/*
* End system adaptation definitions.
*/
#define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino)
#define DOT_OFFSET offsetof(struct dirtemplate, dot_ino)
/*
* Internal function prototypes.
*/
static void check_clear_deps(struct mount *);
static void softdep_error(char *, int);
static int softdep_process_worklist(struct mount *, int);
static int softdep_waitidle(struct mount *, int);
static void drain_output(struct vnode *);
static struct buf *getdirtybuf(struct buf *, struct rwlock *, int);
static int check_inodedep_free(struct inodedep *);
static void clear_remove(struct mount *);
static void clear_inodedeps(struct mount *);
static void unlinked_inodedep(struct mount *, struct inodedep *);
static void clear_unlinked_inodedep(struct inodedep *);
static struct inodedep *first_unlinked_inodedep(struct ufsmount *);
static int flush_pagedep_deps(struct vnode *, struct mount *,
struct diraddhd *);
static int free_pagedep(struct pagedep *);
static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t);
static int flush_inodedep_deps(struct vnode *, struct mount *, ino_t);
static int flush_deplist(struct allocdirectlst *, int, int *);
static int sync_cgs(struct mount *, int);
static int handle_written_filepage(struct pagedep *, struct buf *, int);
static int handle_written_sbdep(struct sbdep *, struct buf *);
static void initiate_write_sbdep(struct sbdep *);
static void diradd_inode_written(struct diradd *, struct inodedep *);
static int handle_written_indirdep(struct indirdep *, struct buf *,
struct buf**, int);
static int handle_written_inodeblock(struct inodedep *, struct buf *, int);
static int jnewblk_rollforward(struct jnewblk *, struct fs *, struct cg *,
uint8_t *);
static int handle_written_bmsafemap(struct bmsafemap *, struct buf *, int);
static void handle_written_jaddref(struct jaddref *);
static void handle_written_jremref(struct jremref *);
static void handle_written_jseg(struct jseg *, struct buf *);
static void handle_written_jnewblk(struct jnewblk *);
static void handle_written_jblkdep(struct jblkdep *);
static void handle_written_jfreefrag(struct jfreefrag *);
static void complete_jseg(struct jseg *);
static void complete_jsegs(struct jseg *);
static void jseg_write(struct ufsmount *ump, struct jseg *, uint8_t *);
static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *);
static void jremref_write(struct jremref *, struct jseg *, uint8_t *);
static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *);
static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *);
static void jfsync_write(struct jfsync *, struct jseg *, uint8_t *data);
static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *);
static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *);
static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *);
static inline void inoref_write(struct inoref *, struct jseg *,
struct jrefrec *);
static void handle_allocdirect_partdone(struct allocdirect *,
struct workhead *);
static struct jnewblk *cancel_newblk(struct newblk *, struct worklist *,
struct workhead *);
static void indirdep_complete(struct indirdep *);
static int indirblk_lookup(struct mount *, ufs2_daddr_t);
static void indirblk_insert(struct freework *);
static void indirblk_remove(struct freework *);
static void handle_allocindir_partdone(struct allocindir *);
static void initiate_write_filepage(struct pagedep *, struct buf *);
static void initiate_write_indirdep(struct indirdep*, struct buf *);
static void handle_written_mkdir(struct mkdir *, int);
static int jnewblk_rollback(struct jnewblk *, struct fs *, struct cg *,
uint8_t *);
static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *);
static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *);
static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *);
static void handle_workitem_freefile(struct freefile *);
static int handle_workitem_remove(struct dirrem *, int);
static struct dirrem *newdirrem(struct buf *, struct inode *,
struct inode *, int, struct dirrem **);
static struct indirdep *indirdep_lookup(struct mount *, struct inode *,
struct buf *);
static void cancel_indirdep(struct indirdep *, struct buf *,
struct freeblks *);
static void free_indirdep(struct indirdep *);
static void free_diradd(struct diradd *, struct workhead *);
static void merge_diradd(struct inodedep *, struct diradd *);
static void complete_diradd(struct diradd *);
static struct diradd *diradd_lookup(struct pagedep *, int);
static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *,
struct jremref *);
static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *,
struct jremref *);
static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *,
struct jremref *, struct jremref *);
static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *,
struct jremref *);
static void cancel_allocindir(struct allocindir *, struct buf *bp,
struct freeblks *, int);
static int setup_trunc_indir(struct freeblks *, struct inode *,
ufs_lbn_t, ufs_lbn_t, ufs2_daddr_t);
static void complete_trunc_indir(struct freework *);
static void trunc_indirdep(struct indirdep *, struct freeblks *, struct buf *,
int);
static void complete_mkdir(struct mkdir *);
static void free_newdirblk(struct newdirblk *);
static void free_jremref(struct jremref *);
static void free_jaddref(struct jaddref *);
static void free_jsegdep(struct jsegdep *);
static void free_jsegs(struct jblocks *);
static void rele_jseg(struct jseg *);
static void free_jseg(struct jseg *, struct jblocks *);
static void free_jnewblk(struct jnewblk *);
static void free_jblkdep(struct jblkdep *);
static void free_jfreefrag(struct jfreefrag *);
static void free_freedep(struct freedep *);
static void journal_jremref(struct dirrem *, struct jremref *,
struct inodedep *);
static void cancel_jnewblk(struct jnewblk *, struct workhead *);
static int cancel_jaddref(struct jaddref *, struct inodedep *,
struct workhead *);
static void cancel_jfreefrag(struct jfreefrag *);
static inline void setup_freedirect(struct freeblks *, struct inode *,
int, int);
static inline void setup_freeext(struct freeblks *, struct inode *, int, int);
static inline void setup_freeindir(struct freeblks *, struct inode *, int,
ufs_lbn_t, int);
static inline struct freeblks *newfreeblks(struct mount *, struct inode *);
static void freeblks_free(struct ufsmount *, struct freeblks *, int);
static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t);
static ufs2_daddr_t blkcount(struct fs *, ufs2_daddr_t, off_t);
static int trunc_check_buf(struct buf *, int *, ufs_lbn_t, int, int);
static void trunc_dependencies(struct inode *, struct freeblks *, ufs_lbn_t,
int, int);
static void trunc_pages(struct inode *, off_t, ufs2_daddr_t, int);
static int cancel_pagedep(struct pagedep *, struct freeblks *, int);
static int deallocate_dependencies(struct buf *, struct freeblks *, int);
static void newblk_freefrag(struct newblk*);
static void free_newblk(struct newblk *);
static void cancel_allocdirect(struct allocdirectlst *,
struct allocdirect *, struct freeblks *);
static int check_inode_unwritten(struct inodedep *);
static int free_inodedep(struct inodedep *);
static void freework_freeblock(struct freework *, u_long);
static void freework_enqueue(struct freework *);
static int handle_workitem_freeblocks(struct freeblks *, int);
static int handle_complete_freeblocks(struct freeblks *, int);
static void handle_workitem_indirblk(struct freework *);
static void handle_written_freework(struct freework *);
static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *);
static struct worklist *jnewblk_merge(struct worklist *, struct worklist *,
struct workhead *);
static struct freefrag *setup_allocindir_phase2(struct buf *, struct inode *,
struct inodedep *, struct allocindir *, ufs_lbn_t);
static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t,
ufs2_daddr_t, ufs_lbn_t);
static void handle_workitem_freefrag(struct freefrag *);
static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long,
ufs_lbn_t, u_long);
static void allocdirect_merge(struct allocdirectlst *,
struct allocdirect *, struct allocdirect *);
static struct freefrag *allocindir_merge(struct allocindir *,
struct allocindir *);
static int bmsafemap_find(struct bmsafemap_hashhead *, int,
struct bmsafemap **);
static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *,
int cg, struct bmsafemap *);
static int newblk_find(struct newblk_hashhead *, ufs2_daddr_t, int,
struct newblk **);
static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **);
static int inodedep_find(struct inodedep_hashhead *, ino_t,
struct inodedep **);
static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **);
static int pagedep_lookup(struct mount *, struct buf *bp, ino_t, ufs_lbn_t,
int, struct pagedep **);
static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t,
struct pagedep **);
static void pause_timer(void *);
static int request_cleanup(struct mount *, int);
static int softdep_request_cleanup_flush(struct mount *, struct ufsmount *);
static void schedule_cleanup(struct mount *);
static void softdep_ast_cleanup_proc(struct thread *);
static struct ufsmount *softdep_bp_to_mp(struct buf *bp);
static int process_worklist_item(struct mount *, int, int);
static void process_removes(struct vnode *);
static void process_truncates(struct vnode *);
static void jwork_move(struct workhead *, struct workhead *);
static void jwork_insert(struct workhead *, struct jsegdep *);
static void add_to_worklist(struct worklist *, int);
static void wake_worklist(struct worklist *);
static void wait_worklist(struct worklist *, char *);
static void remove_from_worklist(struct worklist *);
static void softdep_flush(void *);
static void softdep_flushjournal(struct mount *);
static int softdep_speedup(struct ufsmount *);
static void worklist_speedup(struct mount *);
static int journal_mount(struct mount *, struct fs *, struct ucred *);
static void journal_unmount(struct ufsmount *);
static int journal_space(struct ufsmount *, int);
static void journal_suspend(struct ufsmount *);
static int journal_unsuspend(struct ufsmount *ump);
static void softdep_prelink(struct vnode *, struct vnode *);
static void add_to_journal(struct worklist *);
static void remove_from_journal(struct worklist *);
static bool softdep_excess_items(struct ufsmount *, int);
static void softdep_process_journal(struct mount *, struct worklist *, int);
static struct jremref *newjremref(struct dirrem *, struct inode *,
struct inode *ip, off_t, nlink_t);
static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t,
uint16_t);
static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t,
uint16_t);
static inline struct jsegdep *inoref_jseg(struct inoref *);
static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t);
static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t,
ufs2_daddr_t, int);
static void adjust_newfreework(struct freeblks *, int);
static struct jtrunc *newjtrunc(struct freeblks *, off_t, int);
static void move_newblock_dep(struct jaddref *, struct inodedep *);
static void cancel_jfreeblk(struct freeblks *, ufs2_daddr_t);
static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *,
ufs2_daddr_t, long, ufs_lbn_t);
static struct freework *newfreework(struct ufsmount *, struct freeblks *,
struct freework *, ufs_lbn_t, ufs2_daddr_t, int, int, int);
static int jwait(struct worklist *, int);
static struct inodedep *inodedep_lookup_ip(struct inode *);
static int bmsafemap_backgroundwrite(struct bmsafemap *, struct buf *);
static struct freefile *handle_bufwait(struct inodedep *, struct workhead *);
static void handle_jwork(struct workhead *);
static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *,
struct mkdir **);
static struct jblocks *jblocks_create(void);
static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *);
static void jblocks_free(struct jblocks *, struct mount *, int);
static void jblocks_destroy(struct jblocks *);
static void jblocks_add(struct jblocks *, ufs2_daddr_t, int);
/*
* Exported softdep operations.
*/
static void softdep_disk_io_initiation(struct buf *);
static void softdep_disk_write_complete(struct buf *);
static void softdep_deallocate_dependencies(struct buf *);
static int softdep_count_dependencies(struct buf *bp, int);
/*
* Global lock over all of soft updates.
*/
static struct mtx lk;
MTX_SYSINIT(softdep_lock, &lk, "global softdep", MTX_DEF);
#define ACQUIRE_GBLLOCK(lk) mtx_lock(lk)
#define FREE_GBLLOCK(lk) mtx_unlock(lk)
#define GBLLOCK_OWNED(lk) mtx_assert((lk), MA_OWNED)
/*
* Per-filesystem soft-updates locking.
*/
#define LOCK_PTR(ump) (&(ump)->um_softdep->sd_fslock)
#define TRY_ACQUIRE_LOCK(ump) rw_try_wlock(&(ump)->um_softdep->sd_fslock)
#define ACQUIRE_LOCK(ump) rw_wlock(&(ump)->um_softdep->sd_fslock)
#define FREE_LOCK(ump) rw_wunlock(&(ump)->um_softdep->sd_fslock)
#define LOCK_OWNED(ump) rw_assert(&(ump)->um_softdep->sd_fslock, \
RA_WLOCKED)
#define BUF_AREC(bp) lockallowrecurse(&(bp)->b_lock)
#define BUF_NOREC(bp) lockdisablerecurse(&(bp)->b_lock)
/*
* Worklist queue management.
* These routines require that the lock be held.
*/
#ifndef /* NOT */ INVARIANTS
#define WORKLIST_INSERT(head, item) do { \
(item)->wk_state |= ONWORKLIST; \
LIST_INSERT_HEAD(head, item, wk_list); \
} while (0)
#define WORKLIST_REMOVE(item) do { \
(item)->wk_state &= ~ONWORKLIST; \
LIST_REMOVE(item, wk_list); \
} while (0)
#define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT
#define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE
#else /* INVARIANTS */
static void worklist_insert(struct workhead *, struct worklist *, int,
const char *, int);
static void worklist_remove(struct worklist *, int, const char *, int);
#define WORKLIST_INSERT(head, item) \
worklist_insert(head, item, 1, __func__, __LINE__)
#define WORKLIST_INSERT_UNLOCKED(head, item)\
worklist_insert(head, item, 0, __func__, __LINE__)
#define WORKLIST_REMOVE(item)\
worklist_remove(item, 1, __func__, __LINE__)
#define WORKLIST_REMOVE_UNLOCKED(item)\
worklist_remove(item, 0, __func__, __LINE__)
static void
worklist_insert(head, item, locked, func, line)
struct workhead *head;
struct worklist *item;
int locked;
const char *func;
int line;
{
if (locked)
LOCK_OWNED(VFSTOUFS(item->wk_mp));
if (item->wk_state & ONWORKLIST)
panic("worklist_insert: %p %s(0x%X) already on list, "
"added in function %s at line %d",
item, TYPENAME(item->wk_type), item->wk_state,
item->wk_func, item->wk_line);
item->wk_state |= ONWORKLIST;
item->wk_func = func;
item->wk_line = line;
LIST_INSERT_HEAD(head, item, wk_list);
}
static void
worklist_remove(item, locked, func, line)
struct worklist *item;
int locked;
const char *func;
int line;
{
if (locked)
LOCK_OWNED(VFSTOUFS(item->wk_mp));
if ((item->wk_state & ONWORKLIST) == 0)
panic("worklist_remove: %p %s(0x%X) not on list, "
"removed in function %s at line %d",
item, TYPENAME(item->wk_type), item->wk_state,
item->wk_func, item->wk_line);
item->wk_state &= ~ONWORKLIST;
item->wk_func = func;
item->wk_line = line;
LIST_REMOVE(item, wk_list);
}
#endif /* INVARIANTS */
/*
* Merge two jsegdeps keeping only the oldest one as newer references
* can't be discarded until after older references.
*/
static inline struct jsegdep *
jsegdep_merge(struct jsegdep *one, struct jsegdep *two)
{
struct jsegdep *swp;
if (two == NULL)
return (one);
if (one->jd_seg->js_seq > two->jd_seg->js_seq) {
swp = one;
one = two;
two = swp;
}
WORKLIST_REMOVE(&two->jd_list);
free_jsegdep(two);
return (one);
}
/*
* If two freedeps are compatible free one to reduce list size.
*/
static inline struct freedep *
freedep_merge(struct freedep *one, struct freedep *two)
{
if (two == NULL)
return (one);
if (one->fd_freework == two->fd_freework) {
WORKLIST_REMOVE(&two->fd_list);
free_freedep(two);
}
return (one);
}
/*
* Move journal work from one list to another. Duplicate freedeps and
* jsegdeps are coalesced to keep the lists as small as possible.
*/
static void
jwork_move(dst, src)
struct workhead *dst;
struct workhead *src;
{
struct freedep *freedep;
struct jsegdep *jsegdep;
struct worklist *wkn;
struct worklist *wk;
KASSERT(dst != src,
("jwork_move: dst == src"));
freedep = NULL;
jsegdep = NULL;
LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) {
if (wk->wk_type == D_JSEGDEP)
jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep);
else if (wk->wk_type == D_FREEDEP)
freedep = freedep_merge(WK_FREEDEP(wk), freedep);
}
while ((wk = LIST_FIRST(src)) != NULL) {
WORKLIST_REMOVE(wk);
WORKLIST_INSERT(dst, wk);
if (wk->wk_type == D_JSEGDEP) {
jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep);
continue;
}
if (wk->wk_type == D_FREEDEP)
freedep = freedep_merge(WK_FREEDEP(wk), freedep);
}
}
static void
jwork_insert(dst, jsegdep)
struct workhead *dst;
struct jsegdep *jsegdep;
{
struct jsegdep *jsegdepn;
struct worklist *wk;
LIST_FOREACH(wk, dst, wk_list)
if (wk->wk_type == D_JSEGDEP)
break;
if (wk == NULL) {
WORKLIST_INSERT(dst, &jsegdep->jd_list);
return;
}
jsegdepn = WK_JSEGDEP(wk);
if (jsegdep->jd_seg->js_seq < jsegdepn->jd_seg->js_seq) {
WORKLIST_REMOVE(wk);
free_jsegdep(jsegdepn);
WORKLIST_INSERT(dst, &jsegdep->jd_list);
} else
free_jsegdep(jsegdep);
}
/*
* Routines for tracking and managing workitems.
*/
static void workitem_free(struct worklist *, int);
static void workitem_alloc(struct worklist *, int, struct mount *);
static void workitem_reassign(struct worklist *, int);
#define WORKITEM_FREE(item, type) \
workitem_free((struct worklist *)(item), (type))
#define WORKITEM_REASSIGN(item, type) \
workitem_reassign((struct worklist *)(item), (type))
static void
workitem_free(item, type)
struct worklist *item;
int type;
{
struct ufsmount *ump;
#ifdef INVARIANTS
if (item->wk_state & ONWORKLIST)
panic("workitem_free: %s(0x%X) still on list, "
"added in function %s at line %d",
TYPENAME(item->wk_type), item->wk_state,
item->wk_func, item->wk_line);
if (item->wk_type != type && type != D_NEWBLK)
panic("workitem_free: type mismatch %s != %s",
TYPENAME(item->wk_type), TYPENAME(type));
#endif
if (item->wk_state & IOWAITING)
wakeup(item);
ump = VFSTOUFS(item->wk_mp);
LOCK_OWNED(ump);
KASSERT(ump->softdep_deps > 0,
("workitem_free: %s: softdep_deps going negative",
ump->um_fs->fs_fsmnt));
if (--ump->softdep_deps == 0 && ump->softdep_req)
wakeup(&ump->softdep_deps);
KASSERT(dep_current[item->wk_type] > 0,
("workitem_free: %s: dep_current[%s] going negative",
ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type)));
KASSERT(ump->softdep_curdeps[item->wk_type] > 0,
("workitem_free: %s: softdep_curdeps[%s] going negative",
ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type)));
atomic_subtract_long(&dep_current[item->wk_type], 1);
ump->softdep_curdeps[item->wk_type] -= 1;
#ifdef INVARIANTS
LIST_REMOVE(item, wk_all);
#endif
free(item, DtoM(type));
}
static void
workitem_alloc(item, type, mp)
struct worklist *item;
int type;
struct mount *mp;
{
struct ufsmount *ump;
item->wk_type = type;
item->wk_mp = mp;
item->wk_state = 0;
ump = VFSTOUFS(mp);
ACQUIRE_GBLLOCK(&lk);
dep_current[type]++;
if (dep_current[type] > dep_highuse[type])
dep_highuse[type] = dep_current[type];
dep_total[type]++;
FREE_GBLLOCK(&lk);
ACQUIRE_LOCK(ump);
ump->softdep_curdeps[type] += 1;
ump->softdep_deps++;
ump->softdep_accdeps++;
#ifdef INVARIANTS
LIST_INSERT_HEAD(&ump->softdep_alldeps[type], item, wk_all);
#endif
FREE_LOCK(ump);
}
static void
workitem_reassign(item, newtype)
struct worklist *item;
int newtype;
{
struct ufsmount *ump;
ump = VFSTOUFS(item->wk_mp);
LOCK_OWNED(ump);
KASSERT(ump->softdep_curdeps[item->wk_type] > 0,
("workitem_reassign: %s: softdep_curdeps[%s] going negative",
VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type)));
ump->softdep_curdeps[item->wk_type] -= 1;
ump->softdep_curdeps[newtype] += 1;
KASSERT(dep_current[item->wk_type] > 0,
("workitem_reassign: %s: dep_current[%s] going negative",
VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type)));
ACQUIRE_GBLLOCK(&lk);
dep_current[newtype]++;
dep_current[item->wk_type]--;
if (dep_current[newtype] > dep_highuse[newtype])
dep_highuse[newtype] = dep_current[newtype];
dep_total[newtype]++;
FREE_GBLLOCK(&lk);
item->wk_type = newtype;
}
/*
* Workitem queue management
*/
static int max_softdeps; /* maximum number of structs before slowdown */
static int tickdelay = 2; /* number of ticks to pause during slowdown */
static int proc_waiting; /* tracks whether we have a timeout posted */
static int *stat_countp; /* statistic to count in proc_waiting timeout */
static struct callout softdep_callout;
static int req_clear_inodedeps; /* syncer process flush some inodedeps */
static int req_clear_remove; /* syncer process flush some freeblks */
static int softdep_flushcache = 0; /* Should we do BIO_FLUSH? */
/*
* runtime statistics
*/
static int stat_flush_threads; /* number of softdep flushing threads */
static int stat_worklist_push; /* number of worklist cleanups */
static int stat_blk_limit_push; /* number of times block limit neared */
static int stat_ino_limit_push; /* number of times inode limit neared */
static int stat_blk_limit_hit; /* number of times block slowdown imposed */
static int stat_ino_limit_hit; /* number of times inode slowdown imposed */
static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */
static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */
static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */
static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */
static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */
static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */
static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */
static int stat_journal_min; /* Times hit journal min threshold */
static int stat_journal_low; /* Times hit journal low threshold */
static int stat_journal_wait; /* Times blocked in jwait(). */
static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */
static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */
static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */
static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */
static int stat_cleanup_high_delay; /* Maximum cleanup delay (in ticks) */
static int stat_cleanup_blkrequests; /* Number of block cleanup requests */
static int stat_cleanup_inorequests; /* Number of inode cleanup requests */
static int stat_cleanup_retries; /* Number of cleanups that needed to flush */
static int stat_cleanup_failures; /* Number of cleanup requests that failed */
static int stat_emptyjblocks; /* Number of potentially empty journal blocks */
SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW,
&max_softdeps, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW,
&tickdelay, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, flush_threads, CTLFLAG_RD,
&stat_flush_threads, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push,
CTLFLAG_RW | CTLFLAG_STATS, &stat_worklist_push, 0,"");
SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push,
CTLFLAG_RW | CTLFLAG_STATS, &stat_blk_limit_push, 0,"");
SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push,
CTLFLAG_RW | CTLFLAG_STATS, &stat_ino_limit_push, 0,"");
SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit,
CTLFLAG_RW | CTLFLAG_STATS, &stat_blk_limit_hit, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit,
CTLFLAG_RW | CTLFLAG_STATS, &stat_ino_limit_hit, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit,
CTLFLAG_RW | CTLFLAG_STATS, &stat_sync_limit_hit, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs,
CTLFLAG_RW | CTLFLAG_STATS, &stat_indir_blk_ptrs, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap,
CTLFLAG_RW | CTLFLAG_STATS, &stat_inode_bitmap, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs,
CTLFLAG_RW | CTLFLAG_STATS, &stat_direct_blk_ptrs, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry,
CTLFLAG_RW | CTLFLAG_STATS, &stat_dir_entry, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jaddref, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jnewblk, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low,
CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_low, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min,
CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_min, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait,
CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_wait, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_filepage, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_freeblks, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_inode, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk,
CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_newblk, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_blkrequests,
CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_blkrequests, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_inorequests,
CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_inorequests, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_high_delay,
CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_high_delay, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_retries,
CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_retries, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_failures,
CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_failures, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, flushcache, CTLFLAG_RW,
&softdep_flushcache, 0, "");
SYSCTL_INT(_debug_softdep, OID_AUTO, emptyjblocks, CTLFLAG_RD,
&stat_emptyjblocks, 0, "");
SYSCTL_DECL(_vfs_ffs);
/* Whether to recompute the summary at mount time */
static int compute_summary_at_mount = 0;
SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW,
&compute_summary_at_mount, 0, "Recompute summary at mount");
static int print_threads = 0;
SYSCTL_INT(_debug_softdep, OID_AUTO, print_threads, CTLFLAG_RW,
&print_threads, 0, "Notify flusher thread start/stop");
/* List of all filesystems mounted with soft updates */
static TAILQ_HEAD(, mount_softdeps) softdepmounts;
/*
* This function cleans the worklist for a filesystem.
* Each filesystem running with soft dependencies gets its own
* thread to run in this function. The thread is started up in
* softdep_mount and shutdown in softdep_unmount. They show up
* as part of the kernel "bufdaemon" process whose process
* entry is available in bufdaemonproc.
*/
static int searchfailed;
extern struct proc *bufdaemonproc;
static void
softdep_flush(addr)
void *addr;
{
struct mount *mp;
struct thread *td;
struct ufsmount *ump;
td = curthread;
td->td_pflags |= TDP_NORUNNINGBUF;
mp = (struct mount *)addr;
ump = VFSTOUFS(mp);
atomic_add_int(&stat_flush_threads, 1);
ACQUIRE_LOCK(ump);
ump->softdep_flags &= ~FLUSH_STARTING;
wakeup(&ump->softdep_flushtd);
FREE_LOCK(ump);
if (print_threads) {
if (stat_flush_threads == 1)
printf("Running %s at pid %d\n", bufdaemonproc->p_comm,
bufdaemonproc->p_pid);
printf("Start thread %s\n", td->td_name);
}
for (;;) {
while (softdep_process_worklist(mp, 0) > 0 ||
(MOUNTEDSUJ(mp) &&
VFSTOUFS(mp)->softdep_jblocks->jb_suspended))
kthread_suspend_check();
ACQUIRE_LOCK(ump);
if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0)
msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM,
"sdflush", hz / 2);
ump->softdep_flags &= ~FLUSH_CLEANUP;
/*
* Check to see if we are done and need to exit.
*/
if ((ump->softdep_flags & FLUSH_EXIT) == 0) {
FREE_LOCK(ump);
continue;
}
ump->softdep_flags &= ~FLUSH_EXIT;
FREE_LOCK(ump);
wakeup(&ump->softdep_flags);
if (print_threads)
printf("Stop thread %s: searchfailed %d, did cleanups %d\n", td->td_name, searchfailed, ump->um_softdep->sd_cleanups);
atomic_subtract_int(&stat_flush_threads, 1);
kthread_exit();
panic("kthread_exit failed\n");
}
}
static void
worklist_speedup(mp)
struct mount *mp;
{
struct ufsmount *ump;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0)
ump->softdep_flags |= FLUSH_CLEANUP;
wakeup(&ump->softdep_flushtd);
}
static void
softdep_send_speedup(struct ufsmount *ump, size_t shortage, u_int flags)
{
struct buf *bp;
if ((ump->um_flags & UM_CANSPEEDUP) == 0)
return;
bp = malloc(sizeof(*bp), M_TRIM, M_WAITOK | M_ZERO);
bp->b_iocmd = BIO_SPEEDUP;
bp->b_ioflags = flags;
bp->b_bcount = shortage;
g_vfs_strategy(ump->um_bo, bp);
bufwait(bp);
free(bp, M_TRIM);
}
static int
softdep_speedup(ump)
struct ufsmount *ump;
{
struct ufsmount *altump;
struct mount_softdeps *sdp;
LOCK_OWNED(ump);
worklist_speedup(ump->um_mountp);
bd_speedup();
/*
* If we have global shortages, then we need other
* filesystems to help with the cleanup. Here we wakeup a
* flusher thread for a filesystem that is over its fair
* share of resources.
*/
if (req_clear_inodedeps || req_clear_remove) {
ACQUIRE_GBLLOCK(&lk);
TAILQ_FOREACH(sdp, &softdepmounts, sd_next) {
if ((altump = sdp->sd_ump) == ump)
continue;
if (((req_clear_inodedeps &&
altump->softdep_curdeps[D_INODEDEP] >
max_softdeps / stat_flush_threads) ||
(req_clear_remove &&
altump->softdep_curdeps[D_DIRREM] >
(max_softdeps / 2) / stat_flush_threads)) &&
TRY_ACQUIRE_LOCK(altump))
break;
}
if (sdp == NULL) {
searchfailed++;
FREE_GBLLOCK(&lk);
} else {
/*
* Move to the end of the list so we pick a
* different one on out next try.
*/
TAILQ_REMOVE(&softdepmounts, sdp, sd_next);
TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next);
FREE_GBLLOCK(&lk);
if ((altump->softdep_flags &
(FLUSH_CLEANUP | FLUSH_EXIT)) == 0)
altump->softdep_flags |= FLUSH_CLEANUP;
altump->um_softdep->sd_cleanups++;
wakeup(&altump->softdep_flushtd);
FREE_LOCK(altump);
}
}
return (speedup_syncer());
}
/*
* Add an item to the end of the work queue.
* This routine requires that the lock be held.
* This is the only routine that adds items to the list.
* The following routine is the only one that removes items
* and does so in order from first to last.
*/
#define WK_HEAD 0x0001 /* Add to HEAD. */
#define WK_NODELAY 0x0002 /* Process immediately. */
static void
add_to_worklist(wk, flags)
struct worklist *wk;
int flags;
{
struct ufsmount *ump;
ump = VFSTOUFS(wk->wk_mp);
LOCK_OWNED(ump);
if (wk->wk_state & ONWORKLIST)
panic("add_to_worklist: %s(0x%X) already on list",
TYPENAME(wk->wk_type), wk->wk_state);
wk->wk_state |= ONWORKLIST;
if (ump->softdep_on_worklist == 0) {
LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list);
ump->softdep_worklist_tail = wk;
} else if (flags & WK_HEAD) {
LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list);
} else {
LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list);
ump->softdep_worklist_tail = wk;
}
ump->softdep_on_worklist += 1;
if (flags & WK_NODELAY)
worklist_speedup(wk->wk_mp);
}
/*
* Remove the item to be processed. If we are removing the last
* item on the list, we need to recalculate the tail pointer.
*/
static void
remove_from_worklist(wk)
struct worklist *wk;
{
struct ufsmount *ump;
ump = VFSTOUFS(wk->wk_mp);
if (ump->softdep_worklist_tail == wk)
ump->softdep_worklist_tail =
(struct worklist *)wk->wk_list.le_prev;
WORKLIST_REMOVE(wk);
ump->softdep_on_worklist -= 1;
}
static void
wake_worklist(wk)
struct worklist *wk;
{
if (wk->wk_state & IOWAITING) {
wk->wk_state &= ~IOWAITING;
wakeup(wk);
}
}
static void
wait_worklist(wk, wmesg)
struct worklist *wk;
char *wmesg;
{
struct ufsmount *ump;
ump = VFSTOUFS(wk->wk_mp);
wk->wk_state |= IOWAITING;
msleep(wk, LOCK_PTR(ump), PVM, wmesg, 0);
}
/*
* Process that runs once per second to handle items in the background queue.
*
* Note that we ensure that everything is done in the order in which they
* appear in the queue. The code below depends on this property to ensure
* that blocks of a file are freed before the inode itself is freed. This
* ordering ensures that no new <vfsid, inum, lbn> triples will be generated
* until all the old ones have been purged from the dependency lists.
*/
static int
softdep_process_worklist(mp, full)
struct mount *mp;
int full;
{
int cnt, matchcnt;
struct ufsmount *ump;
long starttime;
KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp"));
if (MOUNTEDSOFTDEP(mp) == 0)
return (0);
matchcnt = 0;
ump = VFSTOUFS(mp);
ACQUIRE_LOCK(ump);
starttime = time_second;
softdep_process_journal(mp, NULL, full ? MNT_WAIT : 0);
check_clear_deps(mp);
while (ump->softdep_on_worklist > 0) {
if ((cnt = process_worklist_item(mp, 10, LK_NOWAIT)) == 0)
break;
else
matchcnt += cnt;
check_clear_deps(mp);
/*
* We do not generally want to stop for buffer space, but if
* we are really being a buffer hog, we will stop and wait.
*/
if (should_yield()) {
FREE_LOCK(ump);
kern_yield(PRI_USER);
bwillwrite();
ACQUIRE_LOCK(ump);
}
/*
* Never allow processing to run for more than one
* second. This gives the syncer thread the opportunity
* to pause if appropriate.
*/
if (!full && starttime != time_second)
break;
}
if (full == 0)
journal_unsuspend(ump);
FREE_LOCK(ump);
return (matchcnt);
}
/*
* Process all removes associated with a vnode if we are running out of
* journal space. Any other process which attempts to flush these will
* be unable as we have the vnodes locked.
*/
static void
process_removes(vp)
struct vnode *vp;
{
struct inodedep *inodedep;
struct dirrem *dirrem;
struct ufsmount *ump;
struct mount *mp;
ino_t inum;
mp = vp->v_mount;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
inum = VTOI(vp)->i_number;
for (;;) {
top:
if (inodedep_lookup(mp, inum, 0, &inodedep) == 0)
return;
LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) {
/*
* If another thread is trying to lock this vnode
* it will fail but we must wait for it to do so
* before we can proceed.
*/
if (dirrem->dm_state & INPROGRESS) {
wait_worklist(&dirrem->dm_list, "pwrwait");
goto top;
}
if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) ==
(COMPLETE | ONWORKLIST))
break;
}
if (dirrem == NULL)
return;
remove_from_worklist(&dirrem->dm_list);
FREE_LOCK(ump);
if (vn_start_secondary_write(NULL, &mp, V_NOWAIT))
panic("process_removes: suspended filesystem");
handle_workitem_remove(dirrem, 0);
vn_finished_secondary_write(mp);
ACQUIRE_LOCK(ump);
}
}
/*
* Process all truncations associated with a vnode if we are running out
* of journal space. This is called when the vnode lock is already held
* and no other process can clear the truncation. This function returns
* a value greater than zero if it did any work.
*/
static void
process_truncates(vp)
struct vnode *vp;
{
struct inodedep *inodedep;
struct freeblks *freeblks;
struct ufsmount *ump;
struct mount *mp;
ino_t inum;
int cgwait;
mp = vp->v_mount;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
inum = VTOI(vp)->i_number;
for (;;) {
if (inodedep_lookup(mp, inum, 0, &inodedep) == 0)
return;
cgwait = 0;
TAILQ_FOREACH(freeblks, &inodedep->id_freeblklst, fb_next) {
/* Journal entries not yet written. */
if (!LIST_EMPTY(&freeblks->fb_jblkdephd)) {
jwait(&LIST_FIRST(
&freeblks->fb_jblkdephd)->jb_list,
MNT_WAIT);
break;
}
/* Another thread is executing this item. */
if (freeblks->fb_state & INPROGRESS) {
wait_worklist(&freeblks->fb_list, "ptrwait");
break;
}
/* Freeblks is waiting on a inode write. */
if ((freeblks->fb_state & COMPLETE) == 0) {
FREE_LOCK(ump);
ffs_update(vp, 1);
ACQUIRE_LOCK(ump);
break;
}
if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST)) ==
(ALLCOMPLETE | ONWORKLIST)) {
remove_from_worklist(&freeblks->fb_list);
freeblks->fb_state |= INPROGRESS;
FREE_LOCK(ump);
if (vn_start_secondary_write(NULL, &mp,
V_NOWAIT))
panic("process_truncates: "
"suspended filesystem");
handle_workitem_freeblocks(freeblks, 0);
vn_finished_secondary_write(mp);
ACQUIRE_LOCK(ump);
break;
}
if (freeblks->fb_cgwait)
cgwait++;
}
if (cgwait) {
FREE_LOCK(ump);
sync_cgs(mp, MNT_WAIT);
ffs_sync_snap(mp, MNT_WAIT);
ACQUIRE_LOCK(ump);
continue;
}
if (freeblks == NULL)
break;
}
return;
}
/*
* Process one item on the worklist.
*/
static int
process_worklist_item(mp, target, flags)
struct mount *mp;
int target;
int flags;
{
struct worklist sentinel;
struct worklist *wk;
struct ufsmount *ump;
int matchcnt;
int error;
KASSERT(mp != NULL, ("process_worklist_item: NULL mp"));
/*
* If we are being called because of a process doing a
* copy-on-write, then it is not safe to write as we may
* recurse into the copy-on-write routine.
*/
if (curthread->td_pflags & TDP_COWINPROGRESS)
return (-1);
PHOLD(curproc); /* Don't let the stack go away. */
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
matchcnt = 0;
sentinel.wk_mp = NULL;
sentinel.wk_type = D_SENTINEL;
LIST_INSERT_HEAD(&ump->softdep_workitem_pending, &sentinel, wk_list);
for (wk = LIST_NEXT(&sentinel, wk_list); wk != NULL;
wk = LIST_NEXT(&sentinel, wk_list)) {
if (wk->wk_type == D_SENTINEL) {
LIST_REMOVE(&sentinel, wk_list);
LIST_INSERT_AFTER(wk, &sentinel, wk_list);
continue;
}
if (wk->wk_state & INPROGRESS)
panic("process_worklist_item: %p already in progress.",
wk);
wk->wk_state |= INPROGRESS;
remove_from_worklist(wk);
FREE_LOCK(ump);
if (vn_start_secondary_write(NULL, &mp, V_NOWAIT))
panic("process_worklist_item: suspended filesystem");
switch (wk->wk_type) {
case D_DIRREM:
/* removal of a directory entry */
error = handle_workitem_remove(WK_DIRREM(wk), flags);
break;
case D_FREEBLKS:
/* releasing blocks and/or fragments from a file */
error = handle_workitem_freeblocks(WK_FREEBLKS(wk),
flags);
break;
case D_FREEFRAG:
/* releasing a fragment when replaced as a file grows */
handle_workitem_freefrag(WK_FREEFRAG(wk));
error = 0;
break;
case D_FREEFILE:
/* releasing an inode when its link count drops to 0 */
handle_workitem_freefile(WK_FREEFILE(wk));
error = 0;
break;
default:
panic("%s_process_worklist: Unknown type %s",
"softdep", TYPENAME(wk->wk_type));
/* NOTREACHED */
}
vn_finished_secondary_write(mp);
ACQUIRE_LOCK(ump);
if (error == 0) {
if (++matchcnt == target)
break;
continue;
}
/*
* We have to retry the worklist item later. Wake up any
* waiters who may be able to complete it immediately and
* add the item back to the head so we don't try to execute
* it again.
*/
wk->wk_state &= ~INPROGRESS;
wake_worklist(wk);
add_to_worklist(wk, WK_HEAD);
}
/* Sentinal could've become the tail from remove_from_worklist. */
if (ump->softdep_worklist_tail == &sentinel)
ump->softdep_worklist_tail =
(struct worklist *)sentinel.wk_list.le_prev;
LIST_REMOVE(&sentinel, wk_list);
PRELE(curproc);
return (matchcnt);
}
/*
* Move dependencies from one buffer to another.
*/
int
softdep_move_dependencies(oldbp, newbp)
struct buf *oldbp;
struct buf *newbp;
{
struct worklist *wk, *wktail;
struct ufsmount *ump;
int dirty;
if ((wk = LIST_FIRST(&oldbp->b_dep)) == NULL)
return (0);
KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0,
("softdep_move_dependencies called on non-softdep filesystem"));
dirty = 0;
wktail = NULL;
ump = VFSTOUFS(wk->wk_mp);
ACQUIRE_LOCK(ump);
while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) {
LIST_REMOVE(wk, wk_list);
if (wk->wk_type == D_BMSAFEMAP &&
bmsafemap_backgroundwrite(WK_BMSAFEMAP(wk), newbp))
dirty = 1;
if (wktail == NULL)
LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list);
else
LIST_INSERT_AFTER(wktail, wk, wk_list);
wktail = wk;
}
FREE_LOCK(ump);
return (dirty);
}
/*
* Purge the work list of all items associated with a particular mount point.
*/
int
softdep_flushworklist(oldmnt, countp, td)
struct mount *oldmnt;
int *countp;
struct thread *td;
{
struct vnode *devvp;
struct ufsmount *ump;
int count, error;
/*
* Alternately flush the block device associated with the mount
* point and process any dependencies that the flushing
* creates. We continue until no more worklist dependencies
* are found.
*/
*countp = 0;
error = 0;
ump = VFSTOUFS(oldmnt);
devvp = ump->um_devvp;
while ((count = softdep_process_worklist(oldmnt, 1)) > 0) {
*countp += count;
vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_FSYNC(devvp, MNT_WAIT, td);
VOP_UNLOCK(devvp);
if (error != 0)
break;
}
return (error);
}
#define SU_WAITIDLE_RETRIES 20
static int
softdep_waitidle(struct mount *mp, int flags __unused)
{
struct ufsmount *ump;
struct vnode *devvp;
struct thread *td;
int error, i;
ump = VFSTOUFS(mp);
devvp = ump->um_devvp;
td = curthread;
error = 0;
ACQUIRE_LOCK(ump);
for (i = 0; i < SU_WAITIDLE_RETRIES && ump->softdep_deps != 0; i++) {
ump->softdep_req = 1;
KASSERT((flags & FORCECLOSE) == 0 ||
ump->softdep_on_worklist == 0,
("softdep_waitidle: work added after flush"));
msleep(&ump->softdep_deps, LOCK_PTR(ump), PVM | PDROP,
"softdeps", 10 * hz);
vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_FSYNC(devvp, MNT_WAIT, td);
VOP_UNLOCK(devvp);
ACQUIRE_LOCK(ump);
if (error != 0)
break;
}
ump->softdep_req = 0;
if (i == SU_WAITIDLE_RETRIES && error == 0 && ump->softdep_deps != 0) {
error = EBUSY;
printf("softdep_waitidle: Failed to flush worklist for %p\n",
mp);
}
FREE_LOCK(ump);
return (error);
}
/*
* Flush all vnodes and worklist items associated with a specified mount point.
*/
int
softdep_flushfiles(oldmnt, flags, td)
struct mount *oldmnt;
int flags;
struct thread *td;
{
#ifdef QUOTA
struct ufsmount *ump;
int i;
#endif
int error, early, depcount, loopcnt, retry_flush_count, retry;
int morework;
KASSERT(MOUNTEDSOFTDEP(oldmnt) != 0,
("softdep_flushfiles called on non-softdep filesystem"));
loopcnt = 10;
retry_flush_count = 3;
retry_flush:
error = 0;
/*
* Alternately flush the vnodes associated with the mount
* point and process any dependencies that the flushing
* creates. In theory, this loop can happen at most twice,
* but we give it a few extra just to be sure.
*/
for (; loopcnt > 0; loopcnt--) {
/*
* Do another flush in case any vnodes were brought in
* as part of the cleanup operations.
*/
early = retry_flush_count == 1 || (oldmnt->mnt_kern_flag &
MNTK_UNMOUNT) == 0 ? 0 : EARLYFLUSH;
if ((error = ffs_flushfiles(oldmnt, flags | early, td)) != 0)
break;
if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 ||
depcount == 0)
break;
}
/*
* If we are unmounting then it is an error to fail. If we
* are simply trying to downgrade to read-only, then filesystem
* activity can keep us busy forever, so we just fail with EBUSY.
*/
if (loopcnt == 0) {
if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT)
panic("softdep_flushfiles: looping");
error = EBUSY;
}
if (!error)
error = softdep_waitidle(oldmnt, flags);
if (!error) {
if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) {
retry = 0;
MNT_ILOCK(oldmnt);
morework = oldmnt->mnt_nvnodelistsize > 0;
#ifdef QUOTA
ump = VFSTOUFS(oldmnt);
UFS_LOCK(ump);
for (i = 0; i < MAXQUOTAS; i++) {
if (ump->um_quotas[i] != NULLVP)
morework = 1;
}
UFS_UNLOCK(ump);
#endif
if (morework) {
if (--retry_flush_count > 0) {
retry = 1;
loopcnt = 3;
} else
error = EBUSY;
}
MNT_IUNLOCK(oldmnt);
if (retry)
goto retry_flush;
}
}
return (error);
}
/*
* Structure hashing.
*
* There are four types of structures that can be looked up:
* 1) pagedep structures identified by mount point, inode number,
* and logical block.
* 2) inodedep structures identified by mount point and inode number.
* 3) newblk structures identified by mount point and
* physical block number.
* 4) bmsafemap structures identified by mount point and
* cylinder group number.
*
* The "pagedep" and "inodedep" dependency structures are hashed
* separately from the file blocks and inodes to which they correspond.
* This separation helps when the in-memory copy of an inode or
* file block must be replaced. It also obviates the need to access
* an inode or file page when simply updating (or de-allocating)
* dependency structures. Lookup of newblk structures is needed to
* find newly allocated blocks when trying to associate them with
* their allocdirect or allocindir structure.
*
* The lookup routines optionally create and hash a new instance when
* an existing entry is not found. The bmsafemap lookup routine always
* allocates a new structure if an existing one is not found.
*/
#define DEPALLOC 0x0001 /* allocate structure if lookup fails */
/*
* Structures and routines associated with pagedep caching.
*/
#define PAGEDEP_HASH(ump, inum, lbn) \
(&(ump)->pagedep_hashtbl[((inum) + (lbn)) & (ump)->pagedep_hash_size])
static int
pagedep_find(pagedephd, ino, lbn, pagedeppp)
struct pagedep_hashhead *pagedephd;
ino_t ino;
ufs_lbn_t lbn;
struct pagedep **pagedeppp;
{
struct pagedep *pagedep;
LIST_FOREACH(pagedep, pagedephd, pd_hash) {
if (ino == pagedep->pd_ino && lbn == pagedep->pd_lbn) {
*pagedeppp = pagedep;
return (1);
}
}
*pagedeppp = NULL;
return (0);
}
/*
* Look up a pagedep. Return 1 if found, 0 otherwise.
* If not found, allocate if DEPALLOC flag is passed.
* Found or allocated entry is returned in pagedeppp.
*/
static int
pagedep_lookup(mp, bp, ino, lbn, flags, pagedeppp)
struct mount *mp;
struct buf *bp;
ino_t ino;
ufs_lbn_t lbn;
int flags;
struct pagedep **pagedeppp;
{
struct pagedep *pagedep;
struct pagedep_hashhead *pagedephd;
struct worklist *wk;
struct ufsmount *ump;
int ret;
int i;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
if (bp) {
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
if (wk->wk_type == D_PAGEDEP) {
*pagedeppp = WK_PAGEDEP(wk);
return (1);
}
}
}
pagedephd = PAGEDEP_HASH(ump, ino, lbn);
ret = pagedep_find(pagedephd, ino, lbn, pagedeppp);
if (ret) {
if (((*pagedeppp)->pd_state & ONWORKLIST) == 0 && bp)
WORKLIST_INSERT(&bp->b_dep, &(*pagedeppp)->pd_list);
return (1);
}
if ((flags & DEPALLOC) == 0)
return (0);
FREE_LOCK(ump);
pagedep = malloc(sizeof(struct pagedep),
M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO);
workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp);
ACQUIRE_LOCK(ump);
ret = pagedep_find(pagedephd, ino, lbn, pagedeppp);
if (*pagedeppp) {
/*
* This should never happen since we only create pagedeps
* with the vnode lock held. Could be an assert.
*/
WORKITEM_FREE(pagedep, D_PAGEDEP);
return (ret);
}
pagedep->pd_ino = ino;
pagedep->pd_lbn = lbn;
LIST_INIT(&pagedep->pd_dirremhd);
LIST_INIT(&pagedep->pd_pendinghd);
for (i = 0; i < DAHASHSZ; i++)
LIST_INIT(&pagedep->pd_diraddhd[i]);
LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash);
WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list);
*pagedeppp = pagedep;
return (0);
}
/*
* Structures and routines associated with inodedep caching.
*/
#define INODEDEP_HASH(ump, inum) \
(&(ump)->inodedep_hashtbl[(inum) & (ump)->inodedep_hash_size])
static int
inodedep_find(inodedephd, inum, inodedeppp)
struct inodedep_hashhead *inodedephd;
ino_t inum;
struct inodedep **inodedeppp;
{
struct inodedep *inodedep;
LIST_FOREACH(inodedep, inodedephd, id_hash)
if (inum == inodedep->id_ino)
break;
if (inodedep) {
*inodedeppp = inodedep;
return (1);
}
*inodedeppp = NULL;
return (0);
}
/*
* Look up an inodedep. Return 1 if found, 0 if not found.
* If not found, allocate if DEPALLOC flag is passed.
* Found or allocated entry is returned in inodedeppp.
*/
static int
inodedep_lookup(mp, inum, flags, inodedeppp)
struct mount *mp;
ino_t inum;
int flags;
struct inodedep **inodedeppp;
{
struct inodedep *inodedep;
struct inodedep_hashhead *inodedephd;
struct ufsmount *ump;
struct fs *fs;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
fs = ump->um_fs;
inodedephd = INODEDEP_HASH(ump, inum);
if (inodedep_find(inodedephd, inum, inodedeppp))
return (1);
if ((flags & DEPALLOC) == 0)
return (0);
/*
* If the system is over its limit and our filesystem is
* responsible for more than our share of that usage and
* we are not in a rush, request some inodedep cleanup.
*/
if (softdep_excess_items(ump, D_INODEDEP))
schedule_cleanup(mp);
else
FREE_LOCK(ump);
inodedep = malloc(sizeof(struct inodedep),
M_INODEDEP, M_SOFTDEP_FLAGS);
workitem_alloc(&inodedep->id_list, D_INODEDEP, mp);
ACQUIRE_LOCK(ump);
if (inodedep_find(inodedephd, inum, inodedeppp)) {
WORKITEM_FREE(inodedep, D_INODEDEP);
return (1);
}
inodedep->id_fs = fs;
inodedep->id_ino = inum;
inodedep->id_state = ALLCOMPLETE;
inodedep->id_nlinkdelta = 0;
+ inodedep->id_nlinkwrote = -1;
inodedep->id_savedino1 = NULL;
inodedep->id_savedsize = -1;
inodedep->id_savedextsize = -1;
inodedep->id_savednlink = -1;
inodedep->id_bmsafemap = NULL;
inodedep->id_mkdiradd = NULL;
LIST_INIT(&inodedep->id_dirremhd);
LIST_INIT(&inodedep->id_pendinghd);
LIST_INIT(&inodedep->id_inowait);
LIST_INIT(&inodedep->id_bufwait);
TAILQ_INIT(&inodedep->id_inoreflst);
TAILQ_INIT(&inodedep->id_inoupdt);
TAILQ_INIT(&inodedep->id_newinoupdt);
TAILQ_INIT(&inodedep->id_extupdt);
TAILQ_INIT(&inodedep->id_newextupdt);
TAILQ_INIT(&inodedep->id_freeblklst);
LIST_INSERT_HEAD(inodedephd, inodedep, id_hash);
*inodedeppp = inodedep;
return (0);
}
/*
* Structures and routines associated with newblk caching.
*/
#define NEWBLK_HASH(ump, inum) \
(&(ump)->newblk_hashtbl[(inum) & (ump)->newblk_hash_size])
static int
newblk_find(newblkhd, newblkno, flags, newblkpp)
struct newblk_hashhead *newblkhd;
ufs2_daddr_t newblkno;
int flags;
struct newblk **newblkpp;
{
struct newblk *newblk;
LIST_FOREACH(newblk, newblkhd, nb_hash) {
if (newblkno != newblk->nb_newblkno)
continue;
/*
* If we're creating a new dependency don't match those that
* have already been converted to allocdirects. This is for
* a frag extend.
*/
if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK)
continue;
break;
}
if (newblk) {
*newblkpp = newblk;
return (1);
}
*newblkpp = NULL;
return (0);
}
/*
* Look up a newblk. Return 1 if found, 0 if not found.
* If not found, allocate if DEPALLOC flag is passed.
* Found or allocated entry is returned in newblkpp.
*/
static int
newblk_lookup(mp, newblkno, flags, newblkpp)
struct mount *mp;
ufs2_daddr_t newblkno;
int flags;
struct newblk **newblkpp;
{
struct newblk *newblk;
struct newblk_hashhead *newblkhd;
struct ufsmount *ump;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
newblkhd = NEWBLK_HASH(ump, newblkno);
if (newblk_find(newblkhd, newblkno, flags, newblkpp))
return (1);
if ((flags & DEPALLOC) == 0)
return (0);
if (softdep_excess_items(ump, D_NEWBLK) ||
softdep_excess_items(ump, D_ALLOCDIRECT) ||
softdep_excess_items(ump, D_ALLOCINDIR))
schedule_cleanup(mp);
else
FREE_LOCK(ump);
newblk = malloc(sizeof(union allblk), M_NEWBLK,
M_SOFTDEP_FLAGS | M_ZERO);
workitem_alloc(&newblk->nb_list, D_NEWBLK, mp);
ACQUIRE_LOCK(ump);
if (newblk_find(newblkhd, newblkno, flags, newblkpp)) {
WORKITEM_FREE(newblk, D_NEWBLK);
return (1);
}
newblk->nb_freefrag = NULL;
LIST_INIT(&newblk->nb_indirdeps);
LIST_INIT(&newblk->nb_newdirblk);
LIST_INIT(&newblk->nb_jwork);
newblk->nb_state = ATTACHED;
newblk->nb_newblkno = newblkno;
LIST_INSERT_HEAD(newblkhd, newblk, nb_hash);
*newblkpp = newblk;
return (0);
}
/*
* Structures and routines associated with freed indirect block caching.
*/
#define INDIR_HASH(ump, blkno) \
(&(ump)->indir_hashtbl[(blkno) & (ump)->indir_hash_size])
/*
* Lookup an indirect block in the indir hash table. The freework is
* removed and potentially freed. The caller must do a blocking journal
* write before writing to the blkno.
*/
static int
indirblk_lookup(mp, blkno)
struct mount *mp;
ufs2_daddr_t blkno;
{
struct freework *freework;
struct indir_hashhead *wkhd;
struct ufsmount *ump;
ump = VFSTOUFS(mp);
wkhd = INDIR_HASH(ump, blkno);
TAILQ_FOREACH(freework, wkhd, fw_next) {
if (freework->fw_blkno != blkno)
continue;
indirblk_remove(freework);
return (1);
}
return (0);
}
/*
* Insert an indirect block represented by freework into the indirblk
* hash table so that it may prevent the block from being re-used prior
* to the journal being written.
*/
static void
indirblk_insert(freework)
struct freework *freework;
{
struct jblocks *jblocks;
struct jseg *jseg;
struct ufsmount *ump;
ump = VFSTOUFS(freework->fw_list.wk_mp);
jblocks = ump->softdep_jblocks;
jseg = TAILQ_LAST(&jblocks->jb_segs, jseglst);
if (jseg == NULL)
return;
LIST_INSERT_HEAD(&jseg->js_indirs, freework, fw_segs);
TAILQ_INSERT_HEAD(INDIR_HASH(ump, freework->fw_blkno), freework,
fw_next);
freework->fw_state &= ~DEPCOMPLETE;
}
static void
indirblk_remove(freework)
struct freework *freework;
{
struct ufsmount *ump;
ump = VFSTOUFS(freework->fw_list.wk_mp);
LIST_REMOVE(freework, fw_segs);
TAILQ_REMOVE(INDIR_HASH(ump, freework->fw_blkno), freework, fw_next);
freework->fw_state |= DEPCOMPLETE;
if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE)
WORKITEM_FREE(freework, D_FREEWORK);
}
/*
* Executed during filesystem system initialization before
* mounting any filesystems.
*/
void
softdep_initialize()
{
TAILQ_INIT(&softdepmounts);
#ifdef __LP64__
max_softdeps = desiredvnodes * 4;
#else
max_softdeps = desiredvnodes * 2;
#endif
/* initialise bioops hack */
bioops.io_start = softdep_disk_io_initiation;
bioops.io_complete = softdep_disk_write_complete;
bioops.io_deallocate = softdep_deallocate_dependencies;
bioops.io_countdeps = softdep_count_dependencies;
softdep_ast_cleanup = softdep_ast_cleanup_proc;
/* Initialize the callout with an mtx. */
callout_init_mtx(&softdep_callout, &lk, 0);
}
/*
* Executed after all filesystems have been unmounted during
* filesystem module unload.
*/
void
softdep_uninitialize()
{
/* clear bioops hack */
bioops.io_start = NULL;
bioops.io_complete = NULL;
bioops.io_deallocate = NULL;
bioops.io_countdeps = NULL;
softdep_ast_cleanup = NULL;
callout_drain(&softdep_callout);
}
/*
* Called at mount time to notify the dependency code that a
* filesystem wishes to use it.
*/
int
softdep_mount(devvp, mp, fs, cred)
struct vnode *devvp;
struct mount *mp;
struct fs *fs;
struct ucred *cred;
{
struct csum_total cstotal;
struct mount_softdeps *sdp;
struct ufsmount *ump;
struct cg *cgp;
struct buf *bp;
u_int cyl, i;
int error;
sdp = malloc(sizeof(struct mount_softdeps), M_MOUNTDATA,
M_WAITOK | M_ZERO);
MNT_ILOCK(mp);
mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP;
if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) {
mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) |
MNTK_SOFTDEP | MNTK_NOASYNC;
}
ump = VFSTOUFS(mp);
ump->um_softdep = sdp;
MNT_IUNLOCK(mp);
rw_init(LOCK_PTR(ump), "per-fs softdep");
sdp->sd_ump = ump;
LIST_INIT(&ump->softdep_workitem_pending);
LIST_INIT(&ump->softdep_journal_pending);
TAILQ_INIT(&ump->softdep_unlinked);
LIST_INIT(&ump->softdep_dirtycg);
ump->softdep_worklist_tail = NULL;
ump->softdep_on_worklist = 0;
ump->softdep_deps = 0;
LIST_INIT(&ump->softdep_mkdirlisthd);
ump->pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP,
&ump->pagedep_hash_size);
ump->pagedep_nextclean = 0;
ump->inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP,
&ump->inodedep_hash_size);
ump->inodedep_nextclean = 0;
ump->newblk_hashtbl = hashinit(max_softdeps / 2, M_NEWBLK,
&ump->newblk_hash_size);
ump->bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP,
&ump->bmsafemap_hash_size);
i = 1 << (ffs(desiredvnodes / 10) - 1);
ump->indir_hashtbl = malloc(i * sizeof(struct indir_hashhead),
M_FREEWORK, M_WAITOK);
ump->indir_hash_size = i - 1;
for (i = 0; i <= ump->indir_hash_size; i++)
TAILQ_INIT(&ump->indir_hashtbl[i]);
#ifdef INVARIANTS
for (i = 0; i <= D_LAST; i++)
LIST_INIT(&ump->softdep_alldeps[i]);
#endif
ACQUIRE_GBLLOCK(&lk);
TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next);
FREE_GBLLOCK(&lk);
if ((fs->fs_flags & FS_SUJ) &&
(error = journal_mount(mp, fs, cred)) != 0) {
printf("Failed to start journal: %d\n", error);
softdep_unmount(mp);
return (error);
}
/*
* Start our flushing thread in the bufdaemon process.
*/
ACQUIRE_LOCK(ump);
ump->softdep_flags |= FLUSH_STARTING;
FREE_LOCK(ump);
kproc_kthread_add(&softdep_flush, mp, &bufdaemonproc,
&ump->softdep_flushtd, 0, 0, "softdepflush", "%s worker",
mp->mnt_stat.f_mntonname);
ACQUIRE_LOCK(ump);
while ((ump->softdep_flags & FLUSH_STARTING) != 0) {
msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM, "sdstart",
hz / 2);
}
FREE_LOCK(ump);
/*
* When doing soft updates, the counters in the
* superblock may have gotten out of sync. Recomputation
* can take a long time and can be deferred for background
* fsck. However, the old behavior of scanning the cylinder
* groups and recalculating them at mount time is available
* by setting vfs.ffs.compute_summary_at_mount to one.
*/
if (compute_summary_at_mount == 0 || fs->fs_clean != 0)
return (0);
bzero(&cstotal, sizeof cstotal);
for (cyl = 0; cyl < fs->fs_ncg; cyl++) {
if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)),
fs->fs_cgsize, cred, &bp)) != 0) {
brelse(bp);
softdep_unmount(mp);
return (error);
}
cgp = (struct cg *)bp->b_data;
cstotal.cs_nffree += cgp->cg_cs.cs_nffree;
cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree;
cstotal.cs_nifree += cgp->cg_cs.cs_nifree;
cstotal.cs_ndir += cgp->cg_cs.cs_ndir;
fs->fs_cs(fs, cyl) = cgp->cg_cs;
brelse(bp);
}
#ifdef INVARIANTS
if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal))
printf("%s: superblock summary recomputed\n", fs->fs_fsmnt);
#endif
bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal);
return (0);
}
void
softdep_unmount(mp)
struct mount *mp;
{
struct ufsmount *ump;
#ifdef INVARIANTS
int i;
#endif
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_unmount called on non-softdep filesystem"));
ump = VFSTOUFS(mp);
MNT_ILOCK(mp);
mp->mnt_flag &= ~MNT_SOFTDEP;
if (MOUNTEDSUJ(mp) == 0) {
MNT_IUNLOCK(mp);
} else {
mp->mnt_flag &= ~MNT_SUJ;
MNT_IUNLOCK(mp);
journal_unmount(ump);
}
/*
* Shut down our flushing thread. Check for NULL is if
* softdep_mount errors out before the thread has been created.
*/
if (ump->softdep_flushtd != NULL) {
ACQUIRE_LOCK(ump);
ump->softdep_flags |= FLUSH_EXIT;
wakeup(&ump->softdep_flushtd);
msleep(&ump->softdep_flags, LOCK_PTR(ump), PVM | PDROP,
"sdwait", 0);
KASSERT((ump->softdep_flags & FLUSH_EXIT) == 0,
("Thread shutdown failed"));
}
/*
* Free up our resources.
*/
ACQUIRE_GBLLOCK(&lk);
TAILQ_REMOVE(&softdepmounts, ump->um_softdep, sd_next);
FREE_GBLLOCK(&lk);
rw_destroy(LOCK_PTR(ump));
hashdestroy(ump->pagedep_hashtbl, M_PAGEDEP, ump->pagedep_hash_size);
hashdestroy(ump->inodedep_hashtbl, M_INODEDEP, ump->inodedep_hash_size);
hashdestroy(ump->newblk_hashtbl, M_NEWBLK, ump->newblk_hash_size);
hashdestroy(ump->bmsafemap_hashtbl, M_BMSAFEMAP,
ump->bmsafemap_hash_size);
free(ump->indir_hashtbl, M_FREEWORK);
#ifdef INVARIANTS
for (i = 0; i <= D_LAST; i++) {
KASSERT(ump->softdep_curdeps[i] == 0,
("Unmount %s: Dep type %s != 0 (%ld)", ump->um_fs->fs_fsmnt,
TYPENAME(i), ump->softdep_curdeps[i]));
KASSERT(LIST_EMPTY(&ump->softdep_alldeps[i]),
("Unmount %s: Dep type %s not empty (%p)", ump->um_fs->fs_fsmnt,
TYPENAME(i), LIST_FIRST(&ump->softdep_alldeps[i])));
}
#endif
free(ump->um_softdep, M_MOUNTDATA);
}
static struct jblocks *
jblocks_create(void)
{
struct jblocks *jblocks;
jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO);
TAILQ_INIT(&jblocks->jb_segs);
jblocks->jb_avail = 10;
jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail,
M_JBLOCKS, M_WAITOK | M_ZERO);
return (jblocks);
}
static ufs2_daddr_t
jblocks_alloc(jblocks, bytes, actual)
struct jblocks *jblocks;
int bytes;
int *actual;
{
ufs2_daddr_t daddr;
struct jextent *jext;
int freecnt;
int blocks;
blocks = bytes / DEV_BSIZE;
jext = &jblocks->jb_extent[jblocks->jb_head];
freecnt = jext->je_blocks - jblocks->jb_off;
if (freecnt == 0) {
jblocks->jb_off = 0;
if (++jblocks->jb_head > jblocks->jb_used)
jblocks->jb_head = 0;
jext = &jblocks->jb_extent[jblocks->jb_head];
freecnt = jext->je_blocks;
}
if (freecnt > blocks)
freecnt = blocks;
*actual = freecnt * DEV_BSIZE;
daddr = jext->je_daddr + jblocks->jb_off;
jblocks->jb_off += freecnt;
jblocks->jb_free -= freecnt;
return (daddr);
}
static void
jblocks_free(jblocks, mp, bytes)
struct jblocks *jblocks;
struct mount *mp;
int bytes;
{
LOCK_OWNED(VFSTOUFS(mp));
jblocks->jb_free += bytes / DEV_BSIZE;
if (jblocks->jb_suspended)
worklist_speedup(mp);
wakeup(jblocks);
}
static void
jblocks_destroy(jblocks)
struct jblocks *jblocks;
{
if (jblocks->jb_extent)
free(jblocks->jb_extent, M_JBLOCKS);
free(jblocks, M_JBLOCKS);
}
static void
jblocks_add(jblocks, daddr, blocks)
struct jblocks *jblocks;
ufs2_daddr_t daddr;
int blocks;
{
struct jextent *jext;
jblocks->jb_blocks += blocks;
jblocks->jb_free += blocks;
jext = &jblocks->jb_extent[jblocks->jb_used];
/* Adding the first block. */
if (jext->je_daddr == 0) {
jext->je_daddr = daddr;
jext->je_blocks = blocks;
return;
}
/* Extending the last extent. */
if (jext->je_daddr + jext->je_blocks == daddr) {
jext->je_blocks += blocks;
return;
}
/* Adding a new extent. */
if (++jblocks->jb_used == jblocks->jb_avail) {
jblocks->jb_avail *= 2;
jext = malloc(sizeof(struct jextent) * jblocks->jb_avail,
M_JBLOCKS, M_WAITOK | M_ZERO);
memcpy(jext, jblocks->jb_extent,
sizeof(struct jextent) * jblocks->jb_used);
free(jblocks->jb_extent, M_JBLOCKS);
jblocks->jb_extent = jext;
}
jext = &jblocks->jb_extent[jblocks->jb_used];
jext->je_daddr = daddr;
jext->je_blocks = blocks;
return;
}
int
softdep_journal_lookup(mp, vpp)
struct mount *mp;
struct vnode **vpp;
{
struct componentname cnp;
struct vnode *dvp;
ino_t sujournal;
int error;
error = VFS_VGET(mp, UFS_ROOTINO, LK_EXCLUSIVE, &dvp);
if (error)
return (error);
bzero(&cnp, sizeof(cnp));
cnp.cn_nameiop = LOOKUP;
cnp.cn_flags = ISLASTCN;
cnp.cn_thread = curthread;
cnp.cn_cred = curthread->td_ucred;
cnp.cn_pnbuf = SUJ_FILE;
cnp.cn_nameptr = SUJ_FILE;
cnp.cn_namelen = strlen(SUJ_FILE);
error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal);
vput(dvp);
if (error != 0)
return (error);
error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp);
return (error);
}
/*
* Open and verify the journal file.
*/
static int
journal_mount(mp, fs, cred)
struct mount *mp;
struct fs *fs;
struct ucred *cred;
{
struct jblocks *jblocks;
struct ufsmount *ump;
struct vnode *vp;
struct inode *ip;
ufs2_daddr_t blkno;
int bcount;
int error;
int i;
ump = VFSTOUFS(mp);
ump->softdep_journal_tail = NULL;
ump->softdep_on_journal = 0;
ump->softdep_accdeps = 0;
ump->softdep_req = 0;
ump->softdep_jblocks = NULL;
error = softdep_journal_lookup(mp, &vp);
if (error != 0) {
printf("Failed to find journal. Use tunefs to create one\n");
return (error);
}
ip = VTOI(vp);
if (ip->i_size < SUJ_MIN) {
error = ENOSPC;
goto out;
}
bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */
jblocks = jblocks_create();
for (i = 0; i < bcount; i++) {
error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL);
if (error)
break;
jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag));
}
if (error) {
jblocks_destroy(jblocks);
goto out;
}
jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */
jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */
ump->softdep_jblocks = jblocks;
out:
if (error == 0) {
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_SUJ;
mp->mnt_flag &= ~MNT_SOFTDEP;
MNT_IUNLOCK(mp);
/*
* Only validate the journal contents if the
* filesystem is clean, otherwise we write the logs
* but they'll never be used. If the filesystem was
* still dirty when we mounted it the journal is
* invalid and a new journal can only be valid if it
* starts from a clean mount.
*/
if (fs->fs_clean) {
DIP_SET(ip, i_modrev, fs->fs_mtime);
ip->i_flags |= IN_MODIFIED;
ffs_update(vp, 1);
}
}
vput(vp);
return (error);
}
static void
journal_unmount(ump)
struct ufsmount *ump;
{
if (ump->softdep_jblocks)
jblocks_destroy(ump->softdep_jblocks);
ump->softdep_jblocks = NULL;
}
/*
* Called when a journal record is ready to be written. Space is allocated
* and the journal entry is created when the journal is flushed to stable
* store.
*/
static void
add_to_journal(wk)
struct worklist *wk;
{
struct ufsmount *ump;
ump = VFSTOUFS(wk->wk_mp);
LOCK_OWNED(ump);
if (wk->wk_state & ONWORKLIST)
panic("add_to_journal: %s(0x%X) already on list",
TYPENAME(wk->wk_type), wk->wk_state);
wk->wk_state |= ONWORKLIST | DEPCOMPLETE;
if (LIST_EMPTY(&ump->softdep_journal_pending)) {
ump->softdep_jblocks->jb_age = ticks;
LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list);
} else
LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list);
ump->softdep_journal_tail = wk;
ump->softdep_on_journal += 1;
}
/*
* Remove an arbitrary item for the journal worklist maintain the tail
* pointer. This happens when a new operation obviates the need to
* journal an old operation.
*/
static void
remove_from_journal(wk)
struct worklist *wk;
{
struct ufsmount *ump;
ump = VFSTOUFS(wk->wk_mp);
LOCK_OWNED(ump);
#ifdef INVARIANTS
{
struct worklist *wkn;
LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list)
if (wkn == wk)
break;
if (wkn == NULL)
panic("remove_from_journal: %p is not in journal", wk);
}
#endif
/*
* We emulate a TAILQ to save space in most structures which do not
* require TAILQ semantics. Here we must update the tail position
* when removing the tail which is not the final entry. This works
* only if the worklist linkage are at the beginning of the structure.
*/
if (ump->softdep_journal_tail == wk)
ump->softdep_journal_tail =
(struct worklist *)wk->wk_list.le_prev;
WORKLIST_REMOVE(wk);
ump->softdep_on_journal -= 1;
}
/*
* Check for journal space as well as dependency limits so the prelink
* code can throttle both journaled and non-journaled filesystems.
* Threshold is 0 for low and 1 for min.
*/
static int
journal_space(ump, thresh)
struct ufsmount *ump;
int thresh;
{
struct jblocks *jblocks;
int limit, avail;
jblocks = ump->softdep_jblocks;
if (jblocks == NULL)
return (1);
/*
* We use a tighter restriction here to prevent request_cleanup()
* running in threads from running into locks we currently hold.
* We have to be over the limit and our filesystem has to be
* responsible for more than our share of that usage.
*/
limit = (max_softdeps / 10) * 9;
if (dep_current[D_INODEDEP] > limit &&
ump->softdep_curdeps[D_INODEDEP] > limit / stat_flush_threads)
return (0);
if (thresh)
thresh = jblocks->jb_min;
else
thresh = jblocks->jb_low;
avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE;
avail = jblocks->jb_free - avail;
return (avail > thresh);
}
static void
journal_suspend(ump)
struct ufsmount *ump;
{
struct jblocks *jblocks;
struct mount *mp;
bool set;
mp = UFSTOVFS(ump);
if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0)
return;
jblocks = ump->softdep_jblocks;
vfs_op_enter(mp);
set = false;
MNT_ILOCK(mp);
if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) {
stat_journal_min++;
mp->mnt_kern_flag |= MNTK_SUSPEND;
mp->mnt_susp_owner = ump->softdep_flushtd;
set = true;
}
jblocks->jb_suspended = 1;
MNT_IUNLOCK(mp);
if (!set)
vfs_op_exit(mp);
}
static int
journal_unsuspend(struct ufsmount *ump)
{
struct jblocks *jblocks;
struct mount *mp;
mp = UFSTOVFS(ump);
jblocks = ump->softdep_jblocks;
if (jblocks != NULL && jblocks->jb_suspended &&
journal_space(ump, jblocks->jb_min)) {
jblocks->jb_suspended = 0;
FREE_LOCK(ump);
mp->mnt_susp_owner = curthread;
vfs_write_resume(mp, 0);
ACQUIRE_LOCK(ump);
return (1);
}
return (0);
}
/*
* Called before any allocation function to be certain that there is
* sufficient space in the journal prior to creating any new records.
* Since in the case of block allocation we may have multiple locked
* buffers at the time of the actual allocation we can not block
* when the journal records are created. Doing so would create a deadlock
* if any of these buffers needed to be flushed to reclaim space. Instead
* we require a sufficiently large amount of available space such that
* each thread in the system could have passed this allocation check and
* still have sufficient free space. With 20% of a minimum journal size
* of 1MB we have 6553 records available.
*/
int
softdep_prealloc(vp, waitok)
struct vnode *vp;
int waitok;
{
struct ufsmount *ump;
KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0,
("softdep_prealloc called on non-softdep filesystem"));
/*
* Nothing to do if we are not running journaled soft updates.
* If we currently hold the snapshot lock, we must avoid
* handling other resources that could cause deadlock. Do not
* touch quotas vnode since it is typically recursed with
* other vnode locks held.
*/
if (DOINGSUJ(vp) == 0 || IS_SNAPSHOT(VTOI(vp)) ||
(vp->v_vflag & VV_SYSTEM) != 0)
return (0);
ump = VFSTOUFS(vp->v_mount);
ACQUIRE_LOCK(ump);
if (journal_space(ump, 0)) {
FREE_LOCK(ump);
return (0);
}
stat_journal_low++;
FREE_LOCK(ump);
if (waitok == MNT_NOWAIT)
return (ENOSPC);
/*
* Attempt to sync this vnode once to flush any journal
* work attached to it.
*/
if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0)
ffs_syncvnode(vp, waitok, 0);
ACQUIRE_LOCK(ump);
process_removes(vp);
process_truncates(vp);
if (journal_space(ump, 0) == 0) {
softdep_speedup(ump);
if (journal_space(ump, 1) == 0)
journal_suspend(ump);
}
FREE_LOCK(ump);
return (0);
}
/*
* Before adjusting a link count on a vnode verify that we have sufficient
* journal space. If not, process operations that depend on the currently
* locked pair of vnodes to try to flush space as the syncer, buf daemon,
* and softdep flush threads can not acquire these locks to reclaim space.
*/
static void
softdep_prelink(dvp, vp)
struct vnode *dvp;
struct vnode *vp;
{
struct ufsmount *ump;
ump = VFSTOUFS(dvp->v_mount);
LOCK_OWNED(ump);
/*
* Nothing to do if we have sufficient journal space.
* If we currently hold the snapshot lock, we must avoid
* handling other resources that could cause deadlock.
*/
if (journal_space(ump, 0) || (vp && IS_SNAPSHOT(VTOI(vp))))
return;
stat_journal_low++;
FREE_LOCK(ump);
if (vp)
ffs_syncvnode(vp, MNT_NOWAIT, 0);
ffs_syncvnode(dvp, MNT_WAIT, 0);
ACQUIRE_LOCK(ump);
/* Process vp before dvp as it may create .. removes. */
if (vp) {
process_removes(vp);
process_truncates(vp);
}
process_removes(dvp);
process_truncates(dvp);
softdep_speedup(ump);
process_worklist_item(UFSTOVFS(ump), 2, LK_NOWAIT);
if (journal_space(ump, 0) == 0) {
softdep_speedup(ump);
if (journal_space(ump, 1) == 0)
journal_suspend(ump);
}
}
static void
jseg_write(ump, jseg, data)
struct ufsmount *ump;
struct jseg *jseg;
uint8_t *data;
{
struct jsegrec *rec;
rec = (struct jsegrec *)data;
rec->jsr_seq = jseg->js_seq;
rec->jsr_oldest = jseg->js_oldseq;
rec->jsr_cnt = jseg->js_cnt;
rec->jsr_blocks = jseg->js_size / ump->um_devvp->v_bufobj.bo_bsize;
rec->jsr_crc = 0;
rec->jsr_time = ump->um_fs->fs_mtime;
}
static inline void
inoref_write(inoref, jseg, rec)
struct inoref *inoref;
struct jseg *jseg;
struct jrefrec *rec;
{
inoref->if_jsegdep->jd_seg = jseg;
rec->jr_ino = inoref->if_ino;
rec->jr_parent = inoref->if_parent;
rec->jr_nlink = inoref->if_nlink;
rec->jr_mode = inoref->if_mode;
rec->jr_diroff = inoref->if_diroff;
}
static void
jaddref_write(jaddref, jseg, data)
struct jaddref *jaddref;
struct jseg *jseg;
uint8_t *data;
{
struct jrefrec *rec;
rec = (struct jrefrec *)data;
rec->jr_op = JOP_ADDREF;
inoref_write(&jaddref->ja_ref, jseg, rec);
}
static void
jremref_write(jremref, jseg, data)
struct jremref *jremref;
struct jseg *jseg;
uint8_t *data;
{
struct jrefrec *rec;
rec = (struct jrefrec *)data;
rec->jr_op = JOP_REMREF;
inoref_write(&jremref->jr_ref, jseg, rec);
}
static void
jmvref_write(jmvref, jseg, data)
struct jmvref *jmvref;
struct jseg *jseg;
uint8_t *data;
{
struct jmvrec *rec;
rec = (struct jmvrec *)data;
rec->jm_op = JOP_MVREF;
rec->jm_ino = jmvref->jm_ino;
rec->jm_parent = jmvref->jm_parent;
rec->jm_oldoff = jmvref->jm_oldoff;
rec->jm_newoff = jmvref->jm_newoff;
}
static void
jnewblk_write(jnewblk, jseg, data)
struct jnewblk *jnewblk;
struct jseg *jseg;
uint8_t *data;
{
struct jblkrec *rec;
jnewblk->jn_jsegdep->jd_seg = jseg;
rec = (struct jblkrec *)data;
rec->jb_op = JOP_NEWBLK;
rec->jb_ino = jnewblk->jn_ino;
rec->jb_blkno = jnewblk->jn_blkno;
rec->jb_lbn = jnewblk->jn_lbn;
rec->jb_frags = jnewblk->jn_frags;
rec->jb_oldfrags = jnewblk->jn_oldfrags;
}
static void
jfreeblk_write(jfreeblk, jseg, data)
struct jfreeblk *jfreeblk;
struct jseg *jseg;
uint8_t *data;
{
struct jblkrec *rec;
jfreeblk->jf_dep.jb_jsegdep->jd_seg = jseg;
rec = (struct jblkrec *)data;
rec->jb_op = JOP_FREEBLK;
rec->jb_ino = jfreeblk->jf_ino;
rec->jb_blkno = jfreeblk->jf_blkno;
rec->jb_lbn = jfreeblk->jf_lbn;
rec->jb_frags = jfreeblk->jf_frags;
rec->jb_oldfrags = 0;
}
static void
jfreefrag_write(jfreefrag, jseg, data)
struct jfreefrag *jfreefrag;
struct jseg *jseg;
uint8_t *data;
{
struct jblkrec *rec;
jfreefrag->fr_jsegdep->jd_seg = jseg;
rec = (struct jblkrec *)data;
rec->jb_op = JOP_FREEBLK;
rec->jb_ino = jfreefrag->fr_ino;
rec->jb_blkno = jfreefrag->fr_blkno;
rec->jb_lbn = jfreefrag->fr_lbn;
rec->jb_frags = jfreefrag->fr_frags;
rec->jb_oldfrags = 0;
}
static void
jtrunc_write(jtrunc, jseg, data)
struct jtrunc *jtrunc;
struct jseg *jseg;
uint8_t *data;
{
struct jtrncrec *rec;
jtrunc->jt_dep.jb_jsegdep->jd_seg = jseg;
rec = (struct jtrncrec *)data;
rec->jt_op = JOP_TRUNC;
rec->jt_ino = jtrunc->jt_ino;
rec->jt_size = jtrunc->jt_size;
rec->jt_extsize = jtrunc->jt_extsize;
}
static void
jfsync_write(jfsync, jseg, data)
struct jfsync *jfsync;
struct jseg *jseg;
uint8_t *data;
{
struct jtrncrec *rec;
rec = (struct jtrncrec *)data;
rec->jt_op = JOP_SYNC;
rec->jt_ino = jfsync->jfs_ino;
rec->jt_size = jfsync->jfs_size;
rec->jt_extsize = jfsync->jfs_extsize;
}
static void
softdep_flushjournal(mp)
struct mount *mp;
{
struct jblocks *jblocks;
struct ufsmount *ump;
if (MOUNTEDSUJ(mp) == 0)
return;
ump = VFSTOUFS(mp);
jblocks = ump->softdep_jblocks;
ACQUIRE_LOCK(ump);
while (ump->softdep_on_journal) {
jblocks->jb_needseg = 1;
softdep_process_journal(mp, NULL, MNT_WAIT);
}
FREE_LOCK(ump);
}
static void softdep_synchronize_completed(struct bio *);
static void softdep_synchronize(struct bio *, struct ufsmount *, void *);
static void
softdep_synchronize_completed(bp)
struct bio *bp;
{
struct jseg *oldest;
struct jseg *jseg;
struct ufsmount *ump;
/*
* caller1 marks the last segment written before we issued the
* synchronize cache.
*/
jseg = bp->bio_caller1;
if (jseg == NULL) {
g_destroy_bio(bp);
return;
}
ump = VFSTOUFS(jseg->js_list.wk_mp);
ACQUIRE_LOCK(ump);
oldest = NULL;
/*
* Mark all the journal entries waiting on the synchronize cache
* as completed so they may continue on.
*/
while (jseg != NULL && (jseg->js_state & COMPLETE) == 0) {
jseg->js_state |= COMPLETE;
oldest = jseg;
jseg = TAILQ_PREV(jseg, jseglst, js_next);
}
/*
* Restart deferred journal entry processing from the oldest
* completed jseg.
*/
if (oldest)
complete_jsegs(oldest);
FREE_LOCK(ump);
g_destroy_bio(bp);
}
/*
* Send BIO_FLUSH/SYNCHRONIZE CACHE to the device to enforce write ordering
* barriers. The journal must be written prior to any blocks that depend
* on it and the journal can not be released until the blocks have be
* written. This code handles both barriers simultaneously.
*/
static void
softdep_synchronize(bp, ump, caller1)
struct bio *bp;
struct ufsmount *ump;
void *caller1;
{
bp->bio_cmd = BIO_FLUSH;
bp->bio_flags |= BIO_ORDERED;
bp->bio_data = NULL;
bp->bio_offset = ump->um_cp->provider->mediasize;
bp->bio_length = 0;
bp->bio_done = softdep_synchronize_completed;
bp->bio_caller1 = caller1;
g_io_request(bp, ump->um_cp);
}
/*
* Flush some journal records to disk.
*/
static void
softdep_process_journal(mp, needwk, flags)
struct mount *mp;
struct worklist *needwk;
int flags;
{
struct jblocks *jblocks;
struct ufsmount *ump;
struct worklist *wk;
struct jseg *jseg;
struct buf *bp;
struct bio *bio;
uint8_t *data;
struct fs *fs;
int shouldflush;
int segwritten;
int jrecmin; /* Minimum records per block. */
int jrecmax; /* Maximum records per block. */
int size;
int cnt;
int off;
int devbsize;
if (MOUNTEDSUJ(mp) == 0)
return;
shouldflush = softdep_flushcache;
bio = NULL;
jseg = NULL;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
fs = ump->um_fs;
jblocks = ump->softdep_jblocks;
devbsize = ump->um_devvp->v_bufobj.bo_bsize;
/*
* We write anywhere between a disk block and fs block. The upper
* bound is picked to prevent buffer cache fragmentation and limit
* processing time per I/O.
*/
jrecmin = (devbsize / JREC_SIZE) - 1; /* -1 for seg header */
jrecmax = (fs->fs_bsize / devbsize) * jrecmin;
segwritten = 0;
for (;;) {
cnt = ump->softdep_on_journal;
/*
* Criteria for writing a segment:
* 1) We have a full block.
* 2) We're called from jwait() and haven't found the
* journal item yet.
* 3) Always write if needseg is set.
* 4) If we are called from process_worklist and have
* not yet written anything we write a partial block
* to enforce a 1 second maximum latency on journal
* entries.
*/
if (cnt < (jrecmax - 1) && needwk == NULL &&
jblocks->jb_needseg == 0 && (segwritten || cnt == 0))
break;
cnt++;
/*
* Verify some free journal space. softdep_prealloc() should
* guarantee that we don't run out so this is indicative of
* a problem with the flow control. Try to recover
* gracefully in any event.
*/
while (jblocks->jb_free == 0) {
if (flags != MNT_WAIT)
break;
printf("softdep: Out of journal space!\n");
softdep_speedup(ump);
msleep(jblocks, LOCK_PTR(ump), PRIBIO, "jblocks", hz);
}
FREE_LOCK(ump);
jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS);
workitem_alloc(&jseg->js_list, D_JSEG, mp);
LIST_INIT(&jseg->js_entries);
LIST_INIT(&jseg->js_indirs);
jseg->js_state = ATTACHED;
if (shouldflush == 0)
jseg->js_state |= COMPLETE;
else if (bio == NULL)
bio = g_alloc_bio();
jseg->js_jblocks = jblocks;
bp = geteblk(fs->fs_bsize, 0);
ACQUIRE_LOCK(ump);
/*
* If there was a race while we were allocating the block
* and jseg the entry we care about was likely written.
* We bail out in both the WAIT and NOWAIT case and assume
* the caller will loop if the entry it cares about is
* not written.
*/
cnt = ump->softdep_on_journal;
if (cnt + jblocks->jb_needseg == 0 || jblocks->jb_free == 0) {
bp->b_flags |= B_INVAL | B_NOCACHE;
WORKITEM_FREE(jseg, D_JSEG);
FREE_LOCK(ump);
brelse(bp);
ACQUIRE_LOCK(ump);
break;
}
/*
* Calculate the disk block size required for the available
* records rounded to the min size.
*/
if (cnt == 0)
size = devbsize;
else if (cnt < jrecmax)
size = howmany(cnt, jrecmin) * devbsize;
else
size = fs->fs_bsize;
/*
* Allocate a disk block for this journal data and account
* for truncation of the requested size if enough contiguous
* space was not available.
*/
bp->b_blkno = jblocks_alloc(jblocks, size, &size);
bp->b_lblkno = bp->b_blkno;
bp->b_offset = bp->b_blkno * DEV_BSIZE;
bp->b_bcount = size;
bp->b_flags &= ~B_INVAL;
bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY;
/*
* Initialize our jseg with cnt records. Assign the next
* sequence number to it and link it in-order.
*/
cnt = MIN(cnt, (size / devbsize) * jrecmin);
jseg->js_buf = bp;
jseg->js_cnt = cnt;
jseg->js_refs = cnt + 1; /* Self ref. */
jseg->js_size = size;
jseg->js_seq = jblocks->jb_nextseq++;
if (jblocks->jb_oldestseg == NULL)
jblocks->jb_oldestseg = jseg;
jseg->js_oldseq = jblocks->jb_oldestseg->js_seq;
TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next);
if (jblocks->jb_writeseg == NULL)
jblocks->jb_writeseg = jseg;
/*
* Start filling in records from the pending list.
*/
data = bp->b_data;
off = 0;
/*
* Always put a header on the first block.
* XXX As with below, there might not be a chance to get
* into the loop. Ensure that something valid is written.
*/
jseg_write(ump, jseg, data);
off += JREC_SIZE;
data = bp->b_data + off;
/*
* XXX Something is wrong here. There's no work to do,
* but we need to perform and I/O and allow it to complete
* anyways.
*/
if (LIST_EMPTY(&ump->softdep_journal_pending))
stat_emptyjblocks++;
while ((wk = LIST_FIRST(&ump->softdep_journal_pending))
!= NULL) {
if (cnt == 0)
break;
/* Place a segment header on every device block. */
if ((off % devbsize) == 0) {
jseg_write(ump, jseg, data);
off += JREC_SIZE;
data = bp->b_data + off;
}
if (wk == needwk)
needwk = NULL;
remove_from_journal(wk);
wk->wk_state |= INPROGRESS;
WORKLIST_INSERT(&jseg->js_entries, wk);
switch (wk->wk_type) {
case D_JADDREF:
jaddref_write(WK_JADDREF(wk), jseg, data);
break;
case D_JREMREF:
jremref_write(WK_JREMREF(wk), jseg, data);
break;
case D_JMVREF:
jmvref_write(WK_JMVREF(wk), jseg, data);
break;
case D_JNEWBLK:
jnewblk_write(WK_JNEWBLK(wk), jseg, data);
break;
case D_JFREEBLK:
jfreeblk_write(WK_JFREEBLK(wk), jseg, data);
break;
case D_JFREEFRAG:
jfreefrag_write(WK_JFREEFRAG(wk), jseg, data);
break;
case D_JTRUNC:
jtrunc_write(WK_JTRUNC(wk), jseg, data);
break;
case D_JFSYNC:
jfsync_write(WK_JFSYNC(wk), jseg, data);
break;
default:
panic("process_journal: Unknown type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
off += JREC_SIZE;
data = bp->b_data + off;
cnt--;
}
/* Clear any remaining space so we don't leak kernel data */
if (size > off)
bzero(data, size - off);
/*
* Write this one buffer and continue.
*/
segwritten = 1;
jblocks->jb_needseg = 0;
WORKLIST_INSERT(&bp->b_dep, &jseg->js_list);
FREE_LOCK(ump);
+ bp->b_xflags |= BX_CVTENXIO;
pbgetvp(ump->um_devvp, bp);
/*
* We only do the blocking wait once we find the journal
* entry we're looking for.
*/
if (needwk == NULL && flags == MNT_WAIT)
bwrite(bp);
else
bawrite(bp);
ACQUIRE_LOCK(ump);
}
/*
* If we wrote a segment issue a synchronize cache so the journal
* is reflected on disk before the data is written. Since reclaiming
* journal space also requires writing a journal record this
* process also enforces a barrier before reclamation.
*/
if (segwritten && shouldflush) {
softdep_synchronize(bio, ump,
TAILQ_LAST(&jblocks->jb_segs, jseglst));
} else if (bio)
g_destroy_bio(bio);
/*
* If we've suspended the filesystem because we ran out of journal
* space either try to sync it here to make some progress or
* unsuspend it if we already have.
*/
if (flags == 0 && jblocks->jb_suspended) {
if (journal_unsuspend(ump))
return;
FREE_LOCK(ump);
VFS_SYNC(mp, MNT_NOWAIT);
ffs_sbupdate(ump, MNT_WAIT, 0);
ACQUIRE_LOCK(ump);
}
}
/*
* Complete a jseg, allowing all dependencies awaiting journal writes
* to proceed. Each journal dependency also attaches a jsegdep to dependent
* structures so that the journal segment can be freed to reclaim space.
*/
static void
complete_jseg(jseg)
struct jseg *jseg;
{
struct worklist *wk;
struct jmvref *jmvref;
#ifdef INVARIANTS
int i = 0;
#endif
while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) {
WORKLIST_REMOVE(wk);
wk->wk_state &= ~INPROGRESS;
wk->wk_state |= COMPLETE;
KASSERT(i++ < jseg->js_cnt,
("handle_written_jseg: overflow %d >= %d",
i - 1, jseg->js_cnt));
switch (wk->wk_type) {
case D_JADDREF:
handle_written_jaddref(WK_JADDREF(wk));
break;
case D_JREMREF:
handle_written_jremref(WK_JREMREF(wk));
break;
case D_JMVREF:
rele_jseg(jseg); /* No jsegdep. */
jmvref = WK_JMVREF(wk);
LIST_REMOVE(jmvref, jm_deps);
if ((jmvref->jm_pagedep->pd_state & ONWORKLIST) == 0)
free_pagedep(jmvref->jm_pagedep);
WORKITEM_FREE(jmvref, D_JMVREF);
break;
case D_JNEWBLK:
handle_written_jnewblk(WK_JNEWBLK(wk));
break;
case D_JFREEBLK:
handle_written_jblkdep(&WK_JFREEBLK(wk)->jf_dep);
break;
case D_JTRUNC:
handle_written_jblkdep(&WK_JTRUNC(wk)->jt_dep);
break;
case D_JFSYNC:
rele_jseg(jseg); /* No jsegdep. */
WORKITEM_FREE(wk, D_JFSYNC);
break;
case D_JFREEFRAG:
handle_written_jfreefrag(WK_JFREEFRAG(wk));
break;
default:
panic("handle_written_jseg: Unknown type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
/* Release the self reference so the structure may be freed. */
rele_jseg(jseg);
}
/*
* Determine which jsegs are ready for completion processing. Waits for
* synchronize cache to complete as well as forcing in-order completion
* of journal entries.
*/
static void
complete_jsegs(jseg)
struct jseg *jseg;
{
struct jblocks *jblocks;
struct jseg *jsegn;
jblocks = jseg->js_jblocks;
/*
* Don't allow out of order completions. If this isn't the first
* block wait for it to write before we're done.
*/
if (jseg != jblocks->jb_writeseg)
return;
/* Iterate through available jsegs processing their entries. */
while (jseg && (jseg->js_state & ALLCOMPLETE) == ALLCOMPLETE) {
jblocks->jb_oldestwrseq = jseg->js_oldseq;
jsegn = TAILQ_NEXT(jseg, js_next);
complete_jseg(jseg);
jseg = jsegn;
}
jblocks->jb_writeseg = jseg;
/*
* Attempt to free jsegs now that oldestwrseq may have advanced.
*/
free_jsegs(jblocks);
}
/*
* Mark a jseg as DEPCOMPLETE and throw away the buffer. Attempt to handle
* the final completions.
*/
static void
handle_written_jseg(jseg, bp)
struct jseg *jseg;
struct buf *bp;
{
if (jseg->js_refs == 0)
panic("handle_written_jseg: No self-reference on %p", jseg);
jseg->js_state |= DEPCOMPLETE;
/*
* We'll never need this buffer again, set flags so it will be
* discarded.
*/
bp->b_flags |= B_INVAL | B_NOCACHE;
pbrelvp(bp);
complete_jsegs(jseg);
}
static inline struct jsegdep *
inoref_jseg(inoref)
struct inoref *inoref;
{
struct jsegdep *jsegdep;
jsegdep = inoref->if_jsegdep;
inoref->if_jsegdep = NULL;
return (jsegdep);
}
/*
* Called once a jremref has made it to stable store. The jremref is marked
* complete and we attempt to free it. Any pagedeps writes sleeping waiting
* for the jremref to complete will be awoken by free_jremref.
*/
static void
handle_written_jremref(jremref)
struct jremref *jremref;
{
struct inodedep *inodedep;
struct jsegdep *jsegdep;
struct dirrem *dirrem;
/* Grab the jsegdep. */
jsegdep = inoref_jseg(&jremref->jr_ref);
/*
* Remove us from the inoref list.
*/
if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino,
0, &inodedep) == 0)
panic("handle_written_jremref: Lost inodedep");
TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps);
/*
* Complete the dirrem.
*/
dirrem = jremref->jr_dirrem;
jremref->jr_dirrem = NULL;
LIST_REMOVE(jremref, jr_deps);
jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT;
jwork_insert(&dirrem->dm_jwork, jsegdep);
if (LIST_EMPTY(&dirrem->dm_jremrefhd) &&
(dirrem->dm_state & COMPLETE) != 0)
add_to_worklist(&dirrem->dm_list, 0);
free_jremref(jremref);
}
/*
* Called once a jaddref has made it to stable store. The dependency is
* marked complete and any dependent structures are added to the inode
* bufwait list to be completed as soon as it is written. If a bitmap write
* depends on this entry we move the inode into the inodedephd of the
* bmsafemap dependency and attempt to remove the jaddref from the bmsafemap.
*/
static void
handle_written_jaddref(jaddref)
struct jaddref *jaddref;
{
struct jsegdep *jsegdep;
struct inodedep *inodedep;
struct diradd *diradd;
struct mkdir *mkdir;
/* Grab the jsegdep. */
jsegdep = inoref_jseg(&jaddref->ja_ref);
mkdir = NULL;
diradd = NULL;
if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino,
0, &inodedep) == 0)
panic("handle_written_jaddref: Lost inodedep.");
if (jaddref->ja_diradd == NULL)
panic("handle_written_jaddref: No dependency");
if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) {
diradd = jaddref->ja_diradd;
WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list);
} else if (jaddref->ja_state & MKDIR_PARENT) {
mkdir = jaddref->ja_mkdir;
WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list);
} else if (jaddref->ja_state & MKDIR_BODY)
mkdir = jaddref->ja_mkdir;
else
panic("handle_written_jaddref: Unknown dependency %p",
jaddref->ja_diradd);
jaddref->ja_diradd = NULL; /* also clears ja_mkdir */
/*
* Remove us from the inode list.
*/
TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps);
/*
* The mkdir may be waiting on the jaddref to clear before freeing.
*/
if (mkdir) {
KASSERT(mkdir->md_list.wk_type == D_MKDIR,
("handle_written_jaddref: Incorrect type for mkdir %s",
TYPENAME(mkdir->md_list.wk_type)));
mkdir->md_jaddref = NULL;
diradd = mkdir->md_diradd;
mkdir->md_state |= DEPCOMPLETE;
complete_mkdir(mkdir);
}
jwork_insert(&diradd->da_jwork, jsegdep);
if (jaddref->ja_state & NEWBLOCK) {
inodedep->id_state |= ONDEPLIST;
LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd,
inodedep, id_deps);
}
free_jaddref(jaddref);
}
/*
* Called once a jnewblk journal is written. The allocdirect or allocindir
* is placed in the bmsafemap to await notification of a written bitmap. If
* the operation was canceled we add the segdep to the appropriate
* dependency to free the journal space once the canceling operation
* completes.
*/
static void
handle_written_jnewblk(jnewblk)
struct jnewblk *jnewblk;
{
struct bmsafemap *bmsafemap;
struct freefrag *freefrag;
struct freework *freework;
struct jsegdep *jsegdep;
struct newblk *newblk;
/* Grab the jsegdep. */
jsegdep = jnewblk->jn_jsegdep;
jnewblk->jn_jsegdep = NULL;
if (jnewblk->jn_dep == NULL)
panic("handle_written_jnewblk: No dependency for the segdep.");
switch (jnewblk->jn_dep->wk_type) {
case D_NEWBLK:
case D_ALLOCDIRECT:
case D_ALLOCINDIR:
/*
* Add the written block to the bmsafemap so it can
* be notified when the bitmap is on disk.
*/
newblk = WK_NEWBLK(jnewblk->jn_dep);
newblk->nb_jnewblk = NULL;
if ((newblk->nb_state & GOINGAWAY) == 0) {
bmsafemap = newblk->nb_bmsafemap;
newblk->nb_state |= ONDEPLIST;
LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk,
nb_deps);
}
jwork_insert(&newblk->nb_jwork, jsegdep);
break;
case D_FREEFRAG:
/*
* A newblock being removed by a freefrag when replaced by
* frag extension.
*/
freefrag = WK_FREEFRAG(jnewblk->jn_dep);
freefrag->ff_jdep = NULL;
jwork_insert(&freefrag->ff_jwork, jsegdep);
break;
case D_FREEWORK:
/*
* A direct block was removed by truncate.
*/
freework = WK_FREEWORK(jnewblk->jn_dep);
freework->fw_jnewblk = NULL;
jwork_insert(&freework->fw_freeblks->fb_jwork, jsegdep);
break;
default:
panic("handle_written_jnewblk: Unknown type %d.",
jnewblk->jn_dep->wk_type);
}
jnewblk->jn_dep = NULL;
free_jnewblk(jnewblk);
}
/*
* Cancel a jfreefrag that won't be needed, probably due to colliding with
* an in-flight allocation that has not yet been committed. Divorce us
* from the freefrag and mark it DEPCOMPLETE so that it may be added
* to the worklist.
*/
static void
cancel_jfreefrag(jfreefrag)
struct jfreefrag *jfreefrag;
{
struct freefrag *freefrag;
if (jfreefrag->fr_jsegdep) {
free_jsegdep(jfreefrag->fr_jsegdep);
jfreefrag->fr_jsegdep = NULL;
}
freefrag = jfreefrag->fr_freefrag;
jfreefrag->fr_freefrag = NULL;
free_jfreefrag(jfreefrag);
freefrag->ff_state |= DEPCOMPLETE;
CTR1(KTR_SUJ, "cancel_jfreefrag: blkno %jd", freefrag->ff_blkno);
}
/*
* Free a jfreefrag when the parent freefrag is rendered obsolete.
*/
static void
free_jfreefrag(jfreefrag)
struct jfreefrag *jfreefrag;
{
if (jfreefrag->fr_state & INPROGRESS)
WORKLIST_REMOVE(&jfreefrag->fr_list);
else if (jfreefrag->fr_state & ONWORKLIST)
remove_from_journal(&jfreefrag->fr_list);
if (jfreefrag->fr_freefrag != NULL)
panic("free_jfreefrag: Still attached to a freefrag.");
WORKITEM_FREE(jfreefrag, D_JFREEFRAG);
}
/*
* Called when the journal write for a jfreefrag completes. The parent
* freefrag is added to the worklist if this completes its dependencies.
*/
static void
handle_written_jfreefrag(jfreefrag)
struct jfreefrag *jfreefrag;
{
struct jsegdep *jsegdep;
struct freefrag *freefrag;
/* Grab the jsegdep. */
jsegdep = jfreefrag->fr_jsegdep;
jfreefrag->fr_jsegdep = NULL;
freefrag = jfreefrag->fr_freefrag;
if (freefrag == NULL)
panic("handle_written_jfreefrag: No freefrag.");
freefrag->ff_state |= DEPCOMPLETE;
freefrag->ff_jdep = NULL;
jwork_insert(&freefrag->ff_jwork, jsegdep);
if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE)
add_to_worklist(&freefrag->ff_list, 0);
jfreefrag->fr_freefrag = NULL;
free_jfreefrag(jfreefrag);
}
/*
* Called when the journal write for a jfreeblk completes. The jfreeblk
* is removed from the freeblks list of pending journal writes and the
* jsegdep is moved to the freeblks jwork to be completed when all blocks
* have been reclaimed.
*/
static void
handle_written_jblkdep(jblkdep)
struct jblkdep *jblkdep;
{
struct freeblks *freeblks;
struct jsegdep *jsegdep;
/* Grab the jsegdep. */
jsegdep = jblkdep->jb_jsegdep;
jblkdep->jb_jsegdep = NULL;
freeblks = jblkdep->jb_freeblks;
LIST_REMOVE(jblkdep, jb_deps);
jwork_insert(&freeblks->fb_jwork, jsegdep);
/*
* If the freeblks is all journaled, we can add it to the worklist.
*/
if (LIST_EMPTY(&freeblks->fb_jblkdephd) &&
(freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE)
add_to_worklist(&freeblks->fb_list, WK_NODELAY);
free_jblkdep(jblkdep);
}
static struct jsegdep *
newjsegdep(struct worklist *wk)
{
struct jsegdep *jsegdep;
jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS);
workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp);
jsegdep->jd_seg = NULL;
return (jsegdep);
}
static struct jmvref *
newjmvref(dp, ino, oldoff, newoff)
struct inode *dp;
ino_t ino;
off_t oldoff;
off_t newoff;
{
struct jmvref *jmvref;
jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS);
workitem_alloc(&jmvref->jm_list, D_JMVREF, ITOVFS(dp));
jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE;
jmvref->jm_parent = dp->i_number;
jmvref->jm_ino = ino;
jmvref->jm_oldoff = oldoff;
jmvref->jm_newoff = newoff;
return (jmvref);
}
/*
* Allocate a new jremref that tracks the removal of ip from dp with the
* directory entry offset of diroff. Mark the entry as ATTACHED and
* DEPCOMPLETE as we have all the information required for the journal write
* and the directory has already been removed from the buffer. The caller
* is responsible for linking the jremref into the pagedep and adding it
* to the journal to write. The MKDIR_PARENT flag is set if we're doing
* a DOTDOT addition so handle_workitem_remove() can properly assign
* the jsegdep when we're done.
*/
static struct jremref *
newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip,
off_t diroff, nlink_t nlink)
{
struct jremref *jremref;
jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS);
workitem_alloc(&jremref->jr_list, D_JREMREF, ITOVFS(dp));
jremref->jr_state = ATTACHED;
newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff,
nlink, ip->i_mode);
jremref->jr_dirrem = dirrem;
return (jremref);
}
static inline void
newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff,
nlink_t nlink, uint16_t mode)
{
inoref->if_jsegdep = newjsegdep(&inoref->if_list);
inoref->if_diroff = diroff;
inoref->if_ino = ino;
inoref->if_parent = parent;
inoref->if_nlink = nlink;
inoref->if_mode = mode;
}
/*
* Allocate a new jaddref to track the addition of ino to dp at diroff. The
* directory offset may not be known until later. The caller is responsible
* adding the entry to the journal when this information is available. nlink
* should be the link count prior to the addition and mode is only required
* to have the correct FMT.
*/
static struct jaddref *
newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink,
uint16_t mode)
{
struct jaddref *jaddref;
jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS);
workitem_alloc(&jaddref->ja_list, D_JADDREF, ITOVFS(dp));
jaddref->ja_state = ATTACHED;
jaddref->ja_mkdir = NULL;
newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode);
return (jaddref);
}
/*
* Create a new free dependency for a freework. The caller is responsible
* for adjusting the reference count when it has the lock held. The freedep
* will track an outstanding bitmap write that will ultimately clear the
* freework to continue.
*/
static struct freedep *
newfreedep(struct freework *freework)
{
struct freedep *freedep;
freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS);
workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp);
freedep->fd_freework = freework;
return (freedep);
}
/*
* Free a freedep structure once the buffer it is linked to is written. If
* this is the last reference to the freework schedule it for completion.
*/
static void
free_freedep(freedep)
struct freedep *freedep;
{
struct freework *freework;
freework = freedep->fd_freework;
freework->fw_freeblks->fb_cgwait--;
if (--freework->fw_ref == 0)
freework_enqueue(freework);
WORKITEM_FREE(freedep, D_FREEDEP);
}
/*
* Allocate a new freework structure that may be a level in an indirect
* when parent is not NULL or a top level block when it is. The top level
* freework structures are allocated without the per-filesystem lock held
* and before the freeblks is visible outside of softdep_setup_freeblocks().
*/
static struct freework *
newfreework(ump, freeblks, parent, lbn, nb, frags, off, journal)
struct ufsmount *ump;
struct freeblks *freeblks;
struct freework *parent;
ufs_lbn_t lbn;
ufs2_daddr_t nb;
int frags;
int off;
int journal;
{
struct freework *freework;
freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS);
workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp);
freework->fw_state = ATTACHED;
freework->fw_jnewblk = NULL;
freework->fw_freeblks = freeblks;
freework->fw_parent = parent;
freework->fw_lbn = lbn;
freework->fw_blkno = nb;
freework->fw_frags = frags;
freework->fw_indir = NULL;
freework->fw_ref = (MOUNTEDSUJ(UFSTOVFS(ump)) == 0 ||
lbn >= -UFS_NXADDR) ? 0 : NINDIR(ump->um_fs) + 1;
freework->fw_start = freework->fw_off = off;
if (journal)
newjfreeblk(freeblks, lbn, nb, frags);
if (parent == NULL) {
ACQUIRE_LOCK(ump);
WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list);
freeblks->fb_ref++;
FREE_LOCK(ump);
}
return (freework);
}
/*
* Eliminate a jfreeblk for a block that does not need journaling.
*/
static void
cancel_jfreeblk(freeblks, blkno)
struct freeblks *freeblks;
ufs2_daddr_t blkno;
{
struct jfreeblk *jfreeblk;
struct jblkdep *jblkdep;
LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) {
if (jblkdep->jb_list.wk_type != D_JFREEBLK)
continue;
jfreeblk = WK_JFREEBLK(&jblkdep->jb_list);
if (jfreeblk->jf_blkno == blkno)
break;
}
if (jblkdep == NULL)
return;
CTR1(KTR_SUJ, "cancel_jfreeblk: blkno %jd", blkno);
free_jsegdep(jblkdep->jb_jsegdep);
LIST_REMOVE(jblkdep, jb_deps);
WORKITEM_FREE(jfreeblk, D_JFREEBLK);
}
/*
* Allocate a new jfreeblk to journal top level block pointer when truncating
* a file. The caller must add this to the worklist when the per-filesystem
* lock is held.
*/
static struct jfreeblk *
newjfreeblk(freeblks, lbn, blkno, frags)
struct freeblks *freeblks;
ufs_lbn_t lbn;
ufs2_daddr_t blkno;
int frags;
{
struct jfreeblk *jfreeblk;
jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS);
workitem_alloc(&jfreeblk->jf_dep.jb_list, D_JFREEBLK,
freeblks->fb_list.wk_mp);
jfreeblk->jf_dep.jb_jsegdep = newjsegdep(&jfreeblk->jf_dep.jb_list);
jfreeblk->jf_dep.jb_freeblks = freeblks;
jfreeblk->jf_ino = freeblks->fb_inum;
jfreeblk->jf_lbn = lbn;
jfreeblk->jf_blkno = blkno;
jfreeblk->jf_frags = frags;
LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jfreeblk->jf_dep, jb_deps);
return (jfreeblk);
}
/*
* The journal is only prepared to handle full-size block numbers, so we
* have to adjust the record to reflect the change to a full-size block.
* For example, suppose we have a block made up of fragments 8-15 and
* want to free its last two fragments. We are given a request that says:
* FREEBLK ino=5, blkno=14, lbn=0, frags=2, oldfrags=0
* where frags are the number of fragments to free and oldfrags are the
* number of fragments to keep. To block align it, we have to change it to
* have a valid full-size blkno, so it becomes:
* FREEBLK ino=5, blkno=8, lbn=0, frags=2, oldfrags=6
*/
static void
adjust_newfreework(freeblks, frag_offset)
struct freeblks *freeblks;
int frag_offset;
{
struct jfreeblk *jfreeblk;
KASSERT((LIST_FIRST(&freeblks->fb_jblkdephd) != NULL &&
LIST_FIRST(&freeblks->fb_jblkdephd)->jb_list.wk_type == D_JFREEBLK),
("adjust_newfreework: Missing freeblks dependency"));
jfreeblk = WK_JFREEBLK(LIST_FIRST(&freeblks->fb_jblkdephd));
jfreeblk->jf_blkno -= frag_offset;
jfreeblk->jf_frags += frag_offset;
}
/*
* Allocate a new jtrunc to track a partial truncation.
*/
static struct jtrunc *
newjtrunc(freeblks, size, extsize)
struct freeblks *freeblks;
off_t size;
int extsize;
{
struct jtrunc *jtrunc;
jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS);
workitem_alloc(&jtrunc->jt_dep.jb_list, D_JTRUNC,
freeblks->fb_list.wk_mp);
jtrunc->jt_dep.jb_jsegdep = newjsegdep(&jtrunc->jt_dep.jb_list);
jtrunc->jt_dep.jb_freeblks = freeblks;
jtrunc->jt_ino = freeblks->fb_inum;
jtrunc->jt_size = size;
jtrunc->jt_extsize = extsize;
LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jtrunc->jt_dep, jb_deps);
return (jtrunc);
}
/*
* If we're canceling a new bitmap we have to search for another ref
* to move into the bmsafemap dep. This might be better expressed
* with another structure.
*/
static void
move_newblock_dep(jaddref, inodedep)
struct jaddref *jaddref;
struct inodedep *inodedep;
{
struct inoref *inoref;
struct jaddref *jaddrefn;
jaddrefn = NULL;
for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref;
inoref = TAILQ_NEXT(inoref, if_deps)) {
if ((jaddref->ja_state & NEWBLOCK) &&
inoref->if_list.wk_type == D_JADDREF) {
jaddrefn = (struct jaddref *)inoref;
break;
}
}
if (jaddrefn == NULL)
return;
jaddrefn->ja_state &= ~(ATTACHED | UNDONE);
jaddrefn->ja_state |= jaddref->ja_state &
(ATTACHED | UNDONE | NEWBLOCK);
jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK);
jaddref->ja_state |= ATTACHED;
LIST_REMOVE(jaddref, ja_bmdeps);
LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn,
ja_bmdeps);
}
/*
* Cancel a jaddref either before it has been written or while it is being
* written. This happens when a link is removed before the add reaches
* the disk. The jaddref dependency is kept linked into the bmsafemap
* and inode to prevent the link count or bitmap from reaching the disk
* until handle_workitem_remove() re-adjusts the counts and bitmaps as
* required.
*
* Returns 1 if the canceled addref requires journaling of the remove and
* 0 otherwise.
*/
static int
cancel_jaddref(jaddref, inodedep, wkhd)
struct jaddref *jaddref;
struct inodedep *inodedep;
struct workhead *wkhd;
{
struct inoref *inoref;
struct jsegdep *jsegdep;
int needsj;
KASSERT((jaddref->ja_state & COMPLETE) == 0,
("cancel_jaddref: Canceling complete jaddref"));
if (jaddref->ja_state & (INPROGRESS | COMPLETE))
needsj = 1;
else
needsj = 0;
if (inodedep == NULL)
if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino,
0, &inodedep) == 0)
panic("cancel_jaddref: Lost inodedep");
/*
* We must adjust the nlink of any reference operation that follows
* us so that it is consistent with the in-memory reference. This
* ensures that inode nlink rollbacks always have the correct link.
*/
if (needsj == 0) {
for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref;
inoref = TAILQ_NEXT(inoref, if_deps)) {
if (inoref->if_state & GOINGAWAY)
break;
inoref->if_nlink--;
}
}
jsegdep = inoref_jseg(&jaddref->ja_ref);
if (jaddref->ja_state & NEWBLOCK)
move_newblock_dep(jaddref, inodedep);
wake_worklist(&jaddref->ja_list);
jaddref->ja_mkdir = NULL;
if (jaddref->ja_state & INPROGRESS) {
jaddref->ja_state &= ~INPROGRESS;
WORKLIST_REMOVE(&jaddref->ja_list);
jwork_insert(wkhd, jsegdep);
} else {
free_jsegdep(jsegdep);
if (jaddref->ja_state & DEPCOMPLETE)
remove_from_journal(&jaddref->ja_list);
}
jaddref->ja_state |= (GOINGAWAY | DEPCOMPLETE);
/*
* Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove
* can arrange for them to be freed with the bitmap. Otherwise we
* no longer need this addref attached to the inoreflst and it
* will incorrectly adjust nlink if we leave it.
*/
if ((jaddref->ja_state & NEWBLOCK) == 0) {
TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref,
if_deps);
jaddref->ja_state |= COMPLETE;
free_jaddref(jaddref);
return (needsj);
}
/*
* Leave the head of the list for jsegdeps for fast merging.
*/
if (LIST_FIRST(wkhd) != NULL) {
jaddref->ja_state |= ONWORKLIST;
LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list);
} else
WORKLIST_INSERT(wkhd, &jaddref->ja_list);
return (needsj);
}
/*
* Attempt to free a jaddref structure when some work completes. This
* should only succeed once the entry is written and all dependencies have
* been notified.
*/
static void
free_jaddref(jaddref)
struct jaddref *jaddref;
{
if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
if (jaddref->ja_ref.if_jsegdep)
panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n",
jaddref, jaddref->ja_state);
if (jaddref->ja_state & NEWBLOCK)
LIST_REMOVE(jaddref, ja_bmdeps);
if (jaddref->ja_state & (INPROGRESS | ONWORKLIST))
panic("free_jaddref: Bad state %p(0x%X)",
jaddref, jaddref->ja_state);
if (jaddref->ja_mkdir != NULL)
panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state);
WORKITEM_FREE(jaddref, D_JADDREF);
}
/*
* Free a jremref structure once it has been written or discarded.
*/
static void
free_jremref(jremref)
struct jremref *jremref;
{
if (jremref->jr_ref.if_jsegdep)
free_jsegdep(jremref->jr_ref.if_jsegdep);
if (jremref->jr_state & INPROGRESS)
panic("free_jremref: IO still pending");
WORKITEM_FREE(jremref, D_JREMREF);
}
/*
* Free a jnewblk structure.
*/
static void
free_jnewblk(jnewblk)
struct jnewblk *jnewblk;
{
if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
LIST_REMOVE(jnewblk, jn_deps);
if (jnewblk->jn_dep != NULL)
panic("free_jnewblk: Dependency still attached.");
WORKITEM_FREE(jnewblk, D_JNEWBLK);
}
/*
* Cancel a jnewblk which has been been made redundant by frag extension.
*/
static void
cancel_jnewblk(jnewblk, wkhd)
struct jnewblk *jnewblk;
struct workhead *wkhd;
{
struct jsegdep *jsegdep;
CTR1(KTR_SUJ, "cancel_jnewblk: blkno %jd", jnewblk->jn_blkno);
jsegdep = jnewblk->jn_jsegdep;
if (jnewblk->jn_jsegdep == NULL || jnewblk->jn_dep == NULL)
panic("cancel_jnewblk: Invalid state");
jnewblk->jn_jsegdep = NULL;
jnewblk->jn_dep = NULL;
jnewblk->jn_state |= GOINGAWAY;
if (jnewblk->jn_state & INPROGRESS) {
jnewblk->jn_state &= ~INPROGRESS;
WORKLIST_REMOVE(&jnewblk->jn_list);
jwork_insert(wkhd, jsegdep);
} else {
free_jsegdep(jsegdep);
remove_from_journal(&jnewblk->jn_list);
}
wake_worklist(&jnewblk->jn_list);
WORKLIST_INSERT(wkhd, &jnewblk->jn_list);
}
static void
free_jblkdep(jblkdep)
struct jblkdep *jblkdep;
{
if (jblkdep->jb_list.wk_type == D_JFREEBLK)
WORKITEM_FREE(jblkdep, D_JFREEBLK);
else if (jblkdep->jb_list.wk_type == D_JTRUNC)
WORKITEM_FREE(jblkdep, D_JTRUNC);
else
panic("free_jblkdep: Unexpected type %s",
TYPENAME(jblkdep->jb_list.wk_type));
}
/*
* Free a single jseg once it is no longer referenced in memory or on
* disk. Reclaim journal blocks and dependencies waiting for the segment
* to disappear.
*/
static void
free_jseg(jseg, jblocks)
struct jseg *jseg;
struct jblocks *jblocks;
{
struct freework *freework;
/*
* Free freework structures that were lingering to indicate freed
* indirect blocks that forced journal write ordering on reallocate.
*/
while ((freework = LIST_FIRST(&jseg->js_indirs)) != NULL)
indirblk_remove(freework);
if (jblocks->jb_oldestseg == jseg)
jblocks->jb_oldestseg = TAILQ_NEXT(jseg, js_next);
TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next);
jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size);
KASSERT(LIST_EMPTY(&jseg->js_entries),
("free_jseg: Freed jseg has valid entries."));
WORKITEM_FREE(jseg, D_JSEG);
}
/*
* Free all jsegs that meet the criteria for being reclaimed and update
* oldestseg.
*/
static void
free_jsegs(jblocks)
struct jblocks *jblocks;
{
struct jseg *jseg;
/*
* Free only those jsegs which have none allocated before them to
* preserve the journal space ordering.
*/
while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) {
/*
* Only reclaim space when nothing depends on this journal
* set and another set has written that it is no longer
* valid.
*/
if (jseg->js_refs != 0) {
jblocks->jb_oldestseg = jseg;
return;
}
if ((jseg->js_state & ALLCOMPLETE) != ALLCOMPLETE)
break;
if (jseg->js_seq > jblocks->jb_oldestwrseq)
break;
/*
* We can free jsegs that didn't write entries when
* oldestwrseq == js_seq.
*/
if (jseg->js_seq == jblocks->jb_oldestwrseq &&
jseg->js_cnt != 0)
break;
free_jseg(jseg, jblocks);
}
/*
* If we exited the loop above we still must discover the
* oldest valid segment.
*/
if (jseg)
for (jseg = jblocks->jb_oldestseg; jseg != NULL;
jseg = TAILQ_NEXT(jseg, js_next))
if (jseg->js_refs != 0)
break;
jblocks->jb_oldestseg = jseg;
/*
* The journal has no valid records but some jsegs may still be
* waiting on oldestwrseq to advance. We force a small record
* out to permit these lingering records to be reclaimed.
*/
if (jblocks->jb_oldestseg == NULL && !TAILQ_EMPTY(&jblocks->jb_segs))
jblocks->jb_needseg = 1;
}
/*
* Release one reference to a jseg and free it if the count reaches 0. This
* should eventually reclaim journal space as well.
*/
static void
rele_jseg(jseg)
struct jseg *jseg;
{
KASSERT(jseg->js_refs > 0,
("free_jseg: Invalid refcnt %d", jseg->js_refs));
if (--jseg->js_refs != 0)
return;
free_jsegs(jseg->js_jblocks);
}
/*
* Release a jsegdep and decrement the jseg count.
*/
static void
free_jsegdep(jsegdep)
struct jsegdep *jsegdep;
{
if (jsegdep->jd_seg)
rele_jseg(jsegdep->jd_seg);
WORKITEM_FREE(jsegdep, D_JSEGDEP);
}
/*
* Wait for a journal item to make it to disk. Initiate journal processing
* if required.
*/
static int
jwait(wk, waitfor)
struct worklist *wk;
int waitfor;
{
LOCK_OWNED(VFSTOUFS(wk->wk_mp));
/*
* Blocking journal waits cause slow synchronous behavior. Record
* stats on the frequency of these blocking operations.
*/
if (waitfor == MNT_WAIT) {
stat_journal_wait++;
switch (wk->wk_type) {
case D_JREMREF:
case D_JMVREF:
stat_jwait_filepage++;
break;
case D_JTRUNC:
case D_JFREEBLK:
stat_jwait_freeblks++;
break;
case D_JNEWBLK:
stat_jwait_newblk++;
break;
case D_JADDREF:
stat_jwait_inode++;
break;
default:
break;
}
}
/*
* If IO has not started we process the journal. We can't mark the
* worklist item as IOWAITING because we drop the lock while
* processing the journal and the worklist entry may be freed after
* this point. The caller may call back in and re-issue the request.
*/
if ((wk->wk_state & INPROGRESS) == 0) {
softdep_process_journal(wk->wk_mp, wk, waitfor);
if (waitfor != MNT_WAIT)
return (EBUSY);
return (0);
}
if (waitfor != MNT_WAIT)
return (EBUSY);
wait_worklist(wk, "jwait");
return (0);
}
/*
* Lookup an inodedep based on an inode pointer and set the nlinkdelta as
* appropriate. This is a convenience function to reduce duplicate code
* for the setup and revert functions below.
*/
static struct inodedep *
inodedep_lookup_ip(ip)
struct inode *ip;
{
struct inodedep *inodedep;
KASSERT(ip->i_nlink >= ip->i_effnlink,
("inodedep_lookup_ip: bad delta"));
(void) inodedep_lookup(ITOVFS(ip), ip->i_number, DEPALLOC,
&inodedep);
inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked"));
return (inodedep);
}
/*
* Called prior to creating a new inode and linking it to a directory. The
* jaddref structure must already be allocated by softdep_setup_inomapdep
* and it is discovered here so we can initialize the mode and update
* nlinkdelta.
*/
void
softdep_setup_create(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_create called on non-softdep filesystem"));
KASSERT(ip->i_nlink == 1,
("softdep_setup_create: Invalid link count."));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(ip);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
("softdep_setup_create: No addref structure present."));
}
softdep_prelink(dvp, NULL);
FREE_LOCK(ITOUMP(dp));
}
/*
* Create a jaddref structure to track the addition of a DOTDOT link when
* we are reparenting an inode as part of a rename. This jaddref will be
* found by softdep_setup_directory_change. Adjusts nlinkdelta for
* non-journaling softdep.
*/
void
softdep_setup_dotdot_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_dotdot_link called on non-softdep filesystem"));
dvp = ITOV(dp);
jaddref = NULL;
/*
* We don't set MKDIR_PARENT as this is not tied to a mkdir and
* is used as a normal link would be.
*/
if (DOINGSUJ(dvp))
jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET,
dp->i_effnlink - 1, dp->i_mode);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(dp);
if (jaddref)
TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
if_deps);
softdep_prelink(dvp, ITOV(ip));
FREE_LOCK(ITOUMP(dp));
}
/*
* Create a jaddref structure to track a new link to an inode. The directory
* offset is not known until softdep_setup_directory_add or
* softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling
* softdep.
*/
void
softdep_setup_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_link called on non-softdep filesystem"));
dvp = ITOV(dp);
jaddref = NULL;
if (DOINGSUJ(dvp))
jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1,
ip->i_mode);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(ip);
if (jaddref)
TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
if_deps);
softdep_prelink(dvp, ITOV(ip));
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to create the jaddref structures to track . and .. references as
* well as lookup and further initialize the incomplete jaddref created
* by softdep_setup_inomapdep when the inode was allocated. Adjusts
* nlinkdelta for non-journaling softdep.
*/
void
softdep_setup_mkdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *dotdotaddref;
struct jaddref *dotaddref;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_mkdir called on non-softdep filesystem"));
dvp = ITOV(dp);
dotaddref = dotdotaddref = NULL;
if (DOINGSUJ(dvp)) {
dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1,
ip->i_mode);
dotaddref->ja_state |= MKDIR_BODY;
dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET,
dp->i_effnlink - 1, dp->i_mode);
dotdotaddref->ja_state |= MKDIR_PARENT;
}
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(ip);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref != NULL,
("softdep_setup_mkdir: No addref structure present."));
KASSERT(jaddref->ja_parent == dp->i_number,
("softdep_setup_mkdir: bad parent %ju",
(uintmax_t)jaddref->ja_parent));
TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref,
if_deps);
}
inodedep = inodedep_lookup_ip(dp);
if (DOINGSUJ(dvp))
TAILQ_INSERT_TAIL(&inodedep->id_inoreflst,
&dotdotaddref->ja_ref, if_deps);
softdep_prelink(ITOV(dp), NULL);
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to track nlinkdelta of the inode and parent directories prior to
* unlinking a directory.
*/
void
softdep_setup_rmdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_rmdir called on non-softdep filesystem"));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
(void) inodedep_lookup_ip(ip);
(void) inodedep_lookup_ip(dp);
softdep_prelink(dvp, ITOV(ip));
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to track nlinkdelta of the inode and parent directories prior to
* unlink.
*/
void
softdep_setup_unlink(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_setup_unlink called on non-softdep filesystem"));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
(void) inodedep_lookup_ip(ip);
(void) inodedep_lookup_ip(dp);
softdep_prelink(dvp, ITOV(ip));
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to release the journal structures created by a failed non-directory
* creation. Adjusts nlinkdelta for non-journaling softdep.
*/
void
softdep_revert_create(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS((dp))) != 0,
("softdep_revert_create called on non-softdep filesystem"));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(ip);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref->ja_parent == dp->i_number,
("softdep_revert_create: addref parent mismatch"));
cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
}
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to release the journal structures created by a failed link
* addition. Adjusts nlinkdelta for non-journaling softdep.
*/
void
softdep_revert_link(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_revert_link called on non-softdep filesystem"));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(ip);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref->ja_parent == dp->i_number,
("softdep_revert_link: addref parent mismatch"));
cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
}
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to release the journal structures created by a failed mkdir
* attempt. Adjusts nlinkdelta for non-journaling softdep.
*/
void
softdep_revert_mkdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct jaddref *dotaddref;
struct vnode *dvp;
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_revert_mkdir called on non-softdep filesystem"));
dvp = ITOV(dp);
ACQUIRE_LOCK(ITOUMP(dp));
inodedep = inodedep_lookup_ip(dp);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref->ja_parent == ip->i_number,
("softdep_revert_mkdir: dotdot addref parent mismatch"));
cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
}
inodedep = inodedep_lookup_ip(ip);
if (DOINGSUJ(dvp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref->ja_parent == dp->i_number,
("softdep_revert_mkdir: addref parent mismatch"));
dotaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref,
inoreflst, if_deps);
cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait);
KASSERT(dotaddref->ja_parent == ip->i_number,
("softdep_revert_mkdir: dot addref parent mismatch"));
cancel_jaddref(dotaddref, inodedep, &inodedep->id_inowait);
}
FREE_LOCK(ITOUMP(dp));
}
/*
* Called to correct nlinkdelta after a failed rmdir.
*/
void
softdep_revert_rmdir(dp, ip)
struct inode *dp;
struct inode *ip;
{
KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0,
("softdep_revert_rmdir called on non-softdep filesystem"));
ACQUIRE_LOCK(ITOUMP(dp));
(void) inodedep_lookup_ip(ip);
(void) inodedep_lookup_ip(dp);
FREE_LOCK(ITOUMP(dp));
}
/*
* Protecting the freemaps (or bitmaps).
*
* To eliminate the need to execute fsck before mounting a filesystem
* after a power failure, one must (conservatively) guarantee that the
* on-disk copy of the bitmaps never indicate that a live inode or block is
* free. So, when a block or inode is allocated, the bitmap should be
* updated (on disk) before any new pointers. When a block or inode is
* freed, the bitmap should not be updated until all pointers have been
* reset. The latter dependency is handled by the delayed de-allocation
* approach described below for block and inode de-allocation. The former
* dependency is handled by calling the following procedure when a block or
* inode is allocated. When an inode is allocated an "inodedep" is created
* with its DEPCOMPLETE flag cleared until its bitmap is written to disk.
* Each "inodedep" is also inserted into the hash indexing structure so
* that any additional link additions can be made dependent on the inode
* allocation.
*
* The ufs filesystem maintains a number of free block counts (e.g., per
* cylinder group, per cylinder and per <cylinder, rotational position> pair)
* in addition to the bitmaps. These counts are used to improve efficiency
* during allocation and therefore must be consistent with the bitmaps.
* There is no convenient way to guarantee post-crash consistency of these
* counts with simple update ordering, for two main reasons: (1) The counts
* and bitmaps for a single cylinder group block are not in the same disk
* sector. If a disk write is interrupted (e.g., by power failure), one may
* be written and the other not. (2) Some of the counts are located in the
* superblock rather than the cylinder group block. So, we focus our soft
* updates implementation on protecting the bitmaps. When mounting a
* filesystem, we recompute the auxiliary counts from the bitmaps.
*/
/*
* Called just after updating the cylinder group block to allocate an inode.
*/
void
softdep_setup_inomapdep(bp, ip, newinum, mode)
struct buf *bp; /* buffer for cylgroup block with inode map */
struct inode *ip; /* inode related to allocation */
ino_t newinum; /* new inode number being allocated */
int mode;
{
struct inodedep *inodedep;
struct bmsafemap *bmsafemap;
struct jaddref *jaddref;
struct mount *mp;
struct fs *fs;
mp = ITOVFS(ip);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_inomapdep called on non-softdep filesystem"));
fs = VFSTOUFS(mp)->um_fs;
jaddref = NULL;
/*
* Allocate the journal reference add structure so that the bitmap
* can be dependent on it.
*/
if (MOUNTEDSUJ(mp)) {
jaddref = newjaddref(ip, newinum, 0, 0, mode);
jaddref->ja_state |= NEWBLOCK;
}
/*
* Create a dependency for the newly allocated inode.
* Panic if it already exists as something is seriously wrong.
* Otherwise add it to the dependency list for the buffer holding
* the cylinder group map from which it was allocated.
*
* We have to preallocate a bmsafemap entry in case it is needed
* in bmsafemap_lookup since once we allocate the inodedep, we
* have to finish initializing it before we can FREE_LOCK().
* By preallocating, we avoid FREE_LOCK() while doing a malloc
* in bmsafemap_lookup. We cannot call bmsafemap_lookup before
* creating the inodedep as it can be freed during the time
* that we FREE_LOCK() while allocating the inodedep. We must
* call workitem_alloc() before entering the locked section as
* it also acquires the lock and we must avoid trying doing so
* recursively.
*/
bmsafemap = malloc(sizeof(struct bmsafemap),
M_BMSAFEMAP, M_SOFTDEP_FLAGS);
workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp);
ACQUIRE_LOCK(ITOUMP(ip));
if ((inodedep_lookup(mp, newinum, DEPALLOC, &inodedep)))
panic("softdep_setup_inomapdep: dependency %p for new"
"inode already exists", inodedep);
bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum), bmsafemap);
if (jaddref) {
LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps);
TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref,
if_deps);
} else {
inodedep->id_state |= ONDEPLIST;
LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps);
}
inodedep->id_bmsafemap = bmsafemap;
inodedep->id_state &= ~DEPCOMPLETE;
FREE_LOCK(ITOUMP(ip));
}
/*
* Called just after updating the cylinder group block to
* allocate block or fragment.
*/
void
softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags)
struct buf *bp; /* buffer for cylgroup block with block map */
struct mount *mp; /* filesystem doing allocation */
ufs2_daddr_t newblkno; /* number of newly allocated block */
int frags; /* Number of fragments. */
int oldfrags; /* Previous number of fragments for extend. */
{
struct newblk *newblk;
struct bmsafemap *bmsafemap;
struct jnewblk *jnewblk;
struct ufsmount *ump;
struct fs *fs;
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_blkmapdep called on non-softdep filesystem"));
ump = VFSTOUFS(mp);
fs = ump->um_fs;
jnewblk = NULL;
/*
* Create a dependency for the newly allocated block.
* Add it to the dependency list for the buffer holding
* the cylinder group map from which it was allocated.
*/
if (MOUNTEDSUJ(mp)) {
jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS);
workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp);
jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list);
jnewblk->jn_state = ATTACHED;
jnewblk->jn_blkno = newblkno;
jnewblk->jn_frags = frags;
jnewblk->jn_oldfrags = oldfrags;
#ifdef INVARIANTS
{
struct cg *cgp;
uint8_t *blksfree;
long bno;
int i;
cgp = (struct cg *)bp->b_data;
blksfree = cg_blksfree(cgp);
bno = dtogd(fs, jnewblk->jn_blkno);
for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags;
i++) {
if (isset(blksfree, bno + i))
panic("softdep_setup_blkmapdep: "
"free fragment %d from %d-%d "
"state 0x%X dep %p", i,
jnewblk->jn_oldfrags,
jnewblk->jn_frags,
jnewblk->jn_state,
jnewblk->jn_dep);
}
}
#endif
}
CTR3(KTR_SUJ,
"softdep_setup_blkmapdep: blkno %jd frags %d oldfrags %d",
newblkno, frags, oldfrags);
ACQUIRE_LOCK(ump);
if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0)
panic("softdep_setup_blkmapdep: found block");
newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp,
dtog(fs, newblkno), NULL);
if (jnewblk) {
jnewblk->jn_dep = (struct worklist *)newblk;
LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps);
} else {
newblk->nb_state |= ONDEPLIST;
LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps);
}
newblk->nb_bmsafemap = bmsafemap;
newblk->nb_jnewblk = jnewblk;
FREE_LOCK(ump);
}
#define BMSAFEMAP_HASH(ump, cg) \
(&(ump)->bmsafemap_hashtbl[(cg) & (ump)->bmsafemap_hash_size])
static int
bmsafemap_find(bmsafemaphd, cg, bmsafemapp)
struct bmsafemap_hashhead *bmsafemaphd;
int cg;
struct bmsafemap **bmsafemapp;
{
struct bmsafemap *bmsafemap;
LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash)
if (bmsafemap->sm_cg == cg)
break;
if (bmsafemap) {
*bmsafemapp = bmsafemap;
return (1);
}
*bmsafemapp = NULL;
return (0);
}
/*
* Find the bmsafemap associated with a cylinder group buffer.
* If none exists, create one. The buffer must be locked when
* this routine is called and this routine must be called with
* the softdep lock held. To avoid giving up the lock while
* allocating a new bmsafemap, a preallocated bmsafemap may be
* provided. If it is provided but not needed, it is freed.
*/
static struct bmsafemap *
bmsafemap_lookup(mp, bp, cg, newbmsafemap)
struct mount *mp;
struct buf *bp;
int cg;
struct bmsafemap *newbmsafemap;
{
struct bmsafemap_hashhead *bmsafemaphd;
struct bmsafemap *bmsafemap, *collision;
struct worklist *wk;
struct ufsmount *ump;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
KASSERT(bp != NULL, ("bmsafemap_lookup: missing buffer"));
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
if (wk->wk_type == D_BMSAFEMAP) {
if (newbmsafemap)
WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP);
return (WK_BMSAFEMAP(wk));
}
}
bmsafemaphd = BMSAFEMAP_HASH(ump, cg);
if (bmsafemap_find(bmsafemaphd, cg, &bmsafemap) == 1) {
if (newbmsafemap)
WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP);
return (bmsafemap);
}
if (newbmsafemap) {
bmsafemap = newbmsafemap;
} else {
FREE_LOCK(ump);
bmsafemap = malloc(sizeof(struct bmsafemap),
M_BMSAFEMAP, M_SOFTDEP_FLAGS);
workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp);
ACQUIRE_LOCK(ump);
}
bmsafemap->sm_buf = bp;
LIST_INIT(&bmsafemap->sm_inodedephd);
LIST_INIT(&bmsafemap->sm_inodedepwr);
LIST_INIT(&bmsafemap->sm_newblkhd);
LIST_INIT(&bmsafemap->sm_newblkwr);
LIST_INIT(&bmsafemap->sm_jaddrefhd);
LIST_INIT(&bmsafemap->sm_jnewblkhd);
LIST_INIT(&bmsafemap->sm_freehd);
LIST_INIT(&bmsafemap->sm_freewr);
if (bmsafemap_find(bmsafemaphd, cg, &collision) == 1) {
WORKITEM_FREE(bmsafemap, D_BMSAFEMAP);
return (collision);
}
bmsafemap->sm_cg = cg;
LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash);
LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next);
WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list);
return (bmsafemap);
}
/*
* Direct block allocation dependencies.
*
* When a new block is allocated, the corresponding disk locations must be
* initialized (with zeros or new data) before the on-disk inode points to
* them. Also, the freemap from which the block was allocated must be
* updated (on disk) before the inode's pointer. These two dependencies are
* independent of each other and are needed for all file blocks and indirect
* blocks that are pointed to directly by the inode. Just before the
* "in-core" version of the inode is updated with a newly allocated block
* number, a procedure (below) is called to setup allocation dependency
* structures. These structures are removed when the corresponding
* dependencies are satisfied or when the block allocation becomes obsolete
* (i.e., the file is deleted, the block is de-allocated, or the block is a
* fragment that gets upgraded). All of these cases are handled in
* procedures described later.
*
* When a file extension causes a fragment to be upgraded, either to a larger
* fragment or to a full block, the on-disk location may change (if the
* previous fragment could not simply be extended). In this case, the old
* fragment must be de-allocated, but not until after the inode's pointer has
* been updated. In most cases, this is handled by later procedures, which
* will construct a "freefrag" structure to be added to the workitem queue
* when the inode update is complete (or obsolete). The main exception to
* this is when an allocation occurs while a pending allocation dependency
* (for the same block pointer) remains. This case is handled in the main
* allocation dependency setup procedure by immediately freeing the
* unreferenced fragments.
*/
void
softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp)
struct inode *ip; /* inode to which block is being added */
ufs_lbn_t off; /* block pointer within inode */
ufs2_daddr_t newblkno; /* disk block number being added */
ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */
long newsize; /* size of new block */
long oldsize; /* size of new block */
struct buf *bp; /* bp for allocated block */
{
struct allocdirect *adp, *oldadp;
struct allocdirectlst *adphead;
struct freefrag *freefrag;
struct inodedep *inodedep;
struct pagedep *pagedep;
struct jnewblk *jnewblk;
struct newblk *newblk;
struct mount *mp;
ufs_lbn_t lbn;
lbn = bp->b_lblkno;
mp = ITOVFS(ip);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_allocdirect called on non-softdep filesystem"));
if (oldblkno && oldblkno != newblkno)
/*
* The usual case is that a smaller fragment that
* was just allocated has been replaced with a bigger
* fragment or a full-size block. If it is marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the block was written
* earlier, but very uncommon. If the block has never
* been written, there is no need to send a BIO_DELETE
* for it when it is freed. The gain from avoiding the
* TRIMs for the common case of unwritten blocks far
* exceeds the cost of the write amplification for the
* uncommon case of failing to send a TRIM for a block
* that had been written.
*/
freefrag = newfreefrag(ip, oldblkno, oldsize, lbn,
(bp->b_flags & B_DELWRI) != 0 ? NOTRIM_KEY : SINGLETON_KEY);
else
freefrag = NULL;
CTR6(KTR_SUJ,
"softdep_setup_allocdirect: ino %d blkno %jd oldblkno %jd "
"off %jd newsize %ld oldsize %d",
ip->i_number, newblkno, oldblkno, off, newsize, oldsize);
ACQUIRE_LOCK(ITOUMP(ip));
if (off >= UFS_NDADDR) {
if (lbn > 0)
panic("softdep_setup_allocdirect: bad lbn %jd, off %jd",
lbn, off);
/* allocating an indirect block */
if (oldblkno != 0)
panic("softdep_setup_allocdirect: non-zero indir");
} else {
if (off != lbn)
panic("softdep_setup_allocdirect: lbn %jd != off %jd",
lbn, off);
/*
* Allocating a direct block.
*
* If we are allocating a directory block, then we must
* allocate an associated pagedep to track additions and
* deletions.
*/
if ((ip->i_mode & IFMT) == IFDIR)
pagedep_lookup(mp, bp, ip->i_number, off, DEPALLOC,
&pagedep);
}
if (newblk_lookup(mp, newblkno, 0, &newblk) == 0)
panic("softdep_setup_allocdirect: lost block");
KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
("softdep_setup_allocdirect: newblk already initialized"));
/*
* Convert the newblk to an allocdirect.
*/
WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT);
adp = (struct allocdirect *)newblk;
newblk->nb_freefrag = freefrag;
adp->ad_offset = off;
adp->ad_oldblkno = oldblkno;
adp->ad_newsize = newsize;
adp->ad_oldsize = oldsize;
/*
* Finish initializing the journal.
*/
if ((jnewblk = newblk->nb_jnewblk) != NULL) {
jnewblk->jn_ino = ip->i_number;
jnewblk->jn_lbn = lbn;
add_to_journal(&jnewblk->jn_list);
}
if (freefrag && freefrag->ff_jdep != NULL &&
freefrag->ff_jdep->wk_type == D_JFREEFRAG)
add_to_journal(freefrag->ff_jdep);
inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
adp->ad_inodedep = inodedep;
WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list);
/*
* The list of allocdirects must be kept in sorted and ascending
* order so that the rollback routines can quickly determine the
* first uncommitted block (the size of the file stored on disk
* ends at the end of the lowest committed fragment, or if there
* are no fragments, at the end of the highest committed block).
* Since files generally grow, the typical case is that the new
* block is to be added at the end of the list. We speed this
* special case by checking against the last allocdirect in the
* list before laboriously traversing the list looking for the
* insertion point.
*/
adphead = &inodedep->id_newinoupdt;
oldadp = TAILQ_LAST(adphead, allocdirectlst);
if (oldadp == NULL || oldadp->ad_offset <= off) {
/* insert at end of list */
TAILQ_INSERT_TAIL(adphead, adp, ad_next);
if (oldadp != NULL && oldadp->ad_offset == off)
allocdirect_merge(adphead, adp, oldadp);
FREE_LOCK(ITOUMP(ip));
return;
}
TAILQ_FOREACH(oldadp, adphead, ad_next) {
if (oldadp->ad_offset >= off)
break;
}
if (oldadp == NULL)
panic("softdep_setup_allocdirect: lost entry");
/* insert in middle of list */
TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
if (oldadp->ad_offset == off)
allocdirect_merge(adphead, adp, oldadp);
FREE_LOCK(ITOUMP(ip));
}
/*
* Merge a newer and older journal record to be stored either in a
* newblock or freefrag. This handles aggregating journal records for
* fragment allocation into a second record as well as replacing a
* journal free with an aborted journal allocation. A segment for the
* oldest record will be placed on wkhd if it has been written. If not
* the segment for the newer record will suffice.
*/
static struct worklist *
jnewblk_merge(new, old, wkhd)
struct worklist *new;
struct worklist *old;
struct workhead *wkhd;
{
struct jnewblk *njnewblk;
struct jnewblk *jnewblk;
/* Handle NULLs to simplify callers. */
if (new == NULL)
return (old);
if (old == NULL)
return (new);
/* Replace a jfreefrag with a jnewblk. */
if (new->wk_type == D_JFREEFRAG) {
if (WK_JNEWBLK(old)->jn_blkno != WK_JFREEFRAG(new)->fr_blkno)
panic("jnewblk_merge: blkno mismatch: %p, %p",
old, new);
cancel_jfreefrag(WK_JFREEFRAG(new));
return (old);
}
if (old->wk_type != D_JNEWBLK || new->wk_type != D_JNEWBLK)
panic("jnewblk_merge: Bad type: old %d new %d\n",
old->wk_type, new->wk_type);
/*
* Handle merging of two jnewblk records that describe
* different sets of fragments in the same block.
*/
jnewblk = WK_JNEWBLK(old);
njnewblk = WK_JNEWBLK(new);
if (jnewblk->jn_blkno != njnewblk->jn_blkno)
panic("jnewblk_merge: Merging disparate blocks.");
/*
* The record may be rolled back in the cg.
*/
if (jnewblk->jn_state & UNDONE) {
jnewblk->jn_state &= ~UNDONE;
njnewblk->jn_state |= UNDONE;
njnewblk->jn_state &= ~ATTACHED;
}
/*
* We modify the newer addref and free the older so that if neither
* has been written the most up-to-date copy will be on disk. If
* both have been written but rolled back we only temporarily need
* one of them to fix the bits when the cg write completes.
*/
jnewblk->jn_state |= ATTACHED | COMPLETE;
njnewblk->jn_oldfrags = jnewblk->jn_oldfrags;
cancel_jnewblk(jnewblk, wkhd);
WORKLIST_REMOVE(&jnewblk->jn_list);
free_jnewblk(jnewblk);
return (new);
}
/*
* Replace an old allocdirect dependency with a newer one.
*/
static void
allocdirect_merge(adphead, newadp, oldadp)
struct allocdirectlst *adphead; /* head of list holding allocdirects */
struct allocdirect *newadp; /* allocdirect being added */
struct allocdirect *oldadp; /* existing allocdirect being checked */
{
struct worklist *wk;
struct freefrag *freefrag;
freefrag = NULL;
LOCK_OWNED(VFSTOUFS(newadp->ad_list.wk_mp));
if (newadp->ad_oldblkno != oldadp->ad_newblkno ||
newadp->ad_oldsize != oldadp->ad_newsize ||
newadp->ad_offset >= UFS_NDADDR)
panic("%s %jd != new %jd || old size %ld != new %ld",
"allocdirect_merge: old blkno",
(intmax_t)newadp->ad_oldblkno,
(intmax_t)oldadp->ad_newblkno,
newadp->ad_oldsize, oldadp->ad_newsize);
newadp->ad_oldblkno = oldadp->ad_oldblkno;
newadp->ad_oldsize = oldadp->ad_oldsize;
/*
* If the old dependency had a fragment to free or had never
* previously had a block allocated, then the new dependency
* can immediately post its freefrag and adopt the old freefrag.
* This action is done by swapping the freefrag dependencies.
* The new dependency gains the old one's freefrag, and the
* old one gets the new one and then immediately puts it on
* the worklist when it is freed by free_newblk. It is
* not possible to do this swap when the old dependency had a
* non-zero size but no previous fragment to free. This condition
* arises when the new block is an extension of the old block.
* Here, the first part of the fragment allocated to the new
* dependency is part of the block currently claimed on disk by
* the old dependency, so cannot legitimately be freed until the
* conditions for the new dependency are fulfilled.
*/
freefrag = newadp->ad_freefrag;
if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) {
newadp->ad_freefrag = oldadp->ad_freefrag;
oldadp->ad_freefrag = freefrag;
}
/*
* If we are tracking a new directory-block allocation,
* move it from the old allocdirect to the new allocdirect.
*/
if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) {
WORKLIST_REMOVE(wk);
if (!LIST_EMPTY(&oldadp->ad_newdirblk))
panic("allocdirect_merge: extra newdirblk");
WORKLIST_INSERT(&newadp->ad_newdirblk, wk);
}
TAILQ_REMOVE(adphead, oldadp, ad_next);
/*
* We need to move any journal dependencies over to the freefrag
* that releases this block if it exists. Otherwise we are
* extending an existing block and we'll wait until that is
* complete to release the journal space and extend the
* new journal to cover this old space as well.
*/
if (freefrag == NULL) {
if (oldadp->ad_newblkno != newadp->ad_newblkno)
panic("allocdirect_merge: %jd != %jd",
oldadp->ad_newblkno, newadp->ad_newblkno);
newadp->ad_block.nb_jnewblk = (struct jnewblk *)
jnewblk_merge(&newadp->ad_block.nb_jnewblk->jn_list,
&oldadp->ad_block.nb_jnewblk->jn_list,
&newadp->ad_block.nb_jwork);
oldadp->ad_block.nb_jnewblk = NULL;
cancel_newblk(&oldadp->ad_block, NULL,
&newadp->ad_block.nb_jwork);
} else {
wk = (struct worklist *) cancel_newblk(&oldadp->ad_block,
&freefrag->ff_list, &freefrag->ff_jwork);
freefrag->ff_jdep = jnewblk_merge(freefrag->ff_jdep, wk,
&freefrag->ff_jwork);
}
free_newblk(&oldadp->ad_block);
}
/*
* Allocate a jfreefrag structure to journal a single block free.
*/
static struct jfreefrag *
newjfreefrag(freefrag, ip, blkno, size, lbn)
struct freefrag *freefrag;
struct inode *ip;
ufs2_daddr_t blkno;
long size;
ufs_lbn_t lbn;
{
struct jfreefrag *jfreefrag;
struct fs *fs;
fs = ITOFS(ip);
jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG,
M_SOFTDEP_FLAGS);
workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, ITOVFS(ip));
jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list);
jfreefrag->fr_state = ATTACHED | DEPCOMPLETE;
jfreefrag->fr_ino = ip->i_number;
jfreefrag->fr_lbn = lbn;
jfreefrag->fr_blkno = blkno;
jfreefrag->fr_frags = numfrags(fs, size);
jfreefrag->fr_freefrag = freefrag;
return (jfreefrag);
}
/*
* Allocate a new freefrag structure.
*/
static struct freefrag *
newfreefrag(ip, blkno, size, lbn, key)
struct inode *ip;
ufs2_daddr_t blkno;
long size;
ufs_lbn_t lbn;
u_long key;
{
struct freefrag *freefrag;
struct ufsmount *ump;
struct fs *fs;
CTR4(KTR_SUJ, "newfreefrag: ino %d blkno %jd size %ld lbn %jd",
ip->i_number, blkno, size, lbn);
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag)
panic("newfreefrag: frag size");
freefrag = malloc(sizeof(struct freefrag),
M_FREEFRAG, M_SOFTDEP_FLAGS);
workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ump));
freefrag->ff_state = ATTACHED;
LIST_INIT(&freefrag->ff_jwork);
freefrag->ff_inum = ip->i_number;
freefrag->ff_vtype = ITOV(ip)->v_type;
freefrag->ff_blkno = blkno;
freefrag->ff_fragsize = size;
freefrag->ff_key = key;
if (MOUNTEDSUJ(UFSTOVFS(ump))) {
freefrag->ff_jdep = (struct worklist *)
newjfreefrag(freefrag, ip, blkno, size, lbn);
} else {
freefrag->ff_state |= DEPCOMPLETE;
freefrag->ff_jdep = NULL;
}
return (freefrag);
}
/*
* This workitem de-allocates fragments that were replaced during
* file block allocation.
*/
static void
handle_workitem_freefrag(freefrag)
struct freefrag *freefrag;
{
struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp);
struct workhead wkhd;
CTR3(KTR_SUJ,
"handle_workitem_freefrag: ino %d blkno %jd size %ld",
freefrag->ff_inum, freefrag->ff_blkno, freefrag->ff_fragsize);
/*
* It would be illegal to add new completion items to the
* freefrag after it was schedule to be done so it must be
* safe to modify the list head here.
*/
LIST_INIT(&wkhd);
ACQUIRE_LOCK(ump);
LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list);
/*
* If the journal has not been written we must cancel it here.
*/
if (freefrag->ff_jdep) {
if (freefrag->ff_jdep->wk_type != D_JNEWBLK)
panic("handle_workitem_freefrag: Unexpected type %d\n",
freefrag->ff_jdep->wk_type);
cancel_jnewblk(WK_JNEWBLK(freefrag->ff_jdep), &wkhd);
}
FREE_LOCK(ump);
ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno,
freefrag->ff_fragsize, freefrag->ff_inum, freefrag->ff_vtype,
&wkhd, freefrag->ff_key);
ACQUIRE_LOCK(ump);
WORKITEM_FREE(freefrag, D_FREEFRAG);
FREE_LOCK(ump);
}
/*
* Set up a dependency structure for an external attributes data block.
* This routine follows much of the structure of softdep_setup_allocdirect.
* See the description of softdep_setup_allocdirect above for details.
*/
void
softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp)
struct inode *ip;
ufs_lbn_t off;
ufs2_daddr_t newblkno;
ufs2_daddr_t oldblkno;
long newsize;
long oldsize;
struct buf *bp;
{
struct allocdirect *adp, *oldadp;
struct allocdirectlst *adphead;
struct freefrag *freefrag;
struct inodedep *inodedep;
struct jnewblk *jnewblk;
struct newblk *newblk;
struct mount *mp;
struct ufsmount *ump;
ufs_lbn_t lbn;
mp = ITOVFS(ip);
ump = VFSTOUFS(mp);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_allocext called on non-softdep filesystem"));
KASSERT(off < UFS_NXADDR,
("softdep_setup_allocext: lbn %lld > UFS_NXADDR", (long long)off));
lbn = bp->b_lblkno;
if (oldblkno && oldblkno != newblkno)
/*
* The usual case is that a smaller fragment that
* was just allocated has been replaced with a bigger
* fragment or a full-size block. If it is marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the block was written
* earlier, but very uncommon. If the block has never
* been written, there is no need to send a BIO_DELETE
* for it when it is freed. The gain from avoiding the
* TRIMs for the common case of unwritten blocks far
* exceeds the cost of the write amplification for the
* uncommon case of failing to send a TRIM for a block
* that had been written.
*/
freefrag = newfreefrag(ip, oldblkno, oldsize, lbn,
(bp->b_flags & B_DELWRI) != 0 ? NOTRIM_KEY : SINGLETON_KEY);
else
freefrag = NULL;
ACQUIRE_LOCK(ump);
if (newblk_lookup(mp, newblkno, 0, &newblk) == 0)
panic("softdep_setup_allocext: lost block");
KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
("softdep_setup_allocext: newblk already initialized"));
/*
* Convert the newblk to an allocdirect.
*/
WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT);
adp = (struct allocdirect *)newblk;
newblk->nb_freefrag = freefrag;
adp->ad_offset = off;
adp->ad_oldblkno = oldblkno;
adp->ad_newsize = newsize;
adp->ad_oldsize = oldsize;
adp->ad_state |= EXTDATA;
/*
* Finish initializing the journal.
*/
if ((jnewblk = newblk->nb_jnewblk) != NULL) {
jnewblk->jn_ino = ip->i_number;
jnewblk->jn_lbn = lbn;
add_to_journal(&jnewblk->jn_list);
}
if (freefrag && freefrag->ff_jdep != NULL &&
freefrag->ff_jdep->wk_type == D_JFREEFRAG)
add_to_journal(freefrag->ff_jdep);
inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
adp->ad_inodedep = inodedep;
WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list);
/*
* The list of allocdirects must be kept in sorted and ascending
* order so that the rollback routines can quickly determine the
* first uncommitted block (the size of the file stored on disk
* ends at the end of the lowest committed fragment, or if there
* are no fragments, at the end of the highest committed block).
* Since files generally grow, the typical case is that the new
* block is to be added at the end of the list. We speed this
* special case by checking against the last allocdirect in the
* list before laboriously traversing the list looking for the
* insertion point.
*/
adphead = &inodedep->id_newextupdt;
oldadp = TAILQ_LAST(adphead, allocdirectlst);
if (oldadp == NULL || oldadp->ad_offset <= off) {
/* insert at end of list */
TAILQ_INSERT_TAIL(adphead, adp, ad_next);
if (oldadp != NULL && oldadp->ad_offset == off)
allocdirect_merge(adphead, adp, oldadp);
FREE_LOCK(ump);
return;
}
TAILQ_FOREACH(oldadp, adphead, ad_next) {
if (oldadp->ad_offset >= off)
break;
}
if (oldadp == NULL)
panic("softdep_setup_allocext: lost entry");
/* insert in middle of list */
TAILQ_INSERT_BEFORE(oldadp, adp, ad_next);
if (oldadp->ad_offset == off)
allocdirect_merge(adphead, adp, oldadp);
FREE_LOCK(ump);
}
/*
* Indirect block allocation dependencies.
*
* The same dependencies that exist for a direct block also exist when
* a new block is allocated and pointed to by an entry in a block of
* indirect pointers. The undo/redo states described above are also
* used here. Because an indirect block contains many pointers that
* may have dependencies, a second copy of the entire in-memory indirect
* block is kept. The buffer cache copy is always completely up-to-date.
* The second copy, which is used only as a source for disk writes,
* contains only the safe pointers (i.e., those that have no remaining
* update dependencies). The second copy is freed when all pointers
* are safe. The cache is not allowed to replace indirect blocks with
* pending update dependencies. If a buffer containing an indirect
* block with dependencies is written, these routines will mark it
* dirty again. It can only be successfully written once all the
* dependencies are removed. The ffs_fsync routine in conjunction with
* softdep_sync_metadata work together to get all the dependencies
* removed so that a file can be successfully written to disk. Three
* procedures are used when setting up indirect block pointer
* dependencies. The division is necessary because of the organization
* of the "balloc" routine and because of the distinction between file
* pages and file metadata blocks.
*/
/*
* Allocate a new allocindir structure.
*/
static struct allocindir *
newallocindir(ip, ptrno, newblkno, oldblkno, lbn)
struct inode *ip; /* inode for file being extended */
int ptrno; /* offset of pointer in indirect block */
ufs2_daddr_t newblkno; /* disk block number being added */
ufs2_daddr_t oldblkno; /* previous block number, 0 if none */
ufs_lbn_t lbn;
{
struct newblk *newblk;
struct allocindir *aip;
struct freefrag *freefrag;
struct jnewblk *jnewblk;
if (oldblkno)
freefrag = newfreefrag(ip, oldblkno, ITOFS(ip)->fs_bsize, lbn,
SINGLETON_KEY);
else
freefrag = NULL;
ACQUIRE_LOCK(ITOUMP(ip));
if (newblk_lookup(ITOVFS(ip), newblkno, 0, &newblk) == 0)
panic("new_allocindir: lost block");
KASSERT(newblk->nb_list.wk_type == D_NEWBLK,
("newallocindir: newblk already initialized"));
WORKITEM_REASSIGN(newblk, D_ALLOCINDIR);
newblk->nb_freefrag = freefrag;
aip = (struct allocindir *)newblk;
aip->ai_offset = ptrno;
aip->ai_oldblkno = oldblkno;
aip->ai_lbn = lbn;
if ((jnewblk = newblk->nb_jnewblk) != NULL) {
jnewblk->jn_ino = ip->i_number;
jnewblk->jn_lbn = lbn;
add_to_journal(&jnewblk->jn_list);
}
if (freefrag && freefrag->ff_jdep != NULL &&
freefrag->ff_jdep->wk_type == D_JFREEFRAG)
add_to_journal(freefrag->ff_jdep);
return (aip);
}
/*
* Called just before setting an indirect block pointer
* to a newly allocated file page.
*/
void
softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp)
struct inode *ip; /* inode for file being extended */
ufs_lbn_t lbn; /* allocated block number within file */
struct buf *bp; /* buffer with indirect blk referencing page */
int ptrno; /* offset of pointer in indirect block */
ufs2_daddr_t newblkno; /* disk block number being added */
ufs2_daddr_t oldblkno; /* previous block number, 0 if none */
struct buf *nbp; /* buffer holding allocated page */
{
struct inodedep *inodedep;
struct freefrag *freefrag;
struct allocindir *aip;
struct pagedep *pagedep;
struct mount *mp;
struct ufsmount *ump;
mp = ITOVFS(ip);
ump = VFSTOUFS(mp);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_allocindir_page called on non-softdep filesystem"));
KASSERT(lbn == nbp->b_lblkno,
("softdep_setup_allocindir_page: lbn %jd != lblkno %jd",
lbn, bp->b_lblkno));
CTR4(KTR_SUJ,
"softdep_setup_allocindir_page: ino %d blkno %jd oldblkno %jd "
"lbn %jd", ip->i_number, newblkno, oldblkno, lbn);
ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page");
aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn);
(void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
/*
* If we are allocating a directory page, then we must
* allocate an associated pagedep to track additions and
* deletions.
*/
if ((ip->i_mode & IFMT) == IFDIR)
pagedep_lookup(mp, nbp, ip->i_number, lbn, DEPALLOC, &pagedep);
WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list);
freefrag = setup_allocindir_phase2(bp, ip, inodedep, aip, lbn);
FREE_LOCK(ump);
if (freefrag)
handle_workitem_freefrag(freefrag);
}
/*
* Called just before setting an indirect block pointer to a
* newly allocated indirect block.
*/
void
softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno)
struct buf *nbp; /* newly allocated indirect block */
struct inode *ip; /* inode for file being extended */
struct buf *bp; /* indirect block referencing allocated block */
int ptrno; /* offset of pointer in indirect block */
ufs2_daddr_t newblkno; /* disk block number being added */
{
struct inodedep *inodedep;
struct allocindir *aip;
struct ufsmount *ump;
ufs_lbn_t lbn;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_setup_allocindir_meta called on non-softdep filesystem"));
CTR3(KTR_SUJ,
"softdep_setup_allocindir_meta: ino %d blkno %jd ptrno %d",
ip->i_number, newblkno, ptrno);
lbn = nbp->b_lblkno;
ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta");
aip = newallocindir(ip, ptrno, newblkno, 0, lbn);
inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep);
WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list);
if (setup_allocindir_phase2(bp, ip, inodedep, aip, lbn))
panic("softdep_setup_allocindir_meta: Block already existed");
FREE_LOCK(ump);
}
static void
indirdep_complete(indirdep)
struct indirdep *indirdep;
{
struct allocindir *aip;
LIST_REMOVE(indirdep, ir_next);
indirdep->ir_state |= DEPCOMPLETE;
while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) {
LIST_REMOVE(aip, ai_next);
free_newblk(&aip->ai_block);
}
/*
* If this indirdep is not attached to a buf it was simply waiting
* on completion to clear completehd. free_indirdep() asserts
* that nothing is dangling.
*/
if ((indirdep->ir_state & ONWORKLIST) == 0)
free_indirdep(indirdep);
}
static struct indirdep *
indirdep_lookup(mp, ip, bp)
struct mount *mp;
struct inode *ip;
struct buf *bp;
{
struct indirdep *indirdep, *newindirdep;
struct newblk *newblk;
struct ufsmount *ump;
struct worklist *wk;
struct fs *fs;
ufs2_daddr_t blkno;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
indirdep = NULL;
newindirdep = NULL;
fs = ump->um_fs;
for (;;) {
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
if (wk->wk_type != D_INDIRDEP)
continue;
indirdep = WK_INDIRDEP(wk);
break;
}
/* Found on the buffer worklist, no new structure to free. */
if (indirdep != NULL && newindirdep == NULL)
return (indirdep);
if (indirdep != NULL && newindirdep != NULL)
panic("indirdep_lookup: simultaneous create");
/* None found on the buffer and a new structure is ready. */
if (indirdep == NULL && newindirdep != NULL)
break;
/* None found and no new structure available. */
FREE_LOCK(ump);
newindirdep = malloc(sizeof(struct indirdep),
M_INDIRDEP, M_SOFTDEP_FLAGS);
workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp);
newindirdep->ir_state = ATTACHED;
if (I_IS_UFS1(ip))
newindirdep->ir_state |= UFS1FMT;
TAILQ_INIT(&newindirdep->ir_trunc);
newindirdep->ir_saveddata = NULL;
LIST_INIT(&newindirdep->ir_deplisthd);
LIST_INIT(&newindirdep->ir_donehd);
LIST_INIT(&newindirdep->ir_writehd);
LIST_INIT(&newindirdep->ir_completehd);
if (bp->b_blkno == bp->b_lblkno) {
ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp,
NULL, NULL);
bp->b_blkno = blkno;
}
newindirdep->ir_freeblks = NULL;
newindirdep->ir_savebp =
getblk(ump->um_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0);
newindirdep->ir_bp = bp;
BUF_KERNPROC(newindirdep->ir_savebp);
bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount);
ACQUIRE_LOCK(ump);
}
indirdep = newindirdep;
WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list);
/*
* If the block is not yet allocated we don't set DEPCOMPLETE so
* that we don't free dependencies until the pointers are valid.
* This could search b_dep for D_ALLOCDIRECT/D_ALLOCINDIR rather
* than using the hash.
*/
if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk))
LIST_INSERT_HEAD(&newblk->nb_indirdeps, indirdep, ir_next);
else
indirdep->ir_state |= DEPCOMPLETE;
return (indirdep);
}
/*
* Called to finish the allocation of the "aip" allocated
* by one of the two routines above.
*/
static struct freefrag *
setup_allocindir_phase2(bp, ip, inodedep, aip, lbn)
struct buf *bp; /* in-memory copy of the indirect block */
struct inode *ip; /* inode for file being extended */
struct inodedep *inodedep; /* Inodedep for ip */
struct allocindir *aip; /* allocindir allocated by the above routines */
ufs_lbn_t lbn; /* Logical block number for this block. */
{
struct fs *fs;
struct indirdep *indirdep;
struct allocindir *oldaip;
struct freefrag *freefrag;
struct mount *mp;
struct ufsmount *ump;
mp = ITOVFS(ip);
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
fs = ump->um_fs;
if (bp->b_lblkno >= 0)
panic("setup_allocindir_phase2: not indir blk");
KASSERT(aip->ai_offset >= 0 && aip->ai_offset < NINDIR(fs),
("setup_allocindir_phase2: Bad offset %d", aip->ai_offset));
indirdep = indirdep_lookup(mp, ip, bp);
KASSERT(indirdep->ir_savebp != NULL,
("setup_allocindir_phase2 NULL ir_savebp"));
aip->ai_indirdep = indirdep;
/*
* Check for an unwritten dependency for this indirect offset. If
* there is, merge the old dependency into the new one. This happens
* as a result of reallocblk only.
*/
freefrag = NULL;
if (aip->ai_oldblkno != 0) {
LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, ai_next) {
if (oldaip->ai_offset == aip->ai_offset) {
freefrag = allocindir_merge(aip, oldaip);
goto done;
}
}
LIST_FOREACH(oldaip, &indirdep->ir_donehd, ai_next) {
if (oldaip->ai_offset == aip->ai_offset) {
freefrag = allocindir_merge(aip, oldaip);
goto done;
}
}
}
done:
LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next);
return (freefrag);
}
/*
* Merge two allocindirs which refer to the same block. Move newblock
* dependencies and setup the freefrags appropriately.
*/
static struct freefrag *
allocindir_merge(aip, oldaip)
struct allocindir *aip;
struct allocindir *oldaip;
{
struct freefrag *freefrag;
struct worklist *wk;
if (oldaip->ai_newblkno != aip->ai_oldblkno)
panic("allocindir_merge: blkno");
aip->ai_oldblkno = oldaip->ai_oldblkno;
freefrag = aip->ai_freefrag;
aip->ai_freefrag = oldaip->ai_freefrag;
oldaip->ai_freefrag = NULL;
KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag"));
/*
* If we are tracking a new directory-block allocation,
* move it from the old allocindir to the new allocindir.
*/
if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) {
WORKLIST_REMOVE(wk);
if (!LIST_EMPTY(&oldaip->ai_newdirblk))
panic("allocindir_merge: extra newdirblk");
WORKLIST_INSERT(&aip->ai_newdirblk, wk);
}
/*
* We can skip journaling for this freefrag and just complete
* any pending journal work for the allocindir that is being
* removed after the freefrag completes.
*/
if (freefrag->ff_jdep)
cancel_jfreefrag(WK_JFREEFRAG(freefrag->ff_jdep));
LIST_REMOVE(oldaip, ai_next);
freefrag->ff_jdep = (struct worklist *)cancel_newblk(&oldaip->ai_block,
&freefrag->ff_list, &freefrag->ff_jwork);
free_newblk(&oldaip->ai_block);
return (freefrag);
}
static inline void
setup_freedirect(freeblks, ip, i, needj)
struct freeblks *freeblks;
struct inode *ip;
int i;
int needj;
{
struct ufsmount *ump;
ufs2_daddr_t blkno;
int frags;
blkno = DIP(ip, i_db[i]);
if (blkno == 0)
return;
DIP_SET(ip, i_db[i], 0);
ump = ITOUMP(ip);
frags = sblksize(ump->um_fs, ip->i_size, i);
frags = numfrags(ump->um_fs, frags);
newfreework(ump, freeblks, NULL, i, blkno, frags, 0, needj);
}
static inline void
setup_freeext(freeblks, ip, i, needj)
struct freeblks *freeblks;
struct inode *ip;
int i;
int needj;
{
struct ufsmount *ump;
ufs2_daddr_t blkno;
int frags;
blkno = ip->i_din2->di_extb[i];
if (blkno == 0)
return;
ip->i_din2->di_extb[i] = 0;
ump = ITOUMP(ip);
frags = sblksize(ump->um_fs, ip->i_din2->di_extsize, i);
frags = numfrags(ump->um_fs, frags);
newfreework(ump, freeblks, NULL, -1 - i, blkno, frags, 0, needj);
}
static inline void
setup_freeindir(freeblks, ip, i, lbn, needj)
struct freeblks *freeblks;
struct inode *ip;
int i;
ufs_lbn_t lbn;
int needj;
{
struct ufsmount *ump;
ufs2_daddr_t blkno;
blkno = DIP(ip, i_ib[i]);
if (blkno == 0)
return;
DIP_SET(ip, i_ib[i], 0);
ump = ITOUMP(ip);
newfreework(ump, freeblks, NULL, lbn, blkno, ump->um_fs->fs_frag,
0, needj);
}
static inline struct freeblks *
newfreeblks(mp, ip)
struct mount *mp;
struct inode *ip;
{
struct freeblks *freeblks;
freeblks = malloc(sizeof(struct freeblks),
M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO);
workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp);
LIST_INIT(&freeblks->fb_jblkdephd);
LIST_INIT(&freeblks->fb_jwork);
freeblks->fb_ref = 0;
freeblks->fb_cgwait = 0;
freeblks->fb_state = ATTACHED;
freeblks->fb_uid = ip->i_uid;
freeblks->fb_inum = ip->i_number;
freeblks->fb_vtype = ITOV(ip)->v_type;
freeblks->fb_modrev = DIP(ip, i_modrev);
freeblks->fb_devvp = ITODEVVP(ip);
freeblks->fb_chkcnt = 0;
freeblks->fb_len = 0;
return (freeblks);
}
static void
trunc_indirdep(indirdep, freeblks, bp, off)
struct indirdep *indirdep;
struct freeblks *freeblks;
struct buf *bp;
int off;
{
struct allocindir *aip, *aipn;
/*
* The first set of allocindirs won't be in savedbp.
*/
LIST_FOREACH_SAFE(aip, &indirdep->ir_deplisthd, ai_next, aipn)
if (aip->ai_offset > off)
cancel_allocindir(aip, bp, freeblks, 1);
LIST_FOREACH_SAFE(aip, &indirdep->ir_donehd, ai_next, aipn)
if (aip->ai_offset > off)
cancel_allocindir(aip, bp, freeblks, 1);
/*
* These will exist in savedbp.
*/
LIST_FOREACH_SAFE(aip, &indirdep->ir_writehd, ai_next, aipn)
if (aip->ai_offset > off)
cancel_allocindir(aip, NULL, freeblks, 0);
LIST_FOREACH_SAFE(aip, &indirdep->ir_completehd, ai_next, aipn)
if (aip->ai_offset > off)
cancel_allocindir(aip, NULL, freeblks, 0);
}
/*
* Follow the chain of indirects down to lastlbn creating a freework
* structure for each. This will be used to start indir_trunc() at
* the right offset and create the journal records for the parrtial
* truncation. A second step will handle the truncated dependencies.
*/
static int
setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno)
struct freeblks *freeblks;
struct inode *ip;
ufs_lbn_t lbn;
ufs_lbn_t lastlbn;
ufs2_daddr_t blkno;
{
struct indirdep *indirdep;
struct indirdep *indirn;
struct freework *freework;
struct newblk *newblk;
struct mount *mp;
struct ufsmount *ump;
struct buf *bp;
uint8_t *start;
uint8_t *end;
ufs_lbn_t lbnadd;
int level;
int error;
int off;
freework = NULL;
if (blkno == 0)
return (0);
mp = freeblks->fb_list.wk_mp;
ump = VFSTOUFS(mp);
/*
* Here, calls to VOP_BMAP() will fail. However, we already have
* the on-disk address, so we just pass it to bread() instead of
* having bread() attempt to calculate it using VOP_BMAP().
*/
- error = breadn_flags(ITOV(ip), lbn, blkptrtodb(ump, blkno),
+ error = ffs_breadz(ump, ITOV(ip), lbn, blkptrtodb(ump, blkno),
(int)mp->mnt_stat.f_iosize, NULL, NULL, 0, NOCRED, 0, NULL, &bp);
if (error)
return (error);
level = lbn_level(lbn);
lbnadd = lbn_offset(ump->um_fs, level);
/*
* Compute the offset of the last block we want to keep. Store
* in the freework the first block we want to completely free.
*/
off = (lastlbn - -(lbn + level)) / lbnadd;
if (off + 1 == NINDIR(ump->um_fs))
goto nowork;
freework = newfreework(ump, freeblks, NULL, lbn, blkno, 0, off + 1, 0);
/*
* Link the freework into the indirdep. This will prevent any new
* allocations from proceeding until we are finished with the
* truncate and the block is written.
*/
ACQUIRE_LOCK(ump);
indirdep = indirdep_lookup(mp, ip, bp);
if (indirdep->ir_freeblks)
panic("setup_trunc_indir: indirdep already truncated.");
TAILQ_INSERT_TAIL(&indirdep->ir_trunc, freework, fw_next);
freework->fw_indir = indirdep;
/*
* Cancel any allocindirs that will not make it to disk.
* We have to do this for all copies of the indirdep that
* live on this newblk.
*/
if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
if (newblk_lookup(mp, dbtofsb(ump->um_fs, bp->b_blkno), 0,
&newblk) == 0)
panic("setup_trunc_indir: lost block");
LIST_FOREACH(indirn, &newblk->nb_indirdeps, ir_next)
trunc_indirdep(indirn, freeblks, bp, off);
} else
trunc_indirdep(indirdep, freeblks, bp, off);
FREE_LOCK(ump);
/*
* Creation is protected by the buf lock. The saveddata is only
* needed if a full truncation follows a partial truncation but it
* is difficult to allocate in that case so we fetch it anyway.
*/
if (indirdep->ir_saveddata == NULL)
indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP,
M_SOFTDEP_FLAGS);
nowork:
/* Fetch the blkno of the child and the zero start offset. */
if (I_IS_UFS1(ip)) {
blkno = ((ufs1_daddr_t *)bp->b_data)[off];
start = (uint8_t *)&((ufs1_daddr_t *)bp->b_data)[off+1];
} else {
blkno = ((ufs2_daddr_t *)bp->b_data)[off];
start = (uint8_t *)&((ufs2_daddr_t *)bp->b_data)[off+1];
}
if (freework) {
/* Zero the truncated pointers. */
end = bp->b_data + bp->b_bcount;
bzero(start, end - start);
bdwrite(bp);
} else
bqrelse(bp);
if (level == 0)
return (0);
lbn++; /* adjust level */
lbn -= (off * lbnadd);
return setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno);
}
/*
* Complete the partial truncation of an indirect block setup by
* setup_trunc_indir(). This zeros the truncated pointers in the saved
* copy and writes them to disk before the freeblks is allowed to complete.
*/
static void
complete_trunc_indir(freework)
struct freework *freework;
{
struct freework *fwn;
struct indirdep *indirdep;
struct ufsmount *ump;
struct buf *bp;
uintptr_t start;
int count;
ump = VFSTOUFS(freework->fw_list.wk_mp);
LOCK_OWNED(ump);
indirdep = freework->fw_indir;
for (;;) {
bp = indirdep->ir_bp;
/* See if the block was discarded. */
if (bp == NULL)
break;
/* Inline part of getdirtybuf(). We dont want bremfree. */
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) == 0)
break;
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
LOCK_PTR(ump)) == 0)
BUF_UNLOCK(bp);
ACQUIRE_LOCK(ump);
}
freework->fw_state |= DEPCOMPLETE;
TAILQ_REMOVE(&indirdep->ir_trunc, freework, fw_next);
/*
* Zero the pointers in the saved copy.
*/
if (indirdep->ir_state & UFS1FMT)
start = sizeof(ufs1_daddr_t);
else
start = sizeof(ufs2_daddr_t);
start *= freework->fw_start;
count = indirdep->ir_savebp->b_bcount - start;
start += (uintptr_t)indirdep->ir_savebp->b_data;
bzero((char *)start, count);
/*
* We need to start the next truncation in the list if it has not
* been started yet.
*/
fwn = TAILQ_FIRST(&indirdep->ir_trunc);
if (fwn != NULL) {
if (fwn->fw_freeblks == indirdep->ir_freeblks)
TAILQ_REMOVE(&indirdep->ir_trunc, fwn, fw_next);
if ((fwn->fw_state & ONWORKLIST) == 0)
freework_enqueue(fwn);
}
/*
* If bp is NULL the block was fully truncated, restore
* the saved block list otherwise free it if it is no
* longer needed.
*/
if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
if (bp == NULL)
bcopy(indirdep->ir_saveddata,
indirdep->ir_savebp->b_data,
indirdep->ir_savebp->b_bcount);
free(indirdep->ir_saveddata, M_INDIRDEP);
indirdep->ir_saveddata = NULL;
}
/*
* When bp is NULL there is a full truncation pending. We
* must wait for this full truncation to be journaled before
* we can release this freework because the disk pointers will
* never be written as zero.
*/
if (bp == NULL) {
if (LIST_EMPTY(&indirdep->ir_freeblks->fb_jblkdephd))
handle_written_freework(freework);
else
WORKLIST_INSERT(&indirdep->ir_freeblks->fb_freeworkhd,
&freework->fw_list);
+ if (fwn == NULL) {
+ freework->fw_indir = (void *)0x0000deadbeef0000;
+ bp = indirdep->ir_savebp;
+ indirdep->ir_savebp = NULL;
+ free_indirdep(indirdep);
+ FREE_LOCK(ump);
+ brelse(bp);
+ ACQUIRE_LOCK(ump);
+ }
} else {
/* Complete when the real copy is written. */
WORKLIST_INSERT(&bp->b_dep, &freework->fw_list);
BUF_UNLOCK(bp);
}
}
/*
* Calculate the number of blocks we are going to release where datablocks
* is the current total and length is the new file size.
*/
static ufs2_daddr_t
blkcount(fs, datablocks, length)
struct fs *fs;
ufs2_daddr_t datablocks;
off_t length;
{
off_t totblks, numblks;
totblks = 0;
numblks = howmany(length, fs->fs_bsize);
if (numblks <= UFS_NDADDR) {
totblks = howmany(length, fs->fs_fsize);
goto out;
}
totblks = blkstofrags(fs, numblks);
numblks -= UFS_NDADDR;
/*
* Count all single, then double, then triple indirects required.
* Subtracting one indirects worth of blocks for each pass
* acknowledges one of each pointed to by the inode.
*/
for (;;) {
totblks += blkstofrags(fs, howmany(numblks, NINDIR(fs)));
numblks -= NINDIR(fs);
if (numblks <= 0)
break;
numblks = howmany(numblks, NINDIR(fs));
}
out:
totblks = fsbtodb(fs, totblks);
/*
* Handle sparse files. We can't reclaim more blocks than the inode
* references. We will correct it later in handle_complete_freeblks()
* when we know the real count.
*/
if (totblks > datablocks)
return (0);
return (datablocks - totblks);
}
/*
* Handle freeblocks for journaled softupdate filesystems.
*
* Contrary to normal softupdates, we must preserve the block pointers in
* indirects until their subordinates are free. This is to avoid journaling
* every block that is freed which may consume more space than the journal
* itself. The recovery program will see the free block journals at the
* base of the truncated area and traverse them to reclaim space. The
* pointers in the inode may be cleared immediately after the journal
* records are written because each direct and indirect pointer in the
* inode is recorded in a journal. This permits full truncation to proceed
* asynchronously. The write order is journal -> inode -> cgs -> indirects.
*
* The algorithm is as follows:
* 1) Traverse the in-memory state and create journal entries to release
* the relevant blocks and full indirect trees.
* 2) Traverse the indirect block chain adding partial truncation freework
* records to indirects in the path to lastlbn. The freework will
* prevent new allocation dependencies from being satisfied in this
* indirect until the truncation completes.
* 3) Read and lock the inode block, performing an update with the new size
* and pointers. This prevents truncated data from becoming valid on
* disk through step 4.
* 4) Reap unsatisfied dependencies that are beyond the truncated area,
* eliminate journal work for those records that do not require it.
* 5) Schedule the journal records to be written followed by the inode block.
* 6) Allocate any necessary frags for the end of file.
* 7) Zero any partially truncated blocks.
*
* From this truncation proceeds asynchronously using the freework and
* indir_trunc machinery. The file will not be extended again into a
* partially truncated indirect block until all work is completed but
* the normal dependency mechanism ensures that it is rolled back/forward
* as appropriate. Further truncation may occur without delay and is
* serialized in indir_trunc().
*/
void
softdep_journal_freeblocks(ip, cred, length, flags)
struct inode *ip; /* The inode whose length is to be reduced */
struct ucred *cred;
off_t length; /* The new length for the file */
int flags; /* IO_EXT and/or IO_NORMAL */
{
struct freeblks *freeblks, *fbn;
struct worklist *wk, *wkn;
struct inodedep *inodedep;
struct jblkdep *jblkdep;
struct allocdirect *adp, *adpn;
struct ufsmount *ump;
struct fs *fs;
struct buf *bp;
struct vnode *vp;
struct mount *mp;
+ daddr_t dbn;
ufs2_daddr_t extblocks, datablocks;
ufs_lbn_t tmpval, lbn, lastlbn;
int frags, lastoff, iboff, allocblock, needj, error, i;
ump = ITOUMP(ip);
mp = UFSTOVFS(ump);
fs = ump->um_fs;
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_journal_freeblocks called on non-softdep filesystem"));
vp = ITOV(ip);
needj = 1;
iboff = -1;
allocblock = 0;
extblocks = 0;
datablocks = 0;
frags = 0;
freeblks = newfreeblks(mp, ip);
ACQUIRE_LOCK(ump);
/*
* If we're truncating a removed file that will never be written
* we don't need to journal the block frees. The canceled journals
* for the allocations will suffice.
*/
inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED &&
length == 0)
needj = 0;
CTR3(KTR_SUJ, "softdep_journal_freeblks: ip %d length %ld needj %d",
ip->i_number, length, needj);
FREE_LOCK(ump);
/*
* Calculate the lbn that we are truncating to. This results in -1
* if we're truncating the 0 bytes. So it is the last lbn we want
* to keep, not the first lbn we want to truncate.
*/
lastlbn = lblkno(fs, length + fs->fs_bsize - 1) - 1;
lastoff = blkoff(fs, length);
/*
* Compute frags we are keeping in lastlbn. 0 means all.
*/
if (lastlbn >= 0 && lastlbn < UFS_NDADDR) {
frags = fragroundup(fs, lastoff);
/* adp offset of last valid allocdirect. */
iboff = lastlbn;
} else if (lastlbn > 0)
iboff = UFS_NDADDR;
if (fs->fs_magic == FS_UFS2_MAGIC)
extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
/*
* Handle normal data blocks and indirects. This section saves
* values used after the inode update to complete frag and indirect
* truncation.
*/
if ((flags & IO_NORMAL) != 0) {
/*
* Handle truncation of whole direct and indirect blocks.
*/
for (i = iboff + 1; i < UFS_NDADDR; i++)
setup_freedirect(freeblks, ip, i, needj);
for (i = 0, tmpval = NINDIR(fs), lbn = UFS_NDADDR;
i < UFS_NIADDR;
i++, lbn += tmpval, tmpval *= NINDIR(fs)) {
/* Release a whole indirect tree. */
if (lbn > lastlbn) {
setup_freeindir(freeblks, ip, i, -lbn -i,
needj);
continue;
}
iboff = i + UFS_NDADDR;
/*
* Traverse partially truncated indirect tree.
*/
if (lbn <= lastlbn && lbn + tmpval - 1 > lastlbn)
setup_trunc_indir(freeblks, ip, -lbn - i,
lastlbn, DIP(ip, i_ib[i]));
}
/*
* Handle partial truncation to a frag boundary.
*/
if (frags) {
ufs2_daddr_t blkno;
long oldfrags;
oldfrags = blksize(fs, ip, lastlbn);
blkno = DIP(ip, i_db[lastlbn]);
if (blkno && oldfrags != frags) {
oldfrags -= frags;
oldfrags = numfrags(fs, oldfrags);
blkno += numfrags(fs, frags);
newfreework(ump, freeblks, NULL, lastlbn,
blkno, oldfrags, 0, needj);
if (needj)
adjust_newfreework(freeblks,
numfrags(fs, frags));
} else if (blkno == 0)
allocblock = 1;
}
/*
* Add a journal record for partial truncate if we are
* handling indirect blocks. Non-indirects need no extra
* journaling.
*/
if (length != 0 && lastlbn >= UFS_NDADDR) {
UFS_INODE_SET_FLAG(ip, IN_TRUNCATED);
newjtrunc(freeblks, length, 0);
}
ip->i_size = length;
DIP_SET(ip, i_size, ip->i_size);
datablocks = DIP(ip, i_blocks) - extblocks;
if (length != 0)
datablocks = blkcount(fs, datablocks, length);
freeblks->fb_len = length;
}
if ((flags & IO_EXT) != 0) {
for (i = 0; i < UFS_NXADDR; i++)
setup_freeext(freeblks, ip, i, needj);
ip->i_din2->di_extsize = 0;
datablocks += extblocks;
}
#ifdef QUOTA
/* Reference the quotas in case the block count is wrong in the end. */
quotaref(vp, freeblks->fb_quota);
(void) chkdq(ip, -datablocks, NOCRED, FORCE);
#endif
freeblks->fb_chkcnt = -datablocks;
UFS_LOCK(ump);
fs->fs_pendingblocks += datablocks;
UFS_UNLOCK(ump);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks);
/*
* Handle truncation of incomplete alloc direct dependencies. We
* hold the inode block locked to prevent incomplete dependencies
* from reaching the disk while we are eliminating those that
* have been truncated. This is a partially inlined ffs_update().
*/
ufs_itimes(vp);
ip->i_flag &= ~(IN_LAZYACCESS | IN_LAZYMOD | IN_MODIFIED);
- error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
- (int)fs->fs_bsize, cred, &bp);
+ dbn = fsbtodb(fs, ino_to_fsba(fs, ip->i_number));
+ error = ffs_breadz(ump, ump->um_devvp, dbn, dbn, (int)fs->fs_bsize,
+ NULL, NULL, 0, cred, 0, NULL, &bp);
if (error) {
softdep_error("softdep_journal_freeblocks", error);
return;
}
if (bp->b_bufsize == fs->fs_bsize)
bp->b_flags |= B_CLUSTEROK;
softdep_update_inodeblock(ip, bp, 0);
if (ump->um_fstype == UFS1) {
*((struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1;
} else {
ffs_update_dinode_ckhash(fs, ip->i_din2);
*((struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2;
}
ACQUIRE_LOCK(ump);
(void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
if ((inodedep->id_state & IOSTARTED) != 0)
panic("softdep_setup_freeblocks: inode busy");
/*
* Add the freeblks structure to the list of operations that
* must await the zero'ed inode being written to disk. If we
* still have a bitmap dependency (needj), then the inode
* has never been written to disk, so we can process the
* freeblks below once we have deleted the dependencies.
*/
if (needj)
WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list);
else
freeblks->fb_state |= COMPLETE;
if ((flags & IO_NORMAL) != 0) {
TAILQ_FOREACH_SAFE(adp, &inodedep->id_inoupdt, ad_next, adpn) {
if (adp->ad_offset > iboff)
cancel_allocdirect(&inodedep->id_inoupdt, adp,
freeblks);
/*
* Truncate the allocdirect. We could eliminate
* or modify journal records as well.
*/
else if (adp->ad_offset == iboff && frags)
adp->ad_newsize = frags;
}
}
if ((flags & IO_EXT) != 0)
while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL)
cancel_allocdirect(&inodedep->id_extupdt, adp,
freeblks);
/*
* Scan the bufwait list for newblock dependencies that will never
* make it to disk.
*/
LIST_FOREACH_SAFE(wk, &inodedep->id_bufwait, wk_list, wkn) {
if (wk->wk_type != D_ALLOCDIRECT)
continue;
adp = WK_ALLOCDIRECT(wk);
if (((flags & IO_NORMAL) != 0 && (adp->ad_offset > iboff)) ||
((flags & IO_EXT) != 0 && (adp->ad_state & EXTDATA))) {
cancel_jfreeblk(freeblks, adp->ad_newblkno);
cancel_newblk(WK_NEWBLK(wk), NULL, &freeblks->fb_jwork);
WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk);
}
}
/*
* Add journal work.
*/
LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps)
add_to_journal(&jblkdep->jb_list);
FREE_LOCK(ump);
bdwrite(bp);
/*
* Truncate dependency structures beyond length.
*/
trunc_dependencies(ip, freeblks, lastlbn, frags, flags);
/*
* This is only set when we need to allocate a fragment because
* none existed at the end of a frag-sized file. It handles only
* allocating a new, zero filled block.
*/
if (allocblock) {
ip->i_size = length - lastoff;
DIP_SET(ip, i_size, ip->i_size);
error = UFS_BALLOC(vp, length - 1, 1, cred, BA_CLRBUF, &bp);
if (error != 0) {
softdep_error("softdep_journal_freeblks", error);
return;
}
ip->i_size = length;
DIP_SET(ip, i_size, length);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allocbuf(bp, frags);
ffs_update(vp, 0);
bawrite(bp);
} else if (lastoff != 0 && vp->v_type != VDIR) {
int size;
/*
* Zero the end of a truncated frag or block.
*/
size = sblksize(fs, length, lastlbn);
error = bread(vp, lastlbn, size, cred, &bp);
- if (error) {
+ if (error == 0) {
+ bzero((char *)bp->b_data + lastoff, size - lastoff);
+ bawrite(bp);
+ } else if (!ffs_fsfail_cleanup(ump, error)) {
softdep_error("softdep_journal_freeblks", error);
return;
}
- bzero((char *)bp->b_data + lastoff, size - lastoff);
- bawrite(bp);
-
}
ACQUIRE_LOCK(ump);
inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
TAILQ_INSERT_TAIL(&inodedep->id_freeblklst, freeblks, fb_next);
freeblks->fb_state |= DEPCOMPLETE | ONDEPLIST;
/*
* We zero earlier truncations so they don't erroneously
* update i_blocks.
*/
if (freeblks->fb_len == 0 && (flags & IO_NORMAL) != 0)
TAILQ_FOREACH(fbn, &inodedep->id_freeblklst, fb_next)
fbn->fb_len = 0;
if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE &&
LIST_EMPTY(&freeblks->fb_jblkdephd))
freeblks->fb_state |= INPROGRESS;
else
freeblks = NULL;
FREE_LOCK(ump);
if (freeblks)
handle_workitem_freeblocks(freeblks, 0);
trunc_pages(ip, length, extblocks, flags);
}
/*
* Flush a JOP_SYNC to the journal.
*/
void
softdep_journal_fsync(ip)
struct inode *ip;
{
struct jfsync *jfsync;
struct ufsmount *ump;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_journal_fsync called on non-softdep filesystem"));
if ((ip->i_flag & IN_TRUNCATED) == 0)
return;
ip->i_flag &= ~IN_TRUNCATED;
jfsync = malloc(sizeof(*jfsync), M_JFSYNC, M_SOFTDEP_FLAGS | M_ZERO);
workitem_alloc(&jfsync->jfs_list, D_JFSYNC, UFSTOVFS(ump));
jfsync->jfs_size = ip->i_size;
jfsync->jfs_ino = ip->i_number;
ACQUIRE_LOCK(ump);
add_to_journal(&jfsync->jfs_list);
jwait(&jfsync->jfs_list, MNT_WAIT);
FREE_LOCK(ump);
}
/*
* Block de-allocation dependencies.
*
* When blocks are de-allocated, the on-disk pointers must be nullified before
* the blocks are made available for use by other files. (The true
* requirement is that old pointers must be nullified before new on-disk
* pointers are set. We chose this slightly more stringent requirement to
* reduce complexity.) Our implementation handles this dependency by updating
* the inode (or indirect block) appropriately but delaying the actual block
* de-allocation (i.e., freemap and free space count manipulation) until
* after the updated versions reach stable storage. After the disk is
* updated, the blocks can be safely de-allocated whenever it is convenient.
* This implementation handles only the common case of reducing a file's
* length to zero. Other cases are handled by the conventional synchronous
* write approach.
*
* The ffs implementation with which we worked double-checks
* the state of the block pointers and file size as it reduces
* a file's length. Some of this code is replicated here in our
* soft updates implementation. The freeblks->fb_chkcnt field is
* used to transfer a part of this information to the procedure
* that eventually de-allocates the blocks.
*
* This routine should be called from the routine that shortens
* a file's length, before the inode's size or block pointers
* are modified. It will save the block pointer information for
* later release and zero the inode so that the calling routine
* can release it.
*/
void
softdep_setup_freeblocks(ip, length, flags)
struct inode *ip; /* The inode whose length is to be reduced */
off_t length; /* The new length for the file */
int flags; /* IO_EXT and/or IO_NORMAL */
{
struct ufs1_dinode *dp1;
struct ufs2_dinode *dp2;
struct freeblks *freeblks;
struct inodedep *inodedep;
struct allocdirect *adp;
struct ufsmount *ump;
struct buf *bp;
struct fs *fs;
ufs2_daddr_t extblocks, datablocks;
struct mount *mp;
int i, delay, error;
ufs_lbn_t tmpval;
ufs_lbn_t lbn;
ump = ITOUMP(ip);
mp = UFSTOVFS(ump);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_freeblocks called on non-softdep filesystem"));
CTR2(KTR_SUJ, "softdep_setup_freeblks: ip %d length %ld",
ip->i_number, length);
KASSERT(length == 0, ("softdep_setup_freeblocks: non-zero length"));
fs = ump->um_fs;
if ((error = bread(ump->um_devvp,
fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
(int)fs->fs_bsize, NOCRED, &bp)) != 0) {
- brelse(bp);
- softdep_error("softdep_setup_freeblocks", error);
+ if (!ffs_fsfail_cleanup(ump, error))
+ softdep_error("softdep_setup_freeblocks", error);
return;
}
freeblks = newfreeblks(mp, ip);
extblocks = 0;
datablocks = 0;
if (fs->fs_magic == FS_UFS2_MAGIC)
extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize));
if ((flags & IO_NORMAL) != 0) {
for (i = 0; i < UFS_NDADDR; i++)
setup_freedirect(freeblks, ip, i, 0);
for (i = 0, tmpval = NINDIR(fs), lbn = UFS_NDADDR;
i < UFS_NIADDR;
i++, lbn += tmpval, tmpval *= NINDIR(fs))
setup_freeindir(freeblks, ip, i, -lbn -i, 0);
ip->i_size = 0;
DIP_SET(ip, i_size, 0);
datablocks = DIP(ip, i_blocks) - extblocks;
}
if ((flags & IO_EXT) != 0) {
for (i = 0; i < UFS_NXADDR; i++)
setup_freeext(freeblks, ip, i, 0);
ip->i_din2->di_extsize = 0;
datablocks += extblocks;
}
#ifdef QUOTA
/* Reference the quotas in case the block count is wrong in the end. */
quotaref(ITOV(ip), freeblks->fb_quota);
(void) chkdq(ip, -datablocks, NOCRED, FORCE);
#endif
freeblks->fb_chkcnt = -datablocks;
UFS_LOCK(ump);
fs->fs_pendingblocks += datablocks;
UFS_UNLOCK(ump);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks);
/*
* Push the zero'ed inode to its disk buffer so that we are free
* to delete its dependencies below. Once the dependencies are gone
* the buffer can be safely released.
*/
if (ump->um_fstype == UFS1) {
dp1 = ((struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number));
ip->i_din1->di_freelink = dp1->di_freelink;
*dp1 = *ip->i_din1;
} else {
dp2 = ((struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number));
ip->i_din2->di_freelink = dp2->di_freelink;
ffs_update_dinode_ckhash(fs, ip->i_din2);
*dp2 = *ip->i_din2;
}
/*
* Find and eliminate any inode dependencies.
*/
ACQUIRE_LOCK(ump);
(void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep);
if ((inodedep->id_state & IOSTARTED) != 0)
panic("softdep_setup_freeblocks: inode busy");
/*
* Add the freeblks structure to the list of operations that
* must await the zero'ed inode being written to disk. If we
* still have a bitmap dependency (delay == 0), then the inode
* has never been written to disk, so we can process the
* freeblks below once we have deleted the dependencies.
*/
delay = (inodedep->id_state & DEPCOMPLETE);
if (delay)
WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list);
else
freeblks->fb_state |= COMPLETE;
/*
* Because the file length has been truncated to zero, any
* pending block allocation dependency structures associated
* with this inode are obsolete and can simply be de-allocated.
* We must first merge the two dependency lists to get rid of
* any duplicate freefrag structures, then purge the merged list.
* If we still have a bitmap dependency, then the inode has never
* been written to disk, so we can free any fragments without delay.
*/
if (flags & IO_NORMAL) {
merge_inode_lists(&inodedep->id_newinoupdt,
&inodedep->id_inoupdt);
while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL)
cancel_allocdirect(&inodedep->id_inoupdt, adp,
freeblks);
}
if (flags & IO_EXT) {
merge_inode_lists(&inodedep->id_newextupdt,
&inodedep->id_extupdt);
while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL)
cancel_allocdirect(&inodedep->id_extupdt, adp,
freeblks);
}
FREE_LOCK(ump);
bdwrite(bp);
trunc_dependencies(ip, freeblks, -1, 0, flags);
ACQUIRE_LOCK(ump);
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0)
(void) free_inodedep(inodedep);
freeblks->fb_state |= DEPCOMPLETE;
/*
* If the inode with zeroed block pointers is now on disk
* we can start freeing blocks.
*/
if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE)
freeblks->fb_state |= INPROGRESS;
else
freeblks = NULL;
FREE_LOCK(ump);
if (freeblks)
handle_workitem_freeblocks(freeblks, 0);
trunc_pages(ip, length, extblocks, flags);
}
/*
* Eliminate pages from the page cache that back parts of this inode and
* adjust the vnode pager's idea of our size. This prevents stale data
* from hanging around in the page cache.
*/
static void
trunc_pages(ip, length, extblocks, flags)
struct inode *ip;
off_t length;
ufs2_daddr_t extblocks;
int flags;
{
struct vnode *vp;
struct fs *fs;
ufs_lbn_t lbn;
off_t end, extend;
vp = ITOV(ip);
fs = ITOFS(ip);
extend = OFF_TO_IDX(lblktosize(fs, -extblocks));
if ((flags & IO_EXT) != 0)
vn_pages_remove(vp, extend, 0);
if ((flags & IO_NORMAL) == 0)
return;
BO_LOCK(&vp->v_bufobj);
drain_output(vp);
BO_UNLOCK(&vp->v_bufobj);
/*
* The vnode pager eliminates file pages we eliminate indirects
* below.
*/
vnode_pager_setsize(vp, length);
/*
* Calculate the end based on the last indirect we want to keep. If
* the block extends into indirects we can just use the negative of
* its lbn. Doubles and triples exist at lower numbers so we must
* be careful not to remove those, if they exist. double and triple
* indirect lbns do not overlap with others so it is not important
* to verify how many levels are required.
*/
lbn = lblkno(fs, length);
if (lbn >= UFS_NDADDR) {
/* Calculate the virtual lbn of the triple indirect. */
lbn = -lbn - (UFS_NIADDR - 1);
end = OFF_TO_IDX(lblktosize(fs, lbn));
} else
end = extend;
vn_pages_remove(vp, OFF_TO_IDX(OFF_MAX), end);
}
/*
* See if the buf bp is in the range eliminated by truncation.
*/
static int
trunc_check_buf(bp, blkoffp, lastlbn, lastoff, flags)
struct buf *bp;
int *blkoffp;
ufs_lbn_t lastlbn;
int lastoff;
int flags;
{
ufs_lbn_t lbn;
*blkoffp = 0;
/* Only match ext/normal blocks as appropriate. */
if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) ||
((flags & IO_NORMAL) == 0 && (bp->b_xflags & BX_ALTDATA) == 0))
return (0);
/* ALTDATA is always a full truncation. */
if ((bp->b_xflags & BX_ALTDATA) != 0)
return (1);
/* -1 is full truncation. */
if (lastlbn == -1)
return (1);
/*
* If this is a partial truncate we only want those
* blocks and indirect blocks that cover the range
* we're after.
*/
lbn = bp->b_lblkno;
if (lbn < 0)
lbn = -(lbn + lbn_level(lbn));
if (lbn < lastlbn)
return (0);
/* Here we only truncate lblkno if it's partial. */
if (lbn == lastlbn) {
if (lastoff == 0)
return (0);
*blkoffp = lastoff;
}
return (1);
}
/*
* Eliminate any dependencies that exist in memory beyond lblkno:off
*/
static void
trunc_dependencies(ip, freeblks, lastlbn, lastoff, flags)
struct inode *ip;
struct freeblks *freeblks;
ufs_lbn_t lastlbn;
int lastoff;
int flags;
{
struct bufobj *bo;
struct vnode *vp;
struct buf *bp;
int blkoff;
/*
* We must wait for any I/O in progress to finish so that
* all potential buffers on the dirty list will be visible.
* Once they are all there, walk the list and get rid of
* any dependencies.
*/
vp = ITOV(ip);
bo = &vp->v_bufobj;
BO_LOCK(bo);
drain_output(vp);
TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
bp->b_vflags &= ~BV_SCANNED;
restart:
TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
if (bp->b_vflags & BV_SCANNED)
continue;
if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) {
bp->b_vflags |= BV_SCANNED;
continue;
}
KASSERT(bp->b_bufobj == bo, ("Wrong object in buffer"));
if ((bp = getdirtybuf(bp, BO_LOCKPTR(bo), MNT_WAIT)) == NULL)
goto restart;
BO_UNLOCK(bo);
if (deallocate_dependencies(bp, freeblks, blkoff))
bqrelse(bp);
else
brelse(bp);
BO_LOCK(bo);
goto restart;
}
/*
* Now do the work of vtruncbuf while also matching indirect blocks.
*/
TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs)
bp->b_vflags &= ~BV_SCANNED;
cleanrestart:
TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) {
if (bp->b_vflags & BV_SCANNED)
continue;
if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) {
bp->b_vflags |= BV_SCANNED;
continue;
}
if (BUF_LOCK(bp,
LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
BO_LOCKPTR(bo)) == ENOLCK) {
BO_LOCK(bo);
goto cleanrestart;
}
bp->b_vflags |= BV_SCANNED;
bremfree(bp);
if (blkoff != 0) {
allocbuf(bp, blkoff);
bqrelse(bp);
} else {
bp->b_flags |= B_INVAL | B_NOCACHE | B_RELBUF;
brelse(bp);
}
BO_LOCK(bo);
goto cleanrestart;
}
drain_output(vp);
BO_UNLOCK(bo);
}
static int
cancel_pagedep(pagedep, freeblks, blkoff)
struct pagedep *pagedep;
struct freeblks *freeblks;
int blkoff;
{
struct jremref *jremref;
struct jmvref *jmvref;
struct dirrem *dirrem, *tmp;
int i;
/*
* Copy any directory remove dependencies to the list
* to be processed after the freeblks proceeds. If
* directory entry never made it to disk they
* can be dumped directly onto the work list.
*/
LIST_FOREACH_SAFE(dirrem, &pagedep->pd_dirremhd, dm_next, tmp) {
/* Skip this directory removal if it is intended to remain. */
if (dirrem->dm_offset < blkoff)
continue;
/*
* If there are any dirrems we wait for the journal write
* to complete and then restart the buf scan as the lock
* has been dropped.
*/
while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) {
jwait(&jremref->jr_list, MNT_WAIT);
return (ERESTART);
}
LIST_REMOVE(dirrem, dm_next);
dirrem->dm_dirinum = pagedep->pd_ino;
WORKLIST_INSERT(&freeblks->fb_freeworkhd, &dirrem->dm_list);
}
while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) {
jwait(&jmvref->jm_list, MNT_WAIT);
return (ERESTART);
}
/*
* When we're partially truncating a pagedep we just want to flush
* journal entries and return. There can not be any adds in the
* truncated portion of the directory and newblk must remain if
* part of the block remains.
*/
if (blkoff != 0) {
struct diradd *dap;
LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist)
if (dap->da_offset > blkoff)
panic("cancel_pagedep: diradd %p off %d > %d",
dap, dap->da_offset, blkoff);
for (i = 0; i < DAHASHSZ; i++)
LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist)
if (dap->da_offset > blkoff)
panic("cancel_pagedep: diradd %p off %d > %d",
dap, dap->da_offset, blkoff);
return (0);
}
/*
* There should be no directory add dependencies present
* as the directory could not be truncated until all
* children were removed.
*/
KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL,
("deallocate_dependencies: pendinghd != NULL"));
for (i = 0; i < DAHASHSZ; i++)
KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL,
("deallocate_dependencies: diraddhd != NULL"));
if ((pagedep->pd_state & NEWBLOCK) != 0)
free_newdirblk(pagedep->pd_newdirblk);
if (free_pagedep(pagedep) == 0)
panic("Failed to free pagedep %p", pagedep);
return (0);
}
/*
* Reclaim any dependency structures from a buffer that is about to
* be reallocated to a new vnode. The buffer must be locked, thus,
* no I/O completion operations can occur while we are manipulating
* its associated dependencies. The mutex is held so that other I/O's
* associated with related dependencies do not occur.
*/
static int
deallocate_dependencies(bp, freeblks, off)
struct buf *bp;
struct freeblks *freeblks;
int off;
{
struct indirdep *indirdep;
struct pagedep *pagedep;
struct worklist *wk, *wkn;
struct ufsmount *ump;
ump = softdep_bp_to_mp(bp);
if (ump == NULL)
goto done;
ACQUIRE_LOCK(ump);
LIST_FOREACH_SAFE(wk, &bp->b_dep, wk_list, wkn) {
switch (wk->wk_type) {
case D_INDIRDEP:
indirdep = WK_INDIRDEP(wk);
if (bp->b_lblkno >= 0 ||
bp->b_blkno != indirdep->ir_savebp->b_lblkno)
panic("deallocate_dependencies: not indir");
cancel_indirdep(indirdep, bp, freeblks);
continue;
case D_PAGEDEP:
pagedep = WK_PAGEDEP(wk);
if (cancel_pagedep(pagedep, freeblks, off)) {
FREE_LOCK(ump);
return (ERESTART);
}
continue;
case D_ALLOCINDIR:
/*
* Simply remove the allocindir, we'll find it via
* the indirdep where we can clear pointers if
* needed.
*/
WORKLIST_REMOVE(wk);
continue;
case D_FREEWORK:
/*
* A truncation is waiting for the zero'd pointers
* to be written. It can be freed when the freeblks
* is journaled.
*/
WORKLIST_REMOVE(wk);
wk->wk_state |= ONDEPLIST;
WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk);
break;
case D_ALLOCDIRECT:
if (off != 0)
continue;
/* FALLTHROUGH */
default:
panic("deallocate_dependencies: Unexpected type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
FREE_LOCK(ump);
done:
/*
* Don't throw away this buf, we were partially truncating and
* some deps may always remain.
*/
if (off) {
allocbuf(bp, off);
bp->b_vflags |= BV_SCANNED;
return (EBUSY);
}
bp->b_flags |= B_INVAL | B_NOCACHE;
return (0);
}
/*
* An allocdirect is being canceled due to a truncate. We must make sure
* the journal entry is released in concert with the blkfree that releases
* the storage. Completed journal entries must not be released until the
* space is no longer pointed to by the inode or in the bitmap.
*/
static void
cancel_allocdirect(adphead, adp, freeblks)
struct allocdirectlst *adphead;
struct allocdirect *adp;
struct freeblks *freeblks;
{
struct freework *freework;
struct newblk *newblk;
struct worklist *wk;
TAILQ_REMOVE(adphead, adp, ad_next);
newblk = (struct newblk *)adp;
freework = NULL;
/*
* Find the correct freework structure.
*/
LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) {
if (wk->wk_type != D_FREEWORK)
continue;
freework = WK_FREEWORK(wk);
if (freework->fw_blkno == newblk->nb_newblkno)
break;
}
if (freework == NULL)
panic("cancel_allocdirect: Freework not found");
/*
* If a newblk exists at all we still have the journal entry that
* initiated the allocation so we do not need to journal the free.
*/
cancel_jfreeblk(freeblks, freework->fw_blkno);
/*
* If the journal hasn't been written the jnewblk must be passed
* to the call to ffs_blkfree that reclaims the space. We accomplish
* this by linking the journal dependency into the freework to be
* freed when freework_freeblock() is called. If the journal has
* been written we can simply reclaim the journal space when the
* freeblks work is complete.
*/
freework->fw_jnewblk = cancel_newblk(newblk, &freework->fw_list,
&freeblks->fb_jwork);
WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list);
}
/*
* Cancel a new block allocation. May be an indirect or direct block. We
* remove it from various lists and return any journal record that needs to
* be resolved by the caller.
*
* A special consideration is made for indirects which were never pointed
* at on disk and will never be found once this block is released.
*/
static struct jnewblk *
cancel_newblk(newblk, wk, wkhd)
struct newblk *newblk;
struct worklist *wk;
struct workhead *wkhd;
{
struct jnewblk *jnewblk;
CTR1(KTR_SUJ, "cancel_newblk: blkno %jd", newblk->nb_newblkno);
newblk->nb_state |= GOINGAWAY;
/*
* Previously we traversed the completedhd on each indirdep
* attached to this newblk to cancel them and gather journal
* work. Since we need only the oldest journal segment and
* the lowest point on the tree will always have the oldest
* journal segment we are free to release the segments
* of any subordinates and may leave the indirdep list to
* indirdep_complete() when this newblk is freed.
*/
if (newblk->nb_state & ONDEPLIST) {
newblk->nb_state &= ~ONDEPLIST;
LIST_REMOVE(newblk, nb_deps);
}
if (newblk->nb_state & ONWORKLIST)
WORKLIST_REMOVE(&newblk->nb_list);
/*
* If the journal entry hasn't been written we save a pointer to
* the dependency that frees it until it is written or the
* superseding operation completes.
*/
jnewblk = newblk->nb_jnewblk;
if (jnewblk != NULL && wk != NULL) {
newblk->nb_jnewblk = NULL;
jnewblk->jn_dep = wk;
}
if (!LIST_EMPTY(&newblk->nb_jwork))
jwork_move(wkhd, &newblk->nb_jwork);
/*
* When truncating we must free the newdirblk early to remove
* the pagedep from the hash before returning.
*/
if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL)
free_newdirblk(WK_NEWDIRBLK(wk));
if (!LIST_EMPTY(&newblk->nb_newdirblk))
panic("cancel_newblk: extra newdirblk");
return (jnewblk);
}
/*
* Schedule the freefrag associated with a newblk to be released once
* the pointers are written and the previous block is no longer needed.
*/
static void
newblk_freefrag(newblk)
struct newblk *newblk;
{
struct freefrag *freefrag;
if (newblk->nb_freefrag == NULL)
return;
freefrag = newblk->nb_freefrag;
newblk->nb_freefrag = NULL;
freefrag->ff_state |= COMPLETE;
if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE)
add_to_worklist(&freefrag->ff_list, 0);
}
/*
* Free a newblk. Generate a new freefrag work request if appropriate.
* This must be called after the inode pointer and any direct block pointers
* are valid or fully removed via truncate or frag extension.
*/
static void
free_newblk(newblk)
struct newblk *newblk;
{
struct indirdep *indirdep;
struct worklist *wk;
KASSERT(newblk->nb_jnewblk == NULL,
("free_newblk: jnewblk %p still attached", newblk->nb_jnewblk));
KASSERT(newblk->nb_list.wk_type != D_NEWBLK,
("free_newblk: unclaimed newblk"));
LOCK_OWNED(VFSTOUFS(newblk->nb_list.wk_mp));
newblk_freefrag(newblk);
if (newblk->nb_state & ONDEPLIST)
LIST_REMOVE(newblk, nb_deps);
if (newblk->nb_state & ONWORKLIST)
WORKLIST_REMOVE(&newblk->nb_list);
LIST_REMOVE(newblk, nb_hash);
if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL)
free_newdirblk(WK_NEWDIRBLK(wk));
if (!LIST_EMPTY(&newblk->nb_newdirblk))
panic("free_newblk: extra newdirblk");
while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL)
indirdep_complete(indirdep);
handle_jwork(&newblk->nb_jwork);
WORKITEM_FREE(newblk, D_NEWBLK);
}
/*
* Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep.
*/
static void
free_newdirblk(newdirblk)
struct newdirblk *newdirblk;
{
struct pagedep *pagedep;
struct diradd *dap;
struct worklist *wk;
LOCK_OWNED(VFSTOUFS(newdirblk->db_list.wk_mp));
WORKLIST_REMOVE(&newdirblk->db_list);
/*
* If the pagedep is still linked onto the directory buffer
* dependency chain, then some of the entries on the
* pd_pendinghd list may not be committed to disk yet. In
* this case, we will simply clear the NEWBLOCK flag and
* let the pd_pendinghd list be processed when the pagedep
* is next written. If the pagedep is no longer on the buffer
* dependency chain, then all the entries on the pd_pending
* list are committed to disk and we can free them here.
*/
pagedep = newdirblk->db_pagedep;
pagedep->pd_state &= ~NEWBLOCK;
if ((pagedep->pd_state & ONWORKLIST) == 0) {
while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
free_diradd(dap, NULL);
/*
* If no dependencies remain, the pagedep will be freed.
*/
free_pagedep(pagedep);
}
/* Should only ever be one item in the list. */
while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) {
WORKLIST_REMOVE(wk);
handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY);
}
WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
}
/*
* Prepare an inode to be freed. The actual free operation is not
* done until the zero'ed inode has been written to disk.
*/
void
softdep_freefile(pvp, ino, mode)
struct vnode *pvp;
ino_t ino;
int mode;
{
struct inode *ip = VTOI(pvp);
struct inodedep *inodedep;
struct freefile *freefile;
struct freeblks *freeblks;
struct ufsmount *ump;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_freefile called on non-softdep filesystem"));
/*
* This sets up the inode de-allocation dependency.
*/
freefile = malloc(sizeof(struct freefile),
M_FREEFILE, M_SOFTDEP_FLAGS);
workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount);
freefile->fx_mode = mode;
freefile->fx_oldinum = ino;
freefile->fx_devvp = ump->um_devvp;
LIST_INIT(&freefile->fx_jwork);
UFS_LOCK(ump);
ump->um_fs->fs_pendinginodes += 1;
UFS_UNLOCK(ump);
/*
* If the inodedep does not exist, then the zero'ed inode has
* been written to disk. If the allocated inode has never been
* written to disk, then the on-disk inode is zero'ed. In either
* case we can free the file immediately. If the journal was
* canceled before being written the inode will never make it to
* disk and we must send the canceled journal entrys to
* ffs_freefile() to be cleared in conjunction with the bitmap.
* Any blocks waiting on the inode to write can be safely freed
* here as it will never been written.
*/
ACQUIRE_LOCK(ump);
inodedep_lookup(pvp->v_mount, ino, 0, &inodedep);
if (inodedep) {
/*
* Clear out freeblks that no longer need to reference
* this inode.
*/
while ((freeblks =
TAILQ_FIRST(&inodedep->id_freeblklst)) != NULL) {
TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks,
fb_next);
freeblks->fb_state &= ~ONDEPLIST;
}
/*
* Remove this inode from the unlinked list.
*/
if (inodedep->id_state & UNLINKED) {
/*
* Save the journal work to be freed with the bitmap
* before we clear UNLINKED. Otherwise it can be lost
* if the inode block is written.
*/
handle_bufwait(inodedep, &freefile->fx_jwork);
clear_unlinked_inodedep(inodedep);
/*
* Re-acquire inodedep as we've dropped the
* per-filesystem lock in clear_unlinked_inodedep().
*/
inodedep_lookup(pvp->v_mount, ino, 0, &inodedep);
}
}
if (inodedep == NULL || check_inode_unwritten(inodedep)) {
FREE_LOCK(ump);
handle_workitem_freefile(freefile);
return;
}
if ((inodedep->id_state & DEPCOMPLETE) == 0)
inodedep->id_state |= GOINGAWAY;
WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list);
FREE_LOCK(ump);
if (ip->i_number == ino)
UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
}
/*
* Check to see if an inode has never been written to disk. If
* so free the inodedep and return success, otherwise return failure.
*
* If we still have a bitmap dependency, then the inode has never
* been written to disk. Drop the dependency as it is no longer
* necessary since the inode is being deallocated. We set the
* ALLCOMPLETE flags since the bitmap now properly shows that the
* inode is not allocated. Even if the inode is actively being
* written, it has been rolled back to its zero'ed state, so we
* are ensured that a zero inode is what is on the disk. For short
* lived files, this change will usually result in removing all the
* dependencies from the inode so that it can be freed immediately.
*/
static int
check_inode_unwritten(inodedep)
struct inodedep *inodedep;
{
LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp));
if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 ||
!LIST_EMPTY(&inodedep->id_dirremhd) ||
!LIST_EMPTY(&inodedep->id_pendinghd) ||
!LIST_EMPTY(&inodedep->id_bufwait) ||
!LIST_EMPTY(&inodedep->id_inowait) ||
!TAILQ_EMPTY(&inodedep->id_inoreflst) ||
!TAILQ_EMPTY(&inodedep->id_inoupdt) ||
!TAILQ_EMPTY(&inodedep->id_newinoupdt) ||
!TAILQ_EMPTY(&inodedep->id_extupdt) ||
!TAILQ_EMPTY(&inodedep->id_newextupdt) ||
!TAILQ_EMPTY(&inodedep->id_freeblklst) ||
inodedep->id_mkdiradd != NULL ||
inodedep->id_nlinkdelta != 0)
return (0);
/*
* Another process might be in initiate_write_inodeblock_ufs[12]
* trying to allocate memory without holding "Softdep Lock".
*/
if ((inodedep->id_state & IOSTARTED) != 0 &&
inodedep->id_savedino1 == NULL)
return (0);
if (inodedep->id_state & ONDEPLIST)
LIST_REMOVE(inodedep, id_deps);
inodedep->id_state &= ~ONDEPLIST;
inodedep->id_state |= ALLCOMPLETE;
inodedep->id_bmsafemap = NULL;
if (inodedep->id_state & ONWORKLIST)
WORKLIST_REMOVE(&inodedep->id_list);
if (inodedep->id_savedino1 != NULL) {
free(inodedep->id_savedino1, M_SAVEDINO);
inodedep->id_savedino1 = NULL;
}
if (free_inodedep(inodedep) == 0)
panic("check_inode_unwritten: busy inode");
return (1);
}
static int
check_inodedep_free(inodedep)
struct inodedep *inodedep;
{
LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp));
if ((inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE ||
!LIST_EMPTY(&inodedep->id_dirremhd) ||
!LIST_EMPTY(&inodedep->id_pendinghd) ||
!LIST_EMPTY(&inodedep->id_bufwait) ||
!LIST_EMPTY(&inodedep->id_inowait) ||
!TAILQ_EMPTY(&inodedep->id_inoreflst) ||
!TAILQ_EMPTY(&inodedep->id_inoupdt) ||
!TAILQ_EMPTY(&inodedep->id_newinoupdt) ||
!TAILQ_EMPTY(&inodedep->id_extupdt) ||
!TAILQ_EMPTY(&inodedep->id_newextupdt) ||
!TAILQ_EMPTY(&inodedep->id_freeblklst) ||
inodedep->id_mkdiradd != NULL ||
inodedep->id_nlinkdelta != 0 ||
inodedep->id_savedino1 != NULL)
return (0);
return (1);
}
/*
* Try to free an inodedep structure. Return 1 if it could be freed.
*/
static int
free_inodedep(inodedep)
struct inodedep *inodedep;
{
LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp));
if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 ||
!check_inodedep_free(inodedep))
return (0);
if (inodedep->id_state & ONDEPLIST)
LIST_REMOVE(inodedep, id_deps);
LIST_REMOVE(inodedep, id_hash);
WORKITEM_FREE(inodedep, D_INODEDEP);
return (1);
}
/*
* Free the block referenced by a freework structure. The parent freeblks
* structure is released and completed when the final cg bitmap reaches
* the disk. This routine may be freeing a jnewblk which never made it to
* disk in which case we do not have to wait as the operation is undone
* in memory immediately.
*/
static void
freework_freeblock(freework, key)
struct freework *freework;
u_long key;
{
struct freeblks *freeblks;
struct jnewblk *jnewblk;
struct ufsmount *ump;
struct workhead wkhd;
struct fs *fs;
int bsize;
int needj;
ump = VFSTOUFS(freework->fw_list.wk_mp);
LOCK_OWNED(ump);
/*
* Handle partial truncate separately.
*/
if (freework->fw_indir) {
complete_trunc_indir(freework);
return;
}
freeblks = freework->fw_freeblks;
fs = ump->um_fs;
needj = MOUNTEDSUJ(freeblks->fb_list.wk_mp) != 0;
bsize = lfragtosize(fs, freework->fw_frags);
LIST_INIT(&wkhd);
/*
* DEPCOMPLETE is cleared in indirblk_insert() if the block lives
* on the indirblk hashtable and prevents premature freeing.
*/
freework->fw_state |= DEPCOMPLETE;
/*
* SUJ needs to wait for the segment referencing freed indirect
* blocks to expire so that we know the checker will not confuse
* a re-allocated indirect block with its old contents.
*/
if (needj && freework->fw_lbn <= -UFS_NDADDR)
indirblk_insert(freework);
/*
* If we are canceling an existing jnewblk pass it to the free
* routine, otherwise pass the freeblk which will ultimately
* release the freeblks. If we're not journaling, we can just
* free the freeblks immediately.
*/
jnewblk = freework->fw_jnewblk;
if (jnewblk != NULL) {
cancel_jnewblk(jnewblk, &wkhd);
needj = 0;
} else if (needj) {
freework->fw_state |= DELAYEDFREE;
freeblks->fb_cgwait++;
WORKLIST_INSERT(&wkhd, &freework->fw_list);
}
FREE_LOCK(ump);
freeblks_free(ump, freeblks, btodb(bsize));
CTR4(KTR_SUJ,
"freework_freeblock: ino %jd blkno %jd lbn %jd size %d",
freeblks->fb_inum, freework->fw_blkno, freework->fw_lbn, bsize);
ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, bsize,
freeblks->fb_inum, freeblks->fb_vtype, &wkhd, key);
ACQUIRE_LOCK(ump);
/*
* The jnewblk will be discarded and the bits in the map never
* made it to disk. We can immediately free the freeblk.
*/
if (needj == 0)
handle_written_freework(freework);
}
/*
* We enqueue freework items that need processing back on the freeblks and
* add the freeblks to the worklist. This makes it easier to find all work
* required to flush a truncation in process_truncates().
*/
static void
freework_enqueue(freework)
struct freework *freework;
{
struct freeblks *freeblks;
freeblks = freework->fw_freeblks;
if ((freework->fw_state & INPROGRESS) == 0)
WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list);
if ((freeblks->fb_state &
(ONWORKLIST | INPROGRESS | ALLCOMPLETE)) == ALLCOMPLETE &&
LIST_EMPTY(&freeblks->fb_jblkdephd))
add_to_worklist(&freeblks->fb_list, WK_NODELAY);
}
/*
* Start, continue, or finish the process of freeing an indirect block tree.
* The free operation may be paused at any point with fw_off containing the
* offset to restart from. This enables us to implement some flow control
* for large truncates which may fan out and generate a huge number of
* dependencies.
*/
static void
handle_workitem_indirblk(freework)
struct freework *freework;
{
struct freeblks *freeblks;
struct ufsmount *ump;
struct fs *fs;
freeblks = freework->fw_freeblks;
ump = VFSTOUFS(freeblks->fb_list.wk_mp);
fs = ump->um_fs;
if (freework->fw_state & DEPCOMPLETE) {
handle_written_freework(freework);
return;
}
if (freework->fw_off == NINDIR(fs)) {
freework_freeblock(freework, SINGLETON_KEY);
return;
}
freework->fw_state |= INPROGRESS;
FREE_LOCK(ump);
indir_trunc(freework, fsbtodb(fs, freework->fw_blkno),
freework->fw_lbn);
ACQUIRE_LOCK(ump);
}
/*
* Called when a freework structure attached to a cg buf is written. The
* ref on either the parent or the freeblks structure is released and
* the freeblks is added back to the worklist if there is more work to do.
*/
static void
handle_written_freework(freework)
struct freework *freework;
{
struct freeblks *freeblks;
struct freework *parent;
freeblks = freework->fw_freeblks;
parent = freework->fw_parent;
if (freework->fw_state & DELAYEDFREE)
freeblks->fb_cgwait--;
freework->fw_state |= COMPLETE;
if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE)
WORKITEM_FREE(freework, D_FREEWORK);
if (parent) {
if (--parent->fw_ref == 0)
freework_enqueue(parent);
return;
}
if (--freeblks->fb_ref != 0)
return;
if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST | INPROGRESS)) ==
ALLCOMPLETE && LIST_EMPTY(&freeblks->fb_jblkdephd))
add_to_worklist(&freeblks->fb_list, WK_NODELAY);
}
/*
* This workitem routine performs the block de-allocation.
* The workitem is added to the pending list after the updated
* inode block has been written to disk. As mentioned above,
* checks regarding the number of blocks de-allocated (compared
* to the number of blocks allocated for the file) are also
* performed in this function.
*/
static int
handle_workitem_freeblocks(freeblks, flags)
struct freeblks *freeblks;
int flags;
{
struct freework *freework;
struct newblk *newblk;
struct allocindir *aip;
struct ufsmount *ump;
struct worklist *wk;
u_long key;
KASSERT(LIST_EMPTY(&freeblks->fb_jblkdephd),
("handle_workitem_freeblocks: Journal entries not written."));
ump = VFSTOUFS(freeblks->fb_list.wk_mp);
key = ffs_blkrelease_start(ump, freeblks->fb_devvp, freeblks->fb_inum);
ACQUIRE_LOCK(ump);
while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) {
WORKLIST_REMOVE(wk);
switch (wk->wk_type) {
case D_DIRREM:
wk->wk_state |= COMPLETE;
add_to_worklist(wk, 0);
continue;
case D_ALLOCDIRECT:
free_newblk(WK_NEWBLK(wk));
continue;
case D_ALLOCINDIR:
aip = WK_ALLOCINDIR(wk);
freework = NULL;
if (aip->ai_state & DELAYEDFREE) {
FREE_LOCK(ump);
freework = newfreework(ump, freeblks, NULL,
aip->ai_lbn, aip->ai_newblkno,
ump->um_fs->fs_frag, 0, 0);
ACQUIRE_LOCK(ump);
}
newblk = WK_NEWBLK(wk);
if (newblk->nb_jnewblk) {
freework->fw_jnewblk = newblk->nb_jnewblk;
newblk->nb_jnewblk->jn_dep = &freework->fw_list;
newblk->nb_jnewblk = NULL;
}
free_newblk(newblk);
continue;
case D_FREEWORK:
freework = WK_FREEWORK(wk);
if (freework->fw_lbn <= -UFS_NDADDR)
handle_workitem_indirblk(freework);
else
freework_freeblock(freework, key);
continue;
default:
panic("handle_workitem_freeblocks: Unknown type %s",
TYPENAME(wk->wk_type));
}
}
if (freeblks->fb_ref != 0) {
freeblks->fb_state &= ~INPROGRESS;
wake_worklist(&freeblks->fb_list);
freeblks = NULL;
}
FREE_LOCK(ump);
ffs_blkrelease_finish(ump, key);
if (freeblks)
return handle_complete_freeblocks(freeblks, flags);
return (0);
}
/*
* Handle completion of block free via truncate. This allows fs_pending
* to track the actual free block count more closely than if we only updated
* it at the end. We must be careful to handle cases where the block count
* on free was incorrect.
*/
static void
freeblks_free(ump, freeblks, blocks)
struct ufsmount *ump;
struct freeblks *freeblks;
int blocks;
{
struct fs *fs;
ufs2_daddr_t remain;
UFS_LOCK(ump);
remain = -freeblks->fb_chkcnt;
freeblks->fb_chkcnt += blocks;
if (remain > 0) {
if (remain < blocks)
blocks = remain;
fs = ump->um_fs;
fs->fs_pendingblocks -= blocks;
}
UFS_UNLOCK(ump);
}
/*
* Once all of the freework workitems are complete we can retire the
* freeblocks dependency and any journal work awaiting completion. This
* can not be called until all other dependencies are stable on disk.
*/
static int
handle_complete_freeblocks(freeblks, flags)
struct freeblks *freeblks;
int flags;
{
struct inodedep *inodedep;
struct inode *ip;
struct vnode *vp;
struct fs *fs;
struct ufsmount *ump;
ufs2_daddr_t spare;
ump = VFSTOUFS(freeblks->fb_list.wk_mp);
fs = ump->um_fs;
flags = LK_EXCLUSIVE | flags;
spare = freeblks->fb_chkcnt;
/*
* If we did not release the expected number of blocks we may have
* to adjust the inode block count here. Only do so if it wasn't
* a truncation to zero and the modrev still matches.
*/
if (spare && freeblks->fb_len != 0) {
if (ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_inum,
flags, &vp, FFSV_FORCEINSMQ) != 0)
return (EBUSY);
ip = VTOI(vp);
if (ip->i_mode == 0) {
vgone(vp);
} else if (DIP(ip, i_modrev) == freeblks->fb_modrev) {
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - spare);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
/*
* We must wait so this happens before the
* journal is reclaimed.
*/
ffs_update(vp, 1);
}
vput(vp);
}
if (spare < 0) {
UFS_LOCK(ump);
fs->fs_pendingblocks += spare;
UFS_UNLOCK(ump);
}
#ifdef QUOTA
/* Handle spare. */
if (spare)
quotaadj(freeblks->fb_quota, ump, -spare);
quotarele(freeblks->fb_quota);
#endif
ACQUIRE_LOCK(ump);
if (freeblks->fb_state & ONDEPLIST) {
inodedep_lookup(freeblks->fb_list.wk_mp, freeblks->fb_inum,
0, &inodedep);
TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, fb_next);
freeblks->fb_state &= ~ONDEPLIST;
if (TAILQ_EMPTY(&inodedep->id_freeblklst))
free_inodedep(inodedep);
}
/*
* All of the freeblock deps must be complete prior to this call
* so it's now safe to complete earlier outstanding journal entries.
*/
handle_jwork(&freeblks->fb_jwork);
WORKITEM_FREE(freeblks, D_FREEBLKS);
FREE_LOCK(ump);
return (0);
}
/*
* Release blocks associated with the freeblks and stored in the indirect
* block dbn. If level is greater than SINGLE, the block is an indirect block
* and recursive calls to indirtrunc must be used to cleanse other indirect
* blocks.
*
* This handles partial and complete truncation of blocks. Partial is noted
* with goingaway == 0. In this case the freework is completed after the
* zero'd indirects are written to disk. For full truncation the freework
* is completed after the block is freed.
*/
static void
indir_trunc(freework, dbn, lbn)
struct freework *freework;
ufs2_daddr_t dbn;
ufs_lbn_t lbn;
{
struct freework *nfreework;
struct workhead wkhd;
struct freeblks *freeblks;
struct buf *bp;
struct fs *fs;
struct indirdep *indirdep;
struct mount *mp;
struct ufsmount *ump;
ufs1_daddr_t *bap1;
ufs2_daddr_t nb, nnb, *bap2;
ufs_lbn_t lbnadd, nlbn;
u_long key;
int nblocks, ufs1fmt, freedblocks;
- int goingaway, freedeps, needj, level, cnt, i;
+ int goingaway, freedeps, needj, level, cnt, i, error;
freeblks = freework->fw_freeblks;
mp = freeblks->fb_list.wk_mp;
ump = VFSTOUFS(mp);
fs = ump->um_fs;
/*
* Get buffer of block pointers to be freed. There are three cases:
*
* 1) Partial truncate caches the indirdep pointer in the freework
* which provides us a back copy to the save bp which holds the
* pointers we want to clear. When this completes the zero
* pointers are written to the real copy.
* 2) The indirect is being completely truncated, cancel_indirdep()
* eliminated the real copy and placed the indirdep on the saved
* copy. The indirdep and buf are discarded when this completes.
* 3) The indirect was not in memory, we read a copy off of the disk
* using the devvp and drop and invalidate the buffer when we're
* done.
*/
goingaway = 1;
indirdep = NULL;
if (freework->fw_indir != NULL) {
goingaway = 0;
indirdep = freework->fw_indir;
bp = indirdep->ir_savebp;
if (bp == NULL || bp->b_blkno != dbn)
panic("indir_trunc: Bad saved buf %p blkno %jd",
bp, (intmax_t)dbn);
} else if ((bp = incore(&freeblks->fb_devvp->v_bufobj, dbn)) != NULL) {
/*
* The lock prevents the buf dep list from changing and
* indirects on devvp should only ever have one dependency.
*/
indirdep = WK_INDIRDEP(LIST_FIRST(&bp->b_dep));
if (indirdep == NULL || (indirdep->ir_state & GOINGAWAY) == 0)
panic("indir_trunc: Bad indirdep %p from buf %p",
indirdep, bp);
- } else if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize,
- NOCRED, &bp) != 0) {
- brelse(bp);
- return;
+ } else {
+ error = ffs_breadz(ump, freeblks->fb_devvp, dbn, dbn,
+ (int)fs->fs_bsize, NULL, NULL, 0, NOCRED, 0, NULL, &bp);
+ if (error)
+ return;
}
ACQUIRE_LOCK(ump);
/* Protects against a race with complete_trunc_indir(). */
freework->fw_state &= ~INPROGRESS;
/*
* If we have an indirdep we need to enforce the truncation order
* and discard it when it is complete.
*/
if (indirdep) {
if (freework != TAILQ_FIRST(&indirdep->ir_trunc) &&
!TAILQ_EMPTY(&indirdep->ir_trunc)) {
/*
* Add the complete truncate to the list on the
* indirdep to enforce in-order processing.
*/
if (freework->fw_indir == NULL)
TAILQ_INSERT_TAIL(&indirdep->ir_trunc,
freework, fw_next);
FREE_LOCK(ump);
return;
}
/*
* If we're goingaway, free the indirdep. Otherwise it will
* linger until the write completes.
*/
if (goingaway) {
KASSERT(indirdep->ir_savebp == bp,
("indir_trunc: losing ir_savebp %p",
indirdep->ir_savebp));
indirdep->ir_savebp = NULL;
free_indirdep(indirdep);
}
}
FREE_LOCK(ump);
/* Initialize pointers depending on block size. */
if (ump->um_fstype == UFS1) {
bap1 = (ufs1_daddr_t *)bp->b_data;
nb = bap1[freework->fw_off];
ufs1fmt = 1;
bap2 = NULL;
} else {
bap2 = (ufs2_daddr_t *)bp->b_data;
nb = bap2[freework->fw_off];
ufs1fmt = 0;
bap1 = NULL;
}
level = lbn_level(lbn);
needj = MOUNTEDSUJ(UFSTOVFS(ump)) != 0;
lbnadd = lbn_offset(fs, level);
nblocks = btodb(fs->fs_bsize);
nfreework = freework;
freedeps = 0;
cnt = 0;
/*
* Reclaim blocks. Traverses into nested indirect levels and
* arranges for the current level to be freed when subordinates
* are free when journaling.
*/
key = ffs_blkrelease_start(ump, freeblks->fb_devvp, freeblks->fb_inum);
for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) {
if (UFS_CHECK_BLKNO(mp, freeblks->fb_inum, nb,
fs->fs_bsize) != 0)
nb = 0;
if (i != NINDIR(fs) - 1) {
if (ufs1fmt)
nnb = bap1[i+1];
else
nnb = bap2[i+1];
} else
nnb = 0;
if (nb == 0)
continue;
cnt++;
if (level != 0) {
nlbn = (lbn + 1) - (i * lbnadd);
if (needj != 0) {
nfreework = newfreework(ump, freeblks, freework,
nlbn, nb, fs->fs_frag, 0, 0);
freedeps++;
}
indir_trunc(nfreework, fsbtodb(fs, nb), nlbn);
} else {
struct freedep *freedep;
/*
* Attempt to aggregate freedep dependencies for
* all blocks being released to the same CG.
*/
LIST_INIT(&wkhd);
if (needj != 0 &&
(nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) {
freedep = newfreedep(freework);
WORKLIST_INSERT_UNLOCKED(&wkhd,
&freedep->fd_list);
freedeps++;
}
CTR3(KTR_SUJ,
"indir_trunc: ino %jd blkno %jd size %d",
freeblks->fb_inum, nb, fs->fs_bsize);
ffs_blkfree(ump, fs, freeblks->fb_devvp, nb,
fs->fs_bsize, freeblks->fb_inum,
freeblks->fb_vtype, &wkhd, key);
}
}
ffs_blkrelease_finish(ump, key);
if (goingaway) {
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
}
freedblocks = 0;
if (level == 0)
freedblocks = (nblocks * cnt);
if (needj == 0)
freedblocks += nblocks;
freeblks_free(ump, freeblks, freedblocks);
/*
* If we are journaling set up the ref counts and offset so this
* indirect can be completed when its children are free.
*/
if (needj) {
ACQUIRE_LOCK(ump);
freework->fw_off = i;
freework->fw_ref += freedeps;
freework->fw_ref -= NINDIR(fs) + 1;
if (level == 0)
freeblks->fb_cgwait += freedeps;
if (freework->fw_ref == 0)
freework_freeblock(freework, SINGLETON_KEY);
FREE_LOCK(ump);
return;
}
/*
* If we're not journaling we can free the indirect now.
*/
dbn = dbtofsb(fs, dbn);
CTR3(KTR_SUJ,
"indir_trunc 2: ino %jd blkno %jd size %d",
freeblks->fb_inum, dbn, fs->fs_bsize);
ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize,
freeblks->fb_inum, freeblks->fb_vtype, NULL, SINGLETON_KEY);
/* Non SUJ softdep does single-threaded truncations. */
if (freework->fw_blkno == dbn) {
freework->fw_state |= ALLCOMPLETE;
ACQUIRE_LOCK(ump);
handle_written_freework(freework);
FREE_LOCK(ump);
}
return;
}
/*
* Cancel an allocindir when it is removed via truncation. When bp is not
* NULL the indirect never appeared on disk and is scheduled to be freed
* independently of the indir so we can more easily track journal work.
*/
static void
cancel_allocindir(aip, bp, freeblks, trunc)
struct allocindir *aip;
struct buf *bp;
struct freeblks *freeblks;
int trunc;
{
struct indirdep *indirdep;
struct freefrag *freefrag;
struct newblk *newblk;
newblk = (struct newblk *)aip;
LIST_REMOVE(aip, ai_next);
/*
* We must eliminate the pointer in bp if it must be freed on its
* own due to partial truncate or pending journal work.
*/
if (bp && (trunc || newblk->nb_jnewblk)) {
/*
* Clear the pointer and mark the aip to be freed
* directly if it never existed on disk.
*/
aip->ai_state |= DELAYEDFREE;
indirdep = aip->ai_indirdep;
if (indirdep->ir_state & UFS1FMT)
((ufs1_daddr_t *)bp->b_data)[aip->ai_offset] = 0;
else
((ufs2_daddr_t *)bp->b_data)[aip->ai_offset] = 0;
}
/*
* When truncating the previous pointer will be freed via
* savedbp. Eliminate the freefrag which would dup free.
*/
if (trunc && (freefrag = newblk->nb_freefrag) != NULL) {
newblk->nb_freefrag = NULL;
if (freefrag->ff_jdep)
cancel_jfreefrag(
WK_JFREEFRAG(freefrag->ff_jdep));
jwork_move(&freeblks->fb_jwork, &freefrag->ff_jwork);
WORKITEM_FREE(freefrag, D_FREEFRAG);
}
/*
* If the journal hasn't been written the jnewblk must be passed
* to the call to ffs_blkfree that reclaims the space. We accomplish
* this by leaving the journal dependency on the newblk to be freed
* when a freework is created in handle_workitem_freeblocks().
*/
cancel_newblk(newblk, NULL, &freeblks->fb_jwork);
WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list);
}
/*
* Create the mkdir dependencies for . and .. in a new directory. Link them
* in to a newdirblk so any subsequent additions are tracked properly. The
* caller is responsible for adding the mkdir1 dependency to the journal
* and updating id_mkdiradd. This function returns with the per-filesystem
* lock held.
*/
static struct mkdir *
setup_newdir(dap, newinum, dinum, newdirbp, mkdirp)
struct diradd *dap;
ino_t newinum;
ino_t dinum;
struct buf *newdirbp;
struct mkdir **mkdirp;
{
struct newblk *newblk;
struct pagedep *pagedep;
struct inodedep *inodedep;
struct newdirblk *newdirblk;
struct mkdir *mkdir1, *mkdir2;
struct worklist *wk;
struct jaddref *jaddref;
struct ufsmount *ump;
struct mount *mp;
mp = dap->da_list.wk_mp;
ump = VFSTOUFS(mp);
newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK,
M_SOFTDEP_FLAGS);
workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp);
LIST_INIT(&newdirblk->db_mkdir);
mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS);
workitem_alloc(&mkdir1->md_list, D_MKDIR, mp);
mkdir1->md_state = ATTACHED | MKDIR_BODY;
mkdir1->md_diradd = dap;
mkdir1->md_jaddref = NULL;
mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS);
workitem_alloc(&mkdir2->md_list, D_MKDIR, mp);
mkdir2->md_state = ATTACHED | MKDIR_PARENT;
mkdir2->md_diradd = dap;
mkdir2->md_jaddref = NULL;
if (MOUNTEDSUJ(mp) == 0) {
mkdir1->md_state |= DEPCOMPLETE;
mkdir2->md_state |= DEPCOMPLETE;
}
/*
* Dependency on "." and ".." being written to disk.
*/
mkdir1->md_buf = newdirbp;
ACQUIRE_LOCK(VFSTOUFS(mp));
LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir1, md_mkdirs);
/*
* We must link the pagedep, allocdirect, and newdirblk for
* the initial file page so the pointer to the new directory
* is not written until the directory contents are live and
* any subsequent additions are not marked live until the
* block is reachable via the inode.
*/
if (pagedep_lookup(mp, newdirbp, newinum, 0, 0, &pagedep) == 0)
panic("setup_newdir: lost pagedep");
LIST_FOREACH(wk, &newdirbp->b_dep, wk_list)
if (wk->wk_type == D_ALLOCDIRECT)
break;
if (wk == NULL)
panic("setup_newdir: lost allocdirect");
if (pagedep->pd_state & NEWBLOCK)
panic("setup_newdir: NEWBLOCK already set");
newblk = WK_NEWBLK(wk);
pagedep->pd_state |= NEWBLOCK;
pagedep->pd_newdirblk = newdirblk;
newdirblk->db_pagedep = pagedep;
WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list);
WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list);
/*
* Look up the inodedep for the parent directory so that we
* can link mkdir2 into the pending dotdot jaddref or
* the inode write if there is none. If the inode is
* ALLCOMPLETE and no jaddref is present all dependencies have
* been satisfied and mkdir2 can be freed.
*/
inodedep_lookup(mp, dinum, 0, &inodedep);
if (MOUNTEDSUJ(mp)) {
if (inodedep == NULL)
panic("setup_newdir: Lost parent.");
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref != NULL && jaddref->ja_parent == newinum &&
(jaddref->ja_state & MKDIR_PARENT),
("setup_newdir: bad dotdot jaddref %p", jaddref));
LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs);
mkdir2->md_jaddref = jaddref;
jaddref->ja_mkdir = mkdir2;
} else if (inodedep == NULL ||
(inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
dap->da_state &= ~MKDIR_PARENT;
WORKITEM_FREE(mkdir2, D_MKDIR);
mkdir2 = NULL;
} else {
LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs);
WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir2->md_list);
}
*mkdirp = mkdir2;
return (mkdir1);
}
/*
* Directory entry addition dependencies.
*
* When adding a new directory entry, the inode (with its incremented link
* count) must be written to disk before the directory entry's pointer to it.
* Also, if the inode is newly allocated, the corresponding freemap must be
* updated (on disk) before the directory entry's pointer. These requirements
* are met via undo/redo on the directory entry's pointer, which consists
* simply of the inode number.
*
* As directory entries are added and deleted, the free space within a
* directory block can become fragmented. The ufs filesystem will compact
* a fragmented directory block to make space for a new entry. When this
* occurs, the offsets of previously added entries change. Any "diradd"
* dependency structures corresponding to these entries must be updated with
* the new offsets.
*/
/*
* This routine is called after the in-memory inode's link
* count has been incremented, but before the directory entry's
* pointer to the inode has been set.
*/
int
softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk)
struct buf *bp; /* buffer containing directory block */
struct inode *dp; /* inode for directory */
off_t diroffset; /* offset of new entry in directory */
ino_t newinum; /* inode referenced by new directory entry */
struct buf *newdirbp; /* non-NULL => contents of new mkdir */
int isnewblk; /* entry is in a newly allocated block */
{
int offset; /* offset of new entry within directory block */
ufs_lbn_t lbn; /* block in directory containing new entry */
struct fs *fs;
struct diradd *dap;
struct newblk *newblk;
struct pagedep *pagedep;
struct inodedep *inodedep;
struct newdirblk *newdirblk;
struct mkdir *mkdir1, *mkdir2;
struct jaddref *jaddref;
struct ufsmount *ump;
struct mount *mp;
int isindir;
mp = ITOVFS(dp);
ump = VFSTOUFS(mp);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_directory_add called on non-softdep filesystem"));
/*
* Whiteouts have no dependencies.
*/
if (newinum == UFS_WINO) {
if (newdirbp != NULL)
bdwrite(newdirbp);
return (0);
}
jaddref = NULL;
mkdir1 = mkdir2 = NULL;
fs = ump->um_fs;
lbn = lblkno(fs, diroffset);
offset = blkoff(fs, diroffset);
dap = malloc(sizeof(struct diradd), M_DIRADD,
M_SOFTDEP_FLAGS|M_ZERO);
workitem_alloc(&dap->da_list, D_DIRADD, mp);
dap->da_offset = offset;
dap->da_newinum = newinum;
dap->da_state = ATTACHED;
LIST_INIT(&dap->da_jwork);
isindir = bp->b_lblkno >= UFS_NDADDR;
newdirblk = NULL;
if (isnewblk &&
(isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) {
newdirblk = malloc(sizeof(struct newdirblk),
M_NEWDIRBLK, M_SOFTDEP_FLAGS);
workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp);
LIST_INIT(&newdirblk->db_mkdir);
}
/*
* If we're creating a new directory setup the dependencies and set
* the dap state to wait for them. Otherwise it's COMPLETE and
* we can move on.
*/
if (newdirbp == NULL) {
dap->da_state |= DEPCOMPLETE;
ACQUIRE_LOCK(ump);
} else {
dap->da_state |= MKDIR_BODY | MKDIR_PARENT;
mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp,
&mkdir2);
}
/*
* Link into parent directory pagedep to await its being written.
*/
pagedep_lookup(mp, bp, dp->i_number, lbn, DEPALLOC, &pagedep);
#ifdef INVARIANTS
if (diradd_lookup(pagedep, offset) != NULL)
panic("softdep_setup_directory_add: %p already at off %d\n",
diradd_lookup(pagedep, offset), offset);
#endif
dap->da_pagedep = pagedep;
LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap,
da_pdlist);
inodedep_lookup(mp, newinum, DEPALLOC, &inodedep);
/*
* If we're journaling, link the diradd into the jaddref so it
* may be completed after the journal entry is written. Otherwise,
* link the diradd into its inodedep. If the inode is not yet
* written place it on the bufwait list, otherwise do the post-inode
* write processing to put it on the id_pendinghd list.
*/
if (MOUNTEDSUJ(mp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
("softdep_setup_directory_add: bad jaddref %p", jaddref));
jaddref->ja_diroff = diroffset;
jaddref->ja_diradd = dap;
add_to_journal(&jaddref->ja_list);
} else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE)
diradd_inode_written(dap, inodedep);
else
WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
/*
* Add the journal entries for . and .. links now that the primary
* link is written.
*/
if (mkdir1 != NULL && MOUNTEDSUJ(mp)) {
jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref,
inoreflst, if_deps);
KASSERT(jaddref != NULL &&
jaddref->ja_ino == jaddref->ja_parent &&
(jaddref->ja_state & MKDIR_BODY),
("softdep_setup_directory_add: bad dot jaddref %p",
jaddref));
mkdir1->md_jaddref = jaddref;
jaddref->ja_mkdir = mkdir1;
/*
* It is important that the dotdot journal entry
* is added prior to the dot entry since dot writes
* both the dot and dotdot links. These both must
* be added after the primary link for the journal
* to remain consistent.
*/
add_to_journal(&mkdir2->md_jaddref->ja_list);
add_to_journal(&jaddref->ja_list);
}
/*
* If we are adding a new directory remember this diradd so that if
* we rename it we can keep the dot and dotdot dependencies. If
* we are adding a new name for an inode that has a mkdiradd we
* must be in rename and we have to move the dot and dotdot
* dependencies to this new name. The old name is being orphaned
* soon.
*/
if (mkdir1 != NULL) {
if (inodedep->id_mkdiradd != NULL)
panic("softdep_setup_directory_add: Existing mkdir");
inodedep->id_mkdiradd = dap;
} else if (inodedep->id_mkdiradd)
merge_diradd(inodedep, dap);
if (newdirblk != NULL) {
/*
* There is nothing to do if we are already tracking
* this block.
*/
if ((pagedep->pd_state & NEWBLOCK) != 0) {
WORKITEM_FREE(newdirblk, D_NEWDIRBLK);
FREE_LOCK(ump);
return (0);
}
if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk)
== 0)
panic("softdep_setup_directory_add: lost entry");
WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list);
pagedep->pd_state |= NEWBLOCK;
pagedep->pd_newdirblk = newdirblk;
newdirblk->db_pagedep = pagedep;
FREE_LOCK(ump);
/*
* If we extended into an indirect signal direnter to sync.
*/
if (isindir)
return (1);
return (0);
}
FREE_LOCK(ump);
return (0);
}
/*
* This procedure is called to change the offset of a directory
* entry when compacting a directory block which must be owned
* exclusively by the caller. Note that the actual entry movement
* must be done in this procedure to ensure that no I/O completions
* occur while the move is in progress.
*/
void
softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize)
struct buf *bp; /* Buffer holding directory block. */
struct inode *dp; /* inode for directory */
caddr_t base; /* address of dp->i_offset */
caddr_t oldloc; /* address of old directory location */
caddr_t newloc; /* address of new directory location */
int entrysize; /* size of directory entry */
{
int offset, oldoffset, newoffset;
struct pagedep *pagedep;
struct jmvref *jmvref;
struct diradd *dap;
struct direct *de;
struct mount *mp;
struct ufsmount *ump;
ufs_lbn_t lbn;
int flags;
mp = ITOVFS(dp);
ump = VFSTOUFS(mp);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_change_directoryentry_offset called on "
"non-softdep filesystem"));
de = (struct direct *)oldloc;
jmvref = NULL;
flags = 0;
/*
* Moves are always journaled as it would be too complex to
* determine if any affected adds or removes are present in the
* journal.
*/
if (MOUNTEDSUJ(mp)) {
flags = DEPALLOC;
jmvref = newjmvref(dp, de->d_ino,
dp->i_offset + (oldloc - base),
dp->i_offset + (newloc - base));
}
lbn = lblkno(ump->um_fs, dp->i_offset);
offset = blkoff(ump->um_fs, dp->i_offset);
oldoffset = offset + (oldloc - base);
newoffset = offset + (newloc - base);
ACQUIRE_LOCK(ump);
if (pagedep_lookup(mp, bp, dp->i_number, lbn, flags, &pagedep) == 0)
goto done;
dap = diradd_lookup(pagedep, oldoffset);
if (dap) {
dap->da_offset = newoffset;
newoffset = DIRADDHASH(newoffset);
oldoffset = DIRADDHASH(oldoffset);
if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE &&
newoffset != oldoffset) {
LIST_REMOVE(dap, da_pdlist);
LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset],
dap, da_pdlist);
}
}
done:
if (jmvref) {
jmvref->jm_pagedep = pagedep;
LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps);
add_to_journal(&jmvref->jm_list);
}
bcopy(oldloc, newloc, entrysize);
FREE_LOCK(ump);
}
/*
* Move the mkdir dependencies and journal work from one diradd to another
* when renaming a directory. The new name must depend on the mkdir deps
* completing as the old name did. Directories can only have one valid link
* at a time so one must be canonical.
*/
static void
merge_diradd(inodedep, newdap)
struct inodedep *inodedep;
struct diradd *newdap;
{
struct diradd *olddap;
struct mkdir *mkdir, *nextmd;
struct ufsmount *ump;
short state;
olddap = inodedep->id_mkdiradd;
inodedep->id_mkdiradd = newdap;
if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
newdap->da_state &= ~DEPCOMPLETE;
ump = VFSTOUFS(inodedep->id_list.wk_mp);
for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir;
mkdir = nextmd) {
nextmd = LIST_NEXT(mkdir, md_mkdirs);
if (mkdir->md_diradd != olddap)
continue;
mkdir->md_diradd = newdap;
state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY);
newdap->da_state |= state;
olddap->da_state &= ~state;
if ((olddap->da_state &
(MKDIR_PARENT | MKDIR_BODY)) == 0)
break;
}
if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0)
panic("merge_diradd: unfound ref");
}
/*
* Any mkdir related journal items are not safe to be freed until
* the new name is stable.
*/
jwork_move(&newdap->da_jwork, &olddap->da_jwork);
olddap->da_state |= DEPCOMPLETE;
complete_diradd(olddap);
}
/*
* Move the diradd to the pending list when all diradd dependencies are
* complete.
*/
static void
complete_diradd(dap)
struct diradd *dap;
{
struct pagedep *pagedep;
if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
if (dap->da_state & DIRCHG)
pagedep = dap->da_previous->dm_pagedep;
else
pagedep = dap->da_pagedep;
LIST_REMOVE(dap, da_pdlist);
LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
}
}
/*
* Cancel a diradd when a dirrem overlaps with it. We must cancel the journal
* add entries and conditonally journal the remove.
*/
static void
cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref)
struct diradd *dap;
struct dirrem *dirrem;
struct jremref *jremref;
struct jremref *dotremref;
struct jremref *dotdotremref;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct inoref *inoref;
struct ufsmount *ump;
struct mkdir *mkdir;
/*
* If no remove references were allocated we're on a non-journaled
* filesystem and can skip the cancel step.
*/
if (jremref == NULL) {
free_diradd(dap, NULL);
return;
}
/*
* Cancel the primary name an free it if it does not require
* journaling.
*/
if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum,
0, &inodedep) != 0) {
/* Abort the addref that reference this diradd. */
TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
if (inoref->if_list.wk_type != D_JADDREF)
continue;
jaddref = (struct jaddref *)inoref;
if (jaddref->ja_diradd != dap)
continue;
if (cancel_jaddref(jaddref, inodedep,
&dirrem->dm_jwork) == 0) {
free_jremref(jremref);
jremref = NULL;
}
break;
}
}
/*
* Cancel subordinate names and free them if they do not require
* journaling.
*/
if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
ump = VFSTOUFS(dap->da_list.wk_mp);
LIST_FOREACH(mkdir, &ump->softdep_mkdirlisthd, md_mkdirs) {
if (mkdir->md_diradd != dap)
continue;
if ((jaddref = mkdir->md_jaddref) == NULL)
continue;
mkdir->md_jaddref = NULL;
if (mkdir->md_state & MKDIR_PARENT) {
if (cancel_jaddref(jaddref, NULL,
&dirrem->dm_jwork) == 0) {
free_jremref(dotdotremref);
dotdotremref = NULL;
}
} else {
if (cancel_jaddref(jaddref, inodedep,
&dirrem->dm_jwork) == 0) {
free_jremref(dotremref);
dotremref = NULL;
}
}
}
}
if (jremref)
journal_jremref(dirrem, jremref, inodedep);
if (dotremref)
journal_jremref(dirrem, dotremref, inodedep);
if (dotdotremref)
journal_jremref(dirrem, dotdotremref, NULL);
jwork_move(&dirrem->dm_jwork, &dap->da_jwork);
free_diradd(dap, &dirrem->dm_jwork);
}
/*
* Free a diradd dependency structure.
*/
static void
free_diradd(dap, wkhd)
struct diradd *dap;
struct workhead *wkhd;
{
struct dirrem *dirrem;
struct pagedep *pagedep;
struct inodedep *inodedep;
struct mkdir *mkdir, *nextmd;
struct ufsmount *ump;
ump = VFSTOUFS(dap->da_list.wk_mp);
LOCK_OWNED(ump);
LIST_REMOVE(dap, da_pdlist);
if (dap->da_state & ONWORKLIST)
WORKLIST_REMOVE(&dap->da_list);
if ((dap->da_state & DIRCHG) == 0) {
pagedep = dap->da_pagedep;
} else {
dirrem = dap->da_previous;
pagedep = dirrem->dm_pagedep;
dirrem->dm_dirinum = pagedep->pd_ino;
dirrem->dm_state |= COMPLETE;
if (LIST_EMPTY(&dirrem->dm_jremrefhd))
add_to_worklist(&dirrem->dm_list, 0);
}
if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum,
0, &inodedep) != 0)
if (inodedep->id_mkdiradd == dap)
inodedep->id_mkdiradd = NULL;
if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) {
for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir;
mkdir = nextmd) {
nextmd = LIST_NEXT(mkdir, md_mkdirs);
if (mkdir->md_diradd != dap)
continue;
dap->da_state &=
~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY));
LIST_REMOVE(mkdir, md_mkdirs);
if (mkdir->md_state & ONWORKLIST)
WORKLIST_REMOVE(&mkdir->md_list);
if (mkdir->md_jaddref != NULL)
panic("free_diradd: Unexpected jaddref");
WORKITEM_FREE(mkdir, D_MKDIR);
if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0)
break;
}
if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0)
panic("free_diradd: unfound ref");
}
if (inodedep)
free_inodedep(inodedep);
/*
* Free any journal segments waiting for the directory write.
*/
handle_jwork(&dap->da_jwork);
WORKITEM_FREE(dap, D_DIRADD);
}
/*
* Directory entry removal dependencies.
*
* When removing a directory entry, the entry's inode pointer must be
* zero'ed on disk before the corresponding inode's link count is decremented
* (possibly freeing the inode for re-use). This dependency is handled by
* updating the directory entry but delaying the inode count reduction until
* after the directory block has been written to disk. After this point, the
* inode count can be decremented whenever it is convenient.
*/
/*
* This routine should be called immediately after removing
* a directory entry. The inode's link count should not be
* decremented by the calling procedure -- the soft updates
* code will do this task when it is safe.
*/
void
softdep_setup_remove(bp, dp, ip, isrmdir)
struct buf *bp; /* buffer containing directory block */
struct inode *dp; /* inode for the directory being modified */
struct inode *ip; /* inode for directory entry being removed */
int isrmdir; /* indicates if doing RMDIR */
{
struct dirrem *dirrem, *prevdirrem;
struct inodedep *inodedep;
struct ufsmount *ump;
int direct;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_setup_remove called on non-softdep filesystem"));
/*
* Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want
* newdirrem() to setup the full directory remove which requires
* isrmdir > 1.
*/
dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
/*
* Add the dirrem to the inodedep's pending remove list for quick
* discovery later.
*/
if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0)
panic("softdep_setup_remove: Lost inodedep.");
KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked"));
dirrem->dm_state |= ONDEPLIST;
LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
/*
* If the COMPLETE flag is clear, then there were no active
* entries and we want to roll back to a zeroed entry until
* the new inode is committed to disk. If the COMPLETE flag is
* set then we have deleted an entry that never made it to
* disk. If the entry we deleted resulted from a name change,
* then the old name still resides on disk. We cannot delete
* its inode (returned to us in prevdirrem) until the zeroed
* directory entry gets to disk. The new inode has never been
* referenced on the disk, so can be deleted immediately.
*/
if ((dirrem->dm_state & COMPLETE) == 0) {
LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem,
dm_next);
FREE_LOCK(ump);
} else {
if (prevdirrem != NULL)
LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd,
prevdirrem, dm_next);
dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino;
direct = LIST_EMPTY(&dirrem->dm_jremrefhd);
FREE_LOCK(ump);
if (direct)
handle_workitem_remove(dirrem, 0);
}
}
/*
* Check for an entry matching 'offset' on both the pd_dirraddhd list and the
* pd_pendinghd list of a pagedep.
*/
static struct diradd *
diradd_lookup(pagedep, offset)
struct pagedep *pagedep;
int offset;
{
struct diradd *dap;
LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist)
if (dap->da_offset == offset)
return (dap);
LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist)
if (dap->da_offset == offset)
return (dap);
return (NULL);
}
/*
* Search for a .. diradd dependency in a directory that is being removed.
* If the directory was renamed to a new parent we have a diradd rather
* than a mkdir for the .. entry. We need to cancel it now before
* it is found in truncate().
*/
static struct jremref *
cancel_diradd_dotdot(ip, dirrem, jremref)
struct inode *ip;
struct dirrem *dirrem;
struct jremref *jremref;
{
struct pagedep *pagedep;
struct diradd *dap;
struct worklist *wk;
if (pagedep_lookup(ITOVFS(ip), NULL, ip->i_number, 0, 0, &pagedep) == 0)
return (jremref);
dap = diradd_lookup(pagedep, DOTDOT_OFFSET);
if (dap == NULL)
return (jremref);
cancel_diradd(dap, dirrem, jremref, NULL, NULL);
/*
* Mark any journal work as belonging to the parent so it is freed
* with the .. reference.
*/
LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list)
wk->wk_state |= MKDIR_PARENT;
return (NULL);
}
/*
* Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to
* replace it with a dirrem/diradd pair as a result of re-parenting a
* directory. This ensures that we don't simultaneously have a mkdir and
* a diradd for the same .. entry.
*/
static struct jremref *
cancel_mkdir_dotdot(ip, dirrem, jremref)
struct inode *ip;
struct dirrem *dirrem;
struct jremref *jremref;
{
struct inodedep *inodedep;
struct jaddref *jaddref;
struct ufsmount *ump;
struct mkdir *mkdir;
struct diradd *dap;
struct mount *mp;
mp = ITOVFS(ip);
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0)
return (jremref);
dap = inodedep->id_mkdiradd;
if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0)
return (jremref);
ump = VFSTOUFS(inodedep->id_list.wk_mp);
for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir;
mkdir = LIST_NEXT(mkdir, md_mkdirs))
if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT)
break;
if (mkdir == NULL)
panic("cancel_mkdir_dotdot: Unable to find mkdir\n");
if ((jaddref = mkdir->md_jaddref) != NULL) {
mkdir->md_jaddref = NULL;
jaddref->ja_state &= ~MKDIR_PARENT;
if (inodedep_lookup(mp, jaddref->ja_ino, 0, &inodedep) == 0)
panic("cancel_mkdir_dotdot: Lost parent inodedep");
if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) {
journal_jremref(dirrem, jremref, inodedep);
jremref = NULL;
}
}
if (mkdir->md_state & ONWORKLIST)
WORKLIST_REMOVE(&mkdir->md_list);
mkdir->md_state |= ALLCOMPLETE;
complete_mkdir(mkdir);
return (jremref);
}
static void
journal_jremref(dirrem, jremref, inodedep)
struct dirrem *dirrem;
struct jremref *jremref;
struct inodedep *inodedep;
{
if (inodedep == NULL)
if (inodedep_lookup(jremref->jr_list.wk_mp,
jremref->jr_ref.if_ino, 0, &inodedep) == 0)
panic("journal_jremref: Lost inodedep");
LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps);
TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps);
add_to_journal(&jremref->jr_list);
}
static void
dirrem_journal(dirrem, jremref, dotremref, dotdotremref)
struct dirrem *dirrem;
struct jremref *jremref;
struct jremref *dotremref;
struct jremref *dotdotremref;
{
struct inodedep *inodedep;
if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0,
&inodedep) == 0)
panic("dirrem_journal: Lost inodedep");
journal_jremref(dirrem, jremref, inodedep);
if (dotremref)
journal_jremref(dirrem, dotremref, inodedep);
if (dotdotremref)
journal_jremref(dirrem, dotdotremref, NULL);
}
/*
* Allocate a new dirrem if appropriate and return it along with
* its associated pagedep. Called without a lock, returns with lock.
*/
static struct dirrem *
newdirrem(bp, dp, ip, isrmdir, prevdirremp)
struct buf *bp; /* buffer containing directory block */
struct inode *dp; /* inode for the directory being modified */
struct inode *ip; /* inode for directory entry being removed */
int isrmdir; /* indicates if doing RMDIR */
struct dirrem **prevdirremp; /* previously referenced inode, if any */
{
int offset;
ufs_lbn_t lbn;
struct diradd *dap;
struct dirrem *dirrem;
struct pagedep *pagedep;
struct jremref *jremref;
struct jremref *dotremref;
struct jremref *dotdotremref;
struct vnode *dvp;
struct ufsmount *ump;
/*
* Whiteouts have no deletion dependencies.
*/
if (ip == NULL)
panic("newdirrem: whiteout");
dvp = ITOV(dp);
ump = ITOUMP(dp);
/*
* If the system is over its limit and our filesystem is
* responsible for more than our share of that usage and
* we are not a snapshot, request some inodedep cleanup.
* Limiting the number of dirrem structures will also limit
* the number of freefile and freeblks structures.
*/
ACQUIRE_LOCK(ump);
if (!IS_SNAPSHOT(ip) && softdep_excess_items(ump, D_DIRREM))
schedule_cleanup(UFSTOVFS(ump));
else
FREE_LOCK(ump);
dirrem = malloc(sizeof(struct dirrem), M_DIRREM, M_SOFTDEP_FLAGS |
M_ZERO);
workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount);
LIST_INIT(&dirrem->dm_jremrefhd);
LIST_INIT(&dirrem->dm_jwork);
dirrem->dm_state = isrmdir ? RMDIR : 0;
dirrem->dm_oldinum = ip->i_number;
*prevdirremp = NULL;
/*
* Allocate remove reference structures to track journal write
* dependencies. We will always have one for the link and
* when doing directories we will always have one more for dot.
* When renaming a directory we skip the dotdot link change so
* this is not needed.
*/
jremref = dotremref = dotdotremref = NULL;
if (DOINGSUJ(dvp)) {
if (isrmdir) {
jremref = newjremref(dirrem, dp, ip, dp->i_offset,
ip->i_effnlink + 2);
dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET,
ip->i_effnlink + 1);
dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET,
dp->i_effnlink + 1);
dotdotremref->jr_state |= MKDIR_PARENT;
} else
jremref = newjremref(dirrem, dp, ip, dp->i_offset,
ip->i_effnlink + 1);
}
ACQUIRE_LOCK(ump);
lbn = lblkno(ump->um_fs, dp->i_offset);
offset = blkoff(ump->um_fs, dp->i_offset);
pagedep_lookup(UFSTOVFS(ump), bp, dp->i_number, lbn, DEPALLOC,
&pagedep);
dirrem->dm_pagedep = pagedep;
dirrem->dm_offset = offset;
/*
* If we're renaming a .. link to a new directory, cancel any
* existing MKDIR_PARENT mkdir. If it has already been canceled
* the jremref is preserved for any potential diradd in this
* location. This can not coincide with a rmdir.
*/
if (dp->i_offset == DOTDOT_OFFSET) {
if (isrmdir)
panic("newdirrem: .. directory change during remove?");
jremref = cancel_mkdir_dotdot(dp, dirrem, jremref);
}
/*
* If we're removing a directory search for the .. dependency now and
* cancel it. Any pending journal work will be added to the dirrem
* to be completed when the workitem remove completes.
*/
if (isrmdir)
dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref);
/*
* Check for a diradd dependency for the same directory entry.
* If present, then both dependencies become obsolete and can
* be de-allocated.
*/
dap = diradd_lookup(pagedep, offset);
if (dap == NULL) {
/*
* Link the jremref structures into the dirrem so they are
* written prior to the pagedep.
*/
if (jremref)
dirrem_journal(dirrem, jremref, dotremref,
dotdotremref);
return (dirrem);
}
/*
* Must be ATTACHED at this point.
*/
if ((dap->da_state & ATTACHED) == 0)
panic("newdirrem: not ATTACHED");
if (dap->da_newinum != ip->i_number)
panic("newdirrem: inum %ju should be %ju",
(uintmax_t)ip->i_number, (uintmax_t)dap->da_newinum);
/*
* If we are deleting a changed name that never made it to disk,
* then return the dirrem describing the previous inode (which
* represents the inode currently referenced from this entry on disk).
*/
if ((dap->da_state & DIRCHG) != 0) {
*prevdirremp = dap->da_previous;
dap->da_state &= ~DIRCHG;
dap->da_pagedep = pagedep;
}
/*
* We are deleting an entry that never made it to disk.
* Mark it COMPLETE so we can delete its inode immediately.
*/
dirrem->dm_state |= COMPLETE;
cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref);
#ifdef INVARIANTS
if (isrmdir == 0) {
struct worklist *wk;
LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list)
if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT))
panic("bad wk %p (0x%X)\n", wk, wk->wk_state);
}
#endif
return (dirrem);
}
/*
* Directory entry change dependencies.
*
* Changing an existing directory entry requires that an add operation
* be completed first followed by a deletion. The semantics for the addition
* are identical to the description of adding a new entry above except
* that the rollback is to the old inode number rather than zero. Once
* the addition dependency is completed, the removal is done as described
* in the removal routine above.
*/
/*
* This routine should be called immediately after changing
* a directory entry. The inode's link count should not be
* decremented by the calling procedure -- the soft updates
* code will perform this task when it is safe.
*/
void
softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir)
struct buf *bp; /* buffer containing directory block */
struct inode *dp; /* inode for the directory being modified */
struct inode *ip; /* inode for directory entry being removed */
ino_t newinum; /* new inode number for changed entry */
int isrmdir; /* indicates if doing RMDIR */
{
int offset;
struct diradd *dap = NULL;
struct dirrem *dirrem, *prevdirrem;
struct pagedep *pagedep;
struct inodedep *inodedep;
struct jaddref *jaddref;
struct mount *mp;
struct ufsmount *ump;
mp = ITOVFS(dp);
ump = VFSTOUFS(mp);
offset = blkoff(ump->um_fs, dp->i_offset);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_directory_change called on non-softdep filesystem"));
/*
* Whiteouts do not need diradd dependencies.
*/
if (newinum != UFS_WINO) {
dap = malloc(sizeof(struct diradd),
M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO);
workitem_alloc(&dap->da_list, D_DIRADD, mp);
dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE;
dap->da_offset = offset;
dap->da_newinum = newinum;
LIST_INIT(&dap->da_jwork);
}
/*
* Allocate a new dirrem and ACQUIRE_LOCK.
*/
dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem);
pagedep = dirrem->dm_pagedep;
/*
* The possible values for isrmdir:
* 0 - non-directory file rename
* 1 - directory rename within same directory
* inum - directory rename to new directory of given inode number
* When renaming to a new directory, we are both deleting and
* creating a new directory entry, so the link count on the new
* directory should not change. Thus we do not need the followup
* dirrem which is usually done in handle_workitem_remove. We set
* the DIRCHG flag to tell handle_workitem_remove to skip the
* followup dirrem.
*/
if (isrmdir > 1)
dirrem->dm_state |= DIRCHG;
/*
* Whiteouts have no additional dependencies,
* so just put the dirrem on the correct list.
*/
if (newinum == UFS_WINO) {
if ((dirrem->dm_state & COMPLETE) == 0) {
LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem,
dm_next);
} else {
dirrem->dm_dirinum = pagedep->pd_ino;
if (LIST_EMPTY(&dirrem->dm_jremrefhd))
add_to_worklist(&dirrem->dm_list, 0);
}
FREE_LOCK(ump);
return;
}
/*
* Add the dirrem to the inodedep's pending remove list for quick
* discovery later. A valid nlinkdelta ensures that this lookup
* will not fail.
*/
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0)
panic("softdep_setup_directory_change: Lost inodedep.");
dirrem->dm_state |= ONDEPLIST;
LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
/*
* If the COMPLETE flag is clear, then there were no active
* entries and we want to roll back to the previous inode until
* the new inode is committed to disk. If the COMPLETE flag is
* set, then we have deleted an entry that never made it to disk.
* If the entry we deleted resulted from a name change, then the old
* inode reference still resides on disk. Any rollback that we do
* needs to be to that old inode (returned to us in prevdirrem). If
* the entry we deleted resulted from a create, then there is
* no entry on the disk, so we want to roll back to zero rather
* than the uncommitted inode. In either of the COMPLETE cases we
* want to immediately free the unwritten and unreferenced inode.
*/
if ((dirrem->dm_state & COMPLETE) == 0) {
dap->da_previous = dirrem;
} else {
if (prevdirrem != NULL) {
dap->da_previous = prevdirrem;
} else {
dap->da_state &= ~DIRCHG;
dap->da_pagedep = pagedep;
}
dirrem->dm_dirinum = pagedep->pd_ino;
if (LIST_EMPTY(&dirrem->dm_jremrefhd))
add_to_worklist(&dirrem->dm_list, 0);
}
/*
* Lookup the jaddref for this journal entry. We must finish
* initializing it and make the diradd write dependent on it.
* If we're not journaling, put it on the id_bufwait list if the
* inode is not yet written. If it is written, do the post-inode
* write processing to put it on the id_pendinghd list.
*/
inodedep_lookup(mp, newinum, DEPALLOC, &inodedep);
if (MOUNTEDSUJ(mp)) {
jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst,
inoreflst);
KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number,
("softdep_setup_directory_change: bad jaddref %p",
jaddref));
jaddref->ja_diroff = dp->i_offset;
jaddref->ja_diradd = dap;
LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)],
dap, da_pdlist);
add_to_journal(&jaddref->ja_list);
} else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) {
dap->da_state |= COMPLETE;
LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist);
WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
} else {
LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)],
dap, da_pdlist);
WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list);
}
/*
* If we're making a new name for a directory that has not been
* committed when need to move the dot and dotdot references to
* this new name.
*/
if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET)
merge_diradd(inodedep, dap);
FREE_LOCK(ump);
}
/*
* Called whenever the link count on an inode is changed.
* It creates an inode dependency so that the new reference(s)
* to the inode cannot be committed to disk until the updated
* inode has been written.
*/
void
softdep_change_linkcnt(ip)
struct inode *ip; /* the inode with the increased link count */
{
struct inodedep *inodedep;
struct ufsmount *ump;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_change_linkcnt called on non-softdep filesystem"));
ACQUIRE_LOCK(ump);
inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep);
if (ip->i_nlink < ip->i_effnlink)
panic("softdep_change_linkcnt: bad delta");
inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
FREE_LOCK(ump);
}
/*
* Attach a sbdep dependency to the superblock buf so that we can keep
* track of the head of the linked list of referenced but unlinked inodes.
*/
void
softdep_setup_sbupdate(ump, fs, bp)
struct ufsmount *ump;
struct fs *fs;
struct buf *bp;
{
struct sbdep *sbdep;
struct worklist *wk;
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_setup_sbupdate called on non-softdep filesystem"));
LIST_FOREACH(wk, &bp->b_dep, wk_list)
if (wk->wk_type == D_SBDEP)
break;
if (wk != NULL)
return;
sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS);
workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump));
sbdep->sb_fs = fs;
sbdep->sb_ump = ump;
ACQUIRE_LOCK(ump);
WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list);
FREE_LOCK(ump);
}
/*
* Return the first unlinked inodedep which is ready to be the head of the
* list. The inodedep and all those after it must have valid next pointers.
*/
static struct inodedep *
first_unlinked_inodedep(ump)
struct ufsmount *ump;
{
struct inodedep *inodedep;
struct inodedep *idp;
LOCK_OWNED(ump);
for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst);
inodedep; inodedep = idp) {
if ((inodedep->id_state & UNLINKNEXT) == 0)
return (NULL);
idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0)
break;
if ((inodedep->id_state & UNLINKPREV) == 0)
break;
}
return (inodedep);
}
/*
* Set the sujfree unlinked head pointer prior to writing a superblock.
*/
static void
initiate_write_sbdep(sbdep)
struct sbdep *sbdep;
{
struct inodedep *inodedep;
struct fs *bpfs;
struct fs *fs;
bpfs = sbdep->sb_fs;
fs = sbdep->sb_ump->um_fs;
inodedep = first_unlinked_inodedep(sbdep->sb_ump);
if (inodedep) {
fs->fs_sujfree = inodedep->id_ino;
inodedep->id_state |= UNLINKPREV;
} else
fs->fs_sujfree = 0;
bpfs->fs_sujfree = fs->fs_sujfree;
/*
* Because we have made changes to the superblock, we need to
* recompute its check-hash.
*/
bpfs->fs_ckhash = ffs_calc_sbhash(bpfs);
}
/*
* After a superblock is written determine whether it must be written again
* due to a changing unlinked list head.
*/
static int
handle_written_sbdep(sbdep, bp)
struct sbdep *sbdep;
struct buf *bp;
{
struct inodedep *inodedep;
struct fs *fs;
LOCK_OWNED(sbdep->sb_ump);
fs = sbdep->sb_fs;
/*
* If the superblock doesn't match the in-memory list start over.
*/
inodedep = first_unlinked_inodedep(sbdep->sb_ump);
if ((inodedep && fs->fs_sujfree != inodedep->id_ino) ||
(inodedep == NULL && fs->fs_sujfree != 0)) {
bdirty(bp);
return (1);
}
WORKITEM_FREE(sbdep, D_SBDEP);
if (fs->fs_sujfree == 0)
return (0);
/*
* Now that we have a record of this inode in stable store allow it
* to be written to free up pending work. Inodes may see a lot of
* write activity after they are unlinked which we must not hold up.
*/
for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) {
if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS)
panic("handle_written_sbdep: Bad inodedep %p (0x%X)",
inodedep, inodedep->id_state);
if (inodedep->id_state & UNLINKONLIST)
break;
inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST;
}
return (0);
}
/*
* Mark an inodedep as unlinked and insert it into the in-memory unlinked list.
*/
static void
unlinked_inodedep(mp, inodedep)
struct mount *mp;
struct inodedep *inodedep;
{
struct ufsmount *ump;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
if (MOUNTEDSUJ(mp) == 0)
return;
ump->um_fs->fs_fmod = 1;
if (inodedep->id_state & UNLINKED)
panic("unlinked_inodedep: %p already unlinked\n", inodedep);
inodedep->id_state |= UNLINKED;
TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked);
}
/*
* Remove an inodedep from the unlinked inodedep list. This may require
* disk writes if the inode has made it that far.
*/
static void
clear_unlinked_inodedep(inodedep)
struct inodedep *inodedep;
{
struct ufs2_dinode *dip;
struct ufsmount *ump;
struct inodedep *idp;
struct inodedep *idn;
struct fs *fs, *bpfs;
struct buf *bp;
+ daddr_t dbn;
ino_t ino;
ino_t nino;
ino_t pino;
int error;
ump = VFSTOUFS(inodedep->id_list.wk_mp);
fs = ump->um_fs;
ino = inodedep->id_ino;
error = 0;
for (;;) {
LOCK_OWNED(ump);
KASSERT((inodedep->id_state & UNLINKED) != 0,
("clear_unlinked_inodedep: inodedep %p not unlinked",
inodedep));
/*
* If nothing has yet been written simply remove us from
* the in memory list and return. This is the most common
* case where handle_workitem_remove() loses the final
* reference.
*/
if ((inodedep->id_state & UNLINKLINKS) == 0)
break;
/*
* If we have a NEXT pointer and no PREV pointer we can simply
* clear NEXT's PREV and remove ourselves from the list. Be
* careful not to clear PREV if the superblock points at
* next as well.
*/
idn = TAILQ_NEXT(inodedep, id_unlinked);
if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) {
if (idn && fs->fs_sujfree != idn->id_ino)
idn->id_state &= ~UNLINKPREV;
break;
}
/*
* Here we have an inodedep which is actually linked into
* the list. We must remove it by forcing a write to the
* link before us, whether it be the superblock or an inode.
* Unfortunately the list may change while we're waiting
* on the buf lock for either resource so we must loop until
* we lock the right one. If both the superblock and an
* inode point to this inode we must clear the inode first
* followed by the superblock.
*/
idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
pino = 0;
if (idp && (idp->id_state & UNLINKNEXT))
pino = idp->id_ino;
FREE_LOCK(ump);
if (pino == 0) {
bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc),
(int)fs->fs_sbsize, 0, 0, 0);
} else {
- error = bread(ump->um_devvp,
- fsbtodb(fs, ino_to_fsba(fs, pino)),
- (int)fs->fs_bsize, NOCRED, &bp);
- if (error)
- brelse(bp);
+ dbn = fsbtodb(fs, ino_to_fsba(fs, pino));
+ error = ffs_breadz(ump, ump->um_devvp, dbn, dbn,
+ (int)fs->fs_bsize, NULL, NULL, 0, NOCRED, 0, NULL,
+ &bp);
}
ACQUIRE_LOCK(ump);
if (error)
break;
/* If the list has changed restart the loop. */
idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked);
nino = 0;
if (idp && (idp->id_state & UNLINKNEXT))
nino = idp->id_ino;
if (nino != pino ||
(inodedep->id_state & UNLINKPREV) != UNLINKPREV) {
FREE_LOCK(ump);
brelse(bp);
ACQUIRE_LOCK(ump);
continue;
}
nino = 0;
idn = TAILQ_NEXT(inodedep, id_unlinked);
if (idn)
nino = idn->id_ino;
/*
* Remove us from the in memory list. After this we cannot
* access the inodedep.
*/
KASSERT((inodedep->id_state & UNLINKED) != 0,
("clear_unlinked_inodedep: inodedep %p not unlinked",
inodedep));
inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST);
TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked);
FREE_LOCK(ump);
/*
* The predecessor's next pointer is manually updated here
* so that the NEXT flag is never cleared for an element
* that is in the list.
*/
if (pino == 0) {
bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize);
bpfs = (struct fs *)bp->b_data;
ffs_oldfscompat_write(bpfs, ump);
softdep_setup_sbupdate(ump, bpfs, bp);
/*
* Because we may have made changes to the superblock,
* we need to recompute its check-hash.
*/
bpfs->fs_ckhash = ffs_calc_sbhash(bpfs);
} else if (fs->fs_magic == FS_UFS1_MAGIC) {
((struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, pino))->di_freelink = nino;
} else {
dip = (struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, pino);
dip->di_freelink = nino;
ffs_update_dinode_ckhash(fs, dip);
}
/*
* If the bwrite fails we have no recourse to recover. The
* filesystem is corrupted already.
*/
bwrite(bp);
ACQUIRE_LOCK(ump);
/*
* If the superblock pointer still needs to be cleared force
* a write here.
*/
if (fs->fs_sujfree == ino) {
FREE_LOCK(ump);
bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc),
(int)fs->fs_sbsize, 0, 0, 0);
bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize);
bpfs = (struct fs *)bp->b_data;
ffs_oldfscompat_write(bpfs, ump);
softdep_setup_sbupdate(ump, bpfs, bp);
/*
* Because we may have made changes to the superblock,
* we need to recompute its check-hash.
*/
bpfs->fs_ckhash = ffs_calc_sbhash(bpfs);
bwrite(bp);
ACQUIRE_LOCK(ump);
}
if (fs->fs_sujfree != ino)
return;
panic("clear_unlinked_inodedep: Failed to clear free head");
}
if (inodedep->id_ino == fs->fs_sujfree)
panic("clear_unlinked_inodedep: Freeing head of free list");
inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST);
TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked);
return;
}
/*
* This workitem decrements the inode's link count.
* If the link count reaches zero, the file is removed.
*/
static int
handle_workitem_remove(dirrem, flags)
struct dirrem *dirrem;
int flags;
{
struct inodedep *inodedep;
struct workhead dotdotwk;
struct worklist *wk;
struct ufsmount *ump;
struct mount *mp;
struct vnode *vp;
struct inode *ip;
ino_t oldinum;
if (dirrem->dm_state & ONWORKLIST)
panic("handle_workitem_remove: dirrem %p still on worklist",
dirrem);
oldinum = dirrem->dm_oldinum;
mp = dirrem->dm_list.wk_mp;
ump = VFSTOUFS(mp);
flags |= LK_EXCLUSIVE;
if (ffs_vgetf(mp, oldinum, flags, &vp, FFSV_FORCEINSMQ) != 0)
return (EBUSY);
ip = VTOI(vp);
MPASS(ip->i_mode != 0);
ACQUIRE_LOCK(ump);
if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0)
panic("handle_workitem_remove: lost inodedep");
if (dirrem->dm_state & ONDEPLIST)
LIST_REMOVE(dirrem, dm_inonext);
KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd),
("handle_workitem_remove: Journal entries not written."));
/*
* Move all dependencies waiting on the remove to complete
* from the dirrem to the inode inowait list to be completed
* after the inode has been updated and written to disk.
*
* Any marked MKDIR_PARENT are saved to be completed when the
* dotdot ref is removed unless DIRCHG is specified. For
* directory change operations there will be no further
* directory writes and the jsegdeps need to be moved along
* with the rest to be completed when the inode is free or
* stable in the inode free list.
*/
LIST_INIT(&dotdotwk);
while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) {
WORKLIST_REMOVE(wk);
if ((dirrem->dm_state & DIRCHG) == 0 &&
wk->wk_state & MKDIR_PARENT) {
wk->wk_state &= ~MKDIR_PARENT;
WORKLIST_INSERT(&dotdotwk, wk);
continue;
}
WORKLIST_INSERT(&inodedep->id_inowait, wk);
}
LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list);
/*
* Normal file deletion.
*/
if ((dirrem->dm_state & RMDIR) == 0) {
ip->i_nlink--;
KASSERT(ip->i_nlink >= 0, ("handle_workitem_remove: file ino "
"%ju negative i_nlink %d", (intmax_t)ip->i_number,
ip->i_nlink));
DIP_SET(ip, i_nlink, ip->i_nlink);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (ip->i_nlink < ip->i_effnlink)
panic("handle_workitem_remove: bad file delta");
if (ip->i_nlink == 0)
unlinked_inodedep(mp, inodedep);
inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
KASSERT(LIST_EMPTY(&dirrem->dm_jwork),
("handle_workitem_remove: worklist not empty. %s",
TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type)));
WORKITEM_FREE(dirrem, D_DIRREM);
FREE_LOCK(ump);
goto out;
}
/*
* Directory deletion. Decrement reference count for both the
* just deleted parent directory entry and the reference for ".".
* Arrange to have the reference count on the parent decremented
* to account for the loss of "..".
*/
ip->i_nlink -= 2;
KASSERT(ip->i_nlink >= 0, ("handle_workitem_remove: directory ino "
"%ju negative i_nlink %d", (intmax_t)ip->i_number, ip->i_nlink));
DIP_SET(ip, i_nlink, ip->i_nlink);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (ip->i_nlink < ip->i_effnlink)
panic("handle_workitem_remove: bad dir delta");
if (ip->i_nlink == 0)
unlinked_inodedep(mp, inodedep);
inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink;
/*
* Rename a directory to a new parent. Since, we are both deleting
* and creating a new directory entry, the link count on the new
* directory should not change. Thus we skip the followup dirrem.
*/
if (dirrem->dm_state & DIRCHG) {
KASSERT(LIST_EMPTY(&dirrem->dm_jwork),
("handle_workitem_remove: DIRCHG and worklist not empty."));
WORKITEM_FREE(dirrem, D_DIRREM);
FREE_LOCK(ump);
goto out;
}
dirrem->dm_state = ONDEPLIST;
dirrem->dm_oldinum = dirrem->dm_dirinum;
/*
* Place the dirrem on the parent's diremhd list.
*/
if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0)
panic("handle_workitem_remove: lost dir inodedep");
LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext);
/*
* If the allocated inode has never been written to disk, then
* the on-disk inode is zero'ed and we can remove the file
* immediately. When journaling if the inode has been marked
* unlinked and not DEPCOMPLETE we know it can never be written.
*/
inodedep_lookup(mp, oldinum, 0, &inodedep);
if (inodedep == NULL ||
(inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED ||
check_inode_unwritten(inodedep)) {
FREE_LOCK(ump);
vput(vp);
return handle_workitem_remove(dirrem, flags);
}
WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list);
FREE_LOCK(ump);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
out:
ffs_update(vp, 0);
vput(vp);
return (0);
}
/*
* Inode de-allocation dependencies.
*
* When an inode's link count is reduced to zero, it can be de-allocated. We
* found it convenient to postpone de-allocation until after the inode is
* written to disk with its new link count (zero). At this point, all of the
* on-disk inode's block pointers are nullified and, with careful dependency
* list ordering, all dependencies related to the inode will be satisfied and
* the corresponding dependency structures de-allocated. So, if/when the
* inode is reused, there will be no mixing of old dependencies with new
* ones. This artificial dependency is set up by the block de-allocation
* procedure above (softdep_setup_freeblocks) and completed by the
* following procedure.
*/
static void
handle_workitem_freefile(freefile)
struct freefile *freefile;
{
struct workhead wkhd;
struct fs *fs;
struct ufsmount *ump;
int error;
#ifdef INVARIANTS
struct inodedep *idp;
#endif
ump = VFSTOUFS(freefile->fx_list.wk_mp);
fs = ump->um_fs;
#ifdef INVARIANTS
ACQUIRE_LOCK(ump);
error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp);
FREE_LOCK(ump);
if (error)
panic("handle_workitem_freefile: inodedep %p survived", idp);
#endif
UFS_LOCK(ump);
fs->fs_pendinginodes -= 1;
UFS_UNLOCK(ump);
LIST_INIT(&wkhd);
LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list);
if ((error = ffs_freefile(ump, fs, freefile->fx_devvp,
freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0)
softdep_error("handle_workitem_freefile", error);
ACQUIRE_LOCK(ump);
WORKITEM_FREE(freefile, D_FREEFILE);
FREE_LOCK(ump);
}
/*
* Helper function which unlinks marker element from work list and returns
* the next element on the list.
*/
static __inline struct worklist *
markernext(struct worklist *marker)
{
struct worklist *next;
next = LIST_NEXT(marker, wk_list);
LIST_REMOVE(marker, wk_list);
return next;
}
/*
* Disk writes.
*
* The dependency structures constructed above are most actively used when file
* system blocks are written to disk. No constraints are placed on when a
* block can be written, but unsatisfied update dependencies are made safe by
* modifying (or replacing) the source memory for the duration of the disk
* write. When the disk write completes, the memory block is again brought
* up-to-date.
*
* In-core inode structure reclamation.
*
* Because there are a finite number of "in-core" inode structures, they are
* reused regularly. By transferring all inode-related dependencies to the
* in-memory inode block and indexing them separately (via "inodedep"s), we
* can allow "in-core" inode structures to be reused at any time and avoid
* any increase in contention.
*
* Called just before entering the device driver to initiate a new disk I/O.
* The buffer must be locked, thus, no I/O completion operations can occur
* while we are manipulating its associated dependencies.
*/
static void
softdep_disk_io_initiation(bp)
struct buf *bp; /* structure describing disk write to occur */
{
struct worklist *wk;
struct worklist marker;
struct inodedep *inodedep;
struct freeblks *freeblks;
struct jblkdep *jblkdep;
struct newblk *newblk;
struct ufsmount *ump;
/*
* We only care about write operations. There should never
* be dependencies for reads.
*/
if (bp->b_iocmd != BIO_WRITE)
panic("softdep_disk_io_initiation: not write");
if (bp->b_vflags & BV_BKGRDINPROG)
panic("softdep_disk_io_initiation: Writing buffer with "
"background write in progress: %p", bp);
ump = softdep_bp_to_mp(bp);
if (ump == NULL)
return;
marker.wk_type = D_LAST + 1; /* Not a normal workitem */
PHOLD(curproc); /* Don't swap out kernel stack */
ACQUIRE_LOCK(ump);
/*
* Do any necessary pre-I/O processing.
*/
for (wk = LIST_FIRST(&bp->b_dep); wk != NULL;
wk = markernext(&marker)) {
LIST_INSERT_AFTER(wk, &marker, wk_list);
switch (wk->wk_type) {
case D_PAGEDEP:
initiate_write_filepage(WK_PAGEDEP(wk), bp);
continue;
case D_INODEDEP:
inodedep = WK_INODEDEP(wk);
if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC)
initiate_write_inodeblock_ufs1(inodedep, bp);
else
initiate_write_inodeblock_ufs2(inodedep, bp);
continue;
case D_INDIRDEP:
initiate_write_indirdep(WK_INDIRDEP(wk), bp);
continue;
case D_BMSAFEMAP:
initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp);
continue;
case D_JSEG:
WK_JSEG(wk)->js_buf = NULL;
continue;
case D_FREEBLKS:
freeblks = WK_FREEBLKS(wk);
jblkdep = LIST_FIRST(&freeblks->fb_jblkdephd);
/*
* We have to wait for the freeblks to be journaled
* before we can write an inodeblock with updated
* pointers. Be careful to arrange the marker so
* we revisit the freeblks if it's not removed by
* the first jwait().
*/
if (jblkdep != NULL) {
LIST_REMOVE(&marker, wk_list);
LIST_INSERT_BEFORE(wk, &marker, wk_list);
jwait(&jblkdep->jb_list, MNT_WAIT);
}
continue;
case D_ALLOCDIRECT:
case D_ALLOCINDIR:
/*
* We have to wait for the jnewblk to be journaled
* before we can write to a block if the contents
* may be confused with an earlier file's indirect
* at recovery time. Handle the marker as described
* above.
*/
newblk = WK_NEWBLK(wk);
if (newblk->nb_jnewblk != NULL &&
indirblk_lookup(newblk->nb_list.wk_mp,
newblk->nb_newblkno)) {
LIST_REMOVE(&marker, wk_list);
LIST_INSERT_BEFORE(wk, &marker, wk_list);
jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
}
continue;
case D_SBDEP:
initiate_write_sbdep(WK_SBDEP(wk));
continue;
case D_MKDIR:
case D_FREEWORK:
case D_FREEDEP:
case D_JSEGDEP:
continue;
default:
panic("handle_disk_io_initiation: Unexpected type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
FREE_LOCK(ump);
PRELE(curproc); /* Allow swapout of kernel stack */
}
/*
* Called from within the procedure above to deal with unsatisfied
* allocation dependencies in a directory. The buffer must be locked,
* thus, no I/O completion operations can occur while we are
* manipulating its associated dependencies.
*/
static void
initiate_write_filepage(pagedep, bp)
struct pagedep *pagedep;
struct buf *bp;
{
struct jremref *jremref;
struct jmvref *jmvref;
struct dirrem *dirrem;
struct diradd *dap;
struct direct *ep;
int i;
if (pagedep->pd_state & IOSTARTED) {
/*
* This can only happen if there is a driver that does not
* understand chaining. Here biodone will reissue the call
* to strategy for the incomplete buffers.
*/
printf("initiate_write_filepage: already started\n");
return;
}
pagedep->pd_state |= IOSTARTED;
/*
* Wait for all journal remove dependencies to hit the disk.
* We can not allow any potentially conflicting directory adds
* to be visible before removes and rollback is too difficult.
* The per-filesystem lock may be dropped and re-acquired, however
* we hold the buf locked so the dependency can not go away.
*/
LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next)
while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL)
jwait(&jremref->jr_list, MNT_WAIT);
while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL)
jwait(&jmvref->jm_list, MNT_WAIT);
for (i = 0; i < DAHASHSZ; i++) {
LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
ep = (struct direct *)
((char *)bp->b_data + dap->da_offset);
if (ep->d_ino != dap->da_newinum)
panic("%s: dir inum %ju != new %ju",
"initiate_write_filepage",
(uintmax_t)ep->d_ino,
(uintmax_t)dap->da_newinum);
if (dap->da_state & DIRCHG)
ep->d_ino = dap->da_previous->dm_oldinum;
else
ep->d_ino = 0;
dap->da_state &= ~ATTACHED;
dap->da_state |= UNDONE;
}
}
}
/*
* Version of initiate_write_inodeblock that handles UFS1 dinodes.
* Note that any bug fixes made to this routine must be done in the
* version found below.
*
* Called from within the procedure above to deal with unsatisfied
* allocation dependencies in an inodeblock. The buffer must be
* locked, thus, no I/O completion operations can occur while we
* are manipulating its associated dependencies.
*/
static void
initiate_write_inodeblock_ufs1(inodedep, bp)
struct inodedep *inodedep;
struct buf *bp; /* The inode block */
{
struct allocdirect *adp, *lastadp;
struct ufs1_dinode *dp;
struct ufs1_dinode *sip;
struct inoref *inoref;
struct ufsmount *ump;
struct fs *fs;
ufs_lbn_t i;
#ifdef INVARIANTS
ufs_lbn_t prevlbn = 0;
#endif
int deplist;
if (inodedep->id_state & IOSTARTED)
panic("initiate_write_inodeblock_ufs1: already started");
inodedep->id_state |= IOSTARTED;
fs = inodedep->id_fs;
ump = VFSTOUFS(inodedep->id_list.wk_mp);
LOCK_OWNED(ump);
dp = (struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, inodedep->id_ino);
/*
* If we're on the unlinked list but have not yet written our
* next pointer initialize it here.
*/
if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) {
struct inodedep *inon;
inon = TAILQ_NEXT(inodedep, id_unlinked);
dp->di_freelink = inon ? inon->id_ino : 0;
}
/*
* If the bitmap is not yet written, then the allocated
* inode cannot be written to disk.
*/
if ((inodedep->id_state & DEPCOMPLETE) == 0) {
if (inodedep->id_savedino1 != NULL)
panic("initiate_write_inodeblock_ufs1: I/O underway");
FREE_LOCK(ump);
sip = malloc(sizeof(struct ufs1_dinode),
M_SAVEDINO, M_SOFTDEP_FLAGS);
ACQUIRE_LOCK(ump);
inodedep->id_savedino1 = sip;
*inodedep->id_savedino1 = *dp;
bzero((caddr_t)dp, sizeof(struct ufs1_dinode));
dp->di_gen = inodedep->id_savedino1->di_gen;
dp->di_freelink = inodedep->id_savedino1->di_freelink;
return;
}
/*
* If no dependencies, then there is nothing to roll back.
*/
inodedep->id_savedsize = dp->di_size;
inodedep->id_savedextsize = 0;
inodedep->id_savednlink = dp->di_nlink;
if (TAILQ_EMPTY(&inodedep->id_inoupdt) &&
TAILQ_EMPTY(&inodedep->id_inoreflst))
return;
/*
* Revert the link count to that of the first unwritten journal entry.
*/
inoref = TAILQ_FIRST(&inodedep->id_inoreflst);
if (inoref)
dp->di_nlink = inoref->if_nlink;
/*
* Set the dependencies to busy.
*/
for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
adp = TAILQ_NEXT(adp, ad_next)) {
#ifdef INVARIANTS
if (deplist != 0 && prevlbn >= adp->ad_offset)
panic("softdep_write_inodeblock: lbn order");
prevlbn = adp->ad_offset;
if (adp->ad_offset < UFS_NDADDR &&
dp->di_db[adp->ad_offset] != adp->ad_newblkno)
panic("initiate_write_inodeblock_ufs1: "
"direct pointer #%jd mismatch %d != %jd",
(intmax_t)adp->ad_offset,
dp->di_db[adp->ad_offset],
(intmax_t)adp->ad_newblkno);
if (adp->ad_offset >= UFS_NDADDR &&
dp->di_ib[adp->ad_offset - UFS_NDADDR] != adp->ad_newblkno)
panic("initiate_write_inodeblock_ufs1: "
"indirect pointer #%jd mismatch %d != %jd",
(intmax_t)adp->ad_offset - UFS_NDADDR,
dp->di_ib[adp->ad_offset - UFS_NDADDR],
(intmax_t)adp->ad_newblkno);
deplist |= 1 << adp->ad_offset;
if ((adp->ad_state & ATTACHED) == 0)
panic("initiate_write_inodeblock_ufs1: "
"Unknown state 0x%x", adp->ad_state);
#endif /* INVARIANTS */
adp->ad_state &= ~ATTACHED;
adp->ad_state |= UNDONE;
}
/*
* The on-disk inode cannot claim to be any larger than the last
* fragment that has been written. Otherwise, the on-disk inode
* might have fragments that were not the last block in the file
* which would corrupt the filesystem.
*/
for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
if (adp->ad_offset >= UFS_NDADDR)
break;
dp->di_db[adp->ad_offset] = adp->ad_oldblkno;
/* keep going until hitting a rollback to a frag */
if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
continue;
dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
for (i = adp->ad_offset + 1; i < UFS_NDADDR; i++) {
#ifdef INVARIANTS
if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
panic("initiate_write_inodeblock_ufs1: "
"lost dep1");
#endif /* INVARIANTS */
dp->di_db[i] = 0;
}
for (i = 0; i < UFS_NIADDR; i++) {
#ifdef INVARIANTS
if (dp->di_ib[i] != 0 &&
(deplist & ((1 << UFS_NDADDR) << i)) == 0)
panic("initiate_write_inodeblock_ufs1: "
"lost dep2");
#endif /* INVARIANTS */
dp->di_ib[i] = 0;
}
return;
}
/*
* If we have zero'ed out the last allocated block of the file,
* roll back the size to the last currently allocated block.
* We know that this last allocated block is a full-sized as
* we already checked for fragments in the loop above.
*/
if (lastadp != NULL &&
dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
for (i = lastadp->ad_offset; i >= 0; i--)
if (dp->di_db[i] != 0)
break;
dp->di_size = (i + 1) * fs->fs_bsize;
}
/*
* The only dependencies are for indirect blocks.
*
* The file size for indirect block additions is not guaranteed.
* Such a guarantee would be non-trivial to achieve. The conventional
* synchronous write implementation also does not make this guarantee.
* Fsck should catch and fix discrepancies. Arguably, the file size
* can be over-estimated without destroying integrity when the file
* moves into the indirect blocks (i.e., is large). If we want to
* postpone fsck, we are stuck with this argument.
*/
for (; adp; adp = TAILQ_NEXT(adp, ad_next))
dp->di_ib[adp->ad_offset - UFS_NDADDR] = 0;
}
/*
* Version of initiate_write_inodeblock that handles UFS2 dinodes.
* Note that any bug fixes made to this routine must be done in the
* version found above.
*
* Called from within the procedure above to deal with unsatisfied
* allocation dependencies in an inodeblock. The buffer must be
* locked, thus, no I/O completion operations can occur while we
* are manipulating its associated dependencies.
*/
static void
initiate_write_inodeblock_ufs2(inodedep, bp)
struct inodedep *inodedep;
struct buf *bp; /* The inode block */
{
struct allocdirect *adp, *lastadp;
struct ufs2_dinode *dp;
struct ufs2_dinode *sip;
struct inoref *inoref;
struct ufsmount *ump;
struct fs *fs;
ufs_lbn_t i;
#ifdef INVARIANTS
ufs_lbn_t prevlbn = 0;
#endif
int deplist;
if (inodedep->id_state & IOSTARTED)
panic("initiate_write_inodeblock_ufs2: already started");
inodedep->id_state |= IOSTARTED;
fs = inodedep->id_fs;
ump = VFSTOUFS(inodedep->id_list.wk_mp);
LOCK_OWNED(ump);
dp = (struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, inodedep->id_ino);
/*
* If we're on the unlinked list but have not yet written our
* next pointer initialize it here.
*/
if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) {
struct inodedep *inon;
inon = TAILQ_NEXT(inodedep, id_unlinked);
dp->di_freelink = inon ? inon->id_ino : 0;
ffs_update_dinode_ckhash(fs, dp);
}
/*
* If the bitmap is not yet written, then the allocated
* inode cannot be written to disk.
*/
if ((inodedep->id_state & DEPCOMPLETE) == 0) {
if (inodedep->id_savedino2 != NULL)
panic("initiate_write_inodeblock_ufs2: I/O underway");
FREE_LOCK(ump);
sip = malloc(sizeof(struct ufs2_dinode),
M_SAVEDINO, M_SOFTDEP_FLAGS);
ACQUIRE_LOCK(ump);
inodedep->id_savedino2 = sip;
*inodedep->id_savedino2 = *dp;
bzero((caddr_t)dp, sizeof(struct ufs2_dinode));
dp->di_gen = inodedep->id_savedino2->di_gen;
dp->di_freelink = inodedep->id_savedino2->di_freelink;
return;
}
/*
* If no dependencies, then there is nothing to roll back.
*/
inodedep->id_savedsize = dp->di_size;
inodedep->id_savedextsize = dp->di_extsize;
inodedep->id_savednlink = dp->di_nlink;
if (TAILQ_EMPTY(&inodedep->id_inoupdt) &&
TAILQ_EMPTY(&inodedep->id_extupdt) &&
TAILQ_EMPTY(&inodedep->id_inoreflst))
return;
/*
* Revert the link count to that of the first unwritten journal entry.
*/
inoref = TAILQ_FIRST(&inodedep->id_inoreflst);
if (inoref)
dp->di_nlink = inoref->if_nlink;
/*
* Set the ext data dependencies to busy.
*/
for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
adp = TAILQ_NEXT(adp, ad_next)) {
#ifdef INVARIANTS
if (deplist != 0 && prevlbn >= adp->ad_offset)
panic("initiate_write_inodeblock_ufs2: lbn order");
prevlbn = adp->ad_offset;
if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno)
panic("initiate_write_inodeblock_ufs2: "
"ext pointer #%jd mismatch %jd != %jd",
(intmax_t)adp->ad_offset,
(intmax_t)dp->di_extb[adp->ad_offset],
(intmax_t)adp->ad_newblkno);
deplist |= 1 << adp->ad_offset;
if ((adp->ad_state & ATTACHED) == 0)
panic("initiate_write_inodeblock_ufs2: Unknown "
"state 0x%x", adp->ad_state);
#endif /* INVARIANTS */
adp->ad_state &= ~ATTACHED;
adp->ad_state |= UNDONE;
}
/*
* The on-disk inode cannot claim to be any larger than the last
* fragment that has been written. Otherwise, the on-disk inode
* might have fragments that were not the last block in the ext
* data which would corrupt the filesystem.
*/
for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp;
lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
dp->di_extb[adp->ad_offset] = adp->ad_oldblkno;
/* keep going until hitting a rollback to a frag */
if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
continue;
dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
for (i = adp->ad_offset + 1; i < UFS_NXADDR; i++) {
#ifdef INVARIANTS
if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0)
panic("initiate_write_inodeblock_ufs2: "
"lost dep1");
#endif /* INVARIANTS */
dp->di_extb[i] = 0;
}
lastadp = NULL;
break;
}
/*
* If we have zero'ed out the last allocated block of the ext
* data, roll back the size to the last currently allocated block.
* We know that this last allocated block is a full-sized as
* we already checked for fragments in the loop above.
*/
if (lastadp != NULL &&
dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
for (i = lastadp->ad_offset; i >= 0; i--)
if (dp->di_extb[i] != 0)
break;
dp->di_extsize = (i + 1) * fs->fs_bsize;
}
/*
* Set the file data dependencies to busy.
*/
for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
adp = TAILQ_NEXT(adp, ad_next)) {
#ifdef INVARIANTS
if (deplist != 0 && prevlbn >= adp->ad_offset)
panic("softdep_write_inodeblock: lbn order");
if ((adp->ad_state & ATTACHED) == 0)
panic("inodedep %p and adp %p not attached", inodedep, adp);
prevlbn = adp->ad_offset;
- if (adp->ad_offset < UFS_NDADDR &&
+ if (!ffs_fsfail_cleanup(ump, 0) &&
+ adp->ad_offset < UFS_NDADDR &&
dp->di_db[adp->ad_offset] != adp->ad_newblkno)
panic("initiate_write_inodeblock_ufs2: "
"direct pointer #%jd mismatch %jd != %jd",
(intmax_t)adp->ad_offset,
(intmax_t)dp->di_db[adp->ad_offset],
(intmax_t)adp->ad_newblkno);
- if (adp->ad_offset >= UFS_NDADDR &&
+ if (!ffs_fsfail_cleanup(ump, 0) &&
+ adp->ad_offset >= UFS_NDADDR &&
dp->di_ib[adp->ad_offset - UFS_NDADDR] != adp->ad_newblkno)
panic("initiate_write_inodeblock_ufs2: "
"indirect pointer #%jd mismatch %jd != %jd",
(intmax_t)adp->ad_offset - UFS_NDADDR,
(intmax_t)dp->di_ib[adp->ad_offset - UFS_NDADDR],
(intmax_t)adp->ad_newblkno);
deplist |= 1 << adp->ad_offset;
if ((adp->ad_state & ATTACHED) == 0)
panic("initiate_write_inodeblock_ufs2: Unknown "
"state 0x%x", adp->ad_state);
#endif /* INVARIANTS */
adp->ad_state &= ~ATTACHED;
adp->ad_state |= UNDONE;
}
/*
* The on-disk inode cannot claim to be any larger than the last
* fragment that has been written. Otherwise, the on-disk inode
* might have fragments that were not the last block in the file
* which would corrupt the filesystem.
*/
for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp;
lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) {
if (adp->ad_offset >= UFS_NDADDR)
break;
dp->di_db[adp->ad_offset] = adp->ad_oldblkno;
/* keep going until hitting a rollback to a frag */
if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize)
continue;
dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize;
for (i = adp->ad_offset + 1; i < UFS_NDADDR; i++) {
#ifdef INVARIANTS
if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0)
panic("initiate_write_inodeblock_ufs2: "
"lost dep2");
#endif /* INVARIANTS */
dp->di_db[i] = 0;
}
for (i = 0; i < UFS_NIADDR; i++) {
#ifdef INVARIANTS
if (dp->di_ib[i] != 0 &&
(deplist & ((1 << UFS_NDADDR) << i)) == 0)
panic("initiate_write_inodeblock_ufs2: "
"lost dep3");
#endif /* INVARIANTS */
dp->di_ib[i] = 0;
}
ffs_update_dinode_ckhash(fs, dp);
return;
}
/*
* If we have zero'ed out the last allocated block of the file,
* roll back the size to the last currently allocated block.
* We know that this last allocated block is a full-sized as
* we already checked for fragments in the loop above.
*/
if (lastadp != NULL &&
dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) {
for (i = lastadp->ad_offset; i >= 0; i--)
if (dp->di_db[i] != 0)
break;
dp->di_size = (i + 1) * fs->fs_bsize;
}
/*
* The only dependencies are for indirect blocks.
*
* The file size for indirect block additions is not guaranteed.
* Such a guarantee would be non-trivial to achieve. The conventional
* synchronous write implementation also does not make this guarantee.
* Fsck should catch and fix discrepancies. Arguably, the file size
* can be over-estimated without destroying integrity when the file
* moves into the indirect blocks (i.e., is large). If we want to
* postpone fsck, we are stuck with this argument.
*/
for (; adp; adp = TAILQ_NEXT(adp, ad_next))
dp->di_ib[adp->ad_offset - UFS_NDADDR] = 0;
ffs_update_dinode_ckhash(fs, dp);
}
/*
* Cancel an indirdep as a result of truncation. Release all of the
* children allocindirs and place their journal work on the appropriate
* list.
*/
static void
cancel_indirdep(indirdep, bp, freeblks)
struct indirdep *indirdep;
struct buf *bp;
struct freeblks *freeblks;
{
struct allocindir *aip;
/*
* None of the indirect pointers will ever be visible,
* so they can simply be tossed. GOINGAWAY ensures
* that allocated pointers will be saved in the buffer
* cache until they are freed. Note that they will
* only be able to be found by their physical address
* since the inode mapping the logical address will
* be gone. The save buffer used for the safe copy
* was allocated in setup_allocindir_phase2 using
* the physical address so it could be used for this
* purpose. Hence we swap the safe copy with the real
* copy, allowing the safe copy to be freed and holding
* on to the real copy for later use in indir_trunc.
*/
if (indirdep->ir_state & GOINGAWAY)
panic("cancel_indirdep: already gone");
if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
indirdep->ir_state |= DEPCOMPLETE;
LIST_REMOVE(indirdep, ir_next);
}
indirdep->ir_state |= GOINGAWAY;
/*
* Pass in bp for blocks still have journal writes
* pending so we can cancel them on their own.
*/
while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != NULL)
cancel_allocindir(aip, bp, freeblks, 0);
while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL)
cancel_allocindir(aip, NULL, freeblks, 0);
while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL)
cancel_allocindir(aip, NULL, freeblks, 0);
while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL)
cancel_allocindir(aip, NULL, freeblks, 0);
/*
* If there are pending partial truncations we need to keep the
* old block copy around until they complete. This is because
* the current b_data is not a perfect superset of the available
* blocks.
*/
if (TAILQ_EMPTY(&indirdep->ir_trunc))
bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount);
else
bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount);
WORKLIST_REMOVE(&indirdep->ir_list);
WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list);
indirdep->ir_bp = NULL;
indirdep->ir_freeblks = freeblks;
}
/*
* Free an indirdep once it no longer has new pointers to track.
*/
static void
free_indirdep(indirdep)
struct indirdep *indirdep;
{
KASSERT(TAILQ_EMPTY(&indirdep->ir_trunc),
("free_indirdep: Indir trunc list not empty."));
KASSERT(LIST_EMPTY(&indirdep->ir_completehd),
("free_indirdep: Complete head not empty."));
KASSERT(LIST_EMPTY(&indirdep->ir_writehd),
("free_indirdep: write head not empty."));
KASSERT(LIST_EMPTY(&indirdep->ir_donehd),
("free_indirdep: done head not empty."));
KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd),
("free_indirdep: deplist head not empty."));
KASSERT((indirdep->ir_state & DEPCOMPLETE),
("free_indirdep: %p still on newblk list.", indirdep));
KASSERT(indirdep->ir_saveddata == NULL,
("free_indirdep: %p still has saved data.", indirdep));
KASSERT(indirdep->ir_savebp == NULL,
("free_indirdep: %p still has savebp buffer.", indirdep));
if (indirdep->ir_state & ONWORKLIST)
WORKLIST_REMOVE(&indirdep->ir_list);
WORKITEM_FREE(indirdep, D_INDIRDEP);
}
/*
* Called before a write to an indirdep. This routine is responsible for
* rolling back pointers to a safe state which includes only those
* allocindirs which have been completed.
*/
static void
initiate_write_indirdep(indirdep, bp)
struct indirdep *indirdep;
struct buf *bp;
{
struct ufsmount *ump;
indirdep->ir_state |= IOSTARTED;
if (indirdep->ir_state & GOINGAWAY)
panic("disk_io_initiation: indirdep gone");
/*
* If there are no remaining dependencies, this will be writing
* the real pointers.
*/
if (LIST_EMPTY(&indirdep->ir_deplisthd) &&
TAILQ_EMPTY(&indirdep->ir_trunc))
return;
/*
* Replace up-to-date version with safe version.
*/
if (indirdep->ir_saveddata == NULL) {
ump = VFSTOUFS(indirdep->ir_list.wk_mp);
LOCK_OWNED(ump);
FREE_LOCK(ump);
indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP,
M_SOFTDEP_FLAGS);
ACQUIRE_LOCK(ump);
}
indirdep->ir_state &= ~ATTACHED;
indirdep->ir_state |= UNDONE;
bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount);
bcopy(indirdep->ir_savebp->b_data, bp->b_data,
bp->b_bcount);
}
/*
* Called when an inode has been cleared in a cg bitmap. This finally
* eliminates any canceled jaddrefs
*/
void
softdep_setup_inofree(mp, bp, ino, wkhd)
struct mount *mp;
struct buf *bp;
ino_t ino;
struct workhead *wkhd;
{
struct worklist *wk, *wkn;
struct inodedep *inodedep;
struct ufsmount *ump;
uint8_t *inosused;
struct cg *cgp;
struct fs *fs;
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_setup_inofree called on non-softdep filesystem"));
ump = VFSTOUFS(mp);
ACQUIRE_LOCK(ump);
- fs = ump->um_fs;
- cgp = (struct cg *)bp->b_data;
- inosused = cg_inosused(cgp);
- if (isset(inosused, ino % fs->fs_ipg))
- panic("softdep_setup_inofree: inode %ju not freed.",
- (uintmax_t)ino);
+ if (!ffs_fsfail_cleanup(ump, 0)) {
+ fs = ump->um_fs;
+ cgp = (struct cg *)bp->b_data;
+ inosused = cg_inosused(cgp);
+ if (isset(inosused, ino % fs->fs_ipg))
+ panic("softdep_setup_inofree: inode %ju not freed.",
+ (uintmax_t)ino);
+ }
if (inodedep_lookup(mp, ino, 0, &inodedep))
panic("softdep_setup_inofree: ino %ju has existing inodedep %p",
(uintmax_t)ino, inodedep);
if (wkhd) {
LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) {
if (wk->wk_type != D_JADDREF)
continue;
WORKLIST_REMOVE(wk);
/*
* We can free immediately even if the jaddref
* isn't attached in a background write as now
* the bitmaps are reconciled.
*/
wk->wk_state |= COMPLETE | ATTACHED;
free_jaddref(WK_JADDREF(wk));
}
jwork_move(&bp->b_dep, wkhd);
}
FREE_LOCK(ump);
}
/*
* Called via ffs_blkfree() after a set of frags has been cleared from a cg
* map. Any dependencies waiting for the write to clear are added to the
* buf's list and any jnewblks that are being canceled are discarded
* immediately.
*/
void
softdep_setup_blkfree(mp, bp, blkno, frags, wkhd)
struct mount *mp;
struct buf *bp;
ufs2_daddr_t blkno;
int frags;
struct workhead *wkhd;
{
struct bmsafemap *bmsafemap;
struct jnewblk *jnewblk;
struct ufsmount *ump;
struct worklist *wk;
struct fs *fs;
#ifdef INVARIANTS
uint8_t *blksfree;
struct cg *cgp;
ufs2_daddr_t jstart;
ufs2_daddr_t jend;
ufs2_daddr_t end;
long bno;
int i;
#endif
CTR3(KTR_SUJ,
"softdep_setup_blkfree: blkno %jd frags %d wk head %p",
blkno, frags, wkhd);
ump = VFSTOUFS(mp);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_setup_blkfree called on non-softdep filesystem"));
ACQUIRE_LOCK(ump);
/* Lookup the bmsafemap so we track when it is dirty. */
fs = ump->um_fs;
bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL);
/*
* Detach any jnewblks which have been canceled. They must linger
* until the bitmap is cleared again by ffs_blkfree() to prevent
* an unjournaled allocation from hitting the disk.
*/
if (wkhd) {
while ((wk = LIST_FIRST(wkhd)) != NULL) {
CTR2(KTR_SUJ,
"softdep_setup_blkfree: blkno %jd wk type %d",
blkno, wk->wk_type);
WORKLIST_REMOVE(wk);
if (wk->wk_type != D_JNEWBLK) {
WORKLIST_INSERT(&bmsafemap->sm_freehd, wk);
continue;
}
jnewblk = WK_JNEWBLK(wk);
KASSERT(jnewblk->jn_state & GOINGAWAY,
("softdep_setup_blkfree: jnewblk not canceled."));
#ifdef INVARIANTS
/*
* Assert that this block is free in the bitmap
* before we discard the jnewblk.
*/
cgp = (struct cg *)bp->b_data;
blksfree = cg_blksfree(cgp);
bno = dtogd(fs, jnewblk->jn_blkno);
for (i = jnewblk->jn_oldfrags;
i < jnewblk->jn_frags; i++) {
if (isset(blksfree, bno + i))
continue;
panic("softdep_setup_blkfree: not free");
}
#endif
/*
* Even if it's not attached we can free immediately
* as the new bitmap is correct.
*/
wk->wk_state |= COMPLETE | ATTACHED;
free_jnewblk(jnewblk);
}
}
#ifdef INVARIANTS
/*
* Assert that we are not freeing a block which has an outstanding
* allocation dependency.
*/
fs = VFSTOUFS(mp)->um_fs;
bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL);
end = blkno + frags;
LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) {
/*
* Don't match against blocks that will be freed when the
* background write is done.
*/
if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) ==
(COMPLETE | DEPCOMPLETE))
continue;
jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags;
jend = jnewblk->jn_blkno + jnewblk->jn_frags;
if ((blkno >= jstart && blkno < jend) ||
(end > jstart && end <= jend)) {
printf("state 0x%X %jd - %d %d dep %p\n",
jnewblk->jn_state, jnewblk->jn_blkno,
jnewblk->jn_oldfrags, jnewblk->jn_frags,
jnewblk->jn_dep);
panic("softdep_setup_blkfree: "
"%jd-%jd(%d) overlaps with %jd-%jd",
blkno, end, frags, jstart, jend);
}
}
#endif
FREE_LOCK(ump);
}
/*
* Revert a block allocation when the journal record that describes it
* is not yet written.
*/
static int
jnewblk_rollback(jnewblk, fs, cgp, blksfree)
struct jnewblk *jnewblk;
struct fs *fs;
struct cg *cgp;
uint8_t *blksfree;
{
ufs1_daddr_t fragno;
long cgbno, bbase;
int frags, blk;
int i;
frags = 0;
cgbno = dtogd(fs, jnewblk->jn_blkno);
/*
* We have to test which frags need to be rolled back. We may
* be operating on a stale copy when doing background writes.
*/
for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++)
if (isclr(blksfree, cgbno + i))
frags++;
if (frags == 0)
return (0);
/*
* This is mostly ffs_blkfree() sans some validation and
* superblock updates.
*/
if (frags == fs->fs_frag) {
fragno = fragstoblks(fs, cgbno);
ffs_setblock(fs, blksfree, fragno);
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
} else {
cgbno += jnewblk->jn_oldfrags;
bbase = cgbno - fragnum(fs, cgbno);
/* Decrement the old frags. */
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
/* Deallocate the fragment */
for (i = 0; i < frags; i++)
setbit(blksfree, cgbno + i);
cgp->cg_cs.cs_nffree += frags;
/* Add back in counts associated with the new frags */
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
/* If a complete block has been reassembled, account for it. */
fragno = fragstoblks(fs, bbase);
if (ffs_isblock(fs, blksfree, fragno)) {
cgp->cg_cs.cs_nffree -= fs->fs_frag;
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
}
}
stat_jnewblk++;
jnewblk->jn_state &= ~ATTACHED;
jnewblk->jn_state |= UNDONE;
return (frags);
}
static void
initiate_write_bmsafemap(bmsafemap, bp)
struct bmsafemap *bmsafemap;
struct buf *bp; /* The cg block. */
{
struct jaddref *jaddref;
struct jnewblk *jnewblk;
uint8_t *inosused;
uint8_t *blksfree;
struct cg *cgp;
struct fs *fs;
ino_t ino;
/*
* If this is a background write, we did this at the time that
* the copy was made, so do not need to do it again.
*/
if (bmsafemap->sm_state & IOSTARTED)
return;
bmsafemap->sm_state |= IOSTARTED;
/*
* Clear any inode allocations which are pending journal writes.
*/
if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) {
cgp = (struct cg *)bp->b_data;
fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
inosused = cg_inosused(cgp);
LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) {
ino = jaddref->ja_ino % fs->fs_ipg;
if (isset(inosused, ino)) {
if ((jaddref->ja_mode & IFMT) == IFDIR)
cgp->cg_cs.cs_ndir--;
cgp->cg_cs.cs_nifree++;
clrbit(inosused, ino);
jaddref->ja_state &= ~ATTACHED;
jaddref->ja_state |= UNDONE;
stat_jaddref++;
} else
panic("initiate_write_bmsafemap: inode %ju "
"marked free", (uintmax_t)jaddref->ja_ino);
}
}
/*
* Clear any block allocations which are pending journal writes.
*/
if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) {
cgp = (struct cg *)bp->b_data;
fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
blksfree = cg_blksfree(cgp);
LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) {
if (jnewblk_rollback(jnewblk, fs, cgp, blksfree))
continue;
panic("initiate_write_bmsafemap: block %jd "
"marked free", jnewblk->jn_blkno);
}
}
/*
* Move allocation lists to the written lists so they can be
* cleared once the block write is complete.
*/
LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr,
inodedep, id_deps);
LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr,
newblk, nb_deps);
LIST_SWAP(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, worklist,
wk_list);
}
+void
+softdep_handle_error(struct buf *bp)
+{
+ struct ufsmount *ump;
+
+ ump = softdep_bp_to_mp(bp);
+ if (ump == NULL)
+ return;
+
+ if (ffs_fsfail_cleanup(ump, bp->b_error)) {
+ /*
+ * No future writes will succeed, so the on-disk image is safe.
+ * Pretend that this write succeeded so that the softdep state
+ * will be cleaned up naturally.
+ */
+ bp->b_ioflags &= ~BIO_ERROR;
+ bp->b_error = 0;
+ }
+}
+
/*
* This routine is called during the completion interrupt
* service routine for a disk write (from the procedure called
* by the device driver to inform the filesystem caches of
* a request completion). It should be called early in this
* procedure, before the block is made available to other
* processes or other routines are called.
*
*/
static void
softdep_disk_write_complete(bp)
struct buf *bp; /* describes the completed disk write */
{
struct worklist *wk;
struct worklist *owk;
struct ufsmount *ump;
struct workhead reattach;
struct freeblks *freeblks;
struct buf *sbp;
ump = softdep_bp_to_mp(bp);
KASSERT(LIST_EMPTY(&bp->b_dep) || ump != NULL,
("softdep_disk_write_complete: softdep_bp_to_mp returned NULL "
"with outstanding dependencies for buffer %p", bp));
if (ump == NULL)
return;
+ if ((bp->b_ioflags & BIO_ERROR) != 0)
+ softdep_handle_error(bp);
/*
* If an error occurred while doing the write, then the data
* has not hit the disk and the dependencies cannot be processed.
* But we do have to go through and roll forward any dependencies
* that were rolled back before the disk write.
*/
sbp = NULL;
ACQUIRE_LOCK(ump);
if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0) {
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
switch (wk->wk_type) {
case D_PAGEDEP:
handle_written_filepage(WK_PAGEDEP(wk), bp, 0);
continue;
case D_INODEDEP:
handle_written_inodeblock(WK_INODEDEP(wk),
bp, 0);
continue;
case D_BMSAFEMAP:
handle_written_bmsafemap(WK_BMSAFEMAP(wk),
bp, 0);
continue;
case D_INDIRDEP:
handle_written_indirdep(WK_INDIRDEP(wk),
bp, &sbp, 0);
continue;
default:
/* nothing to roll forward */
continue;
}
}
FREE_LOCK(ump);
if (sbp)
brelse(sbp);
return;
}
LIST_INIT(&reattach);
/*
* Ump SU lock must not be released anywhere in this code segment.
*/
owk = NULL;
while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) {
WORKLIST_REMOVE(wk);
atomic_add_long(&dep_write[wk->wk_type], 1);
if (wk == owk)
panic("duplicate worklist: %p\n", wk);
owk = wk;
switch (wk->wk_type) {
case D_PAGEDEP:
if (handle_written_filepage(WK_PAGEDEP(wk), bp,
WRITESUCCEEDED))
WORKLIST_INSERT(&reattach, wk);
continue;
case D_INODEDEP:
if (handle_written_inodeblock(WK_INODEDEP(wk), bp,
WRITESUCCEEDED))
WORKLIST_INSERT(&reattach, wk);
continue;
case D_BMSAFEMAP:
if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp,
WRITESUCCEEDED))
WORKLIST_INSERT(&reattach, wk);
continue;
case D_MKDIR:
handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY);
continue;
case D_ALLOCDIRECT:
wk->wk_state |= COMPLETE;
handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL);
continue;
case D_ALLOCINDIR:
wk->wk_state |= COMPLETE;
handle_allocindir_partdone(WK_ALLOCINDIR(wk));
continue;
case D_INDIRDEP:
if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp,
WRITESUCCEEDED))
WORKLIST_INSERT(&reattach, wk);
continue;
case D_FREEBLKS:
wk->wk_state |= COMPLETE;
freeblks = WK_FREEBLKS(wk);
if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE &&
LIST_EMPTY(&freeblks->fb_jblkdephd))
add_to_worklist(wk, WK_NODELAY);
continue;
case D_FREEWORK:
handle_written_freework(WK_FREEWORK(wk));
break;
case D_JSEGDEP:
free_jsegdep(WK_JSEGDEP(wk));
continue;
case D_JSEG:
handle_written_jseg(WK_JSEG(wk), bp);
continue;
case D_SBDEP:
if (handle_written_sbdep(WK_SBDEP(wk), bp))
WORKLIST_INSERT(&reattach, wk);
continue;
case D_FREEDEP:
free_freedep(WK_FREEDEP(wk));
continue;
default:
panic("handle_disk_write_complete: Unknown type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
/*
* Reattach any requests that must be redone.
*/
while ((wk = LIST_FIRST(&reattach)) != NULL) {
WORKLIST_REMOVE(wk);
WORKLIST_INSERT(&bp->b_dep, wk);
}
FREE_LOCK(ump);
if (sbp)
brelse(sbp);
}
/*
* Called from within softdep_disk_write_complete above.
*/
static void
handle_allocdirect_partdone(adp, wkhd)
struct allocdirect *adp; /* the completed allocdirect */
struct workhead *wkhd; /* Work to do when inode is writtne. */
{
struct allocdirectlst *listhead;
struct allocdirect *listadp;
struct inodedep *inodedep;
long bsize;
LOCK_OWNED(VFSTOUFS(adp->ad_block.nb_list.wk_mp));
if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
/*
* The on-disk inode cannot claim to be any larger than the last
* fragment that has been written. Otherwise, the on-disk inode
* might have fragments that were not the last block in the file
* which would corrupt the filesystem. Thus, we cannot free any
* allocdirects after one whose ad_oldblkno claims a fragment as
* these blocks must be rolled back to zero before writing the inode.
* We check the currently active set of allocdirects in id_inoupdt
* or id_extupdt as appropriate.
*/
inodedep = adp->ad_inodedep;
bsize = inodedep->id_fs->fs_bsize;
if (adp->ad_state & EXTDATA)
listhead = &inodedep->id_extupdt;
else
listhead = &inodedep->id_inoupdt;
TAILQ_FOREACH(listadp, listhead, ad_next) {
/* found our block */
if (listadp == adp)
break;
/* continue if ad_oldlbn is not a fragment */
if (listadp->ad_oldsize == 0 ||
listadp->ad_oldsize == bsize)
continue;
/* hit a fragment */
return;
}
/*
* If we have reached the end of the current list without
* finding the just finished dependency, then it must be
* on the future dependency list. Future dependencies cannot
* be freed until they are moved to the current list.
*/
if (listadp == NULL) {
#ifdef INVARIANTS
if (adp->ad_state & EXTDATA)
listhead = &inodedep->id_newextupdt;
else
listhead = &inodedep->id_newinoupdt;
TAILQ_FOREACH(listadp, listhead, ad_next)
/* found our block */
if (listadp == adp)
break;
if (listadp == NULL)
panic("handle_allocdirect_partdone: lost dep");
#endif /* INVARIANTS */
return;
}
/*
* If we have found the just finished dependency, then queue
* it along with anything that follows it that is complete.
* Since the pointer has not yet been written in the inode
* as the dependency prevents it, place the allocdirect on the
* bufwait list where it will be freed once the pointer is
* valid.
*/
if (wkhd == NULL)
wkhd = &inodedep->id_bufwait;
for (; adp; adp = listadp) {
listadp = TAILQ_NEXT(adp, ad_next);
if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
TAILQ_REMOVE(listhead, adp, ad_next);
WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list);
}
}
/*
* Called from within softdep_disk_write_complete above. This routine
* completes successfully written allocindirs.
*/
static void
handle_allocindir_partdone(aip)
struct allocindir *aip; /* the completed allocindir */
{
struct indirdep *indirdep;
if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
indirdep = aip->ai_indirdep;
LIST_REMOVE(aip, ai_next);
/*
* Don't set a pointer while the buffer is undergoing IO or while
* we have active truncations.
*/
if (indirdep->ir_state & UNDONE || !TAILQ_EMPTY(&indirdep->ir_trunc)) {
LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next);
return;
}
if (indirdep->ir_state & UFS1FMT)
((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
aip->ai_newblkno;
else
((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] =
aip->ai_newblkno;
/*
* Await the pointer write before freeing the allocindir.
*/
LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next);
}
/*
* Release segments held on a jwork list.
*/
static void
handle_jwork(wkhd)
struct workhead *wkhd;
{
struct worklist *wk;
while ((wk = LIST_FIRST(wkhd)) != NULL) {
WORKLIST_REMOVE(wk);
switch (wk->wk_type) {
case D_JSEGDEP:
free_jsegdep(WK_JSEGDEP(wk));
continue;
case D_FREEDEP:
free_freedep(WK_FREEDEP(wk));
continue;
case D_FREEFRAG:
rele_jseg(WK_JSEG(WK_FREEFRAG(wk)->ff_jdep));
WORKITEM_FREE(wk, D_FREEFRAG);
continue;
case D_FREEWORK:
handle_written_freework(WK_FREEWORK(wk));
continue;
default:
panic("handle_jwork: Unknown type %s\n",
TYPENAME(wk->wk_type));
}
}
}
/*
* Handle the bufwait list on an inode when it is safe to release items
* held there. This normally happens after an inode block is written but
* may be delayed and handled later if there are pending journal items that
* are not yet safe to be released.
*/
static struct freefile *
handle_bufwait(inodedep, refhd)
struct inodedep *inodedep;
struct workhead *refhd;
{
struct jaddref *jaddref;
struct freefile *freefile;
struct worklist *wk;
freefile = NULL;
while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) {
WORKLIST_REMOVE(wk);
switch (wk->wk_type) {
case D_FREEFILE:
/*
* We defer adding freefile to the worklist
* until all other additions have been made to
* ensure that it will be done after all the
* old blocks have been freed.
*/
if (freefile != NULL)
panic("handle_bufwait: freefile");
freefile = WK_FREEFILE(wk);
continue;
case D_MKDIR:
handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT);
continue;
case D_DIRADD:
diradd_inode_written(WK_DIRADD(wk), inodedep);
continue;
case D_FREEFRAG:
wk->wk_state |= COMPLETE;
if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE)
add_to_worklist(wk, 0);
continue;
case D_DIRREM:
wk->wk_state |= COMPLETE;
add_to_worklist(wk, 0);
continue;
case D_ALLOCDIRECT:
case D_ALLOCINDIR:
free_newblk(WK_NEWBLK(wk));
continue;
case D_JNEWBLK:
wk->wk_state |= COMPLETE;
free_jnewblk(WK_JNEWBLK(wk));
continue;
/*
* Save freed journal segments and add references on
* the supplied list which will delay their release
* until the cg bitmap is cleared on disk.
*/
case D_JSEGDEP:
if (refhd == NULL)
free_jsegdep(WK_JSEGDEP(wk));
else
WORKLIST_INSERT(refhd, wk);
continue;
case D_JADDREF:
jaddref = WK_JADDREF(wk);
TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref,
if_deps);
/*
* Transfer any jaddrefs to the list to be freed with
* the bitmap if we're handling a removed file.
*/
if (refhd == NULL) {
wk->wk_state |= COMPLETE;
free_jaddref(jaddref);
} else
WORKLIST_INSERT(refhd, wk);
continue;
default:
panic("handle_bufwait: Unknown type %p(%s)",
wk, TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
return (freefile);
}
/*
* Called from within softdep_disk_write_complete above to restore
* in-memory inode block contents to their most up-to-date state. Note
* that this routine is always called from interrupt level with further
* interrupts from this device blocked.
*
* If the write did not succeed, we will do all the roll-forward
* operations, but we will not take the actions that will allow its
* dependencies to be processed.
*/
static int
handle_written_inodeblock(inodedep, bp, flags)
struct inodedep *inodedep;
struct buf *bp; /* buffer containing the inode block */
int flags;
{
struct freefile *freefile;
struct allocdirect *adp, *nextadp;
struct ufs1_dinode *dp1 = NULL;
struct ufs2_dinode *dp2 = NULL;
struct workhead wkhd;
int hadchanges, fstype;
ino_t freelink;
LIST_INIT(&wkhd);
hadchanges = 0;
freefile = NULL;
if ((inodedep->id_state & IOSTARTED) == 0)
panic("handle_written_inodeblock: not started");
inodedep->id_state &= ~IOSTARTED;
if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) {
fstype = UFS1;
dp1 = (struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
freelink = dp1->di_freelink;
} else {
fstype = UFS2;
dp2 = (struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(inodedep->id_fs, inodedep->id_ino);
freelink = dp2->di_freelink;
}
/*
* Leave this inodeblock dirty until it's in the list.
*/
if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED &&
(flags & WRITESUCCEEDED)) {
struct inodedep *inon;
inon = TAILQ_NEXT(inodedep, id_unlinked);
if ((inon == NULL && freelink == 0) ||
(inon && inon->id_ino == freelink)) {
if (inon)
inon->id_state |= UNLINKPREV;
inodedep->id_state |= UNLINKNEXT;
}
hadchanges = 1;
}
/*
* If we had to rollback the inode allocation because of
* bitmaps being incomplete, then simply restore it.
* Keep the block dirty so that it will not be reclaimed until
* all associated dependencies have been cleared and the
* corresponding updates written to disk.
*/
if (inodedep->id_savedino1 != NULL) {
hadchanges = 1;
if (fstype == UFS1)
*dp1 = *inodedep->id_savedino1;
else
*dp2 = *inodedep->id_savedino2;
free(inodedep->id_savedino1, M_SAVEDINO);
inodedep->id_savedino1 = NULL;
if ((bp->b_flags & B_DELWRI) == 0)
stat_inode_bitmap++;
bdirty(bp);
/*
* If the inode is clear here and GOINGAWAY it will never
* be written. Process the bufwait and clear any pending
* work which may include the freefile.
*/
if (inodedep->id_state & GOINGAWAY)
goto bufwait;
return (1);
}
if (flags & WRITESUCCEEDED)
inodedep->id_state |= COMPLETE;
/*
* Roll forward anything that had to be rolled back before
* the inode could be updated.
*/
for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) {
nextadp = TAILQ_NEXT(adp, ad_next);
if (adp->ad_state & ATTACHED)
panic("handle_written_inodeblock: new entry");
if (fstype == UFS1) {
if (adp->ad_offset < UFS_NDADDR) {
if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno)
panic("%s %s #%jd mismatch %d != %jd",
"handle_written_inodeblock:",
"direct pointer",
(intmax_t)adp->ad_offset,
dp1->di_db[adp->ad_offset],
(intmax_t)adp->ad_oldblkno);
dp1->di_db[adp->ad_offset] = adp->ad_newblkno;
} else {
if (dp1->di_ib[adp->ad_offset - UFS_NDADDR] !=
0)
panic("%s: %s #%jd allocated as %d",
"handle_written_inodeblock",
"indirect pointer",
(intmax_t)adp->ad_offset -
UFS_NDADDR,
dp1->di_ib[adp->ad_offset -
UFS_NDADDR]);
dp1->di_ib[adp->ad_offset - UFS_NDADDR] =
adp->ad_newblkno;
}
} else {
if (adp->ad_offset < UFS_NDADDR) {
if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno)
panic("%s: %s #%jd %s %jd != %jd",
"handle_written_inodeblock",
"direct pointer",
(intmax_t)adp->ad_offset, "mismatch",
(intmax_t)dp2->di_db[adp->ad_offset],
(intmax_t)adp->ad_oldblkno);
dp2->di_db[adp->ad_offset] = adp->ad_newblkno;
} else {
if (dp2->di_ib[adp->ad_offset - UFS_NDADDR] !=
0)
panic("%s: %s #%jd allocated as %jd",
"handle_written_inodeblock",
"indirect pointer",
(intmax_t)adp->ad_offset -
UFS_NDADDR,
(intmax_t)
dp2->di_ib[adp->ad_offset -
UFS_NDADDR]);
dp2->di_ib[adp->ad_offset - UFS_NDADDR] =
adp->ad_newblkno;
}
}
adp->ad_state &= ~UNDONE;
adp->ad_state |= ATTACHED;
hadchanges = 1;
}
for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) {
nextadp = TAILQ_NEXT(adp, ad_next);
if (adp->ad_state & ATTACHED)
panic("handle_written_inodeblock: new entry");
if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno)
panic("%s: direct pointers #%jd %s %jd != %jd",
"handle_written_inodeblock",
(intmax_t)adp->ad_offset, "mismatch",
(intmax_t)dp2->di_extb[adp->ad_offset],
(intmax_t)adp->ad_oldblkno);
dp2->di_extb[adp->ad_offset] = adp->ad_newblkno;
adp->ad_state &= ~UNDONE;
adp->ad_state |= ATTACHED;
hadchanges = 1;
}
if (hadchanges && (bp->b_flags & B_DELWRI) == 0)
stat_direct_blk_ptrs++;
/*
* Reset the file size to its most up-to-date value.
*/
if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1)
panic("handle_written_inodeblock: bad size");
if (inodedep->id_savednlink > UFS_LINK_MAX)
panic("handle_written_inodeblock: Invalid link count "
"%jd for inodedep %p", (uintmax_t)inodedep->id_savednlink,
inodedep);
if (fstype == UFS1) {
if (dp1->di_nlink != inodedep->id_savednlink) {
dp1->di_nlink = inodedep->id_savednlink;
hadchanges = 1;
}
if (dp1->di_size != inodedep->id_savedsize) {
dp1->di_size = inodedep->id_savedsize;
hadchanges = 1;
}
} else {
if (dp2->di_nlink != inodedep->id_savednlink) {
dp2->di_nlink = inodedep->id_savednlink;
hadchanges = 1;
}
if (dp2->di_size != inodedep->id_savedsize) {
dp2->di_size = inodedep->id_savedsize;
hadchanges = 1;
}
if (dp2->di_extsize != inodedep->id_savedextsize) {
dp2->di_extsize = inodedep->id_savedextsize;
hadchanges = 1;
}
}
inodedep->id_savedsize = -1;
inodedep->id_savedextsize = -1;
inodedep->id_savednlink = -1;
/*
* If there were any rollbacks in the inode block, then it must be
* marked dirty so that its will eventually get written back in
* its correct form.
*/
if (hadchanges) {
if (fstype == UFS2)
ffs_update_dinode_ckhash(inodedep->id_fs, dp2);
bdirty(bp);
}
bufwait:
/*
* If the write did not succeed, we have done all the roll-forward
* operations, but we cannot take the actions that will allow its
* dependencies to be processed.
*/
if ((flags & WRITESUCCEEDED) == 0)
return (hadchanges);
/*
* Process any allocdirects that completed during the update.
*/
if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL)
handle_allocdirect_partdone(adp, &wkhd);
if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL)
handle_allocdirect_partdone(adp, &wkhd);
/*
* Process deallocations that were held pending until the
* inode had been written to disk. Freeing of the inode
* is delayed until after all blocks have been freed to
* avoid creation of new <vfsid, inum, lbn> triples
* before the old ones have been deleted. Completely
* unlinked inodes are not processed until the unlinked
* inode list is written or the last reference is removed.
*/
if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) {
freefile = handle_bufwait(inodedep, NULL);
if (freefile && !LIST_EMPTY(&wkhd)) {
WORKLIST_INSERT(&wkhd, &freefile->fx_list);
freefile = NULL;
}
}
/*
* Move rolled forward dependency completions to the bufwait list
* now that those that were already written have been processed.
*/
if (!LIST_EMPTY(&wkhd) && hadchanges == 0)
panic("handle_written_inodeblock: bufwait but no changes");
jwork_move(&inodedep->id_bufwait, &wkhd);
if (freefile != NULL) {
/*
* If the inode is goingaway it was never written. Fake up
* the state here so free_inodedep() can succeed.
*/
if (inodedep->id_state & GOINGAWAY)
inodedep->id_state |= COMPLETE | DEPCOMPLETE;
if (free_inodedep(inodedep) == 0)
panic("handle_written_inodeblock: live inodedep %p",
inodedep);
add_to_worklist(&freefile->fx_list, 0);
return (0);
}
/*
* If no outstanding dependencies, free it.
*/
if (free_inodedep(inodedep) ||
(TAILQ_FIRST(&inodedep->id_inoreflst) == 0 &&
TAILQ_FIRST(&inodedep->id_inoupdt) == 0 &&
TAILQ_FIRST(&inodedep->id_extupdt) == 0 &&
LIST_FIRST(&inodedep->id_bufwait) == 0))
return (0);
return (hadchanges);
}
/*
* Perform needed roll-forwards and kick off any dependencies that
* can now be processed.
*
* If the write did not succeed, we will do all the roll-forward
* operations, but we will not take the actions that will allow its
* dependencies to be processed.
*/
static int
handle_written_indirdep(indirdep, bp, bpp, flags)
struct indirdep *indirdep;
struct buf *bp;
struct buf **bpp;
int flags;
{
struct allocindir *aip;
struct buf *sbp;
int chgs;
if (indirdep->ir_state & GOINGAWAY)
panic("handle_written_indirdep: indirdep gone");
if ((indirdep->ir_state & IOSTARTED) == 0)
panic("handle_written_indirdep: IO not started");
chgs = 0;
/*
* If there were rollbacks revert them here.
*/
if (indirdep->ir_saveddata) {
bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount);
if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
free(indirdep->ir_saveddata, M_INDIRDEP);
indirdep->ir_saveddata = NULL;
}
chgs = 1;
}
indirdep->ir_state &= ~(UNDONE | IOSTARTED);
indirdep->ir_state |= ATTACHED;
/*
* If the write did not succeed, we have done all the roll-forward
* operations, but we cannot take the actions that will allow its
* dependencies to be processed.
*/
if ((flags & WRITESUCCEEDED) == 0) {
stat_indir_blk_ptrs++;
bdirty(bp);
return (1);
}
/*
* Move allocindirs with written pointers to the completehd if
* the indirdep's pointer is not yet written. Otherwise
* free them here.
*/
while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL) {
LIST_REMOVE(aip, ai_next);
if ((indirdep->ir_state & DEPCOMPLETE) == 0) {
LIST_INSERT_HEAD(&indirdep->ir_completehd, aip,
ai_next);
newblk_freefrag(&aip->ai_block);
continue;
}
free_newblk(&aip->ai_block);
}
/*
* Move allocindirs that have finished dependency processing from
* the done list to the write list after updating the pointers.
*/
if (TAILQ_EMPTY(&indirdep->ir_trunc)) {
while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL) {
handle_allocindir_partdone(aip);
if (aip == LIST_FIRST(&indirdep->ir_donehd))
panic("disk_write_complete: not gone");
chgs = 1;
}
}
/*
* Preserve the indirdep if there were any changes or if it is not
* yet valid on disk.
*/
if (chgs) {
stat_indir_blk_ptrs++;
bdirty(bp);
return (1);
}
/*
* If there were no changes we can discard the savedbp and detach
* ourselves from the buf. We are only carrying completed pointers
* in this case.
*/
sbp = indirdep->ir_savebp;
sbp->b_flags |= B_INVAL | B_NOCACHE;
indirdep->ir_savebp = NULL;
indirdep->ir_bp = NULL;
if (*bpp != NULL)
panic("handle_written_indirdep: bp already exists.");
*bpp = sbp;
/*
* The indirdep may not be freed until its parent points at it.
*/
if (indirdep->ir_state & DEPCOMPLETE)
free_indirdep(indirdep);
return (0);
}
/*
* Process a diradd entry after its dependent inode has been written.
*/
static void
diradd_inode_written(dap, inodedep)
struct diradd *dap;
struct inodedep *inodedep;
{
LOCK_OWNED(VFSTOUFS(dap->da_list.wk_mp));
dap->da_state |= COMPLETE;
complete_diradd(dap);
WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list);
}
/*
* Returns true if the bmsafemap will have rollbacks when written. Must only
* be called with the per-filesystem lock and the buf lock on the cg held.
*/
static int
bmsafemap_backgroundwrite(bmsafemap, bp)
struct bmsafemap *bmsafemap;
struct buf *bp;
{
int dirty;
LOCK_OWNED(VFSTOUFS(bmsafemap->sm_list.wk_mp));
dirty = !LIST_EMPTY(&bmsafemap->sm_jaddrefhd) |
!LIST_EMPTY(&bmsafemap->sm_jnewblkhd);
/*
* If we're initiating a background write we need to process the
* rollbacks as they exist now, not as they exist when IO starts.
* No other consumers will look at the contents of the shadowed
* buf so this is safe to do here.
*/
if (bp->b_xflags & BX_BKGRDMARKER)
initiate_write_bmsafemap(bmsafemap, bp);
return (dirty);
}
/*
* Re-apply an allocation when a cg write is complete.
*/
static int
jnewblk_rollforward(jnewblk, fs, cgp, blksfree)
struct jnewblk *jnewblk;
struct fs *fs;
struct cg *cgp;
uint8_t *blksfree;
{
ufs1_daddr_t fragno;
ufs2_daddr_t blkno;
long cgbno, bbase;
int frags, blk;
int i;
frags = 0;
cgbno = dtogd(fs, jnewblk->jn_blkno);
for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) {
if (isclr(blksfree, cgbno + i))
panic("jnewblk_rollforward: re-allocated fragment");
frags++;
}
if (frags == fs->fs_frag) {
blkno = fragstoblks(fs, cgbno);
ffs_clrblock(fs, blksfree, (long)blkno);
ffs_clusteracct(fs, cgp, blkno, -1);
cgp->cg_cs.cs_nbfree--;
} else {
bbase = cgbno - fragnum(fs, cgbno);
cgbno += jnewblk->jn_oldfrags;
/* If a complete block had been reassembled, account for it. */
fragno = fragstoblks(fs, bbase);
if (ffs_isblock(fs, blksfree, fragno)) {
cgp->cg_cs.cs_nffree += fs->fs_frag;
ffs_clusteracct(fs, cgp, fragno, -1);
cgp->cg_cs.cs_nbfree--;
}
/* Decrement the old frags. */
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
/* Allocate the fragment */
for (i = 0; i < frags; i++)
clrbit(blksfree, cgbno + i);
cgp->cg_cs.cs_nffree -= frags;
/* Add back in counts associated with the new frags */
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
}
return (frags);
}
/*
* Complete a write to a bmsafemap structure. Roll forward any bitmap
* changes if it's not a background write. Set all written dependencies
* to DEPCOMPLETE and free the structure if possible.
*
* If the write did not succeed, we will do all the roll-forward
* operations, but we will not take the actions that will allow its
* dependencies to be processed.
*/
static int
handle_written_bmsafemap(bmsafemap, bp, flags)
struct bmsafemap *bmsafemap;
struct buf *bp;
int flags;
{
struct newblk *newblk;
struct inodedep *inodedep;
struct jaddref *jaddref, *jatmp;
struct jnewblk *jnewblk, *jntmp;
struct ufsmount *ump;
uint8_t *inosused;
uint8_t *blksfree;
struct cg *cgp;
struct fs *fs;
ino_t ino;
int foreground;
int chgs;
if ((bmsafemap->sm_state & IOSTARTED) == 0)
panic("handle_written_bmsafemap: Not started\n");
ump = VFSTOUFS(bmsafemap->sm_list.wk_mp);
chgs = 0;
bmsafemap->sm_state &= ~IOSTARTED;
foreground = (bp->b_xflags & BX_BKGRDMARKER) == 0;
/*
* If write was successful, release journal work that was waiting
* on the write. Otherwise move the work back.
*/
if (flags & WRITESUCCEEDED)
handle_jwork(&bmsafemap->sm_freewr);
else
LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr,
worklist, wk_list);
/*
* Restore unwritten inode allocation pending jaddref writes.
*/
if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) {
cgp = (struct cg *)bp->b_data;
fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
inosused = cg_inosused(cgp);
LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd,
ja_bmdeps, jatmp) {
if ((jaddref->ja_state & UNDONE) == 0)
continue;
ino = jaddref->ja_ino % fs->fs_ipg;
if (isset(inosused, ino))
panic("handle_written_bmsafemap: "
"re-allocated inode");
/* Do the roll-forward only if it's a real copy. */
if (foreground) {
if ((jaddref->ja_mode & IFMT) == IFDIR)
cgp->cg_cs.cs_ndir++;
cgp->cg_cs.cs_nifree--;
setbit(inosused, ino);
chgs = 1;
}
jaddref->ja_state &= ~UNDONE;
jaddref->ja_state |= ATTACHED;
free_jaddref(jaddref);
}
}
/*
* Restore any block allocations which are pending journal writes.
*/
if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) {
cgp = (struct cg *)bp->b_data;
fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs;
blksfree = cg_blksfree(cgp);
LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps,
jntmp) {
if ((jnewblk->jn_state & UNDONE) == 0)
continue;
/* Do the roll-forward only if it's a real copy. */
if (foreground &&
jnewblk_rollforward(jnewblk, fs, cgp, blksfree))
chgs = 1;
jnewblk->jn_state &= ~(UNDONE | NEWBLOCK);
jnewblk->jn_state |= ATTACHED;
free_jnewblk(jnewblk);
}
}
/*
* If the write did not succeed, we have done all the roll-forward
* operations, but we cannot take the actions that will allow its
* dependencies to be processed.
*/
if ((flags & WRITESUCCEEDED) == 0) {
LIST_CONCAT(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr,
newblk, nb_deps);
LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr,
worklist, wk_list);
if (foreground)
bdirty(bp);
return (1);
}
while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) {
newblk->nb_state |= DEPCOMPLETE;
newblk->nb_state &= ~ONDEPLIST;
newblk->nb_bmsafemap = NULL;
LIST_REMOVE(newblk, nb_deps);
if (newblk->nb_list.wk_type == D_ALLOCDIRECT)
handle_allocdirect_partdone(
WK_ALLOCDIRECT(&newblk->nb_list), NULL);
else if (newblk->nb_list.wk_type == D_ALLOCINDIR)
handle_allocindir_partdone(
WK_ALLOCINDIR(&newblk->nb_list));
else if (newblk->nb_list.wk_type != D_NEWBLK)
panic("handle_written_bmsafemap: Unexpected type: %s",
TYPENAME(newblk->nb_list.wk_type));
}
while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) {
inodedep->id_state |= DEPCOMPLETE;
inodedep->id_state &= ~ONDEPLIST;
LIST_REMOVE(inodedep, id_deps);
inodedep->id_bmsafemap = NULL;
}
LIST_REMOVE(bmsafemap, sm_next);
if (chgs == 0 && LIST_EMPTY(&bmsafemap->sm_jaddrefhd) &&
LIST_EMPTY(&bmsafemap->sm_jnewblkhd) &&
LIST_EMPTY(&bmsafemap->sm_newblkhd) &&
LIST_EMPTY(&bmsafemap->sm_inodedephd) &&
LIST_EMPTY(&bmsafemap->sm_freehd)) {
LIST_REMOVE(bmsafemap, sm_hash);
WORKITEM_FREE(bmsafemap, D_BMSAFEMAP);
return (0);
}
LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next);
if (foreground)
bdirty(bp);
return (1);
}
/*
* Try to free a mkdir dependency.
*/
static void
complete_mkdir(mkdir)
struct mkdir *mkdir;
{
struct diradd *dap;
if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE)
return;
LIST_REMOVE(mkdir, md_mkdirs);
dap = mkdir->md_diradd;
dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY));
if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) {
dap->da_state |= DEPCOMPLETE;
complete_diradd(dap);
}
WORKITEM_FREE(mkdir, D_MKDIR);
}
/*
* Handle the completion of a mkdir dependency.
*/
static void
handle_written_mkdir(mkdir, type)
struct mkdir *mkdir;
int type;
{
if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type)
panic("handle_written_mkdir: bad type");
mkdir->md_state |= COMPLETE;
complete_mkdir(mkdir);
}
static int
free_pagedep(pagedep)
struct pagedep *pagedep;
{
int i;
if (pagedep->pd_state & NEWBLOCK)
return (0);
if (!LIST_EMPTY(&pagedep->pd_dirremhd))
return (0);
for (i = 0; i < DAHASHSZ; i++)
if (!LIST_EMPTY(&pagedep->pd_diraddhd[i]))
return (0);
if (!LIST_EMPTY(&pagedep->pd_pendinghd))
return (0);
if (!LIST_EMPTY(&pagedep->pd_jmvrefhd))
return (0);
if (pagedep->pd_state & ONWORKLIST)
WORKLIST_REMOVE(&pagedep->pd_list);
LIST_REMOVE(pagedep, pd_hash);
WORKITEM_FREE(pagedep, D_PAGEDEP);
return (1);
}
/*
* Called from within softdep_disk_write_complete above.
* A write operation was just completed. Removed inodes can
* now be freed and associated block pointers may be committed.
* Note that this routine is always called from interrupt level
* with further interrupts from this device blocked.
*
* If the write did not succeed, we will do all the roll-forward
* operations, but we will not take the actions that will allow its
* dependencies to be processed.
*/
static int
handle_written_filepage(pagedep, bp, flags)
struct pagedep *pagedep;
struct buf *bp; /* buffer containing the written page */
int flags;
{
struct dirrem *dirrem;
struct diradd *dap, *nextdap;
struct direct *ep;
int i, chgs;
if ((pagedep->pd_state & IOSTARTED) == 0)
panic("handle_written_filepage: not started");
pagedep->pd_state &= ~IOSTARTED;
if ((flags & WRITESUCCEEDED) == 0)
goto rollforward;
/*
* Process any directory removals that have been committed.
*/
while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) {
LIST_REMOVE(dirrem, dm_next);
dirrem->dm_state |= COMPLETE;
dirrem->dm_dirinum = pagedep->pd_ino;
KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd),
("handle_written_filepage: Journal entries not written."));
add_to_worklist(&dirrem->dm_list, 0);
}
/*
* Free any directory additions that have been committed.
* If it is a newly allocated block, we have to wait until
* the on-disk directory inode claims the new block.
*/
if ((pagedep->pd_state & NEWBLOCK) == 0)
while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL)
free_diradd(dap, NULL);
rollforward:
/*
* Uncommitted directory entries must be restored.
*/
for (chgs = 0, i = 0; i < DAHASHSZ; i++) {
for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap;
dap = nextdap) {
nextdap = LIST_NEXT(dap, da_pdlist);
if (dap->da_state & ATTACHED)
panic("handle_written_filepage: attached");
ep = (struct direct *)
((char *)bp->b_data + dap->da_offset);
ep->d_ino = dap->da_newinum;
dap->da_state &= ~UNDONE;
dap->da_state |= ATTACHED;
chgs = 1;
/*
* If the inode referenced by the directory has
* been written out, then the dependency can be
* moved to the pending list.
*/
if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) {
LIST_REMOVE(dap, da_pdlist);
LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap,
da_pdlist);
}
}
}
/*
* If there were any rollbacks in the directory, then it must be
* marked dirty so that its will eventually get written back in
* its correct form.
*/
if (chgs || (flags & WRITESUCCEEDED) == 0) {
if ((bp->b_flags & B_DELWRI) == 0)
stat_dir_entry++;
bdirty(bp);
return (1);
}
/*
* If we are not waiting for a new directory block to be
* claimed by its inode, then the pagedep will be freed.
* Otherwise it will remain to track any new entries on
* the page in case they are fsync'ed.
*/
free_pagedep(pagedep);
return (0);
}
/*
* Writing back in-core inode structures.
*
* The filesystem only accesses an inode's contents when it occupies an
* "in-core" inode structure. These "in-core" structures are separate from
* the page frames used to cache inode blocks. Only the latter are
* transferred to/from the disk. So, when the updated contents of the
* "in-core" inode structure are copied to the corresponding in-memory inode
* block, the dependencies are also transferred. The following procedure is
* called when copying a dirty "in-core" inode to a cached inode block.
*/
/*
* Called when an inode is loaded from disk. If the effective link count
* differed from the actual link count when it was last flushed, then we
* need to ensure that the correct effective link count is put back.
*/
void
softdep_load_inodeblock(ip)
struct inode *ip; /* the "in_core" copy of the inode */
{
struct inodedep *inodedep;
struct ufsmount *ump;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_load_inodeblock called on non-softdep filesystem"));
/*
* Check for alternate nlink count.
*/
ip->i_effnlink = ip->i_nlink;
ACQUIRE_LOCK(ump);
if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0) {
FREE_LOCK(ump);
return;
}
+ if (ip->i_nlink != inodedep->id_nlinkwrote &&
+ inodedep->id_nlinkwrote != -1) {
+ KASSERT(ip->i_nlink == 0 &&
+ (ump->um_flags & UM_FSFAIL_CLEANUP) != 0,
+ ("read bad i_nlink value"));
+ ip->i_effnlink = ip->i_nlink = inodedep->id_nlinkwrote;
+ }
ip->i_effnlink -= inodedep->id_nlinkdelta;
KASSERT(ip->i_effnlink >= 0,
("softdep_load_inodeblock: negative i_effnlink"));
FREE_LOCK(ump);
}
/*
* This routine is called just before the "in-core" inode
* information is to be copied to the in-memory inode block.
* Recall that an inode block contains several inodes. If
* the force flag is set, then the dependencies will be
* cleared so that the update can always be made. Note that
* the buffer is locked when this routine is called, so we
* will never be in the middle of writing the inode block
* to disk.
*/
void
softdep_update_inodeblock(ip, bp, waitfor)
struct inode *ip; /* the "in_core" copy of the inode */
struct buf *bp; /* the buffer containing the inode block */
int waitfor; /* nonzero => update must be allowed */
{
struct inodedep *inodedep;
struct inoref *inoref;
struct ufsmount *ump;
struct worklist *wk;
struct mount *mp;
struct buf *ibp;
struct fs *fs;
int error;
ump = ITOUMP(ip);
mp = UFSTOVFS(ump);
KASSERT(MOUNTEDSOFTDEP(mp) != 0,
("softdep_update_inodeblock called on non-softdep filesystem"));
fs = ump->um_fs;
/*
* Preserve the freelink that is on disk. clear_unlinked_inodedep()
* does not have access to the in-core ip so must write directly into
* the inode block buffer when setting freelink.
*/
if (fs->fs_magic == FS_UFS1_MAGIC)
DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number))->di_freelink);
else
DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data +
ino_to_fsbo(fs, ip->i_number))->di_freelink);
/*
* If the effective link count is not equal to the actual link
* count, then we must track the difference in an inodedep while
* the inode is (potentially) tossed out of the cache. Otherwise,
* if there is no existing inodedep, then there are no dependencies
* to track.
*/
ACQUIRE_LOCK(ump);
again:
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) {
FREE_LOCK(ump);
if (ip->i_effnlink != ip->i_nlink)
panic("softdep_update_inodeblock: bad link count");
return;
}
+ KASSERT(ip->i_nlink >= inodedep->id_nlinkdelta,
+ ("softdep_update_inodeblock inconsistent ip %p i_nlink %d "
+ "inodedep %p id_nlinkdelta %jd",
+ ip, ip->i_nlink, inodedep, (intmax_t)inodedep->id_nlinkdelta));
+ inodedep->id_nlinkwrote = ip->i_nlink;
if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink)
panic("softdep_update_inodeblock: bad delta");
/*
* If we're flushing all dependencies we must also move any waiting
* for journal writes onto the bufwait list prior to I/O.
*/
if (waitfor) {
TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
== DEPCOMPLETE) {
jwait(&inoref->if_list, MNT_WAIT);
goto again;
}
}
}
/*
* Changes have been initiated. Anything depending on these
* changes cannot occur until this inode has been written.
*/
inodedep->id_state &= ~COMPLETE;
if ((inodedep->id_state & ONWORKLIST) == 0)
WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list);
/*
* Any new dependencies associated with the incore inode must
* now be moved to the list associated with the buffer holding
* the in-memory copy of the inode. Once merged process any
* allocdirects that are completed by the merger.
*/
merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt);
if (!TAILQ_EMPTY(&inodedep->id_inoupdt))
handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt),
NULL);
merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt);
if (!TAILQ_EMPTY(&inodedep->id_extupdt))
handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt),
NULL);
/*
* Now that the inode has been pushed into the buffer, the
* operations dependent on the inode being written to disk
* can be moved to the id_bufwait so that they will be
* processed when the buffer I/O completes.
*/
while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) {
WORKLIST_REMOVE(wk);
WORKLIST_INSERT(&inodedep->id_bufwait, wk);
}
/*
* Newly allocated inodes cannot be written until the bitmap
* that allocates them have been written (indicated by
* DEPCOMPLETE being set in id_state). If we are doing a
* forced sync (e.g., an fsync on a file), we force the bitmap
* to be written so that the update can be done.
*/
if (waitfor == 0) {
FREE_LOCK(ump);
return;
}
retry:
if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) {
FREE_LOCK(ump);
return;
}
ibp = inodedep->id_bmsafemap->sm_buf;
ibp = getdirtybuf(ibp, LOCK_PTR(ump), MNT_WAIT);
if (ibp == NULL) {
/*
* If ibp came back as NULL, the dependency could have been
* freed while we slept. Look it up again, and check to see
* that it has completed.
*/
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0)
goto retry;
FREE_LOCK(ump);
return;
}
FREE_LOCK(ump);
if ((error = bwrite(ibp)) != 0)
softdep_error("softdep_update_inodeblock: bwrite", error);
}
/*
* Merge the a new inode dependency list (such as id_newinoupdt) into an
* old inode dependency list (such as id_inoupdt).
*/
static void
merge_inode_lists(newlisthead, oldlisthead)
struct allocdirectlst *newlisthead;
struct allocdirectlst *oldlisthead;
{
struct allocdirect *listadp, *newadp;
newadp = TAILQ_FIRST(newlisthead);
if (newadp != NULL)
LOCK_OWNED(VFSTOUFS(newadp->ad_block.nb_list.wk_mp));
for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) {
if (listadp->ad_offset < newadp->ad_offset) {
listadp = TAILQ_NEXT(listadp, ad_next);
continue;
}
TAILQ_REMOVE(newlisthead, newadp, ad_next);
TAILQ_INSERT_BEFORE(listadp, newadp, ad_next);
if (listadp->ad_offset == newadp->ad_offset) {
allocdirect_merge(oldlisthead, newadp,
listadp);
listadp = newadp;
}
newadp = TAILQ_FIRST(newlisthead);
}
while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) {
TAILQ_REMOVE(newlisthead, newadp, ad_next);
TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next);
}
}
/*
* If we are doing an fsync, then we must ensure that any directory
* entries for the inode have been written after the inode gets to disk.
*/
int
softdep_fsync(vp)
struct vnode *vp; /* the "in_core" copy of the inode */
{
struct inodedep *inodedep;
struct pagedep *pagedep;
struct inoref *inoref;
struct ufsmount *ump;
struct worklist *wk;
struct diradd *dap;
struct mount *mp;
struct vnode *pvp;
struct inode *ip;
struct buf *bp;
struct fs *fs;
struct thread *td = curthread;
int error, flushparent, pagedep_new_block;
ino_t parentino;
ufs_lbn_t lbn;
ip = VTOI(vp);
mp = vp->v_mount;
ump = VFSTOUFS(mp);
fs = ump->um_fs;
if (MOUNTEDSOFTDEP(mp) == 0)
return (0);
ACQUIRE_LOCK(ump);
restart:
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) {
FREE_LOCK(ump);
return (0);
}
TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
== DEPCOMPLETE) {
jwait(&inoref->if_list, MNT_WAIT);
goto restart;
}
}
if (!LIST_EMPTY(&inodedep->id_inowait) ||
!TAILQ_EMPTY(&inodedep->id_extupdt) ||
!TAILQ_EMPTY(&inodedep->id_newextupdt) ||
!TAILQ_EMPTY(&inodedep->id_inoupdt) ||
!TAILQ_EMPTY(&inodedep->id_newinoupdt))
panic("softdep_fsync: pending ops %p", inodedep);
for (error = 0, flushparent = 0; ; ) {
if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL)
break;
if (wk->wk_type != D_DIRADD)
panic("softdep_fsync: Unexpected type %s",
TYPENAME(wk->wk_type));
dap = WK_DIRADD(wk);
/*
* Flush our parent if this directory entry has a MKDIR_PARENT
* dependency or is contained in a newly allocated block.
*/
if (dap->da_state & DIRCHG)
pagedep = dap->da_previous->dm_pagedep;
else
pagedep = dap->da_pagedep;
parentino = pagedep->pd_ino;
lbn = pagedep->pd_lbn;
if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE)
panic("softdep_fsync: dirty");
if ((dap->da_state & MKDIR_PARENT) ||
(pagedep->pd_state & NEWBLOCK))
flushparent = 1;
else
flushparent = 0;
/*
* If we are being fsync'ed as part of vgone'ing this vnode,
* then we will not be able to release and recover the
* vnode below, so we just have to give up on writing its
* directory entry out. It will eventually be written, just
* not now, but then the user was not asking to have it
* written, so we are not breaking any promises.
*/
if (VN_IS_DOOMED(vp))
break;
/*
* We prevent deadlock by always fetching inodes from the
* root, moving down the directory tree. Thus, when fetching
* our parent directory, we first try to get the lock. If
* that fails, we must unlock ourselves before requesting
* the lock on our parent. See the comment in ufs_lookup
* for details on possible races.
*/
FREE_LOCK(ump);
if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp,
FFSV_FORCEINSMQ)) {
/*
* Unmount cannot proceed after unlock because
* caller must have called vn_start_write().
*/
VOP_UNLOCK(vp);
error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE,
&pvp, FFSV_FORCEINSMQ);
MPASS(VTOI(pvp)->i_mode != 0);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
if (VN_IS_DOOMED(vp)) {
if (error == 0)
vput(pvp);
error = ENOENT;
}
if (error != 0)
return (error);
}
/*
* All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps
* that are contained in direct blocks will be resolved by
* doing a ffs_update. Pagedeps contained in indirect blocks
* may require a complete sync'ing of the directory. So, we
* try the cheap and fast ffs_update first, and if that fails,
* then we do the slower ffs_syncvnode of the directory.
*/
if (flushparent) {
int locked;
if ((error = ffs_update(pvp, 1)) != 0) {
vput(pvp);
return (error);
}
ACQUIRE_LOCK(ump);
locked = 1;
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) {
if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) {
if (wk->wk_type != D_DIRADD)
panic("softdep_fsync: Unexpected type %s",
TYPENAME(wk->wk_type));
dap = WK_DIRADD(wk);
if (dap->da_state & DIRCHG)
pagedep = dap->da_previous->dm_pagedep;
else
pagedep = dap->da_pagedep;
pagedep_new_block = pagedep->pd_state & NEWBLOCK;
FREE_LOCK(ump);
locked = 0;
if (pagedep_new_block && (error =
ffs_syncvnode(pvp, MNT_WAIT, 0))) {
vput(pvp);
return (error);
}
}
}
if (locked)
FREE_LOCK(ump);
}
/*
* Flush directory page containing the inode's name.
*/
error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred,
&bp);
if (error == 0)
error = bwrite(bp);
else
brelse(bp);
vput(pvp);
- if (error != 0)
+ if (!ffs_fsfail_cleanup(ump, error))
return (error);
ACQUIRE_LOCK(ump);
if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0)
break;
}
FREE_LOCK(ump);
return (0);
}
/*
* Flush all the dirty bitmaps associated with the block device
* before flushing the rest of the dirty blocks so as to reduce
* the number of dependencies that will have to be rolled back.
*
* XXX Unused?
*/
void
softdep_fsync_mountdev(vp)
struct vnode *vp;
{
struct buf *bp, *nbp;
struct worklist *wk;
struct bufobj *bo;
if (!vn_isdisk(vp, NULL))
panic("softdep_fsync_mountdev: vnode not a disk");
bo = &vp->v_bufobj;
restart:
BO_LOCK(bo);
TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
/*
* If it is already scheduled, skip to the next buffer.
*/
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL))
continue;
if ((bp->b_flags & B_DELWRI) == 0)
panic("softdep_fsync_mountdev: not dirty");
/*
* We are only interested in bitmaps with outstanding
* dependencies.
*/
if ((wk = LIST_FIRST(&bp->b_dep)) == NULL ||
wk->wk_type != D_BMSAFEMAP ||
(bp->b_vflags & BV_BKGRDINPROG)) {
BUF_UNLOCK(bp);
continue;
}
BO_UNLOCK(bo);
bremfree(bp);
(void) bawrite(bp);
goto restart;
}
drain_output(vp);
BO_UNLOCK(bo);
}
/*
* Sync all cylinder groups that were dirty at the time this function is
* called. Newly dirtied cgs will be inserted before the sentinel. This
* is used to flush freedep activity that may be holding up writes to a
* indirect block.
*/
static int
sync_cgs(mp, waitfor)
struct mount *mp;
int waitfor;
{
struct bmsafemap *bmsafemap;
struct bmsafemap *sentinel;
struct ufsmount *ump;
struct buf *bp;
int error;
sentinel = malloc(sizeof(*sentinel), M_BMSAFEMAP, M_ZERO | M_WAITOK);
sentinel->sm_cg = -1;
ump = VFSTOUFS(mp);
error = 0;
ACQUIRE_LOCK(ump);
LIST_INSERT_HEAD(&ump->softdep_dirtycg, sentinel, sm_next);
for (bmsafemap = LIST_NEXT(sentinel, sm_next); bmsafemap != NULL;
bmsafemap = LIST_NEXT(sentinel, sm_next)) {
/* Skip sentinels and cgs with no work to release. */
if (bmsafemap->sm_cg == -1 ||
(LIST_EMPTY(&bmsafemap->sm_freehd) &&
LIST_EMPTY(&bmsafemap->sm_freewr))) {
LIST_REMOVE(sentinel, sm_next);
LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next);
continue;
}
/*
* If we don't get the lock and we're waiting try again, if
* not move on to the next buf and try to sync it.
*/
bp = getdirtybuf(bmsafemap->sm_buf, LOCK_PTR(ump), waitfor);
if (bp == NULL && waitfor == MNT_WAIT)
continue;
LIST_REMOVE(sentinel, sm_next);
LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next);
if (bp == NULL)
continue;
FREE_LOCK(ump);
if (waitfor == MNT_NOWAIT)
bawrite(bp);
else
error = bwrite(bp);
ACQUIRE_LOCK(ump);
if (error)
break;
}
LIST_REMOVE(sentinel, sm_next);
FREE_LOCK(ump);
free(sentinel, M_BMSAFEMAP);
return (error);
}
/*
* This routine is called when we are trying to synchronously flush a
* file. This routine must eliminate any filesystem metadata dependencies
* so that the syncing routine can succeed.
*/
int
softdep_sync_metadata(struct vnode *vp)
{
struct inode *ip;
int error;
ip = VTOI(vp);
KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0,
("softdep_sync_metadata called on non-softdep filesystem"));
/*
* Ensure that any direct block dependencies have been cleared,
* truncations are started, and inode references are journaled.
*/
ACQUIRE_LOCK(VFSTOUFS(vp->v_mount));
/*
* Write all journal records to prevent rollbacks on devvp.
*/
if (vp->v_type == VCHR)
softdep_flushjournal(vp->v_mount);
error = flush_inodedep_deps(vp, vp->v_mount, ip->i_number);
/*
* Ensure that all truncates are written so we won't find deps on
* indirect blocks.
*/
process_truncates(vp);
FREE_LOCK(VFSTOUFS(vp->v_mount));
return (error);
}
/*
* This routine is called when we are attempting to sync a buf with
* dependencies. If waitfor is MNT_NOWAIT it attempts to schedule any
* other IO it can but returns EBUSY if the buffer is not yet able to
* be written. Dependencies which will not cause rollbacks will always
* return 0.
*/
int
softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor)
{
struct indirdep *indirdep;
struct pagedep *pagedep;
struct allocindir *aip;
struct newblk *newblk;
struct ufsmount *ump;
struct buf *nbp;
struct worklist *wk;
int i, error;
KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0,
("softdep_sync_buf called on non-softdep filesystem"));
/*
* For VCHR we just don't want to force flush any dependencies that
* will cause rollbacks.
*/
if (vp->v_type == VCHR) {
if (waitfor == MNT_NOWAIT && softdep_count_dependencies(bp, 0))
return (EBUSY);
return (0);
}
ump = VFSTOUFS(vp->v_mount);
ACQUIRE_LOCK(ump);
/*
* As we hold the buffer locked, none of its dependencies
* will disappear.
*/
error = 0;
top:
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
switch (wk->wk_type) {
case D_ALLOCDIRECT:
case D_ALLOCINDIR:
newblk = WK_NEWBLK(wk);
if (newblk->nb_jnewblk != NULL) {
if (waitfor == MNT_NOWAIT) {
error = EBUSY;
goto out_unlock;
}
jwait(&newblk->nb_jnewblk->jn_list, waitfor);
goto top;
}
if (newblk->nb_state & DEPCOMPLETE ||
waitfor == MNT_NOWAIT)
continue;
nbp = newblk->nb_bmsafemap->sm_buf;
nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor);
if (nbp == NULL)
goto top;
FREE_LOCK(ump);
if ((error = bwrite(nbp)) != 0)
goto out;
ACQUIRE_LOCK(ump);
continue;
case D_INDIRDEP:
indirdep = WK_INDIRDEP(wk);
if (waitfor == MNT_NOWAIT) {
if (!TAILQ_EMPTY(&indirdep->ir_trunc) ||
!LIST_EMPTY(&indirdep->ir_deplisthd)) {
error = EBUSY;
goto out_unlock;
}
}
if (!TAILQ_EMPTY(&indirdep->ir_trunc))
panic("softdep_sync_buf: truncation pending.");
restart:
LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) {
newblk = (struct newblk *)aip;
if (newblk->nb_jnewblk != NULL) {
jwait(&newblk->nb_jnewblk->jn_list,
waitfor);
goto restart;
}
if (newblk->nb_state & DEPCOMPLETE)
continue;
nbp = newblk->nb_bmsafemap->sm_buf;
nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor);
if (nbp == NULL)
goto restart;
FREE_LOCK(ump);
if ((error = bwrite(nbp)) != 0)
goto out;
ACQUIRE_LOCK(ump);
goto restart;
}
continue;
case D_PAGEDEP:
/*
* Only flush directory entries in synchronous passes.
*/
if (waitfor != MNT_WAIT) {
error = EBUSY;
goto out_unlock;
}
/*
* While syncing snapshots, we must allow recursive
* lookups.
*/
BUF_AREC(bp);
/*
* We are trying to sync a directory that may
* have dependencies on both its own metadata
* and/or dependencies on the inodes of any
* recently allocated files. We walk its diradd
* lists pushing out the associated inode.
*/
pagedep = WK_PAGEDEP(wk);
for (i = 0; i < DAHASHSZ; i++) {
if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0)
continue;
if ((error = flush_pagedep_deps(vp, wk->wk_mp,
&pagedep->pd_diraddhd[i]))) {
BUF_NOREC(bp);
goto out_unlock;
}
}
BUF_NOREC(bp);
continue;
case D_FREEWORK:
case D_FREEDEP:
case D_JSEGDEP:
case D_JNEWBLK:
continue;
default:
panic("softdep_sync_buf: Unknown type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
out_unlock:
FREE_LOCK(ump);
out:
return (error);
}
/*
* Flush the dependencies associated with an inodedep.
*/
static int
flush_inodedep_deps(vp, mp, ino)
struct vnode *vp;
struct mount *mp;
ino_t ino;
{
struct inodedep *inodedep;
struct inoref *inoref;
struct ufsmount *ump;
int error, waitfor;
/*
* This work is done in two passes. The first pass grabs most
* of the buffers and begins asynchronously writing them. The
* only way to wait for these asynchronous writes is to sleep
* on the filesystem vnode which may stay busy for a long time
* if the filesystem is active. So, instead, we make a second
* pass over the dependencies blocking on each write. In the
* usual case we will be blocking against a write that we
* initiated, so when it is done the dependency will have been
* resolved. Thus the second pass is expected to end quickly.
* We give a brief window at the top of the loop to allow
* any pending I/O to complete.
*/
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
for (error = 0, waitfor = MNT_NOWAIT; ; ) {
if (error)
return (error);
FREE_LOCK(ump);
ACQUIRE_LOCK(ump);
restart:
if (inodedep_lookup(mp, ino, 0, &inodedep) == 0)
return (0);
TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
== DEPCOMPLETE) {
jwait(&inoref->if_list, MNT_WAIT);
goto restart;
}
}
if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) ||
flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) ||
flush_deplist(&inodedep->id_extupdt, waitfor, &error) ||
flush_deplist(&inodedep->id_newextupdt, waitfor, &error))
continue;
/*
* If pass2, we are done, otherwise do pass 2.
*/
if (waitfor == MNT_WAIT)
break;
waitfor = MNT_WAIT;
}
/*
* Try freeing inodedep in case all dependencies have been removed.
*/
if (inodedep_lookup(mp, ino, 0, &inodedep) != 0)
(void) free_inodedep(inodedep);
return (0);
}
/*
* Flush an inode dependency list.
*/
static int
flush_deplist(listhead, waitfor, errorp)
struct allocdirectlst *listhead;
int waitfor;
int *errorp;
{
struct allocdirect *adp;
struct newblk *newblk;
struct ufsmount *ump;
struct buf *bp;
if ((adp = TAILQ_FIRST(listhead)) == NULL)
return (0);
ump = VFSTOUFS(adp->ad_list.wk_mp);
LOCK_OWNED(ump);
TAILQ_FOREACH(adp, listhead, ad_next) {
newblk = (struct newblk *)adp;
if (newblk->nb_jnewblk != NULL) {
jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
return (1);
}
if (newblk->nb_state & DEPCOMPLETE)
continue;
bp = newblk->nb_bmsafemap->sm_buf;
bp = getdirtybuf(bp, LOCK_PTR(ump), waitfor);
if (bp == NULL) {
if (waitfor == MNT_NOWAIT)
continue;
return (1);
}
FREE_LOCK(ump);
if (waitfor == MNT_NOWAIT)
bawrite(bp);
else
*errorp = bwrite(bp);
ACQUIRE_LOCK(ump);
return (1);
}
return (0);
}
/*
* Flush dependencies associated with an allocdirect block.
*/
static int
flush_newblk_dep(vp, mp, lbn)
struct vnode *vp;
struct mount *mp;
ufs_lbn_t lbn;
{
struct newblk *newblk;
struct ufsmount *ump;
struct bufobj *bo;
struct inode *ip;
struct buf *bp;
ufs2_daddr_t blkno;
int error;
error = 0;
bo = &vp->v_bufobj;
ip = VTOI(vp);
blkno = DIP(ip, i_db[lbn]);
if (blkno == 0)
panic("flush_newblk_dep: Missing block");
ump = VFSTOUFS(mp);
ACQUIRE_LOCK(ump);
/*
* Loop until all dependencies related to this block are satisfied.
* We must be careful to restart after each sleep in case a write
* completes some part of this process for us.
*/
for (;;) {
if (newblk_lookup(mp, blkno, 0, &newblk) == 0) {
FREE_LOCK(ump);
break;
}
if (newblk->nb_list.wk_type != D_ALLOCDIRECT)
panic("flush_newblk_dep: Bad newblk %p", newblk);
/*
* Flush the journal.
*/
if (newblk->nb_jnewblk != NULL) {
jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT);
continue;
}
/*
* Write the bitmap dependency.
*/
if ((newblk->nb_state & DEPCOMPLETE) == 0) {
bp = newblk->nb_bmsafemap->sm_buf;
bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT);
if (bp == NULL)
continue;
FREE_LOCK(ump);
error = bwrite(bp);
if (error)
break;
ACQUIRE_LOCK(ump);
continue;
}
/*
* Write the buffer.
*/
FREE_LOCK(ump);
BO_LOCK(bo);
bp = gbincore(bo, lbn);
if (bp != NULL) {
error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL |
LK_INTERLOCK, BO_LOCKPTR(bo));
if (error == ENOLCK) {
ACQUIRE_LOCK(ump);
error = 0;
continue; /* Slept, retry */
}
if (error != 0)
break; /* Failed */
if (bp->b_flags & B_DELWRI) {
bremfree(bp);
error = bwrite(bp);
if (error)
break;
} else
BUF_UNLOCK(bp);
} else
BO_UNLOCK(bo);
/*
* We have to wait for the direct pointers to
* point at the newdirblk before the dependency
* will go away.
*/
error = ffs_update(vp, 1);
if (error)
break;
ACQUIRE_LOCK(ump);
}
return (error);
}
/*
* Eliminate a pagedep dependency by flushing out all its diradd dependencies.
*/
static int
flush_pagedep_deps(pvp, mp, diraddhdp)
struct vnode *pvp;
struct mount *mp;
struct diraddhd *diraddhdp;
{
struct inodedep *inodedep;
struct inoref *inoref;
struct ufsmount *ump;
struct diradd *dap;
struct vnode *vp;
int error = 0;
struct buf *bp;
ino_t inum;
struct diraddhd unfinished;
LIST_INIT(&unfinished);
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
restart:
while ((dap = LIST_FIRST(diraddhdp)) != NULL) {
/*
* Flush ourselves if this directory entry
* has a MKDIR_PARENT dependency.
*/
if (dap->da_state & MKDIR_PARENT) {
FREE_LOCK(ump);
if ((error = ffs_update(pvp, 1)) != 0)
break;
ACQUIRE_LOCK(ump);
/*
* If that cleared dependencies, go on to next.
*/
if (dap != LIST_FIRST(diraddhdp))
continue;
/*
* All MKDIR_PARENT dependencies and all the
* NEWBLOCK pagedeps that are contained in direct
* blocks were resolved by doing above ffs_update.
* Pagedeps contained in indirect blocks may
* require a complete sync'ing of the directory.
* We are in the midst of doing a complete sync,
* so if they are not resolved in this pass we
* defer them for now as they will be sync'ed by
* our caller shortly.
*/
LIST_REMOVE(dap, da_pdlist);
LIST_INSERT_HEAD(&unfinished, dap, da_pdlist);
continue;
}
/*
* A newly allocated directory must have its "." and
* ".." entries written out before its name can be
* committed in its parent.
*/
inum = dap->da_newinum;
if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0)
panic("flush_pagedep_deps: lost inode1");
/*
* Wait for any pending journal adds to complete so we don't
* cause rollbacks while syncing.
*/
TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) {
if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY))
== DEPCOMPLETE) {
jwait(&inoref->if_list, MNT_WAIT);
goto restart;
}
}
if (dap->da_state & MKDIR_BODY) {
FREE_LOCK(ump);
if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp,
FFSV_FORCEINSMQ)))
break;
MPASS(VTOI(vp)->i_mode != 0);
error = flush_newblk_dep(vp, mp, 0);
/*
* If we still have the dependency we might need to
* update the vnode to sync the new link count to
* disk.
*/
if (error == 0 && dap == LIST_FIRST(diraddhdp))
error = ffs_update(vp, 1);
vput(vp);
if (error != 0)
break;
ACQUIRE_LOCK(ump);
/*
* If that cleared dependencies, go on to next.
*/
if (dap != LIST_FIRST(diraddhdp))
continue;
if (dap->da_state & MKDIR_BODY) {
inodedep_lookup(UFSTOVFS(ump), inum, 0,
&inodedep);
panic("flush_pagedep_deps: MKDIR_BODY "
"inodedep %p dap %p vp %p",
inodedep, dap, vp);
}
}
/*
* Flush the inode on which the directory entry depends.
* Having accounted for MKDIR_PARENT and MKDIR_BODY above,
* the only remaining dependency is that the updated inode
* count must get pushed to disk. The inode has already
* been pushed into its inode buffer (via VOP_UPDATE) at
* the time of the reference count change. So we need only
* locate that buffer, ensure that there will be no rollback
* caused by a bitmap dependency, then write the inode buffer.
*/
retry:
if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0)
panic("flush_pagedep_deps: lost inode");
/*
* If the inode still has bitmap dependencies,
* push them to disk.
*/
if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) {
bp = inodedep->id_bmsafemap->sm_buf;
bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT);
if (bp == NULL)
goto retry;
FREE_LOCK(ump);
if ((error = bwrite(bp)) != 0)
break;
ACQUIRE_LOCK(ump);
if (dap != LIST_FIRST(diraddhdp))
continue;
}
/*
* If the inode is still sitting in a buffer waiting
* to be written or waiting for the link count to be
* adjusted update it here to flush it to disk.
*/
if (dap == LIST_FIRST(diraddhdp)) {
FREE_LOCK(ump);
if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp,
FFSV_FORCEINSMQ)))
break;
MPASS(VTOI(vp)->i_mode != 0);
error = ffs_update(vp, 1);
vput(vp);
if (error)
break;
ACQUIRE_LOCK(ump);
}
/*
* If we have failed to get rid of all the dependencies
* then something is seriously wrong.
*/
if (dap == LIST_FIRST(diraddhdp)) {
inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep);
panic("flush_pagedep_deps: failed to flush "
"inodedep %p ino %ju dap %p",
inodedep, (uintmax_t)inum, dap);
}
}
if (error)
ACQUIRE_LOCK(ump);
while ((dap = LIST_FIRST(&unfinished)) != NULL) {
LIST_REMOVE(dap, da_pdlist);
LIST_INSERT_HEAD(diraddhdp, dap, da_pdlist);
}
return (error);
}
/*
* A large burst of file addition or deletion activity can drive the
* memory load excessively high. First attempt to slow things down
* using the techniques below. If that fails, this routine requests
* the offending operations to fall back to running synchronously
* until the memory load returns to a reasonable level.
*/
int
softdep_slowdown(vp)
struct vnode *vp;
{
struct ufsmount *ump;
int jlow;
int max_softdeps_hard;
KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0,
("softdep_slowdown called on non-softdep filesystem"));
ump = VFSTOUFS(vp->v_mount);
ACQUIRE_LOCK(ump);
jlow = 0;
/*
* Check for journal space if needed.
*/
if (DOINGSUJ(vp)) {
if (journal_space(ump, 0) == 0)
jlow = 1;
}
/*
* If the system is under its limits and our filesystem is
* not responsible for more than our share of the usage and
* we are not low on journal space, then no need to slow down.
*/
max_softdeps_hard = max_softdeps * 11 / 10;
if (dep_current[D_DIRREM] < max_softdeps_hard / 2 &&
dep_current[D_INODEDEP] < max_softdeps_hard &&
dep_current[D_INDIRDEP] < max_softdeps_hard / 1000 &&
dep_current[D_FREEBLKS] < max_softdeps_hard && jlow == 0 &&
ump->softdep_curdeps[D_DIRREM] <
(max_softdeps_hard / 2) / stat_flush_threads &&
ump->softdep_curdeps[D_INODEDEP] <
max_softdeps_hard / stat_flush_threads &&
ump->softdep_curdeps[D_INDIRDEP] <
(max_softdeps_hard / 1000) / stat_flush_threads &&
ump->softdep_curdeps[D_FREEBLKS] <
max_softdeps_hard / stat_flush_threads) {
FREE_LOCK(ump);
return (0);
}
/*
* If the journal is low or our filesystem is over its limit
* then speedup the cleanup.
*/
if (ump->softdep_curdeps[D_INDIRDEP] <
(max_softdeps_hard / 1000) / stat_flush_threads || jlow)
softdep_speedup(ump);
stat_sync_limit_hit += 1;
FREE_LOCK(ump);
/*
* We only slow down the rate at which new dependencies are
* generated if we are not using journaling. With journaling,
* the cleanup should always be sufficient to keep things
* under control.
*/
if (DOINGSUJ(vp))
return (0);
return (1);
}
/*
* Called by the allocation routines when they are about to fail
* in the hope that we can free up the requested resource (inodes
* or disk space).
*
* First check to see if the work list has anything on it. If it has,
* clean up entries until we successfully free the requested resource.
* Because this process holds inodes locked, we cannot handle any remove
* requests that might block on a locked inode as that could lead to
* deadlock. If the worklist yields none of the requested resource,
* start syncing out vnodes to free up the needed space.
*/
int
softdep_request_cleanup(fs, vp, cred, resource)
struct fs *fs;
struct vnode *vp;
struct ucred *cred;
int resource;
{
struct ufsmount *ump;
struct mount *mp;
long starttime;
ufs2_daddr_t needed;
int error, failed_vnode;
/*
* If we are being called because of a process doing a
* copy-on-write, then it is not safe to process any
* worklist items as we will recurse into the copyonwrite
* routine. This will result in an incoherent snapshot.
* If the vnode that we hold is a snapshot, we must avoid
* handling other resources that could cause deadlock.
*/
if ((curthread->td_pflags & TDP_COWINPROGRESS) || IS_SNAPSHOT(VTOI(vp)))
return (0);
if (resource == FLUSH_BLOCKS_WAIT)
stat_cleanup_blkrequests += 1;
else
stat_cleanup_inorequests += 1;
mp = vp->v_mount;
ump = VFSTOUFS(mp);
mtx_assert(UFS_MTX(ump), MA_OWNED);
UFS_UNLOCK(ump);
error = ffs_update(vp, 1);
if (error != 0 || MOUNTEDSOFTDEP(mp) == 0) {
UFS_LOCK(ump);
return (0);
}
/*
* If we are in need of resources, start by cleaning up
* any block removals associated with our inode.
*/
ACQUIRE_LOCK(ump);
process_removes(vp);
process_truncates(vp);
FREE_LOCK(ump);
/*
* Now clean up at least as many resources as we will need.
*
* When requested to clean up inodes, the number that are needed
* is set by the number of simultaneous writers (mnt_writeopcount)
* plus a bit of slop (2) in case some more writers show up while
* we are cleaning.
*
* When requested to free up space, the amount of space that
* we need is enough blocks to allocate a full-sized segment
* (fs_contigsumsize). The number of such segments that will
* be needed is set by the number of simultaneous writers
* (mnt_writeopcount) plus a bit of slop (2) in case some more
* writers show up while we are cleaning.
*
* Additionally, if we are unpriviledged and allocating space,
* we need to ensure that we clean up enough blocks to get the
* needed number of blocks over the threshold of the minimum
* number of blocks required to be kept free by the filesystem
* (fs_minfree).
*/
if (resource == FLUSH_INODES_WAIT) {
needed = vfs_mount_fetch_counter(vp->v_mount,
MNT_COUNT_WRITEOPCOUNT) + 2;
} else if (resource == FLUSH_BLOCKS_WAIT) {
needed = (vfs_mount_fetch_counter(vp->v_mount,
MNT_COUNT_WRITEOPCOUNT) + 2) * fs->fs_contigsumsize;
if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE))
needed += fragstoblks(fs,
roundup((fs->fs_dsize * fs->fs_minfree / 100) -
fs->fs_cstotal.cs_nffree, fs->fs_frag));
} else {
printf("softdep_request_cleanup: Unknown resource type %d\n",
resource);
UFS_LOCK(ump);
return (0);
}
starttime = time_second;
retry:
if (resource == FLUSH_BLOCKS_WAIT &&
fs->fs_cstotal.cs_nbfree <= needed)
softdep_send_speedup(ump, needed * fs->fs_bsize,
BIO_SPEEDUP_TRIM);
if ((resource == FLUSH_BLOCKS_WAIT && ump->softdep_on_worklist > 0 &&
fs->fs_cstotal.cs_nbfree <= needed) ||
(resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 &&
fs->fs_cstotal.cs_nifree <= needed)) {
ACQUIRE_LOCK(ump);
if (ump->softdep_on_worklist > 0 &&
process_worklist_item(UFSTOVFS(ump),
ump->softdep_on_worklist, LK_NOWAIT) != 0)
stat_worklist_push += 1;
FREE_LOCK(ump);
}
/*
* If we still need resources and there are no more worklist
* entries to process to obtain them, we have to start flushing
* the dirty vnodes to force the release of additional requests
* to the worklist that we can then process to reap addition
* resources. We walk the vnodes associated with the mount point
* until we get the needed worklist requests that we can reap.
*
* If there are several threads all needing to clean the same
* mount point, only one is allowed to walk the mount list.
* When several threads all try to walk the same mount list,
* they end up competing with each other and often end up in
* livelock. This approach ensures that forward progress is
* made at the cost of occational ENOSPC errors being returned
* that might otherwise have been avoided.
*/
error = 1;
if ((resource == FLUSH_BLOCKS_WAIT &&
fs->fs_cstotal.cs_nbfree <= needed) ||
(resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 &&
fs->fs_cstotal.cs_nifree <= needed)) {
ACQUIRE_LOCK(ump);
if ((ump->um_softdep->sd_flags & FLUSH_RC_ACTIVE) == 0) {
ump->um_softdep->sd_flags |= FLUSH_RC_ACTIVE;
FREE_LOCK(ump);
failed_vnode = softdep_request_cleanup_flush(mp, ump);
ACQUIRE_LOCK(ump);
ump->um_softdep->sd_flags &= ~FLUSH_RC_ACTIVE;
FREE_LOCK(ump);
if (ump->softdep_on_worklist > 0) {
stat_cleanup_retries += 1;
if (!failed_vnode)
goto retry;
}
} else {
FREE_LOCK(ump);
error = 0;
}
stat_cleanup_failures += 1;
}
if (time_second - starttime > stat_cleanup_high_delay)
stat_cleanup_high_delay = time_second - starttime;
UFS_LOCK(ump);
return (error);
}
/*
* Scan the vnodes for the specified mount point flushing out any
* vnodes that can be locked without waiting. Finally, try to flush
* the device associated with the mount point if it can be locked
* without waiting.
*
* We return 0 if we were able to lock every vnode in our scan.
* If we had to skip one or more vnodes, we return 1.
*/
static int
softdep_request_cleanup_flush(mp, ump)
struct mount *mp;
struct ufsmount *ump;
{
struct thread *td;
struct vnode *lvp, *mvp;
int failed_vnode;
failed_vnode = 0;
td = curthread;
MNT_VNODE_FOREACH_ALL(lvp, mp, mvp) {
if (TAILQ_FIRST(&lvp->v_bufobj.bo_dirty.bv_hd) == 0) {
VI_UNLOCK(lvp);
continue;
}
if (vget(lvp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT,
td) != 0) {
failed_vnode = 1;
continue;
}
if (lvp->v_vflag & VV_NOSYNC) { /* unlinked */
vput(lvp);
continue;
}
(void) ffs_syncvnode(lvp, MNT_NOWAIT, 0);
vput(lvp);
}
lvp = ump->um_devvp;
if (vn_lock(lvp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
VOP_FSYNC(lvp, MNT_NOWAIT, td);
VOP_UNLOCK(lvp);
}
return (failed_vnode);
}
static bool
softdep_excess_items(struct ufsmount *ump, int item)
{
KASSERT(item >= 0 && item < D_LAST, ("item %d", item));
return (dep_current[item] > max_softdeps &&
ump->softdep_curdeps[item] > max_softdeps /
stat_flush_threads);
}
static void
schedule_cleanup(struct mount *mp)
{
struct ufsmount *ump;
struct thread *td;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
FREE_LOCK(ump);
td = curthread;
if ((td->td_pflags & TDP_KTHREAD) != 0 &&
(td->td_proc->p_flag2 & P2_AST_SU) == 0) {
/*
* No ast is delivered to kernel threads, so nobody
* would deref the mp. Some kernel threads
* explicitely check for AST, e.g. NFS daemon does
* this in the serving loop.
*/
return;
}
if (td->td_su != NULL)
vfs_rel(td->td_su);
vfs_ref(mp);
td->td_su = mp;
thread_lock(td);
td->td_flags |= TDF_ASTPENDING;
thread_unlock(td);
}
static void
softdep_ast_cleanup_proc(struct thread *td)
{
struct mount *mp;
struct ufsmount *ump;
int error;
bool req;
while ((mp = td->td_su) != NULL) {
td->td_su = NULL;
error = vfs_busy(mp, MBF_NOWAIT);
vfs_rel(mp);
if (error != 0)
return;
if (ffs_own_mount(mp) && MOUNTEDSOFTDEP(mp)) {
ump = VFSTOUFS(mp);
for (;;) {
req = false;
ACQUIRE_LOCK(ump);
if (softdep_excess_items(ump, D_INODEDEP)) {
req = true;
request_cleanup(mp, FLUSH_INODES);
}
if (softdep_excess_items(ump, D_DIRREM)) {
req = true;
request_cleanup(mp, FLUSH_BLOCKS);
}
FREE_LOCK(ump);
if (softdep_excess_items(ump, D_NEWBLK) ||
softdep_excess_items(ump, D_ALLOCDIRECT) ||
softdep_excess_items(ump, D_ALLOCINDIR)) {
error = vn_start_write(NULL, &mp,
V_WAIT);
if (error == 0) {
req = true;
VFS_SYNC(mp, MNT_WAIT);
vn_finished_write(mp);
}
}
if ((td->td_pflags & TDP_KTHREAD) != 0 || !req)
break;
}
}
vfs_unbusy(mp);
}
if ((mp = td->td_su) != NULL) {
td->td_su = NULL;
vfs_rel(mp);
}
}
/*
* If memory utilization has gotten too high, deliberately slow things
* down and speed up the I/O processing.
*/
static int
request_cleanup(mp, resource)
struct mount *mp;
int resource;
{
struct thread *td = curthread;
struct ufsmount *ump;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
/*
* We never hold up the filesystem syncer or buf daemon.
*/
if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF))
return (0);
/*
* First check to see if the work list has gotten backlogged.
* If it has, co-opt this process to help clean up two entries.
* Because this process may hold inodes locked, we cannot
* handle any remove requests that might block on a locked
* inode as that could lead to deadlock. We set TDP_SOFTDEP
* to avoid recursively processing the worklist.
*/
if (ump->softdep_on_worklist > max_softdeps / 10) {
td->td_pflags |= TDP_SOFTDEP;
process_worklist_item(mp, 2, LK_NOWAIT);
td->td_pflags &= ~TDP_SOFTDEP;
stat_worklist_push += 2;
return(1);
}
/*
* Next, we attempt to speed up the syncer process. If that
* is successful, then we allow the process to continue.
*/
if (softdep_speedup(ump) &&
resource != FLUSH_BLOCKS_WAIT &&
resource != FLUSH_INODES_WAIT)
return(0);
/*
* If we are resource constrained on inode dependencies, try
* flushing some dirty inodes. Otherwise, we are constrained
* by file deletions, so try accelerating flushes of directories
* with removal dependencies. We would like to do the cleanup
* here, but we probably hold an inode locked at this point and
* that might deadlock against one that we try to clean. So,
* the best that we can do is request the syncer daemon to do
* the cleanup for us.
*/
switch (resource) {
case FLUSH_INODES:
case FLUSH_INODES_WAIT:
ACQUIRE_GBLLOCK(&lk);
stat_ino_limit_push += 1;
req_clear_inodedeps += 1;
FREE_GBLLOCK(&lk);
stat_countp = &stat_ino_limit_hit;
break;
case FLUSH_BLOCKS:
case FLUSH_BLOCKS_WAIT:
ACQUIRE_GBLLOCK(&lk);
stat_blk_limit_push += 1;
req_clear_remove += 1;
FREE_GBLLOCK(&lk);
stat_countp = &stat_blk_limit_hit;
break;
default:
panic("request_cleanup: unknown type");
}
/*
* Hopefully the syncer daemon will catch up and awaken us.
* We wait at most tickdelay before proceeding in any case.
*/
ACQUIRE_GBLLOCK(&lk);
FREE_LOCK(ump);
proc_waiting += 1;
if (callout_pending(&softdep_callout) == FALSE)
callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2,
pause_timer, 0);
if ((td->td_pflags & TDP_KTHREAD) == 0)
msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0);
proc_waiting -= 1;
FREE_GBLLOCK(&lk);
ACQUIRE_LOCK(ump);
return (1);
}
/*
* Awaken processes pausing in request_cleanup and clear proc_waiting
* to indicate that there is no longer a timer running. Pause_timer
* will be called with the global softdep mutex (&lk) locked.
*/
static void
pause_timer(arg)
void *arg;
{
GBLLOCK_OWNED(&lk);
/*
* The callout_ API has acquired mtx and will hold it around this
* function call.
*/
*stat_countp += proc_waiting;
wakeup(&proc_waiting);
}
/*
* If requested, try removing inode or removal dependencies.
*/
static void
check_clear_deps(mp)
struct mount *mp;
{
struct ufsmount *ump;
bool suj_susp;
/*
* Tell the lower layers that any TRIM or WRITE transactions that have
* been delayed for performance reasons should proceed to help alleviate
* the shortage faster. The race between checking req_* and the softdep
* mutex (lk) is fine since this is an advisory operation that at most
* causes deferred work to be done sooner.
*/
ump = VFSTOUFS(mp);
suj_susp = MOUNTEDSUJ(mp) && ump->softdep_jblocks->jb_suspended;
if (req_clear_remove || req_clear_inodedeps || suj_susp) {
FREE_LOCK(ump);
softdep_send_speedup(ump, 0, BIO_SPEEDUP_TRIM | BIO_SPEEDUP_WRITE);
ACQUIRE_LOCK(ump);
}
/*
* If we are suspended, it may be because of our using
* too many inodedeps, so help clear them out.
*/
if (suj_susp)
clear_inodedeps(mp);
/*
* General requests for cleanup of backed up dependencies
*/
ACQUIRE_GBLLOCK(&lk);
if (req_clear_inodedeps) {
req_clear_inodedeps -= 1;
FREE_GBLLOCK(&lk);
clear_inodedeps(mp);
ACQUIRE_GBLLOCK(&lk);
wakeup(&proc_waiting);
}
if (req_clear_remove) {
req_clear_remove -= 1;
FREE_GBLLOCK(&lk);
clear_remove(mp);
ACQUIRE_GBLLOCK(&lk);
wakeup(&proc_waiting);
}
FREE_GBLLOCK(&lk);
}
/*
* Flush out a directory with at least one removal dependency in an effort to
* reduce the number of dirrem, freefile, and freeblks dependency structures.
*/
static void
clear_remove(mp)
struct mount *mp;
{
struct pagedep_hashhead *pagedephd;
struct pagedep *pagedep;
struct ufsmount *ump;
struct vnode *vp;
struct bufobj *bo;
int error, cnt;
ino_t ino;
ump = VFSTOUFS(mp);
LOCK_OWNED(ump);
for (cnt = 0; cnt <= ump->pagedep_hash_size; cnt++) {
pagedephd = &ump->pagedep_hashtbl[ump->pagedep_nextclean++];
if (ump->pagedep_nextclean > ump->pagedep_hash_size)
ump->pagedep_nextclean = 0;
LIST_FOREACH(pagedep, pagedephd, pd_hash) {
if (LIST_EMPTY(&pagedep->pd_dirremhd))
continue;
ino = pagedep->pd_ino;
if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
continue;
FREE_LOCK(ump);
/*
* Let unmount clear deps
*/
error = vfs_busy(mp, MBF_NOWAIT);
if (error != 0)
goto finish_write;
error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp,
FFSV_FORCEINSMQ);
vfs_unbusy(mp);
if (error != 0) {
softdep_error("clear_remove: vget", error);
goto finish_write;
}
MPASS(VTOI(vp)->i_mode != 0);
if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0)))
softdep_error("clear_remove: fsync", error);
bo = &vp->v_bufobj;
BO_LOCK(bo);
drain_output(vp);
BO_UNLOCK(bo);
vput(vp);
finish_write:
vn_finished_write(mp);
ACQUIRE_LOCK(ump);
return;
}
}
}
/*
* Clear out a block of dirty inodes in an effort to reduce
* the number of inodedep dependency structures.
*/
static void
clear_inodedeps(mp)
struct mount *mp;
{
struct inodedep_hashhead *inodedephd;
struct inodedep *inodedep;
struct ufsmount *ump;
struct vnode *vp;
struct fs *fs;
int error, cnt;
ino_t firstino, lastino, ino;
ump = VFSTOUFS(mp);
fs = ump->um_fs;
LOCK_OWNED(ump);
/*
* Pick a random inode dependency to be cleared.
* We will then gather up all the inodes in its block
* that have dependencies and flush them out.
*/
for (cnt = 0; cnt <= ump->inodedep_hash_size; cnt++) {
inodedephd = &ump->inodedep_hashtbl[ump->inodedep_nextclean++];
if (ump->inodedep_nextclean > ump->inodedep_hash_size)
ump->inodedep_nextclean = 0;
if ((inodedep = LIST_FIRST(inodedephd)) != NULL)
break;
}
if (inodedep == NULL)
return;
/*
* Find the last inode in the block with dependencies.
*/
firstino = rounddown2(inodedep->id_ino, INOPB(fs));
for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--)
if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0)
break;
/*
* Asynchronously push all but the last inode with dependencies.
* Synchronously push the last inode with dependencies to ensure
* that the inode block gets written to free up the inodedeps.
*/
for (ino = firstino; ino <= lastino; ino++) {
if (inodedep_lookup(mp, ino, 0, &inodedep) == 0)
continue;
if (vn_start_write(NULL, &mp, V_NOWAIT) != 0)
continue;
FREE_LOCK(ump);
error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */
if (error != 0) {
vn_finished_write(mp);
ACQUIRE_LOCK(ump);
return;
}
if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp,
FFSV_FORCEINSMQ)) != 0) {
softdep_error("clear_inodedeps: vget", error);
vfs_unbusy(mp);
vn_finished_write(mp);
ACQUIRE_LOCK(ump);
return;
}
vfs_unbusy(mp);
if (VTOI(vp)->i_mode == 0) {
vgone(vp);
} else if (ino == lastino) {
if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)))
softdep_error("clear_inodedeps: fsync1", error);
} else {
if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0)))
softdep_error("clear_inodedeps: fsync2", error);
BO_LOCK(&vp->v_bufobj);
drain_output(vp);
BO_UNLOCK(&vp->v_bufobj);
}
vput(vp);
vn_finished_write(mp);
ACQUIRE_LOCK(ump);
}
}
void
softdep_buf_append(bp, wkhd)
struct buf *bp;
struct workhead *wkhd;
{
struct worklist *wk;
struct ufsmount *ump;
if ((wk = LIST_FIRST(wkhd)) == NULL)
return;
KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0,
("softdep_buf_append called on non-softdep filesystem"));
ump = VFSTOUFS(wk->wk_mp);
ACQUIRE_LOCK(ump);
while ((wk = LIST_FIRST(wkhd)) != NULL) {
WORKLIST_REMOVE(wk);
WORKLIST_INSERT(&bp->b_dep, wk);
}
FREE_LOCK(ump);
}
void
softdep_inode_append(ip, cred, wkhd)
struct inode *ip;
struct ucred *cred;
struct workhead *wkhd;
{
struct buf *bp;
struct fs *fs;
struct ufsmount *ump;
int error;
ump = ITOUMP(ip);
KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0,
("softdep_inode_append called on non-softdep filesystem"));
fs = ump->um_fs;
error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
(int)fs->fs_bsize, cred, &bp);
if (error) {
bqrelse(bp);
softdep_freework(wkhd);
return;
}
softdep_buf_append(bp, wkhd);
bqrelse(bp);
}
void
softdep_freework(wkhd)
struct workhead *wkhd;
{
struct worklist *wk;
struct ufsmount *ump;
if ((wk = LIST_FIRST(wkhd)) == NULL)
return;
KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0,
("softdep_freework called on non-softdep filesystem"));
ump = VFSTOUFS(wk->wk_mp);
ACQUIRE_LOCK(ump);
handle_jwork(wkhd);
FREE_LOCK(ump);
}
static struct ufsmount *
softdep_bp_to_mp(bp)
struct buf *bp;
{
struct mount *mp;
struct vnode *vp;
if (LIST_EMPTY(&bp->b_dep))
return (NULL);
vp = bp->b_vp;
KASSERT(vp != NULL,
("%s, buffer with dependencies lacks vnode", __func__));
/*
* The ump mount point is stable after we get a correct
* pointer, since bp is locked and this prevents unmount from
* proceeding. But to get to it, we cannot dereference bp->b_dep
* head wk_mp, because we do not yet own SU ump lock and
* workitem might be freed while dereferenced.
*/
retry:
switch (vp->v_type) {
case VCHR:
VI_LOCK(vp);
mp = vp->v_type == VCHR ? vp->v_rdev->si_mountpt : NULL;
VI_UNLOCK(vp);
if (mp == NULL)
goto retry;
break;
case VREG:
case VDIR:
case VLNK:
case VFIFO:
case VSOCK:
mp = vp->v_mount;
break;
case VBLK:
vn_printf(vp, "softdep_bp_to_mp: unexpected block device\n");
/* FALLTHROUGH */
case VNON:
case VBAD:
case VMARKER:
mp = NULL;
break;
default:
vn_printf(vp, "unknown vnode type");
mp = NULL;
break;
}
return (VFSTOUFS(mp));
}
/*
* Function to determine if the buffer has outstanding dependencies
* that will cause a roll-back if the buffer is written. If wantcount
* is set, return number of dependencies, otherwise just yes or no.
*/
static int
softdep_count_dependencies(bp, wantcount)
struct buf *bp;
int wantcount;
{
struct worklist *wk;
struct ufsmount *ump;
struct bmsafemap *bmsafemap;
struct freework *freework;
struct inodedep *inodedep;
struct indirdep *indirdep;
struct freeblks *freeblks;
struct allocindir *aip;
struct pagedep *pagedep;
struct dirrem *dirrem;
struct newblk *newblk;
struct mkdir *mkdir;
struct diradd *dap;
int i, retval;
ump = softdep_bp_to_mp(bp);
if (ump == NULL)
return (0);
retval = 0;
ACQUIRE_LOCK(ump);
LIST_FOREACH(wk, &bp->b_dep, wk_list) {
switch (wk->wk_type) {
case D_INODEDEP:
inodedep = WK_INODEDEP(wk);
if ((inodedep->id_state & DEPCOMPLETE) == 0) {
/* bitmap allocation dependency */
retval += 1;
if (!wantcount)
goto out;
}
if (TAILQ_FIRST(&inodedep->id_inoupdt)) {
/* direct block pointer dependency */
retval += 1;
if (!wantcount)
goto out;
}
if (TAILQ_FIRST(&inodedep->id_extupdt)) {
/* direct block pointer dependency */
retval += 1;
if (!wantcount)
goto out;
}
if (TAILQ_FIRST(&inodedep->id_inoreflst)) {
/* Add reference dependency. */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_INDIRDEP:
indirdep = WK_INDIRDEP(wk);
TAILQ_FOREACH(freework, &indirdep->ir_trunc, fw_next) {
/* indirect truncation dependency */
retval += 1;
if (!wantcount)
goto out;
}
LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) {
/* indirect block pointer dependency */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_PAGEDEP:
pagedep = WK_PAGEDEP(wk);
LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) {
if (LIST_FIRST(&dirrem->dm_jremrefhd)) {
/* Journal remove ref dependency. */
retval += 1;
if (!wantcount)
goto out;
}
}
for (i = 0; i < DAHASHSZ; i++) {
LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) {
/* directory entry dependency */
retval += 1;
if (!wantcount)
goto out;
}
}
continue;
case D_BMSAFEMAP:
bmsafemap = WK_BMSAFEMAP(wk);
if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) {
/* Add reference dependency. */
retval += 1;
if (!wantcount)
goto out;
}
if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) {
/* Allocate block dependency. */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_FREEBLKS:
freeblks = WK_FREEBLKS(wk);
if (LIST_FIRST(&freeblks->fb_jblkdephd)) {
/* Freeblk journal dependency. */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_ALLOCDIRECT:
case D_ALLOCINDIR:
newblk = WK_NEWBLK(wk);
if (newblk->nb_jnewblk) {
/* Journal allocate dependency. */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_MKDIR:
mkdir = WK_MKDIR(wk);
if (mkdir->md_jaddref) {
/* Journal reference dependency. */
retval += 1;
if (!wantcount)
goto out;
}
continue;
case D_FREEWORK:
case D_FREEDEP:
case D_JSEGDEP:
case D_JSEG:
case D_SBDEP:
/* never a dependency on these blocks */
continue;
default:
panic("softdep_count_dependencies: Unexpected type %s",
TYPENAME(wk->wk_type));
/* NOTREACHED */
}
}
out:
FREE_LOCK(ump);
return (retval);
}
/*
* Acquire exclusive access to a buffer.
* Must be called with a locked mtx parameter.
* Return acquired buffer or NULL on failure.
*/
static struct buf *
getdirtybuf(bp, lock, waitfor)
struct buf *bp;
struct rwlock *lock;
int waitfor;
{
int error;
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) {
if (waitfor != MNT_WAIT)
return (NULL);
error = BUF_LOCK(bp,
LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, lock);
/*
* Even if we successfully acquire bp here, we have dropped
* lock, which may violates our guarantee.
*/
if (error == 0)
BUF_UNLOCK(bp);
else if (error != ENOLCK)
panic("getdirtybuf: inconsistent lock: %d", error);
rw_wlock(lock);
return (NULL);
}
if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
if (lock != BO_LOCKPTR(bp->b_bufobj) && waitfor == MNT_WAIT) {
rw_wunlock(lock);
BO_LOCK(bp->b_bufobj);
BUF_UNLOCK(bp);
if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
bp->b_vflags |= BV_BKGRDWAIT;
msleep(&bp->b_xflags, BO_LOCKPTR(bp->b_bufobj),
PRIBIO | PDROP, "getbuf", 0);
} else
BO_UNLOCK(bp->b_bufobj);
rw_wlock(lock);
return (NULL);
}
BUF_UNLOCK(bp);
if (waitfor != MNT_WAIT)
return (NULL);
#ifdef DEBUG_VFS_LOCKS
if (bp->b_vp->v_type != VCHR)
ASSERT_BO_WLOCKED(bp->b_bufobj);
#endif
bp->b_vflags |= BV_BKGRDWAIT;
rw_sleep(&bp->b_xflags, lock, PRIBIO, "getbuf", 0);
return (NULL);
}
if ((bp->b_flags & B_DELWRI) == 0) {
BUF_UNLOCK(bp);
return (NULL);
}
bremfree(bp);
return (bp);
}
/*
* Check if it is safe to suspend the file system now. On entry,
* the vnode interlock for devvp should be held. Return 0 with
* the mount interlock held if the file system can be suspended now,
* otherwise return EAGAIN with the mount interlock held.
*/
int
softdep_check_suspend(struct mount *mp,
struct vnode *devvp,
int softdep_depcnt,
int softdep_accdepcnt,
int secondary_writes,
int secondary_accwrites)
{
struct bufobj *bo;
struct ufsmount *ump;
struct inodedep *inodedep;
int error, unlinked;
bo = &devvp->v_bufobj;
ASSERT_BO_WLOCKED(bo);
/*
* If we are not running with soft updates, then we need only
* deal with secondary writes as we try to suspend.
*/
if (MOUNTEDSOFTDEP(mp) == 0) {
MNT_ILOCK(mp);
while (mp->mnt_secondary_writes != 0) {
BO_UNLOCK(bo);
msleep(&mp->mnt_secondary_writes, MNT_MTX(mp),
(PUSER - 1) | PDROP, "secwr", 0);
BO_LOCK(bo);
MNT_ILOCK(mp);
}
/*
* Reasons for needing more work before suspend:
* - Dirty buffers on devvp.
* - Secondary writes occurred after start of vnode sync loop
*/
error = 0;
if (bo->bo_numoutput > 0 ||
bo->bo_dirty.bv_cnt > 0 ||
secondary_writes != 0 ||
mp->mnt_secondary_writes != 0 ||
secondary_accwrites != mp->mnt_secondary_accwrites)
error = EAGAIN;
BO_UNLOCK(bo);
return (error);
}
/*
* If we are running with soft updates, then we need to coordinate
* with them as we try to suspend.
*/
ump = VFSTOUFS(mp);
for (;;) {
if (!TRY_ACQUIRE_LOCK(ump)) {
BO_UNLOCK(bo);
ACQUIRE_LOCK(ump);
FREE_LOCK(ump);
BO_LOCK(bo);
continue;
}
MNT_ILOCK(mp);
if (mp->mnt_secondary_writes != 0) {
FREE_LOCK(ump);
BO_UNLOCK(bo);
msleep(&mp->mnt_secondary_writes,
MNT_MTX(mp),
(PUSER - 1) | PDROP, "secwr", 0);
BO_LOCK(bo);
continue;
}
break;
}
unlinked = 0;
if (MOUNTEDSUJ(mp)) {
for (inodedep = TAILQ_FIRST(&ump->softdep_unlinked);
inodedep != NULL;
inodedep = TAILQ_NEXT(inodedep, id_unlinked)) {
if ((inodedep->id_state & (UNLINKED | UNLINKLINKS |
UNLINKONLIST)) != (UNLINKED | UNLINKLINKS |
UNLINKONLIST) ||
!check_inodedep_free(inodedep))
continue;
unlinked++;
}
}
/*
* Reasons for needing more work before suspend:
* - Dirty buffers on devvp.
* - Softdep activity occurred after start of vnode sync loop
* - Secondary writes occurred after start of vnode sync loop
*/
error = 0;
if (bo->bo_numoutput > 0 ||
bo->bo_dirty.bv_cnt > 0 ||
softdep_depcnt != unlinked ||
ump->softdep_deps != unlinked ||
softdep_accdepcnt != ump->softdep_accdeps ||
secondary_writes != 0 ||
mp->mnt_secondary_writes != 0 ||
secondary_accwrites != mp->mnt_secondary_accwrites)
error = EAGAIN;
FREE_LOCK(ump);
BO_UNLOCK(bo);
return (error);
}
/*
* Get the number of dependency structures for the file system, both
* the current number and the total number allocated. These will
* later be used to detect that softdep processing has occurred.
*/
void
softdep_get_depcounts(struct mount *mp,
int *softdep_depsp,
int *softdep_accdepsp)
{
struct ufsmount *ump;
if (MOUNTEDSOFTDEP(mp) == 0) {
*softdep_depsp = 0;
*softdep_accdepsp = 0;
return;
}
ump = VFSTOUFS(mp);
ACQUIRE_LOCK(ump);
*softdep_depsp = ump->softdep_deps;
*softdep_accdepsp = ump->softdep_accdeps;
FREE_LOCK(ump);
}
/*
* Wait for pending output on a vnode to complete.
*/
static void
drain_output(vp)
struct vnode *vp;
{
ASSERT_VOP_LOCKED(vp, "drain_output");
(void)bufobj_wwait(&vp->v_bufobj, 0, 0);
}
/*
* Called whenever a buffer that is being invalidated or reallocated
* contains dependencies. This should only happen if an I/O error has
* occurred. The routine is called with the buffer locked.
*/
static void
softdep_deallocate_dependencies(bp)
struct buf *bp;
{
if ((bp->b_ioflags & BIO_ERROR) == 0)
panic("softdep_deallocate_dependencies: dangling deps");
if (bp->b_vp != NULL && bp->b_vp->v_mount != NULL)
softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error);
else
printf("softdep_deallocate_dependencies: "
"got error %d while accessing filesystem\n", bp->b_error);
if (bp->b_error != ENXIO)
panic("softdep_deallocate_dependencies: unrecovered I/O error");
}
/*
* Function to handle asynchronous write errors in the filesystem.
*/
static void
softdep_error(func, error)
char *func;
int error;
{
/* XXX should do something better! */
printf("%s: got error %d while accessing filesystem\n", func, error);
}
#ifdef DDB
/* exported to ffs_vfsops.c */
extern void db_print_ffs(struct ufsmount *ump);
void
db_print_ffs(struct ufsmount *ump)
{
db_printf("mp %p (%s) devvp %p\n", ump->um_mountp,
ump->um_mountp->mnt_stat.f_mntonname, ump->um_devvp);
db_printf(" fs %p su_wl %d su_deps %d su_req %d\n",
ump->um_fs, ump->softdep_on_worklist,
ump->softdep_deps, ump->softdep_req);
}
static void
worklist_print(struct worklist *wk, int verbose)
{
if (!verbose) {
db_printf("%s: %p state 0x%b\n", TYPENAME(wk->wk_type), wk,
(u_int)wk->wk_state, PRINT_SOFTDEP_FLAGS);
return;
}
db_printf("worklist: %p type %s state 0x%b next %p\n ", wk,
TYPENAME(wk->wk_type), (u_int)wk->wk_state, PRINT_SOFTDEP_FLAGS,
LIST_NEXT(wk, wk_list));
db_print_ffs(VFSTOUFS(wk->wk_mp));
}
static void
inodedep_print(struct inodedep *inodedep, int verbose)
{
worklist_print(&inodedep->id_list, 0);
db_printf(" fs %p ino %jd inoblk %jd delta %jd nlink %jd\n",
inodedep->id_fs,
(intmax_t)inodedep->id_ino,
(intmax_t)fsbtodb(inodedep->id_fs,
ino_to_fsba(inodedep->id_fs, inodedep->id_ino)),
(intmax_t)inodedep->id_nlinkdelta,
(intmax_t)inodedep->id_savednlink);
if (verbose == 0)
return;
db_printf(" bmsafemap %p, mkdiradd %p, inoreflst %p\n",
inodedep->id_bmsafemap,
inodedep->id_mkdiradd,
TAILQ_FIRST(&inodedep->id_inoreflst));
db_printf(" dirremhd %p, pendinghd %p, bufwait %p\n",
LIST_FIRST(&inodedep->id_dirremhd),
LIST_FIRST(&inodedep->id_pendinghd),
LIST_FIRST(&inodedep->id_bufwait));
db_printf(" inowait %p, inoupdt %p, newinoupdt %p\n",
LIST_FIRST(&inodedep->id_inowait),
TAILQ_FIRST(&inodedep->id_inoupdt),
TAILQ_FIRST(&inodedep->id_newinoupdt));
db_printf(" extupdt %p, newextupdt %p, freeblklst %p\n",
TAILQ_FIRST(&inodedep->id_extupdt),
TAILQ_FIRST(&inodedep->id_newextupdt),
TAILQ_FIRST(&inodedep->id_freeblklst));
db_printf(" saveino %p, savedsize %jd, savedextsize %jd\n",
inodedep->id_savedino1,
(intmax_t)inodedep->id_savedsize,
(intmax_t)inodedep->id_savedextsize);
}
static void
newblk_print(struct newblk *nbp)
{
worklist_print(&nbp->nb_list, 0);
db_printf(" newblkno %jd\n", (intmax_t)nbp->nb_newblkno);
db_printf(" jnewblk %p, bmsafemap %p, freefrag %p\n",
&nbp->nb_jnewblk,
&nbp->nb_bmsafemap,
&nbp->nb_freefrag);
db_printf(" indirdeps %p, newdirblk %p, jwork %p\n",
LIST_FIRST(&nbp->nb_indirdeps),
LIST_FIRST(&nbp->nb_newdirblk),
LIST_FIRST(&nbp->nb_jwork));
}
static void
allocdirect_print(struct allocdirect *adp)
{
newblk_print(&adp->ad_block);
db_printf(" oldblkno %jd, oldsize %ld, newsize %ld\n",
adp->ad_oldblkno, adp->ad_oldsize, adp->ad_newsize);
db_printf(" offset %d, inodedep %p\n",
adp->ad_offset, adp->ad_inodedep);
}
static void
allocindir_print(struct allocindir *aip)
{
newblk_print(&aip->ai_block);
db_printf(" oldblkno %jd, lbn %jd\n",
(intmax_t)aip->ai_oldblkno, (intmax_t)aip->ai_lbn);
db_printf(" offset %d, indirdep %p\n",
aip->ai_offset, aip->ai_indirdep);
}
static void
mkdir_print(struct mkdir *mkdir)
{
worklist_print(&mkdir->md_list, 0);
db_printf(" diradd %p, jaddref %p, buf %p\n",
mkdir->md_diradd, mkdir->md_jaddref, mkdir->md_buf);
}
DB_SHOW_COMMAND(sd_inodedep, db_show_sd_inodedep)
{
if (have_addr == 0) {
db_printf("inodedep address required\n");
return;
}
inodedep_print((struct inodedep*)addr, 1);
}
DB_SHOW_COMMAND(sd_allinodedeps, db_show_sd_allinodedeps)
{
struct inodedep_hashhead *inodedephd;
struct inodedep *inodedep;
struct ufsmount *ump;
int cnt;
if (have_addr == 0) {
db_printf("ufsmount address required\n");
return;
}
ump = (struct ufsmount *)addr;
for (cnt = 0; cnt < ump->inodedep_hash_size; cnt++) {
inodedephd = &ump->inodedep_hashtbl[cnt];
LIST_FOREACH(inodedep, inodedephd, id_hash) {
inodedep_print(inodedep, 0);
}
}
}
DB_SHOW_COMMAND(sd_worklist, db_show_sd_worklist)
{
if (have_addr == 0) {
db_printf("worklist address required\n");
return;
}
worklist_print((struct worklist *)addr, 1);
}
DB_SHOW_COMMAND(sd_workhead, db_show_sd_workhead)
{
struct worklist *wk;
struct workhead *wkhd;
if (have_addr == 0) {
db_printf("worklist address required "
"(for example value in bp->b_dep)\n");
return;
}
/*
* We often do not have the address of the worklist head but
* instead a pointer to its first entry (e.g., we have the
* contents of bp->b_dep rather than &bp->b_dep). But the back
* pointer of bp->b_dep will point at the head of the list, so
* we cheat and use that instead. If we are in the middle of
* a list we will still get the same result, so nothing
* unexpected will result.
*/
wk = (struct worklist *)addr;
if (wk == NULL)
return;
wkhd = (struct workhead *)wk->wk_list.le_prev;
LIST_FOREACH(wk, wkhd, wk_list) {
switch(wk->wk_type) {
case D_INODEDEP:
inodedep_print(WK_INODEDEP(wk), 0);
continue;
case D_ALLOCDIRECT:
allocdirect_print(WK_ALLOCDIRECT(wk));
continue;
case D_ALLOCINDIR:
allocindir_print(WK_ALLOCINDIR(wk));
continue;
case D_MKDIR:
mkdir_print(WK_MKDIR(wk));
continue;
default:
worklist_print(wk, 0);
continue;
}
}
}
DB_SHOW_COMMAND(sd_mkdir, db_show_sd_mkdir)
{
if (have_addr == 0) {
db_printf("mkdir address required\n");
return;
}
mkdir_print((struct mkdir *)addr);
}
DB_SHOW_COMMAND(sd_mkdir_list, db_show_sd_mkdir_list)
{
struct mkdirlist *mkdirlisthd;
struct mkdir *mkdir;
if (have_addr == 0) {
db_printf("mkdir listhead address required\n");
return;
}
mkdirlisthd = (struct mkdirlist *)addr;
LIST_FOREACH(mkdir, mkdirlisthd, md_mkdirs) {
mkdir_print(mkdir);
if (mkdir->md_diradd != NULL) {
db_printf(" ");
worklist_print(&mkdir->md_diradd->da_list, 0);
}
if (mkdir->md_jaddref != NULL) {
db_printf(" ");
worklist_print(&mkdir->md_jaddref->ja_list, 0);
}
}
}
DB_SHOW_COMMAND(sd_allocdirect, db_show_sd_allocdirect)
{
if (have_addr == 0) {
db_printf("allocdirect address required\n");
return;
}
allocdirect_print((struct allocdirect *)addr);
}
DB_SHOW_COMMAND(sd_allocindir, db_show_sd_allocindir)
{
if (have_addr == 0) {
db_printf("allocindir address required\n");
return;
}
allocindir_print((struct allocindir *)addr);
}
#endif /* DDB */
#endif /* SOFTUPDATES */
Index: head/sys/ufs/ffs/ffs_subr.c
===================================================================
--- head/sys/ufs/ffs/ffs_subr.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_subr.c (revision 361491)
@@ -1,739 +1,828 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_subr.c 8.5 (Berkeley) 3/21/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#ifndef _KERNEL
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <sys/errno.h>
#include <ufs/ufs/dinode.h>
#include <ufs/ffs/fs.h>
uint32_t calculate_crc32c(uint32_t, const void *, size_t);
uint32_t ffs_calc_sbhash(struct fs *);
struct malloc_type;
#define UFS_MALLOC(size, type, flags) malloc(size)
#define UFS_FREE(ptr, type) free(ptr)
#define UFS_TIME time(NULL)
/*
* Request standard superblock location in ffs_sbget
*/
#define STDSB -1 /* Fail if check-hash is bad */
#define STDSB_NOHASHFAIL -2 /* Ignore check-hash failure */
#else /* _KERNEL */
#include <sys/systm.h>
#include <sys/gsb_crc32.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/vnode.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/ucred.h>
+#include <sys/taskqueue.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ffs/ffs_extern.h>
#include <ufs/ffs/fs.h>
#define UFS_MALLOC(size, type, flags) malloc(size, type, flags)
#define UFS_FREE(ptr, type) free(ptr, type)
#define UFS_TIME time_second
/*
* Return buffer with the contents of block "offset" from the beginning of
* directory "ip". If "res" is non-zero, fill it in with a pointer to the
* remaining space in the directory.
*/
int
ffs_blkatoff(struct vnode *vp, off_t offset, char **res, struct buf **bpp)
{
struct inode *ip;
struct fs *fs;
struct buf *bp;
ufs_lbn_t lbn;
int bsize, error;
ip = VTOI(vp);
fs = ITOFS(ip);
lbn = lblkno(fs, offset);
bsize = blksize(fs, ip, lbn);
*bpp = NULL;
error = bread(vp, lbn, bsize, NOCRED, &bp);
if (error) {
return (error);
}
if (res)
*res = (char *)bp->b_data + blkoff(fs, offset);
*bpp = bp;
return (0);
}
/*
* Load up the contents of an inode and copy the appropriate pieces
* to the incore copy.
*/
int
ffs_load_inode(struct buf *bp, struct inode *ip, struct fs *fs, ino_t ino)
{
struct ufs1_dinode *dip1;
struct ufs2_dinode *dip2;
int error;
if (I_IS_UFS1(ip)) {
dip1 = ip->i_din1;
*dip1 =
*((struct ufs1_dinode *)bp->b_data + ino_to_fsbo(fs, ino));
ip->i_mode = dip1->di_mode;
ip->i_nlink = dip1->di_nlink;
ip->i_effnlink = dip1->di_nlink;
ip->i_size = dip1->di_size;
ip->i_flags = dip1->di_flags;
ip->i_gen = dip1->di_gen;
ip->i_uid = dip1->di_uid;
ip->i_gid = dip1->di_gid;
return (0);
}
dip2 = ((struct ufs2_dinode *)bp->b_data + ino_to_fsbo(fs, ino));
- if ((error = ffs_verify_dinode_ckhash(fs, dip2)) != 0) {
+ if ((error = ffs_verify_dinode_ckhash(fs, dip2)) != 0 &&
+ !ffs_fsfail_cleanup(ITOUMP(ip), error)) {
printf("%s: inode %jd: check-hash failed\n", fs->fs_fsmnt,
(intmax_t)ino);
return (error);
}
*ip->i_din2 = *dip2;
dip2 = ip->i_din2;
ip->i_mode = dip2->di_mode;
ip->i_nlink = dip2->di_nlink;
ip->i_effnlink = dip2->di_nlink;
ip->i_size = dip2->di_size;
ip->i_flags = dip2->di_flags;
ip->i_gen = dip2->di_gen;
ip->i_uid = dip2->di_uid;
ip->i_gid = dip2->di_gid;
return (0);
}
/*
* Verify that a filesystem block number is a valid data block.
* This routine is only called on untrusted filesystems.
*/
int
ffs_check_blkno(struct mount *mp, ino_t inum, ufs2_daddr_t daddr, int blksize)
{
struct fs *fs;
struct ufsmount *ump;
ufs2_daddr_t end_daddr;
int cg, havemtx;
KASSERT((mp->mnt_flag & MNT_UNTRUSTED) != 0,
("ffs_check_blkno called on a trusted file system"));
ump = VFSTOUFS(mp);
fs = ump->um_fs;
cg = dtog(fs, daddr);
end_daddr = daddr + numfrags(fs, blksize);
/*
* Verify that the block number is a valid data block. Also check
* that it does not point to an inode block or a superblock. Accept
* blocks that are unalloacted (0) or part of snapshot metadata
* (BLK_NOCOPY or BLK_SNAP).
*
* Thus, the block must be in a valid range for the filesystem and
* either in the space before a backup superblock (except the first
* cylinder group where that space is used by the bootstrap code) or
* after the inode blocks and before the end of the cylinder group.
*/
if ((uint64_t)daddr <= BLK_SNAP ||
((uint64_t)end_daddr <= fs->fs_size &&
((cg > 0 && end_daddr <= cgsblock(fs, cg)) ||
(daddr >= cgdmin(fs, cg) &&
end_daddr <= cgbase(fs, cg) + fs->fs_fpg))))
return (0);
if ((havemtx = mtx_owned(UFS_MTX(ump))) == 0)
UFS_LOCK(ump);
if (ppsratecheck(&ump->um_last_integritymsg,
&ump->um_secs_integritymsg, 1)) {
UFS_UNLOCK(ump);
uprintf("\n%s: inode %jd, out-of-range indirect block "
"number %jd\n", mp->mnt_stat.f_mntonname, inum, daddr);
if (havemtx)
UFS_LOCK(ump);
} else if (!havemtx)
UFS_UNLOCK(ump);
return (EINTEGRITY);
+}
+
+/*
+ * Initiate a forcible unmount.
+ * Used to unmount filesystems whose underlying media has gone away.
+ */
+static void
+ffs_fsfail_unmount(void *v, int pending)
+{
+ struct fsfail_task *etp;
+ struct mount *mp;
+
+ etp = v;
+
+ /*
+ * Find our mount and get a ref on it, then try to unmount.
+ */
+ mp = vfs_getvfs(&etp->fsid);
+ if (mp != NULL)
+ dounmount(mp, MNT_FORCE, curthread);
+ free(etp, M_UFSMNT);
+}
+
+/*
+ * On first ENXIO error, start a task that forcibly unmounts the filesystem.
+ *
+ * Return true if a cleanup is in progress.
+ */
+int
+ffs_fsfail_cleanup(struct ufsmount *ump, int error)
+{
+ int retval;
+
+ UFS_LOCK(ump);
+ retval = ffs_fsfail_cleanup_locked(ump, error);
+ UFS_UNLOCK(ump);
+ return (retval);
+}
+
+int
+ffs_fsfail_cleanup_locked(struct ufsmount *ump, int error)
+{
+ struct fsfail_task *etp;
+ struct task *tp;
+
+ mtx_assert(UFS_MTX(ump), MA_OWNED);
+ if (error == ENXIO && (ump->um_flags & UM_FSFAIL_CLEANUP) == 0) {
+ ump->um_flags |= UM_FSFAIL_CLEANUP;
+ /*
+ * Queue an async forced unmount.
+ */
+ etp = ump->um_fsfail_task;
+ ump->um_fsfail_task = NULL;
+ if (etp != NULL) {
+ tp = &etp->task;
+ TASK_INIT(tp, 0, ffs_fsfail_unmount, etp);
+ taskqueue_enqueue(taskqueue_thread, tp);
+ printf("UFS: forcibly unmounting %s from %s\n",
+ ump->um_mountp->mnt_stat.f_mntfromname,
+ ump->um_mountp->mnt_stat.f_mntonname);
+ }
+ }
+ return ((ump->um_flags & UM_FSFAIL_CLEANUP) != 0);
+}
+
+/*
+ * Wrapper used during ENXIO cleanup to allocate empty buffers when
+ * the kernel is unable to read the real one. They are needed so that
+ * the soft updates code can use them to unwind its dependencies.
+ */
+int
+ffs_breadz(struct ufsmount *ump, struct vnode *vp, daddr_t lblkno,
+ daddr_t dblkno, int size, daddr_t *rablkno, int *rabsize, int cnt,
+ struct ucred *cred, int flags, void (*ckhashfunc)(struct buf *),
+ struct buf **bpp)
+{
+ int error;
+
+ flags |= GB_CVTENXIO;
+ error = breadn_flags(vp, lblkno, dblkno, size, rablkno, rabsize, cnt,
+ cred, flags, ckhashfunc, bpp);
+ if (error != 0 && ffs_fsfail_cleanup(ump, error)) {
+ error = getblkx(vp, lblkno, dblkno, size, 0, 0, flags, bpp);
+ KASSERT(error == 0, ("getblkx failed"));
+ vfs_bio_bzero_buf(*bpp, 0, size);
+ }
+ return (error);
}
#endif /* _KERNEL */
/*
* Verify an inode check-hash.
*/
int
ffs_verify_dinode_ckhash(struct fs *fs, struct ufs2_dinode *dip)
{
uint32_t ckhash, save_ckhash;
/*
* Return success if unallocated or we are not doing inode check-hash.
*/
if (dip->di_mode == 0 || (fs->fs_metackhash & CK_INODE) == 0)
return (0);
/*
* Exclude di_ckhash from the crc32 calculation, e.g., always use
* a check-hash value of zero when calculating the check-hash.
*/
save_ckhash = dip->di_ckhash;
dip->di_ckhash = 0;
ckhash = calculate_crc32c(~0L, (void *)dip, sizeof(*dip));
dip->di_ckhash = save_ckhash;
if (save_ckhash == ckhash)
return (0);
return (EINVAL);
}
/*
* Update an inode check-hash.
*/
void
ffs_update_dinode_ckhash(struct fs *fs, struct ufs2_dinode *dip)
{
if (dip->di_mode == 0 || (fs->fs_metackhash & CK_INODE) == 0)
return;
/*
* Exclude old di_ckhash from the crc32 calculation, e.g., always use
* a check-hash value of zero when calculating the new check-hash.
*/
dip->di_ckhash = 0;
dip->di_ckhash = calculate_crc32c(~0L, (void *)dip, sizeof(*dip));
}
/*
* These are the low-level functions that actually read and write
* the superblock and its associated data.
*/
static off_t sblock_try[] = SBLOCKSEARCH;
static int readsuper(void *, struct fs **, off_t, int, int,
int (*)(void *, off_t, void **, int));
/*
* Read a superblock from the devfd device.
*
* If an alternate superblock is specified, it is read. Otherwise the
* set of locations given in the SBLOCKSEARCH list is searched for a
* superblock. Memory is allocated for the superblock by the readfunc and
* is returned. If filltype is non-NULL, additional memory is allocated
* of type filltype and filled in with the superblock summary information.
* All memory is freed when any error is returned.
*
* If a superblock is found, zero is returned. Otherwise one of the
* following error values is returned:
* EIO: non-existent or truncated superblock.
* EIO: error reading summary information.
* ENOENT: no usable known superblock found.
* ENOSPC: failed to allocate space for the superblock.
* EINVAL: The previous newfs operation on this volume did not complete.
* The administrator must complete newfs before using this volume.
*/
int
ffs_sbget(void *devfd, struct fs **fsp, off_t altsblock,
struct malloc_type *filltype,
int (*readfunc)(void *devfd, off_t loc, void **bufp, int size))
{
struct fs *fs;
int i, error, size, blks;
uint8_t *space;
int32_t *lp;
int chkhash;
char *buf;
fs = NULL;
*fsp = NULL;
chkhash = 1;
if (altsblock >= 0) {
if ((error = readsuper(devfd, &fs, altsblock, 1, chkhash,
readfunc)) != 0) {
if (fs != NULL)
UFS_FREE(fs, filltype);
return (error);
}
} else {
if (altsblock == STDSB_NOHASHFAIL)
chkhash = 0;
for (i = 0; sblock_try[i] != -1; i++) {
if ((error = readsuper(devfd, &fs, sblock_try[i], 0,
chkhash, readfunc)) == 0)
break;
if (fs != NULL) {
UFS_FREE(fs, filltype);
fs = NULL;
}
if (error == ENOENT)
continue;
return (error);
}
if (sblock_try[i] == -1)
return (ENOENT);
}
/*
* Read in the superblock summary information.
*/
size = fs->fs_cssize;
blks = howmany(size, fs->fs_fsize);
if (fs->fs_contigsumsize > 0)
size += fs->fs_ncg * sizeof(int32_t);
size += fs->fs_ncg * sizeof(u_int8_t);
/* When running in libufs or libsa, UFS_MALLOC may fail */
if ((space = UFS_MALLOC(size, filltype, M_WAITOK)) == NULL) {
UFS_FREE(fs, filltype);
return (ENOSPC);
}
fs->fs_csp = (struct csum *)space;
for (i = 0; i < blks; i += fs->fs_frag) {
size = fs->fs_bsize;
if (i + fs->fs_frag > blks)
size = (blks - i) * fs->fs_fsize;
buf = NULL;
error = (*readfunc)(devfd,
dbtob(fsbtodb(fs, fs->fs_csaddr + i)), (void **)&buf, size);
if (error) {
if (buf != NULL)
UFS_FREE(buf, filltype);
UFS_FREE(fs->fs_csp, filltype);
UFS_FREE(fs, filltype);
return (error);
}
memcpy(space, buf, size);
UFS_FREE(buf, filltype);
space += size;
}
if (fs->fs_contigsumsize > 0) {
fs->fs_maxcluster = lp = (int32_t *)space;
for (i = 0; i < fs->fs_ncg; i++)
*lp++ = fs->fs_contigsumsize;
space = (uint8_t *)lp;
}
size = fs->fs_ncg * sizeof(u_int8_t);
fs->fs_contigdirs = (u_int8_t *)space;
bzero(fs->fs_contigdirs, size);
*fsp = fs;
return (0);
}
/*
* Try to read a superblock from the location specified by sblockloc.
* Return zero on success or an errno on failure.
*/
static int
readsuper(void *devfd, struct fs **fsp, off_t sblockloc, int isaltsblk,
int chkhash, int (*readfunc)(void *devfd, off_t loc, void **bufp, int size))
{
struct fs *fs;
int error, res;
uint32_t ckhash;
error = (*readfunc)(devfd, sblockloc, (void **)fsp, SBLOCKSIZE);
if (error != 0)
return (error);
fs = *fsp;
if (fs->fs_magic == FS_BAD_MAGIC)
return (EINVAL);
if (((fs->fs_magic == FS_UFS1_MAGIC && (isaltsblk ||
sblockloc <= SBLOCK_UFS1)) ||
(fs->fs_magic == FS_UFS2_MAGIC && (isaltsblk ||
sblockloc == fs->fs_sblockloc))) &&
fs->fs_ncg >= 1 &&
fs->fs_bsize >= MINBSIZE &&
fs->fs_bsize <= MAXBSIZE &&
fs->fs_bsize >= roundup(sizeof(struct fs), DEV_BSIZE) &&
fs->fs_sbsize <= SBLOCKSIZE) {
/*
* If the filesystem has been run on a kernel without
* metadata check hashes, disable them.
*/
if ((fs->fs_flags & FS_METACKHASH) == 0)
fs->fs_metackhash = 0;
if (fs->fs_ckhash != (ckhash = ffs_calc_sbhash(fs))) {
#ifdef _KERNEL
res = uprintf("Superblock check-hash failed: recorded "
"check-hash 0x%x != computed check-hash 0x%x%s\n",
fs->fs_ckhash, ckhash,
chkhash == 0 ? " (Ignored)" : "");
#else
res = 0;
#endif
/*
* Print check-hash failure if no controlling terminal
* in kernel or always if in user-mode (libufs).
*/
if (res == 0)
printf("Superblock check-hash failed: recorded "
"check-hash 0x%x != computed check-hash "
"0x%x%s\n", fs->fs_ckhash, ckhash,
chkhash == 0 ? " (Ignored)" : "");
if (chkhash == 0) {
fs->fs_flags |= FS_NEEDSFSCK;
fs->fs_fmod = 1;
return (0);
}
fs->fs_fmod = 0;
return (EINTEGRITY);
}
/* Have to set for old filesystems that predate this field */
fs->fs_sblockactualloc = sblockloc;
/* Not yet any summary information */
fs->fs_csp = NULL;
return (0);
}
return (ENOENT);
}
/*
* Write a superblock to the devfd device from the memory pointed to by fs.
* Write out the superblock summary information if it is present.
*
* If the write is successful, zero is returned. Otherwise one of the
* following error values is returned:
* EIO: failed to write superblock.
* EIO: failed to write superblock summary information.
*/
int
ffs_sbput(void *devfd, struct fs *fs, off_t loc,
int (*writefunc)(void *devfd, off_t loc, void *buf, int size))
{
int i, error, blks, size;
uint8_t *space;
/*
* If there is summary information, write it first, so if there
* is an error, the superblock will not be marked as clean.
*/
if (fs->fs_csp != NULL) {
blks = howmany(fs->fs_cssize, fs->fs_fsize);
space = (uint8_t *)fs->fs_csp;
for (i = 0; i < blks; i += fs->fs_frag) {
size = fs->fs_bsize;
if (i + fs->fs_frag > blks)
size = (blks - i) * fs->fs_fsize;
if ((error = (*writefunc)(devfd,
dbtob(fsbtodb(fs, fs->fs_csaddr + i)),
space, size)) != 0)
return (error);
space += size;
}
}
fs->fs_fmod = 0;
fs->fs_time = UFS_TIME;
fs->fs_ckhash = ffs_calc_sbhash(fs);
if ((error = (*writefunc)(devfd, loc, fs, fs->fs_sbsize)) != 0)
return (error);
return (0);
}
/*
* Calculate the check-hash for a superblock.
*/
uint32_t
ffs_calc_sbhash(struct fs *fs)
{
uint32_t ckhash, save_ckhash;
/*
* A filesystem that was using a superblock ckhash may be moved
* to an older kernel that does not support ckhashes. The
* older kernel will clear the FS_METACKHASH flag indicating
* that it does not update hashes. When the disk is moved back
* to a kernel capable of ckhashes it disables them on mount:
*
* if ((fs->fs_flags & FS_METACKHASH) == 0)
* fs->fs_metackhash = 0;
*
* This leaves (fs->fs_metackhash & CK_SUPERBLOCK) == 0) with an
* old stale value in the fs->fs_ckhash field. Thus the need to
* just accept what is there.
*/
if ((fs->fs_metackhash & CK_SUPERBLOCK) == 0)
return (fs->fs_ckhash);
save_ckhash = fs->fs_ckhash;
fs->fs_ckhash = 0;
/*
* If newly read from disk, the caller is responsible for
* verifying that fs->fs_sbsize <= SBLOCKSIZE.
*/
ckhash = calculate_crc32c(~0L, (void *)fs, fs->fs_sbsize);
fs->fs_ckhash = save_ckhash;
return (ckhash);
}
/*
* Update the frsum fields to reflect addition or deletion
* of some frags.
*/
void
ffs_fragacct(struct fs *fs, int fragmap, int32_t fraglist[], int cnt)
{
int inblk;
int field, subfield;
int siz, pos;
inblk = (int)(fragtbl[fs->fs_frag][fragmap]) << 1;
fragmap <<= 1;
for (siz = 1; siz < fs->fs_frag; siz++) {
if ((inblk & (1 << (siz + (fs->fs_frag % NBBY)))) == 0)
continue;
field = around[siz];
subfield = inside[siz];
for (pos = siz; pos <= fs->fs_frag; pos++) {
if ((fragmap & field) == subfield) {
fraglist[siz] += cnt;
pos += siz;
field <<= siz;
subfield <<= siz;
}
field <<= 1;
subfield <<= 1;
}
}
}
/*
* block operations
*
* check if a block is available
*/
int
ffs_isblock(struct fs *fs, unsigned char *cp, ufs1_daddr_t h)
{
unsigned char mask;
switch ((int)fs->fs_frag) {
case 8:
return (cp[h] == 0xff);
case 4:
mask = 0x0f << ((h & 0x1) << 2);
return ((cp[h >> 1] & mask) == mask);
case 2:
mask = 0x03 << ((h & 0x3) << 1);
return ((cp[h >> 2] & mask) == mask);
case 1:
mask = 0x01 << (h & 0x7);
return ((cp[h >> 3] & mask) == mask);
default:
#ifdef _KERNEL
panic("ffs_isblock");
#endif
break;
}
return (0);
}
/*
* check if a block is free
*/
int
ffs_isfreeblock(struct fs *fs, u_char *cp, ufs1_daddr_t h)
{
switch ((int)fs->fs_frag) {
case 8:
return (cp[h] == 0);
case 4:
return ((cp[h >> 1] & (0x0f << ((h & 0x1) << 2))) == 0);
case 2:
return ((cp[h >> 2] & (0x03 << ((h & 0x3) << 1))) == 0);
case 1:
return ((cp[h >> 3] & (0x01 << (h & 0x7))) == 0);
default:
#ifdef _KERNEL
panic("ffs_isfreeblock");
#endif
break;
}
return (0);
}
/*
* take a block out of the map
*/
void
ffs_clrblock(struct fs *fs, u_char *cp, ufs1_daddr_t h)
{
switch ((int)fs->fs_frag) {
case 8:
cp[h] = 0;
return;
case 4:
cp[h >> 1] &= ~(0x0f << ((h & 0x1) << 2));
return;
case 2:
cp[h >> 2] &= ~(0x03 << ((h & 0x3) << 1));
return;
case 1:
cp[h >> 3] &= ~(0x01 << (h & 0x7));
return;
default:
#ifdef _KERNEL
panic("ffs_clrblock");
#endif
break;
}
}
/*
* put a block into the map
*/
void
ffs_setblock(struct fs *fs, unsigned char *cp, ufs1_daddr_t h)
{
switch ((int)fs->fs_frag) {
case 8:
cp[h] = 0xff;
return;
case 4:
cp[h >> 1] |= (0x0f << ((h & 0x1) << 2));
return;
case 2:
cp[h >> 2] |= (0x03 << ((h & 0x3) << 1));
return;
case 1:
cp[h >> 3] |= (0x01 << (h & 0x7));
return;
default:
#ifdef _KERNEL
panic("ffs_setblock");
#endif
break;
}
}
/*
* Update the cluster map because of an allocation or free.
*
* Cnt == 1 means free; cnt == -1 means allocating.
*/
void
ffs_clusteracct(struct fs *fs, struct cg *cgp, ufs1_daddr_t blkno, int cnt)
{
int32_t *sump;
int32_t *lp;
u_char *freemapp, *mapp;
int i, start, end, forw, back, map;
u_int bit;
if (fs->fs_contigsumsize <= 0)
return;
freemapp = cg_clustersfree(cgp);
sump = cg_clustersum(cgp);
/*
* Allocate or clear the actual block.
*/
if (cnt > 0)
setbit(freemapp, blkno);
else
clrbit(freemapp, blkno);
/*
* Find the size of the cluster going forward.
*/
start = blkno + 1;
end = start + fs->fs_contigsumsize;
if (end >= cgp->cg_nclusterblks)
end = cgp->cg_nclusterblks;
mapp = &freemapp[start / NBBY];
map = *mapp++;
bit = 1U << (start % NBBY);
for (i = start; i < end; i++) {
if ((map & bit) == 0)
break;
if ((i & (NBBY - 1)) != (NBBY - 1)) {
bit <<= 1;
} else {
map = *mapp++;
bit = 1;
}
}
forw = i - start;
/*
* Find the size of the cluster going backward.
*/
start = blkno - 1;
end = start - fs->fs_contigsumsize;
if (end < 0)
end = -1;
mapp = &freemapp[start / NBBY];
map = *mapp--;
bit = 1U << (start % NBBY);
for (i = start; i > end; i--) {
if ((map & bit) == 0)
break;
if ((i & (NBBY - 1)) != 0) {
bit >>= 1;
} else {
map = *mapp--;
bit = 1U << (NBBY - 1);
}
}
back = start - i;
/*
* Account for old cluster and the possibly new forward and
* back clusters.
*/
i = back + forw + 1;
if (i > fs->fs_contigsumsize)
i = fs->fs_contigsumsize;
sump[i] += cnt;
if (back > 0)
sump[back] -= cnt;
if (forw > 0)
sump[forw] -= cnt;
/*
* Update cluster summary information.
*/
lp = &sump[fs->fs_contigsumsize];
for (i = fs->fs_contigsumsize; i > 0; i--)
if (*lp-- > 0)
break;
fs->fs_maxcluster[cgp->cg_cgx] = i;
}
Index: head/sys/ufs/ffs/ffs_vfsops.c
===================================================================
--- head/sys/ufs/ffs/ffs_vfsops.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_vfsops.c (revision 361491)
@@ -1,2432 +1,2464 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1989, 1991, 1993, 1994
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_vfsops.c 8.31 (Berkeley) 5/20/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_quota.h"
#include "opt_ufs.h"
#include "opt_ffs.h"
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/gsb_crc32.h>
#include <sys/systm.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/taskqueue.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/ioccom.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
+#include <sys/sysctl.h>
#include <sys/vmmeter.h>
#include <security/mac/mac_framework.h>
#include <ufs/ufs/dir.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/gjournal.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
#include <vm/vm.h>
#include <vm/uma.h>
#include <vm/vm_page.h>
#include <geom/geom.h>
#include <geom/geom_vfs.h>
#include <ddb/ddb.h>
static uma_zone_t uma_inode, uma_ufs1, uma_ufs2;
static int ffs_mountfs(struct vnode *, struct mount *, struct thread *);
static void ffs_oldfscompat_read(struct fs *, struct ufsmount *,
ufs2_daddr_t);
static void ffs_ifree(struct ufsmount *ump, struct inode *ip);
static int ffs_sync_lazy(struct mount *mp);
static int ffs_use_bread(void *devfd, off_t loc, void **bufp, int size);
static int ffs_use_bwrite(void *devfd, off_t loc, void *buf, int size);
static vfs_init_t ffs_init;
static vfs_uninit_t ffs_uninit;
static vfs_extattrctl_t ffs_extattrctl;
static vfs_cmount_t ffs_cmount;
static vfs_unmount_t ffs_unmount;
static vfs_mount_t ffs_mount;
static vfs_statfs_t ffs_statfs;
static vfs_fhtovp_t ffs_fhtovp;
static vfs_sync_t ffs_sync;
static struct vfsops ufs_vfsops = {
.vfs_extattrctl = ffs_extattrctl,
.vfs_fhtovp = ffs_fhtovp,
.vfs_init = ffs_init,
.vfs_mount = ffs_mount,
.vfs_cmount = ffs_cmount,
.vfs_quotactl = ufs_quotactl,
.vfs_root = vfs_cache_root,
.vfs_cachedroot = ufs_root,
.vfs_statfs = ffs_statfs,
.vfs_sync = ffs_sync,
.vfs_uninit = ffs_uninit,
.vfs_unmount = ffs_unmount,
.vfs_vget = ffs_vget,
.vfs_susp_clean = process_deferred_inactive,
};
VFS_SET(ufs_vfsops, ufs, 0);
MODULE_VERSION(ufs, 1);
static b_strategy_t ffs_geom_strategy;
static b_write_t ffs_bufwrite;
static struct buf_ops ffs_ops = {
.bop_name = "FFS",
.bop_write = ffs_bufwrite,
.bop_strategy = ffs_geom_strategy,
.bop_sync = bufsync,
#ifdef NO_FFS_SNAPSHOT
.bop_bdflush = bufbdflush,
#else
.bop_bdflush = ffs_bdflush,
#endif
};
/*
* Note that userquota and groupquota options are not currently used
* by UFS/FFS code and generally mount(8) does not pass those options
* from userland, but they can be passed by loader(8) via
* vfs.root.mountfrom.options.
*/
static const char *ffs_opts[] = { "acls", "async", "noatime", "noclusterr",
"noclusterw", "noexec", "export", "force", "from", "groupquota",
"multilabel", "nfsv4acls", "fsckpid", "snapshot", "nosuid", "suiddir",
"nosymfollow", "sync", "union", "userquota", "untrusted", NULL };
+static int ffs_enxio_enable = 1;
+SYSCTL_DECL(_vfs_ffs);
+SYSCTL_INT(_vfs_ffs, OID_AUTO, enxio_enable, CTLFLAG_RWTUN,
+ &ffs_enxio_enable, 0,
+ "enable mapping of other disk I/O errors to ENXIO");
+
static int
ffs_mount(struct mount *mp)
{
struct vnode *devvp, *odevvp;
struct thread *td;
struct ufsmount *ump = NULL;
struct fs *fs;
pid_t fsckpid = 0;
int error, error1, flags;
uint64_t mntorflags, saved_mnt_flag;
accmode_t accmode;
struct nameidata ndp;
char *fspec;
td = curthread;
if (vfs_filteropt(mp->mnt_optnew, ffs_opts))
return (EINVAL);
if (uma_inode == NULL) {
uma_inode = uma_zcreate("FFS inode",
sizeof(struct inode), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
uma_ufs1 = uma_zcreate("FFS1 dinode",
sizeof(struct ufs1_dinode), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
uma_ufs2 = uma_zcreate("FFS2 dinode",
sizeof(struct ufs2_dinode), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
}
vfs_deleteopt(mp->mnt_optnew, "groupquota");
vfs_deleteopt(mp->mnt_optnew, "userquota");
fspec = vfs_getopts(mp->mnt_optnew, "from", &error);
if (error)
return (error);
mntorflags = 0;
if (vfs_getopt(mp->mnt_optnew, "untrusted", NULL, NULL) == 0)
mntorflags |= MNT_UNTRUSTED;
if (vfs_getopt(mp->mnt_optnew, "acls", NULL, NULL) == 0)
mntorflags |= MNT_ACLS;
if (vfs_getopt(mp->mnt_optnew, "snapshot", NULL, NULL) == 0) {
mntorflags |= MNT_SNAPSHOT;
/*
* Once we have set the MNT_SNAPSHOT flag, do not
* persist "snapshot" in the options list.
*/
vfs_deleteopt(mp->mnt_optnew, "snapshot");
vfs_deleteopt(mp->mnt_opt, "snapshot");
}
if (vfs_getopt(mp->mnt_optnew, "fsckpid", NULL, NULL) == 0 &&
vfs_scanopt(mp->mnt_optnew, "fsckpid", "%d", &fsckpid) == 1) {
/*
* Once we have set the restricted PID, do not
* persist "fsckpid" in the options list.
*/
vfs_deleteopt(mp->mnt_optnew, "fsckpid");
vfs_deleteopt(mp->mnt_opt, "fsckpid");
if (mp->mnt_flag & MNT_UPDATE) {
if (VFSTOUFS(mp)->um_fs->fs_ronly == 0 &&
vfs_flagopt(mp->mnt_optnew, "ro", NULL, 0) == 0) {
vfs_mount_error(mp,
"Checker enable: Must be read-only");
return (EINVAL);
}
} else if (vfs_flagopt(mp->mnt_optnew, "ro", NULL, 0) == 0) {
vfs_mount_error(mp,
"Checker enable: Must be read-only");
return (EINVAL);
}
/* Set to -1 if we are done */
if (fsckpid == 0)
fsckpid = -1;
}
if (vfs_getopt(mp->mnt_optnew, "nfsv4acls", NULL, NULL) == 0) {
if (mntorflags & MNT_ACLS) {
vfs_mount_error(mp,
"\"acls\" and \"nfsv4acls\" options "
"are mutually exclusive");
return (EINVAL);
}
mntorflags |= MNT_NFS4ACLS;
}
MNT_ILOCK(mp);
mp->mnt_flag |= mntorflags;
MNT_IUNLOCK(mp);
/*
* If updating, check whether changing from read-only to
* read/write; if there is no device name, that's all we do.
*/
if (mp->mnt_flag & MNT_UPDATE) {
ump = VFSTOUFS(mp);
fs = ump->um_fs;
odevvp = ump->um_odevvp;
devvp = ump->um_devvp;
if (fsckpid == -1 && ump->um_fsckpid > 0) {
if ((error = ffs_flushfiles(mp, WRITECLOSE, td)) != 0 ||
(error = ffs_sbupdate(ump, MNT_WAIT, 0)) != 0)
return (error);
g_topology_lock();
/*
* Return to normal read-only mode.
*/
error = g_access(ump->um_cp, 0, -1, 0);
g_topology_unlock();
ump->um_fsckpid = 0;
}
if (fs->fs_ronly == 0 &&
vfs_flagopt(mp->mnt_optnew, "ro", NULL, 0)) {
/*
* Flush any dirty data and suspend filesystem.
*/
if ((error = vn_start_write(NULL, &mp, V_WAIT)) != 0)
return (error);
error = vfs_write_suspend_umnt(mp);
if (error != 0)
return (error);
/*
* Check for and optionally get rid of files open
* for writing.
*/
flags = WRITECLOSE;
if (mp->mnt_flag & MNT_FORCE)
flags |= FORCECLOSE;
if (MOUNTEDSOFTDEP(mp)) {
error = softdep_flushfiles(mp, flags, td);
} else {
error = ffs_flushfiles(mp, flags, td);
}
if (error) {
vfs_write_resume(mp, 0);
return (error);
}
if (fs->fs_pendingblocks != 0 ||
fs->fs_pendinginodes != 0) {
printf("WARNING: %s Update error: blocks %jd "
"files %d\n", fs->fs_fsmnt,
(intmax_t)fs->fs_pendingblocks,
fs->fs_pendinginodes);
fs->fs_pendingblocks = 0;
fs->fs_pendinginodes = 0;
}
if ((fs->fs_flags & (FS_UNCLEAN | FS_NEEDSFSCK)) == 0)
fs->fs_clean = 1;
if ((error = ffs_sbupdate(ump, MNT_WAIT, 0)) != 0) {
fs->fs_ronly = 0;
fs->fs_clean = 0;
vfs_write_resume(mp, 0);
return (error);
}
if (MOUNTEDSOFTDEP(mp))
softdep_unmount(mp);
g_topology_lock();
/*
* Drop our write and exclusive access.
*/
g_access(ump->um_cp, 0, -1, -1);
g_topology_unlock();
fs->fs_ronly = 1;
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_RDONLY;
MNT_IUNLOCK(mp);
/*
* Allow the writers to note that filesystem
* is ro now.
*/
vfs_write_resume(mp, 0);
}
if ((mp->mnt_flag & MNT_RELOAD) &&
(error = ffs_reload(mp, td, 0)) != 0)
return (error);
if (fs->fs_ronly &&
!vfs_flagopt(mp->mnt_optnew, "ro", NULL, 0)) {
/*
* If we are running a checker, do not allow upgrade.
*/
if (ump->um_fsckpid > 0) {
vfs_mount_error(mp,
"Active checker, cannot upgrade to write");
return (EINVAL);
}
/*
* If upgrade to read-write by non-root, then verify
* that user has necessary permissions on the device.
*/
vn_lock(odevvp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_ACCESS(odevvp, VREAD | VWRITE,
td->td_ucred, td);
if (error)
error = priv_check(td, PRIV_VFS_MOUNT_PERM);
VOP_UNLOCK(odevvp);
if (error) {
return (error);
}
fs->fs_flags &= ~FS_UNCLEAN;
if (fs->fs_clean == 0) {
fs->fs_flags |= FS_UNCLEAN;
if ((mp->mnt_flag & MNT_FORCE) ||
((fs->fs_flags &
(FS_SUJ | FS_NEEDSFSCK)) == 0 &&
(fs->fs_flags & FS_DOSOFTDEP))) {
printf("WARNING: %s was not properly "
"dismounted\n", fs->fs_fsmnt);
} else {
vfs_mount_error(mp,
"R/W mount of %s denied. %s.%s",
fs->fs_fsmnt,
"Filesystem is not clean - run fsck",
(fs->fs_flags & FS_SUJ) == 0 ? "" :
" Forced mount will invalidate"
" journal contents");
return (EPERM);
}
}
g_topology_lock();
/*
* Request exclusive write access.
*/
error = g_access(ump->um_cp, 0, 1, 1);
g_topology_unlock();
if (error)
return (error);
if ((error = vn_start_write(NULL, &mp, V_WAIT)) != 0)
return (error);
error = vfs_write_suspend_umnt(mp);
if (error != 0)
return (error);
fs->fs_ronly = 0;
MNT_ILOCK(mp);
saved_mnt_flag = MNT_RDONLY;
if (MOUNTEDSOFTDEP(mp) && (mp->mnt_flag &
MNT_ASYNC) != 0)
saved_mnt_flag |= MNT_ASYNC;
mp->mnt_flag &= ~saved_mnt_flag;
MNT_IUNLOCK(mp);
fs->fs_mtime = time_second;
/* check to see if we need to start softdep */
if ((fs->fs_flags & FS_DOSOFTDEP) &&
(error = softdep_mount(devvp, mp, fs, td->td_ucred))){
fs->fs_ronly = 1;
MNT_ILOCK(mp);
mp->mnt_flag |= saved_mnt_flag;
MNT_IUNLOCK(mp);
vfs_write_resume(mp, 0);
return (error);
}
fs->fs_clean = 0;
if ((error = ffs_sbupdate(ump, MNT_WAIT, 0)) != 0) {
fs->fs_ronly = 1;
MNT_ILOCK(mp);
mp->mnt_flag |= saved_mnt_flag;
MNT_IUNLOCK(mp);
vfs_write_resume(mp, 0);
return (error);
}
if (fs->fs_snapinum[0] != 0)
ffs_snapshot_mount(mp);
vfs_write_resume(mp, 0);
}
/*
* Soft updates is incompatible with "async",
* so if we are doing softupdates stop the user
* from setting the async flag in an update.
* Softdep_mount() clears it in an initial mount
* or ro->rw remount.
*/
if (MOUNTEDSOFTDEP(mp)) {
/* XXX: Reset too late ? */
MNT_ILOCK(mp);
mp->mnt_flag &= ~MNT_ASYNC;
MNT_IUNLOCK(mp);
}
/*
* Keep MNT_ACLS flag if it is stored in superblock.
*/
if ((fs->fs_flags & FS_ACLS) != 0) {
/* XXX: Set too late ? */
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_ACLS;
MNT_IUNLOCK(mp);
}
if ((fs->fs_flags & FS_NFS4ACLS) != 0) {
/* XXX: Set too late ? */
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_NFS4ACLS;
MNT_IUNLOCK(mp);
}
/*
* If this is a request from fsck to clean up the filesystem,
* then allow the specified pid to proceed.
*/
if (fsckpid > 0) {
if (ump->um_fsckpid != 0) {
vfs_mount_error(mp,
"Active checker already running on %s",
fs->fs_fsmnt);
return (EINVAL);
}
KASSERT(MOUNTEDSOFTDEP(mp) == 0,
("soft updates enabled on read-only file system"));
g_topology_lock();
/*
* Request write access.
*/
error = g_access(ump->um_cp, 0, 1, 0);
g_topology_unlock();
if (error) {
vfs_mount_error(mp,
"Checker activation failed on %s",
fs->fs_fsmnt);
return (error);
}
ump->um_fsckpid = fsckpid;
if (fs->fs_snapinum[0] != 0)
ffs_snapshot_mount(mp);
fs->fs_mtime = time_second;
fs->fs_fmod = 1;
fs->fs_clean = 0;
(void) ffs_sbupdate(ump, MNT_WAIT, 0);
}
/*
* If this is a snapshot request, take the snapshot.
*/
if (mp->mnt_flag & MNT_SNAPSHOT)
return (ffs_snapshot(mp, fspec));
/*
* Must not call namei() while owning busy ref.
*/
vfs_unbusy(mp);
}
/*
* Not an update, or updating the name: look up the name
* and verify that it refers to a sensible disk device.
*/
NDINIT(&ndp, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, fspec, td);
error = namei(&ndp);
if ((mp->mnt_flag & MNT_UPDATE) != 0) {
/*
* Unmount does not start if MNT_UPDATE is set. Mount
* update busies mp before setting MNT_UPDATE. We
* must be able to retain our busy ref succesfully,
* without sleep.
*/
error1 = vfs_busy(mp, MBF_NOWAIT);
MPASS(error1 == 0);
}
if (error != 0)
return (error);
NDFREE(&ndp, NDF_ONLY_PNBUF);
devvp = ndp.ni_vp;
if (!vn_isdisk(devvp, &error)) {
vput(devvp);
return (error);
}
/*
* If mount by non-root, then verify that user has necessary
* permissions on the device.
*/
accmode = VREAD;
if ((mp->mnt_flag & MNT_RDONLY) == 0)
accmode |= VWRITE;
error = VOP_ACCESS(devvp, accmode, td->td_ucred, td);
if (error)
error = priv_check(td, PRIV_VFS_MOUNT_PERM);
if (error) {
vput(devvp);
return (error);
}
if (mp->mnt_flag & MNT_UPDATE) {
/*
* Update only
*
* If it's not the same vnode, or at least the same device
* then it's not correct.
*/
if (devvp->v_rdev != ump->um_devvp->v_rdev)
error = EINVAL; /* needs translation */
vput(devvp);
if (error)
return (error);
} else {
/*
* New mount
*
* We need the name for the mount point (also used for
* "last mounted on") copied in. If an error occurs,
* the mount point is discarded by the upper level code.
* Note that vfs_mount_alloc() populates f_mntonname for us.
*/
if ((error = ffs_mountfs(devvp, mp, td)) != 0) {
vrele(devvp);
return (error);
}
if (fsckpid > 0) {
KASSERT(MOUNTEDSOFTDEP(mp) == 0,
("soft updates enabled on read-only file system"));
ump = VFSTOUFS(mp);
fs = ump->um_fs;
g_topology_lock();
/*
* Request write access.
*/
error = g_access(ump->um_cp, 0, 1, 0);
g_topology_unlock();
if (error) {
printf("WARNING: %s: Checker activation "
"failed\n", fs->fs_fsmnt);
} else {
ump->um_fsckpid = fsckpid;
if (fs->fs_snapinum[0] != 0)
ffs_snapshot_mount(mp);
fs->fs_mtime = time_second;
fs->fs_clean = 0;
(void) ffs_sbupdate(ump, MNT_WAIT, 0);
}
}
}
vfs_mountedfrom(mp, fspec);
return (0);
}
/*
* Compatibility with old mount system call.
*/
static int
ffs_cmount(struct mntarg *ma, void *data, uint64_t flags)
{
struct ufs_args args;
struct export_args exp;
int error;
if (data == NULL)
return (EINVAL);
error = copyin(data, &args, sizeof args);
if (error)
return (error);
vfs_oexport_conv(&args.export, &exp);
ma = mount_argsu(ma, "from", args.fspec, MAXPATHLEN);
ma = mount_arg(ma, "export", &exp, sizeof(exp));
error = kernel_mount(ma, flags);
return (error);
}
/*
* Reload all incore data for a filesystem (used after running fsck on
* the root filesystem and finding things to fix). If the 'force' flag
* is 0, the filesystem must be mounted read-only.
*
* Things to do to update the mount:
* 1) invalidate all cached meta-data.
* 2) re-read superblock from disk.
* 3) re-read summary information from disk.
* 4) invalidate all inactive vnodes.
* 5) clear MNTK_SUSPEND2 and MNTK_SUSPENDED flags, allowing secondary
* writers, if requested.
* 6) invalidate all cached file data.
* 7) re-read inode data for all active vnodes.
*/
int
ffs_reload(struct mount *mp, struct thread *td, int flags)
{
struct vnode *vp, *mvp, *devvp;
struct inode *ip;
void *space;
struct buf *bp;
struct fs *fs, *newfs;
struct ufsmount *ump;
ufs2_daddr_t sblockloc;
int i, blks, error;
u_long size;
int32_t *lp;
ump = VFSTOUFS(mp);
MNT_ILOCK(mp);
if ((mp->mnt_flag & MNT_RDONLY) == 0 && (flags & FFSR_FORCE) == 0) {
MNT_IUNLOCK(mp);
return (EINVAL);
}
MNT_IUNLOCK(mp);
/*
* Step 1: invalidate all cached meta-data.
*/
devvp = VFSTOUFS(mp)->um_devvp;
vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
if (vinvalbuf(devvp, 0, 0, 0) != 0)
panic("ffs_reload: dirty1");
VOP_UNLOCK(devvp);
/*
* Step 2: re-read superblock from disk.
*/
fs = VFSTOUFS(mp)->um_fs;
if ((error = bread(devvp, btodb(fs->fs_sblockloc), fs->fs_sbsize,
NOCRED, &bp)) != 0)
return (error);
newfs = (struct fs *)bp->b_data;
if ((newfs->fs_magic != FS_UFS1_MAGIC &&
newfs->fs_magic != FS_UFS2_MAGIC) ||
newfs->fs_bsize > MAXBSIZE ||
newfs->fs_bsize < sizeof(struct fs)) {
brelse(bp);
return (EIO); /* XXX needs translation */
}
/*
* Copy pointer fields back into superblock before copying in XXX
* new superblock. These should really be in the ufsmount. XXX
* Note that important parameters (eg fs_ncg) are unchanged.
*/
newfs->fs_csp = fs->fs_csp;
newfs->fs_maxcluster = fs->fs_maxcluster;
newfs->fs_contigdirs = fs->fs_contigdirs;
newfs->fs_active = fs->fs_active;
newfs->fs_ronly = fs->fs_ronly;
sblockloc = fs->fs_sblockloc;
bcopy(newfs, fs, (u_int)fs->fs_sbsize);
brelse(bp);
mp->mnt_maxsymlinklen = fs->fs_maxsymlinklen;
ffs_oldfscompat_read(fs, VFSTOUFS(mp), sblockloc);
UFS_LOCK(ump);
if (fs->fs_pendingblocks != 0 || fs->fs_pendinginodes != 0) {
printf("WARNING: %s: reload pending error: blocks %jd "
"files %d\n", fs->fs_fsmnt, (intmax_t)fs->fs_pendingblocks,
fs->fs_pendinginodes);
fs->fs_pendingblocks = 0;
fs->fs_pendinginodes = 0;
}
UFS_UNLOCK(ump);
/*
* Step 3: re-read summary information from disk.
*/
size = fs->fs_cssize;
blks = howmany(size, fs->fs_fsize);
if (fs->fs_contigsumsize > 0)
size += fs->fs_ncg * sizeof(int32_t);
size += fs->fs_ncg * sizeof(u_int8_t);
free(fs->fs_csp, M_UFSMNT);
space = malloc(size, M_UFSMNT, M_WAITOK);
fs->fs_csp = space;
for (i = 0; i < blks; i += fs->fs_frag) {
size = fs->fs_bsize;
if (i + fs->fs_frag > blks)
size = (blks - i) * fs->fs_fsize;
error = bread(devvp, fsbtodb(fs, fs->fs_csaddr + i), size,
NOCRED, &bp);
if (error)
return (error);
bcopy(bp->b_data, space, (u_int)size);
space = (char *)space + size;
brelse(bp);
}
/*
* We no longer know anything about clusters per cylinder group.
*/
if (fs->fs_contigsumsize > 0) {
fs->fs_maxcluster = lp = space;
for (i = 0; i < fs->fs_ncg; i++)
*lp++ = fs->fs_contigsumsize;
space = lp;
}
size = fs->fs_ncg * sizeof(u_int8_t);
fs->fs_contigdirs = (u_int8_t *)space;
bzero(fs->fs_contigdirs, size);
if ((flags & FFSR_UNSUSPEND) != 0) {
MNT_ILOCK(mp);
mp->mnt_kern_flag &= ~(MNTK_SUSPENDED | MNTK_SUSPEND2);
wakeup(&mp->mnt_flag);
MNT_IUNLOCK(mp);
}
loop:
MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
/*
* Skip syncer vnode.
*/
if (vp->v_type == VNON) {
VI_UNLOCK(vp);
continue;
}
/*
* Step 4: invalidate all cached file data.
*/
if (vget(vp, LK_EXCLUSIVE | LK_INTERLOCK, td)) {
MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
goto loop;
}
if (vinvalbuf(vp, 0, 0, 0))
panic("ffs_reload: dirty2");
/*
* Step 5: re-read inode data for all active vnodes.
*/
ip = VTOI(vp);
error =
bread(devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)),
(int)fs->fs_bsize, NOCRED, &bp);
if (error) {
vput(vp);
MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
return (error);
}
if ((error = ffs_load_inode(bp, ip, fs, ip->i_number)) != 0) {
brelse(bp);
vput(vp);
MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
return (error);
}
ip->i_effnlink = ip->i_nlink;
brelse(bp);
vput(vp);
}
return (0);
}
/*
* Common code for mount and mountroot
*/
static int
ffs_mountfs(odevvp, mp, td)
struct vnode *odevvp;
struct mount *mp;
struct thread *td;
{
struct ufsmount *ump;
struct fs *fs;
struct cdev *dev;
int error, i, len, ronly;
struct ucred *cred;
struct g_consumer *cp;
struct mount *nmp;
struct vnode *devvp;
+ struct fsfail_task *etp;
int candelete, canspeedup;
off_t loc;
fs = NULL;
ump = NULL;
cred = td ? td->td_ucred : NOCRED;
ronly = (mp->mnt_flag & MNT_RDONLY) != 0;
devvp = mntfs_allocvp(mp, odevvp);
VOP_UNLOCK(odevvp);
KASSERT(devvp->v_type == VCHR, ("reclaimed devvp"));
dev = devvp->v_rdev;
if (atomic_cmpset_acq_ptr((uintptr_t *)&dev->si_mountpt, 0,
(uintptr_t)mp) == 0) {
mntfs_freevp(devvp);
return (EBUSY);
}
g_topology_lock();
error = g_vfs_open(devvp, &cp, "ffs", ronly ? 0 : 1);
g_topology_unlock();
if (error != 0) {
atomic_store_rel_ptr((uintptr_t *)&dev->si_mountpt, 0);
mntfs_freevp(devvp);
return (error);
}
dev_ref(dev);
devvp->v_bufobj.bo_ops = &ffs_ops;
BO_LOCK(&odevvp->v_bufobj);
odevvp->v_bufobj.bo_flag |= BO_NOBUFS;
BO_UNLOCK(&odevvp->v_bufobj);
if (dev->si_iosize_max != 0)
mp->mnt_iosize_max = dev->si_iosize_max;
if (mp->mnt_iosize_max > MAXPHYS)
mp->mnt_iosize_max = MAXPHYS;
if ((SBLOCKSIZE % cp->provider->sectorsize) != 0) {
error = EINVAL;
vfs_mount_error(mp,
"Invalid sectorsize %d for superblock size %d",
cp->provider->sectorsize, SBLOCKSIZE);
goto out;
}
/* fetch the superblock and summary information */
loc = STDSB;
if ((mp->mnt_flag & MNT_ROOTFS) != 0)
loc = STDSB_NOHASHFAIL;
if ((error = ffs_sbget(devvp, &fs, loc, M_UFSMNT, ffs_use_bread)) != 0)
goto out;
/* none of these types of check-hashes are maintained by this kernel */
fs->fs_metackhash &= ~(CK_INDIR | CK_DIR);
/* no support for any undefined flags */
fs->fs_flags &= FS_SUPPORTED;
fs->fs_flags &= ~FS_UNCLEAN;
if (fs->fs_clean == 0) {
fs->fs_flags |= FS_UNCLEAN;
if (ronly || (mp->mnt_flag & MNT_FORCE) ||
((fs->fs_flags & (FS_SUJ | FS_NEEDSFSCK)) == 0 &&
(fs->fs_flags & FS_DOSOFTDEP))) {
printf("WARNING: %s was not properly dismounted\n",
fs->fs_fsmnt);
} else {
vfs_mount_error(mp, "R/W mount of %s denied. %s%s",
fs->fs_fsmnt, "Filesystem is not clean - run fsck.",
(fs->fs_flags & FS_SUJ) == 0 ? "" :
" Forced mount will invalidate journal contents");
error = EPERM;
goto out;
}
if ((fs->fs_pendingblocks != 0 || fs->fs_pendinginodes != 0) &&
(mp->mnt_flag & MNT_FORCE)) {
printf("WARNING: %s: lost blocks %jd files %d\n",
fs->fs_fsmnt, (intmax_t)fs->fs_pendingblocks,
fs->fs_pendinginodes);
fs->fs_pendingblocks = 0;
fs->fs_pendinginodes = 0;
}
}
if (fs->fs_pendingblocks != 0 || fs->fs_pendinginodes != 0) {
printf("WARNING: %s: mount pending error: blocks %jd "
"files %d\n", fs->fs_fsmnt, (intmax_t)fs->fs_pendingblocks,
fs->fs_pendinginodes);
fs->fs_pendingblocks = 0;
fs->fs_pendinginodes = 0;
}
if ((fs->fs_flags & FS_GJOURNAL) != 0) {
#ifdef UFS_GJOURNAL
/*
* Get journal provider name.
*/
len = 1024;
mp->mnt_gjprovider = malloc((u_long)len, M_UFSMNT, M_WAITOK);
if (g_io_getattr("GJOURNAL::provider", cp, &len,
mp->mnt_gjprovider) == 0) {
mp->mnt_gjprovider = realloc(mp->mnt_gjprovider, len,
M_UFSMNT, M_WAITOK);
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_GJOURNAL;
MNT_IUNLOCK(mp);
} else {
printf("WARNING: %s: GJOURNAL flag on fs "
"but no gjournal provider below\n",
mp->mnt_stat.f_mntonname);
free(mp->mnt_gjprovider, M_UFSMNT);
mp->mnt_gjprovider = NULL;
}
#else
printf("WARNING: %s: GJOURNAL flag on fs but no "
"UFS_GJOURNAL support\n", mp->mnt_stat.f_mntonname);
#endif
} else {
mp->mnt_gjprovider = NULL;
}
ump = malloc(sizeof *ump, M_UFSMNT, M_WAITOK | M_ZERO);
ump->um_cp = cp;
ump->um_bo = &devvp->v_bufobj;
ump->um_fs = fs;
if (fs->fs_magic == FS_UFS1_MAGIC) {
ump->um_fstype = UFS1;
ump->um_balloc = ffs_balloc_ufs1;
} else {
ump->um_fstype = UFS2;
ump->um_balloc = ffs_balloc_ufs2;
}
ump->um_blkatoff = ffs_blkatoff;
ump->um_truncate = ffs_truncate;
ump->um_update = ffs_update;
ump->um_valloc = ffs_valloc;
ump->um_vfree = ffs_vfree;
ump->um_ifree = ffs_ifree;
ump->um_rdonly = ffs_rdonly;
ump->um_snapgone = ffs_snapgone;
if ((mp->mnt_flag & MNT_UNTRUSTED) != 0)
ump->um_check_blkno = ffs_check_blkno;
else
ump->um_check_blkno = NULL;
mtx_init(UFS_MTX(ump), "FFS", "FFS Lock", MTX_DEF);
ffs_oldfscompat_read(fs, ump, fs->fs_sblockloc);
fs->fs_ronly = ronly;
fs->fs_active = NULL;
mp->mnt_data = ump;
mp->mnt_stat.f_fsid.val[0] = fs->fs_id[0];
mp->mnt_stat.f_fsid.val[1] = fs->fs_id[1];
nmp = NULL;
if (fs->fs_id[0] == 0 || fs->fs_id[1] == 0 ||
(nmp = vfs_getvfs(&mp->mnt_stat.f_fsid))) {
if (nmp)
vfs_rel(nmp);
vfs_getnewfsid(mp);
}
mp->mnt_maxsymlinklen = fs->fs_maxsymlinklen;
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_LOCAL;
MNT_IUNLOCK(mp);
if ((fs->fs_flags & FS_MULTILABEL) != 0) {
#ifdef MAC
MNT_ILOCK(mp);
mp->mnt_flag |= MNT_MULTILABEL;
MNT_IUNLOCK(mp);
#else
printf("WARNING: %s: multilabel flag on fs but "
"no MAC support\n", mp->mnt_stat.f_mntonname);
#endif
}
if ((fs->fs_flags & FS_ACLS) != 0) {
#ifdef UFS_ACL
MNT_ILOCK(mp);
if (mp->mnt_flag & MNT_NFS4ACLS)
printf("WARNING: %s: ACLs flag on fs conflicts with "
"\"nfsv4acls\" mount option; option ignored\n",
mp->mnt_stat.f_mntonname);
mp->mnt_flag &= ~MNT_NFS4ACLS;
mp->mnt_flag |= MNT_ACLS;
MNT_IUNLOCK(mp);
#else
printf("WARNING: %s: ACLs flag on fs but no ACLs support\n",
mp->mnt_stat.f_mntonname);
#endif
}
if ((fs->fs_flags & FS_NFS4ACLS) != 0) {
#ifdef UFS_ACL
MNT_ILOCK(mp);
if (mp->mnt_flag & MNT_ACLS)
printf("WARNING: %s: NFSv4 ACLs flag on fs conflicts "
"with \"acls\" mount option; option ignored\n",
mp->mnt_stat.f_mntonname);
mp->mnt_flag &= ~MNT_ACLS;
mp->mnt_flag |= MNT_NFS4ACLS;
MNT_IUNLOCK(mp);
#else
printf("WARNING: %s: NFSv4 ACLs flag on fs but no "
"ACLs support\n", mp->mnt_stat.f_mntonname);
#endif
}
if ((fs->fs_flags & FS_TRIM) != 0) {
len = sizeof(int);
if (g_io_getattr("GEOM::candelete", cp, &len,
&candelete) == 0) {
if (candelete)
ump->um_flags |= UM_CANDELETE;
else
printf("WARNING: %s: TRIM flag on fs but disk "
"does not support TRIM\n",
mp->mnt_stat.f_mntonname);
} else {
printf("WARNING: %s: TRIM flag on fs but disk does "
"not confirm that it supports TRIM\n",
mp->mnt_stat.f_mntonname);
}
if (((ump->um_flags) & UM_CANDELETE) != 0) {
ump->um_trim_tq = taskqueue_create("trim", M_WAITOK,
taskqueue_thread_enqueue, &ump->um_trim_tq);
taskqueue_start_threads(&ump->um_trim_tq, 1, PVFS,
"%s trim", mp->mnt_stat.f_mntonname);
ump->um_trimhash = hashinit(MAXTRIMIO, M_TRIM,
&ump->um_trimlisthashsize);
}
}
len = sizeof(int);
if (g_io_getattr("GEOM::canspeedup", cp, &len, &canspeedup) == 0) {
if (canspeedup)
ump->um_flags |= UM_CANSPEEDUP;
}
ump->um_mountp = mp;
ump->um_dev = dev;
ump->um_devvp = devvp;
ump->um_odevvp = odevvp;
ump->um_nindir = fs->fs_nindir;
ump->um_bptrtodb = fs->fs_fsbtodb;
ump->um_seqinc = fs->fs_frag;
for (i = 0; i < MAXQUOTAS; i++)
ump->um_quotas[i] = NULLVP;
#ifdef UFS_EXTATTR
ufs_extattr_uepm_init(&ump->um_extattr);
#endif
/*
* Set FS local "last mounted on" information (NULL pad)
*/
bzero(fs->fs_fsmnt, MAXMNTLEN);
strlcpy(fs->fs_fsmnt, mp->mnt_stat.f_mntonname, MAXMNTLEN);
mp->mnt_stat.f_iosize = fs->fs_bsize;
if (mp->mnt_flag & MNT_ROOTFS) {
/*
* Root mount; update timestamp in mount structure.
* this will be used by the common root mount code
* to update the system clock.
*/
mp->mnt_time = fs->fs_time;
}
if (ronly == 0) {
fs->fs_mtime = time_second;
if ((fs->fs_flags & FS_DOSOFTDEP) &&
(error = softdep_mount(devvp, mp, fs, cred)) != 0) {
ffs_flushfiles(mp, FORCECLOSE, td);
goto out;
}
if (fs->fs_snapinum[0] != 0)
ffs_snapshot_mount(mp);
fs->fs_fmod = 1;
fs->fs_clean = 0;
(void) ffs_sbupdate(ump, MNT_WAIT, 0);
}
/*
* Initialize filesystem state information in mount struct.
*/
MNT_ILOCK(mp);
mp->mnt_kern_flag |= MNTK_LOOKUP_SHARED | MNTK_EXTENDED_SHARED |
MNTK_NO_IOPF | MNTK_UNMAPPED_BUFS | MNTK_USES_BCACHE;
MNT_IUNLOCK(mp);
#ifdef UFS_EXTATTR
#ifdef UFS_EXTATTR_AUTOSTART
/*
*
* Auto-starting does the following:
* - check for /.attribute in the fs, and extattr_start if so
* - for each file in .attribute, enable that file with
* an attribute of the same name.
* Not clear how to report errors -- probably eat them.
* This would all happen while the filesystem was busy/not
* available, so would effectively be "atomic".
*/
(void) ufs_extattr_autostart(mp, td);
#endif /* !UFS_EXTATTR_AUTOSTART */
#endif /* !UFS_EXTATTR */
+ etp = malloc(sizeof *ump->um_fsfail_task, M_UFSMNT, M_WAITOK | M_ZERO);
+ etp->fsid = mp->mnt_stat.f_fsid;
+ ump->um_fsfail_task = etp;
return (0);
out:
if (fs != NULL) {
free(fs->fs_csp, M_UFSMNT);
free(fs, M_UFSMNT);
}
if (cp != NULL) {
g_topology_lock();
g_vfs_close(cp);
g_topology_unlock();
}
if (ump) {
mtx_destroy(UFS_MTX(ump));
if (mp->mnt_gjprovider != NULL) {
free(mp->mnt_gjprovider, M_UFSMNT);
mp->mnt_gjprovider = NULL;
}
free(ump, M_UFSMNT);
mp->mnt_data = NULL;
}
BO_LOCK(&odevvp->v_bufobj);
odevvp->v_bufobj.bo_flag &= ~BO_NOBUFS;
BO_UNLOCK(&odevvp->v_bufobj);
atomic_store_rel_ptr((uintptr_t *)&dev->si_mountpt, 0);
mntfs_freevp(devvp);
dev_rel(dev);
return (error);
}
/*
* A read function for use by filesystem-layer routines.
*/
static int
ffs_use_bread(void *devfd, off_t loc, void **bufp, int size)
{
struct buf *bp;
int error;
KASSERT(*bufp == NULL, ("ffs_use_bread: non-NULL *bufp %p\n", *bufp));
*bufp = malloc(size, M_UFSMNT, M_WAITOK);
if ((error = bread((struct vnode *)devfd, btodb(loc), size, NOCRED,
&bp)) != 0)
return (error);
bcopy(bp->b_data, *bufp, size);
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
return (0);
}
-#include <sys/sysctl.h>
static int bigcgs = 0;
SYSCTL_INT(_debug, OID_AUTO, bigcgs, CTLFLAG_RW, &bigcgs, 0, "");
/*
* Sanity checks for loading old filesystem superblocks.
* See ffs_oldfscompat_write below for unwound actions.
*
* XXX - Parts get retired eventually.
* Unfortunately new bits get added.
*/
static void
ffs_oldfscompat_read(fs, ump, sblockloc)
struct fs *fs;
struct ufsmount *ump;
ufs2_daddr_t sblockloc;
{
off_t maxfilesize;
/*
* If not yet done, update fs_flags location and value of fs_sblockloc.
*/
if ((fs->fs_old_flags & FS_FLAGS_UPDATED) == 0) {
fs->fs_flags = fs->fs_old_flags;
fs->fs_old_flags |= FS_FLAGS_UPDATED;
fs->fs_sblockloc = sblockloc;
}
/*
* If not yet done, update UFS1 superblock with new wider fields.
*/
if (fs->fs_magic == FS_UFS1_MAGIC && fs->fs_maxbsize != fs->fs_bsize) {
fs->fs_maxbsize = fs->fs_bsize;
fs->fs_time = fs->fs_old_time;
fs->fs_size = fs->fs_old_size;
fs->fs_dsize = fs->fs_old_dsize;
fs->fs_csaddr = fs->fs_old_csaddr;
fs->fs_cstotal.cs_ndir = fs->fs_old_cstotal.cs_ndir;
fs->fs_cstotal.cs_nbfree = fs->fs_old_cstotal.cs_nbfree;
fs->fs_cstotal.cs_nifree = fs->fs_old_cstotal.cs_nifree;
fs->fs_cstotal.cs_nffree = fs->fs_old_cstotal.cs_nffree;
}
if (fs->fs_magic == FS_UFS1_MAGIC &&
fs->fs_old_inodefmt < FS_44INODEFMT) {
fs->fs_maxfilesize = ((uint64_t)1 << 31) - 1;
fs->fs_qbmask = ~fs->fs_bmask;
fs->fs_qfmask = ~fs->fs_fmask;
}
if (fs->fs_magic == FS_UFS1_MAGIC) {
ump->um_savedmaxfilesize = fs->fs_maxfilesize;
maxfilesize = (uint64_t)0x80000000 * fs->fs_bsize - 1;
if (fs->fs_maxfilesize > maxfilesize)
fs->fs_maxfilesize = maxfilesize;
}
/* Compatibility for old filesystems */
if (fs->fs_avgfilesize <= 0)
fs->fs_avgfilesize = AVFILESIZ;
if (fs->fs_avgfpdir <= 0)
fs->fs_avgfpdir = AFPDIR;
if (bigcgs) {
fs->fs_save_cgsize = fs->fs_cgsize;
fs->fs_cgsize = fs->fs_bsize;
}
}
/*
* Unwinding superblock updates for old filesystems.
* See ffs_oldfscompat_read above for details.
*
* XXX - Parts get retired eventually.
* Unfortunately new bits get added.
*/
void
ffs_oldfscompat_write(fs, ump)
struct fs *fs;
struct ufsmount *ump;
{
/*
* Copy back UFS2 updated fields that UFS1 inspects.
*/
if (fs->fs_magic == FS_UFS1_MAGIC) {
fs->fs_old_time = fs->fs_time;
fs->fs_old_cstotal.cs_ndir = fs->fs_cstotal.cs_ndir;
fs->fs_old_cstotal.cs_nbfree = fs->fs_cstotal.cs_nbfree;
fs->fs_old_cstotal.cs_nifree = fs->fs_cstotal.cs_nifree;
fs->fs_old_cstotal.cs_nffree = fs->fs_cstotal.cs_nffree;
fs->fs_maxfilesize = ump->um_savedmaxfilesize;
}
if (bigcgs) {
fs->fs_cgsize = fs->fs_save_cgsize;
fs->fs_save_cgsize = 0;
}
}
/*
* unmount system call
*/
static int
ffs_unmount(mp, mntflags)
struct mount *mp;
int mntflags;
{
struct thread *td;
struct ufsmount *ump = VFSTOUFS(mp);
struct fs *fs;
int error, flags, susp;
#ifdef UFS_EXTATTR
int e_restart;
#endif
flags = 0;
td = curthread;
fs = ump->um_fs;
if (mntflags & MNT_FORCE)
flags |= FORCECLOSE;
susp = fs->fs_ronly == 0;
#ifdef UFS_EXTATTR
if ((error = ufs_extattr_stop(mp, td))) {
if (error != EOPNOTSUPP)
printf("WARNING: unmount %s: ufs_extattr_stop "
"returned errno %d\n", mp->mnt_stat.f_mntonname,
error);
e_restart = 0;
} else {
ufs_extattr_uepm_destroy(&ump->um_extattr);
e_restart = 1;
}
#endif
if (susp) {
error = vfs_write_suspend_umnt(mp);
if (error != 0)
goto fail1;
}
if (MOUNTEDSOFTDEP(mp))
error = softdep_flushfiles(mp, flags, td);
else
error = ffs_flushfiles(mp, flags, td);
- if (error != 0 && error != ENXIO)
+ if (error != 0 && !ffs_fsfail_cleanup(ump, error))
goto fail;
UFS_LOCK(ump);
if (fs->fs_pendingblocks != 0 || fs->fs_pendinginodes != 0) {
printf("WARNING: unmount %s: pending error: blocks %jd "
"files %d\n", fs->fs_fsmnt, (intmax_t)fs->fs_pendingblocks,
fs->fs_pendinginodes);
fs->fs_pendingblocks = 0;
fs->fs_pendinginodes = 0;
}
UFS_UNLOCK(ump);
if (MOUNTEDSOFTDEP(mp))
softdep_unmount(mp);
if (fs->fs_ronly == 0 || ump->um_fsckpid > 0) {
fs->fs_clean = fs->fs_flags & (FS_UNCLEAN|FS_NEEDSFSCK) ? 0 : 1;
error = ffs_sbupdate(ump, MNT_WAIT, 0);
- if (error && error != ENXIO) {
+ if (ffs_fsfail_cleanup(ump, error))
+ error = 0;
+ if (error != 0 && !ffs_fsfail_cleanup(ump, error)) {
fs->fs_clean = 0;
goto fail;
}
}
if (susp)
vfs_write_resume(mp, VR_START_WRITE);
if (ump->um_trim_tq != NULL) {
while (ump->um_trim_inflight != 0)
pause("ufsutr", hz);
taskqueue_drain_all(ump->um_trim_tq);
taskqueue_free(ump->um_trim_tq);
free (ump->um_trimhash, M_TRIM);
}
g_topology_lock();
if (ump->um_fsckpid > 0) {
/*
* Return to normal read-only mode.
*/
error = g_access(ump->um_cp, 0, -1, 0);
ump->um_fsckpid = 0;
}
g_vfs_close(ump->um_cp);
g_topology_unlock();
BO_LOCK(&ump->um_odevvp->v_bufobj);
ump->um_odevvp->v_bufobj.bo_flag &= ~BO_NOBUFS;
BO_UNLOCK(&ump->um_odevvp->v_bufobj);
atomic_store_rel_ptr((uintptr_t *)&ump->um_dev->si_mountpt, 0);
mntfs_freevp(ump->um_devvp);
vrele(ump->um_odevvp);
dev_rel(ump->um_dev);
mtx_destroy(UFS_MTX(ump));
if (mp->mnt_gjprovider != NULL) {
free(mp->mnt_gjprovider, M_UFSMNT);
mp->mnt_gjprovider = NULL;
}
free(fs->fs_csp, M_UFSMNT);
free(fs, M_UFSMNT);
+ if (ump->um_fsfail_task != NULL)
+ free(ump->um_fsfail_task, M_UFSMNT);
free(ump, M_UFSMNT);
mp->mnt_data = NULL;
MNT_ILOCK(mp);
mp->mnt_flag &= ~MNT_LOCAL;
MNT_IUNLOCK(mp);
if (td->td_su == mp) {
td->td_su = NULL;
vfs_rel(mp);
}
return (error);
fail:
if (susp)
vfs_write_resume(mp, VR_START_WRITE);
fail1:
#ifdef UFS_EXTATTR
if (e_restart) {
ufs_extattr_uepm_init(&ump->um_extattr);
#ifdef UFS_EXTATTR_AUTOSTART
(void) ufs_extattr_autostart(mp, td);
#endif
}
#endif
return (error);
}
/*
* Flush out all the files in a filesystem.
*/
int
ffs_flushfiles(mp, flags, td)
struct mount *mp;
int flags;
struct thread *td;
{
struct ufsmount *ump;
int qerror, error;
ump = VFSTOUFS(mp);
qerror = 0;
#ifdef QUOTA
if (mp->mnt_flag & MNT_QUOTA) {
int i;
error = vflush(mp, 0, SKIPSYSTEM|flags, td);
if (error)
return (error);
for (i = 0; i < MAXQUOTAS; i++) {
error = quotaoff(td, mp, i);
if (error != 0) {
if ((flags & EARLYFLUSH) == 0)
return (error);
else
qerror = error;
}
}
/*
* Here we fall through to vflush again to ensure that
* we have gotten rid of all the system vnodes, unless
* quotas must not be closed.
*/
}
#endif
ASSERT_VOP_LOCKED(ump->um_devvp, "ffs_flushfiles");
if (ump->um_devvp->v_vflag & VV_COPYONWRITE) {
if ((error = vflush(mp, 0, SKIPSYSTEM | flags, td)) != 0)
return (error);
ffs_snapshot_unmount(mp);
flags |= FORCECLOSE;
/*
* Here we fall through to vflush again to ensure
* that we have gotten rid of all the system vnodes.
*/
}
/*
* Do not close system files if quotas were not closed, to be
* able to sync the remaining dquots. The freeblks softupdate
* workitems might hold a reference on a dquot, preventing
* quotaoff() from completing. Next round of
* softdep_flushworklist() iteration should process the
* blockers, allowing the next run of quotaoff() to finally
* flush held dquots.
*
* Otherwise, flush all the files.
*/
if (qerror == 0 && (error = vflush(mp, 0, flags, td)) != 0)
return (error);
/*
* Flush filesystem metadata.
*/
vn_lock(ump->um_devvp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_FSYNC(ump->um_devvp, MNT_WAIT, td);
VOP_UNLOCK(ump->um_devvp);
return (error);
}
/*
* Get filesystem statistics.
*/
static int
ffs_statfs(mp, sbp)
struct mount *mp;
struct statfs *sbp;
{
struct ufsmount *ump;
struct fs *fs;
ump = VFSTOUFS(mp);
fs = ump->um_fs;
if (fs->fs_magic != FS_UFS1_MAGIC && fs->fs_magic != FS_UFS2_MAGIC)
panic("ffs_statfs");
sbp->f_version = STATFS_VERSION;
sbp->f_bsize = fs->fs_fsize;
sbp->f_iosize = fs->fs_bsize;
sbp->f_blocks = fs->fs_dsize;
UFS_LOCK(ump);
sbp->f_bfree = fs->fs_cstotal.cs_nbfree * fs->fs_frag +
fs->fs_cstotal.cs_nffree + dbtofsb(fs, fs->fs_pendingblocks);
sbp->f_bavail = freespace(fs, fs->fs_minfree) +
dbtofsb(fs, fs->fs_pendingblocks);
sbp->f_files = fs->fs_ncg * fs->fs_ipg - UFS_ROOTINO;
sbp->f_ffree = fs->fs_cstotal.cs_nifree + fs->fs_pendinginodes;
UFS_UNLOCK(ump);
sbp->f_namemax = UFS_MAXNAMLEN;
return (0);
}
static bool
sync_doupdate(struct inode *ip)
{
return ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_MODIFIED |
IN_UPDATE)) != 0);
}
static int
ffs_sync_lazy_filter(struct vnode *vp, void *arg __unused)
{
struct inode *ip;
/*
* Flags are safe to access because ->v_data invalidation
* is held off by listmtx.
*/
if (vp->v_type == VNON)
return (false);
ip = VTOI(vp);
if (!sync_doupdate(ip) && (vp->v_iflag & VI_OWEINACT) == 0)
return (false);
return (true);
}
/*
* For a lazy sync, we only care about access times, quotas and the
* superblock. Other filesystem changes are already converted to
* cylinder group blocks or inode blocks updates and are written to
* disk by syncer.
*/
static int
ffs_sync_lazy(mp)
struct mount *mp;
{
struct vnode *mvp, *vp;
struct inode *ip;
struct thread *td;
int allerror, error;
allerror = 0;
td = curthread;
if ((mp->mnt_flag & MNT_NOATIME) != 0) {
#ifdef QUOTA
qsync(mp);
#endif
goto sbupdate;
}
MNT_VNODE_FOREACH_LAZY(vp, mp, mvp, ffs_sync_lazy_filter, NULL) {
if (vp->v_type == VNON) {
VI_UNLOCK(vp);
continue;
}
ip = VTOI(vp);
/*
* The IN_ACCESS flag is converted to IN_MODIFIED by
* ufs_close() and ufs_getattr() by the calls to
* ufs_itimes_locked(), without subsequent UFS_UPDATE().
* Test also all the other timestamp flags too, to pick up
* any other cases that could be missed.
*/
if (!sync_doupdate(ip) && (vp->v_iflag & VI_OWEINACT) == 0) {
VI_UNLOCK(vp);
continue;
}
if ((error = vget(vp, LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK,
td)) != 0)
continue;
#ifdef QUOTA
qsyncvp(vp);
#endif
if (sync_doupdate(ip))
error = ffs_update(vp, 0);
if (error != 0)
allerror = error;
vput(vp);
}
sbupdate:
if (VFSTOUFS(mp)->um_fs->fs_fmod != 0 &&
(error = ffs_sbupdate(VFSTOUFS(mp), MNT_LAZY, 0)) != 0)
allerror = error;
return (allerror);
}
/*
* Go through the disk queues to initiate sandbagged IO;
* go through the inodes to write those that have been modified;
* initiate the writing of the super block if it has been modified.
*
* Note: we are always called with the filesystem marked busy using
* vfs_busy().
*/
static int
ffs_sync(mp, waitfor)
struct mount *mp;
int waitfor;
{
struct vnode *mvp, *vp, *devvp;
struct thread *td;
struct inode *ip;
struct ufsmount *ump = VFSTOUFS(mp);
struct fs *fs;
int error, count, lockreq, allerror = 0;
int suspend;
int suspended;
int secondary_writes;
int secondary_accwrites;
int softdep_deps;
int softdep_accdeps;
struct bufobj *bo;
suspend = 0;
suspended = 0;
td = curthread;
fs = ump->um_fs;
if (fs->fs_fmod != 0 && fs->fs_ronly != 0 && ump->um_fsckpid == 0)
panic("%s: ffs_sync: modification on read-only filesystem",
fs->fs_fsmnt);
if (waitfor == MNT_LAZY) {
if (!rebooting)
return (ffs_sync_lazy(mp));
waitfor = MNT_NOWAIT;
}
/*
* Write back each (modified) inode.
*/
lockreq = LK_EXCLUSIVE | LK_NOWAIT;
if (waitfor == MNT_SUSPEND) {
suspend = 1;
waitfor = MNT_WAIT;
}
if (waitfor == MNT_WAIT)
lockreq = LK_EXCLUSIVE;
lockreq |= LK_INTERLOCK | LK_SLEEPFAIL;
loop:
/* Grab snapshot of secondary write counts */
MNT_ILOCK(mp);
secondary_writes = mp->mnt_secondary_writes;
secondary_accwrites = mp->mnt_secondary_accwrites;
MNT_IUNLOCK(mp);
/* Grab snapshot of softdep dependency counts */
softdep_get_depcounts(mp, &softdep_deps, &softdep_accdeps);
MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
/*
* Depend on the vnode interlock to keep things stable enough
* for a quick test. Since there might be hundreds of
* thousands of vnodes, we cannot afford even a subroutine
* call unless there's a good chance that we have work to do.
*/
if (vp->v_type == VNON) {
VI_UNLOCK(vp);
continue;
}
ip = VTOI(vp);
if ((ip->i_flag &
(IN_ACCESS | IN_CHANGE | IN_MODIFIED | IN_UPDATE)) == 0 &&
vp->v_bufobj.bo_dirty.bv_cnt == 0) {
VI_UNLOCK(vp);
continue;
}
if ((error = vget(vp, lockreq, td)) != 0) {
if (error == ENOENT || error == ENOLCK) {
MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
goto loop;
}
continue;
}
#ifdef QUOTA
qsyncvp(vp);
#endif
if ((error = ffs_syncvnode(vp, waitfor, 0)) != 0)
allerror = error;
vput(vp);
}
/*
* Force stale filesystem control information to be flushed.
*/
if (waitfor == MNT_WAIT || rebooting) {
if ((error = softdep_flushworklist(ump->um_mountp, &count, td)))
allerror = error;
+ if (ffs_fsfail_cleanup(ump, allerror))
+ allerror = 0;
/* Flushed work items may create new vnodes to clean */
if (allerror == 0 && count)
goto loop;
}
devvp = ump->um_devvp;
bo = &devvp->v_bufobj;
BO_LOCK(bo);
if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) {
BO_UNLOCK(bo);
vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_FSYNC(devvp, waitfor, td);
VOP_UNLOCK(devvp);
if (MOUNTEDSOFTDEP(mp) && (error == 0 || error == EAGAIN))
error = ffs_sbupdate(ump, waitfor, 0);
if (error != 0)
allerror = error;
+ if (ffs_fsfail_cleanup(ump, allerror))
+ allerror = 0;
if (allerror == 0 && waitfor == MNT_WAIT)
goto loop;
} else if (suspend != 0) {
if (softdep_check_suspend(mp,
devvp,
softdep_deps,
softdep_accdeps,
secondary_writes,
secondary_accwrites) != 0) {
MNT_IUNLOCK(mp);
goto loop; /* More work needed */
}
mtx_assert(MNT_MTX(mp), MA_OWNED);
mp->mnt_kern_flag |= MNTK_SUSPEND2 | MNTK_SUSPENDED;
MNT_IUNLOCK(mp);
suspended = 1;
} else
BO_UNLOCK(bo);
/*
* Write back modified superblock.
*/
if (fs->fs_fmod != 0 &&
(error = ffs_sbupdate(ump, waitfor, suspended)) != 0)
allerror = error;
+ if (ffs_fsfail_cleanup(ump, allerror))
+ allerror = 0;
return (allerror);
}
int
ffs_vget(mp, ino, flags, vpp)
struct mount *mp;
ino_t ino;
int flags;
struct vnode **vpp;
{
return (ffs_vgetf(mp, ino, flags, vpp, 0));
}
int
ffs_vgetf(mp, ino, flags, vpp, ffs_flags)
struct mount *mp;
ino_t ino;
int flags;
struct vnode **vpp;
int ffs_flags;
{
struct fs *fs;
struct inode *ip;
struct ufsmount *ump;
struct buf *bp;
struct vnode *vp;
+ daddr_t dbn;
int error;
MPASS((ffs_flags & FFSV_REPLACE) == 0 || (flags & LK_EXCLUSIVE) != 0);
error = vfs_hash_get(mp, ino, flags, curthread, vpp, NULL, NULL);
if (error != 0)
return (error);
if (*vpp != NULL) {
if ((ffs_flags & FFSV_REPLACE) == 0)
return (0);
vgone(*vpp);
vput(*vpp);
}
/*
* We must promote to an exclusive lock for vnode creation. This
* can happen if lookup is passed LOCKSHARED.
*/
if ((flags & LK_TYPE_MASK) == LK_SHARED) {
flags &= ~LK_TYPE_MASK;
flags |= LK_EXCLUSIVE;
}
/*
* We do not lock vnode creation as it is believed to be too
* expensive for such rare case as simultaneous creation of vnode
* for same ino by different processes. We just allow them to race
* and check later to decide who wins. Let the race begin!
*/
ump = VFSTOUFS(mp);
fs = ump->um_fs;
ip = uma_zalloc(uma_inode, M_WAITOK | M_ZERO);
/* Allocate a new vnode/inode. */
error = getnewvnode("ufs", mp, fs->fs_magic == FS_UFS1_MAGIC ?
&ffs_vnodeops1 : &ffs_vnodeops2, &vp);
if (error) {
*vpp = NULL;
uma_zfree(uma_inode, ip);
return (error);
}
/*
* FFS supports recursive locking.
*/
lockmgr(vp->v_vnlock, LK_EXCLUSIVE, NULL);
VN_LOCK_AREC(vp);
vp->v_data = ip;
vp->v_bufobj.bo_bsize = fs->fs_bsize;
ip->i_vnode = vp;
ip->i_ump = ump;
ip->i_number = ino;
ip->i_ea_refs = 0;
ip->i_nextclustercg = -1;
ip->i_flag = fs->fs_magic == FS_UFS1_MAGIC ? 0 : IN_UFS2;
ip->i_mode = 0; /* ensure error cases below throw away vnode */
#ifdef QUOTA
{
int i;
for (i = 0; i < MAXQUOTAS; i++)
ip->i_dquot[i] = NODQUOT;
}
#endif
if (ffs_flags & FFSV_FORCEINSMQ)
vp->v_vflag |= VV_FORCEINSMQ;
error = insmntque(vp, mp);
if (error != 0) {
uma_zfree(uma_inode, ip);
*vpp = NULL;
return (error);
}
vp->v_vflag &= ~VV_FORCEINSMQ;
error = vfs_hash_insert(vp, ino, flags, curthread, vpp, NULL, NULL);
if (error != 0)
return (error);
if (*vpp != NULL) {
/*
* Calls from ffs_valloc() (i.e. FFSV_REPLACE set)
* operate on empty inode, which must not be found by
* other threads until fully filled. Vnode for empty
* inode must be not re-inserted on the hash by other
* thread, after removal by us at the beginning.
*/
MPASS((ffs_flags & FFSV_REPLACE) == 0);
return (0);
}
/* Read in the disk contents for the inode, copy into the inode. */
- error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ino)),
- (int)fs->fs_bsize, NOCRED, &bp);
- if (error) {
+ dbn = fsbtodb(fs, ino_to_fsba(fs, ino));
+ error = ffs_breadz(ump, ump->um_devvp, dbn, dbn, (int)fs->fs_bsize,
+ NULL, NULL, 0, NOCRED, 0, NULL, &bp);
+ if (error != 0) {
/*
* The inode does not contain anything useful, so it would
* be misleading to leave it on its hash chain. With mode
* still zero, it will be unlinked and returned to the free
* list by vput().
*/
vgone(vp);
vput(vp);
*vpp = NULL;
return (error);
}
if (I_IS_UFS1(ip))
ip->i_din1 = uma_zalloc(uma_ufs1, M_WAITOK);
else
ip->i_din2 = uma_zalloc(uma_ufs2, M_WAITOK);
if ((error = ffs_load_inode(bp, ip, fs, ino)) != 0) {
bqrelse(bp);
vgone(vp);
vput(vp);
*vpp = NULL;
return (error);
}
if (DOINGSOFTDEP(vp))
softdep_load_inodeblock(ip);
else
ip->i_effnlink = ip->i_nlink;
bqrelse(bp);
/*
* Initialize the vnode from the inode, check for aliases.
* Note that the underlying vnode may have changed.
*/
error = ufs_vinit(mp, I_IS_UFS1(ip) ? &ffs_fifoops1 : &ffs_fifoops2,
&vp);
if (error) {
vgone(vp);
vput(vp);
*vpp = NULL;
return (error);
}
/*
* Finish inode initialization.
*/
if (vp->v_type != VFIFO) {
/* FFS supports shared locking for all files except fifos. */
VN_LOCK_ASHARE(vp);
}
/*
* Set up a generation number for this inode if it does not
* already have one. This should only happen on old filesystems.
*/
if (ip->i_gen == 0) {
while (ip->i_gen == 0)
ip->i_gen = arc4random();
if ((vp->v_mount->mnt_flag & MNT_RDONLY) == 0) {
UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
DIP_SET(ip, i_gen, ip->i_gen);
}
}
#ifdef MAC
if ((mp->mnt_flag & MNT_MULTILABEL) && ip->i_mode) {
/*
* If this vnode is already allocated, and we're running
* multi-label, attempt to perform a label association
* from the extended attributes on the inode.
*/
error = mac_vnode_associate_extattr(mp, vp);
if (error) {
/* ufs_inactive will release ip->i_devvp ref. */
vgone(vp);
vput(vp);
*vpp = NULL;
return (error);
}
}
#endif
*vpp = vp;
return (0);
}
/*
* File handle to vnode
*
* Have to be really careful about stale file handles:
* - check that the inode number is valid
* - for UFS2 check that the inode number is initialized
* - call ffs_vget() to get the locked inode
* - check for an unallocated inode (i_mode == 0)
* - check that the given client host has export rights and return
* those rights via. exflagsp and credanonp
*/
static int
ffs_fhtovp(mp, fhp, flags, vpp)
struct mount *mp;
struct fid *fhp;
int flags;
struct vnode **vpp;
{
struct ufid *ufhp;
struct ufsmount *ump;
struct fs *fs;
struct cg *cgp;
struct buf *bp;
ino_t ino;
u_int cg;
int error;
ufhp = (struct ufid *)fhp;
ino = ufhp->ufid_ino;
ump = VFSTOUFS(mp);
fs = ump->um_fs;
if (ino < UFS_ROOTINO || ino >= fs->fs_ncg * fs->fs_ipg)
return (ESTALE);
/*
* Need to check if inode is initialized because UFS2 does lazy
* initialization and nfs_fhtovp can offer arbitrary inode numbers.
*/
if (fs->fs_magic != FS_UFS2_MAGIC)
return (ufs_fhtovp(mp, ufhp, flags, vpp));
cg = ino_to_cg(fs, ino);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0)
return (error);
if (ino >= cg * fs->fs_ipg + cgp->cg_initediblk) {
brelse(bp);
return (ESTALE);
}
brelse(bp);
return (ufs_fhtovp(mp, ufhp, flags, vpp));
}
/*
* Initialize the filesystem.
*/
static int
ffs_init(vfsp)
struct vfsconf *vfsp;
{
ffs_susp_initialize();
softdep_initialize();
return (ufs_init(vfsp));
}
/*
* Undo the work of ffs_init().
*/
static int
ffs_uninit(vfsp)
struct vfsconf *vfsp;
{
int ret;
ret = ufs_uninit(vfsp);
softdep_uninitialize();
ffs_susp_uninitialize();
+ taskqueue_drain_all(taskqueue_thread);
return (ret);
}
/*
* Structure used to pass information from ffs_sbupdate to its
* helper routine ffs_use_bwrite.
*/
struct devfd {
struct ufsmount *ump;
struct buf *sbbp;
int waitfor;
int suspended;
int error;
};
/*
* Write a superblock and associated information back to disk.
*/
int
ffs_sbupdate(ump, waitfor, suspended)
struct ufsmount *ump;
int waitfor;
int suspended;
{
struct fs *fs;
struct buf *sbbp;
struct devfd devfd;
fs = ump->um_fs;
if (fs->fs_ronly == 1 &&
(ump->um_mountp->mnt_flag & (MNT_RDONLY | MNT_UPDATE)) !=
(MNT_RDONLY | MNT_UPDATE) && ump->um_fsckpid == 0)
panic("ffs_sbupdate: write read-only filesystem");
/*
* We use the superblock's buf to serialize calls to ffs_sbupdate().
*/
sbbp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc),
(int)fs->fs_sbsize, 0, 0, 0);
/*
* Initialize info needed for write function.
*/
devfd.ump = ump;
devfd.sbbp = sbbp;
devfd.waitfor = waitfor;
devfd.suspended = suspended;
devfd.error = 0;
return (ffs_sbput(&devfd, fs, fs->fs_sblockloc, ffs_use_bwrite));
}
/*
* Write function for use by filesystem-layer routines.
*/
static int
ffs_use_bwrite(void *devfd, off_t loc, void *buf, int size)
{
struct devfd *devfdp;
struct ufsmount *ump;
struct buf *bp;
struct fs *fs;
int error;
devfdp = devfd;
ump = devfdp->ump;
fs = ump->um_fs;
/*
* Writing the superblock summary information.
*/
if (loc != fs->fs_sblockloc) {
bp = getblk(ump->um_devvp, btodb(loc), size, 0, 0, 0);
bcopy(buf, bp->b_data, (u_int)size);
if (devfdp->suspended)
bp->b_flags |= B_VALIDSUSPWRT;
if (devfdp->waitfor != MNT_WAIT)
bawrite(bp);
else if ((error = bwrite(bp)) != 0)
devfdp->error = error;
return (0);
}
/*
* Writing the superblock itself. We need to do special checks for it.
*/
bp = devfdp->sbbp;
+ if (ffs_fsfail_cleanup(ump, devfdp->error))
+ devfdp->error = 0;
if (devfdp->error != 0) {
brelse(bp);
return (devfdp->error);
}
if (fs->fs_magic == FS_UFS1_MAGIC && fs->fs_sblockloc != SBLOCK_UFS1 &&
(fs->fs_old_flags & FS_FLAGS_UPDATED) == 0) {
printf("WARNING: %s: correcting fs_sblockloc from %jd to %d\n",
fs->fs_fsmnt, fs->fs_sblockloc, SBLOCK_UFS1);
fs->fs_sblockloc = SBLOCK_UFS1;
}
if (fs->fs_magic == FS_UFS2_MAGIC && fs->fs_sblockloc != SBLOCK_UFS2 &&
(fs->fs_old_flags & FS_FLAGS_UPDATED) == 0) {
printf("WARNING: %s: correcting fs_sblockloc from %jd to %d\n",
fs->fs_fsmnt, fs->fs_sblockloc, SBLOCK_UFS2);
fs->fs_sblockloc = SBLOCK_UFS2;
}
if (MOUNTEDSOFTDEP(ump->um_mountp))
softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, bp);
bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize);
fs = (struct fs *)bp->b_data;
ffs_oldfscompat_write(fs, ump);
/*
* Because we may have made changes to the superblock, we need to
* recompute its check-hash.
*/
fs->fs_ckhash = ffs_calc_sbhash(fs);
if (devfdp->suspended)
bp->b_flags |= B_VALIDSUSPWRT;
if (devfdp->waitfor != MNT_WAIT)
bawrite(bp);
else if ((error = bwrite(bp)) != 0)
devfdp->error = error;
return (devfdp->error);
}
static int
ffs_extattrctl(struct mount *mp, int cmd, struct vnode *filename_vp,
int attrnamespace, const char *attrname)
{
#ifdef UFS_EXTATTR
return (ufs_extattrctl(mp, cmd, filename_vp, attrnamespace,
attrname));
#else
return (vfs_stdextattrctl(mp, cmd, filename_vp, attrnamespace,
attrname));
#endif
}
static void
ffs_ifree(struct ufsmount *ump, struct inode *ip)
{
if (ump->um_fstype == UFS1 && ip->i_din1 != NULL)
uma_zfree(uma_ufs1, ip->i_din1);
else if (ip->i_din2 != NULL)
uma_zfree(uma_ufs2, ip->i_din2);
uma_zfree(uma_inode, ip);
}
static int dobkgrdwrite = 1;
SYSCTL_INT(_debug, OID_AUTO, dobkgrdwrite, CTLFLAG_RW, &dobkgrdwrite, 0,
"Do background writes (honoring the BV_BKGRDWRITE flag)?");
/*
* Complete a background write started from bwrite.
*/
static void
ffs_backgroundwritedone(struct buf *bp)
{
struct bufobj *bufobj;
struct buf *origbp;
+#ifdef SOFTUPDATES
+ if (!LIST_EMPTY(&bp->b_dep) && (bp->b_ioflags & BIO_ERROR) != 0)
+ softdep_handle_error(bp);
+#endif
+
/*
* Find the original buffer that we are writing.
*/
bufobj = bp->b_bufobj;
BO_LOCK(bufobj);
if ((origbp = gbincore(bp->b_bufobj, bp->b_lblkno)) == NULL)
panic("backgroundwritedone: lost buffer");
/*
* We should mark the cylinder group buffer origbp as
- * dirty, to not loose the failed write.
+ * dirty, to not lose the failed write.
*/
if ((bp->b_ioflags & BIO_ERROR) != 0)
origbp->b_vflags |= BV_BKGRDERR;
BO_UNLOCK(bufobj);
/*
* Process dependencies then return any unfinished ones.
*/
if (!LIST_EMPTY(&bp->b_dep) && (bp->b_ioflags & BIO_ERROR) == 0)
buf_complete(bp);
#ifdef SOFTUPDATES
if (!LIST_EMPTY(&bp->b_dep))
softdep_move_dependencies(bp, origbp);
#endif
/*
* This buffer is marked B_NOCACHE so when it is released
* by biodone it will be tossed.
*/
bp->b_flags |= B_NOCACHE;
bp->b_flags &= ~B_CACHE;
pbrelvp(bp);
/*
* Prevent brelse() from trying to keep and re-dirtying bp on
* errors. It causes b_bufobj dereference in
* bdirty()/reassignbuf(), and b_bufobj was cleared in
* pbrelvp() above.
*/
if ((bp->b_ioflags & BIO_ERROR) != 0)
bp->b_flags |= B_INVAL;
bufdone(bp);
BO_LOCK(bufobj);
/*
* Clear the BV_BKGRDINPROG flag in the original buffer
* and awaken it if it is waiting for the write to complete.
* If BV_BKGRDINPROG is not set in the original buffer it must
* have been released and re-instantiated - which is not legal.
*/
KASSERT((origbp->b_vflags & BV_BKGRDINPROG),
("backgroundwritedone: lost buffer2"));
origbp->b_vflags &= ~BV_BKGRDINPROG;
if (origbp->b_vflags & BV_BKGRDWAIT) {
origbp->b_vflags &= ~BV_BKGRDWAIT;
wakeup(&origbp->b_xflags);
}
BO_UNLOCK(bufobj);
}
/*
* Write, release buffer on completion. (Done by iodone
* if async). Do not bother writing anything if the buffer
* is invalid.
*
* Note that we set B_CACHE here, indicating that buffer is
* fully valid and thus cacheable. This is true even of NFS
* now so we set it generally. This could be set either here
* or in biodone() since the I/O is synchronous. We put it
* here.
*/
static int
ffs_bufwrite(struct buf *bp)
{
struct buf *newbp;
struct cg *cgp;
CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
if (bp->b_flags & B_INVAL) {
brelse(bp);
return (0);
}
if (!BUF_ISLOCKED(bp))
panic("bufwrite: buffer is not busy???");
/*
* If a background write is already in progress, delay
* writing this block if it is asynchronous. Otherwise
* wait for the background write to complete.
*/
BO_LOCK(bp->b_bufobj);
if (bp->b_vflags & BV_BKGRDINPROG) {
if (bp->b_flags & B_ASYNC) {
BO_UNLOCK(bp->b_bufobj);
bdwrite(bp);
return (0);
}
bp->b_vflags |= BV_BKGRDWAIT;
msleep(&bp->b_xflags, BO_LOCKPTR(bp->b_bufobj), PRIBIO,
"bwrbg", 0);
if (bp->b_vflags & BV_BKGRDINPROG)
panic("bufwrite: still writing");
}
bp->b_vflags &= ~BV_BKGRDERR;
BO_UNLOCK(bp->b_bufobj);
/*
* If this buffer is marked for background writing and we
* do not have to wait for it, make a copy and write the
* copy so as to leave this buffer ready for further use.
*
* This optimization eats a lot of memory. If we have a page
* or buffer shortfall we can't do it.
*/
if (dobkgrdwrite && (bp->b_xflags & BX_BKGRDWRITE) &&
(bp->b_flags & B_ASYNC) &&
!vm_page_count_severe() &&
!buf_dirty_count_severe()) {
KASSERT(bp->b_iodone == NULL,
("bufwrite: needs chained iodone (%p)", bp->b_iodone));
/* get a new block */
newbp = geteblk(bp->b_bufsize, GB_NOWAIT_BD);
if (newbp == NULL)
goto normal_write;
KASSERT(buf_mapped(bp), ("Unmapped cg"));
memcpy(newbp->b_data, bp->b_data, bp->b_bufsize);
BO_LOCK(bp->b_bufobj);
bp->b_vflags |= BV_BKGRDINPROG;
BO_UNLOCK(bp->b_bufobj);
newbp->b_xflags |=
(bp->b_xflags & BX_FSPRIV) | BX_BKGRDMARKER;
newbp->b_lblkno = bp->b_lblkno;
newbp->b_blkno = bp->b_blkno;
newbp->b_offset = bp->b_offset;
newbp->b_iodone = ffs_backgroundwritedone;
newbp->b_flags |= B_ASYNC;
newbp->b_flags &= ~B_INVAL;
pbgetvp(bp->b_vp, newbp);
#ifdef SOFTUPDATES
/*
* Move over the dependencies. If there are rollbacks,
* leave the parent buffer dirtied as it will need to
* be written again.
*/
if (LIST_EMPTY(&bp->b_dep) ||
softdep_move_dependencies(bp, newbp) == 0)
bundirty(bp);
#else
bundirty(bp);
#endif
/*
* Initiate write on the copy, release the original. The
* BKGRDINPROG flag prevents it from going away until
* the background write completes. We have to recalculate
* its check hash in case the buffer gets freed and then
* reconstituted from the buffer cache during a later read.
*/
if ((bp->b_xflags & BX_CYLGRP) != 0) {
cgp = (struct cg *)bp->b_data;
cgp->cg_ckhash = 0;
cgp->cg_ckhash =
calculate_crc32c(~0L, bp->b_data, bp->b_bcount);
}
bqrelse(bp);
bp = newbp;
} else
/* Mark the buffer clean */
bundirty(bp);
/* Let the normal bufwrite do the rest for us */
normal_write:
/*
* If we are writing a cylinder group, update its time.
*/
if ((bp->b_xflags & BX_CYLGRP) != 0) {
cgp = (struct cg *)bp->b_data;
cgp->cg_old_time = cgp->cg_time = time_second;
}
return (bufwrite(bp));
}
static void
ffs_geom_strategy(struct bufobj *bo, struct buf *bp)
{
struct vnode *vp;
struct buf *tbp;
int error, nocopy;
/*
* This is the bufobj strategy for the private VCHR vnodes
* used by FFS to access the underlying storage device.
* We override the default bufobj strategy and thus bypass
* VOP_STRATEGY() for these vnodes.
*/
vp = bo2vnode(bo);
KASSERT(bp->b_vp == NULL || bp->b_vp->v_type != VCHR ||
bp->b_vp->v_rdev == NULL ||
bp->b_vp->v_rdev->si_mountpt == NULL ||
VFSTOUFS(bp->b_vp->v_rdev->si_mountpt) == NULL ||
vp == VFSTOUFS(bp->b_vp->v_rdev->si_mountpt)->um_devvp,
("ffs_geom_strategy() with wrong vp"));
if (bp->b_iocmd == BIO_WRITE) {
if ((bp->b_flags & B_VALIDSUSPWRT) == 0 &&
bp->b_vp != NULL && bp->b_vp->v_mount != NULL &&
(bp->b_vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED) != 0)
panic("ffs_geom_strategy: bad I/O");
nocopy = bp->b_flags & B_NOCOPY;
bp->b_flags &= ~(B_VALIDSUSPWRT | B_NOCOPY);
if ((vp->v_vflag & VV_COPYONWRITE) && nocopy == 0 &&
vp->v_rdev->si_snapdata != NULL) {
if ((bp->b_flags & B_CLUSTER) != 0) {
runningbufwakeup(bp);
TAILQ_FOREACH(tbp, &bp->b_cluster.cluster_head,
b_cluster.cluster_entry) {
error = ffs_copyonwrite(vp, tbp);
if (error != 0 &&
error != EOPNOTSUPP) {
bp->b_error = error;
bp->b_ioflags |= BIO_ERROR;
bufdone(bp);
return;
}
}
bp->b_runningbufspace = bp->b_bufsize;
atomic_add_long(&runningbufspace,
bp->b_runningbufspace);
} else {
error = ffs_copyonwrite(vp, bp);
if (error != 0 && error != EOPNOTSUPP) {
bp->b_error = error;
bp->b_ioflags |= BIO_ERROR;
bufdone(bp);
return;
}
}
}
#ifdef SOFTUPDATES
if ((bp->b_flags & B_CLUSTER) != 0) {
TAILQ_FOREACH(tbp, &bp->b_cluster.cluster_head,
b_cluster.cluster_entry) {
if (!LIST_EMPTY(&tbp->b_dep))
buf_start(tbp);
}
} else {
if (!LIST_EMPTY(&bp->b_dep))
buf_start(bp);
}
#endif
/*
* Check for metadata that needs check-hashes and update them.
*/
switch (bp->b_xflags & BX_FSPRIV) {
case BX_CYLGRP:
((struct cg *)bp->b_data)->cg_ckhash = 0;
((struct cg *)bp->b_data)->cg_ckhash =
calculate_crc32c(~0L, bp->b_data, bp->b_bcount);
break;
case BX_SUPERBLOCK:
case BX_INODE:
case BX_INDIR:
case BX_DIR:
printf("Check-hash write is unimplemented!!!\n");
break;
case 0:
break;
default:
printf("multiple buffer types 0x%b\n",
(u_int)(bp->b_xflags & BX_FSPRIV),
PRINT_UFS_BUF_XFLAGS);
break;
}
}
+ if (bp->b_iocmd != BIO_READ && ffs_enxio_enable)
+ bp->b_xflags |= BX_CVTENXIO;
g_vfs_strategy(bo, bp);
}
int
ffs_own_mount(const struct mount *mp)
{
if (mp->mnt_op == &ufs_vfsops)
return (1);
return (0);
}
#ifdef DDB
#ifdef SOFTUPDATES
/* defined in ffs_softdep.c */
extern void db_print_ffs(struct ufsmount *ump);
DB_SHOW_COMMAND(ffs, db_show_ffs)
{
struct mount *mp;
struct ufsmount *ump;
if (have_addr) {
ump = VFSTOUFS((struct mount *)addr);
db_print_ffs(ump);
return;
}
TAILQ_FOREACH(mp, &mountlist, mnt_list) {
if (!strcmp(mp->mnt_stat.f_fstypename, ufs_vfsconf.vfc_name))
db_print_ffs(VFSTOUFS(mp));
}
}
#endif /* SOFTUPDATES */
#endif /* DDB */
Index: head/sys/ufs/ffs/ffs_vnops.c
===================================================================
--- head/sys/ufs/ffs/ffs_vnops.c (revision 361490)
+++ head/sys/ufs/ffs/ffs_vnops.c (revision 361491)
@@ -1,1805 +1,1818 @@
/*-
* SPDX-License-Identifier: (BSD-2-Clause-FreeBSD AND BSD-3-Clause)
*
* Copyright (c) 2002, 2003 Networks Associates Technology, Inc.
* All rights reserved.
*
* This software was developed for the FreeBSD Project by Marshall
* Kirk McKusick and Network Associates Laboratories, the Security
* Research Division of Network Associates, Inc. under DARPA/SPAWAR
* contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS
* research program
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from: @(#)ufs_readwrite.c 8.11 (Berkeley) 5/8/95
* from: $FreeBSD: .../ufs/ufs_readwrite.c,v 1.96 2002/08/12 09:22:11 phk ...
* @(#)ffs_vnops.c 8.15 (Berkeley) 5/14/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bio.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/extattr.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/priv.h>
#include <sys/rwlock.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_extern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pager.h>
#include <vm/vnode_pager.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
#include "opt_directio.h"
#include "opt_ffs.h"
#define ALIGNED_TO(ptr, s) \
(((uintptr_t)(ptr) & (_Alignof(s) - 1)) == 0)
#ifdef DIRECTIO
extern int ffs_rawread(struct vnode *vp, struct uio *uio, int *workdone);
#endif
static vop_fdatasync_t ffs_fdatasync;
static vop_fsync_t ffs_fsync;
static vop_getpages_t ffs_getpages;
static vop_getpages_async_t ffs_getpages_async;
static vop_lock1_t ffs_lock;
#ifdef INVARIANTS
static vop_unlock_t ffs_unlock_debug;
#endif
static vop_read_t ffs_read;
static vop_write_t ffs_write;
static int ffs_extread(struct vnode *vp, struct uio *uio, int ioflag);
static int ffs_extwrite(struct vnode *vp, struct uio *uio, int ioflag,
struct ucred *cred);
static vop_strategy_t ffsext_strategy;
static vop_closeextattr_t ffs_closeextattr;
static vop_deleteextattr_t ffs_deleteextattr;
static vop_getextattr_t ffs_getextattr;
static vop_listextattr_t ffs_listextattr;
static vop_openextattr_t ffs_openextattr;
static vop_setextattr_t ffs_setextattr;
static vop_vptofh_t ffs_vptofh;
/* Global vfs data structures for ufs. */
struct vop_vector ffs_vnodeops1 = {
.vop_default = &ufs_vnodeops,
.vop_fsync = ffs_fsync,
.vop_fdatasync = ffs_fdatasync,
.vop_getpages = ffs_getpages,
.vop_getpages_async = ffs_getpages_async,
.vop_lock1 = ffs_lock,
#ifdef INVARIANTS
.vop_unlock = ffs_unlock_debug,
#endif
.vop_read = ffs_read,
.vop_reallocblks = ffs_reallocblks,
.vop_write = ffs_write,
.vop_vptofh = ffs_vptofh,
};
VFS_VOP_VECTOR_REGISTER(ffs_vnodeops1);
struct vop_vector ffs_fifoops1 = {
.vop_default = &ufs_fifoops,
.vop_fsync = ffs_fsync,
.vop_fdatasync = ffs_fdatasync,
.vop_lock1 = ffs_lock,
#ifdef INVARIANTS
.vop_unlock = ffs_unlock_debug,
#endif
.vop_vptofh = ffs_vptofh,
};
VFS_VOP_VECTOR_REGISTER(ffs_fifoops1);
/* Global vfs data structures for ufs. */
struct vop_vector ffs_vnodeops2 = {
.vop_default = &ufs_vnodeops,
.vop_fsync = ffs_fsync,
.vop_fdatasync = ffs_fdatasync,
.vop_getpages = ffs_getpages,
.vop_getpages_async = ffs_getpages_async,
.vop_lock1 = ffs_lock,
#ifdef INVARIANTS
.vop_unlock = ffs_unlock_debug,
#endif
.vop_read = ffs_read,
.vop_reallocblks = ffs_reallocblks,
.vop_write = ffs_write,
.vop_closeextattr = ffs_closeextattr,
.vop_deleteextattr = ffs_deleteextattr,
.vop_getextattr = ffs_getextattr,
.vop_listextattr = ffs_listextattr,
.vop_openextattr = ffs_openextattr,
.vop_setextattr = ffs_setextattr,
.vop_vptofh = ffs_vptofh,
};
VFS_VOP_VECTOR_REGISTER(ffs_vnodeops2);
struct vop_vector ffs_fifoops2 = {
.vop_default = &ufs_fifoops,
.vop_fsync = ffs_fsync,
.vop_fdatasync = ffs_fdatasync,
.vop_lock1 = ffs_lock,
#ifdef INVARIANTS
.vop_unlock = ffs_unlock_debug,
#endif
.vop_reallocblks = ffs_reallocblks,
.vop_strategy = ffsext_strategy,
.vop_closeextattr = ffs_closeextattr,
.vop_deleteextattr = ffs_deleteextattr,
.vop_getextattr = ffs_getextattr,
.vop_listextattr = ffs_listextattr,
.vop_openextattr = ffs_openextattr,
.vop_setextattr = ffs_setextattr,
.vop_vptofh = ffs_vptofh,
};
VFS_VOP_VECTOR_REGISTER(ffs_fifoops2);
/*
* Synch an open file.
*/
/* ARGSUSED */
static int
ffs_fsync(struct vop_fsync_args *ap)
{
struct vnode *vp;
struct bufobj *bo;
int error;
vp = ap->a_vp;
bo = &vp->v_bufobj;
retry:
error = ffs_syncvnode(vp, ap->a_waitfor, 0);
if (error)
return (error);
if (ap->a_waitfor == MNT_WAIT && DOINGSOFTDEP(vp)) {
error = softdep_fsync(vp);
if (error)
return (error);
/*
* The softdep_fsync() function may drop vp lock,
* allowing for dirty buffers to reappear on the
* bo_dirty list. Recheck and resync as needed.
*/
BO_LOCK(bo);
if ((vp->v_type == VREG || vp->v_type == VDIR) &&
(bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0)) {
BO_UNLOCK(bo);
goto retry;
}
BO_UNLOCK(bo);
}
+ if (ffs_fsfail_cleanup(VFSTOUFS(vp->v_mount), 0))
+ return (ENXIO);
return (0);
}
int
ffs_syncvnode(struct vnode *vp, int waitfor, int flags)
{
struct inode *ip;
struct bufobj *bo;
+ struct ufsmount *ump;
struct buf *bp, *nbp;
ufs_lbn_t lbn;
int error, passes;
bool still_dirty, wait;
ip = VTOI(vp);
ip->i_flag &= ~IN_NEEDSYNC;
bo = &vp->v_bufobj;
+ ump = VFSTOUFS(vp->v_mount);
/*
* When doing MNT_WAIT we must first flush all dependencies
* on the inode.
*/
if (DOINGSOFTDEP(vp) && waitfor == MNT_WAIT &&
- (error = softdep_sync_metadata(vp)) != 0)
+ (error = softdep_sync_metadata(vp)) != 0) {
+ if (ffs_fsfail_cleanup(ump, error))
+ error = 0;
return (error);
+ }
/*
* Flush all dirty buffers associated with a vnode.
*/
error = 0;
passes = 0;
wait = false; /* Always do an async pass first. */
lbn = lblkno(ITOFS(ip), (ip->i_size + ITOFS(ip)->fs_bsize - 1));
BO_LOCK(bo);
loop:
TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
bp->b_vflags &= ~BV_SCANNED;
TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
/*
* Reasons to skip this buffer: it has already been considered
* on this pass, the buffer has dependencies that will cause
* it to be redirtied and it has not already been deferred,
* or it is already being written.
*/
if ((bp->b_vflags & BV_SCANNED) != 0)
continue;
bp->b_vflags |= BV_SCANNED;
/*
* Flush indirects in order, if requested.
*
* Note that if only datasync is requested, we can
* skip indirect blocks when softupdates are not
* active. Otherwise we must flush them with data,
* since dependencies prevent data block writes.
*/
if (waitfor == MNT_WAIT && bp->b_lblkno <= -UFS_NDADDR &&
(lbn_level(bp->b_lblkno) >= passes ||
((flags & DATA_ONLY) != 0 && !DOINGSOFTDEP(vp))))
continue;
if (bp->b_lblkno > lbn)
panic("ffs_syncvnode: syncing truncated data.");
if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) == 0) {
BO_UNLOCK(bo);
} else if (wait) {
if (BUF_LOCK(bp,
LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
BO_LOCKPTR(bo)) != 0) {
bp->b_vflags &= ~BV_SCANNED;
goto next;
}
} else
continue;
if ((bp->b_flags & B_DELWRI) == 0)
panic("ffs_fsync: not dirty");
/*
* Check for dependencies and potentially complete them.
*/
if (!LIST_EMPTY(&bp->b_dep) &&
(error = softdep_sync_buf(vp, bp,
wait ? MNT_WAIT : MNT_NOWAIT)) != 0) {
/* I/O error. */
if (error != EBUSY) {
BUF_UNLOCK(bp);
return (error);
}
/* If we deferred once, don't defer again. */
if ((bp->b_flags & B_DEFERRED) == 0) {
bp->b_flags |= B_DEFERRED;
BUF_UNLOCK(bp);
goto next;
}
}
if (wait) {
bremfree(bp);
- if ((error = bwrite(bp)) != 0)
+ error = bwrite(bp);
+ if (ffs_fsfail_cleanup(ump, error))
+ error = 0;
+ if (error != 0)
return (error);
} else if ((bp->b_flags & B_CLUSTEROK)) {
(void) vfs_bio_awrite(bp);
} else {
bremfree(bp);
(void) bawrite(bp);
}
next:
/*
* Since we may have slept during the I/O, we need
* to start from a known point.
*/
BO_LOCK(bo);
nbp = TAILQ_FIRST(&bo->bo_dirty.bv_hd);
}
if (waitfor != MNT_WAIT) {
BO_UNLOCK(bo);
if ((flags & NO_INO_UPDT) != 0)
return (0);
else
return (ffs_update(vp, 0));
}
/* Drain IO to see if we're done. */
bufobj_wwait(bo, 0, 0);
/*
* Block devices associated with filesystems may have new I/O
* requests posted for them even if the vnode is locked, so no
* amount of trying will get them clean. We make several passes
* as a best effort.
*
* Regular files may need multiple passes to flush all dependency
* work as it is possible that we must write once per indirect
* level, once for the leaf, and once for the inode and each of
* these will be done with one sync and one async pass.
*/
if (bo->bo_dirty.bv_cnt > 0) {
if ((flags & DATA_ONLY) == 0) {
still_dirty = true;
} else {
/*
* For data-only sync, dirty indirect buffers
* are ignored.
*/
still_dirty = false;
TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
if (bp->b_lblkno > -UFS_NDADDR) {
still_dirty = true;
break;
}
}
}
if (still_dirty) {
/* Write the inode after sync passes to flush deps. */
if (wait && DOINGSOFTDEP(vp) &&
(flags & NO_INO_UPDT) == 0) {
BO_UNLOCK(bo);
ffs_update(vp, 1);
BO_LOCK(bo);
}
/* switch between sync/async. */
wait = !wait;
if (wait || ++passes < UFS_NIADDR + 2)
goto loop;
}
}
BO_UNLOCK(bo);
error = 0;
if ((flags & DATA_ONLY) == 0) {
if ((flags & NO_INO_UPDT) == 0)
error = ffs_update(vp, 1);
if (DOINGSUJ(vp))
softdep_journal_fsync(VTOI(vp));
}
return (error);
}
static int
ffs_fdatasync(struct vop_fdatasync_args *ap)
{
return (ffs_syncvnode(ap->a_vp, MNT_WAIT, DATA_ONLY));
}
static int
ffs_lock(ap)
struct vop_lock1_args /* {
struct vnode *a_vp;
int a_flags;
struct thread *a_td;
char *file;
int line;
} */ *ap;
{
#ifndef NO_FFS_SNAPSHOT
struct vnode *vp;
int flags;
struct lock *lkp;
int result;
switch (ap->a_flags & LK_TYPE_MASK) {
case LK_SHARED:
case LK_UPGRADE:
case LK_EXCLUSIVE:
vp = ap->a_vp;
flags = ap->a_flags;
for (;;) {
#ifdef DEBUG_VFS_LOCKS
VNPASS(vp->v_holdcnt != 0, vp);
#endif
lkp = vp->v_vnlock;
result = lockmgr_lock_flags(lkp, flags,
&VI_MTX(vp)->lock_object, ap->a_file, ap->a_line);
if (lkp == vp->v_vnlock || result != 0)
break;
/*
* Apparent success, except that the vnode
* mutated between snapshot file vnode and
* regular file vnode while this process
* slept. The lock currently held is not the
* right lock. Release it, and try to get the
* new lock.
*/
lockmgr_unlock(lkp);
if ((flags & (LK_INTERLOCK | LK_NOWAIT)) ==
(LK_INTERLOCK | LK_NOWAIT))
return (EBUSY);
if ((flags & LK_TYPE_MASK) == LK_UPGRADE)
flags = (flags & ~LK_TYPE_MASK) | LK_EXCLUSIVE;
flags &= ~LK_INTERLOCK;
}
break;
default:
result = VOP_LOCK1_APV(&ufs_vnodeops, ap);
}
return (result);
#else
return (VOP_LOCK1_APV(&ufs_vnodeops, ap));
#endif
}
#ifdef INVARIANTS
static int
ffs_unlock_debug(struct vop_unlock_args *ap)
{
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
if (ip->i_flag & UFS_INODE_FLAG_LAZY_MASK_ASSERTABLE) {
if ((vp->v_mflag & VMP_LAZYLIST) == 0) {
VI_LOCK(vp);
VNASSERT((vp->v_mflag & VMP_LAZYLIST), vp,
("%s: modified vnode (%x) not on lazy list",
__func__, ip->i_flag));
VI_UNLOCK(vp);
}
}
return (VOP_UNLOCK_APV(&ufs_vnodeops, ap));
}
#endif
static int
ffs_read_hole(struct uio *uio, long xfersize, long *size)
{
ssize_t saved_resid, tlen;
int error;
while (xfersize > 0) {
tlen = min(xfersize, ZERO_REGION_SIZE);
saved_resid = uio->uio_resid;
error = vn_io_fault_uiomove(__DECONST(void *, zero_region),
tlen, uio);
if (error != 0)
return (error);
tlen = saved_resid - uio->uio_resid;
xfersize -= tlen;
*size -= tlen;
}
return (0);
}
/*
* Vnode op for reading.
*/
static int
ffs_read(ap)
struct vop_read_args /* {
struct vnode *a_vp;
struct uio *a_uio;
int a_ioflag;
struct ucred *a_cred;
} */ *ap;
{
struct vnode *vp;
struct inode *ip;
struct uio *uio;
struct fs *fs;
struct buf *bp;
ufs_lbn_t lbn, nextlbn;
off_t bytesinfile;
long size, xfersize, blkoffset;
ssize_t orig_resid;
int bflag, error, ioflag, seqcount;
vp = ap->a_vp;
uio = ap->a_uio;
ioflag = ap->a_ioflag;
if (ap->a_ioflag & IO_EXT)
#ifdef notyet
return (ffs_extread(vp, uio, ioflag));
#else
panic("ffs_read+IO_EXT");
#endif
#ifdef DIRECTIO
if ((ioflag & IO_DIRECT) != 0) {
int workdone;
error = ffs_rawread(vp, uio, &workdone);
if (error != 0 || workdone != 0)
return error;
}
#endif
seqcount = ap->a_ioflag >> IO_SEQSHIFT;
ip = VTOI(vp);
#ifdef INVARIANTS
if (uio->uio_rw != UIO_READ)
panic("ffs_read: mode");
if (vp->v_type == VLNK) {
if ((int)ip->i_size < vp->v_mount->mnt_maxsymlinklen)
panic("ffs_read: short symlink");
} else if (vp->v_type != VREG && vp->v_type != VDIR)
panic("ffs_read: type %d", vp->v_type);
#endif
orig_resid = uio->uio_resid;
KASSERT(orig_resid >= 0, ("ffs_read: uio->uio_resid < 0"));
if (orig_resid == 0)
return (0);
KASSERT(uio->uio_offset >= 0, ("ffs_read: uio->uio_offset < 0"));
fs = ITOFS(ip);
if (uio->uio_offset < ip->i_size &&
uio->uio_offset >= fs->fs_maxfilesize)
return (EOVERFLOW);
bflag = GB_UNMAPPED | (uio->uio_segflg == UIO_NOCOPY ? 0 : GB_NOSPARSE);
for (error = 0, bp = NULL; uio->uio_resid > 0; bp = NULL) {
if ((bytesinfile = ip->i_size - uio->uio_offset) <= 0)
break;
lbn = lblkno(fs, uio->uio_offset);
nextlbn = lbn + 1;
/*
* size of buffer. The buffer representing the
* end of the file is rounded up to the size of
* the block type ( fragment or full block,
* depending ).
*/
size = blksize(fs, ip, lbn);
blkoffset = blkoff(fs, uio->uio_offset);
/*
* The amount we want to transfer in this iteration is
* one FS block less the amount of the data before
* our startpoint (duh!)
*/
xfersize = fs->fs_bsize - blkoffset;
/*
* But if we actually want less than the block,
* or the file doesn't have a whole block more of data,
* then use the lesser number.
*/
if (uio->uio_resid < xfersize)
xfersize = uio->uio_resid;
if (bytesinfile < xfersize)
xfersize = bytesinfile;
if (lblktosize(fs, nextlbn) >= ip->i_size) {
/*
* Don't do readahead if this is the end of the file.
*/
error = bread_gb(vp, lbn, size, NOCRED, bflag, &bp);
} else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) {
/*
* Otherwise if we are allowed to cluster,
* grab as much as we can.
*
* XXX This may not be a win if we are not
* doing sequential access.
*/
error = cluster_read(vp, ip->i_size, lbn,
size, NOCRED, blkoffset + uio->uio_resid,
seqcount, bflag, &bp);
} else if (seqcount > 1) {
/*
* If we are NOT allowed to cluster, then
* if we appear to be acting sequentially,
* fire off a request for a readahead
* as well as a read. Note that the 4th and 5th
* arguments point to arrays of the size specified in
* the 6th argument.
*/
u_int nextsize = blksize(fs, ip, nextlbn);
error = breadn_flags(vp, lbn, lbn, size, &nextlbn,
&nextsize, 1, NOCRED, bflag, NULL, &bp);
} else {
/*
* Failing all of the above, just read what the
* user asked for. Interestingly, the same as
* the first option above.
*/
error = bread_gb(vp, lbn, size, NOCRED, bflag, &bp);
}
if (error == EJUSTRETURN) {
error = ffs_read_hole(uio, xfersize, &size);
if (error == 0)
continue;
}
if (error != 0) {
brelse(bp);
bp = NULL;
break;
}
/*
* We should only get non-zero b_resid when an I/O error
* has occurred, which should cause us to break above.
* However, if the short read did not cause an error,
* then we want to ensure that we do not uiomove bad
* or uninitialized data.
*/
size -= bp->b_resid;
if (size < xfersize) {
if (size == 0)
break;
xfersize = size;
}
if (buf_mapped(bp)) {
error = vn_io_fault_uiomove((char *)bp->b_data +
blkoffset, (int)xfersize, uio);
} else {
error = vn_io_fault_pgmove(bp->b_pages, blkoffset,
(int)xfersize, uio);
}
if (error)
break;
vfs_bio_brelse(bp, ioflag);
}
/*
* This can only happen in the case of an error
* because the loop above resets bp to NULL on each iteration
* and on normal completion has not set a new value into it.
* so it must have come from a 'break' statement
*/
if (bp != NULL)
vfs_bio_brelse(bp, ioflag);
if ((error == 0 || uio->uio_resid != orig_resid) &&
(vp->v_mount->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
UFS_INODE_SET_FLAG_SHARED(ip, IN_ACCESS);
return (error);
}
/*
* Vnode op for writing.
*/
static int
ffs_write(ap)
struct vop_write_args /* {
struct vnode *a_vp;
struct uio *a_uio;
int a_ioflag;
struct ucred *a_cred;
} */ *ap;
{
struct vnode *vp;
struct uio *uio;
struct inode *ip;
struct fs *fs;
struct buf *bp;
ufs_lbn_t lbn;
off_t osize;
ssize_t resid;
int seqcount;
int blkoffset, error, flags, ioflag, size, xfersize;
vp = ap->a_vp;
uio = ap->a_uio;
ioflag = ap->a_ioflag;
if (ap->a_ioflag & IO_EXT)
#ifdef notyet
return (ffs_extwrite(vp, uio, ioflag, ap->a_cred));
#else
panic("ffs_write+IO_EXT");
#endif
seqcount = ap->a_ioflag >> IO_SEQSHIFT;
ip = VTOI(vp);
#ifdef INVARIANTS
if (uio->uio_rw != UIO_WRITE)
panic("ffs_write: mode");
#endif
switch (vp->v_type) {
case VREG:
if (ioflag & IO_APPEND)
uio->uio_offset = ip->i_size;
if ((ip->i_flags & APPEND) && uio->uio_offset != ip->i_size)
return (EPERM);
/* FALLTHROUGH */
case VLNK:
break;
case VDIR:
panic("ffs_write: dir write");
break;
default:
panic("ffs_write: type %p %d (%d,%d)", vp, (int)vp->v_type,
(int)uio->uio_offset,
(int)uio->uio_resid
);
}
KASSERT(uio->uio_resid >= 0, ("ffs_write: uio->uio_resid < 0"));
KASSERT(uio->uio_offset >= 0, ("ffs_write: uio->uio_offset < 0"));
fs = ITOFS(ip);
if ((uoff_t)uio->uio_offset + uio->uio_resid > fs->fs_maxfilesize)
return (EFBIG);
/*
* Maybe this should be above the vnode op call, but so long as
* file servers have no limits, I don't think it matters.
*/
if (vn_rlimit_fsize(vp, uio, uio->uio_td))
return (EFBIG);
resid = uio->uio_resid;
osize = ip->i_size;
if (seqcount > BA_SEQMAX)
flags = BA_SEQMAX << BA_SEQSHIFT;
else
flags = seqcount << BA_SEQSHIFT;
if (ioflag & IO_SYNC)
flags |= IO_SYNC;
flags |= BA_UNMAPPED;
for (error = 0; uio->uio_resid > 0;) {
lbn = lblkno(fs, uio->uio_offset);
blkoffset = blkoff(fs, uio->uio_offset);
xfersize = fs->fs_bsize - blkoffset;
if (uio->uio_resid < xfersize)
xfersize = uio->uio_resid;
if (uio->uio_offset + xfersize > ip->i_size)
vnode_pager_setsize(vp, uio->uio_offset + xfersize);
/*
* We must perform a read-before-write if the transfer size
* does not cover the entire buffer.
*/
if (fs->fs_bsize > xfersize)
flags |= BA_CLRBUF;
else
flags &= ~BA_CLRBUF;
/* XXX is uio->uio_offset the right thing here? */
error = UFS_BALLOC(vp, uio->uio_offset, xfersize,
ap->a_cred, flags, &bp);
if (error != 0) {
vnode_pager_setsize(vp, ip->i_size);
break;
}
if ((ioflag & (IO_SYNC|IO_INVAL)) == (IO_SYNC|IO_INVAL))
bp->b_flags |= B_NOCACHE;
if (uio->uio_offset + xfersize > ip->i_size) {
ip->i_size = uio->uio_offset + xfersize;
DIP_SET(ip, i_size, ip->i_size);
}
size = blksize(fs, ip, lbn) - bp->b_resid;
if (size < xfersize)
xfersize = size;
if (buf_mapped(bp)) {
error = vn_io_fault_uiomove((char *)bp->b_data +
blkoffset, (int)xfersize, uio);
} else {
error = vn_io_fault_pgmove(bp->b_pages, blkoffset,
(int)xfersize, uio);
}
/*
* If the buffer is not already filled and we encounter an
* error while trying to fill it, we have to clear out any
* garbage data from the pages instantiated for the buffer.
* If we do not, a failed uiomove() during a write can leave
* the prior contents of the pages exposed to a userland mmap.
*
* Note that we need only clear buffers with a transfer size
* equal to the block size because buffers with a shorter
* transfer size were cleared above by the call to UFS_BALLOC()
* with the BA_CLRBUF flag set.
*
* If the source region for uiomove identically mmaps the
* buffer, uiomove() performed the NOP copy, and the buffer
* content remains valid because the page fault handler
* validated the pages.
*/
if (error != 0 && (bp->b_flags & B_CACHE) == 0 &&
fs->fs_bsize == xfersize)
vfs_bio_clrbuf(bp);
vfs_bio_set_flags(bp, ioflag);
/*
* If IO_SYNC each buffer is written synchronously. Otherwise
* if we have a severe page deficiency write the buffer
* asynchronously. Otherwise try to cluster, and if that
* doesn't do it then either do an async write (if O_DIRECT),
* or a delayed write (if not).
*/
if (ioflag & IO_SYNC) {
(void)bwrite(bp);
} else if (vm_page_count_severe() ||
buf_dirty_count_severe() ||
(ioflag & IO_ASYNC)) {
bp->b_flags |= B_CLUSTEROK;
bawrite(bp);
} else if (xfersize + blkoffset == fs->fs_bsize) {
if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) {
bp->b_flags |= B_CLUSTEROK;
cluster_write(vp, bp, ip->i_size, seqcount,
GB_UNMAPPED);
} else {
bawrite(bp);
}
} else if (ioflag & IO_DIRECT) {
bp->b_flags |= B_CLUSTEROK;
bawrite(bp);
} else {
bp->b_flags |= B_CLUSTEROK;
bdwrite(bp);
}
if (error || xfersize == 0)
break;
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
}
/*
* If we successfully wrote any data, and we are not the superuser
* we clear the setuid and setgid bits as a precaution against
* tampering.
*/
if ((ip->i_mode & (ISUID | ISGID)) && resid > uio->uio_resid &&
ap->a_cred) {
if (priv_check_cred(ap->a_cred, PRIV_VFS_RETAINSUGID)) {
ip->i_mode &= ~(ISUID | ISGID);
DIP_SET(ip, i_mode, ip->i_mode);
}
}
if (error) {
if (ioflag & IO_UNIT) {
(void)ffs_truncate(vp, osize,
IO_NORMAL | (ioflag & IO_SYNC), ap->a_cred);
uio->uio_offset -= resid - uio->uio_resid;
uio->uio_resid = resid;
}
- } else if (resid > uio->uio_resid && (ioflag & IO_SYNC))
+ } else if (resid > uio->uio_resid && (ioflag & IO_SYNC)) {
error = ffs_update(vp, 1);
+ if (ffs_fsfail_cleanup(VFSTOUFS(vp->v_mount), error))
+ error = ENXIO;
+ }
return (error);
}
/*
* Extended attribute area reading.
*/
static int
ffs_extread(struct vnode *vp, struct uio *uio, int ioflag)
{
struct inode *ip;
struct ufs2_dinode *dp;
struct fs *fs;
struct buf *bp;
ufs_lbn_t lbn, nextlbn;
off_t bytesinfile;
long size, xfersize, blkoffset;
ssize_t orig_resid;
int error;
ip = VTOI(vp);
fs = ITOFS(ip);
dp = ip->i_din2;
#ifdef INVARIANTS
if (uio->uio_rw != UIO_READ || fs->fs_magic != FS_UFS2_MAGIC)
panic("ffs_extread: mode");
#endif
orig_resid = uio->uio_resid;
KASSERT(orig_resid >= 0, ("ffs_extread: uio->uio_resid < 0"));
if (orig_resid == 0)
return (0);
KASSERT(uio->uio_offset >= 0, ("ffs_extread: uio->uio_offset < 0"));
for (error = 0, bp = NULL; uio->uio_resid > 0; bp = NULL) {
if ((bytesinfile = dp->di_extsize - uio->uio_offset) <= 0)
break;
lbn = lblkno(fs, uio->uio_offset);
nextlbn = lbn + 1;
/*
* size of buffer. The buffer representing the
* end of the file is rounded up to the size of
* the block type ( fragment or full block,
* depending ).
*/
size = sblksize(fs, dp->di_extsize, lbn);
blkoffset = blkoff(fs, uio->uio_offset);
/*
* The amount we want to transfer in this iteration is
* one FS block less the amount of the data before
* our startpoint (duh!)
*/
xfersize = fs->fs_bsize - blkoffset;
/*
* But if we actually want less than the block,
* or the file doesn't have a whole block more of data,
* then use the lesser number.
*/
if (uio->uio_resid < xfersize)
xfersize = uio->uio_resid;
if (bytesinfile < xfersize)
xfersize = bytesinfile;
if (lblktosize(fs, nextlbn) >= dp->di_extsize) {
/*
* Don't do readahead if this is the end of the info.
*/
error = bread(vp, -1 - lbn, size, NOCRED, &bp);
} else {
/*
* If we have a second block, then
* fire off a request for a readahead
* as well as a read. Note that the 4th and 5th
* arguments point to arrays of the size specified in
* the 6th argument.
*/
u_int nextsize = sblksize(fs, dp->di_extsize, nextlbn);
nextlbn = -1 - nextlbn;
error = breadn(vp, -1 - lbn,
size, &nextlbn, &nextsize, 1, NOCRED, &bp);
}
if (error) {
brelse(bp);
bp = NULL;
break;
}
/*
* We should only get non-zero b_resid when an I/O error
* has occurred, which should cause us to break above.
* However, if the short read did not cause an error,
* then we want to ensure that we do not uiomove bad
* or uninitialized data.
*/
size -= bp->b_resid;
if (size < xfersize) {
if (size == 0)
break;
xfersize = size;
}
error = uiomove((char *)bp->b_data + blkoffset,
(int)xfersize, uio);
if (error)
break;
vfs_bio_brelse(bp, ioflag);
}
/*
* This can only happen in the case of an error
* because the loop above resets bp to NULL on each iteration
* and on normal completion has not set a new value into it.
* so it must have come from a 'break' statement
*/
if (bp != NULL)
vfs_bio_brelse(bp, ioflag);
return (error);
}
/*
* Extended attribute area writing.
*/
static int
ffs_extwrite(struct vnode *vp, struct uio *uio, int ioflag, struct ucred *ucred)
{
struct inode *ip;
struct ufs2_dinode *dp;
struct fs *fs;
struct buf *bp;
ufs_lbn_t lbn;
off_t osize;
ssize_t resid;
int blkoffset, error, flags, size, xfersize;
ip = VTOI(vp);
fs = ITOFS(ip);
dp = ip->i_din2;
#ifdef INVARIANTS
if (uio->uio_rw != UIO_WRITE || fs->fs_magic != FS_UFS2_MAGIC)
panic("ffs_extwrite: mode");
#endif
if (ioflag & IO_APPEND)
uio->uio_offset = dp->di_extsize;
KASSERT(uio->uio_offset >= 0, ("ffs_extwrite: uio->uio_offset < 0"));
KASSERT(uio->uio_resid >= 0, ("ffs_extwrite: uio->uio_resid < 0"));
if ((uoff_t)uio->uio_offset + uio->uio_resid >
UFS_NXADDR * fs->fs_bsize)
return (EFBIG);
resid = uio->uio_resid;
osize = dp->di_extsize;
flags = IO_EXT;
if (ioflag & IO_SYNC)
flags |= IO_SYNC;
for (error = 0; uio->uio_resid > 0;) {
lbn = lblkno(fs, uio->uio_offset);
blkoffset = blkoff(fs, uio->uio_offset);
xfersize = fs->fs_bsize - blkoffset;
if (uio->uio_resid < xfersize)
xfersize = uio->uio_resid;
/*
* We must perform a read-before-write if the transfer size
* does not cover the entire buffer.
*/
if (fs->fs_bsize > xfersize)
flags |= BA_CLRBUF;
else
flags &= ~BA_CLRBUF;
error = UFS_BALLOC(vp, uio->uio_offset, xfersize,
ucred, flags, &bp);
if (error != 0)
break;
/*
* If the buffer is not valid we have to clear out any
* garbage data from the pages instantiated for the buffer.
* If we do not, a failed uiomove() during a write can leave
* the prior contents of the pages exposed to a userland
* mmap(). XXX deal with uiomove() errors a better way.
*/
if ((bp->b_flags & B_CACHE) == 0 && fs->fs_bsize <= xfersize)
vfs_bio_clrbuf(bp);
if (uio->uio_offset + xfersize > dp->di_extsize)
dp->di_extsize = uio->uio_offset + xfersize;
size = sblksize(fs, dp->di_extsize, lbn) - bp->b_resid;
if (size < xfersize)
xfersize = size;
error =
uiomove((char *)bp->b_data + blkoffset, (int)xfersize, uio);
vfs_bio_set_flags(bp, ioflag);
/*
* If IO_SYNC each buffer is written synchronously. Otherwise
* if we have a severe page deficiency write the buffer
* asynchronously. Otherwise try to cluster, and if that
* doesn't do it then either do an async write (if O_DIRECT),
* or a delayed write (if not).
*/
if (ioflag & IO_SYNC) {
(void)bwrite(bp);
} else if (vm_page_count_severe() ||
buf_dirty_count_severe() ||
xfersize + blkoffset == fs->fs_bsize ||
(ioflag & (IO_ASYNC | IO_DIRECT)))
bawrite(bp);
else
bdwrite(bp);
if (error || xfersize == 0)
break;
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
}
/*
* If we successfully wrote any data, and we are not the superuser
* we clear the setuid and setgid bits as a precaution against
* tampering.
*/
if ((ip->i_mode & (ISUID | ISGID)) && resid > uio->uio_resid && ucred) {
if (priv_check_cred(ucred, PRIV_VFS_RETAINSUGID)) {
ip->i_mode &= ~(ISUID | ISGID);
dp->di_mode = ip->i_mode;
}
}
if (error) {
if (ioflag & IO_UNIT) {
(void)ffs_truncate(vp, osize,
IO_EXT | (ioflag&IO_SYNC), ucred);
uio->uio_offset -= resid - uio->uio_resid;
uio->uio_resid = resid;
}
} else if (resid > uio->uio_resid && (ioflag & IO_SYNC))
error = ffs_update(vp, 1);
return (error);
}
/*
* Vnode operating to retrieve a named extended attribute.
*
* Locate a particular EA (nspace:name) in the area (ptr:length), and return
* the length of the EA, and possibly the pointer to the entry and to the data.
*/
static int
ffs_findextattr(u_char *ptr, u_int length, int nspace, const char *name,
struct extattr **eapp, u_char **eac)
{
struct extattr *eap, *eaend;
size_t nlen;
nlen = strlen(name);
KASSERT(ALIGNED_TO(ptr, struct extattr), ("unaligned"));
eap = (struct extattr *)ptr;
eaend = (struct extattr *)(ptr + length);
for (; eap < eaend; eap = EXTATTR_NEXT(eap)) {
/* make sure this entry is complete */
if (EXTATTR_NEXT(eap) > eaend)
break;
if (eap->ea_namespace != nspace || eap->ea_namelength != nlen
|| memcmp(eap->ea_name, name, nlen) != 0)
continue;
if (eapp != NULL)
*eapp = eap;
if (eac != NULL)
*eac = EXTATTR_CONTENT(eap);
return (EXTATTR_CONTENT_SIZE(eap));
}
return (-1);
}
static int
ffs_rdextattr(u_char **p, struct vnode *vp, struct thread *td, int extra)
{
struct inode *ip;
struct ufs2_dinode *dp;
struct fs *fs;
struct uio luio;
struct iovec liovec;
u_int easize;
int error;
u_char *eae;
ip = VTOI(vp);
fs = ITOFS(ip);
dp = ip->i_din2;
easize = dp->di_extsize;
if ((uoff_t)easize + extra > UFS_NXADDR * fs->fs_bsize)
return (EFBIG);
eae = malloc(easize + extra, M_TEMP, M_WAITOK);
liovec.iov_base = eae;
liovec.iov_len = easize;
luio.uio_iov = &liovec;
luio.uio_iovcnt = 1;
luio.uio_offset = 0;
luio.uio_resid = easize;
luio.uio_segflg = UIO_SYSSPACE;
luio.uio_rw = UIO_READ;
luio.uio_td = td;
error = ffs_extread(vp, &luio, IO_EXT | IO_SYNC);
if (error) {
free(eae, M_TEMP);
return(error);
}
*p = eae;
return (0);
}
static void
ffs_lock_ea(struct vnode *vp)
{
struct inode *ip;
ip = VTOI(vp);
VI_LOCK(vp);
while (ip->i_flag & IN_EA_LOCKED) {
UFS_INODE_SET_FLAG(ip, IN_EA_LOCKWAIT);
msleep(&ip->i_ea_refs, &vp->v_interlock, PINOD + 2, "ufs_ea",
0);
}
UFS_INODE_SET_FLAG(ip, IN_EA_LOCKED);
VI_UNLOCK(vp);
}
static void
ffs_unlock_ea(struct vnode *vp)
{
struct inode *ip;
ip = VTOI(vp);
VI_LOCK(vp);
if (ip->i_flag & IN_EA_LOCKWAIT)
wakeup(&ip->i_ea_refs);
ip->i_flag &= ~(IN_EA_LOCKED | IN_EA_LOCKWAIT);
VI_UNLOCK(vp);
}
static int
ffs_open_ea(struct vnode *vp, struct ucred *cred, struct thread *td)
{
struct inode *ip;
struct ufs2_dinode *dp;
int error;
ip = VTOI(vp);
ffs_lock_ea(vp);
if (ip->i_ea_area != NULL) {
ip->i_ea_refs++;
ffs_unlock_ea(vp);
return (0);
}
dp = ip->i_din2;
error = ffs_rdextattr(&ip->i_ea_area, vp, td, 0);
if (error) {
ffs_unlock_ea(vp);
return (error);
}
ip->i_ea_len = dp->di_extsize;
ip->i_ea_error = 0;
ip->i_ea_refs++;
ffs_unlock_ea(vp);
return (0);
}
/*
* Vnode extattr transaction commit/abort
*/
static int
ffs_close_ea(struct vnode *vp, int commit, struct ucred *cred, struct thread *td)
{
struct inode *ip;
struct uio luio;
struct iovec liovec;
int error;
struct ufs2_dinode *dp;
ip = VTOI(vp);
ffs_lock_ea(vp);
if (ip->i_ea_area == NULL) {
ffs_unlock_ea(vp);
return (EINVAL);
}
dp = ip->i_din2;
error = ip->i_ea_error;
if (commit && error == 0) {
ASSERT_VOP_ELOCKED(vp, "ffs_close_ea commit");
if (cred == NOCRED)
cred = vp->v_mount->mnt_cred;
liovec.iov_base = ip->i_ea_area;
liovec.iov_len = ip->i_ea_len;
luio.uio_iov = &liovec;
luio.uio_iovcnt = 1;
luio.uio_offset = 0;
luio.uio_resid = ip->i_ea_len;
luio.uio_segflg = UIO_SYSSPACE;
luio.uio_rw = UIO_WRITE;
luio.uio_td = td;
/* XXX: I'm not happy about truncating to zero size */
if (ip->i_ea_len < dp->di_extsize)
error = ffs_truncate(vp, 0, IO_EXT, cred);
error = ffs_extwrite(vp, &luio, IO_EXT | IO_SYNC, cred);
}
if (--ip->i_ea_refs == 0) {
free(ip->i_ea_area, M_TEMP);
ip->i_ea_area = NULL;
ip->i_ea_len = 0;
ip->i_ea_error = 0;
}
ffs_unlock_ea(vp);
return (error);
}
/*
* Vnode extattr strategy routine for fifos.
*
* We need to check for a read or write of the external attributes.
* Otherwise we just fall through and do the usual thing.
*/
static int
ffsext_strategy(struct vop_strategy_args *ap)
/*
struct vop_strategy_args {
struct vnodeop_desc *a_desc;
struct vnode *a_vp;
struct buf *a_bp;
};
*/
{
struct vnode *vp;
daddr_t lbn;
vp = ap->a_vp;
lbn = ap->a_bp->b_lblkno;
if (I_IS_UFS2(VTOI(vp)) && lbn < 0 && lbn >= -UFS_NXADDR)
return (VOP_STRATEGY_APV(&ufs_vnodeops, ap));
if (vp->v_type == VFIFO)
return (VOP_STRATEGY_APV(&ufs_fifoops, ap));
panic("spec nodes went here");
}
/*
* Vnode extattr transaction commit/abort
*/
static int
ffs_openextattr(struct vop_openextattr_args *ap)
/*
struct vop_openextattr_args {
struct vnodeop_desc *a_desc;
struct vnode *a_vp;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
return (ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td));
}
/*
* Vnode extattr transaction commit/abort
*/
static int
ffs_closeextattr(struct vop_closeextattr_args *ap)
/*
struct vop_closeextattr_args {
struct vnodeop_desc *a_desc;
struct vnode *a_vp;
int a_commit;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
if (ap->a_commit && (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY))
return (EROFS);
return (ffs_close_ea(ap->a_vp, ap->a_commit, ap->a_cred, ap->a_td));
}
/*
* Vnode operation to remove a named attribute.
*/
static int
ffs_deleteextattr(struct vop_deleteextattr_args *ap)
/*
vop_deleteextattr {
IN struct vnode *a_vp;
IN int a_attrnamespace;
IN const char *a_name;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
struct inode *ip;
struct extattr *eap;
uint32_t ul;
int olen, error, i, easize;
u_char *eae;
void *tmp;
ip = VTOI(ap->a_vp);
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
if (strlen(ap->a_name) == 0)
return (EINVAL);
if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
ap->a_cred, ap->a_td, VWRITE);
if (error) {
/*
* ffs_lock_ea is not needed there, because the vnode
* must be exclusively locked.
*/
if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
ip->i_ea_error = error;
return (error);
}
error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
if (error)
return (error);
/* CEM: delete could be done in-place instead */
eae = malloc(ip->i_ea_len, M_TEMP, M_WAITOK);
bcopy(ip->i_ea_area, eae, ip->i_ea_len);
easize = ip->i_ea_len;
olen = ffs_findextattr(eae, easize, ap->a_attrnamespace, ap->a_name,
&eap, NULL);
if (olen == -1) {
/* delete but nonexistent */
free(eae, M_TEMP);
ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
return (ENOATTR);
}
ul = eap->ea_length;
i = (u_char *)EXTATTR_NEXT(eap) - eae;
bcopy(EXTATTR_NEXT(eap), eap, easize - i);
easize -= ul;
tmp = ip->i_ea_area;
ip->i_ea_area = eae;
ip->i_ea_len = easize;
free(tmp, M_TEMP);
error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
return (error);
}
/*
* Vnode operation to retrieve a named extended attribute.
*/
static int
ffs_getextattr(struct vop_getextattr_args *ap)
/*
vop_getextattr {
IN struct vnode *a_vp;
IN int a_attrnamespace;
IN const char *a_name;
INOUT struct uio *a_uio;
OUT size_t *a_size;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
struct inode *ip;
u_char *eae, *p;
unsigned easize;
int error, ealen;
ip = VTOI(ap->a_vp);
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
ap->a_cred, ap->a_td, VREAD);
if (error)
return (error);
error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
if (error)
return (error);
eae = ip->i_ea_area;
easize = ip->i_ea_len;
ealen = ffs_findextattr(eae, easize, ap->a_attrnamespace, ap->a_name,
NULL, &p);
if (ealen >= 0) {
error = 0;
if (ap->a_size != NULL)
*ap->a_size = ealen;
else if (ap->a_uio != NULL)
error = uiomove(p, ealen, ap->a_uio);
} else
error = ENOATTR;
ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
return (error);
}
/*
* Vnode operation to retrieve extended attributes on a vnode.
*/
static int
ffs_listextattr(struct vop_listextattr_args *ap)
/*
vop_listextattr {
IN struct vnode *a_vp;
IN int a_attrnamespace;
INOUT struct uio *a_uio;
OUT size_t *a_size;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
struct inode *ip;
struct extattr *eap, *eaend;
int error, ealen;
ip = VTOI(ap->a_vp);
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
ap->a_cred, ap->a_td, VREAD);
if (error)
return (error);
error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
if (error)
return (error);
error = 0;
if (ap->a_size != NULL)
*ap->a_size = 0;
KASSERT(ALIGNED_TO(ip->i_ea_area, struct extattr), ("unaligned"));
eap = (struct extattr *)ip->i_ea_area;
eaend = (struct extattr *)(ip->i_ea_area + ip->i_ea_len);
for (; error == 0 && eap < eaend; eap = EXTATTR_NEXT(eap)) {
/* make sure this entry is complete */
if (EXTATTR_NEXT(eap) > eaend)
break;
if (eap->ea_namespace != ap->a_attrnamespace)
continue;
ealen = eap->ea_namelength;
if (ap->a_size != NULL)
*ap->a_size += ealen + 1;
else if (ap->a_uio != NULL)
error = uiomove(&eap->ea_namelength, ealen + 1,
ap->a_uio);
}
ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
return (error);
}
/*
* Vnode operation to set a named attribute.
*/
static int
ffs_setextattr(struct vop_setextattr_args *ap)
/*
vop_setextattr {
IN struct vnode *a_vp;
IN int a_attrnamespace;
IN const char *a_name;
INOUT struct uio *a_uio;
IN struct ucred *a_cred;
IN struct thread *a_td;
};
*/
{
struct inode *ip;
struct fs *fs;
struct extattr *eap;
uint32_t ealength, ul;
ssize_t ealen;
int olen, eapad1, eapad2, error, i, easize;
u_char *eae;
void *tmp;
ip = VTOI(ap->a_vp);
fs = ITOFS(ip);
if (ap->a_vp->v_type == VCHR || ap->a_vp->v_type == VBLK)
return (EOPNOTSUPP);
if (strlen(ap->a_name) == 0)
return (EINVAL);
/* XXX Now unsupported API to delete EAs using NULL uio. */
if (ap->a_uio == NULL)
return (EOPNOTSUPP);
if (ap->a_vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
ealen = ap->a_uio->uio_resid;
if (ealen < 0 || ealen > lblktosize(fs, UFS_NXADDR))
return (EINVAL);
error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
ap->a_cred, ap->a_td, VWRITE);
if (error) {
/*
* ffs_lock_ea is not needed there, because the vnode
* must be exclusively locked.
*/
if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
ip->i_ea_error = error;
return (error);
}
error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
if (error)
return (error);
ealength = sizeof(uint32_t) + 3 + strlen(ap->a_name);
eapad1 = roundup2(ealength, 8) - ealength;
eapad2 = roundup2(ealen, 8) - ealen;
ealength += eapad1 + ealen + eapad2;
/*
* CEM: rewrites of the same size or smaller could be done in-place
* instead. (We don't acquire any fine-grained locks in here either,
* so we could also do bigger writes in-place.)
*/
eae = malloc(ip->i_ea_len + ealength, M_TEMP, M_WAITOK);
bcopy(ip->i_ea_area, eae, ip->i_ea_len);
easize = ip->i_ea_len;
olen = ffs_findextattr(eae, easize, ap->a_attrnamespace, ap->a_name,
&eap, NULL);
if (olen == -1) {
/* new, append at end */
KASSERT(ALIGNED_TO(eae + easize, struct extattr),
("unaligned"));
eap = (struct extattr *)(eae + easize);
easize += ealength;
} else {
ul = eap->ea_length;
i = (u_char *)EXTATTR_NEXT(eap) - eae;
if (ul != ealength) {
bcopy(EXTATTR_NEXT(eap), (u_char *)eap + ealength,
easize - i);
easize += (ealength - ul);
}
}
if (easize > lblktosize(fs, UFS_NXADDR)) {
free(eae, M_TEMP);
ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
ip->i_ea_error = ENOSPC;
return (ENOSPC);
}
eap->ea_length = ealength;
eap->ea_namespace = ap->a_attrnamespace;
eap->ea_contentpadlen = eapad2;
eap->ea_namelength = strlen(ap->a_name);
memcpy(eap->ea_name, ap->a_name, strlen(ap->a_name));
bzero(&eap->ea_name[strlen(ap->a_name)], eapad1);
error = uiomove(EXTATTR_CONTENT(eap), ealen, ap->a_uio);
if (error) {
free(eae, M_TEMP);
ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
ip->i_ea_error = error;
return (error);
}
bzero((u_char *)EXTATTR_CONTENT(eap) + ealen, eapad2);
tmp = ip->i_ea_area;
ip->i_ea_area = eae;
ip->i_ea_len = easize;
free(tmp, M_TEMP);
error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
return (error);
}
/*
* Vnode pointer to File handle
*/
static int
ffs_vptofh(struct vop_vptofh_args *ap)
/*
vop_vptofh {
IN struct vnode *a_vp;
IN struct fid *a_fhp;
};
*/
{
struct inode *ip;
struct ufid *ufhp;
ip = VTOI(ap->a_vp);
ufhp = (struct ufid *)ap->a_fhp;
ufhp->ufid_len = sizeof(struct ufid);
ufhp->ufid_ino = ip->i_number;
ufhp->ufid_gen = ip->i_gen;
return (0);
}
SYSCTL_DECL(_vfs_ffs);
static int use_buf_pager = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, use_buf_pager, CTLFLAG_RWTUN, &use_buf_pager, 0,
"Always use buffer pager instead of bmap");
static daddr_t
ffs_gbp_getblkno(struct vnode *vp, vm_ooffset_t off)
{
return (lblkno(VFSTOUFS(vp->v_mount)->um_fs, off));
}
static int
ffs_gbp_getblksz(struct vnode *vp, daddr_t lbn)
{
return (blksize(VFSTOUFS(vp->v_mount)->um_fs, VTOI(vp), lbn));
}
static int
ffs_getpages(struct vop_getpages_args *ap)
{
struct vnode *vp;
struct ufsmount *um;
vp = ap->a_vp;
um = VFSTOUFS(vp->v_mount);
if (!use_buf_pager && um->um_devvp->v_bufobj.bo_bsize <= PAGE_SIZE)
return (vnode_pager_generic_getpages(vp, ap->a_m, ap->a_count,
ap->a_rbehind, ap->a_rahead, NULL, NULL));
return (vfs_bio_getpages(vp, ap->a_m, ap->a_count, ap->a_rbehind,
ap->a_rahead, ffs_gbp_getblkno, ffs_gbp_getblksz));
}
static int
ffs_getpages_async(struct vop_getpages_async_args *ap)
{
struct vnode *vp;
struct ufsmount *um;
bool do_iodone;
int error;
vp = ap->a_vp;
um = VFSTOUFS(vp->v_mount);
do_iodone = true;
if (um->um_devvp->v_bufobj.bo_bsize <= PAGE_SIZE) {
error = vnode_pager_generic_getpages(vp, ap->a_m, ap->a_count,
ap->a_rbehind, ap->a_rahead, ap->a_iodone, ap->a_arg);
if (error == 0)
do_iodone = false;
} else {
error = vfs_bio_getpages(vp, ap->a_m, ap->a_count,
ap->a_rbehind, ap->a_rahead, ffs_gbp_getblkno,
ffs_gbp_getblksz);
}
if (do_iodone && ap->a_iodone != NULL)
ap->a_iodone(ap->a_arg, ap->a_m, ap->a_count, error);
return (error);
}
Index: head/sys/ufs/ffs/softdep.h
===================================================================
--- head/sys/ufs/ffs/softdep.h (revision 361490)
+++ head/sys/ufs/ffs/softdep.h (revision 361491)
@@ -1,1120 +1,1121 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright 1998, 2000 Marshall Kirk McKusick. All Rights Reserved.
*
* The soft updates code is derived from the appendix of a University
* of Michigan technical report (Gregory R. Ganger and Yale N. Patt,
* "Soft Updates: A Solution to the Metadata Update Problem in File
* Systems", CSE-TR-254-95, August 1995).
*
* Further information about soft updates can be obtained from:
*
* Marshall Kirk McKusick http://www.mckusick.com/softdep/
* 1614 Oxford Street mckusick@mckusick.com
* Berkeley, CA 94709-1608 +1-510-843-9542
* USA
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY MARSHALL KIRK MCKUSICK ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL MARSHALL KIRK MCKUSICK BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)softdep.h 9.7 (McKusick) 6/21/00
* $FreeBSD$
*/
#include <sys/queue.h>
/*
* Allocation dependencies are handled with undo/redo on the in-memory
* copy of the data. A particular data dependency is eliminated when
* it is ALLCOMPLETE: that is ATTACHED, DEPCOMPLETE, and COMPLETE.
*
* The ATTACHED flag means that the data is not currently being written
* to disk.
*
* The UNDONE flag means that the data has been rolled back to a safe
* state for writing to the disk. When the I/O completes, the data is
* restored to its current form and the state reverts to ATTACHED.
* The data must be locked throughout the rollback, I/O, and roll
* forward so that the rolled back information is never visible to
* user processes.
*
* The COMPLETE flag indicates that the item has been written. For example,
* a dependency that requires that an inode be written will be marked
* COMPLETE after the inode has been written to disk.
*
* The DEPCOMPLETE flag indicates the completion of any other
* dependencies such as the writing of a cylinder group map has been
* completed. A dependency structure may be freed only when both it
* and its dependencies have completed and any rollbacks that are in
* progress have finished as indicated by the set of ALLCOMPLETE flags
* all being set.
*
* The two MKDIR flags indicate additional dependencies that must be done
* when creating a new directory. MKDIR_BODY is cleared when the directory
* data block containing the "." and ".." entries has been written.
* MKDIR_PARENT is cleared when the parent inode with the increased link
* count for ".." has been written. When both MKDIR flags have been
* cleared, the DEPCOMPLETE flag is set to indicate that the directory
* dependencies have been completed. The writing of the directory inode
* itself sets the COMPLETE flag which then allows the directory entry for
* the new directory to be written to disk. The RMDIR flag marks a dirrem
* structure as representing the removal of a directory rather than a
* file. When the removal dependencies are completed, additional work needs
* to be done* (an additional decrement of the associated inode, and a
* decrement of the parent inode).
*
* The DIRCHG flag marks a diradd structure as representing the changing
* of an existing entry rather than the addition of a new one. When
* the update is complete the dirrem associated with the inode for
* the old name must be added to the worklist to do the necessary
* reference count decrement.
*
* The GOINGAWAY flag indicates that the data structure is frozen from
* further change until its dependencies have been completed and its
* resources freed after which it will be discarded.
*
* The IOSTARTED flag prevents multiple calls to the I/O start routine from
* doing multiple rollbacks.
*
* The NEWBLOCK flag marks pagedep structures that have just been allocated,
* so must be claimed by the inode before all dependencies are complete.
*
* The INPROGRESS flag marks worklist structures that are still on the
* worklist, but are being considered for action by some process.
*
* The UFS1FMT flag indicates that the inode being processed is a ufs1 format.
*
* The EXTDATA flag indicates that the allocdirect describes an
* extended-attributes dependency.
*
* The ONWORKLIST flag shows whether the structure is currently linked
* onto a worklist.
*
* The UNLINK* flags track the progress of updating the on-disk linked
* list of active but unlinked inodes. When an inode is first unlinked
* it is marked as UNLINKED. When its on-disk di_freelink has been
* written its UNLINKNEXT flags is set. When its predecessor in the
* list has its di_freelink pointing at us its UNLINKPREV is set.
* When the on-disk list can reach it from the superblock, its
* UNLINKONLIST flag is set. Once all of these flags are set, it
* is safe to let its last name be removed.
*/
#define ATTACHED 0x000001
#define UNDONE 0x000002
#define COMPLETE 0x000004
#define DEPCOMPLETE 0x000008
#define MKDIR_PARENT 0x000010 /* diradd, mkdir, jaddref, jsegdep only */
#define MKDIR_BODY 0x000020 /* diradd, mkdir, jaddref only */
#define RMDIR 0x000040 /* dirrem only */
#define DIRCHG 0x000080 /* diradd, dirrem only */
#define GOINGAWAY 0x000100 /* indirdep, jremref only */
#define IOSTARTED 0x000200 /* inodedep, pagedep, bmsafemap only */
#define DELAYEDFREE 0x000400 /* allocindirect free delayed. */
#define NEWBLOCK 0x000800 /* pagedep, jaddref only */
#define INPROGRESS 0x001000 /* dirrem, freeblks, freefrag, freefile only */
#define UFS1FMT 0x002000 /* indirdep only */
#define EXTDATA 0x004000 /* allocdirect only */
#define ONWORKLIST 0x008000
#define IOWAITING 0x010000 /* Thread is waiting for IO to complete. */
#define ONDEPLIST 0x020000 /* Structure is on a dependency list. */
#define UNLINKED 0x040000 /* inodedep has been unlinked. */
#define UNLINKNEXT 0x080000 /* inodedep has valid di_freelink */
#define UNLINKPREV 0x100000 /* inodedep is pointed at in the unlink list */
#define UNLINKONLIST 0x200000 /* inodedep is in the unlinked list on disk */
#define UNLINKLINKS (UNLINKNEXT | UNLINKPREV)
#define WRITESUCCEEDED 0x400000 /* the disk write completed successfully */
#define ALLCOMPLETE (ATTACHED | COMPLETE | DEPCOMPLETE)
#define PRINT_SOFTDEP_FLAGS "\20\27writesucceeded\26unlinkonlist" \
"\25unlinkprev\24unlinknext\23unlinked\22ondeplist\21iowaiting" \
"\20onworklist\17extdata\16ufs1fmt\15inprogress\14newblock" \
"\13delayedfree\12iostarted\11goingaway\10dirchg\7rmdir\6mkdir_body" \
"\5mkdir_parent\4depcomplete\3complete\2undone\1attached"
/*
* Values for each of the soft dependency types.
*/
#define D_UNUSED 0
#define D_FIRST D_PAGEDEP
#define D_PAGEDEP 1
#define D_INODEDEP 2
#define D_BMSAFEMAP 3
#define D_NEWBLK 4
#define D_ALLOCDIRECT 5
#define D_INDIRDEP 6
#define D_ALLOCINDIR 7
#define D_FREEFRAG 8
#define D_FREEBLKS 9
#define D_FREEFILE 10
#define D_DIRADD 11
#define D_MKDIR 12
#define D_DIRREM 13
#define D_NEWDIRBLK 14
#define D_FREEWORK 15
#define D_FREEDEP 16
#define D_JADDREF 17
#define D_JREMREF 18
#define D_JMVREF 19
#define D_JNEWBLK 20
#define D_JFREEBLK 21
#define D_JFREEFRAG 22
#define D_JSEG 23
#define D_JSEGDEP 24
#define D_SBDEP 25
#define D_JTRUNC 26
#define D_JFSYNC 27
#define D_SENTINEL 28
#define D_LAST D_SENTINEL
/*
* The workitem queue.
*
* It is sometimes useful and/or necessary to clean up certain dependencies
* in the background rather than during execution of an application process
* or interrupt service routine. To realize this, we append dependency
* structures corresponding to such tasks to a "workitem" queue. In a soft
* updates implementation, most pending workitems should not wait for more
* than a couple of seconds, so the filesystem syncer process awakens once
* per second to process the items on the queue.
*/
/* LIST_HEAD(workhead, worklist); -- declared in buf.h */
/*
* Each request can be linked onto a work queue through its worklist structure.
* To avoid the need for a pointer to the structure itself, this structure
* MUST be declared FIRST in each type in which it appears! If more than one
* worklist is needed in the structure, then a wk_data field must be added
* and the macros below changed to use it.
*/
struct worklist {
LIST_ENTRY(worklist) wk_list; /* list of work requests */
struct mount *wk_mp; /* Mount we live in */
unsigned int wk_type:8, /* type of request */
wk_state:24; /* state flags */
#ifdef INVARIANTS
const char *wk_func; /* func where added / removed */
int wk_line; /* line where added / removed */
LIST_ENTRY(worklist) wk_all; /* list of deps of this type */
#endif
};
#define WK_DATA(wk) ((void *)(wk))
#define WK_PAGEDEP(wk) ((struct pagedep *)(wk))
#define WK_INODEDEP(wk) ((struct inodedep *)(wk))
#define WK_BMSAFEMAP(wk) ((struct bmsafemap *)(wk))
#define WK_NEWBLK(wk) ((struct newblk *)(wk))
#define WK_ALLOCDIRECT(wk) ((struct allocdirect *)(wk))
#define WK_INDIRDEP(wk) ((struct indirdep *)(wk))
#define WK_ALLOCINDIR(wk) ((struct allocindir *)(wk))
#define WK_FREEFRAG(wk) ((struct freefrag *)(wk))
#define WK_FREEBLKS(wk) ((struct freeblks *)(wk))
#define WK_FREEWORK(wk) ((struct freework *)(wk))
#define WK_FREEFILE(wk) ((struct freefile *)(wk))
#define WK_DIRADD(wk) ((struct diradd *)(wk))
#define WK_MKDIR(wk) ((struct mkdir *)(wk))
#define WK_DIRREM(wk) ((struct dirrem *)(wk))
#define WK_NEWDIRBLK(wk) ((struct newdirblk *)(wk))
#define WK_JADDREF(wk) ((struct jaddref *)(wk))
#define WK_JREMREF(wk) ((struct jremref *)(wk))
#define WK_JMVREF(wk) ((struct jmvref *)(wk))
#define WK_JSEGDEP(wk) ((struct jsegdep *)(wk))
#define WK_JSEG(wk) ((struct jseg *)(wk))
#define WK_JNEWBLK(wk) ((struct jnewblk *)(wk))
#define WK_JFREEBLK(wk) ((struct jfreeblk *)(wk))
#define WK_FREEDEP(wk) ((struct freedep *)(wk))
#define WK_JFREEFRAG(wk) ((struct jfreefrag *)(wk))
#define WK_SBDEP(wk) ((struct sbdep *)(wk))
#define WK_JTRUNC(wk) ((struct jtrunc *)(wk))
#define WK_JFSYNC(wk) ((struct jfsync *)(wk))
/*
* Various types of lists
*/
LIST_HEAD(dirremhd, dirrem);
LIST_HEAD(diraddhd, diradd);
LIST_HEAD(newblkhd, newblk);
LIST_HEAD(inodedephd, inodedep);
LIST_HEAD(allocindirhd, allocindir);
LIST_HEAD(allocdirecthd, allocdirect);
TAILQ_HEAD(allocdirectlst, allocdirect);
LIST_HEAD(indirdephd, indirdep);
LIST_HEAD(jaddrefhd, jaddref);
LIST_HEAD(jremrefhd, jremref);
LIST_HEAD(jmvrefhd, jmvref);
LIST_HEAD(jnewblkhd, jnewblk);
LIST_HEAD(jblkdephd, jblkdep);
LIST_HEAD(freeworkhd, freework);
TAILQ_HEAD(freeworklst, freework);
TAILQ_HEAD(jseglst, jseg);
TAILQ_HEAD(inoreflst, inoref);
TAILQ_HEAD(freeblklst, freeblks);
/*
* The "pagedep" structure tracks the various dependencies related to
* a particular directory page. If a directory page has any dependencies,
* it will have a pagedep linked to its associated buffer. The
* pd_dirremhd list holds the list of dirrem requests which decrement
* inode reference counts. These requests are processed after the
* directory page with the corresponding zero'ed entries has been
* written. The pd_diraddhd list maintains the list of diradd requests
* which cannot be committed until their corresponding inode has been
* written to disk. Because a directory may have many new entries
* being created, several lists are maintained hashed on bits of the
* offset of the entry into the directory page to keep the lists from
* getting too long. Once a new directory entry has been cleared to
* be written, it is moved to the pd_pendinghd list. After the new
* entry has been written to disk it is removed from the pd_pendinghd
* list, any removed operations are done, and the dependency structure
* is freed.
*/
#define DAHASHSZ 5
#define DIRADDHASH(offset) (((offset) >> 2) % DAHASHSZ)
struct pagedep {
struct worklist pd_list; /* page buffer */
# define pd_state pd_list.wk_state /* check for multiple I/O starts */
LIST_ENTRY(pagedep) pd_hash; /* hashed lookup */
ino_t pd_ino; /* associated file */
ufs_lbn_t pd_lbn; /* block within file */
struct newdirblk *pd_newdirblk; /* associated newdirblk if NEWBLOCK */
struct dirremhd pd_dirremhd; /* dirrem's waiting for page */
struct diraddhd pd_diraddhd[DAHASHSZ]; /* diradd dir entry updates */
struct diraddhd pd_pendinghd; /* directory entries awaiting write */
struct jmvrefhd pd_jmvrefhd; /* Dependent journal writes. */
};
/*
* The "inodedep" structure tracks the set of dependencies associated
* with an inode. One task that it must manage is delayed operations
* (i.e., work requests that must be held until the inodedep's associated
* inode has been written to disk). Getting an inode from its incore
* state to the disk requires two steps to be taken by the filesystem
* in this order: first the inode must be copied to its disk buffer by
* the VOP_UPDATE operation; second the inode's buffer must be written
* to disk. To ensure that both operations have happened in the required
* order, the inodedep maintains two lists. Delayed operations are
* placed on the id_inowait list. When the VOP_UPDATE is done, all
* operations on the id_inowait list are moved to the id_bufwait list.
* When the buffer is written, the items on the id_bufwait list can be
* safely moved to the work queue to be processed. A second task of the
* inodedep structure is to track the status of block allocation within
* the inode. Each block that is allocated is represented by an
* "allocdirect" structure (see below). It is linked onto the id_newinoupdt
* list until both its contents and its allocation in the cylinder
* group map have been written to disk. Once these dependencies have been
* satisfied, it is removed from the id_newinoupdt list and any followup
* actions such as releasing the previous block or fragment are placed
* on the id_inowait list. When an inode is updated (a VOP_UPDATE is
* done), the "inodedep" structure is linked onto the buffer through
* its worklist. Thus, it will be notified when the buffer is about
* to be written and when it is done. At the update time, all the
* elements on the id_newinoupdt list are moved to the id_inoupdt list
* since those changes are now relevant to the copy of the inode in the
* buffer. Also at update time, the tasks on the id_inowait list are
* moved to the id_bufwait list so that they will be executed when
* the updated inode has been written to disk. When the buffer containing
* the inode is written to disk, any updates listed on the id_inoupdt
* list are rolled back as they are not yet safe. Following the write,
* the changes are once again rolled forward and any actions on the
* id_bufwait list are processed (since those actions are now safe).
* The entries on the id_inoupdt and id_newinoupdt lists must be kept
* sorted by logical block number to speed the calculation of the size
* of the rolled back inode (see explanation in initiate_write_inodeblock).
* When a directory entry is created, it is represented by a diradd.
* The diradd is added to the id_inowait list as it cannot be safely
* written to disk until the inode that it represents is on disk. After
* the inode is written, the id_bufwait list is processed and the diradd
* entries are moved to the id_pendinghd list where they remain until
* the directory block containing the name has been written to disk.
* The purpose of keeping the entries on the id_pendinghd list is so that
* the softdep_fsync function can find and push the inode's directory
* name(s) as part of the fsync operation for that file.
*/
struct inodedep {
struct worklist id_list; /* buffer holding inode block */
# define id_state id_list.wk_state /* inode dependency state */
LIST_ENTRY(inodedep) id_hash; /* hashed lookup */
TAILQ_ENTRY(inodedep) id_unlinked; /* Unlinked but ref'd inodes */
struct fs *id_fs; /* associated filesystem */
ino_t id_ino; /* dependent inode */
nlink_t id_nlinkdelta; /* saved effective link count */
+ nlink_t id_nlinkwrote; /* i_nlink that we wrote to disk */
nlink_t id_savednlink; /* Link saved during rollback */
LIST_ENTRY(inodedep) id_deps; /* bmsafemap's list of inodedep's */
struct bmsafemap *id_bmsafemap; /* related bmsafemap (if pending) */
struct diradd *id_mkdiradd; /* diradd for a mkdir. */
struct inoreflst id_inoreflst; /* Inode reference adjustments. */
long id_savedextsize; /* ext size saved during rollback */
off_t id_savedsize; /* file size saved during rollback */
struct dirremhd id_dirremhd; /* Removals pending. */
struct workhead id_pendinghd; /* entries awaiting directory write */
struct workhead id_bufwait; /* operations after inode written */
struct workhead id_inowait; /* operations waiting inode update */
struct allocdirectlst id_inoupdt; /* updates before inode written */
struct allocdirectlst id_newinoupdt; /* updates when inode written */
struct allocdirectlst id_extupdt; /* extdata updates pre-inode write */
struct allocdirectlst id_newextupdt; /* extdata updates at ino write */
struct freeblklst id_freeblklst; /* List of partial truncates. */
union {
struct ufs1_dinode *idu_savedino1; /* saved ufs1_dinode contents */
struct ufs2_dinode *idu_savedino2; /* saved ufs2_dinode contents */
} id_un;
};
#define id_savedino1 id_un.idu_savedino1
#define id_savedino2 id_un.idu_savedino2
/*
* A "bmsafemap" structure maintains a list of dependency structures
* that depend on the update of a particular cylinder group map.
* It has lists for newblks, allocdirects, allocindirs, and inodedeps.
* It is attached to the buffer of a cylinder group block when any of
* these things are allocated from the cylinder group. It is freed
* after the cylinder group map is written and the state of its
* dependencies are updated with DEPCOMPLETE to indicate that it has
* been processed.
*/
struct bmsafemap {
struct worklist sm_list; /* cylgrp buffer */
# define sm_state sm_list.wk_state
LIST_ENTRY(bmsafemap) sm_hash; /* Hash links. */
LIST_ENTRY(bmsafemap) sm_next; /* Mount list. */
int sm_cg;
struct buf *sm_buf; /* associated buffer */
struct allocdirecthd sm_allocdirecthd; /* allocdirect deps */
struct allocdirecthd sm_allocdirectwr; /* writing allocdirect deps */
struct allocindirhd sm_allocindirhd; /* allocindir deps */
struct allocindirhd sm_allocindirwr; /* writing allocindir deps */
struct inodedephd sm_inodedephd; /* inodedep deps */
struct inodedephd sm_inodedepwr; /* writing inodedep deps */
struct newblkhd sm_newblkhd; /* newblk deps */
struct newblkhd sm_newblkwr; /* writing newblk deps */
struct jaddrefhd sm_jaddrefhd; /* Pending inode allocations. */
struct jnewblkhd sm_jnewblkhd; /* Pending block allocations. */
struct workhead sm_freehd; /* Freedep deps. */
struct workhead sm_freewr; /* Written freedeps. */
};
/*
* A "newblk" structure is attached to a bmsafemap structure when a block
* or fragment is allocated from a cylinder group. Its state is set to
* DEPCOMPLETE when its cylinder group map is written. It is converted to
* an allocdirect or allocindir allocation once the allocator calls the
* appropriate setup function. It will initially be linked onto a bmsafemap
* list. Once converted it can be linked onto the lists described for
* allocdirect or allocindir as described below.
*/
struct newblk {
struct worklist nb_list; /* See comment above. */
# define nb_state nb_list.wk_state
LIST_ENTRY(newblk) nb_hash; /* Hashed lookup. */
LIST_ENTRY(newblk) nb_deps; /* Bmsafemap's list of newblks. */
struct jnewblk *nb_jnewblk; /* New block journal entry. */
struct bmsafemap *nb_bmsafemap;/* Cylgrp dep (if pending). */
struct freefrag *nb_freefrag; /* Fragment to be freed (if any). */
struct indirdephd nb_indirdeps; /* Children indirect blocks. */
struct workhead nb_newdirblk; /* Dir block to notify when written. */
struct workhead nb_jwork; /* Journal work pending. */
ufs2_daddr_t nb_newblkno; /* New value of block pointer. */
};
/*
* An "allocdirect" structure is attached to an "inodedep" when a new block
* or fragment is allocated and pointed to by the inode described by
* "inodedep". The worklist is linked to the buffer that holds the block.
* When the block is first allocated, it is linked to the bmsafemap
* structure associated with the buffer holding the cylinder group map
* from which it was allocated. When the cylinder group map is written
* to disk, ad_state has the DEPCOMPLETE flag set. When the block itself
* is written, the COMPLETE flag is set. Once both the cylinder group map
* and the data itself have been written, it is safe to write the inode
* that claims the block. If there was a previous fragment that had been
* allocated before the file was increased in size, the old fragment may
* be freed once the inode claiming the new block is written to disk.
* This ad_fragfree request is attached to the id_inowait list of the
* associated inodedep (pointed to by ad_inodedep) for processing after
* the inode is written. When a block is allocated to a directory, an
* fsync of a file whose name is within that block must ensure not only
* that the block containing the file name has been written, but also
* that the on-disk inode references that block. When a new directory
* block is created, we allocate a newdirblk structure which is linked
* to the associated allocdirect (on its ad_newdirblk list). When the
* allocdirect has been satisfied, the newdirblk structure is moved to
* the inodedep id_bufwait list of its directory to await the inode
* being written. When the inode is written, the directory entries are
* fully committed and can be deleted from their pagedep->id_pendinghd
* and inodedep->id_pendinghd lists.
*/
struct allocdirect {
struct newblk ad_block; /* Common block logic */
# define ad_list ad_block.nb_list /* block pointer worklist */
# define ad_state ad_list.wk_state /* block pointer state */
TAILQ_ENTRY(allocdirect) ad_next; /* inodedep's list of allocdirect's */
struct inodedep *ad_inodedep; /* associated inodedep */
ufs2_daddr_t ad_oldblkno; /* old value of block pointer */
int ad_offset; /* Pointer offset in parent. */
long ad_newsize; /* size of new block */
long ad_oldsize; /* size of old block */
};
#define ad_newblkno ad_block.nb_newblkno
#define ad_freefrag ad_block.nb_freefrag
#define ad_newdirblk ad_block.nb_newdirblk
/*
* A single "indirdep" structure manages all allocation dependencies for
* pointers in an indirect block. The up-to-date state of the indirect
* block is stored in ir_savedata. The set of pointers that may be safely
* written to the disk is stored in ir_savebp. The state field is used
* only to track whether the buffer is currently being written (in which
* case it is not safe to update ir_savebp). Ir_deplisthd contains the
* list of allocindir structures, one for each block that needs to be
* written to disk. Once the block and its bitmap allocation have been
* written the safecopy can be updated to reflect the allocation and the
* allocindir structure freed. If ir_state indicates that an I/O on the
* indirect block is in progress when ir_savebp is to be updated, the
* update is deferred by placing the allocindir on the ir_donehd list.
* When the I/O on the indirect block completes, the entries on the
* ir_donehd list are processed by updating their corresponding ir_savebp
* pointers and then freeing the allocindir structure.
*/
struct indirdep {
struct worklist ir_list; /* buffer holding indirect block */
# define ir_state ir_list.wk_state /* indirect block pointer state */
LIST_ENTRY(indirdep) ir_next; /* alloc{direct,indir} list */
TAILQ_HEAD(, freework) ir_trunc; /* List of truncations. */
caddr_t ir_saveddata; /* buffer cache contents */
struct buf *ir_savebp; /* buffer holding safe copy */
struct buf *ir_bp; /* buffer holding live copy */
struct allocindirhd ir_completehd; /* waiting for indirdep complete */
struct allocindirhd ir_writehd; /* Waiting for the pointer write. */
struct allocindirhd ir_donehd; /* done waiting to update safecopy */
struct allocindirhd ir_deplisthd; /* allocindir deps for this block */
struct freeblks *ir_freeblks; /* Freeblks that frees this indir. */
};
/*
* An "allocindir" structure is attached to an "indirdep" when a new block
* is allocated and pointed to by the indirect block described by the
* "indirdep". The worklist is linked to the buffer that holds the new block.
* When the block is first allocated, it is linked to the bmsafemap
* structure associated with the buffer holding the cylinder group map
* from which it was allocated. When the cylinder group map is written
* to disk, ai_state has the DEPCOMPLETE flag set. When the block itself
* is written, the COMPLETE flag is set. Once both the cylinder group map
* and the data itself have been written, it is safe to write the entry in
* the indirect block that claims the block; the "allocindir" dependency
* can then be freed as it is no longer applicable.
*/
struct allocindir {
struct newblk ai_block; /* Common block area */
# define ai_state ai_block.nb_list.wk_state /* indirect pointer state */
LIST_ENTRY(allocindir) ai_next; /* indirdep's list of allocindir's */
struct indirdep *ai_indirdep; /* address of associated indirdep */
ufs2_daddr_t ai_oldblkno; /* old value of block pointer */
ufs_lbn_t ai_lbn; /* Logical block number. */
int ai_offset; /* Pointer offset in parent. */
};
#define ai_newblkno ai_block.nb_newblkno
#define ai_freefrag ai_block.nb_freefrag
#define ai_newdirblk ai_block.nb_newdirblk
/*
* The allblk union is used to size the newblk structure on allocation so
* that it may be any one of three types.
*/
union allblk {
struct allocindir ab_allocindir;
struct allocdirect ab_allocdirect;
struct newblk ab_newblk;
};
/*
* A "freefrag" structure is attached to an "inodedep" when a previously
* allocated fragment is replaced with a larger fragment, rather than extended.
* The "freefrag" structure is constructed and attached when the replacement
* block is first allocated. It is processed after the inode claiming the
* bigger block that replaces it has been written to disk.
*/
struct freefrag {
struct worklist ff_list; /* id_inowait or delayed worklist */
# define ff_state ff_list.wk_state
struct worklist *ff_jdep; /* Associated journal entry. */
struct workhead ff_jwork; /* Journal work pending. */
ufs2_daddr_t ff_blkno; /* fragment physical block number */
long ff_fragsize; /* size of fragment being deleted */
ino_t ff_inum; /* owning inode number */
enum vtype ff_vtype; /* owning inode's file type */
int ff_key; /* trim key when deleted */
};
/*
* A "freeblks" structure is attached to an "inodedep" when the
* corresponding file's length is reduced to zero. It records all
* the information needed to free the blocks of a file after its
* zero'ed inode has been written to disk. The actual work is done
* by child freework structures which are responsible for individual
* inode pointers while freeblks is responsible for retiring the
* entire operation when it is complete and holding common members.
*/
struct freeblks {
struct worklist fb_list; /* id_inowait or delayed worklist */
# define fb_state fb_list.wk_state /* inode and dirty block state */
TAILQ_ENTRY(freeblks) fb_next; /* List of inode truncates. */
struct jblkdephd fb_jblkdephd; /* Journal entries pending */
struct workhead fb_freeworkhd; /* Work items pending */
struct workhead fb_jwork; /* Journal work pending */
struct vnode *fb_devvp; /* filesystem device vnode */
#ifdef QUOTA
struct dquot *fb_quota[MAXQUOTAS]; /* quotas to be adjusted */
#endif
uint64_t fb_modrev; /* Inode revision at start of trunc. */
off_t fb_len; /* Length we're truncating to. */
ufs2_daddr_t fb_chkcnt; /* Blocks released. */
ino_t fb_inum; /* inode owner of blocks */
enum vtype fb_vtype; /* inode owner's file type */
uid_t fb_uid; /* uid of previous owner of blocks */
int fb_ref; /* Children outstanding. */
int fb_cgwait; /* cg writes outstanding. */
};
/*
* A "freework" structure handles the release of a tree of blocks or a single
* block. Each indirect block in a tree is allocated its own freework
* structure so that the indirect block may be freed only when all of its
* children are freed. In this way we enforce the rule that an allocated
* block must have a valid path to a root that is journaled. Each child
* block acquires a reference and when the ref hits zero the parent ref
* is decremented. If there is no parent the freeblks ref is decremented.
*/
struct freework {
struct worklist fw_list; /* Delayed worklist. */
# define fw_state fw_list.wk_state
LIST_ENTRY(freework) fw_segs; /* Seg list. */
TAILQ_ENTRY(freework) fw_next; /* Hash/Trunc list. */
struct jnewblk *fw_jnewblk; /* Journal entry to cancel. */
struct freeblks *fw_freeblks; /* Root of operation. */
struct freework *fw_parent; /* Parent indirect. */
struct indirdep *fw_indir; /* indirect block. */
ufs2_daddr_t fw_blkno; /* Our block #. */
ufs_lbn_t fw_lbn; /* Original lbn before free. */
uint16_t fw_frags; /* Number of frags. */
uint16_t fw_ref; /* Number of children out. */
uint16_t fw_off; /* Current working position. */
uint16_t fw_start; /* Start of partial truncate. */
};
/*
* A "freedep" structure is allocated to track the completion of a bitmap
* write for a freework. One freedep may cover many freed blocks so long
* as they reside in the same cylinder group. When the cg is written
* the freedep decrements the ref on the freework which may permit it
* to be freed as well.
*/
struct freedep {
struct worklist fd_list; /* Delayed worklist. */
struct freework *fd_freework; /* Parent freework. */
};
/*
* A "freefile" structure is attached to an inode when its
* link count is reduced to zero. It marks the inode as free in
* the cylinder group map after the zero'ed inode has been written
* to disk and any associated blocks and fragments have been freed.
*/
struct freefile {
struct worklist fx_list; /* id_inowait or delayed worklist */
mode_t fx_mode; /* mode of inode */
ino_t fx_oldinum; /* inum of the unlinked file */
struct vnode *fx_devvp; /* filesystem device vnode */
struct workhead fx_jwork; /* journal work pending. */
};
/*
* A "diradd" structure is linked to an "inodedep" id_inowait list when a
* new directory entry is allocated that references the inode described
* by "inodedep". When the inode itself is written (either the initial
* allocation for new inodes or with the increased link count for
* existing inodes), the COMPLETE flag is set in da_state. If the entry
* is for a newly allocated inode, the "inodedep" structure is associated
* with a bmsafemap which prevents the inode from being written to disk
* until the cylinder group has been updated. Thus the da_state COMPLETE
* flag cannot be set until the inode bitmap dependency has been removed.
* When creating a new file, it is safe to write the directory entry that
* claims the inode once the referenced inode has been written. Since
* writing the inode clears the bitmap dependencies, the DEPCOMPLETE flag
* in the diradd can be set unconditionally when creating a file. When
* creating a directory, there are two additional dependencies described by
* mkdir structures (see their description below). When these dependencies
* are resolved the DEPCOMPLETE flag is set in the diradd structure.
* If there are multiple links created to the same inode, there will be
* a separate diradd structure created for each link. The diradd is
* linked onto the pg_diraddhd list of the pagedep for the directory
* page that contains the entry. When a directory page is written,
* the pg_diraddhd list is traversed to rollback any entries that are
* not yet ready to be written to disk. If a directory entry is being
* changed (by rename) rather than added, the DIRCHG flag is set and
* the da_previous entry points to the entry that will be "removed"
* once the new entry has been committed. During rollback, entries
* with da_previous are replaced with the previous inode number rather
* than zero.
*
* The overlaying of da_pagedep and da_previous is done to keep the
* structure down. If a da_previous entry is present, the pointer to its
* pagedep is available in the associated dirrem entry. If the DIRCHG flag
* is set, the da_previous entry is valid; if not set the da_pagedep entry
* is valid. The DIRCHG flag never changes; it is set when the structure
* is created if appropriate and is never cleared.
*/
struct diradd {
struct worklist da_list; /* id_inowait or id_pendinghd list */
# define da_state da_list.wk_state /* state of the new directory entry */
LIST_ENTRY(diradd) da_pdlist; /* pagedep holding directory block */
doff_t da_offset; /* offset of new dir entry in dir blk */
ino_t da_newinum; /* inode number for the new dir entry */
union {
struct dirrem *dau_previous; /* entry being replaced in dir change */
struct pagedep *dau_pagedep; /* pagedep dependency for addition */
} da_un;
struct workhead da_jwork; /* Journal work awaiting completion. */
};
#define da_previous da_un.dau_previous
#define da_pagedep da_un.dau_pagedep
/*
* Two "mkdir" structures are needed to track the additional dependencies
* associated with creating a new directory entry. Normally a directory
* addition can be committed as soon as the newly referenced inode has been
* written to disk with its increased link count. When a directory is
* created there are two additional dependencies: writing the directory
* data block containing the "." and ".." entries (MKDIR_BODY) and writing
* the parent inode with the increased link count for ".." (MKDIR_PARENT).
* These additional dependencies are tracked by two mkdir structures that
* reference the associated "diradd" structure. When they have completed,
* they set the DEPCOMPLETE flag on the diradd so that it knows that its
* extra dependencies have been completed. The md_state field is used only
* to identify which type of dependency the mkdir structure is tracking.
* It is not used in the mainline code for any purpose other than consistency
* checking. All the mkdir structures in the system are linked together on
* a list. This list is needed so that a diradd can find its associated
* mkdir structures and deallocate them if it is prematurely freed (as for
* example if a mkdir is immediately followed by a rmdir of the same directory).
* Here, the free of the diradd must traverse the list to find the associated
* mkdir structures that reference it. The deletion would be faster if the
* diradd structure were simply augmented to have two pointers that referenced
* the associated mkdir's. However, this would increase the size of the diradd
* structure to speed a very infrequent operation.
*/
struct mkdir {
struct worklist md_list; /* id_inowait or buffer holding dir */
# define md_state md_list.wk_state /* type: MKDIR_PARENT or MKDIR_BODY */
struct diradd *md_diradd; /* associated diradd */
struct jaddref *md_jaddref; /* dependent jaddref. */
struct buf *md_buf; /* MKDIR_BODY: buffer holding dir */
LIST_ENTRY(mkdir) md_mkdirs; /* list of all mkdirs */
};
/*
* A "dirrem" structure describes an operation to decrement the link
* count on an inode. The dirrem structure is attached to the pg_dirremhd
* list of the pagedep for the directory page that contains the entry.
* It is processed after the directory page with the deleted entry has
* been written to disk.
*/
struct dirrem {
struct worklist dm_list; /* delayed worklist */
# define dm_state dm_list.wk_state /* state of the old directory entry */
LIST_ENTRY(dirrem) dm_next; /* pagedep's list of dirrem's */
LIST_ENTRY(dirrem) dm_inonext; /* inodedep's list of dirrem's */
struct jremrefhd dm_jremrefhd; /* Pending remove reference deps. */
ino_t dm_oldinum; /* inum of the removed dir entry */
doff_t dm_offset; /* offset of removed dir entry in blk */
union {
struct pagedep *dmu_pagedep; /* pagedep dependency for remove */
ino_t dmu_dirinum; /* parent inode number (for rmdir) */
} dm_un;
struct workhead dm_jwork; /* Journal work awaiting completion. */
};
#define dm_pagedep dm_un.dmu_pagedep
#define dm_dirinum dm_un.dmu_dirinum
/*
* A "newdirblk" structure tracks the progress of a newly allocated
* directory block from its creation until it is claimed by its on-disk
* inode. When a block is allocated to a directory, an fsync of a file
* whose name is within that block must ensure not only that the block
* containing the file name has been written, but also that the on-disk
* inode references that block. When a new directory block is created,
* we allocate a newdirblk structure which is linked to the associated
* allocdirect (on its ad_newdirblk list). When the allocdirect has been
* satisfied, the newdirblk structure is moved to the inodedep id_bufwait
* list of its directory to await the inode being written. When the inode
* is written, the directory entries are fully committed and can be
* deleted from their pagedep->id_pendinghd and inodedep->id_pendinghd
* lists. Note that we could track directory blocks allocated to indirect
* blocks using a similar scheme with the allocindir structures. Rather
* than adding this level of complexity, we simply write those newly
* allocated indirect blocks synchronously as such allocations are rare.
* In the case of a new directory the . and .. links are tracked with
* a mkdir rather than a pagedep. In this case we track the mkdir
* so it can be released when it is written. A workhead is used
* to simplify canceling a mkdir that is removed by a subsequent dirrem.
*/
struct newdirblk {
struct worklist db_list; /* id_inowait or pg_newdirblk */
# define db_state db_list.wk_state
struct pagedep *db_pagedep; /* associated pagedep */
struct workhead db_mkdir;
};
/*
* The inoref structure holds the elements common to jaddref and jremref
* so they may easily be queued in-order on the inodedep.
*/
struct inoref {
struct worklist if_list; /* Journal pending or jseg entries. */
# define if_state if_list.wk_state
TAILQ_ENTRY(inoref) if_deps; /* Links for inodedep. */
struct jsegdep *if_jsegdep; /* Will track our journal record. */
off_t if_diroff; /* Directory offset. */
ino_t if_ino; /* Inode number. */
ino_t if_parent; /* Parent inode number. */
nlink_t if_nlink; /* nlink before addition. */
uint16_t if_mode; /* File mode, needed for IFMT. */
};
/*
* A "jaddref" structure tracks a new reference (link count) on an inode
* and prevents the link count increase and bitmap allocation until a
* journal entry can be written. Once the journal entry is written,
* the inode is put on the pendinghd of the bmsafemap and a diradd or
* mkdir entry is placed on the bufwait list of the inode. The DEPCOMPLETE
* flag is used to indicate that all of the required information for writing
* the journal entry is present. MKDIR_BODY and MKDIR_PARENT are used to
* differentiate . and .. links from regular file names. NEWBLOCK indicates
* a bitmap is still pending. If a new reference is canceled by a delete
* prior to writing the journal the jaddref write is canceled and the
* structure persists to prevent any disk-visible changes until it is
* ultimately released when the file is freed or the link is dropped again.
*/
struct jaddref {
struct inoref ja_ref; /* see inoref above. */
# define ja_list ja_ref.if_list /* Jrnl pending, id_inowait, dm_jwork.*/
# define ja_state ja_ref.if_list.wk_state
LIST_ENTRY(jaddref) ja_bmdeps; /* Links for bmsafemap. */
union {
struct diradd *jau_diradd; /* Pending diradd. */
struct mkdir *jau_mkdir; /* MKDIR_{PARENT,BODY} */
} ja_un;
};
#define ja_diradd ja_un.jau_diradd
#define ja_mkdir ja_un.jau_mkdir
#define ja_diroff ja_ref.if_diroff
#define ja_ino ja_ref.if_ino
#define ja_parent ja_ref.if_parent
#define ja_mode ja_ref.if_mode
/*
* A "jremref" structure tracks a removed reference (unlink) on an
* inode and prevents the directory remove from proceeding until the
* journal entry is written. Once the journal has been written the remove
* may proceed as normal.
*/
struct jremref {
struct inoref jr_ref; /* see inoref above. */
# define jr_list jr_ref.if_list /* Linked to softdep_journal_pending. */
# define jr_state jr_ref.if_list.wk_state
LIST_ENTRY(jremref) jr_deps; /* Links for dirrem. */
struct dirrem *jr_dirrem; /* Back pointer to dirrem. */
};
/*
* A "jmvref" structure tracks a name relocations within the same
* directory block that occur as a result of directory compaction.
* It prevents the updated directory entry from being written to disk
* until the journal entry is written. Once the journal has been
* written the compacted directory may be written to disk.
*/
struct jmvref {
struct worklist jm_list; /* Linked to softdep_journal_pending. */
LIST_ENTRY(jmvref) jm_deps; /* Jmvref on pagedep. */
struct pagedep *jm_pagedep; /* Back pointer to pagedep. */
ino_t jm_parent; /* Containing directory inode number. */
ino_t jm_ino; /* Inode number of our entry. */
off_t jm_oldoff; /* Our old offset in directory. */
off_t jm_newoff; /* Our new offset in directory. */
};
/*
* A "jnewblk" structure tracks a newly allocated block or fragment and
* prevents the direct or indirect block pointer as well as the cg bitmap
* from being written until it is logged. After it is logged the jsegdep
* is attached to the allocdirect or allocindir until the operation is
* completed or reverted. If the operation is reverted prior to the journal
* write the jnewblk structure is maintained to prevent the bitmaps from
* reaching the disk. Ultimately the jnewblk structure will be passed
* to the free routine as the in memory cg is modified back to the free
* state at which time it can be released. It may be held on any of the
* fx_jwork, fw_jwork, fb_jwork, ff_jwork, nb_jwork, or ir_jwork lists.
*/
struct jnewblk {
struct worklist jn_list; /* See lists above. */
# define jn_state jn_list.wk_state
struct jsegdep *jn_jsegdep; /* Will track our journal record. */
LIST_ENTRY(jnewblk) jn_deps; /* Jnewblks on sm_jnewblkhd. */
struct worklist *jn_dep; /* Dependency to ref completed seg. */
ufs_lbn_t jn_lbn; /* Lbn to which allocated. */
ufs2_daddr_t jn_blkno; /* Blkno allocated */
ino_t jn_ino; /* Ino to which allocated. */
int jn_oldfrags; /* Previous fragments when extended. */
int jn_frags; /* Number of fragments. */
};
/*
* A "jblkdep" structure tracks jfreeblk and jtrunc records attached to a
* freeblks structure.
*/
struct jblkdep {
struct worklist jb_list; /* For softdep journal pending. */
struct jsegdep *jb_jsegdep; /* Reference to the jseg. */
struct freeblks *jb_freeblks; /* Back pointer to freeblks. */
LIST_ENTRY(jblkdep) jb_deps; /* Dep list on freeblks. */
};
/*
* A "jfreeblk" structure tracks the journal write for freeing a block
* or tree of blocks. The block pointer must not be cleared in the inode
* or indirect prior to the jfreeblk being written to the journal.
*/
struct jfreeblk {
struct jblkdep jf_dep; /* freeblks linkage. */
ufs_lbn_t jf_lbn; /* Lbn from which blocks freed. */
ufs2_daddr_t jf_blkno; /* Blkno being freed. */
ino_t jf_ino; /* Ino from which blocks freed. */
int jf_frags; /* Number of frags being freed. */
};
/*
* A "jfreefrag" tracks the freeing of a single block when a fragment is
* extended or an indirect page is replaced. It is not part of a larger
* freeblks operation.
*/
struct jfreefrag {
struct worklist fr_list; /* Linked to softdep_journal_pending. */
# define fr_state fr_list.wk_state
struct jsegdep *fr_jsegdep; /* Will track our journal record. */
struct freefrag *fr_freefrag; /* Back pointer to freefrag. */
ufs_lbn_t fr_lbn; /* Lbn from which frag freed. */
ufs2_daddr_t fr_blkno; /* Blkno being freed. */
ino_t fr_ino; /* Ino from which frag freed. */
int fr_frags; /* Size of frag being freed. */
};
/*
* A "jtrunc" journals the intent to truncate an inode's data or extent area.
*/
struct jtrunc {
struct jblkdep jt_dep; /* freeblks linkage. */
off_t jt_size; /* Final file size. */
int jt_extsize; /* Final extent size. */
ino_t jt_ino; /* Ino being truncated. */
};
/*
* A "jfsync" journals the completion of an fsync which invalidates earlier
* jtrunc records in the journal.
*/
struct jfsync {
struct worklist jfs_list; /* For softdep journal pending. */
off_t jfs_size; /* Sync file size. */
int jfs_extsize; /* Sync extent size. */
ino_t jfs_ino; /* ino being synced. */
};
/*
* A "jsegdep" structure tracks a single reference to a written journal
* segment so the journal space can be reclaimed when all dependencies
* have been written. It can hang off of id_inowait, dm_jwork, da_jwork,
* nb_jwork, ff_jwork, or fb_jwork lists.
*/
struct jsegdep {
struct worklist jd_list; /* See above for lists. */
# define jd_state jd_list.wk_state
struct jseg *jd_seg; /* Our journal record. */
};
/*
* A "jseg" structure contains all of the journal records written in a
* single disk write. The jaddref and jremref structures are linked into
* js_entries so thay may be completed when the write completes. The
* js_entries also include the write dependency structures: jmvref,
* jnewblk, jfreeblk, jfreefrag, and jtrunc. The js_refs field counts
* the number of entries on the js_entries list. Thus there is a single
* jseg entry to describe each journal write.
*/
struct jseg {
struct worklist js_list; /* b_deps link for journal */
# define js_state js_list.wk_state
struct workhead js_entries; /* Entries awaiting write */
LIST_HEAD(, freework) js_indirs;/* List of indirects in this seg. */
TAILQ_ENTRY(jseg) js_next; /* List of all unfinished segments. */
struct jblocks *js_jblocks; /* Back pointer to block/seg list */
struct buf *js_buf; /* Buffer while unwritten */
uint64_t js_seq; /* Journal record sequence number. */
uint64_t js_oldseq; /* Oldest valid sequence number. */
int js_size; /* Size of journal record in bytes. */
int js_cnt; /* Total items allocated. */
int js_refs; /* Count of js_entries items. */
};
/*
* A 'sbdep' structure tracks the head of the free inode list and
* superblock writes. This makes sure the superblock is always pointing at
* the first possible unlinked inode for the suj recovery process. If a
* block write completes and we discover a new head is available the buf
* is dirtied and the dep is kept. See the description of the UNLINK*
* flags above for more details.
*/
struct sbdep {
struct worklist sb_list; /* b_dep linkage */
struct fs *sb_fs; /* Filesystem pointer within buf. */
struct ufsmount *sb_ump; /* Our mount structure */
};
/*
* Private journaling structures.
*/
struct jblocks {
struct jseglst jb_segs; /* TAILQ of current segments. */
struct jseg *jb_writeseg; /* Next write to complete. */
struct jseg *jb_oldestseg; /* Oldest segment with valid entries. */
struct jextent *jb_extent; /* Extent array. */
uint64_t jb_nextseq; /* Next sequence number. */
uint64_t jb_oldestwrseq; /* Oldest written sequence number. */
uint8_t jb_needseg; /* Need a forced segment. */
uint8_t jb_suspended; /* Did journal suspend writes? */
int jb_avail; /* Available extents. */
int jb_used; /* Last used extent. */
int jb_head; /* Allocator head. */
int jb_off; /* Allocator extent offset. */
int jb_blocks; /* Total disk blocks covered. */
int jb_free; /* Total disk blocks free. */
int jb_min; /* Minimum free space. */
int jb_low; /* Low on space. */
int jb_age; /* Insertion time of oldest rec. */
};
struct jextent {
ufs2_daddr_t je_daddr; /* Disk block address. */
int je_blocks; /* Disk block count. */
};
/*
* Hash table declarations.
*/
LIST_HEAD(mkdirlist, mkdir);
LIST_HEAD(pagedep_hashhead, pagedep);
LIST_HEAD(inodedep_hashhead, inodedep);
LIST_HEAD(newblk_hashhead, newblk);
LIST_HEAD(bmsafemap_hashhead, bmsafemap);
TAILQ_HEAD(indir_hashhead, freework);
/*
* Per-filesystem soft dependency data.
* Allocated at mount and freed at unmount.
*/
struct mount_softdeps {
struct rwlock sd_fslock; /* softdep lock */
struct workhead sd_workitem_pending; /* softdep work queue */
struct worklist *sd_worklist_tail; /* Tail pointer for above */
struct workhead sd_journal_pending; /* journal work queue */
struct worklist *sd_journal_tail; /* Tail pointer for above */
struct jblocks *sd_jblocks; /* Journal block information */
struct inodedeplst sd_unlinked; /* Unlinked inodes */
struct bmsafemaphd sd_dirtycg; /* Dirty CGs */
struct mkdirlist sd_mkdirlisthd; /* Track mkdirs */
struct pagedep_hashhead *sd_pdhash; /* pagedep hash table */
u_long sd_pdhashsize; /* pagedep hash table size-1 */
long sd_pdnextclean; /* next hash bucket to clean */
struct inodedep_hashhead *sd_idhash; /* inodedep hash table */
u_long sd_idhashsize; /* inodedep hash table size-1 */
long sd_idnextclean; /* next hash bucket to clean */
struct newblk_hashhead *sd_newblkhash; /* newblk hash table */
u_long sd_newblkhashsize; /* newblk hash table size-1 */
struct bmsafemap_hashhead *sd_bmhash; /* bmsafemap hash table */
u_long sd_bmhashsize; /* bmsafemap hash table size-1*/
struct indir_hashhead *sd_indirhash; /* indir hash table */
u_long sd_indirhashsize; /* indir hash table size-1 */
int sd_on_journal; /* Items on the journal list */
int sd_on_worklist; /* Items on the worklist */
int sd_deps; /* Total dependency count */
int sd_accdeps; /* accumulated dep count */
int sd_req; /* Wakeup when deps hits 0. */
int sd_flags; /* comm with flushing thread */
int sd_cleanups; /* Calls to cleanup */
struct thread *sd_flushtd; /* thread handling flushing */
TAILQ_ENTRY(mount_softdeps) sd_next; /* List of softdep filesystem */
struct ufsmount *sd_ump; /* our ufsmount structure */
u_long sd_curdeps[D_LAST + 1]; /* count of current deps */
#ifdef INVARIANTS
struct workhead sd_alldeps[D_LAST + 1];/* Lists of all deps */
#endif
};
/*
* Flags for communicating with the syncer thread.
*/
#define FLUSH_EXIT 0x0001 /* time to exit */
#define FLUSH_CLEANUP 0x0002 /* need to clear out softdep structures */
#define FLUSH_STARTING 0x0004 /* flush thread not yet started */
#define FLUSH_RC_ACTIVE 0x0008 /* a thread is flushing the mount point */
/*
* Keep the old names from when these were in the ufsmount structure.
*/
#define softdep_workitem_pending um_softdep->sd_workitem_pending
#define softdep_worklist_tail um_softdep->sd_worklist_tail
#define softdep_journal_pending um_softdep->sd_journal_pending
#define softdep_journal_tail um_softdep->sd_journal_tail
#define softdep_jblocks um_softdep->sd_jblocks
#define softdep_unlinked um_softdep->sd_unlinked
#define softdep_dirtycg um_softdep->sd_dirtycg
#define softdep_mkdirlisthd um_softdep->sd_mkdirlisthd
#define pagedep_hashtbl um_softdep->sd_pdhash
#define pagedep_hash_size um_softdep->sd_pdhashsize
#define pagedep_nextclean um_softdep->sd_pdnextclean
#define inodedep_hashtbl um_softdep->sd_idhash
#define inodedep_hash_size um_softdep->sd_idhashsize
#define inodedep_nextclean um_softdep->sd_idnextclean
#define newblk_hashtbl um_softdep->sd_newblkhash
#define newblk_hash_size um_softdep->sd_newblkhashsize
#define bmsafemap_hashtbl um_softdep->sd_bmhash
#define bmsafemap_hash_size um_softdep->sd_bmhashsize
#define indir_hashtbl um_softdep->sd_indirhash
#define indir_hash_size um_softdep->sd_indirhashsize
#define softdep_on_journal um_softdep->sd_on_journal
#define softdep_on_worklist um_softdep->sd_on_worklist
#define softdep_deps um_softdep->sd_deps
#define softdep_accdeps um_softdep->sd_accdeps
#define softdep_req um_softdep->sd_req
#define softdep_flags um_softdep->sd_flags
#define softdep_flushtd um_softdep->sd_flushtd
#define softdep_curdeps um_softdep->sd_curdeps
#define softdep_alldeps um_softdep->sd_alldeps
Index: head/sys/ufs/ufs/ufs_vnops.c
===================================================================
--- head/sys/ufs/ufs/ufs_vnops.c (revision 361490)
+++ head/sys/ufs/ufs/ufs_vnops.c (revision 361491)
@@ -1,2810 +1,2813 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993, 1995
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ufs_vnops.c 8.27 (Berkeley) 5/27/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_quota.h"
#include "opt_suiddir.h"
#include "opt_ufs.h"
#include "opt_ffs.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/namei.h>
#include <sys/kernel.h>
#include <sys/fcntl.h>
#include <sys/filio.h>
#include <sys/stat.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/mount.h>
#include <sys/priv.h>
#include <sys/refcount.h>
#include <sys/unistd.h>
#include <sys/vnode.h>
#include <sys/dirent.h>
#include <sys/lockf.h>
#include <sys/conf.h>
#include <sys/acl.h>
#include <security/mac/mac_framework.h>
#include <sys/file.h> /* XXX */
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <ufs/ufs/acl.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/dir.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ufs/ufs_extern.h>
#ifdef UFS_DIRHASH
#include <ufs/ufs/dirhash.h>
#endif
#ifdef UFS_GJOURNAL
#include <ufs/ufs/gjournal.h>
FEATURE(ufs_gjournal, "Journaling support through GEOM for UFS");
#endif
#ifdef QUOTA
FEATURE(ufs_quota, "UFS disk quotas support");
FEATURE(ufs_quota64, "64bit UFS disk quotas support");
#endif
#ifdef SUIDDIR
FEATURE(suiddir,
"Give all new files in directory the same ownership as the directory");
#endif
#include <ufs/ffs/ffs_extern.h>
static vop_accessx_t ufs_accessx;
static int ufs_chmod(struct vnode *, int, struct ucred *, struct thread *);
static int ufs_chown(struct vnode *, uid_t, gid_t, struct ucred *, struct thread *);
static vop_close_t ufs_close;
static vop_create_t ufs_create;
static vop_getattr_t ufs_getattr;
static vop_ioctl_t ufs_ioctl;
static vop_link_t ufs_link;
static int ufs_makeinode(int mode, struct vnode *, struct vnode **, struct componentname *, const char *);
static vop_mmapped_t ufs_mmapped;
static vop_mkdir_t ufs_mkdir;
static vop_mknod_t ufs_mknod;
static vop_open_t ufs_open;
static vop_pathconf_t ufs_pathconf;
static vop_print_t ufs_print;
static vop_readlink_t ufs_readlink;
static vop_remove_t ufs_remove;
static vop_rename_t ufs_rename;
static vop_rmdir_t ufs_rmdir;
static vop_setattr_t ufs_setattr;
static vop_strategy_t ufs_strategy;
static vop_symlink_t ufs_symlink;
static vop_whiteout_t ufs_whiteout;
static vop_close_t ufsfifo_close;
static vop_kqfilter_t ufsfifo_kqfilter;
SYSCTL_NODE(_vfs, OID_AUTO, ufs, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"UFS filesystem");
/*
* A virgin directory (no blushing please).
*/
static struct dirtemplate mastertemplate = {
0, 12, DT_DIR, 1, ".",
0, DIRBLKSIZ - 12, DT_DIR, 2, ".."
};
static struct odirtemplate omastertemplate = {
0, 12, 1, ".",
0, DIRBLKSIZ - 12, 2, ".."
};
static void
ufs_itimes_locked(struct vnode *vp)
{
struct inode *ip;
struct timespec ts;
ASSERT_VI_LOCKED(vp, __func__);
ip = VTOI(vp);
if (UFS_RDONLY(ip))
goto out;
if ((ip->i_flag & (IN_ACCESS | IN_CHANGE | IN_UPDATE)) == 0)
return;
if ((vp->v_type == VBLK || vp->v_type == VCHR) && !DOINGSOFTDEP(vp))
UFS_INODE_SET_FLAG(ip, IN_LAZYMOD);
else if (((vp->v_mount->mnt_kern_flag &
(MNTK_SUSPENDED | MNTK_SUSPEND)) == 0) ||
(ip->i_flag & (IN_CHANGE | IN_UPDATE)))
UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
else if (ip->i_flag & IN_ACCESS)
UFS_INODE_SET_FLAG(ip, IN_LAZYACCESS);
vfs_timestamp(&ts);
if (ip->i_flag & IN_ACCESS) {
DIP_SET(ip, i_atime, ts.tv_sec);
DIP_SET(ip, i_atimensec, ts.tv_nsec);
}
if (ip->i_flag & IN_UPDATE) {
DIP_SET(ip, i_mtime, ts.tv_sec);
DIP_SET(ip, i_mtimensec, ts.tv_nsec);
}
if (ip->i_flag & IN_CHANGE) {
DIP_SET(ip, i_ctime, ts.tv_sec);
DIP_SET(ip, i_ctimensec, ts.tv_nsec);
DIP_SET(ip, i_modrev, DIP(ip, i_modrev) + 1);
}
out:
ip->i_flag &= ~(IN_ACCESS | IN_CHANGE | IN_UPDATE);
}
void
ufs_itimes(struct vnode *vp)
{
VI_LOCK(vp);
ufs_itimes_locked(vp);
VI_UNLOCK(vp);
}
/*
* Create a regular file
*/
static int
ufs_create(ap)
struct vop_create_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
} */ *ap;
{
int error;
error =
ufs_makeinode(MAKEIMODE(ap->a_vap->va_type, ap->a_vap->va_mode),
ap->a_dvp, ap->a_vpp, ap->a_cnp, "ufs_create");
if (error != 0)
return (error);
if ((ap->a_cnp->cn_flags & MAKEENTRY) != 0)
cache_enter(ap->a_dvp, *ap->a_vpp, ap->a_cnp);
return (0);
}
/*
* Mknod vnode call
*/
/* ARGSUSED */
static int
ufs_mknod(ap)
struct vop_mknod_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
} */ *ap;
{
struct vattr *vap = ap->a_vap;
struct vnode **vpp = ap->a_vpp;
struct inode *ip;
ino_t ino;
int error;
error = ufs_makeinode(MAKEIMODE(vap->va_type, vap->va_mode),
ap->a_dvp, vpp, ap->a_cnp, "ufs_mknod");
if (error)
return (error);
ip = VTOI(*vpp);
UFS_INODE_SET_FLAG(ip, IN_ACCESS | IN_CHANGE | IN_UPDATE);
if (vap->va_rdev != VNOVAL) {
/*
* Want to be able to use this to make badblock
* inodes, so don't truncate the dev number.
*/
DIP_SET(ip, i_rdev, vap->va_rdev);
}
/*
* Remove inode, then reload it through VFS_VGET so it is
* checked to see if it is an alias of an existing entry in
* the inode cache. XXX I don't believe this is necessary now.
*/
(*vpp)->v_type = VNON;
ino = ip->i_number; /* Save this before vgone() invalidates ip. */
vgone(*vpp);
vput(*vpp);
error = VFS_VGET(ap->a_dvp->v_mount, ino, LK_EXCLUSIVE, vpp);
if (error) {
*vpp = NULL;
return (error);
}
return (0);
}
/*
* Open called.
*/
/* ARGSUSED */
static int
ufs_open(struct vop_open_args *ap)
{
struct vnode *vp = ap->a_vp;
struct inode *ip;
if (vp->v_type == VCHR || vp->v_type == VBLK)
return (EOPNOTSUPP);
ip = VTOI(vp);
/*
* Files marked append-only must be opened for appending.
*/
if ((ip->i_flags & APPEND) &&
(ap->a_mode & (FWRITE | O_APPEND)) == FWRITE)
return (EPERM);
vnode_create_vobject(vp, DIP(ip, i_size), ap->a_td);
return (0);
}
/*
* Close called.
*
* Update the times on the inode.
*/
/* ARGSUSED */
static int
ufs_close(ap)
struct vop_close_args /* {
struct vnode *a_vp;
int a_fflag;
struct ucred *a_cred;
struct thread *a_td;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
int usecount;
VI_LOCK(vp);
usecount = vp->v_usecount;
if (usecount > 1)
ufs_itimes_locked(vp);
VI_UNLOCK(vp);
return (0);
}
static int
ufs_accessx(ap)
struct vop_accessx_args /* {
struct vnode *a_vp;
accmode_t a_accmode;
struct ucred *a_cred;
struct thread *a_td;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
accmode_t accmode = ap->a_accmode;
int error;
#ifdef UFS_ACL
struct acl *acl;
acl_type_t type;
#endif
/*
* Disallow write attempts on read-only filesystems;
* unless the file is a socket, fifo, or a block or
* character device resident on the filesystem.
*/
if (accmode & VMODIFY_PERMS) {
switch (vp->v_type) {
case VDIR:
case VLNK:
case VREG:
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
#ifdef QUOTA
/*
* Inode is accounted in the quotas only if struct
* dquot is attached to it. VOP_ACCESS() is called
* from vn_open_cred() and provides a convenient
* point to call getinoquota(). The lock mode is
* exclusive when the file is opening for write.
*/
if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE) {
error = getinoquota(ip);
if (error != 0)
return (error);
}
#endif
break;
default:
break;
}
}
/*
* If immutable bit set, nobody gets to write it. "& ~VADMIN_PERMS"
* permits the owner of the file to remove the IMMUTABLE flag.
*/
if ((accmode & (VMODIFY_PERMS & ~VADMIN_PERMS)) &&
(ip->i_flags & (IMMUTABLE | SF_SNAPSHOT)))
return (EPERM);
#ifdef UFS_ACL
if ((vp->v_mount->mnt_flag & (MNT_ACLS | MNT_NFS4ACLS)) != 0) {
if (vp->v_mount->mnt_flag & MNT_NFS4ACLS)
type = ACL_TYPE_NFS4;
else
type = ACL_TYPE_ACCESS;
acl = acl_alloc(M_WAITOK);
if (type == ACL_TYPE_NFS4)
error = ufs_getacl_nfs4_internal(vp, acl, ap->a_td);
else
error = VOP_GETACL(vp, type, acl, ap->a_cred, ap->a_td);
switch (error) {
case 0:
if (type == ACL_TYPE_NFS4) {
error = vaccess_acl_nfs4(vp->v_type, ip->i_uid,
ip->i_gid, acl, accmode, ap->a_cred, NULL);
} else {
error = vfs_unixify_accmode(&accmode);
if (error == 0)
error = vaccess_acl_posix1e(vp->v_type, ip->i_uid,
ip->i_gid, acl, accmode, ap->a_cred, NULL);
}
break;
default:
if (error != EOPNOTSUPP)
printf(
"ufs_accessx(): Error retrieving ACL on object (%d).\n",
error);
/*
* XXX: Fall back until debugged. Should
* eventually possibly log an error, and return
* EPERM for safety.
*/
error = vfs_unixify_accmode(&accmode);
if (error == 0)
error = vaccess(vp->v_type, ip->i_mode, ip->i_uid,
ip->i_gid, accmode, ap->a_cred, NULL);
}
acl_free(acl);
return (error);
}
#endif /* !UFS_ACL */
error = vfs_unixify_accmode(&accmode);
if (error == 0)
error = vaccess(vp->v_type, ip->i_mode, ip->i_uid, ip->i_gid,
accmode, ap->a_cred, NULL);
return (error);
}
/* ARGSUSED */
static int
ufs_getattr(ap)
struct vop_getattr_args /* {
struct vnode *a_vp;
struct vattr *a_vap;
struct ucred *a_cred;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
struct vattr *vap = ap->a_vap;
VI_LOCK(vp);
ufs_itimes_locked(vp);
if (I_IS_UFS1(ip)) {
vap->va_atime.tv_sec = ip->i_din1->di_atime;
vap->va_atime.tv_nsec = ip->i_din1->di_atimensec;
} else {
vap->va_atime.tv_sec = ip->i_din2->di_atime;
vap->va_atime.tv_nsec = ip->i_din2->di_atimensec;
}
VI_UNLOCK(vp);
/*
* Copy from inode table
*/
vap->va_fsid = dev2udev(ITOUMP(ip)->um_dev);
vap->va_fileid = ip->i_number;
vap->va_mode = ip->i_mode & ~IFMT;
vap->va_nlink = ip->i_effnlink;
vap->va_uid = ip->i_uid;
vap->va_gid = ip->i_gid;
if (I_IS_UFS1(ip)) {
vap->va_rdev = ip->i_din1->di_rdev;
vap->va_size = ip->i_din1->di_size;
vap->va_mtime.tv_sec = ip->i_din1->di_mtime;
vap->va_mtime.tv_nsec = ip->i_din1->di_mtimensec;
vap->va_ctime.tv_sec = ip->i_din1->di_ctime;
vap->va_ctime.tv_nsec = ip->i_din1->di_ctimensec;
vap->va_bytes = dbtob((u_quad_t)ip->i_din1->di_blocks);
vap->va_filerev = ip->i_din1->di_modrev;
} else {
vap->va_rdev = ip->i_din2->di_rdev;
vap->va_size = ip->i_din2->di_size;
vap->va_mtime.tv_sec = ip->i_din2->di_mtime;
vap->va_mtime.tv_nsec = ip->i_din2->di_mtimensec;
vap->va_ctime.tv_sec = ip->i_din2->di_ctime;
vap->va_ctime.tv_nsec = ip->i_din2->di_ctimensec;
vap->va_birthtime.tv_sec = ip->i_din2->di_birthtime;
vap->va_birthtime.tv_nsec = ip->i_din2->di_birthnsec;
vap->va_bytes = dbtob((u_quad_t)ip->i_din2->di_blocks);
vap->va_filerev = ip->i_din2->di_modrev;
}
vap->va_flags = ip->i_flags;
vap->va_gen = ip->i_gen;
vap->va_blocksize = vp->v_mount->mnt_stat.f_iosize;
vap->va_type = IFTOVT(ip->i_mode);
return (0);
}
/*
* Set attribute vnode op. called from several syscalls
*/
static int
ufs_setattr(ap)
struct vop_setattr_args /* {
struct vnode *a_vp;
struct vattr *a_vap;
struct ucred *a_cred;
} */ *ap;
{
struct vattr *vap = ap->a_vap;
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
struct ucred *cred = ap->a_cred;
struct thread *td = curthread;
int error;
/*
* Check for unsettable attributes.
*/
if ((vap->va_type != VNON) || (vap->va_nlink != VNOVAL) ||
(vap->va_fsid != VNOVAL) || (vap->va_fileid != VNOVAL) ||
(vap->va_blocksize != VNOVAL) || (vap->va_rdev != VNOVAL) ||
((int)vap->va_bytes != VNOVAL) || (vap->va_gen != VNOVAL)) {
return (EINVAL);
}
if (vap->va_flags != VNOVAL) {
if ((vap->va_flags & ~(SF_APPEND | SF_ARCHIVED | SF_IMMUTABLE |
SF_NOUNLINK | SF_SNAPSHOT | UF_APPEND | UF_ARCHIVE |
UF_HIDDEN | UF_IMMUTABLE | UF_NODUMP | UF_NOUNLINK |
UF_OFFLINE | UF_OPAQUE | UF_READONLY | UF_REPARSE |
UF_SPARSE | UF_SYSTEM)) != 0)
return (EOPNOTSUPP);
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
/*
* Callers may only modify the file flags on objects they
* have VADMIN rights for.
*/
if ((error = VOP_ACCESS(vp, VADMIN, cred, td)))
return (error);
/*
* Unprivileged processes are not permitted to unset system
* flags, or modify flags if any system flags are set.
* Privileged non-jail processes may not modify system flags
* if securelevel > 0 and any existing system flags are set.
* Privileged jail processes behave like privileged non-jail
* processes if the PR_ALLOW_CHFLAGS permission bit is set;
* otherwise, they behave like unprivileged processes.
*/
if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS)) {
if (ip->i_flags &
(SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) {
error = securelevel_gt(cred, 0);
if (error)
return (error);
}
/* The snapshot flag cannot be toggled. */
if ((vap->va_flags ^ ip->i_flags) & SF_SNAPSHOT)
return (EPERM);
} else {
if (ip->i_flags &
(SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) ||
((vap->va_flags ^ ip->i_flags) & SF_SETTABLE))
return (EPERM);
}
ip->i_flags = vap->va_flags;
DIP_SET(ip, i_flags, vap->va_flags);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
error = UFS_UPDATE(vp, 0);
if (ip->i_flags & (IMMUTABLE | APPEND))
return (error);
}
/*
* If immutable or append, no one can change any of its attributes
* except the ones already handled (in some cases, file flags
* including the immutability flags themselves for the superuser).
*/
if (ip->i_flags & (IMMUTABLE | APPEND))
return (EPERM);
/*
* Go through the fields and update iff not VNOVAL.
*/
if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) {
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
if ((error = ufs_chown(vp, vap->va_uid, vap->va_gid, cred,
td)) != 0)
return (error);
}
if (vap->va_size != VNOVAL) {
/*
* XXX most of the following special cases should be in
* callers instead of in N filesystems. The VDIR check
* mostly already is.
*/
switch (vp->v_type) {
case VDIR:
return (EISDIR);
case VLNK:
case VREG:
/*
* Truncation should have an effect in these cases.
* Disallow it if the filesystem is read-only or
* the file is being snapshotted.
*/
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
if ((ip->i_flags & SF_SNAPSHOT) != 0)
return (EPERM);
break;
default:
/*
* According to POSIX, the result is unspecified
* for file types other than regular files,
* directories and shared memory objects. We
* don't support shared memory objects in the file
* system, and have dubious support for truncating
* symlinks. Just ignore the request in other cases.
*/
return (0);
}
if ((error = UFS_TRUNCATE(vp, vap->va_size, IO_NORMAL |
((vap->va_vaflags & VA_SYNC) != 0 ? IO_SYNC : 0),
cred)) != 0)
return (error);
}
if (vap->va_atime.tv_sec != VNOVAL ||
vap->va_mtime.tv_sec != VNOVAL ||
vap->va_birthtime.tv_sec != VNOVAL) {
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
if ((ip->i_flags & SF_SNAPSHOT) != 0)
return (EPERM);
error = vn_utimes_perm(vp, vap, cred, td);
if (error != 0)
return (error);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
if (vap->va_atime.tv_sec != VNOVAL) {
ip->i_flag &= ~IN_ACCESS;
DIP_SET(ip, i_atime, vap->va_atime.tv_sec);
DIP_SET(ip, i_atimensec, vap->va_atime.tv_nsec);
}
if (vap->va_mtime.tv_sec != VNOVAL) {
ip->i_flag &= ~IN_UPDATE;
DIP_SET(ip, i_mtime, vap->va_mtime.tv_sec);
DIP_SET(ip, i_mtimensec, vap->va_mtime.tv_nsec);
}
if (vap->va_birthtime.tv_sec != VNOVAL && I_IS_UFS2(ip)) {
ip->i_din2->di_birthtime = vap->va_birthtime.tv_sec;
ip->i_din2->di_birthnsec = vap->va_birthtime.tv_nsec;
}
error = UFS_UPDATE(vp, 0);
if (error)
return (error);
}
error = 0;
if (vap->va_mode != (mode_t)VNOVAL) {
if (vp->v_mount->mnt_flag & MNT_RDONLY)
return (EROFS);
if ((ip->i_flags & SF_SNAPSHOT) != 0 && (vap->va_mode &
(S_IXUSR | S_IWUSR | S_IXGRP | S_IWGRP | S_IXOTH | S_IWOTH)))
return (EPERM);
error = ufs_chmod(vp, (int)vap->va_mode, cred, td);
}
return (error);
}
#ifdef UFS_ACL
static int
ufs_update_nfs4_acl_after_mode_change(struct vnode *vp, int mode,
int file_owner_id, struct ucred *cred, struct thread *td)
{
int error;
struct acl *aclp;
aclp = acl_alloc(M_WAITOK);
error = ufs_getacl_nfs4_internal(vp, aclp, td);
/*
* We don't have to handle EOPNOTSUPP here, as the filesystem claims
* it supports ACLs.
*/
if (error)
goto out;
acl_nfs4_sync_acl_from_mode(aclp, mode, file_owner_id);
error = ufs_setacl_nfs4_internal(vp, aclp, td);
out:
acl_free(aclp);
return (error);
}
#endif /* UFS_ACL */
static int
ufs_mmapped(ap)
struct vop_mmapped_args /* {
struct vnode *a_vp;
} */ *ap;
{
struct vnode *vp;
struct inode *ip;
struct mount *mp;
vp = ap->a_vp;
ip = VTOI(vp);
mp = vp->v_mount;
if ((mp->mnt_flag & (MNT_NOATIME | MNT_RDONLY)) == 0)
UFS_INODE_SET_FLAG_SHARED(ip, IN_ACCESS);
/*
* XXXKIB No UFS_UPDATE(ap->a_vp, 0) there.
*/
return (0);
}
/*
* Change the mode on a file.
* Inode must be locked before calling.
*/
static int
ufs_chmod(vp, mode, cred, td)
struct vnode *vp;
int mode;
struct ucred *cred;
struct thread *td;
{
struct inode *ip = VTOI(vp);
int error;
/*
* To modify the permissions on a file, must possess VADMIN
* for that file.
*/
if ((error = VOP_ACCESSX(vp, VWRITE_ACL, cred, td)))
return (error);
/*
* Privileged processes may set the sticky bit on non-directories,
* as well as set the setgid bit on a file with a group that the
* process is not a member of. Both of these are allowed in
* jail(8).
*/
if (vp->v_type != VDIR && (mode & S_ISTXT)) {
if (priv_check_cred(cred, PRIV_VFS_STICKYFILE))
return (EFTYPE);
}
if (!groupmember(ip->i_gid, cred) && (mode & ISGID)) {
error = priv_check_cred(cred, PRIV_VFS_SETGID);
if (error)
return (error);
}
/*
* Deny setting setuid if we are not the file owner.
*/
if ((mode & ISUID) && ip->i_uid != cred->cr_uid) {
error = priv_check_cred(cred, PRIV_VFS_ADMIN);
if (error)
return (error);
}
ip->i_mode &= ~ALLPERMS;
ip->i_mode |= (mode & ALLPERMS);
DIP_SET(ip, i_mode, ip->i_mode);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
#ifdef UFS_ACL
if ((vp->v_mount->mnt_flag & MNT_NFS4ACLS) != 0)
error = ufs_update_nfs4_acl_after_mode_change(vp, mode, ip->i_uid, cred, td);
#endif
if (error == 0 && (ip->i_flag & IN_CHANGE) != 0)
error = UFS_UPDATE(vp, 0);
return (error);
}
/*
* Perform chown operation on inode ip;
* inode must be locked prior to call.
*/
static int
ufs_chown(vp, uid, gid, cred, td)
struct vnode *vp;
uid_t uid;
gid_t gid;
struct ucred *cred;
struct thread *td;
{
struct inode *ip = VTOI(vp);
uid_t ouid;
gid_t ogid;
int error = 0;
#ifdef QUOTA
int i;
ufs2_daddr_t change;
#endif
if (uid == (uid_t)VNOVAL)
uid = ip->i_uid;
if (gid == (gid_t)VNOVAL)
gid = ip->i_gid;
/*
* To modify the ownership of a file, must possess VADMIN for that
* file.
*/
if ((error = VOP_ACCESSX(vp, VWRITE_OWNER, cred, td)))
return (error);
/*
* To change the owner of a file, or change the group of a file to a
* group of which we are not a member, the caller must have
* privilege.
*/
if (((uid != ip->i_uid && uid != cred->cr_uid) ||
(gid != ip->i_gid && !groupmember(gid, cred))) &&
(error = priv_check_cred(cred, PRIV_VFS_CHOWN)))
return (error);
ogid = ip->i_gid;
ouid = ip->i_uid;
#ifdef QUOTA
if ((error = getinoquota(ip)) != 0)
return (error);
if (ouid == uid) {
dqrele(vp, ip->i_dquot[USRQUOTA]);
ip->i_dquot[USRQUOTA] = NODQUOT;
}
if (ogid == gid) {
dqrele(vp, ip->i_dquot[GRPQUOTA]);
ip->i_dquot[GRPQUOTA] = NODQUOT;
}
change = DIP(ip, i_blocks);
(void) chkdq(ip, -change, cred, CHOWN|FORCE);
(void) chkiq(ip, -1, cred, CHOWN|FORCE);
for (i = 0; i < MAXQUOTAS; i++) {
dqrele(vp, ip->i_dquot[i]);
ip->i_dquot[i] = NODQUOT;
}
#endif
ip->i_gid = gid;
DIP_SET(ip, i_gid, gid);
ip->i_uid = uid;
DIP_SET(ip, i_uid, uid);
#ifdef QUOTA
if ((error = getinoquota(ip)) == 0) {
if (ouid == uid) {
dqrele(vp, ip->i_dquot[USRQUOTA]);
ip->i_dquot[USRQUOTA] = NODQUOT;
}
if (ogid == gid) {
dqrele(vp, ip->i_dquot[GRPQUOTA]);
ip->i_dquot[GRPQUOTA] = NODQUOT;
}
if ((error = chkdq(ip, change, cred, CHOWN)) == 0) {
if ((error = chkiq(ip, 1, cred, CHOWN)) == 0)
goto good;
else
(void) chkdq(ip, -change, cred, CHOWN|FORCE);
}
for (i = 0; i < MAXQUOTAS; i++) {
dqrele(vp, ip->i_dquot[i]);
ip->i_dquot[i] = NODQUOT;
}
}
ip->i_gid = ogid;
DIP_SET(ip, i_gid, ogid);
ip->i_uid = ouid;
DIP_SET(ip, i_uid, ouid);
if (getinoquota(ip) == 0) {
if (ouid == uid) {
dqrele(vp, ip->i_dquot[USRQUOTA]);
ip->i_dquot[USRQUOTA] = NODQUOT;
}
if (ogid == gid) {
dqrele(vp, ip->i_dquot[GRPQUOTA]);
ip->i_dquot[GRPQUOTA] = NODQUOT;
}
(void) chkdq(ip, change, cred, FORCE|CHOWN);
(void) chkiq(ip, 1, cred, FORCE|CHOWN);
(void) getinoquota(ip);
}
return (error);
good:
if (getinoquota(ip))
panic("ufs_chown: lost quota");
#endif /* QUOTA */
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if ((ip->i_mode & (ISUID | ISGID)) && (ouid != uid || ogid != gid)) {
if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID)) {
ip->i_mode &= ~(ISUID | ISGID);
DIP_SET(ip, i_mode, ip->i_mode);
}
}
error = UFS_UPDATE(vp, 0);
return (error);
}
static int
ufs_remove(ap)
struct vop_remove_args /* {
struct vnode *a_dvp;
struct vnode *a_vp;
struct componentname *a_cnp;
} */ *ap;
{
struct inode *ip;
struct vnode *vp = ap->a_vp;
struct vnode *dvp = ap->a_dvp;
int error;
struct thread *td;
td = curthread;
ip = VTOI(vp);
if ((ip->i_flags & (NOUNLINK | IMMUTABLE | APPEND)) ||
(VTOI(dvp)->i_flags & APPEND)) {
error = EPERM;
goto out;
}
#ifdef UFS_GJOURNAL
ufs_gjournal_orphan(vp);
#endif
error = ufs_dirremove(dvp, ip, ap->a_cnp->cn_flags, 0);
if (ip->i_nlink <= 0)
vp->v_vflag |= VV_NOSYNC;
if ((ip->i_flags & SF_SNAPSHOT) != 0) {
/*
* Avoid deadlock where another thread is trying to
* update the inodeblock for dvp and is waiting on
* snaplk. Temporary unlock the vnode lock for the
* unlinked file and sync the directory. This should
* allow vput() of the directory to not block later on
* while holding the snapshot vnode locked, assuming
* that the directory hasn't been unlinked too.
*/
VOP_UNLOCK(vp);
(void) VOP_FSYNC(dvp, MNT_WAIT, td);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
}
out:
return (error);
}
static void
print_bad_link_count(const char *funcname, struct vnode *dvp)
{
struct inode *dip;
dip = VTOI(dvp);
uprintf("%s: Bad link count %d on parent inode %jd in file system %s\n",
funcname, dip->i_effnlink, (intmax_t)dip->i_number,
dvp->v_mount->mnt_stat.f_mntonname);
}
/*
* link vnode call
*/
static int
ufs_link(ap)
struct vop_link_args /* {
struct vnode *a_tdvp;
struct vnode *a_vp;
struct componentname *a_cnp;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct vnode *tdvp = ap->a_tdvp;
struct componentname *cnp = ap->a_cnp;
struct inode *ip;
struct direct newdir;
int error;
#ifdef INVARIANTS
if ((cnp->cn_flags & HASBUF) == 0)
panic("ufs_link: no name");
#endif
if (VTOI(tdvp)->i_effnlink < 2) {
print_bad_link_count("ufs_link", tdvp);
error = EINVAL;
goto out;
}
ip = VTOI(vp);
if (ip->i_nlink >= UFS_LINK_MAX) {
error = EMLINK;
goto out;
}
/*
* The file may have been removed after namei droped the original
* lock.
*/
if (ip->i_effnlink == 0) {
error = ENOENT;
goto out;
}
if (ip->i_flags & (IMMUTABLE | APPEND)) {
error = EPERM;
goto out;
}
ip->i_effnlink++;
ip->i_nlink++;
DIP_SET(ip, i_nlink, ip->i_nlink);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (DOINGSOFTDEP(vp))
softdep_setup_link(VTOI(tdvp), ip);
error = UFS_UPDATE(vp, !DOINGSOFTDEP(vp) && !DOINGASYNC(vp));
if (!error) {
ufs_makedirentry(ip, cnp, &newdir);
error = ufs_direnter(tdvp, vp, &newdir, cnp, NULL, 0);
}
if (error) {
ip->i_effnlink--;
ip->i_nlink--;
DIP_SET(ip, i_nlink, ip->i_nlink);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (DOINGSOFTDEP(vp))
softdep_revert_link(VTOI(tdvp), ip);
}
out:
return (error);
}
/*
* whiteout vnode call
*/
static int
ufs_whiteout(ap)
struct vop_whiteout_args /* {
struct vnode *a_dvp;
struct componentname *a_cnp;
int a_flags;
} */ *ap;
{
struct vnode *dvp = ap->a_dvp;
struct componentname *cnp = ap->a_cnp;
struct direct newdir;
int error = 0;
switch (ap->a_flags) {
case LOOKUP:
/* 4.4 format directories support whiteout operations */
if (dvp->v_mount->mnt_maxsymlinklen > 0)
return (0);
return (EOPNOTSUPP);
case CREATE:
/* create a new directory whiteout */
#ifdef INVARIANTS
if ((cnp->cn_flags & SAVENAME) == 0)
panic("ufs_whiteout: missing name");
if (dvp->v_mount->mnt_maxsymlinklen <= 0)
panic("ufs_whiteout: old format filesystem");
#endif
newdir.d_ino = UFS_WINO;
newdir.d_namlen = cnp->cn_namelen;
bcopy(cnp->cn_nameptr, newdir.d_name, (unsigned)cnp->cn_namelen + 1);
newdir.d_type = DT_WHT;
error = ufs_direnter(dvp, NULL, &newdir, cnp, NULL, 0);
break;
case DELETE:
/* remove an existing directory whiteout */
#ifdef INVARIANTS
if (dvp->v_mount->mnt_maxsymlinklen <= 0)
panic("ufs_whiteout: old format filesystem");
#endif
cnp->cn_flags &= ~DOWHITEOUT;
error = ufs_dirremove(dvp, NULL, cnp->cn_flags, 0);
break;
default:
panic("ufs_whiteout: unknown op");
}
return (error);
}
static volatile int rename_restarts;
SYSCTL_INT(_vfs_ufs, OID_AUTO, rename_restarts, CTLFLAG_RD,
__DEVOLATILE(int *, &rename_restarts), 0,
"Times rename had to restart due to lock contention");
/*
* Rename system call.
* rename("foo", "bar");
* is essentially
* unlink("bar");
* link("foo", "bar");
* unlink("foo");
* but ``atomically''. Can't do full commit without saving state in the
* inode on disk which isn't feasible at this time. Best we can do is
* always guarantee the target exists.
*
* Basic algorithm is:
*
* 1) Bump link count on source while we're linking it to the
* target. This also ensure the inode won't be deleted out
* from underneath us while we work (it may be truncated by
* a concurrent `trunc' or `open' for creation).
* 2) Link source to destination. If destination already exists,
* delete it first.
* 3) Unlink source reference to inode if still around. If a
* directory was moved and the parent of the destination
* is different from the source, patch the ".." entry in the
* directory.
*/
static int
ufs_rename(ap)
struct vop_rename_args /* {
struct vnode *a_fdvp;
struct vnode *a_fvp;
struct componentname *a_fcnp;
struct vnode *a_tdvp;
struct vnode *a_tvp;
struct componentname *a_tcnp;
} */ *ap;
{
struct vnode *tvp = ap->a_tvp;
struct vnode *tdvp = ap->a_tdvp;
struct vnode *fvp = ap->a_fvp;
struct vnode *fdvp = ap->a_fdvp;
struct vnode *nvp;
struct componentname *tcnp = ap->a_tcnp;
struct componentname *fcnp = ap->a_fcnp;
struct thread *td = fcnp->cn_thread;
struct inode *fip, *tip, *tdp, *fdp;
struct direct newdir;
off_t endoff;
int doingdirectory, newparent;
int error = 0;
struct mount *mp;
ino_t ino;
#ifdef INVARIANTS
if ((tcnp->cn_flags & HASBUF) == 0 ||
(fcnp->cn_flags & HASBUF) == 0)
panic("ufs_rename: no name");
#endif
endoff = 0;
mp = tdvp->v_mount;
VOP_UNLOCK(tdvp);
if (tvp && tvp != tdvp)
VOP_UNLOCK(tvp);
/*
* Check for cross-device rename.
*/
if ((fvp->v_mount != tdvp->v_mount) ||
(tvp && (fvp->v_mount != tvp->v_mount))) {
error = EXDEV;
mp = NULL;
goto releout;
}
relock:
/*
* We need to acquire 2 to 4 locks depending on whether tvp is NULL
* and fdvp and tdvp are the same directory. Subsequently we need
* to double-check all paths and in the directory rename case we
* need to verify that we are not creating a directory loop. To
* handle this we acquire all but fdvp using non-blocking
* acquisitions. If we fail to acquire any lock in the path we will
* drop all held locks, acquire the new lock in a blocking fashion,
* and then release it and restart the rename. This acquire/release
* step ensures that we do not spin on a lock waiting for release.
*/
error = vn_lock(fdvp, LK_EXCLUSIVE);
if (error)
goto releout;
if (vn_lock(tdvp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
VOP_UNLOCK(fdvp);
error = vn_lock(tdvp, LK_EXCLUSIVE);
if (error)
goto releout;
VOP_UNLOCK(tdvp);
atomic_add_int(&rename_restarts, 1);
goto relock;
}
/*
* Re-resolve fvp to be certain it still exists and fetch the
* correct vnode.
*/
error = ufs_lookup_ino(fdvp, NULL, fcnp, &ino);
if (error) {
VOP_UNLOCK(fdvp);
VOP_UNLOCK(tdvp);
goto releout;
}
error = VFS_VGET(mp, ino, LK_EXCLUSIVE | LK_NOWAIT, &nvp);
if (error) {
VOP_UNLOCK(fdvp);
VOP_UNLOCK(tdvp);
if (error != EBUSY)
goto releout;
error = VFS_VGET(mp, ino, LK_EXCLUSIVE, &nvp);
if (error != 0)
goto releout;
VOP_UNLOCK(nvp);
vrele(fvp);
fvp = nvp;
atomic_add_int(&rename_restarts, 1);
goto relock;
}
vrele(fvp);
fvp = nvp;
/*
* Re-resolve tvp and acquire the vnode lock if present.
*/
error = ufs_lookup_ino(tdvp, NULL, tcnp, &ino);
if (error != 0 && error != EJUSTRETURN) {
VOP_UNLOCK(fdvp);
VOP_UNLOCK(tdvp);
VOP_UNLOCK(fvp);
goto releout;
}
/*
* If tvp disappeared we just carry on.
*/
if (error == EJUSTRETURN && tvp != NULL) {
vrele(tvp);
tvp = NULL;
}
/*
* Get the tvp ino if the lookup succeeded. We may have to restart
* if the non-blocking acquire fails.
*/
if (error == 0) {
nvp = NULL;
error = VFS_VGET(mp, ino, LK_EXCLUSIVE | LK_NOWAIT, &nvp);
if (tvp)
vrele(tvp);
tvp = nvp;
if (error) {
VOP_UNLOCK(fdvp);
VOP_UNLOCK(tdvp);
VOP_UNLOCK(fvp);
if (error != EBUSY)
goto releout;
error = VFS_VGET(mp, ino, LK_EXCLUSIVE, &nvp);
if (error != 0)
goto releout;
vput(nvp);
atomic_add_int(&rename_restarts, 1);
goto relock;
}
}
fdp = VTOI(fdvp);
fip = VTOI(fvp);
tdp = VTOI(tdvp);
tip = NULL;
if (tvp)
tip = VTOI(tvp);
if (tvp && ((VTOI(tvp)->i_flags & (NOUNLINK | IMMUTABLE | APPEND)) ||
(VTOI(tdvp)->i_flags & APPEND))) {
error = EPERM;
goto unlockout;
}
/*
* Renaming a file to itself has no effect. The upper layers should
* not call us in that case. However, things could change after
* we drop the locks above.
*/
if (fvp == tvp) {
error = 0;
goto unlockout;
}
doingdirectory = 0;
newparent = 0;
ino = fip->i_number;
if (fip->i_nlink >= UFS_LINK_MAX) {
error = EMLINK;
goto unlockout;
}
if ((fip->i_flags & (NOUNLINK | IMMUTABLE | APPEND))
|| (fdp->i_flags & APPEND)) {
error = EPERM;
goto unlockout;
}
if ((fip->i_mode & IFMT) == IFDIR) {
/*
* Avoid ".", "..", and aliases of "." for obvious reasons.
*/
if ((fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.') ||
fdp == fip ||
(fcnp->cn_flags | tcnp->cn_flags) & ISDOTDOT) {
error = EINVAL;
goto unlockout;
}
if (fdp->i_number != tdp->i_number)
newparent = tdp->i_number;
doingdirectory = 1;
}
if ((fvp->v_type == VDIR && fvp->v_mountedhere != NULL) ||
(tvp != NULL && tvp->v_type == VDIR &&
tvp->v_mountedhere != NULL)) {
error = EXDEV;
goto unlockout;
}
/*
* If ".." must be changed (ie the directory gets a new
* parent) then the source directory must not be in the
* directory hierarchy above the target, as this would
* orphan everything below the source directory. Also
* the user must have write permission in the source so
* as to be able to change "..".
*/
if (doingdirectory && newparent) {
error = VOP_ACCESS(fvp, VWRITE, tcnp->cn_cred, tcnp->cn_thread);
if (error)
goto unlockout;
error = ufs_checkpath(ino, fdp->i_number, tdp, tcnp->cn_cred,
&ino);
/*
* We encountered a lock that we have to wait for. Unlock
* everything else and VGET before restarting.
*/
if (ino) {
VOP_UNLOCK(fdvp);
VOP_UNLOCK(fvp);
VOP_UNLOCK(tdvp);
if (tvp)
VOP_UNLOCK(tvp);
error = VFS_VGET(mp, ino, LK_SHARED, &nvp);
if (error == 0)
vput(nvp);
atomic_add_int(&rename_restarts, 1);
goto relock;
}
if (error)
goto unlockout;
if ((tcnp->cn_flags & SAVESTART) == 0)
panic("ufs_rename: lost to startdir");
}
if (fip->i_effnlink == 0 || fdp->i_effnlink == 0 ||
tdp->i_effnlink == 0)
panic("Bad effnlink fip %p, fdp %p, tdp %p", fip, fdp, tdp);
/*
* 1) Bump link count while we're moving stuff
* around. If we crash somewhere before
* completing our work, the link count
* may be wrong, but correctable.
*/
fip->i_effnlink++;
fip->i_nlink++;
DIP_SET(fip, i_nlink, fip->i_nlink);
UFS_INODE_SET_FLAG(fip, IN_CHANGE);
if (DOINGSOFTDEP(fvp))
softdep_setup_link(tdp, fip);
error = UFS_UPDATE(fvp, !DOINGSOFTDEP(fvp) && !DOINGASYNC(fvp));
if (error)
goto bad;
/*
* 2) If target doesn't exist, link the target
* to the source and unlink the source.
* Otherwise, rewrite the target directory
* entry to reference the source inode and
* expunge the original entry's existence.
*/
if (tip == NULL) {
if (ITODEV(tdp) != ITODEV(fip))
panic("ufs_rename: EXDEV");
if (doingdirectory && newparent) {
/*
* Account for ".." in new directory.
* When source and destination have the same
* parent we don't adjust the link count. The
* actual link modification is completed when
* .. is rewritten below.
*/
if (tdp->i_nlink >= UFS_LINK_MAX) {
error = EMLINK;
goto bad;
}
}
ufs_makedirentry(fip, tcnp, &newdir);
error = ufs_direnter(tdvp, NULL, &newdir, tcnp, NULL, 1);
if (error)
goto bad;
/* Setup tdvp for directory compaction if needed. */
if (tdp->i_count && tdp->i_endoff &&
tdp->i_endoff < tdp->i_size)
endoff = tdp->i_endoff;
} else {
if (ITODEV(tip) != ITODEV(tdp) || ITODEV(tip) != ITODEV(fip))
panic("ufs_rename: EXDEV");
/*
* Short circuit rename(foo, foo).
*/
if (tip->i_number == fip->i_number)
panic("ufs_rename: same file");
/*
* If the parent directory is "sticky", then the caller
* must possess VADMIN for the parent directory, or the
* destination of the rename. This implements append-only
* directories.
*/
if ((tdp->i_mode & S_ISTXT) &&
VOP_ACCESS(tdvp, VADMIN, tcnp->cn_cred, td) &&
VOP_ACCESS(tvp, VADMIN, tcnp->cn_cred, td)) {
error = EPERM;
goto bad;
}
/*
* Target must be empty if a directory and have no links
* to it. Also, ensure source and target are compatible
* (both directories, or both not directories).
*/
if ((tip->i_mode & IFMT) == IFDIR) {
if ((tip->i_effnlink > 2) ||
!ufs_dirempty(tip, tdp->i_number, tcnp->cn_cred)) {
error = ENOTEMPTY;
goto bad;
}
if (!doingdirectory) {
error = ENOTDIR;
goto bad;
}
cache_purge(tdvp);
} else if (doingdirectory) {
error = EISDIR;
goto bad;
}
if (doingdirectory) {
if (!newparent) {
tdp->i_effnlink--;
if (DOINGSOFTDEP(tdvp))
softdep_change_linkcnt(tdp);
}
tip->i_effnlink--;
if (DOINGSOFTDEP(tvp))
softdep_change_linkcnt(tip);
}
error = ufs_dirrewrite(tdp, tip, fip->i_number,
IFTODT(fip->i_mode),
(doingdirectory && newparent) ? newparent : doingdirectory);
if (error) {
if (doingdirectory) {
if (!newparent) {
tdp->i_effnlink++;
if (DOINGSOFTDEP(tdvp))
softdep_change_linkcnt(tdp);
}
tip->i_effnlink++;
if (DOINGSOFTDEP(tvp))
softdep_change_linkcnt(tip);
}
+ goto bad;
}
if (doingdirectory && !DOINGSOFTDEP(tvp)) {
/*
* The only stuff left in the directory is "."
* and "..". The "." reference is inconsequential
* since we are quashing it. We have removed the "."
* reference and the reference in the parent directory,
* but there may be other hard links. The soft
* dependency code will arrange to do these operations
* after the parent directory entry has been deleted on
* disk, so when running with that code we avoid doing
* them now.
*/
if (!newparent) {
tdp->i_nlink--;
DIP_SET(tdp, i_nlink, tdp->i_nlink);
UFS_INODE_SET_FLAG(tdp, IN_CHANGE);
}
tip->i_nlink--;
DIP_SET(tip, i_nlink, tip->i_nlink);
UFS_INODE_SET_FLAG(tip, IN_CHANGE);
}
}
/*
* 3) Unlink the source. We have to resolve the path again to
* fixup the directory offset and count for ufs_dirremove.
*/
if (fdvp == tdvp) {
error = ufs_lookup_ino(fdvp, NULL, fcnp, &ino);
if (error)
panic("ufs_rename: from entry went away!");
if (ino != fip->i_number)
panic("ufs_rename: ino mismatch %ju != %ju\n",
(uintmax_t)ino, (uintmax_t)fip->i_number);
}
/*
* If the source is a directory with a
* new parent, the link count of the old
* parent directory must be decremented
* and ".." set to point to the new parent.
*/
if (doingdirectory && newparent) {
/*
* If tip exists we simply use its link, otherwise we must
* add a new one.
*/
if (tip == NULL) {
tdp->i_effnlink++;
tdp->i_nlink++;
DIP_SET(tdp, i_nlink, tdp->i_nlink);
UFS_INODE_SET_FLAG(tdp, IN_CHANGE);
if (DOINGSOFTDEP(tdvp))
softdep_setup_dotdot_link(tdp, fip);
error = UFS_UPDATE(tdvp, !DOINGSOFTDEP(tdvp) &&
!DOINGASYNC(tdvp));
/* Don't go to bad here as the new link exists. */
if (error)
goto unlockout;
} else if (DOINGSUJ(tdvp))
/* Journal must account for each new link. */
softdep_setup_dotdot_link(tdp, fip);
fip->i_offset = mastertemplate.dot_reclen;
ufs_dirrewrite(fip, fdp, newparent, DT_DIR, 0);
cache_purge(fdvp);
}
error = ufs_dirremove(fdvp, fip, fcnp->cn_flags, 0);
/*
* The kern_renameat() looks up the fvp using the DELETE flag, which
* causes the removal of the name cache entry for fvp.
* As the relookup of the fvp is done in two steps:
* ufs_lookup_ino() and then VFS_VGET(), another thread might do a
* normal lookup of the from name just before the VFS_VGET() call,
* causing the cache entry to be re-instantiated.
*
* The same issue also applies to tvp if it exists as
* otherwise we may have a stale name cache entry for the new
* name that references the old i-node if it has other links
* or open file descriptors.
*/
cache_purge(fvp);
if (tvp)
cache_purge(tvp);
cache_purge_negative(tdvp);
unlockout:
vput(fdvp);
vput(fvp);
if (tvp)
vput(tvp);
/*
* If compaction or fsync was requested do it now that other locks
* are no longer needed.
*/
if (error == 0 && endoff != 0) {
error = UFS_TRUNCATE(tdvp, endoff, IO_NORMAL |
(DOINGASYNC(tdvp) ? 0 : IO_SYNC), tcnp->cn_cred);
- if (error != 0)
+ if (error != 0 && !ffs_fsfail_cleanup(VFSTOUFS(mp), error))
vn_printf(tdvp,
"ufs_rename: failed to truncate, error %d\n",
error);
#ifdef UFS_DIRHASH
+ if (error != 0)
+ ufsdirhash_free(tdp);
else if (tdp->i_dirhash != NULL)
ufsdirhash_dirtrunc(tdp, endoff);
#endif
/*
* Even if the directory compaction failed, rename was
* succesful. Do not propagate a UFS_TRUNCATE() error
* to the caller.
*/
error = 0;
}
if (error == 0 && tdp->i_flag & IN_NEEDSYNC)
error = VOP_FSYNC(tdvp, MNT_WAIT, td);
vput(tdvp);
return (error);
bad:
fip->i_effnlink--;
fip->i_nlink--;
DIP_SET(fip, i_nlink, fip->i_nlink);
UFS_INODE_SET_FLAG(fip, IN_CHANGE);
if (DOINGSOFTDEP(fvp))
softdep_revert_link(tdp, fip);
goto unlockout;
releout:
vrele(fdvp);
vrele(fvp);
vrele(tdvp);
if (tvp)
vrele(tvp);
return (error);
}
#ifdef UFS_ACL
static int
ufs_do_posix1e_acl_inheritance_dir(struct vnode *dvp, struct vnode *tvp,
mode_t dmode, struct ucred *cred, struct thread *td)
{
int error;
struct inode *ip = VTOI(tvp);
struct acl *dacl, *acl;
acl = acl_alloc(M_WAITOK);
dacl = acl_alloc(M_WAITOK);
/*
* Retrieve default ACL from parent, if any.
*/
error = VOP_GETACL(dvp, ACL_TYPE_DEFAULT, acl, cred, td);
switch (error) {
case 0:
/*
* Retrieved a default ACL, so merge mode and ACL if
* necessary. If the ACL is empty, fall through to
* the "not defined or available" case.
*/
if (acl->acl_cnt != 0) {
dmode = acl_posix1e_newfilemode(dmode, acl);
ip->i_mode = dmode;
DIP_SET(ip, i_mode, dmode);
*dacl = *acl;
ufs_sync_acl_from_inode(ip, acl);
break;
}
/* FALLTHROUGH */
case EOPNOTSUPP:
/*
* Just use the mode as-is.
*/
ip->i_mode = dmode;
DIP_SET(ip, i_mode, dmode);
error = 0;
goto out;
default:
goto out;
}
/*
* XXX: If we abort now, will Soft Updates notify the extattr
* code that the EAs for the file need to be released?
*/
error = VOP_SETACL(tvp, ACL_TYPE_ACCESS, acl, cred, td);
if (error == 0)
error = VOP_SETACL(tvp, ACL_TYPE_DEFAULT, dacl, cred, td);
switch (error) {
case 0:
break;
case EOPNOTSUPP:
/*
* XXX: This should not happen, as EOPNOTSUPP above
* was supposed to free acl.
*/
printf("ufs_mkdir: VOP_GETACL() but no VOP_SETACL()\n");
/*
panic("ufs_mkdir: VOP_GETACL() but no VOP_SETACL()");
*/
break;
default:
goto out;
}
out:
acl_free(acl);
acl_free(dacl);
return (error);
}
static int
ufs_do_posix1e_acl_inheritance_file(struct vnode *dvp, struct vnode *tvp,
mode_t mode, struct ucred *cred, struct thread *td)
{
int error;
struct inode *ip = VTOI(tvp);
struct acl *acl;
acl = acl_alloc(M_WAITOK);
/*
* Retrieve default ACL for parent, if any.
*/
error = VOP_GETACL(dvp, ACL_TYPE_DEFAULT, acl, cred, td);
switch (error) {
case 0:
/*
* Retrieved a default ACL, so merge mode and ACL if
* necessary.
*/
if (acl->acl_cnt != 0) {
/*
* Two possible ways for default ACL to not
* be present. First, the EA can be
* undefined, or second, the default ACL can
* be blank. If it's blank, fall through to
* the it's not defined case.
*/
mode = acl_posix1e_newfilemode(mode, acl);
ip->i_mode = mode;
DIP_SET(ip, i_mode, mode);
ufs_sync_acl_from_inode(ip, acl);
break;
}
/* FALLTHROUGH */
case EOPNOTSUPP:
/*
* Just use the mode as-is.
*/
ip->i_mode = mode;
DIP_SET(ip, i_mode, mode);
error = 0;
goto out;
default:
goto out;
}
/*
* XXX: If we abort now, will Soft Updates notify the extattr
* code that the EAs for the file need to be released?
*/
error = VOP_SETACL(tvp, ACL_TYPE_ACCESS, acl, cred, td);
switch (error) {
case 0:
break;
case EOPNOTSUPP:
/*
* XXX: This should not happen, as EOPNOTSUPP above was
* supposed to free acl.
*/
printf("ufs_do_posix1e_acl_inheritance_file: VOP_GETACL() "
"but no VOP_SETACL()\n");
/* panic("ufs_do_posix1e_acl_inheritance_file: VOP_GETACL() "
"but no VOP_SETACL()"); */
break;
default:
goto out;
}
out:
acl_free(acl);
return (error);
}
static int
ufs_do_nfs4_acl_inheritance(struct vnode *dvp, struct vnode *tvp,
mode_t child_mode, struct ucred *cred, struct thread *td)
{
int error;
struct acl *parent_aclp, *child_aclp;
parent_aclp = acl_alloc(M_WAITOK);
child_aclp = acl_alloc(M_WAITOK | M_ZERO);
error = ufs_getacl_nfs4_internal(dvp, parent_aclp, td);
if (error)
goto out;
acl_nfs4_compute_inherited_acl(parent_aclp, child_aclp,
child_mode, VTOI(tvp)->i_uid, tvp->v_type == VDIR);
error = ufs_setacl_nfs4_internal(tvp, child_aclp, td);
if (error)
goto out;
out:
acl_free(parent_aclp);
acl_free(child_aclp);
return (error);
}
#endif
/*
* Mkdir system call
*/
static int
ufs_mkdir(ap)
struct vop_mkdir_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
} */ *ap;
{
struct vnode *dvp = ap->a_dvp;
struct vattr *vap = ap->a_vap;
struct componentname *cnp = ap->a_cnp;
struct inode *ip, *dp;
struct vnode *tvp;
struct buf *bp;
struct dirtemplate dirtemplate, *dtp;
struct direct newdir;
int error, dmode;
long blkoff;
#ifdef INVARIANTS
if ((cnp->cn_flags & HASBUF) == 0)
panic("ufs_mkdir: no name");
#endif
dp = VTOI(dvp);
if (dp->i_nlink >= UFS_LINK_MAX) {
error = EMLINK;
goto out;
}
dmode = vap->va_mode & 0777;
dmode |= IFDIR;
/*
* Must simulate part of ufs_makeinode here to acquire the inode,
* but not have it entered in the parent directory. The entry is
* made later after writing "." and ".." entries.
*/
if (dp->i_effnlink < 2) {
print_bad_link_count("ufs_mkdir", dvp);
error = EINVAL;
goto out;
}
error = UFS_VALLOC(dvp, dmode, cnp->cn_cred, &tvp);
if (error)
goto out;
ip = VTOI(tvp);
ip->i_gid = dp->i_gid;
DIP_SET(ip, i_gid, dp->i_gid);
#ifdef SUIDDIR
{
#ifdef QUOTA
struct ucred ucred, *ucp;
gid_t ucred_group;
ucp = cnp->cn_cred;
#endif
/*
* If we are hacking owners here, (only do this where told to)
* and we are not giving it TO root, (would subvert quotas)
* then go ahead and give it to the other user.
* The new directory also inherits the SUID bit.
* If user's UID and dir UID are the same,
* 'give it away' so that the SUID is still forced on.
*/
if ((dvp->v_mount->mnt_flag & MNT_SUIDDIR) &&
(dp->i_mode & ISUID) && dp->i_uid) {
dmode |= ISUID;
ip->i_uid = dp->i_uid;
DIP_SET(ip, i_uid, dp->i_uid);
#ifdef QUOTA
if (dp->i_uid != cnp->cn_cred->cr_uid) {
/*
* Make sure the correct user gets charged
* for the space.
* Make a dummy credential for the victim.
* XXX This seems to never be accessed out of
* our context so a stack variable is ok.
*/
refcount_init(&ucred.cr_ref, 1);
ucred.cr_uid = ip->i_uid;
ucred.cr_ngroups = 1;
ucred.cr_groups = &ucred_group;
ucred.cr_groups[0] = dp->i_gid;
ucp = &ucred;
}
#endif
} else {
ip->i_uid = cnp->cn_cred->cr_uid;
DIP_SET(ip, i_uid, ip->i_uid);
}
#ifdef QUOTA
if ((error = getinoquota(ip)) ||
(error = chkiq(ip, 1, ucp, 0))) {
if (DOINGSOFTDEP(tvp))
softdep_revert_link(dp, ip);
UFS_VFREE(tvp, ip->i_number, dmode);
vgone(tvp);
vput(tvp);
return (error);
}
#endif
}
#else /* !SUIDDIR */
ip->i_uid = cnp->cn_cred->cr_uid;
DIP_SET(ip, i_uid, ip->i_uid);
#ifdef QUOTA
if ((error = getinoquota(ip)) ||
(error = chkiq(ip, 1, cnp->cn_cred, 0))) {
if (DOINGSOFTDEP(tvp))
softdep_revert_link(dp, ip);
UFS_VFREE(tvp, ip->i_number, dmode);
vgone(tvp);
vput(tvp);
return (error);
}
#endif
#endif /* !SUIDDIR */
UFS_INODE_SET_FLAG(ip, IN_ACCESS | IN_CHANGE | IN_UPDATE);
ip->i_mode = dmode;
DIP_SET(ip, i_mode, dmode);
tvp->v_type = VDIR; /* Rest init'd in getnewvnode(). */
ip->i_effnlink = 2;
ip->i_nlink = 2;
DIP_SET(ip, i_nlink, 2);
if (cnp->cn_flags & ISWHITEOUT) {
ip->i_flags |= UF_OPAQUE;
DIP_SET(ip, i_flags, ip->i_flags);
}
/*
* Bump link count in parent directory to reflect work done below.
* Should be done before reference is created so cleanup is
* possible if we crash.
*/
dp->i_effnlink++;
dp->i_nlink++;
DIP_SET(dp, i_nlink, dp->i_nlink);
UFS_INODE_SET_FLAG(dp, IN_CHANGE);
if (DOINGSOFTDEP(dvp))
softdep_setup_mkdir(dp, ip);
error = UFS_UPDATE(dvp, !DOINGSOFTDEP(dvp) && !DOINGASYNC(dvp));
if (error)
goto bad;
#ifdef MAC
if (dvp->v_mount->mnt_flag & MNT_MULTILABEL) {
error = mac_vnode_create_extattr(cnp->cn_cred, dvp->v_mount,
dvp, tvp, cnp);
if (error)
goto bad;
}
#endif
#ifdef UFS_ACL
if (dvp->v_mount->mnt_flag & MNT_ACLS) {
error = ufs_do_posix1e_acl_inheritance_dir(dvp, tvp, dmode,
cnp->cn_cred, cnp->cn_thread);
if (error)
goto bad;
} else if (dvp->v_mount->mnt_flag & MNT_NFS4ACLS) {
error = ufs_do_nfs4_acl_inheritance(dvp, tvp, dmode,
cnp->cn_cred, cnp->cn_thread);
if (error)
goto bad;
}
#endif /* !UFS_ACL */
/*
* Initialize directory with "." and ".." from static template.
*/
if (dvp->v_mount->mnt_maxsymlinklen > 0)
dtp = &mastertemplate;
else
dtp = (struct dirtemplate *)&omastertemplate;
dirtemplate = *dtp;
dirtemplate.dot_ino = ip->i_number;
dirtemplate.dotdot_ino = dp->i_number;
vnode_pager_setsize(tvp, DIRBLKSIZ);
if ((error = UFS_BALLOC(tvp, (off_t)0, DIRBLKSIZ, cnp->cn_cred,
BA_CLRBUF, &bp)) != 0)
goto bad;
ip->i_size = DIRBLKSIZ;
DIP_SET(ip, i_size, DIRBLKSIZ);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
bcopy((caddr_t)&dirtemplate, (caddr_t)bp->b_data, sizeof dirtemplate);
if (DOINGSOFTDEP(tvp)) {
/*
* Ensure that the entire newly allocated block is a
* valid directory so that future growth within the
* block does not have to ensure that the block is
* written before the inode.
*/
blkoff = DIRBLKSIZ;
while (blkoff < bp->b_bcount) {
((struct direct *)
(bp->b_data + blkoff))->d_reclen = DIRBLKSIZ;
blkoff += DIRBLKSIZ;
}
}
if ((error = UFS_UPDATE(tvp, !DOINGSOFTDEP(tvp) &&
!DOINGASYNC(tvp))) != 0) {
(void)bwrite(bp);
goto bad;
}
/*
* Directory set up, now install its entry in the parent directory.
*
* If we are not doing soft dependencies, then we must write out the
* buffer containing the new directory body before entering the new
* name in the parent. If we are doing soft dependencies, then the
* buffer containing the new directory body will be passed to and
* released in the soft dependency code after the code has attached
* an appropriate ordering dependency to the buffer which ensures that
* the buffer is written before the new name is written in the parent.
*/
if (DOINGASYNC(dvp))
bdwrite(bp);
else if (!DOINGSOFTDEP(dvp) && ((error = bwrite(bp))))
goto bad;
ufs_makedirentry(ip, cnp, &newdir);
error = ufs_direnter(dvp, tvp, &newdir, cnp, bp, 0);
bad:
if (error == 0) {
*ap->a_vpp = tvp;
} else {
dp->i_effnlink--;
dp->i_nlink--;
DIP_SET(dp, i_nlink, dp->i_nlink);
UFS_INODE_SET_FLAG(dp, IN_CHANGE);
/*
* No need to do an explicit VOP_TRUNCATE here, vrele will
* do this for us because we set the link count to 0.
*/
ip->i_effnlink = 0;
ip->i_nlink = 0;
DIP_SET(ip, i_nlink, 0);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (DOINGSOFTDEP(tvp))
softdep_revert_mkdir(dp, ip);
vgone(tvp);
vput(tvp);
}
out:
return (error);
}
/*
* Rmdir system call.
*/
static int
ufs_rmdir(ap)
struct vop_rmdir_args /* {
struct vnode *a_dvp;
struct vnode *a_vp;
struct componentname *a_cnp;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct vnode *dvp = ap->a_dvp;
struct componentname *cnp = ap->a_cnp;
struct inode *ip, *dp;
int error;
ip = VTOI(vp);
dp = VTOI(dvp);
/*
* Do not remove a directory that is in the process of being renamed.
* Verify the directory is empty (and valid). Rmdir ".." will not be
* valid since ".." will contain a reference to the current directory
* and thus be non-empty. Do not allow the removal of mounted on
* directories (this can happen when an NFS exported filesystem
* tries to remove a locally mounted on directory).
*/
error = 0;
if (dp->i_effnlink <= 2) {
if (dp->i_effnlink == 2)
print_bad_link_count("ufs_rmdir", dvp);
error = EINVAL;
goto out;
}
if (!ufs_dirempty(ip, dp->i_number, cnp->cn_cred)) {
error = ENOTEMPTY;
goto out;
}
if ((dp->i_flags & APPEND)
|| (ip->i_flags & (NOUNLINK | IMMUTABLE | APPEND))) {
error = EPERM;
goto out;
}
if (vp->v_mountedhere != 0) {
error = EINVAL;
goto out;
}
#ifdef UFS_GJOURNAL
ufs_gjournal_orphan(vp);
#endif
/*
* Delete reference to directory before purging
* inode. If we crash in between, the directory
* will be reattached to lost+found,
*/
dp->i_effnlink--;
ip->i_effnlink--;
if (DOINGSOFTDEP(vp))
softdep_setup_rmdir(dp, ip);
error = ufs_dirremove(dvp, ip, cnp->cn_flags, 1);
if (error) {
dp->i_effnlink++;
ip->i_effnlink++;
if (DOINGSOFTDEP(vp))
softdep_revert_rmdir(dp, ip);
goto out;
}
cache_purge(dvp);
/*
* The only stuff left in the directory is "." and "..". The "."
* reference is inconsequential since we are quashing it. The soft
* dependency code will arrange to do these operations after
* the parent directory entry has been deleted on disk, so
* when running with that code we avoid doing them now.
*/
if (!DOINGSOFTDEP(vp)) {
dp->i_nlink--;
DIP_SET(dp, i_nlink, dp->i_nlink);
UFS_INODE_SET_FLAG(dp, IN_CHANGE);
error = UFS_UPDATE(dvp, 0);
ip->i_nlink--;
DIP_SET(ip, i_nlink, ip->i_nlink);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
}
cache_purge(vp);
#ifdef UFS_DIRHASH
/* Kill any active hash; i_effnlink == 0, so it will not come back. */
if (ip->i_dirhash != NULL)
ufsdirhash_free(ip);
#endif
out:
return (error);
}
/*
* symlink -- make a symbolic link
*/
static int
ufs_symlink(ap)
struct vop_symlink_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
const char *a_target;
} */ *ap;
{
struct vnode *vp, **vpp = ap->a_vpp;
struct inode *ip;
int len, error;
error = ufs_makeinode(IFLNK | ap->a_vap->va_mode, ap->a_dvp,
vpp, ap->a_cnp, "ufs_symlink");
if (error)
return (error);
vp = *vpp;
len = strlen(ap->a_target);
if (len < vp->v_mount->mnt_maxsymlinklen) {
ip = VTOI(vp);
bcopy(ap->a_target, SHORTLINK(ip), len);
ip->i_size = len;
DIP_SET(ip, i_size, len);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
error = UFS_UPDATE(vp, 0);
} else
error = vn_rdwr(UIO_WRITE, vp, __DECONST(void *, ap->a_target),
len, (off_t)0, UIO_SYSSPACE, IO_NODELOCKED | IO_NOMACCHECK,
ap->a_cnp->cn_cred, NOCRED, NULL, NULL);
if (error)
vput(vp);
return (error);
}
/*
* Vnode op for reading directories.
*/
int
ufs_readdir(ap)
struct vop_readdir_args /* {
struct vnode *a_vp;
struct uio *a_uio;
struct ucred *a_cred;
int *a_eofflag;
int *a_ncookies;
u_long **a_cookies;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct uio *uio = ap->a_uio;
struct buf *bp;
struct inode *ip;
struct direct *dp, *edp;
u_long *cookies;
struct dirent dstdp;
off_t offset, startoffset;
size_t readcnt, skipcnt;
ssize_t startresid;
u_int ncookies;
int error;
if (uio->uio_offset < 0)
return (EINVAL);
ip = VTOI(vp);
if (ip->i_effnlink == 0)
return (0);
if (ap->a_ncookies != NULL) {
if (uio->uio_resid < 0)
ncookies = 0;
else
ncookies = uio->uio_resid;
if (uio->uio_offset >= ip->i_size)
ncookies = 0;
else if (ip->i_size - uio->uio_offset < ncookies)
ncookies = ip->i_size - uio->uio_offset;
ncookies = ncookies / (offsetof(struct direct, d_name) + 4) + 1;
cookies = malloc(ncookies * sizeof(*cookies), M_TEMP, M_WAITOK);
*ap->a_ncookies = ncookies;
*ap->a_cookies = cookies;
} else {
ncookies = 0;
cookies = NULL;
}
offset = startoffset = uio->uio_offset;
startresid = uio->uio_resid;
error = 0;
while (error == 0 && uio->uio_resid > 0 &&
uio->uio_offset < ip->i_size) {
error = ffs_blkatoff(vp, uio->uio_offset, NULL, &bp);
if (error)
break;
if (bp->b_offset + bp->b_bcount > ip->i_size)
readcnt = ip->i_size - bp->b_offset;
else
readcnt = bp->b_bcount;
skipcnt = (size_t)(uio->uio_offset - bp->b_offset) &
~(size_t)(DIRBLKSIZ - 1);
offset = bp->b_offset + skipcnt;
dp = (struct direct *)&bp->b_data[skipcnt];
edp = (struct direct *)&bp->b_data[readcnt];
while (error == 0 && uio->uio_resid > 0 && dp < edp) {
if (dp->d_reclen <= offsetof(struct direct, d_name) ||
(caddr_t)dp + dp->d_reclen > (caddr_t)edp) {
error = EIO;
break;
}
#if BYTE_ORDER == LITTLE_ENDIAN
/* Old filesystem format. */
if (vp->v_mount->mnt_maxsymlinklen <= 0) {
dstdp.d_namlen = dp->d_type;
dstdp.d_type = dp->d_namlen;
} else
#endif
{
dstdp.d_namlen = dp->d_namlen;
dstdp.d_type = dp->d_type;
}
if (offsetof(struct direct, d_name) + dstdp.d_namlen >
dp->d_reclen) {
error = EIO;
break;
}
if (offset < startoffset || dp->d_ino == 0)
goto nextentry;
dstdp.d_fileno = dp->d_ino;
dstdp.d_reclen = GENERIC_DIRSIZ(&dstdp);
bcopy(dp->d_name, dstdp.d_name, dstdp.d_namlen);
/* NOTE: d_off is the offset of the *next* entry. */
dstdp.d_off = offset + dp->d_reclen;
dirent_terminate(&dstdp);
if (dstdp.d_reclen > uio->uio_resid) {
if (uio->uio_resid == startresid)
error = EINVAL;
else
error = EJUSTRETURN;
break;
}
/* Advance dp. */
error = uiomove((caddr_t)&dstdp, dstdp.d_reclen, uio);
if (error)
break;
if (cookies != NULL) {
KASSERT(ncookies > 0,
("ufs_readdir: cookies buffer too small"));
*cookies = offset + dp->d_reclen;
cookies++;
ncookies--;
}
nextentry:
offset += dp->d_reclen;
dp = (struct direct *)((caddr_t)dp + dp->d_reclen);
}
bqrelse(bp);
uio->uio_offset = offset;
}
/* We need to correct uio_offset. */
uio->uio_offset = offset;
if (error == EJUSTRETURN)
error = 0;
if (ap->a_ncookies != NULL) {
if (error == 0) {
ap->a_ncookies -= ncookies;
} else {
free(*ap->a_cookies, M_TEMP);
*ap->a_ncookies = 0;
*ap->a_cookies = NULL;
}
}
if (error == 0 && ap->a_eofflag)
*ap->a_eofflag = ip->i_size <= uio->uio_offset;
return (error);
}
/*
* Return target name of a symbolic link
*/
static int
ufs_readlink(ap)
struct vop_readlink_args /* {
struct vnode *a_vp;
struct uio *a_uio;
struct ucred *a_cred;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
doff_t isize;
isize = ip->i_size;
if ((isize < vp->v_mount->mnt_maxsymlinklen) ||
DIP(ip, i_blocks) == 0) { /* XXX - for old fastlink support */
return (uiomove(SHORTLINK(ip), isize, ap->a_uio));
}
return (VOP_READ(vp, ap->a_uio, 0, ap->a_cred));
}
/*
* Calculate the logical to physical mapping if not done already,
* then call the device strategy routine.
*
* In order to be able to swap to a file, the ufs_bmaparray() operation may not
* deadlock on memory. See ufs_bmap() for details.
*/
static int
ufs_strategy(ap)
struct vop_strategy_args /* {
struct vnode *a_vp;
struct buf *a_bp;
} */ *ap;
{
struct buf *bp = ap->a_bp;
struct vnode *vp = ap->a_vp;
ufs2_daddr_t blkno;
int error;
if (bp->b_blkno == bp->b_lblkno) {
error = ufs_bmaparray(vp, bp->b_lblkno, &blkno, bp, NULL, NULL);
bp->b_blkno = blkno;
if (error) {
bp->b_error = error;
bp->b_ioflags |= BIO_ERROR;
bufdone(bp);
return (0);
}
if ((long)bp->b_blkno == -1)
vfs_bio_clrbuf(bp);
}
if ((long)bp->b_blkno == -1) {
bufdone(bp);
return (0);
}
bp->b_iooffset = dbtob(bp->b_blkno);
BO_STRATEGY(VFSTOUFS(vp->v_mount)->um_bo, bp);
return (0);
}
/*
* Print out the contents of an inode.
*/
static int
ufs_print(ap)
struct vop_print_args /* {
struct vnode *a_vp;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
struct inode *ip = VTOI(vp);
printf("\tnlink=%d, effnlink=%d, size=%jd", ip->i_nlink,
ip->i_effnlink, (intmax_t)ip->i_size);
if (I_IS_UFS2(ip))
printf(", extsize %d", ip->i_din2->di_extsize);
printf("\n\tgeneration=%jx, uid=%d, gid=%d, flags=0x%b\n",
(uintmax_t)ip->i_gen, ip->i_uid, ip->i_gid,
(u_int)ip->i_flags, PRINT_INODE_FLAGS);
printf("\tino %lu, on dev %s", (u_long)ip->i_number,
devtoname(ITODEV(ip)));
if (vp->v_type == VFIFO)
fifo_printinfo(vp);
printf("\n");
return (0);
}
/*
* Close wrapper for fifos.
*
* Update the times on the inode then do device close.
*/
static int
ufsfifo_close(ap)
struct vop_close_args /* {
struct vnode *a_vp;
int a_fflag;
struct ucred *a_cred;
struct thread *a_td;
} */ *ap;
{
struct vnode *vp = ap->a_vp;
int usecount;
VI_LOCK(vp);
usecount = vp->v_usecount;
if (usecount > 1)
ufs_itimes_locked(vp);
VI_UNLOCK(vp);
return (fifo_specops.vop_close(ap));
}
/*
* Kqfilter wrapper for fifos.
*
* Fall through to ufs kqfilter routines if needed
*/
static int
ufsfifo_kqfilter(ap)
struct vop_kqfilter_args *ap;
{
int error;
error = fifo_specops.vop_kqfilter(ap);
if (error)
error = vfs_kqfilter(ap);
return (error);
}
/*
* Return POSIX pathconf information applicable to ufs filesystems.
*/
static int
ufs_pathconf(ap)
struct vop_pathconf_args /* {
struct vnode *a_vp;
int a_name;
int *a_retval;
} */ *ap;
{
int error;
error = 0;
switch (ap->a_name) {
case _PC_LINK_MAX:
*ap->a_retval = UFS_LINK_MAX;
break;
case _PC_NAME_MAX:
*ap->a_retval = UFS_MAXNAMLEN;
break;
case _PC_PIPE_BUF:
if (ap->a_vp->v_type == VDIR || ap->a_vp->v_type == VFIFO)
*ap->a_retval = PIPE_BUF;
else
error = EINVAL;
break;
case _PC_CHOWN_RESTRICTED:
*ap->a_retval = 1;
break;
case _PC_NO_TRUNC:
*ap->a_retval = 1;
break;
#ifdef UFS_ACL
case _PC_ACL_EXTENDED:
if (ap->a_vp->v_mount->mnt_flag & MNT_ACLS)
*ap->a_retval = 1;
else
*ap->a_retval = 0;
break;
case _PC_ACL_NFS4:
if (ap->a_vp->v_mount->mnt_flag & MNT_NFS4ACLS)
*ap->a_retval = 1;
else
*ap->a_retval = 0;
break;
#endif
case _PC_ACL_PATH_MAX:
#ifdef UFS_ACL
if (ap->a_vp->v_mount->mnt_flag & (MNT_ACLS | MNT_NFS4ACLS))
*ap->a_retval = ACL_MAX_ENTRIES;
else
*ap->a_retval = 3;
#else
*ap->a_retval = 3;
#endif
break;
#ifdef MAC
case _PC_MAC_PRESENT:
if (ap->a_vp->v_mount->mnt_flag & MNT_MULTILABEL)
*ap->a_retval = 1;
else
*ap->a_retval = 0;
break;
#endif
case _PC_MIN_HOLE_SIZE:
*ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize;
break;
case _PC_PRIO_IO:
*ap->a_retval = 0;
break;
case _PC_SYNC_IO:
*ap->a_retval = 0;
break;
case _PC_ALLOC_SIZE_MIN:
*ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_bsize;
break;
case _PC_FILESIZEBITS:
*ap->a_retval = 64;
break;
case _PC_REC_INCR_XFER_SIZE:
*ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize;
break;
case _PC_REC_MAX_XFER_SIZE:
*ap->a_retval = -1; /* means ``unlimited'' */
break;
case _PC_REC_MIN_XFER_SIZE:
*ap->a_retval = ap->a_vp->v_mount->mnt_stat.f_iosize;
break;
case _PC_REC_XFER_ALIGN:
*ap->a_retval = PAGE_SIZE;
break;
case _PC_SYMLINK_MAX:
*ap->a_retval = MAXPATHLEN;
break;
default:
error = vop_stdpathconf(ap);
break;
}
return (error);
}
/*
* Initialize the vnode associated with a new inode, handle aliased
* vnodes.
*/
int
ufs_vinit(mntp, fifoops, vpp)
struct mount *mntp;
struct vop_vector *fifoops;
struct vnode **vpp;
{
struct inode *ip;
struct vnode *vp;
vp = *vpp;
ASSERT_VOP_LOCKED(vp, "ufs_vinit");
ip = VTOI(vp);
vp->v_type = IFTOVT(ip->i_mode);
/*
* Only unallocated inodes should be of type VNON.
*/
if (ip->i_mode != 0 && vp->v_type == VNON)
return (EINVAL);
if (vp->v_type == VFIFO)
vp->v_op = fifoops;
if (ip->i_number == UFS_ROOTINO)
vp->v_vflag |= VV_ROOT;
*vpp = vp;
return (0);
}
/*
* Allocate a new inode.
* Vnode dvp must be locked.
*/
static int
ufs_makeinode(mode, dvp, vpp, cnp, callfunc)
int mode;
struct vnode *dvp;
struct vnode **vpp;
struct componentname *cnp;
const char *callfunc;
{
struct inode *ip, *pdir;
struct direct newdir;
struct vnode *tvp;
int error;
pdir = VTOI(dvp);
#ifdef INVARIANTS
if ((cnp->cn_flags & HASBUF) == 0)
panic("%s: no name", callfunc);
#endif
*vpp = NULL;
if ((mode & IFMT) == 0)
mode |= IFREG;
if (pdir->i_effnlink < 2) {
print_bad_link_count(callfunc, dvp);
return (EINVAL);
}
error = UFS_VALLOC(dvp, mode, cnp->cn_cred, &tvp);
if (error)
return (error);
ip = VTOI(tvp);
ip->i_gid = pdir->i_gid;
DIP_SET(ip, i_gid, pdir->i_gid);
#ifdef SUIDDIR
{
#ifdef QUOTA
struct ucred ucred, *ucp;
gid_t ucred_group;
ucp = cnp->cn_cred;
#endif
/*
* If we are not the owner of the directory,
* and we are hacking owners here, (only do this where told to)
* and we are not giving it TO root, (would subvert quotas)
* then go ahead and give it to the other user.
* Note that this drops off the execute bits for security.
*/
if ((dvp->v_mount->mnt_flag & MNT_SUIDDIR) &&
(pdir->i_mode & ISUID) &&
(pdir->i_uid != cnp->cn_cred->cr_uid) && pdir->i_uid) {
ip->i_uid = pdir->i_uid;
DIP_SET(ip, i_uid, ip->i_uid);
mode &= ~07111;
#ifdef QUOTA
/*
* Make sure the correct user gets charged
* for the space.
* Quickly knock up a dummy credential for the victim.
* XXX This seems to never be accessed out of our
* context so a stack variable is ok.
*/
refcount_init(&ucred.cr_ref, 1);
ucred.cr_uid = ip->i_uid;
ucred.cr_ngroups = 1;
ucred.cr_groups = &ucred_group;
ucred.cr_groups[0] = pdir->i_gid;
ucp = &ucred;
#endif
} else {
ip->i_uid = cnp->cn_cred->cr_uid;
DIP_SET(ip, i_uid, ip->i_uid);
}
#ifdef QUOTA
if ((error = getinoquota(ip)) ||
(error = chkiq(ip, 1, ucp, 0))) {
if (DOINGSOFTDEP(tvp))
softdep_revert_link(pdir, ip);
UFS_VFREE(tvp, ip->i_number, mode);
vgone(tvp);
vput(tvp);
return (error);
}
#endif
}
#else /* !SUIDDIR */
ip->i_uid = cnp->cn_cred->cr_uid;
DIP_SET(ip, i_uid, ip->i_uid);
#ifdef QUOTA
if ((error = getinoquota(ip)) ||
(error = chkiq(ip, 1, cnp->cn_cred, 0))) {
if (DOINGSOFTDEP(tvp))
softdep_revert_link(pdir, ip);
UFS_VFREE(tvp, ip->i_number, mode);
vgone(tvp);
vput(tvp);
return (error);
}
#endif
#endif /* !SUIDDIR */
UFS_INODE_SET_FLAG(ip, IN_ACCESS | IN_CHANGE | IN_UPDATE);
ip->i_mode = mode;
DIP_SET(ip, i_mode, mode);
tvp->v_type = IFTOVT(mode); /* Rest init'd in getnewvnode(). */
ip->i_effnlink = 1;
ip->i_nlink = 1;
DIP_SET(ip, i_nlink, 1);
if (DOINGSOFTDEP(tvp))
softdep_setup_create(VTOI(dvp), ip);
if ((ip->i_mode & ISGID) && !groupmember(ip->i_gid, cnp->cn_cred) &&
priv_check_cred(cnp->cn_cred, PRIV_VFS_SETGID)) {
ip->i_mode &= ~ISGID;
DIP_SET(ip, i_mode, ip->i_mode);
}
if (cnp->cn_flags & ISWHITEOUT) {
ip->i_flags |= UF_OPAQUE;
DIP_SET(ip, i_flags, ip->i_flags);
}
/*
* Make sure inode goes to disk before directory entry.
*/
error = UFS_UPDATE(tvp, !DOINGSOFTDEP(tvp) && !DOINGASYNC(tvp));
if (error)
goto bad;
#ifdef MAC
if (dvp->v_mount->mnt_flag & MNT_MULTILABEL) {
error = mac_vnode_create_extattr(cnp->cn_cred, dvp->v_mount,
dvp, tvp, cnp);
if (error)
goto bad;
}
#endif
#ifdef UFS_ACL
if (dvp->v_mount->mnt_flag & MNT_ACLS) {
error = ufs_do_posix1e_acl_inheritance_file(dvp, tvp, mode,
cnp->cn_cred, cnp->cn_thread);
if (error)
goto bad;
} else if (dvp->v_mount->mnt_flag & MNT_NFS4ACLS) {
error = ufs_do_nfs4_acl_inheritance(dvp, tvp, mode,
cnp->cn_cred, cnp->cn_thread);
if (error)
goto bad;
}
#endif /* !UFS_ACL */
ufs_makedirentry(ip, cnp, &newdir);
error = ufs_direnter(dvp, tvp, &newdir, cnp, NULL, 0);
if (error)
goto bad;
*vpp = tvp;
return (0);
bad:
/*
* Write error occurred trying to update the inode
* or the directory so must deallocate the inode.
*/
ip->i_effnlink = 0;
ip->i_nlink = 0;
DIP_SET(ip, i_nlink, 0);
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
if (DOINGSOFTDEP(tvp))
softdep_revert_create(VTOI(dvp), ip);
vgone(tvp);
vput(tvp);
return (error);
}
static int
ufs_ioctl(struct vop_ioctl_args *ap)
{
struct vnode *vp;
int error;
vp = ap->a_vp;
switch (ap->a_command) {
case FIOSEEKDATA:
error = vn_lock(vp, LK_SHARED);
if (error == 0) {
error = ufs_bmap_seekdata(vp, (off_t *)ap->a_data);
VOP_UNLOCK(vp);
} else
error = EBADF;
return (error);
case FIOSEEKHOLE:
return (vn_bmap_seekhole(vp, ap->a_command, (off_t *)ap->a_data,
ap->a_cred));
default:
return (ENOTTY);
}
}
/* Global vfs data structures for ufs. */
struct vop_vector ufs_vnodeops = {
.vop_default = &default_vnodeops,
.vop_fsync = VOP_PANIC,
.vop_read = VOP_PANIC,
.vop_reallocblks = VOP_PANIC,
.vop_write = VOP_PANIC,
.vop_accessx = ufs_accessx,
.vop_bmap = ufs_bmap,
.vop_cachedlookup = ufs_lookup,
.vop_close = ufs_close,
.vop_create = ufs_create,
.vop_getattr = ufs_getattr,
.vop_inactive = ufs_inactive,
.vop_ioctl = ufs_ioctl,
.vop_link = ufs_link,
.vop_lookup = vfs_cache_lookup,
.vop_mmapped = ufs_mmapped,
.vop_mkdir = ufs_mkdir,
.vop_mknod = ufs_mknod,
.vop_need_inactive = ufs_need_inactive,
.vop_open = ufs_open,
.vop_pathconf = ufs_pathconf,
.vop_poll = vop_stdpoll,
.vop_print = ufs_print,
.vop_readdir = ufs_readdir,
.vop_readlink = ufs_readlink,
.vop_reclaim = ufs_reclaim,
.vop_remove = ufs_remove,
.vop_rename = ufs_rename,
.vop_rmdir = ufs_rmdir,
.vop_setattr = ufs_setattr,
#ifdef MAC
.vop_setlabel = vop_stdsetlabel_ea,
#endif
.vop_strategy = ufs_strategy,
.vop_symlink = ufs_symlink,
.vop_whiteout = ufs_whiteout,
#ifdef UFS_EXTATTR
.vop_getextattr = ufs_getextattr,
.vop_deleteextattr = ufs_deleteextattr,
.vop_setextattr = ufs_setextattr,
#endif
#ifdef UFS_ACL
.vop_getacl = ufs_getacl,
.vop_setacl = ufs_setacl,
.vop_aclcheck = ufs_aclcheck,
#endif
};
VFS_VOP_VECTOR_REGISTER(ufs_vnodeops);
struct vop_vector ufs_fifoops = {
.vop_default = &fifo_specops,
.vop_fsync = VOP_PANIC,
.vop_accessx = ufs_accessx,
.vop_close = ufsfifo_close,
.vop_getattr = ufs_getattr,
.vop_inactive = ufs_inactive,
.vop_kqfilter = ufsfifo_kqfilter,
.vop_pathconf = ufs_pathconf,
.vop_print = ufs_print,
.vop_read = VOP_PANIC,
.vop_reclaim = ufs_reclaim,
.vop_setattr = ufs_setattr,
#ifdef MAC
.vop_setlabel = vop_stdsetlabel_ea,
#endif
.vop_write = VOP_PANIC,
#ifdef UFS_EXTATTR
.vop_getextattr = ufs_getextattr,
.vop_deleteextattr = ufs_deleteextattr,
.vop_setextattr = ufs_setextattr,
#endif
#ifdef UFS_ACL
.vop_getacl = ufs_getacl,
.vop_setacl = ufs_setacl,
.vop_aclcheck = ufs_aclcheck,
#endif
};
VFS_VOP_VECTOR_REGISTER(ufs_fifoops);
Index: head/sys/ufs/ufs/ufsmount.h
===================================================================
--- head/sys/ufs/ufs/ufsmount.h (revision 361490)
+++ head/sys/ufs/ufs/ufsmount.h (revision 361491)
@@ -1,187 +1,195 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ufsmount.h 8.6 (Berkeley) 3/30/95
* $FreeBSD$
*/
#ifndef _UFS_UFS_UFSMOUNT_H_
#define _UFS_UFS_UFSMOUNT_H_
/*
* Arguments to mount UFS-based filesystems
*/
struct ufs_args {
char *fspec; /* block special device to mount */
struct oexport_args export; /* network export information */
};
#ifdef _KERNEL
+#include <sys/_task.h>
+
#ifdef MALLOC_DECLARE
MALLOC_DECLARE(M_UFSMNT);
MALLOC_DECLARE(M_TRIM);
#endif
struct buf;
struct inode;
struct nameidata;
struct taskqueue;
struct timeval;
struct ucred;
struct uio;
struct vnode;
struct ufs_extattr_per_mount;
struct jblocks;
struct inodedep;
TAILQ_HEAD(inodedeplst, inodedep);
LIST_HEAD(bmsafemaphd, bmsafemap);
LIST_HEAD(trimlist_hashhead, ffs_blkfree_trim_params);
+struct fsfail_task {
+ struct task task;
+ fsid_t fsid;
+};
/*
* This structure describes the UFS specific mount structure data.
* The function operators are used to support different versions of
* UFS (UFS1, UFS2, etc).
*
* Lock reference:
* c - set at allocation then constant until freed
* i - ufsmount interlock (UFS_LOCK / UFS_UNLOCK)
* q - associated quota file is locked
* r - ref to parent mount structure is held (vfs_busy / vfs_unbusy)
* u - managed by user process fsck_ufs
*/
struct ufsmount {
struct mount *um_mountp; /* (r) filesystem vfs struct */
struct cdev *um_dev; /* (r) device mounted */
struct g_consumer *um_cp; /* (r) GEOM access point */
struct bufobj *um_bo; /* (r) Buffer cache object */
struct vnode *um_odevvp; /* (r) devfs dev vnode */
struct vnode *um_devvp; /* (r) mntfs private vnode */
u_long um_fstype; /* (c) type of filesystem */
struct fs *um_fs; /* (r) pointer to superblock */
struct ufs_extattr_per_mount um_extattr; /* (c) extended attrs */
u_long um_nindir; /* (c) indirect ptrs per blk */
u_long um_bptrtodb; /* (c) indir disk block ptr */
u_long um_seqinc; /* (c) inc between seq blocks */
struct mtx um_lock; /* (c) Protects ufsmount & fs */
pid_t um_fsckpid; /* (u) PID can do fsck sysctl */
struct mount_softdeps *um_softdep; /* (c) softdep mgmt structure */
struct vnode *um_quotas[MAXQUOTAS]; /* (q) pointer to quota files */
struct ucred *um_cred[MAXQUOTAS]; /* (q) quota file access cred */
time_t um_btime[MAXQUOTAS]; /* (q) block quota time limit */
time_t um_itime[MAXQUOTAS]; /* (q) inode quota time limit */
char um_qflags[MAXQUOTAS]; /* (i) quota specific flags */
int64_t um_savedmaxfilesize; /* (c) track maxfilesize */
u_int um_flags; /* (i) filesystem flags */
struct timeval um_last_fullmsg; /* (i) last full msg time */
int um_secs_fullmsg; /* (i) seconds since full msg */
struct timeval um_last_integritymsg; /* (i) last integrity msg */
int um_secs_integritymsg; /* (i) secs since integ msg */
u_int um_trim_inflight; /* (i) outstanding trim count */
u_int um_trim_inflight_blks; /* (i) outstanding trim blks */
u_long um_trim_total; /* (i) total trim count */
u_long um_trim_total_blks; /* (i) total trim block count */
struct taskqueue *um_trim_tq; /* (c) trim request queue */
struct trimlist_hashhead *um_trimhash; /* (i) trimlist hash table */
u_long um_trimlisthashsize; /* (i) trim hash table size-1 */
+ struct fsfail_task *um_fsfail_task; /* (i) task for fsfail cleanup*/
/* (c) - below function ptrs */
int (*um_balloc)(struct vnode *, off_t, int, struct ucred *,
int, struct buf **);
int (*um_blkatoff)(struct vnode *, off_t, char **, struct buf **);
int (*um_truncate)(struct vnode *, off_t, int, struct ucred *);
int (*um_update)(struct vnode *, int);
int (*um_valloc)(struct vnode *, int, struct ucred *,
struct vnode **);
int (*um_vfree)(struct vnode *, ino_t, int);
void (*um_ifree)(struct ufsmount *, struct inode *);
int (*um_rdonly)(struct inode *);
void (*um_snapgone)(struct inode *);
int (*um_check_blkno)(struct mount *, ino_t, daddr_t, int);
};
/*
* filesystem flags
*/
#define UM_CANDELETE 0x00000001 /* devvp supports TRIM */
#define UM_WRITESUSPENDED 0x00000002 /* suspension in progress */
#define UM_CANSPEEDUP 0x00000004 /* devvp supports SPEEDUP */
-
+#define UM_FSFAIL_CLEANUP 0x00000008 /* need cleanup after
+ unrecoverable error */
/*
* function prototypes
*/
#define UFS_BALLOC(aa, bb, cc, dd, ee, ff) \
VFSTOUFS((aa)->v_mount)->um_balloc(aa, bb, cc, dd, ee, ff)
#define UFS_BLKATOFF(aa, bb, cc, dd) \
VFSTOUFS((aa)->v_mount)->um_blkatoff(aa, bb, cc, dd)
#define UFS_TRUNCATE(aa, bb, cc, dd) \
VFSTOUFS((aa)->v_mount)->um_truncate(aa, bb, cc, dd)
#define UFS_UPDATE(aa, bb) VFSTOUFS((aa)->v_mount)->um_update(aa, bb)
#define UFS_VALLOC(aa, bb, cc, dd) \
VFSTOUFS((aa)->v_mount)->um_valloc(aa, bb, cc, dd)
#define UFS_VFREE(aa, bb, cc) VFSTOUFS((aa)->v_mount)->um_vfree(aa, bb, cc)
#define UFS_IFREE(aa, bb) ((aa)->um_ifree(aa, bb))
#define UFS_RDONLY(aa) (ITOUMP(aa)->um_rdonly(aa))
#define UFS_SNAPGONE(aa) (ITOUMP(aa)->um_snapgone(aa))
#define UFS_CHECK_BLKNO(aa, bb, cc, dd) \
(VFSTOUFS(aa)->um_check_blkno == NULL ? 0 : \
VFSTOUFS(aa)->um_check_blkno(aa, bb, cc, dd))
#define UFS_LOCK(aa) mtx_lock(&(aa)->um_lock)
#define UFS_UNLOCK(aa) mtx_unlock(&(aa)->um_lock)
#define UFS_MTX(aa) (&(aa)->um_lock)
/*
* Filesystem types
*/
#define UFS1 1
#define UFS2 2
/*
* Flags describing the state of quotas.
*/
#define QTF_OPENING 0x01 /* Q_QUOTAON in progress */
#define QTF_CLOSING 0x02 /* Q_QUOTAOFF in progress */
#define QTF_64BIT 0x04 /* 64-bit quota file */
/* Convert mount ptr to ufsmount ptr. */
#define VFSTOUFS(mp) ((struct ufsmount *)((mp)->mnt_data))
#define UFSTOVFS(ump) (ump)->um_mountp
/*
* Macros to access filesystem parameters in the ufsmount structure.
* Used by ufs_bmap.
*/
#define MNINDIR(ump) ((ump)->um_nindir)
#define blkptrtodb(ump, b) ((b) << (ump)->um_bptrtodb)
#define is_sequential(ump, a, b) ((b) == (a) + ump->um_seqinc)
#endif /* _KERNEL */
#endif