diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -798,58 +798,82 @@ } #if OFF_MAX <= LONG_MAX +static void +file_v_lock(struct file *fp, short lock_bit, short lock_wait_bit) +{ + volatile short *flagsp; + short state; + + flagsp = &fp->f_vflags; + state = atomic_load_16(flagsp); + if ((state & lock_bit) == 0 && + atomic_cmpset_acq_16(flagsp, state, state | lock_bit)) + return; + + sleepq_lock(&fp->f_vflags); + state = atomic_load_16(flagsp); + for (;;) { + if ((state & lock_bit) == 0) { + if (!atomic_fcmpset_acq_16(flagsp, &state, + state | lock_bit)) + continue; + break; + } + if ((state & lock_wait_bit) == 0) { + if (!atomic_fcmpset_acq_16(flagsp, &state, + state | lock_wait_bit)) + continue; + } + DROP_GIANT(); + sleepq_add(&fp->f_vflags, NULL, "vofflock", 0, 0); + sleepq_wait(&fp->f_vflags, PRI_MAX_KERN); + PICKUP_GIANT(); + sleepq_lock(&fp->f_vflags); + state = atomic_load_16(flagsp); + } + sleepq_release(&fp->f_vflags); +} + +static void +file_v_unlock(struct file *fp, short lock_bit, short lock_wait_bit) +{ + volatile short *flagsp; + short state; + + flagsp = &fp->f_vflags; + state = atomic_load_16(flagsp); + if ((state & lock_wait_bit) == 0 && + atomic_cmpset_rel_16(flagsp, state, state & ~lock_bit)) + return; + + sleepq_lock(&fp->f_vflags); + MPASS((fp->f_vflags & lock_bit) != 0); + MPASS((fp->f_vflags & lock_wait_bit) != 0); + atomic_clear_16(&fp->f_vflags, lock_bit | lock_wait_bit); + sleepq_broadcast(&fp->f_vflags, SLEEPQ_SLEEP, 0, 0); + sleepq_release(&fp->f_vflags); +} + off_t foffset_lock(struct file *fp, int flags) { - volatile short *flagsp; - off_t res; - short state; - KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed")); - if ((flags & FOF_NOLOCK) != 0) - return (atomic_load_long(&fp->f_offset)); + if ((flags & FOF_NOLOCK) == 0) { + file_v_lock(fp, FILE_V_FOFFSET_LOCKED, + FILE_V_FOFFSET_LOCK_WAITING); + } /* * According to McKusick the vn lock was protecting f_offset here. * It is now protected by the FOFFSET_LOCKED flag. */ - flagsp = &fp->f_vnread_flags; - if (atomic_cmpset_acq_16(flagsp, 0, FOFFSET_LOCKED)) - return (atomic_load_long(&fp->f_offset)); - - sleepq_lock(&fp->f_vnread_flags); - state = atomic_load_16(flagsp); - for (;;) { - if ((state & FOFFSET_LOCKED) == 0) { - if (!atomic_fcmpset_acq_16(flagsp, &state, - FOFFSET_LOCKED)) - continue; - break; - } - if ((state & FOFFSET_LOCK_WAITING) == 0) { - if (!atomic_fcmpset_acq_16(flagsp, &state, - state | FOFFSET_LOCK_WAITING)) - continue; - } - DROP_GIANT(); - sleepq_add(&fp->f_vnread_flags, NULL, "vofflock", 0, 0); - sleepq_wait(&fp->f_vnread_flags, PRI_MAX_KERN); - PICKUP_GIANT(); - sleepq_lock(&fp->f_vnread_flags); - state = atomic_load_16(flagsp); - } - res = atomic_load_long(&fp->f_offset); - sleepq_release(&fp->f_vnread_flags); - return (res); + return (atomic_load_long(&fp->f_offset)); } void foffset_unlock(struct file *fp, off_t val, int flags) { - volatile short *flagsp; - short state; - KASSERT((flags & FOF_OFFSET) == 0, ("FOF_OFFSET passed")); if ((flags & FOF_NOUPDATE) == 0) @@ -859,21 +883,10 @@ if ((flags & FOF_NEXTOFF_W) != 0) fp->f_nextoff[UIO_WRITE] = val; - if ((flags & FOF_NOLOCK) != 0) - return; - - flagsp = &fp->f_vnread_flags; - state = atomic_load_16(flagsp); - if ((state & FOFFSET_LOCK_WAITING) == 0 && - atomic_cmpset_rel_16(flagsp, state, 0)) - return; - - sleepq_lock(&fp->f_vnread_flags); - MPASS((fp->f_vnread_flags & FOFFSET_LOCKED) != 0); - MPASS((fp->f_vnread_flags & FOFFSET_LOCK_WAITING) != 0); - fp->f_vnread_flags = 0; - sleepq_broadcast(&fp->f_vnread_flags, SLEEPQ_SLEEP, 0, 0); - sleepq_release(&fp->f_vnread_flags); + if ((flags & FOF_NOLOCK) == 0) { + file_v_unlock(fp, FILE_V_FOFFSET_LOCKED, + FILE_V_FOFFSET_LOCK_WAITING); + } } static off_t @@ -882,7 +895,35 @@ return (atomic_load_long(&fp->f_offset)); } -#else + +#else /* OFF_MAX <= LONG_MAX */ + +static void +file_v_lock_mtxp(struct file *fp, struct mtx *mtxp, short lock_bit, + short lock_wait_bit) +{ + mtx_assert(mtxp, MA_OWNED); + + while ((fp->f_vflags & lock_bit) != 0) { + fp->f_vflags |= lock_wait_bit; + msleep(&fp->f_vflags, mtxp, PRI_MAX_KERN, + "vofflock", 0); + } + fp->f_vflags |= lock_bit; +} + +static void +file_v_unlock_mtxp(struct file *fp, struct mtx *mtxp, short lock_bit, + short lock_wait_bit) +{ + mtx_assert(mtxp, MA_OWNED); + + KASSERT((fp->f_vflags & lock_bit) != 0, ("Lost lock_bit")); + if ((fp->f_vflags & lock_wait_bit) != 0) + wakeup(&fp->f_vflags); + fp->f_vflags &= ~(lock_bit | lock_wait_bit); +} + off_t foffset_lock(struct file *fp, int flags) { @@ -894,12 +935,8 @@ mtxp = mtx_pool_find(mtxpool_sleep, fp); mtx_lock(mtxp); if ((flags & FOF_NOLOCK) == 0) { - while (fp->f_vnread_flags & FOFFSET_LOCKED) { - fp->f_vnread_flags |= FOFFSET_LOCK_WAITING; - msleep(&fp->f_vnread_flags, mtxp, PRI_MAX_KERN, - "vofflock", 0); - } - fp->f_vnread_flags |= FOFFSET_LOCKED; + file_v_lock_mtxp(fp, mtxp, FILE_V_FOFFSET_LOCKED, + FILE_V_FOFFSET_LOCK_WAITING); } res = fp->f_offset; mtx_unlock(mtxp); @@ -922,11 +959,8 @@ if ((flags & FOF_NEXTOFF_W) != 0) fp->f_nextoff[UIO_WRITE] = val; if ((flags & FOF_NOLOCK) == 0) { - KASSERT((fp->f_vnread_flags & FOFFSET_LOCKED) != 0, - ("Lost FOFFSET_LOCKED")); - if (fp->f_vnread_flags & FOFFSET_LOCK_WAITING) - wakeup(&fp->f_vnread_flags); - fp->f_vnread_flags = 0; + file_v_unlock_mtxp(fp, mtxp, FILE_V_FOFFSET_LOCKED, + FILE_V_FOFFSET_LOCK_WAITING); } mtx_unlock(mtxp); } diff --git a/sys/sys/file.h b/sys/sys/file.h --- a/sys/sys/file.h +++ b/sys/sys/file.h @@ -197,7 +197,7 @@ struct vnode *f_vnode; /* NULL or applicable vnode */ struct ucred *f_cred; /* associated credentials. */ short f_type; /* descriptor type */ - short f_vnread_flags; /* (f) Sleep lock for f_offset */ + short f_vflags; /* (f) Sleep lock flags for members */ /* * DTYPE_VNODE specific fields. */ @@ -220,8 +220,8 @@ #define f_cdevpriv f_vnun.fvn_cdevpriv #define f_advice f_vnun.fvn_advice -#define FOFFSET_LOCKED 0x1 -#define FOFFSET_LOCK_WAITING 0x2 +#define FILE_V_FOFFSET_LOCKED 0x0001 +#define FILE_V_FOFFSET_LOCK_WAITING 0x0002 #endif /* __BSD_VISIBLE */ #endif /* _KERNEL || _WANT_FILE */