diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 03498d5a9ce9..88a91b9e6fcf 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -1,5261 +1,5334 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_descrip.c 8.6 (Berkeley) 4/19/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_ddb.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/filio.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/selinfo.h>
#include <sys/poll.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/sbuf.h>
#include <sys/signalvar.h>
#include <sys/kdb.h>
#include <sys/smr.h>
#include <sys/stat.h>
#include <sys/sx.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/unistd.h>
#include <sys/user.h>
#include <sys/vnode.h>
#include <sys/ktrace.h>
#include <net/vnet.h>
#include <security/audit/audit.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <ddb/ddb.h>
static MALLOC_DEFINE(M_FILEDESC, "filedesc", "Open file descriptor table");
static MALLOC_DEFINE(M_PWD, "pwd", "Descriptor table vnodes");
static MALLOC_DEFINE(M_PWDDESC, "pwddesc", "Pwd descriptors");
static MALLOC_DEFINE(M_FILEDESC_TO_LEADER, "filedesc_to_leader",
"file desc to leader structures");
static MALLOC_DEFINE(M_SIGIO, "sigio", "sigio structures");
MALLOC_DEFINE(M_FILECAPS, "filecaps", "descriptor capabilities");
MALLOC_DECLARE(M_FADVISE);
static __read_mostly uma_zone_t file_zone;
static __read_mostly uma_zone_t filedesc0_zone;
__read_mostly uma_zone_t pwd_zone;
VFS_SMR_DECLARE;
static int closefp(struct filedesc *fdp, int fd, struct file *fp,
struct thread *td, bool holdleaders, bool audit);
static void export_file_to_kinfo(struct file *fp, int fd,
cap_rights_t *rightsp, struct kinfo_file *kif,
struct filedesc *fdp, int flags);
static int fd_first_free(struct filedesc *fdp, int low, int size);
static void fdgrowtable(struct filedesc *fdp, int nfd);
static void fdgrowtable_exp(struct filedesc *fdp, int nfd);
static void fdunused(struct filedesc *fdp, int fd);
static void fdused(struct filedesc *fdp, int fd);
static int fget_unlocked_seq(struct filedesc *fdp, int fd,
cap_rights_t *needrightsp, struct file **fpp, seqc_t *seqp);
static int getmaxfd(struct thread *td);
static u_long *filecaps_copy_prep(const struct filecaps *src);
static void filecaps_copy_finish(const struct filecaps *src,
struct filecaps *dst, u_long *ioctls);
static u_long *filecaps_free_prep(struct filecaps *fcaps);
static void filecaps_free_finish(u_long *ioctls);
static struct pwd *pwd_alloc(void);
/*
* Each process has:
*
* - An array of open file descriptors (fd_ofiles)
* - An array of file flags (fd_ofileflags)
* - A bitmap recording which descriptors are in use (fd_map)
*
* A process starts out with NDFILE descriptors. The value of NDFILE has
* been selected based the historical limit of 20 open files, and an
* assumption that the majority of processes, especially short-lived
* processes like shells, will never need more.
*
* If this initial allocation is exhausted, a larger descriptor table and
* map are allocated dynamically, and the pointers in the process's struct
* filedesc are updated to point to those. This is repeated every time
* the process runs out of file descriptors (provided it hasn't hit its
* resource limit).
*
* Since threads may hold references to individual descriptor table
* entries, the tables are never freed. Instead, they are placed on a
* linked list and freed only when the struct filedesc is released.
*/
#define NDFILE 20
#define NDSLOTSIZE sizeof(NDSLOTTYPE)
#define NDENTRIES (NDSLOTSIZE * __CHAR_BIT)
#define NDSLOT(x) ((x) / NDENTRIES)
#define NDBIT(x) ((NDSLOTTYPE)1 << ((x) % NDENTRIES))
#define NDSLOTS(x) (((x) + NDENTRIES - 1) / NDENTRIES)
/*
* SLIST entry used to keep track of ofiles which must be reclaimed when
* the process exits.
*/
struct freetable {
struct fdescenttbl *ft_table;
SLIST_ENTRY(freetable) ft_next;
};
/*
* Initial allocation: a filedesc structure + the head of SLIST used to
* keep track of old ofiles + enough space for NDFILE descriptors.
*/
struct fdescenttbl0 {
int fdt_nfiles;
struct filedescent fdt_ofiles[NDFILE];
};
struct filedesc0 {
struct filedesc fd_fd;
SLIST_HEAD(, freetable) fd_free;
struct fdescenttbl0 fd_dfiles;
NDSLOTTYPE fd_dmap[NDSLOTS(NDFILE)];
};
/*
* Descriptor management.
*/
static int __exclusive_cache_line openfiles; /* actual number of open files */
struct mtx sigio_lock; /* mtx to protect pointers to sigio */
void __read_mostly (*mq_fdclose)(struct thread *td, int fd, struct file *fp);
/*
* If low >= size, just return low. Otherwise find the first zero bit in the
* given bitmap, starting at low and not exceeding size - 1. Return size if
* not found.
*/
static int
fd_first_free(struct filedesc *fdp, int low, int size)
{
NDSLOTTYPE *map = fdp->fd_map;
NDSLOTTYPE mask;
int off, maxoff;
if (low >= size)
return (low);
off = NDSLOT(low);
if (low % NDENTRIES) {
mask = ~(~(NDSLOTTYPE)0 >> (NDENTRIES - (low % NDENTRIES)));
if ((mask &= ~map[off]) != 0UL)
return (off * NDENTRIES + ffsl(mask) - 1);
++off;
}
for (maxoff = NDSLOTS(size); off < maxoff; ++off)
if (map[off] != ~0UL)
return (off * NDENTRIES + ffsl(~map[off]) - 1);
return (size);
}
/*
* Find the last used fd.
*
* Call this variant if fdp can't be modified by anyone else (e.g, during exec).
* Otherwise use fdlastfile.
*/
int
fdlastfile_single(struct filedesc *fdp)
{
NDSLOTTYPE *map = fdp->fd_map;
int off, minoff;
off = NDSLOT(fdp->fd_nfiles - 1);
for (minoff = NDSLOT(0); off >= minoff; --off)
if (map[off] != 0)
return (off * NDENTRIES + flsl(map[off]) - 1);
return (-1);
}
int
fdlastfile(struct filedesc *fdp)
{
FILEDESC_LOCK_ASSERT(fdp);
return (fdlastfile_single(fdp));
}
static int
fdisused(struct filedesc *fdp, int fd)
{
KASSERT(fd >= 0 && fd < fdp->fd_nfiles,
("file descriptor %d out of range (0, %d)", fd, fdp->fd_nfiles));
return ((fdp->fd_map[NDSLOT(fd)] & NDBIT(fd)) != 0);
}
/*
* Mark a file descriptor as used.
*/
static void
fdused_init(struct filedesc *fdp, int fd)
{
KASSERT(!fdisused(fdp, fd), ("fd=%d is already used", fd));
fdp->fd_map[NDSLOT(fd)] |= NDBIT(fd);
}
static void
fdused(struct filedesc *fdp, int fd)
{
FILEDESC_XLOCK_ASSERT(fdp);
fdused_init(fdp, fd);
if (fd == fdp->fd_freefile)
fdp->fd_freefile++;
}
/*
* Mark a file descriptor as unused.
*/
static void
fdunused(struct filedesc *fdp, int fd)
{
FILEDESC_XLOCK_ASSERT(fdp);
KASSERT(fdisused(fdp, fd), ("fd=%d is already unused", fd));
KASSERT(fdp->fd_ofiles[fd].fde_file == NULL,
("fd=%d is still in use", fd));
fdp->fd_map[NDSLOT(fd)] &= ~NDBIT(fd);
if (fd < fdp->fd_freefile)
fdp->fd_freefile = fd;
}
/*
* Free a file descriptor.
*
* Avoid some work if fdp is about to be destroyed.
*/
static inline void
fdefree_last(struct filedescent *fde)
{
filecaps_free(&fde->fde_caps);
}
static inline void
fdfree(struct filedesc *fdp, int fd)
{
struct filedescent *fde;
FILEDESC_XLOCK_ASSERT(fdp);
fde = &fdp->fd_ofiles[fd];
#ifdef CAPABILITIES
seqc_write_begin(&fde->fde_seqc);
#endif
fde->fde_file = NULL;
#ifdef CAPABILITIES
seqc_write_end(&fde->fde_seqc);
#endif
fdefree_last(fde);
fdunused(fdp, fd);
}
/*
* System calls on descriptors.
*/
#ifndef _SYS_SYSPROTO_H_
struct getdtablesize_args {
int dummy;
};
#endif
/* ARGSUSED */
int
sys_getdtablesize(struct thread *td, struct getdtablesize_args *uap)
{
#ifdef RACCT
uint64_t lim;
#endif
td->td_retval[0] = getmaxfd(td);
#ifdef RACCT
PROC_LOCK(td->td_proc);
lim = racct_get_limit(td->td_proc, RACCT_NOFILE);
PROC_UNLOCK(td->td_proc);
if (lim < td->td_retval[0])
td->td_retval[0] = lim;
#endif
return (0);
}
/*
* Duplicate a file descriptor to a particular value.
*
* Note: keep in mind that a potential race condition exists when closing
* descriptors from a shared descriptor table (via rfork).
*/
#ifndef _SYS_SYSPROTO_H_
struct dup2_args {
u_int from;
u_int to;
};
#endif
/* ARGSUSED */
int
sys_dup2(struct thread *td, struct dup2_args *uap)
{
return (kern_dup(td, FDDUP_FIXED, 0, (int)uap->from, (int)uap->to));
}
/*
* Duplicate a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct dup_args {
u_int fd;
};
#endif
/* ARGSUSED */
int
sys_dup(struct thread *td, struct dup_args *uap)
{
return (kern_dup(td, FDDUP_NORMAL, 0, (int)uap->fd, 0));
}
/*
* The file control system call.
*/
#ifndef _SYS_SYSPROTO_H_
struct fcntl_args {
int fd;
int cmd;
long arg;
};
#endif
/* ARGSUSED */
int
sys_fcntl(struct thread *td, struct fcntl_args *uap)
{
return (kern_fcntl_freebsd(td, uap->fd, uap->cmd, uap->arg));
}
int
kern_fcntl_freebsd(struct thread *td, int fd, int cmd, long arg)
{
struct flock fl;
struct __oflock ofl;
intptr_t arg1;
int error, newcmd;
error = 0;
newcmd = cmd;
switch (cmd) {
case F_OGETLK:
case F_OSETLK:
case F_OSETLKW:
/*
* Convert old flock structure to new.
*/
error = copyin((void *)(intptr_t)arg, &ofl, sizeof(ofl));
fl.l_start = ofl.l_start;
fl.l_len = ofl.l_len;
fl.l_pid = ofl.l_pid;
fl.l_type = ofl.l_type;
fl.l_whence = ofl.l_whence;
fl.l_sysid = 0;
switch (cmd) {
case F_OGETLK:
newcmd = F_GETLK;
break;
case F_OSETLK:
newcmd = F_SETLK;
break;
case F_OSETLKW:
newcmd = F_SETLKW;
break;
}
arg1 = (intptr_t)&fl;
break;
case F_GETLK:
case F_SETLK:
case F_SETLKW:
case F_SETLK_REMOTE:
error = copyin((void *)(intptr_t)arg, &fl, sizeof(fl));
arg1 = (intptr_t)&fl;
break;
default:
arg1 = arg;
break;
}
if (error)
return (error);
error = kern_fcntl(td, fd, newcmd, arg1);
if (error)
return (error);
if (cmd == F_OGETLK) {
ofl.l_start = fl.l_start;
ofl.l_len = fl.l_len;
ofl.l_pid = fl.l_pid;
ofl.l_type = fl.l_type;
ofl.l_whence = fl.l_whence;
error = copyout(&ofl, (void *)(intptr_t)arg, sizeof(ofl));
} else if (cmd == F_GETLK) {
error = copyout(&fl, (void *)(intptr_t)arg, sizeof(fl));
}
return (error);
}
int
kern_fcntl(struct thread *td, int fd, int cmd, intptr_t arg)
{
struct filedesc *fdp;
struct flock *flp;
struct file *fp, *fp2;
struct filedescent *fde;
struct proc *p;
struct vnode *vp;
struct mount *mp;
struct kinfo_file *kif;
int error, flg, kif_sz, seals, tmp;
uint64_t bsize;
off_t foffset;
error = 0;
flg = F_POSIX;
p = td->td_proc;
fdp = p->p_fd;
AUDIT_ARG_FD(cmd);
AUDIT_ARG_CMD(cmd);
switch (cmd) {
case F_DUPFD:
tmp = arg;
error = kern_dup(td, FDDUP_FCNTL, 0, fd, tmp);
break;
case F_DUPFD_CLOEXEC:
tmp = arg;
error = kern_dup(td, FDDUP_FCNTL, FDDUP_FLAG_CLOEXEC, fd, tmp);
break;
case F_DUP2FD:
tmp = arg;
error = kern_dup(td, FDDUP_FIXED, 0, fd, tmp);
break;
case F_DUP2FD_CLOEXEC:
tmp = arg;
error = kern_dup(td, FDDUP_FIXED, FDDUP_FLAG_CLOEXEC, fd, tmp);
break;
case F_GETFD:
error = EBADF;
FILEDESC_SLOCK(fdp);
fde = fdeget_locked(fdp, fd);
if (fde != NULL) {
td->td_retval[0] =
(fde->fde_flags & UF_EXCLOSE) ? FD_CLOEXEC : 0;
error = 0;
}
FILEDESC_SUNLOCK(fdp);
break;
case F_SETFD:
error = EBADF;
FILEDESC_XLOCK(fdp);
fde = fdeget_locked(fdp, fd);
if (fde != NULL) {
fde->fde_flags = (fde->fde_flags & ~UF_EXCLOSE) |
(arg & FD_CLOEXEC ? UF_EXCLOSE : 0);
error = 0;
}
FILEDESC_XUNLOCK(fdp);
break;
case F_GETFL:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_GETFL, &fp);
if (error != 0)
break;
td->td_retval[0] = OFLAGS(fp->f_flag);
fdrop(fp, td);
break;
case F_SETFL:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_SETFL, &fp);
if (error != 0)
break;
if (fp->f_ops == &path_fileops) {
fdrop(fp, td);
error = EBADF;
break;
}
do {
tmp = flg = fp->f_flag;
tmp &= ~FCNTLFLAGS;
tmp |= FFLAGS(arg & ~O_ACCMODE) & FCNTLFLAGS;
} while (atomic_cmpset_int(&fp->f_flag, flg, tmp) == 0);
tmp = fp->f_flag & FNONBLOCK;
error = fo_ioctl(fp, FIONBIO, &tmp, td->td_ucred, td);
if (error != 0) {
fdrop(fp, td);
break;
}
tmp = fp->f_flag & FASYNC;
error = fo_ioctl(fp, FIOASYNC, &tmp, td->td_ucred, td);
if (error == 0) {
fdrop(fp, td);
break;
}
atomic_clear_int(&fp->f_flag, FNONBLOCK);
tmp = 0;
(void)fo_ioctl(fp, FIONBIO, &tmp, td->td_ucred, td);
fdrop(fp, td);
break;
case F_GETOWN:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_GETOWN, &fp);
if (error != 0)
break;
error = fo_ioctl(fp, FIOGETOWN, &tmp, td->td_ucred, td);
if (error == 0)
td->td_retval[0] = tmp;
fdrop(fp, td);
break;
case F_SETOWN:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_SETOWN, &fp);
if (error != 0)
break;
tmp = arg;
error = fo_ioctl(fp, FIOSETOWN, &tmp, td->td_ucred, td);
fdrop(fp, td);
break;
case F_SETLK_REMOTE:
error = priv_check(td, PRIV_NFS_LOCKD);
if (error != 0)
return (error);
flg = F_REMOTE;
goto do_setlk;
case F_SETLKW:
flg |= F_WAIT;
/* FALLTHROUGH F_SETLK */
case F_SETLK:
do_setlk:
flp = (struct flock *)arg;
if ((flg & F_REMOTE) != 0 && flp->l_sysid == 0) {
error = EINVAL;
break;
}
error = fget_unlocked(fdp, fd, &cap_flock_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE || fp->f_ops == &path_fileops) {
error = EBADF;
fdrop(fp, td);
break;
}
if (flp->l_whence == SEEK_CUR) {
foffset = foffset_get(fp);
if (foffset < 0 ||
(flp->l_start > 0 &&
foffset > OFF_MAX - flp->l_start)) {
error = EOVERFLOW;
fdrop(fp, td);
break;
}
flp->l_start += foffset;
}
vp = fp->f_vnode;
switch (flp->l_type) {
case F_RDLCK:
if ((fp->f_flag & FREAD) == 0) {
error = EBADF;
break;
}
if ((p->p_leader->p_flag & P_ADVLOCK) == 0) {
PROC_LOCK(p->p_leader);
p->p_leader->p_flag |= P_ADVLOCK;
PROC_UNLOCK(p->p_leader);
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_SETLK,
flp, flg);
break;
case F_WRLCK:
if ((fp->f_flag & FWRITE) == 0) {
error = EBADF;
break;
}
if ((p->p_leader->p_flag & P_ADVLOCK) == 0) {
PROC_LOCK(p->p_leader);
p->p_leader->p_flag |= P_ADVLOCK;
PROC_UNLOCK(p->p_leader);
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_SETLK,
flp, flg);
break;
case F_UNLCK:
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_UNLCK,
flp, flg);
break;
case F_UNLCKSYS:
if (flg != F_REMOTE) {
error = EINVAL;
break;
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader,
F_UNLCKSYS, flp, flg);
break;
default:
error = EINVAL;
break;
}
if (error != 0 || flp->l_type == F_UNLCK ||
flp->l_type == F_UNLCKSYS) {
fdrop(fp, td);
break;
}
/*
* Check for a race with close.
*
* The vnode is now advisory locked (or unlocked, but this case
* is not really important) as the caller requested.
* We had to drop the filedesc lock, so we need to recheck if
* the descriptor is still valid, because if it was closed
* in the meantime we need to remove advisory lock from the
* vnode - close on any descriptor leading to an advisory
* locked vnode, removes that lock.
* We will return 0 on purpose in that case, as the result of
* successful advisory lock might have been externally visible
* already. This is fine - effectively we pretend to the caller
* that the closing thread was a bit slower and that the
* advisory lock succeeded before the close.
*/
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp2);
if (error != 0) {
fdrop(fp, td);
break;
}
if (fp != fp2) {
flp->l_whence = SEEK_SET;
flp->l_start = 0;
flp->l_len = 0;
flp->l_type = F_UNLCK;
(void) VOP_ADVLOCK(vp, (caddr_t)p->p_leader,
F_UNLCK, flp, F_POSIX);
}
fdrop(fp, td);
fdrop(fp2, td);
break;
case F_GETLK:
error = fget_unlocked(fdp, fd, &cap_flock_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE || fp->f_ops == &path_fileops) {
error = EBADF;
fdrop(fp, td);
break;
}
flp = (struct flock *)arg;
if (flp->l_type != F_RDLCK && flp->l_type != F_WRLCK &&
flp->l_type != F_UNLCK) {
error = EINVAL;
fdrop(fp, td);
break;
}
if (flp->l_whence == SEEK_CUR) {
foffset = foffset_get(fp);
if ((flp->l_start > 0 &&
foffset > OFF_MAX - flp->l_start) ||
(flp->l_start < 0 &&
foffset < OFF_MIN - flp->l_start)) {
error = EOVERFLOW;
fdrop(fp, td);
break;
}
flp->l_start += foffset;
}
vp = fp->f_vnode;
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_GETLK, flp,
F_POSIX);
fdrop(fp, td);
break;
case F_ADD_SEALS:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
error = fo_add_seals(fp, arg);
fdrop(fp, td);
break;
case F_GET_SEALS:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fo_get_seals(fp, &seals) == 0)
td->td_retval[0] = seals;
else
error = EINVAL;
fdrop(fp, td);
break;
case F_RDAHEAD:
arg = arg ? 128 * 1024: 0;
/* FALLTHROUGH */
case F_READAHEAD:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE || fp->f_ops == &path_fileops) {
fdrop(fp, td);
error = EBADF;
break;
}
vp = fp->f_vnode;
if (vp->v_type != VREG) {
fdrop(fp, td);
error = ENOTTY;
break;
}
/*
* Exclusive lock synchronizes against f_seqcount reads and
* writes in sequential_heuristic().
*/
error = vn_lock(vp, LK_EXCLUSIVE);
if (error != 0) {
fdrop(fp, td);
break;
}
if (arg >= 0) {
bsize = fp->f_vnode->v_mount->mnt_stat.f_iosize;
arg = MIN(arg, INT_MAX - bsize + 1);
fp->f_seqcount[UIO_READ] = MIN(IO_SEQMAX,
(arg + bsize - 1) / bsize);
atomic_set_int(&fp->f_flag, FRDAHEAD);
} else {
atomic_clear_int(&fp->f_flag, FRDAHEAD);
}
VOP_UNLOCK(vp);
fdrop(fp, td);
break;
case F_ISUNIONSTACK:
/*
* Check if the vnode is part of a union stack (either the
* "union" flag from mount(2) or unionfs).
*
* Prior to introduction of this op libc's readdir would call
* fstatfs(2), in effect unnecessarily copying kilobytes of
* data just to check fs name and a mount flag.
*
* Fixing the code to handle everything in the kernel instead
* is a non-trivial endeavor and has low priority, thus this
* horrible kludge facilitates the current behavior in a much
* cheaper manner until someone(tm) sorts this out.
*/
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE) {
fdrop(fp, td);
error = EBADF;
break;
}
vp = fp->f_vnode;
/*
* Since we don't prevent dooming the vnode even non-null mp
* found can become immediately stale. This is tolerable since
* mount points are type-stable (providing safe memory access)
* and any vfs op on this vnode going forward will return an
* error (meaning return value in this case is meaningless).
*/
mp = atomic_load_ptr(&vp->v_mount);
if (__predict_false(mp == NULL)) {
fdrop(fp, td);
error = EBADF;
break;
}
td->td_retval[0] = 0;
if (mp->mnt_kern_flag & MNTK_UNIONFS ||
mp->mnt_flag & MNT_UNION)
td->td_retval[0] = 1;
fdrop(fp, td);
break;
case F_KINFO:
#ifdef CAPABILITY_MODE
if (IN_CAPABILITY_MODE(td)) {
error = ECAPMODE;
break;
}
#endif
error = copyin((void *)arg, &kif_sz, sizeof(kif_sz));
if (error != 0)
break;
if (kif_sz != sizeof(*kif)) {
error = EINVAL;
break;
}
kif = malloc(sizeof(*kif), M_TEMP, M_WAITOK | M_ZERO);
FILEDESC_SLOCK(fdp);
error = fget_cap_locked(fdp, fd, &cap_fcntl_rights, &fp, NULL);
if (error == 0 && fhold(fp)) {
export_file_to_kinfo(fp, fd, NULL, kif, fdp, 0);
FILEDESC_SUNLOCK(fdp);
fdrop(fp, td);
if ((kif->kf_status & KF_ATTR_VALID) != 0) {
kif->kf_structsize = sizeof(*kif);
error = copyout(kif, (void *)arg, sizeof(*kif));
} else {
error = EBADF;
}
} else {
FILEDESC_SUNLOCK(fdp);
if (error == 0)
error = EBADF;
}
free(kif, M_TEMP);
break;
default:
error = EINVAL;
break;
}
return (error);
}
static int
getmaxfd(struct thread *td)
{
return (min((int)lim_cur(td, RLIMIT_NOFILE), maxfilesperproc));
}
/*
* Common code for dup, dup2, fcntl(F_DUPFD) and fcntl(F_DUP2FD).
*/
int
kern_dup(struct thread *td, u_int mode, int flags, int old, int new)
{
struct filedesc *fdp;
struct filedescent *oldfde, *newfde;
struct proc *p;
struct file *delfp, *oldfp;
u_long *oioctls, *nioctls;
int error, maxfd;
p = td->td_proc;
fdp = p->p_fd;
oioctls = NULL;
MPASS((flags & ~(FDDUP_FLAG_CLOEXEC)) == 0);
MPASS(mode < FDDUP_LASTMODE);
AUDIT_ARG_FD(old);
/* XXXRW: if (flags & FDDUP_FIXED) AUDIT_ARG_FD2(new); */
/*
* Verify we have a valid descriptor to dup from and possibly to
* dup to. Unlike dup() and dup2(), fcntl()'s F_DUPFD should
* return EINVAL when the new descriptor is out of bounds.
*/
if (old < 0)
return (EBADF);
if (new < 0)
return (mode == FDDUP_FCNTL ? EINVAL : EBADF);
maxfd = getmaxfd(td);
if (new >= maxfd)
return (mode == FDDUP_FCNTL ? EINVAL : EBADF);
error = EBADF;
FILEDESC_XLOCK(fdp);
if (fget_locked(fdp, old) == NULL)
goto unlock;
if ((mode == FDDUP_FIXED || mode == FDDUP_MUSTREPLACE) && old == new) {
td->td_retval[0] = new;
if (flags & FDDUP_FLAG_CLOEXEC)
fdp->fd_ofiles[new].fde_flags |= UF_EXCLOSE;
error = 0;
goto unlock;
}
oldfde = &fdp->fd_ofiles[old];
oldfp = oldfde->fde_file;
if (!fhold(oldfp))
goto unlock;
/*
* If the caller specified a file descriptor, make sure the file
* table is large enough to hold it, and grab it. Otherwise, just
* allocate a new descriptor the usual way.
*/
switch (mode) {
case FDDUP_NORMAL:
case FDDUP_FCNTL:
if ((error = fdalloc(td, new, &new)) != 0) {
fdrop(oldfp, td);
goto unlock;
}
break;
case FDDUP_MUSTREPLACE:
/* Target file descriptor must exist. */
if (fget_locked(fdp, new) == NULL) {
fdrop(oldfp, td);
goto unlock;
}
break;
case FDDUP_FIXED:
if (new >= fdp->fd_nfiles) {
/*
* The resource limits are here instead of e.g.
* fdalloc(), because the file descriptor table may be
* shared between processes, so we can't really use
* racct_add()/racct_sub(). Instead of counting the
* number of actually allocated descriptors, just put
* the limit on the size of the file descriptor table.
*/
#ifdef RACCT
if (RACCT_ENABLED()) {
error = racct_set_unlocked(p, RACCT_NOFILE, new + 1);
if (error != 0) {
error = EMFILE;
fdrop(oldfp, td);
goto unlock;
}
}
#endif
fdgrowtable_exp(fdp, new + 1);
}
if (!fdisused(fdp, new))
fdused(fdp, new);
break;
default:
KASSERT(0, ("%s unsupported mode %d", __func__, mode));
}
KASSERT(old != new, ("new fd is same as old"));
/* Refetch oldfde because the table may have grown and old one freed. */
oldfde = &fdp->fd_ofiles[old];
KASSERT(oldfp == oldfde->fde_file,
("fdt_ofiles shift from growth observed at fd %d",
old));
newfde = &fdp->fd_ofiles[new];
delfp = newfde->fde_file;
nioctls = filecaps_copy_prep(&oldfde->fde_caps);
/*
* Duplicate the source descriptor.
*/
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
oioctls = filecaps_free_prep(&newfde->fde_caps);
memcpy(newfde, oldfde, fde_change_size);
filecaps_copy_finish(&oldfde->fde_caps, &newfde->fde_caps,
nioctls);
if ((flags & FDDUP_FLAG_CLOEXEC) != 0)
newfde->fde_flags = oldfde->fde_flags | UF_EXCLOSE;
else
newfde->fde_flags = oldfde->fde_flags & ~UF_EXCLOSE;
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
td->td_retval[0] = new;
error = 0;
if (delfp != NULL) {
(void) closefp(fdp, new, delfp, td, true, false);
FILEDESC_UNLOCK_ASSERT(fdp);
} else {
unlock:
FILEDESC_XUNLOCK(fdp);
}
filecaps_free_finish(oioctls);
return (error);
}
static void
sigiofree(struct sigio *sigio)
{
crfree(sigio->sio_ucred);
free(sigio, M_SIGIO);
}
static struct sigio *
funsetown_locked(struct sigio *sigio)
{
struct proc *p;
struct pgrp *pg;
SIGIO_ASSERT_LOCKED();
if (sigio == NULL)
return (NULL);
*sigio->sio_myref = NULL;
if (sigio->sio_pgid < 0) {
pg = sigio->sio_pgrp;
PGRP_LOCK(pg);
SLIST_REMOVE(&pg->pg_sigiolst, sigio, sigio, sio_pgsigio);
PGRP_UNLOCK(pg);
} else {
p = sigio->sio_proc;
PROC_LOCK(p);
SLIST_REMOVE(&p->p_sigiolst, sigio, sigio, sio_pgsigio);
PROC_UNLOCK(p);
}
return (sigio);
}
/*
* If sigio is on the list associated with a process or process group,
* disable signalling from the device, remove sigio from the list and
* free sigio.
*/
void
funsetown(struct sigio **sigiop)
{
struct sigio *sigio;
/* Racy check, consumers must provide synchronization. */
if (*sigiop == NULL)
return;
SIGIO_LOCK();
sigio = funsetown_locked(*sigiop);
SIGIO_UNLOCK();
if (sigio != NULL)
sigiofree(sigio);
}
/*
* Free a list of sigio structures. The caller must ensure that new sigio
* structures cannot be added after this point. For process groups this is
* guaranteed using the proctree lock; for processes, the P_WEXIT flag serves
* as an interlock.
*/
void
funsetownlst(struct sigiolst *sigiolst)
{
struct proc *p;
struct pgrp *pg;
struct sigio *sigio, *tmp;
/* Racy check. */
sigio = SLIST_FIRST(sigiolst);
if (sigio == NULL)
return;
p = NULL;
pg = NULL;
SIGIO_LOCK();
sigio = SLIST_FIRST(sigiolst);
if (sigio == NULL) {
SIGIO_UNLOCK();
return;
}
/*
* Every entry of the list should belong to a single proc or pgrp.
*/
if (sigio->sio_pgid < 0) {
pg = sigio->sio_pgrp;
sx_assert(&proctree_lock, SX_XLOCKED);
PGRP_LOCK(pg);
} else /* if (sigio->sio_pgid > 0) */ {
p = sigio->sio_proc;
PROC_LOCK(p);
KASSERT((p->p_flag & P_WEXIT) != 0,
("%s: process %p is not exiting", __func__, p));
}
SLIST_FOREACH(sigio, sigiolst, sio_pgsigio) {
*sigio->sio_myref = NULL;
if (pg != NULL) {
KASSERT(sigio->sio_pgid < 0,
("Proc sigio in pgrp sigio list"));
KASSERT(sigio->sio_pgrp == pg,
("Bogus pgrp in sigio list"));
} else /* if (p != NULL) */ {
KASSERT(sigio->sio_pgid > 0,
("Pgrp sigio in proc sigio list"));
KASSERT(sigio->sio_proc == p,
("Bogus proc in sigio list"));
}
}
if (pg != NULL)
PGRP_UNLOCK(pg);
else
PROC_UNLOCK(p);
SIGIO_UNLOCK();
SLIST_FOREACH_SAFE(sigio, sigiolst, sio_pgsigio, tmp)
sigiofree(sigio);
}
/*
* This is common code for FIOSETOWN ioctl called by fcntl(fd, F_SETOWN, arg).
*
* After permission checking, add a sigio structure to the sigio list for
* the process or process group.
*/
int
fsetown(pid_t pgid, struct sigio **sigiop)
{
struct proc *proc;
struct pgrp *pgrp;
struct sigio *osigio, *sigio;
int ret;
if (pgid == 0) {
funsetown(sigiop);
return (0);
}
sigio = malloc(sizeof(struct sigio), M_SIGIO, M_WAITOK);
sigio->sio_pgid = pgid;
sigio->sio_ucred = crhold(curthread->td_ucred);
sigio->sio_myref = sigiop;
ret = 0;
if (pgid > 0) {
ret = pget(pgid, PGET_NOTWEXIT | PGET_NOTID | PGET_HOLD, &proc);
SIGIO_LOCK();
osigio = funsetown_locked(*sigiop);
if (ret == 0) {
PROC_LOCK(proc);
_PRELE(proc);
if ((proc->p_flag & P_WEXIT) != 0) {
ret = ESRCH;
} else if (proc->p_session !=
curthread->td_proc->p_session) {
/*
* Policy - Don't allow a process to FSETOWN a
* process in another session.
*
* Remove this test to allow maximum flexibility
* or restrict FSETOWN to the current process or
* process group for maximum safety.
*/
ret = EPERM;
} else {
sigio->sio_proc = proc;
SLIST_INSERT_HEAD(&proc->p_sigiolst, sigio,
sio_pgsigio);
}
PROC_UNLOCK(proc);
}
} else /* if (pgid < 0) */ {
sx_slock(&proctree_lock);
SIGIO_LOCK();
osigio = funsetown_locked(*sigiop);
pgrp = pgfind(-pgid);
if (pgrp == NULL) {
ret = ESRCH;
} else {
if (pgrp->pg_session != curthread->td_proc->p_session) {
/*
* Policy - Don't allow a process to FSETOWN a
* process in another session.
*
* Remove this test to allow maximum flexibility
* or restrict FSETOWN to the current process or
* process group for maximum safety.
*/
ret = EPERM;
} else {
sigio->sio_pgrp = pgrp;
SLIST_INSERT_HEAD(&pgrp->pg_sigiolst, sigio,
sio_pgsigio);
}
PGRP_UNLOCK(pgrp);
}
sx_sunlock(&proctree_lock);
}
if (ret == 0)
*sigiop = sigio;
SIGIO_UNLOCK();
if (osigio != NULL)
sigiofree(osigio);
return (ret);
}
/*
* This is common code for FIOGETOWN ioctl called by fcntl(fd, F_GETOWN, arg).
*/
pid_t
fgetown(struct sigio **sigiop)
{
pid_t pgid;
SIGIO_LOCK();
pgid = (*sigiop != NULL) ? (*sigiop)->sio_pgid : 0;
SIGIO_UNLOCK();
return (pgid);
}
static int
closefp_impl(struct filedesc *fdp, int fd, struct file *fp, struct thread *td,
bool audit)
{
int error;
FILEDESC_XLOCK_ASSERT(fdp);
/*
* We now hold the fp reference that used to be owned by the
* descriptor array. We have to unlock the FILEDESC *AFTER*
* knote_fdclose to prevent a race of the fd getting opened, a knote
* added, and deleteing a knote for the new fd.
*/
if (__predict_false(!TAILQ_EMPTY(&fdp->fd_kqlist)))
knote_fdclose(td, fd);
/*
* We need to notify mqueue if the object is of type mqueue.
*/
if (__predict_false(fp->f_type == DTYPE_MQUEUE))
mq_fdclose(td, fd, fp);
FILEDESC_XUNLOCK(fdp);
#ifdef AUDIT
if (AUDITING_TD(td) && audit)
audit_sysclose(td, fd, fp);
#endif
error = closef(fp, td);
/*
* All paths leading up to closefp() will have already removed or
* replaced the fd in the filedesc table, so a restart would not
* operate on the same file.
*/
if (error == ERESTART)
error = EINTR;
return (error);
}
static int
closefp_hl(struct filedesc *fdp, int fd, struct file *fp, struct thread *td,
bool holdleaders, bool audit)
{
int error;
FILEDESC_XLOCK_ASSERT(fdp);
if (holdleaders) {
if (td->td_proc->p_fdtol != NULL) {
/*
* Ask fdfree() to sleep to ensure that all relevant
* process leaders can be traversed in closef().
*/
fdp->fd_holdleaderscount++;
} else {
holdleaders = false;
}
}
error = closefp_impl(fdp, fd, fp, td, audit);
if (holdleaders) {
FILEDESC_XLOCK(fdp);
fdp->fd_holdleaderscount--;
if (fdp->fd_holdleaderscount == 0 &&
fdp->fd_holdleaderswakeup != 0) {
fdp->fd_holdleaderswakeup = 0;
wakeup(&fdp->fd_holdleaderscount);
}
FILEDESC_XUNLOCK(fdp);
}
return (error);
}
static int
closefp(struct filedesc *fdp, int fd, struct file *fp, struct thread *td,
bool holdleaders, bool audit)
{
FILEDESC_XLOCK_ASSERT(fdp);
if (__predict_false(td->td_proc->p_fdtol != NULL)) {
return (closefp_hl(fdp, fd, fp, td, holdleaders, audit));
} else {
return (closefp_impl(fdp, fd, fp, td, audit));
}
}
/*
* Close a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct close_args {
int fd;
};
#endif
/* ARGSUSED */
int
sys_close(struct thread *td, struct close_args *uap)
{
return (kern_close(td, uap->fd));
}
int
kern_close(struct thread *td, int fd)
{
struct filedesc *fdp;
struct file *fp;
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
if ((fp = fget_locked(fdp, fd)) == NULL) {
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
fdfree(fdp, fd);
/* closefp() drops the FILEDESC lock for us. */
return (closefp(fdp, fd, fp, td, true, true));
}
static int
close_range_cloexec(struct thread *td, u_int lowfd, u_int highfd)
{
struct filedesc *fdp;
struct fdescenttbl *fdt;
struct filedescent *fde;
int fd;
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
fdt = atomic_load_ptr(&fdp->fd_files);
highfd = MIN(highfd, fdt->fdt_nfiles - 1);
fd = lowfd;
if (__predict_false(fd > highfd)) {
goto out_locked;
}
for (; fd <= highfd; fd++) {
fde = &fdt->fdt_ofiles[fd];
if (fde->fde_file != NULL)
fde->fde_flags |= UF_EXCLOSE;
}
out_locked:
FILEDESC_XUNLOCK(fdp);
return (0);
}
static int
close_range_impl(struct thread *td, u_int lowfd, u_int highfd)
{
struct filedesc *fdp;
const struct fdescenttbl *fdt;
struct file *fp;
int fd;
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
fdt = atomic_load_ptr(&fdp->fd_files);
highfd = MIN(highfd, fdt->fdt_nfiles - 1);
fd = lowfd;
if (__predict_false(fd > highfd)) {
goto out_locked;
}
for (;;) {
fp = fdt->fdt_ofiles[fd].fde_file;
if (fp == NULL) {
if (fd == highfd)
goto out_locked;
} else {
fdfree(fdp, fd);
(void) closefp(fdp, fd, fp, td, true, true);
if (fd == highfd)
goto out_unlocked;
FILEDESC_XLOCK(fdp);
fdt = atomic_load_ptr(&fdp->fd_files);
}
fd++;
}
out_locked:
FILEDESC_XUNLOCK(fdp);
out_unlocked:
return (0);
}
int
kern_close_range(struct thread *td, int flags, u_int lowfd, u_int highfd)
{
/*
* Check this prior to clamping; closefrom(3) with only fd 0, 1, and 2
* open should not be a usage error. From a close_range() perspective,
* close_range(3, ~0U, 0) in the same scenario should also likely not
* be a usage error as all fd above 3 are in-fact already closed.
*/
if (highfd < lowfd) {
return (EINVAL);
}
if ((flags & CLOSE_RANGE_CLOEXEC) != 0)
return (close_range_cloexec(td, lowfd, highfd));
return (close_range_impl(td, lowfd, highfd));
}
#ifndef _SYS_SYSPROTO_H_
struct close_range_args {
u_int lowfd;
u_int highfd;
int flags;
};
#endif
int
sys_close_range(struct thread *td, struct close_range_args *uap)
{
AUDIT_ARG_FD(uap->lowfd);
AUDIT_ARG_CMD(uap->highfd);
AUDIT_ARG_FFLAGS(uap->flags);
if ((uap->flags & ~(CLOSE_RANGE_CLOEXEC)) != 0)
return (EINVAL);
return (kern_close_range(td, uap->flags, uap->lowfd, uap->highfd));
}
#ifdef COMPAT_FREEBSD12
/*
* Close open file descriptors.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd12_closefrom_args {
int lowfd;
};
#endif
/* ARGSUSED */
int
freebsd12_closefrom(struct thread *td, struct freebsd12_closefrom_args *uap)
{
u_int lowfd;
AUDIT_ARG_FD(uap->lowfd);
/*
* Treat negative starting file descriptor values identical to
* closefrom(0) which closes all files.
*/
lowfd = MAX(0, uap->lowfd);
return (kern_close_range(td, 0, lowfd, ~0U));
}
#endif /* COMPAT_FREEBSD12 */
#if defined(COMPAT_43)
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct ofstat_args {
int fd;
struct ostat *sb;
};
#endif
/* ARGSUSED */
int
ofstat(struct thread *td, struct ofstat_args *uap)
{
struct ostat oub;
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0) {
cvtstat(&ub, &oub);
error = copyout(&oub, uap->sb, sizeof(oub));
}
return (error);
}
#endif /* COMPAT_43 */
#if defined(COMPAT_FREEBSD11)
int
freebsd11_fstat(struct thread *td, struct freebsd11_fstat_args *uap)
{
struct stat sb;
struct freebsd11_stat osb;
int error;
error = kern_fstat(td, uap->fd, &sb);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->sb, sizeof(osb));
return (error);
}
#endif /* COMPAT_FREEBSD11 */
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fstat_args {
int fd;
struct stat *sb;
};
#endif
/* ARGSUSED */
int
sys_fstat(struct thread *td, struct fstat_args *uap)
{
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0)
error = copyout(&ub, uap->sb, sizeof(ub));
return (error);
}
int
kern_fstat(struct thread *td, int fd, struct stat *sbp)
{
struct file *fp;
int error;
AUDIT_ARG_FD(fd);
error = fget(td, fd, &cap_fstat_rights, &fp);
if (__predict_false(error != 0))
return (error);
AUDIT_ARG_FILE(td->td_proc, fp);
error = fo_stat(fp, sbp, td->td_ucred, td);
fdrop(fp, td);
#ifdef __STAT_TIME_T_EXT
sbp->st_atim_ext = 0;
sbp->st_mtim_ext = 0;
sbp->st_ctim_ext = 0;
sbp->st_btim_ext = 0;
#endif
#ifdef KTRACE
if (KTRPOINT(td, KTR_STRUCT))
ktrstat_error(sbp, error);
#endif
return (error);
}
#if defined(COMPAT_FREEBSD11)
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd11_nfstat_args {
int fd;
struct nstat *sb;
};
#endif
/* ARGSUSED */
int
freebsd11_nfstat(struct thread *td, struct freebsd11_nfstat_args *uap)
{
struct nstat nub;
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0) {
freebsd11_cvtnstat(&ub, &nub);
error = copyout(&nub, uap->sb, sizeof(nub));
}
return (error);
}
#endif /* COMPAT_FREEBSD11 */
/*
* Return pathconf information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fpathconf_args {
int fd;
int name;
};
#endif
/* ARGSUSED */
int
sys_fpathconf(struct thread *td, struct fpathconf_args *uap)
{
long value;
int error;
error = kern_fpathconf(td, uap->fd, uap->name, &value);
if (error == 0)
td->td_retval[0] = value;
return (error);
}
int
kern_fpathconf(struct thread *td, int fd, int name, long *valuep)
{
struct file *fp;
struct vnode *vp;
int error;
error = fget(td, fd, &cap_fpathconf_rights, &fp);
if (error != 0)
return (error);
if (name == _PC_ASYNC_IO) {
*valuep = _POSIX_ASYNCHRONOUS_IO;
goto out;
}
vp = fp->f_vnode;
if (vp != NULL) {
vn_lock(vp, LK_SHARED | LK_RETRY);
error = VOP_PATHCONF(vp, name, valuep);
VOP_UNLOCK(vp);
} else if (fp->f_type == DTYPE_PIPE || fp->f_type == DTYPE_SOCKET) {
if (name != _PC_PIPE_BUF) {
error = EINVAL;
} else {
*valuep = PIPE_BUF;
error = 0;
}
} else {
error = EOPNOTSUPP;
}
out:
fdrop(fp, td);
return (error);
}
/*
* Copy filecaps structure allocating memory for ioctls array if needed.
*
* The last parameter indicates whether the fdtable is locked. If it is not and
* ioctls are encountered, copying fails and the caller must lock the table.
*
* Note that if the table was not locked, the caller has to check the relevant
* sequence counter to determine whether the operation was successful.
*/
bool
filecaps_copy(const struct filecaps *src, struct filecaps *dst, bool locked)
{
size_t size;
if (src->fc_ioctls != NULL && !locked)
return (false);
memcpy(dst, src, sizeof(*src));
if (src->fc_ioctls == NULL)
return (true);
KASSERT(src->fc_nioctls > 0,
("fc_ioctls != NULL, but fc_nioctls=%hd", src->fc_nioctls));
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
dst->fc_ioctls = malloc(size, M_FILECAPS, M_WAITOK);
memcpy(dst->fc_ioctls, src->fc_ioctls, size);
return (true);
}
static u_long *
filecaps_copy_prep(const struct filecaps *src)
{
u_long *ioctls;
size_t size;
if (__predict_true(src->fc_ioctls == NULL))
return (NULL);
KASSERT(src->fc_nioctls > 0,
("fc_ioctls != NULL, but fc_nioctls=%hd", src->fc_nioctls));
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
ioctls = malloc(size, M_FILECAPS, M_WAITOK);
return (ioctls);
}
static void
filecaps_copy_finish(const struct filecaps *src, struct filecaps *dst,
u_long *ioctls)
{
size_t size;
*dst = *src;
if (__predict_true(src->fc_ioctls == NULL)) {
MPASS(ioctls == NULL);
return;
}
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
dst->fc_ioctls = ioctls;
bcopy(src->fc_ioctls, dst->fc_ioctls, size);
}
/*
* Move filecaps structure to the new place and clear the old place.
*/
void
filecaps_move(struct filecaps *src, struct filecaps *dst)
{
*dst = *src;
bzero(src, sizeof(*src));
}
/*
* Fill the given filecaps structure with full rights.
*/
static void
filecaps_fill(struct filecaps *fcaps)
{
CAP_ALL(&fcaps->fc_rights);
fcaps->fc_ioctls = NULL;
fcaps->fc_nioctls = -1;
fcaps->fc_fcntls = CAP_FCNTL_ALL;
}
/*
* Free memory allocated within filecaps structure.
*/
+static void
+filecaps_free_ioctl(struct filecaps *fcaps)
+{
+
+ free(fcaps->fc_ioctls, M_FILECAPS);
+ fcaps->fc_ioctls = NULL;
+}
+
void
filecaps_free(struct filecaps *fcaps)
{
- free(fcaps->fc_ioctls, M_FILECAPS);
+ filecaps_free_ioctl(fcaps);
bzero(fcaps, sizeof(*fcaps));
}
static u_long *
filecaps_free_prep(struct filecaps *fcaps)
{
u_long *ioctls;
ioctls = fcaps->fc_ioctls;
bzero(fcaps, sizeof(*fcaps));
return (ioctls);
}
static void
filecaps_free_finish(u_long *ioctls)
{
free(ioctls, M_FILECAPS);
}
/*
* Validate the given filecaps structure.
*/
static void
filecaps_validate(const struct filecaps *fcaps, const char *func)
{
KASSERT(cap_rights_is_valid(&fcaps->fc_rights),
("%s: invalid rights", func));
KASSERT((fcaps->fc_fcntls & ~CAP_FCNTL_ALL) == 0,
("%s: invalid fcntls", func));
KASSERT(fcaps->fc_fcntls == 0 ||
cap_rights_is_set(&fcaps->fc_rights, CAP_FCNTL),
("%s: fcntls without CAP_FCNTL", func));
KASSERT(fcaps->fc_ioctls != NULL ? fcaps->fc_nioctls > 0 :
(fcaps->fc_nioctls == -1 || fcaps->fc_nioctls == 0),
("%s: invalid ioctls", func));
KASSERT(fcaps->fc_nioctls == 0 ||
cap_rights_is_set(&fcaps->fc_rights, CAP_IOCTL),
("%s: ioctls without CAP_IOCTL", func));
}
static void
fdgrowtable_exp(struct filedesc *fdp, int nfd)
{
int nfd1;
FILEDESC_XLOCK_ASSERT(fdp);
nfd1 = fdp->fd_nfiles * 2;
if (nfd1 < nfd)
nfd1 = nfd;
fdgrowtable(fdp, nfd1);
}
/*
* Grow the file table to accommodate (at least) nfd descriptors.
*/
static void
fdgrowtable(struct filedesc *fdp, int nfd)
{
struct filedesc0 *fdp0;
struct freetable *ft;
struct fdescenttbl *ntable;
struct fdescenttbl *otable;
int nnfiles, onfiles;
NDSLOTTYPE *nmap, *omap;
KASSERT(fdp->fd_nfiles > 0, ("zero-length file table"));
/* save old values */
onfiles = fdp->fd_nfiles;
otable = fdp->fd_files;
omap = fdp->fd_map;
/* compute the size of the new table */
nnfiles = NDSLOTS(nfd) * NDENTRIES; /* round up */
if (nnfiles <= onfiles)
/* the table is already large enough */
return;
/*
* Allocate a new table. We need enough space for the number of
* entries, file entries themselves and the struct freetable we will use
* when we decommission the table and place it on the freelist.
* We place the struct freetable in the middle so we don't have
* to worry about padding.
*/
ntable = malloc(offsetof(struct fdescenttbl, fdt_ofiles) +
nnfiles * sizeof(ntable->fdt_ofiles[0]) +
sizeof(struct freetable),
M_FILEDESC, M_ZERO | M_WAITOK);
/* copy the old data */
ntable->fdt_nfiles = nnfiles;
memcpy(ntable->fdt_ofiles, otable->fdt_ofiles,
onfiles * sizeof(ntable->fdt_ofiles[0]));
/*
* Allocate a new map only if the old is not large enough. It will
* grow at a slower rate than the table as it can map more
* entries than the table can hold.
*/
if (NDSLOTS(nnfiles) > NDSLOTS(onfiles)) {
nmap = malloc(NDSLOTS(nnfiles) * NDSLOTSIZE, M_FILEDESC,
M_ZERO | M_WAITOK);
/* copy over the old data and update the pointer */
memcpy(nmap, omap, NDSLOTS(onfiles) * sizeof(*omap));
fdp->fd_map = nmap;
}
/*
* Make sure that ntable is correctly initialized before we replace
* fd_files poiner. Otherwise fget_unlocked() may see inconsistent
* data.
*/
atomic_store_rel_ptr((volatile void *)&fdp->fd_files, (uintptr_t)ntable);
/*
* Free the old file table when not shared by other threads or processes.
* The old file table is considered to be shared when either are true:
* - The process has more than one thread.
* - The file descriptor table has been shared via fdshare().
*
* When shared, the old file table will be placed on a freelist
* which will be processed when the struct filedesc is released.
*
* Note that if onfiles == NDFILE, we're dealing with the original
* static allocation contained within (struct filedesc0 *)fdp,
* which must not be freed.
*/
if (onfiles > NDFILE) {
/*
* Note we may be called here from fdinit while allocating a
* table for a new process in which case ->p_fd points
* elsewhere.
*/
if (curproc->p_fd != fdp || FILEDESC_IS_ONLY_USER(fdp)) {
free(otable, M_FILEDESC);
} else {
ft = (struct freetable *)&otable->fdt_ofiles[onfiles];
fdp0 = (struct filedesc0 *)fdp;
ft->ft_table = otable;
SLIST_INSERT_HEAD(&fdp0->fd_free, ft, ft_next);
}
}
/*
* The map does not have the same possibility of threads still
* holding references to it. So always free it as long as it
* does not reference the original static allocation.
*/
if (NDSLOTS(onfiles) > NDSLOTS(NDFILE))
free(omap, M_FILEDESC);
}
/*
* Allocate a file descriptor for the process.
*/
int
fdalloc(struct thread *td, int minfd, int *result)
{
struct proc *p = td->td_proc;
struct filedesc *fdp = p->p_fd;
int fd, maxfd, allocfd;
#ifdef RACCT
int error;
#endif
FILEDESC_XLOCK_ASSERT(fdp);
if (fdp->fd_freefile > minfd)
minfd = fdp->fd_freefile;
maxfd = getmaxfd(td);
/*
* Search the bitmap for a free descriptor starting at minfd.
* If none is found, grow the file table.
*/
fd = fd_first_free(fdp, minfd, fdp->fd_nfiles);
if (__predict_false(fd >= maxfd))
return (EMFILE);
if (__predict_false(fd >= fdp->fd_nfiles)) {
allocfd = min(fd * 2, maxfd);
#ifdef RACCT
if (RACCT_ENABLED()) {
error = racct_set_unlocked(p, RACCT_NOFILE, allocfd);
if (error != 0)
return (EMFILE);
}
#endif
/*
* fd is already equal to first free descriptor >= minfd, so
* we only need to grow the table and we are done.
*/
fdgrowtable_exp(fdp, allocfd);
}
/*
* Perform some sanity checks, then mark the file descriptor as
* used and return it to the caller.
*/
KASSERT(fd >= 0 && fd < min(maxfd, fdp->fd_nfiles),
("invalid descriptor %d", fd));
KASSERT(!fdisused(fdp, fd),
("fd_first_free() returned non-free descriptor"));
KASSERT(fdp->fd_ofiles[fd].fde_file == NULL,
("file descriptor isn't free"));
fdused(fdp, fd);
*result = fd;
return (0);
}
/*
* Allocate n file descriptors for the process.
*/
int
fdallocn(struct thread *td, int minfd, int *fds, int n)
{
struct proc *p = td->td_proc;
struct filedesc *fdp = p->p_fd;
int i;
FILEDESC_XLOCK_ASSERT(fdp);
for (i = 0; i < n; i++)
if (fdalloc(td, 0, &fds[i]) != 0)
break;
if (i < n) {
for (i--; i >= 0; i--)
fdunused(fdp, fds[i]);
return (EMFILE);
}
return (0);
}
/*
* Create a new open file structure and allocate a file descriptor for the
* process that refers to it. We add one reference to the file for the
* descriptor table and one reference for resultfp. This is to prevent us
* being preempted and the entry in the descriptor table closed after we
* release the FILEDESC lock.
*/
int
falloc_caps(struct thread *td, struct file **resultfp, int *resultfd, int flags,
struct filecaps *fcaps)
{
struct file *fp;
int error, fd;
MPASS(resultfp != NULL);
MPASS(resultfd != NULL);
error = _falloc_noinstall(td, &fp, 2);
if (__predict_false(error != 0)) {
return (error);
}
error = finstall_refed(td, fp, &fd, flags, fcaps);
if (__predict_false(error != 0)) {
falloc_abort(td, fp);
return (error);
}
*resultfp = fp;
*resultfd = fd;
return (0);
}
/*
* Create a new open file structure without allocating a file descriptor.
*/
int
_falloc_noinstall(struct thread *td, struct file **resultfp, u_int n)
{
struct file *fp;
int maxuserfiles = maxfiles - (maxfiles / 20);
int openfiles_new;
static struct timeval lastfail;
static int curfail;
KASSERT(resultfp != NULL, ("%s: resultfp == NULL", __func__));
MPASS(n > 0);
openfiles_new = atomic_fetchadd_int(&openfiles, 1) + 1;
if ((openfiles_new >= maxuserfiles &&
priv_check(td, PRIV_MAXFILES) != 0) ||
openfiles_new >= maxfiles) {
atomic_subtract_int(&openfiles, 1);
if (ppsratecheck(&lastfail, &curfail, 1)) {
printf("kern.maxfiles limit exceeded by uid %i, (%s) "
"please see tuning(7).\n", td->td_ucred->cr_ruid, td->td_proc->p_comm);
}
return (ENFILE);
}
fp = uma_zalloc(file_zone, M_WAITOK);
bzero(fp, sizeof(*fp));
refcount_init(&fp->f_count, n);
fp->f_cred = crhold(td->td_ucred);
fp->f_ops = &badfileops;
*resultfp = fp;
return (0);
}
void
falloc_abort(struct thread *td, struct file *fp)
{
/*
* For assertion purposes.
*/
refcount_init(&fp->f_count, 0);
_fdrop(fp, td);
}
/*
* Install a file in a file descriptor table.
*/
void
_finstall(struct filedesc *fdp, struct file *fp, int fd, int flags,
struct filecaps *fcaps)
{
struct filedescent *fde;
MPASS(fp != NULL);
if (fcaps != NULL)
filecaps_validate(fcaps, __func__);
FILEDESC_XLOCK_ASSERT(fdp);
fde = &fdp->fd_ofiles[fd];
#ifdef CAPABILITIES
seqc_write_begin(&fde->fde_seqc);
#endif
fde->fde_file = fp;
fde->fde_flags = (flags & O_CLOEXEC) != 0 ? UF_EXCLOSE : 0;
if (fcaps != NULL)
filecaps_move(fcaps, &fde->fde_caps);
else
filecaps_fill(&fde->fde_caps);
#ifdef CAPABILITIES
seqc_write_end(&fde->fde_seqc);
#endif
}
int
finstall_refed(struct thread *td, struct file *fp, int *fd, int flags,
struct filecaps *fcaps)
{
struct filedesc *fdp = td->td_proc->p_fd;
int error;
MPASS(fd != NULL);
FILEDESC_XLOCK(fdp);
error = fdalloc(td, 0, fd);
if (__predict_true(error == 0)) {
_finstall(fdp, fp, *fd, flags, fcaps);
}
FILEDESC_XUNLOCK(fdp);
return (error);
}
int
finstall(struct thread *td, struct file *fp, int *fd, int flags,
struct filecaps *fcaps)
{
int error;
MPASS(fd != NULL);
if (!fhold(fp))
return (EBADF);
error = finstall_refed(td, fp, fd, flags, fcaps);
if (__predict_false(error != 0)) {
fdrop(fp, td);
}
return (error);
}
/*
* Build a new filedesc structure from another.
*
* If fdp is not NULL, return with it shared locked.
*/
struct filedesc *
fdinit(struct filedesc *fdp, bool prepfiles, int *lastfile)
{
struct filedesc0 *newfdp0;
struct filedesc *newfdp;
if (prepfiles)
MPASS(lastfile != NULL);
else
MPASS(lastfile == NULL);
newfdp0 = uma_zalloc(filedesc0_zone, M_WAITOK | M_ZERO);
newfdp = &newfdp0->fd_fd;
/* Create the file descriptor table. */
FILEDESC_LOCK_INIT(newfdp);
refcount_init(&newfdp->fd_refcnt, 1);
refcount_init(&newfdp->fd_holdcnt, 1);
newfdp->fd_map = newfdp0->fd_dmap;
newfdp->fd_files = (struct fdescenttbl *)&newfdp0->fd_dfiles;
newfdp->fd_files->fdt_nfiles = NDFILE;
if (fdp == NULL)
return (newfdp);
FILEDESC_SLOCK(fdp);
if (!prepfiles) {
FILEDESC_SUNLOCK(fdp);
return (newfdp);
}
for (;;) {
*lastfile = fdlastfile(fdp);
if (*lastfile < newfdp->fd_nfiles)
break;
FILEDESC_SUNLOCK(fdp);
fdgrowtable(newfdp, *lastfile + 1);
FILEDESC_SLOCK(fdp);
}
return (newfdp);
}
/*
* Build a pwddesc structure from another.
* Copy the current, root, and jail root vnode references.
*
* If pdp is not NULL, return with it shared locked.
*/
struct pwddesc *
pdinit(struct pwddesc *pdp, bool keeplock)
{
struct pwddesc *newpdp;
struct pwd *newpwd;
newpdp = malloc(sizeof(*newpdp), M_PWDDESC, M_WAITOK | M_ZERO);
PWDDESC_LOCK_INIT(newpdp);
refcount_init(&newpdp->pd_refcount, 1);
newpdp->pd_cmask = CMASK;
if (pdp == NULL) {
newpwd = pwd_alloc();
smr_serialized_store(&newpdp->pd_pwd, newpwd, true);
return (newpdp);
}
PWDDESC_XLOCK(pdp);
newpwd = pwd_hold_pwddesc(pdp);
smr_serialized_store(&newpdp->pd_pwd, newpwd, true);
if (!keeplock)
PWDDESC_XUNLOCK(pdp);
return (newpdp);
}
/*
* Hold either filedesc or pwddesc of the passed process.
*
* The process lock is used to synchronize against the target exiting and
* freeing the data.
*
* Clearing can be ilustrated in 3 steps:
* 1. set the pointer to NULL. Either routine can race against it, hence
* atomic_load_ptr.
* 2. observe the process lock as not taken. Until then fdhold/pdhold can
* race to either still see the pointer or find NULL. It is still safe to
* grab a reference as clearing is stalled.
* 3. after the lock is observed as not taken, any fdhold/pdhold calls are
* guaranteed to see NULL, making it safe to finish clearing
*/
static struct filedesc *
fdhold(struct proc *p)
{
struct filedesc *fdp;
PROC_LOCK_ASSERT(p, MA_OWNED);
fdp = atomic_load_ptr(&p->p_fd);
if (fdp != NULL)
refcount_acquire(&fdp->fd_holdcnt);
return (fdp);
}
static struct pwddesc *
pdhold(struct proc *p)
{
struct pwddesc *pdp;
PROC_LOCK_ASSERT(p, MA_OWNED);
pdp = atomic_load_ptr(&p->p_pd);
if (pdp != NULL)
refcount_acquire(&pdp->pd_refcount);
return (pdp);
}
static void
fddrop(struct filedesc *fdp)
{
if (refcount_load(&fdp->fd_holdcnt) > 1) {
if (refcount_release(&fdp->fd_holdcnt) == 0)
return;
}
FILEDESC_LOCK_DESTROY(fdp);
uma_zfree(filedesc0_zone, fdp);
}
static void
pddrop(struct pwddesc *pdp)
{
struct pwd *pwd;
if (refcount_release_if_not_last(&pdp->pd_refcount))
return;
PWDDESC_XLOCK(pdp);
if (refcount_release(&pdp->pd_refcount) == 0) {
PWDDESC_XUNLOCK(pdp);
return;
}
pwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
pwd_set(pdp, NULL);
PWDDESC_XUNLOCK(pdp);
pwd_drop(pwd);
PWDDESC_LOCK_DESTROY(pdp);
free(pdp, M_PWDDESC);
}
/*
* Share a filedesc structure.
*/
struct filedesc *
fdshare(struct filedesc *fdp)
{
refcount_acquire(&fdp->fd_refcnt);
return (fdp);
}
/*
* Share a pwddesc structure.
*/
struct pwddesc *
pdshare(struct pwddesc *pdp)
{
refcount_acquire(&pdp->pd_refcount);
return (pdp);
}
/*
* Unshare a filedesc structure, if necessary by making a copy
*/
void
fdunshare(struct thread *td)
{
struct filedesc *tmp;
struct proc *p = td->td_proc;
if (refcount_load(&p->p_fd->fd_refcnt) == 1)
return;
tmp = fdcopy(p->p_fd);
fdescfree(td);
p->p_fd = tmp;
}
/*
* Unshare a pwddesc structure.
*/
void
pdunshare(struct thread *td)
{
struct pwddesc *pdp;
struct proc *p;
p = td->td_proc;
/* Not shared. */
if (p->p_pd->pd_refcount == 1)
return;
pdp = pdcopy(p->p_pd);
pdescfree(td);
p->p_pd = pdp;
}
void
fdinstall_remapped(struct thread *td, struct filedesc *fdp)
{
fdescfree(td);
td->td_proc->p_fd = fdp;
}
/*
* Copy a filedesc structure. A NULL pointer in returns a NULL reference,
* this is to ease callers, not catch errors.
*/
struct filedesc *
fdcopy(struct filedesc *fdp)
{
struct filedesc *newfdp;
struct filedescent *nfde, *ofde;
int i, lastfile;
MPASS(fdp != NULL);
newfdp = fdinit(fdp, true, &lastfile);
/* copy all passable descriptors (i.e. not kqueue) */
newfdp->fd_freefile = -1;
for (i = 0; i <= lastfile; ++i) {
ofde = &fdp->fd_ofiles[i];
if (ofde->fde_file == NULL ||
(ofde->fde_file->f_ops->fo_flags & DFLAG_PASSABLE) == 0 ||
!fhold(ofde->fde_file)) {
if (newfdp->fd_freefile == -1)
newfdp->fd_freefile = i;
continue;
}
nfde = &newfdp->fd_ofiles[i];
*nfde = *ofde;
filecaps_copy(&ofde->fde_caps, &nfde->fde_caps, true);
fdused_init(newfdp, i);
}
if (newfdp->fd_freefile == -1)
newfdp->fd_freefile = i;
FILEDESC_SUNLOCK(fdp);
return (newfdp);
}
/*
* Copy a pwddesc structure.
*/
struct pwddesc *
pdcopy(struct pwddesc *pdp)
{
struct pwddesc *newpdp;
MPASS(pdp != NULL);
newpdp = pdinit(pdp, true);
newpdp->pd_cmask = pdp->pd_cmask;
PWDDESC_XUNLOCK(pdp);
return (newpdp);
}
/*
* Copies a filedesc structure, while remapping all file descriptors
* stored inside using a translation table.
*
* File descriptors are copied over to the new file descriptor table,
* regardless of whether the close-on-exec flag is set.
*/
int
fdcopy_remapped(struct filedesc *fdp, const int *fds, size_t nfds,
struct filedesc **ret)
{
struct filedesc *newfdp;
struct filedescent *nfde, *ofde;
int error, i, lastfile;
MPASS(fdp != NULL);
newfdp = fdinit(fdp, true, &lastfile);
if (nfds > lastfile + 1) {
/* New table cannot be larger than the old one. */
error = E2BIG;
goto bad;
}
/* Copy all passable descriptors (i.e. not kqueue). */
newfdp->fd_freefile = nfds;
for (i = 0; i < nfds; ++i) {
if (fds[i] < 0 || fds[i] > lastfile) {
/* File descriptor out of bounds. */
error = EBADF;
goto bad;
}
ofde = &fdp->fd_ofiles[fds[i]];
if (ofde->fde_file == NULL) {
/* Unused file descriptor. */
error = EBADF;
goto bad;
}
if ((ofde->fde_file->f_ops->fo_flags & DFLAG_PASSABLE) == 0) {
/* File descriptor cannot be passed. */
error = EINVAL;
goto bad;
}
if (!fhold(ofde->fde_file)) {
error = EBADF;
goto bad;
}
nfde = &newfdp->fd_ofiles[i];
*nfde = *ofde;
filecaps_copy(&ofde->fde_caps, &nfde->fde_caps, true);
fdused_init(newfdp, i);
}
FILEDESC_SUNLOCK(fdp);
*ret = newfdp;
return (0);
bad:
FILEDESC_SUNLOCK(fdp);
fdescfree_remapped(newfdp);
return (error);
}
/*
* Clear POSIX style locks. This is only used when fdp looses a reference (i.e.
* one of processes using it exits) and the table used to be shared.
*/
static void
fdclearlocks(struct thread *td)
{
struct filedesc *fdp;
struct filedesc_to_leader *fdtol;
struct flock lf;
struct file *fp;
struct proc *p;
struct vnode *vp;
int i, lastfile;
p = td->td_proc;
fdp = p->p_fd;
fdtol = p->p_fdtol;
MPASS(fdtol != NULL);
FILEDESC_XLOCK(fdp);
KASSERT(fdtol->fdl_refcount > 0,
("filedesc_to_refcount botch: fdl_refcount=%d",
fdtol->fdl_refcount));
if (fdtol->fdl_refcount == 1 &&
(p->p_leader->p_flag & P_ADVLOCK) != 0) {
lastfile = fdlastfile(fdp);
for (i = 0; i <= lastfile; i++) {
fp = fdp->fd_ofiles[i].fde_file;
if (fp == NULL || fp->f_type != DTYPE_VNODE ||
!fhold(fp))
continue;
FILEDESC_XUNLOCK(fdp);
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
vp = fp->f_vnode;
(void) VOP_ADVLOCK(vp,
(caddr_t)p->p_leader, F_UNLCK,
&lf, F_POSIX);
FILEDESC_XLOCK(fdp);
fdrop(fp, td);
}
}
retry:
if (fdtol->fdl_refcount == 1) {
if (fdp->fd_holdleaderscount > 0 &&
(p->p_leader->p_flag & P_ADVLOCK) != 0) {
/*
* close() or kern_dup() has cleared a reference
* in a shared file descriptor table.
*/
fdp->fd_holdleaderswakeup = 1;
sx_sleep(&fdp->fd_holdleaderscount,
FILEDESC_LOCK(fdp), PLOCK, "fdlhold", 0);
goto retry;
}
if (fdtol->fdl_holdcount > 0) {
/*
* Ensure that fdtol->fdl_leader remains
* valid in closef().
*/
fdtol->fdl_wakeup = 1;
sx_sleep(fdtol, FILEDESC_LOCK(fdp), PLOCK,
"fdlhold", 0);
goto retry;
}
}
fdtol->fdl_refcount--;
if (fdtol->fdl_refcount == 0 &&
fdtol->fdl_holdcount == 0) {
fdtol->fdl_next->fdl_prev = fdtol->fdl_prev;
fdtol->fdl_prev->fdl_next = fdtol->fdl_next;
} else
fdtol = NULL;
p->p_fdtol = NULL;
FILEDESC_XUNLOCK(fdp);
if (fdtol != NULL)
free(fdtol, M_FILEDESC_TO_LEADER);
}
/*
* Release a filedesc structure.
*/
static void
fdescfree_fds(struct thread *td, struct filedesc *fdp, bool needclose)
{
struct filedesc0 *fdp0;
struct freetable *ft, *tft;
struct filedescent *fde;
struct file *fp;
int i, lastfile;
KASSERT(refcount_load(&fdp->fd_refcnt) == 0,
("%s: fd table %p carries references", __func__, fdp));
/*
* Serialize with threads iterating over the table, if any.
*/
if (refcount_load(&fdp->fd_holdcnt) > 1) {
FILEDESC_XLOCK(fdp);
FILEDESC_XUNLOCK(fdp);
}
lastfile = fdlastfile_single(fdp);
for (i = 0; i <= lastfile; i++) {
fde = &fdp->fd_ofiles[i];
fp = fde->fde_file;
if (fp != NULL) {
fdefree_last(fde);
if (needclose)
(void) closef(fp, td);
else
fdrop(fp, td);
}
}
if (NDSLOTS(fdp->fd_nfiles) > NDSLOTS(NDFILE))
free(fdp->fd_map, M_FILEDESC);
if (fdp->fd_nfiles > NDFILE)
free(fdp->fd_files, M_FILEDESC);
fdp0 = (struct filedesc0 *)fdp;
SLIST_FOREACH_SAFE(ft, &fdp0->fd_free, ft_next, tft)
free(ft->ft_table, M_FILEDESC);
fddrop(fdp);
}
void
fdescfree(struct thread *td)
{
struct proc *p;
struct filedesc *fdp;
p = td->td_proc;
fdp = p->p_fd;
MPASS(fdp != NULL);
#ifdef RACCT
if (RACCT_ENABLED())
racct_set_unlocked(p, RACCT_NOFILE, 0);
#endif
if (p->p_fdtol != NULL)
fdclearlocks(td);
/*
* Check fdhold for an explanation.
*/
atomic_store_ptr(&p->p_fd, NULL);
atomic_thread_fence_seq_cst();
PROC_WAIT_UNLOCKED(p);
if (refcount_release(&fdp->fd_refcnt) == 0)
return;
fdescfree_fds(td, fdp, 1);
}
void
pdescfree(struct thread *td)
{
struct proc *p;
struct pwddesc *pdp;
p = td->td_proc;
pdp = p->p_pd;
MPASS(pdp != NULL);
/*
* Check pdhold for an explanation.
*/
atomic_store_ptr(&p->p_pd, NULL);
atomic_thread_fence_seq_cst();
PROC_WAIT_UNLOCKED(p);
pddrop(pdp);
}
void
fdescfree_remapped(struct filedesc *fdp)
{
#ifdef INVARIANTS
/* fdescfree_fds() asserts that fd_refcnt == 0. */
if (!refcount_release(&fdp->fd_refcnt))
panic("%s: fd table %p has extra references", __func__, fdp);
#endif
fdescfree_fds(curthread, fdp, 0);
}
/*
* For setugid programs, we don't want to people to use that setugidness
* to generate error messages which write to a file which otherwise would
* otherwise be off-limits to the process. We check for filesystems where
* the vnode can change out from under us after execve (like [lin]procfs).
*
* Since fdsetugidsafety calls this only for fd 0, 1 and 2, this check is
* sufficient. We also don't check for setugidness since we know we are.
*/
static bool
is_unsafe(struct file *fp)
{
struct vnode *vp;
if (fp->f_type != DTYPE_VNODE)
return (false);
vp = fp->f_vnode;
return ((vp->v_vflag & VV_PROCDEP) != 0);
}
/*
* Make this setguid thing safe, if at all possible.
*/
void
fdsetugidsafety(struct thread *td)
{
struct filedesc *fdp;
struct file *fp;
int i;
fdp = td->td_proc->p_fd;
KASSERT(refcount_load(&fdp->fd_refcnt) == 1,
("the fdtable should not be shared"));
MPASS(fdp->fd_nfiles >= 3);
for (i = 0; i <= 2; i++) {
fp = fdp->fd_ofiles[i].fde_file;
if (fp != NULL && is_unsafe(fp)) {
FILEDESC_XLOCK(fdp);
knote_fdclose(td, i);
/*
* NULL-out descriptor prior to close to avoid
* a race while close blocks.
*/
fdfree(fdp, i);
FILEDESC_XUNLOCK(fdp);
(void) closef(fp, td);
}
}
}
/*
* If a specific file object occupies a specific file descriptor, close the
* file descriptor entry and drop a reference on the file object. This is a
* convenience function to handle a subsequent error in a function that calls
* falloc() that handles the race that another thread might have closed the
* file descriptor out from under the thread creating the file object.
*/
void
fdclose(struct thread *td, struct file *fp, int idx)
{
struct filedesc *fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
if (fdp->fd_ofiles[idx].fde_file == fp) {
fdfree(fdp, idx);
FILEDESC_XUNLOCK(fdp);
fdrop(fp, td);
} else
FILEDESC_XUNLOCK(fdp);
}
/*
* Close any files on exec?
*/
void
fdcloseexec(struct thread *td)
{
struct filedesc *fdp;
struct filedescent *fde;
struct file *fp;
int i, lastfile;
fdp = td->td_proc->p_fd;
KASSERT(refcount_load(&fdp->fd_refcnt) == 1,
("the fdtable should not be shared"));
lastfile = fdlastfile_single(fdp);
for (i = 0; i <= lastfile; i++) {
fde = &fdp->fd_ofiles[i];
fp = fde->fde_file;
if (fp != NULL && (fp->f_type == DTYPE_MQUEUE ||
(fde->fde_flags & UF_EXCLOSE))) {
FILEDESC_XLOCK(fdp);
fdfree(fdp, i);
(void) closefp(fdp, i, fp, td, false, false);
FILEDESC_UNLOCK_ASSERT(fdp);
}
}
}
/*
* It is unsafe for set[ug]id processes to be started with file
* descriptors 0..2 closed, as these descriptors are given implicit
* significance in the Standard C library. fdcheckstd() will create a
* descriptor referencing /dev/null for each of stdin, stdout, and
* stderr that is not already open.
*/
int
fdcheckstd(struct thread *td)
{
struct filedesc *fdp;
register_t save;
int i, error, devnull;
fdp = td->td_proc->p_fd;
KASSERT(refcount_load(&fdp->fd_refcnt) == 1,
("the fdtable should not be shared"));
MPASS(fdp->fd_nfiles >= 3);
devnull = -1;
for (i = 0; i <= 2; i++) {
if (fdp->fd_ofiles[i].fde_file != NULL)
continue;
save = td->td_retval[0];
if (devnull != -1) {
error = kern_dup(td, FDDUP_FIXED, 0, devnull, i);
} else {
error = kern_openat(td, AT_FDCWD, "/dev/null",
UIO_SYSSPACE, O_RDWR, 0);
if (error == 0) {
devnull = td->td_retval[0];
KASSERT(devnull == i, ("we didn't get our fd"));
}
}
td->td_retval[0] = save;
if (error != 0)
return (error);
}
return (0);
}
/*
* Internal form of close. Decrement reference count on file structure.
* Note: td may be NULL when closing a file that was being passed in a
* message.
*/
int
closef(struct file *fp, struct thread *td)
{
struct vnode *vp;
struct flock lf;
struct filedesc_to_leader *fdtol;
struct filedesc *fdp;
MPASS(td != NULL);
/*
* POSIX record locking dictates that any close releases ALL
* locks owned by this process. This is handled by setting
* a flag in the unlock to free ONLY locks obeying POSIX
* semantics, and not to free BSD-style file locks.
* If the descriptor was in a message, POSIX-style locks
* aren't passed with the descriptor, and the thread pointer
* will be NULL. Callers should be careful only to pass a
* NULL thread pointer when there really is no owning
* context that might have locks, or the locks will be
* leaked.
*/
if (fp->f_type == DTYPE_VNODE) {
vp = fp->f_vnode;
if ((td->td_proc->p_leader->p_flag & P_ADVLOCK) != 0) {
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
(void) VOP_ADVLOCK(vp, (caddr_t)td->td_proc->p_leader,
F_UNLCK, &lf, F_POSIX);
}
fdtol = td->td_proc->p_fdtol;
if (fdtol != NULL) {
/*
* Handle special case where file descriptor table is
* shared between multiple process leaders.
*/
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
for (fdtol = fdtol->fdl_next;
fdtol != td->td_proc->p_fdtol;
fdtol = fdtol->fdl_next) {
if ((fdtol->fdl_leader->p_flag &
P_ADVLOCK) == 0)
continue;
fdtol->fdl_holdcount++;
FILEDESC_XUNLOCK(fdp);
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
vp = fp->f_vnode;
(void) VOP_ADVLOCK(vp,
(caddr_t)fdtol->fdl_leader, F_UNLCK, &lf,
F_POSIX);
FILEDESC_XLOCK(fdp);
fdtol->fdl_holdcount--;
if (fdtol->fdl_holdcount == 0 &&
fdtol->fdl_wakeup != 0) {
fdtol->fdl_wakeup = 0;
wakeup(fdtol);
}
}
FILEDESC_XUNLOCK(fdp);
}
}
return (fdrop_close(fp, td));
}
/*
* Hack for file descriptor passing code.
*/
void
closef_nothread(struct file *fp)
{
fdrop(fp, NULL);
}
/*
* Initialize the file pointer with the specified properties.
*
* The ops are set with release semantics to be certain that the flags, type,
* and data are visible when ops is. This is to prevent ops methods from being
* called with bad data.
*/
void
finit(struct file *fp, u_int flag, short type, void *data, struct fileops *ops)
{
fp->f_data = data;
fp->f_flag = flag;
fp->f_type = type;
atomic_store_rel_ptr((volatile uintptr_t *)&fp->f_ops, (uintptr_t)ops);
}
void
finit_vnode(struct file *fp, u_int flag, void *data, struct fileops *ops)
{
fp->f_seqcount[UIO_READ] = 1;
fp->f_seqcount[UIO_WRITE] = 1;
finit(fp, (flag & FMASK) | (fp->f_flag & FHASLOCK), DTYPE_VNODE,
data, ops);
}
int
fget_cap_locked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp)
{
struct filedescent *fde;
int error;
FILEDESC_LOCK_ASSERT(fdp);
*fpp = NULL;
fde = fdeget_locked(fdp, fd);
if (fde == NULL) {
error = EBADF;
goto out;
}
#ifdef CAPABILITIES
error = cap_check(cap_rights_fde_inline(fde), needrightsp);
if (error != 0)
goto out;
#endif
if (havecapsp != NULL)
filecaps_copy(&fde->fde_caps, havecapsp, true);
*fpp = fde->fde_file;
error = 0;
out:
return (error);
}
int
fget_cap(struct thread *td, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp)
{
struct filedesc *fdp = td->td_proc->p_fd;
int error;
#ifndef CAPABILITIES
error = fget_unlocked(fdp, fd, needrightsp, fpp);
if (havecapsp != NULL && error == 0)
filecaps_fill(havecapsp);
#else
struct file *fp;
seqc_t seq;
*fpp = NULL;
for (;;) {
error = fget_unlocked_seq(fdp, fd, needrightsp, &fp, &seq);
if (error != 0)
return (error);
if (havecapsp != NULL) {
if (!filecaps_copy(&fdp->fd_ofiles[fd].fde_caps,
havecapsp, false)) {
fdrop(fp, td);
goto get_locked;
}
}
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
*fpp = fp;
return (0);
get_locked:
FILEDESC_SLOCK(fdp);
error = fget_cap_locked(fdp, fd, needrightsp, fpp, havecapsp);
if (error == 0 && !fhold(*fpp))
error = EBADF;
FILEDESC_SUNLOCK(fdp);
#endif
return (error);
}
#ifdef CAPABILITIES
int
fgetvp_lookup_smr(int fd, struct nameidata *ndp, struct vnode **vpp, bool *fsearch)
{
const struct filedescent *fde;
const struct fdescenttbl *fdt;
struct filedesc *fdp;
struct file *fp;
struct vnode *vp;
const cap_rights_t *haverights;
cap_rights_t rights;
seqc_t seq;
VFS_SMR_ASSERT_ENTERED();
rights = *ndp->ni_rightsneeded;
cap_rights_set_one(&rights, CAP_LOOKUP);
fdp = curproc->p_fd;
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles))
return (EBADF);
seq = seqc_read_notmodify(fd_seqc(fdt, fd));
fde = &fdt->fdt_ofiles[fd];
haverights = cap_rights_fde_inline(fde);
fp = fde->fde_file;
if (__predict_false(fp == NULL))
return (EAGAIN);
if (__predict_false(cap_check_inline_transient(haverights, &rights)))
return (EAGAIN);
*fsearch = ((fp->f_flag & FSEARCH) != 0);
vp = fp->f_vnode;
if (__predict_false(vp == NULL)) {
return (EAGAIN);
}
if (!filecaps_copy(&fde->fde_caps, &ndp->ni_filecaps, false)) {
return (EAGAIN);
}
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
if (__predict_false(!seqc_consistent_nomb(fd_seqc(fdt, fd), seq)))
return (EAGAIN);
/*
* If file descriptor doesn't have all rights,
* all lookups relative to it must also be
* strictly relative.
*
* Not yet supported by fast path.
*/
CAP_ALL(&rights);
if (!cap_rights_contains(&ndp->ni_filecaps.fc_rights, &rights) ||
ndp->ni_filecaps.fc_fcntls != CAP_FCNTL_ALL ||
ndp->ni_filecaps.fc_nioctls != -1) {
#ifdef notyet
ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
#else
return (EAGAIN);
#endif
}
*vpp = vp;
return (0);
}
#else
int
fgetvp_lookup_smr(int fd, struct nameidata *ndp, struct vnode **vpp, bool *fsearch)
{
const struct fdescenttbl *fdt;
struct filedesc *fdp;
struct file *fp;
struct vnode *vp;
VFS_SMR_ASSERT_ENTERED();
fdp = curproc->p_fd;
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles))
return (EBADF);
fp = fdt->fdt_ofiles[fd].fde_file;
if (__predict_false(fp == NULL))
return (EAGAIN);
*fsearch = ((fp->f_flag & FSEARCH) != 0);
vp = fp->f_vnode;
if (__predict_false(vp == NULL || vp->v_type != VDIR)) {
return (EAGAIN);
}
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
if (__predict_false(fp != fdt->fdt_ofiles[fd].fde_file))
return (EAGAIN);
filecaps_fill(&ndp->ni_filecaps);
*vpp = vp;
return (0);
}
#endif
+int
+fgetvp_lookup(int fd, struct nameidata *ndp, struct vnode **vpp)
+{
+ struct thread *td;
+ struct file *fp;
+ struct vnode *vp;
+ struct componentname *cnp;
+ cap_rights_t rights;
+ int error;
+
+ td = curthread;
+ rights = *ndp->ni_rightsneeded;
+ cap_rights_set_one(&rights, CAP_LOOKUP);
+ cnp = &ndp->ni_cnd;
+
+ error = fget_cap(td, ndp->ni_dirfd, &rights, &fp, &ndp->ni_filecaps);
+ if (__predict_false(error != 0))
+ return (error);
+ if (__predict_false(fp->f_ops == &badfileops)) {
+ error = EBADF;
+ goto out_free;
+ }
+ vp = fp->f_vnode;
+ if (__predict_false(vp == NULL)) {
+ error = ENOTDIR;
+ goto out_free;
+ }
+ vref(vp);
+ /*
+ * XXX does not check for VDIR, handled by namei_setup
+ */
+ if ((fp->f_flag & FSEARCH) != 0)
+ cnp->cn_flags |= NOEXECCHECK;
+ fdrop(fp, td);
+
+#ifdef CAPABILITIES
+ /*
+ * If file descriptor doesn't have all rights,
+ * all lookups relative to it must also be
+ * strictly relative.
+ */
+ CAP_ALL(&rights);
+ if (!cap_rights_contains(&ndp->ni_filecaps.fc_rights, &rights) ||
+ ndp->ni_filecaps.fc_fcntls != CAP_FCNTL_ALL ||
+ ndp->ni_filecaps.fc_nioctls != -1) {
+ ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
+ ndp->ni_resflags |= NIRES_STRICTREL;
+ }
+#endif
+
+ /*
+ * TODO: avoid copying ioctl caps if it can be helped to begin with
+ */
+ if ((cnp->cn_flags & WANTIOCTLCAPS) == 0)
+ filecaps_free_ioctl(&ndp->ni_filecaps);
+
+ *vpp = vp;
+ return (0);
+
+out_free:
+ filecaps_free(&ndp->ni_filecaps);
+ fdrop(fp, td);
+ return (error);
+}
+
static int
fget_unlocked_seq(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, seqc_t *seqp)
{
#ifdef CAPABILITIES
const struct filedescent *fde;
#endif
const struct fdescenttbl *fdt;
struct file *fp;
#ifdef CAPABILITIES
seqc_t seq;
cap_rights_t haverights;
int error;
#endif
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles))
return (EBADF);
/*
* Fetch the descriptor locklessly. We avoid fdrop() races by
* never raising a refcount above 0. To accomplish this we have
* to use a cmpset loop rather than an atomic_add. The descriptor
* must be re-verified once we acquire a reference to be certain
* that the identity is still correct and we did not lose a race
* due to preemption.
*/
for (;;) {
#ifdef CAPABILITIES
seq = seqc_read_notmodify(fd_seqc(fdt, fd));
fde = &fdt->fdt_ofiles[fd];
haverights = *cap_rights_fde_inline(fde);
fp = fde->fde_file;
if (!seqc_consistent(fd_seqc(fdt, fd), seq))
continue;
#else
fp = fdt->fdt_ofiles[fd].fde_file;
#endif
if (fp == NULL)
return (EBADF);
#ifdef CAPABILITIES
error = cap_check_inline(&haverights, needrightsp);
if (error != 0)
return (error);
#endif
if (__predict_false(!refcount_acquire_if_not_zero(&fp->f_count))) {
/*
* Force a reload. Other thread could reallocate the
* table before this fd was closed, so it is possible
* that there is a stale fp pointer in cached version.
*/
fdt = atomic_load_ptr(&fdp->fd_files);
continue;
}
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
#ifdef CAPABILITIES
if (seqc_consistent_nomb(fd_seqc(fdt, fd), seq))
#else
if (fp == fdt->fdt_ofiles[fd].fde_file)
#endif
break;
fdrop(fp, curthread);
}
*fpp = fp;
if (seqp != NULL) {
#ifdef CAPABILITIES
*seqp = seq;
#endif
}
return (0);
}
/*
* See the comments in fget_unlocked_seq for an explanation of how this works.
*
* This is a simplified variant which bails out to the aforementioned routine
* if anything goes wrong. In practice this only happens when userspace is
* racing with itself.
*/
int
fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp)
{
#ifdef CAPABILITIES
const struct filedescent *fde;
#endif
const struct fdescenttbl *fdt;
struct file *fp;
#ifdef CAPABILITIES
seqc_t seq;
const cap_rights_t *haverights;
#endif
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles)) {
*fpp = NULL;
return (EBADF);
}
#ifdef CAPABILITIES
seq = seqc_read_notmodify(fd_seqc(fdt, fd));
fde = &fdt->fdt_ofiles[fd];
haverights = cap_rights_fde_inline(fde);
fp = fde->fde_file;
#else
fp = fdt->fdt_ofiles[fd].fde_file;
#endif
if (__predict_false(fp == NULL))
goto out_fallback;
#ifdef CAPABILITIES
if (__predict_false(cap_check_inline_transient(haverights, needrightsp)))
goto out_fallback;
#endif
if (__predict_false(!refcount_acquire_if_not_zero(&fp->f_count)))
goto out_fallback;
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
#ifdef CAPABILITIES
if (__predict_false(!seqc_consistent_nomb(fd_seqc(fdt, fd), seq)))
#else
if (__predict_false(fp != fdt->fdt_ofiles[fd].fde_file))
#endif
goto out_fdrop;
*fpp = fp;
return (0);
out_fdrop:
fdrop(fp, curthread);
out_fallback:
*fpp = NULL;
return (fget_unlocked_seq(fdp, fd, needrightsp, fpp, NULL));
}
/*
* Translate fd -> file when the caller guarantees the file descriptor table
* can't be changed by others.
*
* Note this does not mean the file object itself is only visible to the caller,
* merely that it wont disappear without having to be referenced.
*
* Must be paired with fput_only_user.
*/
#ifdef CAPABILITIES
int
fget_only_user(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp)
{
const struct filedescent *fde;
const struct fdescenttbl *fdt;
const cap_rights_t *haverights;
struct file *fp;
int error;
MPASS(FILEDESC_IS_ONLY_USER(fdp));
*fpp = NULL;
if (__predict_false(fd >= fdp->fd_nfiles))
return (EBADF);
fdt = fdp->fd_files;
fde = &fdt->fdt_ofiles[fd];
fp = fde->fde_file;
if (__predict_false(fp == NULL))
return (EBADF);
MPASS(refcount_load(&fp->f_count) > 0);
haverights = cap_rights_fde_inline(fde);
error = cap_check_inline(haverights, needrightsp);
if (__predict_false(error != 0))
return (error);
*fpp = fp;
return (0);
}
#else
int
fget_only_user(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp)
{
struct file *fp;
MPASS(FILEDESC_IS_ONLY_USER(fdp));
*fpp = NULL;
if (__predict_false(fd >= fdp->fd_nfiles))
return (EBADF);
fp = fdp->fd_ofiles[fd].fde_file;
if (__predict_false(fp == NULL))
return (EBADF);
MPASS(refcount_load(&fp->f_count) > 0);
*fpp = fp;
return (0);
}
#endif
/*
* Extract the file pointer associated with the specified descriptor for the
* current user process.
*
* If the descriptor doesn't exist or doesn't match 'flags', EBADF is
* returned.
*
* File's rights will be checked against the capability rights mask.
*
* If an error occurred the non-zero error is returned and *fpp is set to
* NULL. Otherwise *fpp is held and set and zero is returned. Caller is
* responsible for fdrop().
*/
static __inline int
_fget(struct thread *td, int fd, struct file **fpp, int flags,
cap_rights_t *needrightsp)
{
struct filedesc *fdp;
struct file *fp;
int error;
*fpp = NULL;
fdp = td->td_proc->p_fd;
error = fget_unlocked(fdp, fd, needrightsp, &fp);
if (__predict_false(error != 0))
return (error);
if (__predict_false(fp->f_ops == &badfileops)) {
fdrop(fp, td);
return (EBADF);
}
/*
* FREAD and FWRITE failure return EBADF as per POSIX.
*/
error = 0;
switch (flags) {
case FREAD:
case FWRITE:
if ((fp->f_flag & flags) == 0)
error = EBADF;
break;
case FEXEC:
if (fp->f_ops != &path_fileops &&
((fp->f_flag & (FREAD | FEXEC)) == 0 ||
(fp->f_flag & FWRITE) != 0))
error = EBADF;
break;
case 0:
break;
default:
KASSERT(0, ("wrong flags"));
}
if (error != 0) {
fdrop(fp, td);
return (error);
}
*fpp = fp;
return (0);
}
int
fget(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, 0, rightsp));
}
int
fget_mmap(struct thread *td, int fd, cap_rights_t *rightsp, vm_prot_t *maxprotp,
struct file **fpp)
{
int error;
#ifndef CAPABILITIES
error = _fget(td, fd, fpp, 0, rightsp);
if (maxprotp != NULL)
*maxprotp = VM_PROT_ALL;
return (error);
#else
cap_rights_t fdrights;
struct filedesc *fdp;
struct file *fp;
seqc_t seq;
*fpp = NULL;
fdp = td->td_proc->p_fd;
MPASS(cap_rights_is_set(rightsp, CAP_MMAP));
for (;;) {
error = fget_unlocked_seq(fdp, fd, rightsp, &fp, &seq);
if (__predict_false(error != 0))
return (error);
if (__predict_false(fp->f_ops == &badfileops)) {
fdrop(fp, td);
return (EBADF);
}
if (maxprotp != NULL)
fdrights = *cap_rights(fdp, fd);
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
/*
* If requested, convert capability rights to access flags.
*/
if (maxprotp != NULL)
*maxprotp = cap_rights_to_vmprot(&fdrights);
*fpp = fp;
return (0);
#endif
}
int
fget_read(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, FREAD, rightsp));
}
int
fget_write(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, FWRITE, rightsp));
}
int
fget_fcntl(struct thread *td, int fd, cap_rights_t *rightsp, int needfcntl,
struct file **fpp)
{
struct filedesc *fdp = td->td_proc->p_fd;
#ifndef CAPABILITIES
return (fget_unlocked(fdp, fd, rightsp, fpp));
#else
struct file *fp;
int error;
seqc_t seq;
*fpp = NULL;
MPASS(cap_rights_is_set(rightsp, CAP_FCNTL));
for (;;) {
error = fget_unlocked_seq(fdp, fd, rightsp, &fp, &seq);
if (error != 0)
return (error);
error = cap_fcntl_check(fdp, fd, needfcntl);
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
if (error != 0) {
fdrop(fp, td);
return (error);
}
*fpp = fp;
return (0);
#endif
}
/*
* Like fget() but loads the underlying vnode, or returns an error if the
* descriptor does not represent a vnode. Note that pipes use vnodes but
* never have VM objects. The returned vnode will be vref()'d.
*
* XXX: what about the unused flags ?
*/
static __inline int
_fgetvp(struct thread *td, int fd, int flags, cap_rights_t *needrightsp,
struct vnode **vpp)
{
struct file *fp;
int error;
*vpp = NULL;
error = _fget(td, fd, &fp, flags, needrightsp);
if (error != 0)
return (error);
if (fp->f_vnode == NULL) {
error = EINVAL;
} else {
*vpp = fp->f_vnode;
vref(*vpp);
}
fdrop(fp, td);
return (error);
}
int
fgetvp(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, 0, rightsp, vpp));
}
int
fgetvp_rights(struct thread *td, int fd, cap_rights_t *needrightsp,
struct filecaps *havecaps, struct vnode **vpp)
{
struct filecaps caps;
struct file *fp;
int error;
error = fget_cap(td, fd, needrightsp, &fp, &caps);
if (error != 0)
return (error);
if (fp->f_ops == &badfileops) {
error = EBADF;
goto out;
}
if (fp->f_vnode == NULL) {
error = EINVAL;
goto out;
}
*havecaps = caps;
*vpp = fp->f_vnode;
vref(*vpp);
fdrop(fp, td);
return (0);
out:
filecaps_free(&caps);
fdrop(fp, td);
return (error);
}
int
fgetvp_read(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, FREAD, rightsp, vpp));
}
int
fgetvp_exec(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, FEXEC, rightsp, vpp));
}
#ifdef notyet
int
fgetvp_write(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp)
{
return (_fgetvp(td, fd, FWRITE, rightsp, vpp));
}
#endif
/*
* Handle the last reference to a file being closed.
*
* Without the noinline attribute clang keeps inlining the func thorough this
* file when fdrop is used.
*/
int __noinline
_fdrop(struct file *fp, struct thread *td)
{
int error;
#ifdef INVARIANTS
int count;
count = refcount_load(&fp->f_count);
if (count != 0)
panic("fdrop: fp %p count %d", fp, count);
#endif
error = fo_close(fp, td);
atomic_subtract_int(&openfiles, 1);
crfree(fp->f_cred);
free(fp->f_advice, M_FADVISE);
uma_zfree(file_zone, fp);
return (error);
}
/*
* Apply an advisory lock on a file descriptor.
*
* Just attempt to get a record lock of the requested type on the entire file
* (l_whence = SEEK_SET, l_start = 0, l_len = 0).
*/
#ifndef _SYS_SYSPROTO_H_
struct flock_args {
int fd;
int how;
};
#endif
/* ARGSUSED */
int
sys_flock(struct thread *td, struct flock_args *uap)
{
struct file *fp;
struct vnode *vp;
struct flock lf;
int error;
error = fget(td, uap->fd, &cap_flock_rights, &fp);
if (error != 0)
return (error);
error = EOPNOTSUPP;
if (fp->f_type != DTYPE_VNODE && fp->f_type != DTYPE_FIFO) {
goto done;
}
if (fp->f_ops == &path_fileops) {
goto done;
}
error = 0;
vp = fp->f_vnode;
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
if (uap->how & LOCK_UN) {
lf.l_type = F_UNLCK;
atomic_clear_int(&fp->f_flag, FHASLOCK);
error = VOP_ADVLOCK(vp, (caddr_t)fp, F_UNLCK, &lf, F_FLOCK);
goto done;
}
if (uap->how & LOCK_EX)
lf.l_type = F_WRLCK;
else if (uap->how & LOCK_SH)
lf.l_type = F_RDLCK;
else {
error = EBADF;
goto done;
}
atomic_set_int(&fp->f_flag, FHASLOCK);
error = VOP_ADVLOCK(vp, (caddr_t)fp, F_SETLK, &lf,
(uap->how & LOCK_NB) ? F_FLOCK : F_FLOCK | F_WAIT);
done:
fdrop(fp, td);
return (error);
}
/*
* Duplicate the specified descriptor to a free descriptor.
*/
int
dupfdopen(struct thread *td, struct filedesc *fdp, int dfd, int mode,
int openerror, int *indxp)
{
struct filedescent *newfde, *oldfde;
struct file *fp;
u_long *ioctls;
int error, indx;
KASSERT(openerror == ENODEV || openerror == ENXIO,
("unexpected error %d in %s", openerror, __func__));
/*
* If the to-be-dup'd fd number is greater than the allowed number
* of file descriptors, or the fd to be dup'd has already been
* closed, then reject.
*/
FILEDESC_XLOCK(fdp);
if ((fp = fget_locked(fdp, dfd)) == NULL) {
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
error = fdalloc(td, 0, &indx);
if (error != 0) {
FILEDESC_XUNLOCK(fdp);
return (error);
}
/*
* There are two cases of interest here.
*
* For ENODEV simply dup (dfd) to file descriptor (indx) and return.
*
* For ENXIO steal away the file structure from (dfd) and store it in
* (indx). (dfd) is effectively closed by this operation.
*/
switch (openerror) {
case ENODEV:
/*
* Check that the mode the file is being opened for is a
* subset of the mode of the existing descriptor.
*/
if (((mode & (FREAD|FWRITE)) | fp->f_flag) != fp->f_flag) {
fdunused(fdp, indx);
FILEDESC_XUNLOCK(fdp);
return (EACCES);
}
if (!fhold(fp)) {
fdunused(fdp, indx);
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
newfde = &fdp->fd_ofiles[indx];
oldfde = &fdp->fd_ofiles[dfd];
ioctls = filecaps_copy_prep(&oldfde->fde_caps);
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
memcpy(newfde, oldfde, fde_change_size);
filecaps_copy_finish(&oldfde->fde_caps, &newfde->fde_caps,
ioctls);
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
break;
case ENXIO:
/*
* Steal away the file pointer from dfd and stuff it into indx.
*/
newfde = &fdp->fd_ofiles[indx];
oldfde = &fdp->fd_ofiles[dfd];
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
memcpy(newfde, oldfde, fde_change_size);
oldfde->fde_file = NULL;
fdunused(fdp, dfd);
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
break;
}
FILEDESC_XUNLOCK(fdp);
*indxp = indx;
return (0);
}
/*
* This sysctl determines if we will allow a process to chroot(2) if it
* has a directory open:
* 0: disallowed for all processes.
* 1: allowed for processes that were not already chroot(2)'ed.
* 2: allowed for all processes.
*/
static int chroot_allow_open_directories = 1;
SYSCTL_INT(_kern, OID_AUTO, chroot_allow_open_directories, CTLFLAG_RW,
&chroot_allow_open_directories, 0,
"Allow a process to chroot(2) if it has a directory open");
/*
* Helper function for raised chroot(2) security function: Refuse if
* any filedescriptors are open directories.
*/
static int
chroot_refuse_vdir_fds(struct filedesc *fdp)
{
struct vnode *vp;
struct file *fp;
int fd, lastfile;
FILEDESC_LOCK_ASSERT(fdp);
lastfile = fdlastfile(fdp);
for (fd = 0; fd <= lastfile; fd++) {
fp = fget_locked(fdp, fd);
if (fp == NULL)
continue;
if (fp->f_type == DTYPE_VNODE) {
vp = fp->f_vnode;
if (vp->v_type == VDIR)
return (EPERM);
}
}
return (0);
}
static void
pwd_fill(struct pwd *oldpwd, struct pwd *newpwd)
{
if (newpwd->pwd_cdir == NULL && oldpwd->pwd_cdir != NULL) {
vrefact(oldpwd->pwd_cdir);
newpwd->pwd_cdir = oldpwd->pwd_cdir;
}
if (newpwd->pwd_rdir == NULL && oldpwd->pwd_rdir != NULL) {
vrefact(oldpwd->pwd_rdir);
newpwd->pwd_rdir = oldpwd->pwd_rdir;
}
if (newpwd->pwd_jdir == NULL && oldpwd->pwd_jdir != NULL) {
vrefact(oldpwd->pwd_jdir);
newpwd->pwd_jdir = oldpwd->pwd_jdir;
}
}
struct pwd *
pwd_hold_pwddesc(struct pwddesc *pdp)
{
struct pwd *pwd;
PWDDESC_ASSERT_XLOCKED(pdp);
pwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
if (pwd != NULL)
refcount_acquire(&pwd->pwd_refcount);
return (pwd);
}
bool
pwd_hold_smr(struct pwd *pwd)
{
MPASS(pwd != NULL);
if (__predict_true(refcount_acquire_if_not_zero(&pwd->pwd_refcount))) {
return (true);
}
return (false);
}
struct pwd *
pwd_hold(struct thread *td)
{
struct pwddesc *pdp;
struct pwd *pwd;
pdp = td->td_proc->p_pd;
vfs_smr_enter();
pwd = vfs_smr_entered_load(&pdp->pd_pwd);
if (pwd_hold_smr(pwd)) {
vfs_smr_exit();
return (pwd);
}
vfs_smr_exit();
PWDDESC_XLOCK(pdp);
pwd = pwd_hold_pwddesc(pdp);
MPASS(pwd != NULL);
PWDDESC_XUNLOCK(pdp);
return (pwd);
}
struct pwd *
pwd_hold_proc(struct proc *p)
{
struct pwddesc *pdp;
struct pwd *pwd;
PROC_ASSERT_HELD(p);
PROC_LOCK(p);
pdp = pdhold(p);
MPASS(pdp != NULL);
PROC_UNLOCK(p);
PWDDESC_XLOCK(pdp);
pwd = pwd_hold_pwddesc(pdp);
MPASS(pwd != NULL);
PWDDESC_XUNLOCK(pdp);
pddrop(pdp);
return (pwd);
}
static struct pwd *
pwd_alloc(void)
{
struct pwd *pwd;
pwd = uma_zalloc_smr(pwd_zone, M_WAITOK);
bzero(pwd, sizeof(*pwd));
refcount_init(&pwd->pwd_refcount, 1);
return (pwd);
}
void
pwd_drop(struct pwd *pwd)
{
if (!refcount_release(&pwd->pwd_refcount))
return;
if (pwd->pwd_cdir != NULL)
vrele(pwd->pwd_cdir);
if (pwd->pwd_rdir != NULL)
vrele(pwd->pwd_rdir);
if (pwd->pwd_jdir != NULL)
vrele(pwd->pwd_jdir);
uma_zfree_smr(pwd_zone, pwd);
}
/*
* The caller is responsible for invoking priv_check() and
* mac_vnode_check_chroot() to authorize this operation.
*/
int
pwd_chroot(struct thread *td, struct vnode *vp)
{
struct pwddesc *pdp;
struct filedesc *fdp;
struct pwd *newpwd, *oldpwd;
int error;
fdp = td->td_proc->p_fd;
pdp = td->td_proc->p_pd;
newpwd = pwd_alloc();
FILEDESC_SLOCK(fdp);
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
if (chroot_allow_open_directories == 0 ||
(chroot_allow_open_directories == 1 &&
oldpwd->pwd_rdir != rootvnode)) {
error = chroot_refuse_vdir_fds(fdp);
FILEDESC_SUNLOCK(fdp);
if (error != 0) {
PWDDESC_XUNLOCK(pdp);
pwd_drop(newpwd);
return (error);
}
} else {
FILEDESC_SUNLOCK(fdp);
}
vrefact(vp);
newpwd->pwd_rdir = vp;
if (oldpwd->pwd_jdir == NULL) {
vrefact(vp);
newpwd->pwd_jdir = vp;
}
pwd_fill(oldpwd, newpwd);
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
return (0);
}
void
pwd_chdir(struct thread *td, struct vnode *vp)
{
struct pwddesc *pdp;
struct pwd *newpwd, *oldpwd;
VNPASS(vp->v_usecount > 0, vp);
newpwd = pwd_alloc();
pdp = td->td_proc->p_pd;
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
newpwd->pwd_cdir = vp;
pwd_fill(oldpwd, newpwd);
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
}
/*
* jail_attach(2) changes both root and working directories.
*/
int
pwd_chroot_chdir(struct thread *td, struct vnode *vp)
{
struct pwddesc *pdp;
struct filedesc *fdp;
struct pwd *newpwd, *oldpwd;
int error;
fdp = td->td_proc->p_fd;
pdp = td->td_proc->p_pd;
newpwd = pwd_alloc();
FILEDESC_SLOCK(fdp);
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
error = chroot_refuse_vdir_fds(fdp);
FILEDESC_SUNLOCK(fdp);
if (error != 0) {
PWDDESC_XUNLOCK(pdp);
pwd_drop(newpwd);
return (error);
}
vrefact(vp);
newpwd->pwd_rdir = vp;
vrefact(vp);
newpwd->pwd_cdir = vp;
if (oldpwd->pwd_jdir == NULL) {
vrefact(vp);
newpwd->pwd_jdir = vp;
}
pwd_fill(oldpwd, newpwd);
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
return (0);
}
void
pwd_ensure_dirs(void)
{
struct pwddesc *pdp;
struct pwd *oldpwd, *newpwd;
pdp = curproc->p_pd;
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
if (oldpwd->pwd_cdir != NULL && oldpwd->pwd_rdir != NULL) {
PWDDESC_XUNLOCK(pdp);
return;
}
PWDDESC_XUNLOCK(pdp);
newpwd = pwd_alloc();
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
pwd_fill(oldpwd, newpwd);
if (newpwd->pwd_cdir == NULL) {
vrefact(rootvnode);
newpwd->pwd_cdir = rootvnode;
}
if (newpwd->pwd_rdir == NULL) {
vrefact(rootvnode);
newpwd->pwd_rdir = rootvnode;
}
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
}
void
pwd_set_rootvnode(void)
{
struct pwddesc *pdp;
struct pwd *oldpwd, *newpwd;
pdp = curproc->p_pd;
newpwd = pwd_alloc();
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
vrefact(rootvnode);
newpwd->pwd_cdir = rootvnode;
vrefact(rootvnode);
newpwd->pwd_rdir = rootvnode;
pwd_fill(oldpwd, newpwd);
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
}
/*
* Scan all active processes and prisons to see if any of them have a current
* or root directory of `olddp'. If so, replace them with the new mount point.
*/
void
mountcheckdirs(struct vnode *olddp, struct vnode *newdp)
{
struct pwddesc *pdp;
struct pwd *newpwd, *oldpwd;
struct prison *pr;
struct proc *p;
int nrele;
if (vrefcnt(olddp) == 1)
return;
nrele = 0;
newpwd = pwd_alloc();
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
pdp = pdhold(p);
PROC_UNLOCK(p);
if (pdp == NULL)
continue;
PWDDESC_XLOCK(pdp);
oldpwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
if (oldpwd == NULL ||
(oldpwd->pwd_cdir != olddp &&
oldpwd->pwd_rdir != olddp &&
oldpwd->pwd_jdir != olddp)) {
PWDDESC_XUNLOCK(pdp);
pddrop(pdp);
continue;
}
if (oldpwd->pwd_cdir == olddp) {
vrefact(newdp);
newpwd->pwd_cdir = newdp;
}
if (oldpwd->pwd_rdir == olddp) {
vrefact(newdp);
newpwd->pwd_rdir = newdp;
}
if (oldpwd->pwd_jdir == olddp) {
vrefact(newdp);
newpwd->pwd_jdir = newdp;
}
pwd_fill(oldpwd, newpwd);
pwd_set(pdp, newpwd);
PWDDESC_XUNLOCK(pdp);
pwd_drop(oldpwd);
pddrop(pdp);
newpwd = pwd_alloc();
}
sx_sunlock(&allproc_lock);
pwd_drop(newpwd);
if (rootvnode == olddp) {
vrefact(newdp);
rootvnode = newdp;
nrele++;
}
mtx_lock(&prison0.pr_mtx);
if (prison0.pr_root == olddp) {
vrefact(newdp);
prison0.pr_root = newdp;
nrele++;
}
mtx_unlock(&prison0.pr_mtx);
sx_slock(&allprison_lock);
TAILQ_FOREACH(pr, &allprison, pr_list) {
mtx_lock(&pr->pr_mtx);
if (pr->pr_root == olddp) {
vrefact(newdp);
pr->pr_root = newdp;
nrele++;
}
mtx_unlock(&pr->pr_mtx);
}
sx_sunlock(&allprison_lock);
while (nrele--)
vrele(olddp);
}
struct filedesc_to_leader *
filedesc_to_leader_alloc(struct filedesc_to_leader *old, struct filedesc *fdp, struct proc *leader)
{
struct filedesc_to_leader *fdtol;
fdtol = malloc(sizeof(struct filedesc_to_leader),
M_FILEDESC_TO_LEADER, M_WAITOK);
fdtol->fdl_refcount = 1;
fdtol->fdl_holdcount = 0;
fdtol->fdl_wakeup = 0;
fdtol->fdl_leader = leader;
if (old != NULL) {
FILEDESC_XLOCK(fdp);
fdtol->fdl_next = old->fdl_next;
fdtol->fdl_prev = old;
old->fdl_next = fdtol;
fdtol->fdl_next->fdl_prev = fdtol;
FILEDESC_XUNLOCK(fdp);
} else {
fdtol->fdl_next = fdtol;
fdtol->fdl_prev = fdtol;
}
return (fdtol);
}
static int
sysctl_kern_proc_nfds(SYSCTL_HANDLER_ARGS)
{
NDSLOTTYPE *map;
struct filedesc *fdp;
u_int namelen;
int count, off, minoff;
namelen = arg2;
if (namelen != 1)
return (EINVAL);
if (*(int *)arg1 != 0)
return (EINVAL);
fdp = curproc->p_fd;
count = 0;
FILEDESC_SLOCK(fdp);
map = fdp->fd_map;
off = NDSLOT(fdp->fd_nfiles - 1);
for (minoff = NDSLOT(0); off >= minoff; --off)
count += bitcountl(map[off]);
FILEDESC_SUNLOCK(fdp);
return (SYSCTL_OUT(req, &count, sizeof(count)));
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_NFDS, nfds,
CTLFLAG_RD|CTLFLAG_CAPRD|CTLFLAG_MPSAFE, sysctl_kern_proc_nfds,
"Number of open file descriptors");
/*
* Get file structures globally.
*/
static int
sysctl_kern_file(SYSCTL_HANDLER_ARGS)
{
struct xfile xf;
struct filedesc *fdp;
struct file *fp;
struct proc *p;
int error, n, lastfile;
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
if (req->oldptr == NULL) {
n = 0;
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
continue;
/* overestimates sparse tables. */
n += fdp->fd_nfiles;
fddrop(fdp);
}
sx_sunlock(&allproc_lock);
return (SYSCTL_OUT(req, 0, n * sizeof(xf)));
}
error = 0;
bzero(&xf, sizeof(xf));
xf.xf_size = sizeof(xf);
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
if (p_cansee(req->td, p) != 0) {
PROC_UNLOCK(p);
continue;
}
xf.xf_pid = p->p_pid;
xf.xf_uid = p->p_ucred->cr_uid;
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
continue;
FILEDESC_SLOCK(fdp);
lastfile = fdlastfile(fdp);
for (n = 0; refcount_load(&fdp->fd_refcnt) > 0 && n <= lastfile;
n++) {
if ((fp = fdp->fd_ofiles[n].fde_file) == NULL)
continue;
xf.xf_fd = n;
xf.xf_file = (uintptr_t)fp;
xf.xf_data = (uintptr_t)fp->f_data;
xf.xf_vnode = (uintptr_t)fp->f_vnode;
xf.xf_type = (uintptr_t)fp->f_type;
xf.xf_count = refcount_load(&fp->f_count);
xf.xf_msgcount = 0;
xf.xf_offset = foffset_get(fp);
xf.xf_flag = fp->f_flag;
error = SYSCTL_OUT(req, &xf, sizeof(xf));
if (error)
break;
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
if (error)
break;
}
sx_sunlock(&allproc_lock);
return (error);
}
SYSCTL_PROC(_kern, KERN_FILE, file, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE,
0, 0, sysctl_kern_file, "S,xfile", "Entire file table");
#ifdef KINFO_FILE_SIZE
CTASSERT(sizeof(struct kinfo_file) == KINFO_FILE_SIZE);
#endif
static int
xlate_fflags(int fflags)
{
static const struct {
int fflag;
int kf_fflag;
} fflags_table[] = {
{ FAPPEND, KF_FLAG_APPEND },
{ FASYNC, KF_FLAG_ASYNC },
{ FFSYNC, KF_FLAG_FSYNC },
{ FHASLOCK, KF_FLAG_HASLOCK },
{ FNONBLOCK, KF_FLAG_NONBLOCK },
{ FREAD, KF_FLAG_READ },
{ FWRITE, KF_FLAG_WRITE },
{ O_CREAT, KF_FLAG_CREAT },
{ O_DIRECT, KF_FLAG_DIRECT },
{ O_EXCL, KF_FLAG_EXCL },
{ O_EXEC, KF_FLAG_EXEC },
{ O_EXLOCK, KF_FLAG_EXLOCK },
{ O_NOFOLLOW, KF_FLAG_NOFOLLOW },
{ O_SHLOCK, KF_FLAG_SHLOCK },
{ O_TRUNC, KF_FLAG_TRUNC }
};
unsigned int i;
int kflags;
kflags = 0;
for (i = 0; i < nitems(fflags_table); i++)
if (fflags & fflags_table[i].fflag)
kflags |= fflags_table[i].kf_fflag;
return (kflags);
}
/* Trim unused data from kf_path by truncating the structure size. */
void
pack_kinfo(struct kinfo_file *kif)
{
kif->kf_structsize = offsetof(struct kinfo_file, kf_path) +
strlen(kif->kf_path) + 1;
kif->kf_structsize = roundup(kif->kf_structsize, sizeof(uint64_t));
}
static void
export_file_to_kinfo(struct file *fp, int fd, cap_rights_t *rightsp,
struct kinfo_file *kif, struct filedesc *fdp, int flags)
{
int error;
bzero(kif, sizeof(*kif));
/* Set a default type to allow for empty fill_kinfo() methods. */
kif->kf_type = KF_TYPE_UNKNOWN;
kif->kf_flags = xlate_fflags(fp->f_flag);
if (rightsp != NULL)
kif->kf_cap_rights = *rightsp;
else
cap_rights_init_zero(&kif->kf_cap_rights);
kif->kf_fd = fd;
kif->kf_ref_count = refcount_load(&fp->f_count);
kif->kf_offset = foffset_get(fp);
/*
* This may drop the filedesc lock, so the 'fp' cannot be
* accessed after this call.
*/
error = fo_fill_kinfo(fp, kif, fdp);
if (error == 0)
kif->kf_status |= KF_ATTR_VALID;
if ((flags & KERN_FILEDESC_PACK_KINFO) != 0)
pack_kinfo(kif);
else
kif->kf_structsize = roundup2(sizeof(*kif), sizeof(uint64_t));
}
static void
export_vnode_to_kinfo(struct vnode *vp, int fd, int fflags,
struct kinfo_file *kif, int flags)
{
int error;
bzero(kif, sizeof(*kif));
kif->kf_type = KF_TYPE_VNODE;
error = vn_fill_kinfo_vnode(vp, kif);
if (error == 0)
kif->kf_status |= KF_ATTR_VALID;
kif->kf_flags = xlate_fflags(fflags);
cap_rights_init_zero(&kif->kf_cap_rights);
kif->kf_fd = fd;
kif->kf_ref_count = -1;
kif->kf_offset = -1;
if ((flags & KERN_FILEDESC_PACK_KINFO) != 0)
pack_kinfo(kif);
else
kif->kf_structsize = roundup2(sizeof(*kif), sizeof(uint64_t));
vrele(vp);
}
struct export_fd_buf {
struct filedesc *fdp;
struct pwddesc *pdp;
struct sbuf *sb;
ssize_t remainder;
struct kinfo_file kif;
int flags;
};
static int
export_kinfo_to_sb(struct export_fd_buf *efbuf)
{
struct kinfo_file *kif;
kif = &efbuf->kif;
if (efbuf->remainder != -1) {
if (efbuf->remainder < kif->kf_structsize)
return (ENOMEM);
efbuf->remainder -= kif->kf_structsize;
}
if (sbuf_bcat(efbuf->sb, kif, kif->kf_structsize) != 0)
return (sbuf_error(efbuf->sb));
return (0);
}
static int
export_file_to_sb(struct file *fp, int fd, cap_rights_t *rightsp,
struct export_fd_buf *efbuf)
{
int error;
if (efbuf->remainder == 0)
return (ENOMEM);
export_file_to_kinfo(fp, fd, rightsp, &efbuf->kif, efbuf->fdp,
efbuf->flags);
FILEDESC_SUNLOCK(efbuf->fdp);
error = export_kinfo_to_sb(efbuf);
FILEDESC_SLOCK(efbuf->fdp);
return (error);
}
static int
export_vnode_to_sb(struct vnode *vp, int fd, int fflags,
struct export_fd_buf *efbuf)
{
int error;
if (efbuf->remainder == 0)
return (ENOMEM);
if (efbuf->pdp != NULL)
PWDDESC_XUNLOCK(efbuf->pdp);
export_vnode_to_kinfo(vp, fd, fflags, &efbuf->kif, efbuf->flags);
error = export_kinfo_to_sb(efbuf);
if (efbuf->pdp != NULL)
PWDDESC_XLOCK(efbuf->pdp);
return (error);
}
/*
* Store a process file descriptor information to sbuf.
*
* Takes a locked proc as argument, and returns with the proc unlocked.
*/
int
kern_proc_filedesc_out(struct proc *p, struct sbuf *sb, ssize_t maxlen,
int flags)
{
struct file *fp;
struct filedesc *fdp;
struct pwddesc *pdp;
struct export_fd_buf *efbuf;
struct vnode *cttyvp, *textvp, *tracevp;
struct pwd *pwd;
int error, i, lastfile;
cap_rights_t rights;
PROC_LOCK_ASSERT(p, MA_OWNED);
/* ktrace vnode */
tracevp = ktr_get_tracevp(p, true);
/* text vnode */
textvp = p->p_textvp;
if (textvp != NULL)
vrefact(textvp);
/* Controlling tty. */
cttyvp = NULL;
if (p->p_pgrp != NULL && p->p_pgrp->pg_session != NULL) {
cttyvp = p->p_pgrp->pg_session->s_ttyvp;
if (cttyvp != NULL)
vrefact(cttyvp);
}
fdp = fdhold(p);
pdp = pdhold(p);
PROC_UNLOCK(p);
efbuf = malloc(sizeof(*efbuf), M_TEMP, M_WAITOK);
efbuf->fdp = NULL;
efbuf->pdp = NULL;
efbuf->sb = sb;
efbuf->remainder = maxlen;
efbuf->flags = flags;
error = 0;
if (tracevp != NULL)
error = export_vnode_to_sb(tracevp, KF_FD_TYPE_TRACE,
FREAD | FWRITE, efbuf);
if (error == 0 && textvp != NULL)
error = export_vnode_to_sb(textvp, KF_FD_TYPE_TEXT, FREAD,
efbuf);
if (error == 0 && cttyvp != NULL)
error = export_vnode_to_sb(cttyvp, KF_FD_TYPE_CTTY,
FREAD | FWRITE, efbuf);
if (error != 0 || pdp == NULL || fdp == NULL)
goto fail;
efbuf->fdp = fdp;
efbuf->pdp = pdp;
PWDDESC_XLOCK(pdp);
pwd = pwd_hold_pwddesc(pdp);
if (pwd != NULL) {
/* working directory */
if (pwd->pwd_cdir != NULL) {
vrefact(pwd->pwd_cdir);
error = export_vnode_to_sb(pwd->pwd_cdir,
KF_FD_TYPE_CWD, FREAD, efbuf);
}
/* root directory */
if (error == 0 && pwd->pwd_rdir != NULL) {
vrefact(pwd->pwd_rdir);
error = export_vnode_to_sb(pwd->pwd_rdir,
KF_FD_TYPE_ROOT, FREAD, efbuf);
}
/* jail directory */
if (error == 0 && pwd->pwd_jdir != NULL) {
vrefact(pwd->pwd_jdir);
error = export_vnode_to_sb(pwd->pwd_jdir,
KF_FD_TYPE_JAIL, FREAD, efbuf);
}
}
PWDDESC_XUNLOCK(pdp);
if (error != 0)
goto fail;
if (pwd != NULL)
pwd_drop(pwd);
FILEDESC_SLOCK(fdp);
lastfile = fdlastfile(fdp);
for (i = 0; refcount_load(&fdp->fd_refcnt) > 0 && i <= lastfile; i++) {
if ((fp = fdp->fd_ofiles[i].fde_file) == NULL)
continue;
#ifdef CAPABILITIES
rights = *cap_rights(fdp, i);
#else /* !CAPABILITIES */
rights = cap_no_rights;
#endif
/*
* Create sysctl entry. It is OK to drop the filedesc
* lock inside of export_file_to_sb() as we will
* re-validate and re-evaluate its properties when the
* loop continues.
*/
error = export_file_to_sb(fp, i, &rights, efbuf);
if (error != 0)
break;
}
FILEDESC_SUNLOCK(fdp);
fail:
if (fdp != NULL)
fddrop(fdp);
if (pdp != NULL)
pddrop(pdp);
free(efbuf, M_TEMP);
return (error);
}
#define FILEDESC_SBUF_SIZE (sizeof(struct kinfo_file) * 5)
/*
* Get per-process file descriptors for use by procstat(1), et al.
*/
static int
sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
{
struct sbuf sb;
struct proc *p;
ssize_t maxlen;
u_int namelen;
int error, error2, *name;
namelen = arg2;
if (namelen != 1)
return (EINVAL);
name = (int *)arg1;
sbuf_new_for_sysctl(&sb, NULL, FILEDESC_SBUF_SIZE, req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0) {
sbuf_delete(&sb);
return (error);
}
maxlen = req->oldptr != NULL ? req->oldlen : -1;
error = kern_proc_filedesc_out(p, &sb, maxlen,
KERN_FILEDESC_PACK_KINFO);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
#ifdef COMPAT_FREEBSD7
#ifdef KINFO_OFILE_SIZE
CTASSERT(sizeof(struct kinfo_ofile) == KINFO_OFILE_SIZE);
#endif
static void
kinfo_to_okinfo(struct kinfo_file *kif, struct kinfo_ofile *okif)
{
okif->kf_structsize = sizeof(*okif);
okif->kf_type = kif->kf_type;
okif->kf_fd = kif->kf_fd;
okif->kf_ref_count = kif->kf_ref_count;
okif->kf_flags = kif->kf_flags & (KF_FLAG_READ | KF_FLAG_WRITE |
KF_FLAG_APPEND | KF_FLAG_ASYNC | KF_FLAG_FSYNC | KF_FLAG_NONBLOCK |
KF_FLAG_DIRECT | KF_FLAG_HASLOCK);
okif->kf_offset = kif->kf_offset;
if (kif->kf_type == KF_TYPE_VNODE)
okif->kf_vnode_type = kif->kf_un.kf_file.kf_file_type;
else
okif->kf_vnode_type = KF_VTYPE_VNON;
strlcpy(okif->kf_path, kif->kf_path, sizeof(okif->kf_path));
if (kif->kf_type == KF_TYPE_SOCKET) {
okif->kf_sock_domain = kif->kf_un.kf_sock.kf_sock_domain0;
okif->kf_sock_type = kif->kf_un.kf_sock.kf_sock_type0;
okif->kf_sock_protocol = kif->kf_un.kf_sock.kf_sock_protocol0;
okif->kf_sa_local = kif->kf_un.kf_sock.kf_sa_local;
okif->kf_sa_peer = kif->kf_un.kf_sock.kf_sa_peer;
} else {
okif->kf_sa_local.ss_family = AF_UNSPEC;
okif->kf_sa_peer.ss_family = AF_UNSPEC;
}
}
static int
export_vnode_for_osysctl(struct vnode *vp, int type, struct kinfo_file *kif,
struct kinfo_ofile *okif, struct pwddesc *pdp, struct sysctl_req *req)
{
int error;
vrefact(vp);
PWDDESC_XUNLOCK(pdp);
export_vnode_to_kinfo(vp, type, 0, kif, KERN_FILEDESC_PACK_KINFO);
kinfo_to_okinfo(kif, okif);
error = SYSCTL_OUT(req, okif, sizeof(*okif));
PWDDESC_XLOCK(pdp);
return (error);
}
/*
* Get per-process file descriptors for use by procstat(1), et al.
*/
static int
sysctl_kern_proc_ofiledesc(SYSCTL_HANDLER_ARGS)
{
struct kinfo_ofile *okif;
struct kinfo_file *kif;
struct filedesc *fdp;
struct pwddesc *pdp;
struct pwd *pwd;
u_int namelen;
int error, i, lastfile, *name;
struct file *fp;
struct proc *p;
namelen = arg2;
if (namelen != 1)
return (EINVAL);
name = (int *)arg1;
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0)
return (error);
fdp = fdhold(p);
if (fdp != NULL)
pdp = pdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL || pdp == NULL) {
if (fdp != NULL)
fddrop(fdp);
return (ENOENT);
}
kif = malloc(sizeof(*kif), M_TEMP, M_WAITOK);
okif = malloc(sizeof(*okif), M_TEMP, M_WAITOK);
PWDDESC_XLOCK(pdp);
pwd = pwd_hold_pwddesc(pdp);
if (pwd != NULL) {
if (pwd->pwd_cdir != NULL)
export_vnode_for_osysctl(pwd->pwd_cdir, KF_FD_TYPE_CWD, kif,
okif, pdp, req);
if (pwd->pwd_rdir != NULL)
export_vnode_for_osysctl(pwd->pwd_rdir, KF_FD_TYPE_ROOT, kif,
okif, pdp, req);
if (pwd->pwd_jdir != NULL)
export_vnode_for_osysctl(pwd->pwd_jdir, KF_FD_TYPE_JAIL, kif,
okif, pdp, req);
}
PWDDESC_XUNLOCK(pdp);
if (pwd != NULL)
pwd_drop(pwd);
FILEDESC_SLOCK(fdp);
lastfile = fdlastfile(fdp);
for (i = 0; refcount_load(&fdp->fd_refcnt) > 0 && i <= lastfile; i++) {
if ((fp = fdp->fd_ofiles[i].fde_file) == NULL)
continue;
export_file_to_kinfo(fp, i, NULL, kif, fdp,
KERN_FILEDESC_PACK_KINFO);
FILEDESC_SUNLOCK(fdp);
kinfo_to_okinfo(kif, okif);
error = SYSCTL_OUT(req, okif, sizeof(*okif));
FILEDESC_SLOCK(fdp);
if (error)
break;
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
pddrop(pdp);
free(kif, M_TEMP);
free(okif, M_TEMP);
return (0);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_OFILEDESC, ofiledesc,
CTLFLAG_RD|CTLFLAG_MPSAFE, sysctl_kern_proc_ofiledesc,
"Process ofiledesc entries");
#endif /* COMPAT_FREEBSD7 */
int
vntype_to_kinfo(int vtype)
{
struct {
int vtype;
int kf_vtype;
} vtypes_table[] = {
{ VBAD, KF_VTYPE_VBAD },
{ VBLK, KF_VTYPE_VBLK },
{ VCHR, KF_VTYPE_VCHR },
{ VDIR, KF_VTYPE_VDIR },
{ VFIFO, KF_VTYPE_VFIFO },
{ VLNK, KF_VTYPE_VLNK },
{ VNON, KF_VTYPE_VNON },
{ VREG, KF_VTYPE_VREG },
{ VSOCK, KF_VTYPE_VSOCK }
};
unsigned int i;
/*
* Perform vtype translation.
*/
for (i = 0; i < nitems(vtypes_table); i++)
if (vtypes_table[i].vtype == vtype)
return (vtypes_table[i].kf_vtype);
return (KF_VTYPE_UNKNOWN);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_FILEDESC, filedesc,
CTLFLAG_RD|CTLFLAG_MPSAFE, sysctl_kern_proc_filedesc,
"Process filedesc entries");
/*
* Store a process current working directory information to sbuf.
*
* Takes a locked proc as argument, and returns with the proc unlocked.
*/
int
kern_proc_cwd_out(struct proc *p, struct sbuf *sb, ssize_t maxlen)
{
struct pwddesc *pdp;
struct pwd *pwd;
struct export_fd_buf *efbuf;
struct vnode *cdir;
int error;
PROC_LOCK_ASSERT(p, MA_OWNED);
pdp = pdhold(p);
PROC_UNLOCK(p);
if (pdp == NULL)
return (EINVAL);
efbuf = malloc(sizeof(*efbuf), M_TEMP, M_WAITOK);
efbuf->fdp = NULL;
efbuf->pdp = pdp;
efbuf->sb = sb;
efbuf->remainder = maxlen;
efbuf->flags = 0;
PWDDESC_XLOCK(pdp);
pwd = PWDDESC_XLOCKED_LOAD_PWD(pdp);
cdir = pwd->pwd_cdir;
if (cdir == NULL) {
error = EINVAL;
} else {
vrefact(cdir);
error = export_vnode_to_sb(cdir, KF_FD_TYPE_CWD, FREAD, efbuf);
}
PWDDESC_XUNLOCK(pdp);
pddrop(pdp);
free(efbuf, M_TEMP);
return (error);
}
/*
* Get per-process current working directory.
*/
static int
sysctl_kern_proc_cwd(SYSCTL_HANDLER_ARGS)
{
struct sbuf sb;
struct proc *p;
ssize_t maxlen;
u_int namelen;
int error, error2, *name;
namelen = arg2;
if (namelen != 1)
return (EINVAL);
name = (int *)arg1;
sbuf_new_for_sysctl(&sb, NULL, sizeof(struct kinfo_file), req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0) {
sbuf_delete(&sb);
return (error);
}
maxlen = req->oldptr != NULL ? req->oldlen : -1;
error = kern_proc_cwd_out(p, &sb, maxlen);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_CWD, cwd, CTLFLAG_RD|CTLFLAG_MPSAFE,
sysctl_kern_proc_cwd, "Process current working directory");
#ifdef DDB
/*
* For the purposes of debugging, generate a human-readable string for the
* file type.
*/
static const char *
file_type_to_name(short type)
{
switch (type) {
case 0:
return ("zero");
case DTYPE_VNODE:
return ("vnode");
case DTYPE_SOCKET:
return ("socket");
case DTYPE_PIPE:
return ("pipe");
case DTYPE_FIFO:
return ("fifo");
case DTYPE_KQUEUE:
return ("kqueue");
case DTYPE_CRYPTO:
return ("crypto");
case DTYPE_MQUEUE:
return ("mqueue");
case DTYPE_SHM:
return ("shm");
case DTYPE_SEM:
return ("ksem");
case DTYPE_PTS:
return ("pts");
case DTYPE_DEV:
return ("dev");
case DTYPE_PROCDESC:
return ("proc");
case DTYPE_EVENTFD:
return ("eventfd");
case DTYPE_LINUXTFD:
return ("ltimer");
default:
return ("unkn");
}
}
/*
* For the purposes of debugging, identify a process (if any, perhaps one of
* many) that references the passed file in its file descriptor array. Return
* NULL if none.
*/
static struct proc *
file_to_first_proc(struct file *fp)
{
struct filedesc *fdp;
struct proc *p;
int n;
FOREACH_PROC_IN_SYSTEM(p) {
if (p->p_state == PRS_NEW)
continue;
fdp = p->p_fd;
if (fdp == NULL)
continue;
for (n = 0; n < fdp->fd_nfiles; n++) {
if (fp == fdp->fd_ofiles[n].fde_file)
return (p);
}
}
return (NULL);
}
static void
db_print_file(struct file *fp, int header)
{
#define XPTRWIDTH ((int)howmany(sizeof(void *) * NBBY, 4))
struct proc *p;
if (header)
db_printf("%*s %6s %*s %8s %4s %5s %6s %*s %5s %s\n",
XPTRWIDTH, "File", "Type", XPTRWIDTH, "Data", "Flag",
"GCFl", "Count", "MCount", XPTRWIDTH, "Vnode", "FPID",
"FCmd");
p = file_to_first_proc(fp);
db_printf("%*p %6s %*p %08x %04x %5d %6d %*p %5d %s\n", XPTRWIDTH,
fp, file_type_to_name(fp->f_type), XPTRWIDTH, fp->f_data,
fp->f_flag, 0, refcount_load(&fp->f_count), 0, XPTRWIDTH, fp->f_vnode,
p != NULL ? p->p_pid : -1, p != NULL ? p->p_comm : "-");
#undef XPTRWIDTH
}
DB_SHOW_COMMAND(file, db_show_file)
{
struct file *fp;
if (!have_addr) {
db_printf("usage: show file <addr>\n");
return;
}
fp = (struct file *)addr;
db_print_file(fp, 1);
}
DB_SHOW_COMMAND(files, db_show_files)
{
struct filedesc *fdp;
struct file *fp;
struct proc *p;
int header;
int n;
header = 1;
FOREACH_PROC_IN_SYSTEM(p) {
if (p->p_state == PRS_NEW)
continue;
if ((fdp = p->p_fd) == NULL)
continue;
for (n = 0; n < fdp->fd_nfiles; ++n) {
if ((fp = fdp->fd_ofiles[n].fde_file) == NULL)
continue;
db_print_file(fp, header);
header = 0;
}
}
}
#endif
SYSCTL_INT(_kern, KERN_MAXFILESPERPROC, maxfilesperproc, CTLFLAG_RW,
&maxfilesperproc, 0, "Maximum files allowed open per process");
SYSCTL_INT(_kern, KERN_MAXFILES, maxfiles, CTLFLAG_RW,
&maxfiles, 0, "Maximum number of files");
SYSCTL_INT(_kern, OID_AUTO, openfiles, CTLFLAG_RD,
&openfiles, 0, "System-wide number of open files");
/* ARGSUSED*/
static void
filelistinit(void *dummy)
{
file_zone = uma_zcreate("Files", sizeof(struct file), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
filedesc0_zone = uma_zcreate("filedesc0", sizeof(struct filedesc0),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
pwd_zone = uma_zcreate("PWD", sizeof(struct pwd), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_SMR);
/*
* XXXMJG this is a temporary hack due to boot ordering issues against
* the vnode zone.
*/
vfs_smr = uma_zone_get_smr(pwd_zone);
mtx_init(&sigio_lock, "sigio lock", NULL, MTX_DEF);
}
SYSINIT(select, SI_SUB_LOCK, SI_ORDER_FIRST, filelistinit, NULL);
/*-------------------------------------------------------------------*/
static int
badfo_readwrite(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return (EBADF);
}
static int
badfo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
static int
badfo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return (0);
}
static int
badfo_kqfilter(struct file *fp, struct knote *kn)
{
return (EBADF);
}
static int
badfo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_close(struct file *fp, struct thread *td)
{
return (0);
}
static int
badfo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
return (EBADF);
}
static int
badfo_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
{
return (0);
}
struct fileops badfileops = {
.fo_read = badfo_readwrite,
.fo_write = badfo_readwrite,
.fo_truncate = badfo_truncate,
.fo_ioctl = badfo_ioctl,
.fo_poll = badfo_poll,
.fo_kqfilter = badfo_kqfilter,
.fo_stat = badfo_stat,
.fo_close = badfo_close,
.fo_chmod = badfo_chmod,
.fo_chown = badfo_chown,
.fo_sendfile = badfo_sendfile,
.fo_fill_kinfo = badfo_fill_kinfo,
};
static int
path_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return (POLLNVAL);
}
static int
path_close(struct file *fp, struct thread *td)
{
MPASS(fp->f_type == DTYPE_VNODE);
fp->f_ops = &badfileops;
vdrop(fp->f_vnode);
return (0);
}
struct fileops path_fileops = {
.fo_read = badfo_readwrite,
.fo_write = badfo_readwrite,
.fo_truncate = badfo_truncate,
.fo_ioctl = badfo_ioctl,
.fo_poll = path_poll,
.fo_kqfilter = vn_kqfilter_opath,
.fo_stat = vn_statfile,
.fo_close = path_close,
.fo_chmod = badfo_chmod,
.fo_chown = badfo_chown,
.fo_sendfile = badfo_sendfile,
.fo_fill_kinfo = vn_fill_kinfo,
.fo_flags = DFLAG_PASSABLE,
};
int
invfo_rdwr(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return (EOPNOTSUPP);
}
int
invfo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_ioctl(struct file *fp, u_long com, void *data,
struct ucred *active_cred, struct thread *td)
{
return (ENOTTY);
}
int
invfo_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return (poll_no_poll(events));
}
int
invfo_kqfilter(struct file *fp, struct knote *kn)
{
return (EINVAL);
}
int
invfo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
return (EINVAL);
}
/*-------------------------------------------------------------------*/
/*
* File Descriptor pseudo-device driver (/dev/fd/).
*
* Opening minor device N dup()s the file (if any) connected to file
* descriptor N belonging to the calling process. Note that this driver
* consists of only the ``open()'' routine, because all subsequent
* references to this file will be direct to the other driver.
*
* XXX: we could give this one a cloning event handler if necessary.
*/
/* ARGSUSED */
static int
fdopen(struct cdev *dev, int mode, int type, struct thread *td)
{
/*
* XXX Kludge: set curthread->td_dupfd to contain the value of the
* the file descriptor being sought for duplication. The error
* return ensures that the vnode for this device will be released
* by vn_open. Open will detect this special error and take the
* actions in dupfdopen below. Other callers of vn_open or VOP_OPEN
* will simply report the error.
*/
td->td_dupfd = dev2unit(dev);
return (ENODEV);
}
static struct cdevsw fildesc_cdevsw = {
.d_version = D_VERSION,
.d_open = fdopen,
.d_name = "FD",
};
static void
fildesc_drvinit(void *unused)
{
struct cdev *dev;
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 0, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/0");
make_dev_alias(dev, "stdin");
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 1, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/1");
make_dev_alias(dev, "stdout");
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 2, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/2");
make_dev_alias(dev, "stderr");
}
SYSINIT(fildescdev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, fildesc_drvinit, NULL);
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index 52ed0756db25..1847b070c426 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -1,6131 +1,6132 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1989, 1993, 1995
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Poul-Henning Kamp of the FreeBSD Project.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_cache.c 8.5 (Berkeley) 3/22/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/counter.h>
#include <sys/filedesc.h>
#include <sys/fnv_hash.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/fcntl.h>
#include <sys/jail.h>
#include <sys/mount.h>
#include <sys/namei.h>
#include <sys/proc.h>
#include <sys/seqc.h>
#include <sys/sdt.h>
#include <sys/smr.h>
#include <sys/smp.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vnode.h>
#include <ck_queue.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#ifdef INVARIANTS
#include <machine/_inttypes.h>
#endif
#include <sys/capsicum.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <vm/uma.h>
/*
* High level overview of name caching in the VFS layer.
*
* Originally caching was implemented as part of UFS, later extracted to allow
* use by other filesystems. A decision was made to make it optional and
* completely detached from the rest of the kernel, which comes with limitations
* outlined near the end of this comment block.
*
* This fundamental choice needs to be revisited. In the meantime, the current
* state is described below. Significance of all notable routines is explained
* in comments placed above their implementation. Scattered thoroughout the
* file are TODO comments indicating shortcomings which can be fixed without
* reworking everything (most of the fixes will likely be reusable). Various
* details are omitted from this explanation to not clutter the overview, they
* have to be checked by reading the code and associated commentary.
*
* Keep in mind that it's individual path components which are cached, not full
* paths. That is, for a fully cached path "foo/bar/baz" there are 3 entries,
* one for each name.
*
* I. Data organization
*
* Entries are described by "struct namecache" objects and stored in a hash
* table. See cache_get_hash for more information.
*
* "struct vnode" contains pointers to source entries (names which can be found
* when traversing through said vnode), destination entries (names of that
* vnode (see "Limitations" for a breakdown on the subject) and a pointer to
* the parent vnode.
*
* The (directory vnode; name) tuple reliably determines the target entry if
* it exists.
*
* Since there are no small locks at this time (all are 32 bytes in size on
* LP64), the code works around the problem by introducing lock arrays to
* protect hash buckets and vnode lists.
*
* II. Filesystem integration
*
* Filesystems participating in name caching do the following:
* - set vop_lookup routine to vfs_cache_lookup
* - set vop_cachedlookup to whatever can perform the lookup if the above fails
* - if they support lockless lookup (see below), vop_fplookup_vexec and
* vop_fplookup_symlink are set along with the MNTK_FPLOOKUP flag on the
* mount point
* - call cache_purge or cache_vop_* routines to eliminate stale entries as
* applicable
* - call cache_enter to add entries depending on the MAKEENTRY flag
*
* With the above in mind, there are 2 entry points when doing lookups:
* - ... -> namei -> cache_fplookup -- this is the default
* - ... -> VOP_LOOKUP -> vfs_cache_lookup -- normally only called by namei
* should the above fail
*
* Example code flow how an entry is added:
* ... -> namei -> cache_fplookup -> cache_fplookup_noentry -> VOP_LOOKUP ->
* vfs_cache_lookup -> VOP_CACHEDLOOKUP -> ufs_lookup_ino -> cache_enter
*
* III. Performance considerations
*
* For lockless case forward lookup avoids any writes to shared areas apart
* from the terminal path component. In other words non-modifying lookups of
* different files don't suffer any scalability problems in the namecache.
* Looking up the same file is limited by VFS and goes beyond the scope of this
* file.
*
* At least on amd64 the single-threaded bottleneck for long paths is hashing
* (see cache_get_hash). There are cases where the code issues acquire fence
* multiple times, they can be combined on architectures which suffer from it.
*
* For locked case each encountered vnode has to be referenced and locked in
* order to be handed out to the caller (normally that's namei). This
* introduces significant hit single-threaded and serialization multi-threaded.
*
* Reverse lookup (e.g., "getcwd") fully scales provided it is fully cached --
* avoids any writes to shared areas to any components.
*
* Unrelated insertions are partially serialized on updating the global entry
* counter and possibly serialized on colliding bucket or vnode locks.
*
* IV. Observability
*
* Note not everything has an explicit dtrace probe nor it should have, thus
* some of the one-liners below depend on implementation details.
*
* Examples:
*
* # Check what lookups failed to be handled in a lockless manner. Column 1 is
* # line number, column 2 is status code (see cache_fpl_status)
* dtrace -n 'vfs:fplookup:lookup:done { @[arg1, arg2] = count(); }'
*
* # Lengths of names added by binary name
* dtrace -n 'fbt::cache_enter_time:entry { @[execname] = quantize(args[2]->cn_namelen); }'
*
* # Same as above but only those which exceed 64 characters
* dtrace -n 'fbt::cache_enter_time:entry /args[2]->cn_namelen > 64/ { @[execname] = quantize(args[2]->cn_namelen); }'
*
* # Who is performing lookups with spurious slashes (e.g., "foo//bar") and what
* # path is it
* dtrace -n 'fbt::cache_fplookup_skip_slashes:entry { @[execname, stringof(args[0]->cnp->cn_pnbuf)] = count(); }'
*
* V. Limitations and implementation defects
*
* - since it is possible there is no entry for an open file, tools like
* "procstat" may fail to resolve fd -> vnode -> path to anything
* - even if a filesystem adds an entry, it may get purged (e.g., due to memory
* shortage) in which case the above problem applies
* - hardlinks are not tracked, thus if a vnode is reachable in more than one
* way, resolving a name may return a different path than the one used to
* open it (even if said path is still valid)
* - by default entries are not added for newly created files
* - adding an entry may need to evict negative entry first, which happens in 2
* distinct places (evicting on lookup, adding in a later VOP) making it
* impossible to simply reuse it
* - there is a simple scheme to evict negative entries as the cache is approaching
* its capacity, but it is very unclear if doing so is a good idea to begin with
* - vnodes are subject to being recycled even if target inode is left in memory,
* which loses the name cache entries when it perhaps should not. in case of tmpfs
* names get duplicated -- kept by filesystem itself and namecache separately
* - struct namecache has a fixed size and comes in 2 variants, often wasting space.
* now hard to replace with malloc due to dependence on SMR.
* - lack of better integration with the kernel also turns nullfs into a layered
* filesystem instead of something which can take advantage of caching
*/
static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache");
SDT_PROVIDER_DECLARE(vfs);
SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE3(vfs, namecache, enter, duplicate, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
"char *");
SDT_PROBE_DEFINE2(vfs, namecache, fullpath_smr, hit, "struct vnode *",
"const char *");
SDT_PROBE_DEFINE4(vfs, namecache, fullpath_smr, miss, "struct vnode *",
"struct namecache *", "int", "int");
SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
"char *", "struct vnode *");
SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
"struct vnode *", "char *");
SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
"struct vnode *", "char *");
SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
"char *");
SDT_PROBE_DEFINE2(vfs, namecache, removecnp, hit, "struct vnode *",
"struct componentname *");
SDT_PROBE_DEFINE2(vfs, namecache, removecnp, miss, "struct vnode *",
"struct componentname *");
SDT_PROBE_DEFINE3(vfs, namecache, purge, done, "struct vnode *", "size_t", "size_t");
SDT_PROBE_DEFINE1(vfs, namecache, purge, batch, "int");
SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
"char *");
SDT_PROBE_DEFINE2(vfs, namecache, evict_negative, done, "struct vnode *",
"char *");
SDT_PROBE_DEFINE1(vfs, namecache, symlink, alloc__fail, "size_t");
SDT_PROBE_DEFINE3(vfs, fplookup, lookup, done, "struct nameidata", "int", "bool");
SDT_PROBE_DECLARE(vfs, namei, lookup, entry);
SDT_PROBE_DECLARE(vfs, namei, lookup, return);
static char __read_frequently cache_fast_lookup_enabled = true;
/*
* This structure describes the elements in the cache of recent
* names looked up by namei.
*/
struct negstate {
u_char neg_flag;
u_char neg_hit;
};
_Static_assert(sizeof(struct negstate) <= sizeof(struct vnode *),
"the state must fit in a union with a pointer without growing it");
struct namecache {
LIST_ENTRY(namecache) nc_src; /* source vnode list */
TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */
CK_SLIST_ENTRY(namecache) nc_hash;/* hash chain */
struct vnode *nc_dvp; /* vnode of parent of name */
union {
struct vnode *nu_vp; /* vnode the name refers to */
struct negstate nu_neg;/* negative entry state */
} n_un;
u_char nc_flag; /* flag bits */
u_char nc_nlen; /* length of name */
char nc_name[0]; /* segment name + nul */
};
/*
* struct namecache_ts repeats struct namecache layout up to the
* nc_nlen member.
* struct namecache_ts is used in place of struct namecache when time(s) need
* to be stored. The nc_dotdottime field is used when a cache entry is mapping
* both a non-dotdot directory name plus dotdot for the directory's
* parent.
*
* See below for alignment requirement.
*/
struct namecache_ts {
struct timespec nc_time; /* timespec provided by fs */
struct timespec nc_dotdottime; /* dotdot timespec provided by fs */
int nc_ticks; /* ticks value when entry was added */
int nc_pad;
struct namecache nc_nc;
};
TAILQ_HEAD(cache_freebatch, namecache);
/*
* At least mips n32 performs 64-bit accesses to timespec as found
* in namecache_ts and requires them to be aligned. Since others
* may be in the same spot suffer a little bit and enforce the
* alignment for everyone. Note this is a nop for 64-bit platforms.
*/
#define CACHE_ZONE_ALIGNMENT UMA_ALIGNOF(time_t)
/*
* TODO: the initial value of CACHE_PATH_CUTOFF was inherited from the
* 4.4 BSD codebase. Later on struct namecache was tweaked to become
* smaller and the value was bumped to retain the total size, but it
* was never re-evaluated for suitability. A simple test counting
* lengths during package building shows that the value of 45 covers
* about 86% of all added entries, reaching 99% at 65.
*
* Regardless of the above, use of dedicated zones instead of malloc may be
* inducing additional waste. This may be hard to address as said zones are
* tied to VFS SMR. Even if retaining them, the current split should be
* re-evaluated.
*/
#ifdef __LP64__
#define CACHE_PATH_CUTOFF 45
#define CACHE_LARGE_PAD 6
#else
#define CACHE_PATH_CUTOFF 41
#define CACHE_LARGE_PAD 2
#endif
#define CACHE_ZONE_SMALL_SIZE (offsetof(struct namecache, nc_name) + CACHE_PATH_CUTOFF + 1)
#define CACHE_ZONE_SMALL_TS_SIZE (offsetof(struct namecache_ts, nc_nc) + CACHE_ZONE_SMALL_SIZE)
#define CACHE_ZONE_LARGE_SIZE (offsetof(struct namecache, nc_name) + NAME_MAX + 1 + CACHE_LARGE_PAD)
#define CACHE_ZONE_LARGE_TS_SIZE (offsetof(struct namecache_ts, nc_nc) + CACHE_ZONE_LARGE_SIZE)
_Static_assert((CACHE_ZONE_SMALL_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
_Static_assert((CACHE_ZONE_SMALL_TS_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
_Static_assert((CACHE_ZONE_LARGE_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
_Static_assert((CACHE_ZONE_LARGE_TS_SIZE % (CACHE_ZONE_ALIGNMENT + 1)) == 0, "bad zone size");
#define nc_vp n_un.nu_vp
#define nc_neg n_un.nu_neg
/*
* Flags in namecache.nc_flag
*/
#define NCF_WHITE 0x01
#define NCF_ISDOTDOT 0x02
#define NCF_TS 0x04
#define NCF_DTS 0x08
#define NCF_DVDROP 0x10
#define NCF_NEGATIVE 0x20
#define NCF_INVALID 0x40
#define NCF_WIP 0x80
/*
* Flags in negstate.neg_flag
*/
#define NEG_HOT 0x01
static bool cache_neg_evict_cond(u_long lnumcache);
/*
* Mark an entry as invalid.
*
* This is called before it starts getting deconstructed.
*/
static void
cache_ncp_invalidate(struct namecache *ncp)
{
KASSERT((ncp->nc_flag & NCF_INVALID) == 0,
("%s: entry %p already invalid", __func__, ncp));
atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_INVALID);
atomic_thread_fence_rel();
}
/*
* Check whether the entry can be safely used.
*
* All places which elide locks are supposed to call this after they are
* done with reading from an entry.
*/
#define cache_ncp_canuse(ncp) ({ \
struct namecache *_ncp = (ncp); \
u_char _nc_flag; \
\
atomic_thread_fence_acq(); \
_nc_flag = atomic_load_char(&_ncp->nc_flag); \
__predict_true((_nc_flag & (NCF_INVALID | NCF_WIP)) == 0); \
})
/*
* Like the above but also checks NCF_WHITE.
*/
#define cache_fpl_neg_ncp_canuse(ncp) ({ \
struct namecache *_ncp = (ncp); \
u_char _nc_flag; \
\
atomic_thread_fence_acq(); \
_nc_flag = atomic_load_char(&_ncp->nc_flag); \
__predict_true((_nc_flag & (NCF_INVALID | NCF_WIP | NCF_WHITE)) == 0); \
})
VFS_SMR_DECLARE;
static SYSCTL_NODE(_vfs_cache, OID_AUTO, param, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache parameters");
static u_int __read_mostly ncsize; /* the size as computed on creation or resizing */
SYSCTL_UINT(_vfs_cache_param, OID_AUTO, size, CTLFLAG_RW, &ncsize, 0,
"Total namecache capacity");
u_int ncsizefactor = 2;
SYSCTL_UINT(_vfs_cache_param, OID_AUTO, sizefactor, CTLFLAG_RW, &ncsizefactor, 0,
"Size factor for namecache");
static u_long __read_mostly ncnegfactor = 5; /* ratio of negative entries */
SYSCTL_ULONG(_vfs_cache_param, OID_AUTO, negfactor, CTLFLAG_RW, &ncnegfactor, 0,
"Ratio of negative namecache entries");
/*
* Negative entry % of namecache capacity above which automatic eviction is allowed.
*
* Check cache_neg_evict_cond for details.
*/
static u_int ncnegminpct = 3;
static u_int __read_mostly neg_min; /* the above recomputed against ncsize */
SYSCTL_UINT(_vfs_cache_param, OID_AUTO, negmin, CTLFLAG_RD, &neg_min, 0,
"Negative entry count above which automatic eviction is allowed");
/*
* Structures associated with name caching.
*/
#define NCHHASH(hash) \
(&nchashtbl[(hash) & nchash])
static __read_mostly CK_SLIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */
static u_long __read_mostly nchash; /* size of hash table */
SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
"Size of namecache hash table");
static u_long __exclusive_cache_line numneg; /* number of negative entries allocated */
static u_long __exclusive_cache_line numcache;/* number of cache entries allocated */
struct nchstats nchstats; /* cache effectiveness statistics */
static bool __read_mostly cache_rename_add = true;
SYSCTL_BOOL(_vfs, OID_AUTO, cache_rename_add, CTLFLAG_RW,
&cache_rename_add, 0, "");
static u_int __exclusive_cache_line neg_cycle;
#define ncneghash 3
#define numneglists (ncneghash + 1)
struct neglist {
struct mtx nl_evict_lock;
struct mtx nl_lock __aligned(CACHE_LINE_SIZE);
TAILQ_HEAD(, namecache) nl_list;
TAILQ_HEAD(, namecache) nl_hotlist;
u_long nl_hotnum;
} __aligned(CACHE_LINE_SIZE);
static struct neglist neglists[numneglists];
static inline struct neglist *
NCP2NEGLIST(struct namecache *ncp)
{
return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
}
static inline struct negstate *
NCP2NEGSTATE(struct namecache *ncp)
{
MPASS(atomic_load_char(&ncp->nc_flag) & NCF_NEGATIVE);
return (&ncp->nc_neg);
}
#define numbucketlocks (ncbuckethash + 1)
static u_int __read_mostly ncbuckethash;
static struct mtx_padalign __read_mostly *bucketlocks;
#define HASH2BUCKETLOCK(hash) \
((struct mtx *)(&bucketlocks[((hash) & ncbuckethash)]))
#define numvnodelocks (ncvnodehash + 1)
static u_int __read_mostly ncvnodehash;
static struct mtx __read_mostly *vnodelocks;
static inline struct mtx *
VP2VNODELOCK(struct vnode *vp)
{
return (&vnodelocks[(((uintptr_t)(vp) >> 8) & ncvnodehash)]);
}
static void
cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
{
struct namecache_ts *ncp_ts;
KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
(tsp == NULL && ticksp == NULL),
("No NCF_TS"));
if (tsp == NULL)
return;
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
*tsp = ncp_ts->nc_time;
*ticksp = ncp_ts->nc_ticks;
}
#ifdef DEBUG_CACHE
static int __read_mostly doingcache = 1; /* 1 => enable the cache */
SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
"VFS namecache enabled");
#endif
/* Export size information to userland */
SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
sizeof(struct namecache), "sizeof(struct namecache)");
/*
* The new name cache statistics
*/
static SYSCTL_NODE(_vfs_cache, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache statistics");
#define STATNODE_ULONG(name, varname, descr) \
SYSCTL_ULONG(_vfs_cache_stats, OID_AUTO, name, CTLFLAG_RD, &varname, 0, descr);
#define STATNODE_COUNTER(name, varname, descr) \
static COUNTER_U64_DEFINE_EARLY(varname); \
SYSCTL_COUNTER_U64(_vfs_cache_stats, OID_AUTO, name, CTLFLAG_RD, &varname, \
descr);
STATNODE_ULONG(neg, numneg, "Number of negative cache entries");
STATNODE_ULONG(count, numcache, "Number of cache entries");
STATNODE_COUNTER(heldvnodes, numcachehv, "Number of namecache entries with vnodes held");
STATNODE_COUNTER(drops, numdrops, "Number of dropped entries due to reaching the limit");
STATNODE_COUNTER(dothits, dothits, "Number of '.' hits");
STATNODE_COUNTER(dotdothis, dotdothits, "Number of '..' hits");
STATNODE_COUNTER(miss, nummiss, "Number of cache misses");
STATNODE_COUNTER(misszap, nummisszap, "Number of cache misses we do not want to cache");
STATNODE_COUNTER(posszaps, numposzaps,
"Number of cache hits (positive) we do not want to cache");
STATNODE_COUNTER(poshits, numposhits, "Number of cache hits (positive)");
STATNODE_COUNTER(negzaps, numnegzaps,
"Number of cache hits (negative) we do not want to cache");
STATNODE_COUNTER(neghits, numneghits, "Number of cache hits (negative)");
/* These count for vn_getcwd(), too. */
STATNODE_COUNTER(fullpathcalls, numfullpathcalls, "Number of fullpath search calls");
STATNODE_COUNTER(fullpathfail1, numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
STATNODE_COUNTER(fullpathfail2, numfullpathfail2,
"Number of fullpath search errors (VOP_VPTOCNP failures)");
STATNODE_COUNTER(fullpathfail4, numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
STATNODE_COUNTER(fullpathfound, numfullpathfound, "Number of successful fullpath calls");
STATNODE_COUNTER(symlinktoobig, symlinktoobig, "Number of times symlink did not fit the cache");
/*
* Debug or developer statistics.
*/
static SYSCTL_NODE(_vfs_cache, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache debugging");
#define DEBUGNODE_ULONG(name, varname, descr) \
SYSCTL_ULONG(_vfs_cache_debug, OID_AUTO, name, CTLFLAG_RD, &varname, 0, descr);
#define DEBUGNODE_COUNTER(name, varname, descr) \
static COUNTER_U64_DEFINE_EARLY(varname); \
SYSCTL_COUNTER_U64(_vfs_cache_debug, OID_AUTO, name, CTLFLAG_RD, &varname, \
descr);
DEBUGNODE_COUNTER(zap_bucket_relock_success, zap_bucket_relock_success,
"Number of successful removals after relocking");
static long zap_bucket_fail;
DEBUGNODE_ULONG(zap_bucket_fail, zap_bucket_fail, "");
static long zap_bucket_fail2;
DEBUGNODE_ULONG(zap_bucket_fail2, zap_bucket_fail2, "");
static long cache_lock_vnodes_cel_3_failures;
DEBUGNODE_ULONG(vnodes_cel_3_failures, cache_lock_vnodes_cel_3_failures,
"Number of times 3-way vnode locking failed");
static void cache_zap_locked(struct namecache *ncp);
static int vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf,
char **retbuf, size_t *buflen, size_t addend);
static int vn_fullpath_any(struct vnode *vp, struct vnode *rdir, char *buf,
char **retbuf, size_t *buflen);
static int vn_fullpath_dir(struct vnode *vp, struct vnode *rdir, char *buf,
char **retbuf, size_t *len, size_t addend);
static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
static inline void
cache_assert_vlp_locked(struct mtx *vlp)
{
if (vlp != NULL)
mtx_assert(vlp, MA_OWNED);
}
static inline void
cache_assert_vnode_locked(struct vnode *vp)
{
struct mtx *vlp;
vlp = VP2VNODELOCK(vp);
cache_assert_vlp_locked(vlp);
}
/*
* Directory vnodes with entries are held for two reasons:
* 1. make them less of a target for reclamation in vnlru
* 2. suffer smaller performance penalty in locked lookup as requeieing is avoided
*
* It will be feasible to stop doing it altogether if all filesystems start
* supporting lockless lookup.
*/
static void
cache_hold_vnode(struct vnode *vp)
{
cache_assert_vnode_locked(vp);
VNPASS(LIST_EMPTY(&vp->v_cache_src), vp);
vhold(vp);
counter_u64_add(numcachehv, 1);
}
static void
cache_drop_vnode(struct vnode *vp)
{
/*
* Called after all locks are dropped, meaning we can't assert
* on the state of v_cache_src.
*/
vdrop(vp);
counter_u64_add(numcachehv, -1);
}
/*
* UMA zones.
*/
static uma_zone_t __read_mostly cache_zone_small;
static uma_zone_t __read_mostly cache_zone_small_ts;
static uma_zone_t __read_mostly cache_zone_large;
static uma_zone_t __read_mostly cache_zone_large_ts;
char *
cache_symlink_alloc(size_t size, int flags)
{
if (size < CACHE_ZONE_SMALL_SIZE) {
return (uma_zalloc_smr(cache_zone_small, flags));
}
if (size < CACHE_ZONE_LARGE_SIZE) {
return (uma_zalloc_smr(cache_zone_large, flags));
}
counter_u64_add(symlinktoobig, 1);
SDT_PROBE1(vfs, namecache, symlink, alloc__fail, size);
return (NULL);
}
void
cache_symlink_free(char *string, size_t size)
{
MPASS(string != NULL);
KASSERT(size < CACHE_ZONE_LARGE_SIZE,
("%s: size %zu too big", __func__, size));
if (size < CACHE_ZONE_SMALL_SIZE) {
uma_zfree_smr(cache_zone_small, string);
return;
}
if (size < CACHE_ZONE_LARGE_SIZE) {
uma_zfree_smr(cache_zone_large, string);
return;
}
__assert_unreachable();
}
static struct namecache *
cache_alloc_uma(int len, bool ts)
{
struct namecache_ts *ncp_ts;
struct namecache *ncp;
if (__predict_false(ts)) {
if (len <= CACHE_PATH_CUTOFF)
ncp_ts = uma_zalloc_smr(cache_zone_small_ts, M_WAITOK);
else
ncp_ts = uma_zalloc_smr(cache_zone_large_ts, M_WAITOK);
ncp = &ncp_ts->nc_nc;
} else {
if (len <= CACHE_PATH_CUTOFF)
ncp = uma_zalloc_smr(cache_zone_small, M_WAITOK);
else
ncp = uma_zalloc_smr(cache_zone_large, M_WAITOK);
}
return (ncp);
}
static void
cache_free_uma(struct namecache *ncp)
{
struct namecache_ts *ncp_ts;
if (__predict_false(ncp->nc_flag & NCF_TS)) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
uma_zfree_smr(cache_zone_small_ts, ncp_ts);
else
uma_zfree_smr(cache_zone_large_ts, ncp_ts);
} else {
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
uma_zfree_smr(cache_zone_small, ncp);
else
uma_zfree_smr(cache_zone_large, ncp);
}
}
static struct namecache *
cache_alloc(int len, bool ts)
{
u_long lnumcache;
/*
* Avoid blowout in namecache entries.
*
* Bugs:
* 1. filesystems may end up trying to add an already existing entry
* (for example this can happen after a cache miss during concurrent
* lookup), in which case we will call cache_neg_evict despite not
* adding anything.
* 2. the routine may fail to free anything and no provisions are made
* to make it try harder (see the inside for failure modes)
* 3. it only ever looks at negative entries.
*/
lnumcache = atomic_fetchadd_long(&numcache, 1) + 1;
if (cache_neg_evict_cond(lnumcache)) {
lnumcache = atomic_load_long(&numcache);
}
if (__predict_false(lnumcache >= ncsize)) {
atomic_subtract_long(&numcache, 1);
counter_u64_add(numdrops, 1);
return (NULL);
}
return (cache_alloc_uma(len, ts));
}
static void
cache_free(struct namecache *ncp)
{
MPASS(ncp != NULL);
if ((ncp->nc_flag & NCF_DVDROP) != 0) {
cache_drop_vnode(ncp->nc_dvp);
}
cache_free_uma(ncp);
atomic_subtract_long(&numcache, 1);
}
static void
cache_free_batch(struct cache_freebatch *batch)
{
struct namecache *ncp, *nnp;
int i;
i = 0;
if (TAILQ_EMPTY(batch))
goto out;
TAILQ_FOREACH_SAFE(ncp, batch, nc_dst, nnp) {
if ((ncp->nc_flag & NCF_DVDROP) != 0) {
cache_drop_vnode(ncp->nc_dvp);
}
cache_free_uma(ncp);
i++;
}
atomic_subtract_long(&numcache, i);
out:
SDT_PROBE1(vfs, namecache, purge, batch, i);
}
/*
* Hashing.
*
* The code was made to use FNV in 2001 and this choice needs to be revisited.
*
* Short summary of the difficulty:
* The longest name which can be inserted is NAME_MAX characters in length (or
* 255 at the time of writing this comment), while majority of names used in
* practice are significantly shorter (mostly below 10). More importantly
* majority of lookups performed find names are even shorter than that.
*
* This poses a problem where hashes which do better than FNV past word size
* (or so) tend to come with additional overhead when finalizing the result,
* making them noticeably slower for the most commonly used range.
*
* Consider a path like: /usr/obj/usr/src/sys/amd64/GENERIC/vnode_if.c
*
* When looking it up the most time consuming part by a large margin (at least
* on amd64) is hashing. Replacing FNV with something which pessimizes short
* input would make the slowest part stand out even more.
*/
/*
* TODO: With the value stored we can do better than computing the hash based
* on the address.
*/
static void
cache_prehash(struct vnode *vp)
{
vp->v_nchash = fnv_32_buf(&vp, sizeof(vp), FNV1_32_INIT);
}
static uint32_t
cache_get_hash(char *name, u_char len, struct vnode *dvp)
{
return (fnv_32_buf(name, len, dvp->v_nchash));
}
static uint32_t
cache_get_hash_iter_start(struct vnode *dvp)
{
return (dvp->v_nchash);
}
static uint32_t
cache_get_hash_iter(char c, uint32_t hash)
{
return (fnv_32_buf(&c, 1, hash));
}
static uint32_t
cache_get_hash_iter_finish(uint32_t hash)
{
return (hash);
}
static inline struct nchashhead *
NCP2BUCKET(struct namecache *ncp)
{
uint32_t hash;
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
return (NCHHASH(hash));
}
static inline struct mtx *
NCP2BUCKETLOCK(struct namecache *ncp)
{
uint32_t hash;
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
return (HASH2BUCKETLOCK(hash));
}
#ifdef INVARIANTS
static void
cache_assert_bucket_locked(struct namecache *ncp)
{
struct mtx *blp;
blp = NCP2BUCKETLOCK(ncp);
mtx_assert(blp, MA_OWNED);
}
static void
cache_assert_bucket_unlocked(struct namecache *ncp)
{
struct mtx *blp;
blp = NCP2BUCKETLOCK(ncp);
mtx_assert(blp, MA_NOTOWNED);
}
#else
#define cache_assert_bucket_locked(x) do { } while (0)
#define cache_assert_bucket_unlocked(x) do { } while (0)
#endif
#define cache_sort_vnodes(x, y) _cache_sort_vnodes((void **)(x), (void **)(y))
static void
_cache_sort_vnodes(void **p1, void **p2)
{
void *tmp;
MPASS(*p1 != NULL || *p2 != NULL);
if (*p1 > *p2) {
tmp = *p2;
*p2 = *p1;
*p1 = tmp;
}
}
static void
cache_lock_all_buckets(void)
{
u_int i;
for (i = 0; i < numbucketlocks; i++)
mtx_lock(&bucketlocks[i]);
}
static void
cache_unlock_all_buckets(void)
{
u_int i;
for (i = 0; i < numbucketlocks; i++)
mtx_unlock(&bucketlocks[i]);
}
static void
cache_lock_all_vnodes(void)
{
u_int i;
for (i = 0; i < numvnodelocks; i++)
mtx_lock(&vnodelocks[i]);
}
static void
cache_unlock_all_vnodes(void)
{
u_int i;
for (i = 0; i < numvnodelocks; i++)
mtx_unlock(&vnodelocks[i]);
}
static int
cache_trylock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 != NULL) {
if (!mtx_trylock(vlp1))
return (EAGAIN);
}
if (!mtx_trylock(vlp2)) {
if (vlp1 != NULL)
mtx_unlock(vlp1);
return (EAGAIN);
}
return (0);
}
static void
cache_lock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
MPASS(vlp1 != NULL || vlp2 != NULL);
MPASS(vlp1 <= vlp2);
if (vlp1 != NULL)
mtx_lock(vlp1);
if (vlp2 != NULL)
mtx_lock(vlp2);
}
static void
cache_unlock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
MPASS(vlp1 != NULL || vlp2 != NULL);
if (vlp1 != NULL)
mtx_unlock(vlp1);
if (vlp2 != NULL)
mtx_unlock(vlp2);
}
static int
sysctl_nchstats(SYSCTL_HANDLER_ARGS)
{
struct nchstats snap;
if (req->oldptr == NULL)
return (SYSCTL_OUT(req, 0, sizeof(snap)));
snap = nchstats;
snap.ncs_goodhits = counter_u64_fetch(numposhits);
snap.ncs_neghits = counter_u64_fetch(numneghits);
snap.ncs_badhits = counter_u64_fetch(numposzaps) +
counter_u64_fetch(numnegzaps);
snap.ncs_miss = counter_u64_fetch(nummisszap) +
counter_u64_fetch(nummiss);
return (SYSCTL_OUT(req, &snap, sizeof(snap)));
}
SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE | CTLFLAG_RD |
CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU",
"VFS cache effectiveness statistics");
static void
cache_recalc_neg_min(u_int val)
{
neg_min = (ncsize * val) / 100;
}
static int
sysctl_negminpct(SYSCTL_HANDLER_ARGS)
{
u_int val;
int error;
val = ncnegminpct;
error = sysctl_handle_int(oidp, &val, 0, req);
if (error != 0 || req->newptr == NULL)
return (error);
if (val == ncnegminpct)
return (0);
if (val < 0 || val > 99)
return (EINVAL);
ncnegminpct = val;
cache_recalc_neg_min(val);
return (0);
}
SYSCTL_PROC(_vfs_cache_param, OID_AUTO, negminpct,
CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, NULL, 0, sysctl_negminpct,
"I", "Negative entry \% of namecache capacity above which automatic eviction is allowed");
#ifdef DIAGNOSTIC
/*
* Grab an atomic snapshot of the name cache hash chain lengths
*/
static SYSCTL_NODE(_debug, OID_AUTO, hashstat,
CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
"hash table stats");
static int
sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
{
struct nchashhead *ncpp;
struct namecache *ncp;
int i, error, n_nchash, *cntbuf;
retry:
n_nchash = nchash + 1; /* nchash is max index, not count */
if (req->oldptr == NULL)
return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK);
cache_lock_all_buckets();
if (n_nchash != nchash + 1) {
cache_unlock_all_buckets();
free(cntbuf, M_TEMP);
goto retry;
}
/* Scan hash tables counting entries */
for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
CK_SLIST_FOREACH(ncp, ncpp, nc_hash)
cntbuf[i]++;
cache_unlock_all_buckets();
for (error = 0, i = 0; i < n_nchash; i++)
if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
break;
free(cntbuf, M_TEMP);
return (error);
}
SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD|
CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
"nchash chain lengths");
static int
sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
{
int error;
struct nchashhead *ncpp;
struct namecache *ncp;
int n_nchash;
int count, maxlength, used, pct;
if (!req->oldptr)
return SYSCTL_OUT(req, 0, 4 * sizeof(int));
cache_lock_all_buckets();
n_nchash = nchash + 1; /* nchash is max index, not count */
used = 0;
maxlength = 0;
/* Scan hash tables for applicable entries */
for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
count = 0;
CK_SLIST_FOREACH(ncp, ncpp, nc_hash) {
count++;
}
if (count)
used++;
if (maxlength < count)
maxlength = count;
}
n_nchash = nchash + 1;
cache_unlock_all_buckets();
pct = (used * 100) / (n_nchash / 100);
error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
if (error)
return (error);
error = SYSCTL_OUT(req, &used, sizeof(used));
if (error)
return (error);
error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
if (error)
return (error);
error = SYSCTL_OUT(req, &pct, sizeof(pct));
if (error)
return (error);
return (0);
}
SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD|
CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
"nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
#endif
/*
* Negative entries management
*
* Various workloads create plenty of negative entries and barely use them
* afterwards. Moreover malicious users can keep performing bogus lookups
* adding even more entries. For example "make tinderbox" as of writing this
* comment ends up with 2.6M namecache entries in total, 1.2M of which are
* negative.
*
* As such, a rather aggressive eviction method is needed. The currently
* employed method is a placeholder.
*
* Entries are split over numneglists separate lists, each of which is further
* split into hot and cold entries. Entries get promoted after getting a hit.
* Eviction happens on addition of new entry.
*/
static SYSCTL_NODE(_vfs_cache, OID_AUTO, neg, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache negative entry statistics");
SYSCTL_ULONG(_vfs_cache_neg, OID_AUTO, count, CTLFLAG_RD, &numneg, 0,
"Number of negative cache entries");
static COUNTER_U64_DEFINE_EARLY(neg_created);
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, created, CTLFLAG_RD, &neg_created,
"Number of created negative entries");
static COUNTER_U64_DEFINE_EARLY(neg_evicted);
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evicted, CTLFLAG_RD, &neg_evicted,
"Number of evicted negative entries");
static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_empty);
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_empty, CTLFLAG_RD,
&neg_evict_skipped_empty,
"Number of times evicting failed due to lack of entries");
static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_missed);
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_missed, CTLFLAG_RD,
&neg_evict_skipped_missed,
"Number of times evicting failed due to target entry disappearing");
static COUNTER_U64_DEFINE_EARLY(neg_evict_skipped_contended);
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, evict_skipped_contended, CTLFLAG_RD,
&neg_evict_skipped_contended,
"Number of times evicting failed due to contention");
SYSCTL_COUNTER_U64(_vfs_cache_neg, OID_AUTO, hits, CTLFLAG_RD, &numneghits,
"Number of cache hits (negative)");
static int
sysctl_neg_hot(SYSCTL_HANDLER_ARGS)
{
int i, out;
out = 0;
for (i = 0; i < numneglists; i++)
out += neglists[i].nl_hotnum;
return (SYSCTL_OUT(req, &out, sizeof(out)));
}
SYSCTL_PROC(_vfs_cache_neg, OID_AUTO, hot, CTLTYPE_INT | CTLFLAG_RD |
CTLFLAG_MPSAFE, 0, 0, sysctl_neg_hot, "I",
"Number of hot negative entries");
static void
cache_neg_init(struct namecache *ncp)
{
struct negstate *ns;
ncp->nc_flag |= NCF_NEGATIVE;
ns = NCP2NEGSTATE(ncp);
ns->neg_flag = 0;
ns->neg_hit = 0;
counter_u64_add(neg_created, 1);
}
#define CACHE_NEG_PROMOTION_THRESH 2
static bool
cache_neg_hit_prep(struct namecache *ncp)
{
struct negstate *ns;
u_char n;
ns = NCP2NEGSTATE(ncp);
n = atomic_load_char(&ns->neg_hit);
for (;;) {
if (n >= CACHE_NEG_PROMOTION_THRESH)
return (false);
if (atomic_fcmpset_8(&ns->neg_hit, &n, n + 1))
break;
}
return (n + 1 == CACHE_NEG_PROMOTION_THRESH);
}
/*
* Nothing to do here but it is provided for completeness as some
* cache_neg_hit_prep callers may end up returning without even
* trying to promote.
*/
#define cache_neg_hit_abort(ncp) do { } while (0)
static void
cache_neg_hit_finish(struct namecache *ncp)
{
SDT_PROBE2(vfs, namecache, lookup, hit__negative, ncp->nc_dvp, ncp->nc_name);
counter_u64_add(numneghits, 1);
}
/*
* Move a negative entry to the hot list.
*/
static void
cache_neg_promote_locked(struct namecache *ncp)
{
struct neglist *nl;
struct negstate *ns;
ns = NCP2NEGSTATE(ncp);
nl = NCP2NEGLIST(ncp);
mtx_assert(&nl->nl_lock, MA_OWNED);
if ((ns->neg_flag & NEG_HOT) == 0) {
TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
TAILQ_INSERT_TAIL(&nl->nl_hotlist, ncp, nc_dst);
nl->nl_hotnum++;
ns->neg_flag |= NEG_HOT;
}
}
/*
* Move a hot negative entry to the cold list.
*/
static void
cache_neg_demote_locked(struct namecache *ncp)
{
struct neglist *nl;
struct negstate *ns;
ns = NCP2NEGSTATE(ncp);
nl = NCP2NEGLIST(ncp);
mtx_assert(&nl->nl_lock, MA_OWNED);
MPASS(ns->neg_flag & NEG_HOT);
TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
nl->nl_hotnum--;
ns->neg_flag &= ~NEG_HOT;
atomic_store_char(&ns->neg_hit, 0);
}
/*
* Move a negative entry to the hot list if it matches the lookup.
*
* We have to take locks, but they may be contended and in the worst
* case we may need to go off CPU. We don't want to spin within the
* smr section and we can't block with it. Exiting the section means
* the found entry could have been evicted. We are going to look it
* up again.
*/
static bool
cache_neg_promote_cond(struct vnode *dvp, struct componentname *cnp,
struct namecache *oncp, uint32_t hash)
{
struct namecache *ncp;
struct neglist *nl;
u_char nc_flag;
nl = NCP2NEGLIST(oncp);
mtx_lock(&nl->nl_lock);
/*
* For hash iteration.
*/
vfs_smr_enter();
/*
* Avoid all surprises by only succeeding if we got the same entry and
* bailing completely otherwise.
* XXX There are no provisions to keep the vnode around, meaning we may
* end up promoting a negative entry for a *new* vnode and returning
* ENOENT on its account. This is the error we want to return anyway
* and promotion is harmless.
*
* In particular at this point there can be a new ncp which matches the
* search but hashes to a different neglist.
*/
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp == oncp)
break;
}
/*
* No match to begin with.
*/
if (__predict_false(ncp == NULL)) {
goto out_abort;
}
/*
* The newly found entry may be something different...
*/
if (!(ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))) {
goto out_abort;
}
/*
* ... and not even negative.
*/
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_NEGATIVE) == 0) {
goto out_abort;
}
if (!cache_ncp_canuse(ncp)) {
goto out_abort;
}
cache_neg_promote_locked(ncp);
cache_neg_hit_finish(ncp);
vfs_smr_exit();
mtx_unlock(&nl->nl_lock);
return (true);
out_abort:
vfs_smr_exit();
mtx_unlock(&nl->nl_lock);
return (false);
}
static void
cache_neg_promote(struct namecache *ncp)
{
struct neglist *nl;
nl = NCP2NEGLIST(ncp);
mtx_lock(&nl->nl_lock);
cache_neg_promote_locked(ncp);
mtx_unlock(&nl->nl_lock);
}
static void
cache_neg_insert(struct namecache *ncp)
{
struct neglist *nl;
MPASS(ncp->nc_flag & NCF_NEGATIVE);
cache_assert_bucket_locked(ncp);
nl = NCP2NEGLIST(ncp);
mtx_lock(&nl->nl_lock);
TAILQ_INSERT_TAIL(&nl->nl_list, ncp, nc_dst);
mtx_unlock(&nl->nl_lock);
atomic_add_long(&numneg, 1);
}
static void
cache_neg_remove(struct namecache *ncp)
{
struct neglist *nl;
struct negstate *ns;
cache_assert_bucket_locked(ncp);
nl = NCP2NEGLIST(ncp);
ns = NCP2NEGSTATE(ncp);
mtx_lock(&nl->nl_lock);
if ((ns->neg_flag & NEG_HOT) != 0) {
TAILQ_REMOVE(&nl->nl_hotlist, ncp, nc_dst);
nl->nl_hotnum--;
} else {
TAILQ_REMOVE(&nl->nl_list, ncp, nc_dst);
}
mtx_unlock(&nl->nl_lock);
atomic_subtract_long(&numneg, 1);
}
static struct neglist *
cache_neg_evict_select_list(void)
{
struct neglist *nl;
u_int c;
c = atomic_fetchadd_int(&neg_cycle, 1) + 1;
nl = &neglists[c % numneglists];
if (!mtx_trylock(&nl->nl_evict_lock)) {
counter_u64_add(neg_evict_skipped_contended, 1);
return (NULL);
}
return (nl);
}
static struct namecache *
cache_neg_evict_select_entry(struct neglist *nl)
{
struct namecache *ncp, *lncp;
struct negstate *ns, *lns;
int i;
mtx_assert(&nl->nl_evict_lock, MA_OWNED);
mtx_assert(&nl->nl_lock, MA_OWNED);
ncp = TAILQ_FIRST(&nl->nl_list);
if (ncp == NULL)
return (NULL);
lncp = ncp;
lns = NCP2NEGSTATE(lncp);
for (i = 1; i < 4; i++) {
ncp = TAILQ_NEXT(ncp, nc_dst);
if (ncp == NULL)
break;
ns = NCP2NEGSTATE(ncp);
if (ns->neg_hit < lns->neg_hit) {
lncp = ncp;
lns = ns;
}
}
return (lncp);
}
static bool
cache_neg_evict(void)
{
struct namecache *ncp, *ncp2;
struct neglist *nl;
struct vnode *dvp;
struct mtx *dvlp;
struct mtx *blp;
uint32_t hash;
u_char nlen;
bool evicted;
nl = cache_neg_evict_select_list();
if (nl == NULL) {
return (false);
}
mtx_lock(&nl->nl_lock);
ncp = TAILQ_FIRST(&nl->nl_hotlist);
if (ncp != NULL) {
cache_neg_demote_locked(ncp);
}
ncp = cache_neg_evict_select_entry(nl);
if (ncp == NULL) {
counter_u64_add(neg_evict_skipped_empty, 1);
mtx_unlock(&nl->nl_lock);
mtx_unlock(&nl->nl_evict_lock);
return (false);
}
nlen = ncp->nc_nlen;
dvp = ncp->nc_dvp;
hash = cache_get_hash(ncp->nc_name, nlen, dvp);
dvlp = VP2VNODELOCK(dvp);
blp = HASH2BUCKETLOCK(hash);
mtx_unlock(&nl->nl_lock);
mtx_unlock(&nl->nl_evict_lock);
mtx_lock(dvlp);
mtx_lock(blp);
/*
* Note that since all locks were dropped above, the entry may be
* gone or reallocated to be something else.
*/
CK_SLIST_FOREACH(ncp2, (NCHHASH(hash)), nc_hash) {
if (ncp2 == ncp && ncp2->nc_dvp == dvp &&
ncp2->nc_nlen == nlen && (ncp2->nc_flag & NCF_NEGATIVE) != 0)
break;
}
if (ncp2 == NULL) {
counter_u64_add(neg_evict_skipped_missed, 1);
ncp = NULL;
evicted = false;
} else {
MPASS(dvlp == VP2VNODELOCK(ncp->nc_dvp));
MPASS(blp == NCP2BUCKETLOCK(ncp));
SDT_PROBE2(vfs, namecache, evict_negative, done, ncp->nc_dvp,
ncp->nc_name);
cache_zap_locked(ncp);
counter_u64_add(neg_evicted, 1);
evicted = true;
}
mtx_unlock(blp);
mtx_unlock(dvlp);
if (ncp != NULL)
cache_free(ncp);
return (evicted);
}
/*
* Maybe evict a negative entry to create more room.
*
* The ncnegfactor parameter limits what fraction of the total count
* can comprise of negative entries. However, if the cache is just
* warming up this leads to excessive evictions. As such, ncnegminpct
* (recomputed to neg_min) dictates whether the above should be
* applied.
*
* Try evicting if the cache is close to full capacity regardless of
* other considerations.
*/
static bool
cache_neg_evict_cond(u_long lnumcache)
{
u_long lnumneg;
if (ncsize - 1000 < lnumcache)
goto out_evict;
lnumneg = atomic_load_long(&numneg);
if (lnumneg < neg_min)
return (false);
if (lnumneg * ncnegfactor < lnumcache)
return (false);
out_evict:
return (cache_neg_evict());
}
/*
* cache_zap_locked():
*
* Removes a namecache entry from cache, whether it contains an actual
* pointer to a vnode or if it is just a negative cache entry.
*/
static void
cache_zap_locked(struct namecache *ncp)
{
struct nchashhead *ncpp;
struct vnode *dvp, *vp;
dvp = ncp->nc_dvp;
vp = ncp->nc_vp;
if (!(ncp->nc_flag & NCF_NEGATIVE))
cache_assert_vnode_locked(vp);
cache_assert_vnode_locked(dvp);
cache_assert_bucket_locked(ncp);
cache_ncp_invalidate(ncp);
ncpp = NCP2BUCKET(ncp);
CK_SLIST_REMOVE(ncpp, ncp, namecache, nc_hash);
if (!(ncp->nc_flag & NCF_NEGATIVE)) {
SDT_PROBE3(vfs, namecache, zap, done, dvp, ncp->nc_name, vp);
TAILQ_REMOVE(&vp->v_cache_dst, ncp, nc_dst);
if (ncp == vp->v_cache_dd) {
atomic_store_ptr(&vp->v_cache_dd, NULL);
}
} else {
SDT_PROBE2(vfs, namecache, zap_negative, done, dvp, ncp->nc_name);
cache_neg_remove(ncp);
}
if (ncp->nc_flag & NCF_ISDOTDOT) {
if (ncp == dvp->v_cache_dd) {
atomic_store_ptr(&dvp->v_cache_dd, NULL);
}
} else {
LIST_REMOVE(ncp, nc_src);
if (LIST_EMPTY(&dvp->v_cache_src)) {
ncp->nc_flag |= NCF_DVDROP;
}
}
}
static void
cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp)
{
struct mtx *blp;
MPASS(ncp->nc_dvp == vp);
MPASS(ncp->nc_flag & NCF_NEGATIVE);
cache_assert_vnode_locked(vp);
blp = NCP2BUCKETLOCK(ncp);
mtx_lock(blp);
cache_zap_locked(ncp);
mtx_unlock(blp);
}
static bool
cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp,
struct mtx **vlpp)
{
struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
struct mtx *blp;
MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
cache_assert_vnode_locked(vp);
if (ncp->nc_flag & NCF_NEGATIVE) {
if (*vlpp != NULL) {
mtx_unlock(*vlpp);
*vlpp = NULL;
}
cache_zap_negative_locked_vnode_kl(ncp, vp);
return (true);
}
pvlp = VP2VNODELOCK(vp);
blp = NCP2BUCKETLOCK(ncp);
vlp1 = VP2VNODELOCK(ncp->nc_dvp);
vlp2 = VP2VNODELOCK(ncp->nc_vp);
if (*vlpp == vlp1 || *vlpp == vlp2) {
to_unlock = *vlpp;
*vlpp = NULL;
} else {
if (*vlpp != NULL) {
mtx_unlock(*vlpp);
*vlpp = NULL;
}
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 == pvlp) {
mtx_lock(vlp2);
to_unlock = vlp2;
} else {
if (!mtx_trylock(vlp1))
goto out_relock;
to_unlock = vlp1;
}
}
mtx_lock(blp);
cache_zap_locked(ncp);
mtx_unlock(blp);
if (to_unlock != NULL)
mtx_unlock(to_unlock);
return (true);
out_relock:
mtx_unlock(vlp2);
mtx_lock(vlp1);
mtx_lock(vlp2);
MPASS(*vlpp == NULL);
*vlpp = vlp1;
return (false);
}
/*
* If trylocking failed we can get here. We know enough to take all needed locks
* in the right order and re-lookup the entry.
*/
static int
cache_zap_unlocked_bucket(struct namecache *ncp, struct componentname *cnp,
struct vnode *dvp, struct mtx *dvlp, struct mtx *vlp, uint32_t hash,
struct mtx *blp)
{
struct namecache *rncp;
cache_assert_bucket_unlocked(ncp);
cache_sort_vnodes(&dvlp, &vlp);
cache_lock_vnodes(dvlp, vlp);
mtx_lock(blp);
CK_SLIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) {
if (rncp == ncp && rncp->nc_dvp == dvp &&
rncp->nc_nlen == cnp->cn_namelen &&
!bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen))
break;
}
if (rncp != NULL) {
cache_zap_locked(rncp);
mtx_unlock(blp);
cache_unlock_vnodes(dvlp, vlp);
counter_u64_add(zap_bucket_relock_success, 1);
return (0);
}
mtx_unlock(blp);
cache_unlock_vnodes(dvlp, vlp);
return (EAGAIN);
}
static int __noinline
cache_zap_locked_bucket(struct namecache *ncp, struct componentname *cnp,
uint32_t hash, struct mtx *blp)
{
struct mtx *dvlp, *vlp;
struct vnode *dvp;
cache_assert_bucket_locked(ncp);
dvlp = VP2VNODELOCK(ncp->nc_dvp);
vlp = NULL;
if (!(ncp->nc_flag & NCF_NEGATIVE))
vlp = VP2VNODELOCK(ncp->nc_vp);
if (cache_trylock_vnodes(dvlp, vlp) == 0) {
cache_zap_locked(ncp);
mtx_unlock(blp);
cache_unlock_vnodes(dvlp, vlp);
return (0);
}
dvp = ncp->nc_dvp;
mtx_unlock(blp);
return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
}
static __noinline int
cache_remove_cnp(struct vnode *dvp, struct componentname *cnp)
{
struct namecache *ncp;
struct mtx *blp;
struct mtx *dvlp, *dvlp2;
uint32_t hash;
int error;
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
dvlp = VP2VNODELOCK(dvp);
dvlp2 = NULL;
mtx_lock(dvlp);
retry_dotdot:
ncp = dvp->v_cache_dd;
if (ncp == NULL) {
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
SDT_PROBE2(vfs, namecache, removecnp, miss, dvp, cnp);
return (0);
}
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
if (!cache_zap_locked_vnode_kl2(ncp, dvp, &dvlp2))
goto retry_dotdot;
MPASS(dvp->v_cache_dd == NULL);
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
cache_free(ncp);
} else {
atomic_store_ptr(&dvp->v_cache_dd, NULL);
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
}
SDT_PROBE2(vfs, namecache, removecnp, hit, dvp, cnp);
return (1);
}
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
blp = HASH2BUCKETLOCK(hash);
retry:
if (CK_SLIST_EMPTY(NCHHASH(hash)))
goto out_no_entry;
mtx_lock(blp);
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
if (ncp == NULL) {
mtx_unlock(blp);
goto out_no_entry;
}
error = cache_zap_locked_bucket(ncp, cnp, hash, blp);
if (__predict_false(error != 0)) {
zap_bucket_fail++;
goto retry;
}
counter_u64_add(numposzaps, 1);
SDT_PROBE2(vfs, namecache, removecnp, hit, dvp, cnp);
cache_free(ncp);
return (1);
out_no_entry:
counter_u64_add(nummisszap, 1);
SDT_PROBE2(vfs, namecache, removecnp, miss, dvp, cnp);
return (0);
}
static int __noinline
cache_lookup_dot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
int ltype;
*vpp = dvp;
counter_u64_add(dothits, 1);
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
if (tsp != NULL)
timespecclear(tsp);
if (ticksp != NULL)
*ticksp = ticks;
vrefact(*vpp);
/*
* When we lookup "." we still can be asked to lock it
* differently...
*/
ltype = cnp->cn_lkflags & LK_TYPE_MASK;
if (ltype != VOP_ISLOCKED(*vpp)) {
if (ltype == LK_EXCLUSIVE) {
vn_lock(*vpp, LK_UPGRADE | LK_RETRY);
if (VN_IS_DOOMED((*vpp))) {
/* forced unmount */
vrele(*vpp);
*vpp = NULL;
return (ENOENT);
}
} else
vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
}
return (-1);
}
static int __noinline
cache_lookup_dotdot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
struct namecache_ts *ncp_ts;
struct namecache *ncp;
struct mtx *dvlp;
enum vgetstate vs;
int error, ltype;
bool whiteout;
MPASS((cnp->cn_flags & ISDOTDOT) != 0);
if ((cnp->cn_flags & MAKEENTRY) == 0) {
cache_remove_cnp(dvp, cnp);
return (0);
}
counter_u64_add(dotdothits, 1);
retry:
dvlp = VP2VNODELOCK(dvp);
mtx_lock(dvlp);
ncp = dvp->v_cache_dd;
if (ncp == NULL) {
SDT_PROBE2(vfs, namecache, lookup, miss, dvp, "..");
mtx_unlock(dvlp);
return (0);
}
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
if (ncp->nc_flag & NCF_NEGATIVE)
*vpp = NULL;
else
*vpp = ncp->nc_vp;
} else
*vpp = ncp->nc_dvp;
if (*vpp == NULL)
goto negative_success;
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..", *vpp);
cache_out_ts(ncp, tsp, ticksp);
if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
NCF_DTS && tsp != NULL) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
*tsp = ncp_ts->nc_dotdottime;
}
MPASS(dvp != *vpp);
ltype = VOP_ISLOCKED(dvp);
VOP_UNLOCK(dvp);
vs = vget_prep(*vpp);
mtx_unlock(dvlp);
error = vget_finish(*vpp, cnp->cn_lkflags, vs);
vn_lock(dvp, ltype | LK_RETRY);
if (VN_IS_DOOMED(dvp)) {
if (error == 0)
vput(*vpp);
*vpp = NULL;
return (ENOENT);
}
if (error) {
*vpp = NULL;
goto retry;
}
return (-1);
negative_success:
if (__predict_false(cnp->cn_nameiop == CREATE)) {
if (cnp->cn_flags & ISLASTCN) {
counter_u64_add(numnegzaps, 1);
cache_zap_negative_locked_vnode_kl(ncp, dvp);
mtx_unlock(dvlp);
cache_free(ncp);
return (0);
}
}
whiteout = (ncp->nc_flag & NCF_WHITE);
cache_out_ts(ncp, tsp, ticksp);
if (cache_neg_hit_prep(ncp))
cache_neg_promote(ncp);
else
cache_neg_hit_finish(ncp);
mtx_unlock(dvlp);
if (whiteout)
cnp->cn_flags |= ISWHITEOUT;
return (ENOENT);
}
/**
* Lookup a name in the name cache
*
* # Arguments
*
* - dvp: Parent directory in which to search.
* - vpp: Return argument. Will contain desired vnode on cache hit.
* - cnp: Parameters of the name search. The most interesting bits of
* the cn_flags field have the following meanings:
* - MAKEENTRY: If clear, free an entry from the cache rather than look
* it up.
* - ISDOTDOT: Must be set if and only if cn_nameptr == ".."
* - tsp: Return storage for cache timestamp. On a successful (positive
* or negative) lookup, tsp will be filled with any timespec that
* was stored when this cache entry was created. However, it will
* be clear for "." entries.
* - ticks: Return storage for alternate cache timestamp. On a successful
* (positive or negative) lookup, it will contain the ticks value
* that was current when the cache entry was created, unless cnp
* was ".".
*
* Either both tsp and ticks have to be provided or neither of them.
*
* # Returns
*
* - -1: A positive cache hit. vpp will contain the desired vnode.
* - ENOENT: A negative cache hit, or dvp was recycled out from under us due
* to a forced unmount. vpp will not be modified. If the entry
* is a whiteout, then the ISWHITEOUT flag will be set in
* cnp->cn_flags.
* - 0: A cache miss. vpp will not be modified.
*
* # Locking
*
* On a cache hit, vpp will be returned locked and ref'd. If we're looking up
* .., dvp is unlocked. If we're looking up . an extra ref is taken, but the
* lock is not recursively acquired.
*/
static int __noinline
cache_lookup_fallback(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
struct namecache *ncp;
struct mtx *blp;
uint32_t hash;
enum vgetstate vs;
int error;
bool whiteout;
MPASS((cnp->cn_flags & ISDOTDOT) == 0);
MPASS((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) != 0);
retry:
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
blp = HASH2BUCKETLOCK(hash);
mtx_lock(blp);
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
if (__predict_false(ncp == NULL)) {
mtx_unlock(blp);
SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
counter_u64_add(nummiss, 1);
return (0);
}
if (ncp->nc_flag & NCF_NEGATIVE)
goto negative_success;
counter_u64_add(numposhits, 1);
*vpp = ncp->nc_vp;
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
cache_out_ts(ncp, tsp, ticksp);
MPASS(dvp != *vpp);
vs = vget_prep(*vpp);
mtx_unlock(blp);
error = vget_finish(*vpp, cnp->cn_lkflags, vs);
if (error) {
*vpp = NULL;
goto retry;
}
return (-1);
negative_success:
/*
* We don't get here with regular lookup apart from corner cases.
*/
if (__predict_true(cnp->cn_nameiop == CREATE)) {
if (cnp->cn_flags & ISLASTCN) {
counter_u64_add(numnegzaps, 1);
error = cache_zap_locked_bucket(ncp, cnp, hash, blp);
if (__predict_false(error != 0)) {
zap_bucket_fail2++;
goto retry;
}
cache_free(ncp);
return (0);
}
}
whiteout = (ncp->nc_flag & NCF_WHITE);
cache_out_ts(ncp, tsp, ticksp);
if (cache_neg_hit_prep(ncp))
cache_neg_promote(ncp);
else
cache_neg_hit_finish(ncp);
mtx_unlock(blp);
if (whiteout)
cnp->cn_flags |= ISWHITEOUT;
return (ENOENT);
}
int
cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
struct namecache *ncp;
uint32_t hash;
enum vgetstate vs;
int error;
bool whiteout, neg_promote;
u_short nc_flag;
MPASS((tsp == NULL && ticksp == NULL) || (tsp != NULL && ticksp != NULL));
#ifdef DEBUG_CACHE
if (__predict_false(!doingcache)) {
cnp->cn_flags &= ~MAKEENTRY;
return (0);
}
#endif
if (__predict_false(cnp->cn_nameptr[0] == '.')) {
if (cnp->cn_namelen == 1)
return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp));
if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.')
return (cache_lookup_dotdot(dvp, vpp, cnp, tsp, ticksp));
}
MPASS((cnp->cn_flags & ISDOTDOT) == 0);
if ((cnp->cn_flags & (MAKEENTRY | NC_KEEPPOSENTRY)) == 0) {
cache_remove_cnp(dvp, cnp);
return (0);
}
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
vfs_smr_enter();
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
if (__predict_false(ncp == NULL)) {
vfs_smr_exit();
SDT_PROBE2(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr);
counter_u64_add(nummiss, 1);
return (0);
}
nc_flag = atomic_load_char(&ncp->nc_flag);
if (nc_flag & NCF_NEGATIVE)
goto negative_success;
counter_u64_add(numposhits, 1);
*vpp = ncp->nc_vp;
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, *vpp);
cache_out_ts(ncp, tsp, ticksp);
MPASS(dvp != *vpp);
if (!cache_ncp_canuse(ncp)) {
vfs_smr_exit();
*vpp = NULL;
goto out_fallback;
}
vs = vget_prep_smr(*vpp);
vfs_smr_exit();
if (__predict_false(vs == VGET_NONE)) {
*vpp = NULL;
goto out_fallback;
}
error = vget_finish(*vpp, cnp->cn_lkflags, vs);
if (error) {
*vpp = NULL;
goto out_fallback;
}
return (-1);
negative_success:
if (cnp->cn_nameiop == CREATE) {
if (cnp->cn_flags & ISLASTCN) {
vfs_smr_exit();
goto out_fallback;
}
}
cache_out_ts(ncp, tsp, ticksp);
whiteout = (atomic_load_char(&ncp->nc_flag) & NCF_WHITE);
neg_promote = cache_neg_hit_prep(ncp);
if (!cache_ncp_canuse(ncp)) {
cache_neg_hit_abort(ncp);
vfs_smr_exit();
goto out_fallback;
}
if (neg_promote) {
vfs_smr_exit();
if (!cache_neg_promote_cond(dvp, cnp, ncp, hash))
goto out_fallback;
} else {
cache_neg_hit_finish(ncp);
vfs_smr_exit();
}
if (whiteout)
cnp->cn_flags |= ISWHITEOUT;
return (ENOENT);
out_fallback:
return (cache_lookup_fallback(dvp, vpp, cnp, tsp, ticksp));
}
struct celockstate {
struct mtx *vlp[3];
struct mtx *blp[2];
};
CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));
static inline void
cache_celockstate_init(struct celockstate *cel)
{
bzero(cel, sizeof(*cel));
}
static void
cache_lock_vnodes_cel(struct celockstate *cel, struct vnode *vp,
struct vnode *dvp)
{
struct mtx *vlp1, *vlp2;
MPASS(cel->vlp[0] == NULL);
MPASS(cel->vlp[1] == NULL);
MPASS(cel->vlp[2] == NULL);
MPASS(vp != NULL || dvp != NULL);
vlp1 = VP2VNODELOCK(vp);
vlp2 = VP2VNODELOCK(dvp);
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 != NULL) {
mtx_lock(vlp1);
cel->vlp[0] = vlp1;
}
mtx_lock(vlp2);
cel->vlp[1] = vlp2;
}
static void
cache_unlock_vnodes_cel(struct celockstate *cel)
{
MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL);
if (cel->vlp[0] != NULL)
mtx_unlock(cel->vlp[0]);
if (cel->vlp[1] != NULL)
mtx_unlock(cel->vlp[1]);
if (cel->vlp[2] != NULL)
mtx_unlock(cel->vlp[2]);
}
static bool
cache_lock_vnodes_cel_3(struct celockstate *cel, struct vnode *vp)
{
struct mtx *vlp;
bool ret;
cache_assert_vlp_locked(cel->vlp[0]);
cache_assert_vlp_locked(cel->vlp[1]);
MPASS(cel->vlp[2] == NULL);
MPASS(vp != NULL);
vlp = VP2VNODELOCK(vp);
ret = true;
if (vlp >= cel->vlp[1]) {
mtx_lock(vlp);
} else {
if (mtx_trylock(vlp))
goto out;
cache_lock_vnodes_cel_3_failures++;
cache_unlock_vnodes_cel(cel);
if (vlp < cel->vlp[0]) {
mtx_lock(vlp);
mtx_lock(cel->vlp[0]);
mtx_lock(cel->vlp[1]);
} else {
if (cel->vlp[0] != NULL)
mtx_lock(cel->vlp[0]);
mtx_lock(vlp);
mtx_lock(cel->vlp[1]);
}
ret = false;
}
out:
cel->vlp[2] = vlp;
return (ret);
}
static void
cache_lock_buckets_cel(struct celockstate *cel, struct mtx *blp1,
struct mtx *blp2)
{
MPASS(cel->blp[0] == NULL);
MPASS(cel->blp[1] == NULL);
cache_sort_vnodes(&blp1, &blp2);
if (blp1 != NULL) {
mtx_lock(blp1);
cel->blp[0] = blp1;
}
mtx_lock(blp2);
cel->blp[1] = blp2;
}
static void
cache_unlock_buckets_cel(struct celockstate *cel)
{
if (cel->blp[0] != NULL)
mtx_unlock(cel->blp[0]);
mtx_unlock(cel->blp[1]);
}
/*
* Lock part of the cache affected by the insertion.
*
* This means vnodelocks for dvp, vp and the relevant bucketlock.
* However, insertion can result in removal of an old entry. In this
* case we have an additional vnode and bucketlock pair to lock.
*
* That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while
* preserving the locking order (smaller address first).
*/
static void
cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
uint32_t hash)
{
struct namecache *ncp;
struct mtx *blps[2];
u_char nc_flag;
blps[0] = HASH2BUCKETLOCK(hash);
for (;;) {
blps[1] = NULL;
cache_lock_vnodes_cel(cel, dvp, vp);
if (vp == NULL || vp->v_type != VDIR)
break;
ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
if (ncp == NULL)
break;
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_ISDOTDOT) == 0)
break;
MPASS(ncp->nc_dvp == vp);
blps[1] = NCP2BUCKETLOCK(ncp);
if ((nc_flag & NCF_NEGATIVE) != 0)
break;
if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
break;
/*
* All vnodes got re-locked. Re-validate the state and if
* nothing changed we are done. Otherwise restart.
*/
if (ncp == vp->v_cache_dd &&
(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
blps[1] == NCP2BUCKETLOCK(ncp) &&
VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
break;
cache_unlock_vnodes_cel(cel);
cel->vlp[0] = NULL;
cel->vlp[1] = NULL;
cel->vlp[2] = NULL;
}
cache_lock_buckets_cel(cel, blps[0], blps[1]);
}
static void
cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
uint32_t hash)
{
struct namecache *ncp;
struct mtx *blps[2];
u_char nc_flag;
blps[0] = HASH2BUCKETLOCK(hash);
for (;;) {
blps[1] = NULL;
cache_lock_vnodes_cel(cel, dvp, vp);
ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
if (ncp == NULL)
break;
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_ISDOTDOT) == 0)
break;
MPASS(ncp->nc_dvp == dvp);
blps[1] = NCP2BUCKETLOCK(ncp);
if ((nc_flag & NCF_NEGATIVE) != 0)
break;
if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
break;
if (ncp == dvp->v_cache_dd &&
(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
blps[1] == NCP2BUCKETLOCK(ncp) &&
VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
break;
cache_unlock_vnodes_cel(cel);
cel->vlp[0] = NULL;
cel->vlp[1] = NULL;
cel->vlp[2] = NULL;
}
cache_lock_buckets_cel(cel, blps[0], blps[1]);
}
static void
cache_enter_unlock(struct celockstate *cel)
{
cache_unlock_buckets_cel(cel);
cache_unlock_vnodes_cel(cel);
}
static void __noinline
cache_enter_dotdot_prep(struct vnode *dvp, struct vnode *vp,
struct componentname *cnp)
{
struct celockstate cel;
struct namecache *ncp;
uint32_t hash;
int len;
if (atomic_load_ptr(&dvp->v_cache_dd) == NULL)
return;
len = cnp->cn_namelen;
cache_celockstate_init(&cel);
hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
cache_enter_lock_dd(&cel, dvp, vp, hash);
ncp = dvp->v_cache_dd;
if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) {
KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent"));
cache_zap_locked(ncp);
} else {
ncp = NULL;
}
atomic_store_ptr(&dvp->v_cache_dd, NULL);
cache_enter_unlock(&cel);
if (ncp != NULL)
cache_free(ncp);
}
/*
* Add an entry to the cache.
*/
void
cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
struct timespec *tsp, struct timespec *dtsp)
{
struct celockstate cel;
struct namecache *ncp, *n2, *ndd;
struct namecache_ts *ncp_ts;
struct nchashhead *ncpp;
uint32_t hash;
int flag;
int len;
KASSERT(cnp->cn_namelen <= NAME_MAX,
("%s: passed len %ld exceeds NAME_MAX (%d)", __func__, cnp->cn_namelen,
NAME_MAX));
VNPASS(!VN_IS_DOOMED(dvp), dvp);
VNPASS(dvp->v_type != VNON, dvp);
if (vp != NULL) {
VNPASS(!VN_IS_DOOMED(vp), vp);
VNPASS(vp->v_type != VNON, vp);
}
if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.') {
KASSERT(dvp == vp,
("%s: different vnodes for dot entry (%p; %p)\n", __func__,
dvp, vp));
} else {
KASSERT(dvp != vp,
("%s: same vnode for non-dot entry [%s] (%p)\n", __func__,
cnp->cn_nameptr, dvp));
}
#ifdef DEBUG_CACHE
if (__predict_false(!doingcache))
return;
#endif
flag = 0;
if (__predict_false(cnp->cn_nameptr[0] == '.')) {
if (cnp->cn_namelen == 1)
return;
if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
cache_enter_dotdot_prep(dvp, vp, cnp);
flag = NCF_ISDOTDOT;
}
}
ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
if (ncp == NULL)
return;
cache_celockstate_init(&cel);
ndd = NULL;
ncp_ts = NULL;
/*
* Calculate the hash key and setup as much of the new
* namecache entry as possible before acquiring the lock.
*/
ncp->nc_flag = flag | NCF_WIP;
ncp->nc_vp = vp;
if (vp == NULL)
cache_neg_init(ncp);
ncp->nc_dvp = dvp;
if (tsp != NULL) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
ncp_ts->nc_time = *tsp;
ncp_ts->nc_ticks = ticks;
ncp_ts->nc_nc.nc_flag |= NCF_TS;
if (dtsp != NULL) {
ncp_ts->nc_dotdottime = *dtsp;
ncp_ts->nc_nc.nc_flag |= NCF_DTS;
}
}
len = ncp->nc_nlen = cnp->cn_namelen;
hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
memcpy(ncp->nc_name, cnp->cn_nameptr, len);
ncp->nc_name[len] = '\0';
cache_enter_lock(&cel, dvp, vp, hash);
/*
* See if this vnode or negative entry is already in the cache
* with this name. This can happen with concurrent lookups of
* the same path name.
*/
ncpp = NCHHASH(hash);
CK_SLIST_FOREACH(n2, ncpp, nc_hash) {
if (n2->nc_dvp == dvp &&
n2->nc_nlen == cnp->cn_namelen &&
!bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) {
MPASS(cache_ncp_canuse(n2));
if ((n2->nc_flag & NCF_NEGATIVE) != 0)
KASSERT(vp == NULL,
("%s: found entry pointing to a different vnode (%p != %p) ; name [%s]",
__func__, NULL, vp, cnp->cn_nameptr));
else
KASSERT(n2->nc_vp == vp,
("%s: found entry pointing to a different vnode (%p != %p) ; name [%s]",
__func__, n2->nc_vp, vp, cnp->cn_nameptr));
/*
* Entries are supposed to be immutable unless in the
* process of getting destroyed. Accommodating for
* changing timestamps is possible but not worth it.
* This should be harmless in terms of correctness, in
* the worst case resulting in an earlier expiration.
* Alternatively, the found entry can be replaced
* altogether.
*/
MPASS((n2->nc_flag & (NCF_TS | NCF_DTS)) == (ncp->nc_flag & (NCF_TS | NCF_DTS)));
#if 0
if (tsp != NULL) {
KASSERT((n2->nc_flag & NCF_TS) != 0,
("no NCF_TS"));
n2_ts = __containerof(n2, struct namecache_ts, nc_nc);
n2_ts->nc_time = ncp_ts->nc_time;
n2_ts->nc_ticks = ncp_ts->nc_ticks;
if (dtsp != NULL) {
n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
n2_ts->nc_nc.nc_flag |= NCF_DTS;
}
}
#endif
SDT_PROBE3(vfs, namecache, enter, duplicate, dvp, ncp->nc_name,
vp);
goto out_unlock_free;
}
}
if (flag == NCF_ISDOTDOT) {
/*
* See if we are trying to add .. entry, but some other lookup
* has populated v_cache_dd pointer already.
*/
if (dvp->v_cache_dd != NULL)
goto out_unlock_free;
KASSERT(vp == NULL || vp->v_type == VDIR,
("wrong vnode type %p", vp));
atomic_thread_fence_rel();
atomic_store_ptr(&dvp->v_cache_dd, ncp);
}
if (vp != NULL) {
if (flag != NCF_ISDOTDOT) {
/*
* For this case, the cache entry maps both the
* directory name in it and the name ".." for the
* directory's parent.
*/
if ((ndd = vp->v_cache_dd) != NULL) {
if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
cache_zap_locked(ndd);
else
ndd = NULL;
}
atomic_thread_fence_rel();
atomic_store_ptr(&vp->v_cache_dd, ncp);
} else if (vp->v_type != VDIR) {
if (vp->v_cache_dd != NULL) {
atomic_store_ptr(&vp->v_cache_dd, NULL);
}
}
}
if (flag != NCF_ISDOTDOT) {
if (LIST_EMPTY(&dvp->v_cache_src)) {
cache_hold_vnode(dvp);
}
LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
}
/*
* If the entry is "negative", we place it into the
* "negative" cache queue, otherwise, we place it into the
* destination vnode's cache entries queue.
*/
if (vp != NULL) {
TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
vp);
} else {
if (cnp->cn_flags & ISWHITEOUT)
atomic_store_char(&ncp->nc_flag, ncp->nc_flag | NCF_WHITE);
cache_neg_insert(ncp);
SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
ncp->nc_name);
}
/*
* Insert the new namecache entry into the appropriate chain
* within the cache entries table.
*/
CK_SLIST_INSERT_HEAD(ncpp, ncp, nc_hash);
atomic_thread_fence_rel();
/*
* Mark the entry as fully constructed.
* It is immutable past this point until its removal.
*/
atomic_store_char(&ncp->nc_flag, ncp->nc_flag & ~NCF_WIP);
cache_enter_unlock(&cel);
if (ndd != NULL)
cache_free(ndd);
return;
out_unlock_free:
cache_enter_unlock(&cel);
cache_free(ncp);
return;
}
/*
* A variant of the above accepting flags.
*
* - VFS_CACHE_DROPOLD -- if a conflicting entry is found, drop it.
*
* TODO: this routine is a hack. It blindly removes the old entry, even if it
* happens to match and it is doing it in an inefficient manner. It was added
* to accomodate NFS which runs into a case where the target for a given name
* may change from under it. Note this does nothing to solve the following
* race: 2 callers of cache_enter_time_flags pass a different target vnode for
* the same [dvp, cnp]. It may be argued that code doing this is broken.
*/
void
cache_enter_time_flags(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
struct timespec *tsp, struct timespec *dtsp, int flags)
{
MPASS((flags & ~(VFS_CACHE_DROPOLD)) == 0);
if (flags & VFS_CACHE_DROPOLD)
cache_remove_cnp(dvp, cnp);
cache_enter_time(dvp, vp, cnp, tsp, dtsp);
}
static u_int
cache_roundup_2(u_int val)
{
u_int res;
for (res = 1; res <= val; res <<= 1)
continue;
return (res);
}
static struct nchashhead *
nchinittbl(u_long elements, u_long *hashmask)
{
struct nchashhead *hashtbl;
u_long hashsize, i;
hashsize = cache_roundup_2(elements) / 2;
hashtbl = malloc((u_long)hashsize * sizeof(*hashtbl), M_VFSCACHE, M_WAITOK);
for (i = 0; i < hashsize; i++)
CK_SLIST_INIT(&hashtbl[i]);
*hashmask = hashsize - 1;
return (hashtbl);
}
static void
ncfreetbl(struct nchashhead *hashtbl)
{
free(hashtbl, M_VFSCACHE);
}
/*
* Name cache initialization, from vfs_init() when we are booting
*/
static void
nchinit(void *dummy __unused)
{
u_int i;
cache_zone_small = uma_zcreate("S VFS Cache", CACHE_ZONE_SMALL_SIZE,
NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
cache_zone_small_ts = uma_zcreate("STS VFS Cache", CACHE_ZONE_SMALL_TS_SIZE,
NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
cache_zone_large = uma_zcreate("L VFS Cache", CACHE_ZONE_LARGE_SIZE,
NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
cache_zone_large_ts = uma_zcreate("LTS VFS Cache", CACHE_ZONE_LARGE_TS_SIZE,
NULL, NULL, NULL, NULL, CACHE_ZONE_ALIGNMENT, UMA_ZONE_ZINIT);
VFS_SMR_ZONE_SET(cache_zone_small);
VFS_SMR_ZONE_SET(cache_zone_small_ts);
VFS_SMR_ZONE_SET(cache_zone_large);
VFS_SMR_ZONE_SET(cache_zone_large_ts);
ncsize = desiredvnodes * ncsizefactor;
cache_recalc_neg_min(ncnegminpct);
nchashtbl = nchinittbl(desiredvnodes * 2, &nchash);
ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1;
if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */
ncbuckethash = 7;
if (ncbuckethash > nchash)
ncbuckethash = nchash;
bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE,
M_WAITOK | M_ZERO);
for (i = 0; i < numbucketlocks; i++)
mtx_init(&bucketlocks[i], "ncbuc", NULL, MTX_DUPOK | MTX_RECURSE);
ncvnodehash = ncbuckethash;
vnodelocks = malloc(sizeof(*vnodelocks) * numvnodelocks, M_VFSCACHE,
M_WAITOK | M_ZERO);
for (i = 0; i < numvnodelocks; i++)
mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK | MTX_RECURSE);
for (i = 0; i < numneglists; i++) {
mtx_init(&neglists[i].nl_evict_lock, "ncnege", NULL, MTX_DEF);
mtx_init(&neglists[i].nl_lock, "ncnegl", NULL, MTX_DEF);
TAILQ_INIT(&neglists[i].nl_list);
TAILQ_INIT(&neglists[i].nl_hotlist);
}
}
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);
void
cache_vnode_init(struct vnode *vp)
{
LIST_INIT(&vp->v_cache_src);
TAILQ_INIT(&vp->v_cache_dst);
vp->v_cache_dd = NULL;
cache_prehash(vp);
}
/*
* Induce transient cache misses for lockless operation in cache_lookup() by
* using a temporary hash table.
*
* This will force a fs lookup.
*
* Synchronisation is done in 2 steps, calling vfs_smr_synchronize each time
* to observe all CPUs not performing the lookup.
*/
static void
cache_changesize_set_temp(struct nchashhead *temptbl, u_long temphash)
{
MPASS(temphash < nchash);
/*
* Change the size. The new size is smaller and can safely be used
* against the existing table. All lookups which now hash wrong will
* result in a cache miss, which all callers are supposed to know how
* to handle.
*/
atomic_store_long(&nchash, temphash);
atomic_thread_fence_rel();
vfs_smr_synchronize();
/*
* At this point everyone sees the updated hash value, but they still
* see the old table.
*/
atomic_store_ptr(&nchashtbl, temptbl);
atomic_thread_fence_rel();
vfs_smr_synchronize();
/*
* At this point everyone sees the updated table pointer and size pair.
*/
}
/*
* Set the new hash table.
*
* Similarly to cache_changesize_set_temp(), this has to synchronize against
* lockless operation in cache_lookup().
*/
static void
cache_changesize_set_new(struct nchashhead *new_tbl, u_long new_hash)
{
MPASS(nchash < new_hash);
/*
* Change the pointer first. This wont result in out of bounds access
* since the temporary table is guaranteed to be smaller.
*/
atomic_store_ptr(&nchashtbl, new_tbl);
atomic_thread_fence_rel();
vfs_smr_synchronize();
/*
* At this point everyone sees the updated pointer value, but they
* still see the old size.
*/
atomic_store_long(&nchash, new_hash);
atomic_thread_fence_rel();
vfs_smr_synchronize();
/*
* At this point everyone sees the updated table pointer and size pair.
*/
}
void
cache_changesize(u_long newmaxvnodes)
{
struct nchashhead *new_nchashtbl, *old_nchashtbl, *temptbl;
u_long new_nchash, old_nchash, temphash;
struct namecache *ncp;
uint32_t hash;
u_long newncsize;
int i;
newncsize = newmaxvnodes * ncsizefactor;
newmaxvnodes = cache_roundup_2(newmaxvnodes * 2);
if (newmaxvnodes < numbucketlocks)
newmaxvnodes = numbucketlocks;
new_nchashtbl = nchinittbl(newmaxvnodes, &new_nchash);
/* If same hash table size, nothing to do */
if (nchash == new_nchash) {
ncfreetbl(new_nchashtbl);
return;
}
temptbl = nchinittbl(1, &temphash);
/*
* Move everything from the old hash table to the new table.
* None of the namecache entries in the table can be removed
* because to do so, they have to be removed from the hash table.
*/
cache_lock_all_vnodes();
cache_lock_all_buckets();
old_nchashtbl = nchashtbl;
old_nchash = nchash;
cache_changesize_set_temp(temptbl, temphash);
for (i = 0; i <= old_nchash; i++) {
while ((ncp = CK_SLIST_FIRST(&old_nchashtbl[i])) != NULL) {
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
ncp->nc_dvp);
CK_SLIST_REMOVE(&old_nchashtbl[i], ncp, namecache, nc_hash);
CK_SLIST_INSERT_HEAD(&new_nchashtbl[hash & new_nchash], ncp, nc_hash);
}
}
ncsize = newncsize;
cache_recalc_neg_min(ncnegminpct);
cache_changesize_set_new(new_nchashtbl, new_nchash);
cache_unlock_all_buckets();
cache_unlock_all_vnodes();
ncfreetbl(old_nchashtbl);
ncfreetbl(temptbl);
}
/*
* Remove all entries from and to a particular vnode.
*/
static void
cache_purge_impl(struct vnode *vp)
{
struct cache_freebatch batch;
struct namecache *ncp;
struct mtx *vlp, *vlp2;
TAILQ_INIT(&batch);
vlp = VP2VNODELOCK(vp);
vlp2 = NULL;
mtx_lock(vlp);
retry:
while (!LIST_EMPTY(&vp->v_cache_src)) {
ncp = LIST_FIRST(&vp->v_cache_src);
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
}
while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
ncp = TAILQ_FIRST(&vp->v_cache_dst);
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
}
ncp = vp->v_cache_dd;
if (ncp != NULL) {
KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
("lost dotdot link"));
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
}
KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
mtx_unlock(vlp);
if (vlp2 != NULL)
mtx_unlock(vlp2);
cache_free_batch(&batch);
}
/*
* Opportunistic check to see if there is anything to do.
*/
static bool
cache_has_entries(struct vnode *vp)
{
if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
atomic_load_ptr(&vp->v_cache_dd) == NULL)
return (false);
return (true);
}
void
cache_purge(struct vnode *vp)
{
SDT_PROBE1(vfs, namecache, purge, done, vp);
if (!cache_has_entries(vp))
return;
cache_purge_impl(vp);
}
/*
* Only to be used by vgone.
*/
void
cache_purge_vgone(struct vnode *vp)
{
struct mtx *vlp;
VNPASS(VN_IS_DOOMED(vp), vp);
if (cache_has_entries(vp)) {
cache_purge_impl(vp);
return;
}
/*
* Serialize against a potential thread doing cache_purge.
*/
vlp = VP2VNODELOCK(vp);
mtx_wait_unlocked(vlp);
if (cache_has_entries(vp)) {
cache_purge_impl(vp);
return;
}
return;
}
/*
* Remove all negative entries for a particular directory vnode.
*/
void
cache_purge_negative(struct vnode *vp)
{
struct cache_freebatch batch;
struct namecache *ncp, *nnp;
struct mtx *vlp;
SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
if (LIST_EMPTY(&vp->v_cache_src))
return;
TAILQ_INIT(&batch);
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
if (!(ncp->nc_flag & NCF_NEGATIVE))
continue;
cache_zap_negative_locked_vnode_kl(ncp, vp);
TAILQ_INSERT_TAIL(&batch, ncp, nc_dst);
}
mtx_unlock(vlp);
cache_free_batch(&batch);
}
/*
* Entry points for modifying VOP operations.
*/
void
cache_vop_rename(struct vnode *fdvp, struct vnode *fvp, struct vnode *tdvp,
struct vnode *tvp, struct componentname *fcnp, struct componentname *tcnp)
{
ASSERT_VOP_IN_SEQC(fdvp);
ASSERT_VOP_IN_SEQC(fvp);
ASSERT_VOP_IN_SEQC(tdvp);
if (tvp != NULL)
ASSERT_VOP_IN_SEQC(tvp);
cache_purge(fvp);
if (tvp != NULL) {
cache_purge(tvp);
KASSERT(!cache_remove_cnp(tdvp, tcnp),
("%s: lingering negative entry", __func__));
} else {
cache_remove_cnp(tdvp, tcnp);
}
/*
* TODO
*
* Historically renaming was always purging all revelang entries,
* but that's quite wasteful. In particular turns out that in many cases
* the target file is immediately accessed after rename, inducing a cache
* miss.
*
* Recode this to reduce relocking and reuse the existing entry (if any)
* instead of just removing it above and allocating a new one here.
*/
if (cache_rename_add) {
cache_enter(tdvp, fvp, tcnp);
}
}
void
cache_vop_rmdir(struct vnode *dvp, struct vnode *vp)
{
ASSERT_VOP_IN_SEQC(dvp);
ASSERT_VOP_IN_SEQC(vp);
cache_purge(vp);
}
#ifdef INVARIANTS
/*
* Validate that if an entry exists it matches.
*/
void
cache_validate(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
{
struct namecache *ncp;
struct mtx *blp;
uint32_t hash;
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
if (CK_SLIST_EMPTY(NCHHASH(hash)))
return;
blp = HASH2BUCKETLOCK(hash);
mtx_lock(blp);
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen)) {
if (ncp->nc_vp != vp)
panic("%s: mismatch (%p != %p); ncp %p [%s] dvp %p\n",
__func__, vp, ncp->nc_vp, ncp, ncp->nc_name, ncp->nc_dvp);
}
}
mtx_unlock(blp);
}
#endif
/*
* Flush all entries referencing a particular filesystem.
*/
void
cache_purgevfs(struct mount *mp)
{
struct vnode *vp, *mvp;
size_t visited, purged;
visited = purged = 0;
/*
* Somewhat wasteful iteration over all vnodes. Would be better to
* support filtering and avoid the interlock to begin with.
*/
MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
visited++;
if (!cache_has_entries(vp)) {
VI_UNLOCK(vp);
continue;
}
vholdl(vp);
VI_UNLOCK(vp);
cache_purge(vp);
purged++;
vdrop(vp);
}
SDT_PROBE3(vfs, namecache, purgevfs, done, mp, visited, purged);
}
/*
* Perform canonical checks and cache lookup and pass on to filesystem
* through the vop_cachedlookup only if needed.
*/
int
vfs_cache_lookup(struct vop_lookup_args *ap)
{
struct vnode *dvp;
int error;
struct vnode **vpp = ap->a_vpp;
struct componentname *cnp = ap->a_cnp;
int flags = cnp->cn_flags;
*vpp = NULL;
dvp = ap->a_dvp;
if (dvp->v_type != VDIR)
return (ENOTDIR);
if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
return (EROFS);
error = vn_dir_check_exec(dvp, cnp);
if (error != 0)
return (error);
error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
if (error == 0)
return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
if (error == -1)
return (0);
return (error);
}
/* Implementation of the getcwd syscall. */
int
sys___getcwd(struct thread *td, struct __getcwd_args *uap)
{
char *buf, *retbuf;
size_t buflen;
int error;
buflen = uap->buflen;
if (__predict_false(buflen < 2))
return (EINVAL);
if (buflen > MAXPATHLEN)
buflen = MAXPATHLEN;
buf = uma_zalloc(namei_zone, M_WAITOK);
error = vn_getcwd(buf, &retbuf, &buflen);
if (error == 0)
error = copyout(retbuf, uap->buf, buflen);
uma_zfree(namei_zone, buf);
return (error);
}
int
vn_getcwd(char *buf, char **retbuf, size_t *buflen)
{
struct pwd *pwd;
int error;
vfs_smr_enter();
pwd = pwd_get_smr();
error = vn_fullpath_any_smr(pwd->pwd_cdir, pwd->pwd_rdir, buf, retbuf,
buflen, 0);
VFS_SMR_ASSERT_NOT_ENTERED();
if (error < 0) {
pwd = pwd_hold(curthread);
error = vn_fullpath_any(pwd->pwd_cdir, pwd->pwd_rdir, buf,
retbuf, buflen);
pwd_drop(pwd);
}
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_NAMEI) && error == 0)
ktrnamei(*retbuf);
#endif
return (error);
}
static int
kern___realpathat(struct thread *td, int fd, const char *path, char *buf,
size_t size, int flags, enum uio_seg pathseg)
{
struct nameidata nd;
char *retbuf, *freebuf;
int error;
if (flags != 0)
return (EINVAL);
NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | SAVENAME | WANTPARENT | AUDITVNODE1,
pathseg, path, fd, &cap_fstat_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
error = vn_fullpath_hardlink(nd.ni_vp, nd.ni_dvp, nd.ni_cnd.cn_nameptr,
nd.ni_cnd.cn_namelen, &retbuf, &freebuf, &size);
if (error == 0) {
error = copyout(retbuf, buf, size);
free(freebuf, M_TEMP);
}
NDFREE(&nd, 0);
return (error);
}
int
sys___realpathat(struct thread *td, struct __realpathat_args *uap)
{
return (kern___realpathat(td, uap->fd, uap->path, uap->buf, uap->size,
uap->flags, UIO_USERSPACE));
}
/*
* Retrieve the full filesystem path that correspond to a vnode from the name
* cache (if available)
*/
int
vn_fullpath(struct vnode *vp, char **retbuf, char **freebuf)
{
struct pwd *pwd;
char *buf;
size_t buflen;
int error;
if (__predict_false(vp == NULL))
return (EINVAL);
buflen = MAXPATHLEN;
buf = malloc(buflen, M_TEMP, M_WAITOK);
vfs_smr_enter();
pwd = pwd_get_smr();
error = vn_fullpath_any_smr(vp, pwd->pwd_rdir, buf, retbuf, &buflen, 0);
VFS_SMR_ASSERT_NOT_ENTERED();
if (error < 0) {
pwd = pwd_hold(curthread);
error = vn_fullpath_any(vp, pwd->pwd_rdir, buf, retbuf, &buflen);
pwd_drop(pwd);
}
if (error == 0)
*freebuf = buf;
else
free(buf, M_TEMP);
return (error);
}
/*
* This function is similar to vn_fullpath, but it attempts to lookup the
* pathname relative to the global root mount point. This is required for the
* auditing sub-system, as audited pathnames must be absolute, relative to the
* global root mount point.
*/
int
vn_fullpath_global(struct vnode *vp, char **retbuf, char **freebuf)
{
char *buf;
size_t buflen;
int error;
if (__predict_false(vp == NULL))
return (EINVAL);
buflen = MAXPATHLEN;
buf = malloc(buflen, M_TEMP, M_WAITOK);
vfs_smr_enter();
error = vn_fullpath_any_smr(vp, rootvnode, buf, retbuf, &buflen, 0);
VFS_SMR_ASSERT_NOT_ENTERED();
if (error < 0) {
error = vn_fullpath_any(vp, rootvnode, buf, retbuf, &buflen);
}
if (error == 0)
*freebuf = buf;
else
free(buf, M_TEMP);
return (error);
}
static struct namecache *
vn_dd_from_dst(struct vnode *vp)
{
struct namecache *ncp;
cache_assert_vnode_locked(vp);
TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst) {
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
return (ncp);
}
return (NULL);
}
int
vn_vptocnp(struct vnode **vp, char *buf, size_t *buflen)
{
struct vnode *dvp;
struct namecache *ncp;
struct mtx *vlp;
int error;
vlp = VP2VNODELOCK(*vp);
mtx_lock(vlp);
ncp = (*vp)->v_cache_dd;
if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT) == 0) {
KASSERT(ncp == vn_dd_from_dst(*vp),
("%s: mismatch for dd entry (%p != %p)", __func__,
ncp, vn_dd_from_dst(*vp)));
} else {
ncp = vn_dd_from_dst(*vp);
}
if (ncp != NULL) {
if (*buflen < ncp->nc_nlen) {
mtx_unlock(vlp);
vrele(*vp);
counter_u64_add(numfullpathfail4, 1);
error = ENOMEM;
SDT_PROBE3(vfs, namecache, fullpath, return, error,
vp, NULL);
return (error);
}
*buflen -= ncp->nc_nlen;
memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
ncp->nc_name, vp);
dvp = *vp;
*vp = ncp->nc_dvp;
vref(*vp);
mtx_unlock(vlp);
vrele(dvp);
return (0);
}
SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
mtx_unlock(vlp);
vn_lock(*vp, LK_SHARED | LK_RETRY);
error = VOP_VPTOCNP(*vp, &dvp, buf, buflen);
vput(*vp);
if (error) {
counter_u64_add(numfullpathfail2, 1);
SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
return (error);
}
*vp = dvp;
if (VN_IS_DOOMED(dvp)) {
/* forced unmount */
vrele(dvp);
error = ENOENT;
SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
return (error);
}
/*
* *vp has its use count incremented still.
*/
return (0);
}
/*
* Resolve a directory to a pathname.
*
* The name of the directory can always be found in the namecache or fetched
* from the filesystem. There is also guaranteed to be only one parent, meaning
* we can just follow vnodes up until we find the root.
*
* The vnode must be referenced.
*/
static int
vn_fullpath_dir(struct vnode *vp, struct vnode *rdir, char *buf, char **retbuf,
size_t *len, size_t addend)
{
#ifdef KDTRACE_HOOKS
struct vnode *startvp = vp;
#endif
struct vnode *vp1;
size_t buflen;
int error;
bool slash_prefixed;
VNPASS(vp->v_type == VDIR || VN_IS_DOOMED(vp), vp);
VNPASS(vp->v_usecount > 0, vp);
buflen = *len;
slash_prefixed = true;
if (addend == 0) {
MPASS(*len >= 2);
buflen--;
buf[buflen] = '\0';
slash_prefixed = false;
}
error = 0;
SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
counter_u64_add(numfullpathcalls, 1);
while (vp != rdir && vp != rootvnode) {
/*
* The vp vnode must be already fully constructed,
* since it is either found in namecache or obtained
* from VOP_VPTOCNP(). We may test for VV_ROOT safely
* without obtaining the vnode lock.
*/
if ((vp->v_vflag & VV_ROOT) != 0) {
vn_lock(vp, LK_RETRY | LK_SHARED);
/*
* With the vnode locked, check for races with
* unmount, forced or not. Note that we
* already verified that vp is not equal to
* the root vnode, which means that
* mnt_vnodecovered can be NULL only for the
* case of unmount.
*/
if (VN_IS_DOOMED(vp) ||
(vp1 = vp->v_mount->mnt_vnodecovered) == NULL ||
vp1->v_mountedhere != vp->v_mount) {
vput(vp);
error = ENOENT;
SDT_PROBE3(vfs, namecache, fullpath, return,
error, vp, NULL);
break;
}
vref(vp1);
vput(vp);
vp = vp1;
continue;
}
if (vp->v_type != VDIR) {
vrele(vp);
counter_u64_add(numfullpathfail1, 1);
error = ENOTDIR;
SDT_PROBE3(vfs, namecache, fullpath, return,
error, vp, NULL);
break;
}
error = vn_vptocnp(&vp, buf, &buflen);
if (error)
break;
if (buflen == 0) {
vrele(vp);
error = ENOMEM;
SDT_PROBE3(vfs, namecache, fullpath, return, error,
startvp, NULL);
break;
}
buf[--buflen] = '/';
slash_prefixed = true;
}
if (error)
return (error);
if (!slash_prefixed) {
if (buflen == 0) {
vrele(vp);
counter_u64_add(numfullpathfail4, 1);
SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
startvp, NULL);
return (ENOMEM);
}
buf[--buflen] = '/';
}
counter_u64_add(numfullpathfound, 1);
vrele(vp);
*retbuf = buf + buflen;
SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, *retbuf);
*len -= buflen;
*len += addend;
return (0);
}
/*
* Resolve an arbitrary vnode to a pathname.
*
* Note 2 caveats:
* - hardlinks are not tracked, thus if the vnode is not a directory this can
* resolve to a different path than the one used to find it
* - namecache is not mandatory, meaning names are not guaranteed to be added
* (in which case resolving fails)
*/
static void __inline
cache_rev_failed_impl(int *reason, int line)
{
*reason = line;
}
#define cache_rev_failed(var) cache_rev_failed_impl((var), __LINE__)
static int
vn_fullpath_any_smr(struct vnode *vp, struct vnode *rdir, char *buf,
char **retbuf, size_t *buflen, size_t addend)
{
#ifdef KDTRACE_HOOKS
struct vnode *startvp = vp;
#endif
struct vnode *tvp;
struct mount *mp;
struct namecache *ncp;
size_t orig_buflen;
int reason;
int error;
#ifdef KDTRACE_HOOKS
int i;
#endif
seqc_t vp_seqc, tvp_seqc;
u_char nc_flag;
VFS_SMR_ASSERT_ENTERED();
if (!atomic_load_char(&cache_fast_lookup_enabled)) {
vfs_smr_exit();
return (-1);
}
orig_buflen = *buflen;
if (addend == 0) {
MPASS(*buflen >= 2);
*buflen -= 1;
buf[*buflen] = '\0';
}
if (vp == rdir || vp == rootvnode) {
if (addend == 0) {
*buflen -= 1;
buf[*buflen] = '/';
}
goto out_ok;
}
#ifdef KDTRACE_HOOKS
i = 0;
#endif
error = -1;
ncp = NULL; /* for sdt probe down below */
vp_seqc = vn_seqc_read_any(vp);
if (seqc_in_modify(vp_seqc)) {
cache_rev_failed(&reason);
goto out_abort;
}
for (;;) {
#ifdef KDTRACE_HOOKS
i++;
#endif
if ((vp->v_vflag & VV_ROOT) != 0) {
mp = atomic_load_ptr(&vp->v_mount);
if (mp == NULL) {
cache_rev_failed(&reason);
goto out_abort;
}
tvp = atomic_load_ptr(&mp->mnt_vnodecovered);
tvp_seqc = vn_seqc_read_any(tvp);
if (seqc_in_modify(tvp_seqc)) {
cache_rev_failed(&reason);
goto out_abort;
}
if (!vn_seqc_consistent(vp, vp_seqc)) {
cache_rev_failed(&reason);
goto out_abort;
}
vp = tvp;
vp_seqc = tvp_seqc;
continue;
}
ncp = atomic_load_consume_ptr(&vp->v_cache_dd);
if (ncp == NULL) {
cache_rev_failed(&reason);
goto out_abort;
}
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_ISDOTDOT) != 0) {
cache_rev_failed(&reason);
goto out_abort;
}
if (ncp->nc_nlen >= *buflen) {
cache_rev_failed(&reason);
error = ENOMEM;
goto out_abort;
}
*buflen -= ncp->nc_nlen;
memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
*buflen -= 1;
buf[*buflen] = '/';
tvp = ncp->nc_dvp;
tvp_seqc = vn_seqc_read_any(tvp);
if (seqc_in_modify(tvp_seqc)) {
cache_rev_failed(&reason);
goto out_abort;
}
if (!vn_seqc_consistent(vp, vp_seqc)) {
cache_rev_failed(&reason);
goto out_abort;
}
/*
* Acquire fence provided by vn_seqc_read_any above.
*/
if (__predict_false(atomic_load_ptr(&vp->v_cache_dd) != ncp)) {
cache_rev_failed(&reason);
goto out_abort;
}
if (!cache_ncp_canuse(ncp)) {
cache_rev_failed(&reason);
goto out_abort;
}
vp = tvp;
vp_seqc = tvp_seqc;
if (vp == rdir || vp == rootvnode)
break;
}
out_ok:
vfs_smr_exit();
*retbuf = buf + *buflen;
*buflen = orig_buflen - *buflen + addend;
SDT_PROBE2(vfs, namecache, fullpath_smr, hit, startvp, *retbuf);
return (0);
out_abort:
*buflen = orig_buflen;
SDT_PROBE4(vfs, namecache, fullpath_smr, miss, startvp, ncp, reason, i);
vfs_smr_exit();
return (error);
}
static int
vn_fullpath_any(struct vnode *vp, struct vnode *rdir, char *buf, char **retbuf,
size_t *buflen)
{
size_t orig_buflen, addend;
int error;
if (*buflen < 2)
return (EINVAL);
orig_buflen = *buflen;
vref(vp);
addend = 0;
if (vp->v_type != VDIR) {
*buflen -= 1;
buf[*buflen] = '\0';
error = vn_vptocnp(&vp, buf, buflen);
if (error)
return (error);
if (*buflen == 0) {
vrele(vp);
return (ENOMEM);
}
*buflen -= 1;
buf[*buflen] = '/';
addend = orig_buflen - *buflen;
}
return (vn_fullpath_dir(vp, rdir, buf, retbuf, buflen, addend));
}
/*
* Resolve an arbitrary vnode to a pathname (taking care of hardlinks).
*
* Since the namecache does not track hardlinks, the caller is
* expected to first look up the target vnode with SAVENAME |
* WANTPARENT flags passed to namei to get dvp and vp.
*
* Then we have 2 cases:
* - if the found vnode is a directory, the path can be constructed just by
* following names up the chain
* - otherwise we populate the buffer with the saved name and start resolving
* from the parent
*/
int
vn_fullpath_hardlink(struct vnode *vp, struct vnode *dvp,
const char *hrdl_name, size_t hrdl_name_length,
char **retbuf, char **freebuf, size_t *buflen)
{
char *buf, *tmpbuf;
struct pwd *pwd;
size_t addend;
int error;
enum vtype type;
if (*buflen < 2)
return (EINVAL);
if (*buflen > MAXPATHLEN)
*buflen = MAXPATHLEN;
buf = malloc(*buflen, M_TEMP, M_WAITOK);
addend = 0;
/*
* Check for VBAD to work around the vp_crossmp bug in lookup().
*
* For example consider tmpfs on /tmp and realpath /tmp. ni_vp will be
* set to mount point's root vnode while ni_dvp will be vp_crossmp.
* If the type is VDIR (like in this very case) we can skip looking
* at ni_dvp in the first place. However, since vnodes get passed here
* unlocked the target may transition to doomed state (type == VBAD)
* before we get to evaluate the condition. If this happens, we will
* populate part of the buffer and descend to vn_fullpath_dir with
* vp == vp_crossmp. Prevent the problem by checking for VBAD.
*
* This should be atomic_load(&vp->v_type) but it is illegal to take
* an address of a bit field, even if said field is sized to char.
* Work around the problem by reading the value into a full-sized enum
* and then re-reading it with atomic_load which will still prevent
* the compiler from re-reading down the road.
*/
type = vp->v_type;
type = atomic_load_int(&type);
if (type == VBAD) {
error = ENOENT;
goto out_bad;
}
if (type != VDIR) {
addend = hrdl_name_length + 2;
if (*buflen < addend) {
error = ENOMEM;
goto out_bad;
}
*buflen -= addend;
tmpbuf = buf + *buflen;
tmpbuf[0] = '/';
memcpy(&tmpbuf[1], hrdl_name, hrdl_name_length);
tmpbuf[addend - 1] = '\0';
vp = dvp;
}
vfs_smr_enter();
pwd = pwd_get_smr();
error = vn_fullpath_any_smr(vp, pwd->pwd_rdir, buf, retbuf, buflen,
addend);
VFS_SMR_ASSERT_NOT_ENTERED();
if (error < 0) {
pwd = pwd_hold(curthread);
vref(vp);
error = vn_fullpath_dir(vp, pwd->pwd_rdir, buf, retbuf, buflen,
addend);
pwd_drop(pwd);
}
if (error != 0)
goto out_bad;
*freebuf = buf;
return (0);
out_bad:
free(buf, M_TEMP);
return (error);
}
struct vnode *
vn_dir_dd_ino(struct vnode *vp)
{
struct namecache *ncp;
struct vnode *ddvp;
struct mtx *vlp;
enum vgetstate vs;
ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
continue;
ddvp = ncp->nc_dvp;
vs = vget_prep(ddvp);
mtx_unlock(vlp);
if (vget_finish(ddvp, LK_SHARED | LK_NOWAIT, vs))
return (NULL);
return (ddvp);
}
mtx_unlock(vlp);
return (NULL);
}
int
vn_commname(struct vnode *vp, char *buf, u_int buflen)
{
struct namecache *ncp;
struct mtx *vlp;
int l;
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
break;
if (ncp == NULL) {
mtx_unlock(vlp);
return (ENOENT);
}
l = min(ncp->nc_nlen, buflen - 1);
memcpy(buf, ncp->nc_name, l);
mtx_unlock(vlp);
buf[l] = '\0';
return (0);
}
/*
* This function updates path string to vnode's full global path
* and checks the size of the new path string against the pathlen argument.
*
* Requires a locked, referenced vnode.
* Vnode is re-locked on success or ENODEV, otherwise unlocked.
*
* If vp is a directory, the call to vn_fullpath_global() always succeeds
* because it falls back to the ".." lookup if the namecache lookup fails.
*/
int
vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path,
u_int pathlen)
{
struct nameidata nd;
struct vnode *vp1;
char *rpath, *fbuf;
int error;
ASSERT_VOP_ELOCKED(vp, __func__);
/* Construct global filesystem path from vp. */
VOP_UNLOCK(vp);
error = vn_fullpath_global(vp, &rpath, &fbuf);
if (error != 0) {
vrele(vp);
return (error);
}
if (strlen(rpath) >= pathlen) {
vrele(vp);
error = ENAMETOOLONG;
goto out;
}
/*
* Re-lookup the vnode by path to detect a possible rename.
* As a side effect, the vnode is relocked.
* If vnode was renamed, return ENOENT.
*/
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
UIO_SYSSPACE, path, td);
error = namei(&nd);
if (error != 0) {
vrele(vp);
goto out;
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vp1 = nd.ni_vp;
vrele(vp);
if (vp1 == vp)
strcpy(path, rpath);
else {
vput(vp1);
error = ENOENT;
}
out:
free(fbuf, M_TEMP);
return (error);
}
#ifdef DDB
static void
db_print_vpath(struct vnode *vp)
{
while (vp != NULL) {
db_printf("%p: ", vp);
if (vp == rootvnode) {
db_printf("/");
vp = NULL;
} else {
if (vp->v_vflag & VV_ROOT) {
db_printf("<mount point>");
vp = vp->v_mount->mnt_vnodecovered;
} else {
struct namecache *ncp;
char *ncn;
int i;
ncp = TAILQ_FIRST(&vp->v_cache_dst);
if (ncp != NULL) {
ncn = ncp->nc_name;
for (i = 0; i < ncp->nc_nlen; i++)
db_printf("%c", *ncn++);
vp = ncp->nc_dvp;
} else {
vp = NULL;
}
}
}
db_printf("\n");
}
return;
}
DB_SHOW_COMMAND(vpath, db_show_vpath)
{
struct vnode *vp;
if (!have_addr) {
db_printf("usage: show vpath <struct vnode *>\n");
return;
}
vp = (struct vnode *)addr;
db_print_vpath(vp);
}
#endif
static int cache_fast_lookup = 1;
#define CACHE_FPL_FAILED -2020
void
cache_fast_lookup_enabled_recalc(void)
{
int lookup_flag;
int mac_on;
#ifdef MAC
mac_on = mac_vnode_check_lookup_enabled();
mac_on |= mac_vnode_check_readlink_enabled();
#else
mac_on = 0;
#endif
lookup_flag = atomic_load_int(&cache_fast_lookup);
if (lookup_flag && !mac_on) {
atomic_store_char(&cache_fast_lookup_enabled, true);
} else {
atomic_store_char(&cache_fast_lookup_enabled, false);
}
}
static int
syscal_vfs_cache_fast_lookup(SYSCTL_HANDLER_ARGS)
{
int error, old;
old = atomic_load_int(&cache_fast_lookup);
error = sysctl_handle_int(oidp, arg1, arg2, req);
if (error == 0 && req->newptr && old != atomic_load_int(&cache_fast_lookup))
cache_fast_lookup_enabled_recalc();
return (error);
}
SYSCTL_PROC(_vfs, OID_AUTO, cache_fast_lookup, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE,
&cache_fast_lookup, 0, syscal_vfs_cache_fast_lookup, "IU", "");
/*
* Components of nameidata (or objects it can point to) which may
* need restoring in case fast path lookup fails.
*/
struct nameidata_outer {
size_t ni_pathlen;
int cn_flags;
};
struct nameidata_saved {
#ifdef INVARIANTS
char *cn_nameptr;
size_t ni_pathlen;
#endif
};
#ifdef INVARIANTS
struct cache_fpl_debug {
size_t ni_pathlen;
};
#endif
struct cache_fpl {
struct nameidata *ndp;
struct componentname *cnp;
char *nulchar;
struct vnode *dvp;
struct vnode *tvp;
seqc_t dvp_seqc;
seqc_t tvp_seqc;
uint32_t hash;
struct nameidata_saved snd;
struct nameidata_outer snd_outer;
int line;
enum cache_fpl_status status:8;
bool in_smr;
bool fsearch;
bool savename;
struct pwd **pwd;
#ifdef INVARIANTS
struct cache_fpl_debug debug;
#endif
};
static bool cache_fplookup_mp_supported(struct mount *mp);
static bool cache_fplookup_is_mp(struct cache_fpl *fpl);
static int cache_fplookup_cross_mount(struct cache_fpl *fpl);
static int cache_fplookup_partial_setup(struct cache_fpl *fpl);
static int cache_fplookup_skip_slashes(struct cache_fpl *fpl);
static int cache_fplookup_trailingslash(struct cache_fpl *fpl);
static void cache_fpl_pathlen_dec(struct cache_fpl *fpl);
static void cache_fpl_pathlen_inc(struct cache_fpl *fpl);
static void cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n);
static void cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n);
static void
cache_fpl_cleanup_cnp(struct componentname *cnp)
{
uma_zfree(namei_zone, cnp->cn_pnbuf);
#ifdef DIAGNOSTIC
cnp->cn_pnbuf = NULL;
cnp->cn_nameptr = NULL;
#endif
}
static struct vnode *
cache_fpl_handle_root(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
ndp = fpl->ndp;
cnp = fpl->cnp;
MPASS(*(cnp->cn_nameptr) == '/');
cnp->cn_nameptr++;
cache_fpl_pathlen_dec(fpl);
if (__predict_false(*(cnp->cn_nameptr) == '/')) {
do {
cnp->cn_nameptr++;
cache_fpl_pathlen_dec(fpl);
} while (*(cnp->cn_nameptr) == '/');
}
return (ndp->ni_rootdir);
}
static void
cache_fpl_checkpoint_outer(struct cache_fpl *fpl)
{
fpl->snd_outer.ni_pathlen = fpl->ndp->ni_pathlen;
fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags;
}
static void
cache_fpl_checkpoint(struct cache_fpl *fpl)
{
#ifdef INVARIANTS
fpl->snd.cn_nameptr = fpl->ndp->ni_cnd.cn_nameptr;
fpl->snd.ni_pathlen = fpl->debug.ni_pathlen;
#endif
}
static void
cache_fpl_restore_partial(struct cache_fpl *fpl)
{
fpl->ndp->ni_cnd.cn_flags = fpl->snd_outer.cn_flags;
#ifdef INVARIANTS
fpl->debug.ni_pathlen = fpl->snd.ni_pathlen;
#endif
}
static void
cache_fpl_restore_abort(struct cache_fpl *fpl)
{
cache_fpl_restore_partial(fpl);
/*
* It is 0 on entry by API contract.
*/
fpl->ndp->ni_resflags = 0;
fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf;
fpl->ndp->ni_pathlen = fpl->snd_outer.ni_pathlen;
}
#ifdef INVARIANTS
#define cache_fpl_smr_assert_entered(fpl) ({ \
struct cache_fpl *_fpl = (fpl); \
MPASS(_fpl->in_smr == true); \
VFS_SMR_ASSERT_ENTERED(); \
})
#define cache_fpl_smr_assert_not_entered(fpl) ({ \
struct cache_fpl *_fpl = (fpl); \
MPASS(_fpl->in_smr == false); \
VFS_SMR_ASSERT_NOT_ENTERED(); \
})
static void
cache_fpl_assert_status(struct cache_fpl *fpl)
{
switch (fpl->status) {
case CACHE_FPL_STATUS_UNSET:
__assert_unreachable();
break;
case CACHE_FPL_STATUS_DESTROYED:
case CACHE_FPL_STATUS_ABORTED:
case CACHE_FPL_STATUS_PARTIAL:
case CACHE_FPL_STATUS_HANDLED:
break;
}
}
#else
#define cache_fpl_smr_assert_entered(fpl) do { } while (0)
#define cache_fpl_smr_assert_not_entered(fpl) do { } while (0)
#define cache_fpl_assert_status(fpl) do { } while (0)
#endif
#define cache_fpl_smr_enter_initial(fpl) ({ \
struct cache_fpl *_fpl = (fpl); \
vfs_smr_enter(); \
_fpl->in_smr = true; \
})
#define cache_fpl_smr_enter(fpl) ({ \
struct cache_fpl *_fpl = (fpl); \
MPASS(_fpl->in_smr == false); \
vfs_smr_enter(); \
_fpl->in_smr = true; \
})
#define cache_fpl_smr_exit(fpl) ({ \
struct cache_fpl *_fpl = (fpl); \
MPASS(_fpl->in_smr == true); \
vfs_smr_exit(); \
_fpl->in_smr = false; \
})
static int
cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line)
{
if (fpl->status != CACHE_FPL_STATUS_UNSET) {
KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
("%s: converting to abort from %d at %d, set at %d\n",
__func__, fpl->status, line, fpl->line));
}
cache_fpl_smr_assert_not_entered(fpl);
fpl->status = CACHE_FPL_STATUS_ABORTED;
fpl->line = line;
return (CACHE_FPL_FAILED);
}
#define cache_fpl_aborted_early(x) cache_fpl_aborted_early_impl((x), __LINE__)
static int __noinline
cache_fpl_aborted_impl(struct cache_fpl *fpl, int line)
{
struct nameidata *ndp;
struct componentname *cnp;
ndp = fpl->ndp;
cnp = fpl->cnp;
if (fpl->status != CACHE_FPL_STATUS_UNSET) {
KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL,
("%s: converting to abort from %d at %d, set at %d\n",
__func__, fpl->status, line, fpl->line));
}
fpl->status = CACHE_FPL_STATUS_ABORTED;
fpl->line = line;
if (fpl->in_smr)
cache_fpl_smr_exit(fpl);
cache_fpl_restore_abort(fpl);
/*
* Resolving symlinks overwrites data passed by the caller.
* Let namei know.
*/
if (ndp->ni_loopcnt > 0) {
fpl->status = CACHE_FPL_STATUS_DESTROYED;
cache_fpl_cleanup_cnp(cnp);
}
return (CACHE_FPL_FAILED);
}
#define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__)
static int __noinline
cache_fpl_partial_impl(struct cache_fpl *fpl, int line)
{
KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
("%s: setting to partial at %d, but already set to %d at %d\n",
__func__, line, fpl->status, fpl->line));
cache_fpl_smr_assert_entered(fpl);
fpl->status = CACHE_FPL_STATUS_PARTIAL;
fpl->line = line;
return (cache_fplookup_partial_setup(fpl));
}
#define cache_fpl_partial(x) cache_fpl_partial_impl((x), __LINE__)
static int
cache_fpl_handled_impl(struct cache_fpl *fpl, int line)
{
KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
("%s: setting to handled at %d, but already set to %d at %d\n",
__func__, line, fpl->status, fpl->line));
cache_fpl_smr_assert_not_entered(fpl);
fpl->status = CACHE_FPL_STATUS_HANDLED;
fpl->line = line;
return (0);
}
#define cache_fpl_handled(x) cache_fpl_handled_impl((x), __LINE__)
static int
cache_fpl_handled_error_impl(struct cache_fpl *fpl, int error, int line)
{
KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET,
("%s: setting to handled at %d, but already set to %d at %d\n",
__func__, line, fpl->status, fpl->line));
MPASS(error != 0);
MPASS(error != CACHE_FPL_FAILED);
cache_fpl_smr_assert_not_entered(fpl);
fpl->status = CACHE_FPL_STATUS_HANDLED;
fpl->line = line;
fpl->dvp = NULL;
fpl->tvp = NULL;
fpl->savename = false;
return (error);
}
#define cache_fpl_handled_error(x, e) cache_fpl_handled_error_impl((x), (e), __LINE__)
static bool
cache_fpl_terminated(struct cache_fpl *fpl)
{
return (fpl->status != CACHE_FPL_STATUS_UNSET);
}
#define CACHE_FPL_SUPPORTED_CN_FLAGS \
(NC_NOMAKEENTRY | NC_KEEPPOSENTRY | LOCKLEAF | LOCKPARENT | WANTPARENT | \
FAILIFEXISTS | FOLLOW | EMPTYPATH | LOCKSHARED | SAVENAME | SAVESTART | \
- WILLBEDIR | ISOPEN | NOMACCHECK | AUDITVNODE1 | AUDITVNODE2 | NOCAPCHECK)
+ WILLBEDIR | ISOPEN | NOMACCHECK | AUDITVNODE1 | AUDITVNODE2 | NOCAPCHECK | \
+ WANTIOCTLCAPS)
#define CACHE_FPL_INTERNAL_CN_FLAGS \
(ISDOTDOT | MAKEENTRY | ISLASTCN)
_Static_assert((CACHE_FPL_SUPPORTED_CN_FLAGS & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
"supported and internal flags overlap");
static bool
cache_fpl_islastcn(struct nameidata *ndp)
{
return (*ndp->ni_next == 0);
}
static bool
cache_fpl_istrailingslash(struct cache_fpl *fpl)
{
MPASS(fpl->nulchar > fpl->cnp->cn_pnbuf);
return (*(fpl->nulchar - 1) == '/');
}
static bool
cache_fpl_isdotdot(struct componentname *cnp)
{
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
return (true);
return (false);
}
static bool
cache_can_fplookup(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
struct thread *td;
ndp = fpl->ndp;
cnp = fpl->cnp;
td = cnp->cn_thread;
if (!atomic_load_char(&cache_fast_lookup_enabled)) {
cache_fpl_aborted_early(fpl);
return (false);
}
if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) {
cache_fpl_aborted_early(fpl);
return (false);
}
if (IN_CAPABILITY_MODE(td)) {
cache_fpl_aborted_early(fpl);
return (false);
}
if (AUDITING_TD(td)) {
cache_fpl_aborted_early(fpl);
return (false);
}
if (ndp->ni_startdir != NULL) {
cache_fpl_aborted_early(fpl);
return (false);
}
return (true);
}
static int __noinline
cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp)
{
struct nameidata *ndp;
struct componentname *cnp;
int error;
bool fsearch;
ndp = fpl->ndp;
cnp = fpl->cnp;
error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch);
if (__predict_false(error != 0)) {
return (cache_fpl_aborted(fpl));
}
fpl->fsearch = fsearch;
if ((*vpp)->v_type != VDIR) {
if (!((cnp->cn_flags & EMPTYPATH) != 0 && cnp->cn_pnbuf[0] == '\0')) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENOTDIR));
}
}
return (0);
}
static int __noinline
cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp,
uint32_t hash)
{
struct componentname *cnp;
struct vnode *dvp;
cnp = fpl->cnp;
dvp = fpl->dvp;
cache_fpl_smr_exit(fpl);
if (cache_neg_promote_cond(dvp, cnp, oncp, hash))
return (cache_fpl_handled_error(fpl, ENOENT));
else
return (cache_fpl_aborted(fpl));
}
/*
* The target vnode is not supported, prepare for the slow path to take over.
*/
static int __noinline
cache_fplookup_partial_setup(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
enum vgetstate dvs;
struct vnode *dvp;
struct pwd *pwd;
seqc_t dvp_seqc;
ndp = fpl->ndp;
cnp = fpl->cnp;
pwd = *(fpl->pwd);
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
if (!pwd_hold_smr(pwd)) {
return (cache_fpl_aborted(fpl));
}
/*
* Note that seqc is checked before the vnode is locked, so by
* the time regular lookup gets to it it may have moved.
*
* Ultimately this does not affect correctness, any lookup errors
* are userspace racing with itself. It is guaranteed that any
* path which ultimately gets found could also have been found
* by regular lookup going all the way in absence of concurrent
* modifications.
*/
dvs = vget_prep_smr(dvp);
cache_fpl_smr_exit(fpl);
if (__predict_false(dvs == VGET_NONE)) {
pwd_drop(pwd);
return (cache_fpl_aborted(fpl));
}
vget_finish_ref(dvp, dvs);
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
vrele(dvp);
pwd_drop(pwd);
return (cache_fpl_aborted(fpl));
}
cache_fpl_restore_partial(fpl);
#ifdef INVARIANTS
if (cnp->cn_nameptr != fpl->snd.cn_nameptr) {
panic("%s: cn_nameptr mismatch (%p != %p) full [%s]\n", __func__,
cnp->cn_nameptr, fpl->snd.cn_nameptr, cnp->cn_pnbuf);
}
#endif
ndp->ni_startdir = dvp;
cnp->cn_flags |= MAKEENTRY;
if (cache_fpl_islastcn(ndp))
cnp->cn_flags |= ISLASTCN;
if (cache_fpl_isdotdot(cnp))
cnp->cn_flags |= ISDOTDOT;
/*
* Skip potential extra slashes parsing did not take care of.
* cache_fplookup_skip_slashes explains the mechanism.
*/
if (__predict_false(*(cnp->cn_nameptr) == '/')) {
do {
cnp->cn_nameptr++;
cache_fpl_pathlen_dec(fpl);
} while (*(cnp->cn_nameptr) == '/');
}
ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
#ifdef INVARIANTS
if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
__func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
}
#endif
return (0);
}
static int
cache_fplookup_final_child(struct cache_fpl *fpl, enum vgetstate tvs)
{
struct componentname *cnp;
struct vnode *tvp;
seqc_t tvp_seqc;
int error, lkflags;
cnp = fpl->cnp;
tvp = fpl->tvp;
tvp_seqc = fpl->tvp_seqc;
if ((cnp->cn_flags & LOCKLEAF) != 0) {
lkflags = LK_SHARED;
if ((cnp->cn_flags & LOCKSHARED) == 0)
lkflags = LK_EXCLUSIVE;
error = vget_finish(tvp, lkflags, tvs);
if (__predict_false(error != 0)) {
return (cache_fpl_aborted(fpl));
}
} else {
vget_finish_ref(tvp, tvs);
}
if (!vn_seqc_consistent(tvp, tvp_seqc)) {
if ((cnp->cn_flags & LOCKLEAF) != 0)
vput(tvp);
else
vrele(tvp);
return (cache_fpl_aborted(fpl));
}
return (cache_fpl_handled(fpl));
}
/*
* They want to possibly modify the state of the namecache.
*/
static int __noinline
cache_fplookup_final_modifying(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
enum vgetstate dvs;
struct vnode *dvp, *tvp;
struct mount *mp;
seqc_t dvp_seqc;
int error;
bool docache;
ndp = fpl->ndp;
cnp = fpl->cnp;
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
MPASS(*(cnp->cn_nameptr) != '/');
MPASS(cache_fpl_islastcn(ndp));
if ((cnp->cn_flags & LOCKPARENT) == 0)
MPASS((cnp->cn_flags & WANTPARENT) != 0);
MPASS((cnp->cn_flags & TRAILINGSLASH) == 0);
MPASS(cnp->cn_nameiop == CREATE || cnp->cn_nameiop == DELETE ||
cnp->cn_nameiop == RENAME);
MPASS((cnp->cn_flags & MAKEENTRY) == 0);
MPASS((cnp->cn_flags & ISDOTDOT) == 0);
docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
if (cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)
docache = false;
/*
* Regular lookup nulifies the slash, which we don't do here.
* Don't take chances with filesystem routines seeing it for
* the last entry.
*/
if (cache_fpl_istrailingslash(fpl)) {
return (cache_fpl_partial(fpl));
}
mp = atomic_load_ptr(&dvp->v_mount);
if (__predict_false(mp == NULL)) {
return (cache_fpl_aborted(fpl));
}
if (__predict_false(mp->mnt_flag & MNT_RDONLY)) {
cache_fpl_smr_exit(fpl);
/*
* Original code keeps not checking for CREATE which
* might be a bug. For now let the old lookup decide.
*/
if (cnp->cn_nameiop == CREATE) {
return (cache_fpl_aborted(fpl));
}
return (cache_fpl_handled_error(fpl, EROFS));
}
if (fpl->tvp != NULL && (cnp->cn_flags & FAILIFEXISTS) != 0) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, EEXIST));
}
/*
* Secure access to dvp; check cache_fplookup_partial_setup for
* reasoning.
*
* XXX At least UFS requires its lookup routine to be called for
* the last path component, which leads to some level of complication
* and inefficiency:
* - the target routine always locks the target vnode, but our caller
* may not need it locked
* - some of the VOP machinery asserts that the parent is locked, which
* once more may be not required
*
* TODO: add a flag for filesystems which don't need this.
*/
dvs = vget_prep_smr(dvp);
cache_fpl_smr_exit(fpl);
if (__predict_false(dvs == VGET_NONE)) {
return (cache_fpl_aborted(fpl));
}
vget_finish_ref(dvp, dvs);
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
vrele(dvp);
return (cache_fpl_aborted(fpl));
}
error = vn_lock(dvp, LK_EXCLUSIVE);
if (__predict_false(error != 0)) {
vrele(dvp);
return (cache_fpl_aborted(fpl));
}
tvp = NULL;
cnp->cn_flags |= ISLASTCN;
if (docache)
cnp->cn_flags |= MAKEENTRY;
if (cache_fpl_isdotdot(cnp))
cnp->cn_flags |= ISDOTDOT;
cnp->cn_lkflags = LK_EXCLUSIVE;
error = VOP_LOOKUP(dvp, &tvp, cnp);
switch (error) {
case EJUSTRETURN:
case 0:
break;
case ENOTDIR:
case ENOENT:
vput(dvp);
return (cache_fpl_handled_error(fpl, error));
default:
vput(dvp);
return (cache_fpl_aborted(fpl));
}
fpl->tvp = tvp;
fpl->savename = (cnp->cn_flags & SAVENAME) != 0;
if (tvp == NULL) {
if ((cnp->cn_flags & SAVESTART) != 0) {
ndp->ni_startdir = dvp;
vrefact(ndp->ni_startdir);
cnp->cn_flags |= SAVENAME;
fpl->savename = true;
}
MPASS(error == EJUSTRETURN);
if ((cnp->cn_flags & LOCKPARENT) == 0) {
VOP_UNLOCK(dvp);
}
return (cache_fpl_handled(fpl));
}
/*
* There are very hairy corner cases concerning various flag combinations
* and locking state. In particular here we only hold one lock instead of
* two.
*
* Skip the complexity as it is of no significance for normal workloads.
*/
if (__predict_false(tvp == dvp)) {
vput(dvp);
vrele(tvp);
return (cache_fpl_aborted(fpl));
}
/*
* If they want the symlink itself we are fine, but if they want to
* follow it regular lookup has to be engaged.
*/
if (tvp->v_type == VLNK) {
if ((cnp->cn_flags & FOLLOW) != 0) {
vput(dvp);
vput(tvp);
return (cache_fpl_aborted(fpl));
}
}
/*
* Since we expect this to be the terminal vnode it should almost never
* be a mount point.
*/
if (__predict_false(cache_fplookup_is_mp(fpl))) {
vput(dvp);
vput(tvp);
return (cache_fpl_aborted(fpl));
}
if ((cnp->cn_flags & FAILIFEXISTS) != 0) {
vput(dvp);
vput(tvp);
return (cache_fpl_handled_error(fpl, EEXIST));
}
if ((cnp->cn_flags & LOCKLEAF) == 0) {
VOP_UNLOCK(tvp);
}
if ((cnp->cn_flags & LOCKPARENT) == 0) {
VOP_UNLOCK(dvp);
}
if ((cnp->cn_flags & SAVESTART) != 0) {
ndp->ni_startdir = dvp;
vrefact(ndp->ni_startdir);
cnp->cn_flags |= SAVENAME;
fpl->savename = true;
}
return (cache_fpl_handled(fpl));
}
static int __noinline
cache_fplookup_modifying(struct cache_fpl *fpl)
{
struct nameidata *ndp;
ndp = fpl->ndp;
if (!cache_fpl_islastcn(ndp)) {
return (cache_fpl_partial(fpl));
}
return (cache_fplookup_final_modifying(fpl));
}
static int __noinline
cache_fplookup_final_withparent(struct cache_fpl *fpl)
{
struct componentname *cnp;
enum vgetstate dvs, tvs;
struct vnode *dvp, *tvp;
seqc_t dvp_seqc;
int error;
cnp = fpl->cnp;
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
tvp = fpl->tvp;
MPASS((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0);
/*
* This is less efficient than it can be for simplicity.
*/
dvs = vget_prep_smr(dvp);
if (__predict_false(dvs == VGET_NONE)) {
return (cache_fpl_aborted(fpl));
}
tvs = vget_prep_smr(tvp);
if (__predict_false(tvs == VGET_NONE)) {
cache_fpl_smr_exit(fpl);
vget_abort(dvp, dvs);
return (cache_fpl_aborted(fpl));
}
cache_fpl_smr_exit(fpl);
if ((cnp->cn_flags & LOCKPARENT) != 0) {
error = vget_finish(dvp, LK_EXCLUSIVE, dvs);
if (__predict_false(error != 0)) {
vget_abort(tvp, tvs);
return (cache_fpl_aborted(fpl));
}
} else {
vget_finish_ref(dvp, dvs);
}
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
vget_abort(tvp, tvs);
if ((cnp->cn_flags & LOCKPARENT) != 0)
vput(dvp);
else
vrele(dvp);
return (cache_fpl_aborted(fpl));
}
error = cache_fplookup_final_child(fpl, tvs);
if (__predict_false(error != 0)) {
MPASS(fpl->status == CACHE_FPL_STATUS_ABORTED ||
fpl->status == CACHE_FPL_STATUS_DESTROYED);
if ((cnp->cn_flags & LOCKPARENT) != 0)
vput(dvp);
else
vrele(dvp);
return (error);
}
MPASS(fpl->status == CACHE_FPL_STATUS_HANDLED);
return (0);
}
static int
cache_fplookup_final(struct cache_fpl *fpl)
{
struct componentname *cnp;
enum vgetstate tvs;
struct vnode *dvp, *tvp;
seqc_t dvp_seqc;
cnp = fpl->cnp;
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
tvp = fpl->tvp;
MPASS(*(cnp->cn_nameptr) != '/');
if (cnp->cn_nameiop != LOOKUP) {
return (cache_fplookup_final_modifying(fpl));
}
if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0)
return (cache_fplookup_final_withparent(fpl));
tvs = vget_prep_smr(tvp);
if (__predict_false(tvs == VGET_NONE)) {
return (cache_fpl_partial(fpl));
}
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
cache_fpl_smr_exit(fpl);
vget_abort(tvp, tvs);
return (cache_fpl_aborted(fpl));
}
cache_fpl_smr_exit(fpl);
return (cache_fplookup_final_child(fpl, tvs));
}
/*
* Comment from locked lookup:
* Check for degenerate name (e.g. / or "") which is a way of talking about a
* directory, e.g. like "/." or ".".
*/
static int __noinline
cache_fplookup_degenerate(struct cache_fpl *fpl)
{
struct componentname *cnp;
struct vnode *dvp;
enum vgetstate dvs;
int error, lkflags;
#ifdef INVARIANTS
char *cp;
#endif
fpl->tvp = fpl->dvp;
fpl->tvp_seqc = fpl->dvp_seqc;
cnp = fpl->cnp;
dvp = fpl->dvp;
#ifdef INVARIANTS
for (cp = cnp->cn_pnbuf; *cp != '\0'; cp++) {
KASSERT(*cp == '/',
("%s: encountered non-slash; string [%s]\n", __func__,
cnp->cn_pnbuf));
}
#endif
if (__predict_false(cnp->cn_nameiop != LOOKUP)) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, EISDIR));
}
MPASS((cnp->cn_flags & SAVESTART) == 0);
if ((cnp->cn_flags & (LOCKPARENT|WANTPARENT)) != 0) {
return (cache_fplookup_final_withparent(fpl));
}
dvs = vget_prep_smr(dvp);
cache_fpl_smr_exit(fpl);
if (__predict_false(dvs == VGET_NONE)) {
return (cache_fpl_aborted(fpl));
}
if ((cnp->cn_flags & LOCKLEAF) != 0) {
lkflags = LK_SHARED;
if ((cnp->cn_flags & LOCKSHARED) == 0)
lkflags = LK_EXCLUSIVE;
error = vget_finish(dvp, lkflags, dvs);
if (__predict_false(error != 0)) {
return (cache_fpl_aborted(fpl));
}
} else {
vget_finish_ref(dvp, dvs);
}
return (cache_fpl_handled(fpl));
}
static int __noinline
cache_fplookup_emptypath(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
enum vgetstate tvs;
struct vnode *tvp;
int error, lkflags;
fpl->tvp = fpl->dvp;
fpl->tvp_seqc = fpl->dvp_seqc;
ndp = fpl->ndp;
cnp = fpl->cnp;
tvp = fpl->tvp;
MPASS(*cnp->cn_pnbuf == '\0');
if (__predict_false((cnp->cn_flags & EMPTYPATH) == 0)) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENOENT));
}
MPASS((cnp->cn_flags & (LOCKPARENT | WANTPARENT)) == 0);
tvs = vget_prep_smr(tvp);
cache_fpl_smr_exit(fpl);
if (__predict_false(tvs == VGET_NONE)) {
return (cache_fpl_aborted(fpl));
}
if ((cnp->cn_flags & LOCKLEAF) != 0) {
lkflags = LK_SHARED;
if ((cnp->cn_flags & LOCKSHARED) == 0)
lkflags = LK_EXCLUSIVE;
error = vget_finish(tvp, lkflags, tvs);
if (__predict_false(error != 0)) {
return (cache_fpl_aborted(fpl));
}
} else {
vget_finish_ref(tvp, tvs);
}
ndp->ni_resflags |= NIRES_EMPTYPATH;
return (cache_fpl_handled(fpl));
}
static int __noinline
cache_fplookup_noentry(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
enum vgetstate dvs;
struct vnode *dvp, *tvp;
seqc_t dvp_seqc;
int error;
ndp = fpl->ndp;
cnp = fpl->cnp;
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
MPASS((cnp->cn_flags & MAKEENTRY) == 0);
MPASS((cnp->cn_flags & ISDOTDOT) == 0);
if (cnp->cn_nameiop == LOOKUP)
MPASS((cnp->cn_flags & NOCACHE) == 0);
MPASS(!cache_fpl_isdotdot(cnp));
/*
* Hack: delayed name len checking.
*/
if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENAMETOOLONG));
}
if (cnp->cn_nameptr[0] == '/') {
return (cache_fplookup_skip_slashes(fpl));
}
if (cnp->cn_pnbuf[0] == '\0') {
return (cache_fplookup_emptypath(fpl));
}
if (cnp->cn_nameptr[0] == '\0') {
if (fpl->tvp == NULL) {
return (cache_fplookup_degenerate(fpl));
}
return (cache_fplookup_trailingslash(fpl));
}
if (cnp->cn_nameiop != LOOKUP) {
fpl->tvp = NULL;
return (cache_fplookup_modifying(fpl));
}
MPASS((cnp->cn_flags & SAVESTART) == 0);
/*
* Only try to fill in the component if it is the last one,
* otherwise not only there may be several to handle but the
* walk may be complicated.
*/
if (!cache_fpl_islastcn(ndp)) {
return (cache_fpl_partial(fpl));
}
/*
* Regular lookup nulifies the slash, which we don't do here.
* Don't take chances with filesystem routines seeing it for
* the last entry.
*/
if (cache_fpl_istrailingslash(fpl)) {
return (cache_fpl_partial(fpl));
}
/*
* Secure access to dvp; check cache_fplookup_partial_setup for
* reasoning.
*/
dvs = vget_prep_smr(dvp);
cache_fpl_smr_exit(fpl);
if (__predict_false(dvs == VGET_NONE)) {
return (cache_fpl_aborted(fpl));
}
vget_finish_ref(dvp, dvs);
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
vrele(dvp);
return (cache_fpl_aborted(fpl));
}
error = vn_lock(dvp, LK_SHARED);
if (__predict_false(error != 0)) {
vrele(dvp);
return (cache_fpl_aborted(fpl));
}
tvp = NULL;
/*
* TODO: provide variants which don't require locking either vnode.
*/
cnp->cn_flags |= ISLASTCN | MAKEENTRY;
cnp->cn_lkflags = LK_SHARED;
if ((cnp->cn_flags & LOCKSHARED) == 0) {
cnp->cn_lkflags = LK_EXCLUSIVE;
}
error = VOP_LOOKUP(dvp, &tvp, cnp);
switch (error) {
case EJUSTRETURN:
case 0:
break;
case ENOTDIR:
case ENOENT:
vput(dvp);
return (cache_fpl_handled_error(fpl, error));
default:
vput(dvp);
return (cache_fpl_aborted(fpl));
}
fpl->tvp = tvp;
if (!fpl->savename) {
MPASS((cnp->cn_flags & SAVENAME) == 0);
}
if (tvp == NULL) {
MPASS(error == EJUSTRETURN);
if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
vput(dvp);
} else if ((cnp->cn_flags & LOCKPARENT) == 0) {
VOP_UNLOCK(dvp);
}
return (cache_fpl_handled(fpl));
}
if (tvp->v_type == VLNK) {
if ((cnp->cn_flags & FOLLOW) != 0) {
vput(dvp);
vput(tvp);
return (cache_fpl_aborted(fpl));
}
}
if (__predict_false(cache_fplookup_is_mp(fpl))) {
vput(dvp);
vput(tvp);
return (cache_fpl_aborted(fpl));
}
if ((cnp->cn_flags & LOCKLEAF) == 0) {
VOP_UNLOCK(tvp);
}
if ((cnp->cn_flags & (WANTPARENT | LOCKPARENT)) == 0) {
vput(dvp);
} else if ((cnp->cn_flags & LOCKPARENT) == 0) {
VOP_UNLOCK(dvp);
}
return (cache_fpl_handled(fpl));
}
static int __noinline
cache_fplookup_dot(struct cache_fpl *fpl)
{
int error;
MPASS(!seqc_in_modify(fpl->dvp_seqc));
/*
* Just re-assign the value. seqc will be checked later for the first
* non-dot path component in line and/or before deciding to return the
* vnode.
*/
fpl->tvp = fpl->dvp;
fpl->tvp_seqc = fpl->dvp_seqc;
counter_u64_add(dothits, 1);
SDT_PROBE3(vfs, namecache, lookup, hit, fpl->dvp, ".", fpl->dvp);
error = 0;
if (cache_fplookup_is_mp(fpl)) {
error = cache_fplookup_cross_mount(fpl);
}
return (error);
}
static int __noinline
cache_fplookup_dotdot(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
struct namecache *ncp;
struct vnode *dvp;
struct prison *pr;
u_char nc_flag;
ndp = fpl->ndp;
cnp = fpl->cnp;
dvp = fpl->dvp;
MPASS(cache_fpl_isdotdot(cnp));
/*
* XXX this is racy the same way regular lookup is
*/
for (pr = cnp->cn_cred->cr_prison; pr != NULL;
pr = pr->pr_parent)
if (dvp == pr->pr_root)
break;
if (dvp == ndp->ni_rootdir ||
dvp == ndp->ni_topdir ||
dvp == rootvnode ||
pr != NULL) {
fpl->tvp = dvp;
fpl->tvp_seqc = vn_seqc_read_any(dvp);
if (seqc_in_modify(fpl->tvp_seqc)) {
return (cache_fpl_aborted(fpl));
}
return (0);
}
if ((dvp->v_vflag & VV_ROOT) != 0) {
/*
* TODO
* The opposite of climb mount is needed here.
*/
return (cache_fpl_partial(fpl));
}
ncp = atomic_load_consume_ptr(&dvp->v_cache_dd);
if (ncp == NULL) {
return (cache_fpl_aborted(fpl));
}
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_ISDOTDOT) != 0) {
if ((nc_flag & NCF_NEGATIVE) != 0)
return (cache_fpl_aborted(fpl));
fpl->tvp = ncp->nc_vp;
} else {
fpl->tvp = ncp->nc_dvp;
}
fpl->tvp_seqc = vn_seqc_read_any(fpl->tvp);
if (seqc_in_modify(fpl->tvp_seqc)) {
return (cache_fpl_partial(fpl));
}
/*
* Acquire fence provided by vn_seqc_read_any above.
*/
if (__predict_false(atomic_load_ptr(&dvp->v_cache_dd) != ncp)) {
return (cache_fpl_aborted(fpl));
}
if (!cache_ncp_canuse(ncp)) {
return (cache_fpl_aborted(fpl));
}
counter_u64_add(dotdothits, 1);
return (0);
}
static int __noinline
cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash)
{
u_char nc_flag __diagused;
bool neg_promote;
#ifdef INVARIANTS
nc_flag = atomic_load_char(&ncp->nc_flag);
MPASS((nc_flag & NCF_NEGATIVE) != 0);
#endif
/*
* If they want to create an entry we need to replace this one.
*/
if (__predict_false(fpl->cnp->cn_nameiop != LOOKUP)) {
fpl->tvp = NULL;
return (cache_fplookup_modifying(fpl));
}
neg_promote = cache_neg_hit_prep(ncp);
if (!cache_fpl_neg_ncp_canuse(ncp)) {
cache_neg_hit_abort(ncp);
return (cache_fpl_partial(fpl));
}
if (neg_promote) {
return (cache_fplookup_negative_promote(fpl, ncp, hash));
}
cache_neg_hit_finish(ncp);
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENOENT));
}
/*
* Resolve a symlink. Called by filesystem-specific routines.
*
* Code flow is:
* ... -> cache_fplookup_symlink -> VOP_FPLOOKUP_SYMLINK -> cache_symlink_resolve
*/
int
cache_symlink_resolve(struct cache_fpl *fpl, const char *string, size_t len)
{
struct nameidata *ndp;
struct componentname *cnp;
size_t adjust;
ndp = fpl->ndp;
cnp = fpl->cnp;
if (__predict_false(len == 0)) {
return (ENOENT);
}
if (__predict_false(len > MAXPATHLEN - 2)) {
if (cache_fpl_istrailingslash(fpl)) {
return (EAGAIN);
}
}
ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr - cnp->cn_namelen + 1;
#ifdef INVARIANTS
if (ndp->ni_pathlen != fpl->debug.ni_pathlen) {
panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
__func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
}
#endif
if (__predict_false(len + ndp->ni_pathlen > MAXPATHLEN)) {
return (ENAMETOOLONG);
}
if (__predict_false(ndp->ni_loopcnt++ >= MAXSYMLINKS)) {
return (ELOOP);
}
adjust = len;
if (ndp->ni_pathlen > 1) {
bcopy(ndp->ni_next, cnp->cn_pnbuf + len, ndp->ni_pathlen);
} else {
if (cache_fpl_istrailingslash(fpl)) {
adjust = len + 1;
cnp->cn_pnbuf[len] = '/';
cnp->cn_pnbuf[len + 1] = '\0';
} else {
cnp->cn_pnbuf[len] = '\0';
}
}
bcopy(string, cnp->cn_pnbuf, len);
ndp->ni_pathlen += adjust;
cache_fpl_pathlen_add(fpl, adjust);
cnp->cn_nameptr = cnp->cn_pnbuf;
fpl->nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
fpl->tvp = NULL;
return (0);
}
static int __noinline
cache_fplookup_symlink(struct cache_fpl *fpl)
{
struct mount *mp;
struct nameidata *ndp;
struct componentname *cnp;
struct vnode *dvp, *tvp;
int error;
ndp = fpl->ndp;
cnp = fpl->cnp;
dvp = fpl->dvp;
tvp = fpl->tvp;
if (cache_fpl_islastcn(ndp)) {
if ((cnp->cn_flags & FOLLOW) == 0) {
return (cache_fplookup_final(fpl));
}
}
mp = atomic_load_ptr(&dvp->v_mount);
if (__predict_false(mp == NULL)) {
return (cache_fpl_aborted(fpl));
}
/*
* Note this check races against setting the flag just like regular
* lookup.
*/
if (__predict_false((mp->mnt_flag & MNT_NOSYMFOLLOW) != 0)) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, EACCES));
}
error = VOP_FPLOOKUP_SYMLINK(tvp, fpl);
if (__predict_false(error != 0)) {
switch (error) {
case EAGAIN:
return (cache_fpl_partial(fpl));
case ENOENT:
case ENAMETOOLONG:
case ELOOP:
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, error));
default:
return (cache_fpl_aborted(fpl));
}
}
if (*(cnp->cn_nameptr) == '/') {
fpl->dvp = cache_fpl_handle_root(fpl);
fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
if (seqc_in_modify(fpl->dvp_seqc)) {
return (cache_fpl_aborted(fpl));
}
/*
* The main loop assumes that ->dvp points to a vnode belonging
* to a filesystem which can do lockless lookup, but the absolute
* symlink can be wandering off to one which does not.
*/
mp = atomic_load_ptr(&fpl->dvp->v_mount);
if (__predict_false(mp == NULL)) {
return (cache_fpl_aborted(fpl));
}
if (!cache_fplookup_mp_supported(mp)) {
cache_fpl_checkpoint(fpl);
return (cache_fpl_partial(fpl));
}
}
return (0);
}
static int
cache_fplookup_next(struct cache_fpl *fpl)
{
struct componentname *cnp;
struct namecache *ncp;
struct vnode *dvp, *tvp;
u_char nc_flag;
uint32_t hash;
int error;
cnp = fpl->cnp;
dvp = fpl->dvp;
hash = fpl->hash;
if (__predict_false(cnp->cn_nameptr[0] == '.')) {
if (cnp->cn_namelen == 1) {
return (cache_fplookup_dot(fpl));
}
if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
return (cache_fplookup_dotdot(fpl));
}
}
MPASS(!cache_fpl_isdotdot(cnp));
CK_SLIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
if (__predict_false(ncp == NULL)) {
return (cache_fplookup_noentry(fpl));
}
tvp = atomic_load_ptr(&ncp->nc_vp);
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_NEGATIVE) != 0) {
return (cache_fplookup_neg(fpl, ncp, hash));
}
if (!cache_ncp_canuse(ncp)) {
return (cache_fpl_partial(fpl));
}
fpl->tvp = tvp;
fpl->tvp_seqc = vn_seqc_read_any(tvp);
if (seqc_in_modify(fpl->tvp_seqc)) {
return (cache_fpl_partial(fpl));
}
counter_u64_add(numposhits, 1);
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name, tvp);
error = 0;
if (cache_fplookup_is_mp(fpl)) {
error = cache_fplookup_cross_mount(fpl);
}
return (error);
}
static bool
cache_fplookup_mp_supported(struct mount *mp)
{
MPASS(mp != NULL);
if ((mp->mnt_kern_flag & MNTK_FPLOOKUP) == 0)
return (false);
return (true);
}
/*
* Walk up the mount stack (if any).
*
* Correctness is provided in the following ways:
* - all vnodes are protected from freeing with SMR
* - struct mount objects are type stable making them always safe to access
* - stability of the particular mount is provided by busying it
* - relationship between the vnode which is mounted on and the mount is
* verified with the vnode sequence counter after busying
* - association between root vnode of the mount and the mount is protected
* by busy
*
* From that point on we can read the sequence counter of the root vnode
* and get the next mount on the stack (if any) using the same protection.
*
* By the end of successful walk we are guaranteed the reached state was
* indeed present at least at some point which matches the regular lookup.
*/
static int __noinline
cache_fplookup_climb_mount(struct cache_fpl *fpl)
{
struct mount *mp, *prev_mp;
struct mount_pcpu *mpcpu, *prev_mpcpu;
struct vnode *vp;
seqc_t vp_seqc;
vp = fpl->tvp;
vp_seqc = fpl->tvp_seqc;
VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp);
mp = atomic_load_ptr(&vp->v_mountedhere);
if (__predict_false(mp == NULL)) {
return (0);
}
prev_mp = NULL;
for (;;) {
if (!vfs_op_thread_enter_crit(mp, mpcpu)) {
if (prev_mp != NULL)
vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
return (cache_fpl_partial(fpl));
}
if (prev_mp != NULL)
vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
if (!vn_seqc_consistent(vp, vp_seqc)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
if (!cache_fplookup_mp_supported(mp)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
vp = atomic_load_ptr(&mp->mnt_rootvnode);
if (vp == NULL) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
vp_seqc = vn_seqc_read_any(vp);
if (seqc_in_modify(vp_seqc)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
prev_mp = mp;
prev_mpcpu = mpcpu;
mp = atomic_load_ptr(&vp->v_mountedhere);
if (mp == NULL)
break;
}
vfs_op_thread_exit_crit(prev_mp, prev_mpcpu);
fpl->tvp = vp;
fpl->tvp_seqc = vp_seqc;
return (0);
}
static int __noinline
cache_fplookup_cross_mount(struct cache_fpl *fpl)
{
struct mount *mp;
struct mount_pcpu *mpcpu;
struct vnode *vp;
seqc_t vp_seqc;
vp = fpl->tvp;
vp_seqc = fpl->tvp_seqc;
VNPASS(vp->v_type == VDIR || vp->v_type == VBAD, vp);
mp = atomic_load_ptr(&vp->v_mountedhere);
if (__predict_false(mp == NULL)) {
return (0);
}
if (!vfs_op_thread_enter_crit(mp, mpcpu)) {
return (cache_fpl_partial(fpl));
}
if (!vn_seqc_consistent(vp, vp_seqc)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
if (!cache_fplookup_mp_supported(mp)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
vp = atomic_load_ptr(&mp->mnt_rootvnode);
if (__predict_false(vp == NULL)) {
vfs_op_thread_exit_crit(mp, mpcpu);
return (cache_fpl_partial(fpl));
}
vp_seqc = vn_seqc_read_any(vp);
vfs_op_thread_exit_crit(mp, mpcpu);
if (seqc_in_modify(vp_seqc)) {
return (cache_fpl_partial(fpl));
}
mp = atomic_load_ptr(&vp->v_mountedhere);
if (__predict_false(mp != NULL)) {
/*
* There are possibly more mount points on top.
* Normally this does not happen so for simplicity just start
* over.
*/
return (cache_fplookup_climb_mount(fpl));
}
fpl->tvp = vp;
fpl->tvp_seqc = vp_seqc;
return (0);
}
/*
* Check if a vnode is mounted on.
*/
static bool
cache_fplookup_is_mp(struct cache_fpl *fpl)
{
struct vnode *vp;
vp = fpl->tvp;
return ((vn_irflag_read(vp) & VIRF_MOUNTPOINT) != 0);
}
/*
* Parse the path.
*
* The code was originally copy-pasted from regular lookup and despite
* clean ups leaves performance on the table. Any modifications here
* must take into account that in case off fallback the resulting
* nameidata state has to be compatible with the original.
*/
/*
* Debug ni_pathlen tracking.
*/
#ifdef INVARIANTS
static void
cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n)
{
fpl->debug.ni_pathlen += n;
KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
("%s: pathlen overflow to %zd\n", __func__, fpl->debug.ni_pathlen));
}
static void
cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n)
{
fpl->debug.ni_pathlen -= n;
KASSERT(fpl->debug.ni_pathlen <= PATH_MAX,
("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen));
}
static void
cache_fpl_pathlen_inc(struct cache_fpl *fpl)
{
cache_fpl_pathlen_add(fpl, 1);
}
static void
cache_fpl_pathlen_dec(struct cache_fpl *fpl)
{
cache_fpl_pathlen_sub(fpl, 1);
}
#else
static void
cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n)
{
}
static void
cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n)
{
}
static void
cache_fpl_pathlen_inc(struct cache_fpl *fpl)
{
}
static void
cache_fpl_pathlen_dec(struct cache_fpl *fpl)
{
}
#endif
static void
cache_fplookup_parse(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
struct vnode *dvp;
char *cp;
uint32_t hash;
ndp = fpl->ndp;
cnp = fpl->cnp;
dvp = fpl->dvp;
/*
* Find the end of this path component, it is either / or nul.
*
* Store / as a temporary sentinel so that we only have one character
* to test for. Pathnames tend to be short so this should not be
* resulting in cache misses.
*
* TODO: fix this to be word-sized.
*/
MPASS(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] >= cnp->cn_pnbuf);
KASSERT(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] == fpl->nulchar,
("%s: mismatch between pathlen (%zu) and nulchar (%p != %p), string [%s]\n",
__func__, fpl->debug.ni_pathlen, &cnp->cn_nameptr[fpl->debug.ni_pathlen - 1],
fpl->nulchar, cnp->cn_pnbuf));
KASSERT(*fpl->nulchar == '\0',
("%s: expected nul at %p; string [%s]\n", __func__, fpl->nulchar,
cnp->cn_pnbuf));
hash = cache_get_hash_iter_start(dvp);
*fpl->nulchar = '/';
for (cp = cnp->cn_nameptr; *cp != '/'; cp++) {
KASSERT(*cp != '\0',
("%s: encountered unexpected nul; string [%s]\n", __func__,
cnp->cn_nameptr));
hash = cache_get_hash_iter(*cp, hash);
continue;
}
*fpl->nulchar = '\0';
fpl->hash = cache_get_hash_iter_finish(hash);
cnp->cn_namelen = cp - cnp->cn_nameptr;
cache_fpl_pathlen_sub(fpl, cnp->cn_namelen);
#ifdef INVARIANTS
/*
* cache_get_hash only accepts lengths up to NAME_MAX. This is fine since
* we are going to fail this lookup with ENAMETOOLONG (see below).
*/
if (cnp->cn_namelen <= NAME_MAX) {
if (fpl->hash != cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp)) {
panic("%s: mismatched hash for [%s] len %ld", __func__,
cnp->cn_nameptr, cnp->cn_namelen);
}
}
#endif
/*
* Hack: we have to check if the found path component's length exceeds
* NAME_MAX. However, the condition is very rarely true and check can
* be elided in the common case -- if an entry was found in the cache,
* then it could not have been too long to begin with.
*/
ndp->ni_next = cp;
}
static void
cache_fplookup_parse_advance(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
ndp = fpl->ndp;
cnp = fpl->cnp;
cnp->cn_nameptr = ndp->ni_next;
KASSERT(*(cnp->cn_nameptr) == '/',
("%s: should have seen slash at %p ; buf %p [%s]\n", __func__,
cnp->cn_nameptr, cnp->cn_pnbuf, cnp->cn_pnbuf));
cnp->cn_nameptr++;
cache_fpl_pathlen_dec(fpl);
}
/*
* Skip spurious slashes in a pathname (e.g., "foo///bar") and retry.
*
* Lockless lookup tries to elide checking for spurious slashes and should they
* be present is guaranteed to fail to find an entry. In this case the caller
* must check if the name starts with a slash and call this routine. It is
* going to fast forward across the spurious slashes and set the state up for
* retry.
*/
static int __noinline
cache_fplookup_skip_slashes(struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
ndp = fpl->ndp;
cnp = fpl->cnp;
MPASS(*(cnp->cn_nameptr) == '/');
do {
cnp->cn_nameptr++;
cache_fpl_pathlen_dec(fpl);
} while (*(cnp->cn_nameptr) == '/');
/*
* Go back to one slash so that cache_fplookup_parse_advance has
* something to skip.
*/
cnp->cn_nameptr--;
cache_fpl_pathlen_inc(fpl);
/*
* cache_fplookup_parse_advance starts from ndp->ni_next
*/
ndp->ni_next = cnp->cn_nameptr;
/*
* See cache_fplookup_dot.
*/
fpl->tvp = fpl->dvp;
fpl->tvp_seqc = fpl->dvp_seqc;
return (0);
}
/*
* Handle trailing slashes (e.g., "foo/").
*
* If a trailing slash is found the terminal vnode must be a directory.
* Regular lookup shortens the path by nulifying the first trailing slash and
* sets the TRAILINGSLASH flag to denote this took place. There are several
* checks on it performed later.
*
* Similarly to spurious slashes, lockless lookup handles this in a speculative
* manner relying on an invariant that a non-directory vnode will get a miss.
* In this case cn_nameptr[0] == '\0' and cn_namelen == 0.
*
* Thus for a path like "foo/bar/" the code unwinds the state back to "bar/"
* and denotes this is the last path component, which avoids looping back.
*
* Only plain lookups are supported for now to restrict corner cases to handle.
*/
static int __noinline
cache_fplookup_trailingslash(struct cache_fpl *fpl)
{
#ifdef INVARIANTS
size_t ni_pathlen;
#endif
struct nameidata *ndp;
struct componentname *cnp;
struct namecache *ncp;
struct vnode *tvp;
char *cn_nameptr_orig, *cn_nameptr_slash;
seqc_t tvp_seqc;
u_char nc_flag;
ndp = fpl->ndp;
cnp = fpl->cnp;
tvp = fpl->tvp;
tvp_seqc = fpl->tvp_seqc;
MPASS(fpl->dvp == fpl->tvp);
KASSERT(cache_fpl_istrailingslash(fpl),
("%s: expected trailing slash at %p; string [%s]\n", __func__, fpl->nulchar - 1,
cnp->cn_pnbuf));
KASSERT(cnp->cn_nameptr[0] == '\0',
("%s: expected nul char at %p; string [%s]\n", __func__, &cnp->cn_nameptr[0],
cnp->cn_pnbuf));
KASSERT(cnp->cn_namelen == 0,
("%s: namelen 0 but got %ld; string [%s]\n", __func__, cnp->cn_namelen,
cnp->cn_pnbuf));
MPASS(cnp->cn_nameptr > cnp->cn_pnbuf);
if (cnp->cn_nameiop != LOOKUP) {
return (cache_fpl_aborted(fpl));
}
if (__predict_false(tvp->v_type != VDIR)) {
if (!vn_seqc_consistent(tvp, tvp_seqc)) {
return (cache_fpl_aborted(fpl));
}
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENOTDIR));
}
/*
* Denote the last component.
*/
ndp->ni_next = &cnp->cn_nameptr[0];
MPASS(cache_fpl_islastcn(ndp));
/*
* Unwind trailing slashes.
*/
cn_nameptr_orig = cnp->cn_nameptr;
while (cnp->cn_nameptr >= cnp->cn_pnbuf) {
cnp->cn_nameptr--;
if (cnp->cn_nameptr[0] != '/') {
break;
}
}
/*
* Unwind to the beginning of the path component.
*
* Note the path may or may not have started with a slash.
*/
cn_nameptr_slash = cnp->cn_nameptr;
while (cnp->cn_nameptr > cnp->cn_pnbuf) {
cnp->cn_nameptr--;
if (cnp->cn_nameptr[0] == '/') {
break;
}
}
if (cnp->cn_nameptr[0] == '/') {
cnp->cn_nameptr++;
}
cnp->cn_namelen = cn_nameptr_slash - cnp->cn_nameptr + 1;
cache_fpl_pathlen_add(fpl, cn_nameptr_orig - cnp->cn_nameptr);
cache_fpl_checkpoint(fpl);
#ifdef INVARIANTS
ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1;
if (ni_pathlen != fpl->debug.ni_pathlen) {
panic("%s: mismatch (%zu != %zu) nulchar %p nameptr %p [%s] ; full string [%s]\n",
__func__, ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar,
cnp->cn_nameptr, cnp->cn_nameptr, cnp->cn_pnbuf);
}
#endif
/*
* If this was a "./" lookup the parent directory is already correct.
*/
if (cnp->cn_nameptr[0] == '.' && cnp->cn_namelen == 1) {
return (0);
}
/*
* Otherwise we need to look it up.
*/
tvp = fpl->tvp;
ncp = atomic_load_consume_ptr(&tvp->v_cache_dd);
if (__predict_false(ncp == NULL)) {
return (cache_fpl_aborted(fpl));
}
nc_flag = atomic_load_char(&ncp->nc_flag);
if ((nc_flag & NCF_ISDOTDOT) != 0) {
return (cache_fpl_aborted(fpl));
}
fpl->dvp = ncp->nc_dvp;
fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp);
if (seqc_in_modify(fpl->dvp_seqc)) {
return (cache_fpl_aborted(fpl));
}
return (0);
}
/*
* See the API contract for VOP_FPLOOKUP_VEXEC.
*/
static int __noinline
cache_fplookup_failed_vexec(struct cache_fpl *fpl, int error)
{
struct componentname *cnp;
struct vnode *dvp;
seqc_t dvp_seqc;
cnp = fpl->cnp;
dvp = fpl->dvp;
dvp_seqc = fpl->dvp_seqc;
/*
* Hack: delayed empty path checking.
*/
if (cnp->cn_pnbuf[0] == '\0') {
return (cache_fplookup_emptypath(fpl));
}
/*
* TODO: Due to ignoring trailing slashes lookup will perform a
* permission check on the last dir when it should not be doing it. It
* may fail, but said failure should be ignored. It is possible to fix
* it up fully without resorting to regular lookup, but for now just
* abort.
*/
if (cache_fpl_istrailingslash(fpl)) {
return (cache_fpl_aborted(fpl));
}
/*
* Hack: delayed degenerate path checking.
*/
if (cnp->cn_nameptr[0] == '\0' && fpl->tvp == NULL) {
return (cache_fplookup_degenerate(fpl));
}
/*
* Hack: delayed name len checking.
*/
if (__predict_false(cnp->cn_namelen > NAME_MAX)) {
cache_fpl_smr_exit(fpl);
return (cache_fpl_handled_error(fpl, ENAMETOOLONG));
}
/*
* Hack: they may be looking up foo/bar, where foo is not a directory.
* In such a case we need to return ENOTDIR, but we may happen to get
* here with a different error.
*/
if (dvp->v_type != VDIR) {
error = ENOTDIR;
}
/*
* Hack: handle O_SEARCH.
*
* Open Group Base Specifications Issue 7, 2018 edition states:
* <quote>
* If the access mode of the open file description associated with the
* file descriptor is not O_SEARCH, the function shall check whether
* directory searches are permitted using the current permissions of
* the directory underlying the file descriptor. If the access mode is
* O_SEARCH, the function shall not perform the check.
* </quote>
*
* Regular lookup tests for the NOEXECCHECK flag for every path
* component to decide whether to do the permission check. However,
* since most lookups never have the flag (and when they do it is only
* present for the first path component), lockless lookup only acts on
* it if there is a permission problem. Here the flag is represented
* with a boolean so that we don't have to clear it on the way out.
*
* For simplicity this always aborts.
* TODO: check if this is the first lookup and ignore the permission
* problem. Note the flag has to survive fallback (if it happens to be
* performed).
*/
if (fpl->fsearch) {
return (cache_fpl_aborted(fpl));
}
switch (error) {
case EAGAIN:
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
error = cache_fpl_aborted(fpl);
} else {
cache_fpl_partial(fpl);
}
break;
default:
if (!vn_seqc_consistent(dvp, dvp_seqc)) {
error = cache_fpl_aborted(fpl);
} else {
cache_fpl_smr_exit(fpl);
cache_fpl_handled_error(fpl, error);
}
break;
}
return (error);
}
static int
cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl)
{
struct nameidata *ndp;
struct componentname *cnp;
struct mount *mp;
int error;
ndp = fpl->ndp;
cnp = fpl->cnp;
cache_fpl_checkpoint(fpl);
/*
* The vnode at hand is almost always stable, skip checking for it.
* Worst case this postpones the check towards the end of the iteration
* of the main loop.
*/
fpl->dvp = dvp;
fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp);
mp = atomic_load_ptr(&dvp->v_mount);
if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) {
return (cache_fpl_aborted(fpl));
}
MPASS(fpl->tvp == NULL);
for (;;) {
cache_fplookup_parse(fpl);
error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred);
if (__predict_false(error != 0)) {
error = cache_fplookup_failed_vexec(fpl, error);
break;
}
error = cache_fplookup_next(fpl);
if (__predict_false(cache_fpl_terminated(fpl))) {
break;
}
VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp);
if (fpl->tvp->v_type == VLNK) {
error = cache_fplookup_symlink(fpl);
if (cache_fpl_terminated(fpl)) {
break;
}
} else {
if (cache_fpl_islastcn(ndp)) {
error = cache_fplookup_final(fpl);
break;
}
if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) {
error = cache_fpl_aborted(fpl);
break;
}
fpl->dvp = fpl->tvp;
fpl->dvp_seqc = fpl->tvp_seqc;
cache_fplookup_parse_advance(fpl);
}
cache_fpl_checkpoint(fpl);
}
return (error);
}
/*
* Fast path lookup protected with SMR and sequence counters.
*
* Note: all VOP_FPLOOKUP_VEXEC routines have a comment referencing this one.
*
* Filesystems can opt in by setting the MNTK_FPLOOKUP flag and meeting criteria
* outlined below.
*
* Traditional vnode lookup conceptually looks like this:
*
* vn_lock(current);
* for (;;) {
* next = find();
* vn_lock(next);
* vn_unlock(current);
* current = next;
* if (last)
* break;
* }
* return (current);
*
* Each jump to the next vnode is safe memory-wise and atomic with respect to
* any modifications thanks to holding respective locks.
*
* The same guarantee can be provided with a combination of safe memory
* reclamation and sequence counters instead. If all operations which affect
* the relationship between the current vnode and the one we are looking for
* also modify the counter, we can verify whether all the conditions held as
* we made the jump. This includes things like permissions, mount points etc.
* Counter modification is provided by enclosing relevant places in
* vn_seqc_write_begin()/end() calls.
*
* Thus this translates to:
*
* vfs_smr_enter();
* dvp_seqc = seqc_read_any(dvp);
* if (seqc_in_modify(dvp_seqc)) // someone is altering the vnode
* abort();
* for (;;) {
* tvp = find();
* tvp_seqc = seqc_read_any(tvp);
* if (seqc_in_modify(tvp_seqc)) // someone is altering the target vnode
* abort();
* if (!seqc_consistent(dvp, dvp_seqc) // someone is altering the vnode
* abort();
* dvp = tvp; // we know nothing of importance has changed
* dvp_seqc = tvp_seqc; // store the counter for the tvp iteration
* if (last)
* break;
* }
* vget(); // secure the vnode
* if (!seqc_consistent(tvp, tvp_seqc) // final check
* abort();
* // at this point we know nothing has changed for any parent<->child pair
* // as they were crossed during the lookup, meaning we matched the guarantee
* // of the locked variant
* return (tvp);
*
* The API contract for VOP_FPLOOKUP_VEXEC routines is as follows:
* - they are called while within vfs_smr protection which they must never exit
* - EAGAIN can be returned to denote checking could not be performed, it is
* always valid to return it
* - if the sequence counter has not changed the result must be valid
* - if the sequence counter has changed both false positives and false negatives
* are permitted (since the result will be rejected later)
* - for simple cases of unix permission checks vaccess_vexec_smr can be used
*
* Caveats to watch out for:
* - vnodes are passed unlocked and unreferenced with nothing stopping
* VOP_RECLAIM, in turn meaning that ->v_data can become NULL. It is advised
* to use atomic_load_ptr to fetch it.
* - the aforementioned object can also get freed, meaning absent other means it
* should be protected with vfs_smr
* - either safely checking permissions as they are modified or guaranteeing
* their stability is left to the routine
*/
int
cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
struct pwd **pwdp)
{
struct cache_fpl fpl;
struct pwd *pwd;
struct vnode *dvp;
struct componentname *cnp;
int error;
fpl.status = CACHE_FPL_STATUS_UNSET;
fpl.in_smr = false;
fpl.ndp = ndp;
fpl.cnp = cnp = &ndp->ni_cnd;
MPASS(ndp->ni_lcf == 0);
MPASS(curthread == cnp->cn_thread);
KASSERT ((cnp->cn_flags & CACHE_FPL_INTERNAL_CN_FLAGS) == 0,
("%s: internal flags found in cn_flags %" PRIx64, __func__,
cnp->cn_flags));
if ((cnp->cn_flags & SAVESTART) != 0) {
MPASS(cnp->cn_nameiop != LOOKUP);
}
MPASS(cnp->cn_nameptr == cnp->cn_pnbuf);
if (__predict_false(!cache_can_fplookup(&fpl))) {
*status = fpl.status;
SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
return (EOPNOTSUPP);
}
cache_fpl_checkpoint_outer(&fpl);
cache_fpl_smr_enter_initial(&fpl);
#ifdef INVARIANTS
fpl.debug.ni_pathlen = ndp->ni_pathlen;
#endif
fpl.nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
fpl.fsearch = false;
fpl.savename = (cnp->cn_flags & SAVENAME) != 0;
fpl.tvp = NULL; /* for degenerate path handling */
fpl.pwd = pwdp;
pwd = pwd_get_smr();
*(fpl.pwd) = pwd;
ndp->ni_rootdir = pwd->pwd_rdir;
ndp->ni_topdir = pwd->pwd_jdir;
if (cnp->cn_pnbuf[0] == '/') {
dvp = cache_fpl_handle_root(&fpl);
MPASS(ndp->ni_resflags == 0);
ndp->ni_resflags = NIRES_ABS;
} else {
if (ndp->ni_dirfd == AT_FDCWD) {
dvp = pwd->pwd_cdir;
} else {
error = cache_fplookup_dirfd(&fpl, &dvp);
if (__predict_false(error != 0)) {
goto out;
}
}
}
SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true);
error = cache_fplookup_impl(dvp, &fpl);
out:
cache_fpl_smr_assert_not_entered(&fpl);
cache_fpl_assert_status(&fpl);
*status = fpl.status;
if (SDT_PROBES_ENABLED()) {
SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status);
if (fpl.status == CACHE_FPL_STATUS_HANDLED)
SDT_PROBE4(vfs, namei, lookup, return, error, ndp->ni_vp, true,
ndp);
}
if (__predict_true(fpl.status == CACHE_FPL_STATUS_HANDLED)) {
MPASS(error != CACHE_FPL_FAILED);
if (error != 0) {
MPASS(fpl.dvp == NULL);
MPASS(fpl.tvp == NULL);
MPASS(fpl.savename == false);
}
ndp->ni_dvp = fpl.dvp;
ndp->ni_vp = fpl.tvp;
if (fpl.savename) {
cnp->cn_flags |= HASBUF;
} else {
cache_fpl_cleanup_cnp(cnp);
}
}
return (error);
}
diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c
index 95c5599ab232..7fee9d2c488f 100644
--- a/sys/kern/vfs_lookup.c
+++ b/sys/kern/vfs_lookup.c
@@ -1,1782 +1,1736 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_lookup.c 8.4 (Berkeley) 2/16/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/dirent.h>
#include <sys/kernel.h>
#include <sys/capsicum.h>
#include <sys/fcntl.h>
#include <sys/jail.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/filedesc.h>
#include <sys/proc.h>
#include <sys/sdt.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#ifdef INVARIANTS
#include <machine/_inttypes.h>
#endif
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/uma.h>
#define NAMEI_DIAGNOSTIC 1
#undef NAMEI_DIAGNOSTIC
SDT_PROVIDER_DEFINE(vfs);
SDT_PROBE_DEFINE4(vfs, namei, lookup, entry, "struct vnode *", "char *",
"unsigned long", "bool");
SDT_PROBE_DEFINE4(vfs, namei, lookup, return, "int", "struct vnode *", "bool",
"struct nameidata");
/* Allocation zone for namei. */
uma_zone_t namei_zone;
/* Placeholder vnode for mp traversal. */
static struct vnode *vp_crossmp;
static int
crossmp_vop_islocked(struct vop_islocked_args *ap)
{
return (LK_SHARED);
}
static int
crossmp_vop_lock1(struct vop_lock1_args *ap)
{
struct vnode *vp;
struct lock *lk __unused;
const char *file __unused;
int flags, line __unused;
vp = ap->a_vp;
lk = vp->v_vnlock;
flags = ap->a_flags;
file = ap->a_file;
line = ap->a_line;
if ((flags & LK_SHARED) == 0)
panic("invalid lock request for crossmp");
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, file, line,
flags & LK_INTERLOCK ? &VI_MTX(vp)->lock_object : NULL);
WITNESS_LOCK(&lk->lock_object, 0, file, line);
if ((flags & LK_INTERLOCK) != 0)
VI_UNLOCK(vp);
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, ap->a_file, line);
return (0);
}
static int
crossmp_vop_unlock(struct vop_unlock_args *ap)
{
struct vnode *vp;
struct lock *lk __unused;
vp = ap->a_vp;
lk = vp->v_vnlock;
WITNESS_UNLOCK(&lk->lock_object, 0, LOCK_FILE, LOCK_LINE);
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, LOCK_FILE,
LOCK_LINE);
return (0);
}
static struct vop_vector crossmp_vnodeops = {
.vop_default = &default_vnodeops,
.vop_islocked = crossmp_vop_islocked,
.vop_lock1 = crossmp_vop_lock1,
.vop_unlock = crossmp_vop_unlock,
};
/*
* VFS_VOP_VECTOR_REGISTER(crossmp_vnodeops) is not used here since the vnode
* gets allocated early. See nameiinit for the direct call below.
*/
struct nameicap_tracker {
struct vnode *dp;
TAILQ_ENTRY(nameicap_tracker) nm_link;
};
/* Zone for cap mode tracker elements used for dotdot capability checks. */
MALLOC_DEFINE(M_NAMEITRACKER, "namei_tracker", "namei tracking for dotdot");
static void
nameiinit(void *dummy __unused)
{
namei_zone = uma_zcreate("NAMEI", MAXPATHLEN, NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
vfs_vector_op_register(&crossmp_vnodeops);
getnewvnode("crossmp", NULL, &crossmp_vnodeops, &vp_crossmp);
}
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nameiinit, NULL);
static int lookup_cap_dotdot = 1;
SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot, CTLFLAG_RWTUN,
&lookup_cap_dotdot, 0,
"enables \"..\" components in path lookup in capability mode");
static int lookup_cap_dotdot_nonlocal = 1;
SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot_nonlocal, CTLFLAG_RWTUN,
&lookup_cap_dotdot_nonlocal, 0,
"enables \"..\" components in path lookup in capability mode "
"on non-local mount");
static void
nameicap_tracker_add(struct nameidata *ndp, struct vnode *dp)
{
struct nameicap_tracker *nt;
struct componentname *cnp;
if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0 || dp->v_type != VDIR)
return;
cnp = &ndp->ni_cnd;
nt = TAILQ_LAST(&ndp->ni_cap_tracker, nameicap_tracker_head);
if (nt != NULL && nt->dp == dp)
return;
nt = malloc(sizeof(*nt), M_NAMEITRACKER, M_WAITOK);
vhold(dp);
nt->dp = dp;
TAILQ_INSERT_TAIL(&ndp->ni_cap_tracker, nt, nm_link);
}
static void
nameicap_cleanup_from(struct nameidata *ndp, struct nameicap_tracker *first)
{
struct nameicap_tracker *nt, *nt1;
nt = first;
TAILQ_FOREACH_FROM_SAFE(nt, &ndp->ni_cap_tracker, nm_link, nt1) {
TAILQ_REMOVE(&ndp->ni_cap_tracker, nt, nm_link);
vdrop(nt->dp);
free(nt, M_NAMEITRACKER);
}
}
static void
nameicap_cleanup(struct nameidata *ndp)
{
KASSERT(TAILQ_EMPTY(&ndp->ni_cap_tracker) ||
(ndp->ni_lcf & NI_LCF_CAP_DOTDOT) != 0, ("not strictrelative"));
nameicap_cleanup_from(ndp, NULL);
}
/*
* For dotdot lookups in capability mode, only allow the component
* lookup to succeed if the resulting directory was already traversed
* during the operation. This catches situations where already
* traversed directory is moved to different parent, and then we walk
* over it with dotdots.
*
* Also allow to force failure of dotdot lookups for non-local
* filesystems, where external agents might assist local lookups to
* escape the compartment.
*/
static int
nameicap_check_dotdot(struct nameidata *ndp, struct vnode *dp)
{
struct nameicap_tracker *nt;
struct mount *mp;
if (dp == NULL || dp->v_type != VDIR || (ndp->ni_lcf &
NI_LCF_STRICTRELATIVE) == 0)
return (0);
if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0)
return (ENOTCAPABLE);
mp = dp->v_mount;
if (lookup_cap_dotdot_nonlocal == 0 && mp != NULL &&
(mp->mnt_flag & MNT_LOCAL) == 0)
return (ENOTCAPABLE);
TAILQ_FOREACH_REVERSE(nt, &ndp->ni_cap_tracker, nameicap_tracker_head,
nm_link) {
if (dp == nt->dp) {
nt = TAILQ_NEXT(nt, nm_link);
if (nt != NULL)
nameicap_cleanup_from(ndp, nt);
return (0);
}
}
return (ENOTCAPABLE);
}
static void
namei_cleanup_cnp(struct componentname *cnp)
{
uma_zfree(namei_zone, cnp->cn_pnbuf);
#ifdef DIAGNOSTIC
cnp->cn_pnbuf = NULL;
cnp->cn_nameptr = NULL;
#endif
}
static int
namei_handle_root(struct nameidata *ndp, struct vnode **dpp)
{
struct componentname *cnp;
cnp = &ndp->ni_cnd;
if ((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
return (ENOTCAPABLE);
}
while (*(cnp->cn_nameptr) == '/') {
cnp->cn_nameptr++;
ndp->ni_pathlen--;
}
*dpp = ndp->ni_rootdir;
vrefact(*dpp);
return (0);
}
static int
namei_setup(struct nameidata *ndp, struct vnode **dpp, struct pwd **pwdp)
{
struct componentname *cnp;
- struct file *dfp;
struct thread *td;
struct pwd *pwd;
- cap_rights_t rights;
int error;
bool startdir_used;
cnp = &ndp->ni_cnd;
td = cnp->cn_thread;
startdir_used = false;
*pwdp = NULL;
*dpp = NULL;
#ifdef CAPABILITY_MODE
/*
* In capability mode, lookups must be restricted to happen in
* the subtree with the root specified by the file descriptor:
* - The root must be real file descriptor, not the pseudo-descriptor
* AT_FDCWD.
* - The passed path must be relative and not absolute.
* - If lookup_cap_dotdot is disabled, path must not contain the
* '..' components.
* - If lookup_cap_dotdot is enabled, we verify that all '..'
* components lookups result in the directories which were
* previously walked by us, which prevents an escape from
* the relative root.
*/
if (IN_CAPABILITY_MODE(td) && (cnp->cn_flags & NOCAPCHECK) == 0) {
ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
ndp->ni_resflags |= NIRES_STRICTREL;
if (ndp->ni_dirfd == AT_FDCWD) {
#ifdef KTRACE
if (KTRPOINT(td, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
return (ECAPMODE);
}
}
#endif
error = 0;
/*
* Get starting point for the translation.
*/
pwd = pwd_hold(td);
/*
* The reference on ni_rootdir is acquired in the block below to avoid
* back-to-back atomics for absolute lookups.
*/
ndp->ni_rootdir = pwd->pwd_rdir;
ndp->ni_topdir = pwd->pwd_jdir;
if (cnp->cn_pnbuf[0] == '/') {
ndp->ni_resflags |= NIRES_ABS;
error = namei_handle_root(ndp, dpp);
} else {
if (ndp->ni_startdir != NULL) {
*dpp = ndp->ni_startdir;
startdir_used = true;
} else if (ndp->ni_dirfd == AT_FDCWD) {
*dpp = pwd->pwd_cdir;
vrefact(*dpp);
} else {
- rights = *ndp->ni_rightsneeded;
- cap_rights_set_one(&rights, CAP_LOOKUP);
-
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_ATFD1(ndp->ni_dirfd);
if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_ATFD2(ndp->ni_dirfd);
- /*
- * Effectively inlined fgetvp_rights, because
- * we need to inspect the file as well as
- * grabbing the vnode. No check for O_PATH,
- * files to implement its semantic.
- */
- error = fget_cap(td, ndp->ni_dirfd, &rights,
- &dfp, &ndp->ni_filecaps);
- if (error != 0) {
- /*
- * Preserve the error; it should either be EBADF
- * or capability-related, both of which can be
- * safely returned to the caller.
- */
- } else {
- if (dfp->f_ops == &badfileops) {
- error = EBADF;
- } else if (dfp->f_vnode == NULL) {
- error = ENOTDIR;
- } else {
- *dpp = dfp->f_vnode;
- vref(*dpp);
-
- if ((dfp->f_flag & FSEARCH) != 0)
- cnp->cn_flags |= NOEXECCHECK;
- }
- fdrop(dfp, td);
- }
-#ifdef CAPABILITIES
- /*
- * If file descriptor doesn't have all rights,
- * all lookups relative to it must also be
- * strictly relative.
- */
- CAP_ALL(&rights);
- if (!cap_rights_contains(&ndp->ni_filecaps.fc_rights,
- &rights) ||
- ndp->ni_filecaps.fc_fcntls != CAP_FCNTL_ALL ||
- ndp->ni_filecaps.fc_nioctls != -1) {
- ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
- ndp->ni_resflags |= NIRES_STRICTREL;
- }
-#endif
+
+ error = fgetvp_lookup(ndp->ni_dirfd, ndp, dpp);
}
if (error == 0 && (*dpp)->v_type != VDIR &&
(cnp->cn_pnbuf[0] != '\0' ||
(cnp->cn_flags & EMPTYPATH) == 0))
error = ENOTDIR;
}
if (error == 0 && (cnp->cn_flags & RBENEATH) != 0) {
if (cnp->cn_pnbuf[0] == '/') {
error = ENOTCAPABLE;
} else if ((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) == 0) {
ndp->ni_lcf |= NI_LCF_STRICTRELATIVE |
NI_LCF_CAP_DOTDOT;
}
}
/*
* If we are auditing the kernel pathname, save the user pathname.
*/
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_UPATH1_VP(td, ndp->ni_rootdir, *dpp, cnp->cn_pnbuf);
if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_UPATH2_VP(td, ndp->ni_rootdir, *dpp, cnp->cn_pnbuf);
if (ndp->ni_startdir != NULL && !startdir_used)
vrele(ndp->ni_startdir);
if (error != 0) {
if (*dpp != NULL)
vrele(*dpp);
pwd_drop(pwd);
return (error);
}
if ((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0 &&
lookup_cap_dotdot != 0)
ndp->ni_lcf |= NI_LCF_CAP_DOTDOT;
SDT_PROBE4(vfs, namei, lookup, entry, *dpp, cnp->cn_pnbuf,
cnp->cn_flags, false);
*pwdp = pwd;
return (0);
}
static int
namei_getpath(struct nameidata *ndp)
{
struct componentname *cnp;
int error;
cnp = &ndp->ni_cnd;
/*
* Get a buffer for the name to be translated, and copy the
* name into the buffer.
*/
cnp->cn_pnbuf = uma_zalloc(namei_zone, M_WAITOK);
if (ndp->ni_segflg == UIO_SYSSPACE) {
error = copystr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
&ndp->ni_pathlen);
} else {
error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
&ndp->ni_pathlen);
}
if (__predict_false(error != 0))
return (error);
cnp->cn_nameptr = cnp->cn_pnbuf;
return (0);
}
static int
namei_emptypath(struct nameidata *ndp)
{
struct componentname *cnp;
struct pwd *pwd;
struct vnode *dp;
int error;
cnp = &ndp->ni_cnd;
MPASS(*cnp->cn_pnbuf == '\0');
MPASS((cnp->cn_flags & EMPTYPATH) != 0);
MPASS((cnp->cn_flags & (LOCKPARENT | WANTPARENT)) == 0);
ndp->ni_resflags |= NIRES_EMPTYPATH;
error = namei_setup(ndp, &dp, &pwd);
if (error != 0) {
namei_cleanup_cnp(cnp);
goto errout;
}
/*
* Usecount on dp already provided by namei_setup.
*/
ndp->ni_vp = dp;
namei_cleanup_cnp(cnp);
pwd_drop(pwd);
NDVALIDATE(ndp);
if ((cnp->cn_flags & LOCKLEAF) != 0) {
VOP_LOCK(dp, (cnp->cn_flags & LOCKSHARED) != 0 ?
LK_SHARED : LK_EXCLUSIVE);
if (VN_IS_DOOMED(dp)) {
vput(dp);
error = ENOENT;
goto errout;
}
}
SDT_PROBE4(vfs, namei, lookup, return, 0, ndp->ni_vp, false, ndp);
return (0);
errout:
SDT_PROBE4(vfs, namei, lookup, return, error, NULL, false, ndp);
return (error);
}
/*
* Convert a pathname into a pointer to a locked vnode.
*
* The FOLLOW flag is set when symbolic links are to be followed
* when they occur at the end of the name translation process.
* Symbolic links are always followed for all other pathname
* components other than the last.
*
* The segflg defines whether the name is to be copied from user
* space or kernel space.
*
* Overall outline of namei:
*
* copy in name
* get starting directory
* while (!done && !error) {
* call lookup to search path.
* if symbolic link, massage name in buffer and continue
* }
*/
int
namei(struct nameidata *ndp)
{
char *cp; /* pointer into pathname argument */
struct vnode *dp; /* the directory we are searching */
struct iovec aiov; /* uio for reading symbolic links */
struct componentname *cnp;
struct thread *td;
struct pwd *pwd;
struct uio auio;
int error, linklen;
enum cache_fpl_status status;
cnp = &ndp->ni_cnd;
td = cnp->cn_thread;
#ifdef INVARIANTS
KASSERT(cnp->cn_thread == curthread,
("namei not using curthread"));
KASSERT((ndp->ni_debugflags & NAMEI_DBG_CALLED) == 0,
("%s: repeated call to namei without NDREINIT", __func__));
KASSERT(ndp->ni_debugflags == NAMEI_DBG_INITED,
("%s: bad debugflags %d", __func__, ndp->ni_debugflags));
ndp->ni_debugflags |= NAMEI_DBG_CALLED;
if (ndp->ni_startdir != NULL)
ndp->ni_debugflags |= NAMEI_DBG_HADSTARTDIR;
if (cnp->cn_flags & FAILIFEXISTS) {
KASSERT(cnp->cn_nameiop == CREATE,
("%s: FAILIFEXISTS passed for op %d", __func__, cnp->cn_nameiop));
/*
* The limitation below is to restrict hairy corner cases.
*/
KASSERT((cnp->cn_flags & (LOCKPARENT | LOCKLEAF)) == LOCKPARENT,
("%s: FAILIFEXISTS must be passed with LOCKPARENT and without LOCKLEAF",
__func__));
}
/*
* For NDVALIDATE.
*
* While NDINIT may seem like a more natural place to do it, there are
* callers which directly modify flags past invoking init.
*/
cnp->cn_origflags = cnp->cn_flags;
#endif
ndp->ni_cnd.cn_cred = ndp->ni_cnd.cn_thread->td_ucred;
KASSERT(ndp->ni_resflags == 0, ("%s: garbage in ni_resflags: %x\n",
__func__, ndp->ni_resflags));
KASSERT(cnp->cn_cred && td->td_proc, ("namei: bad cred/proc"));
KASSERT((cnp->cn_flags & NAMEI_INTERNAL_FLAGS) == 0,
("namei: unexpected flags: %" PRIx64 "\n",
cnp->cn_flags & NAMEI_INTERNAL_FLAGS));
if (cnp->cn_flags & NOCACHE)
KASSERT(cnp->cn_nameiop != LOOKUP,
("%s: NOCACHE passed with LOOKUP", __func__));
MPASS(ndp->ni_startdir == NULL || ndp->ni_startdir->v_type == VDIR ||
ndp->ni_startdir->v_type == VBAD);
ndp->ni_lcf = 0;
ndp->ni_loopcnt = 0;
ndp->ni_vp = NULL;
error = namei_getpath(ndp);
if (__predict_false(error != 0)) {
namei_cleanup_cnp(cnp);
SDT_PROBE4(vfs, namei, lookup, return, error, NULL,
false, ndp);
return (error);
}
#ifdef KTRACE
if (KTRPOINT(td, KTR_NAMEI)) {
ktrnamei(cnp->cn_pnbuf);
}
#endif
TSNAMEI(curthread->td_proc->p_pid, cnp->cn_pnbuf);
/*
* First try looking up the target without locking any vnodes.
*
* We may need to start from scratch or pick up where it left off.
*/
error = cache_fplookup(ndp, &status, &pwd);
switch (status) {
case CACHE_FPL_STATUS_UNSET:
__assert_unreachable();
break;
case CACHE_FPL_STATUS_HANDLED:
if (error == 0)
NDVALIDATE(ndp);
return (error);
case CACHE_FPL_STATUS_PARTIAL:
TAILQ_INIT(&ndp->ni_cap_tracker);
dp = ndp->ni_startdir;
break;
case CACHE_FPL_STATUS_DESTROYED:
ndp->ni_loopcnt = 0;
error = namei_getpath(ndp);
if (__predict_false(error != 0)) {
namei_cleanup_cnp(cnp);
return (error);
}
/* FALLTHROUGH */
case CACHE_FPL_STATUS_ABORTED:
TAILQ_INIT(&ndp->ni_cap_tracker);
MPASS(ndp->ni_lcf == 0);
if (*cnp->cn_pnbuf == '\0') {
if ((cnp->cn_flags & EMPTYPATH) != 0) {
return (namei_emptypath(ndp));
}
namei_cleanup_cnp(cnp);
SDT_PROBE4(vfs, namei, lookup, return, ENOENT, NULL,
false, ndp);
return (ENOENT);
}
error = namei_setup(ndp, &dp, &pwd);
if (error != 0) {
namei_cleanup_cnp(cnp);
return (error);
}
break;
}
/*
* Locked lookup.
*/
for (;;) {
ndp->ni_startdir = dp;
error = lookup(ndp);
if (error != 0)
goto out;
/*
* If not a symbolic link, we're done.
*/
if ((cnp->cn_flags & ISSYMLINK) == 0) {
SDT_PROBE4(vfs, namei, lookup, return, error,
(error == 0 ? ndp->ni_vp : NULL), false, ndp);
if ((cnp->cn_flags & (SAVENAME | SAVESTART)) == 0) {
namei_cleanup_cnp(cnp);
} else
cnp->cn_flags |= HASBUF;
nameicap_cleanup(ndp);
pwd_drop(pwd);
if (error == 0)
NDVALIDATE(ndp);
return (error);
}
if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
error = ELOOP;
break;
}
#ifdef MAC
if ((cnp->cn_flags & NOMACCHECK) == 0) {
error = mac_vnode_check_readlink(td->td_ucred,
ndp->ni_vp);
if (error != 0)
break;
}
#endif
if (ndp->ni_pathlen > 1)
cp = uma_zalloc(namei_zone, M_WAITOK);
else
cp = cnp->cn_pnbuf;
aiov.iov_base = cp;
aiov.iov_len = MAXPATHLEN;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = 0;
auio.uio_rw = UIO_READ;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_td = td;
auio.uio_resid = MAXPATHLEN;
error = VOP_READLINK(ndp->ni_vp, &auio, cnp->cn_cred);
if (error != 0) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
break;
}
linklen = MAXPATHLEN - auio.uio_resid;
if (linklen == 0) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
error = ENOENT;
break;
}
if (linklen + ndp->ni_pathlen > MAXPATHLEN) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
error = ENAMETOOLONG;
break;
}
if (ndp->ni_pathlen > 1) {
bcopy(ndp->ni_next, cp + linklen, ndp->ni_pathlen);
uma_zfree(namei_zone, cnp->cn_pnbuf);
cnp->cn_pnbuf = cp;
} else
cnp->cn_pnbuf[linklen] = '\0';
ndp->ni_pathlen += linklen;
vput(ndp->ni_vp);
dp = ndp->ni_dvp;
/*
* Check if root directory should replace current directory.
*/
cnp->cn_nameptr = cnp->cn_pnbuf;
if (*(cnp->cn_nameptr) == '/') {
vrele(dp);
error = namei_handle_root(ndp, &dp);
if (error != 0)
goto out;
}
}
vput(ndp->ni_vp);
ndp->ni_vp = NULL;
vrele(ndp->ni_dvp);
out:
MPASS(error != 0);
SDT_PROBE4(vfs, namei, lookup, return, error, NULL, false, ndp);
namei_cleanup_cnp(cnp);
nameicap_cleanup(ndp);
pwd_drop(pwd);
return (error);
}
static int
compute_cn_lkflags(struct mount *mp, int lkflags, int cnflags)
{
if (mp == NULL || ((lkflags & LK_SHARED) &&
(!(mp->mnt_kern_flag & MNTK_LOOKUP_SHARED) ||
((cnflags & ISDOTDOT) &&
(mp->mnt_kern_flag & MNTK_LOOKUP_EXCL_DOTDOT))))) {
lkflags &= ~LK_SHARED;
lkflags |= LK_EXCLUSIVE;
}
lkflags |= LK_NODDLKTREAT;
return (lkflags);
}
static __inline int
needs_exclusive_leaf(struct mount *mp, int flags)
{
/*
* Intermediate nodes can use shared locks, we only need to
* force an exclusive lock for leaf nodes.
*/
if ((flags & (ISLASTCN | LOCKLEAF)) != (ISLASTCN | LOCKLEAF))
return (0);
/* Always use exclusive locks if LOCKSHARED isn't set. */
if (!(flags & LOCKSHARED))
return (1);
/*
* For lookups during open(), if the mount point supports
* extended shared operations, then use a shared lock for the
* leaf node, otherwise use an exclusive lock.
*/
if ((flags & ISOPEN) != 0)
return (!MNT_EXTENDED_SHARED(mp));
/*
* Lookup requests outside of open() that specify LOCKSHARED
* only need a shared lock on the leaf vnode.
*/
return (0);
}
/*
* Various filesystems expect to be able to copy a name component with length
* bounded by NAME_MAX into a directory entry buffer of size MAXNAMLEN. Make
* sure that these are the same size.
*/
_Static_assert(MAXNAMLEN == NAME_MAX,
"MAXNAMLEN and NAME_MAX have different values");
/*
* Search a pathname.
* This is a very central and rather complicated routine.
*
* The pathname is pointed to by ni_ptr and is of length ni_pathlen.
* The starting directory is taken from ni_startdir. The pathname is
* descended until done, or a symbolic link is encountered. The variable
* ni_more is clear if the path is completed; it is set to one if a
* symbolic link needing interpretation is encountered.
*
* The flag argument is LOOKUP, CREATE, RENAME, or DELETE depending on
* whether the name is to be looked up, created, renamed, or deleted.
* When CREATE, RENAME, or DELETE is specified, information usable in
* creating, renaming, or deleting a directory entry may be calculated.
* If flag has LOCKPARENT or'ed into it, the parent directory is returned
* locked. If flag has WANTPARENT or'ed into it, the parent directory is
* returned unlocked. Otherwise the parent directory is not returned. If
* the target of the pathname exists and LOCKLEAF is or'ed into the flag
* the target is returned locked, otherwise it is returned unlocked.
* When creating or renaming and LOCKPARENT is specified, the target may not
* be ".". When deleting and LOCKPARENT is specified, the target may be ".".
*
* Overall outline of lookup:
*
* dirloop:
* identify next component of name at ndp->ni_ptr
* handle degenerate case where name is null string
* if .. and crossing mount points and on mounted filesys, find parent
* call VOP_LOOKUP routine for next component name
* directory vnode returned in ni_dvp, unlocked unless LOCKPARENT set
* component vnode returned in ni_vp (if it exists), locked.
* if result vnode is mounted on and crossing mount points,
* find mounted on vnode
* if more components of name, do next level at dirloop
* return the answer in ni_vp, locked if LOCKLEAF set
* if LOCKPARENT set, return locked parent in ni_dvp
* if WANTPARENT set, return unlocked parent in ni_dvp
*/
int
lookup(struct nameidata *ndp)
{
char *cp; /* pointer into pathname argument */
char *prev_ni_next; /* saved ndp->ni_next */
char *nulchar; /* location of '\0' in cn_pnbuf */
struct vnode *dp = NULL; /* the directory we are searching */
struct vnode *tdp; /* saved dp */
struct mount *mp; /* mount table entry */
struct prison *pr;
size_t prev_ni_pathlen; /* saved ndp->ni_pathlen */
int docache; /* == 0 do not cache last component */
int wantparent; /* 1 => wantparent or lockparent flag */
int rdonly; /* lookup read-only flag bit */
int error = 0;
int dpunlocked = 0; /* dp has already been unlocked */
int relookup = 0; /* do not consume the path component */
struct componentname *cnp = &ndp->ni_cnd;
int lkflags_save;
int ni_dvp_unlocked;
/*
* Setup: break out flag bits into variables.
*/
ni_dvp_unlocked = 0;
wantparent = cnp->cn_flags & (LOCKPARENT | WANTPARENT);
KASSERT(cnp->cn_nameiop == LOOKUP || wantparent,
("CREATE, DELETE, RENAME require LOCKPARENT or WANTPARENT."));
/*
* When set to zero, docache causes the last component of the
* pathname to be deleted from the cache and the full lookup
* of the name to be done (via VOP_CACHEDLOOKUP()). Often
* filesystems need some pre-computed values that are made
* during the full lookup, for instance UFS sets dp->i_offset.
*
* The docache variable is set to zero when requested by the
* NOCACHE flag and for all modifying operations except CREATE.
*/
docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
if (cnp->cn_nameiop == DELETE ||
(wantparent && cnp->cn_nameiop != CREATE &&
cnp->cn_nameiop != LOOKUP))
docache = 0;
rdonly = cnp->cn_flags & RDONLY;
cnp->cn_flags &= ~ISSYMLINK;
ndp->ni_dvp = NULL;
/*
* We use shared locks until we hit the parent of the last cn then
* we adjust based on the requesting flags.
*/
cnp->cn_lkflags = LK_SHARED;
dp = ndp->ni_startdir;
ndp->ni_startdir = NULLVP;
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags | LK_RETRY,
cnp->cn_flags));
dirloop:
/*
* Search a new directory.
*
* The last component of the filename is left accessible via
* cnp->cn_nameptr for callers that need the name. Callers needing
* the name set the SAVENAME flag. When done, they assume
* responsibility for freeing the pathname buffer.
*
* Store / as a temporary sentinel so that we only have one character
* to test for. Pathnames tend to be short so this should not be
* resulting in cache misses.
*/
nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1];
KASSERT(*nulchar == '\0',
("%s: expected nul at %p; string [%s]\n", __func__, nulchar,
cnp->cn_pnbuf));
*nulchar = '/';
for (cp = cnp->cn_nameptr; *cp != '/'; cp++) {
KASSERT(*cp != '\0',
("%s: encountered unexpected nul; string [%s]\n", __func__,
cnp->cn_nameptr));
continue;
}
*nulchar = '\0';
cnp->cn_namelen = cp - cnp->cn_nameptr;
if (cnp->cn_namelen > NAME_MAX) {
error = ENAMETOOLONG;
goto bad;
}
#ifdef NAMEI_DIAGNOSTIC
{ char c = *cp;
*cp = '\0';
printf("{%s}: ", cnp->cn_nameptr);
*cp = c; }
#endif
prev_ni_pathlen = ndp->ni_pathlen;
ndp->ni_pathlen -= cnp->cn_namelen;
KASSERT(ndp->ni_pathlen <= PATH_MAX,
("%s: ni_pathlen underflow to %zd\n", __func__, ndp->ni_pathlen));
prev_ni_next = ndp->ni_next;
ndp->ni_next = cp;
/*
* Replace multiple slashes by a single slash and trailing slashes
* by a null. This must be done before VOP_LOOKUP() because some
* fs's don't know about trailing slashes. Remember if there were
* trailing slashes to handle symlinks, existing non-directories
* and non-existing files that won't be directories specially later.
*/
while (*cp == '/' && (cp[1] == '/' || cp[1] == '\0')) {
cp++;
ndp->ni_pathlen--;
if (*cp == '\0') {
*ndp->ni_next = '\0';
cnp->cn_flags |= TRAILINGSLASH;
}
}
ndp->ni_next = cp;
cnp->cn_flags |= MAKEENTRY;
if (*cp == '\0' && docache == 0)
cnp->cn_flags &= ~MAKEENTRY;
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
cnp->cn_flags |= ISDOTDOT;
else
cnp->cn_flags &= ~ISDOTDOT;
if (*ndp->ni_next == 0)
cnp->cn_flags |= ISLASTCN;
else
cnp->cn_flags &= ~ISLASTCN;
if ((cnp->cn_flags & ISLASTCN) != 0 &&
cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.' &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EINVAL;
goto bad;
}
nameicap_tracker_add(ndp, dp);
/*
* Check for degenerate name (e.g. / or "")
* which is a way of talking about a directory,
* e.g. like "/." or ".".
*/
if (cnp->cn_nameptr[0] == '\0') {
if (dp->v_type != VDIR) {
error = ENOTDIR;
goto bad;
}
if (cnp->cn_nameiop != LOOKUP) {
error = EISDIR;
goto bad;
}
if (wantparent) {
ndp->ni_dvp = dp;
VREF(dp);
}
ndp->ni_vp = dp;
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_VNODE1(dp);
else if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_VNODE2(dp);
if (!(cnp->cn_flags & (LOCKPARENT | LOCKLEAF)))
VOP_UNLOCK(dp);
/* XXX This should probably move to the top of function. */
if (cnp->cn_flags & SAVESTART)
panic("lookup: SAVESTART");
goto success;
}
/*
* Handle "..": five special cases.
* 0. If doing a capability lookup and lookup_cap_dotdot is
* disabled, return ENOTCAPABLE.
* 1. Return an error if this is the last component of
* the name and the operation is DELETE or RENAME.
* 2. If at root directory (e.g. after chroot)
* or at absolute root directory
* then ignore it so can't get out.
* 3. If this vnode is the root of a mounted
* filesystem, then replace it with the
* vnode which was mounted on so we take the
* .. in the other filesystem.
* 4. If the vnode is the top directory of
* the jail or chroot, don't let them out.
* 5. If doing a capability lookup and lookup_cap_dotdot is
* enabled, return ENOTCAPABLE if the lookup would escape
* from the initial file descriptor directory. Checks are
* done by ensuring that namei() already traversed the
* result of dotdot lookup.
*/
if (cnp->cn_flags & ISDOTDOT) {
if ((ndp->ni_lcf & (NI_LCF_STRICTRELATIVE | NI_LCF_CAP_DOTDOT))
== NI_LCF_STRICTRELATIVE) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
error = ENOTCAPABLE;
goto bad;
}
if ((cnp->cn_flags & ISLASTCN) != 0 &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EINVAL;
goto bad;
}
for (;;) {
for (pr = cnp->cn_cred->cr_prison; pr != NULL;
pr = pr->pr_parent)
if (dp == pr->pr_root)
break;
if (dp == ndp->ni_rootdir ||
dp == ndp->ni_topdir ||
dp == rootvnode ||
pr != NULL ||
((dp->v_vflag & VV_ROOT) != 0 &&
(cnp->cn_flags & NOCROSSMOUNT) != 0)) {
ndp->ni_dvp = dp;
ndp->ni_vp = dp;
VREF(dp);
goto nextname;
}
if ((dp->v_vflag & VV_ROOT) == 0)
break;
if (VN_IS_DOOMED(dp)) { /* forced unmount */
error = ENOENT;
goto bad;
}
tdp = dp;
dp = dp->v_mount->mnt_vnodecovered;
VREF(dp);
vput(tdp);
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags |
LK_RETRY, ISDOTDOT));
error = nameicap_check_dotdot(ndp, dp);
if (error != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
goto bad;
}
}
}
/*
* We now have a segment name to search for, and a directory to search.
*/
unionlookup:
#ifdef MAC
error = mac_vnode_check_lookup(cnp->cn_thread->td_ucred, dp, cnp);
if (error)
goto bad;
#endif
ndp->ni_dvp = dp;
ndp->ni_vp = NULL;
ASSERT_VOP_LOCKED(dp, "lookup");
/*
* If we have a shared lock we may need to upgrade the lock for the
* last operation.
*/
if ((cnp->cn_flags & LOCKPARENT) && (cnp->cn_flags & ISLASTCN) &&
dp != vp_crossmp && VOP_ISLOCKED(dp) == LK_SHARED)
vn_lock(dp, LK_UPGRADE|LK_RETRY);
if (VN_IS_DOOMED(dp)) {
error = ENOENT;
goto bad;
}
/*
* If we're looking up the last component and we need an exclusive
* lock, adjust our lkflags.
*/
if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags))
cnp->cn_lkflags = LK_EXCLUSIVE;
#ifdef NAMEI_DIAGNOSTIC
vn_printf(dp, "lookup in ");
#endif
lkflags_save = cnp->cn_lkflags;
cnp->cn_lkflags = compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags,
cnp->cn_flags);
error = VOP_LOOKUP(dp, &ndp->ni_vp, cnp);
cnp->cn_lkflags = lkflags_save;
if (error != 0) {
KASSERT(ndp->ni_vp == NULL, ("leaf should be empty"));
#ifdef NAMEI_DIAGNOSTIC
printf("not found\n");
#endif
if ((error == ENOENT) &&
(dp->v_vflag & VV_ROOT) && (dp->v_mount != NULL) &&
(dp->v_mount->mnt_flag & MNT_UNION)) {
tdp = dp;
dp = dp->v_mount->mnt_vnodecovered;
VREF(dp);
vput(tdp);
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags |
LK_RETRY, cnp->cn_flags));
nameicap_tracker_add(ndp, dp);
goto unionlookup;
}
if (error == ERELOOKUP) {
vref(dp);
ndp->ni_vp = dp;
error = 0;
relookup = 1;
goto good;
}
if (error != EJUSTRETURN)
goto bad;
/*
* At this point, we know we're at the end of the
* pathname. If creating / renaming, we can consider
* allowing the file or directory to be created / renamed,
* provided we're not on a read-only filesystem.
*/
if (rdonly) {
error = EROFS;
goto bad;
}
/* trailing slash only allowed for directories */
if ((cnp->cn_flags & TRAILINGSLASH) &&
!(cnp->cn_flags & WILLBEDIR)) {
error = ENOENT;
goto bad;
}
if ((cnp->cn_flags & LOCKPARENT) == 0)
VOP_UNLOCK(dp);
/*
* We return with ni_vp NULL to indicate that the entry
* doesn't currently exist, leaving a pointer to the
* (possibly locked) directory vnode in ndp->ni_dvp.
*/
if (cnp->cn_flags & SAVESTART) {
ndp->ni_startdir = ndp->ni_dvp;
VREF(ndp->ni_startdir);
}
goto success;
}
good:
#ifdef NAMEI_DIAGNOSTIC
printf("found\n");
#endif
dp = ndp->ni_vp;
/*
* Check to see if the vnode has been mounted on;
* if so find the root of the mounted filesystem.
*/
while (dp->v_type == VDIR && (mp = dp->v_mountedhere) &&
(cnp->cn_flags & NOCROSSMOUNT) == 0) {
if (vfs_busy(mp, 0))
continue;
vput(dp);
if (dp != ndp->ni_dvp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
vrefact(vp_crossmp);
ndp->ni_dvp = vp_crossmp;
error = VFS_ROOT(mp, compute_cn_lkflags(mp, cnp->cn_lkflags,
cnp->cn_flags), &tdp);
vfs_unbusy(mp);
if (vn_lock(vp_crossmp, LK_SHARED | LK_NOWAIT))
panic("vp_crossmp exclusively locked or reclaimed");
if (error) {
dpunlocked = 1;
goto bad2;
}
ndp->ni_vp = dp = tdp;
}
/*
* Check for symbolic link
*/
if ((dp->v_type == VLNK) &&
((cnp->cn_flags & FOLLOW) || (cnp->cn_flags & TRAILINGSLASH) ||
*ndp->ni_next == '/')) {
cnp->cn_flags |= ISSYMLINK;
if (VN_IS_DOOMED(dp)) {
/*
* We can't know whether the directory was mounted with
* NOSYMFOLLOW, so we can't follow safely.
*/
error = ENOENT;
goto bad2;
}
if (dp->v_mount->mnt_flag & MNT_NOSYMFOLLOW) {
error = EACCES;
goto bad2;
}
/*
* Symlink code always expects an unlocked dvp.
*/
if (ndp->ni_dvp != ndp->ni_vp) {
VOP_UNLOCK(ndp->ni_dvp);
ni_dvp_unlocked = 1;
}
goto success;
}
nextname:
/*
* Not a symbolic link that we will follow. Continue with the
* next component if there is any; otherwise, we're done.
*/
KASSERT((cnp->cn_flags & ISLASTCN) || *ndp->ni_next == '/',
("lookup: invalid path state."));
if (relookup) {
relookup = 0;
ndp->ni_pathlen = prev_ni_pathlen;
ndp->ni_next = prev_ni_next;
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
goto dirloop;
}
if (cnp->cn_flags & ISDOTDOT) {
error = nameicap_check_dotdot(ndp, ndp->ni_vp);
if (error != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
goto bad2;
}
}
if (*ndp->ni_next == '/') {
cnp->cn_nameptr = ndp->ni_next;
while (*cnp->cn_nameptr == '/') {
cnp->cn_nameptr++;
ndp->ni_pathlen--;
}
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
goto dirloop;
}
/*
* If we're processing a path with a trailing slash,
* check that the end result is a directory.
*/
if ((cnp->cn_flags & TRAILINGSLASH) && dp->v_type != VDIR) {
error = ENOTDIR;
goto bad2;
}
/*
* Disallow directory write attempts on read-only filesystems.
*/
if (rdonly &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EROFS;
goto bad2;
}
if (cnp->cn_flags & SAVESTART) {
ndp->ni_startdir = ndp->ni_dvp;
VREF(ndp->ni_startdir);
}
if (!wantparent) {
ni_dvp_unlocked = 2;
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
} else if ((cnp->cn_flags & LOCKPARENT) == 0 && ndp->ni_dvp != dp) {
VOP_UNLOCK(ndp->ni_dvp);
ni_dvp_unlocked = 1;
}
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_VNODE1(dp);
else if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_VNODE2(dp);
if ((cnp->cn_flags & LOCKLEAF) == 0)
VOP_UNLOCK(dp);
success:
/*
* FIXME: for lookups which only cross a mount point to fetch the
* root vnode, ni_dvp will be set to vp_crossmp. This can be a problem
* if either WANTPARENT or LOCKPARENT is set.
*/
/*
* Because of shared lookup we may have the vnode shared locked, but
* the caller may want it to be exclusively locked.
*/
if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags) &&
VOP_ISLOCKED(dp) != LK_EXCLUSIVE) {
vn_lock(dp, LK_UPGRADE | LK_RETRY);
if (VN_IS_DOOMED(dp)) {
error = ENOENT;
goto bad2;
}
}
if (ndp->ni_vp != NULL) {
if ((cnp->cn_flags & ISDOTDOT) == 0)
nameicap_tracker_add(ndp, ndp->ni_vp);
if ((cnp->cn_flags & (FAILIFEXISTS | ISSYMLINK)) == FAILIFEXISTS)
goto bad_eexist;
}
return (0);
bad2:
if (ni_dvp_unlocked != 2) {
if (dp != ndp->ni_dvp && !ni_dvp_unlocked)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
}
bad:
if (!dpunlocked)
vput(dp);
ndp->ni_vp = NULL;
return (error);
bad_eexist:
/*
* FAILIFEXISTS handling.
*
* XXX namei called with LOCKPARENT but not LOCKLEAF has the strange
* behaviour of leaving the vnode unlocked if the target is the same
* vnode as the parent.
*/
MPASS((cnp->cn_flags & ISSYMLINK) == 0);
if (ndp->ni_vp == ndp->ni_dvp)
vrele(ndp->ni_dvp);
else
vput(ndp->ni_dvp);
vrele(ndp->ni_vp);
ndp->ni_dvp = NULL;
ndp->ni_vp = NULL;
NDFREE(ndp, NDF_ONLY_PNBUF);
return (EEXIST);
}
/*
* relookup - lookup a path name component
* Used by lookup to re-acquire things.
*/
int
relookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp)
{
struct vnode *dp = NULL; /* the directory we are searching */
int rdonly; /* lookup read-only flag bit */
int error = 0;
KASSERT(cnp->cn_flags & ISLASTCN,
("relookup: Not given last component."));
/*
* Setup: break out flag bits into variables.
*/
KASSERT((cnp->cn_flags & (LOCKPARENT | WANTPARENT)) != 0,
("relookup: parent not wanted"));
rdonly = cnp->cn_flags & RDONLY;
cnp->cn_flags &= ~ISSYMLINK;
dp = dvp;
cnp->cn_lkflags = LK_EXCLUSIVE;
vn_lock(dp, LK_EXCLUSIVE | LK_RETRY);
/*
* Search a new directory.
*
* The last component of the filename is left accessible via
* cnp->cn_nameptr for callers that need the name. Callers needing
* the name set the SAVENAME flag. When done, they assume
* responsibility for freeing the pathname buffer.
*/
#ifdef NAMEI_DIAGNOSTIC
printf("{%s}: ", cnp->cn_nameptr);
#endif
/*
* Check for "" which represents the root directory after slash
* removal.
*/
if (cnp->cn_nameptr[0] == '\0') {
/*
* Support only LOOKUP for "/" because lookup()
* can't succeed for CREATE, DELETE and RENAME.
*/
KASSERT(cnp->cn_nameiop == LOOKUP, ("nameiop must be LOOKUP"));
KASSERT(dp->v_type == VDIR, ("dp is not a directory"));
if (!(cnp->cn_flags & LOCKLEAF))
VOP_UNLOCK(dp);
*vpp = dp;
/* XXX This should probably move to the top of function. */
if (cnp->cn_flags & SAVESTART)
panic("lookup: SAVESTART");
return (0);
}
if (cnp->cn_flags & ISDOTDOT)
panic ("relookup: lookup on dot-dot");
/*
* We now have a segment name to search for, and a directory to search.
*/
#ifdef NAMEI_DIAGNOSTIC
vn_printf(dp, "search in ");
#endif
if ((error = VOP_LOOKUP(dp, vpp, cnp)) != 0) {
KASSERT(*vpp == NULL, ("leaf should be empty"));
if (error != EJUSTRETURN)
goto bad;
/*
* If creating and at end of pathname, then can consider
* allowing file to be created.
*/
if (rdonly) {
error = EROFS;
goto bad;
}
/* ASSERT(dvp == ndp->ni_startdir) */
if (cnp->cn_flags & SAVESTART)
VREF(dvp);
if ((cnp->cn_flags & LOCKPARENT) == 0)
VOP_UNLOCK(dp);
/*
* We return with ni_vp NULL to indicate that the entry
* doesn't currently exist, leaving a pointer to the
* (possibly locked) directory vnode in ndp->ni_dvp.
*/
return (0);
}
dp = *vpp;
/*
* Disallow directory write attempts on read-only filesystems.
*/
if (rdonly &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
if (dvp == dp)
vrele(dvp);
else
vput(dvp);
error = EROFS;
goto bad;
}
/*
* Set the parent lock/ref state to the requested state.
*/
if ((cnp->cn_flags & LOCKPARENT) == 0 && dvp != dp)
VOP_UNLOCK(dvp);
/*
* Check for symbolic link
*/
KASSERT(dp->v_type != VLNK || !(cnp->cn_flags & FOLLOW),
("relookup: symlink found.\n"));
/* ASSERT(dvp == ndp->ni_startdir) */
if (cnp->cn_flags & SAVESTART)
VREF(dvp);
if ((cnp->cn_flags & LOCKLEAF) == 0)
VOP_UNLOCK(dp);
return (0);
bad:
vput(dp);
*vpp = NULL;
return (error);
}
/*
* Free data allocated by namei(); see namei(9) for details.
*/
void
NDFREE_PNBUF(struct nameidata *ndp)
{
if ((ndp->ni_cnd.cn_flags & HASBUF) != 0) {
MPASS((ndp->ni_cnd.cn_flags & (SAVENAME | SAVESTART)) != 0);
uma_zfree(namei_zone, ndp->ni_cnd.cn_pnbuf);
ndp->ni_cnd.cn_flags &= ~HASBUF;
}
}
/*
* NDFREE_PNBUF replacement for callers that know there is no buffer.
*
* This is a hack. Preferably the VFS layer would not produce anything more
* than it was asked to do. Unfortunately several non-LOOKUP cases can add the
* HASBUF flag to the result. Even then an interface could be implemented where
* the caller specifies what they expected to see in the result and what they
* are going to take care of.
*
* In the meantime provide this kludge as a trivial replacement for NDFREE_PNBUF
* calls scattered throughout the kernel where we know for a fact the flag must not
* be seen.
*/
#ifdef INVARIANTS
void
NDFREE_NOTHING(struct nameidata *ndp)
{
struct componentname *cnp;
cnp = &ndp->ni_cnd;
KASSERT(cnp->cn_nameiop == LOOKUP, ("%s: got non-LOOKUP op %d\n",
__func__, cnp->cn_nameiop));
KASSERT((cnp->cn_flags & (SAVENAME | HASBUF)) == 0,
("%s: bad flags \%" PRIx64 "\n", __func__, cnp->cn_flags));
}
#endif
void
(NDFREE)(struct nameidata *ndp, const u_int flags)
{
int unlock_dvp;
int unlock_vp;
unlock_dvp = 0;
unlock_vp = 0;
if (!(flags & NDF_NO_FREE_PNBUF)) {
NDFREE_PNBUF(ndp);
}
if (!(flags & NDF_NO_VP_UNLOCK) &&
(ndp->ni_cnd.cn_flags & LOCKLEAF) && ndp->ni_vp)
unlock_vp = 1;
if (!(flags & NDF_NO_DVP_UNLOCK) &&
(ndp->ni_cnd.cn_flags & LOCKPARENT) &&
ndp->ni_dvp != ndp->ni_vp)
unlock_dvp = 1;
if (!(flags & NDF_NO_VP_RELE) && ndp->ni_vp) {
if (unlock_vp) {
vput(ndp->ni_vp);
unlock_vp = 0;
} else
vrele(ndp->ni_vp);
ndp->ni_vp = NULL;
}
if (unlock_vp)
VOP_UNLOCK(ndp->ni_vp);
if (!(flags & NDF_NO_DVP_RELE) &&
(ndp->ni_cnd.cn_flags & (LOCKPARENT|WANTPARENT))) {
if (unlock_dvp) {
vput(ndp->ni_dvp);
unlock_dvp = 0;
} else
vrele(ndp->ni_dvp);
ndp->ni_dvp = NULL;
}
if (unlock_dvp)
VOP_UNLOCK(ndp->ni_dvp);
if (!(flags & NDF_NO_STARTDIR_RELE) &&
(ndp->ni_cnd.cn_flags & SAVESTART)) {
vrele(ndp->ni_startdir);
ndp->ni_startdir = NULL;
}
}
#ifdef INVARIANTS
/*
* Validate the final state of ndp after the lookup.
*
* Historically filesystems were allowed to modify cn_flags. Most notably they
* can add SAVENAME to the request, resulting in HASBUF and pushing subsequent
* clean up to the consumer. In practice this seems to only concern != LOOKUP
* operations.
*
* As a step towards stricter API contract this routine validates the state to
* clean up. Note validation is a work in progress with the intent of becoming
* stricter over time.
*/
#define NDMODIFYINGFLAGS (LOCKLEAF | LOCKPARENT | WANTPARENT | SAVENAME | SAVESTART | HASBUF)
void
NDVALIDATE(struct nameidata *ndp)
{
struct componentname *cnp;
u_int64_t used, orig;
cnp = &ndp->ni_cnd;
orig = cnp->cn_origflags;
used = cnp->cn_flags;
switch (cnp->cn_nameiop) {
case LOOKUP:
/*
* For plain lookup we require strict conformance -- nothing
* to clean up if it was not requested by the caller.
*/
orig &= NDMODIFYINGFLAGS;
used &= NDMODIFYINGFLAGS;
if ((orig & (SAVENAME | SAVESTART)) != 0)
orig |= HASBUF;
if (orig != used) {
goto out_mismatch;
}
break;
case CREATE:
case DELETE:
case RENAME:
/*
* Some filesystems set SAVENAME to provoke HASBUF, accomodate
* for it until it gets fixed.
*/
orig &= NDMODIFYINGFLAGS;
orig |= (SAVENAME | HASBUF);
used &= NDMODIFYINGFLAGS;
used |= (SAVENAME | HASBUF);
if (orig != used) {
goto out_mismatch;
}
break;
}
return;
out_mismatch:
panic("%s: mismatched flags for op %d: added %" PRIx64 ", "
"removed %" PRIx64" (%" PRIx64" != %" PRIx64"; stored %" PRIx64" != %" PRIx64")",
__func__, cnp->cn_nameiop, used & ~orig, orig &~ used,
orig, used, cnp->cn_origflags, cnp->cn_flags);
}
#endif
/*
* Determine if there is a suitable alternate filename under the specified
* prefix for the specified path. If the create flag is set, then the
* alternate prefix will be used so long as the parent directory exists.
* This is used by the various compatibility ABIs so that Linux binaries prefer
* files under /compat/linux for example. The chosen path (whether under
* the prefix or under /) is returned in a kernel malloc'd buffer pointed
* to by pathbuf. The caller is responsible for free'ing the buffer from
* the M_TEMP bucket if one is returned.
*/
int
kern_alternate_path(struct thread *td, const char *prefix, const char *path,
enum uio_seg pathseg, char **pathbuf, int create, int dirfd)
{
struct nameidata nd, ndroot;
char *ptr, *buf, *cp;
size_t len, sz;
int error;
buf = (char *) malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
*pathbuf = buf;
/* Copy the prefix into the new pathname as a starting point. */
len = strlcpy(buf, prefix, MAXPATHLEN);
if (len >= MAXPATHLEN) {
*pathbuf = NULL;
free(buf, M_TEMP);
return (EINVAL);
}
sz = MAXPATHLEN - len;
ptr = buf + len;
/* Append the filename to the prefix. */
if (pathseg == UIO_SYSSPACE)
error = copystr(path, ptr, sz, &len);
else
error = copyinstr(path, ptr, sz, &len);
if (error) {
*pathbuf = NULL;
free(buf, M_TEMP);
return (error);
}
/* Only use a prefix with absolute pathnames. */
if (*ptr != '/') {
error = EINVAL;
goto keeporig;
}
if (dirfd != AT_FDCWD) {
/*
* We want the original because the "prefix" is
* included in the already opened dirfd.
*/
bcopy(ptr, buf, len);
return (0);
}
/*
* We know that there is a / somewhere in this pathname.
* Search backwards for it, to find the file's parent dir
* to see if it exists in the alternate tree. If it does,
* and we want to create a file (cflag is set). We don't
* need to worry about the root comparison in this case.
*/
if (create) {
for (cp = &ptr[len] - 1; *cp != '/'; cp--);
*cp = '\0';
NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);
error = namei(&nd);
*cp = '/';
if (error != 0)
goto keeporig;
} else {
NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);
error = namei(&nd);
if (error != 0)
goto keeporig;
/*
* We now compare the vnode of the prefix to the one
* vnode asked. If they resolve to be the same, then we
* ignore the match so that the real root gets used.
* This avoids the problem of traversing "../.." to find the
* root directory and never finding it, because "/" resolves
* to the emulation root directory. This is expensive :-(
*/
NDINIT(&ndroot, LOOKUP, FOLLOW, UIO_SYSSPACE, prefix,
td);
/* We shouldn't ever get an error from this namei(). */
error = namei(&ndroot);
if (error == 0) {
if (nd.ni_vp == ndroot.ni_vp)
error = ENOENT;
NDFREE(&ndroot, NDF_ONLY_PNBUF);
vrele(ndroot.ni_vp);
}
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vrele(nd.ni_vp);
keeporig:
/* If there was an error, use the original path name. */
if (error)
bcopy(ptr, buf, len);
return (error);
}
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 19a32a175895..ed75316f8add 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -1,5013 +1,5013 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1989, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_syscalls.c 8.13 (Berkeley) 4/15/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/capsicum.h>
#include <sys/disk.h>
#include <sys/sysent.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/sysproto.h>
#include <sys/namei.h>
#include <sys/filedesc.h>
#include <sys/kernel.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/filio.h>
#include <sys/limits.h>
#include <sys/linker.h>
#include <sys/rwlock.h>
#include <sys/sdt.h>
#include <sys/stat.h>
#include <sys/sx.h>
#include <sys/unistd.h>
#include <sys/vnode.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/dirent.h>
#include <sys/jail.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#include <machine/stdarg.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/uma.h>
#include <fs/devfs/devfs.h>
#include <ufs/ufs/quota.h>
MALLOC_DEFINE(M_FADVISE, "fadvise", "posix_fadvise(2) information");
static int kern_chflagsat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, u_long flags, int atflag);
static int setfflags(struct thread *td, struct vnode *, u_long);
static int getutimes(const struct timeval *, enum uio_seg, struct timespec *);
static int getutimens(const struct timespec *, enum uio_seg,
struct timespec *, int *);
static int setutimes(struct thread *td, struct vnode *,
const struct timespec *, int, int);
static int vn_access(struct vnode *vp, int user_flags, struct ucred *cred,
struct thread *td);
static int kern_fhlinkat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, fhandle_t *fhp);
static int kern_readlink_vp(struct vnode *vp, char *buf, enum uio_seg bufseg,
size_t count, struct thread *td);
static int kern_linkat_vp(struct thread *td, struct vnode *vp, int fd,
const char *path, enum uio_seg segflag);
static uint64_t
at2cnpflags(u_int at_flags, u_int mask)
{
u_int64_t res;
MPASS((at_flags & (AT_SYMLINK_FOLLOW | AT_SYMLINK_NOFOLLOW)) !=
(AT_SYMLINK_FOLLOW | AT_SYMLINK_NOFOLLOW));
res = 0;
at_flags &= mask;
if ((at_flags & AT_RESOLVE_BENEATH) != 0)
res |= RBENEATH;
if ((at_flags & AT_SYMLINK_FOLLOW) != 0)
res |= FOLLOW;
/* NOFOLLOW is pseudo flag */
if ((mask & AT_SYMLINK_NOFOLLOW) != 0) {
res |= (at_flags & AT_SYMLINK_NOFOLLOW) != 0 ? NOFOLLOW :
FOLLOW;
}
if ((mask & AT_EMPTY_PATH) != 0 && (at_flags & AT_EMPTY_PATH) != 0)
res |= EMPTYPATH;
return (res);
}
int
kern_sync(struct thread *td)
{
struct mount *mp, *nmp;
int save;
mtx_lock(&mountlist_mtx);
for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
if ((mp->mnt_flag & MNT_RDONLY) == 0 &&
vn_start_write(NULL, &mp, V_NOWAIT) == 0) {
save = curthread_pflags_set(TDP_SYNCIO);
vfs_periodic(mp, MNT_NOWAIT);
VFS_SYNC(mp, MNT_NOWAIT);
curthread_pflags_restore(save);
vn_finished_write(mp);
}
mtx_lock(&mountlist_mtx);
nmp = TAILQ_NEXT(mp, mnt_list);
vfs_unbusy(mp);
}
mtx_unlock(&mountlist_mtx);
return (0);
}
/*
* Sync each mounted filesystem.
*/
#ifndef _SYS_SYSPROTO_H_
struct sync_args {
int dummy;
};
#endif
/* ARGSUSED */
int
sys_sync(struct thread *td, struct sync_args *uap)
{
return (kern_sync(td));
}
/*
* Change filesystem quotas.
*/
#ifndef _SYS_SYSPROTO_H_
struct quotactl_args {
char *path;
int cmd;
int uid;
caddr_t arg;
};
#endif
int
sys_quotactl(struct thread *td, struct quotactl_args *uap)
{
struct mount *mp;
struct nameidata nd;
int error;
AUDIT_ARG_CMD(uap->cmd);
AUDIT_ARG_UID(uap->uid);
if (!prison_allow(td->td_ucred, PR_ALLOW_QUOTAS))
return (EPERM);
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
uap->path, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE(&nd, NDF_ONLY_PNBUF);
mp = nd.ni_vp->v_mount;
vfs_ref(mp);
vput(nd.ni_vp);
error = vfs_busy(mp, 0);
if (error != 0) {
vfs_rel(mp);
return (error);
}
error = VFS_QUOTACTL(mp, uap->cmd, uap->uid, uap->arg);
/*
* Since quota on operation typically needs to open quota
* file, the Q_QUOTAON handler needs to unbusy the mount point
* before calling into namei. Otherwise, unmount might be
* started between two vfs_busy() invocations (first is our,
* second is from mount point cross-walk code in lookup()),
* causing deadlock.
*
* Require that Q_QUOTAON handles the vfs_busy() reference on
* its own, always returning with ubusied mount point.
*/
if ((uap->cmd >> SUBCMDSHIFT) != Q_QUOTAON &&
(uap->cmd >> SUBCMDSHIFT) != Q_QUOTAOFF)
vfs_unbusy(mp);
vfs_rel(mp);
return (error);
}
/*
* Used by statfs conversion routines to scale the block size up if
* necessary so that all of the block counts are <= 'max_size'. Note
* that 'max_size' should be a bitmask, i.e. 2^n - 1 for some non-zero
* value of 'n'.
*/
void
statfs_scale_blocks(struct statfs *sf, long max_size)
{
uint64_t count;
int shift;
KASSERT(powerof2(max_size + 1), ("%s: invalid max_size", __func__));
/*
* Attempt to scale the block counts to give a more accurate
* overview to userland of the ratio of free space to used
* space. To do this, find the largest block count and compute
* a divisor that lets it fit into a signed integer <= max_size.
*/
if (sf->f_bavail < 0)
count = -sf->f_bavail;
else
count = sf->f_bavail;
count = MAX(sf->f_blocks, MAX(sf->f_bfree, count));
if (count <= max_size)
return;
count >>= flsl(max_size);
shift = 0;
while (count > 0) {
shift++;
count >>=1;
}
sf->f_bsize <<= shift;
sf->f_blocks >>= shift;
sf->f_bfree >>= shift;
sf->f_bavail >>= shift;
}
static int
kern_do_statfs(struct thread *td, struct mount *mp, struct statfs *buf)
{
int error;
if (mp == NULL)
return (EBADF);
error = vfs_busy(mp, 0);
vfs_rel(mp);
if (error != 0)
return (error);
#ifdef MAC
error = mac_mount_check_stat(td->td_ucred, mp);
if (error != 0)
goto out;
#endif
error = VFS_STATFS(mp, buf);
if (error != 0)
goto out;
if (priv_check_cred_vfs_generation(td->td_ucred)) {
buf->f_fsid.val[0] = buf->f_fsid.val[1] = 0;
prison_enforce_statfs(td->td_ucred, mp, buf);
}
out:
vfs_unbusy(mp);
return (error);
}
/*
* Get filesystem statistics.
*/
#ifndef _SYS_SYSPROTO_H_
struct statfs_args {
char *path;
struct statfs *buf;
};
#endif
int
sys_statfs(struct thread *td, struct statfs_args *uap)
{
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
if (error == 0)
error = copyout(sfp, uap->buf, sizeof(struct statfs));
free(sfp, M_STATFS);
return (error);
}
int
kern_statfs(struct thread *td, const char *path, enum uio_seg pathseg,
struct statfs *buf)
{
struct mount *mp;
struct nameidata nd;
int error;
NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, td);
error = namei(&nd);
if (error != 0)
return (error);
mp = vfs_ref_from_vp(nd.ni_vp);
NDFREE_NOTHING(&nd);
vrele(nd.ni_vp);
return (kern_do_statfs(td, mp, buf));
}
/*
* Get filesystem statistics.
*/
#ifndef _SYS_SYSPROTO_H_
struct fstatfs_args {
int fd;
struct statfs *buf;
};
#endif
int
sys_fstatfs(struct thread *td, struct fstatfs_args *uap)
{
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fstatfs(td, uap->fd, sfp);
if (error == 0)
error = copyout(sfp, uap->buf, sizeof(struct statfs));
free(sfp, M_STATFS);
return (error);
}
int
kern_fstatfs(struct thread *td, int fd, struct statfs *buf)
{
struct file *fp;
struct mount *mp;
struct vnode *vp;
int error;
AUDIT_ARG_FD(fd);
error = getvnode_path(td, fd, &cap_fstatfs_rights, &fp);
if (error != 0)
return (error);
vp = fp->f_vnode;
#ifdef AUDIT
if (AUDITING_TD(td)) {
vn_lock(vp, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(vp);
VOP_UNLOCK(vp);
}
#endif
mp = vfs_ref_from_vp(vp);
fdrop(fp, td);
return (kern_do_statfs(td, mp, buf));
}
/*
* Get statistics on all filesystems.
*/
#ifndef _SYS_SYSPROTO_H_
struct getfsstat_args {
struct statfs *buf;
long bufsize;
int mode;
};
#endif
int
sys_getfsstat(struct thread *td, struct getfsstat_args *uap)
{
size_t count;
int error;
if (uap->bufsize < 0 || uap->bufsize > SIZE_MAX)
return (EINVAL);
error = kern_getfsstat(td, &uap->buf, uap->bufsize, &count,
UIO_USERSPACE, uap->mode);
if (error == 0)
td->td_retval[0] = count;
return (error);
}
/*
* If (bufsize > 0 && bufseg == UIO_SYSSPACE)
* The caller is responsible for freeing memory which will be allocated
* in '*buf'.
*/
int
kern_getfsstat(struct thread *td, struct statfs **buf, size_t bufsize,
size_t *countp, enum uio_seg bufseg, int mode)
{
struct mount *mp, *nmp;
struct statfs *sfsp, *sp, *sptmp, *tofree;
size_t count, maxcount;
int error;
switch (mode) {
case MNT_WAIT:
case MNT_NOWAIT:
break;
default:
if (bufseg == UIO_SYSSPACE)
*buf = NULL;
return (EINVAL);
}
restart:
maxcount = bufsize / sizeof(struct statfs);
if (bufsize == 0) {
sfsp = NULL;
tofree = NULL;
} else if (bufseg == UIO_USERSPACE) {
sfsp = *buf;
tofree = NULL;
} else /* if (bufseg == UIO_SYSSPACE) */ {
count = 0;
mtx_lock(&mountlist_mtx);
TAILQ_FOREACH(mp, &mountlist, mnt_list) {
count++;
}
mtx_unlock(&mountlist_mtx);
if (maxcount > count)
maxcount = count;
tofree = sfsp = *buf = malloc(maxcount * sizeof(struct statfs),
M_STATFS, M_WAITOK);
}
count = 0;
/*
* If there is no target buffer they only want the count.
*
* This could be TAILQ_FOREACH but it is open-coded to match the original
* code below.
*/
if (sfsp == NULL) {
mtx_lock(&mountlist_mtx);
for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
if (prison_canseemount(td->td_ucred, mp) != 0) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
#ifdef MAC
if (mac_mount_check_stat(td->td_ucred, mp) != 0) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
#endif
count++;
nmp = TAILQ_NEXT(mp, mnt_list);
}
mtx_unlock(&mountlist_mtx);
*countp = count;
return (0);
}
/*
* They want the entire thing.
*
* Short-circuit the corner case of no room for anything, avoids
* relocking below.
*/
if (maxcount < 1) {
goto out;
}
mtx_lock(&mountlist_mtx);
for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) {
if (prison_canseemount(td->td_ucred, mp) != 0) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
#ifdef MAC
if (mac_mount_check_stat(td->td_ucred, mp) != 0) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
#endif
if (mode == MNT_WAIT) {
if (vfs_busy(mp, MBF_MNTLSTLOCK) != 0) {
/*
* If vfs_busy() failed, and MBF_NOWAIT
* wasn't passed, then the mp is gone.
* Furthermore, because of MBF_MNTLSTLOCK,
* the mountlist_mtx was dropped. We have
* no other choice than to start over.
*/
mtx_unlock(&mountlist_mtx);
free(tofree, M_STATFS);
goto restart;
}
} else {
if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK) != 0) {
nmp = TAILQ_NEXT(mp, mnt_list);
continue;
}
}
sp = &mp->mnt_stat;
/*
* If MNT_NOWAIT is specified, do not refresh
* the fsstat cache.
*/
if (mode != MNT_NOWAIT) {
error = VFS_STATFS(mp, sp);
if (error != 0) {
mtx_lock(&mountlist_mtx);
nmp = TAILQ_NEXT(mp, mnt_list);
vfs_unbusy(mp);
continue;
}
}
if (priv_check_cred_vfs_generation(td->td_ucred)) {
sptmp = malloc(sizeof(struct statfs), M_STATFS,
M_WAITOK);
*sptmp = *sp;
sptmp->f_fsid.val[0] = sptmp->f_fsid.val[1] = 0;
prison_enforce_statfs(td->td_ucred, mp, sptmp);
sp = sptmp;
} else
sptmp = NULL;
if (bufseg == UIO_SYSSPACE) {
bcopy(sp, sfsp, sizeof(*sp));
free(sptmp, M_STATFS);
} else /* if (bufseg == UIO_USERSPACE) */ {
error = copyout(sp, sfsp, sizeof(*sp));
free(sptmp, M_STATFS);
if (error != 0) {
vfs_unbusy(mp);
return (error);
}
}
sfsp++;
count++;
if (count == maxcount) {
vfs_unbusy(mp);
goto out;
}
mtx_lock(&mountlist_mtx);
nmp = TAILQ_NEXT(mp, mnt_list);
vfs_unbusy(mp);
}
mtx_unlock(&mountlist_mtx);
out:
*countp = count;
return (0);
}
#ifdef COMPAT_FREEBSD4
/*
* Get old format filesystem statistics.
*/
static void freebsd4_cvtstatfs(struct statfs *, struct ostatfs *);
#ifndef _SYS_SYSPROTO_H_
struct freebsd4_statfs_args {
char *path;
struct ostatfs *buf;
};
#endif
int
freebsd4_statfs(struct thread *td, struct freebsd4_statfs_args *uap)
{
struct ostatfs osb;
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
if (error == 0) {
freebsd4_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Get filesystem statistics.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd4_fstatfs_args {
int fd;
struct ostatfs *buf;
};
#endif
int
freebsd4_fstatfs(struct thread *td, struct freebsd4_fstatfs_args *uap)
{
struct ostatfs osb;
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fstatfs(td, uap->fd, sfp);
if (error == 0) {
freebsd4_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Get statistics on all filesystems.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd4_getfsstat_args {
struct ostatfs *buf;
long bufsize;
int mode;
};
#endif
int
freebsd4_getfsstat(struct thread *td, struct freebsd4_getfsstat_args *uap)
{
struct statfs *buf, *sp;
struct ostatfs osb;
size_t count, size;
int error;
if (uap->bufsize < 0)
return (EINVAL);
count = uap->bufsize / sizeof(struct ostatfs);
if (count > SIZE_MAX / sizeof(struct statfs))
return (EINVAL);
size = count * sizeof(struct statfs);
error = kern_getfsstat(td, &buf, size, &count, UIO_SYSSPACE,
uap->mode);
if (error == 0)
td->td_retval[0] = count;
if (size != 0) {
sp = buf;
while (count != 0 && error == 0) {
freebsd4_cvtstatfs(sp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
sp++;
uap->buf++;
count--;
}
free(buf, M_STATFS);
}
return (error);
}
/*
* Implement fstatfs() for (NFS) file handles.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd4_fhstatfs_args {
struct fhandle *u_fhp;
struct ostatfs *buf;
};
#endif
int
freebsd4_fhstatfs(struct thread *td, struct freebsd4_fhstatfs_args *uap)
{
struct ostatfs osb;
struct statfs *sfp;
fhandle_t fh;
int error;
error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
if (error != 0)
return (error);
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fhstatfs(td, fh, sfp);
if (error == 0) {
freebsd4_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Convert a new format statfs structure to an old format statfs structure.
*/
static void
freebsd4_cvtstatfs(struct statfs *nsp, struct ostatfs *osp)
{
statfs_scale_blocks(nsp, LONG_MAX);
bzero(osp, sizeof(*osp));
osp->f_bsize = nsp->f_bsize;
osp->f_iosize = MIN(nsp->f_iosize, LONG_MAX);
osp->f_blocks = nsp->f_blocks;
osp->f_bfree = nsp->f_bfree;
osp->f_bavail = nsp->f_bavail;
osp->f_files = MIN(nsp->f_files, LONG_MAX);
osp->f_ffree = MIN(nsp->f_ffree, LONG_MAX);
osp->f_owner = nsp->f_owner;
osp->f_type = nsp->f_type;
osp->f_flags = nsp->f_flags;
osp->f_syncwrites = MIN(nsp->f_syncwrites, LONG_MAX);
osp->f_asyncwrites = MIN(nsp->f_asyncwrites, LONG_MAX);
osp->f_syncreads = MIN(nsp->f_syncreads, LONG_MAX);
osp->f_asyncreads = MIN(nsp->f_asyncreads, LONG_MAX);
strlcpy(osp->f_fstypename, nsp->f_fstypename,
MIN(MFSNAMELEN, OMFSNAMELEN));
strlcpy(osp->f_mntonname, nsp->f_mntonname,
MIN(MNAMELEN, OMNAMELEN));
strlcpy(osp->f_mntfromname, nsp->f_mntfromname,
MIN(MNAMELEN, OMNAMELEN));
osp->f_fsid = nsp->f_fsid;
}
#endif /* COMPAT_FREEBSD4 */
#if defined(COMPAT_FREEBSD11)
/*
* Get old format filesystem statistics.
*/
static void freebsd11_cvtstatfs(struct statfs *, struct freebsd11_statfs *);
int
freebsd11_statfs(struct thread *td, struct freebsd11_statfs_args *uap)
{
struct freebsd11_statfs osb;
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_statfs(td, uap->path, UIO_USERSPACE, sfp);
if (error == 0) {
freebsd11_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Get filesystem statistics.
*/
int
freebsd11_fstatfs(struct thread *td, struct freebsd11_fstatfs_args *uap)
{
struct freebsd11_statfs osb;
struct statfs *sfp;
int error;
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fstatfs(td, uap->fd, sfp);
if (error == 0) {
freebsd11_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Get statistics on all filesystems.
*/
int
freebsd11_getfsstat(struct thread *td, struct freebsd11_getfsstat_args *uap)
{
struct freebsd11_statfs osb;
struct statfs *buf, *sp;
size_t count, size;
int error;
count = uap->bufsize / sizeof(struct ostatfs);
size = count * sizeof(struct statfs);
error = kern_getfsstat(td, &buf, size, &count, UIO_SYSSPACE,
uap->mode);
if (error == 0)
td->td_retval[0] = count;
if (size > 0) {
sp = buf;
while (count > 0 && error == 0) {
freebsd11_cvtstatfs(sp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
sp++;
uap->buf++;
count--;
}
free(buf, M_STATFS);
}
return (error);
}
/*
* Implement fstatfs() for (NFS) file handles.
*/
int
freebsd11_fhstatfs(struct thread *td, struct freebsd11_fhstatfs_args *uap)
{
struct freebsd11_statfs osb;
struct statfs *sfp;
fhandle_t fh;
int error;
error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
if (error)
return (error);
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fhstatfs(td, fh, sfp);
if (error == 0) {
freebsd11_cvtstatfs(sfp, &osb);
error = copyout(&osb, uap->buf, sizeof(osb));
}
free(sfp, M_STATFS);
return (error);
}
/*
* Convert a new format statfs structure to an old format statfs structure.
*/
static void
freebsd11_cvtstatfs(struct statfs *nsp, struct freebsd11_statfs *osp)
{
bzero(osp, sizeof(*osp));
osp->f_version = FREEBSD11_STATFS_VERSION;
osp->f_type = nsp->f_type;
osp->f_flags = nsp->f_flags;
osp->f_bsize = nsp->f_bsize;
osp->f_iosize = nsp->f_iosize;
osp->f_blocks = nsp->f_blocks;
osp->f_bfree = nsp->f_bfree;
osp->f_bavail = nsp->f_bavail;
osp->f_files = nsp->f_files;
osp->f_ffree = nsp->f_ffree;
osp->f_syncwrites = nsp->f_syncwrites;
osp->f_asyncwrites = nsp->f_asyncwrites;
osp->f_syncreads = nsp->f_syncreads;
osp->f_asyncreads = nsp->f_asyncreads;
osp->f_namemax = nsp->f_namemax;
osp->f_owner = nsp->f_owner;
osp->f_fsid = nsp->f_fsid;
strlcpy(osp->f_fstypename, nsp->f_fstypename,
MIN(MFSNAMELEN, sizeof(osp->f_fstypename)));
strlcpy(osp->f_mntonname, nsp->f_mntonname,
MIN(MNAMELEN, sizeof(osp->f_mntonname)));
strlcpy(osp->f_mntfromname, nsp->f_mntfromname,
MIN(MNAMELEN, sizeof(osp->f_mntfromname)));
}
#endif /* COMPAT_FREEBSD11 */
/*
* Change current working directory to a given file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fchdir_args {
int fd;
};
#endif
int
sys_fchdir(struct thread *td, struct fchdir_args *uap)
{
struct vnode *vp, *tdp;
struct mount *mp;
struct file *fp;
int error;
AUDIT_ARG_FD(uap->fd);
error = getvnode_path(td, uap->fd, &cap_fchdir_rights,
&fp);
if (error != 0)
return (error);
vp = fp->f_vnode;
vref(vp);
fdrop(fp, td);
vn_lock(vp, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(vp);
error = change_dir(vp, td);
while (!error && (mp = vp->v_mountedhere) != NULL) {
if (vfs_busy(mp, 0))
continue;
error = VFS_ROOT(mp, LK_SHARED, &tdp);
vfs_unbusy(mp);
if (error != 0)
break;
vput(vp);
vp = tdp;
}
if (error != 0) {
vput(vp);
return (error);
}
VOP_UNLOCK(vp);
pwd_chdir(td, vp);
return (0);
}
/*
* Change current working directory (``.'').
*/
#ifndef _SYS_SYSPROTO_H_
struct chdir_args {
char *path;
};
#endif
int
sys_chdir(struct thread *td, struct chdir_args *uap)
{
return (kern_chdir(td, uap->path, UIO_USERSPACE));
}
int
kern_chdir(struct thread *td, const char *path, enum uio_seg pathseg)
{
struct nameidata nd;
int error;
NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1,
pathseg, path, td);
if ((error = namei(&nd)) != 0)
return (error);
if ((error = change_dir(nd.ni_vp, td)) != 0) {
vput(nd.ni_vp);
NDFREE_NOTHING(&nd);
return (error);
}
VOP_UNLOCK(nd.ni_vp);
NDFREE_NOTHING(&nd);
pwd_chdir(td, nd.ni_vp);
return (0);
}
static int unprivileged_chroot = 0;
SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_chroot, CTLFLAG_RW,
&unprivileged_chroot, 0,
"Unprivileged processes can use chroot(2)");
/*
* Change notion of root (``/'') directory.
*/
#ifndef _SYS_SYSPROTO_H_
struct chroot_args {
char *path;
};
#endif
int
sys_chroot(struct thread *td, struct chroot_args *uap)
{
struct nameidata nd;
struct proc *p;
int error;
error = priv_check(td, PRIV_VFS_CHROOT);
if (error != 0) {
p = td->td_proc;
PROC_LOCK(p);
if (unprivileged_chroot == 0 ||
(p->p_flag2 & P2_NO_NEW_PRIVS) == 0) {
PROC_UNLOCK(p);
return (error);
}
PROC_UNLOCK(p);
}
NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1,
UIO_USERSPACE, uap->path, td);
error = namei(&nd);
if (error != 0)
goto error;
error = change_dir(nd.ni_vp, td);
if (error != 0)
goto e_vunlock;
#ifdef MAC
error = mac_vnode_check_chroot(td->td_ucred, nd.ni_vp);
if (error != 0)
goto e_vunlock;
#endif
VOP_UNLOCK(nd.ni_vp);
error = pwd_chroot(td, nd.ni_vp);
vrele(nd.ni_vp);
NDFREE_NOTHING(&nd);
return (error);
e_vunlock:
vput(nd.ni_vp);
error:
NDFREE_NOTHING(&nd);
return (error);
}
/*
* Common routine for chroot and chdir. Callers must provide a locked vnode
* instance.
*/
int
change_dir(struct vnode *vp, struct thread *td)
{
#ifdef MAC
int error;
#endif
ASSERT_VOP_LOCKED(vp, "change_dir(): vp not locked");
if (vp->v_type != VDIR)
return (ENOTDIR);
#ifdef MAC
error = mac_vnode_check_chdir(td->td_ucred, vp);
if (error != 0)
return (error);
#endif
return (VOP_ACCESS(vp, VEXEC, td->td_ucred, td));
}
static __inline void
flags_to_rights(int flags, cap_rights_t *rightsp)
{
if (flags & O_EXEC) {
cap_rights_set_one(rightsp, CAP_FEXECVE);
if (flags & O_PATH)
return;
} else {
switch ((flags & O_ACCMODE)) {
case O_RDONLY:
cap_rights_set_one(rightsp, CAP_READ);
break;
case O_RDWR:
cap_rights_set_one(rightsp, CAP_READ);
/* FALLTHROUGH */
case O_WRONLY:
cap_rights_set_one(rightsp, CAP_WRITE);
if (!(flags & (O_APPEND | O_TRUNC)))
cap_rights_set_one(rightsp, CAP_SEEK);
break;
}
}
if (flags & O_CREAT)
cap_rights_set_one(rightsp, CAP_CREATE);
if (flags & O_TRUNC)
cap_rights_set_one(rightsp, CAP_FTRUNCATE);
if (flags & (O_SYNC | O_FSYNC))
cap_rights_set_one(rightsp, CAP_FSYNC);
if (flags & (O_EXLOCK | O_SHLOCK))
cap_rights_set_one(rightsp, CAP_FLOCK);
}
/*
* Check permissions, allocate an open file structure, and call the device
* open routine if any.
*/
#ifndef _SYS_SYSPROTO_H_
struct open_args {
char *path;
int flags;
int mode;
};
#endif
int
sys_open(struct thread *td, struct open_args *uap)
{
return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->flags, uap->mode));
}
#ifndef _SYS_SYSPROTO_H_
struct openat_args {
int fd;
char *path;
int flag;
int mode;
};
#endif
int
sys_openat(struct thread *td, struct openat_args *uap)
{
AUDIT_ARG_FD(uap->fd);
return (kern_openat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
uap->mode));
}
int
kern_openat(struct thread *td, int fd, const char *path, enum uio_seg pathseg,
int flags, int mode)
{
struct proc *p = td->td_proc;
struct filedesc *fdp;
struct pwddesc *pdp;
struct file *fp;
struct vnode *vp;
struct nameidata nd;
cap_rights_t rights;
int cmode, error, indx;
indx = -1;
fdp = p->p_fd;
pdp = p->p_pd;
AUDIT_ARG_FFLAGS(flags);
AUDIT_ARG_MODE(mode);
cap_rights_init_one(&rights, CAP_LOOKUP);
flags_to_rights(flags, &rights);
/*
* Only one of the O_EXEC, O_RDONLY, O_WRONLY and O_RDWR flags
* may be specified. On the other hand, for O_PATH any mode
* except O_EXEC is ignored.
*/
if ((flags & O_PATH) != 0) {
flags &= ~(O_CREAT | O_ACCMODE);
} else if ((flags & O_EXEC) != 0) {
if (flags & O_ACCMODE)
return (EINVAL);
} else if ((flags & O_ACCMODE) == O_ACCMODE) {
return (EINVAL);
} else {
flags = FFLAGS(flags);
}
/*
* Allocate a file structure. The descriptor to reference it
* is allocated and used by finstall_refed() below.
*/
error = falloc_noinstall(td, &fp);
if (error != 0)
return (error);
/* Set the flags early so the finit in devfs can pick them up. */
fp->f_flag = flags & FMASK;
cmode = ((mode & ~pdp->pd_cmask) & ALLPERMS) & ~S_ISTXT;
- NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, fd,
- &rights, td);
+ NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1 | WANTIOCTLCAPS,
+ pathseg, path, fd, &rights, td);
td->td_dupfd = -1; /* XXX check for fdopen */
error = vn_open(&nd, &flags, cmode, fp);
if (error != 0) {
/*
* If the vn_open replaced the method vector, something
* wonderous happened deep below and we just pass it up
* pretending we know what we do.
*/
if (error == ENXIO && fp->f_ops != &badfileops) {
MPASS((flags & O_PATH) == 0);
goto success;
}
/*
* Handle special fdopen() case. bleh.
*
* Don't do this for relative (capability) lookups; we don't
* understand exactly what would happen, and we don't think
* that it ever should.
*/
if ((nd.ni_resflags & NIRES_STRICTREL) == 0 &&
(error == ENODEV || error == ENXIO) &&
td->td_dupfd >= 0) {
error = dupfdopen(td, fdp, td->td_dupfd, flags, error,
&indx);
if (error == 0)
goto success;
}
goto bad;
}
td->td_dupfd = 0;
NDFREE(&nd, NDF_ONLY_PNBUF);
vp = nd.ni_vp;
/*
* Store the vnode, for any f_type. Typically, the vnode use
* count is decremented by direct call to vn_closefile() for
* files that switched type in the cdevsw fdopen() method.
*/
fp->f_vnode = vp;
/*
* If the file wasn't claimed by devfs bind it to the normal
* vnode operations here.
*/
if (fp->f_ops == &badfileops) {
KASSERT(vp->v_type != VFIFO || (flags & O_PATH) != 0,
("Unexpected fifo fp %p vp %p", fp, vp));
if ((flags & O_PATH) != 0) {
finit(fp, (flags & FMASK) | (fp->f_flag & FKQALLOWED),
DTYPE_VNODE, NULL, &path_fileops);
vhold(vp);
vunref(vp);
} else {
finit_vnode(fp, flags, NULL, &vnops);
}
}
VOP_UNLOCK(vp);
if (flags & O_TRUNC) {
error = fo_truncate(fp, 0, td->td_ucred, td);
if (error != 0)
goto bad;
}
success:
/*
* If we haven't already installed the FD (for dupfdopen), do so now.
*/
if (indx == -1) {
struct filecaps *fcaps;
#ifdef CAPABILITIES
if ((nd.ni_resflags & NIRES_STRICTREL) != 0)
fcaps = &nd.ni_filecaps;
else
#endif
fcaps = NULL;
error = finstall_refed(td, fp, &indx, flags, fcaps);
/* On success finstall_refed() consumes fcaps. */
if (error != 0) {
- filecaps_free(&nd.ni_filecaps);
goto bad;
}
} else {
- filecaps_free(&nd.ni_filecaps);
+ NDFREE_IOCTLCAPS(&nd);
falloc_abort(td, fp);
}
td->td_retval[0] = indx;
return (0);
bad:
KASSERT(indx == -1, ("indx=%d, should be -1", indx));
+ NDFREE_IOCTLCAPS(&nd);
falloc_abort(td, fp);
return (error);
}
#ifdef COMPAT_43
/*
* Create a file.
*/
#ifndef _SYS_SYSPROTO_H_
struct ocreat_args {
char *path;
int mode;
};
#endif
int
ocreat(struct thread *td, struct ocreat_args *uap)
{
return (kern_openat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
O_WRONLY | O_CREAT | O_TRUNC, uap->mode));
}
#endif /* COMPAT_43 */
/*
* Create a special file.
*/
#ifndef _SYS_SYSPROTO_H_
struct mknodat_args {
int fd;
char *path;
mode_t mode;
dev_t dev;
};
#endif
int
sys_mknodat(struct thread *td, struct mknodat_args *uap)
{
return (kern_mknodat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode,
uap->dev));
}
#if defined(COMPAT_FREEBSD11)
int
freebsd11_mknod(struct thread *td,
struct freebsd11_mknod_args *uap)
{
return (kern_mknodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->mode, uap->dev));
}
int
freebsd11_mknodat(struct thread *td,
struct freebsd11_mknodat_args *uap)
{
return (kern_mknodat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode,
uap->dev));
}
#endif /* COMPAT_FREEBSD11 */
int
kern_mknodat(struct thread *td, int fd, const char *path, enum uio_seg pathseg,
int mode, dev_t dev)
{
struct vnode *vp;
struct mount *mp;
struct vattr vattr;
struct nameidata nd;
int error, whiteout = 0;
AUDIT_ARG_MODE(mode);
AUDIT_ARG_DEV(dev);
switch (mode & S_IFMT) {
case S_IFCHR:
case S_IFBLK:
error = priv_check(td, PRIV_VFS_MKNOD_DEV);
if (error == 0 && dev == VNOVAL)
error = EINVAL;
break;
case S_IFWHT:
error = priv_check(td, PRIV_VFS_MKNOD_WHT);
break;
case S_IFIFO:
if (dev == 0)
return (kern_mkfifoat(td, fd, path, pathseg, mode));
/* FALLTHROUGH */
default:
error = EINVAL;
break;
}
if (error != 0)
return (error);
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1 |
NOCACHE, pathseg, path, fd, &cap_mknodat_rights,
td);
if ((error = namei(&nd)) != 0)
return (error);
vp = nd.ni_vp;
if (vp != NULL) {
NDFREE(&nd, NDF_ONLY_PNBUF);
if (vp == nd.ni_dvp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
vrele(vp);
return (EEXIST);
} else {
VATTR_NULL(&vattr);
vattr.va_mode = (mode & ALLPERMS) &
~td->td_proc->p_pd->pd_cmask;
vattr.va_rdev = dev;
whiteout = 0;
switch (mode & S_IFMT) {
case S_IFCHR:
vattr.va_type = VCHR;
break;
case S_IFBLK:
vattr.va_type = VBLK;
break;
case S_IFWHT:
whiteout = 1;
break;
default:
panic("kern_mknod: invalid mode");
}
}
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
return (error);
goto restart;
}
#ifdef MAC
if (error == 0 && !whiteout)
error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp,
&nd.ni_cnd, &vattr);
#endif
if (error == 0) {
if (whiteout)
error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, CREATE);
else {
error = VOP_MKNOD(nd.ni_dvp, &nd.ni_vp,
&nd.ni_cnd, &vattr);
}
}
VOP_VPUT_PAIR(nd.ni_dvp, error == 0 && !whiteout ? &nd.ni_vp : NULL,
true);
vn_finished_write(mp);
NDFREE(&nd, NDF_ONLY_PNBUF);
if (error == ERELOOKUP)
goto restart;
return (error);
}
/*
* Create a named pipe.
*/
#ifndef _SYS_SYSPROTO_H_
struct mkfifo_args {
char *path;
int mode;
};
#endif
int
sys_mkfifo(struct thread *td, struct mkfifo_args *uap)
{
return (kern_mkfifoat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->mode));
}
#ifndef _SYS_SYSPROTO_H_
struct mkfifoat_args {
int fd;
char *path;
mode_t mode;
};
#endif
int
sys_mkfifoat(struct thread *td, struct mkfifoat_args *uap)
{
return (kern_mkfifoat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->mode));
}
int
kern_mkfifoat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, int mode)
{
struct mount *mp;
struct vattr vattr;
struct nameidata nd;
int error;
AUDIT_ARG_MODE(mode);
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1 |
NOCACHE, pathseg, path, fd, &cap_mkfifoat_rights,
td);
if ((error = namei(&nd)) != 0)
return (error);
if (nd.ni_vp != NULL) {
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_vp == nd.ni_dvp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
vrele(nd.ni_vp);
return (EEXIST);
}
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
return (error);
goto restart;
}
VATTR_NULL(&vattr);
vattr.va_type = VFIFO;
vattr.va_mode = (mode & ALLPERMS) & ~td->td_proc->p_pd->pd_cmask;
#ifdef MAC
error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
&vattr);
if (error != 0)
goto out;
#endif
error = VOP_MKNOD(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
#ifdef MAC
out:
#endif
VOP_VPUT_PAIR(nd.ni_dvp, error == 0 ? &nd.ni_vp : NULL, true);
vn_finished_write(mp);
NDFREE(&nd, NDF_ONLY_PNBUF);
if (error == ERELOOKUP)
goto restart;
return (error);
}
/*
* Make a hard file link.
*/
#ifndef _SYS_SYSPROTO_H_
struct link_args {
char *path;
char *link;
};
#endif
int
sys_link(struct thread *td, struct link_args *uap)
{
return (kern_linkat(td, AT_FDCWD, AT_FDCWD, uap->path, uap->link,
UIO_USERSPACE, FOLLOW));
}
#ifndef _SYS_SYSPROTO_H_
struct linkat_args {
int fd1;
char *path1;
int fd2;
char *path2;
int flag;
};
#endif
int
sys_linkat(struct thread *td, struct linkat_args *uap)
{
int flag;
flag = uap->flag;
if ((flag & ~(AT_SYMLINK_FOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
return (kern_linkat(td, uap->fd1, uap->fd2, uap->path1, uap->path2,
UIO_USERSPACE, at2cnpflags(flag, AT_SYMLINK_FOLLOW |
AT_RESOLVE_BENEATH | AT_EMPTY_PATH)));
}
int hardlink_check_uid = 0;
SYSCTL_INT(_security_bsd, OID_AUTO, hardlink_check_uid, CTLFLAG_RW,
&hardlink_check_uid, 0,
"Unprivileged processes cannot create hard links to files owned by other "
"users");
static int hardlink_check_gid = 0;
SYSCTL_INT(_security_bsd, OID_AUTO, hardlink_check_gid, CTLFLAG_RW,
&hardlink_check_gid, 0,
"Unprivileged processes cannot create hard links to files owned by other "
"groups");
static int
can_hardlink(struct vnode *vp, struct ucred *cred)
{
struct vattr va;
int error;
if (!hardlink_check_uid && !hardlink_check_gid)
return (0);
error = VOP_GETATTR(vp, &va, cred);
if (error != 0)
return (error);
if (hardlink_check_uid && cred->cr_uid != va.va_uid) {
error = priv_check_cred(cred, PRIV_VFS_LINK);
if (error != 0)
return (error);
}
if (hardlink_check_gid && !groupmember(va.va_gid, cred)) {
error = priv_check_cred(cred, PRIV_VFS_LINK);
if (error != 0)
return (error);
}
return (0);
}
int
kern_linkat(struct thread *td, int fd1, int fd2, const char *path1,
const char *path2, enum uio_seg segflag, int follow)
{
struct nameidata nd;
int error;
NDPREINIT(&nd);
do {
bwillwrite();
NDINIT_ATRIGHTS(&nd, LOOKUP, follow | AUDITVNODE1, segflag,
path1, fd1, &cap_linkat_source_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE(&nd, NDF_ONLY_PNBUF);
if ((nd.ni_resflags & NIRES_EMPTYPATH) != 0) {
error = priv_check(td, PRIV_VFS_FHOPEN);
if (error != 0) {
vrele(nd.ni_vp);
return (error);
}
}
error = kern_linkat_vp(td, nd.ni_vp, fd2, path2, segflag);
} while (error == EAGAIN || error == ERELOOKUP);
return (error);
}
static int
kern_linkat_vp(struct thread *td, struct vnode *vp, int fd, const char *path,
enum uio_seg segflag)
{
struct nameidata nd;
struct mount *mp;
int error;
if (vp->v_type == VDIR) {
vrele(vp);
return (EPERM); /* POSIX */
}
NDINIT_ATRIGHTS(&nd, CREATE,
LOCKPARENT | SAVENAME | AUDITVNODE2 | NOCACHE, segflag, path, fd,
&cap_linkat_target_rights, td);
if ((error = namei(&nd)) == 0) {
if (nd.ni_vp != NULL) {
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_dvp == nd.ni_vp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
vrele(nd.ni_vp);
vrele(vp);
return (EEXIST);
} else if (nd.ni_dvp->v_mount != vp->v_mount) {
/*
* Cross-device link. No need to recheck
* vp->v_type, since it cannot change, except
* to VBAD.
*/
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
vrele(vp);
return (EXDEV);
} else if ((error = vn_lock(vp, LK_EXCLUSIVE)) == 0) {
error = can_hardlink(vp, td->td_ucred);
#ifdef MAC
if (error == 0)
error = mac_vnode_check_link(td->td_ucred,
nd.ni_dvp, vp, &nd.ni_cnd);
#endif
if (error != 0) {
vput(vp);
vput(nd.ni_dvp);
NDFREE(&nd, NDF_ONLY_PNBUF);
return (error);
}
error = vn_start_write(vp, &mp, V_NOWAIT);
if (error != 0) {
vput(vp);
vput(nd.ni_dvp);
NDFREE(&nd, NDF_ONLY_PNBUF);
error = vn_start_write(NULL, &mp,
V_XSLEEP | PCATCH);
if (error != 0)
return (error);
return (EAGAIN);
}
error = VOP_LINK(nd.ni_dvp, vp, &nd.ni_cnd);
VOP_VPUT_PAIR(nd.ni_dvp, &vp, true);
vn_finished_write(mp);
NDFREE(&nd, NDF_ONLY_PNBUF);
vp = NULL;
} else {
vput(nd.ni_dvp);
NDFREE(&nd, NDF_ONLY_PNBUF);
vrele(vp);
return (EAGAIN);
}
}
if (vp != NULL)
vrele(vp);
return (error);
}
/*
* Make a symbolic link.
*/
#ifndef _SYS_SYSPROTO_H_
struct symlink_args {
char *path;
char *link;
};
#endif
int
sys_symlink(struct thread *td, struct symlink_args *uap)
{
return (kern_symlinkat(td, uap->path, AT_FDCWD, uap->link,
UIO_USERSPACE));
}
#ifndef _SYS_SYSPROTO_H_
struct symlinkat_args {
char *path;
int fd;
char *path2;
};
#endif
int
sys_symlinkat(struct thread *td, struct symlinkat_args *uap)
{
return (kern_symlinkat(td, uap->path1, uap->fd, uap->path2,
UIO_USERSPACE));
}
int
kern_symlinkat(struct thread *td, const char *path1, int fd, const char *path2,
enum uio_seg segflg)
{
struct mount *mp;
struct vattr vattr;
const char *syspath;
char *tmppath;
struct nameidata nd;
int error;
if (segflg == UIO_SYSSPACE) {
syspath = path1;
} else {
tmppath = uma_zalloc(namei_zone, M_WAITOK);
if ((error = copyinstr(path1, tmppath, MAXPATHLEN, NULL)) != 0)
goto out;
syspath = tmppath;
}
AUDIT_ARG_TEXT(syspath);
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1 |
NOCACHE, segflg, path2, fd, &cap_symlinkat_rights,
td);
if ((error = namei(&nd)) != 0)
goto out;
if (nd.ni_vp) {
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_vp == nd.ni_dvp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
vrele(nd.ni_vp);
nd.ni_vp = NULL;
error = EEXIST;
goto out;
}
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
goto out;
goto restart;
}
VATTR_NULL(&vattr);
vattr.va_mode = ACCESSPERMS &~ td->td_proc->p_pd->pd_cmask;
#ifdef MAC
vattr.va_type = VLNK;
error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
&vattr);
if (error != 0)
goto out2;
#endif
error = VOP_SYMLINK(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr, syspath);
#ifdef MAC
out2:
#endif
VOP_VPUT_PAIR(nd.ni_dvp, error == 0 ? &nd.ni_vp : NULL, true);
vn_finished_write(mp);
NDFREE(&nd, NDF_ONLY_PNBUF);
if (error == ERELOOKUP)
goto restart;
out:
if (segflg != UIO_SYSSPACE)
uma_zfree(namei_zone, tmppath);
return (error);
}
/*
* Delete a whiteout from the filesystem.
*/
#ifndef _SYS_SYSPROTO_H_
struct undelete_args {
char *path;
};
#endif
int
sys_undelete(struct thread *td, struct undelete_args *uap)
{
struct mount *mp;
struct nameidata nd;
int error;
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT(&nd, DELETE, LOCKPARENT | DOWHITEOUT | AUDITVNODE1,
UIO_USERSPACE, uap->path, td);
error = namei(&nd);
if (error != 0)
return (error);
if (nd.ni_vp != NULLVP || !(nd.ni_cnd.cn_flags & ISWHITEOUT)) {
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_vp == nd.ni_dvp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
if (nd.ni_vp)
vrele(nd.ni_vp);
return (EEXIST);
}
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
return (error);
goto restart;
}
error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, DELETE);
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
vn_finished_write(mp);
if (error == ERELOOKUP)
goto restart;
return (error);
}
/*
* Delete a name from the filesystem.
*/
#ifndef _SYS_SYSPROTO_H_
struct unlink_args {
char *path;
};
#endif
int
sys_unlink(struct thread *td, struct unlink_args *uap)
{
return (kern_funlinkat(td, AT_FDCWD, uap->path, FD_NONE, UIO_USERSPACE,
0, 0));
}
static int
kern_funlinkat_ex(struct thread *td, int dfd, const char *path, int fd,
int flag, enum uio_seg pathseg, ino_t oldinum)
{
if ((flag & ~(AT_REMOVEDIR | AT_RESOLVE_BENEATH)) != 0)
return (EINVAL);
if ((flag & AT_REMOVEDIR) != 0)
return (kern_frmdirat(td, dfd, path, fd, UIO_USERSPACE, 0));
return (kern_funlinkat(td, dfd, path, fd, UIO_USERSPACE, 0, 0));
}
#ifndef _SYS_SYSPROTO_H_
struct unlinkat_args {
int fd;
char *path;
int flag;
};
#endif
int
sys_unlinkat(struct thread *td, struct unlinkat_args *uap)
{
return (kern_funlinkat_ex(td, uap->fd, uap->path, FD_NONE, uap->flag,
UIO_USERSPACE, 0));
}
#ifndef _SYS_SYSPROTO_H_
struct funlinkat_args {
int dfd;
const char *path;
int fd;
int flag;
};
#endif
int
sys_funlinkat(struct thread *td, struct funlinkat_args *uap)
{
return (kern_funlinkat_ex(td, uap->dfd, uap->path, uap->fd, uap->flag,
UIO_USERSPACE, 0));
}
int
kern_funlinkat(struct thread *td, int dfd, const char *path, int fd,
enum uio_seg pathseg, int flag, ino_t oldinum)
{
struct mount *mp;
struct file *fp;
struct vnode *vp;
struct nameidata nd;
struct stat sb;
int error;
fp = NULL;
if (fd != FD_NONE) {
error = getvnode_path(td, fd, &cap_no_rights, &fp);
if (error != 0)
return (error);
}
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF | AUDITVNODE1 |
at2cnpflags(flag, AT_RESOLVE_BENEATH),
pathseg, path, dfd, &cap_unlinkat_rights, td);
if ((error = namei(&nd)) != 0) {
if (error == EINVAL)
error = EPERM;
goto fdout;
}
vp = nd.ni_vp;
if (vp->v_type == VDIR && oldinum == 0) {
error = EPERM; /* POSIX */
} else if (oldinum != 0 &&
((error = VOP_STAT(vp, &sb, td->td_ucred, NOCRED, td)) == 0) &&
sb.st_ino != oldinum) {
error = EIDRM; /* Identifier removed */
} else if (fp != NULL && fp->f_vnode != vp) {
if (VN_IS_DOOMED(fp->f_vnode))
error = EBADF;
else
error = EDEADLK;
} else {
/*
* The root of a mounted filesystem cannot be deleted.
*
* XXX: can this only be a VDIR case?
*/
if (vp->v_vflag & VV_ROOT)
error = EBUSY;
}
if (error == 0) {
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if (vp == nd.ni_dvp)
vrele(vp);
else
vput(vp);
if ((error = vn_start_write(NULL, &mp,
V_XSLEEP | PCATCH)) != 0) {
goto fdout;
}
goto restart;
}
#ifdef MAC
error = mac_vnode_check_unlink(td->td_ucred, nd.ni_dvp, vp,
&nd.ni_cnd);
if (error != 0)
goto out;
#endif
vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK);
error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd);
#ifdef MAC
out:
#endif
vn_finished_write(mp);
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if (vp == nd.ni_dvp)
vrele(vp);
else
vput(vp);
if (error == ERELOOKUP)
goto restart;
fdout:
if (fp != NULL)
fdrop(fp, td);
return (error);
}
/*
* Reposition read/write file offset.
*/
#ifndef _SYS_SYSPROTO_H_
struct lseek_args {
int fd;
int pad;
off_t offset;
int whence;
};
#endif
int
sys_lseek(struct thread *td, struct lseek_args *uap)
{
return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
}
int
kern_lseek(struct thread *td, int fd, off_t offset, int whence)
{
struct file *fp;
int error;
AUDIT_ARG_FD(fd);
error = fget(td, fd, &cap_seek_rights, &fp);
if (error != 0)
return (error);
error = (fp->f_ops->fo_flags & DFLAG_SEEKABLE) != 0 ?
fo_seek(fp, offset, whence, td) : ESPIPE;
fdrop(fp, td);
return (error);
}
#if defined(COMPAT_43)
/*
* Reposition read/write file offset.
*/
#ifndef _SYS_SYSPROTO_H_
struct olseek_args {
int fd;
long offset;
int whence;
};
#endif
int
olseek(struct thread *td, struct olseek_args *uap)
{
return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
}
#endif /* COMPAT_43 */
#if defined(COMPAT_FREEBSD6)
/* Version with the 'pad' argument */
int
freebsd6_lseek(struct thread *td, struct freebsd6_lseek_args *uap)
{
return (kern_lseek(td, uap->fd, uap->offset, uap->whence));
}
#endif
/*
* Check access permissions using passed credentials.
*/
static int
vn_access(struct vnode *vp, int user_flags, struct ucred *cred,
struct thread *td)
{
accmode_t accmode;
int error;
/* Flags == 0 means only check for existence. */
if (user_flags == 0)
return (0);
accmode = 0;
if (user_flags & R_OK)
accmode |= VREAD;
if (user_flags & W_OK)
accmode |= VWRITE;
if (user_flags & X_OK)
accmode |= VEXEC;
#ifdef MAC
error = mac_vnode_check_access(cred, vp, accmode);
if (error != 0)
return (error);
#endif
if ((accmode & VWRITE) == 0 || (error = vn_writechk(vp)) == 0)
error = VOP_ACCESS(vp, accmode, cred, td);
return (error);
}
/*
* Check access permissions using "real" credentials.
*/
#ifndef _SYS_SYSPROTO_H_
struct access_args {
char *path;
int amode;
};
#endif
int
sys_access(struct thread *td, struct access_args *uap)
{
return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
0, uap->amode));
}
#ifndef _SYS_SYSPROTO_H_
struct faccessat_args {
int dirfd;
char *path;
int amode;
int flag;
}
#endif
int
sys_faccessat(struct thread *td, struct faccessat_args *uap)
{
return (kern_accessat(td, uap->fd, uap->path, UIO_USERSPACE, uap->flag,
uap->amode));
}
int
kern_accessat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, int flag, int amode)
{
struct ucred *cred, *usecred;
struct vnode *vp;
struct nameidata nd;
int error;
if ((flag & ~(AT_EACCESS | AT_RESOLVE_BENEATH | AT_EMPTY_PATH)) != 0)
return (EINVAL);
if (amode != F_OK && (amode & ~(R_OK | W_OK | X_OK)) != 0)
return (EINVAL);
/*
* Create and modify a temporary credential instead of one that
* is potentially shared (if we need one).
*/
cred = td->td_ucred;
if ((flag & AT_EACCESS) == 0 &&
((cred->cr_uid != cred->cr_ruid ||
cred->cr_rgid != cred->cr_groups[0]))) {
usecred = crdup(cred);
usecred->cr_uid = cred->cr_ruid;
usecred->cr_groups[0] = cred->cr_rgid;
td->td_ucred = usecred;
} else
usecred = cred;
AUDIT_ARG_VALUE(amode);
NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF |
AUDITVNODE1 | at2cnpflags(flag, AT_RESOLVE_BENEATH |
AT_EMPTY_PATH), pathseg, path, fd, &cap_fstat_rights, td);
if ((error = namei(&nd)) != 0)
goto out;
vp = nd.ni_vp;
error = vn_access(vp, amode, usecred, td);
NDFREE_NOTHING(&nd);
vput(vp);
out:
if (usecred != cred) {
td->td_ucred = cred;
crfree(usecred);
}
return (error);
}
/*
* Check access permissions using "effective" credentials.
*/
#ifndef _SYS_SYSPROTO_H_
struct eaccess_args {
char *path;
int amode;
};
#endif
int
sys_eaccess(struct thread *td, struct eaccess_args *uap)
{
return (kern_accessat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
AT_EACCESS, uap->amode));
}
#if defined(COMPAT_43)
/*
* Get file status; this version follows links.
*/
#ifndef _SYS_SYSPROTO_H_
struct ostat_args {
char *path;
struct ostat *ub;
};
#endif
int
ostat(struct thread *td, struct ostat_args *uap)
{
struct stat sb;
struct ostat osb;
int error;
error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
&sb, NULL);
if (error != 0)
return (error);
cvtstat(&sb, &osb);
return (copyout(&osb, uap->ub, sizeof (osb)));
}
/*
* Get file status; this version does not follow links.
*/
#ifndef _SYS_SYSPROTO_H_
struct olstat_args {
char *path;
struct ostat *ub;
};
#endif
int
olstat(struct thread *td, struct olstat_args *uap)
{
struct stat sb;
struct ostat osb;
int error;
error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
UIO_USERSPACE, &sb, NULL);
if (error != 0)
return (error);
cvtstat(&sb, &osb);
return (copyout(&osb, uap->ub, sizeof (osb)));
}
/*
* Convert from an old to a new stat structure.
* XXX: many values are blindly truncated.
*/
void
cvtstat(struct stat *st, struct ostat *ost)
{
bzero(ost, sizeof(*ost));
ost->st_dev = st->st_dev;
ost->st_ino = st->st_ino;
ost->st_mode = st->st_mode;
ost->st_nlink = st->st_nlink;
ost->st_uid = st->st_uid;
ost->st_gid = st->st_gid;
ost->st_rdev = st->st_rdev;
ost->st_size = MIN(st->st_size, INT32_MAX);
ost->st_atim = st->st_atim;
ost->st_mtim = st->st_mtim;
ost->st_ctim = st->st_ctim;
ost->st_blksize = st->st_blksize;
ost->st_blocks = st->st_blocks;
ost->st_flags = st->st_flags;
ost->st_gen = st->st_gen;
}
#endif /* COMPAT_43 */
#if defined(COMPAT_43) || defined(COMPAT_FREEBSD11)
int ino64_trunc_error;
SYSCTL_INT(_vfs, OID_AUTO, ino64_trunc_error, CTLFLAG_RW,
&ino64_trunc_error, 0,
"Error on truncation of device, file or inode number, or link count");
int
freebsd11_cvtstat(struct stat *st, struct freebsd11_stat *ost)
{
ost->st_dev = st->st_dev;
if (ost->st_dev != st->st_dev) {
switch (ino64_trunc_error) {
default:
/*
* Since dev_t is almost raw, don't clamp to the
* maximum for case 2, but ignore the error.
*/
break;
case 1:
return (EOVERFLOW);
}
}
ost->st_ino = st->st_ino;
if (ost->st_ino != st->st_ino) {
switch (ino64_trunc_error) {
default:
case 0:
break;
case 1:
return (EOVERFLOW);
case 2:
ost->st_ino = UINT32_MAX;
break;
}
}
ost->st_mode = st->st_mode;
ost->st_nlink = st->st_nlink;
if (ost->st_nlink != st->st_nlink) {
switch (ino64_trunc_error) {
default:
case 0:
break;
case 1:
return (EOVERFLOW);
case 2:
ost->st_nlink = UINT16_MAX;
break;
}
}
ost->st_uid = st->st_uid;
ost->st_gid = st->st_gid;
ost->st_rdev = st->st_rdev;
if (ost->st_rdev != st->st_rdev) {
switch (ino64_trunc_error) {
default:
break;
case 1:
return (EOVERFLOW);
}
}
ost->st_atim = st->st_atim;
ost->st_mtim = st->st_mtim;
ost->st_ctim = st->st_ctim;
ost->st_size = st->st_size;
ost->st_blocks = st->st_blocks;
ost->st_blksize = st->st_blksize;
ost->st_flags = st->st_flags;
ost->st_gen = st->st_gen;
ost->st_lspare = 0;
ost->st_birthtim = st->st_birthtim;
bzero((char *)&ost->st_birthtim + sizeof(ost->st_birthtim),
sizeof(*ost) - offsetof(struct freebsd11_stat,
st_birthtim) - sizeof(ost->st_birthtim));
return (0);
}
int
freebsd11_stat(struct thread *td, struct freebsd11_stat_args* uap)
{
struct stat sb;
struct freebsd11_stat osb;
int error;
error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
&sb, NULL);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->ub, sizeof(osb));
return (error);
}
int
freebsd11_lstat(struct thread *td, struct freebsd11_lstat_args* uap)
{
struct stat sb;
struct freebsd11_stat osb;
int error;
error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
UIO_USERSPACE, &sb, NULL);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->ub, sizeof(osb));
return (error);
}
int
freebsd11_fhstat(struct thread *td, struct freebsd11_fhstat_args* uap)
{
struct fhandle fh;
struct stat sb;
struct freebsd11_stat osb;
int error;
error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
if (error != 0)
return (error);
error = kern_fhstat(td, fh, &sb);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->sb, sizeof(osb));
return (error);
}
int
freebsd11_fstatat(struct thread *td, struct freebsd11_fstatat_args* uap)
{
struct stat sb;
struct freebsd11_stat osb;
int error;
error = kern_statat(td, uap->flag, uap->fd, uap->path,
UIO_USERSPACE, &sb, NULL);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->buf, sizeof(osb));
return (error);
}
#endif /* COMPAT_FREEBSD11 */
/*
* Get file status
*/
#ifndef _SYS_SYSPROTO_H_
struct fstatat_args {
int fd;
char *path;
struct stat *buf;
int flag;
}
#endif
int
sys_fstatat(struct thread *td, struct fstatat_args *uap)
{
struct stat sb;
int error;
error = kern_statat(td, uap->flag, uap->fd, uap->path,
UIO_USERSPACE, &sb, NULL);
if (error == 0)
error = copyout(&sb, uap->buf, sizeof (sb));
return (error);
}
int
kern_statat(struct thread *td, int flag, int fd, const char *path,
enum uio_seg pathseg, struct stat *sbp,
void (*hook)(struct vnode *vp, struct stat *sbp))
{
struct nameidata nd;
int error;
if ((flag & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
NDINIT_ATRIGHTS(&nd, LOOKUP, at2cnpflags(flag, AT_RESOLVE_BENEATH |
AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH) | LOCKSHARED | LOCKLEAF |
AUDITVNODE1, pathseg, path, fd, &cap_fstat_rights, td);
if ((error = namei(&nd)) != 0) {
if (error == ENOTDIR &&
(nd.ni_resflags & NIRES_EMPTYPATH) != 0)
error = kern_fstat(td, fd, sbp);
return (error);
}
error = VOP_STAT(nd.ni_vp, sbp, td->td_ucred, NOCRED, td);
if (error == 0) {
if (__predict_false(hook != NULL))
hook(nd.ni_vp, sbp);
}
NDFREE_NOTHING(&nd);
vput(nd.ni_vp);
#ifdef __STAT_TIME_T_EXT
sbp->st_atim_ext = 0;
sbp->st_mtim_ext = 0;
sbp->st_ctim_ext = 0;
sbp->st_btim_ext = 0;
#endif
#ifdef KTRACE
if (KTRPOINT(td, KTR_STRUCT))
ktrstat_error(sbp, error);
#endif
return (error);
}
#if defined(COMPAT_FREEBSD11)
/*
* Implementation of the NetBSD [l]stat() functions.
*/
void
freebsd11_cvtnstat(struct stat *sb, struct nstat *nsb)
{
bzero(nsb, sizeof(*nsb));
nsb->st_dev = sb->st_dev;
nsb->st_ino = sb->st_ino;
nsb->st_mode = sb->st_mode;
nsb->st_nlink = sb->st_nlink;
nsb->st_uid = sb->st_uid;
nsb->st_gid = sb->st_gid;
nsb->st_rdev = sb->st_rdev;
nsb->st_atim = sb->st_atim;
nsb->st_mtim = sb->st_mtim;
nsb->st_ctim = sb->st_ctim;
nsb->st_size = sb->st_size;
nsb->st_blocks = sb->st_blocks;
nsb->st_blksize = sb->st_blksize;
nsb->st_flags = sb->st_flags;
nsb->st_gen = sb->st_gen;
nsb->st_birthtim = sb->st_birthtim;
}
#ifndef _SYS_SYSPROTO_H_
struct freebsd11_nstat_args {
char *path;
struct nstat *ub;
};
#endif
int
freebsd11_nstat(struct thread *td, struct freebsd11_nstat_args *uap)
{
struct stat sb;
struct nstat nsb;
int error;
error = kern_statat(td, 0, AT_FDCWD, uap->path, UIO_USERSPACE,
&sb, NULL);
if (error != 0)
return (error);
freebsd11_cvtnstat(&sb, &nsb);
return (copyout(&nsb, uap->ub, sizeof (nsb)));
}
/*
* NetBSD lstat. Get file status; this version does not follow links.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd11_nlstat_args {
char *path;
struct nstat *ub;
};
#endif
int
freebsd11_nlstat(struct thread *td, struct freebsd11_nlstat_args *uap)
{
struct stat sb;
struct nstat nsb;
int error;
error = kern_statat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->path,
UIO_USERSPACE, &sb, NULL);
if (error != 0)
return (error);
freebsd11_cvtnstat(&sb, &nsb);
return (copyout(&nsb, uap->ub, sizeof (nsb)));
}
#endif /* COMPAT_FREEBSD11 */
/*
* Get configurable pathname variables.
*/
#ifndef _SYS_SYSPROTO_H_
struct pathconf_args {
char *path;
int name;
};
#endif
int
sys_pathconf(struct thread *td, struct pathconf_args *uap)
{
long value;
int error;
error = kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name, FOLLOW,
&value);
if (error == 0)
td->td_retval[0] = value;
return (error);
}
#ifndef _SYS_SYSPROTO_H_
struct lpathconf_args {
char *path;
int name;
};
#endif
int
sys_lpathconf(struct thread *td, struct lpathconf_args *uap)
{
long value;
int error;
error = kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name,
NOFOLLOW, &value);
if (error == 0)
td->td_retval[0] = value;
return (error);
}
int
kern_pathconf(struct thread *td, const char *path, enum uio_seg pathseg,
int name, u_long flags, long *valuep)
{
struct nameidata nd;
int error;
NDINIT(&nd, LOOKUP, LOCKSHARED | LOCKLEAF | AUDITVNODE1 | flags,
pathseg, path, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = VOP_PATHCONF(nd.ni_vp, name, valuep);
vput(nd.ni_vp);
return (error);
}
/*
* Return target name of a symbolic link.
*/
#ifndef _SYS_SYSPROTO_H_
struct readlink_args {
char *path;
char *buf;
size_t count;
};
#endif
int
sys_readlink(struct thread *td, struct readlink_args *uap)
{
return (kern_readlinkat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->buf, UIO_USERSPACE, uap->count));
}
#ifndef _SYS_SYSPROTO_H_
struct readlinkat_args {
int fd;
char *path;
char *buf;
size_t bufsize;
};
#endif
int
sys_readlinkat(struct thread *td, struct readlinkat_args *uap)
{
return (kern_readlinkat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->buf, UIO_USERSPACE, uap->bufsize));
}
int
kern_readlinkat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, char *buf, enum uio_seg bufseg, size_t count)
{
struct vnode *vp;
struct nameidata nd;
int error;
if (count > IOSIZE_MAX)
return (EINVAL);
NDINIT_AT(&nd, LOOKUP, NOFOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1 |
EMPTYPATH, pathseg, path, fd, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
vp = nd.ni_vp;
error = kern_readlink_vp(vp, buf, bufseg, count, td);
vput(vp);
return (error);
}
/*
* Helper function to readlink from a vnode
*/
static int
kern_readlink_vp(struct vnode *vp, char *buf, enum uio_seg bufseg, size_t count,
struct thread *td)
{
struct iovec aiov;
struct uio auio;
int error;
ASSERT_VOP_LOCKED(vp, "kern_readlink_vp(): vp not locked");
#ifdef MAC
error = mac_vnode_check_readlink(td->td_ucred, vp);
if (error != 0)
return (error);
#endif
if (vp->v_type != VLNK && (vp->v_vflag & VV_READLINK) == 0)
return (EINVAL);
aiov.iov_base = buf;
aiov.iov_len = count;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = 0;
auio.uio_rw = UIO_READ;
auio.uio_segflg = bufseg;
auio.uio_td = td;
auio.uio_resid = count;
error = VOP_READLINK(vp, &auio, td->td_ucred);
td->td_retval[0] = count - auio.uio_resid;
return (error);
}
/*
* Common implementation code for chflags() and fchflags().
*/
static int
setfflags(struct thread *td, struct vnode *vp, u_long flags)
{
struct mount *mp;
struct vattr vattr;
int error;
/* We can't support the value matching VNOVAL. */
if (flags == VNOVAL)
return (EOPNOTSUPP);
/*
* Prevent non-root users from setting flags on devices. When
* a device is reused, users can retain ownership of the device
* if they are allowed to set flags and programs assume that
* chown can't fail when done as root.
*/
if (vp->v_type == VCHR || vp->v_type == VBLK) {
error = priv_check(td, PRIV_VFS_CHFLAGS_DEV);
if (error != 0)
return (error);
}
if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
return (error);
VATTR_NULL(&vattr);
vattr.va_flags = flags;
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
#ifdef MAC
error = mac_vnode_check_setflags(td->td_ucred, vp, vattr.va_flags);
if (error == 0)
#endif
error = VOP_SETATTR(vp, &vattr, td->td_ucred);
VOP_UNLOCK(vp);
vn_finished_write(mp);
return (error);
}
/*
* Change flags of a file given a path name.
*/
#ifndef _SYS_SYSPROTO_H_
struct chflags_args {
const char *path;
u_long flags;
};
#endif
int
sys_chflags(struct thread *td, struct chflags_args *uap)
{
return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->flags, 0));
}
#ifndef _SYS_SYSPROTO_H_
struct chflagsat_args {
int fd;
const char *path;
u_long flags;
int atflag;
}
#endif
int
sys_chflagsat(struct thread *td, struct chflagsat_args *uap)
{
if ((uap->atflag & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
return (kern_chflagsat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->flags, uap->atflag));
}
/*
* Same as chflags() but doesn't follow symlinks.
*/
#ifndef _SYS_SYSPROTO_H_
struct lchflags_args {
const char *path;
u_long flags;
};
#endif
int
sys_lchflags(struct thread *td, struct lchflags_args *uap)
{
return (kern_chflagsat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->flags, AT_SYMLINK_NOFOLLOW));
}
static int
kern_chflagsat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, u_long flags, int atflag)
{
struct nameidata nd;
int error;
AUDIT_ARG_FFLAGS(flags);
NDINIT_ATRIGHTS(&nd, LOOKUP, at2cnpflags(atflag, AT_SYMLINK_NOFOLLOW |
AT_RESOLVE_BENEATH | AT_EMPTY_PATH) | AUDITVNODE1, pathseg, path,
fd, &cap_fchflags_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = setfflags(td, nd.ni_vp, flags);
vrele(nd.ni_vp);
return (error);
}
/*
* Change flags of a file given a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fchflags_args {
int fd;
u_long flags;
};
#endif
int
sys_fchflags(struct thread *td, struct fchflags_args *uap)
{
struct file *fp;
int error;
AUDIT_ARG_FD(uap->fd);
AUDIT_ARG_FFLAGS(uap->flags);
error = getvnode(td, uap->fd, &cap_fchflags_rights,
&fp);
if (error != 0)
return (error);
#ifdef AUDIT
if (AUDITING_TD(td)) {
vn_lock(fp->f_vnode, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(fp->f_vnode);
VOP_UNLOCK(fp->f_vnode);
}
#endif
error = setfflags(td, fp->f_vnode, uap->flags);
fdrop(fp, td);
return (error);
}
/*
* Common implementation code for chmod(), lchmod() and fchmod().
*/
int
setfmode(struct thread *td, struct ucred *cred, struct vnode *vp, int mode)
{
struct mount *mp;
struct vattr vattr;
int error;
if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
return (error);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
VATTR_NULL(&vattr);
vattr.va_mode = mode & ALLPERMS;
#ifdef MAC
error = mac_vnode_check_setmode(cred, vp, vattr.va_mode);
if (error == 0)
#endif
error = VOP_SETATTR(vp, &vattr, cred);
VOP_UNLOCK(vp);
vn_finished_write(mp);
return (error);
}
/*
* Change mode of a file given path name.
*/
#ifndef _SYS_SYSPROTO_H_
struct chmod_args {
char *path;
int mode;
};
#endif
int
sys_chmod(struct thread *td, struct chmod_args *uap)
{
return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->mode, 0));
}
#ifndef _SYS_SYSPROTO_H_
struct fchmodat_args {
int dirfd;
char *path;
mode_t mode;
int flag;
}
#endif
int
sys_fchmodat(struct thread *td, struct fchmodat_args *uap)
{
if ((uap->flag & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
return (kern_fchmodat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->mode, uap->flag));
}
/*
* Change mode of a file given path name (don't follow links.)
*/
#ifndef _SYS_SYSPROTO_H_
struct lchmod_args {
char *path;
int mode;
};
#endif
int
sys_lchmod(struct thread *td, struct lchmod_args *uap)
{
return (kern_fchmodat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->mode, AT_SYMLINK_NOFOLLOW));
}
int
kern_fchmodat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, mode_t mode, int flag)
{
struct nameidata nd;
int error;
AUDIT_ARG_MODE(mode);
NDINIT_ATRIGHTS(&nd, LOOKUP, at2cnpflags(flag, AT_SYMLINK_NOFOLLOW |
AT_RESOLVE_BENEATH | AT_EMPTY_PATH) | AUDITVNODE1, pathseg, path,
fd, &cap_fchmod_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = setfmode(td, td->td_ucred, nd.ni_vp, mode);
vrele(nd.ni_vp);
return (error);
}
/*
* Change mode of a file given a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fchmod_args {
int fd;
int mode;
};
#endif
int
sys_fchmod(struct thread *td, struct fchmod_args *uap)
{
struct file *fp;
int error;
AUDIT_ARG_FD(uap->fd);
AUDIT_ARG_MODE(uap->mode);
error = fget(td, uap->fd, &cap_fchmod_rights, &fp);
if (error != 0)
return (error);
error = fo_chmod(fp, uap->mode, td->td_ucred, td);
fdrop(fp, td);
return (error);
}
/*
* Common implementation for chown(), lchown(), and fchown()
*/
int
setfown(struct thread *td, struct ucred *cred, struct vnode *vp, uid_t uid,
gid_t gid)
{
struct mount *mp;
struct vattr vattr;
int error;
if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
return (error);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
VATTR_NULL(&vattr);
vattr.va_uid = uid;
vattr.va_gid = gid;
#ifdef MAC
error = mac_vnode_check_setowner(cred, vp, vattr.va_uid,
vattr.va_gid);
if (error == 0)
#endif
error = VOP_SETATTR(vp, &vattr, cred);
VOP_UNLOCK(vp);
vn_finished_write(mp);
return (error);
}
/*
* Set ownership given a path name.
*/
#ifndef _SYS_SYSPROTO_H_
struct chown_args {
char *path;
int uid;
int gid;
};
#endif
int
sys_chown(struct thread *td, struct chown_args *uap)
{
return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE, uap->uid,
uap->gid, 0));
}
#ifndef _SYS_SYSPROTO_H_
struct fchownat_args {
int fd;
const char * path;
uid_t uid;
gid_t gid;
int flag;
};
#endif
int
sys_fchownat(struct thread *td, struct fchownat_args *uap)
{
if ((uap->flag & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
return (kern_fchownat(td, uap->fd, uap->path, UIO_USERSPACE, uap->uid,
uap->gid, uap->flag));
}
int
kern_fchownat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, int uid, int gid, int flag)
{
struct nameidata nd;
int error;
AUDIT_ARG_OWNER(uid, gid);
NDINIT_ATRIGHTS(&nd, LOOKUP, at2cnpflags(flag, AT_SYMLINK_NOFOLLOW |
AT_RESOLVE_BENEATH | AT_EMPTY_PATH) | AUDITVNODE1, pathseg, path,
fd, &cap_fchown_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = setfown(td, td->td_ucred, nd.ni_vp, uid, gid);
vrele(nd.ni_vp);
return (error);
}
/*
* Set ownership given a path name, do not cross symlinks.
*/
#ifndef _SYS_SYSPROTO_H_
struct lchown_args {
char *path;
int uid;
int gid;
};
#endif
int
sys_lchown(struct thread *td, struct lchown_args *uap)
{
return (kern_fchownat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->uid, uap->gid, AT_SYMLINK_NOFOLLOW));
}
/*
* Set ownership given a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fchown_args {
int fd;
int uid;
int gid;
};
#endif
int
sys_fchown(struct thread *td, struct fchown_args *uap)
{
struct file *fp;
int error;
AUDIT_ARG_FD(uap->fd);
AUDIT_ARG_OWNER(uap->uid, uap->gid);
error = fget(td, uap->fd, &cap_fchown_rights, &fp);
if (error != 0)
return (error);
error = fo_chown(fp, uap->uid, uap->gid, td->td_ucred, td);
fdrop(fp, td);
return (error);
}
/*
* Common implementation code for utimes(), lutimes(), and futimes().
*/
static int
getutimes(const struct timeval *usrtvp, enum uio_seg tvpseg,
struct timespec *tsp)
{
struct timeval tv[2];
const struct timeval *tvp;
int error;
if (usrtvp == NULL) {
vfs_timestamp(&tsp[0]);
tsp[1] = tsp[0];
} else {
if (tvpseg == UIO_SYSSPACE) {
tvp = usrtvp;
} else {
if ((error = copyin(usrtvp, tv, sizeof(tv))) != 0)
return (error);
tvp = tv;
}
if (tvp[0].tv_usec < 0 || tvp[0].tv_usec >= 1000000 ||
tvp[1].tv_usec < 0 || tvp[1].tv_usec >= 1000000)
return (EINVAL);
TIMEVAL_TO_TIMESPEC(&tvp[0], &tsp[0]);
TIMEVAL_TO_TIMESPEC(&tvp[1], &tsp[1]);
}
return (0);
}
/*
* Common implementation code for futimens(), utimensat().
*/
#define UTIMENS_NULL 0x1
#define UTIMENS_EXIT 0x2
static int
getutimens(const struct timespec *usrtsp, enum uio_seg tspseg,
struct timespec *tsp, int *retflags)
{
struct timespec tsnow;
int error;
vfs_timestamp(&tsnow);
*retflags = 0;
if (usrtsp == NULL) {
tsp[0] = tsnow;
tsp[1] = tsnow;
*retflags |= UTIMENS_NULL;
return (0);
}
if (tspseg == UIO_SYSSPACE) {
tsp[0] = usrtsp[0];
tsp[1] = usrtsp[1];
} else if ((error = copyin(usrtsp, tsp, sizeof(*tsp) * 2)) != 0)
return (error);
if (tsp[0].tv_nsec == UTIME_OMIT && tsp[1].tv_nsec == UTIME_OMIT)
*retflags |= UTIMENS_EXIT;
if (tsp[0].tv_nsec == UTIME_NOW && tsp[1].tv_nsec == UTIME_NOW)
*retflags |= UTIMENS_NULL;
if (tsp[0].tv_nsec == UTIME_OMIT)
tsp[0].tv_sec = VNOVAL;
else if (tsp[0].tv_nsec == UTIME_NOW)
tsp[0] = tsnow;
else if (tsp[0].tv_nsec < 0 || tsp[0].tv_nsec >= 1000000000L)
return (EINVAL);
if (tsp[1].tv_nsec == UTIME_OMIT)
tsp[1].tv_sec = VNOVAL;
else if (tsp[1].tv_nsec == UTIME_NOW)
tsp[1] = tsnow;
else if (tsp[1].tv_nsec < 0 || tsp[1].tv_nsec >= 1000000000L)
return (EINVAL);
return (0);
}
/*
* Common implementation code for utimes(), lutimes(), futimes(), futimens(),
* and utimensat().
*/
static int
setutimes(struct thread *td, struct vnode *vp, const struct timespec *ts,
int numtimes, int nullflag)
{
struct mount *mp;
struct vattr vattr;
int error;
bool setbirthtime;
setbirthtime = false;
vattr.va_birthtime.tv_sec = VNOVAL;
vattr.va_birthtime.tv_nsec = 0;
if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
return (error);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
if (numtimes < 3 && VOP_GETATTR(vp, &vattr, td->td_ucred) == 0 &&
timespeccmp(&ts[1], &vattr.va_birthtime, < ))
setbirthtime = true;
VATTR_NULL(&vattr);
vattr.va_atime = ts[0];
vattr.va_mtime = ts[1];
if (setbirthtime)
vattr.va_birthtime = ts[1];
if (numtimes > 2)
vattr.va_birthtime = ts[2];
if (nullflag)
vattr.va_vaflags |= VA_UTIMES_NULL;
#ifdef MAC
error = mac_vnode_check_setutimes(td->td_ucred, vp, vattr.va_atime,
vattr.va_mtime);
#endif
if (error == 0)
error = VOP_SETATTR(vp, &vattr, td->td_ucred);
VOP_UNLOCK(vp);
vn_finished_write(mp);
return (error);
}
/*
* Set the access and modification times of a file.
*/
#ifndef _SYS_SYSPROTO_H_
struct utimes_args {
char *path;
struct timeval *tptr;
};
#endif
int
sys_utimes(struct thread *td, struct utimes_args *uap)
{
return (kern_utimesat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->tptr, UIO_USERSPACE));
}
#ifndef _SYS_SYSPROTO_H_
struct futimesat_args {
int fd;
const char * path;
const struct timeval * times;
};
#endif
int
sys_futimesat(struct thread *td, struct futimesat_args *uap)
{
return (kern_utimesat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->times, UIO_USERSPACE));
}
int
kern_utimesat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, struct timeval *tptr, enum uio_seg tptrseg)
{
struct nameidata nd;
struct timespec ts[2];
int error;
if ((error = getutimes(tptr, tptrseg, ts)) != 0)
return (error);
NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, fd,
&cap_futimes_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = setutimes(td, nd.ni_vp, ts, 2, tptr == NULL);
vrele(nd.ni_vp);
return (error);
}
/*
* Set the access and modification times of a file.
*/
#ifndef _SYS_SYSPROTO_H_
struct lutimes_args {
char *path;
struct timeval *tptr;
};
#endif
int
sys_lutimes(struct thread *td, struct lutimes_args *uap)
{
return (kern_lutimes(td, uap->path, UIO_USERSPACE, uap->tptr,
UIO_USERSPACE));
}
int
kern_lutimes(struct thread *td, const char *path, enum uio_seg pathseg,
struct timeval *tptr, enum uio_seg tptrseg)
{
struct timespec ts[2];
struct nameidata nd;
int error;
if ((error = getutimes(tptr, tptrseg, ts)) != 0)
return (error);
NDINIT(&nd, LOOKUP, NOFOLLOW | AUDITVNODE1, pathseg, path, td);
if ((error = namei(&nd)) != 0)
return (error);
NDFREE_NOTHING(&nd);
error = setutimes(td, nd.ni_vp, ts, 2, tptr == NULL);
vrele(nd.ni_vp);
return (error);
}
/*
* Set the access and modification times of a file.
*/
#ifndef _SYS_SYSPROTO_H_
struct futimes_args {
int fd;
struct timeval *tptr;
};
#endif
int
sys_futimes(struct thread *td, struct futimes_args *uap)
{
return (kern_futimes(td, uap->fd, uap->tptr, UIO_USERSPACE));
}
int
kern_futimes(struct thread *td, int fd, struct timeval *tptr,
enum uio_seg tptrseg)
{
struct timespec ts[2];
struct file *fp;
int error;
AUDIT_ARG_FD(fd);
error = getutimes(tptr, tptrseg, ts);
if (error != 0)
return (error);
error = getvnode(td, fd, &cap_futimes_rights, &fp);
if (error != 0)
return (error);
#ifdef AUDIT
if (AUDITING_TD(td)) {
vn_lock(fp->f_vnode, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(fp->f_vnode);
VOP_UNLOCK(fp->f_vnode);
}
#endif
error = setutimes(td, fp->f_vnode, ts, 2, tptr == NULL);
fdrop(fp, td);
return (error);
}
int
sys_futimens(struct thread *td, struct futimens_args *uap)
{
return (kern_futimens(td, uap->fd, uap->times, UIO_USERSPACE));
}
int
kern_futimens(struct thread *td, int fd, struct timespec *tptr,
enum uio_seg tptrseg)
{
struct timespec ts[2];
struct file *fp;
int error, flags;
AUDIT_ARG_FD(fd);
error = getutimens(tptr, tptrseg, ts, &flags);
if (error != 0)
return (error);
if (flags & UTIMENS_EXIT)
return (0);
error = getvnode(td, fd, &cap_futimes_rights, &fp);
if (error != 0)
return (error);
#ifdef AUDIT
if (AUDITING_TD(td)) {
vn_lock(fp->f_vnode, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(fp->f_vnode);
VOP_UNLOCK(fp->f_vnode);
}
#endif
error = setutimes(td, fp->f_vnode, ts, 2, flags & UTIMENS_NULL);
fdrop(fp, td);
return (error);
}
int
sys_utimensat(struct thread *td, struct utimensat_args *uap)
{
return (kern_utimensat(td, uap->fd, uap->path, UIO_USERSPACE,
uap->times, UIO_USERSPACE, uap->flag));
}
int
kern_utimensat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, struct timespec *tptr, enum uio_seg tptrseg,
int flag)
{
struct nameidata nd;
struct timespec ts[2];
int error, flags;
if ((flag & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH |
AT_EMPTY_PATH)) != 0)
return (EINVAL);
if ((error = getutimens(tptr, tptrseg, ts, &flags)) != 0)
return (error);
NDINIT_ATRIGHTS(&nd, LOOKUP, at2cnpflags(flag, AT_SYMLINK_NOFOLLOW |
AT_RESOLVE_BENEATH | AT_EMPTY_PATH) | AUDITVNODE1,
pathseg, path, fd, &cap_futimes_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
/*
* We are allowed to call namei() regardless of 2xUTIME_OMIT.
* POSIX states:
* "If both tv_nsec fields are UTIME_OMIT... EACCESS may be detected."
* "Search permission is denied by a component of the path prefix."
*/
NDFREE_NOTHING(&nd);
if ((flags & UTIMENS_EXIT) == 0)
error = setutimes(td, nd.ni_vp, ts, 2, flags & UTIMENS_NULL);
vrele(nd.ni_vp);
return (error);
}
/*
* Truncate a file given its path name.
*/
#ifndef _SYS_SYSPROTO_H_
struct truncate_args {
char *path;
int pad;
off_t length;
};
#endif
int
sys_truncate(struct thread *td, struct truncate_args *uap)
{
return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
}
int
kern_truncate(struct thread *td, const char *path, enum uio_seg pathseg,
off_t length)
{
struct mount *mp;
struct vnode *vp;
void *rl_cookie;
struct vattr vattr;
struct nameidata nd;
int error;
if (length < 0)
return (EINVAL);
NDPREINIT(&nd);
retry:
NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, td);
if ((error = namei(&nd)) != 0)
return (error);
vp = nd.ni_vp;
rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0) {
vn_rangelock_unlock(vp, rl_cookie);
vrele(vp);
return (error);
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
if (vp->v_type == VDIR)
error = EISDIR;
#ifdef MAC
else if ((error = mac_vnode_check_write(td->td_ucred, NOCRED, vp))) {
}
#endif
else if ((error = vn_writechk(vp)) == 0 &&
(error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td)) == 0) {
VATTR_NULL(&vattr);
vattr.va_size = length;
error = VOP_SETATTR(vp, &vattr, td->td_ucred);
}
VOP_UNLOCK(vp);
vn_finished_write(mp);
vn_rangelock_unlock(vp, rl_cookie);
vrele(vp);
if (error == ERELOOKUP)
goto retry;
return (error);
}
#if defined(COMPAT_43)
/*
* Truncate a file given its path name.
*/
#ifndef _SYS_SYSPROTO_H_
struct otruncate_args {
char *path;
long length;
};
#endif
int
otruncate(struct thread *td, struct otruncate_args *uap)
{
return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
}
#endif /* COMPAT_43 */
#if defined(COMPAT_FREEBSD6)
/* Versions with the pad argument */
int
freebsd6_truncate(struct thread *td, struct freebsd6_truncate_args *uap)
{
return (kern_truncate(td, uap->path, UIO_USERSPACE, uap->length));
}
int
freebsd6_ftruncate(struct thread *td, struct freebsd6_ftruncate_args *uap)
{
return (kern_ftruncate(td, uap->fd, uap->length));
}
#endif
int
kern_fsync(struct thread *td, int fd, bool fullsync)
{
struct vnode *vp;
struct mount *mp;
struct file *fp;
int error;
AUDIT_ARG_FD(fd);
error = getvnode(td, fd, &cap_fsync_rights, &fp);
if (error != 0)
return (error);
vp = fp->f_vnode;
#if 0
if (!fullsync)
/* XXXKIB: compete outstanding aio writes */;
#endif
retry:
error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
if (error != 0)
goto drop;
vn_lock(vp, vn_lktype_write(mp, vp) | LK_RETRY);
AUDIT_ARG_VNODE1(vp);
if (vp->v_object != NULL) {
VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, 0, 0, 0);
VM_OBJECT_WUNLOCK(vp->v_object);
}
error = fullsync ? VOP_FSYNC(vp, MNT_WAIT, td) : VOP_FDATASYNC(vp, td);
VOP_UNLOCK(vp);
vn_finished_write(mp);
if (error == ERELOOKUP)
goto retry;
drop:
fdrop(fp, td);
return (error);
}
/*
* Sync an open file.
*/
#ifndef _SYS_SYSPROTO_H_
struct fsync_args {
int fd;
};
#endif
int
sys_fsync(struct thread *td, struct fsync_args *uap)
{
return (kern_fsync(td, uap->fd, true));
}
int
sys_fdatasync(struct thread *td, struct fdatasync_args *uap)
{
return (kern_fsync(td, uap->fd, false));
}
/*
* Rename files. Source and destination must either both be directories, or
* both not be directories. If target is a directory, it must be empty.
*/
#ifndef _SYS_SYSPROTO_H_
struct rename_args {
char *from;
char *to;
};
#endif
int
sys_rename(struct thread *td, struct rename_args *uap)
{
return (kern_renameat(td, AT_FDCWD, uap->from, AT_FDCWD,
uap->to, UIO_USERSPACE));
}
#ifndef _SYS_SYSPROTO_H_
struct renameat_args {
int oldfd;
char *old;
int newfd;
char *new;
};
#endif
int
sys_renameat(struct thread *td, struct renameat_args *uap)
{
return (kern_renameat(td, uap->oldfd, uap->old, uap->newfd, uap->new,
UIO_USERSPACE));
}
#ifdef MAC
static int
kern_renameat_mac(struct thread *td, int oldfd, const char *old, int newfd,
const char *new, enum uio_seg pathseg, struct nameidata *fromnd)
{
int error;
NDINIT_ATRIGHTS(fromnd, DELETE, LOCKPARENT | LOCKLEAF | SAVESTART |
AUDITVNODE1, pathseg, old, oldfd, &cap_renameat_source_rights, td);
if ((error = namei(fromnd)) != 0)
return (error);
error = mac_vnode_check_rename_from(td->td_ucred, fromnd->ni_dvp,
fromnd->ni_vp, &fromnd->ni_cnd);
VOP_UNLOCK(fromnd->ni_dvp);
if (fromnd->ni_dvp != fromnd->ni_vp)
VOP_UNLOCK(fromnd->ni_vp);
if (error != 0) {
NDFREE(fromnd, NDF_ONLY_PNBUF);
vrele(fromnd->ni_dvp);
vrele(fromnd->ni_vp);
if (fromnd->ni_startdir)
vrele(fromnd->ni_startdir);
}
return (error);
}
#endif
int
kern_renameat(struct thread *td, int oldfd, const char *old, int newfd,
const char *new, enum uio_seg pathseg)
{
struct mount *mp = NULL;
struct vnode *tvp, *fvp, *tdvp;
struct nameidata fromnd, tond;
u_int64_t tondflags;
int error;
again:
bwillwrite();
#ifdef MAC
if (mac_vnode_check_rename_from_enabled()) {
error = kern_renameat_mac(td, oldfd, old, newfd, new, pathseg,
&fromnd);
if (error != 0)
return (error);
} else {
#endif
NDINIT_ATRIGHTS(&fromnd, DELETE, WANTPARENT | SAVESTART | AUDITVNODE1,
pathseg, old, oldfd, &cap_renameat_source_rights, td);
if ((error = namei(&fromnd)) != 0)
return (error);
#ifdef MAC
}
#endif
fvp = fromnd.ni_vp;
tondflags = LOCKPARENT | LOCKLEAF | NOCACHE | SAVESTART | AUDITVNODE2;
if (fromnd.ni_vp->v_type == VDIR)
tondflags |= WILLBEDIR;
NDINIT_ATRIGHTS(&tond, RENAME, tondflags, pathseg, new, newfd,
&cap_renameat_target_rights, td);
if ((error = namei(&tond)) != 0) {
/* Translate error code for rename("dir1", "dir2/."). */
if (error == EISDIR && fvp->v_type == VDIR)
error = EINVAL;
NDFREE(&fromnd, NDF_ONLY_PNBUF);
vrele(fromnd.ni_dvp);
vrele(fvp);
goto out1;
}
tdvp = tond.ni_dvp;
tvp = tond.ni_vp;
error = vn_start_write(fvp, &mp, V_NOWAIT);
if (error != 0) {
NDFREE(&fromnd, NDF_ONLY_PNBUF);
NDFREE(&tond, NDF_ONLY_PNBUF);
if (tvp != NULL)
vput(tvp);
if (tdvp == tvp)
vrele(tdvp);
else
vput(tdvp);
vrele(fromnd.ni_dvp);
vrele(fvp);
vrele(tond.ni_startdir);
if (fromnd.ni_startdir != NULL)
vrele(fromnd.ni_startdir);
error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH);
if (error != 0)
return (error);
goto again;
}
if (tvp != NULL) {
if (fvp->v_type == VDIR && tvp->v_type != VDIR) {
error = ENOTDIR;
goto out;
} else if (fvp->v_type != VDIR && tvp->v_type == VDIR) {
error = EISDIR;
goto out;
}
#ifdef CAPABILITIES
if (newfd != AT_FDCWD && (tond.ni_resflags & NIRES_ABS) == 0) {
/*
* If the target already exists we require CAP_UNLINKAT
* from 'newfd', when newfd was used for the lookup.
*/
error = cap_check(&tond.ni_filecaps.fc_rights,
&cap_unlinkat_rights);
if (error != 0)
goto out;
}
#endif
}
if (fvp == tdvp) {
error = EINVAL;
goto out;
}
/*
* If the source is the same as the destination (that is, if they
* are links to the same vnode), then there is nothing to do.
*/
if (fvp == tvp)
error = ERESTART;
#ifdef MAC
else
error = mac_vnode_check_rename_to(td->td_ucred, tdvp,
tond.ni_vp, fromnd.ni_dvp == tdvp, &tond.ni_cnd);
#endif
out:
if (error == 0) {
error = VOP_RENAME(fromnd.ni_dvp, fromnd.ni_vp, &fromnd.ni_cnd,
tond.ni_dvp, tond.ni_vp, &tond.ni_cnd);
NDFREE(&fromnd, NDF_ONLY_PNBUF);
NDFREE(&tond, NDF_ONLY_PNBUF);
} else {
NDFREE(&fromnd, NDF_ONLY_PNBUF);
NDFREE(&tond, NDF_ONLY_PNBUF);
if (tvp != NULL)
vput(tvp);
if (tdvp == tvp)
vrele(tdvp);
else
vput(tdvp);
vrele(fromnd.ni_dvp);
vrele(fvp);
}
vrele(tond.ni_startdir);
vn_finished_write(mp);
out1:
if (fromnd.ni_startdir)
vrele(fromnd.ni_startdir);
if (error == ERESTART)
return (0);
if (error == ERELOOKUP)
goto again;
return (error);
}
/*
* Make a directory file.
*/
#ifndef _SYS_SYSPROTO_H_
struct mkdir_args {
char *path;
int mode;
};
#endif
int
sys_mkdir(struct thread *td, struct mkdir_args *uap)
{
return (kern_mkdirat(td, AT_FDCWD, uap->path, UIO_USERSPACE,
uap->mode));
}
#ifndef _SYS_SYSPROTO_H_
struct mkdirat_args {
int fd;
char *path;
mode_t mode;
};
#endif
int
sys_mkdirat(struct thread *td, struct mkdirat_args *uap)
{
return (kern_mkdirat(td, uap->fd, uap->path, UIO_USERSPACE, uap->mode));
}
int
kern_mkdirat(struct thread *td, int fd, const char *path, enum uio_seg segflg,
int mode)
{
struct mount *mp;
struct vattr vattr;
struct nameidata nd;
int error;
AUDIT_ARG_MODE(mode);
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1 |
NC_NOMAKEENTRY | NC_KEEPPOSENTRY | FAILIFEXISTS | WILLBEDIR,
segflg, path, fd, &cap_mkdirat_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
return (error);
goto restart;
}
VATTR_NULL(&vattr);
vattr.va_type = VDIR;
vattr.va_mode = (mode & ACCESSPERMS) &~ td->td_proc->p_pd->pd_cmask;
#ifdef MAC
error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
&vattr);
if (error != 0)
goto out;
#endif
error = VOP_MKDIR(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
#ifdef MAC
out:
#endif
NDFREE(&nd, NDF_ONLY_PNBUF);
VOP_VPUT_PAIR(nd.ni_dvp, error == 0 ? &nd.ni_vp : NULL, true);
vn_finished_write(mp);
if (error == ERELOOKUP)
goto restart;
return (error);
}
/*
* Remove a directory file.
*/
#ifndef _SYS_SYSPROTO_H_
struct rmdir_args {
char *path;
};
#endif
int
sys_rmdir(struct thread *td, struct rmdir_args *uap)
{
return (kern_frmdirat(td, AT_FDCWD, uap->path, FD_NONE, UIO_USERSPACE,
0));
}
int
kern_frmdirat(struct thread *td, int dfd, const char *path, int fd,
enum uio_seg pathseg, int flag)
{
struct mount *mp;
struct vnode *vp;
struct file *fp;
struct nameidata nd;
cap_rights_t rights;
int error;
fp = NULL;
if (fd != FD_NONE) {
error = getvnode(td, fd, cap_rights_init_one(&rights,
CAP_LOOKUP), &fp);
if (error != 0)
return (error);
}
NDPREINIT(&nd);
restart:
bwillwrite();
NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF | AUDITVNODE1 |
at2cnpflags(flag, AT_RESOLVE_BENEATH),
pathseg, path, dfd, &cap_unlinkat_rights, td);
if ((error = namei(&nd)) != 0)
goto fdout;
vp = nd.ni_vp;
if (vp->v_type != VDIR) {
error = ENOTDIR;
goto out;
}
/*
* No rmdir "." please.
*/
if (nd.ni_dvp == vp) {
error = EINVAL;
goto out;
}
/*
* The root of a mounted filesystem cannot be deleted.
*/
if (vp->v_vflag & VV_ROOT) {
error = EBUSY;
goto out;
}
if (fp != NULL && fp->f_vnode != vp) {
if (VN_IS_DOOMED(fp->f_vnode))
error = EBADF;
else
error = EDEADLK;
goto out;
}
#ifdef MAC
error = mac_vnode_check_unlink(td->td_ucred, nd.ni_dvp, vp,
&nd.ni_cnd);
if (error != 0)
goto out;
#endif
if (vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(vp);
if (nd.ni_dvp == vp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
if ((error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH)) != 0)
goto fdout;
goto restart;
}
vfs_notify_upper(vp, VFS_NOTIFY_UPPER_UNLINK);
error = VOP_RMDIR(nd.ni_dvp, nd.ni_vp, &nd.ni_cnd);
vn_finished_write(mp);
out:
NDFREE(&nd, NDF_ONLY_PNBUF);
vput(vp);
if (nd.ni_dvp == vp)
vrele(nd.ni_dvp);
else
vput(nd.ni_dvp);
if (error == ERELOOKUP)
goto restart;
fdout:
if (fp != NULL)
fdrop(fp, td);
return (error);
}
#if defined(COMPAT_43) || defined(COMPAT_FREEBSD11)
int
freebsd11_kern_getdirentries(struct thread *td, int fd, char *ubuf, u_int count,
long *basep, void (*func)(struct freebsd11_dirent *))
{
struct freebsd11_dirent dstdp;
struct dirent *dp, *edp;
char *dirbuf;
off_t base;
ssize_t resid, ucount;
int error;
/* XXX arbitrary sanity limit on `count'. */
count = min(count, 64 * 1024);
dirbuf = malloc(count, M_TEMP, M_WAITOK);
error = kern_getdirentries(td, fd, dirbuf, count, &base, &resid,
UIO_SYSSPACE);
if (error != 0)
goto done;
if (basep != NULL)
*basep = base;
ucount = 0;
for (dp = (struct dirent *)dirbuf,
edp = (struct dirent *)&dirbuf[count - resid];
ucount < count && dp < edp; ) {
if (dp->d_reclen == 0)
break;
MPASS(dp->d_reclen >= _GENERIC_DIRLEN(0));
if (dp->d_namlen >= sizeof(dstdp.d_name))
continue;
dstdp.d_type = dp->d_type;
dstdp.d_namlen = dp->d_namlen;
dstdp.d_fileno = dp->d_fileno; /* truncate */
if (dstdp.d_fileno != dp->d_fileno) {
switch (ino64_trunc_error) {
default:
case 0:
break;
case 1:
error = EOVERFLOW;
goto done;
case 2:
dstdp.d_fileno = UINT32_MAX;
break;
}
}
dstdp.d_reclen = sizeof(dstdp) - sizeof(dstdp.d_name) +
((dp->d_namlen + 1 + 3) &~ 3);
bcopy(dp->d_name, dstdp.d_name, dstdp.d_namlen);
bzero(dstdp.d_name + dstdp.d_namlen,
dstdp.d_reclen - offsetof(struct freebsd11_dirent, d_name) -
dstdp.d_namlen);
MPASS(dstdp.d_reclen <= dp->d_reclen);
MPASS(ucount + dstdp.d_reclen <= count);
if (func != NULL)
func(&dstdp);
error = copyout(&dstdp, ubuf + ucount, dstdp.d_reclen);
if (error != 0)
break;
dp = (struct dirent *)((char *)dp + dp->d_reclen);
ucount += dstdp.d_reclen;
}
done:
free(dirbuf, M_TEMP);
if (error == 0)
td->td_retval[0] = ucount;
return (error);
}
#endif /* COMPAT */
#ifdef COMPAT_43
static void
ogetdirentries_cvt(struct freebsd11_dirent *dp)
{
#if (BYTE_ORDER == LITTLE_ENDIAN)
/*
* The expected low byte of dp->d_namlen is our dp->d_type.
* The high MBZ byte of dp->d_namlen is our dp->d_namlen.
*/
dp->d_type = dp->d_namlen;
dp->d_namlen = 0;
#else
/*
* The dp->d_type is the high byte of the expected dp->d_namlen,
* so must be zero'ed.
*/
dp->d_type = 0;
#endif
}
/*
* Read a block of directory entries in a filesystem independent format.
*/
#ifndef _SYS_SYSPROTO_H_
struct ogetdirentries_args {
int fd;
char *buf;
u_int count;
long *basep;
};
#endif
int
ogetdirentries(struct thread *td, struct ogetdirentries_args *uap)
{
long loff;
int error;
error = kern_ogetdirentries(td, uap, &loff);
if (error == 0)
error = copyout(&loff, uap->basep, sizeof(long));
return (error);
}
int
kern_ogetdirentries(struct thread *td, struct ogetdirentries_args *uap,
long *ploff)
{
long base;
int error;
/* XXX arbitrary sanity limit on `count'. */
if (uap->count > 64 * 1024)
return (EINVAL);
error = freebsd11_kern_getdirentries(td, uap->fd, uap->buf, uap->count,
&base, ogetdirentries_cvt);
if (error == 0 && uap->basep != NULL)
error = copyout(&base, uap->basep, sizeof(long));
return (error);
}
#endif /* COMPAT_43 */
#if defined(COMPAT_FREEBSD11)
#ifndef _SYS_SYSPROTO_H_
struct freebsd11_getdirentries_args {
int fd;
char *buf;
u_int count;
long *basep;
};
#endif
int
freebsd11_getdirentries(struct thread *td,
struct freebsd11_getdirentries_args *uap)
{
long base;
int error;
error = freebsd11_kern_getdirentries(td, uap->fd, uap->buf, uap->count,
&base, NULL);
if (error == 0 && uap->basep != NULL)
error = copyout(&base, uap->basep, sizeof(long));
return (error);
}
int
freebsd11_getdents(struct thread *td, struct freebsd11_getdents_args *uap)
{
struct freebsd11_getdirentries_args ap;
ap.fd = uap->fd;
ap.buf = uap->buf;
ap.count = uap->count;
ap.basep = NULL;
return (freebsd11_getdirentries(td, &ap));
}
#endif /* COMPAT_FREEBSD11 */
/*
* Read a block of directory entries in a filesystem independent format.
*/
int
sys_getdirentries(struct thread *td, struct getdirentries_args *uap)
{
off_t base;
int error;
error = kern_getdirentries(td, uap->fd, uap->buf, uap->count, &base,
NULL, UIO_USERSPACE);
if (error != 0)
return (error);
if (uap->basep != NULL)
error = copyout(&base, uap->basep, sizeof(off_t));
return (error);
}
int
kern_getdirentries(struct thread *td, int fd, char *buf, size_t count,
off_t *basep, ssize_t *residp, enum uio_seg bufseg)
{
struct vnode *vp;
struct file *fp;
struct uio auio;
struct iovec aiov;
off_t loff;
int error, eofflag;
off_t foffset;
AUDIT_ARG_FD(fd);
if (count > IOSIZE_MAX)
return (EINVAL);
auio.uio_resid = count;
error = getvnode(td, fd, &cap_read_rights, &fp);
if (error != 0)
return (error);
if ((fp->f_flag & FREAD) == 0) {
fdrop(fp, td);
return (EBADF);
}
vp = fp->f_vnode;
foffset = foffset_lock(fp, 0);
unionread:
if (vp->v_type != VDIR) {
error = EINVAL;
goto fail;
}
aiov.iov_base = buf;
aiov.iov_len = count;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_rw = UIO_READ;
auio.uio_segflg = bufseg;
auio.uio_td = td;
vn_lock(vp, LK_SHARED | LK_RETRY);
AUDIT_ARG_VNODE1(vp);
loff = auio.uio_offset = foffset;
#ifdef MAC
error = mac_vnode_check_readdir(td->td_ucred, vp);
if (error == 0)
#endif
error = VOP_READDIR(vp, &auio, fp->f_cred, &eofflag, NULL,
NULL);
foffset = auio.uio_offset;
if (error != 0) {
VOP_UNLOCK(vp);
goto fail;
}
if (count == auio.uio_resid &&
(vp->v_vflag & VV_ROOT) &&
(vp->v_mount->mnt_flag & MNT_UNION)) {
struct vnode *tvp = vp;
vp = vp->v_mount->mnt_vnodecovered;
VREF(vp);
fp->f_vnode = vp;
foffset = 0;
vput(tvp);
goto unionread;
}
VOP_UNLOCK(vp);
*basep = loff;
if (residp != NULL)
*residp = auio.uio_resid;
td->td_retval[0] = count - auio.uio_resid;
fail:
foffset_unlock(fp, foffset, 0);
fdrop(fp, td);
return (error);
}
/*
* Set the mode mask for creation of filesystem nodes.
*/
#ifndef _SYS_SYSPROTO_H_
struct umask_args {
int newmask;
};
#endif
int
sys_umask(struct thread *td, struct umask_args *uap)
{
struct pwddesc *pdp;
pdp = td->td_proc->p_pd;
PWDDESC_XLOCK(pdp);
td->td_retval[0] = pdp->pd_cmask;
pdp->pd_cmask = uap->newmask & ALLPERMS;
PWDDESC_XUNLOCK(pdp);
return (0);
}
/*
* Void all references to file by ripping underlying filesystem away from
* vnode.
*/
#ifndef _SYS_SYSPROTO_H_
struct revoke_args {
char *path;
};
#endif
int
sys_revoke(struct thread *td, struct revoke_args *uap)
{
struct vnode *vp;
struct vattr vattr;
struct nameidata nd;
int error;
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
uap->path, td);
if ((error = namei(&nd)) != 0)
return (error);
vp = nd.ni_vp;
NDFREE_NOTHING(&nd);
if (vp->v_type != VCHR || vp->v_rdev == NULL) {
error = EINVAL;
goto out;
}
#ifdef MAC
error = mac_vnode_check_revoke(td->td_ucred, vp);
if (error != 0)
goto out;
#endif
error = VOP_GETATTR(vp, &vattr, td->td_ucred);
if (error != 0)
goto out;
if (td->td_ucred->cr_uid != vattr.va_uid) {
error = priv_check(td, PRIV_VFS_ADMIN);
if (error != 0)
goto out;
}
if (devfs_usecount(vp) > 0)
VOP_REVOKE(vp, REVOKEALL);
out:
vput(vp);
return (error);
}
/*
* This variant of getvnode() allows O_PATH files. Caller should
* ensure that returned file and vnode are only used for compatible
* semantics.
*/
int
getvnode_path(struct thread *td, int fd, cap_rights_t *rightsp,
struct file **fpp)
{
struct file *fp;
int error;
error = fget_unlocked(td->td_proc->p_fd, fd, rightsp, &fp);
if (error != 0)
return (error);
/*
* The file could be not of the vnode type, or it may be not
* yet fully initialized, in which case the f_vnode pointer
* may be set, but f_ops is still badfileops. E.g.,
* devfs_open() transiently create such situation to
* facilitate csw d_fdopen().
*
* Dupfdopen() handling in kern_openat() installs the
* half-baked file into the process descriptor table, allowing
* other thread to dereference it. Guard against the race by
* checking f_ops.
*/
if (fp->f_vnode == NULL || fp->f_ops == &badfileops) {
fdrop(fp, td);
*fpp = NULL;
return (EINVAL);
}
*fpp = fp;
return (0);
}
/*
* Convert a user file descriptor to a kernel file entry and check
* that, if it is a capability, the correct rights are present.
* A reference on the file entry is held upon returning.
*/
int
getvnode(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
int error;
error = getvnode_path(td, fd, rightsp, fpp);
/*
* Filter out O_PATH file descriptors, most getvnode() callers
* do not call fo_ methods.
*/
if (error == 0 && (*fpp)->f_ops == &path_fileops) {
fdrop(*fpp, td);
*fpp = NULL;
error = EBADF;
}
return (error);
}
/*
* Get an (NFS) file handle.
*/
#ifndef _SYS_SYSPROTO_H_
struct lgetfh_args {
char *fname;
fhandle_t *fhp;
};
#endif
int
sys_lgetfh(struct thread *td, struct lgetfh_args *uap)
{
return (kern_getfhat(td, AT_SYMLINK_NOFOLLOW, AT_FDCWD, uap->fname,
UIO_USERSPACE, uap->fhp, UIO_USERSPACE));
}
#ifndef _SYS_SYSPROTO_H_
struct getfh_args {
char *fname;
fhandle_t *fhp;
};
#endif
int
sys_getfh(struct thread *td, struct getfh_args *uap)
{
return (kern_getfhat(td, 0, AT_FDCWD, uap->fname, UIO_USERSPACE,
uap->fhp, UIO_USERSPACE));
}
/*
* syscall for the rpc.lockd to use to translate an open descriptor into
* a NFS file handle.
*
* warning: do not remove the priv_check() call or this becomes one giant
* security hole.
*/
#ifndef _SYS_SYSPROTO_H_
struct getfhat_args {
int fd;
char *path;
fhandle_t *fhp;
int flags;
};
#endif
int
sys_getfhat(struct thread *td, struct getfhat_args *uap)
{
if ((uap->flags & ~(AT_SYMLINK_NOFOLLOW | AT_RESOLVE_BENEATH)) != 0)
return (EINVAL);
return (kern_getfhat(td, uap->flags, uap->fd, uap->path, UIO_USERSPACE,
uap->fhp, UIO_USERSPACE));
}
int
kern_getfhat(struct thread *td, int flags, int fd, const char *path,
enum uio_seg pathseg, fhandle_t *fhp, enum uio_seg fhseg)
{
struct nameidata nd;
fhandle_t fh;
struct vnode *vp;
int error;
error = priv_check(td, PRIV_VFS_GETFH);
if (error != 0)
return (error);
NDINIT_AT(&nd, LOOKUP, at2cnpflags(flags, AT_SYMLINK_NOFOLLOW |
AT_RESOLVE_BENEATH) | LOCKLEAF | AUDITVNODE1, pathseg, path,
fd, td);
error = namei(&nd);
if (error != 0)
return (error);
NDFREE_NOTHING(&nd);
vp = nd.ni_vp;
bzero(&fh, sizeof(fh));
fh.fh_fsid = vp->v_mount->mnt_stat.f_fsid;
error = VOP_VPTOFH(vp, &fh.fh_fid);
vput(vp);
if (error == 0) {
if (fhseg == UIO_USERSPACE)
error = copyout(&fh, fhp, sizeof (fh));
else
memcpy(fhp, &fh, sizeof(fh));
}
return (error);
}
#ifndef _SYS_SYSPROTO_H_
struct fhlink_args {
fhandle_t *fhp;
const char *to;
};
#endif
int
sys_fhlink(struct thread *td, struct fhlink_args *uap)
{
return (kern_fhlinkat(td, AT_FDCWD, uap->to, UIO_USERSPACE, uap->fhp));
}
#ifndef _SYS_SYSPROTO_H_
struct fhlinkat_args {
fhandle_t *fhp;
int tofd;
const char *to;
};
#endif
int
sys_fhlinkat(struct thread *td, struct fhlinkat_args *uap)
{
return (kern_fhlinkat(td, uap->tofd, uap->to, UIO_USERSPACE, uap->fhp));
}
static int
kern_fhlinkat(struct thread *td, int fd, const char *path,
enum uio_seg pathseg, fhandle_t *fhp)
{
fhandle_t fh;
struct mount *mp;
struct vnode *vp;
int error;
error = priv_check(td, PRIV_VFS_GETFH);
if (error != 0)
return (error);
error = copyin(fhp, &fh, sizeof(fh));
if (error != 0)
return (error);
do {
bwillwrite();
if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
return (ESTALE);
error = VFS_FHTOVP(mp, &fh.fh_fid, LK_SHARED, &vp);
vfs_unbusy(mp);
if (error != 0)
return (error);
VOP_UNLOCK(vp);
error = kern_linkat_vp(td, vp, fd, path, pathseg);
} while (error == EAGAIN || error == ERELOOKUP);
return (error);
}
#ifndef _SYS_SYSPROTO_H_
struct fhreadlink_args {
fhandle_t *fhp;
char *buf;
size_t bufsize;
};
#endif
int
sys_fhreadlink(struct thread *td, struct fhreadlink_args *uap)
{
fhandle_t fh;
struct mount *mp;
struct vnode *vp;
int error;
error = priv_check(td, PRIV_VFS_GETFH);
if (error != 0)
return (error);
if (uap->bufsize > IOSIZE_MAX)
return (EINVAL);
error = copyin(uap->fhp, &fh, sizeof(fh));
if (error != 0)
return (error);
if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
return (ESTALE);
error = VFS_FHTOVP(mp, &fh.fh_fid, LK_SHARED, &vp);
vfs_unbusy(mp);
if (error != 0)
return (error);
error = kern_readlink_vp(vp, uap->buf, UIO_USERSPACE, uap->bufsize, td);
vput(vp);
return (error);
}
/*
* syscall for the rpc.lockd to use to translate a NFS file handle into an
* open descriptor.
*
* warning: do not remove the priv_check() call or this becomes one giant
* security hole.
*/
#ifndef _SYS_SYSPROTO_H_
struct fhopen_args {
const struct fhandle *u_fhp;
int flags;
};
#endif
int
sys_fhopen(struct thread *td, struct fhopen_args *uap)
{
return (kern_fhopen(td, uap->u_fhp, uap->flags));
}
int
kern_fhopen(struct thread *td, const struct fhandle *u_fhp, int flags)
{
struct mount *mp;
struct vnode *vp;
struct fhandle fhp;
struct file *fp;
int fmode, error;
int indx;
error = priv_check(td, PRIV_VFS_FHOPEN);
if (error != 0)
return (error);
indx = -1;
fmode = FFLAGS(flags);
/* why not allow a non-read/write open for our lockd? */
if (((fmode & (FREAD | FWRITE)) == 0) || (fmode & O_CREAT))
return (EINVAL);
error = copyin(u_fhp, &fhp, sizeof(fhp));
if (error != 0)
return(error);
/* find the mount point */
mp = vfs_busyfs(&fhp.fh_fsid);
if (mp == NULL)
return (ESTALE);
/* now give me my vnode, it gets returned to me locked */
error = VFS_FHTOVP(mp, &fhp.fh_fid, LK_EXCLUSIVE, &vp);
vfs_unbusy(mp);
if (error != 0)
return (error);
error = falloc_noinstall(td, &fp);
if (error != 0) {
vput(vp);
return (error);
}
/*
* An extra reference on `fp' has been held for us by
* falloc_noinstall().
*/
#ifdef INVARIANTS
td->td_dupfd = -1;
#endif
error = vn_open_vnode(vp, fmode, td->td_ucred, td, fp);
if (error != 0) {
KASSERT(fp->f_ops == &badfileops,
("VOP_OPEN in fhopen() set f_ops"));
KASSERT(td->td_dupfd < 0,
("fhopen() encountered fdopen()"));
vput(vp);
goto bad;
}
#ifdef INVARIANTS
td->td_dupfd = 0;
#endif
fp->f_vnode = vp;
finit_vnode(fp, fmode, NULL, &vnops);
VOP_UNLOCK(vp);
if ((fmode & O_TRUNC) != 0) {
error = fo_truncate(fp, 0, td->td_ucred, td);
if (error != 0)
goto bad;
}
error = finstall(td, fp, &indx, fmode, NULL);
bad:
fdrop(fp, td);
td->td_retval[0] = indx;
return (error);
}
/*
* Stat an (NFS) file handle.
*/
#ifndef _SYS_SYSPROTO_H_
struct fhstat_args {
struct fhandle *u_fhp;
struct stat *sb;
};
#endif
int
sys_fhstat(struct thread *td, struct fhstat_args *uap)
{
struct stat sb;
struct fhandle fh;
int error;
error = copyin(uap->u_fhp, &fh, sizeof(fh));
if (error != 0)
return (error);
error = kern_fhstat(td, fh, &sb);
if (error == 0)
error = copyout(&sb, uap->sb, sizeof(sb));
return (error);
}
int
kern_fhstat(struct thread *td, struct fhandle fh, struct stat *sb)
{
struct mount *mp;
struct vnode *vp;
int error;
error = priv_check(td, PRIV_VFS_FHSTAT);
if (error != 0)
return (error);
if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
return (ESTALE);
error = VFS_FHTOVP(mp, &fh.fh_fid, LK_EXCLUSIVE, &vp);
vfs_unbusy(mp);
if (error != 0)
return (error);
error = VOP_STAT(vp, sb, td->td_ucred, NOCRED, td);
vput(vp);
return (error);
}
/*
* Implement fstatfs() for (NFS) file handles.
*/
#ifndef _SYS_SYSPROTO_H_
struct fhstatfs_args {
struct fhandle *u_fhp;
struct statfs *buf;
};
#endif
int
sys_fhstatfs(struct thread *td, struct fhstatfs_args *uap)
{
struct statfs *sfp;
fhandle_t fh;
int error;
error = copyin(uap->u_fhp, &fh, sizeof(fhandle_t));
if (error != 0)
return (error);
sfp = malloc(sizeof(struct statfs), M_STATFS, M_WAITOK);
error = kern_fhstatfs(td, fh, sfp);
if (error == 0)
error = copyout(sfp, uap->buf, sizeof(*sfp));
free(sfp, M_STATFS);
return (error);
}
int
kern_fhstatfs(struct thread *td, fhandle_t fh, struct statfs *buf)
{
struct mount *mp;
struct vnode *vp;
int error;
error = priv_check(td, PRIV_VFS_FHSTATFS);
if (error != 0)
return (error);
if ((mp = vfs_busyfs(&fh.fh_fsid)) == NULL)
return (ESTALE);
error = VFS_FHTOVP(mp, &fh.fh_fid, LK_EXCLUSIVE, &vp);
if (error != 0) {
vfs_unbusy(mp);
return (error);
}
vput(vp);
error = prison_canseemount(td->td_ucred, mp);
if (error != 0)
goto out;
#ifdef MAC
error = mac_mount_check_stat(td->td_ucred, mp);
if (error != 0)
goto out;
#endif
error = VFS_STATFS(mp, buf);
out:
vfs_unbusy(mp);
return (error);
}
/*
* Unlike madvise(2), we do not make a best effort to remember every
* possible caching hint. Instead, we remember the last setting with
* the exception that we will allow POSIX_FADV_NORMAL to adjust the
* region of any current setting.
*/
int
kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len,
int advice)
{
struct fadvise_info *fa, *new;
struct file *fp;
struct vnode *vp;
off_t end;
int error;
if (offset < 0 || len < 0 || offset > OFF_MAX - len)
return (EINVAL);
AUDIT_ARG_VALUE(advice);
switch (advice) {
case POSIX_FADV_SEQUENTIAL:
case POSIX_FADV_RANDOM:
case POSIX_FADV_NOREUSE:
new = malloc(sizeof(*fa), M_FADVISE, M_WAITOK);
break;
case POSIX_FADV_NORMAL:
case POSIX_FADV_WILLNEED:
case POSIX_FADV_DONTNEED:
new = NULL;
break;
default:
return (EINVAL);
}
/* XXX: CAP_POSIX_FADVISE? */
AUDIT_ARG_FD(fd);
error = fget(td, fd, &cap_no_rights, &fp);
if (error != 0)
goto out;
AUDIT_ARG_FILE(td->td_proc, fp);
if ((fp->f_ops->fo_flags & DFLAG_SEEKABLE) == 0) {
error = ESPIPE;
goto out;
}
if (fp->f_type != DTYPE_VNODE) {
error = ENODEV;
goto out;
}
vp = fp->f_vnode;
if (vp->v_type != VREG) {
error = ENODEV;
goto out;
}
if (len == 0)
end = OFF_MAX;
else
end = offset + len - 1;
switch (advice) {
case POSIX_FADV_SEQUENTIAL:
case POSIX_FADV_RANDOM:
case POSIX_FADV_NOREUSE:
/*
* Try to merge any existing non-standard region with
* this new region if possible, otherwise create a new
* non-standard region for this request.
*/
mtx_pool_lock(mtxpool_sleep, fp);
fa = fp->f_advice;
if (fa != NULL && fa->fa_advice == advice &&
((fa->fa_start <= end && fa->fa_end >= offset) ||
(end != OFF_MAX && fa->fa_start == end + 1) ||
(fa->fa_end != OFF_MAX && fa->fa_end + 1 == offset))) {
if (offset < fa->fa_start)
fa->fa_start = offset;
if (end > fa->fa_end)
fa->fa_end = end;
} else {
new->fa_advice = advice;
new->fa_start = offset;
new->fa_end = end;
fp->f_advice = new;
new = fa;
}
mtx_pool_unlock(mtxpool_sleep, fp);
break;
case POSIX_FADV_NORMAL:
/*
* If a the "normal" region overlaps with an existing
* non-standard region, trim or remove the
* non-standard region.
*/
mtx_pool_lock(mtxpool_sleep, fp);
fa = fp->f_advice;
if (fa != NULL) {
if (offset <= fa->fa_start && end >= fa->fa_end) {
new = fa;
fp->f_advice = NULL;
} else if (offset <= fa->fa_start &&
end >= fa->fa_start)
fa->fa_start = end + 1;
else if (offset <= fa->fa_end && end >= fa->fa_end)
fa->fa_end = offset - 1;
else if (offset >= fa->fa_start && end <= fa->fa_end) {
/*
* If the "normal" region is a middle
* portion of the existing
* non-standard region, just remove
* the whole thing rather than picking
* one side or the other to
* preserve.
*/
new = fa;
fp->f_advice = NULL;
}
}
mtx_pool_unlock(mtxpool_sleep, fp);
break;
case POSIX_FADV_WILLNEED:
case POSIX_FADV_DONTNEED:
error = VOP_ADVISE(vp, offset, end, advice);
break;
}
out:
if (fp != NULL)
fdrop(fp, td);
free(new, M_FADVISE);
return (error);
}
int
sys_posix_fadvise(struct thread *td, struct posix_fadvise_args *uap)
{
int error;
error = kern_posix_fadvise(td, uap->fd, uap->offset, uap->len,
uap->advice);
return (kern_posix_error(td, error));
}
int
kern_copy_file_range(struct thread *td, int infd, off_t *inoffp, int outfd,
off_t *outoffp, size_t len, unsigned int flags)
{
struct file *infp, *outfp;
struct vnode *invp, *outvp;
int error;
size_t retlen;
void *rl_rcookie, *rl_wcookie;
off_t savinoff, savoutoff;
infp = outfp = NULL;
rl_rcookie = rl_wcookie = NULL;
savinoff = -1;
error = 0;
retlen = 0;
if (flags != 0) {
error = EINVAL;
goto out;
}
if (len > SSIZE_MAX)
/*
* Although the len argument is size_t, the return argument
* is ssize_t (which is signed). Therefore a size that won't
* fit in ssize_t can't be returned.
*/
len = SSIZE_MAX;
/* Get the file structures for the file descriptors. */
error = fget_read(td, infd, &cap_read_rights, &infp);
if (error != 0)
goto out;
if (infp->f_ops == &badfileops) {
error = EBADF;
goto out;
}
if (infp->f_vnode == NULL) {
error = EINVAL;
goto out;
}
error = fget_write(td, outfd, &cap_write_rights, &outfp);
if (error != 0)
goto out;
if (outfp->f_ops == &badfileops) {
error = EBADF;
goto out;
}
if (outfp->f_vnode == NULL) {
error = EINVAL;
goto out;
}
/* Set the offset pointers to the correct place. */
if (inoffp == NULL)
inoffp = &infp->f_offset;
if (outoffp == NULL)
outoffp = &outfp->f_offset;
savinoff = *inoffp;
savoutoff = *outoffp;
invp = infp->f_vnode;
outvp = outfp->f_vnode;
/* Sanity check the f_flag bits. */
if ((outfp->f_flag & (FWRITE | FAPPEND)) != FWRITE ||
(infp->f_flag & FREAD) == 0) {
error = EBADF;
goto out;
}
/* If len == 0, just return 0. */
if (len == 0)
goto out;
/*
* If infp and outfp refer to the same file, the byte ranges cannot
* overlap.
*/
if (invp == outvp && ((savinoff <= savoutoff && savinoff + len >
savoutoff) || (savinoff > savoutoff && savoutoff + len >
savinoff))) {
error = EINVAL;
goto out;
}
/* Range lock the byte ranges for both invp and outvp. */
for (;;) {
rl_wcookie = vn_rangelock_wlock(outvp, *outoffp, *outoffp +
len);
rl_rcookie = vn_rangelock_tryrlock(invp, *inoffp, *inoffp +
len);
if (rl_rcookie != NULL)
break;
vn_rangelock_unlock(outvp, rl_wcookie);
rl_rcookie = vn_rangelock_rlock(invp, *inoffp, *inoffp + len);
vn_rangelock_unlock(invp, rl_rcookie);
}
retlen = len;
error = vn_copy_file_range(invp, inoffp, outvp, outoffp, &retlen,
flags, infp->f_cred, outfp->f_cred, td);
out:
if (rl_rcookie != NULL)
vn_rangelock_unlock(invp, rl_rcookie);
if (rl_wcookie != NULL)
vn_rangelock_unlock(outvp, rl_wcookie);
if (savinoff != -1 && (error == EINTR || error == ERESTART)) {
*inoffp = savinoff;
*outoffp = savoutoff;
}
if (outfp != NULL)
fdrop(outfp, td);
if (infp != NULL)
fdrop(infp, td);
td->td_retval[0] = retlen;
return (error);
}
int
sys_copy_file_range(struct thread *td, struct copy_file_range_args *uap)
{
off_t inoff, outoff, *inoffp, *outoffp;
int error;
inoffp = outoffp = NULL;
if (uap->inoffp != NULL) {
error = copyin(uap->inoffp, &inoff, sizeof(off_t));
if (error != 0)
return (error);
inoffp = &inoff;
}
if (uap->outoffp != NULL) {
error = copyin(uap->outoffp, &outoff, sizeof(off_t));
if (error != 0)
return (error);
outoffp = &outoff;
}
error = kern_copy_file_range(td, uap->infd, inoffp, uap->outfd,
outoffp, uap->len, uap->flags);
if (error == 0 && uap->inoffp != NULL)
error = copyout(inoffp, uap->inoffp, sizeof(off_t));
if (error == 0 && uap->outoffp != NULL)
error = copyout(outoffp, uap->outoffp, sizeof(off_t));
return (error);
}
diff --git a/sys/sys/file.h b/sys/sys/file.h
index c97841d1a108..66b50c418953 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -1,481 +1,482 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)file.h 8.3 (Berkeley) 1/9/95
* $FreeBSD$
*/
#ifndef _SYS_FILE_H_
#define _SYS_FILE_H_
#ifndef _KERNEL
#include <sys/types.h> /* XXX */
#include <sys/fcntl.h>
#include <sys/unistd.h>
#else
#include <sys/queue.h>
#include <sys/refcount.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <vm/vm.h>
struct filedesc;
struct stat;
struct thread;
struct uio;
struct knote;
struct vnode;
struct nameidata;
#endif /* _KERNEL */
#define DTYPE_NONE 0 /* not yet initialized */
#define DTYPE_VNODE 1 /* file */
#define DTYPE_SOCKET 2 /* communications endpoint */
#define DTYPE_PIPE 3 /* pipe */
#define DTYPE_FIFO 4 /* fifo (named pipe) */
#define DTYPE_KQUEUE 5 /* event queue */
#define DTYPE_CRYPTO 6 /* crypto */
#define DTYPE_MQUEUE 7 /* posix message queue */
#define DTYPE_SHM 8 /* swap-backed shared memory */
#define DTYPE_SEM 9 /* posix semaphore */
#define DTYPE_PTS 10 /* pseudo teletype master device */
#define DTYPE_DEV 11 /* Device specific fd type */
#define DTYPE_PROCDESC 12 /* process descriptor */
#define DTYPE_EVENTFD 13 /* eventfd */
#define DTYPE_LINUXTFD 14 /* emulation timerfd type */
#ifdef _KERNEL
struct file;
struct filecaps;
struct kaiocb;
struct kinfo_file;
struct ucred;
#define FOF_OFFSET 0x01 /* Use the offset in uio argument */
#define FOF_NOLOCK 0x02 /* Do not take FOFFSET_LOCK */
#define FOF_NEXTOFF_R 0x04 /* Also update f_nextoff[UIO_READ] */
#define FOF_NEXTOFF_W 0x08 /* Also update f_nextoff[UIO_WRITE] */
#define FOF_NOUPDATE 0x10 /* Do not update f_offset */
off_t foffset_lock(struct file *fp, int flags);
void foffset_lock_uio(struct file *fp, struct uio *uio, int flags);
void foffset_unlock(struct file *fp, off_t val, int flags);
void foffset_unlock_uio(struct file *fp, struct uio *uio, int flags);
static inline off_t
foffset_get(struct file *fp)
{
return (foffset_lock(fp, FOF_NOLOCK));
}
typedef int fo_rdwr_t(struct file *fp, struct uio *uio,
struct ucred *active_cred, int flags,
struct thread *td);
typedef int fo_truncate_t(struct file *fp, off_t length,
struct ucred *active_cred, struct thread *td);
typedef int fo_ioctl_t(struct file *fp, u_long com, void *data,
struct ucred *active_cred, struct thread *td);
typedef int fo_poll_t(struct file *fp, int events,
struct ucred *active_cred, struct thread *td);
typedef int fo_kqfilter_t(struct file *fp, struct knote *kn);
typedef int fo_stat_t(struct file *fp, struct stat *sb,
struct ucred *active_cred, struct thread *td);
typedef int fo_close_t(struct file *fp, struct thread *td);
typedef int fo_chmod_t(struct file *fp, mode_t mode,
struct ucred *active_cred, struct thread *td);
typedef int fo_chown_t(struct file *fp, uid_t uid, gid_t gid,
struct ucred *active_cred, struct thread *td);
typedef int fo_sendfile_t(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes,
off_t *sent, int flags, struct thread *td);
typedef int fo_seek_t(struct file *fp, off_t offset, int whence,
struct thread *td);
typedef int fo_fill_kinfo_t(struct file *fp, struct kinfo_file *kif,
struct filedesc *fdp);
typedef int fo_mmap_t(struct file *fp, vm_map_t map, vm_offset_t *addr,
vm_size_t size, vm_prot_t prot, vm_prot_t cap_maxprot,
int flags, vm_ooffset_t foff, struct thread *td);
typedef int fo_aio_queue_t(struct file *fp, struct kaiocb *job);
typedef int fo_add_seals_t(struct file *fp, int flags);
typedef int fo_get_seals_t(struct file *fp, int *flags);
typedef int fo_fallocate_t(struct file *fp, off_t offset, off_t len,
struct thread *td);
typedef int fo_flags_t;
struct fileops {
fo_rdwr_t *fo_read;
fo_rdwr_t *fo_write;
fo_truncate_t *fo_truncate;
fo_ioctl_t *fo_ioctl;
fo_poll_t *fo_poll;
fo_kqfilter_t *fo_kqfilter;
fo_stat_t *fo_stat;
fo_close_t *fo_close;
fo_chmod_t *fo_chmod;
fo_chown_t *fo_chown;
fo_sendfile_t *fo_sendfile;
fo_seek_t *fo_seek;
fo_fill_kinfo_t *fo_fill_kinfo;
fo_mmap_t *fo_mmap;
fo_aio_queue_t *fo_aio_queue;
fo_add_seals_t *fo_add_seals;
fo_get_seals_t *fo_get_seals;
fo_fallocate_t *fo_fallocate;
fo_flags_t fo_flags; /* DFLAG_* below */
};
#define DFLAG_PASSABLE 0x01 /* may be passed via unix sockets. */
#define DFLAG_SEEKABLE 0x02 /* seekable / nonsequential */
#endif /* _KERNEL */
#if defined(_KERNEL) || defined(_WANT_FILE)
/*
* Kernel descriptor table.
* One entry for each open kernel vnode and socket.
*
* Below is the list of locks that protects members in struct file.
*
* (a) f_vnode lock required (shared allows both reads and writes)
* (f) updated with atomics and blocking on sleepq
* (d) cdevpriv_mtx
* none not locked
*/
#if __BSD_VISIBLE
struct fadvise_info {
int fa_advice; /* (f) FADV_* type. */
off_t fa_start; /* (f) Region start. */
off_t fa_end; /* (f) Region end. */
};
struct file {
volatile u_int f_flag; /* see fcntl.h */
volatile u_int f_count; /* reference count */
void *f_data; /* file descriptor specific data */
struct fileops *f_ops; /* File operations */
struct vnode *f_vnode; /* NULL or applicable vnode */
struct ucred *f_cred; /* associated credentials. */
short f_type; /* descriptor type */
short f_vnread_flags; /* (f) Sleep lock for f_offset */
/*
* DTYPE_VNODE specific fields.
*/
union {
int16_t f_seqcount[2]; /* (a) Count of seq. reads and writes. */
int f_pipegen;
};
off_t f_nextoff[2]; /* next expected read/write offset. */
union {
struct cdev_privdata *fvn_cdevpriv;
/* (d) Private data for the cdev. */
struct fadvise_info *fvn_advice;
} f_vnun;
/*
* DFLAG_SEEKABLE specific fields
*/
off_t f_offset;
};
#define f_cdevpriv f_vnun.fvn_cdevpriv
#define f_advice f_vnun.fvn_advice
#define FOFFSET_LOCKED 0x1
#define FOFFSET_LOCK_WAITING 0x2
#endif /* __BSD_VISIBLE */
#endif /* _KERNEL || _WANT_FILE */
/*
* Userland version of struct file, for sysctl
*/
#if __BSD_VISIBLE
struct xfile {
ksize_t xf_size; /* size of struct xfile */
pid_t xf_pid; /* owning process */
uid_t xf_uid; /* effective uid of owning process */
int xf_fd; /* descriptor number */
int _xf_int_pad1;
kvaddr_t xf_file; /* address of struct file */
short xf_type; /* descriptor type */
short _xf_short_pad1;
int xf_count; /* reference count */
int xf_msgcount; /* references from message queue */
int _xf_int_pad2;
off_t xf_offset; /* file offset */
kvaddr_t xf_data; /* file descriptor specific data */
kvaddr_t xf_vnode; /* vnode pointer */
u_int xf_flag; /* flags (see fcntl.h) */
int _xf_int_pad3;
int64_t _xf_int64_pad[6];
};
#endif /* __BSD_VISIBLE */
#ifdef _KERNEL
extern struct fileops vnops;
extern struct fileops badfileops;
extern struct fileops path_fileops;
extern struct fileops socketops;
extern int maxfiles; /* kernel limit on number of open files */
extern int maxfilesperproc; /* per process limit on number of open files */
int fget(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp);
int fget_mmap(struct thread *td, int fd, cap_rights_t *rightsp,
vm_prot_t *maxprotp, struct file **fpp);
int fget_read(struct thread *td, int fd, cap_rights_t *rightsp,
struct file **fpp);
int fget_write(struct thread *td, int fd, cap_rights_t *rightsp,
struct file **fpp);
int fget_fcntl(struct thread *td, int fd, cap_rights_t *rightsp,
int needfcntl, struct file **fpp);
int _fdrop(struct file *fp, struct thread *td);
fo_rdwr_t invfo_rdwr;
fo_truncate_t invfo_truncate;
fo_ioctl_t invfo_ioctl;
fo_poll_t invfo_poll;
fo_kqfilter_t invfo_kqfilter;
fo_chmod_t invfo_chmod;
fo_chown_t invfo_chown;
fo_sendfile_t invfo_sendfile;
fo_stat_t vn_statfile;
fo_sendfile_t vn_sendfile;
fo_seek_t vn_seek;
fo_fill_kinfo_t vn_fill_kinfo;
fo_kqfilter_t vn_kqfilter_opath;
int vn_fill_kinfo_vnode(struct vnode *vp, struct kinfo_file *kif);
void finit(struct file *, u_int, short, void *, struct fileops *);
void finit_vnode(struct file *, u_int, void *, struct fileops *);
int fgetvp(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp);
int fgetvp_exec(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp);
int fgetvp_rights(struct thread *td, int fd, cap_rights_t *needrightsp,
struct filecaps *havecaps, struct vnode **vpp);
int fgetvp_read(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp);
int fgetvp_write(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp);
int fgetvp_lookup_smr(int fd, struct nameidata *ndp, struct vnode **vpp, bool *fsearch);
+int fgetvp_lookup(int fd, struct nameidata *ndp, struct vnode **vpp);
static __inline __result_use_check bool
fhold(struct file *fp)
{
return (refcount_acquire_checked(&fp->f_count));
}
#define fdrop(fp, td) ({ \
struct file *_fp; \
int _error; \
\
_error = 0; \
_fp = (fp); \
if (__predict_false(refcount_release(&_fp->f_count))) \
_error = _fdrop(_fp, td); \
_error; \
})
#define fdrop_close(fp, td) ({ \
struct file *_fp; \
int _error; \
\
_error = 0; \
_fp = (fp); \
if (__predict_true(refcount_release(&_fp->f_count))) \
_error = _fdrop(_fp, td); \
_error; \
})
static __inline fo_rdwr_t fo_read;
static __inline fo_rdwr_t fo_write;
static __inline fo_truncate_t fo_truncate;
static __inline fo_ioctl_t fo_ioctl;
static __inline fo_poll_t fo_poll;
static __inline fo_kqfilter_t fo_kqfilter;
static __inline fo_stat_t fo_stat;
static __inline fo_close_t fo_close;
static __inline fo_chmod_t fo_chmod;
static __inline fo_chown_t fo_chown;
static __inline fo_sendfile_t fo_sendfile;
static __inline int
fo_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return ((*fp->f_ops->fo_read)(fp, uio, active_cred, flags, td));
}
static __inline int
fo_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return ((*fp->f_ops->fo_write)(fp, uio, active_cred, flags, td));
}
static __inline int
fo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_truncate)(fp, length, active_cred, td));
}
static __inline int
fo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_ioctl)(fp, com, data, active_cred, td));
}
static __inline int
fo_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_poll)(fp, events, active_cred, td));
}
static __inline int
fo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_stat)(fp, sb, active_cred, td));
}
static __inline int
fo_close(struct file *fp, struct thread *td)
{
return ((*fp->f_ops->fo_close)(fp, td));
}
static __inline int
fo_kqfilter(struct file *fp, struct knote *kn)
{
return ((*fp->f_ops->fo_kqfilter)(fp, kn));
}
static __inline int
fo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_chmod)(fp, mode, active_cred, td));
}
static __inline int
fo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
struct thread *td)
{
return ((*fp->f_ops->fo_chown)(fp, uid, gid, active_cred, td));
}
static __inline int
fo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
return ((*fp->f_ops->fo_sendfile)(fp, sockfd, hdr_uio, trl_uio, offset,
nbytes, sent, flags, td));
}
static __inline int
fo_seek(struct file *fp, off_t offset, int whence, struct thread *td)
{
return ((*fp->f_ops->fo_seek)(fp, offset, whence, td));
}
static __inline int
fo_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
{
return ((*fp->f_ops->fo_fill_kinfo)(fp, kif, fdp));
}
static __inline int
fo_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size,
vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff,
struct thread *td)
{
if (fp->f_ops->fo_mmap == NULL)
return (ENODEV);
return ((*fp->f_ops->fo_mmap)(fp, map, addr, size, prot, cap_maxprot,
flags, foff, td));
}
static __inline int
fo_aio_queue(struct file *fp, struct kaiocb *job)
{
return ((*fp->f_ops->fo_aio_queue)(fp, job));
}
static __inline int
fo_add_seals(struct file *fp, int seals)
{
if (fp->f_ops->fo_add_seals == NULL)
return (EINVAL);
return ((*fp->f_ops->fo_add_seals)(fp, seals));
}
static __inline int
fo_get_seals(struct file *fp, int *seals)
{
if (fp->f_ops->fo_get_seals == NULL)
return (EINVAL);
return ((*fp->f_ops->fo_get_seals)(fp, seals));
}
static __inline int
fo_fallocate(struct file *fp, off_t offset, off_t len, struct thread *td)
{
if (fp->f_ops->fo_fallocate == NULL)
return (ENODEV);
return ((*fp->f_ops->fo_fallocate)(fp, offset, len, td));
}
#endif /* _KERNEL */
#endif /* !SYS_FILE_H */
diff --git a/sys/sys/namei.h b/sys/sys/namei.h
index 9e0a82ea1659..d22864a3c2c8 100644
--- a/sys/sys/namei.h
+++ b/sys/sys/namei.h
@@ -1,331 +1,336 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1985, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)namei.h 8.5 (Berkeley) 1/9/95
* $FreeBSD$
*/
#ifndef _SYS_NAMEI_H_
#define _SYS_NAMEI_H_
#include <sys/caprights.h>
#include <sys/filedesc.h>
#include <sys/queue.h>
#include <sys/_seqc.h>
#include <sys/_uio.h>
enum nameiop { LOOKUP, CREATE, DELETE, RENAME };
struct componentname {
/*
* Arguments to lookup.
*/
u_int64_t cn_origflags; /* flags to namei */
u_int64_t cn_flags; /* flags to namei */
struct thread *cn_thread;/* thread requesting lookup */
struct ucred *cn_cred; /* credentials */
enum nameiop cn_nameiop; /* namei operation */
int cn_lkflags; /* Lock flags LK_EXCLUSIVE or LK_SHARED */
/*
* Shared between lookup and commit routines.
*/
char *cn_pnbuf; /* pathname buffer */
char *cn_nameptr; /* pointer to looked up name */
long cn_namelen; /* length of looked up component */
};
struct nameicap_tracker;
TAILQ_HEAD(nameicap_tracker_head, nameicap_tracker);
/*
* Encapsulation of namei parameters.
*/
struct nameidata {
/*
* Arguments to namei/lookup.
*/
const char *ni_dirp; /* pathname pointer */
enum uio_seg ni_segflg; /* location of pathname */
cap_rights_t *ni_rightsneeded; /* rights required to look up vnode */
/*
* Arguments to lookup.
*/
struct vnode *ni_startdir; /* starting directory */
struct vnode *ni_rootdir; /* logical root directory */
struct vnode *ni_topdir; /* logical top directory */
int ni_dirfd; /* starting directory for *at functions */
int ni_lcf; /* local call flags */
/*
* Results: returned from namei
*/
struct filecaps ni_filecaps; /* rights the *at base has */
/*
* Results: returned from/manipulated by lookup
*/
struct vnode *ni_vp; /* vnode of result */
struct vnode *ni_dvp; /* vnode of intermediate directory */
/*
* Results: flags returned from namei
*/
u_int ni_resflags;
/*
* Debug for validating API use by the callers.
*/
u_short ni_debugflags;
/*
* Shared between namei and lookup/commit routines.
*/
u_short ni_loopcnt; /* count of symlinks encountered */
size_t ni_pathlen; /* remaining chars in path */
char *ni_next; /* next location in pathname */
/*
* Lookup parameters: this structure describes the subset of
* information from the nameidata structure that is passed
* through the VOP interface.
*/
struct componentname ni_cnd;
struct nameicap_tracker_head ni_cap_tracker;
/*
* Private helper data for UFS, must be at the end. See
* NDINIT_PREFILL().
*/
seqc_t ni_dvp_seqc;
seqc_t ni_vp_seqc;
};
#ifdef _KERNEL
enum cache_fpl_status { CACHE_FPL_STATUS_DESTROYED, CACHE_FPL_STATUS_ABORTED,
CACHE_FPL_STATUS_PARTIAL, CACHE_FPL_STATUS_HANDLED, CACHE_FPL_STATUS_UNSET };
int cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status,
struct pwd **pwdp);
/*
* Flags for namei.
*
* If modifying the list make sure to check whether NDVALIDATE needs updating.
*/
/*
* Debug.
*/
#define NAMEI_DBG_INITED 0x0001
#define NAMEI_DBG_CALLED 0x0002
#define NAMEI_DBG_HADSTARTDIR 0x0004
/*
* namei operational modifier flags, stored in ni_cnd.flags
*/
#define NC_NOMAKEENTRY 0x0001 /* name must not be added to cache */
#define NC_KEEPPOSENTRY 0x0002 /* don't evict a positive entry */
#define NOCACHE NC_NOMAKEENTRY /* for compatibility with older code */
#define LOCKLEAF 0x0004 /* lock vnode on return */
#define LOCKPARENT 0x0008 /* want parent vnode returned locked */
#define WANTPARENT 0x0010 /* want parent vnode returned unlocked */
#define FAILIFEXISTS 0x0020 /* return EEXIST if found */
#define FOLLOW 0x0040 /* follow symbolic links */
#define EMPTYPATH 0x0080 /* Allow empty path for *at */
#define LOCKSHARED 0x0100 /* Shared lock leaf */
#define NOFOLLOW 0x0000 /* do not follow symbolic links (pseudo) */
#define RBENEATH 0x100000000ULL /* No escape, even tmp, from start dir */
#define MODMASK 0xf000001ffULL /* mask of operational modifiers */
/*
* Namei parameter descriptors.
*
* SAVENAME may be set by either the callers of namei or by VOP_LOOKUP.
* If the caller of namei sets the flag (for example execve wants to
* know the name of the program that is being executed), then it must
* free the buffer. If VOP_LOOKUP sets the flag, then the buffer must
* be freed by either the commit routine or the VOP_ABORT routine.
* SAVESTART is set only by the callers of namei. It implies SAVENAME
* plus the addition of saving the parent directory that contains the
* name in ni_startdir. It allows repeated calls to lookup for the
* name being sought. The caller is responsible for releasing the
* buffer and for vrele'ing ni_startdir.
*/
#define RDONLY 0x00000200 /* lookup with read-only semantics */
#define SAVENAME 0x00000400 /* save pathname buffer */
#define SAVESTART 0x00000800 /* save starting directory */
#define ISWHITEOUT 0x00001000 /* found whiteout */
#define DOWHITEOUT 0x00002000 /* do whiteouts */
#define WILLBEDIR 0x00004000 /* new files will be dirs; allow trailing / */
#define ISOPEN 0x00008000 /* caller is opening; return a real vnode. */
#define NOCROSSMOUNT 0x00010000 /* do not cross mount points */
#define NOMACCHECK 0x00020000 /* do not perform MAC checks */
#define AUDITVNODE1 0x00040000 /* audit the looked up vnode information */
#define AUDITVNODE2 0x00080000 /* audit the looked up vnode information */
#define NOCAPCHECK 0x00100000 /* do not perform capability checks */
/* UNUSED 0x00200000 */
/* UNUSED 0x00400000 */
-/* UNUSED 0x00800000 */
+#define WANTIOCTLCAPS 0x00800000 /* leave ioctl caps for the caller */
#define HASBUF 0x01000000 /* has allocated pathname buffer */
#define NOEXECCHECK 0x02000000 /* do not perform exec check on dir */
#define MAKEENTRY 0x04000000 /* entry is to be added to name cache */
#define ISSYMLINK 0x08000000 /* symlink needs interpretation */
#define ISLASTCN 0x10000000 /* this is last component of pathname */
#define ISDOTDOT 0x20000000 /* current component name is .. */
#define TRAILINGSLASH 0x40000000 /* path ended in a slash */
#define PARAMASK 0x7ffffe00 /* mask of parameter descriptors */
/*
* Flags which must not be passed in by callers.
*/
#define NAMEI_INTERNAL_FLAGS \
(HASBUF | NOEXECCHECK | MAKEENTRY | ISSYMLINK | ISLASTCN | ISDOTDOT | \
TRAILINGSLASH)
/*
* Namei results flags
*/
#define NIRES_ABS 0x00000001 /* Path was absolute */
#define NIRES_STRICTREL 0x00000002 /* Restricted lookup result */
#define NIRES_EMPTYPATH 0x00000004 /* EMPTYPATH used */
/*
* Flags in ni_lcf, valid for the duration of the namei call.
*/
#define NI_LCF_STRICTRELATIVE 0x0001 /* relative lookup only */
#define NI_LCF_CAP_DOTDOT 0x0002 /* ".." in strictrelative case */
/*
* Initialization of a nameidata structure.
*/
#define NDINIT(ndp, op, flags, segflg, namep, td) \
NDINIT_ALL(ndp, op, flags, segflg, namep, AT_FDCWD, NULL, &cap_no_rights, td)
#define NDINIT_AT(ndp, op, flags, segflg, namep, dirfd, td) \
NDINIT_ALL(ndp, op, flags, segflg, namep, dirfd, NULL, &cap_no_rights, td)
#define NDINIT_ATRIGHTS(ndp, op, flags, segflg, namep, dirfd, rightsp, td) \
NDINIT_ALL(ndp, op, flags, segflg, namep, dirfd, NULL, rightsp, td)
#define NDINIT_ATVP(ndp, op, flags, segflg, namep, vp, td) \
NDINIT_ALL(ndp, op, flags, segflg, namep, AT_FDCWD, vp, &cap_no_rights, td)
/*
* Note the constant pattern may *hide* bugs.
*/
#ifdef INVARIANTS
#define NDINIT_PREFILL(arg) memset(arg, 0xff, offsetof(struct nameidata, \
ni_dvp_seqc))
#define NDINIT_DBG(arg) { (arg)->ni_debugflags = NAMEI_DBG_INITED; }
#define NDREINIT_DBG(arg) { \
if (((arg)->ni_debugflags & NAMEI_DBG_INITED) == 0) \
panic("namei data not inited"); \
if (((arg)->ni_debugflags & NAMEI_DBG_HADSTARTDIR) != 0) \
panic("NDREINIT on namei data with NAMEI_DBG_HADSTARTDIR"); \
(arg)->ni_debugflags = NAMEI_DBG_INITED; \
}
#else
#define NDINIT_PREFILL(arg) do { } while (0)
#define NDINIT_DBG(arg) do { } while (0)
#define NDREINIT_DBG(arg) do { } while (0)
#endif
#define NDINIT_ALL(ndp, op, flags, segflg, namep, dirfd, startdir, rightsp, td) \
do { \
struct nameidata *_ndp = (ndp); \
cap_rights_t *_rightsp = (rightsp); \
MPASS(_rightsp != NULL); \
NDINIT_PREFILL(_ndp); \
NDINIT_DBG(_ndp); \
_ndp->ni_cnd.cn_nameiop = op; \
_ndp->ni_cnd.cn_flags = flags; \
_ndp->ni_segflg = segflg; \
_ndp->ni_dirp = namep; \
_ndp->ni_dirfd = dirfd; \
_ndp->ni_startdir = startdir; \
_ndp->ni_resflags = 0; \
filecaps_init(&_ndp->ni_filecaps); \
_ndp->ni_cnd.cn_thread = td; \
_ndp->ni_rightsneeded = _rightsp; \
} while (0)
#define NDREINIT(ndp) do { \
struct nameidata *_ndp = (ndp); \
NDREINIT_DBG(_ndp); \
+ filecaps_free(&_ndp->ni_filecaps); \
_ndp->ni_resflags = 0; \
_ndp->ni_startdir = NULL; \
} while (0)
#define NDPREINIT(ndp) do { \
(ndp)->ni_dvp_seqc = SEQC_MOD; \
(ndp)->ni_vp_seqc = SEQC_MOD; \
} while (0)
#define NDF_NO_DVP_RELE 0x00000001
#define NDF_NO_DVP_UNLOCK 0x00000002
#define NDF_NO_DVP_PUT 0x00000003
#define NDF_NO_VP_RELE 0x00000004
#define NDF_NO_VP_UNLOCK 0x00000008
#define NDF_NO_VP_PUT 0x0000000c
#define NDF_NO_STARTDIR_RELE 0x00000010
#define NDF_NO_FREE_PNBUF 0x00000020
#define NDF_ONLY_PNBUF (~NDF_NO_FREE_PNBUF)
+#define NDFREE_IOCTLCAPS(ndp) do { \
+ struct nameidata *_ndp = (ndp); \
+ filecaps_free(&_ndp->ni_filecaps); \
+} while (0)
void NDFREE_PNBUF(struct nameidata *);
void NDFREE(struct nameidata *, const u_int);
#define NDFREE(ndp, flags) do { \
struct nameidata *_ndp = (ndp); \
if (__builtin_constant_p(flags) && flags == NDF_ONLY_PNBUF) \
NDFREE_PNBUF(_ndp); \
else \
NDFREE(_ndp, flags); \
} while (0)
#ifdef INVARIANTS
void NDFREE_NOTHING(struct nameidata *);
void NDVALIDATE(struct nameidata *);
#else
#define NDFREE_NOTHING(ndp) do { } while (0)
#define NDVALIDATE(ndp) do { } while (0)
#endif
int namei(struct nameidata *ndp);
int lookup(struct nameidata *ndp);
int relookup(struct vnode *dvp, struct vnode **vpp,
struct componentname *cnp);
#endif
/*
* Stats on usefulness of namei caches.
*/
struct nchstats {
long ncs_goodhits; /* hits that we can really use */
long ncs_neghits; /* negative hits that we can use */
long ncs_badhits; /* hits we must drop */
long ncs_falsehits; /* hits with id mismatch */
long ncs_miss; /* misses */
long ncs_long; /* long names that ignore cache */
long ncs_pass2; /* names found with passes == 2 */
long ncs_2passes; /* number of times we attempt it */
};
extern struct nchstats nchstats;
#endif /* !_SYS_NAMEI_H_ */