Index: head/lib/libprocstat/libprocstat.c
===================================================================
--- head/lib/libprocstat/libprocstat.c (revision 358502)
+++ head/lib/libprocstat/libprocstat.c (revision 358503)
@@ -1,2618 +1,2631 @@
/*-
* SPDX-License-Identifier: BSD-4-Clause
*
* Copyright (c) 2017 Dell EMC
* Copyright (c) 2009 Stanislav Sedov <stas@FreeBSD.org>
* Copyright (c) 1988, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/elf.h>
#include <sys/time.h>
#include <sys/resourcevar.h>
#define _WANT_UCRED
#include <sys/ucred.h>
#undef _WANT_UCRED
#include <sys/proc.h>
#include <sys/user.h>
#include <sys/stat.h>
#include <sys/vnode.h>
#include <sys/socket.h>
#define _WANT_SOCKET
#include <sys/socketvar.h>
#include <sys/domain.h>
#include <sys/protosw.h>
#include <sys/un.h>
#define _WANT_UNPCB
#include <sys/unpcb.h>
#include <sys/sysctl.h>
#include <sys/tty.h>
#include <sys/filedesc.h>
#include <sys/queue.h>
#define _WANT_FILE
#include <sys/file.h>
#include <sys/conf.h>
#include <sys/ksem.h>
#include <sys/mman.h>
#include <sys/capsicum.h>
#include <sys/ptrace.h>
#define _KERNEL
#include <sys/mount.h>
#include <sys/pipe.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <fs/devfs/devfs.h>
#include <fs/devfs/devfs_int.h>
#undef _KERNEL
#include <nfs/nfsproto.h>
#include <nfsclient/nfs.h>
#include <nfsclient/nfsnode.h>
#include <vm/vm.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#define _WANT_INPCB
#include <netinet/in_pcb.h>
#include <assert.h>
#include <ctype.h>
#include <err.h>
#include <fcntl.h>
#include <kvm.h>
#include <libutil.h>
#include <limits.h>
#include <paths.h>
#include <pwd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>
#include <unistd.h>
#include <netdb.h>
#include <libprocstat.h>
#include "libprocstat_internal.h"
#include "common_kvm.h"
#include "core.h"
int statfs(const char *, struct statfs *); /* XXX */
#define PROCSTAT_KVM 1
#define PROCSTAT_SYSCTL 2
#define PROCSTAT_CORE 3
static char **getargv(struct procstat *procstat, struct kinfo_proc *kp,
size_t nchr, int env);
static char *getmnton(kvm_t *kd, struct mount *m);
static struct kinfo_vmentry * kinfo_getvmmap_core(struct procstat_core *core,
int *cntp);
static Elf_Auxinfo *procstat_getauxv_core(struct procstat_core *core,
unsigned int *cntp);
static Elf_Auxinfo *procstat_getauxv_sysctl(pid_t pid, unsigned int *cntp);
static struct filestat_list *procstat_getfiles_kvm(
struct procstat *procstat, struct kinfo_proc *kp, int mmapped);
static struct filestat_list *procstat_getfiles_sysctl(
struct procstat *procstat, struct kinfo_proc *kp, int mmapped);
static int procstat_get_pipe_info_sysctl(struct filestat *fst,
struct pipestat *pipe, char *errbuf);
static int procstat_get_pipe_info_kvm(kvm_t *kd, struct filestat *fst,
struct pipestat *pipe, char *errbuf);
static int procstat_get_pts_info_sysctl(struct filestat *fst,
struct ptsstat *pts, char *errbuf);
static int procstat_get_pts_info_kvm(kvm_t *kd, struct filestat *fst,
struct ptsstat *pts, char *errbuf);
static int procstat_get_sem_info_sysctl(struct filestat *fst,
struct semstat *sem, char *errbuf);
static int procstat_get_sem_info_kvm(kvm_t *kd, struct filestat *fst,
struct semstat *sem, char *errbuf);
static int procstat_get_shm_info_sysctl(struct filestat *fst,
struct shmstat *shm, char *errbuf);
static int procstat_get_shm_info_kvm(kvm_t *kd, struct filestat *fst,
struct shmstat *shm, char *errbuf);
static int procstat_get_socket_info_sysctl(struct filestat *fst,
struct sockstat *sock, char *errbuf);
static int procstat_get_socket_info_kvm(kvm_t *kd, struct filestat *fst,
struct sockstat *sock, char *errbuf);
static int to_filestat_flags(int flags);
static int procstat_get_vnode_info_kvm(kvm_t *kd, struct filestat *fst,
struct vnstat *vn, char *errbuf);
static int procstat_get_vnode_info_sysctl(struct filestat *fst,
struct vnstat *vn, char *errbuf);
static gid_t *procstat_getgroups_core(struct procstat_core *core,
unsigned int *count);
static gid_t * procstat_getgroups_kvm(kvm_t *kd, struct kinfo_proc *kp,
unsigned int *count);
static gid_t *procstat_getgroups_sysctl(pid_t pid, unsigned int *count);
static struct kinfo_kstack *procstat_getkstack_sysctl(pid_t pid,
int *cntp);
static int procstat_getosrel_core(struct procstat_core *core,
int *osrelp);
static int procstat_getosrel_kvm(kvm_t *kd, struct kinfo_proc *kp,
int *osrelp);
static int procstat_getosrel_sysctl(pid_t pid, int *osrelp);
static int procstat_getpathname_core(struct procstat_core *core,
char *pathname, size_t maxlen);
static int procstat_getpathname_sysctl(pid_t pid, char *pathname,
size_t maxlen);
static int procstat_getrlimit_core(struct procstat_core *core, int which,
struct rlimit* rlimit);
static int procstat_getrlimit_kvm(kvm_t *kd, struct kinfo_proc *kp,
int which, struct rlimit* rlimit);
static int procstat_getrlimit_sysctl(pid_t pid, int which,
struct rlimit* rlimit);
static int procstat_getumask_core(struct procstat_core *core,
unsigned short *maskp);
static int procstat_getumask_kvm(kvm_t *kd, struct kinfo_proc *kp,
unsigned short *maskp);
static int procstat_getumask_sysctl(pid_t pid, unsigned short *maskp);
static int vntype2psfsttype(int type);
void
procstat_close(struct procstat *procstat)
{
assert(procstat);
if (procstat->type == PROCSTAT_KVM)
kvm_close(procstat->kd);
else if (procstat->type == PROCSTAT_CORE)
procstat_core_close(procstat->core);
procstat_freeargv(procstat);
procstat_freeenvv(procstat);
free(procstat);
}
struct procstat *
procstat_open_sysctl(void)
{
struct procstat *procstat;
procstat = calloc(1, sizeof(*procstat));
if (procstat == NULL) {
warn("malloc()");
return (NULL);
}
procstat->type = PROCSTAT_SYSCTL;
return (procstat);
}
struct procstat *
procstat_open_kvm(const char *nlistf, const char *memf)
{
struct procstat *procstat;
kvm_t *kd;
char buf[_POSIX2_LINE_MAX];
procstat = calloc(1, sizeof(*procstat));
if (procstat == NULL) {
warn("malloc()");
return (NULL);
}
kd = kvm_openfiles(nlistf, memf, NULL, O_RDONLY, buf);
if (kd == NULL) {
warnx("kvm_openfiles(): %s", buf);
free(procstat);
return (NULL);
}
procstat->type = PROCSTAT_KVM;
procstat->kd = kd;
return (procstat);
}
struct procstat *
procstat_open_core(const char *filename)
{
struct procstat *procstat;
struct procstat_core *core;
procstat = calloc(1, sizeof(*procstat));
if (procstat == NULL) {
warn("malloc()");
return (NULL);
}
core = procstat_core_open(filename);
if (core == NULL) {
free(procstat);
return (NULL);
}
procstat->type = PROCSTAT_CORE;
procstat->core = core;
return (procstat);
}
struct kinfo_proc *
procstat_getprocs(struct procstat *procstat, int what, int arg,
unsigned int *count)
{
struct kinfo_proc *p0, *p;
size_t len, olen;
int name[4];
int cnt;
int error;
assert(procstat);
assert(count);
p = NULL;
if (procstat->type == PROCSTAT_KVM) {
*count = 0;
p0 = kvm_getprocs(procstat->kd, what, arg, &cnt);
if (p0 == NULL || cnt <= 0)
return (NULL);
*count = cnt;
len = *count * sizeof(*p);
p = malloc(len);
if (p == NULL) {
warnx("malloc(%zu)", len);
goto fail;
}
bcopy(p0, p, len);
return (p);
} else if (procstat->type == PROCSTAT_SYSCTL) {
len = 0;
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = what;
name[3] = arg;
error = sysctl(name, nitems(name), NULL, &len, NULL, 0);
if (error < 0 && errno != EPERM) {
warn("sysctl(kern.proc)");
goto fail;
}
if (len == 0) {
warnx("no processes?");
goto fail;
}
do {
len += len / 10;
p = reallocf(p, len);
if (p == NULL) {
warnx("reallocf(%zu)", len);
goto fail;
}
olen = len;
error = sysctl(name, nitems(name), p, &len, NULL, 0);
} while (error < 0 && errno == ENOMEM && olen == len);
if (error < 0 && errno != EPERM) {
warn("sysctl(kern.proc)");
goto fail;
}
/* Perform simple consistency checks. */
if ((len % sizeof(*p)) != 0 || p->ki_structsize != sizeof(*p)) {
warnx("kinfo_proc structure size mismatch (len = %zu)", len);
goto fail;
}
*count = len / sizeof(*p);
return (p);
} else if (procstat->type == PROCSTAT_CORE) {
p = procstat_core_get(procstat->core, PSC_TYPE_PROC, NULL,
&len);
if ((len % sizeof(*p)) != 0 || p->ki_structsize != sizeof(*p)) {
warnx("kinfo_proc structure size mismatch");
goto fail;
}
*count = len / sizeof(*p);
return (p);
} else {
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
fail:
if (p)
free(p);
return (NULL);
}
void
procstat_freeprocs(struct procstat *procstat __unused, struct kinfo_proc *p)
{
if (p != NULL)
free(p);
p = NULL;
}
struct filestat_list *
procstat_getfiles(struct procstat *procstat, struct kinfo_proc *kp, int mmapped)
{
switch(procstat->type) {
case PROCSTAT_KVM:
return (procstat_getfiles_kvm(procstat, kp, mmapped));
case PROCSTAT_SYSCTL:
case PROCSTAT_CORE:
return (procstat_getfiles_sysctl(procstat, kp, mmapped));
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freefiles(struct procstat *procstat, struct filestat_list *head)
{
struct filestat *fst, *tmp;
STAILQ_FOREACH_SAFE(fst, head, next, tmp) {
if (fst->fs_path != NULL)
free(fst->fs_path);
free(fst);
}
free(head);
if (procstat->vmentries != NULL) {
free(procstat->vmentries);
procstat->vmentries = NULL;
}
if (procstat->files != NULL) {
free(procstat->files);
procstat->files = NULL;
}
}
static struct filestat *
filestat_new_entry(void *typedep, int type, int fd, int fflags, int uflags,
int refcount, off_t offset, char *path, cap_rights_t *cap_rightsp)
{
struct filestat *entry;
entry = calloc(1, sizeof(*entry));
if (entry == NULL) {
warn("malloc()");
return (NULL);
}
entry->fs_typedep = typedep;
entry->fs_fflags = fflags;
entry->fs_uflags = uflags;
entry->fs_fd = fd;
entry->fs_type = type;
entry->fs_ref_count = refcount;
entry->fs_offset = offset;
entry->fs_path = path;
if (cap_rightsp != NULL)
entry->fs_cap_rights = *cap_rightsp;
else
cap_rights_init(&entry->fs_cap_rights);
return (entry);
}
static struct vnode *
getctty(kvm_t *kd, struct kinfo_proc *kp)
{
struct pgrp pgrp;
struct proc proc;
struct session sess;
int error;
assert(kp);
error = kvm_read_all(kd, (unsigned long)kp->ki_paddr, &proc,
sizeof(proc));
if (error == 0) {
warnx("can't read proc struct at %p for pid %d",
kp->ki_paddr, kp->ki_pid);
return (NULL);
}
if (proc.p_pgrp == NULL)
return (NULL);
error = kvm_read_all(kd, (unsigned long)proc.p_pgrp, &pgrp,
sizeof(pgrp));
if (error == 0) {
warnx("can't read pgrp struct at %p for pid %d",
proc.p_pgrp, kp->ki_pid);
return (NULL);
}
error = kvm_read_all(kd, (unsigned long)pgrp.pg_session, &sess,
sizeof(sess));
if (error == 0) {
warnx("can't read session struct at %p for pid %d",
pgrp.pg_session, kp->ki_pid);
return (NULL);
}
return (sess.s_ttyvp);
}
static int
procstat_vm_map_reader(void *token, vm_map_entry_t addr, vm_map_entry_t dest)
{
kvm_t *kd;
kd = (kvm_t *)token;
return (kvm_read_all(kd, (unsigned long)addr, dest, sizeof(*dest)));
}
static struct filestat_list *
procstat_getfiles_kvm(struct procstat *procstat, struct kinfo_proc *kp, int mmapped)
{
struct file file;
struct filedesc filed;
+ struct pwd pwd;
struct vm_map_entry vmentry;
struct vm_object object;
struct vmspace vmspace;
vm_map_entry_t entryp;
vm_object_t objp;
struct vnode *vp;
struct file **ofiles;
struct filestat *entry;
struct filestat_list *head;
kvm_t *kd;
void *data;
int i, fflags;
int prot, type;
unsigned int nfiles;
+ bool haspwd;
assert(procstat);
kd = procstat->kd;
if (kd == NULL)
return (NULL);
if (kp->ki_fd == NULL)
return (NULL);
if (!kvm_read_all(kd, (unsigned long)kp->ki_fd, &filed,
sizeof(filed))) {
warnx("can't read filedesc at %p", (void *)kp->ki_fd);
return (NULL);
}
+ haspwd = false;
+ if (filed.fd_pwd != NULL) {
+ if (!kvm_read_all(kd, (unsigned long)filed.fd_pwd, &pwd,
+ sizeof(pwd))) {
+ warnx("can't read fd_pwd at %p", (void *)filed.fd_pwd);
+ return (NULL);
+ }
+ haspwd = true;
+ }
/*
* Allocate list head.
*/
head = malloc(sizeof(*head));
if (head == NULL)
return (NULL);
STAILQ_INIT(head);
/* root directory vnode, if one. */
- if (filed.fd_rdir) {
- entry = filestat_new_entry(filed.fd_rdir, PS_FST_TYPE_VNODE, -1,
- PS_FST_FFLAG_READ, PS_FST_UFLAG_RDIR, 0, 0, NULL, NULL);
- if (entry != NULL)
- STAILQ_INSERT_TAIL(head, entry, next);
- }
- /* current working directory vnode. */
- if (filed.fd_cdir) {
- entry = filestat_new_entry(filed.fd_cdir, PS_FST_TYPE_VNODE, -1,
- PS_FST_FFLAG_READ, PS_FST_UFLAG_CDIR, 0, 0, NULL, NULL);
- if (entry != NULL)
- STAILQ_INSERT_TAIL(head, entry, next);
- }
- /* jail root, if any. */
- if (filed.fd_jdir) {
- entry = filestat_new_entry(filed.fd_jdir, PS_FST_TYPE_VNODE, -1,
- PS_FST_FFLAG_READ, PS_FST_UFLAG_JAIL, 0, 0, NULL, NULL);
- if (entry != NULL)
- STAILQ_INSERT_TAIL(head, entry, next);
+ if (haspwd) {
+ if (pwd.pwd_rdir) {
+ entry = filestat_new_entry(pwd.pwd_rdir, PS_FST_TYPE_VNODE, -1,
+ PS_FST_FFLAG_READ, PS_FST_UFLAG_RDIR, 0, 0, NULL, NULL);
+ if (entry != NULL)
+ STAILQ_INSERT_TAIL(head, entry, next);
+ }
+ /* current working directory vnode. */
+ if (pwd.pwd_cdir) {
+ entry = filestat_new_entry(pwd.pwd_cdir, PS_FST_TYPE_VNODE, -1,
+ PS_FST_FFLAG_READ, PS_FST_UFLAG_CDIR, 0, 0, NULL, NULL);
+ if (entry != NULL)
+ STAILQ_INSERT_TAIL(head, entry, next);
+ }
+ /* jail root, if any. */
+ if (pwd.pwd_jdir) {
+ entry = filestat_new_entry(pwd.pwd_jdir, PS_FST_TYPE_VNODE, -1,
+ PS_FST_FFLAG_READ, PS_FST_UFLAG_JAIL, 0, 0, NULL, NULL);
+ if (entry != NULL)
+ STAILQ_INSERT_TAIL(head, entry, next);
+ }
}
/* ktrace vnode, if one */
if (kp->ki_tracep) {
entry = filestat_new_entry(kp->ki_tracep, PS_FST_TYPE_VNODE, -1,
PS_FST_FFLAG_READ | PS_FST_FFLAG_WRITE,
PS_FST_UFLAG_TRACE, 0, 0, NULL, NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
/* text vnode, if one */
if (kp->ki_textvp) {
entry = filestat_new_entry(kp->ki_textvp, PS_FST_TYPE_VNODE, -1,
PS_FST_FFLAG_READ, PS_FST_UFLAG_TEXT, 0, 0, NULL, NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
/* Controlling terminal. */
if ((vp = getctty(kd, kp)) != NULL) {
entry = filestat_new_entry(vp, PS_FST_TYPE_VNODE, -1,
PS_FST_FFLAG_READ | PS_FST_FFLAG_WRITE,
PS_FST_UFLAG_CTTY, 0, 0, NULL, NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
nfiles = filed.fd_lastfile + 1;
ofiles = malloc(nfiles * sizeof(struct file *));
if (ofiles == NULL) {
warn("malloc(%zu)", nfiles * sizeof(struct file *));
goto do_mmapped;
}
if (!kvm_read_all(kd, (unsigned long)filed.fd_ofiles, ofiles,
nfiles * sizeof(struct file *))) {
warnx("cannot read file structures at %p",
(void *)filed.fd_ofiles);
free(ofiles);
goto do_mmapped;
}
for (i = 0; i <= filed.fd_lastfile; i++) {
if (ofiles[i] == NULL)
continue;
if (!kvm_read_all(kd, (unsigned long)ofiles[i], &file,
sizeof(struct file))) {
warnx("can't read file %d at %p", i,
(void *)ofiles[i]);
continue;
}
switch (file.f_type) {
case DTYPE_VNODE:
type = PS_FST_TYPE_VNODE;
data = file.f_vnode;
break;
case DTYPE_SOCKET:
type = PS_FST_TYPE_SOCKET;
data = file.f_data;
break;
case DTYPE_PIPE:
type = PS_FST_TYPE_PIPE;
data = file.f_data;
break;
case DTYPE_FIFO:
type = PS_FST_TYPE_FIFO;
data = file.f_vnode;
break;
#ifdef DTYPE_PTS
case DTYPE_PTS:
type = PS_FST_TYPE_PTS;
data = file.f_data;
break;
#endif
case DTYPE_SEM:
type = PS_FST_TYPE_SEM;
data = file.f_data;
break;
case DTYPE_SHM:
type = PS_FST_TYPE_SHM;
data = file.f_data;
break;
case DTYPE_PROCDESC:
type = PS_FST_TYPE_PROCDESC;
data = file.f_data;
break;
case DTYPE_DEV:
type = PS_FST_TYPE_DEV;
data = file.f_data;
break;
default:
continue;
}
/* XXXRW: No capability rights support for kvm yet. */
entry = filestat_new_entry(data, type, i,
to_filestat_flags(file.f_flag), 0, 0, 0, NULL, NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
free(ofiles);
do_mmapped:
/*
* Process mmapped files if requested.
*/
if (mmapped) {
if (!kvm_read_all(kd, (unsigned long)kp->ki_vmspace, &vmspace,
sizeof(vmspace))) {
warnx("can't read vmspace at %p",
(void *)kp->ki_vmspace);
goto exit;
}
vmentry = vmspace.vm_map.header;
for (entryp = vm_map_entry_read_succ(kd, &vmentry, procstat_vm_map_reader);
entryp != NULL && entryp != &kp->ki_vmspace->vm_map.header;
entryp = vm_map_entry_read_succ(kd, &vmentry, procstat_vm_map_reader)) {
if (vmentry.eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
if ((objp = vmentry.object.vm_object) == NULL)
continue;
for (; objp; objp = object.backing_object) {
if (!kvm_read_all(kd, (unsigned long)objp,
&object, sizeof(object))) {
warnx("can't read vm_object at %p",
(void *)objp);
break;
}
}
/* We want only vnode objects. */
if (object.type != OBJT_VNODE)
continue;
prot = vmentry.protection;
fflags = 0;
if (prot & VM_PROT_READ)
fflags = PS_FST_FFLAG_READ;
if ((vmentry.eflags & MAP_ENTRY_COW) == 0 &&
prot & VM_PROT_WRITE)
fflags |= PS_FST_FFLAG_WRITE;
/*
* Create filestat entry.
*/
entry = filestat_new_entry(object.handle,
PS_FST_TYPE_VNODE, -1, fflags,
PS_FST_UFLAG_MMAP, 0, 0, NULL, NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
if (entryp == NULL)
warnx("can't read vm_map_entry");
}
exit:
return (head);
}
/*
* kinfo types to filestat translation.
*/
static int
kinfo_type2fst(int kftype)
{
static struct {
int kf_type;
int fst_type;
} kftypes2fst[] = {
{ KF_TYPE_PROCDESC, PS_FST_TYPE_PROCDESC },
{ KF_TYPE_CRYPTO, PS_FST_TYPE_CRYPTO },
{ KF_TYPE_DEV, PS_FST_TYPE_DEV },
{ KF_TYPE_FIFO, PS_FST_TYPE_FIFO },
{ KF_TYPE_KQUEUE, PS_FST_TYPE_KQUEUE },
{ KF_TYPE_MQUEUE, PS_FST_TYPE_MQUEUE },
{ KF_TYPE_NONE, PS_FST_TYPE_NONE },
{ KF_TYPE_PIPE, PS_FST_TYPE_PIPE },
{ KF_TYPE_PTS, PS_FST_TYPE_PTS },
{ KF_TYPE_SEM, PS_FST_TYPE_SEM },
{ KF_TYPE_SHM, PS_FST_TYPE_SHM },
{ KF_TYPE_SOCKET, PS_FST_TYPE_SOCKET },
{ KF_TYPE_VNODE, PS_FST_TYPE_VNODE },
{ KF_TYPE_UNKNOWN, PS_FST_TYPE_UNKNOWN }
};
#define NKFTYPES (sizeof(kftypes2fst) / sizeof(*kftypes2fst))
unsigned int i;
for (i = 0; i < NKFTYPES; i++)
if (kftypes2fst[i].kf_type == kftype)
break;
if (i == NKFTYPES)
return (PS_FST_TYPE_UNKNOWN);
return (kftypes2fst[i].fst_type);
}
/*
* kinfo flags to filestat translation.
*/
static int
kinfo_fflags2fst(int kfflags)
{
static struct {
int kf_flag;
int fst_flag;
} kfflags2fst[] = {
{ KF_FLAG_APPEND, PS_FST_FFLAG_APPEND },
{ KF_FLAG_ASYNC, PS_FST_FFLAG_ASYNC },
{ KF_FLAG_CREAT, PS_FST_FFLAG_CREAT },
{ KF_FLAG_DIRECT, PS_FST_FFLAG_DIRECT },
{ KF_FLAG_EXCL, PS_FST_FFLAG_EXCL },
{ KF_FLAG_EXEC, PS_FST_FFLAG_EXEC },
{ KF_FLAG_EXLOCK, PS_FST_FFLAG_EXLOCK },
{ KF_FLAG_FSYNC, PS_FST_FFLAG_SYNC },
{ KF_FLAG_HASLOCK, PS_FST_FFLAG_HASLOCK },
{ KF_FLAG_NOFOLLOW, PS_FST_FFLAG_NOFOLLOW },
{ KF_FLAG_NONBLOCK, PS_FST_FFLAG_NONBLOCK },
{ KF_FLAG_READ, PS_FST_FFLAG_READ },
{ KF_FLAG_SHLOCK, PS_FST_FFLAG_SHLOCK },
{ KF_FLAG_TRUNC, PS_FST_FFLAG_TRUNC },
{ KF_FLAG_WRITE, PS_FST_FFLAG_WRITE }
};
#define NKFFLAGS (sizeof(kfflags2fst) / sizeof(*kfflags2fst))
unsigned int i;
int flags;
flags = 0;
for (i = 0; i < NKFFLAGS; i++)
if ((kfflags & kfflags2fst[i].kf_flag) != 0)
flags |= kfflags2fst[i].fst_flag;
return (flags);
}
static int
kinfo_uflags2fst(int fd)
{
switch (fd) {
case KF_FD_TYPE_CTTY:
return (PS_FST_UFLAG_CTTY);
case KF_FD_TYPE_CWD:
return (PS_FST_UFLAG_CDIR);
case KF_FD_TYPE_JAIL:
return (PS_FST_UFLAG_JAIL);
case KF_FD_TYPE_TEXT:
return (PS_FST_UFLAG_TEXT);
case KF_FD_TYPE_TRACE:
return (PS_FST_UFLAG_TRACE);
case KF_FD_TYPE_ROOT:
return (PS_FST_UFLAG_RDIR);
}
return (0);
}
static struct kinfo_file *
kinfo_getfile_core(struct procstat_core *core, int *cntp)
{
int cnt;
size_t len;
char *buf, *bp, *eb;
struct kinfo_file *kif, *kp, *kf;
buf = procstat_core_get(core, PSC_TYPE_FILES, NULL, &len);
if (buf == NULL)
return (NULL);
/*
* XXXMG: The code below is just copy&past from libutil.
* The code duplication can be avoided if libutil
* is extended to provide something like:
* struct kinfo_file *kinfo_getfile_from_buf(const char *buf,
* size_t len, int *cntp);
*/
/* Pass 1: count items */
cnt = 0;
bp = buf;
eb = buf + len;
while (bp < eb) {
kf = (struct kinfo_file *)(uintptr_t)bp;
if (kf->kf_structsize == 0)
break;
bp += kf->kf_structsize;
cnt++;
}
kif = calloc(cnt, sizeof(*kif));
if (kif == NULL) {
free(buf);
return (NULL);
}
bp = buf;
eb = buf + len;
kp = kif;
/* Pass 2: unpack */
while (bp < eb) {
kf = (struct kinfo_file *)(uintptr_t)bp;
if (kf->kf_structsize == 0)
break;
/* Copy/expand into pre-zeroed buffer */
memcpy(kp, kf, kf->kf_structsize);
/* Advance to next packed record */
bp += kf->kf_structsize;
/* Set field size to fixed length, advance */
kp->kf_structsize = sizeof(*kp);
kp++;
}
free(buf);
*cntp = cnt;
return (kif); /* Caller must free() return value */
}
static struct filestat_list *
procstat_getfiles_sysctl(struct procstat *procstat, struct kinfo_proc *kp,
int mmapped)
{
struct kinfo_file *kif, *files;
struct kinfo_vmentry *kve, *vmentries;
struct filestat_list *head;
struct filestat *entry;
char *path;
off_t offset;
int cnt, fd, fflags;
int i, type, uflags;
int refcount;
cap_rights_t cap_rights;
assert(kp);
if (kp->ki_fd == NULL)
return (NULL);
switch(procstat->type) {
case PROCSTAT_SYSCTL:
files = kinfo_getfile(kp->ki_pid, &cnt);
break;
case PROCSTAT_CORE:
files = kinfo_getfile_core(procstat->core, &cnt);
break;
default:
assert(!"invalid type");
}
if (files == NULL && errno != EPERM) {
warn("kinfo_getfile()");
return (NULL);
}
procstat->files = files;
/*
* Allocate list head.
*/
head = malloc(sizeof(*head));
if (head == NULL)
return (NULL);
STAILQ_INIT(head);
for (i = 0; i < cnt; i++) {
kif = &files[i];
type = kinfo_type2fst(kif->kf_type);
fd = kif->kf_fd >= 0 ? kif->kf_fd : -1;
fflags = kinfo_fflags2fst(kif->kf_flags);
uflags = kinfo_uflags2fst(kif->kf_fd);
refcount = kif->kf_ref_count;
offset = kif->kf_offset;
if (*kif->kf_path != '\0')
path = strdup(kif->kf_path);
else
path = NULL;
cap_rights = kif->kf_cap_rights;
/*
* Create filestat entry.
*/
entry = filestat_new_entry(kif, type, fd, fflags, uflags,
refcount, offset, path, &cap_rights);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
if (mmapped != 0) {
vmentries = procstat_getvmmap(procstat, kp, &cnt);
procstat->vmentries = vmentries;
if (vmentries == NULL || cnt == 0)
goto fail;
for (i = 0; i < cnt; i++) {
kve = &vmentries[i];
if (kve->kve_type != KVME_TYPE_VNODE)
continue;
fflags = 0;
if (kve->kve_protection & KVME_PROT_READ)
fflags = PS_FST_FFLAG_READ;
if ((kve->kve_flags & KVME_FLAG_COW) == 0 &&
kve->kve_protection & KVME_PROT_WRITE)
fflags |= PS_FST_FFLAG_WRITE;
offset = kve->kve_offset;
refcount = kve->kve_ref_count;
if (*kve->kve_path != '\0')
path = strdup(kve->kve_path);
else
path = NULL;
entry = filestat_new_entry(kve, PS_FST_TYPE_VNODE, -1,
fflags, PS_FST_UFLAG_MMAP, refcount, offset, path,
NULL);
if (entry != NULL)
STAILQ_INSERT_TAIL(head, entry, next);
}
}
fail:
return (head);
}
int
procstat_get_pipe_info(struct procstat *procstat, struct filestat *fst,
struct pipestat *ps, char *errbuf)
{
assert(ps);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_pipe_info_kvm(procstat->kd, fst, ps,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_pipe_info_sysctl(fst, ps, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_pipe_info_kvm(kvm_t *kd, struct filestat *fst,
struct pipestat *ps, char *errbuf)
{
struct pipe pi;
void *pipep;
assert(kd);
assert(ps);
assert(fst);
bzero(ps, sizeof(*ps));
pipep = fst->fs_typedep;
if (pipep == NULL)
goto fail;
if (!kvm_read_all(kd, (unsigned long)pipep, &pi, sizeof(struct pipe))) {
warnx("can't read pipe at %p", (void *)pipep);
goto fail;
}
ps->addr = (uintptr_t)pipep;
ps->peer = (uintptr_t)pi.pipe_peer;
ps->buffer_cnt = pi.pipe_buffer.cnt;
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
static int
procstat_get_pipe_info_sysctl(struct filestat *fst, struct pipestat *ps,
char *errbuf __unused)
{
struct kinfo_file *kif;
assert(ps);
assert(fst);
bzero(ps, sizeof(*ps));
kif = fst->fs_typedep;
if (kif == NULL)
return (1);
ps->addr = kif->kf_un.kf_pipe.kf_pipe_addr;
ps->peer = kif->kf_un.kf_pipe.kf_pipe_peer;
ps->buffer_cnt = kif->kf_un.kf_pipe.kf_pipe_buffer_cnt;
return (0);
}
int
procstat_get_pts_info(struct procstat *procstat, struct filestat *fst,
struct ptsstat *pts, char *errbuf)
{
assert(pts);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_pts_info_kvm(procstat->kd, fst, pts,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_pts_info_sysctl(fst, pts, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_pts_info_kvm(kvm_t *kd, struct filestat *fst,
struct ptsstat *pts, char *errbuf)
{
struct tty tty;
void *ttyp;
assert(kd);
assert(pts);
assert(fst);
bzero(pts, sizeof(*pts));
ttyp = fst->fs_typedep;
if (ttyp == NULL)
goto fail;
if (!kvm_read_all(kd, (unsigned long)ttyp, &tty, sizeof(struct tty))) {
warnx("can't read tty at %p", (void *)ttyp);
goto fail;
}
pts->dev = dev2udev(kd, tty.t_dev);
(void)kdevtoname(kd, tty.t_dev, pts->devname);
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
static int
procstat_get_pts_info_sysctl(struct filestat *fst, struct ptsstat *pts,
char *errbuf __unused)
{
struct kinfo_file *kif;
assert(pts);
assert(fst);
bzero(pts, sizeof(*pts));
kif = fst->fs_typedep;
if (kif == NULL)
return (0);
pts->dev = kif->kf_un.kf_pts.kf_pts_dev;
strlcpy(pts->devname, kif->kf_path, sizeof(pts->devname));
return (0);
}
int
procstat_get_sem_info(struct procstat *procstat, struct filestat *fst,
struct semstat *sem, char *errbuf)
{
assert(sem);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_sem_info_kvm(procstat->kd, fst, sem,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_sem_info_sysctl(fst, sem, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_sem_info_kvm(kvm_t *kd, struct filestat *fst,
struct semstat *sem, char *errbuf)
{
struct ksem ksem;
void *ksemp;
char *path;
int i;
assert(kd);
assert(sem);
assert(fst);
bzero(sem, sizeof(*sem));
ksemp = fst->fs_typedep;
if (ksemp == NULL)
goto fail;
if (!kvm_read_all(kd, (unsigned long)ksemp, &ksem,
sizeof(struct ksem))) {
warnx("can't read ksem at %p", (void *)ksemp);
goto fail;
}
sem->mode = S_IFREG | ksem.ks_mode;
sem->value = ksem.ks_value;
if (fst->fs_path == NULL && ksem.ks_path != NULL) {
path = malloc(MAXPATHLEN);
for (i = 0; i < MAXPATHLEN - 1; i++) {
if (!kvm_read_all(kd, (unsigned long)ksem.ks_path + i,
path + i, 1))
break;
if (path[i] == '\0')
break;
}
path[i] = '\0';
if (i == 0)
free(path);
else
fst->fs_path = path;
}
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
static int
procstat_get_sem_info_sysctl(struct filestat *fst, struct semstat *sem,
char *errbuf __unused)
{
struct kinfo_file *kif;
assert(sem);
assert(fst);
bzero(sem, sizeof(*sem));
kif = fst->fs_typedep;
if (kif == NULL)
return (0);
sem->value = kif->kf_un.kf_sem.kf_sem_value;
sem->mode = kif->kf_un.kf_sem.kf_sem_mode;
return (0);
}
int
procstat_get_shm_info(struct procstat *procstat, struct filestat *fst,
struct shmstat *shm, char *errbuf)
{
assert(shm);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_shm_info_kvm(procstat->kd, fst, shm,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_shm_info_sysctl(fst, shm, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_shm_info_kvm(kvm_t *kd, struct filestat *fst,
struct shmstat *shm, char *errbuf)
{
struct shmfd shmfd;
void *shmfdp;
char *path;
int i;
assert(kd);
assert(shm);
assert(fst);
bzero(shm, sizeof(*shm));
shmfdp = fst->fs_typedep;
if (shmfdp == NULL)
goto fail;
if (!kvm_read_all(kd, (unsigned long)shmfdp, &shmfd,
sizeof(struct shmfd))) {
warnx("can't read shmfd at %p", (void *)shmfdp);
goto fail;
}
shm->mode = S_IFREG | shmfd.shm_mode;
shm->size = shmfd.shm_size;
if (fst->fs_path == NULL && shmfd.shm_path != NULL) {
path = malloc(MAXPATHLEN);
for (i = 0; i < MAXPATHLEN - 1; i++) {
if (!kvm_read_all(kd, (unsigned long)shmfd.shm_path + i,
path + i, 1))
break;
if (path[i] == '\0')
break;
}
path[i] = '\0';
if (i == 0)
free(path);
else
fst->fs_path = path;
}
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
static int
procstat_get_shm_info_sysctl(struct filestat *fst, struct shmstat *shm,
char *errbuf __unused)
{
struct kinfo_file *kif;
assert(shm);
assert(fst);
bzero(shm, sizeof(*shm));
kif = fst->fs_typedep;
if (kif == NULL)
return (0);
shm->size = kif->kf_un.kf_file.kf_file_size;
shm->mode = kif->kf_un.kf_file.kf_file_mode;
return (0);
}
int
procstat_get_vnode_info(struct procstat *procstat, struct filestat *fst,
struct vnstat *vn, char *errbuf)
{
assert(vn);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_vnode_info_kvm(procstat->kd, fst, vn,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_vnode_info_sysctl(fst, vn, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_vnode_info_kvm(kvm_t *kd, struct filestat *fst,
struct vnstat *vn, char *errbuf)
{
/* Filesystem specific handlers. */
#define FSTYPE(fst) {#fst, fst##_filestat}
struct {
const char *tag;
int (*handler)(kvm_t *kd, struct vnode *vp,
struct vnstat *vn);
} fstypes[] = {
FSTYPE(devfs),
FSTYPE(isofs),
FSTYPE(msdosfs),
FSTYPE(nfs),
FSTYPE(smbfs),
FSTYPE(udf),
FSTYPE(ufs),
#ifdef LIBPROCSTAT_ZFS
FSTYPE(zfs),
#endif
};
#define NTYPES (sizeof(fstypes) / sizeof(*fstypes))
struct vnode vnode;
char tagstr[12];
void *vp;
int error;
unsigned int i;
assert(kd);
assert(vn);
assert(fst);
vp = fst->fs_typedep;
if (vp == NULL)
goto fail;
error = kvm_read_all(kd, (unsigned long)vp, &vnode, sizeof(vnode));
if (error == 0) {
warnx("can't read vnode at %p", (void *)vp);
goto fail;
}
bzero(vn, sizeof(*vn));
vn->vn_type = vntype2psfsttype(vnode.v_type);
if (vnode.v_type == VNON || vnode.v_type == VBAD)
return (0);
error = kvm_read_all(kd, (unsigned long)vnode.v_lock.lock_object.lo_name,
tagstr, sizeof(tagstr));
if (error == 0) {
warnx("can't read lo_name at %p", (void *)vp);
goto fail;
}
tagstr[sizeof(tagstr) - 1] = '\0';
/*
* Find appropriate handler.
*/
for (i = 0; i < NTYPES; i++)
if (!strcmp(fstypes[i].tag, tagstr)) {
if (fstypes[i].handler(kd, &vnode, vn) != 0) {
goto fail;
}
break;
}
if (i == NTYPES) {
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "?(%s)", tagstr);
return (1);
}
vn->vn_mntdir = getmnton(kd, vnode.v_mount);
if ((vnode.v_type == VBLK || vnode.v_type == VCHR) &&
vnode.v_rdev != NULL){
vn->vn_dev = dev2udev(kd, vnode.v_rdev);
(void)kdevtoname(kd, vnode.v_rdev, vn->vn_devname);
} else {
vn->vn_dev = -1;
}
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
/*
* kinfo vnode type to filestat translation.
*/
static int
kinfo_vtype2fst(int kfvtype)
{
static struct {
int kf_vtype;
int fst_vtype;
} kfvtypes2fst[] = {
{ KF_VTYPE_VBAD, PS_FST_VTYPE_VBAD },
{ KF_VTYPE_VBLK, PS_FST_VTYPE_VBLK },
{ KF_VTYPE_VCHR, PS_FST_VTYPE_VCHR },
{ KF_VTYPE_VDIR, PS_FST_VTYPE_VDIR },
{ KF_VTYPE_VFIFO, PS_FST_VTYPE_VFIFO },
{ KF_VTYPE_VLNK, PS_FST_VTYPE_VLNK },
{ KF_VTYPE_VNON, PS_FST_VTYPE_VNON },
{ KF_VTYPE_VREG, PS_FST_VTYPE_VREG },
{ KF_VTYPE_VSOCK, PS_FST_VTYPE_VSOCK }
};
#define NKFVTYPES (sizeof(kfvtypes2fst) / sizeof(*kfvtypes2fst))
unsigned int i;
for (i = 0; i < NKFVTYPES; i++)
if (kfvtypes2fst[i].kf_vtype == kfvtype)
break;
if (i == NKFVTYPES)
return (PS_FST_VTYPE_UNKNOWN);
return (kfvtypes2fst[i].fst_vtype);
}
static int
procstat_get_vnode_info_sysctl(struct filestat *fst, struct vnstat *vn,
char *errbuf)
{
struct statfs stbuf;
struct kinfo_file *kif;
struct kinfo_vmentry *kve;
char *name, *path;
uint64_t fileid;
uint64_t size;
uint64_t fsid;
uint64_t rdev;
uint16_t mode;
int vntype;
int status;
assert(fst);
assert(vn);
bzero(vn, sizeof(*vn));
if (fst->fs_typedep == NULL)
return (1);
if (fst->fs_uflags & PS_FST_UFLAG_MMAP) {
kve = fst->fs_typedep;
fileid = kve->kve_vn_fileid;
fsid = kve->kve_vn_fsid;
mode = kve->kve_vn_mode;
path = kve->kve_path;
rdev = kve->kve_vn_rdev;
size = kve->kve_vn_size;
vntype = kinfo_vtype2fst(kve->kve_vn_type);
status = kve->kve_status;
} else {
kif = fst->fs_typedep;
fileid = kif->kf_un.kf_file.kf_file_fileid;
fsid = kif->kf_un.kf_file.kf_file_fsid;
mode = kif->kf_un.kf_file.kf_file_mode;
path = kif->kf_path;
rdev = kif->kf_un.kf_file.kf_file_rdev;
size = kif->kf_un.kf_file.kf_file_size;
vntype = kinfo_vtype2fst(kif->kf_vnode_type);
status = kif->kf_status;
}
vn->vn_type = vntype;
if (vntype == PS_FST_VTYPE_VNON || vntype == PS_FST_VTYPE_VBAD)
return (0);
if ((status & KF_ATTR_VALID) == 0) {
if (errbuf != NULL) {
snprintf(errbuf, _POSIX2_LINE_MAX,
"? (no info available)");
}
return (1);
}
if (path && *path) {
statfs(path, &stbuf);
vn->vn_mntdir = strdup(stbuf.f_mntonname);
} else
vn->vn_mntdir = strdup("-");
vn->vn_dev = rdev;
if (vntype == PS_FST_VTYPE_VBLK) {
name = devname(rdev, S_IFBLK);
if (name != NULL)
strlcpy(vn->vn_devname, name,
sizeof(vn->vn_devname));
} else if (vntype == PS_FST_VTYPE_VCHR) {
name = devname(vn->vn_dev, S_IFCHR);
if (name != NULL)
strlcpy(vn->vn_devname, name,
sizeof(vn->vn_devname));
}
vn->vn_fsid = fsid;
vn->vn_fileid = fileid;
vn->vn_size = size;
vn->vn_mode = mode;
return (0);
}
int
procstat_get_socket_info(struct procstat *procstat, struct filestat *fst,
struct sockstat *sock, char *errbuf)
{
assert(sock);
if (procstat->type == PROCSTAT_KVM) {
return (procstat_get_socket_info_kvm(procstat->kd, fst, sock,
errbuf));
} else if (procstat->type == PROCSTAT_SYSCTL ||
procstat->type == PROCSTAT_CORE) {
return (procstat_get_socket_info_sysctl(fst, sock, errbuf));
} else {
warnx("unknown access method: %d", procstat->type);
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
}
static int
procstat_get_socket_info_kvm(kvm_t *kd, struct filestat *fst,
struct sockstat *sock, char *errbuf)
{
struct domain dom;
struct inpcb inpcb;
struct protosw proto;
struct socket s;
struct unpcb unpcb;
ssize_t len;
void *so;
assert(kd);
assert(sock);
assert(fst);
bzero(sock, sizeof(*sock));
so = fst->fs_typedep;
if (so == NULL)
goto fail;
sock->so_addr = (uintptr_t)so;
/* fill in socket */
if (!kvm_read_all(kd, (unsigned long)so, &s,
sizeof(struct socket))) {
warnx("can't read sock at %p", (void *)so);
goto fail;
}
/* fill in protosw entry */
if (!kvm_read_all(kd, (unsigned long)s.so_proto, &proto,
sizeof(struct protosw))) {
warnx("can't read protosw at %p", (void *)s.so_proto);
goto fail;
}
/* fill in domain */
if (!kvm_read_all(kd, (unsigned long)proto.pr_domain, &dom,
sizeof(struct domain))) {
warnx("can't read domain at %p",
(void *)proto.pr_domain);
goto fail;
}
if ((len = kvm_read(kd, (unsigned long)dom.dom_name, sock->dname,
sizeof(sock->dname) - 1)) < 0) {
warnx("can't read domain name at %p", (void *)dom.dom_name);
sock->dname[0] = '\0';
}
else
sock->dname[len] = '\0';
/*
* Fill in known data.
*/
sock->type = s.so_type;
sock->proto = proto.pr_protocol;
sock->dom_family = dom.dom_family;
sock->so_pcb = (uintptr_t)s.so_pcb;
/*
* Protocol specific data.
*/
switch(dom.dom_family) {
case AF_INET:
case AF_INET6:
if (proto.pr_protocol == IPPROTO_TCP) {
if (s.so_pcb) {
if (kvm_read(kd, (u_long)s.so_pcb,
(char *)&inpcb, sizeof(struct inpcb))
!= sizeof(struct inpcb)) {
warnx("can't read inpcb at %p",
(void *)s.so_pcb);
} else
sock->inp_ppcb =
(uintptr_t)inpcb.inp_ppcb;
sock->sendq = s.so_snd.sb_ccc;
sock->recvq = s.so_rcv.sb_ccc;
}
}
break;
case AF_UNIX:
if (s.so_pcb) {
if (kvm_read(kd, (u_long)s.so_pcb, (char *)&unpcb,
sizeof(struct unpcb)) != sizeof(struct unpcb)){
warnx("can't read unpcb at %p",
(void *)s.so_pcb);
} else if (unpcb.unp_conn) {
sock->so_rcv_sb_state = s.so_rcv.sb_state;
sock->so_snd_sb_state = s.so_snd.sb_state;
sock->unp_conn = (uintptr_t)unpcb.unp_conn;
sock->sendq = s.so_snd.sb_ccc;
sock->recvq = s.so_rcv.sb_ccc;
}
}
break;
default:
break;
}
return (0);
fail:
if (errbuf != NULL)
snprintf(errbuf, _POSIX2_LINE_MAX, "error");
return (1);
}
static int
procstat_get_socket_info_sysctl(struct filestat *fst, struct sockstat *sock,
char *errbuf __unused)
{
struct kinfo_file *kif;
assert(sock);
assert(fst);
bzero(sock, sizeof(*sock));
kif = fst->fs_typedep;
if (kif == NULL)
return (0);
/*
* Fill in known data.
*/
sock->type = kif->kf_sock_type;
sock->proto = kif->kf_sock_protocol;
sock->dom_family = kif->kf_sock_domain;
sock->so_pcb = kif->kf_un.kf_sock.kf_sock_pcb;
strlcpy(sock->dname, kif->kf_path, sizeof(sock->dname));
bcopy(&kif->kf_un.kf_sock.kf_sa_local, &sock->sa_local,
kif->kf_un.kf_sock.kf_sa_local.ss_len);
bcopy(&kif->kf_un.kf_sock.kf_sa_peer, &sock->sa_peer,
kif->kf_un.kf_sock.kf_sa_peer.ss_len);
/*
* Protocol specific data.
*/
switch(sock->dom_family) {
case AF_INET:
case AF_INET6:
if (sock->proto == IPPROTO_TCP) {
sock->inp_ppcb = kif->kf_un.kf_sock.kf_sock_inpcb;
sock->sendq = kif->kf_un.kf_sock.kf_sock_sendq;
sock->recvq = kif->kf_un.kf_sock.kf_sock_recvq;
}
break;
case AF_UNIX:
if (kif->kf_un.kf_sock.kf_sock_unpconn != 0) {
sock->so_rcv_sb_state =
kif->kf_un.kf_sock.kf_sock_rcv_sb_state;
sock->so_snd_sb_state =
kif->kf_un.kf_sock.kf_sock_snd_sb_state;
sock->unp_conn =
kif->kf_un.kf_sock.kf_sock_unpconn;
sock->sendq = kif->kf_un.kf_sock.kf_sock_sendq;
sock->recvq = kif->kf_un.kf_sock.kf_sock_recvq;
}
break;
default:
break;
}
return (0);
}
/*
* Descriptor flags to filestat translation.
*/
static int
to_filestat_flags(int flags)
{
static struct {
int flag;
int fst_flag;
} fstflags[] = {
{ FREAD, PS_FST_FFLAG_READ },
{ FWRITE, PS_FST_FFLAG_WRITE },
{ O_APPEND, PS_FST_FFLAG_APPEND },
{ O_ASYNC, PS_FST_FFLAG_ASYNC },
{ O_CREAT, PS_FST_FFLAG_CREAT },
{ O_DIRECT, PS_FST_FFLAG_DIRECT },
{ O_EXCL, PS_FST_FFLAG_EXCL },
{ O_EXEC, PS_FST_FFLAG_EXEC },
{ O_EXLOCK, PS_FST_FFLAG_EXLOCK },
{ O_NOFOLLOW, PS_FST_FFLAG_NOFOLLOW },
{ O_NONBLOCK, PS_FST_FFLAG_NONBLOCK },
{ O_SHLOCK, PS_FST_FFLAG_SHLOCK },
{ O_SYNC, PS_FST_FFLAG_SYNC },
{ O_TRUNC, PS_FST_FFLAG_TRUNC }
};
#define NFSTFLAGS (sizeof(fstflags) / sizeof(*fstflags))
int fst_flags;
unsigned int i;
fst_flags = 0;
for (i = 0; i < NFSTFLAGS; i++)
if (flags & fstflags[i].flag)
fst_flags |= fstflags[i].fst_flag;
return (fst_flags);
}
/*
* Vnode type to filestate translation.
*/
static int
vntype2psfsttype(int type)
{
static struct {
int vtype;
int fst_vtype;
} vt2fst[] = {
{ VBAD, PS_FST_VTYPE_VBAD },
{ VBLK, PS_FST_VTYPE_VBLK },
{ VCHR, PS_FST_VTYPE_VCHR },
{ VDIR, PS_FST_VTYPE_VDIR },
{ VFIFO, PS_FST_VTYPE_VFIFO },
{ VLNK, PS_FST_VTYPE_VLNK },
{ VNON, PS_FST_VTYPE_VNON },
{ VREG, PS_FST_VTYPE_VREG },
{ VSOCK, PS_FST_VTYPE_VSOCK }
};
#define NVFTYPES (sizeof(vt2fst) / sizeof(*vt2fst))
unsigned int i, fst_type;
fst_type = PS_FST_VTYPE_UNKNOWN;
for (i = 0; i < NVFTYPES; i++) {
if (type == vt2fst[i].vtype) {
fst_type = vt2fst[i].fst_vtype;
break;
}
}
return (fst_type);
}
static char *
getmnton(kvm_t *kd, struct mount *m)
{
struct mount mnt;
static struct mtab {
struct mtab *next;
struct mount *m;
char mntonname[MNAMELEN + 1];
} *mhead = NULL;
struct mtab *mt;
for (mt = mhead; mt != NULL; mt = mt->next)
if (m == mt->m)
return (mt->mntonname);
if (!kvm_read_all(kd, (unsigned long)m, &mnt, sizeof(struct mount))) {
warnx("can't read mount table at %p", (void *)m);
return (NULL);
}
if ((mt = malloc(sizeof (struct mtab))) == NULL)
err(1, NULL);
mt->m = m;
bcopy(&mnt.mnt_stat.f_mntonname[0], &mt->mntonname[0], MNAMELEN);
mt->mntonname[MNAMELEN] = '\0';
mt->next = mhead;
mhead = mt;
return (mt->mntonname);
}
/*
* Auxiliary structures and functions to get process environment or
* command line arguments.
*/
struct argvec {
char *buf;
size_t bufsize;
char **argv;
size_t argc;
};
static struct argvec *
argvec_alloc(size_t bufsize)
{
struct argvec *av;
av = malloc(sizeof(*av));
if (av == NULL)
return (NULL);
av->bufsize = bufsize;
av->buf = malloc(av->bufsize);
if (av->buf == NULL) {
free(av);
return (NULL);
}
av->argc = 32;
av->argv = malloc(sizeof(char *) * av->argc);
if (av->argv == NULL) {
free(av->buf);
free(av);
return (NULL);
}
return av;
}
static void
argvec_free(struct argvec * av)
{
free(av->argv);
free(av->buf);
free(av);
}
static char **
getargv(struct procstat *procstat, struct kinfo_proc *kp, size_t nchr, int env)
{
int error, name[4], argc, i;
struct argvec *av, **avp;
enum psc_type type;
size_t len;
char *p, **argv;
assert(procstat);
assert(kp);
if (procstat->type == PROCSTAT_KVM) {
warnx("can't use kvm access method");
return (NULL);
}
if (procstat->type != PROCSTAT_SYSCTL &&
procstat->type != PROCSTAT_CORE) {
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
if (nchr == 0 || nchr > ARG_MAX)
nchr = ARG_MAX;
avp = (struct argvec **)(env ? &procstat->argv : &procstat->envv);
av = *avp;
if (av == NULL)
{
av = argvec_alloc(nchr);
if (av == NULL)
{
warn("malloc(%zu)", nchr);
return (NULL);
}
*avp = av;
} else if (av->bufsize < nchr) {
av->buf = reallocf(av->buf, nchr);
if (av->buf == NULL) {
warn("malloc(%zu)", nchr);
return (NULL);
}
}
if (procstat->type == PROCSTAT_SYSCTL) {
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = env ? KERN_PROC_ENV : KERN_PROC_ARGS;
name[3] = kp->ki_pid;
len = nchr;
error = sysctl(name, nitems(name), av->buf, &len, NULL, 0);
if (error != 0 && errno != ESRCH && errno != EPERM)
warn("sysctl(kern.proc.%s)", env ? "env" : "args");
if (error != 0 || len == 0)
return (NULL);
} else /* procstat->type == PROCSTAT_CORE */ {
type = env ? PSC_TYPE_ENVV : PSC_TYPE_ARGV;
len = nchr;
if (procstat_core_get(procstat->core, type, av->buf, &len)
== NULL) {
return (NULL);
}
}
argv = av->argv;
argc = av->argc;
i = 0;
for (p = av->buf; p < av->buf + len; p += strlen(p) + 1) {
argv[i++] = p;
if (i < argc)
continue;
/* Grow argv. */
argc += argc;
argv = realloc(argv, sizeof(char *) * argc);
if (argv == NULL) {
warn("malloc(%zu)", sizeof(char *) * argc);
return (NULL);
}
av->argv = argv;
av->argc = argc;
}
argv[i] = NULL;
return (argv);
}
/*
* Return process command line arguments.
*/
char **
procstat_getargv(struct procstat *procstat, struct kinfo_proc *p, size_t nchr)
{
return (getargv(procstat, p, nchr, 0));
}
/*
* Free the buffer allocated by procstat_getargv().
*/
void
procstat_freeargv(struct procstat *procstat)
{
if (procstat->argv != NULL) {
argvec_free(procstat->argv);
procstat->argv = NULL;
}
}
/*
* Return process environment.
*/
char **
procstat_getenvv(struct procstat *procstat, struct kinfo_proc *p, size_t nchr)
{
return (getargv(procstat, p, nchr, 1));
}
/*
* Free the buffer allocated by procstat_getenvv().
*/
void
procstat_freeenvv(struct procstat *procstat)
{
if (procstat->envv != NULL) {
argvec_free(procstat->envv);
procstat->envv = NULL;
}
}
static struct kinfo_vmentry *
kinfo_getvmmap_core(struct procstat_core *core, int *cntp)
{
int cnt;
size_t len;
char *buf, *bp, *eb;
struct kinfo_vmentry *kiv, *kp, *kv;
buf = procstat_core_get(core, PSC_TYPE_VMMAP, NULL, &len);
if (buf == NULL)
return (NULL);
/*
* XXXMG: The code below is just copy&past from libutil.
* The code duplication can be avoided if libutil
* is extended to provide something like:
* struct kinfo_vmentry *kinfo_getvmmap_from_buf(const char *buf,
* size_t len, int *cntp);
*/
/* Pass 1: count items */
cnt = 0;
bp = buf;
eb = buf + len;
while (bp < eb) {
kv = (struct kinfo_vmentry *)(uintptr_t)bp;
if (kv->kve_structsize == 0)
break;
bp += kv->kve_structsize;
cnt++;
}
kiv = calloc(cnt, sizeof(*kiv));
if (kiv == NULL) {
free(buf);
return (NULL);
}
bp = buf;
eb = buf + len;
kp = kiv;
/* Pass 2: unpack */
while (bp < eb) {
kv = (struct kinfo_vmentry *)(uintptr_t)bp;
if (kv->kve_structsize == 0)
break;
/* Copy/expand into pre-zeroed buffer */
memcpy(kp, kv, kv->kve_structsize);
/* Advance to next packed record */
bp += kv->kve_structsize;
/* Set field size to fixed length, advance */
kp->kve_structsize = sizeof(*kp);
kp++;
}
free(buf);
*cntp = cnt;
return (kiv); /* Caller must free() return value */
}
struct kinfo_vmentry *
procstat_getvmmap(struct procstat *procstat, struct kinfo_proc *kp,
unsigned int *cntp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
warnx("kvm method is not supported");
return (NULL);
case PROCSTAT_SYSCTL:
return (kinfo_getvmmap(kp->ki_pid, cntp));
case PROCSTAT_CORE:
return (kinfo_getvmmap_core(procstat->core, cntp));
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freevmmap(struct procstat *procstat __unused,
struct kinfo_vmentry *vmmap)
{
free(vmmap);
}
static gid_t *
procstat_getgroups_kvm(kvm_t *kd, struct kinfo_proc *kp, unsigned int *cntp)
{
struct proc proc;
struct ucred ucred;
gid_t *groups;
size_t len;
assert(kd != NULL);
assert(kp != NULL);
if (!kvm_read_all(kd, (unsigned long)kp->ki_paddr, &proc,
sizeof(proc))) {
warnx("can't read proc struct at %p for pid %d",
kp->ki_paddr, kp->ki_pid);
return (NULL);
}
if (proc.p_ucred == NOCRED)
return (NULL);
if (!kvm_read_all(kd, (unsigned long)proc.p_ucred, &ucred,
sizeof(ucred))) {
warnx("can't read ucred struct at %p for pid %d",
proc.p_ucred, kp->ki_pid);
return (NULL);
}
len = ucred.cr_ngroups * sizeof(gid_t);
groups = malloc(len);
if (groups == NULL) {
warn("malloc(%zu)", len);
return (NULL);
}
if (!kvm_read_all(kd, (unsigned long)ucred.cr_groups, groups, len)) {
warnx("can't read groups at %p for pid %d",
ucred.cr_groups, kp->ki_pid);
free(groups);
return (NULL);
}
*cntp = ucred.cr_ngroups;
return (groups);
}
static gid_t *
procstat_getgroups_sysctl(pid_t pid, unsigned int *cntp)
{
int mib[4];
size_t len;
gid_t *groups;
mib[0] = CTL_KERN;
mib[1] = KERN_PROC;
mib[2] = KERN_PROC_GROUPS;
mib[3] = pid;
len = (sysconf(_SC_NGROUPS_MAX) + 1) * sizeof(gid_t);
groups = malloc(len);
if (groups == NULL) {
warn("malloc(%zu)", len);
return (NULL);
}
if (sysctl(mib, nitems(mib), groups, &len, NULL, 0) == -1) {
warn("sysctl: kern.proc.groups: %d", pid);
free(groups);
return (NULL);
}
*cntp = len / sizeof(gid_t);
return (groups);
}
static gid_t *
procstat_getgroups_core(struct procstat_core *core, unsigned int *cntp)
{
size_t len;
gid_t *groups;
groups = procstat_core_get(core, PSC_TYPE_GROUPS, NULL, &len);
if (groups == NULL)
return (NULL);
*cntp = len / sizeof(gid_t);
return (groups);
}
gid_t *
procstat_getgroups(struct procstat *procstat, struct kinfo_proc *kp,
unsigned int *cntp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
return (procstat_getgroups_kvm(procstat->kd, kp, cntp));
case PROCSTAT_SYSCTL:
return (procstat_getgroups_sysctl(kp->ki_pid, cntp));
case PROCSTAT_CORE:
return (procstat_getgroups_core(procstat->core, cntp));
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freegroups(struct procstat *procstat __unused, gid_t *groups)
{
free(groups);
}
static int
procstat_getumask_kvm(kvm_t *kd, struct kinfo_proc *kp, unsigned short *maskp)
{
struct filedesc fd;
assert(kd != NULL);
assert(kp != NULL);
if (kp->ki_fd == NULL)
return (-1);
if (!kvm_read_all(kd, (unsigned long)kp->ki_fd, &fd, sizeof(fd))) {
warnx("can't read filedesc at %p for pid %d", kp->ki_fd,
kp->ki_pid);
return (-1);
}
*maskp = fd.fd_cmask;
return (0);
}
static int
procstat_getumask_sysctl(pid_t pid, unsigned short *maskp)
{
int error;
int mib[4];
size_t len;
mib[0] = CTL_KERN;
mib[1] = KERN_PROC;
mib[2] = KERN_PROC_UMASK;
mib[3] = pid;
len = sizeof(*maskp);
error = sysctl(mib, nitems(mib), maskp, &len, NULL, 0);
if (error != 0 && errno != ESRCH && errno != EPERM)
warn("sysctl: kern.proc.umask: %d", pid);
return (error);
}
static int
procstat_getumask_core(struct procstat_core *core, unsigned short *maskp)
{
size_t len;
unsigned short *buf;
buf = procstat_core_get(core, PSC_TYPE_UMASK, NULL, &len);
if (buf == NULL)
return (-1);
if (len < sizeof(*maskp)) {
free(buf);
return (-1);
}
*maskp = *buf;
free(buf);
return (0);
}
int
procstat_getumask(struct procstat *procstat, struct kinfo_proc *kp,
unsigned short *maskp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
return (procstat_getumask_kvm(procstat->kd, kp, maskp));
case PROCSTAT_SYSCTL:
return (procstat_getumask_sysctl(kp->ki_pid, maskp));
case PROCSTAT_CORE:
return (procstat_getumask_core(procstat->core, maskp));
default:
warnx("unknown access method: %d", procstat->type);
return (-1);
}
}
static int
procstat_getrlimit_kvm(kvm_t *kd, struct kinfo_proc *kp, int which,
struct rlimit* rlimit)
{
struct proc proc;
unsigned long offset;
assert(kd != NULL);
assert(kp != NULL);
assert(which >= 0 && which < RLIM_NLIMITS);
if (!kvm_read_all(kd, (unsigned long)kp->ki_paddr, &proc,
sizeof(proc))) {
warnx("can't read proc struct at %p for pid %d",
kp->ki_paddr, kp->ki_pid);
return (-1);
}
if (proc.p_limit == NULL)
return (-1);
offset = (unsigned long)proc.p_limit + sizeof(struct rlimit) * which;
if (!kvm_read_all(kd, offset, rlimit, sizeof(*rlimit))) {
warnx("can't read rlimit struct at %p for pid %d",
(void *)offset, kp->ki_pid);
return (-1);
}
return (0);
}
static int
procstat_getrlimit_sysctl(pid_t pid, int which, struct rlimit* rlimit)
{
int error, name[5];
size_t len;
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_RLIMIT;
name[3] = pid;
name[4] = which;
len = sizeof(struct rlimit);
error = sysctl(name, nitems(name), rlimit, &len, NULL, 0);
if (error < 0 && errno != ESRCH) {
warn("sysctl: kern.proc.rlimit: %d", pid);
return (-1);
}
if (error < 0 || len != sizeof(struct rlimit))
return (-1);
return (0);
}
static int
procstat_getrlimit_core(struct procstat_core *core, int which,
struct rlimit* rlimit)
{
size_t len;
struct rlimit* rlimits;
if (which < 0 || which >= RLIM_NLIMITS) {
errno = EINVAL;
warn("getrlimit: which");
return (-1);
}
rlimits = procstat_core_get(core, PSC_TYPE_RLIMIT, NULL, &len);
if (rlimits == NULL)
return (-1);
if (len < sizeof(struct rlimit) * RLIM_NLIMITS) {
free(rlimits);
return (-1);
}
*rlimit = rlimits[which];
free(rlimits);
return (0);
}
int
procstat_getrlimit(struct procstat *procstat, struct kinfo_proc *kp, int which,
struct rlimit* rlimit)
{
switch(procstat->type) {
case PROCSTAT_KVM:
return (procstat_getrlimit_kvm(procstat->kd, kp, which,
rlimit));
case PROCSTAT_SYSCTL:
return (procstat_getrlimit_sysctl(kp->ki_pid, which, rlimit));
case PROCSTAT_CORE:
return (procstat_getrlimit_core(procstat->core, which, rlimit));
default:
warnx("unknown access method: %d", procstat->type);
return (-1);
}
}
static int
procstat_getpathname_sysctl(pid_t pid, char *pathname, size_t maxlen)
{
int error, name[4];
size_t len;
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_PATHNAME;
name[3] = pid;
len = maxlen;
error = sysctl(name, nitems(name), pathname, &len, NULL, 0);
if (error != 0 && errno != ESRCH)
warn("sysctl: kern.proc.pathname: %d", pid);
if (len == 0)
pathname[0] = '\0';
return (error);
}
static int
procstat_getpathname_core(struct procstat_core *core, char *pathname,
size_t maxlen)
{
struct kinfo_file *files;
int cnt, i, result;
files = kinfo_getfile_core(core, &cnt);
if (files == NULL)
return (-1);
result = -1;
for (i = 0; i < cnt; i++) {
if (files[i].kf_fd != KF_FD_TYPE_TEXT)
continue;
strncpy(pathname, files[i].kf_path, maxlen);
result = 0;
break;
}
free(files);
return (result);
}
int
procstat_getpathname(struct procstat *procstat, struct kinfo_proc *kp,
char *pathname, size_t maxlen)
{
switch(procstat->type) {
case PROCSTAT_KVM:
/* XXX: Return empty string. */
if (maxlen > 0)
pathname[0] = '\0';
return (0);
case PROCSTAT_SYSCTL:
return (procstat_getpathname_sysctl(kp->ki_pid, pathname,
maxlen));
case PROCSTAT_CORE:
return (procstat_getpathname_core(procstat->core, pathname,
maxlen));
default:
warnx("unknown access method: %d", procstat->type);
return (-1);
}
}
static int
procstat_getosrel_kvm(kvm_t *kd, struct kinfo_proc *kp, int *osrelp)
{
struct proc proc;
assert(kd != NULL);
assert(kp != NULL);
if (!kvm_read_all(kd, (unsigned long)kp->ki_paddr, &proc,
sizeof(proc))) {
warnx("can't read proc struct at %p for pid %d",
kp->ki_paddr, kp->ki_pid);
return (-1);
}
*osrelp = proc.p_osrel;
return (0);
}
static int
procstat_getosrel_sysctl(pid_t pid, int *osrelp)
{
int error, name[4];
size_t len;
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_OSREL;
name[3] = pid;
len = sizeof(*osrelp);
error = sysctl(name, nitems(name), osrelp, &len, NULL, 0);
if (error != 0 && errno != ESRCH)
warn("sysctl: kern.proc.osrel: %d", pid);
return (error);
}
static int
procstat_getosrel_core(struct procstat_core *core, int *osrelp)
{
size_t len;
int *buf;
buf = procstat_core_get(core, PSC_TYPE_OSREL, NULL, &len);
if (buf == NULL)
return (-1);
if (len < sizeof(*osrelp)) {
free(buf);
return (-1);
}
*osrelp = *buf;
free(buf);
return (0);
}
int
procstat_getosrel(struct procstat *procstat, struct kinfo_proc *kp, int *osrelp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
return (procstat_getosrel_kvm(procstat->kd, kp, osrelp));
case PROCSTAT_SYSCTL:
return (procstat_getosrel_sysctl(kp->ki_pid, osrelp));
case PROCSTAT_CORE:
return (procstat_getosrel_core(procstat->core, osrelp));
default:
warnx("unknown access method: %d", procstat->type);
return (-1);
}
}
#define PROC_AUXV_MAX 256
#if __ELF_WORD_SIZE == 64
static const char *elf32_sv_names[] = {
"Linux ELF32",
"FreeBSD ELF32",
};
static int
is_elf32_sysctl(pid_t pid)
{
int error, name[4];
size_t len, i;
static char sv_name[256];
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_SV_NAME;
name[3] = pid;
len = sizeof(sv_name);
error = sysctl(name, nitems(name), sv_name, &len, NULL, 0);
if (error != 0 || len == 0)
return (0);
for (i = 0; i < sizeof(elf32_sv_names) / sizeof(*elf32_sv_names); i++) {
if (strncmp(sv_name, elf32_sv_names[i], sizeof(sv_name)) == 0)
return (1);
}
return (0);
}
static Elf_Auxinfo *
procstat_getauxv32_sysctl(pid_t pid, unsigned int *cntp)
{
Elf_Auxinfo *auxv;
Elf32_Auxinfo *auxv32;
void *ptr;
size_t len;
unsigned int i, count;
int name[4];
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_AUXV;
name[3] = pid;
len = PROC_AUXV_MAX * sizeof(Elf32_Auxinfo);
auxv = NULL;
auxv32 = malloc(len);
if (auxv32 == NULL) {
warn("malloc(%zu)", len);
goto out;
}
if (sysctl(name, nitems(name), auxv32, &len, NULL, 0) == -1) {
if (errno != ESRCH && errno != EPERM)
warn("sysctl: kern.proc.auxv: %d: %d", pid, errno);
goto out;
}
count = len / sizeof(Elf_Auxinfo);
auxv = malloc(count * sizeof(Elf_Auxinfo));
if (auxv == NULL) {
warn("malloc(%zu)", count * sizeof(Elf_Auxinfo));
goto out;
}
for (i = 0; i < count; i++) {
/*
* XXX: We expect that values for a_type on a 32-bit platform
* are directly mapped to values on 64-bit one, which is not
* necessarily true.
*/
auxv[i].a_type = auxv32[i].a_type;
ptr = &auxv32[i].a_un;
auxv[i].a_un.a_val = *((uint32_t *)ptr);
}
*cntp = count;
out:
free(auxv32);
return (auxv);
}
#endif /* __ELF_WORD_SIZE == 64 */
static Elf_Auxinfo *
procstat_getauxv_sysctl(pid_t pid, unsigned int *cntp)
{
Elf_Auxinfo *auxv;
int name[4];
size_t len;
#if __ELF_WORD_SIZE == 64
if (is_elf32_sysctl(pid))
return (procstat_getauxv32_sysctl(pid, cntp));
#endif
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_AUXV;
name[3] = pid;
len = PROC_AUXV_MAX * sizeof(Elf_Auxinfo);
auxv = malloc(len);
if (auxv == NULL) {
warn("malloc(%zu)", len);
return (NULL);
}
if (sysctl(name, nitems(name), auxv, &len, NULL, 0) == -1) {
if (errno != ESRCH && errno != EPERM)
warn("sysctl: kern.proc.auxv: %d: %d", pid, errno);
free(auxv);
return (NULL);
}
*cntp = len / sizeof(Elf_Auxinfo);
return (auxv);
}
static Elf_Auxinfo *
procstat_getauxv_core(struct procstat_core *core, unsigned int *cntp)
{
Elf_Auxinfo *auxv;
size_t len;
auxv = procstat_core_get(core, PSC_TYPE_AUXV, NULL, &len);
if (auxv == NULL)
return (NULL);
*cntp = len / sizeof(Elf_Auxinfo);
return (auxv);
}
Elf_Auxinfo *
procstat_getauxv(struct procstat *procstat, struct kinfo_proc *kp,
unsigned int *cntp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
warnx("kvm method is not supported");
return (NULL);
case PROCSTAT_SYSCTL:
return (procstat_getauxv_sysctl(kp->ki_pid, cntp));
case PROCSTAT_CORE:
return (procstat_getauxv_core(procstat->core, cntp));
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freeauxv(struct procstat *procstat __unused, Elf_Auxinfo *auxv)
{
free(auxv);
}
static struct ptrace_lwpinfo *
procstat_getptlwpinfo_core(struct procstat_core *core, unsigned int *cntp)
{
void *buf;
struct ptrace_lwpinfo *pl;
unsigned int cnt;
size_t len;
cnt = procstat_core_note_count(core, PSC_TYPE_PTLWPINFO);
if (cnt == 0)
return (NULL);
len = cnt * sizeof(*pl);
buf = calloc(1, len);
pl = procstat_core_get(core, PSC_TYPE_PTLWPINFO, buf, &len);
if (pl == NULL) {
free(buf);
return (NULL);
}
*cntp = len / sizeof(*pl);
return (pl);
}
struct ptrace_lwpinfo *
procstat_getptlwpinfo(struct procstat *procstat, unsigned int *cntp)
{
switch (procstat->type) {
case PROCSTAT_KVM:
warnx("kvm method is not supported");
return (NULL);
case PROCSTAT_SYSCTL:
warnx("sysctl method is not supported");
return (NULL);
case PROCSTAT_CORE:
return (procstat_getptlwpinfo_core(procstat->core, cntp));
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freeptlwpinfo(struct procstat *procstat __unused,
struct ptrace_lwpinfo *pl)
{
free(pl);
}
static struct kinfo_kstack *
procstat_getkstack_sysctl(pid_t pid, int *cntp)
{
struct kinfo_kstack *kkstp;
int error, name[4];
size_t len;
name[0] = CTL_KERN;
name[1] = KERN_PROC;
name[2] = KERN_PROC_KSTACK;
name[3] = pid;
len = 0;
error = sysctl(name, nitems(name), NULL, &len, NULL, 0);
if (error < 0 && errno != ESRCH && errno != EPERM && errno != ENOENT) {
warn("sysctl: kern.proc.kstack: %d", pid);
return (NULL);
}
if (error == -1 && errno == ENOENT) {
warnx("sysctl: kern.proc.kstack unavailable"
" (options DDB or options STACK required in kernel)");
return (NULL);
}
if (error == -1)
return (NULL);
kkstp = malloc(len);
if (kkstp == NULL) {
warn("malloc(%zu)", len);
return (NULL);
}
if (sysctl(name, nitems(name), kkstp, &len, NULL, 0) == -1) {
warn("sysctl: kern.proc.pid: %d", pid);
free(kkstp);
return (NULL);
}
*cntp = len / sizeof(*kkstp);
return (kkstp);
}
struct kinfo_kstack *
procstat_getkstack(struct procstat *procstat, struct kinfo_proc *kp,
unsigned int *cntp)
{
switch(procstat->type) {
case PROCSTAT_KVM:
warnx("kvm method is not supported");
return (NULL);
case PROCSTAT_SYSCTL:
return (procstat_getkstack_sysctl(kp->ki_pid, cntp));
case PROCSTAT_CORE:
warnx("core method is not supported");
return (NULL);
default:
warnx("unknown access method: %d", procstat->type);
return (NULL);
}
}
void
procstat_freekstack(struct procstat *procstat __unused,
struct kinfo_kstack *kkstp)
{
free(kkstp);
}
Index: head/sys/compat/linprocfs/linprocfs.c
===================================================================
--- head/sys/compat/linprocfs/linprocfs.c (revision 358502)
+++ head/sys/compat/linprocfs/linprocfs.c (revision 358503)
@@ -1,1831 +1,1819 @@
/*-
* SPDX-License-Identifier: BSD-4-Clause
*
* Copyright (c) 2000 Dag-Erling Coïdan Smørgrav
* Copyright (c) 1999 Pierre Beyssac
* Copyright (c) 1993 Jan-Simon Pendry
* Copyright (c) 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Jan-Simon Pendry.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)procfs_status.c 8.4 (Berkeley) 6/15/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/queue.h>
#include <sys/blist.h>
#include <sys/conf.h>
#include <sys/exec.h>
#include <sys/fcntl.h>
#include <sys/filedesc.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/linker.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/msg.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/proc.h>
#include <sys/ptrace.h>
#include <sys/resourcevar.h>
#include <sys/resource.h>
#include <sys/sbuf.h>
#include <sys/sem.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/time.h>
#include <sys/tty.h>
#include <sys/user.h>
#include <sys/uuid.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <sys/bus.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/if_types.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_param.h>
#include <vm/vm_object.h>
#include <vm/swap_pager.h>
#include <machine/clock.h>
#include <geom/geom.h>
#include <geom/geom_int.h>
#if defined(__i386__) || defined(__amd64__)
#include <machine/cputypes.h>
#include <machine/md_var.h>
#endif /* __i386__ || __amd64__ */
#include <compat/linux/linux.h>
#include <compat/linux/linux_mib.h>
#include <compat/linux/linux_misc.h>
#include <compat/linux/linux_util.h>
#include <fs/pseudofs/pseudofs.h>
#include <fs/procfs/procfs.h>
/*
* Various conversion macros
*/
#define T2J(x) ((long)(((x) * 100ULL) / (stathz ? stathz : hz))) /* ticks to jiffies */
#define T2CS(x) ((unsigned long)(((x) * 100ULL) / (stathz ? stathz : hz))) /* ticks to centiseconds */
#define T2S(x) ((x) / (stathz ? stathz : hz)) /* ticks to seconds */
#define B2K(x) ((x) >> 10) /* bytes to kbytes */
#define B2P(x) ((x) >> PAGE_SHIFT) /* bytes to pages */
#define P2B(x) ((x) << PAGE_SHIFT) /* pages to bytes */
#define P2K(x) ((x) << (PAGE_SHIFT - 10)) /* pages to kbytes */
#define TV2J(x) ((x)->tv_sec * 100UL + (x)->tv_usec / 10000)
/**
* @brief Mapping of ki_stat in struct kinfo_proc to the linux state
*
* The linux procfs state field displays one of the characters RSDZTW to
* denote running, sleeping in an interruptible wait, waiting in an
* uninterruptible disk sleep, a zombie process, process is being traced
* or stopped, or process is paging respectively.
*
* Our struct kinfo_proc contains the variable ki_stat which contains a
* value out of SIDL, SRUN, SSLEEP, SSTOP, SZOMB, SWAIT and SLOCK.
*
* This character array is used with ki_stati-1 as an index and tries to
* map our states to suitable linux states.
*/
static char linux_state[] = "RRSTZDD";
/*
* Filler function for proc/meminfo
*/
static int
linprocfs_domeminfo(PFS_FILL_ARGS)
{
unsigned long memtotal; /* total memory in bytes */
unsigned long memused; /* used memory in bytes */
unsigned long memfree; /* free memory in bytes */
unsigned long buffers, cached; /* buffer / cache memory ??? */
unsigned long long swaptotal; /* total swap space in bytes */
unsigned long long swapused; /* used swap space in bytes */
unsigned long long swapfree; /* free swap space in bytes */
int i, j;
memtotal = physmem * PAGE_SIZE;
/*
* The correct thing here would be:
*
memfree = vm_free_count() * PAGE_SIZE;
memused = memtotal - memfree;
*
* but it might mislead linux binaries into thinking there
* is very little memory left, so we cheat and tell them that
* all memory that isn't wired down is free.
*/
memused = vm_wire_count() * PAGE_SIZE;
memfree = memtotal - memused;
swap_pager_status(&i, &j);
swaptotal = (unsigned long long)i * PAGE_SIZE;
swapused = (unsigned long long)j * PAGE_SIZE;
swapfree = swaptotal - swapused;
/*
* We'd love to be able to write:
*
buffers = bufspace;
*
* but bufspace is internal to vfs_bio.c and we don't feel
* like unstaticizing it just for linprocfs's sake.
*/
buffers = 0;
cached = vm_inactive_count() * PAGE_SIZE;
sbuf_printf(sb,
"MemTotal: %9lu kB\n"
"MemFree: %9lu kB\n"
"Buffers: %9lu kB\n"
"Cached: %9lu kB\n"
"SwapTotal:%9llu kB\n"
"SwapFree: %9llu kB\n",
B2K(memtotal), B2K(memfree), B2K(buffers),
B2K(cached), B2K(swaptotal), B2K(swapfree));
return (0);
}
#if defined(__i386__) || defined(__amd64__)
/*
* Filler function for proc/cpuinfo (i386 & amd64 version)
*/
static int
linprocfs_docpuinfo(PFS_FILL_ARGS)
{
int hw_model[2];
char model[128];
uint64_t freq;
size_t size;
u_int cache_size[4];
int fqmhz, fqkhz;
int i, j;
/*
* We default the flags to include all non-conflicting flags,
* and the Intel versions of conflicting flags.
*/
static char *cpu_feature_names[] = {
/* 0 */ "fpu", "vme", "de", "pse",
/* 4 */ "tsc", "msr", "pae", "mce",
/* 8 */ "cx8", "apic", "", "sep",
/* 12 */ "mtrr", "pge", "mca", "cmov",
/* 16 */ "pat", "pse36", "pn", "clflush",
/* 20 */ "", "dts", "acpi", "mmx",
/* 24 */ "fxsr", "sse", "sse2", "ss",
/* 28 */ "ht", "tm", "ia64", "pbe"
};
static char *amd_feature_names[] = {
/* 0 */ "", "", "", "",
/* 4 */ "", "", "", "",
/* 8 */ "", "", "", "syscall",
/* 12 */ "", "", "", "",
/* 16 */ "", "", "", "mp",
/* 20 */ "nx", "", "mmxext", "",
/* 24 */ "", "fxsr_opt", "pdpe1gb", "rdtscp",
/* 28 */ "", "lm", "3dnowext", "3dnow"
};
static char *cpu_feature2_names[] = {
/* 0 */ "pni", "pclmulqdq", "dtes3", "monitor",
/* 4 */ "ds_cpl", "vmx", "smx", "est",
/* 8 */ "tm2", "ssse3", "cid", "sdbg",
/* 12 */ "fma", "cx16", "xptr", "pdcm",
/* 16 */ "", "pcid", "dca", "sse4_1",
/* 20 */ "sse4_2", "x2apic", "movbe", "popcnt",
/* 24 */ "tsc_deadline_timer", "aes", "xsave", "",
/* 28 */ "avx", "f16c", "rdrand", "hypervisor"
};
static char *amd_feature2_names[] = {
/* 0 */ "lahf_lm", "cmp_legacy", "svm", "extapic",
/* 4 */ "cr8_legacy", "abm", "sse4a", "misalignsse",
/* 8 */ "3dnowprefetch", "osvw", "ibs", "xop",
/* 12 */ "skinit", "wdt", "", "lwp",
/* 16 */ "fma4", "tce", "", "nodeid_msr",
/* 20 */ "", "tbm", "topoext", "perfctr_core",
/* 24 */ "perfctr_nb", "", "bpext", "ptsc",
/* 28 */ "perfctr_llc", "mwaitx", "", ""
};
static char *cpu_stdext_feature_names[] = {
/* 0 */ "fsgsbase", "tsc_adjust", "", "bmi1",
/* 4 */ "hle", "avx2", "", "smep",
/* 8 */ "bmi2", "erms", "invpcid", "rtm",
/* 12 */ "cqm", "", "mpx", "rdt_a",
/* 16 */ "avx512f", "avx512dq", "rdseed", "adx",
/* 20 */ "smap", "avx512ifma", "", "clflushopt",
/* 24 */ "clwb", "intel_pt", "avx512pf", "avx512er",
/* 28 */ "avx512cd", "sha_ni", "avx512bw", "avx512vl"
};
static char *power_flags[] = {
"ts", "fid", "vid",
"ttp", "tm", "stc",
"100mhzsteps", "hwpstate", "",
"cpb", "eff_freq_ro", "proc_feedback",
"acc_power",
};
hw_model[0] = CTL_HW;
hw_model[1] = HW_MODEL;
model[0] = '\0';
size = sizeof(model);
if (kernel_sysctl(td, hw_model, 2, &model, &size, 0, 0, 0, 0) != 0)
strcpy(model, "unknown");
#ifdef __i386__
switch (cpu_vendor_id) {
case CPU_VENDOR_AMD:
if (cpu_class < CPUCLASS_686)
cpu_feature_names[16] = "fcmov";
break;
case CPU_VENDOR_CYRIX:
cpu_feature_names[24] = "cxmmx";
break;
}
#endif
if (cpu_exthigh >= 0x80000006)
do_cpuid(0x80000006, cache_size);
else
memset(cache_size, 0, sizeof(cache_size));
for (i = 0; i < mp_ncpus; ++i) {
fqmhz = 0;
fqkhz = 0;
freq = atomic_load_acq_64(&tsc_freq);
if (freq != 0) {
fqmhz = (freq + 4999) / 1000000;
fqkhz = ((freq + 4999) / 10000) % 100;
}
sbuf_printf(sb,
"processor\t: %d\n"
"vendor_id\t: %.20s\n"
"cpu family\t: %u\n"
"model\t\t: %u\n"
"model name\t: %s\n"
"stepping\t: %u\n"
"cpu MHz\t\t: %d.%02d\n"
"cache size\t: %d KB\n"
"physical id\t: %d\n"
"siblings\t: %d\n"
"core id\t\t: %d\n"
"cpu cores\t: %d\n"
"apicid\t\t: %d\n"
"initial apicid\t: %d\n"
"fpu\t\t: %s\n"
"fpu_exception\t: %s\n"
"cpuid level\t: %d\n"
"wp\t\t: %s\n",
i, cpu_vendor, CPUID_TO_FAMILY(cpu_id),
CPUID_TO_MODEL(cpu_id), model, cpu_id & CPUID_STEPPING,
fqmhz, fqkhz,
(cache_size[2] >> 16), 0, mp_ncpus, i, mp_ncpus,
i, i, /*cpu_id & CPUID_LOCAL_APIC_ID ??*/
(cpu_feature & CPUID_FPU) ? "yes" : "no", "yes",
CPUID_TO_FAMILY(cpu_id), "yes");
sbuf_cat(sb, "flags\t\t:");
for (j = 0; j < nitems(cpu_feature_names); j++)
if (cpu_feature & (1 << j) &&
cpu_feature_names[j][0] != '\0')
sbuf_printf(sb, " %s", cpu_feature_names[j]);
for (j = 0; j < nitems(amd_feature_names); j++)
if (amd_feature & (1 << j) &&
amd_feature_names[j][0] != '\0')
sbuf_printf(sb, " %s", amd_feature_names[j]);
for (j = 0; j < nitems(cpu_feature2_names); j++)
if (cpu_feature2 & (1 << j) &&
cpu_feature2_names[j][0] != '\0')
sbuf_printf(sb, " %s", cpu_feature2_names[j]);
for (j = 0; j < nitems(amd_feature2_names); j++)
if (amd_feature2 & (1 << j) &&
amd_feature2_names[j][0] != '\0')
sbuf_printf(sb, " %s", amd_feature2_names[j]);
for (j = 0; j < nitems(cpu_stdext_feature_names); j++)
if (cpu_stdext_feature & (1 << j) &&
cpu_stdext_feature_names[j][0] != '\0')
sbuf_printf(sb, " %s",
cpu_stdext_feature_names[j]);
sbuf_cat(sb, "\n");
sbuf_printf(sb,
"bugs\t\t: %s\n"
"bogomips\t: %d.%02d\n"
"clflush size\t: %d\n"
"cache_alignment\t: %d\n"
"address sizes\t: %d bits physical, %d bits virtual\n",
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
(has_f00f_bug) ? "Intel F00F" : "",
#else
"",
#endif
fqmhz, fqkhz,
cpu_clflush_line_size, cpu_clflush_line_size,
cpu_maxphyaddr,
(cpu_maxphyaddr > 32) ? 48 : 0);
sbuf_cat(sb, "power management: ");
for (j = 0; j < nitems(power_flags); j++)
if (amd_pminfo & (1 << j))
sbuf_printf(sb, " %s", power_flags[j]);
sbuf_cat(sb, "\n\n");
/* XXX per-cpu vendor / class / model / id? */
}
sbuf_cat(sb, "\n");
return (0);
}
#else
/* ARM64TODO: implement non-stubbed linprocfs_docpuinfo */
static int
linprocfs_docpuinfo(PFS_FILL_ARGS)
{
int i;
for (i = 0; i < mp_ncpus; ++i) {
sbuf_printf(sb,
"processor\t: %d\n"
"BogoMIPS\t: %d.%02d\n",
i, 0, 0);
sbuf_cat(sb, "Features\t: ");
sbuf_cat(sb, "\n");
sbuf_printf(sb,
"CPU implementer\t: \n"
"CPU architecture: \n"
"CPU variant\t: 0x%x\n"
"CPU part\t: 0x%x\n"
"CPU revision\t: %d\n",
0, 0, 0);
sbuf_cat(sb, "\n");
}
return (0);
}
#endif /* __i386__ || __amd64__ */
/*
* Filler function for proc/mtab
*
* This file doesn't exist in Linux' procfs, but is included here so
* users can symlink /compat/linux/etc/mtab to /proc/mtab
*/
static int
linprocfs_domtab(PFS_FILL_ARGS)
{
struct nameidata nd;
const char *lep;
char *dlep, *flep, *mntto, *mntfrom, *fstype;
size_t lep_len;
int error;
struct statfs *buf, *sp;
size_t count;
/* resolve symlinks etc. in the emulation tree prefix */
NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, linux_emul_path, td);
flep = NULL;
error = namei(&nd);
lep = linux_emul_path;
if (error == 0) {
if (vn_fullpath(td, nd.ni_vp, &dlep, &flep) == 0)
lep = dlep;
vrele(nd.ni_vp);
}
lep_len = strlen(lep);
buf = NULL;
error = kern_getfsstat(td, &buf, SIZE_T_MAX, &count,
UIO_SYSSPACE, MNT_WAIT);
if (error != 0) {
free(buf, M_TEMP);
free(flep, M_TEMP);
return (error);
}
for (sp = buf; count > 0; sp++, count--) {
/* determine device name */
mntfrom = sp->f_mntfromname;
/* determine mount point */
mntto = sp->f_mntonname;
if (strncmp(mntto, lep, lep_len) == 0 && mntto[lep_len] == '/')
mntto += lep_len;
/* determine fs type */
fstype = sp->f_fstypename;
if (strcmp(fstype, pn->pn_info->pi_name) == 0)
mntfrom = fstype = "proc";
else if (strcmp(fstype, "procfs") == 0)
continue;
if (strcmp(fstype, "autofs") == 0) {
/*
* FreeBSD uses eg "map -hosts", whereas Linux
* expects just "-hosts".
*/
if (strncmp(mntfrom, "map ", 4) == 0)
mntfrom += 4;
}
if (strcmp(fstype, "linsysfs") == 0) {
sbuf_printf(sb, "/sys %s sysfs %s", mntto,
sp->f_flags & MNT_RDONLY ? "ro" : "rw");
} else {
/* For Linux msdosfs is called vfat */
if (strcmp(fstype, "msdosfs") == 0)
fstype = "vfat";
sbuf_printf(sb, "%s %s %s %s", mntfrom, mntto, fstype,
sp->f_flags & MNT_RDONLY ? "ro" : "rw");
}
#define ADD_OPTION(opt, name) \
if (sp->f_flags & (opt)) sbuf_printf(sb, "," name);
ADD_OPTION(MNT_SYNCHRONOUS, "sync");
ADD_OPTION(MNT_NOEXEC, "noexec");
ADD_OPTION(MNT_NOSUID, "nosuid");
ADD_OPTION(MNT_UNION, "union");
ADD_OPTION(MNT_ASYNC, "async");
ADD_OPTION(MNT_SUIDDIR, "suiddir");
ADD_OPTION(MNT_NOSYMFOLLOW, "nosymfollow");
ADD_OPTION(MNT_NOATIME, "noatime");
#undef ADD_OPTION
/* a real Linux mtab will also show NFS options */
sbuf_printf(sb, " 0 0\n");
}
free(buf, M_TEMP);
free(flep, M_TEMP);
return (error);
}
/*
* Filler function for proc/partitions
*/
static int
linprocfs_dopartitions(PFS_FILL_ARGS)
{
struct g_class *cp;
struct g_geom *gp;
struct g_provider *pp;
int major, minor;
g_topology_lock();
sbuf_printf(sb, "major minor #blocks name rio rmerge rsect "
"ruse wio wmerge wsect wuse running use aveq\n");
LIST_FOREACH(cp, &g_classes, class) {
if (strcmp(cp->name, "DISK") == 0 ||
strcmp(cp->name, "PART") == 0)
LIST_FOREACH(gp, &cp->geom, geom) {
LIST_FOREACH(pp, &gp->provider, provider) {
if (linux_driver_get_major_minor(
pp->name, &major, &minor) != 0) {
major = 0;
minor = 0;
}
sbuf_printf(sb, "%d %d %lld %s "
"%d %d %d %d %d "
"%d %d %d %d %d %d\n",
major, minor,
(long long)pp->mediasize, pp->name,
0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0);
}
}
}
g_topology_unlock();
return (0);
}
/*
* Filler function for proc/stat
*
* Output depends on kernel version:
*
* v2.5.40 <=
* user nice system idle
* v2.5.41
* user nice system idle iowait
* v2.6.11
* user nice system idle iowait irq softirq steal
* v2.6.24
* user nice system idle iowait irq softirq steal guest
* v2.6.33 >=
* user nice system idle iowait irq softirq steal guest guest_nice
*/
static int
linprocfs_dostat(PFS_FILL_ARGS)
{
struct pcpu *pcpu;
long cp_time[CPUSTATES];
long *cp;
struct timeval boottime;
int i;
char *zero_pad;
bool has_intr = true;
if (linux_kernver(td) >= LINUX_KERNVER(2,6,33)) {
zero_pad = " 0 0 0 0\n";
} else if (linux_kernver(td) >= LINUX_KERNVER(2,6,24)) {
zero_pad = " 0 0 0\n";
} else if (linux_kernver(td) >= LINUX_KERNVER(2,6,11)) {
zero_pad = " 0 0\n";
} else if (linux_kernver(td) >= LINUX_KERNVER(2,5,41)) {
has_intr = false;
zero_pad = " 0\n";
} else {
has_intr = false;
zero_pad = "\n";
}
read_cpu_time(cp_time);
getboottime(&boottime);
/* Parameters common to all versions */
sbuf_printf(sb, "cpu %lu %lu %lu %lu",
T2J(cp_time[CP_USER]),
T2J(cp_time[CP_NICE]),
T2J(cp_time[CP_SYS]),
T2J(cp_time[CP_IDLE]));
/* Print interrupt stats if available */
if (has_intr) {
sbuf_printf(sb, " 0 %lu", T2J(cp_time[CP_INTR]));
}
/* Pad out remaining fields depending on version */
sbuf_printf(sb, "%s", zero_pad);
CPU_FOREACH(i) {
pcpu = pcpu_find(i);
cp = pcpu->pc_cp_time;
sbuf_printf(sb, "cpu%d %lu %lu %lu %lu", i,
T2J(cp[CP_USER]),
T2J(cp[CP_NICE]),
T2J(cp[CP_SYS]),
T2J(cp[CP_IDLE]));
if (has_intr) {
sbuf_printf(sb, " 0 %lu", T2J(cp[CP_INTR]));
}
sbuf_printf(sb, "%s", zero_pad);
}
sbuf_printf(sb,
"disk 0 0 0 0\n"
"page %ju %ju\n"
"swap %ju %ju\n"
"intr %ju\n"
"ctxt %ju\n"
"btime %lld\n",
(uintmax_t)VM_CNT_FETCH(v_vnodepgsin),
(uintmax_t)VM_CNT_FETCH(v_vnodepgsout),
(uintmax_t)VM_CNT_FETCH(v_swappgsin),
(uintmax_t)VM_CNT_FETCH(v_swappgsout),
(uintmax_t)VM_CNT_FETCH(v_intr),
(uintmax_t)VM_CNT_FETCH(v_swtch),
(long long)boottime.tv_sec);
return (0);
}
static int
linprocfs_doswaps(PFS_FILL_ARGS)
{
struct xswdev xsw;
uintmax_t total, used;
int n;
char devname[SPECNAMELEN + 1];
sbuf_printf(sb, "Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
for (n = 0; ; n++) {
if (swap_dev_info(n, &xsw, devname, sizeof(devname)) != 0)
break;
total = (uintmax_t)xsw.xsw_nblks * PAGE_SIZE / 1024;
used = (uintmax_t)xsw.xsw_used * PAGE_SIZE / 1024;
/*
* The space and not tab after the device name is on
* purpose. Linux does so.
*/
sbuf_printf(sb, "/dev/%-34s unknown\t\t%jd\t%jd\t-1\n",
devname, total, used);
}
return (0);
}
/*
* Filler function for proc/uptime
*/
static int
linprocfs_douptime(PFS_FILL_ARGS)
{
long cp_time[CPUSTATES];
struct timeval tv;
getmicrouptime(&tv);
read_cpu_time(cp_time);
sbuf_printf(sb, "%lld.%02ld %ld.%02lu\n",
(long long)tv.tv_sec, tv.tv_usec / 10000,
T2S(cp_time[CP_IDLE] / mp_ncpus),
T2CS(cp_time[CP_IDLE] / mp_ncpus) % 100);
return (0);
}
/*
* Get OS build date
*/
static void
linprocfs_osbuild(struct thread *td, struct sbuf *sb)
{
#if 0
char osbuild[256];
char *cp1, *cp2;
strncpy(osbuild, version, 256);
osbuild[255] = '\0';
cp1 = strstr(osbuild, "\n");
cp2 = strstr(osbuild, ":");
if (cp1 && cp2) {
*cp1 = *cp2 = '\0';
cp1 = strstr(osbuild, "#");
} else
cp1 = NULL;
if (cp1)
sbuf_printf(sb, "%s%s", cp1, cp2 + 1);
else
#endif
sbuf_cat(sb, "#4 Sun Dec 18 04:30:00 CET 1977");
}
/*
* Get OS builder
*/
static void
linprocfs_osbuilder(struct thread *td, struct sbuf *sb)
{
#if 0
char builder[256];
char *cp;
cp = strstr(version, "\n ");
if (cp) {
strncpy(builder, cp + 5, 256);
builder[255] = '\0';
cp = strstr(builder, ":");
if (cp)
*cp = '\0';
}
if (cp)
sbuf_cat(sb, builder);
else
#endif
sbuf_cat(sb, "des@freebsd.org");
}
/*
* Filler function for proc/version
*/
static int
linprocfs_doversion(PFS_FILL_ARGS)
{
char osname[LINUX_MAX_UTSNAME];
char osrelease[LINUX_MAX_UTSNAME];
linux_get_osname(td, osname);
linux_get_osrelease(td, osrelease);
sbuf_printf(sb, "%s version %s (", osname, osrelease);
linprocfs_osbuilder(td, sb);
sbuf_cat(sb, ") (gcc version " __VERSION__ ") ");
linprocfs_osbuild(td, sb);
sbuf_cat(sb, "\n");
return (0);
}
/*
* Filler function for proc/loadavg
*/
static int
linprocfs_doloadavg(PFS_FILL_ARGS)
{
sbuf_printf(sb,
"%d.%02d %d.%02d %d.%02d %d/%d %d\n",
(int)(averunnable.ldavg[0] / averunnable.fscale),
(int)(averunnable.ldavg[0] * 100 / averunnable.fscale % 100),
(int)(averunnable.ldavg[1] / averunnable.fscale),
(int)(averunnable.ldavg[1] * 100 / averunnable.fscale % 100),
(int)(averunnable.ldavg[2] / averunnable.fscale),
(int)(averunnable.ldavg[2] * 100 / averunnable.fscale % 100),
1, /* number of running tasks */
nprocs, /* number of tasks */
lastpid /* the last pid */
);
return (0);
}
/*
* Filler function for proc/pid/stat
*/
static int
linprocfs_doprocstat(PFS_FILL_ARGS)
{
struct kinfo_proc kp;
struct timeval boottime;
char state;
static int ratelimit = 0;
vm_offset_t startcode, startdata;
getboottime(&boottime);
sx_slock(&proctree_lock);
PROC_LOCK(p);
fill_kinfo_proc(p, &kp);
sx_sunlock(&proctree_lock);
if (p->p_vmspace) {
startcode = (vm_offset_t)p->p_vmspace->vm_taddr;
startdata = (vm_offset_t)p->p_vmspace->vm_daddr;
} else {
startcode = 0;
startdata = 0;
}
sbuf_printf(sb, "%d", p->p_pid);
#define PS_ADD(name, fmt, arg) sbuf_printf(sb, " " fmt, arg)
PS_ADD("comm", "(%s)", p->p_comm);
if (kp.ki_stat > sizeof(linux_state)) {
state = 'R';
if (ratelimit == 0) {
printf("linprocfs: don't know how to handle unknown FreeBSD state %d/%zd, mapping to R\n",
kp.ki_stat, sizeof(linux_state));
++ratelimit;
}
} else
state = linux_state[kp.ki_stat - 1];
PS_ADD("state", "%c", state);
PS_ADD("ppid", "%d", p->p_pptr ? p->p_pptr->p_pid : 0);
PS_ADD("pgrp", "%d", p->p_pgid);
PS_ADD("session", "%d", p->p_session->s_sid);
PROC_UNLOCK(p);
if (kp.ki_tdev == NODEV)
PS_ADD("tty", "%s", "-1");
else
PS_ADD("tty", "%ju", (uintmax_t)kp.ki_tdev);
PS_ADD("tpgid", "%d", kp.ki_tpgid);
PS_ADD("flags", "%u", 0); /* XXX */
PS_ADD("minflt", "%lu", kp.ki_rusage.ru_minflt);
PS_ADD("cminflt", "%lu", kp.ki_rusage_ch.ru_minflt);
PS_ADD("majflt", "%lu", kp.ki_rusage.ru_majflt);
PS_ADD("cmajflt", "%lu", kp.ki_rusage_ch.ru_majflt);
PS_ADD("utime", "%ld", TV2J(&kp.ki_rusage.ru_utime));
PS_ADD("stime", "%ld", TV2J(&kp.ki_rusage.ru_stime));
PS_ADD("cutime", "%ld", TV2J(&kp.ki_rusage_ch.ru_utime));
PS_ADD("cstime", "%ld", TV2J(&kp.ki_rusage_ch.ru_stime));
PS_ADD("priority", "%d", kp.ki_pri.pri_user);
PS_ADD("nice", "%d", kp.ki_nice); /* 19 (nicest) to -19 */
PS_ADD("0", "%d", 0); /* removed field */
PS_ADD("itrealvalue", "%d", 0); /* XXX */
PS_ADD("starttime", "%lu", TV2J(&kp.ki_start) - TV2J(&boottime));
PS_ADD("vsize", "%ju", P2K((uintmax_t)kp.ki_size));
PS_ADD("rss", "%ju", (uintmax_t)kp.ki_rssize);
PS_ADD("rlim", "%lu", kp.ki_rusage.ru_maxrss);
PS_ADD("startcode", "%ju", (uintmax_t)startcode);
PS_ADD("endcode", "%ju", (uintmax_t)startdata);
PS_ADD("startstack", "%u", 0); /* XXX */
PS_ADD("kstkesp", "%u", 0); /* XXX */
PS_ADD("kstkeip", "%u", 0); /* XXX */
PS_ADD("signal", "%u", 0); /* XXX */
PS_ADD("blocked", "%u", 0); /* XXX */
PS_ADD("sigignore", "%u", 0); /* XXX */
PS_ADD("sigcatch", "%u", 0); /* XXX */
PS_ADD("wchan", "%u", 0); /* XXX */
PS_ADD("nswap", "%lu", kp.ki_rusage.ru_nswap);
PS_ADD("cnswap", "%lu", kp.ki_rusage_ch.ru_nswap);
PS_ADD("exitsignal", "%d", 0); /* XXX */
PS_ADD("processor", "%u", kp.ki_lastcpu);
PS_ADD("rt_priority", "%u", 0); /* XXX */ /* >= 2.5.19 */
PS_ADD("policy", "%u", kp.ki_pri.pri_class); /* >= 2.5.19 */
#undef PS_ADD
sbuf_putc(sb, '\n');
return (0);
}
/*
* Filler function for proc/pid/statm
*/
static int
linprocfs_doprocstatm(PFS_FILL_ARGS)
{
struct kinfo_proc kp;
segsz_t lsize;
sx_slock(&proctree_lock);
PROC_LOCK(p);
fill_kinfo_proc(p, &kp);
PROC_UNLOCK(p);
sx_sunlock(&proctree_lock);
/*
* See comments in linprocfs_doprocstatus() regarding the
* computation of lsize.
*/
/* size resident share trs drs lrs dt */
sbuf_printf(sb, "%ju ", B2P((uintmax_t)kp.ki_size));
sbuf_printf(sb, "%ju ", (uintmax_t)kp.ki_rssize);
sbuf_printf(sb, "%ju ", (uintmax_t)0); /* XXX */
sbuf_printf(sb, "%ju ", (uintmax_t)kp.ki_tsize);
sbuf_printf(sb, "%ju ", (uintmax_t)(kp.ki_dsize + kp.ki_ssize));
lsize = B2P(kp.ki_size) - kp.ki_dsize -
kp.ki_ssize - kp.ki_tsize - 1;
sbuf_printf(sb, "%ju ", (uintmax_t)lsize);
sbuf_printf(sb, "%ju\n", (uintmax_t)0); /* XXX */
return (0);
}
/*
* Filler function for proc/pid/status
*/
static int
linprocfs_doprocstatus(PFS_FILL_ARGS)
{
struct kinfo_proc kp;
char *state;
segsz_t lsize;
struct thread *td2;
struct sigacts *ps;
l_sigset_t siglist, sigignore, sigcatch;
int i;
sx_slock(&proctree_lock);
PROC_LOCK(p);
td2 = FIRST_THREAD_IN_PROC(p); /* XXXKSE pretend only one thread */
if (P_SHOULDSTOP(p)) {
state = "T (stopped)";
} else {
switch(p->p_state) {
case PRS_NEW:
state = "I (idle)";
break;
case PRS_NORMAL:
if (p->p_flag & P_WEXIT) {
state = "X (exiting)";
break;
}
switch(td2->td_state) {
case TDS_INHIBITED:
state = "S (sleeping)";
break;
case TDS_RUNQ:
case TDS_RUNNING:
state = "R (running)";
break;
default:
state = "? (unknown)";
break;
}
break;
case PRS_ZOMBIE:
state = "Z (zombie)";
break;
default:
state = "? (unknown)";
break;
}
}
fill_kinfo_proc(p, &kp);
sx_sunlock(&proctree_lock);
sbuf_printf(sb, "Name:\t%s\n", p->p_comm); /* XXX escape */
sbuf_printf(sb, "State:\t%s\n", state);
/*
* Credentials
*/
sbuf_printf(sb, "Tgid:\t%d\n", p->p_pid);
sbuf_printf(sb, "Pid:\t%d\n", p->p_pid);
sbuf_printf(sb, "PPid:\t%d\n", kp.ki_ppid );
sbuf_printf(sb, "TracerPid:\t%d\n", kp.ki_tracer );
sbuf_printf(sb, "Uid:\t%d %d %d %d\n", p->p_ucred->cr_ruid,
p->p_ucred->cr_uid,
p->p_ucred->cr_svuid,
/* FreeBSD doesn't have fsuid */
p->p_ucred->cr_uid);
sbuf_printf(sb, "Gid:\t%d %d %d %d\n", p->p_ucred->cr_rgid,
p->p_ucred->cr_gid,
p->p_ucred->cr_svgid,
/* FreeBSD doesn't have fsgid */
p->p_ucred->cr_gid);
sbuf_cat(sb, "Groups:\t");
for (i = 0; i < p->p_ucred->cr_ngroups; i++)
sbuf_printf(sb, "%d ", p->p_ucred->cr_groups[i]);
PROC_UNLOCK(p);
sbuf_putc(sb, '\n');
/*
* Memory
*
* While our approximation of VmLib may not be accurate (I
* don't know of a simple way to verify it, and I'm not sure
* it has much meaning anyway), I believe it's good enough.
*
* The same code that could (I think) accurately compute VmLib
* could also compute VmLck, but I don't really care enough to
* implement it. Submissions are welcome.
*/
sbuf_printf(sb, "VmSize:\t%8ju kB\n", B2K((uintmax_t)kp.ki_size));
sbuf_printf(sb, "VmLck:\t%8u kB\n", P2K(0)); /* XXX */
sbuf_printf(sb, "VmRSS:\t%8ju kB\n", P2K((uintmax_t)kp.ki_rssize));
sbuf_printf(sb, "VmData:\t%8ju kB\n", P2K((uintmax_t)kp.ki_dsize));
sbuf_printf(sb, "VmStk:\t%8ju kB\n", P2K((uintmax_t)kp.ki_ssize));
sbuf_printf(sb, "VmExe:\t%8ju kB\n", P2K((uintmax_t)kp.ki_tsize));
lsize = B2P(kp.ki_size) - kp.ki_dsize -
kp.ki_ssize - kp.ki_tsize - 1;
sbuf_printf(sb, "VmLib:\t%8ju kB\n", P2K((uintmax_t)lsize));
/*
* Signal masks
*/
PROC_LOCK(p);
bsd_to_linux_sigset(&p->p_siglist, &siglist);
ps = p->p_sigacts;
mtx_lock(&ps->ps_mtx);
bsd_to_linux_sigset(&ps->ps_sigignore, &sigignore);
bsd_to_linux_sigset(&ps->ps_sigcatch, &sigcatch);
mtx_unlock(&ps->ps_mtx);
PROC_UNLOCK(p);
sbuf_printf(sb, "SigPnd:\t%016jx\n", siglist.__mask);
/*
* XXX. SigBlk - target thread's signal mask, td_sigmask.
* To implement SigBlk pseudofs should support proc/tid dir entries.
*/
sbuf_printf(sb, "SigBlk:\t%016x\n", 0);
sbuf_printf(sb, "SigIgn:\t%016jx\n", sigignore.__mask);
sbuf_printf(sb, "SigCgt:\t%016jx\n", sigcatch.__mask);
/*
* Linux also prints the capability masks, but we don't have
* capabilities yet, and when we do get them they're likely to
* be meaningless to Linux programs, so we lie. XXX
*/
sbuf_printf(sb, "CapInh:\t%016x\n", 0);
sbuf_printf(sb, "CapPrm:\t%016x\n", 0);
sbuf_printf(sb, "CapEff:\t%016x\n", 0);
return (0);
}
/*
* Filler function for proc/pid/cwd
*/
static int
linprocfs_doproccwd(PFS_FILL_ARGS)
{
- struct filedesc *fdp;
- struct vnode *vp;
+ struct pwd *pwd;
char *fullpath = "unknown";
char *freepath = NULL;
- fdp = p->p_fd;
- FILEDESC_SLOCK(fdp);
- vp = fdp->fd_cdir;
- if (vp != NULL)
- VREF(vp);
- FILEDESC_SUNLOCK(fdp);
- vn_fullpath(td, vp, &fullpath, &freepath);
- if (vp != NULL)
- vrele(vp);
+ pwd = pwd_hold(td);
+ vn_fullpath(td, pwd->pwd_cdir, &fullpath, &freepath);
sbuf_printf(sb, "%s", fullpath);
if (freepath)
free(freepath, M_TEMP);
+ pwd_drop(pwd);
return (0);
}
/*
* Filler function for proc/pid/root
*/
static int
linprocfs_doprocroot(PFS_FILL_ARGS)
{
- struct filedesc *fdp;
+ struct pwd *pwd;
struct vnode *vp;
char *fullpath = "unknown";
char *freepath = NULL;
- fdp = p->p_fd;
- FILEDESC_SLOCK(fdp);
- vp = jailed(p->p_ucred) ? fdp->fd_jdir : fdp->fd_rdir;
- if (vp != NULL)
- VREF(vp);
- FILEDESC_SUNLOCK(fdp);
+ pwd = pwd_hold(td);
+ vp = jailed(p->p_ucred) ? pwd->pwd_jdir : pwd->pwd_rdir;
vn_fullpath(td, vp, &fullpath, &freepath);
- if (vp != NULL)
- vrele(vp);
sbuf_printf(sb, "%s", fullpath);
if (freepath)
free(freepath, M_TEMP);
+ pwd_drop(pwd);
return (0);
}
/*
* Filler function for proc/pid/cmdline
*/
static int
linprocfs_doproccmdline(PFS_FILL_ARGS)
{
int ret;
PROC_LOCK(p);
if ((ret = p_cansee(td, p)) != 0) {
PROC_UNLOCK(p);
return (ret);
}
/*
* Mimic linux behavior and pass only processes with usermode
* address space as valid. Return zero silently otherwize.
*/
if (p->p_vmspace == &vmspace0) {
PROC_UNLOCK(p);
return (0);
}
if (p->p_args != NULL) {
sbuf_bcpy(sb, p->p_args->ar_args, p->p_args->ar_length);
PROC_UNLOCK(p);
return (0);
}
if ((p->p_flag & P_SYSTEM) != 0) {
PROC_UNLOCK(p);
return (0);
}
PROC_UNLOCK(p);
ret = proc_getargv(td, p, sb);
return (ret);
}
/*
* Filler function for proc/pid/environ
*/
static int
linprocfs_doprocenviron(PFS_FILL_ARGS)
{
/*
* Mimic linux behavior and pass only processes with usermode
* address space as valid. Return zero silently otherwize.
*/
if (p->p_vmspace == &vmspace0)
return (0);
return (proc_getenvv(td, p, sb));
}
static char l32_map_str[] = "%08lx-%08lx %s%s%s%s %08lx %02x:%02x %lu%s%s\n";
static char l64_map_str[] = "%016lx-%016lx %s%s%s%s %08lx %02x:%02x %lu%s%s\n";
static char vdso_str[] = " [vdso]";
static char stack_str[] = " [stack]";
/*
* Filler function for proc/pid/maps
*/
static int
linprocfs_doprocmaps(PFS_FILL_ARGS)
{
struct vmspace *vm;
vm_map_t map;
vm_map_entry_t entry, tmp_entry;
vm_object_t obj, tobj, lobj;
vm_offset_t e_start, e_end;
vm_ooffset_t off;
vm_prot_t e_prot;
unsigned int last_timestamp;
char *name = "", *freename = NULL;
const char *l_map_str;
ino_t ino;
int ref_count, shadow_count, flags;
int error;
struct vnode *vp;
struct vattr vat;
bool private;
PROC_LOCK(p);
error = p_candebug(td, p);
PROC_UNLOCK(p);
if (error)
return (error);
if (uio->uio_rw != UIO_READ)
return (EOPNOTSUPP);
error = 0;
vm = vmspace_acquire_ref(p);
if (vm == NULL)
return (ESRCH);
if (SV_CURPROC_FLAG(SV_LP64))
l_map_str = l64_map_str;
else
l_map_str = l32_map_str;
map = &vm->vm_map;
vm_map_lock_read(map);
VM_MAP_ENTRY_FOREACH(entry, map) {
name = "";
freename = NULL;
if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
e_prot = entry->protection;
e_start = entry->start;
e_end = entry->end;
obj = entry->object.vm_object;
off = entry->offset;
for (lobj = tobj = obj; tobj != NULL;
lobj = tobj, tobj = tobj->backing_object) {
VM_OBJECT_RLOCK(tobj);
off += lobj->backing_object_offset;
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
}
private = (entry->eflags & MAP_ENTRY_COW) != 0 || obj == NULL ||
(obj->flags & OBJ_ANON) != 0;
last_timestamp = map->timestamp;
vm_map_unlock_read(map);
ino = 0;
if (lobj) {
vp = vm_object_vnode(lobj);
if (vp != NULL)
vref(vp);
if (lobj != obj)
VM_OBJECT_RUNLOCK(lobj);
flags = obj->flags;
ref_count = obj->ref_count;
shadow_count = obj->shadow_count;
VM_OBJECT_RUNLOCK(obj);
if (vp != NULL) {
vn_fullpath(td, vp, &name, &freename);
vn_lock(vp, LK_SHARED | LK_RETRY);
VOP_GETATTR(vp, &vat, td->td_ucred);
ino = vat.va_fileid;
vput(vp);
} else if (SV_PROC_ABI(p) == SV_ABI_LINUX) {
if (e_start == p->p_sysent->sv_shared_page_base)
name = vdso_str;
if (e_end == p->p_sysent->sv_usrstack)
name = stack_str;
}
} else {
flags = 0;
ref_count = 0;
shadow_count = 0;
}
/*
* format:
* start, end, access, offset, major, minor, inode, name.
*/
error = sbuf_printf(sb, l_map_str,
(u_long)e_start, (u_long)e_end,
(e_prot & VM_PROT_READ)?"r":"-",
(e_prot & VM_PROT_WRITE)?"w":"-",
(e_prot & VM_PROT_EXECUTE)?"x":"-",
private ? "p" : "s",
(u_long)off,
0,
0,
(u_long)ino,
*name ? " " : "",
name
);
if (freename)
free(freename, M_TEMP);
vm_map_lock_read(map);
if (error == -1) {
error = 0;
break;
}
if (last_timestamp != map->timestamp) {
/*
* Look again for the entry because the map was
* modified while it was unlocked. Specifically,
* the entry may have been clipped, merged, or deleted.
*/
vm_map_lookup_entry(map, e_end - 1, &tmp_entry);
entry = tmp_entry;
}
}
vm_map_unlock_read(map);
vmspace_free(vm);
return (error);
}
/*
* Criteria for interface name translation
*/
#define IFP_IS_ETH(ifp) (ifp->if_type == IFT_ETHER)
static int
linux_ifname(struct ifnet *ifp, char *buffer, size_t buflen)
{
struct ifnet *ifscan;
int ethno;
IFNET_RLOCK_ASSERT();
/* Short-circuit non ethernet interfaces */
if (!IFP_IS_ETH(ifp))
return (strlcpy(buffer, ifp->if_xname, buflen));
/* Determine the (relative) unit number for ethernet interfaces */
ethno = 0;
CK_STAILQ_FOREACH(ifscan, &V_ifnet, if_link) {
if (ifscan == ifp)
return (snprintf(buffer, buflen, "eth%d", ethno));
if (IFP_IS_ETH(ifscan))
ethno++;
}
return (0);
}
/*
* Filler function for proc/net/dev
*/
static int
linprocfs_donetdev(PFS_FILL_ARGS)
{
char ifname[16]; /* XXX LINUX_IFNAMSIZ */
struct ifnet *ifp;
sbuf_printf(sb, "%6s|%58s|%s\n"
"%6s|%58s|%58s\n",
"Inter-", " Receive", " Transmit",
" face",
"bytes packets errs drop fifo frame compressed multicast",
"bytes packets errs drop fifo colls carrier compressed");
CURVNET_SET(TD_TO_VNET(curthread));
IFNET_RLOCK();
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
linux_ifname(ifp, ifname, sizeof ifname);
sbuf_printf(sb, "%6.6s: ", ifname);
sbuf_printf(sb, "%7ju %7ju %4ju %4ju %4lu %5lu %10lu %9ju ",
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_IBYTES),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_IPACKETS),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_IERRORS),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_IQDROPS),
/* rx_missed_errors */
0UL, /* rx_fifo_errors */
0UL, /* rx_length_errors +
* rx_over_errors +
* rx_crc_errors +
* rx_frame_errors */
0UL, /* rx_compressed */
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_IMCASTS));
/* XXX-BZ rx only? */
sbuf_printf(sb, "%8ju %7ju %4ju %4ju %4lu %5ju %7lu %10lu\n",
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_OBYTES),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_OPACKETS),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_OERRORS),
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_OQDROPS),
0UL, /* tx_fifo_errors */
(uintmax_t )ifp->if_get_counter(ifp, IFCOUNTER_COLLISIONS),
0UL, /* tx_carrier_errors +
* tx_aborted_errors +
* tx_window_errors +
* tx_heartbeat_errors*/
0UL); /* tx_compressed */
}
IFNET_RUNLOCK();
CURVNET_RESTORE();
return (0);
}
/*
* Filler function for proc/sys/kernel/osrelease
*/
static int
linprocfs_doosrelease(PFS_FILL_ARGS)
{
char osrelease[LINUX_MAX_UTSNAME];
linux_get_osrelease(td, osrelease);
sbuf_printf(sb, "%s\n", osrelease);
return (0);
}
/*
* Filler function for proc/sys/kernel/ostype
*/
static int
linprocfs_doostype(PFS_FILL_ARGS)
{
char osname[LINUX_MAX_UTSNAME];
linux_get_osname(td, osname);
sbuf_printf(sb, "%s\n", osname);
return (0);
}
/*
* Filler function for proc/sys/kernel/version
*/
static int
linprocfs_doosbuild(PFS_FILL_ARGS)
{
linprocfs_osbuild(td, sb);
sbuf_cat(sb, "\n");
return (0);
}
/*
* Filler function for proc/sys/kernel/msgmni
*/
static int
linprocfs_domsgmni(PFS_FILL_ARGS)
{
sbuf_printf(sb, "%d\n", msginfo.msgmni);
return (0);
}
/*
* Filler function for proc/sys/kernel/pid_max
*/
static int
linprocfs_dopid_max(PFS_FILL_ARGS)
{
sbuf_printf(sb, "%i\n", PID_MAX);
return (0);
}
/*
* Filler function for proc/sys/kernel/sem
*/
static int
linprocfs_dosem(PFS_FILL_ARGS)
{
sbuf_printf(sb, "%d %d %d %d\n", seminfo.semmsl, seminfo.semmns,
seminfo.semopm, seminfo.semmni);
return (0);
}
/*
* Filler function for proc/sys/vm/min_free_kbytes
*
* This mirrors the approach in illumos to return zero for reads. Effectively,
* it says, no memory is kept in reserve for "atomic allocations". This class
* of allocation can be used at times when a thread cannot be suspended.
*/
static int
linprocfs_dominfree(PFS_FILL_ARGS)
{
sbuf_printf(sb, "%d\n", 0);
return (0);
}
/*
* Filler function for proc/scsi/device_info
*/
static int
linprocfs_doscsidevinfo(PFS_FILL_ARGS)
{
return (0);
}
/*
* Filler function for proc/scsi/scsi
*/
static int
linprocfs_doscsiscsi(PFS_FILL_ARGS)
{
return (0);
}
/*
* Filler function for proc/devices
*/
static int
linprocfs_dodevices(PFS_FILL_ARGS)
{
char *char_devices;
sbuf_printf(sb, "Character devices:\n");
char_devices = linux_get_char_devices();
sbuf_printf(sb, "%s", char_devices);
linux_free_get_char_devices(char_devices);
sbuf_printf(sb, "\nBlock devices:\n");
return (0);
}
/*
* Filler function for proc/cmdline
*/
static int
linprocfs_docmdline(PFS_FILL_ARGS)
{
sbuf_printf(sb, "BOOT_IMAGE=%s", kernelname);
sbuf_printf(sb, " ro root=302\n");
return (0);
}
/*
* Filler function for proc/filesystems
*/
static int
linprocfs_dofilesystems(PFS_FILL_ARGS)
{
struct vfsconf *vfsp;
vfsconf_slock();
TAILQ_FOREACH(vfsp, &vfsconf, vfc_list) {
if (vfsp->vfc_flags & VFCF_SYNTHETIC)
sbuf_printf(sb, "nodev");
sbuf_printf(sb, "\t%s\n", vfsp->vfc_name);
}
vfsconf_sunlock();
return(0);
}
/*
* Filler function for proc/modules
*/
static int
linprocfs_domodules(PFS_FILL_ARGS)
{
#if 0
struct linker_file *lf;
TAILQ_FOREACH(lf, &linker_files, link) {
sbuf_printf(sb, "%-20s%8lu%4d\n", lf->filename,
(unsigned long)lf->size, lf->refs);
}
#endif
return (0);
}
/*
* Filler function for proc/pid/fd
*/
static int
linprocfs_dofdescfs(PFS_FILL_ARGS)
{
if (p == curproc)
sbuf_printf(sb, "/dev/fd");
else
sbuf_printf(sb, "unknown");
return (0);
}
/*
* Filler function for proc/pid/limits
*/
static const struct linux_rlimit_ident {
const char *desc;
const char *unit;
unsigned int rlim_id;
} linux_rlimits_ident[] = {
{ "Max cpu time", "seconds", RLIMIT_CPU },
{ "Max file size", "bytes", RLIMIT_FSIZE },
{ "Max data size", "bytes", RLIMIT_DATA },
{ "Max stack size", "bytes", RLIMIT_STACK },
{ "Max core file size", "bytes", RLIMIT_CORE },
{ "Max resident set", "bytes", RLIMIT_RSS },
{ "Max processes", "processes", RLIMIT_NPROC },
{ "Max open files", "files", RLIMIT_NOFILE },
{ "Max locked memory", "bytes", RLIMIT_MEMLOCK },
{ "Max address space", "bytes", RLIMIT_AS },
{ "Max file locks", "locks", LINUX_RLIMIT_LOCKS },
{ "Max pending signals", "signals", LINUX_RLIMIT_SIGPENDING },
{ "Max msgqueue size", "bytes", LINUX_RLIMIT_MSGQUEUE },
{ "Max nice priority", "", LINUX_RLIMIT_NICE },
{ "Max realtime priority", "", LINUX_RLIMIT_RTPRIO },
{ "Max realtime timeout", "us", LINUX_RLIMIT_RTTIME },
{ 0, 0, 0 }
};
static int
linprocfs_doproclimits(PFS_FILL_ARGS)
{
const struct linux_rlimit_ident *li;
struct plimit *limp;
struct rlimit rl;
ssize_t size;
int res, error;
error = 0;
PROC_LOCK(p);
limp = lim_hold(p->p_limit);
PROC_UNLOCK(p);
size = sizeof(res);
sbuf_printf(sb, "%-26s%-21s%-21s%-21s\n", "Limit", "Soft Limit",
"Hard Limit", "Units");
for (li = linux_rlimits_ident; li->desc != NULL; ++li) {
switch (li->rlim_id)
{
case LINUX_RLIMIT_LOCKS:
/* FALLTHROUGH */
case LINUX_RLIMIT_RTTIME:
rl.rlim_cur = RLIM_INFINITY;
break;
case LINUX_RLIMIT_SIGPENDING:
error = kernel_sysctlbyname(td,
"kern.sigqueue.max_pending_per_proc",
&res, &size, 0, 0, 0, 0);
if (error != 0)
goto out;
rl.rlim_cur = res;
rl.rlim_max = res;
break;
case LINUX_RLIMIT_MSGQUEUE:
error = kernel_sysctlbyname(td,
"kern.ipc.msgmnb", &res, &size, 0, 0, 0, 0);
if (error != 0)
goto out;
rl.rlim_cur = res;
rl.rlim_max = res;
break;
case LINUX_RLIMIT_NICE:
/* FALLTHROUGH */
case LINUX_RLIMIT_RTPRIO:
rl.rlim_cur = 0;
rl.rlim_max = 0;
break;
default:
rl = limp->pl_rlimit[li->rlim_id];
break;
}
if (rl.rlim_cur == RLIM_INFINITY)
sbuf_printf(sb, "%-26s%-21s%-21s%-10s\n",
li->desc, "unlimited", "unlimited", li->unit);
else
sbuf_printf(sb, "%-26s%-21llu%-21llu%-10s\n",
li->desc, (unsigned long long)rl.rlim_cur,
(unsigned long long)rl.rlim_max, li->unit);
}
out:
lim_free(limp);
return (error);
}
/*
* Filler function for proc/sys/kernel/random/uuid
*/
static int
linprocfs_douuid(PFS_FILL_ARGS)
{
struct uuid uuid;
kern_uuidgen(&uuid, 1);
sbuf_printf_uuid(sb, &uuid);
sbuf_printf(sb, "\n");
return(0);
}
/*
* Filler function for proc/pid/auxv
*/
static int
linprocfs_doauxv(PFS_FILL_ARGS)
{
struct sbuf *asb;
off_t buflen, resid;
int error;
/*
* Mimic linux behavior and pass only processes with usermode
* address space as valid. Return zero silently otherwise.
*/
if (p->p_vmspace == &vmspace0)
return (0);
if (uio->uio_resid == 0)
return (0);
if (uio->uio_offset < 0 || uio->uio_resid < 0)
return (EINVAL);
asb = sbuf_new_auto();
if (asb == NULL)
return (ENOMEM);
error = proc_getauxv(td, p, asb);
if (error == 0)
error = sbuf_finish(asb);
resid = sbuf_len(asb) - uio->uio_offset;
if (resid > uio->uio_resid)
buflen = uio->uio_resid;
else
buflen = resid;
if (buflen > IOSIZE_MAX)
return (EINVAL);
if (buflen > MAXPHYS)
buflen = MAXPHYS;
if (resid <= 0)
return (0);
if (error == 0)
error = uiomove(sbuf_data(asb) + uio->uio_offset, buflen, uio);
sbuf_delete(asb);
return (error);
}
/*
* Constructor
*/
static int
linprocfs_init(PFS_INIT_ARGS)
{
struct pfs_node *root;
struct pfs_node *dir;
struct pfs_node *sys;
root = pi->pi_root;
/* /proc/... */
pfs_create_file(root, "cmdline", &linprocfs_docmdline,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "cpuinfo", &linprocfs_docpuinfo,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "devices", &linprocfs_dodevices,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "filesystems", &linprocfs_dofilesystems,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "loadavg", &linprocfs_doloadavg,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "meminfo", &linprocfs_domeminfo,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "modules", &linprocfs_domodules,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "mounts", &linprocfs_domtab,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "mtab", &linprocfs_domtab,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "partitions", &linprocfs_dopartitions,
NULL, NULL, NULL, PFS_RD);
pfs_create_link(root, "self", &procfs_docurproc,
NULL, NULL, NULL, 0);
pfs_create_file(root, "stat", &linprocfs_dostat,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "swaps", &linprocfs_doswaps,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "uptime", &linprocfs_douptime,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(root, "version", &linprocfs_doversion,
NULL, NULL, NULL, PFS_RD);
/* /proc/net/... */
dir = pfs_create_dir(root, "net", NULL, NULL, NULL, 0);
pfs_create_file(dir, "dev", &linprocfs_donetdev,
NULL, NULL, NULL, PFS_RD);
/* /proc/<pid>/... */
dir = pfs_create_dir(root, "pid", NULL, NULL, NULL, PFS_PROCDEP);
pfs_create_file(dir, "cmdline", &linprocfs_doproccmdline,
NULL, NULL, NULL, PFS_RD);
pfs_create_link(dir, "cwd", &linprocfs_doproccwd,
NULL, NULL, NULL, 0);
pfs_create_file(dir, "environ", &linprocfs_doprocenviron,
NULL, &procfs_candebug, NULL, PFS_RD);
pfs_create_link(dir, "exe", &procfs_doprocfile,
NULL, &procfs_notsystem, NULL, 0);
pfs_create_file(dir, "maps", &linprocfs_doprocmaps,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "mem", &procfs_doprocmem,
procfs_attr_rw, &procfs_candebug, NULL, PFS_RDWR | PFS_RAW);
pfs_create_file(dir, "mounts", &linprocfs_domtab,
NULL, NULL, NULL, PFS_RD);
pfs_create_link(dir, "root", &linprocfs_doprocroot,
NULL, NULL, NULL, 0);
pfs_create_file(dir, "stat", &linprocfs_doprocstat,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "statm", &linprocfs_doprocstatm,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "status", &linprocfs_doprocstatus,
NULL, NULL, NULL, PFS_RD);
pfs_create_link(dir, "fd", &linprocfs_dofdescfs,
NULL, NULL, NULL, 0);
pfs_create_file(dir, "auxv", &linprocfs_doauxv,
NULL, &procfs_candebug, NULL, PFS_RD|PFS_RAWRD);
pfs_create_file(dir, "limits", &linprocfs_doproclimits,
NULL, NULL, NULL, PFS_RD);
/* /proc/scsi/... */
dir = pfs_create_dir(root, "scsi", NULL, NULL, NULL, 0);
pfs_create_file(dir, "device_info", &linprocfs_doscsidevinfo,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "scsi", &linprocfs_doscsiscsi,
NULL, NULL, NULL, PFS_RD);
/* /proc/sys/... */
sys = pfs_create_dir(root, "sys", NULL, NULL, NULL, 0);
/* /proc/sys/kernel/... */
dir = pfs_create_dir(sys, "kernel", NULL, NULL, NULL, 0);
pfs_create_file(dir, "osrelease", &linprocfs_doosrelease,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "ostype", &linprocfs_doostype,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "version", &linprocfs_doosbuild,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "msgmni", &linprocfs_domsgmni,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "pid_max", &linprocfs_dopid_max,
NULL, NULL, NULL, PFS_RD);
pfs_create_file(dir, "sem", &linprocfs_dosem,
NULL, NULL, NULL, PFS_RD);
/* /proc/sys/kernel/random/... */
dir = pfs_create_dir(dir, "random", NULL, NULL, NULL, 0);
pfs_create_file(dir, "uuid", &linprocfs_douuid,
NULL, NULL, NULL, PFS_RD);
/* /proc/sys/vm/.... */
dir = pfs_create_dir(sys, "vm", NULL, NULL, NULL, 0);
pfs_create_file(dir, "min_free_kbytes", &linprocfs_dominfree,
NULL, NULL, NULL, PFS_RD);
return (0);
}
/*
* Destructor
*/
static int
linprocfs_uninit(PFS_INIT_ARGS)
{
/* nothing to do, pseudofs will GC */
return (0);
}
PSEUDOFS(linprocfs, 1, VFCF_JAIL);
#if defined(__aarch64__) || defined(__amd64__)
MODULE_DEPEND(linprocfs, linux_common, 1, 1, 1);
#else
MODULE_DEPEND(linprocfs, linux, 1, 1, 1);
#endif
MODULE_DEPEND(linprocfs, procfs, 1, 1, 1);
MODULE_DEPEND(linprocfs, sysvmsg, 1, 1, 1);
MODULE_DEPEND(linprocfs, sysvsem, 1, 1, 1);
Index: head/sys/kern/kern_descrip.c
===================================================================
--- head/sys/kern/kern_descrip.c (revision 358502)
+++ head/sys/kern/kern_descrip.c (revision 358503)
@@ -1,4402 +1,4505 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1991, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)kern_descrip.c 8.6 (Berkeley) 4/19/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_ddb.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/filio.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/selinfo.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/sbuf.h>
#include <sys/signalvar.h>
#include <sys/kdb.h>
#include <sys/stat.h>
#include <sys/sx.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/unistd.h>
#include <sys/user.h>
#include <sys/vnode.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#include <net/vnet.h>
#include <security/audit/audit.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <ddb/ddb.h>
static MALLOC_DEFINE(M_FILEDESC, "filedesc", "Open file descriptor table");
+static MALLOC_DEFINE(M_PWD, "pwd", "Descriptor table vnodes");
static MALLOC_DEFINE(M_FILEDESC_TO_LEADER, "filedesc_to_leader",
"file desc to leader structures");
static MALLOC_DEFINE(M_SIGIO, "sigio", "sigio structures");
MALLOC_DEFINE(M_FILECAPS, "filecaps", "descriptor capabilities");
MALLOC_DECLARE(M_FADVISE);
static __read_mostly uma_zone_t file_zone;
static __read_mostly uma_zone_t filedesc0_zone;
static int closefp(struct filedesc *fdp, int fd, struct file *fp,
struct thread *td, int holdleaders);
static int fd_first_free(struct filedesc *fdp, int low, int size);
static int fd_last_used(struct filedesc *fdp, int size);
static void fdgrowtable(struct filedesc *fdp, int nfd);
static void fdgrowtable_exp(struct filedesc *fdp, int nfd);
static void fdunused(struct filedesc *fdp, int fd);
static void fdused(struct filedesc *fdp, int fd);
static int getmaxfd(struct thread *td);
static u_long *filecaps_copy_prep(const struct filecaps *src);
static void filecaps_copy_finish(const struct filecaps *src,
struct filecaps *dst, u_long *ioctls);
static u_long *filecaps_free_prep(struct filecaps *fcaps);
static void filecaps_free_finish(u_long *ioctls);
+static struct pwd *pwd_alloc(void);
+
/*
* Each process has:
*
* - An array of open file descriptors (fd_ofiles)
* - An array of file flags (fd_ofileflags)
* - A bitmap recording which descriptors are in use (fd_map)
*
* A process starts out with NDFILE descriptors. The value of NDFILE has
* been selected based the historical limit of 20 open files, and an
* assumption that the majority of processes, especially short-lived
* processes like shells, will never need more.
*
* If this initial allocation is exhausted, a larger descriptor table and
* map are allocated dynamically, and the pointers in the process's struct
* filedesc are updated to point to those. This is repeated every time
* the process runs out of file descriptors (provided it hasn't hit its
* resource limit).
*
* Since threads may hold references to individual descriptor table
* entries, the tables are never freed. Instead, they are placed on a
* linked list and freed only when the struct filedesc is released.
*/
#define NDFILE 20
#define NDSLOTSIZE sizeof(NDSLOTTYPE)
#define NDENTRIES (NDSLOTSIZE * __CHAR_BIT)
#define NDSLOT(x) ((x) / NDENTRIES)
#define NDBIT(x) ((NDSLOTTYPE)1 << ((x) % NDENTRIES))
#define NDSLOTS(x) (((x) + NDENTRIES - 1) / NDENTRIES)
/*
* SLIST entry used to keep track of ofiles which must be reclaimed when
* the process exits.
*/
struct freetable {
struct fdescenttbl *ft_table;
SLIST_ENTRY(freetable) ft_next;
};
/*
* Initial allocation: a filedesc structure + the head of SLIST used to
* keep track of old ofiles + enough space for NDFILE descriptors.
*/
struct fdescenttbl0 {
int fdt_nfiles;
struct filedescent fdt_ofiles[NDFILE];
};
struct filedesc0 {
struct filedesc fd_fd;
SLIST_HEAD(, freetable) fd_free;
struct fdescenttbl0 fd_dfiles;
NDSLOTTYPE fd_dmap[NDSLOTS(NDFILE)];
};
/*
* Descriptor management.
*/
static int __exclusive_cache_line openfiles; /* actual number of open files */
struct mtx sigio_lock; /* mtx to protect pointers to sigio */
void __read_mostly (*mq_fdclose)(struct thread *td, int fd, struct file *fp);
/*
* If low >= size, just return low. Otherwise find the first zero bit in the
* given bitmap, starting at low and not exceeding size - 1. Return size if
* not found.
*/
static int
fd_first_free(struct filedesc *fdp, int low, int size)
{
NDSLOTTYPE *map = fdp->fd_map;
NDSLOTTYPE mask;
int off, maxoff;
if (low >= size)
return (low);
off = NDSLOT(low);
if (low % NDENTRIES) {
mask = ~(~(NDSLOTTYPE)0 >> (NDENTRIES - (low % NDENTRIES)));
if ((mask &= ~map[off]) != 0UL)
return (off * NDENTRIES + ffsl(mask) - 1);
++off;
}
for (maxoff = NDSLOTS(size); off < maxoff; ++off)
if (map[off] != ~0UL)
return (off * NDENTRIES + ffsl(~map[off]) - 1);
return (size);
}
/*
* Find the highest non-zero bit in the given bitmap, starting at 0 and
* not exceeding size - 1. Return -1 if not found.
*/
static int
fd_last_used(struct filedesc *fdp, int size)
{
NDSLOTTYPE *map = fdp->fd_map;
NDSLOTTYPE mask;
int off, minoff;
off = NDSLOT(size);
if (size % NDENTRIES) {
mask = ~(~(NDSLOTTYPE)0 << (size % NDENTRIES));
if ((mask &= map[off]) != 0)
return (off * NDENTRIES + flsl(mask) - 1);
--off;
}
for (minoff = NDSLOT(0); off >= minoff; --off)
if (map[off] != 0)
return (off * NDENTRIES + flsl(map[off]) - 1);
return (-1);
}
static int
fdisused(struct filedesc *fdp, int fd)
{
KASSERT(fd >= 0 && fd < fdp->fd_nfiles,
("file descriptor %d out of range (0, %d)", fd, fdp->fd_nfiles));
return ((fdp->fd_map[NDSLOT(fd)] & NDBIT(fd)) != 0);
}
/*
* Mark a file descriptor as used.
*/
static void
fdused_init(struct filedesc *fdp, int fd)
{
KASSERT(!fdisused(fdp, fd), ("fd=%d is already used", fd));
fdp->fd_map[NDSLOT(fd)] |= NDBIT(fd);
}
static void
fdused(struct filedesc *fdp, int fd)
{
FILEDESC_XLOCK_ASSERT(fdp);
fdused_init(fdp, fd);
if (fd > fdp->fd_lastfile)
fdp->fd_lastfile = fd;
if (fd == fdp->fd_freefile)
fdp->fd_freefile++;
}
/*
* Mark a file descriptor as unused.
*/
static void
fdunused(struct filedesc *fdp, int fd)
{
FILEDESC_XLOCK_ASSERT(fdp);
KASSERT(fdisused(fdp, fd), ("fd=%d is already unused", fd));
KASSERT(fdp->fd_ofiles[fd].fde_file == NULL,
("fd=%d is still in use", fd));
fdp->fd_map[NDSLOT(fd)] &= ~NDBIT(fd);
if (fd < fdp->fd_freefile)
fdp->fd_freefile = fd;
if (fd == fdp->fd_lastfile)
fdp->fd_lastfile = fd_last_used(fdp, fd);
}
/*
* Free a file descriptor.
*
* Avoid some work if fdp is about to be destroyed.
*/
static inline void
fdefree_last(struct filedescent *fde)
{
filecaps_free(&fde->fde_caps);
}
static inline void
fdfree(struct filedesc *fdp, int fd)
{
struct filedescent *fde;
fde = &fdp->fd_ofiles[fd];
#ifdef CAPABILITIES
seqc_write_begin(&fde->fde_seqc);
#endif
fde->fde_file = NULL;
#ifdef CAPABILITIES
seqc_write_end(&fde->fde_seqc);
#endif
fdefree_last(fde);
fdunused(fdp, fd);
}
-void
-pwd_ensure_dirs(void)
-{
- struct filedesc *fdp;
-
- fdp = curproc->p_fd;
- FILEDESC_XLOCK(fdp);
- if (fdp->fd_cdir == NULL) {
- fdp->fd_cdir = rootvnode;
- vrefact(rootvnode);
- }
- if (fdp->fd_rdir == NULL) {
- fdp->fd_rdir = rootvnode;
- vrefact(rootvnode);
- }
- FILEDESC_XUNLOCK(fdp);
-}
-
/*
* System calls on descriptors.
*/
#ifndef _SYS_SYSPROTO_H_
struct getdtablesize_args {
int dummy;
};
#endif
/* ARGSUSED */
int
sys_getdtablesize(struct thread *td, struct getdtablesize_args *uap)
{
#ifdef RACCT
uint64_t lim;
#endif
td->td_retval[0] = getmaxfd(td);
#ifdef RACCT
PROC_LOCK(td->td_proc);
lim = racct_get_limit(td->td_proc, RACCT_NOFILE);
PROC_UNLOCK(td->td_proc);
if (lim < td->td_retval[0])
td->td_retval[0] = lim;
#endif
return (0);
}
/*
* Duplicate a file descriptor to a particular value.
*
* Note: keep in mind that a potential race condition exists when closing
* descriptors from a shared descriptor table (via rfork).
*/
#ifndef _SYS_SYSPROTO_H_
struct dup2_args {
u_int from;
u_int to;
};
#endif
/* ARGSUSED */
int
sys_dup2(struct thread *td, struct dup2_args *uap)
{
return (kern_dup(td, FDDUP_FIXED, 0, (int)uap->from, (int)uap->to));
}
/*
* Duplicate a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct dup_args {
u_int fd;
};
#endif
/* ARGSUSED */
int
sys_dup(struct thread *td, struct dup_args *uap)
{
return (kern_dup(td, FDDUP_NORMAL, 0, (int)uap->fd, 0));
}
/*
* The file control system call.
*/
#ifndef _SYS_SYSPROTO_H_
struct fcntl_args {
int fd;
int cmd;
long arg;
};
#endif
/* ARGSUSED */
int
sys_fcntl(struct thread *td, struct fcntl_args *uap)
{
return (kern_fcntl_freebsd(td, uap->fd, uap->cmd, uap->arg));
}
int
kern_fcntl_freebsd(struct thread *td, int fd, int cmd, long arg)
{
struct flock fl;
struct __oflock ofl;
intptr_t arg1;
int error, newcmd;
error = 0;
newcmd = cmd;
switch (cmd) {
case F_OGETLK:
case F_OSETLK:
case F_OSETLKW:
/*
* Convert old flock structure to new.
*/
error = copyin((void *)(intptr_t)arg, &ofl, sizeof(ofl));
fl.l_start = ofl.l_start;
fl.l_len = ofl.l_len;
fl.l_pid = ofl.l_pid;
fl.l_type = ofl.l_type;
fl.l_whence = ofl.l_whence;
fl.l_sysid = 0;
switch (cmd) {
case F_OGETLK:
newcmd = F_GETLK;
break;
case F_OSETLK:
newcmd = F_SETLK;
break;
case F_OSETLKW:
newcmd = F_SETLKW;
break;
}
arg1 = (intptr_t)&fl;
break;
case F_GETLK:
case F_SETLK:
case F_SETLKW:
case F_SETLK_REMOTE:
error = copyin((void *)(intptr_t)arg, &fl, sizeof(fl));
arg1 = (intptr_t)&fl;
break;
default:
arg1 = arg;
break;
}
if (error)
return (error);
error = kern_fcntl(td, fd, newcmd, arg1);
if (error)
return (error);
if (cmd == F_OGETLK) {
ofl.l_start = fl.l_start;
ofl.l_len = fl.l_len;
ofl.l_pid = fl.l_pid;
ofl.l_type = fl.l_type;
ofl.l_whence = fl.l_whence;
error = copyout(&ofl, (void *)(intptr_t)arg, sizeof(ofl));
} else if (cmd == F_GETLK) {
error = copyout(&fl, (void *)(intptr_t)arg, sizeof(fl));
}
return (error);
}
int
kern_fcntl(struct thread *td, int fd, int cmd, intptr_t arg)
{
struct filedesc *fdp;
struct flock *flp;
struct file *fp, *fp2;
struct filedescent *fde;
struct proc *p;
struct vnode *vp;
struct mount *mp;
int error, flg, seals, tmp;
uint64_t bsize;
off_t foffset;
error = 0;
flg = F_POSIX;
p = td->td_proc;
fdp = p->p_fd;
AUDIT_ARG_FD(cmd);
AUDIT_ARG_CMD(cmd);
switch (cmd) {
case F_DUPFD:
tmp = arg;
error = kern_dup(td, FDDUP_FCNTL, 0, fd, tmp);
break;
case F_DUPFD_CLOEXEC:
tmp = arg;
error = kern_dup(td, FDDUP_FCNTL, FDDUP_FLAG_CLOEXEC, fd, tmp);
break;
case F_DUP2FD:
tmp = arg;
error = kern_dup(td, FDDUP_FIXED, 0, fd, tmp);
break;
case F_DUP2FD_CLOEXEC:
tmp = arg;
error = kern_dup(td, FDDUP_FIXED, FDDUP_FLAG_CLOEXEC, fd, tmp);
break;
case F_GETFD:
error = EBADF;
FILEDESC_SLOCK(fdp);
fde = fdeget_locked(fdp, fd);
if (fde != NULL) {
td->td_retval[0] =
(fde->fde_flags & UF_EXCLOSE) ? FD_CLOEXEC : 0;
error = 0;
}
FILEDESC_SUNLOCK(fdp);
break;
case F_SETFD:
error = EBADF;
FILEDESC_XLOCK(fdp);
fde = fdeget_locked(fdp, fd);
if (fde != NULL) {
fde->fde_flags = (fde->fde_flags & ~UF_EXCLOSE) |
(arg & FD_CLOEXEC ? UF_EXCLOSE : 0);
error = 0;
}
FILEDESC_XUNLOCK(fdp);
break;
case F_GETFL:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_GETFL, &fp);
if (error != 0)
break;
td->td_retval[0] = OFLAGS(fp->f_flag);
fdrop(fp, td);
break;
case F_SETFL:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_SETFL, &fp);
if (error != 0)
break;
do {
tmp = flg = fp->f_flag;
tmp &= ~FCNTLFLAGS;
tmp |= FFLAGS(arg & ~O_ACCMODE) & FCNTLFLAGS;
} while(atomic_cmpset_int(&fp->f_flag, flg, tmp) == 0);
tmp = fp->f_flag & FNONBLOCK;
error = fo_ioctl(fp, FIONBIO, &tmp, td->td_ucred, td);
if (error != 0) {
fdrop(fp, td);
break;
}
tmp = fp->f_flag & FASYNC;
error = fo_ioctl(fp, FIOASYNC, &tmp, td->td_ucred, td);
if (error == 0) {
fdrop(fp, td);
break;
}
atomic_clear_int(&fp->f_flag, FNONBLOCK);
tmp = 0;
(void)fo_ioctl(fp, FIONBIO, &tmp, td->td_ucred, td);
fdrop(fp, td);
break;
case F_GETOWN:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_GETOWN, &fp);
if (error != 0)
break;
error = fo_ioctl(fp, FIOGETOWN, &tmp, td->td_ucred, td);
if (error == 0)
td->td_retval[0] = tmp;
fdrop(fp, td);
break;
case F_SETOWN:
error = fget_fcntl(td, fd, &cap_fcntl_rights, F_SETOWN, &fp);
if (error != 0)
break;
tmp = arg;
error = fo_ioctl(fp, FIOSETOWN, &tmp, td->td_ucred, td);
fdrop(fp, td);
break;
case F_SETLK_REMOTE:
error = priv_check(td, PRIV_NFS_LOCKD);
if (error != 0)
return (error);
flg = F_REMOTE;
goto do_setlk;
case F_SETLKW:
flg |= F_WAIT;
/* FALLTHROUGH F_SETLK */
case F_SETLK:
do_setlk:
flp = (struct flock *)arg;
if ((flg & F_REMOTE) != 0 && flp->l_sysid == 0) {
error = EINVAL;
break;
}
error = fget_unlocked(fdp, fd, &cap_flock_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE) {
error = EBADF;
fdrop(fp, td);
break;
}
if (flp->l_whence == SEEK_CUR) {
foffset = foffset_get(fp);
if (foffset < 0 ||
(flp->l_start > 0 &&
foffset > OFF_MAX - flp->l_start)) {
error = EOVERFLOW;
fdrop(fp, td);
break;
}
flp->l_start += foffset;
}
vp = fp->f_vnode;
switch (flp->l_type) {
case F_RDLCK:
if ((fp->f_flag & FREAD) == 0) {
error = EBADF;
break;
}
if ((p->p_leader->p_flag & P_ADVLOCK) == 0) {
PROC_LOCK(p->p_leader);
p->p_leader->p_flag |= P_ADVLOCK;
PROC_UNLOCK(p->p_leader);
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_SETLK,
flp, flg);
break;
case F_WRLCK:
if ((fp->f_flag & FWRITE) == 0) {
error = EBADF;
break;
}
if ((p->p_leader->p_flag & P_ADVLOCK) == 0) {
PROC_LOCK(p->p_leader);
p->p_leader->p_flag |= P_ADVLOCK;
PROC_UNLOCK(p->p_leader);
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_SETLK,
flp, flg);
break;
case F_UNLCK:
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_UNLCK,
flp, flg);
break;
case F_UNLCKSYS:
if (flg != F_REMOTE) {
error = EINVAL;
break;
}
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader,
F_UNLCKSYS, flp, flg);
break;
default:
error = EINVAL;
break;
}
if (error != 0 || flp->l_type == F_UNLCK ||
flp->l_type == F_UNLCKSYS) {
fdrop(fp, td);
break;
}
/*
* Check for a race with close.
*
* The vnode is now advisory locked (or unlocked, but this case
* is not really important) as the caller requested.
* We had to drop the filedesc lock, so we need to recheck if
* the descriptor is still valid, because if it was closed
* in the meantime we need to remove advisory lock from the
* vnode - close on any descriptor leading to an advisory
* locked vnode, removes that lock.
* We will return 0 on purpose in that case, as the result of
* successful advisory lock might have been externally visible
* already. This is fine - effectively we pretend to the caller
* that the closing thread was a bit slower and that the
* advisory lock succeeded before the close.
*/
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp2);
if (error != 0) {
fdrop(fp, td);
break;
}
if (fp != fp2) {
flp->l_whence = SEEK_SET;
flp->l_start = 0;
flp->l_len = 0;
flp->l_type = F_UNLCK;
(void) VOP_ADVLOCK(vp, (caddr_t)p->p_leader,
F_UNLCK, flp, F_POSIX);
}
fdrop(fp, td);
fdrop(fp2, td);
break;
case F_GETLK:
error = fget_unlocked(fdp, fd, &cap_flock_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE) {
error = EBADF;
fdrop(fp, td);
break;
}
flp = (struct flock *)arg;
if (flp->l_type != F_RDLCK && flp->l_type != F_WRLCK &&
flp->l_type != F_UNLCK) {
error = EINVAL;
fdrop(fp, td);
break;
}
if (flp->l_whence == SEEK_CUR) {
foffset = foffset_get(fp);
if ((flp->l_start > 0 &&
foffset > OFF_MAX - flp->l_start) ||
(flp->l_start < 0 &&
foffset < OFF_MIN - flp->l_start)) {
error = EOVERFLOW;
fdrop(fp, td);
break;
}
flp->l_start += foffset;
}
vp = fp->f_vnode;
error = VOP_ADVLOCK(vp, (caddr_t)p->p_leader, F_GETLK, flp,
F_POSIX);
fdrop(fp, td);
break;
case F_ADD_SEALS:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
error = fo_add_seals(fp, arg);
fdrop(fp, td);
break;
case F_GET_SEALS:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fo_get_seals(fp, &seals) == 0)
td->td_retval[0] = seals;
else
error = EINVAL;
fdrop(fp, td);
break;
case F_RDAHEAD:
arg = arg ? 128 * 1024: 0;
/* FALLTHROUGH */
case F_READAHEAD:
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE) {
fdrop(fp, td);
error = EBADF;
break;
}
vp = fp->f_vnode;
if (vp->v_type != VREG) {
fdrop(fp, td);
error = ENOTTY;
break;
}
/*
* Exclusive lock synchronizes against f_seqcount reads and
* writes in sequential_heuristic().
*/
error = vn_lock(vp, LK_EXCLUSIVE);
if (error != 0) {
fdrop(fp, td);
break;
}
if (arg >= 0) {
bsize = fp->f_vnode->v_mount->mnt_stat.f_iosize;
arg = MIN(arg, INT_MAX - bsize + 1);
fp->f_seqcount = MIN(IO_SEQMAX,
(arg + bsize - 1) / bsize);
atomic_set_int(&fp->f_flag, FRDAHEAD);
} else {
atomic_clear_int(&fp->f_flag, FRDAHEAD);
}
VOP_UNLOCK(vp);
fdrop(fp, td);
break;
case F_ISUNIONSTACK:
/*
* Check if the vnode is part of a union stack (either the
* "union" flag from mount(2) or unionfs).
*
* Prior to introduction of this op libc's readdir would call
* fstatfs(2), in effect unnecessarily copying kilobytes of
* data just to check fs name and a mount flag.
*
* Fixing the code to handle everything in the kernel instead
* is a non-trivial endeavor and has low priority, thus this
* horrible kludge facilitates the current behavior in a much
* cheaper manner until someone(tm) sorts this out.
*/
error = fget_unlocked(fdp, fd, &cap_no_rights, &fp);
if (error != 0)
break;
if (fp->f_type != DTYPE_VNODE) {
fdrop(fp, td);
error = EBADF;
break;
}
vp = fp->f_vnode;
/*
* Since we don't prevent dooming the vnode even non-null mp
* found can become immediately stale. This is tolerable since
* mount points are type-stable (providing safe memory access)
* and any vfs op on this vnode going forward will return an
* error (meaning return value in this case is meaningless).
*/
mp = atomic_load_ptr(&vp->v_mount);
if (__predict_false(mp == NULL)) {
fdrop(fp, td);
error = EBADF;
break;
}
td->td_retval[0] = 0;
if (mp->mnt_kern_flag & MNTK_UNIONFS ||
mp->mnt_flag & MNT_UNION)
td->td_retval[0] = 1;
fdrop(fp, td);
break;
default:
error = EINVAL;
break;
}
return (error);
}
static int
getmaxfd(struct thread *td)
{
return (min((int)lim_cur(td, RLIMIT_NOFILE), maxfilesperproc));
}
/*
* Common code for dup, dup2, fcntl(F_DUPFD) and fcntl(F_DUP2FD).
*/
int
kern_dup(struct thread *td, u_int mode, int flags, int old, int new)
{
struct filedesc *fdp;
struct filedescent *oldfde, *newfde;
struct proc *p;
struct file *delfp;
u_long *oioctls, *nioctls;
int error, maxfd;
p = td->td_proc;
fdp = p->p_fd;
oioctls = NULL;
MPASS((flags & ~(FDDUP_FLAG_CLOEXEC)) == 0);
MPASS(mode < FDDUP_LASTMODE);
AUDIT_ARG_FD(old);
/* XXXRW: if (flags & FDDUP_FIXED) AUDIT_ARG_FD2(new); */
/*
* Verify we have a valid descriptor to dup from and possibly to
* dup to. Unlike dup() and dup2(), fcntl()'s F_DUPFD should
* return EINVAL when the new descriptor is out of bounds.
*/
if (old < 0)
return (EBADF);
if (new < 0)
return (mode == FDDUP_FCNTL ? EINVAL : EBADF);
maxfd = getmaxfd(td);
if (new >= maxfd)
return (mode == FDDUP_FCNTL ? EINVAL : EBADF);
error = EBADF;
FILEDESC_XLOCK(fdp);
if (fget_locked(fdp, old) == NULL)
goto unlock;
if ((mode == FDDUP_FIXED || mode == FDDUP_MUSTREPLACE) && old == new) {
td->td_retval[0] = new;
if (flags & FDDUP_FLAG_CLOEXEC)
fdp->fd_ofiles[new].fde_flags |= UF_EXCLOSE;
error = 0;
goto unlock;
}
oldfde = &fdp->fd_ofiles[old];
if (!fhold(oldfde->fde_file))
goto unlock;
/*
* If the caller specified a file descriptor, make sure the file
* table is large enough to hold it, and grab it. Otherwise, just
* allocate a new descriptor the usual way.
*/
switch (mode) {
case FDDUP_NORMAL:
case FDDUP_FCNTL:
if ((error = fdalloc(td, new, &new)) != 0) {
fdrop(oldfde->fde_file, td);
goto unlock;
}
break;
case FDDUP_MUSTREPLACE:
/* Target file descriptor must exist. */
if (fget_locked(fdp, new) == NULL) {
fdrop(oldfde->fde_file, td);
goto unlock;
}
break;
case FDDUP_FIXED:
if (new >= fdp->fd_nfiles) {
/*
* The resource limits are here instead of e.g.
* fdalloc(), because the file descriptor table may be
* shared between processes, so we can't really use
* racct_add()/racct_sub(). Instead of counting the
* number of actually allocated descriptors, just put
* the limit on the size of the file descriptor table.
*/
#ifdef RACCT
if (RACCT_ENABLED()) {
error = racct_set_unlocked(p, RACCT_NOFILE, new + 1);
if (error != 0) {
error = EMFILE;
fdrop(oldfde->fde_file, td);
goto unlock;
}
}
#endif
fdgrowtable_exp(fdp, new + 1);
}
if (!fdisused(fdp, new))
fdused(fdp, new);
break;
default:
KASSERT(0, ("%s unsupported mode %d", __func__, mode));
}
KASSERT(old != new, ("new fd is same as old"));
newfde = &fdp->fd_ofiles[new];
delfp = newfde->fde_file;
oioctls = filecaps_free_prep(&newfde->fde_caps);
nioctls = filecaps_copy_prep(&oldfde->fde_caps);
/*
* Duplicate the source descriptor.
*/
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
memcpy(newfde, oldfde, fde_change_size);
filecaps_copy_finish(&oldfde->fde_caps, &newfde->fde_caps,
nioctls);
if ((flags & FDDUP_FLAG_CLOEXEC) != 0)
newfde->fde_flags = oldfde->fde_flags | UF_EXCLOSE;
else
newfde->fde_flags = oldfde->fde_flags & ~UF_EXCLOSE;
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
td->td_retval[0] = new;
error = 0;
if (delfp != NULL) {
(void) closefp(fdp, new, delfp, td, 1);
FILEDESC_UNLOCK_ASSERT(fdp);
} else {
unlock:
FILEDESC_XUNLOCK(fdp);
}
filecaps_free_finish(oioctls);
return (error);
}
/*
* If sigio is on the list associated with a process or process group,
* disable signalling from the device, remove sigio from the list and
* free sigio.
*/
void
funsetown(struct sigio **sigiop)
{
struct sigio *sigio;
if (*sigiop == NULL)
return;
SIGIO_LOCK();
sigio = *sigiop;
if (sigio == NULL) {
SIGIO_UNLOCK();
return;
}
*(sigio->sio_myref) = NULL;
if ((sigio)->sio_pgid < 0) {
struct pgrp *pg = (sigio)->sio_pgrp;
PGRP_LOCK(pg);
SLIST_REMOVE(&sigio->sio_pgrp->pg_sigiolst, sigio,
sigio, sio_pgsigio);
PGRP_UNLOCK(pg);
} else {
struct proc *p = (sigio)->sio_proc;
PROC_LOCK(p);
SLIST_REMOVE(&sigio->sio_proc->p_sigiolst, sigio,
sigio, sio_pgsigio);
PROC_UNLOCK(p);
}
SIGIO_UNLOCK();
crfree(sigio->sio_ucred);
free(sigio, M_SIGIO);
}
/*
* Free a list of sigio structures.
* We only need to lock the SIGIO_LOCK because we have made ourselves
* inaccessible to callers of fsetown and therefore do not need to lock
* the proc or pgrp struct for the list manipulation.
*/
void
funsetownlst(struct sigiolst *sigiolst)
{
struct proc *p;
struct pgrp *pg;
struct sigio *sigio;
sigio = SLIST_FIRST(sigiolst);
if (sigio == NULL)
return;
p = NULL;
pg = NULL;
/*
* Every entry of the list should belong
* to a single proc or pgrp.
*/
if (sigio->sio_pgid < 0) {
pg = sigio->sio_pgrp;
PGRP_LOCK_ASSERT(pg, MA_NOTOWNED);
} else /* if (sigio->sio_pgid > 0) */ {
p = sigio->sio_proc;
PROC_LOCK_ASSERT(p, MA_NOTOWNED);
}
SIGIO_LOCK();
while ((sigio = SLIST_FIRST(sigiolst)) != NULL) {
*(sigio->sio_myref) = NULL;
if (pg != NULL) {
KASSERT(sigio->sio_pgid < 0,
("Proc sigio in pgrp sigio list"));
KASSERT(sigio->sio_pgrp == pg,
("Bogus pgrp in sigio list"));
PGRP_LOCK(pg);
SLIST_REMOVE(&pg->pg_sigiolst, sigio, sigio,
sio_pgsigio);
PGRP_UNLOCK(pg);
} else /* if (p != NULL) */ {
KASSERT(sigio->sio_pgid > 0,
("Pgrp sigio in proc sigio list"));
KASSERT(sigio->sio_proc == p,
("Bogus proc in sigio list"));
PROC_LOCK(p);
SLIST_REMOVE(&p->p_sigiolst, sigio, sigio,
sio_pgsigio);
PROC_UNLOCK(p);
}
SIGIO_UNLOCK();
crfree(sigio->sio_ucred);
free(sigio, M_SIGIO);
SIGIO_LOCK();
}
SIGIO_UNLOCK();
}
/*
* This is common code for FIOSETOWN ioctl called by fcntl(fd, F_SETOWN, arg).
*
* After permission checking, add a sigio structure to the sigio list for
* the process or process group.
*/
int
fsetown(pid_t pgid, struct sigio **sigiop)
{
struct proc *proc;
struct pgrp *pgrp;
struct sigio *sigio;
int ret;
if (pgid == 0) {
funsetown(sigiop);
return (0);
}
ret = 0;
/* Allocate and fill in the new sigio out of locks. */
sigio = malloc(sizeof(struct sigio), M_SIGIO, M_WAITOK);
sigio->sio_pgid = pgid;
sigio->sio_ucred = crhold(curthread->td_ucred);
sigio->sio_myref = sigiop;
sx_slock(&proctree_lock);
if (pgid > 0) {
proc = pfind(pgid);
if (proc == NULL) {
ret = ESRCH;
goto fail;
}
/*
* Policy - Don't allow a process to FSETOWN a process
* in another session.
*
* Remove this test to allow maximum flexibility or
* restrict FSETOWN to the current process or process
* group for maximum safety.
*/
PROC_UNLOCK(proc);
if (proc->p_session != curthread->td_proc->p_session) {
ret = EPERM;
goto fail;
}
pgrp = NULL;
} else /* if (pgid < 0) */ {
pgrp = pgfind(-pgid);
if (pgrp == NULL) {
ret = ESRCH;
goto fail;
}
PGRP_UNLOCK(pgrp);
/*
* Policy - Don't allow a process to FSETOWN a process
* in another session.
*
* Remove this test to allow maximum flexibility or
* restrict FSETOWN to the current process or process
* group for maximum safety.
*/
if (pgrp->pg_session != curthread->td_proc->p_session) {
ret = EPERM;
goto fail;
}
proc = NULL;
}
funsetown(sigiop);
if (pgid > 0) {
PROC_LOCK(proc);
/*
* Since funsetownlst() is called without the proctree
* locked, we need to check for P_WEXIT.
* XXX: is ESRCH correct?
*/
if ((proc->p_flag & P_WEXIT) != 0) {
PROC_UNLOCK(proc);
ret = ESRCH;
goto fail;
}
SLIST_INSERT_HEAD(&proc->p_sigiolst, sigio, sio_pgsigio);
sigio->sio_proc = proc;
PROC_UNLOCK(proc);
} else {
PGRP_LOCK(pgrp);
SLIST_INSERT_HEAD(&pgrp->pg_sigiolst, sigio, sio_pgsigio);
sigio->sio_pgrp = pgrp;
PGRP_UNLOCK(pgrp);
}
sx_sunlock(&proctree_lock);
SIGIO_LOCK();
*sigiop = sigio;
SIGIO_UNLOCK();
return (0);
fail:
sx_sunlock(&proctree_lock);
crfree(sigio->sio_ucred);
free(sigio, M_SIGIO);
return (ret);
}
/*
* This is common code for FIOGETOWN ioctl called by fcntl(fd, F_GETOWN, arg).
*/
pid_t
fgetown(struct sigio **sigiop)
{
pid_t pgid;
SIGIO_LOCK();
pgid = (*sigiop != NULL) ? (*sigiop)->sio_pgid : 0;
SIGIO_UNLOCK();
return (pgid);
}
/*
* Function drops the filedesc lock on return.
*/
static int
closefp(struct filedesc *fdp, int fd, struct file *fp, struct thread *td,
int holdleaders)
{
int error;
FILEDESC_XLOCK_ASSERT(fdp);
if (holdleaders) {
if (td->td_proc->p_fdtol != NULL) {
/*
* Ask fdfree() to sleep to ensure that all relevant
* process leaders can be traversed in closef().
*/
fdp->fd_holdleaderscount++;
} else {
holdleaders = 0;
}
}
/*
* We now hold the fp reference that used to be owned by the
* descriptor array. We have to unlock the FILEDESC *AFTER*
* knote_fdclose to prevent a race of the fd getting opened, a knote
* added, and deleteing a knote for the new fd.
*/
if (__predict_false(!TAILQ_EMPTY(&fdp->fd_kqlist)))
knote_fdclose(td, fd);
/*
* We need to notify mqueue if the object is of type mqueue.
*/
if (__predict_false(fp->f_type == DTYPE_MQUEUE))
mq_fdclose(td, fd, fp);
FILEDESC_XUNLOCK(fdp);
error = closef(fp, td);
if (holdleaders) {
FILEDESC_XLOCK(fdp);
fdp->fd_holdleaderscount--;
if (fdp->fd_holdleaderscount == 0 &&
fdp->fd_holdleaderswakeup != 0) {
fdp->fd_holdleaderswakeup = 0;
wakeup(&fdp->fd_holdleaderscount);
}
FILEDESC_XUNLOCK(fdp);
}
return (error);
}
/*
* Close a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct close_args {
int fd;
};
#endif
/* ARGSUSED */
int
sys_close(struct thread *td, struct close_args *uap)
{
return (kern_close(td, uap->fd));
}
int
kern_close(struct thread *td, int fd)
{
struct filedesc *fdp;
struct file *fp;
fdp = td->td_proc->p_fd;
AUDIT_SYSCLOSE(td, fd);
FILEDESC_XLOCK(fdp);
if ((fp = fget_locked(fdp, fd)) == NULL) {
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
fdfree(fdp, fd);
/* closefp() drops the FILEDESC lock for us. */
return (closefp(fdp, fd, fp, td, 1));
}
/*
* Close open file descriptors.
*/
#ifndef _SYS_SYSPROTO_H_
struct closefrom_args {
int lowfd;
};
#endif
/* ARGSUSED */
int
sys_closefrom(struct thread *td, struct closefrom_args *uap)
{
struct filedesc *fdp;
int fd;
fdp = td->td_proc->p_fd;
AUDIT_ARG_FD(uap->lowfd);
/*
* Treat negative starting file descriptor values identical to
* closefrom(0) which closes all files.
*/
if (uap->lowfd < 0)
uap->lowfd = 0;
FILEDESC_SLOCK(fdp);
for (fd = uap->lowfd; fd <= fdp->fd_lastfile; fd++) {
if (fdp->fd_ofiles[fd].fde_file != NULL) {
FILEDESC_SUNLOCK(fdp);
(void)kern_close(td, fd);
FILEDESC_SLOCK(fdp);
}
}
FILEDESC_SUNLOCK(fdp);
return (0);
}
#if defined(COMPAT_43)
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct ofstat_args {
int fd;
struct ostat *sb;
};
#endif
/* ARGSUSED */
int
ofstat(struct thread *td, struct ofstat_args *uap)
{
struct ostat oub;
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0) {
cvtstat(&ub, &oub);
error = copyout(&oub, uap->sb, sizeof(oub));
}
return (error);
}
#endif /* COMPAT_43 */
#if defined(COMPAT_FREEBSD11)
int
freebsd11_fstat(struct thread *td, struct freebsd11_fstat_args *uap)
{
struct stat sb;
struct freebsd11_stat osb;
int error;
error = kern_fstat(td, uap->fd, &sb);
if (error != 0)
return (error);
error = freebsd11_cvtstat(&sb, &osb);
if (error == 0)
error = copyout(&osb, uap->sb, sizeof(osb));
return (error);
}
#endif /* COMPAT_FREEBSD11 */
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fstat_args {
int fd;
struct stat *sb;
};
#endif
/* ARGSUSED */
int
sys_fstat(struct thread *td, struct fstat_args *uap)
{
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0)
error = copyout(&ub, uap->sb, sizeof(ub));
return (error);
}
int
kern_fstat(struct thread *td, int fd, struct stat *sbp)
{
struct file *fp;
int error;
AUDIT_ARG_FD(fd);
error = fget(td, fd, &cap_fstat_rights, &fp);
if (__predict_false(error != 0))
return (error);
AUDIT_ARG_FILE(td->td_proc, fp);
error = fo_stat(fp, sbp, td->td_ucred, td);
fdrop(fp, td);
#ifdef __STAT_TIME_T_EXT
sbp->st_atim_ext = 0;
sbp->st_mtim_ext = 0;
sbp->st_ctim_ext = 0;
sbp->st_btim_ext = 0;
#endif
#ifdef KTRACE
if (KTRPOINT(td, KTR_STRUCT))
ktrstat_error(sbp, error);
#endif
return (error);
}
#if defined(COMPAT_FREEBSD11)
/*
* Return status information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct freebsd11_nfstat_args {
int fd;
struct nstat *sb;
};
#endif
/* ARGSUSED */
int
freebsd11_nfstat(struct thread *td, struct freebsd11_nfstat_args *uap)
{
struct nstat nub;
struct stat ub;
int error;
error = kern_fstat(td, uap->fd, &ub);
if (error == 0) {
freebsd11_cvtnstat(&ub, &nub);
error = copyout(&nub, uap->sb, sizeof(nub));
}
return (error);
}
#endif /* COMPAT_FREEBSD11 */
/*
* Return pathconf information about a file descriptor.
*/
#ifndef _SYS_SYSPROTO_H_
struct fpathconf_args {
int fd;
int name;
};
#endif
/* ARGSUSED */
int
sys_fpathconf(struct thread *td, struct fpathconf_args *uap)
{
long value;
int error;
error = kern_fpathconf(td, uap->fd, uap->name, &value);
if (error == 0)
td->td_retval[0] = value;
return (error);
}
int
kern_fpathconf(struct thread *td, int fd, int name, long *valuep)
{
struct file *fp;
struct vnode *vp;
int error;
error = fget(td, fd, &cap_fpathconf_rights, &fp);
if (error != 0)
return (error);
if (name == _PC_ASYNC_IO) {
*valuep = _POSIX_ASYNCHRONOUS_IO;
goto out;
}
vp = fp->f_vnode;
if (vp != NULL) {
vn_lock(vp, LK_SHARED | LK_RETRY);
error = VOP_PATHCONF(vp, name, valuep);
VOP_UNLOCK(vp);
} else if (fp->f_type == DTYPE_PIPE || fp->f_type == DTYPE_SOCKET) {
if (name != _PC_PIPE_BUF) {
error = EINVAL;
} else {
*valuep = PIPE_BUF;
error = 0;
}
} else {
error = EOPNOTSUPP;
}
out:
fdrop(fp, td);
return (error);
}
/*
* Copy filecaps structure allocating memory for ioctls array if needed.
*
* The last parameter indicates whether the fdtable is locked. If it is not and
* ioctls are encountered, copying fails and the caller must lock the table.
*
* Note that if the table was not locked, the caller has to check the relevant
* sequence counter to determine whether the operation was successful.
*/
bool
filecaps_copy(const struct filecaps *src, struct filecaps *dst, bool locked)
{
size_t size;
if (src->fc_ioctls != NULL && !locked)
return (false);
memcpy(dst, src, sizeof(*src));
if (src->fc_ioctls == NULL)
return (true);
KASSERT(src->fc_nioctls > 0,
("fc_ioctls != NULL, but fc_nioctls=%hd", src->fc_nioctls));
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
dst->fc_ioctls = malloc(size, M_FILECAPS, M_WAITOK);
memcpy(dst->fc_ioctls, src->fc_ioctls, size);
return (true);
}
static u_long *
filecaps_copy_prep(const struct filecaps *src)
{
u_long *ioctls;
size_t size;
if (__predict_true(src->fc_ioctls == NULL))
return (NULL);
KASSERT(src->fc_nioctls > 0,
("fc_ioctls != NULL, but fc_nioctls=%hd", src->fc_nioctls));
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
ioctls = malloc(size, M_FILECAPS, M_WAITOK);
return (ioctls);
}
static void
filecaps_copy_finish(const struct filecaps *src, struct filecaps *dst,
u_long *ioctls)
{
size_t size;
*dst = *src;
if (__predict_true(src->fc_ioctls == NULL)) {
MPASS(ioctls == NULL);
return;
}
size = sizeof(src->fc_ioctls[0]) * src->fc_nioctls;
dst->fc_ioctls = ioctls;
bcopy(src->fc_ioctls, dst->fc_ioctls, size);
}
/*
* Move filecaps structure to the new place and clear the old place.
*/
void
filecaps_move(struct filecaps *src, struct filecaps *dst)
{
*dst = *src;
bzero(src, sizeof(*src));
}
/*
* Fill the given filecaps structure with full rights.
*/
static void
filecaps_fill(struct filecaps *fcaps)
{
CAP_ALL(&fcaps->fc_rights);
fcaps->fc_ioctls = NULL;
fcaps->fc_nioctls = -1;
fcaps->fc_fcntls = CAP_FCNTL_ALL;
}
/*
* Free memory allocated within filecaps structure.
*/
void
filecaps_free(struct filecaps *fcaps)
{
free(fcaps->fc_ioctls, M_FILECAPS);
bzero(fcaps, sizeof(*fcaps));
}
static u_long *
filecaps_free_prep(struct filecaps *fcaps)
{
u_long *ioctls;
ioctls = fcaps->fc_ioctls;
bzero(fcaps, sizeof(*fcaps));
return (ioctls);
}
static void
filecaps_free_finish(u_long *ioctls)
{
free(ioctls, M_FILECAPS);
}
/*
* Validate the given filecaps structure.
*/
static void
filecaps_validate(const struct filecaps *fcaps, const char *func)
{
KASSERT(cap_rights_is_valid(&fcaps->fc_rights),
("%s: invalid rights", func));
KASSERT((fcaps->fc_fcntls & ~CAP_FCNTL_ALL) == 0,
("%s: invalid fcntls", func));
KASSERT(fcaps->fc_fcntls == 0 ||
cap_rights_is_set(&fcaps->fc_rights, CAP_FCNTL),
("%s: fcntls without CAP_FCNTL", func));
KASSERT(fcaps->fc_ioctls != NULL ? fcaps->fc_nioctls > 0 :
(fcaps->fc_nioctls == -1 || fcaps->fc_nioctls == 0),
("%s: invalid ioctls", func));
KASSERT(fcaps->fc_nioctls == 0 ||
cap_rights_is_set(&fcaps->fc_rights, CAP_IOCTL),
("%s: ioctls without CAP_IOCTL", func));
}
static void
fdgrowtable_exp(struct filedesc *fdp, int nfd)
{
int nfd1;
FILEDESC_XLOCK_ASSERT(fdp);
nfd1 = fdp->fd_nfiles * 2;
if (nfd1 < nfd)
nfd1 = nfd;
fdgrowtable(fdp, nfd1);
}
/*
* Grow the file table to accommodate (at least) nfd descriptors.
*/
static void
fdgrowtable(struct filedesc *fdp, int nfd)
{
struct filedesc0 *fdp0;
struct freetable *ft;
struct fdescenttbl *ntable;
struct fdescenttbl *otable;
int nnfiles, onfiles;
NDSLOTTYPE *nmap, *omap;
/*
* If lastfile is -1 this struct filedesc was just allocated and we are
* growing it to accommodate for the one we are going to copy from. There
* is no need to have a lock on this one as it's not visible to anyone.
*/
if (fdp->fd_lastfile != -1)
FILEDESC_XLOCK_ASSERT(fdp);
KASSERT(fdp->fd_nfiles > 0, ("zero-length file table"));
/* save old values */
onfiles = fdp->fd_nfiles;
otable = fdp->fd_files;
omap = fdp->fd_map;
/* compute the size of the new table */
nnfiles = NDSLOTS(nfd) * NDENTRIES; /* round up */
if (nnfiles <= onfiles)
/* the table is already large enough */
return;
/*
* Allocate a new table. We need enough space for the number of
* entries, file entries themselves and the struct freetable we will use
* when we decommission the table and place it on the freelist.
* We place the struct freetable in the middle so we don't have
* to worry about padding.
*/
ntable = malloc(offsetof(struct fdescenttbl, fdt_ofiles) +
nnfiles * sizeof(ntable->fdt_ofiles[0]) +
sizeof(struct freetable),
M_FILEDESC, M_ZERO | M_WAITOK);
/* copy the old data */
ntable->fdt_nfiles = nnfiles;
memcpy(ntable->fdt_ofiles, otable->fdt_ofiles,
onfiles * sizeof(ntable->fdt_ofiles[0]));
/*
* Allocate a new map only if the old is not large enough. It will
* grow at a slower rate than the table as it can map more
* entries than the table can hold.
*/
if (NDSLOTS(nnfiles) > NDSLOTS(onfiles)) {
nmap = malloc(NDSLOTS(nnfiles) * NDSLOTSIZE, M_FILEDESC,
M_ZERO | M_WAITOK);
/* copy over the old data and update the pointer */
memcpy(nmap, omap, NDSLOTS(onfiles) * sizeof(*omap));
fdp->fd_map = nmap;
}
/*
* Make sure that ntable is correctly initialized before we replace
* fd_files poiner. Otherwise fget_unlocked() may see inconsistent
* data.
*/
atomic_store_rel_ptr((volatile void *)&fdp->fd_files, (uintptr_t)ntable);
/*
* Do not free the old file table, as some threads may still
* reference entries within it. Instead, place it on a freelist
* which will be processed when the struct filedesc is released.
*
* Note that if onfiles == NDFILE, we're dealing with the original
* static allocation contained within (struct filedesc0 *)fdp,
* which must not be freed.
*/
if (onfiles > NDFILE) {
ft = (struct freetable *)&otable->fdt_ofiles[onfiles];
fdp0 = (struct filedesc0 *)fdp;
ft->ft_table = otable;
SLIST_INSERT_HEAD(&fdp0->fd_free, ft, ft_next);
}
/*
* The map does not have the same possibility of threads still
* holding references to it. So always free it as long as it
* does not reference the original static allocation.
*/
if (NDSLOTS(onfiles) > NDSLOTS(NDFILE))
free(omap, M_FILEDESC);
}
/*
* Allocate a file descriptor for the process.
*/
int
fdalloc(struct thread *td, int minfd, int *result)
{
struct proc *p = td->td_proc;
struct filedesc *fdp = p->p_fd;
int fd, maxfd, allocfd;
#ifdef RACCT
int error;
#endif
FILEDESC_XLOCK_ASSERT(fdp);
if (fdp->fd_freefile > minfd)
minfd = fdp->fd_freefile;
maxfd = getmaxfd(td);
/*
* Search the bitmap for a free descriptor starting at minfd.
* If none is found, grow the file table.
*/
fd = fd_first_free(fdp, minfd, fdp->fd_nfiles);
if (fd >= maxfd)
return (EMFILE);
if (fd >= fdp->fd_nfiles) {
allocfd = min(fd * 2, maxfd);
#ifdef RACCT
if (RACCT_ENABLED()) {
error = racct_set_unlocked(p, RACCT_NOFILE, allocfd);
if (error != 0)
return (EMFILE);
}
#endif
/*
* fd is already equal to first free descriptor >= minfd, so
* we only need to grow the table and we are done.
*/
fdgrowtable_exp(fdp, allocfd);
}
/*
* Perform some sanity checks, then mark the file descriptor as
* used and return it to the caller.
*/
KASSERT(fd >= 0 && fd < min(maxfd, fdp->fd_nfiles),
("invalid descriptor %d", fd));
KASSERT(!fdisused(fdp, fd),
("fd_first_free() returned non-free descriptor"));
KASSERT(fdp->fd_ofiles[fd].fde_file == NULL,
("file descriptor isn't free"));
fdused(fdp, fd);
*result = fd;
return (0);
}
/*
* Allocate n file descriptors for the process.
*/
int
fdallocn(struct thread *td, int minfd, int *fds, int n)
{
struct proc *p = td->td_proc;
struct filedesc *fdp = p->p_fd;
int i;
FILEDESC_XLOCK_ASSERT(fdp);
for (i = 0; i < n; i++)
if (fdalloc(td, 0, &fds[i]) != 0)
break;
if (i < n) {
for (i--; i >= 0; i--)
fdunused(fdp, fds[i]);
return (EMFILE);
}
return (0);
}
/*
* Create a new open file structure and allocate a file descriptor for the
* process that refers to it. We add one reference to the file for the
* descriptor table and one reference for resultfp. This is to prevent us
* being preempted and the entry in the descriptor table closed after we
* release the FILEDESC lock.
*/
int
falloc_caps(struct thread *td, struct file **resultfp, int *resultfd, int flags,
struct filecaps *fcaps)
{
struct file *fp;
int error, fd;
error = falloc_noinstall(td, &fp);
if (error)
return (error); /* no reference held on error */
error = finstall(td, fp, &fd, flags, fcaps);
if (error) {
fdrop(fp, td); /* one reference (fp only) */
return (error);
}
if (resultfp != NULL)
*resultfp = fp; /* copy out result */
else
fdrop(fp, td); /* release local reference */
if (resultfd != NULL)
*resultfd = fd;
return (0);
}
/*
* Create a new open file structure without allocating a file descriptor.
*/
int
falloc_noinstall(struct thread *td, struct file **resultfp)
{
struct file *fp;
int maxuserfiles = maxfiles - (maxfiles / 20);
int openfiles_new;
static struct timeval lastfail;
static int curfail;
KASSERT(resultfp != NULL, ("%s: resultfp == NULL", __func__));
openfiles_new = atomic_fetchadd_int(&openfiles, 1) + 1;
if ((openfiles_new >= maxuserfiles &&
priv_check(td, PRIV_MAXFILES) != 0) ||
openfiles_new >= maxfiles) {
atomic_subtract_int(&openfiles, 1);
if (ppsratecheck(&lastfail, &curfail, 1)) {
printf("kern.maxfiles limit exceeded by uid %i, (%s) "
"please see tuning(7).\n", td->td_ucred->cr_ruid, td->td_proc->p_comm);
}
return (ENFILE);
}
fp = uma_zalloc(file_zone, M_WAITOK);
bzero(fp, sizeof(*fp));
refcount_init(&fp->f_count, 1);
fp->f_cred = crhold(td->td_ucred);
fp->f_ops = &badfileops;
*resultfp = fp;
return (0);
}
/*
* Install a file in a file descriptor table.
*/
void
_finstall(struct filedesc *fdp, struct file *fp, int fd, int flags,
struct filecaps *fcaps)
{
struct filedescent *fde;
MPASS(fp != NULL);
if (fcaps != NULL)
filecaps_validate(fcaps, __func__);
FILEDESC_XLOCK_ASSERT(fdp);
fde = &fdp->fd_ofiles[fd];
#ifdef CAPABILITIES
seqc_write_begin(&fde->fde_seqc);
#endif
fde->fde_file = fp;
fde->fde_flags = (flags & O_CLOEXEC) != 0 ? UF_EXCLOSE : 0;
if (fcaps != NULL)
filecaps_move(fcaps, &fde->fde_caps);
else
filecaps_fill(&fde->fde_caps);
#ifdef CAPABILITIES
seqc_write_end(&fde->fde_seqc);
#endif
}
int
finstall(struct thread *td, struct file *fp, int *fd, int flags,
struct filecaps *fcaps)
{
struct filedesc *fdp = td->td_proc->p_fd;
int error;
MPASS(fd != NULL);
if (!fhold(fp))
return (EBADF);
FILEDESC_XLOCK(fdp);
error = fdalloc(td, 0, fd);
if (__predict_false(error != 0)) {
FILEDESC_XUNLOCK(fdp);
fdrop(fp, td);
return (error);
}
_finstall(fdp, fp, *fd, flags, fcaps);
FILEDESC_XUNLOCK(fdp);
return (0);
}
/*
* Build a new filedesc structure from another.
* Copy the current, root, and jail root vnode references.
*
* If fdp is not NULL, return with it shared locked.
*/
struct filedesc *
fdinit(struct filedesc *fdp, bool prepfiles)
{
struct filedesc0 *newfdp0;
struct filedesc *newfdp;
newfdp0 = uma_zalloc(filedesc0_zone, M_WAITOK | M_ZERO);
newfdp = &newfdp0->fd_fd;
/* Create the file descriptor table. */
FILEDESC_LOCK_INIT(newfdp);
refcount_init(&newfdp->fd_refcnt, 1);
refcount_init(&newfdp->fd_holdcnt, 1);
newfdp->fd_cmask = CMASK;
newfdp->fd_map = newfdp0->fd_dmap;
newfdp->fd_lastfile = -1;
newfdp->fd_files = (struct fdescenttbl *)&newfdp0->fd_dfiles;
newfdp->fd_files->fdt_nfiles = NDFILE;
- if (fdp == NULL)
+ if (fdp == NULL) {
+ newfdp->fd_pwd = pwd_alloc();
return (newfdp);
+ }
if (prepfiles && fdp->fd_lastfile >= newfdp->fd_nfiles)
fdgrowtable(newfdp, fdp->fd_lastfile + 1);
FILEDESC_SLOCK(fdp);
- newfdp->fd_cdir = fdp->fd_cdir;
- if (newfdp->fd_cdir)
- vrefact(newfdp->fd_cdir);
- newfdp->fd_rdir = fdp->fd_rdir;
- if (newfdp->fd_rdir)
- vrefact(newfdp->fd_rdir);
- newfdp->fd_jdir = fdp->fd_jdir;
- if (newfdp->fd_jdir)
- vrefact(newfdp->fd_jdir);
+ newfdp->fd_pwd = pwd_hold_filedesc(fdp);
if (!prepfiles) {
FILEDESC_SUNLOCK(fdp);
} else {
while (fdp->fd_lastfile >= newfdp->fd_nfiles) {
FILEDESC_SUNLOCK(fdp);
fdgrowtable(newfdp, fdp->fd_lastfile + 1);
FILEDESC_SLOCK(fdp);
}
}
return (newfdp);
}
static struct filedesc *
fdhold(struct proc *p)
{
struct filedesc *fdp;
PROC_LOCK_ASSERT(p, MA_OWNED);
fdp = p->p_fd;
if (fdp != NULL)
refcount_acquire(&fdp->fd_holdcnt);
return (fdp);
}
static void
fddrop(struct filedesc *fdp)
{
if (fdp->fd_holdcnt > 1) {
if (refcount_release(&fdp->fd_holdcnt) == 0)
return;
}
FILEDESC_LOCK_DESTROY(fdp);
uma_zfree(filedesc0_zone, fdp);
}
/*
* Share a filedesc structure.
*/
struct filedesc *
fdshare(struct filedesc *fdp)
{
refcount_acquire(&fdp->fd_refcnt);
return (fdp);
}
/*
* Unshare a filedesc structure, if necessary by making a copy
*/
void
fdunshare(struct thread *td)
{
struct filedesc *tmp;
struct proc *p = td->td_proc;
if (p->p_fd->fd_refcnt == 1)
return;
tmp = fdcopy(p->p_fd);
fdescfree(td);
p->p_fd = tmp;
}
void
fdinstall_remapped(struct thread *td, struct filedesc *fdp)
{
fdescfree(td);
td->td_proc->p_fd = fdp;
}
/*
* Copy a filedesc structure. A NULL pointer in returns a NULL reference,
* this is to ease callers, not catch errors.
*/
struct filedesc *
fdcopy(struct filedesc *fdp)
{
struct filedesc *newfdp;
struct filedescent *nfde, *ofde;
int i;
MPASS(fdp != NULL);
newfdp = fdinit(fdp, true);
/* copy all passable descriptors (i.e. not kqueue) */
newfdp->fd_freefile = -1;
for (i = 0; i <= fdp->fd_lastfile; ++i) {
ofde = &fdp->fd_ofiles[i];
if (ofde->fde_file == NULL ||
(ofde->fde_file->f_ops->fo_flags & DFLAG_PASSABLE) == 0 ||
!fhold(ofde->fde_file)) {
if (newfdp->fd_freefile == -1)
newfdp->fd_freefile = i;
continue;
}
nfde = &newfdp->fd_ofiles[i];
*nfde = *ofde;
filecaps_copy(&ofde->fde_caps, &nfde->fde_caps, true);
fdused_init(newfdp, i);
newfdp->fd_lastfile = i;
}
if (newfdp->fd_freefile == -1)
newfdp->fd_freefile = i;
newfdp->fd_cmask = fdp->fd_cmask;
FILEDESC_SUNLOCK(fdp);
return (newfdp);
}
/*
* Copies a filedesc structure, while remapping all file descriptors
* stored inside using a translation table.
*
* File descriptors are copied over to the new file descriptor table,
* regardless of whether the close-on-exec flag is set.
*/
int
fdcopy_remapped(struct filedesc *fdp, const int *fds, size_t nfds,
struct filedesc **ret)
{
struct filedesc *newfdp;
struct filedescent *nfde, *ofde;
int error, i;
MPASS(fdp != NULL);
newfdp = fdinit(fdp, true);
if (nfds > fdp->fd_lastfile + 1) {
/* New table cannot be larger than the old one. */
error = E2BIG;
goto bad;
}
/* Copy all passable descriptors (i.e. not kqueue). */
newfdp->fd_freefile = nfds;
for (i = 0; i < nfds; ++i) {
if (fds[i] < 0 || fds[i] > fdp->fd_lastfile) {
/* File descriptor out of bounds. */
error = EBADF;
goto bad;
}
ofde = &fdp->fd_ofiles[fds[i]];
if (ofde->fde_file == NULL) {
/* Unused file descriptor. */
error = EBADF;
goto bad;
}
if ((ofde->fde_file->f_ops->fo_flags & DFLAG_PASSABLE) == 0) {
/* File descriptor cannot be passed. */
error = EINVAL;
goto bad;
}
if (!fhold(nfde->fde_file)) {
error = EBADF;
goto bad;
}
nfde = &newfdp->fd_ofiles[i];
*nfde = *ofde;
filecaps_copy(&ofde->fde_caps, &nfde->fde_caps, true);
fdused_init(newfdp, i);
newfdp->fd_lastfile = i;
}
newfdp->fd_cmask = fdp->fd_cmask;
FILEDESC_SUNLOCK(fdp);
*ret = newfdp;
return (0);
bad:
FILEDESC_SUNLOCK(fdp);
fdescfree_remapped(newfdp);
return (error);
}
/*
* Clear POSIX style locks. This is only used when fdp looses a reference (i.e.
* one of processes using it exits) and the table used to be shared.
*/
static void
fdclearlocks(struct thread *td)
{
struct filedesc *fdp;
struct filedesc_to_leader *fdtol;
struct flock lf;
struct file *fp;
struct proc *p;
struct vnode *vp;
int i;
p = td->td_proc;
fdp = p->p_fd;
fdtol = p->p_fdtol;
MPASS(fdtol != NULL);
FILEDESC_XLOCK(fdp);
KASSERT(fdtol->fdl_refcount > 0,
("filedesc_to_refcount botch: fdl_refcount=%d",
fdtol->fdl_refcount));
if (fdtol->fdl_refcount == 1 &&
(p->p_leader->p_flag & P_ADVLOCK) != 0) {
for (i = 0; i <= fdp->fd_lastfile; i++) {
fp = fdp->fd_ofiles[i].fde_file;
if (fp == NULL || fp->f_type != DTYPE_VNODE ||
!fhold(fp))
continue;
FILEDESC_XUNLOCK(fdp);
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
vp = fp->f_vnode;
(void) VOP_ADVLOCK(vp,
(caddr_t)p->p_leader, F_UNLCK,
&lf, F_POSIX);
FILEDESC_XLOCK(fdp);
fdrop(fp, td);
}
}
retry:
if (fdtol->fdl_refcount == 1) {
if (fdp->fd_holdleaderscount > 0 &&
(p->p_leader->p_flag & P_ADVLOCK) != 0) {
/*
* close() or kern_dup() has cleared a reference
* in a shared file descriptor table.
*/
fdp->fd_holdleaderswakeup = 1;
sx_sleep(&fdp->fd_holdleaderscount,
FILEDESC_LOCK(fdp), PLOCK, "fdlhold", 0);
goto retry;
}
if (fdtol->fdl_holdcount > 0) {
/*
* Ensure that fdtol->fdl_leader remains
* valid in closef().
*/
fdtol->fdl_wakeup = 1;
sx_sleep(fdtol, FILEDESC_LOCK(fdp), PLOCK,
"fdlhold", 0);
goto retry;
}
}
fdtol->fdl_refcount--;
if (fdtol->fdl_refcount == 0 &&
fdtol->fdl_holdcount == 0) {
fdtol->fdl_next->fdl_prev = fdtol->fdl_prev;
fdtol->fdl_prev->fdl_next = fdtol->fdl_next;
} else
fdtol = NULL;
p->p_fdtol = NULL;
FILEDESC_XUNLOCK(fdp);
if (fdtol != NULL)
free(fdtol, M_FILEDESC_TO_LEADER);
}
/*
* Release a filedesc structure.
*/
static void
fdescfree_fds(struct thread *td, struct filedesc *fdp, bool needclose)
{
struct filedesc0 *fdp0;
struct freetable *ft, *tft;
struct filedescent *fde;
struct file *fp;
int i;
for (i = 0; i <= fdp->fd_lastfile; i++) {
fde = &fdp->fd_ofiles[i];
fp = fde->fde_file;
if (fp != NULL) {
fdefree_last(fde);
if (needclose)
(void) closef(fp, td);
else
fdrop(fp, td);
}
}
if (NDSLOTS(fdp->fd_nfiles) > NDSLOTS(NDFILE))
free(fdp->fd_map, M_FILEDESC);
if (fdp->fd_nfiles > NDFILE)
free(fdp->fd_files, M_FILEDESC);
fdp0 = (struct filedesc0 *)fdp;
SLIST_FOREACH_SAFE(ft, &fdp0->fd_free, ft_next, tft)
free(ft->ft_table, M_FILEDESC);
fddrop(fdp);
}
void
fdescfree(struct thread *td)
{
struct proc *p;
struct filedesc *fdp;
- struct vnode *cdir, *jdir, *rdir;
+ struct pwd *pwd;
p = td->td_proc;
fdp = p->p_fd;
MPASS(fdp != NULL);
#ifdef RACCT
if (RACCT_ENABLED())
racct_set_unlocked(p, RACCT_NOFILE, 0);
#endif
if (p->p_fdtol != NULL)
fdclearlocks(td);
PROC_LOCK(p);
p->p_fd = NULL;
PROC_UNLOCK(p);
if (refcount_release(&fdp->fd_refcnt) == 0)
return;
FILEDESC_XLOCK(fdp);
- cdir = fdp->fd_cdir;
- fdp->fd_cdir = NULL;
- rdir = fdp->fd_rdir;
- fdp->fd_rdir = NULL;
- jdir = fdp->fd_jdir;
- fdp->fd_jdir = NULL;
+ pwd = fdp->fd_pwd;
+ pwd_set(fdp, NULL);
FILEDESC_XUNLOCK(fdp);
- if (cdir != NULL)
- vrele(cdir);
- if (rdir != NULL)
- vrele(rdir);
- if (jdir != NULL)
- vrele(jdir);
+ pwd_drop(pwd);
fdescfree_fds(td, fdp, 1);
}
void
fdescfree_remapped(struct filedesc *fdp)
{
- if (fdp->fd_cdir != NULL)
- vrele(fdp->fd_cdir);
- if (fdp->fd_rdir != NULL)
- vrele(fdp->fd_rdir);
- if (fdp->fd_jdir != NULL)
- vrele(fdp->fd_jdir);
-
+ pwd_drop(fdp->fd_pwd);
fdescfree_fds(curthread, fdp, 0);
}
/*
* For setugid programs, we don't want to people to use that setugidness
* to generate error messages which write to a file which otherwise would
* otherwise be off-limits to the process. We check for filesystems where
* the vnode can change out from under us after execve (like [lin]procfs).
*
* Since fdsetugidsafety calls this only for fd 0, 1 and 2, this check is
* sufficient. We also don't check for setugidness since we know we are.
*/
static bool
is_unsafe(struct file *fp)
{
struct vnode *vp;
if (fp->f_type != DTYPE_VNODE)
return (false);
vp = fp->f_vnode;
return ((vp->v_vflag & VV_PROCDEP) != 0);
}
/*
* Make this setguid thing safe, if at all possible.
*/
void
fdsetugidsafety(struct thread *td)
{
struct filedesc *fdp;
struct file *fp;
int i;
fdp = td->td_proc->p_fd;
KASSERT(fdp->fd_refcnt == 1, ("the fdtable should not be shared"));
MPASS(fdp->fd_nfiles >= 3);
for (i = 0; i <= 2; i++) {
fp = fdp->fd_ofiles[i].fde_file;
if (fp != NULL && is_unsafe(fp)) {
FILEDESC_XLOCK(fdp);
knote_fdclose(td, i);
/*
* NULL-out descriptor prior to close to avoid
* a race while close blocks.
*/
fdfree(fdp, i);
FILEDESC_XUNLOCK(fdp);
(void) closef(fp, td);
}
}
}
/*
* If a specific file object occupies a specific file descriptor, close the
* file descriptor entry and drop a reference on the file object. This is a
* convenience function to handle a subsequent error in a function that calls
* falloc() that handles the race that another thread might have closed the
* file descriptor out from under the thread creating the file object.
*/
void
fdclose(struct thread *td, struct file *fp, int idx)
{
struct filedesc *fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
if (fdp->fd_ofiles[idx].fde_file == fp) {
fdfree(fdp, idx);
FILEDESC_XUNLOCK(fdp);
fdrop(fp, td);
} else
FILEDESC_XUNLOCK(fdp);
}
/*
* Close any files on exec?
*/
void
fdcloseexec(struct thread *td)
{
struct filedesc *fdp;
struct filedescent *fde;
struct file *fp;
int i;
fdp = td->td_proc->p_fd;
KASSERT(fdp->fd_refcnt == 1, ("the fdtable should not be shared"));
for (i = 0; i <= fdp->fd_lastfile; i++) {
fde = &fdp->fd_ofiles[i];
fp = fde->fde_file;
if (fp != NULL && (fp->f_type == DTYPE_MQUEUE ||
(fde->fde_flags & UF_EXCLOSE))) {
FILEDESC_XLOCK(fdp);
fdfree(fdp, i);
(void) closefp(fdp, i, fp, td, 0);
FILEDESC_UNLOCK_ASSERT(fdp);
}
}
}
/*
* It is unsafe for set[ug]id processes to be started with file
* descriptors 0..2 closed, as these descriptors are given implicit
* significance in the Standard C library. fdcheckstd() will create a
* descriptor referencing /dev/null for each of stdin, stdout, and
* stderr that is not already open.
*/
int
fdcheckstd(struct thread *td)
{
struct filedesc *fdp;
register_t save;
int i, error, devnull;
fdp = td->td_proc->p_fd;
KASSERT(fdp->fd_refcnt == 1, ("the fdtable should not be shared"));
MPASS(fdp->fd_nfiles >= 3);
devnull = -1;
for (i = 0; i <= 2; i++) {
if (fdp->fd_ofiles[i].fde_file != NULL)
continue;
save = td->td_retval[0];
if (devnull != -1) {
error = kern_dup(td, FDDUP_FIXED, 0, devnull, i);
} else {
error = kern_openat(td, AT_FDCWD, "/dev/null",
UIO_SYSSPACE, O_RDWR, 0);
if (error == 0) {
devnull = td->td_retval[0];
KASSERT(devnull == i, ("we didn't get our fd"));
}
}
td->td_retval[0] = save;
if (error != 0)
return (error);
}
return (0);
}
/*
* Internal form of close. Decrement reference count on file structure.
* Note: td may be NULL when closing a file that was being passed in a
* message.
*/
int
closef(struct file *fp, struct thread *td)
{
struct vnode *vp;
struct flock lf;
struct filedesc_to_leader *fdtol;
struct filedesc *fdp;
/*
* POSIX record locking dictates that any close releases ALL
* locks owned by this process. This is handled by setting
* a flag in the unlock to free ONLY locks obeying POSIX
* semantics, and not to free BSD-style file locks.
* If the descriptor was in a message, POSIX-style locks
* aren't passed with the descriptor, and the thread pointer
* will be NULL. Callers should be careful only to pass a
* NULL thread pointer when there really is no owning
* context that might have locks, or the locks will be
* leaked.
*/
if (fp->f_type == DTYPE_VNODE && td != NULL) {
vp = fp->f_vnode;
if ((td->td_proc->p_leader->p_flag & P_ADVLOCK) != 0) {
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
(void) VOP_ADVLOCK(vp, (caddr_t)td->td_proc->p_leader,
F_UNLCK, &lf, F_POSIX);
}
fdtol = td->td_proc->p_fdtol;
if (fdtol != NULL) {
/*
* Handle special case where file descriptor table is
* shared between multiple process leaders.
*/
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
for (fdtol = fdtol->fdl_next;
fdtol != td->td_proc->p_fdtol;
fdtol = fdtol->fdl_next) {
if ((fdtol->fdl_leader->p_flag &
P_ADVLOCK) == 0)
continue;
fdtol->fdl_holdcount++;
FILEDESC_XUNLOCK(fdp);
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
lf.l_type = F_UNLCK;
vp = fp->f_vnode;
(void) VOP_ADVLOCK(vp,
(caddr_t)fdtol->fdl_leader, F_UNLCK, &lf,
F_POSIX);
FILEDESC_XLOCK(fdp);
fdtol->fdl_holdcount--;
if (fdtol->fdl_holdcount == 0 &&
fdtol->fdl_wakeup != 0) {
fdtol->fdl_wakeup = 0;
wakeup(fdtol);
}
}
FILEDESC_XUNLOCK(fdp);
}
}
return (fdrop(fp, td));
}
/*
* Initialize the file pointer with the specified properties.
*
* The ops are set with release semantics to be certain that the flags, type,
* and data are visible when ops is. This is to prevent ops methods from being
* called with bad data.
*/
void
finit(struct file *fp, u_int flag, short type, void *data, struct fileops *ops)
{
fp->f_data = data;
fp->f_flag = flag;
fp->f_type = type;
atomic_store_rel_ptr((volatile uintptr_t *)&fp->f_ops, (uintptr_t)ops);
}
int
fget_cap_locked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp)
{
struct filedescent *fde;
int error;
FILEDESC_LOCK_ASSERT(fdp);
fde = fdeget_locked(fdp, fd);
if (fde == NULL) {
error = EBADF;
goto out;
}
#ifdef CAPABILITIES
error = cap_check(cap_rights_fde_inline(fde), needrightsp);
if (error != 0)
goto out;
#endif
if (havecapsp != NULL)
filecaps_copy(&fde->fde_caps, havecapsp, true);
*fpp = fde->fde_file;
error = 0;
out:
return (error);
}
int
fget_cap(struct thread *td, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp)
{
struct filedesc *fdp = td->td_proc->p_fd;
int error;
#ifndef CAPABILITIES
error = fget_unlocked(fdp, fd, needrightsp, fpp);
if (havecapsp != NULL && error == 0)
filecaps_fill(havecapsp);
#else
struct file *fp;
seqc_t seq;
*fpp = NULL;
for (;;) {
error = fget_unlocked_seq(fdp, fd, needrightsp, &fp, &seq);
if (error != 0)
return (error);
if (havecapsp != NULL) {
if (!filecaps_copy(&fdp->fd_ofiles[fd].fde_caps,
havecapsp, false)) {
fdrop(fp, td);
goto get_locked;
}
}
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
*fpp = fp;
return (0);
get_locked:
FILEDESC_SLOCK(fdp);
error = fget_cap_locked(fdp, fd, needrightsp, fpp, havecapsp);
if (error == 0 && !fhold(*fpp))
error = EBADF;
FILEDESC_SUNLOCK(fdp);
#endif
return (error);
}
int
fget_unlocked_seq(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, seqc_t *seqp)
{
#ifdef CAPABILITIES
const struct filedescent *fde;
#endif
const struct fdescenttbl *fdt;
struct file *fp;
#ifdef CAPABILITIES
seqc_t seq;
cap_rights_t haverights;
int error;
#endif
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles))
return (EBADF);
/*
* Fetch the descriptor locklessly. We avoid fdrop() races by
* never raising a refcount above 0. To accomplish this we have
* to use a cmpset loop rather than an atomic_add. The descriptor
* must be re-verified once we acquire a reference to be certain
* that the identity is still correct and we did not lose a race
* due to preemption.
*/
for (;;) {
#ifdef CAPABILITIES
seq = seqc_read(fd_seqc(fdt, fd));
fde = &fdt->fdt_ofiles[fd];
haverights = *cap_rights_fde_inline(fde);
fp = fde->fde_file;
if (!seqc_consistent(fd_seqc(fdt, fd), seq))
continue;
#else
fp = fdt->fdt_ofiles[fd].fde_file;
#endif
if (fp == NULL)
return (EBADF);
#ifdef CAPABILITIES
error = cap_check_inline(&haverights, needrightsp);
if (error != 0)
return (error);
#endif
if (__predict_false(!refcount_acquire_if_not_zero(&fp->f_count))) {
/*
* The count was found either saturated or zero.
* This re-read is not any more racy than using the
* return value from fcmpset.
*/
if (fp->f_count != 0)
return (EBADF);
/*
* Force a reload. Other thread could reallocate the
* table before this fd was closed, so it is possible
* that there is a stale fp pointer in cached version.
*/
fdt = atomic_load_ptr(&fdp->fd_files);
continue;
}
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
#ifdef CAPABILITIES
if (seqc_consistent_nomb(fd_seqc(fdt, fd), seq))
#else
if (fp == fdt->fdt_ofiles[fd].fde_file)
#endif
break;
fdrop(fp, curthread);
}
*fpp = fp;
if (seqp != NULL) {
#ifdef CAPABILITIES
*seqp = seq;
#endif
}
return (0);
}
/*
* See the comments in fget_unlocked_seq for an explanation of how this works.
*
* This is a simplified variant which bails out to the aforementioned routine
* if anything goes wrong. In practice this only happens when userspace is
* racing with itself.
*/
int
fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp)
{
#ifdef CAPABILITIES
const struct filedescent *fde;
#endif
const struct fdescenttbl *fdt;
struct file *fp;
#ifdef CAPABILITIES
seqc_t seq;
const cap_rights_t *haverights;
#endif
fdt = fdp->fd_files;
if (__predict_false((u_int)fd >= fdt->fdt_nfiles))
return (EBADF);
#ifdef CAPABILITIES
seq = seqc_read_any(fd_seqc(fdt, fd));
if (__predict_false(seqc_in_modify(seq)))
goto out_fallback;
fde = &fdt->fdt_ofiles[fd];
haverights = cap_rights_fde_inline(fde);
fp = fde->fde_file;
#else
fp = fdt->fdt_ofiles[fd].fde_file;
#endif
if (__predict_false(fp == NULL))
goto out_fallback;
#ifdef CAPABILITIES
if (__predict_false(cap_check_inline_transient(haverights, needrightsp)))
goto out_fallback;
#endif
if (__predict_false(!refcount_acquire_if_not_zero(&fp->f_count)))
goto out_fallback;
/*
* Use an acquire barrier to force re-reading of fdt so it is
* refreshed for verification.
*/
atomic_thread_fence_acq();
fdt = fdp->fd_files;
#ifdef CAPABILITIES
if (__predict_false(!seqc_consistent_nomb(fd_seqc(fdt, fd), seq)))
#else
if (__predict_false(fp != fdt->fdt_ofiles[fd].fde_file))
#endif
goto out_fdrop;
*fpp = fp;
return (0);
out_fdrop:
fdrop(fp, curthread);
out_fallback:
return (fget_unlocked_seq(fdp, fd, needrightsp, fpp, NULL));
}
/*
* Extract the file pointer associated with the specified descriptor for the
* current user process.
*
* If the descriptor doesn't exist or doesn't match 'flags', EBADF is
* returned.
*
* File's rights will be checked against the capability rights mask.
*
* If an error occurred the non-zero error is returned and *fpp is set to
* NULL. Otherwise *fpp is held and set and zero is returned. Caller is
* responsible for fdrop().
*/
static __inline int
_fget(struct thread *td, int fd, struct file **fpp, int flags,
cap_rights_t *needrightsp)
{
struct filedesc *fdp;
struct file *fp;
int error;
*fpp = NULL;
fdp = td->td_proc->p_fd;
error = fget_unlocked(fdp, fd, needrightsp, &fp);
if (__predict_false(error != 0))
return (error);
if (__predict_false(fp->f_ops == &badfileops)) {
fdrop(fp, td);
return (EBADF);
}
/*
* FREAD and FWRITE failure return EBADF as per POSIX.
*/
error = 0;
switch (flags) {
case FREAD:
case FWRITE:
if ((fp->f_flag & flags) == 0)
error = EBADF;
break;
case FEXEC:
if ((fp->f_flag & (FREAD | FEXEC)) == 0 ||
((fp->f_flag & FWRITE) != 0))
error = EBADF;
break;
case 0:
break;
default:
KASSERT(0, ("wrong flags"));
}
if (error != 0) {
fdrop(fp, td);
return (error);
}
*fpp = fp;
return (0);
}
int
fget(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, 0, rightsp));
}
int
fget_mmap(struct thread *td, int fd, cap_rights_t *rightsp, vm_prot_t *maxprotp,
struct file **fpp)
{
int error;
#ifndef CAPABILITIES
error = _fget(td, fd, fpp, 0, rightsp);
if (maxprotp != NULL)
*maxprotp = VM_PROT_ALL;
return (error);
#else
cap_rights_t fdrights;
struct filedesc *fdp;
struct file *fp;
seqc_t seq;
*fpp = NULL;
fdp = td->td_proc->p_fd;
MPASS(cap_rights_is_set(rightsp, CAP_MMAP));
for (;;) {
error = fget_unlocked_seq(fdp, fd, rightsp, &fp, &seq);
if (__predict_false(error != 0))
return (error);
if (__predict_false(fp->f_ops == &badfileops)) {
fdrop(fp, td);
return (EBADF);
}
if (maxprotp != NULL)
fdrights = *cap_rights(fdp, fd);
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
/*
* If requested, convert capability rights to access flags.
*/
if (maxprotp != NULL)
*maxprotp = cap_rights_to_vmprot(&fdrights);
*fpp = fp;
return (0);
#endif
}
int
fget_read(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, FREAD, rightsp));
}
int
fget_write(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
{
return (_fget(td, fd, fpp, FWRITE, rightsp));
}
int
fget_fcntl(struct thread *td, int fd, cap_rights_t *rightsp, int needfcntl,
struct file **fpp)
{
struct filedesc *fdp = td->td_proc->p_fd;
#ifndef CAPABILITIES
return (fget_unlocked(fdp, fd, rightsp, fpp));
#else
struct file *fp;
int error;
seqc_t seq;
*fpp = NULL;
MPASS(cap_rights_is_set(rightsp, CAP_FCNTL));
for (;;) {
error = fget_unlocked_seq(fdp, fd, rightsp, &fp, &seq);
if (error != 0)
return (error);
error = cap_fcntl_check(fdp, fd, needfcntl);
if (!fd_modified(fdp, fd, seq))
break;
fdrop(fp, td);
}
if (error != 0) {
fdrop(fp, td);
return (error);
}
*fpp = fp;
return (0);
#endif
}
/*
* Like fget() but loads the underlying vnode, or returns an error if the
* descriptor does not represent a vnode. Note that pipes use vnodes but
* never have VM objects. The returned vnode will be vref()'d.
*
* XXX: what about the unused flags ?
*/
static __inline int
_fgetvp(struct thread *td, int fd, int flags, cap_rights_t *needrightsp,
struct vnode **vpp)
{
struct file *fp;
int error;
*vpp = NULL;
error = _fget(td, fd, &fp, flags, needrightsp);
if (error != 0)
return (error);
if (fp->f_vnode == NULL) {
error = EINVAL;
} else {
*vpp = fp->f_vnode;
vrefact(*vpp);
}
fdrop(fp, td);
return (error);
}
int
fgetvp(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, 0, rightsp, vpp));
}
int
fgetvp_rights(struct thread *td, int fd, cap_rights_t *needrightsp,
struct filecaps *havecaps, struct vnode **vpp)
{
struct filecaps caps;
struct file *fp;
int error;
error = fget_cap(td, fd, needrightsp, &fp, &caps);
if (error != 0)
return (error);
if (fp->f_ops == &badfileops) {
error = EBADF;
goto out;
}
if (fp->f_vnode == NULL) {
error = EINVAL;
goto out;
}
*havecaps = caps;
*vpp = fp->f_vnode;
vrefact(*vpp);
fdrop(fp, td);
return (0);
out:
filecaps_free(&caps);
fdrop(fp, td);
return (error);
}
int
fgetvp_read(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, FREAD, rightsp, vpp));
}
int
fgetvp_exec(struct thread *td, int fd, cap_rights_t *rightsp, struct vnode **vpp)
{
return (_fgetvp(td, fd, FEXEC, rightsp, vpp));
}
#ifdef notyet
int
fgetvp_write(struct thread *td, int fd, cap_rights_t *rightsp,
struct vnode **vpp)
{
return (_fgetvp(td, fd, FWRITE, rightsp, vpp));
}
#endif
/*
* Handle the last reference to a file being closed.
*
* Without the noinline attribute clang keeps inlining the func thorough this
* file when fdrop is used.
*/
int __noinline
_fdrop(struct file *fp, struct thread *td)
{
int error;
if (fp->f_count != 0)
panic("fdrop: count %d", fp->f_count);
error = fo_close(fp, td);
atomic_subtract_int(&openfiles, 1);
crfree(fp->f_cred);
free(fp->f_advice, M_FADVISE);
uma_zfree(file_zone, fp);
return (error);
}
/*
* Apply an advisory lock on a file descriptor.
*
* Just attempt to get a record lock of the requested type on the entire file
* (l_whence = SEEK_SET, l_start = 0, l_len = 0).
*/
#ifndef _SYS_SYSPROTO_H_
struct flock_args {
int fd;
int how;
};
#endif
/* ARGSUSED */
int
sys_flock(struct thread *td, struct flock_args *uap)
{
struct file *fp;
struct vnode *vp;
struct flock lf;
int error;
error = fget(td, uap->fd, &cap_flock_rights, &fp);
if (error != 0)
return (error);
if (fp->f_type != DTYPE_VNODE) {
fdrop(fp, td);
return (EOPNOTSUPP);
}
vp = fp->f_vnode;
lf.l_whence = SEEK_SET;
lf.l_start = 0;
lf.l_len = 0;
if (uap->how & LOCK_UN) {
lf.l_type = F_UNLCK;
atomic_clear_int(&fp->f_flag, FHASLOCK);
error = VOP_ADVLOCK(vp, (caddr_t)fp, F_UNLCK, &lf, F_FLOCK);
goto done2;
}
if (uap->how & LOCK_EX)
lf.l_type = F_WRLCK;
else if (uap->how & LOCK_SH)
lf.l_type = F_RDLCK;
else {
error = EBADF;
goto done2;
}
atomic_set_int(&fp->f_flag, FHASLOCK);
error = VOP_ADVLOCK(vp, (caddr_t)fp, F_SETLK, &lf,
(uap->how & LOCK_NB) ? F_FLOCK : F_FLOCK | F_WAIT);
done2:
fdrop(fp, td);
return (error);
}
/*
* Duplicate the specified descriptor to a free descriptor.
*/
int
dupfdopen(struct thread *td, struct filedesc *fdp, int dfd, int mode,
int openerror, int *indxp)
{
struct filedescent *newfde, *oldfde;
struct file *fp;
u_long *ioctls;
int error, indx;
KASSERT(openerror == ENODEV || openerror == ENXIO,
("unexpected error %d in %s", openerror, __func__));
/*
* If the to-be-dup'd fd number is greater than the allowed number
* of file descriptors, or the fd to be dup'd has already been
* closed, then reject.
*/
FILEDESC_XLOCK(fdp);
if ((fp = fget_locked(fdp, dfd)) == NULL) {
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
error = fdalloc(td, 0, &indx);
if (error != 0) {
FILEDESC_XUNLOCK(fdp);
return (error);
}
/*
* There are two cases of interest here.
*
* For ENODEV simply dup (dfd) to file descriptor (indx) and return.
*
* For ENXIO steal away the file structure from (dfd) and store it in
* (indx). (dfd) is effectively closed by this operation.
*/
switch (openerror) {
case ENODEV:
/*
* Check that the mode the file is being opened for is a
* subset of the mode of the existing descriptor.
*/
if (((mode & (FREAD|FWRITE)) | fp->f_flag) != fp->f_flag) {
fdunused(fdp, indx);
FILEDESC_XUNLOCK(fdp);
return (EACCES);
}
if (!fhold(fp)) {
fdunused(fdp, indx);
FILEDESC_XUNLOCK(fdp);
return (EBADF);
}
newfde = &fdp->fd_ofiles[indx];
oldfde = &fdp->fd_ofiles[dfd];
ioctls = filecaps_copy_prep(&oldfde->fde_caps);
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
memcpy(newfde, oldfde, fde_change_size);
filecaps_copy_finish(&oldfde->fde_caps, &newfde->fde_caps,
ioctls);
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
break;
case ENXIO:
/*
* Steal away the file pointer from dfd and stuff it into indx.
*/
newfde = &fdp->fd_ofiles[indx];
oldfde = &fdp->fd_ofiles[dfd];
#ifdef CAPABILITIES
seqc_write_begin(&newfde->fde_seqc);
#endif
memcpy(newfde, oldfde, fde_change_size);
oldfde->fde_file = NULL;
fdunused(fdp, dfd);
#ifdef CAPABILITIES
seqc_write_end(&newfde->fde_seqc);
#endif
break;
}
FILEDESC_XUNLOCK(fdp);
*indxp = indx;
return (0);
}
/*
* This sysctl determines if we will allow a process to chroot(2) if it
* has a directory open:
* 0: disallowed for all processes.
* 1: allowed for processes that were not already chroot(2)'ed.
* 2: allowed for all processes.
*/
static int chroot_allow_open_directories = 1;
SYSCTL_INT(_kern, OID_AUTO, chroot_allow_open_directories, CTLFLAG_RW,
&chroot_allow_open_directories, 0,
"Allow a process to chroot(2) if it has a directory open");
/*
* Helper function for raised chroot(2) security function: Refuse if
* any filedescriptors are open directories.
*/
static int
chroot_refuse_vdir_fds(struct filedesc *fdp)
{
struct vnode *vp;
struct file *fp;
int fd;
FILEDESC_LOCK_ASSERT(fdp);
for (fd = 0; fd <= fdp->fd_lastfile; fd++) {
fp = fget_locked(fdp, fd);
if (fp == NULL)
continue;
if (fp->f_type == DTYPE_VNODE) {
vp = fp->f_vnode;
if (vp->v_type == VDIR)
return (EPERM);
}
}
return (0);
}
+static void
+pwd_fill(struct pwd *oldpwd, struct pwd *newpwd)
+{
+
+ if (newpwd->pwd_cdir == NULL && oldpwd->pwd_cdir != NULL) {
+ vrefact(oldpwd->pwd_cdir);
+ newpwd->pwd_cdir = oldpwd->pwd_cdir;
+ }
+
+ if (newpwd->pwd_rdir == NULL && oldpwd->pwd_rdir != NULL) {
+ vrefact(oldpwd->pwd_rdir);
+ newpwd->pwd_rdir = oldpwd->pwd_rdir;
+ }
+
+ if (newpwd->pwd_jdir == NULL && oldpwd->pwd_jdir != NULL) {
+ vrefact(oldpwd->pwd_jdir);
+ newpwd->pwd_jdir = oldpwd->pwd_jdir;
+ }
+}
+
+struct pwd *
+pwd_hold_filedesc(struct filedesc *fdp)
+{
+ struct pwd *pwd;
+
+ FILEDESC_LOCK_ASSERT(fdp);
+ pwd = fdp->fd_pwd;
+ if (pwd != NULL)
+ refcount_acquire(&pwd->pwd_refcount);
+ return (pwd);
+}
+
+struct pwd *
+pwd_hold(struct thread *td)
+{
+ struct filedesc *fdp;
+ struct pwd *pwd;
+
+ fdp = td->td_proc->p_fd;
+
+ FILEDESC_SLOCK(fdp);
+ pwd = fdp->fd_pwd;
+ MPASS(pwd != NULL);
+ refcount_acquire(&pwd->pwd_refcount);
+ FILEDESC_SUNLOCK(fdp);
+ return (pwd);
+}
+
+static struct pwd *
+pwd_alloc(void)
+{
+ struct pwd *pwd;
+
+ pwd = malloc(sizeof(*pwd), M_PWD, M_WAITOK | M_ZERO);
+ refcount_init(&pwd->pwd_refcount, 1);
+ return (pwd);
+}
+
+void
+pwd_drop(struct pwd *pwd)
+{
+
+ if (!refcount_release(&pwd->pwd_refcount))
+ return;
+
+ if (pwd->pwd_cdir != NULL)
+ vrele(pwd->pwd_cdir);
+ if (pwd->pwd_rdir != NULL)
+ vrele(pwd->pwd_rdir);
+ if (pwd->pwd_jdir != NULL)
+ vrele(pwd->pwd_jdir);
+ free(pwd, M_PWD);
+}
+
/*
- * Common routine for kern_chroot() and jail_attach(). The caller is
- * responsible for invoking priv_check() and mac_vnode_check_chroot() to
- * authorize this operation.
- */
+* Common routine for kern_chroot() and jail_attach(). The caller is
+* responsible for invoking priv_check() and mac_vnode_check_chroot() to
+* authorize this operation.
+*/
int
pwd_chroot(struct thread *td, struct vnode *vp)
{
struct filedesc *fdp;
- struct vnode *oldvp;
+ struct pwd *newpwd, *oldpwd;
int error;
fdp = td->td_proc->p_fd;
+ newpwd = pwd_alloc();
FILEDESC_XLOCK(fdp);
+ oldpwd = fdp->fd_pwd;
if (chroot_allow_open_directories == 0 ||
- (chroot_allow_open_directories == 1 && fdp->fd_rdir != rootvnode)) {
+ (chroot_allow_open_directories == 1 &&
+ oldpwd->pwd_rdir != rootvnode)) {
error = chroot_refuse_vdir_fds(fdp);
if (error != 0) {
FILEDESC_XUNLOCK(fdp);
+ pwd_drop(newpwd);
return (error);
}
}
- oldvp = fdp->fd_rdir;
+
vrefact(vp);
- fdp->fd_rdir = vp;
- if (fdp->fd_jdir == NULL) {
+ newpwd->pwd_rdir = vp;
+ if (oldpwd->pwd_jdir == NULL) {
vrefact(vp);
- fdp->fd_jdir = vp;
+ newpwd->pwd_jdir = vp;
}
+ pwd_fill(oldpwd, newpwd);
+ pwd_set(fdp, newpwd);
FILEDESC_XUNLOCK(fdp);
- vrele(oldvp);
+ pwd_drop(oldpwd);
return (0);
}
void
pwd_chdir(struct thread *td, struct vnode *vp)
{
struct filedesc *fdp;
- struct vnode *oldvp;
+ struct pwd *newpwd, *oldpwd;
+ VNPASS(vp->v_usecount > 0, vp);
+
+ newpwd = pwd_alloc();
fdp = td->td_proc->p_fd;
FILEDESC_XLOCK(fdp);
- VNASSERT(vp->v_usecount > 0, vp,
- ("chdir to a vnode with zero usecount"));
- oldvp = fdp->fd_cdir;
- fdp->fd_cdir = vp;
+ oldpwd = fdp->fd_pwd;
+ newpwd->pwd_cdir = vp;
+ pwd_fill(oldpwd, newpwd);
+ pwd_set(fdp, newpwd);
FILEDESC_XUNLOCK(fdp);
- vrele(oldvp);
+ pwd_drop(oldpwd);
}
+void
+pwd_ensure_dirs(void)
+{
+ struct filedesc *fdp;
+ struct pwd *oldpwd, *newpwd;
+
+ fdp = curproc->p_fd;
+ FILEDESC_XLOCK(fdp);
+ oldpwd = fdp->fd_pwd;
+ if (oldpwd->pwd_cdir != NULL && oldpwd->pwd_rdir != NULL) {
+ FILEDESC_XUNLOCK(fdp);
+ return;
+ }
+ FILEDESC_XUNLOCK(fdp);
+
+ newpwd = pwd_alloc();
+ FILEDESC_XLOCK(fdp);
+ oldpwd = fdp->fd_pwd;
+ pwd_fill(oldpwd, newpwd);
+ if (newpwd->pwd_cdir == NULL) {
+ vrefact(rootvnode);
+ newpwd->pwd_cdir = rootvnode;
+ }
+ if (newpwd->pwd_rdir == NULL) {
+ vrefact(rootvnode);
+ newpwd->pwd_rdir = rootvnode;
+ }
+ pwd_set(fdp, newpwd);
+ FILEDESC_XUNLOCK(fdp);
+ pwd_drop(oldpwd);
+}
+
/*
* Scan all active processes and prisons to see if any of them have a current
* or root directory of `olddp'. If so, replace them with the new mount point.
*/
void
mountcheckdirs(struct vnode *olddp, struct vnode *newdp)
{
struct filedesc *fdp;
+ struct pwd *newpwd, *oldpwd;
struct prison *pr;
struct proc *p;
int nrele;
if (vrefcnt(olddp) == 1)
return;
nrele = 0;
+ newpwd = pwd_alloc();
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
continue;
FILEDESC_XLOCK(fdp);
- if (fdp->fd_cdir == olddp) {
+ oldpwd = fdp->fd_pwd;
+ if (oldpwd == NULL ||
+ (oldpwd->pwd_cdir != olddp &&
+ oldpwd->pwd_rdir != olddp &&
+ oldpwd->pwd_jdir != olddp)) {
+ FILEDESC_XUNLOCK(fdp);
+ fddrop(fdp);
+ continue;
+ }
+ if (oldpwd->pwd_cdir == olddp) {
vrefact(newdp);
- fdp->fd_cdir = newdp;
- nrele++;
+ newpwd->pwd_cdir = newdp;
}
- if (fdp->fd_rdir == olddp) {
+ if (oldpwd->pwd_rdir == olddp) {
vrefact(newdp);
- fdp->fd_rdir = newdp;
- nrele++;
+ newpwd->pwd_rdir = newdp;
}
- if (fdp->fd_jdir == olddp) {
+ if (oldpwd->pwd_jdir == olddp) {
vrefact(newdp);
- fdp->fd_jdir = newdp;
- nrele++;
+ newpwd->pwd_jdir = newdp;
}
+ pwd_fill(oldpwd, newpwd);
+ pwd_set(fdp, newpwd);
FILEDESC_XUNLOCK(fdp);
+ pwd_drop(oldpwd);
fddrop(fdp);
+ newpwd = pwd_alloc();
}
sx_sunlock(&allproc_lock);
+ pwd_drop(newpwd);
if (rootvnode == olddp) {
vrefact(newdp);
rootvnode = newdp;
nrele++;
}
mtx_lock(&prison0.pr_mtx);
if (prison0.pr_root == olddp) {
vrefact(newdp);
prison0.pr_root = newdp;
nrele++;
}
mtx_unlock(&prison0.pr_mtx);
sx_slock(&allprison_lock);
TAILQ_FOREACH(pr, &allprison, pr_list) {
mtx_lock(&pr->pr_mtx);
if (pr->pr_root == olddp) {
vrefact(newdp);
pr->pr_root = newdp;
nrele++;
}
mtx_unlock(&pr->pr_mtx);
}
sx_sunlock(&allprison_lock);
while (nrele--)
vrele(olddp);
}
struct filedesc_to_leader *
filedesc_to_leader_alloc(struct filedesc_to_leader *old, struct filedesc *fdp, struct proc *leader)
{
struct filedesc_to_leader *fdtol;
fdtol = malloc(sizeof(struct filedesc_to_leader),
M_FILEDESC_TO_LEADER, M_WAITOK);
fdtol->fdl_refcount = 1;
fdtol->fdl_holdcount = 0;
fdtol->fdl_wakeup = 0;
fdtol->fdl_leader = leader;
if (old != NULL) {
FILEDESC_XLOCK(fdp);
fdtol->fdl_next = old->fdl_next;
fdtol->fdl_prev = old;
old->fdl_next = fdtol;
fdtol->fdl_next->fdl_prev = fdtol;
FILEDESC_XUNLOCK(fdp);
} else {
fdtol->fdl_next = fdtol;
fdtol->fdl_prev = fdtol;
}
return (fdtol);
}
static int
sysctl_kern_proc_nfds(SYSCTL_HANDLER_ARGS)
{
struct filedesc *fdp;
int i, count, slots;
if (*(int *)arg1 != 0)
return (EINVAL);
fdp = curproc->p_fd;
count = 0;
FILEDESC_SLOCK(fdp);
slots = NDSLOTS(fdp->fd_lastfile + 1);
for (i = 0; i < slots; i++)
count += bitcountl(fdp->fd_map[i]);
FILEDESC_SUNLOCK(fdp);
return (SYSCTL_OUT(req, &count, sizeof(count)));
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_NFDS, nfds,
CTLFLAG_RD|CTLFLAG_CAPRD|CTLFLAG_MPSAFE, sysctl_kern_proc_nfds,
"Number of open file descriptors");
/*
* Get file structures globally.
*/
static int
sysctl_kern_file(SYSCTL_HANDLER_ARGS)
{
struct xfile xf;
struct filedesc *fdp;
struct file *fp;
struct proc *p;
int error, n;
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
if (req->oldptr == NULL) {
n = 0;
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
continue;
/* overestimates sparse tables. */
if (fdp->fd_lastfile > 0)
n += fdp->fd_lastfile;
fddrop(fdp);
}
sx_sunlock(&allproc_lock);
return (SYSCTL_OUT(req, 0, n * sizeof(xf)));
}
error = 0;
bzero(&xf, sizeof(xf));
xf.xf_size = sizeof(xf);
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
PROC_LOCK(p);
if (p->p_state == PRS_NEW) {
PROC_UNLOCK(p);
continue;
}
if (p_cansee(req->td, p) != 0) {
PROC_UNLOCK(p);
continue;
}
xf.xf_pid = p->p_pid;
xf.xf_uid = p->p_ucred->cr_uid;
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
continue;
FILEDESC_SLOCK(fdp);
for (n = 0; fdp->fd_refcnt > 0 && n <= fdp->fd_lastfile; ++n) {
if ((fp = fdp->fd_ofiles[n].fde_file) == NULL)
continue;
xf.xf_fd = n;
xf.xf_file = (uintptr_t)fp;
xf.xf_data = (uintptr_t)fp->f_data;
xf.xf_vnode = (uintptr_t)fp->f_vnode;
xf.xf_type = (uintptr_t)fp->f_type;
xf.xf_count = fp->f_count;
xf.xf_msgcount = 0;
xf.xf_offset = foffset_get(fp);
xf.xf_flag = fp->f_flag;
error = SYSCTL_OUT(req, &xf, sizeof(xf));
if (error)
break;
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
if (error)
break;
}
sx_sunlock(&allproc_lock);
return (error);
}
SYSCTL_PROC(_kern, KERN_FILE, file, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE,
0, 0, sysctl_kern_file, "S,xfile", "Entire file table");
#ifdef KINFO_FILE_SIZE
CTASSERT(sizeof(struct kinfo_file) == KINFO_FILE_SIZE);
#endif
static int
xlate_fflags(int fflags)
{
static const struct {
int fflag;
int kf_fflag;
} fflags_table[] = {
{ FAPPEND, KF_FLAG_APPEND },
{ FASYNC, KF_FLAG_ASYNC },
{ FFSYNC, KF_FLAG_FSYNC },
{ FHASLOCK, KF_FLAG_HASLOCK },
{ FNONBLOCK, KF_FLAG_NONBLOCK },
{ FREAD, KF_FLAG_READ },
{ FWRITE, KF_FLAG_WRITE },
{ O_CREAT, KF_FLAG_CREAT },
{ O_DIRECT, KF_FLAG_DIRECT },
{ O_EXCL, KF_FLAG_EXCL },
{ O_EXEC, KF_FLAG_EXEC },
{ O_EXLOCK, KF_FLAG_EXLOCK },
{ O_NOFOLLOW, KF_FLAG_NOFOLLOW },
{ O_SHLOCK, KF_FLAG_SHLOCK },
{ O_TRUNC, KF_FLAG_TRUNC }
};
unsigned int i;
int kflags;
kflags = 0;
for (i = 0; i < nitems(fflags_table); i++)
if (fflags & fflags_table[i].fflag)
kflags |= fflags_table[i].kf_fflag;
return (kflags);
}
/* Trim unused data from kf_path by truncating the structure size. */
void
pack_kinfo(struct kinfo_file *kif)
{
kif->kf_structsize = offsetof(struct kinfo_file, kf_path) +
strlen(kif->kf_path) + 1;
kif->kf_structsize = roundup(kif->kf_structsize, sizeof(uint64_t));
}
static void
export_file_to_kinfo(struct file *fp, int fd, cap_rights_t *rightsp,
struct kinfo_file *kif, struct filedesc *fdp, int flags)
{
int error;
bzero(kif, sizeof(*kif));
/* Set a default type to allow for empty fill_kinfo() methods. */
kif->kf_type = KF_TYPE_UNKNOWN;
kif->kf_flags = xlate_fflags(fp->f_flag);
if (rightsp != NULL)
kif->kf_cap_rights = *rightsp;
else
cap_rights_init_zero(&kif->kf_cap_rights);
kif->kf_fd = fd;
kif->kf_ref_count = fp->f_count;
kif->kf_offset = foffset_get(fp);
/*
* This may drop the filedesc lock, so the 'fp' cannot be
* accessed after this call.
*/
error = fo_fill_kinfo(fp, kif, fdp);
if (error == 0)
kif->kf_status |= KF_ATTR_VALID;
if ((flags & KERN_FILEDESC_PACK_KINFO) != 0)
pack_kinfo(kif);
else
kif->kf_structsize = roundup2(sizeof(*kif), sizeof(uint64_t));
}
static void
export_vnode_to_kinfo(struct vnode *vp, int fd, int fflags,
struct kinfo_file *kif, int flags)
{
int error;
bzero(kif, sizeof(*kif));
kif->kf_type = KF_TYPE_VNODE;
error = vn_fill_kinfo_vnode(vp, kif);
if (error == 0)
kif->kf_status |= KF_ATTR_VALID;
kif->kf_flags = xlate_fflags(fflags);
cap_rights_init_zero(&kif->kf_cap_rights);
kif->kf_fd = fd;
kif->kf_ref_count = -1;
kif->kf_offset = -1;
if ((flags & KERN_FILEDESC_PACK_KINFO) != 0)
pack_kinfo(kif);
else
kif->kf_structsize = roundup2(sizeof(*kif), sizeof(uint64_t));
vrele(vp);
}
struct export_fd_buf {
struct filedesc *fdp;
struct sbuf *sb;
ssize_t remainder;
struct kinfo_file kif;
int flags;
};
static int
export_kinfo_to_sb(struct export_fd_buf *efbuf)
{
struct kinfo_file *kif;
kif = &efbuf->kif;
if (efbuf->remainder != -1) {
if (efbuf->remainder < kif->kf_structsize) {
/* Terminate export. */
efbuf->remainder = 0;
return (0);
}
efbuf->remainder -= kif->kf_structsize;
}
return (sbuf_bcat(efbuf->sb, kif, kif->kf_structsize) == 0 ? 0 : ENOMEM);
}
static int
export_file_to_sb(struct file *fp, int fd, cap_rights_t *rightsp,
struct export_fd_buf *efbuf)
{
int error;
if (efbuf->remainder == 0)
return (0);
export_file_to_kinfo(fp, fd, rightsp, &efbuf->kif, efbuf->fdp,
efbuf->flags);
FILEDESC_SUNLOCK(efbuf->fdp);
error = export_kinfo_to_sb(efbuf);
FILEDESC_SLOCK(efbuf->fdp);
return (error);
}
static int
export_vnode_to_sb(struct vnode *vp, int fd, int fflags,
struct export_fd_buf *efbuf)
{
int error;
if (efbuf->remainder == 0)
return (0);
if (efbuf->fdp != NULL)
FILEDESC_SUNLOCK(efbuf->fdp);
export_vnode_to_kinfo(vp, fd, fflags, &efbuf->kif, efbuf->flags);
error = export_kinfo_to_sb(efbuf);
if (efbuf->fdp != NULL)
FILEDESC_SLOCK(efbuf->fdp);
return (error);
}
/*
* Store a process file descriptor information to sbuf.
*
* Takes a locked proc as argument, and returns with the proc unlocked.
*/
int
kern_proc_filedesc_out(struct proc *p, struct sbuf *sb, ssize_t maxlen,
int flags)
{
struct file *fp;
struct filedesc *fdp;
struct export_fd_buf *efbuf;
struct vnode *cttyvp, *textvp, *tracevp;
+ struct pwd *pwd;
int error, i;
cap_rights_t rights;
PROC_LOCK_ASSERT(p, MA_OWNED);
/* ktrace vnode */
tracevp = p->p_tracevp;
if (tracevp != NULL)
vrefact(tracevp);
/* text vnode */
textvp = p->p_textvp;
if (textvp != NULL)
vrefact(textvp);
/* Controlling tty. */
cttyvp = NULL;
if (p->p_pgrp != NULL && p->p_pgrp->pg_session != NULL) {
cttyvp = p->p_pgrp->pg_session->s_ttyvp;
if (cttyvp != NULL)
vrefact(cttyvp);
}
fdp = fdhold(p);
PROC_UNLOCK(p);
efbuf = malloc(sizeof(*efbuf), M_TEMP, M_WAITOK);
efbuf->fdp = NULL;
efbuf->sb = sb;
efbuf->remainder = maxlen;
efbuf->flags = flags;
if (tracevp != NULL)
export_vnode_to_sb(tracevp, KF_FD_TYPE_TRACE, FREAD | FWRITE,
efbuf);
if (textvp != NULL)
export_vnode_to_sb(textvp, KF_FD_TYPE_TEXT, FREAD, efbuf);
if (cttyvp != NULL)
export_vnode_to_sb(cttyvp, KF_FD_TYPE_CTTY, FREAD | FWRITE,
efbuf);
error = 0;
if (fdp == NULL)
goto fail;
efbuf->fdp = fdp;
FILEDESC_SLOCK(fdp);
- /* working directory */
- if (fdp->fd_cdir != NULL) {
- vrefact(fdp->fd_cdir);
- export_vnode_to_sb(fdp->fd_cdir, KF_FD_TYPE_CWD, FREAD, efbuf);
+ pwd = pwd_hold_filedesc(fdp);
+ if (pwd != NULL) {
+ /* working directory */
+ if (pwd->pwd_cdir != NULL) {
+ vrefact(pwd->pwd_cdir);
+ export_vnode_to_sb(pwd->pwd_cdir, KF_FD_TYPE_CWD, FREAD, efbuf);
+ }
+ /* root directory */
+ if (pwd->pwd_rdir != NULL) {
+ vrefact(pwd->pwd_rdir);
+ export_vnode_to_sb(pwd->pwd_rdir, KF_FD_TYPE_ROOT, FREAD, efbuf);
+ }
+ /* jail directory */
+ if (pwd->pwd_jdir != NULL) {
+ vrefact(pwd->pwd_jdir);
+ export_vnode_to_sb(pwd->pwd_jdir, KF_FD_TYPE_JAIL, FREAD, efbuf);
+ }
+ pwd_drop(pwd);
}
- /* root directory */
- if (fdp->fd_rdir != NULL) {
- vrefact(fdp->fd_rdir);
- export_vnode_to_sb(fdp->fd_rdir, KF_FD_TYPE_ROOT, FREAD, efbuf);
- }
- /* jail directory */
- if (fdp->fd_jdir != NULL) {
- vrefact(fdp->fd_jdir);
- export_vnode_to_sb(fdp->fd_jdir, KF_FD_TYPE_JAIL, FREAD, efbuf);
- }
for (i = 0; fdp->fd_refcnt > 0 && i <= fdp->fd_lastfile; i++) {
if ((fp = fdp->fd_ofiles[i].fde_file) == NULL)
continue;
#ifdef CAPABILITIES
rights = *cap_rights(fdp, i);
#else /* !CAPABILITIES */
rights = cap_no_rights;
#endif
/*
* Create sysctl entry. It is OK to drop the filedesc
* lock inside of export_file_to_sb() as we will
* re-validate and re-evaluate its properties when the
* loop continues.
*/
error = export_file_to_sb(fp, i, &rights, efbuf);
if (error != 0 || efbuf->remainder == 0)
break;
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
fail:
free(efbuf, M_TEMP);
return (error);
}
#define FILEDESC_SBUF_SIZE (sizeof(struct kinfo_file) * 5)
/*
* Get per-process file descriptors for use by procstat(1), et al.
*/
static int
sysctl_kern_proc_filedesc(SYSCTL_HANDLER_ARGS)
{
struct sbuf sb;
struct proc *p;
ssize_t maxlen;
int error, error2, *name;
name = (int *)arg1;
sbuf_new_for_sysctl(&sb, NULL, FILEDESC_SBUF_SIZE, req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0) {
sbuf_delete(&sb);
return (error);
}
maxlen = req->oldptr != NULL ? req->oldlen : -1;
error = kern_proc_filedesc_out(p, &sb, maxlen,
KERN_FILEDESC_PACK_KINFO);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
#ifdef COMPAT_FREEBSD7
#ifdef KINFO_OFILE_SIZE
CTASSERT(sizeof(struct kinfo_ofile) == KINFO_OFILE_SIZE);
#endif
static void
kinfo_to_okinfo(struct kinfo_file *kif, struct kinfo_ofile *okif)
{
okif->kf_structsize = sizeof(*okif);
okif->kf_type = kif->kf_type;
okif->kf_fd = kif->kf_fd;
okif->kf_ref_count = kif->kf_ref_count;
okif->kf_flags = kif->kf_flags & (KF_FLAG_READ | KF_FLAG_WRITE |
KF_FLAG_APPEND | KF_FLAG_ASYNC | KF_FLAG_FSYNC | KF_FLAG_NONBLOCK |
KF_FLAG_DIRECT | KF_FLAG_HASLOCK);
okif->kf_offset = kif->kf_offset;
if (kif->kf_type == KF_TYPE_VNODE)
okif->kf_vnode_type = kif->kf_un.kf_file.kf_file_type;
else
okif->kf_vnode_type = KF_VTYPE_VNON;
strlcpy(okif->kf_path, kif->kf_path, sizeof(okif->kf_path));
if (kif->kf_type == KF_TYPE_SOCKET) {
okif->kf_sock_domain = kif->kf_un.kf_sock.kf_sock_domain0;
okif->kf_sock_type = kif->kf_un.kf_sock.kf_sock_type0;
okif->kf_sock_protocol = kif->kf_un.kf_sock.kf_sock_protocol0;
okif->kf_sa_local = kif->kf_un.kf_sock.kf_sa_local;
okif->kf_sa_peer = kif->kf_un.kf_sock.kf_sa_peer;
} else {
okif->kf_sa_local.ss_family = AF_UNSPEC;
okif->kf_sa_peer.ss_family = AF_UNSPEC;
}
}
static int
export_vnode_for_osysctl(struct vnode *vp, int type, struct kinfo_file *kif,
struct kinfo_ofile *okif, struct filedesc *fdp, struct sysctl_req *req)
{
int error;
vrefact(vp);
FILEDESC_SUNLOCK(fdp);
export_vnode_to_kinfo(vp, type, 0, kif, KERN_FILEDESC_PACK_KINFO);
kinfo_to_okinfo(kif, okif);
error = SYSCTL_OUT(req, okif, sizeof(*okif));
FILEDESC_SLOCK(fdp);
return (error);
}
/*
* Get per-process file descriptors for use by procstat(1), et al.
*/
static int
sysctl_kern_proc_ofiledesc(SYSCTL_HANDLER_ARGS)
{
struct kinfo_ofile *okif;
struct kinfo_file *kif;
struct filedesc *fdp;
+ struct pwd *pwd;
int error, i, *name;
struct file *fp;
struct proc *p;
name = (int *)arg1;
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0)
return (error);
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
return (ENOENT);
kif = malloc(sizeof(*kif), M_TEMP, M_WAITOK);
okif = malloc(sizeof(*okif), M_TEMP, M_WAITOK);
FILEDESC_SLOCK(fdp);
- if (fdp->fd_cdir != NULL)
- export_vnode_for_osysctl(fdp->fd_cdir, KF_FD_TYPE_CWD, kif,
- okif, fdp, req);
- if (fdp->fd_rdir != NULL)
- export_vnode_for_osysctl(fdp->fd_rdir, KF_FD_TYPE_ROOT, kif,
- okif, fdp, req);
- if (fdp->fd_jdir != NULL)
- export_vnode_for_osysctl(fdp->fd_jdir, KF_FD_TYPE_JAIL, kif,
- okif, fdp, req);
+ pwd = pwd_hold_filedesc(fdp);
+ if (pwd != NULL) {
+ if (pwd->pwd_cdir != NULL)
+ export_vnode_for_osysctl(pwd->pwd_cdir, KF_FD_TYPE_CWD, kif,
+ okif, fdp, req);
+ if (pwd->pwd_rdir != NULL)
+ export_vnode_for_osysctl(pwd->pwd_rdir, KF_FD_TYPE_ROOT, kif,
+ okif, fdp, req);
+ if (pwd->pwd_jdir != NULL)
+ export_vnode_for_osysctl(pwd->pwd_jdir, KF_FD_TYPE_JAIL, kif,
+ okif, fdp, req);
+ pwd_drop(pwd);
+ }
for (i = 0; fdp->fd_refcnt > 0 && i <= fdp->fd_lastfile; i++) {
if ((fp = fdp->fd_ofiles[i].fde_file) == NULL)
continue;
export_file_to_kinfo(fp, i, NULL, kif, fdp,
KERN_FILEDESC_PACK_KINFO);
FILEDESC_SUNLOCK(fdp);
kinfo_to_okinfo(kif, okif);
error = SYSCTL_OUT(req, okif, sizeof(*okif));
FILEDESC_SLOCK(fdp);
if (error)
break;
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
free(kif, M_TEMP);
free(okif, M_TEMP);
return (0);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_OFILEDESC, ofiledesc,
CTLFLAG_RD|CTLFLAG_MPSAFE, sysctl_kern_proc_ofiledesc,
"Process ofiledesc entries");
#endif /* COMPAT_FREEBSD7 */
int
vntype_to_kinfo(int vtype)
{
struct {
int vtype;
int kf_vtype;
} vtypes_table[] = {
{ VBAD, KF_VTYPE_VBAD },
{ VBLK, KF_VTYPE_VBLK },
{ VCHR, KF_VTYPE_VCHR },
{ VDIR, KF_VTYPE_VDIR },
{ VFIFO, KF_VTYPE_VFIFO },
{ VLNK, KF_VTYPE_VLNK },
{ VNON, KF_VTYPE_VNON },
{ VREG, KF_VTYPE_VREG },
{ VSOCK, KF_VTYPE_VSOCK }
};
unsigned int i;
/*
* Perform vtype translation.
*/
for (i = 0; i < nitems(vtypes_table); i++)
if (vtypes_table[i].vtype == vtype)
return (vtypes_table[i].kf_vtype);
return (KF_VTYPE_UNKNOWN);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_FILEDESC, filedesc,
CTLFLAG_RD|CTLFLAG_MPSAFE, sysctl_kern_proc_filedesc,
"Process filedesc entries");
/*
* Store a process current working directory information to sbuf.
*
* Takes a locked proc as argument, and returns with the proc unlocked.
*/
int
kern_proc_cwd_out(struct proc *p, struct sbuf *sb, ssize_t maxlen)
{
struct filedesc *fdp;
struct export_fd_buf *efbuf;
+ struct vnode *cdir;
int error;
PROC_LOCK_ASSERT(p, MA_OWNED);
fdp = fdhold(p);
PROC_UNLOCK(p);
if (fdp == NULL)
return (EINVAL);
efbuf = malloc(sizeof(*efbuf), M_TEMP, M_WAITOK);
efbuf->fdp = fdp;
efbuf->sb = sb;
efbuf->remainder = maxlen;
FILEDESC_SLOCK(fdp);
- if (fdp->fd_cdir == NULL)
+ cdir = fdp->fd_pwd->pwd_cdir;
+ if (cdir == NULL) {
error = EINVAL;
- else {
- vrefact(fdp->fd_cdir);
- error = export_vnode_to_sb(fdp->fd_cdir, KF_FD_TYPE_CWD,
- FREAD, efbuf);
+ } else {
+ vrefact(cdir);
+ error = export_vnode_to_sb(cdir, KF_FD_TYPE_CWD, FREAD, efbuf);
}
FILEDESC_SUNLOCK(fdp);
fddrop(fdp);
free(efbuf, M_TEMP);
return (error);
}
/*
* Get per-process current working directory.
*/
static int
sysctl_kern_proc_cwd(SYSCTL_HANDLER_ARGS)
{
struct sbuf sb;
struct proc *p;
ssize_t maxlen;
int error, error2, *name;
name = (int *)arg1;
sbuf_new_for_sysctl(&sb, NULL, sizeof(struct kinfo_file), req);
sbuf_clear_flags(&sb, SBUF_INCLUDENUL);
error = pget((pid_t)name[0], PGET_CANDEBUG | PGET_NOTWEXIT, &p);
if (error != 0) {
sbuf_delete(&sb);
return (error);
}
maxlen = req->oldptr != NULL ? req->oldlen : -1;
error = kern_proc_cwd_out(p, &sb, maxlen);
error2 = sbuf_finish(&sb);
sbuf_delete(&sb);
return (error != 0 ? error : error2);
}
static SYSCTL_NODE(_kern_proc, KERN_PROC_CWD, cwd, CTLFLAG_RD|CTLFLAG_MPSAFE,
sysctl_kern_proc_cwd, "Process current working directory");
#ifdef DDB
/*
* For the purposes of debugging, generate a human-readable string for the
* file type.
*/
static const char *
file_type_to_name(short type)
{
switch (type) {
case 0:
return ("zero");
case DTYPE_VNODE:
return ("vnode");
case DTYPE_SOCKET:
return ("socket");
case DTYPE_PIPE:
return ("pipe");
case DTYPE_FIFO:
return ("fifo");
case DTYPE_KQUEUE:
return ("kqueue");
case DTYPE_CRYPTO:
return ("crypto");
case DTYPE_MQUEUE:
return ("mqueue");
case DTYPE_SHM:
return ("shm");
case DTYPE_SEM:
return ("ksem");
case DTYPE_PTS:
return ("pts");
case DTYPE_DEV:
return ("dev");
case DTYPE_PROCDESC:
return ("proc");
case DTYPE_LINUXEFD:
return ("levent");
case DTYPE_LINUXTFD:
return ("ltimer");
default:
return ("unkn");
}
}
/*
* For the purposes of debugging, identify a process (if any, perhaps one of
* many) that references the passed file in its file descriptor array. Return
* NULL if none.
*/
static struct proc *
file_to_first_proc(struct file *fp)
{
struct filedesc *fdp;
struct proc *p;
int n;
FOREACH_PROC_IN_SYSTEM(p) {
if (p->p_state == PRS_NEW)
continue;
fdp = p->p_fd;
if (fdp == NULL)
continue;
for (n = 0; n <= fdp->fd_lastfile; n++) {
if (fp == fdp->fd_ofiles[n].fde_file)
return (p);
}
}
return (NULL);
}
static void
db_print_file(struct file *fp, int header)
{
#define XPTRWIDTH ((int)howmany(sizeof(void *) * NBBY, 4))
struct proc *p;
if (header)
db_printf("%*s %6s %*s %8s %4s %5s %6s %*s %5s %s\n",
XPTRWIDTH, "File", "Type", XPTRWIDTH, "Data", "Flag",
"GCFl", "Count", "MCount", XPTRWIDTH, "Vnode", "FPID",
"FCmd");
p = file_to_first_proc(fp);
db_printf("%*p %6s %*p %08x %04x %5d %6d %*p %5d %s\n", XPTRWIDTH,
fp, file_type_to_name(fp->f_type), XPTRWIDTH, fp->f_data,
fp->f_flag, 0, fp->f_count, 0, XPTRWIDTH, fp->f_vnode,
p != NULL ? p->p_pid : -1, p != NULL ? p->p_comm : "-");
#undef XPTRWIDTH
}
DB_SHOW_COMMAND(file, db_show_file)
{
struct file *fp;
if (!have_addr) {
db_printf("usage: show file <addr>\n");
return;
}
fp = (struct file *)addr;
db_print_file(fp, 1);
}
DB_SHOW_COMMAND(files, db_show_files)
{
struct filedesc *fdp;
struct file *fp;
struct proc *p;
int header;
int n;
header = 1;
FOREACH_PROC_IN_SYSTEM(p) {
if (p->p_state == PRS_NEW)
continue;
if ((fdp = p->p_fd) == NULL)
continue;
for (n = 0; n <= fdp->fd_lastfile; ++n) {
if ((fp = fdp->fd_ofiles[n].fde_file) == NULL)
continue;
db_print_file(fp, header);
header = 0;
}
}
}
#endif
SYSCTL_INT(_kern, KERN_MAXFILESPERPROC, maxfilesperproc, CTLFLAG_RW,
&maxfilesperproc, 0, "Maximum files allowed open per process");
SYSCTL_INT(_kern, KERN_MAXFILES, maxfiles, CTLFLAG_RW,
&maxfiles, 0, "Maximum number of files");
SYSCTL_INT(_kern, OID_AUTO, openfiles, CTLFLAG_RD,
&openfiles, 0, "System-wide number of open files");
/* ARGSUSED*/
static void
filelistinit(void *dummy)
{
file_zone = uma_zcreate("Files", sizeof(struct file), NULL, NULL,
NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
filedesc0_zone = uma_zcreate("filedesc0", sizeof(struct filedesc0),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
mtx_init(&sigio_lock, "sigio lock", NULL, MTX_DEF);
}
SYSINIT(select, SI_SUB_LOCK, SI_ORDER_FIRST, filelistinit, NULL);
/*-------------------------------------------------------------------*/
static int
badfo_readwrite(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return (EBADF);
}
static int
badfo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
static int
badfo_ioctl(struct file *fp, u_long com, void *data, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return (0);
}
static int
badfo_kqfilter(struct file *fp, struct knote *kn)
{
return (EBADF);
}
static int
badfo_stat(struct file *fp, struct stat *sb, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_close(struct file *fp, struct thread *td)
{
return (0);
}
static int
badfo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
struct thread *td)
{
return (EBADF);
}
static int
badfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
return (EBADF);
}
static int
badfo_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
{
return (0);
}
struct fileops badfileops = {
.fo_read = badfo_readwrite,
.fo_write = badfo_readwrite,
.fo_truncate = badfo_truncate,
.fo_ioctl = badfo_ioctl,
.fo_poll = badfo_poll,
.fo_kqfilter = badfo_kqfilter,
.fo_stat = badfo_stat,
.fo_close = badfo_close,
.fo_chmod = badfo_chmod,
.fo_chown = badfo_chown,
.fo_sendfile = badfo_sendfile,
.fo_fill_kinfo = badfo_fill_kinfo,
};
int
invfo_rdwr(struct file *fp, struct uio *uio, struct ucred *active_cred,
int flags, struct thread *td)
{
return (EOPNOTSUPP);
}
int
invfo_truncate(struct file *fp, off_t length, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_ioctl(struct file *fp, u_long com, void *data,
struct ucred *active_cred, struct thread *td)
{
return (ENOTTY);
}
int
invfo_poll(struct file *fp, int events, struct ucred *active_cred,
struct thread *td)
{
return (poll_no_poll(events));
}
int
invfo_kqfilter(struct file *fp, struct knote *kn)
{
return (EINVAL);
}
int
invfo_chmod(struct file *fp, mode_t mode, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
struct thread *td)
{
return (EINVAL);
}
int
invfo_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio,
struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags,
struct thread *td)
{
return (EINVAL);
}
/*-------------------------------------------------------------------*/
/*
* File Descriptor pseudo-device driver (/dev/fd/).
*
* Opening minor device N dup()s the file (if any) connected to file
* descriptor N belonging to the calling process. Note that this driver
* consists of only the ``open()'' routine, because all subsequent
* references to this file will be direct to the other driver.
*
* XXX: we could give this one a cloning event handler if necessary.
*/
/* ARGSUSED */
static int
fdopen(struct cdev *dev, int mode, int type, struct thread *td)
{
/*
* XXX Kludge: set curthread->td_dupfd to contain the value of the
* the file descriptor being sought for duplication. The error
* return ensures that the vnode for this device will be released
* by vn_open. Open will detect this special error and take the
* actions in dupfdopen below. Other callers of vn_open or VOP_OPEN
* will simply report the error.
*/
td->td_dupfd = dev2unit(dev);
return (ENODEV);
}
static struct cdevsw fildesc_cdevsw = {
.d_version = D_VERSION,
.d_open = fdopen,
.d_name = "FD",
};
static void
fildesc_drvinit(void *unused)
{
struct cdev *dev;
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 0, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/0");
make_dev_alias(dev, "stdin");
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 1, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/1");
make_dev_alias(dev, "stdout");
dev = make_dev_credf(MAKEDEV_ETERNAL, &fildesc_cdevsw, 2, NULL,
UID_ROOT, GID_WHEEL, 0666, "fd/2");
make_dev_alias(dev, "stderr");
}
SYSINIT(fildescdev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, fildesc_drvinit, NULL);
Index: head/sys/kern/kern_linker.c
===================================================================
--- head/sys/kern/kern_linker.c (revision 358502)
+++ head/sys/kern/kern_linker.c (revision 358503)
@@ -1,2266 +1,2266 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 1997-2000 Doug Rabson
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_kld.h"
#include "opt_hwpmc_hooks.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysproto.h>
#include <sys/sysent.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/module.h>
#include <sys/mount.h>
#include <sys/linker.h>
#include <sys/eventhandler.h>
#include <sys/fcntl.h>
#include <sys/jail.h>
#include <sys/libkern.h>
#include <sys/namei.h>
#include <sys/vnode.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <net/vnet.h>
#include <security/mac/mac_framework.h>
#include "linker_if.h"
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
#endif
#ifdef KLD_DEBUG
int kld_debug = 0;
SYSCTL_INT(_debug, OID_AUTO, kld_debug, CTLFLAG_RWTUN,
&kld_debug, 0, "Set various levels of KLD debug");
#endif
/* These variables are used by kernel debuggers to enumerate loaded files. */
const int kld_off_address = offsetof(struct linker_file, address);
const int kld_off_filename = offsetof(struct linker_file, filename);
const int kld_off_pathname = offsetof(struct linker_file, pathname);
const int kld_off_next = offsetof(struct linker_file, link.tqe_next);
/*
* static char *linker_search_path(const char *name, struct mod_depend
* *verinfo);
*/
static const char *linker_basename(const char *path);
/*
* Find a currently loaded file given its filename.
*/
static linker_file_t linker_find_file_by_name(const char* _filename);
/*
* Find a currently loaded file given its file id.
*/
static linker_file_t linker_find_file_by_id(int _fileid);
/* Metadata from the static kernel */
SET_DECLARE(modmetadata_set, struct mod_metadata);
MALLOC_DEFINE(M_LINKER, "linker", "kernel linker");
linker_file_t linker_kernel_file;
static struct sx kld_sx; /* kernel linker lock */
/*
* Load counter used by clients to determine if a linker file has been
* re-loaded. This counter is incremented for each file load.
*/
static int loadcnt;
static linker_class_list_t classes;
static linker_file_list_t linker_files;
static int next_file_id = 1;
static int linker_no_more_classes = 0;
#define LINKER_GET_NEXT_FILE_ID(a) do { \
linker_file_t lftmp; \
\
if (!cold) \
sx_assert(&kld_sx, SA_XLOCKED); \
retry: \
TAILQ_FOREACH(lftmp, &linker_files, link) { \
if (next_file_id == lftmp->id) { \
next_file_id++; \
goto retry; \
} \
} \
(a) = next_file_id; \
} while(0)
/* XXX wrong name; we're looking at version provision tags here, not modules */
typedef TAILQ_HEAD(, modlist) modlisthead_t;
struct modlist {
TAILQ_ENTRY(modlist) link; /* chain together all modules */
linker_file_t container;
const char *name;
int version;
};
typedef struct modlist *modlist_t;
static modlisthead_t found_modules;
static int linker_file_add_dependency(linker_file_t file,
linker_file_t dep);
static caddr_t linker_file_lookup_symbol_internal(linker_file_t file,
const char* name, int deps);
static int linker_load_module(const char *kldname,
const char *modname, struct linker_file *parent,
const struct mod_depend *verinfo, struct linker_file **lfpp);
static modlist_t modlist_lookup2(const char *name, const struct mod_depend *verinfo);
static void
linker_init(void *arg)
{
sx_init(&kld_sx, "kernel linker");
TAILQ_INIT(&classes);
TAILQ_INIT(&linker_files);
}
SYSINIT(linker, SI_SUB_KLD, SI_ORDER_FIRST, linker_init, NULL);
static void
linker_stop_class_add(void *arg)
{
linker_no_more_classes = 1;
}
SYSINIT(linker_class, SI_SUB_KLD, SI_ORDER_ANY, linker_stop_class_add, NULL);
int
linker_add_class(linker_class_t lc)
{
/*
* We disallow any class registration past SI_ORDER_ANY
* of SI_SUB_KLD. We bump the reference count to keep the
* ops from being freed.
*/
if (linker_no_more_classes == 1)
return (EPERM);
kobj_class_compile((kobj_class_t) lc);
((kobj_class_t)lc)->refs++; /* XXX: kobj_mtx */
TAILQ_INSERT_TAIL(&classes, lc, link);
return (0);
}
static void
linker_file_sysinit(linker_file_t lf)
{
struct sysinit **start, **stop, **sipp, **xipp, *save;
KLD_DPF(FILE, ("linker_file_sysinit: calling SYSINITs for %s\n",
lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "sysinit_set", &start, &stop, NULL) != 0)
return;
/*
* Perform a bubble sort of the system initialization objects by
* their subsystem (primary key) and order (secondary key).
*
* Since some things care about execution order, this is the operation
* which ensures continued function.
*/
for (sipp = start; sipp < stop; sipp++) {
for (xipp = sipp + 1; xipp < stop; xipp++) {
if ((*sipp)->subsystem < (*xipp)->subsystem ||
((*sipp)->subsystem == (*xipp)->subsystem &&
(*sipp)->order <= (*xipp)->order))
continue; /* skip */
save = *sipp;
*sipp = *xipp;
*xipp = save;
}
}
/*
* Traverse the (now) ordered list of system initialization tasks.
* Perform each task, and continue on to the next task.
*/
sx_xunlock(&kld_sx);
mtx_lock(&Giant);
for (sipp = start; sipp < stop; sipp++) {
if ((*sipp)->subsystem == SI_SUB_DUMMY)
continue; /* skip dummy task(s) */
/* Call function */
(*((*sipp)->func)) ((*sipp)->udata);
}
mtx_unlock(&Giant);
sx_xlock(&kld_sx);
}
static void
linker_file_sysuninit(linker_file_t lf)
{
struct sysinit **start, **stop, **sipp, **xipp, *save;
KLD_DPF(FILE, ("linker_file_sysuninit: calling SYSUNINITs for %s\n",
lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "sysuninit_set", &start, &stop,
NULL) != 0)
return;
/*
* Perform a reverse bubble sort of the system initialization objects
* by their subsystem (primary key) and order (secondary key).
*
* Since some things care about execution order, this is the operation
* which ensures continued function.
*/
for (sipp = start; sipp < stop; sipp++) {
for (xipp = sipp + 1; xipp < stop; xipp++) {
if ((*sipp)->subsystem > (*xipp)->subsystem ||
((*sipp)->subsystem == (*xipp)->subsystem &&
(*sipp)->order >= (*xipp)->order))
continue; /* skip */
save = *sipp;
*sipp = *xipp;
*xipp = save;
}
}
/*
* Traverse the (now) ordered list of system initialization tasks.
* Perform each task, and continue on to the next task.
*/
sx_xunlock(&kld_sx);
mtx_lock(&Giant);
for (sipp = start; sipp < stop; sipp++) {
if ((*sipp)->subsystem == SI_SUB_DUMMY)
continue; /* skip dummy task(s) */
/* Call function */
(*((*sipp)->func)) ((*sipp)->udata);
}
mtx_unlock(&Giant);
sx_xlock(&kld_sx);
}
static void
linker_file_register_sysctls(linker_file_t lf, bool enable)
{
struct sysctl_oid **start, **stop, **oidp;
KLD_DPF(FILE,
("linker_file_register_sysctls: registering SYSCTLs for %s\n",
lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
return;
sx_xunlock(&kld_sx);
sysctl_wlock();
for (oidp = start; oidp < stop; oidp++) {
if (enable)
sysctl_register_oid(*oidp);
else
sysctl_register_disabled_oid(*oidp);
}
sysctl_wunlock();
sx_xlock(&kld_sx);
}
static void
linker_file_enable_sysctls(linker_file_t lf)
{
struct sysctl_oid **start, **stop, **oidp;
KLD_DPF(FILE,
("linker_file_enable_sysctls: enable SYSCTLs for %s\n",
lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
return;
sx_xunlock(&kld_sx);
sysctl_wlock();
for (oidp = start; oidp < stop; oidp++)
sysctl_enable_oid(*oidp);
sysctl_wunlock();
sx_xlock(&kld_sx);
}
static void
linker_file_unregister_sysctls(linker_file_t lf)
{
struct sysctl_oid **start, **stop, **oidp;
KLD_DPF(FILE, ("linker_file_unregister_sysctls: unregistering SYSCTLs"
" for %s\n", lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0)
return;
sx_xunlock(&kld_sx);
sysctl_wlock();
for (oidp = start; oidp < stop; oidp++)
sysctl_unregister_oid(*oidp);
sysctl_wunlock();
sx_xlock(&kld_sx);
}
static int
linker_file_register_modules(linker_file_t lf)
{
struct mod_metadata **start, **stop, **mdp;
const moduledata_t *moddata;
int first_error, error;
KLD_DPF(FILE, ("linker_file_register_modules: registering modules"
" in %s\n", lf->filename));
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_file_lookup_set(lf, "modmetadata_set", &start,
&stop, NULL) != 0) {
/*
* This fallback should be unnecessary, but if we get booted
* from boot2 instead of loader and we are missing our
* metadata then we have to try the best we can.
*/
if (lf == linker_kernel_file) {
start = SET_BEGIN(modmetadata_set);
stop = SET_LIMIT(modmetadata_set);
} else
return (0);
}
first_error = 0;
for (mdp = start; mdp < stop; mdp++) {
if ((*mdp)->md_type != MDT_MODULE)
continue;
moddata = (*mdp)->md_data;
KLD_DPF(FILE, ("Registering module %s in %s\n",
moddata->name, lf->filename));
error = module_register(moddata, lf);
if (error) {
printf("Module %s failed to register: %d\n",
moddata->name, error);
if (first_error == 0)
first_error = error;
}
}
return (first_error);
}
static void
linker_init_kernel_modules(void)
{
sx_xlock(&kld_sx);
linker_file_register_modules(linker_kernel_file);
sx_xunlock(&kld_sx);
}
SYSINIT(linker_kernel, SI_SUB_KLD, SI_ORDER_ANY, linker_init_kernel_modules,
NULL);
static int
linker_load_file(const char *filename, linker_file_t *result)
{
linker_class_t lc;
linker_file_t lf;
int foundfile, error, modules;
/* Refuse to load modules if securelevel raised */
if (prison0.pr_securelevel > 0)
return (EPERM);
sx_assert(&kld_sx, SA_XLOCKED);
lf = linker_find_file_by_name(filename);
if (lf) {
KLD_DPF(FILE, ("linker_load_file: file %s is already loaded,"
" incrementing refs\n", filename));
*result = lf;
lf->refs++;
return (0);
}
foundfile = 0;
error = 0;
/*
* We do not need to protect (lock) classes here because there is
* no class registration past startup (SI_SUB_KLD, SI_ORDER_ANY)
* and there is no class deregistration mechanism at this time.
*/
TAILQ_FOREACH(lc, &classes, link) {
KLD_DPF(FILE, ("linker_load_file: trying to load %s\n",
filename));
error = LINKER_LOAD_FILE(lc, filename, &lf);
/*
* If we got something other than ENOENT, then it exists but
* we cannot load it for some other reason.
*/
if (error != ENOENT)
foundfile = 1;
if (lf) {
error = linker_file_register_modules(lf);
if (error == EEXIST) {
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
return (error);
}
modules = !TAILQ_EMPTY(&lf->modules);
linker_file_register_sysctls(lf, false);
linker_file_sysinit(lf);
lf->flags |= LINKER_FILE_LINKED;
/*
* If all of the modules in this file failed
* to load, unload the file and return an
* error of ENOEXEC.
*/
if (modules && TAILQ_EMPTY(&lf->modules)) {
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
return (ENOEXEC);
}
linker_file_enable_sysctls(lf);
EVENTHANDLER_INVOKE(kld_load, lf);
*result = lf;
return (0);
}
}
/*
* Less than ideal, but tells the user whether it failed to load or
* the module was not found.
*/
if (foundfile) {
/*
* If the file type has not been recognized by the last try
* printout a message before to fail.
*/
if (error == ENOSYS)
printf("%s: %s - unsupported file type\n",
__func__, filename);
/*
* Format not recognized or otherwise unloadable.
* When loading a module that is statically built into
* the kernel EEXIST percolates back up as the return
* value. Preserve this so that apps like sysinstall
* can recognize this special case and not post bogus
* dialog boxes.
*/
if (error != EEXIST)
error = ENOEXEC;
} else
error = ENOENT; /* Nothing found */
return (error);
}
int
linker_reference_module(const char *modname, struct mod_depend *verinfo,
linker_file_t *result)
{
modlist_t mod;
int error;
sx_xlock(&kld_sx);
if ((mod = modlist_lookup2(modname, verinfo)) != NULL) {
*result = mod->container;
(*result)->refs++;
sx_xunlock(&kld_sx);
return (0);
}
error = linker_load_module(NULL, modname, NULL, verinfo, result);
sx_xunlock(&kld_sx);
return (error);
}
int
linker_release_module(const char *modname, struct mod_depend *verinfo,
linker_file_t lf)
{
modlist_t mod;
int error;
sx_xlock(&kld_sx);
if (lf == NULL) {
KASSERT(modname != NULL,
("linker_release_module: no file or name"));
mod = modlist_lookup2(modname, verinfo);
if (mod == NULL) {
sx_xunlock(&kld_sx);
return (ESRCH);
}
lf = mod->container;
} else
KASSERT(modname == NULL && verinfo == NULL,
("linker_release_module: both file and name"));
error = linker_file_unload(lf, LINKER_UNLOAD_NORMAL);
sx_xunlock(&kld_sx);
return (error);
}
static linker_file_t
linker_find_file_by_name(const char *filename)
{
linker_file_t lf;
char *koname;
koname = malloc(strlen(filename) + 4, M_LINKER, M_WAITOK);
sprintf(koname, "%s.ko", filename);
sx_assert(&kld_sx, SA_XLOCKED);
TAILQ_FOREACH(lf, &linker_files, link) {
if (strcmp(lf->filename, koname) == 0)
break;
if (strcmp(lf->filename, filename) == 0)
break;
}
free(koname, M_LINKER);
return (lf);
}
static linker_file_t
linker_find_file_by_id(int fileid)
{
linker_file_t lf;
sx_assert(&kld_sx, SA_XLOCKED);
TAILQ_FOREACH(lf, &linker_files, link)
if (lf->id == fileid && lf->flags & LINKER_FILE_LINKED)
break;
return (lf);
}
int
linker_file_foreach(linker_predicate_t *predicate, void *context)
{
linker_file_t lf;
int retval = 0;
sx_xlock(&kld_sx);
TAILQ_FOREACH(lf, &linker_files, link) {
retval = predicate(lf, context);
if (retval != 0)
break;
}
sx_xunlock(&kld_sx);
return (retval);
}
linker_file_t
linker_make_file(const char *pathname, linker_class_t lc)
{
linker_file_t lf;
const char *filename;
if (!cold)
sx_assert(&kld_sx, SA_XLOCKED);
filename = linker_basename(pathname);
KLD_DPF(FILE, ("linker_make_file: new file, filename='%s' for pathname='%s'\n", filename, pathname));
lf = (linker_file_t)kobj_create((kobj_class_t)lc, M_LINKER, M_WAITOK);
if (lf == NULL)
return (NULL);
lf->ctors_addr = 0;
lf->ctors_size = 0;
lf->refs = 1;
lf->userrefs = 0;
lf->flags = 0;
lf->filename = strdup(filename, M_LINKER);
lf->pathname = strdup(pathname, M_LINKER);
LINKER_GET_NEXT_FILE_ID(lf->id);
lf->ndeps = 0;
lf->deps = NULL;
lf->loadcnt = ++loadcnt;
#ifdef __arm__
lf->exidx_addr = 0;
lf->exidx_size = 0;
#endif
STAILQ_INIT(&lf->common);
TAILQ_INIT(&lf->modules);
TAILQ_INSERT_TAIL(&linker_files, lf, link);
return (lf);
}
int
linker_file_unload(linker_file_t file, int flags)
{
module_t mod, next;
modlist_t ml, nextml;
struct common_symbol *cp;
int error, i;
/* Refuse to unload modules if securelevel raised. */
if (prison0.pr_securelevel > 0)
return (EPERM);
sx_assert(&kld_sx, SA_XLOCKED);
KLD_DPF(FILE, ("linker_file_unload: lf->refs=%d\n", file->refs));
/* Easy case of just dropping a reference. */
if (file->refs > 1) {
file->refs--;
return (0);
}
/* Give eventhandlers a chance to prevent the unload. */
error = 0;
EVENTHANDLER_INVOKE(kld_unload_try, file, &error);
if (error != 0)
return (EBUSY);
KLD_DPF(FILE, ("linker_file_unload: file is unloading,"
" informing modules\n"));
/*
* Quiesce all the modules to give them a chance to veto the unload.
*/
MOD_SLOCK;
for (mod = TAILQ_FIRST(&file->modules); mod;
mod = module_getfnext(mod)) {
error = module_quiesce(mod);
if (error != 0 && flags != LINKER_UNLOAD_FORCE) {
KLD_DPF(FILE, ("linker_file_unload: module %s"
" vetoed unload\n", module_getname(mod)));
/*
* XXX: Do we need to tell all the quiesced modules
* that they can resume work now via a new module
* event?
*/
MOD_SUNLOCK;
return (error);
}
}
MOD_SUNLOCK;
/*
* Inform any modules associated with this file that they are
* being unloaded.
*/
MOD_XLOCK;
for (mod = TAILQ_FIRST(&file->modules); mod; mod = next) {
next = module_getfnext(mod);
MOD_XUNLOCK;
/*
* Give the module a chance to veto the unload.
*/
if ((error = module_unload(mod)) != 0) {
#ifdef KLD_DEBUG
MOD_SLOCK;
KLD_DPF(FILE, ("linker_file_unload: module %s"
" failed unload\n", module_getname(mod)));
MOD_SUNLOCK;
#endif
return (error);
}
MOD_XLOCK;
module_release(mod);
}
MOD_XUNLOCK;
TAILQ_FOREACH_SAFE(ml, &found_modules, link, nextml) {
if (ml->container == file) {
TAILQ_REMOVE(&found_modules, ml, link);
free(ml, M_LINKER);
}
}
/*
* Don't try to run SYSUNINITs if we are unloaded due to a
* link error.
*/
if (file->flags & LINKER_FILE_LINKED) {
file->flags &= ~LINKER_FILE_LINKED;
linker_file_unregister_sysctls(file);
linker_file_sysuninit(file);
}
TAILQ_REMOVE(&linker_files, file, link);
if (file->deps) {
for (i = 0; i < file->ndeps; i++)
linker_file_unload(file->deps[i], flags);
free(file->deps, M_LINKER);
file->deps = NULL;
}
while ((cp = STAILQ_FIRST(&file->common)) != NULL) {
STAILQ_REMOVE_HEAD(&file->common, link);
free(cp, M_LINKER);
}
LINKER_UNLOAD(file);
EVENTHANDLER_INVOKE(kld_unload, file->filename, file->address,
file->size);
if (file->filename) {
free(file->filename, M_LINKER);
file->filename = NULL;
}
if (file->pathname) {
free(file->pathname, M_LINKER);
file->pathname = NULL;
}
kobj_delete((kobj_t) file, M_LINKER);
return (0);
}
int
linker_ctf_get(linker_file_t file, linker_ctf_t *lc)
{
return (LINKER_CTF_GET(file, lc));
}
static int
linker_file_add_dependency(linker_file_t file, linker_file_t dep)
{
linker_file_t *newdeps;
sx_assert(&kld_sx, SA_XLOCKED);
file->deps = realloc(file->deps, (file->ndeps + 1) * sizeof(*newdeps),
M_LINKER, M_WAITOK | M_ZERO);
file->deps[file->ndeps] = dep;
file->ndeps++;
KLD_DPF(FILE, ("linker_file_add_dependency:"
" adding %s as dependency for %s\n",
dep->filename, file->filename));
return (0);
}
/*
* Locate a linker set and its contents. This is a helper function to avoid
* linker_if.h exposure elsewhere. Note: firstp and lastp are really void **.
* This function is used in this file so we can avoid having lots of (void **)
* casts.
*/
int
linker_file_lookup_set(linker_file_t file, const char *name,
void *firstp, void *lastp, int *countp)
{
sx_assert(&kld_sx, SA_LOCKED);
return (LINKER_LOOKUP_SET(file, name, firstp, lastp, countp));
}
/*
* List all functions in a file.
*/
int
linker_file_function_listall(linker_file_t lf,
linker_function_nameval_callback_t callback_func, void *arg)
{
return (LINKER_EACH_FUNCTION_NAMEVAL(lf, callback_func, arg));
}
caddr_t
linker_file_lookup_symbol(linker_file_t file, const char *name, int deps)
{
caddr_t sym;
int locked;
locked = sx_xlocked(&kld_sx);
if (!locked)
sx_xlock(&kld_sx);
sym = linker_file_lookup_symbol_internal(file, name, deps);
if (!locked)
sx_xunlock(&kld_sx);
return (sym);
}
static caddr_t
linker_file_lookup_symbol_internal(linker_file_t file, const char *name,
int deps)
{
c_linker_sym_t sym;
linker_symval_t symval;
caddr_t address;
size_t common_size = 0;
int i;
sx_assert(&kld_sx, SA_XLOCKED);
KLD_DPF(SYM, ("linker_file_lookup_symbol: file=%p, name=%s, deps=%d\n",
file, name, deps));
if (LINKER_LOOKUP_SYMBOL(file, name, &sym) == 0) {
LINKER_SYMBOL_VALUES(file, sym, &symval);
if (symval.value == 0)
/*
* For commons, first look them up in the
* dependencies and only allocate space if not found
* there.
*/
common_size = symval.size;
else {
KLD_DPF(SYM, ("linker_file_lookup_symbol: symbol"
".value=%p\n", symval.value));
return (symval.value);
}
}
if (deps) {
for (i = 0; i < file->ndeps; i++) {
address = linker_file_lookup_symbol_internal(
file->deps[i], name, 0);
if (address) {
KLD_DPF(SYM, ("linker_file_lookup_symbol:"
" deps value=%p\n", address));
return (address);
}
}
}
if (common_size > 0) {
/*
* This is a common symbol which was not found in the
* dependencies. We maintain a simple common symbol table in
* the file object.
*/
struct common_symbol *cp;
STAILQ_FOREACH(cp, &file->common, link) {
if (strcmp(cp->name, name) == 0) {
KLD_DPF(SYM, ("linker_file_lookup_symbol:"
" old common value=%p\n", cp->address));
return (cp->address);
}
}
/*
* Round the symbol size up to align.
*/
common_size = (common_size + sizeof(int) - 1) & -sizeof(int);
cp = malloc(sizeof(struct common_symbol)
+ common_size + strlen(name) + 1, M_LINKER,
M_WAITOK | M_ZERO);
cp->address = (caddr_t)(cp + 1);
cp->name = cp->address + common_size;
strcpy(cp->name, name);
bzero(cp->address, common_size);
STAILQ_INSERT_TAIL(&file->common, cp, link);
KLD_DPF(SYM, ("linker_file_lookup_symbol: new common"
" value=%p\n", cp->address));
return (cp->address);
}
KLD_DPF(SYM, ("linker_file_lookup_symbol: fail\n"));
return (0);
}
/*
* Both DDB and stack(9) rely on the kernel linker to provide forward and
* backward lookup of symbols. However, DDB and sometimes stack(9) need to
* do this in a lockfree manner. We provide a set of internal helper
* routines to perform these operations without locks, and then wrappers that
* optionally lock.
*
* linker_debug_lookup() is ifdef DDB as currently it's only used by DDB.
*/
#ifdef DDB
static int
linker_debug_lookup(const char *symstr, c_linker_sym_t *sym)
{
linker_file_t lf;
TAILQ_FOREACH(lf, &linker_files, link) {
if (LINKER_LOOKUP_SYMBOL(lf, symstr, sym) == 0)
return (0);
}
return (ENOENT);
}
#endif
static int
linker_debug_search_symbol(caddr_t value, c_linker_sym_t *sym, long *diffp)
{
linker_file_t lf;
c_linker_sym_t best, es;
u_long diff, bestdiff, off;
best = 0;
off = (uintptr_t)value;
bestdiff = off;
TAILQ_FOREACH(lf, &linker_files, link) {
if (LINKER_SEARCH_SYMBOL(lf, value, &es, &diff) != 0)
continue;
if (es != 0 && diff < bestdiff) {
best = es;
bestdiff = diff;
}
if (bestdiff == 0)
break;
}
if (best) {
*sym = best;
*diffp = bestdiff;
return (0);
} else {
*sym = 0;
*diffp = off;
return (ENOENT);
}
}
static int
linker_debug_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
{
linker_file_t lf;
TAILQ_FOREACH(lf, &linker_files, link) {
if (LINKER_SYMBOL_VALUES(lf, sym, symval) == 0)
return (0);
}
return (ENOENT);
}
static int
linker_debug_search_symbol_name(caddr_t value, char *buf, u_int buflen,
long *offset)
{
linker_symval_t symval;
c_linker_sym_t sym;
int error;
*offset = 0;
error = linker_debug_search_symbol(value, &sym, offset);
if (error)
return (error);
error = linker_debug_symbol_values(sym, &symval);
if (error)
return (error);
strlcpy(buf, symval.name, buflen);
return (0);
}
/*
* DDB Helpers. DDB has to look across multiple files with their own symbol
* tables and string tables.
*
* Note that we do not obey list locking protocols here. We really don't need
* DDB to hang because somebody's got the lock held. We'll take the chance
* that the files list is inconsistent instead.
*/
#ifdef DDB
int
linker_ddb_lookup(const char *symstr, c_linker_sym_t *sym)
{
return (linker_debug_lookup(symstr, sym));
}
#endif
int
linker_ddb_search_symbol(caddr_t value, c_linker_sym_t *sym, long *diffp)
{
return (linker_debug_search_symbol(value, sym, diffp));
}
int
linker_ddb_symbol_values(c_linker_sym_t sym, linker_symval_t *symval)
{
return (linker_debug_symbol_values(sym, symval));
}
int
linker_ddb_search_symbol_name(caddr_t value, char *buf, u_int buflen,
long *offset)
{
return (linker_debug_search_symbol_name(value, buf, buflen, offset));
}
/*
* stack(9) helper for non-debugging environemnts. Unlike DDB helpers, we do
* obey locking protocols, and offer a significantly less complex interface.
*/
int
linker_search_symbol_name_flags(caddr_t value, char *buf, u_int buflen,
long *offset, int flags)
{
int error;
KASSERT((flags & (M_NOWAIT | M_WAITOK)) != 0 &&
(flags & (M_NOWAIT | M_WAITOK)) != (M_NOWAIT | M_WAITOK),
("%s: bad flags: 0x%x", __func__, flags));
if (flags & M_NOWAIT) {
if (!sx_try_slock(&kld_sx))
return (EWOULDBLOCK);
} else
sx_slock(&kld_sx);
error = linker_debug_search_symbol_name(value, buf, buflen, offset);
sx_sunlock(&kld_sx);
return (error);
}
int
linker_search_symbol_name(caddr_t value, char *buf, u_int buflen,
long *offset)
{
return (linker_search_symbol_name_flags(value, buf, buflen, offset,
M_WAITOK));
}
/*
* Syscalls.
*/
int
kern_kldload(struct thread *td, const char *file, int *fileid)
{
const char *kldname, *modname;
linker_file_t lf;
int error;
if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
return (error);
if ((error = priv_check(td, PRIV_KLD_LOAD)) != 0)
return (error);
/*
* It is possible that kldloaded module will attach a new ifnet,
* so vnet context must be set when this ocurs.
*/
CURVNET_SET(TD_TO_VNET(td));
/*
* If file does not contain a qualified name or any dot in it
* (kldname.ko, or kldname.ver.ko) treat it as an interface
* name.
*/
if (strchr(file, '/') || strchr(file, '.')) {
kldname = file;
modname = NULL;
} else {
kldname = NULL;
modname = file;
}
sx_xlock(&kld_sx);
error = linker_load_module(kldname, modname, NULL, NULL, &lf);
if (error) {
sx_xunlock(&kld_sx);
goto done;
}
lf->userrefs++;
if (fileid != NULL)
*fileid = lf->id;
sx_xunlock(&kld_sx);
done:
CURVNET_RESTORE();
return (error);
}
int
sys_kldload(struct thread *td, struct kldload_args *uap)
{
char *pathname = NULL;
int error, fileid;
td->td_retval[0] = -1;
pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL);
if (error == 0) {
error = kern_kldload(td, pathname, &fileid);
if (error == 0)
td->td_retval[0] = fileid;
}
free(pathname, M_TEMP);
return (error);
}
int
kern_kldunload(struct thread *td, int fileid, int flags)
{
linker_file_t lf;
int error = 0;
if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
return (error);
if ((error = priv_check(td, PRIV_KLD_UNLOAD)) != 0)
return (error);
CURVNET_SET(TD_TO_VNET(td));
sx_xlock(&kld_sx);
lf = linker_find_file_by_id(fileid);
if (lf) {
KLD_DPF(FILE, ("kldunload: lf->userrefs=%d\n", lf->userrefs));
if (lf->userrefs == 0) {
/*
* XXX: maybe LINKER_UNLOAD_FORCE should override ?
*/
printf("kldunload: attempt to unload file that was"
" loaded by the kernel\n");
error = EBUSY;
} else {
lf->userrefs--;
error = linker_file_unload(lf, flags);
if (error)
lf->userrefs++;
}
} else
error = ENOENT;
sx_xunlock(&kld_sx);
CURVNET_RESTORE();
return (error);
}
int
sys_kldunload(struct thread *td, struct kldunload_args *uap)
{
return (kern_kldunload(td, uap->fileid, LINKER_UNLOAD_NORMAL));
}
int
sys_kldunloadf(struct thread *td, struct kldunloadf_args *uap)
{
if (uap->flags != LINKER_UNLOAD_NORMAL &&
uap->flags != LINKER_UNLOAD_FORCE)
return (EINVAL);
return (kern_kldunload(td, uap->fileid, uap->flags));
}
int
sys_kldfind(struct thread *td, struct kldfind_args *uap)
{
char *pathname;
const char *filename;
linker_file_t lf;
int error;
#ifdef MAC
error = mac_kld_check_stat(td->td_ucred);
if (error)
return (error);
#endif
td->td_retval[0] = -1;
pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
if ((error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL)) != 0)
goto out;
filename = linker_basename(pathname);
sx_xlock(&kld_sx);
lf = linker_find_file_by_name(filename);
if (lf)
td->td_retval[0] = lf->id;
else
error = ENOENT;
sx_xunlock(&kld_sx);
out:
free(pathname, M_TEMP);
return (error);
}
int
sys_kldnext(struct thread *td, struct kldnext_args *uap)
{
linker_file_t lf;
int error = 0;
#ifdef MAC
error = mac_kld_check_stat(td->td_ucred);
if (error)
return (error);
#endif
sx_xlock(&kld_sx);
if (uap->fileid == 0)
lf = TAILQ_FIRST(&linker_files);
else {
lf = linker_find_file_by_id(uap->fileid);
if (lf == NULL) {
error = ENOENT;
goto out;
}
lf = TAILQ_NEXT(lf, link);
}
/* Skip partially loaded files. */
while (lf != NULL && !(lf->flags & LINKER_FILE_LINKED))
lf = TAILQ_NEXT(lf, link);
if (lf)
td->td_retval[0] = lf->id;
else
td->td_retval[0] = 0;
out:
sx_xunlock(&kld_sx);
return (error);
}
int
sys_kldstat(struct thread *td, struct kldstat_args *uap)
{
struct kld_file_stat *stat;
int error, version;
/*
* Check the version of the user's structure.
*/
if ((error = copyin(&uap->stat->version, &version, sizeof(version)))
!= 0)
return (error);
if (version != sizeof(struct kld_file_stat_1) &&
version != sizeof(struct kld_file_stat))
return (EINVAL);
stat = malloc(sizeof(*stat), M_TEMP, M_WAITOK | M_ZERO);
error = kern_kldstat(td, uap->fileid, stat);
if (error == 0)
error = copyout(stat, uap->stat, version);
free(stat, M_TEMP);
return (error);
}
int
kern_kldstat(struct thread *td, int fileid, struct kld_file_stat *stat)
{
linker_file_t lf;
int namelen;
#ifdef MAC
int error;
error = mac_kld_check_stat(td->td_ucred);
if (error)
return (error);
#endif
sx_xlock(&kld_sx);
lf = linker_find_file_by_id(fileid);
if (lf == NULL) {
sx_xunlock(&kld_sx);
return (ENOENT);
}
/* Version 1 fields: */
namelen = strlen(lf->filename) + 1;
if (namelen > sizeof(stat->name))
namelen = sizeof(stat->name);
bcopy(lf->filename, &stat->name[0], namelen);
stat->refs = lf->refs;
stat->id = lf->id;
stat->address = lf->address;
stat->size = lf->size;
/* Version 2 fields: */
namelen = strlen(lf->pathname) + 1;
if (namelen > sizeof(stat->pathname))
namelen = sizeof(stat->pathname);
bcopy(lf->pathname, &stat->pathname[0], namelen);
sx_xunlock(&kld_sx);
td->td_retval[0] = 0;
return (0);
}
#ifdef DDB
DB_COMMAND(kldstat, db_kldstat)
{
linker_file_t lf;
#define POINTER_WIDTH ((int)(sizeof(void *) * 2 + 2))
db_printf("Id Refs Address%*c Size Name\n", POINTER_WIDTH - 7, ' ');
#undef POINTER_WIDTH
TAILQ_FOREACH(lf, &linker_files, link) {
if (db_pager_quit)
return;
db_printf("%2d %4d %p %-8zx %s\n", lf->id, lf->refs,
lf->address, lf->size, lf->filename);
}
}
#endif /* DDB */
int
sys_kldfirstmod(struct thread *td, struct kldfirstmod_args *uap)
{
linker_file_t lf;
module_t mp;
int error = 0;
#ifdef MAC
error = mac_kld_check_stat(td->td_ucred);
if (error)
return (error);
#endif
sx_xlock(&kld_sx);
lf = linker_find_file_by_id(uap->fileid);
if (lf) {
MOD_SLOCK;
mp = TAILQ_FIRST(&lf->modules);
if (mp != NULL)
td->td_retval[0] = module_getid(mp);
else
td->td_retval[0] = 0;
MOD_SUNLOCK;
} else
error = ENOENT;
sx_xunlock(&kld_sx);
return (error);
}
int
sys_kldsym(struct thread *td, struct kldsym_args *uap)
{
char *symstr = NULL;
c_linker_sym_t sym;
linker_symval_t symval;
linker_file_t lf;
struct kld_sym_lookup lookup;
int error = 0;
#ifdef MAC
error = mac_kld_check_stat(td->td_ucred);
if (error)
return (error);
#endif
if ((error = copyin(uap->data, &lookup, sizeof(lookup))) != 0)
return (error);
if (lookup.version != sizeof(lookup) ||
uap->cmd != KLDSYM_LOOKUP)
return (EINVAL);
symstr = malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
if ((error = copyinstr(lookup.symname, symstr, MAXPATHLEN, NULL)) != 0)
goto out;
sx_xlock(&kld_sx);
if (uap->fileid != 0) {
lf = linker_find_file_by_id(uap->fileid);
if (lf == NULL)
error = ENOENT;
else if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 &&
LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) {
lookup.symvalue = (uintptr_t) symval.value;
lookup.symsize = symval.size;
error = copyout(&lookup, uap->data, sizeof(lookup));
} else
error = ENOENT;
} else {
TAILQ_FOREACH(lf, &linker_files, link) {
if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 &&
LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) {
lookup.symvalue = (uintptr_t)symval.value;
lookup.symsize = symval.size;
error = copyout(&lookup, uap->data,
sizeof(lookup));
break;
}
}
if (lf == NULL)
error = ENOENT;
}
sx_xunlock(&kld_sx);
out:
free(symstr, M_TEMP);
return (error);
}
/*
* Preloaded module support
*/
static modlist_t
modlist_lookup(const char *name, int ver)
{
modlist_t mod;
TAILQ_FOREACH(mod, &found_modules, link) {
if (strcmp(mod->name, name) == 0 &&
(ver == 0 || mod->version == ver))
return (mod);
}
return (NULL);
}
static modlist_t
modlist_lookup2(const char *name, const struct mod_depend *verinfo)
{
modlist_t mod, bestmod;
int ver;
if (verinfo == NULL)
return (modlist_lookup(name, 0));
bestmod = NULL;
TAILQ_FOREACH(mod, &found_modules, link) {
if (strcmp(mod->name, name) != 0)
continue;
ver = mod->version;
if (ver == verinfo->md_ver_preferred)
return (mod);
if (ver >= verinfo->md_ver_minimum &&
ver <= verinfo->md_ver_maximum &&
(bestmod == NULL || ver > bestmod->version))
bestmod = mod;
}
return (bestmod);
}
static modlist_t
modlist_newmodule(const char *modname, int version, linker_file_t container)
{
modlist_t mod;
mod = malloc(sizeof(struct modlist), M_LINKER, M_NOWAIT | M_ZERO);
if (mod == NULL)
panic("no memory for module list");
mod->container = container;
mod->name = modname;
mod->version = version;
TAILQ_INSERT_TAIL(&found_modules, mod, link);
return (mod);
}
static void
linker_addmodules(linker_file_t lf, struct mod_metadata **start,
struct mod_metadata **stop, int preload)
{
struct mod_metadata *mp, **mdp;
const char *modname;
int ver;
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_VERSION)
continue;
modname = mp->md_cval;
ver = ((const struct mod_version *)mp->md_data)->mv_version;
if (modlist_lookup(modname, ver) != NULL) {
printf("module %s already present!\n", modname);
/* XXX what can we do? this is a build error. :-( */
continue;
}
modlist_newmodule(modname, ver, lf);
}
}
static void
linker_preload(void *arg)
{
caddr_t modptr;
const char *modname, *nmodname;
char *modtype;
linker_file_t lf, nlf;
linker_class_t lc;
int error;
linker_file_list_t loaded_files;
linker_file_list_t depended_files;
struct mod_metadata *mp, *nmp;
struct mod_metadata **start, **stop, **mdp, **nmdp;
const struct mod_depend *verinfo;
int nver;
int resolves;
modlist_t mod;
struct sysinit **si_start, **si_stop;
TAILQ_INIT(&loaded_files);
TAILQ_INIT(&depended_files);
TAILQ_INIT(&found_modules);
error = 0;
modptr = NULL;
sx_xlock(&kld_sx);
while ((modptr = preload_search_next_name(modptr)) != NULL) {
modname = (char *)preload_search_info(modptr, MODINFO_NAME);
modtype = (char *)preload_search_info(modptr, MODINFO_TYPE);
if (modname == NULL) {
printf("Preloaded module at %p does not have a"
" name!\n", modptr);
continue;
}
if (modtype == NULL) {
printf("Preloaded module at %p does not have a type!\n",
modptr);
continue;
}
if (bootverbose)
printf("Preloaded %s \"%s\" at %p.\n", modtype, modname,
modptr);
lf = NULL;
TAILQ_FOREACH(lc, &classes, link) {
error = LINKER_LINK_PRELOAD(lc, modname, &lf);
if (!error)
break;
lf = NULL;
}
if (lf)
TAILQ_INSERT_TAIL(&loaded_files, lf, loaded);
}
/*
* First get a list of stuff in the kernel.
*/
if (linker_file_lookup_set(linker_kernel_file, MDT_SETNAME, &start,
&stop, NULL) == 0)
linker_addmodules(linker_kernel_file, start, stop, 1);
/*
* This is a once-off kinky bubble sort to resolve relocation
* dependency requirements.
*/
restart:
TAILQ_FOREACH(lf, &loaded_files, loaded) {
error = linker_file_lookup_set(lf, MDT_SETNAME, &start,
&stop, NULL);
/*
* First, look to see if we would successfully link with this
* stuff.
*/
resolves = 1; /* unless we know otherwise */
if (!error) {
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_DEPEND)
continue;
modname = mp->md_cval;
verinfo = mp->md_data;
for (nmdp = start; nmdp < stop; nmdp++) {
nmp = *nmdp;
if (nmp->md_type != MDT_VERSION)
continue;
nmodname = nmp->md_cval;
if (strcmp(modname, nmodname) == 0)
break;
}
if (nmdp < stop) /* it's a self reference */
continue;
/*
* ok, the module isn't here yet, we
* are not finished
*/
if (modlist_lookup2(modname, verinfo) == NULL)
resolves = 0;
}
}
/*
* OK, if we found our modules, we can link. So, "provide"
* the modules inside and add it to the end of the link order
* list.
*/
if (resolves) {
if (!error) {
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_VERSION)
continue;
modname = mp->md_cval;
nver = ((const struct mod_version *)
mp->md_data)->mv_version;
if (modlist_lookup(modname,
nver) != NULL) {
printf("module %s already"
" present!\n", modname);
TAILQ_REMOVE(&loaded_files,
lf, loaded);
linker_file_unload(lf,
LINKER_UNLOAD_FORCE);
/* we changed tailq next ptr */
goto restart;
}
modlist_newmodule(modname, nver, lf);
}
}
TAILQ_REMOVE(&loaded_files, lf, loaded);
TAILQ_INSERT_TAIL(&depended_files, lf, loaded);
/*
* Since we provided modules, we need to restart the
* sort so that the previous files that depend on us
* have a chance. Also, we've busted the tailq next
* pointer with the REMOVE.
*/
goto restart;
}
}
/*
* At this point, we check to see what could not be resolved..
*/
while ((lf = TAILQ_FIRST(&loaded_files)) != NULL) {
TAILQ_REMOVE(&loaded_files, lf, loaded);
printf("KLD file %s is missing dependencies\n", lf->filename);
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
}
/*
* We made it. Finish off the linking in the order we determined.
*/
TAILQ_FOREACH_SAFE(lf, &depended_files, loaded, nlf) {
if (linker_kernel_file) {
linker_kernel_file->refs++;
error = linker_file_add_dependency(lf,
linker_kernel_file);
if (error)
panic("cannot add dependency");
}
error = linker_file_lookup_set(lf, MDT_SETNAME, &start,
&stop, NULL);
if (!error) {
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_DEPEND)
continue;
modname = mp->md_cval;
verinfo = mp->md_data;
mod = modlist_lookup2(modname, verinfo);
if (mod == NULL) {
printf("KLD file %s - cannot find "
"dependency \"%s\"\n",
lf->filename, modname);
goto fail;
}
/* Don't count self-dependencies */
if (lf == mod->container)
continue;
mod->container->refs++;
error = linker_file_add_dependency(lf,
mod->container);
if (error)
panic("cannot add dependency");
}
}
/*
* Now do relocation etc using the symbol search paths
* established by the dependencies
*/
error = LINKER_LINK_PRELOAD_FINISH(lf);
if (error) {
printf("KLD file %s - could not finalize loading\n",
lf->filename);
goto fail;
}
linker_file_register_modules(lf);
if (!TAILQ_EMPTY(&lf->modules))
lf->flags |= LINKER_FILE_MODULES;
if (linker_file_lookup_set(lf, "sysinit_set", &si_start,
&si_stop, NULL) == 0)
sysinit_add(si_start, si_stop);
linker_file_register_sysctls(lf, true);
lf->flags |= LINKER_FILE_LINKED;
continue;
fail:
TAILQ_REMOVE(&depended_files, lf, loaded);
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
}
sx_xunlock(&kld_sx);
/* woohoo! we made it! */
}
SYSINIT(preload, SI_SUB_KLD, SI_ORDER_MIDDLE, linker_preload, NULL);
/*
* Handle preload files that failed to load any modules.
*/
static void
linker_preload_finish(void *arg)
{
linker_file_t lf, nlf;
sx_xlock(&kld_sx);
TAILQ_FOREACH_SAFE(lf, &linker_files, link, nlf) {
/*
* If all of the modules in this file failed to load, unload
* the file and return an error of ENOEXEC. (Parity with
* linker_load_file.)
*/
if ((lf->flags & LINKER_FILE_MODULES) != 0 &&
TAILQ_EMPTY(&lf->modules)) {
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
continue;
}
lf->flags &= ~LINKER_FILE_MODULES;
lf->userrefs++; /* so we can (try to) kldunload it */
}
sx_xunlock(&kld_sx);
}
/*
* Attempt to run after all DECLARE_MODULE SYSINITs. Unfortunately they can be
* scheduled at any subsystem and order, so run this as late as possible. init
* becomes runnable in SI_SUB_KTHREAD_INIT, so go slightly before that.
*/
SYSINIT(preload_finish, SI_SUB_KTHREAD_INIT - 100, SI_ORDER_MIDDLE,
linker_preload_finish, NULL);
/*
* Search for a not-loaded module by name.
*
* Modules may be found in the following locations:
*
* - preloaded (result is just the module name) - on disk (result is full path
* to module)
*
* If the module name is qualified in any way (contains path, etc.) the we
* simply return a copy of it.
*
* The search path can be manipulated via sysctl. Note that we use the ';'
* character as a separator to be consistent with the bootloader.
*/
static char linker_hintfile[] = "linker.hints";
static char linker_path[MAXPATHLEN] = "/boot/kernel;/boot/modules";
SYSCTL_STRING(_kern, OID_AUTO, module_path, CTLFLAG_RWTUN, linker_path,
sizeof(linker_path), "module load search path");
TUNABLE_STR("module_path", linker_path, sizeof(linker_path));
static const char * const linker_ext_list[] = {
"",
".ko",
NULL
};
/*
* Check if file actually exists either with or without extension listed in
* the linker_ext_list. (probably should be generic for the rest of the
* kernel)
*/
static char *
linker_lookup_file(const char *path, int pathlen, const char *name,
int namelen, struct vattr *vap)
{
struct nameidata nd;
struct thread *td = curthread; /* XXX */
const char * const *cpp, *sep;
char *result;
int error, len, extlen, reclen, flags;
enum vtype type;
extlen = 0;
for (cpp = linker_ext_list; *cpp; cpp++) {
len = strlen(*cpp);
if (len > extlen)
extlen = len;
}
extlen++; /* trailing '\0' */
sep = (path[pathlen - 1] != '/') ? "/" : "";
reclen = pathlen + strlen(sep) + namelen + extlen + 1;
result = malloc(reclen, M_LINKER, M_WAITOK);
for (cpp = linker_ext_list; *cpp; cpp++) {
snprintf(result, reclen, "%.*s%s%.*s%s", pathlen, path, sep,
namelen, name, *cpp);
/*
* Attempt to open the file, and return the path if
* we succeed and it's a regular file.
*/
NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, result, td);
flags = FREAD;
error = vn_open(&nd, &flags, 0, NULL);
if (error == 0) {
NDFREE(&nd, NDF_ONLY_PNBUF);
type = nd.ni_vp->v_type;
if (vap)
VOP_GETATTR(nd.ni_vp, vap, td->td_ucred);
VOP_UNLOCK(nd.ni_vp);
vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
if (type == VREG)
return (result);
}
}
free(result, M_LINKER);
return (NULL);
}
#define INT_ALIGN(base, ptr) ptr = \
(base) + roundup2((ptr) - (base), sizeof(int))
/*
* Lookup KLD which contains requested module in the "linker.hints" file. If
* version specification is available, then try to find the best KLD.
* Otherwise just find the latest one.
*/
static char *
linker_hints_lookup(const char *path, int pathlen, const char *modname,
int modnamelen, const struct mod_depend *verinfo)
{
struct thread *td = curthread; /* XXX */
struct ucred *cred = td ? td->td_ucred : NULL;
struct nameidata nd;
struct vattr vattr, mattr;
const char *best, *sep;
u_char *hints = NULL;
u_char *cp, *recptr, *bufend, *result, *pathbuf;
int error, ival, bestver, *intp, found, flags, clen, blen;
ssize_t reclen;
result = NULL;
bestver = found = 0;
sep = (path[pathlen - 1] != '/') ? "/" : "";
reclen = imax(modnamelen, strlen(linker_hintfile)) + pathlen +
strlen(sep) + 1;
pathbuf = malloc(reclen, M_LINKER, M_WAITOK);
snprintf(pathbuf, reclen, "%.*s%s%s", pathlen, path, sep,
linker_hintfile);
NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, pathbuf, td);
flags = FREAD;
error = vn_open(&nd, &flags, 0, NULL);
if (error)
goto bad;
NDFREE(&nd, NDF_ONLY_PNBUF);
if (nd.ni_vp->v_type != VREG)
goto bad;
best = cp = NULL;
error = VOP_GETATTR(nd.ni_vp, &vattr, cred);
if (error)
goto bad;
/*
* XXX: we need to limit this number to some reasonable value
*/
if (vattr.va_size > LINKER_HINTS_MAX) {
printf("hints file too large %ld\n", (long)vattr.va_size);
goto bad;
}
hints = malloc(vattr.va_size, M_TEMP, M_WAITOK);
error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)hints, vattr.va_size, 0,
UIO_SYSSPACE, IO_NODELOCKED, cred, NOCRED, &reclen, td);
if (error)
goto bad;
VOP_UNLOCK(nd.ni_vp);
vn_close(nd.ni_vp, FREAD, cred, td);
nd.ni_vp = NULL;
if (reclen != 0) {
printf("can't read %zd\n", reclen);
goto bad;
}
intp = (int *)hints;
ival = *intp++;
if (ival != LINKER_HINTS_VERSION) {
printf("hints file version mismatch %d\n", ival);
goto bad;
}
bufend = hints + vattr.va_size;
recptr = (u_char *)intp;
clen = blen = 0;
while (recptr < bufend && !found) {
intp = (int *)recptr;
reclen = *intp++;
ival = *intp++;
cp = (char *)intp;
switch (ival) {
case MDT_VERSION:
clen = *cp++;
if (clen != modnamelen || bcmp(cp, modname, clen) != 0)
break;
cp += clen;
INT_ALIGN(hints, cp);
ival = *(int *)cp;
cp += sizeof(int);
clen = *cp++;
if (verinfo == NULL ||
ival == verinfo->md_ver_preferred) {
found = 1;
break;
}
if (ival >= verinfo->md_ver_minimum &&
ival <= verinfo->md_ver_maximum &&
ival > bestver) {
bestver = ival;
best = cp;
blen = clen;
}
break;
default:
break;
}
recptr += reclen + sizeof(int);
}
/*
* Finally check if KLD is in the place
*/
if (found)
result = linker_lookup_file(path, pathlen, cp, clen, &mattr);
else if (best)
result = linker_lookup_file(path, pathlen, best, blen, &mattr);
/*
* KLD is newer than hints file. What we should do now?
*/
if (result && timespeccmp(&mattr.va_mtime, &vattr.va_mtime, >))
printf("warning: KLD '%s' is newer than the linker.hints"
" file\n", result);
bad:
free(pathbuf, M_LINKER);
if (hints)
free(hints, M_TEMP);
if (nd.ni_vp != NULL) {
VOP_UNLOCK(nd.ni_vp);
vn_close(nd.ni_vp, FREAD, cred, td);
}
/*
* If nothing found or hints is absent - fallback to the old
* way by using "kldname[.ko]" as module name.
*/
if (!found && !bestver && result == NULL)
result = linker_lookup_file(path, pathlen, modname,
modnamelen, NULL);
return (result);
}
/*
* Lookup KLD which contains requested module in the all directories.
*/
static char *
linker_search_module(const char *modname, int modnamelen,
const struct mod_depend *verinfo)
{
char *cp, *ep, *result;
/*
* traverse the linker path
*/
for (cp = linker_path; *cp; cp = ep + 1) {
/* find the end of this component */
for (ep = cp; (*ep != 0) && (*ep != ';'); ep++);
result = linker_hints_lookup(cp, ep - cp, modname,
modnamelen, verinfo);
if (result != NULL)
return (result);
if (*ep == 0)
break;
}
return (NULL);
}
/*
* Search for module in all directories listed in the linker_path.
*/
static char *
linker_search_kld(const char *name)
{
char *cp, *ep, *result;
int len;
/* qualified at all? */
if (strchr(name, '/'))
return (strdup(name, M_LINKER));
/* traverse the linker path */
len = strlen(name);
for (ep = linker_path; *ep; ep++) {
cp = ep;
/* find the end of this component */
for (; *ep != 0 && *ep != ';'; ep++);
result = linker_lookup_file(cp, ep - cp, name, len, NULL);
if (result != NULL)
return (result);
}
return (NULL);
}
static const char *
linker_basename(const char *path)
{
const char *filename;
filename = strrchr(path, '/');
if (filename == NULL)
return path;
if (filename[1])
filename++;
return (filename);
}
#ifdef HWPMC_HOOKS
/*
* Inform hwpmc about the set of kernel modules currently loaded.
*/
void *
linker_hwpmc_list_objects(void)
{
linker_file_t lf;
struct pmckern_map_in *kobase;
int i, nmappings;
nmappings = 0;
sx_slock(&kld_sx);
TAILQ_FOREACH(lf, &linker_files, link)
nmappings++;
/* Allocate nmappings + 1 entries. */
kobase = malloc((nmappings + 1) * sizeof(struct pmckern_map_in),
M_LINKER, M_WAITOK | M_ZERO);
i = 0;
TAILQ_FOREACH(lf, &linker_files, link) {
/* Save the info for this linker file. */
kobase[i].pm_file = lf->filename;
kobase[i].pm_address = (uintptr_t)lf->address;
i++;
}
sx_sunlock(&kld_sx);
KASSERT(i > 0, ("linker_hpwmc_list_objects: no kernel objects?"));
/* The last entry of the malloced area comprises of all zeros. */
KASSERT(kobase[i].pm_file == NULL,
("linker_hwpmc_list_objects: last object not NULL"));
return ((void *)kobase);
}
#endif
/*
* Find a file which contains given module and load it, if "parent" is not
* NULL, register a reference to it.
*/
static int
linker_load_module(const char *kldname, const char *modname,
struct linker_file *parent, const struct mod_depend *verinfo,
struct linker_file **lfpp)
{
linker_file_t lfdep;
const char *filename;
char *pathname;
int error;
sx_assert(&kld_sx, SA_XLOCKED);
if (modname == NULL) {
/*
* We have to load KLD
*/
KASSERT(verinfo == NULL, ("linker_load_module: verinfo"
" is not NULL"));
/* check if root file system is not mounted */
- if (rootvnode == NULL || curproc->p_fd->fd_rdir == NULL)
+ if (rootvnode == NULL || curproc->p_fd->fd_pwd->pwd_rdir == NULL)
return (ENXIO);
pathname = linker_search_kld(kldname);
} else {
if (modlist_lookup2(modname, verinfo) != NULL)
return (EEXIST);
/* check if root file system is not mounted */
- if (rootvnode == NULL || curproc->p_fd->fd_rdir == NULL)
+ if (rootvnode == NULL || curproc->p_fd->fd_pwd->pwd_rdir == NULL)
return (ENXIO);
if (kldname != NULL)
pathname = strdup(kldname, M_LINKER);
else
/*
* Need to find a KLD with required module
*/
pathname = linker_search_module(modname,
strlen(modname), verinfo);
}
if (pathname == NULL)
return (ENOENT);
/*
* Can't load more than one file with the same basename XXX:
* Actually it should be possible to have multiple KLDs with
* the same basename but different path because they can
* provide different versions of the same modules.
*/
filename = linker_basename(pathname);
if (linker_find_file_by_name(filename))
error = EEXIST;
else do {
error = linker_load_file(pathname, &lfdep);
if (error)
break;
if (modname && verinfo &&
modlist_lookup2(modname, verinfo) == NULL) {
linker_file_unload(lfdep, LINKER_UNLOAD_FORCE);
error = ENOENT;
break;
}
if (parent) {
error = linker_file_add_dependency(parent, lfdep);
if (error)
break;
}
if (lfpp)
*lfpp = lfdep;
} while (0);
free(pathname, M_LINKER);
return (error);
}
/*
* This routine is responsible for finding dependencies of userland initiated
* kldload(2)'s of files.
*/
int
linker_load_dependencies(linker_file_t lf)
{
linker_file_t lfdep;
struct mod_metadata **start, **stop, **mdp, **nmdp;
struct mod_metadata *mp, *nmp;
const struct mod_depend *verinfo;
modlist_t mod;
const char *modname, *nmodname;
int ver, error = 0;
/*
* All files are dependent on /kernel.
*/
sx_assert(&kld_sx, SA_XLOCKED);
if (linker_kernel_file) {
linker_kernel_file->refs++;
error = linker_file_add_dependency(lf, linker_kernel_file);
if (error)
return (error);
}
if (linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop,
NULL) != 0)
return (0);
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_VERSION)
continue;
modname = mp->md_cval;
ver = ((const struct mod_version *)mp->md_data)->mv_version;
mod = modlist_lookup(modname, ver);
if (mod != NULL) {
printf("interface %s.%d already present in the KLD"
" '%s'!\n", modname, ver,
mod->container->filename);
return (EEXIST);
}
}
for (mdp = start; mdp < stop; mdp++) {
mp = *mdp;
if (mp->md_type != MDT_DEPEND)
continue;
modname = mp->md_cval;
verinfo = mp->md_data;
nmodname = NULL;
for (nmdp = start; nmdp < stop; nmdp++) {
nmp = *nmdp;
if (nmp->md_type != MDT_VERSION)
continue;
nmodname = nmp->md_cval;
if (strcmp(modname, nmodname) == 0)
break;
}
if (nmdp < stop)/* early exit, it's a self reference */
continue;
mod = modlist_lookup2(modname, verinfo);
if (mod) { /* woohoo, it's loaded already */
lfdep = mod->container;
lfdep->refs++;
error = linker_file_add_dependency(lf, lfdep);
if (error)
break;
continue;
}
error = linker_load_module(NULL, modname, lf, verinfo, NULL);
if (error) {
printf("KLD %s: depends on %s - not available or"
" version mismatch\n", lf->filename, modname);
break;
}
}
if (error)
return (error);
linker_addmodules(lf, start, stop, 0);
return (error);
}
static int
sysctl_kern_function_list_iterate(const char *name, void *opaque)
{
struct sysctl_req *req;
req = opaque;
return (SYSCTL_OUT(req, name, strlen(name) + 1));
}
/*
* Export a nul-separated, double-nul-terminated list of all function names
* in the kernel.
*/
static int
sysctl_kern_function_list(SYSCTL_HANDLER_ARGS)
{
linker_file_t lf;
int error;
#ifdef MAC
error = mac_kld_check_stat(req->td->td_ucred);
if (error)
return (error);
#endif
error = sysctl_wire_old_buffer(req, 0);
if (error != 0)
return (error);
sx_xlock(&kld_sx);
TAILQ_FOREACH(lf, &linker_files, link) {
error = LINKER_EACH_FUNCTION_NAME(lf,
sysctl_kern_function_list_iterate, req);
if (error) {
sx_xunlock(&kld_sx);
return (error);
}
}
sx_xunlock(&kld_sx);
return (SYSCTL_OUT(req, "", 1));
}
SYSCTL_PROC(_kern, OID_AUTO, function_list,
CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
sysctl_kern_function_list, "",
"kernel function list");
Index: head/sys/kern/vfs_cache.c
===================================================================
--- head/sys/kern/vfs_cache.c (revision 358502)
+++ head/sys/kern/vfs_cache.c (revision 358503)
@@ -1,2759 +1,2742 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1989, 1993, 1995
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Poul-Henning Kamp of the FreeBSD Project.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_cache.c 8.5 (Berkeley) 3/22/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/counter.h>
#include <sys/filedesc.h>
#include <sys/fnv_hash.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/fcntl.h>
#include <sys/mount.h>
#include <sys/namei.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vnode.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#ifdef DDB
#include <ddb/ddb.h>
#endif
#include <vm/uma.h>
SDT_PROVIDER_DECLARE(vfs);
SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
"char *");
SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
"char *", "struct vnode *");
SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int",
"struct vnode *", "char *");
SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit__negative,
"struct vnode *", "char *");
SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
"char *");
SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
"struct vnode *");
SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
"char *");
SDT_PROBE_DEFINE2(vfs, namecache, shrink_negative, done, "struct vnode *",
"char *");
/*
* This structure describes the elements in the cache of recent
* names looked up by namei.
*/
struct namecache {
LIST_ENTRY(namecache) nc_hash; /* hash chain */
LIST_ENTRY(namecache) nc_src; /* source vnode list */
TAILQ_ENTRY(namecache) nc_dst; /* destination vnode list */
struct vnode *nc_dvp; /* vnode of parent of name */
union {
struct vnode *nu_vp; /* vnode the name refers to */
} n_un;
u_char nc_flag; /* flag bits */
u_char nc_nlen; /* length of name */
char nc_name[0]; /* segment name + nul */
};
/*
* struct namecache_ts repeats struct namecache layout up to the
* nc_nlen member.
* struct namecache_ts is used in place of struct namecache when time(s) need
* to be stored. The nc_dotdottime field is used when a cache entry is mapping
* both a non-dotdot directory name plus dotdot for the directory's
* parent.
*/
struct namecache_ts {
struct timespec nc_time; /* timespec provided by fs */
struct timespec nc_dotdottime; /* dotdot timespec provided by fs */
int nc_ticks; /* ticks value when entry was added */
struct namecache nc_nc;
};
#define nc_vp n_un.nu_vp
/*
* Flags in namecache.nc_flag
*/
#define NCF_WHITE 0x01
#define NCF_ISDOTDOT 0x02
#define NCF_TS 0x04
#define NCF_DTS 0x08
#define NCF_DVDROP 0x10
#define NCF_NEGATIVE 0x20
#define NCF_HOTNEGATIVE 0x40
/*
* Name caching works as follows:
*
* Names found by directory scans are retained in a cache
* for future reference. It is managed LRU, so frequently
* used names will hang around. Cache is indexed by hash value
* obtained from (dvp, name) where dvp refers to the directory
* containing name.
*
* If it is a "negative" entry, (i.e. for a name that is known NOT to
* exist) the vnode pointer will be NULL.
*
* Upon reaching the last segment of a path, if the reference
* is for DELETE, or NOCACHE is set (rewrite), and the
* name is located in the cache, it will be dropped.
*
* These locks are used (in the order in which they can be taken):
* NAME TYPE ROLE
* vnodelock mtx vnode lists and v_cache_dd field protection
* bucketlock rwlock for access to given set of hash buckets
* neglist mtx negative entry LRU management
*
* Additionally, ncneg_shrink_lock mtx is used to have at most one thread
* shrinking the LRU list.
*
* It is legal to take multiple vnodelock and bucketlock locks. The locking
* order is lower address first. Both are recursive.
*
* "." lookups are lockless.
*
* ".." and vnode -> name lookups require vnodelock.
*
* name -> vnode lookup requires the relevant bucketlock to be held for reading.
*
* Insertions and removals of entries require involved vnodes and bucketlocks
* to be write-locked to prevent other threads from seeing the entry.
*
* Some lookups result in removal of the found entry (e.g. getting rid of a
* negative entry with the intent to create a positive one), which poses a
* problem when multiple threads reach the state. Similarly, two different
* threads can purge two different vnodes and try to remove the same name.
*
* If the already held vnode lock is lower than the second required lock, we
* can just take the other lock. However, in the opposite case, this could
* deadlock. As such, this is resolved by trylocking and if that fails unlocking
* the first node, locking everything in order and revalidating the state.
*/
/*
* Structures associated with name caching.
*/
#define NCHHASH(hash) \
(&nchashtbl[(hash) & nchash])
static __read_mostly LIST_HEAD(nchashhead, namecache) *nchashtbl;/* Hash Table */
static u_long __read_mostly nchash; /* size of hash table */
SYSCTL_ULONG(_debug, OID_AUTO, nchash, CTLFLAG_RD, &nchash, 0,
"Size of namecache hash table");
static u_long __read_mostly ncnegfactor = 5; /* ratio of negative entries */
SYSCTL_ULONG(_vfs, OID_AUTO, ncnegfactor, CTLFLAG_RW, &ncnegfactor, 0,
"Ratio of negative namecache entries");
static u_long __exclusive_cache_line numneg; /* number of negative entries allocated */
static u_long __exclusive_cache_line numcache;/* number of cache entries allocated */
u_int ncsizefactor = 2;
SYSCTL_UINT(_vfs, OID_AUTO, ncsizefactor, CTLFLAG_RW, &ncsizefactor, 0,
"Size factor for namecache");
static u_int __read_mostly ncpurgeminvnodes;
SYSCTL_UINT(_vfs, OID_AUTO, ncpurgeminvnodes, CTLFLAG_RW, &ncpurgeminvnodes, 0,
"Number of vnodes below which purgevfs ignores the request");
static u_int __read_mostly ncsize; /* the size as computed on creation or resizing */
struct nchstats nchstats; /* cache effectiveness statistics */
static struct mtx __exclusive_cache_line ncneg_shrink_lock;
static int shrink_list_turn;
struct neglist {
struct mtx nl_lock;
TAILQ_HEAD(, namecache) nl_list;
} __aligned(CACHE_LINE_SIZE);
static struct neglist __read_mostly *neglists;
static struct neglist ncneg_hot;
static u_long numhotneg;
#define numneglists (ncneghash + 1)
static u_int __read_mostly ncneghash;
static inline struct neglist *
NCP2NEGLIST(struct namecache *ncp)
{
return (&neglists[(((uintptr_t)(ncp) >> 8) & ncneghash)]);
}
#define numbucketlocks (ncbuckethash + 1)
static u_int __read_mostly ncbuckethash;
static struct rwlock_padalign __read_mostly *bucketlocks;
#define HASH2BUCKETLOCK(hash) \
((struct rwlock *)(&bucketlocks[((hash) & ncbuckethash)]))
#define numvnodelocks (ncvnodehash + 1)
static u_int __read_mostly ncvnodehash;
static struct mtx __read_mostly *vnodelocks;
static inline struct mtx *
VP2VNODELOCK(struct vnode *vp)
{
return (&vnodelocks[(((uintptr_t)(vp) >> 8) & ncvnodehash)]);
}
/*
* UMA zones for the VFS cache.
*
* The small cache is used for entries with short names, which are the
* most common. The large cache is used for entries which are too big to
* fit in the small cache.
*/
static uma_zone_t __read_mostly cache_zone_small;
static uma_zone_t __read_mostly cache_zone_small_ts;
static uma_zone_t __read_mostly cache_zone_large;
static uma_zone_t __read_mostly cache_zone_large_ts;
#define CACHE_PATH_CUTOFF 35
static struct namecache *
cache_alloc(int len, int ts)
{
struct namecache_ts *ncp_ts;
struct namecache *ncp;
if (__predict_false(ts)) {
if (len <= CACHE_PATH_CUTOFF)
ncp_ts = uma_zalloc(cache_zone_small_ts, M_WAITOK);
else
ncp_ts = uma_zalloc(cache_zone_large_ts, M_WAITOK);
ncp = &ncp_ts->nc_nc;
} else {
if (len <= CACHE_PATH_CUTOFF)
ncp = uma_zalloc(cache_zone_small, M_WAITOK);
else
ncp = uma_zalloc(cache_zone_large, M_WAITOK);
}
return (ncp);
}
static void
cache_free(struct namecache *ncp)
{
struct namecache_ts *ncp_ts;
if (ncp == NULL)
return;
if ((ncp->nc_flag & NCF_DVDROP) != 0)
vdrop(ncp->nc_dvp);
if (__predict_false(ncp->nc_flag & NCF_TS)) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
uma_zfree(cache_zone_small_ts, ncp_ts);
else
uma_zfree(cache_zone_large_ts, ncp_ts);
} else {
if (ncp->nc_nlen <= CACHE_PATH_CUTOFF)
uma_zfree(cache_zone_small, ncp);
else
uma_zfree(cache_zone_large, ncp);
}
}
static void
cache_out_ts(struct namecache *ncp, struct timespec *tsp, int *ticksp)
{
struct namecache_ts *ncp_ts;
KASSERT((ncp->nc_flag & NCF_TS) != 0 ||
(tsp == NULL && ticksp == NULL),
("No NCF_TS"));
if (tsp == NULL && ticksp == NULL)
return;
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
if (tsp != NULL)
*tsp = ncp_ts->nc_time;
if (ticksp != NULL)
*ticksp = ncp_ts->nc_ticks;
}
#ifdef DEBUG_CACHE
static int __read_mostly doingcache = 1; /* 1 => enable the cache */
SYSCTL_INT(_debug, OID_AUTO, vfscache, CTLFLAG_RW, &doingcache, 0,
"VFS namecache enabled");
#endif
/* Export size information to userland */
SYSCTL_INT(_debug_sizeof, OID_AUTO, namecache, CTLFLAG_RD, SYSCTL_NULL_INT_PTR,
sizeof(struct namecache), "sizeof(struct namecache)");
/*
* The new name cache statistics
*/
static SYSCTL_NODE(_vfs, OID_AUTO, cache, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"Name cache statistics");
#define STATNODE_ULONG(name, descr) \
SYSCTL_ULONG(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, 0, descr);
#define STATNODE_COUNTER(name, descr) \
static counter_u64_t __read_mostly name; \
SYSCTL_COUNTER_U64(_vfs_cache, OID_AUTO, name, CTLFLAG_RD, &name, descr);
STATNODE_ULONG(numneg, "Number of negative cache entries");
STATNODE_ULONG(numcache, "Number of cache entries");
STATNODE_COUNTER(numcachehv, "Number of namecache entries with vnodes held");
STATNODE_COUNTER(numcalls, "Number of cache lookups");
STATNODE_COUNTER(dothits, "Number of '.' hits");
STATNODE_COUNTER(dotdothits, "Number of '..' hits");
STATNODE_COUNTER(numchecks, "Number of checks in lookup");
STATNODE_COUNTER(nummiss, "Number of cache misses");
STATNODE_COUNTER(nummisszap, "Number of cache misses we do not want to cache");
STATNODE_COUNTER(numposzaps,
"Number of cache hits (positive) we do not want to cache");
STATNODE_COUNTER(numposhits, "Number of cache hits (positive)");
STATNODE_COUNTER(numnegzaps,
"Number of cache hits (negative) we do not want to cache");
STATNODE_COUNTER(numneghits, "Number of cache hits (negative)");
/* These count for vn_getcwd(), too. */
STATNODE_COUNTER(numfullpathcalls, "Number of fullpath search calls");
STATNODE_COUNTER(numfullpathfail1, "Number of fullpath search errors (ENOTDIR)");
STATNODE_COUNTER(numfullpathfail2,
"Number of fullpath search errors (VOP_VPTOCNP failures)");
STATNODE_COUNTER(numfullpathfail4, "Number of fullpath search errors (ENOMEM)");
STATNODE_COUNTER(numfullpathfound, "Number of successful fullpath calls");
STATNODE_COUNTER(zap_and_exit_bucket_relock_success,
"Number of successful removals after relocking");
static long zap_and_exit_bucket_fail; STATNODE_ULONG(zap_and_exit_bucket_fail,
"Number of times zap_and_exit failed to lock");
static long zap_and_exit_bucket_fail2; STATNODE_ULONG(zap_and_exit_bucket_fail2,
"Number of times zap_and_exit failed to lock");
static long cache_lock_vnodes_cel_3_failures;
STATNODE_ULONG(cache_lock_vnodes_cel_3_failures,
"Number of times 3-way vnode locking failed");
STATNODE_ULONG(numhotneg, "Number of hot negative entries");
STATNODE_COUNTER(numneg_evicted,
"Number of negative entries evicted when adding a new entry");
STATNODE_COUNTER(shrinking_skipped,
"Number of times shrinking was already in progress");
static void cache_zap_locked(struct namecache *ncp, bool neg_locked);
static int vn_fullpath_hardlink(struct thread *td, struct nameidata *ndp, char **retbuf,
char **freebuf, size_t *buflen);
static int vn_fullpath_any(struct thread *td, struct vnode *vp, struct vnode *rdir,
char *buf, char **retbuf, size_t *buflen);
static int vn_fullpath_dir(struct thread *td, struct vnode *vp, struct vnode *rdir,
char *buf, char **retbuf, size_t *len, bool slash_prefixed, size_t addend);
static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
static int cache_yield;
SYSCTL_INT(_vfs_cache, OID_AUTO, yield, CTLFLAG_RD, &cache_yield, 0,
"Number of times cache called yield");
static void __noinline
cache_maybe_yield(void)
{
if (should_yield()) {
cache_yield++;
kern_yield(PRI_USER);
}
}
static inline void
cache_assert_vlp_locked(struct mtx *vlp)
{
if (vlp != NULL)
mtx_assert(vlp, MA_OWNED);
}
static inline void
cache_assert_vnode_locked(struct vnode *vp)
{
struct mtx *vlp;
vlp = VP2VNODELOCK(vp);
cache_assert_vlp_locked(vlp);
}
static uint32_t
cache_get_hash(char *name, u_char len, struct vnode *dvp)
{
uint32_t hash;
hash = fnv_32_buf(name, len, FNV1_32_INIT);
hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
return (hash);
}
static inline struct rwlock *
NCP2BUCKETLOCK(struct namecache *ncp)
{
uint32_t hash;
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen, ncp->nc_dvp);
return (HASH2BUCKETLOCK(hash));
}
#ifdef INVARIANTS
static void
cache_assert_bucket_locked(struct namecache *ncp, int mode)
{
struct rwlock *blp;
blp = NCP2BUCKETLOCK(ncp);
rw_assert(blp, mode);
}
#else
#define cache_assert_bucket_locked(x, y) do { } while (0)
#endif
#define cache_sort_vnodes(x, y) _cache_sort_vnodes((void **)(x), (void **)(y))
static void
_cache_sort_vnodes(void **p1, void **p2)
{
void *tmp;
MPASS(*p1 != NULL || *p2 != NULL);
if (*p1 > *p2) {
tmp = *p2;
*p2 = *p1;
*p1 = tmp;
}
}
static void
cache_lock_all_buckets(void)
{
u_int i;
for (i = 0; i < numbucketlocks; i++)
rw_wlock(&bucketlocks[i]);
}
static void
cache_unlock_all_buckets(void)
{
u_int i;
for (i = 0; i < numbucketlocks; i++)
rw_wunlock(&bucketlocks[i]);
}
static void
cache_lock_all_vnodes(void)
{
u_int i;
for (i = 0; i < numvnodelocks; i++)
mtx_lock(&vnodelocks[i]);
}
static void
cache_unlock_all_vnodes(void)
{
u_int i;
for (i = 0; i < numvnodelocks; i++)
mtx_unlock(&vnodelocks[i]);
}
static int
cache_trylock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 != NULL) {
if (!mtx_trylock(vlp1))
return (EAGAIN);
}
if (!mtx_trylock(vlp2)) {
if (vlp1 != NULL)
mtx_unlock(vlp1);
return (EAGAIN);
}
return (0);
}
static void
cache_lock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
MPASS(vlp1 != NULL || vlp2 != NULL);
MPASS(vlp1 <= vlp2);
if (vlp1 != NULL)
mtx_lock(vlp1);
if (vlp2 != NULL)
mtx_lock(vlp2);
}
static void
cache_unlock_vnodes(struct mtx *vlp1, struct mtx *vlp2)
{
MPASS(vlp1 != NULL || vlp2 != NULL);
if (vlp1 != NULL)
mtx_unlock(vlp1);
if (vlp2 != NULL)
mtx_unlock(vlp2);
}
static int
sysctl_nchstats(SYSCTL_HANDLER_ARGS)
{
struct nchstats snap;
if (req->oldptr == NULL)
return (SYSCTL_OUT(req, 0, sizeof(snap)));
snap = nchstats;
snap.ncs_goodhits = counter_u64_fetch(numposhits);
snap.ncs_neghits = counter_u64_fetch(numneghits);
snap.ncs_badhits = counter_u64_fetch(numposzaps) +
counter_u64_fetch(numnegzaps);
snap.ncs_miss = counter_u64_fetch(nummisszap) +
counter_u64_fetch(nummiss);
return (SYSCTL_OUT(req, &snap, sizeof(snap)));
}
SYSCTL_PROC(_vfs_cache, OID_AUTO, nchstats, CTLTYPE_OPAQUE | CTLFLAG_RD |
CTLFLAG_MPSAFE, 0, 0, sysctl_nchstats, "LU",
"VFS cache effectiveness statistics");
#ifdef DIAGNOSTIC
/*
* Grab an atomic snapshot of the name cache hash chain lengths
*/
static SYSCTL_NODE(_debug, OID_AUTO, hashstat,
CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
"hash table stats");
static int
sysctl_debug_hashstat_rawnchash(SYSCTL_HANDLER_ARGS)
{
struct nchashhead *ncpp;
struct namecache *ncp;
int i, error, n_nchash, *cntbuf;
retry:
n_nchash = nchash + 1; /* nchash is max index, not count */
if (req->oldptr == NULL)
return SYSCTL_OUT(req, 0, n_nchash * sizeof(int));
cntbuf = malloc(n_nchash * sizeof(int), M_TEMP, M_ZERO | M_WAITOK);
cache_lock_all_buckets();
if (n_nchash != nchash + 1) {
cache_unlock_all_buckets();
free(cntbuf, M_TEMP);
goto retry;
}
/* Scan hash tables counting entries */
for (ncpp = nchashtbl, i = 0; i < n_nchash; ncpp++, i++)
LIST_FOREACH(ncp, ncpp, nc_hash)
cntbuf[i]++;
cache_unlock_all_buckets();
for (error = 0, i = 0; i < n_nchash; i++)
if ((error = SYSCTL_OUT(req, &cntbuf[i], sizeof(int))) != 0)
break;
free(cntbuf, M_TEMP);
return (error);
}
SYSCTL_PROC(_debug_hashstat, OID_AUTO, rawnchash, CTLTYPE_INT|CTLFLAG_RD|
CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_rawnchash, "S,int",
"nchash chain lengths");
static int
sysctl_debug_hashstat_nchash(SYSCTL_HANDLER_ARGS)
{
int error;
struct nchashhead *ncpp;
struct namecache *ncp;
int n_nchash;
int count, maxlength, used, pct;
if (!req->oldptr)
return SYSCTL_OUT(req, 0, 4 * sizeof(int));
cache_lock_all_buckets();
n_nchash = nchash + 1; /* nchash is max index, not count */
used = 0;
maxlength = 0;
/* Scan hash tables for applicable entries */
for (ncpp = nchashtbl; n_nchash > 0; n_nchash--, ncpp++) {
count = 0;
LIST_FOREACH(ncp, ncpp, nc_hash) {
count++;
}
if (count)
used++;
if (maxlength < count)
maxlength = count;
}
n_nchash = nchash + 1;
cache_unlock_all_buckets();
pct = (used * 100) / (n_nchash / 100);
error = SYSCTL_OUT(req, &n_nchash, sizeof(n_nchash));
if (error)
return (error);
error = SYSCTL_OUT(req, &used, sizeof(used));
if (error)
return (error);
error = SYSCTL_OUT(req, &maxlength, sizeof(maxlength));
if (error)
return (error);
error = SYSCTL_OUT(req, &pct, sizeof(pct));
if (error)
return (error);
return (0);
}
SYSCTL_PROC(_debug_hashstat, OID_AUTO, nchash, CTLTYPE_INT|CTLFLAG_RD|
CTLFLAG_MPSAFE, 0, 0, sysctl_debug_hashstat_nchash, "I",
"nchash statistics (number of total/used buckets, maximum chain length, usage percentage)");
#endif
/*
* Negative entries management
*
* A variation of LRU scheme is used. New entries are hashed into one of
* numneglists cold lists. Entries get promoted to the hot list on first hit.
*
* The shrinker will demote hot list head and evict from the cold list in a
* round-robin manner.
*/
static void
cache_negative_hit(struct namecache *ncp)
{
struct neglist *neglist;
MPASS(ncp->nc_flag & NCF_NEGATIVE);
if (ncp->nc_flag & NCF_HOTNEGATIVE)
return;
neglist = NCP2NEGLIST(ncp);
mtx_lock(&ncneg_hot.nl_lock);
mtx_lock(&neglist->nl_lock);
if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
numhotneg++;
TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
TAILQ_INSERT_TAIL(&ncneg_hot.nl_list, ncp, nc_dst);
ncp->nc_flag |= NCF_HOTNEGATIVE;
}
mtx_unlock(&neglist->nl_lock);
mtx_unlock(&ncneg_hot.nl_lock);
}
static void
cache_negative_insert(struct namecache *ncp, bool neg_locked)
{
struct neglist *neglist;
MPASS(ncp->nc_flag & NCF_NEGATIVE);
cache_assert_bucket_locked(ncp, RA_WLOCKED);
neglist = NCP2NEGLIST(ncp);
if (!neg_locked) {
mtx_lock(&neglist->nl_lock);
} else {
mtx_assert(&neglist->nl_lock, MA_OWNED);
}
TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
if (!neg_locked)
mtx_unlock(&neglist->nl_lock);
atomic_add_rel_long(&numneg, 1);
}
static void
cache_negative_remove(struct namecache *ncp, bool neg_locked)
{
struct neglist *neglist;
bool hot_locked = false;
bool list_locked = false;
MPASS(ncp->nc_flag & NCF_NEGATIVE);
cache_assert_bucket_locked(ncp, RA_WLOCKED);
neglist = NCP2NEGLIST(ncp);
if (!neg_locked) {
if (ncp->nc_flag & NCF_HOTNEGATIVE) {
hot_locked = true;
mtx_lock(&ncneg_hot.nl_lock);
if (!(ncp->nc_flag & NCF_HOTNEGATIVE)) {
list_locked = true;
mtx_lock(&neglist->nl_lock);
}
} else {
list_locked = true;
mtx_lock(&neglist->nl_lock);
}
}
if (ncp->nc_flag & NCF_HOTNEGATIVE) {
mtx_assert(&ncneg_hot.nl_lock, MA_OWNED);
TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
numhotneg--;
} else {
mtx_assert(&neglist->nl_lock, MA_OWNED);
TAILQ_REMOVE(&neglist->nl_list, ncp, nc_dst);
}
if (list_locked)
mtx_unlock(&neglist->nl_lock);
if (hot_locked)
mtx_unlock(&ncneg_hot.nl_lock);
atomic_subtract_rel_long(&numneg, 1);
}
static void
cache_negative_shrink_select(int start, struct namecache **ncpp,
struct neglist **neglistpp)
{
struct neglist *neglist;
struct namecache *ncp;
int i;
*ncpp = ncp = NULL;
neglist = NULL;
for (i = start; i < numneglists; i++) {
neglist = &neglists[i];
if (TAILQ_FIRST(&neglist->nl_list) == NULL)
continue;
mtx_lock(&neglist->nl_lock);
ncp = TAILQ_FIRST(&neglist->nl_list);
if (ncp != NULL)
break;
mtx_unlock(&neglist->nl_lock);
}
*neglistpp = neglist;
*ncpp = ncp;
}
static void
cache_negative_zap_one(void)
{
struct namecache *ncp, *ncp2;
struct neglist *neglist;
struct mtx *dvlp;
struct rwlock *blp;
if (mtx_owner(&ncneg_shrink_lock) != NULL ||
!mtx_trylock(&ncneg_shrink_lock)) {
counter_u64_add(shrinking_skipped, 1);
return;
}
mtx_lock(&ncneg_hot.nl_lock);
ncp = TAILQ_FIRST(&ncneg_hot.nl_list);
if (ncp != NULL) {
neglist = NCP2NEGLIST(ncp);
mtx_lock(&neglist->nl_lock);
TAILQ_REMOVE(&ncneg_hot.nl_list, ncp, nc_dst);
TAILQ_INSERT_TAIL(&neglist->nl_list, ncp, nc_dst);
ncp->nc_flag &= ~NCF_HOTNEGATIVE;
numhotneg--;
mtx_unlock(&neglist->nl_lock);
}
mtx_unlock(&ncneg_hot.nl_lock);
cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
shrink_list_turn++;
if (shrink_list_turn == numneglists)
shrink_list_turn = 0;
if (ncp == NULL && shrink_list_turn == 0)
cache_negative_shrink_select(shrink_list_turn, &ncp, &neglist);
mtx_unlock(&ncneg_shrink_lock);
if (ncp == NULL)
return;
MPASS(ncp->nc_flag & NCF_NEGATIVE);
dvlp = VP2VNODELOCK(ncp->nc_dvp);
blp = NCP2BUCKETLOCK(ncp);
mtx_unlock(&neglist->nl_lock);
mtx_lock(dvlp);
rw_wlock(blp);
mtx_lock(&neglist->nl_lock);
ncp2 = TAILQ_FIRST(&neglist->nl_list);
if (ncp != ncp2 || dvlp != VP2VNODELOCK(ncp2->nc_dvp) ||
blp != NCP2BUCKETLOCK(ncp2) || !(ncp2->nc_flag & NCF_NEGATIVE)) {
ncp = NULL;
} else {
SDT_PROBE2(vfs, namecache, shrink_negative, done, ncp->nc_dvp,
ncp->nc_name);
cache_zap_locked(ncp, true);
counter_u64_add(numneg_evicted, 1);
}
mtx_unlock(&neglist->nl_lock);
rw_wunlock(blp);
mtx_unlock(dvlp);
cache_free(ncp);
}
/*
* cache_zap_locked():
*
* Removes a namecache entry from cache, whether it contains an actual
* pointer to a vnode or if it is just a negative cache entry.
*/
static void
cache_zap_locked(struct namecache *ncp, bool neg_locked)
{
if (!(ncp->nc_flag & NCF_NEGATIVE))
cache_assert_vnode_locked(ncp->nc_vp);
cache_assert_vnode_locked(ncp->nc_dvp);
cache_assert_bucket_locked(ncp, RA_WLOCKED);
CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp,
(ncp->nc_flag & NCF_NEGATIVE) ? NULL : ncp->nc_vp);
LIST_REMOVE(ncp, nc_hash);
if (!(ncp->nc_flag & NCF_NEGATIVE)) {
SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
ncp->nc_name, ncp->nc_vp);
TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
if (ncp == ncp->nc_vp->v_cache_dd)
ncp->nc_vp->v_cache_dd = NULL;
} else {
SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp,
ncp->nc_name);
cache_negative_remove(ncp, neg_locked);
}
if (ncp->nc_flag & NCF_ISDOTDOT) {
if (ncp == ncp->nc_dvp->v_cache_dd)
ncp->nc_dvp->v_cache_dd = NULL;
} else {
LIST_REMOVE(ncp, nc_src);
if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
ncp->nc_flag |= NCF_DVDROP;
counter_u64_add(numcachehv, -1);
}
}
atomic_subtract_rel_long(&numcache, 1);
}
static void
cache_zap_negative_locked_vnode_kl(struct namecache *ncp, struct vnode *vp)
{
struct rwlock *blp;
MPASS(ncp->nc_dvp == vp);
MPASS(ncp->nc_flag & NCF_NEGATIVE);
cache_assert_vnode_locked(vp);
blp = NCP2BUCKETLOCK(ncp);
rw_wlock(blp);
cache_zap_locked(ncp, false);
rw_wunlock(blp);
}
static bool
cache_zap_locked_vnode_kl2(struct namecache *ncp, struct vnode *vp,
struct mtx **vlpp)
{
struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
struct rwlock *blp;
MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
cache_assert_vnode_locked(vp);
if (ncp->nc_flag & NCF_NEGATIVE) {
if (*vlpp != NULL) {
mtx_unlock(*vlpp);
*vlpp = NULL;
}
cache_zap_negative_locked_vnode_kl(ncp, vp);
return (true);
}
pvlp = VP2VNODELOCK(vp);
blp = NCP2BUCKETLOCK(ncp);
vlp1 = VP2VNODELOCK(ncp->nc_dvp);
vlp2 = VP2VNODELOCK(ncp->nc_vp);
if (*vlpp == vlp1 || *vlpp == vlp2) {
to_unlock = *vlpp;
*vlpp = NULL;
} else {
if (*vlpp != NULL) {
mtx_unlock(*vlpp);
*vlpp = NULL;
}
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 == pvlp) {
mtx_lock(vlp2);
to_unlock = vlp2;
} else {
if (!mtx_trylock(vlp1))
goto out_relock;
to_unlock = vlp1;
}
}
rw_wlock(blp);
cache_zap_locked(ncp, false);
rw_wunlock(blp);
if (to_unlock != NULL)
mtx_unlock(to_unlock);
return (true);
out_relock:
mtx_unlock(vlp2);
mtx_lock(vlp1);
mtx_lock(vlp2);
MPASS(*vlpp == NULL);
*vlpp = vlp1;
return (false);
}
static int __noinline
cache_zap_locked_vnode(struct namecache *ncp, struct vnode *vp)
{
struct mtx *pvlp, *vlp1, *vlp2, *to_unlock;
struct rwlock *blp;
int error = 0;
MPASS(vp == ncp->nc_dvp || vp == ncp->nc_vp);
cache_assert_vnode_locked(vp);
pvlp = VP2VNODELOCK(vp);
if (ncp->nc_flag & NCF_NEGATIVE) {
cache_zap_negative_locked_vnode_kl(ncp, vp);
goto out;
}
blp = NCP2BUCKETLOCK(ncp);
vlp1 = VP2VNODELOCK(ncp->nc_dvp);
vlp2 = VP2VNODELOCK(ncp->nc_vp);
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 == pvlp) {
mtx_lock(vlp2);
to_unlock = vlp2;
} else {
if (!mtx_trylock(vlp1)) {
error = EAGAIN;
goto out;
}
to_unlock = vlp1;
}
rw_wlock(blp);
cache_zap_locked(ncp, false);
rw_wunlock(blp);
mtx_unlock(to_unlock);
out:
mtx_unlock(pvlp);
return (error);
}
/*
* If trylocking failed we can get here. We know enough to take all needed locks
* in the right order and re-lookup the entry.
*/
static int
cache_zap_unlocked_bucket(struct namecache *ncp, struct componentname *cnp,
struct vnode *dvp, struct mtx *dvlp, struct mtx *vlp, uint32_t hash,
struct rwlock *blp)
{
struct namecache *rncp;
cache_assert_bucket_locked(ncp, RA_UNLOCKED);
cache_sort_vnodes(&dvlp, &vlp);
cache_lock_vnodes(dvlp, vlp);
rw_wlock(blp);
LIST_FOREACH(rncp, (NCHHASH(hash)), nc_hash) {
if (rncp == ncp && rncp->nc_dvp == dvp &&
rncp->nc_nlen == cnp->cn_namelen &&
!bcmp(rncp->nc_name, cnp->cn_nameptr, rncp->nc_nlen))
break;
}
if (rncp != NULL) {
cache_zap_locked(rncp, false);
rw_wunlock(blp);
cache_unlock_vnodes(dvlp, vlp);
counter_u64_add(zap_and_exit_bucket_relock_success, 1);
return (0);
}
rw_wunlock(blp);
cache_unlock_vnodes(dvlp, vlp);
return (EAGAIN);
}
static int __noinline
cache_zap_wlocked_bucket(struct namecache *ncp, struct componentname *cnp,
uint32_t hash, struct rwlock *blp)
{
struct mtx *dvlp, *vlp;
struct vnode *dvp;
cache_assert_bucket_locked(ncp, RA_WLOCKED);
dvlp = VP2VNODELOCK(ncp->nc_dvp);
vlp = NULL;
if (!(ncp->nc_flag & NCF_NEGATIVE))
vlp = VP2VNODELOCK(ncp->nc_vp);
if (cache_trylock_vnodes(dvlp, vlp) == 0) {
cache_zap_locked(ncp, false);
rw_wunlock(blp);
cache_unlock_vnodes(dvlp, vlp);
return (0);
}
dvp = ncp->nc_dvp;
rw_wunlock(blp);
return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
}
static int __noinline
cache_zap_rlocked_bucket(struct namecache *ncp, struct componentname *cnp,
uint32_t hash, struct rwlock *blp)
{
struct mtx *dvlp, *vlp;
struct vnode *dvp;
cache_assert_bucket_locked(ncp, RA_RLOCKED);
dvlp = VP2VNODELOCK(ncp->nc_dvp);
vlp = NULL;
if (!(ncp->nc_flag & NCF_NEGATIVE))
vlp = VP2VNODELOCK(ncp->nc_vp);
if (cache_trylock_vnodes(dvlp, vlp) == 0) {
rw_runlock(blp);
rw_wlock(blp);
cache_zap_locked(ncp, false);
rw_wunlock(blp);
cache_unlock_vnodes(dvlp, vlp);
return (0);
}
dvp = ncp->nc_dvp;
rw_runlock(blp);
return (cache_zap_unlocked_bucket(ncp, cnp, dvp, dvlp, vlp, hash, blp));
}
static int
cache_zap_wlocked_bucket_kl(struct namecache *ncp, struct rwlock *blp,
struct mtx **vlpp1, struct mtx **vlpp2)
{
struct mtx *dvlp, *vlp;
cache_assert_bucket_locked(ncp, RA_WLOCKED);
dvlp = VP2VNODELOCK(ncp->nc_dvp);
vlp = NULL;
if (!(ncp->nc_flag & NCF_NEGATIVE))
vlp = VP2VNODELOCK(ncp->nc_vp);
cache_sort_vnodes(&dvlp, &vlp);
if (*vlpp1 == dvlp && *vlpp2 == vlp) {
cache_zap_locked(ncp, false);
cache_unlock_vnodes(dvlp, vlp);
*vlpp1 = NULL;
*vlpp2 = NULL;
return (0);
}
if (*vlpp1 != NULL)
mtx_unlock(*vlpp1);
if (*vlpp2 != NULL)
mtx_unlock(*vlpp2);
*vlpp1 = NULL;
*vlpp2 = NULL;
if (cache_trylock_vnodes(dvlp, vlp) == 0) {
cache_zap_locked(ncp, false);
cache_unlock_vnodes(dvlp, vlp);
return (0);
}
rw_wunlock(blp);
*vlpp1 = dvlp;
*vlpp2 = vlp;
if (*vlpp1 != NULL)
mtx_lock(*vlpp1);
mtx_lock(*vlpp2);
rw_wlock(blp);
return (EAGAIN);
}
static void
cache_lookup_unlock(struct rwlock *blp, struct mtx *vlp)
{
if (blp != NULL) {
rw_runlock(blp);
} else {
mtx_unlock(vlp);
}
}
static int __noinline
cache_lookup_dot(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
int ltype;
*vpp = dvp;
CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
dvp, cnp->cn_nameptr);
counter_u64_add(dothits, 1);
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
if (tsp != NULL)
timespecclear(tsp);
if (ticksp != NULL)
*ticksp = ticks;
vrefact(*vpp);
/*
* When we lookup "." we still can be asked to lock it
* differently...
*/
ltype = cnp->cn_lkflags & LK_TYPE_MASK;
if (ltype != VOP_ISLOCKED(*vpp)) {
if (ltype == LK_EXCLUSIVE) {
vn_lock(*vpp, LK_UPGRADE | LK_RETRY);
if (VN_IS_DOOMED((*vpp))) {
/* forced unmount */
vrele(*vpp);
*vpp = NULL;
return (ENOENT);
}
} else
vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
}
return (-1);
}
static __noinline int
cache_lookup_nomakeentry(struct vnode *dvp, struct vnode **vpp,
struct componentname *cnp, struct timespec *tsp, int *ticksp)
{
struct namecache *ncp;
struct rwlock *blp;
struct mtx *dvlp, *dvlp2;
uint32_t hash;
int error;
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
counter_u64_add(dotdothits, 1);
dvlp = VP2VNODELOCK(dvp);
dvlp2 = NULL;
mtx_lock(dvlp);
retry_dotdot:
ncp = dvp->v_cache_dd;
if (ncp == NULL) {
SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
"..", NULL);
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
return (0);
}
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
if (ncp->nc_dvp != dvp)
panic("dvp %p v_cache_dd %p\n", dvp, ncp);
if (!cache_zap_locked_vnode_kl2(ncp,
dvp, &dvlp2))
goto retry_dotdot;
MPASS(dvp->v_cache_dd == NULL);
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
cache_free(ncp);
} else {
dvp->v_cache_dd = NULL;
mtx_unlock(dvlp);
if (dvlp2 != NULL)
mtx_unlock(dvlp2);
}
return (0);
}
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
blp = HASH2BUCKETLOCK(hash);
retry:
if (LIST_EMPTY(NCHHASH(hash)))
goto out_no_entry;
rw_wlock(blp);
LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
counter_u64_add(numchecks, 1);
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
/* We failed to find an entry */
if (ncp == NULL) {
rw_wunlock(blp);
goto out_no_entry;
}
error = cache_zap_wlocked_bucket(ncp, cnp, hash, blp);
if (__predict_false(error != 0)) {
zap_and_exit_bucket_fail++;
cache_maybe_yield();
goto retry;
}
counter_u64_add(numposzaps, 1);
cache_free(ncp);
return (0);
out_no_entry:
SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr, NULL);
counter_u64_add(nummisszap, 1);
return (0);
}
/**
* Lookup a name in the name cache
*
* # Arguments
*
* - dvp: Parent directory in which to search.
* - vpp: Return argument. Will contain desired vnode on cache hit.
* - cnp: Parameters of the name search. The most interesting bits of
* the cn_flags field have the following meanings:
* - MAKEENTRY: If clear, free an entry from the cache rather than look
* it up.
* - ISDOTDOT: Must be set if and only if cn_nameptr == ".."
* - tsp: Return storage for cache timestamp. On a successful (positive
* or negative) lookup, tsp will be filled with any timespec that
* was stored when this cache entry was created. However, it will
* be clear for "." entries.
* - ticks: Return storage for alternate cache timestamp. On a successful
* (positive or negative) lookup, it will contain the ticks value
* that was current when the cache entry was created, unless cnp
* was ".".
*
* # Returns
*
* - -1: A positive cache hit. vpp will contain the desired vnode.
* - ENOENT: A negative cache hit, or dvp was recycled out from under us due
* to a forced unmount. vpp will not be modified. If the entry
* is a whiteout, then the ISWHITEOUT flag will be set in
* cnp->cn_flags.
* - 0: A cache miss. vpp will not be modified.
*
* # Locking
*
* On a cache hit, vpp will be returned locked and ref'd. If we're looking up
* .., dvp is unlocked. If we're looking up . an extra ref is taken, but the
* lock is not recursively acquired.
*/
int
cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
struct timespec *tsp, int *ticksp)
{
struct namecache_ts *ncp_ts;
struct namecache *ncp;
struct rwlock *blp;
struct mtx *dvlp;
uint32_t hash;
enum vgetstate vs;
int error, ltype;
#ifdef DEBUG_CACHE
if (__predict_false(!doingcache)) {
cnp->cn_flags &= ~MAKEENTRY;
return (0);
}
#endif
counter_u64_add(numcalls, 1);
if (__predict_false(cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.'))
return (cache_lookup_dot(dvp, vpp, cnp, tsp, ticksp));
if ((cnp->cn_flags & MAKEENTRY) == 0)
return (cache_lookup_nomakeentry(dvp, vpp, cnp, tsp, ticksp));
retry:
blp = NULL;
dvlp = NULL;
error = 0;
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[0] == '.' && cnp->cn_nameptr[1] == '.') {
counter_u64_add(dotdothits, 1);
dvlp = VP2VNODELOCK(dvp);
mtx_lock(dvlp);
ncp = dvp->v_cache_dd;
if (ncp == NULL) {
SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
"..", NULL);
mtx_unlock(dvlp);
return (0);
}
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0) {
if (ncp->nc_flag & NCF_NEGATIVE)
*vpp = NULL;
else
*vpp = ncp->nc_vp;
} else
*vpp = ncp->nc_dvp;
/* Return failure if negative entry was found. */
if (*vpp == NULL)
goto negative_success;
CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
dvp, cnp->cn_nameptr, *vpp);
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..",
*vpp);
cache_out_ts(ncp, tsp, ticksp);
if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
NCF_DTS && tsp != NULL) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
*tsp = ncp_ts->nc_dotdottime;
}
goto success;
}
hash = cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp);
blp = HASH2BUCKETLOCK(hash);
rw_rlock(blp);
LIST_FOREACH(ncp, (NCHHASH(hash)), nc_hash) {
counter_u64_add(numchecks, 1);
if (ncp->nc_dvp == dvp && ncp->nc_nlen == cnp->cn_namelen &&
!bcmp(ncp->nc_name, cnp->cn_nameptr, ncp->nc_nlen))
break;
}
/* We failed to find an entry */
if (__predict_false(ncp == NULL)) {
rw_runlock(blp);
SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
NULL);
counter_u64_add(nummiss, 1);
return (0);
}
if (ncp->nc_flag & NCF_NEGATIVE)
goto negative_success;
/* We found a "positive" match, return the vnode */
counter_u64_add(numposhits, 1);
*vpp = ncp->nc_vp;
CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
dvp, cnp->cn_nameptr, *vpp, ncp);
SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
*vpp);
cache_out_ts(ncp, tsp, ticksp);
success:
/*
* On success we return a locked and ref'd vnode as per the lookup
* protocol.
*/
MPASS(dvp != *vpp);
ltype = 0; /* silence gcc warning */
if (cnp->cn_flags & ISDOTDOT) {
ltype = VOP_ISLOCKED(dvp);
VOP_UNLOCK(dvp);
}
vs = vget_prep(*vpp);
cache_lookup_unlock(blp, dvlp);
error = vget_finish(*vpp, cnp->cn_lkflags, vs);
if (cnp->cn_flags & ISDOTDOT) {
vn_lock(dvp, ltype | LK_RETRY);
if (VN_IS_DOOMED(dvp)) {
if (error == 0)
vput(*vpp);
*vpp = NULL;
return (ENOENT);
}
}
if (error) {
*vpp = NULL;
goto retry;
}
if ((cnp->cn_flags & ISLASTCN) &&
(cnp->cn_lkflags & LK_TYPE_MASK) == LK_EXCLUSIVE) {
ASSERT_VOP_ELOCKED(*vpp, "cache_lookup");
}
return (-1);
negative_success:
/* We found a negative match, and want to create it, so purge */
if (cnp->cn_nameiop == CREATE) {
counter_u64_add(numnegzaps, 1);
goto zap_and_exit;
}
counter_u64_add(numneghits, 1);
cache_negative_hit(ncp);
if (ncp->nc_flag & NCF_WHITE)
cnp->cn_flags |= ISWHITEOUT;
SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp,
ncp->nc_name);
cache_out_ts(ncp, tsp, ticksp);
cache_lookup_unlock(blp, dvlp);
return (ENOENT);
zap_and_exit:
if (blp != NULL)
error = cache_zap_rlocked_bucket(ncp, cnp, hash, blp);
else
error = cache_zap_locked_vnode(ncp, dvp);
if (__predict_false(error != 0)) {
zap_and_exit_bucket_fail2++;
cache_maybe_yield();
goto retry;
}
cache_free(ncp);
return (0);
}
struct celockstate {
struct mtx *vlp[3];
struct rwlock *blp[2];
};
CTASSERT((nitems(((struct celockstate *)0)->vlp) == 3));
CTASSERT((nitems(((struct celockstate *)0)->blp) == 2));
static inline void
cache_celockstate_init(struct celockstate *cel)
{
bzero(cel, sizeof(*cel));
}
static void
cache_lock_vnodes_cel(struct celockstate *cel, struct vnode *vp,
struct vnode *dvp)
{
struct mtx *vlp1, *vlp2;
MPASS(cel->vlp[0] == NULL);
MPASS(cel->vlp[1] == NULL);
MPASS(cel->vlp[2] == NULL);
MPASS(vp != NULL || dvp != NULL);
vlp1 = VP2VNODELOCK(vp);
vlp2 = VP2VNODELOCK(dvp);
cache_sort_vnodes(&vlp1, &vlp2);
if (vlp1 != NULL) {
mtx_lock(vlp1);
cel->vlp[0] = vlp1;
}
mtx_lock(vlp2);
cel->vlp[1] = vlp2;
}
static void
cache_unlock_vnodes_cel(struct celockstate *cel)
{
MPASS(cel->vlp[0] != NULL || cel->vlp[1] != NULL);
if (cel->vlp[0] != NULL)
mtx_unlock(cel->vlp[0]);
if (cel->vlp[1] != NULL)
mtx_unlock(cel->vlp[1]);
if (cel->vlp[2] != NULL)
mtx_unlock(cel->vlp[2]);
}
static bool
cache_lock_vnodes_cel_3(struct celockstate *cel, struct vnode *vp)
{
struct mtx *vlp;
bool ret;
cache_assert_vlp_locked(cel->vlp[0]);
cache_assert_vlp_locked(cel->vlp[1]);
MPASS(cel->vlp[2] == NULL);
MPASS(vp != NULL);
vlp = VP2VNODELOCK(vp);
ret = true;
if (vlp >= cel->vlp[1]) {
mtx_lock(vlp);
} else {
if (mtx_trylock(vlp))
goto out;
cache_lock_vnodes_cel_3_failures++;
cache_unlock_vnodes_cel(cel);
if (vlp < cel->vlp[0]) {
mtx_lock(vlp);
mtx_lock(cel->vlp[0]);
mtx_lock(cel->vlp[1]);
} else {
if (cel->vlp[0] != NULL)
mtx_lock(cel->vlp[0]);
mtx_lock(vlp);
mtx_lock(cel->vlp[1]);
}
ret = false;
}
out:
cel->vlp[2] = vlp;
return (ret);
}
static void
cache_lock_buckets_cel(struct celockstate *cel, struct rwlock *blp1,
struct rwlock *blp2)
{
MPASS(cel->blp[0] == NULL);
MPASS(cel->blp[1] == NULL);
cache_sort_vnodes(&blp1, &blp2);
if (blp1 != NULL) {
rw_wlock(blp1);
cel->blp[0] = blp1;
}
rw_wlock(blp2);
cel->blp[1] = blp2;
}
static void
cache_unlock_buckets_cel(struct celockstate *cel)
{
if (cel->blp[0] != NULL)
rw_wunlock(cel->blp[0]);
rw_wunlock(cel->blp[1]);
}
/*
* Lock part of the cache affected by the insertion.
*
* This means vnodelocks for dvp, vp and the relevant bucketlock.
* However, insertion can result in removal of an old entry. In this
* case we have an additional vnode and bucketlock pair to lock. If the
* entry is negative, ncelock is locked instead of the vnode.
*
* That is, in the worst case we have to lock 3 vnodes and 2 bucketlocks, while
* preserving the locking order (smaller address first).
*/
static void
cache_enter_lock(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
uint32_t hash)
{
struct namecache *ncp;
struct rwlock *blps[2];
blps[0] = HASH2BUCKETLOCK(hash);
for (;;) {
blps[1] = NULL;
cache_lock_vnodes_cel(cel, dvp, vp);
if (vp == NULL || vp->v_type != VDIR)
break;
ncp = vp->v_cache_dd;
if (ncp == NULL)
break;
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
break;
MPASS(ncp->nc_dvp == vp);
blps[1] = NCP2BUCKETLOCK(ncp);
if (ncp->nc_flag & NCF_NEGATIVE)
break;
if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
break;
/*
* All vnodes got re-locked. Re-validate the state and if
* nothing changed we are done. Otherwise restart.
*/
if (ncp == vp->v_cache_dd &&
(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
blps[1] == NCP2BUCKETLOCK(ncp) &&
VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
break;
cache_unlock_vnodes_cel(cel);
cel->vlp[0] = NULL;
cel->vlp[1] = NULL;
cel->vlp[2] = NULL;
}
cache_lock_buckets_cel(cel, blps[0], blps[1]);
}
static void
cache_enter_lock_dd(struct celockstate *cel, struct vnode *dvp, struct vnode *vp,
uint32_t hash)
{
struct namecache *ncp;
struct rwlock *blps[2];
blps[0] = HASH2BUCKETLOCK(hash);
for (;;) {
blps[1] = NULL;
cache_lock_vnodes_cel(cel, dvp, vp);
ncp = dvp->v_cache_dd;
if (ncp == NULL)
break;
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
break;
MPASS(ncp->nc_dvp == dvp);
blps[1] = NCP2BUCKETLOCK(ncp);
if (ncp->nc_flag & NCF_NEGATIVE)
break;
if (cache_lock_vnodes_cel_3(cel, ncp->nc_vp))
break;
if (ncp == dvp->v_cache_dd &&
(ncp->nc_flag & NCF_ISDOTDOT) != 0 &&
blps[1] == NCP2BUCKETLOCK(ncp) &&
VP2VNODELOCK(ncp->nc_vp) == cel->vlp[2])
break;
cache_unlock_vnodes_cel(cel);
cel->vlp[0] = NULL;
cel->vlp[1] = NULL;
cel->vlp[2] = NULL;
}
cache_lock_buckets_cel(cel, blps[0], blps[1]);
}
static void
cache_enter_unlock(struct celockstate *cel)
{
cache_unlock_buckets_cel(cel);
cache_unlock_vnodes_cel(cel);
}
static void __noinline
cache_enter_dotdot_prep(struct vnode *dvp, struct vnode *vp,
struct componentname *cnp)
{
struct celockstate cel;
struct namecache *ncp;
uint32_t hash;
int len;
if (dvp->v_cache_dd == NULL)
return;
len = cnp->cn_namelen;
cache_celockstate_init(&cel);
hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
cache_enter_lock_dd(&cel, dvp, vp, hash);
ncp = dvp->v_cache_dd;
if (ncp != NULL && (ncp->nc_flag & NCF_ISDOTDOT)) {
KASSERT(ncp->nc_dvp == dvp, ("wrong isdotdot parent"));
cache_zap_locked(ncp, false);
} else {
ncp = NULL;
}
dvp->v_cache_dd = NULL;
cache_enter_unlock(&cel);
cache_free(ncp);
}
/*
* Add an entry to the cache.
*/
void
cache_enter_time(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
struct timespec *tsp, struct timespec *dtsp)
{
struct celockstate cel;
struct namecache *ncp, *n2, *ndd;
struct namecache_ts *ncp_ts, *n2_ts;
struct nchashhead *ncpp;
uint32_t hash;
int flag;
int len;
u_long lnumcache;
CTR3(KTR_VFS, "cache_enter(%p, %p, %s)", dvp, vp, cnp->cn_nameptr);
VNASSERT(vp == NULL || !VN_IS_DOOMED(vp), vp,
("cache_enter: Adding a doomed vnode"));
VNASSERT(dvp == NULL || !VN_IS_DOOMED(dvp), dvp,
("cache_enter: Doomed vnode used as src"));
#ifdef DEBUG_CACHE
if (__predict_false(!doingcache))
return;
#endif
flag = 0;
if (__predict_false(cnp->cn_nameptr[0] == '.')) {
if (cnp->cn_namelen == 1)
return;
if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
cache_enter_dotdot_prep(dvp, vp, cnp);
flag = NCF_ISDOTDOT;
}
}
/*
* Avoid blowout in namecache entries.
*/
lnumcache = atomic_fetchadd_long(&numcache, 1) + 1;
if (__predict_false(lnumcache >= ncsize)) {
atomic_add_long(&numcache, -1);
return;
}
cache_celockstate_init(&cel);
ndd = NULL;
ncp_ts = NULL;
/*
* Calculate the hash key and setup as much of the new
* namecache entry as possible before acquiring the lock.
*/
ncp = cache_alloc(cnp->cn_namelen, tsp != NULL);
ncp->nc_flag = flag;
ncp->nc_vp = vp;
if (vp == NULL)
ncp->nc_flag |= NCF_NEGATIVE;
ncp->nc_dvp = dvp;
if (tsp != NULL) {
ncp_ts = __containerof(ncp, struct namecache_ts, nc_nc);
ncp_ts->nc_time = *tsp;
ncp_ts->nc_ticks = ticks;
ncp_ts->nc_nc.nc_flag |= NCF_TS;
if (dtsp != NULL) {
ncp_ts->nc_dotdottime = *dtsp;
ncp_ts->nc_nc.nc_flag |= NCF_DTS;
}
}
len = ncp->nc_nlen = cnp->cn_namelen;
hash = cache_get_hash(cnp->cn_nameptr, len, dvp);
strlcpy(ncp->nc_name, cnp->cn_nameptr, len + 1);
cache_enter_lock(&cel, dvp, vp, hash);
/*
* See if this vnode or negative entry is already in the cache
* with this name. This can happen with concurrent lookups of
* the same path name.
*/
ncpp = NCHHASH(hash);
LIST_FOREACH(n2, ncpp, nc_hash) {
if (n2->nc_dvp == dvp &&
n2->nc_nlen == cnp->cn_namelen &&
!bcmp(n2->nc_name, cnp->cn_nameptr, n2->nc_nlen)) {
if (tsp != NULL) {
KASSERT((n2->nc_flag & NCF_TS) != 0,
("no NCF_TS"));
n2_ts = __containerof(n2, struct namecache_ts, nc_nc);
n2_ts->nc_time = ncp_ts->nc_time;
n2_ts->nc_ticks = ncp_ts->nc_ticks;
if (dtsp != NULL) {
n2_ts->nc_dotdottime = ncp_ts->nc_dotdottime;
if (ncp->nc_flag & NCF_NEGATIVE)
mtx_lock(&ncneg_hot.nl_lock);
n2_ts->nc_nc.nc_flag |= NCF_DTS;
if (ncp->nc_flag & NCF_NEGATIVE)
mtx_unlock(&ncneg_hot.nl_lock);
}
}
goto out_unlock_free;
}
}
if (flag == NCF_ISDOTDOT) {
/*
* See if we are trying to add .. entry, but some other lookup
* has populated v_cache_dd pointer already.
*/
if (dvp->v_cache_dd != NULL)
goto out_unlock_free;
KASSERT(vp == NULL || vp->v_type == VDIR,
("wrong vnode type %p", vp));
dvp->v_cache_dd = ncp;
}
if (vp != NULL) {
if (vp->v_type == VDIR) {
if (flag != NCF_ISDOTDOT) {
/*
* For this case, the cache entry maps both the
* directory name in it and the name ".." for the
* directory's parent.
*/
if ((ndd = vp->v_cache_dd) != NULL) {
if ((ndd->nc_flag & NCF_ISDOTDOT) != 0)
cache_zap_locked(ndd, false);
else
ndd = NULL;
}
vp->v_cache_dd = ncp;
}
} else {
vp->v_cache_dd = NULL;
}
}
if (flag != NCF_ISDOTDOT) {
if (LIST_EMPTY(&dvp->v_cache_src)) {
vhold(dvp);
counter_u64_add(numcachehv, 1);
}
LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
}
/*
* Insert the new namecache entry into the appropriate chain
* within the cache entries table.
*/
LIST_INSERT_HEAD(ncpp, ncp, nc_hash);
/*
* If the entry is "negative", we place it into the
* "negative" cache queue, otherwise, we place it into the
* destination vnode's cache entries queue.
*/
if (vp != NULL) {
TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
SDT_PROBE3(vfs, namecache, enter, done, dvp, ncp->nc_name,
vp);
} else {
if (cnp->cn_flags & ISWHITEOUT)
ncp->nc_flag |= NCF_WHITE;
cache_negative_insert(ncp, false);
SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
ncp->nc_name);
}
cache_enter_unlock(&cel);
if (numneg * ncnegfactor > lnumcache)
cache_negative_zap_one();
cache_free(ndd);
return;
out_unlock_free:
cache_enter_unlock(&cel);
cache_free(ncp);
return;
}
static u_int
cache_roundup_2(u_int val)
{
u_int res;
for (res = 1; res <= val; res <<= 1)
continue;
return (res);
}
/*
* Name cache initialization, from vfs_init() when we are booting
*/
static void
nchinit(void *dummy __unused)
{
u_int i;
cache_zone_small = uma_zcreate("S VFS Cache",
sizeof(struct namecache) + CACHE_PATH_CUTOFF + 1,
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
UMA_ZONE_ZINIT);
cache_zone_small_ts = uma_zcreate("STS VFS Cache",
sizeof(struct namecache_ts) + CACHE_PATH_CUTOFF + 1,
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
UMA_ZONE_ZINIT);
cache_zone_large = uma_zcreate("L VFS Cache",
sizeof(struct namecache) + NAME_MAX + 1,
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache),
UMA_ZONE_ZINIT);
cache_zone_large_ts = uma_zcreate("LTS VFS Cache",
sizeof(struct namecache_ts) + NAME_MAX + 1,
NULL, NULL, NULL, NULL, UMA_ALIGNOF(struct namecache_ts),
UMA_ZONE_ZINIT);
ncsize = desiredvnodes * ncsizefactor;
nchashtbl = hashinit(desiredvnodes * 2, M_VFSCACHE, &nchash);
ncbuckethash = cache_roundup_2(mp_ncpus * mp_ncpus) - 1;
if (ncbuckethash < 7) /* arbitrarily chosen to avoid having one lock */
ncbuckethash = 7;
if (ncbuckethash > nchash)
ncbuckethash = nchash;
bucketlocks = malloc(sizeof(*bucketlocks) * numbucketlocks, M_VFSCACHE,
M_WAITOK | M_ZERO);
for (i = 0; i < numbucketlocks; i++)
rw_init_flags(&bucketlocks[i], "ncbuc", RW_DUPOK | RW_RECURSE);
ncvnodehash = ncbuckethash;
vnodelocks = malloc(sizeof(*vnodelocks) * numvnodelocks, M_VFSCACHE,
M_WAITOK | M_ZERO);
for (i = 0; i < numvnodelocks; i++)
mtx_init(&vnodelocks[i], "ncvn", NULL, MTX_DUPOK | MTX_RECURSE);
ncpurgeminvnodes = numbucketlocks * 2;
ncneghash = 3;
neglists = malloc(sizeof(*neglists) * numneglists, M_VFSCACHE,
M_WAITOK | M_ZERO);
for (i = 0; i < numneglists; i++) {
mtx_init(&neglists[i].nl_lock, "ncnegl", NULL, MTX_DEF);
TAILQ_INIT(&neglists[i].nl_list);
}
mtx_init(&ncneg_hot.nl_lock, "ncneglh", NULL, MTX_DEF);
TAILQ_INIT(&ncneg_hot.nl_list);
mtx_init(&ncneg_shrink_lock, "ncnegs", NULL, MTX_DEF);
numcachehv = counter_u64_alloc(M_WAITOK);
numcalls = counter_u64_alloc(M_WAITOK);
dothits = counter_u64_alloc(M_WAITOK);
dotdothits = counter_u64_alloc(M_WAITOK);
numchecks = counter_u64_alloc(M_WAITOK);
nummiss = counter_u64_alloc(M_WAITOK);
nummisszap = counter_u64_alloc(M_WAITOK);
numposzaps = counter_u64_alloc(M_WAITOK);
numposhits = counter_u64_alloc(M_WAITOK);
numnegzaps = counter_u64_alloc(M_WAITOK);
numneghits = counter_u64_alloc(M_WAITOK);
numfullpathcalls = counter_u64_alloc(M_WAITOK);
numfullpathfail1 = counter_u64_alloc(M_WAITOK);
numfullpathfail2 = counter_u64_alloc(M_WAITOK);
numfullpathfail4 = counter_u64_alloc(M_WAITOK);
numfullpathfound = counter_u64_alloc(M_WAITOK);
zap_and_exit_bucket_relock_success = counter_u64_alloc(M_WAITOK);
numneg_evicted = counter_u64_alloc(M_WAITOK);
shrinking_skipped = counter_u64_alloc(M_WAITOK);
}
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nchinit, NULL);
void
cache_changesize(u_long newmaxvnodes)
{
struct nchashhead *new_nchashtbl, *old_nchashtbl;
u_long new_nchash, old_nchash;
struct namecache *ncp;
uint32_t hash;
u_long newncsize;
int i;
newncsize = newmaxvnodes * ncsizefactor;
newmaxvnodes = cache_roundup_2(newmaxvnodes * 2);
if (newmaxvnodes < numbucketlocks)
newmaxvnodes = numbucketlocks;
new_nchashtbl = hashinit(newmaxvnodes, M_VFSCACHE, &new_nchash);
/* If same hash table size, nothing to do */
if (nchash == new_nchash) {
free(new_nchashtbl, M_VFSCACHE);
return;
}
/*
* Move everything from the old hash table to the new table.
* None of the namecache entries in the table can be removed
* because to do so, they have to be removed from the hash table.
*/
cache_lock_all_vnodes();
cache_lock_all_buckets();
old_nchashtbl = nchashtbl;
old_nchash = nchash;
nchashtbl = new_nchashtbl;
nchash = new_nchash;
for (i = 0; i <= old_nchash; i++) {
while ((ncp = LIST_FIRST(&old_nchashtbl[i])) != NULL) {
hash = cache_get_hash(ncp->nc_name, ncp->nc_nlen,
ncp->nc_dvp);
LIST_REMOVE(ncp, nc_hash);
LIST_INSERT_HEAD(NCHHASH(hash), ncp, nc_hash);
}
}
ncsize = newncsize;
cache_unlock_all_buckets();
cache_unlock_all_vnodes();
free(old_nchashtbl, M_VFSCACHE);
}
/*
* Invalidate all entries from and to a particular vnode.
*/
void
cache_purge(struct vnode *vp)
{
TAILQ_HEAD(, namecache) ncps;
struct namecache *ncp, *nnp;
struct mtx *vlp, *vlp2;
CTR1(KTR_VFS, "cache_purge(%p)", vp);
SDT_PROBE1(vfs, namecache, purge, done, vp);
if (LIST_EMPTY(&vp->v_cache_src) && TAILQ_EMPTY(&vp->v_cache_dst) &&
vp->v_cache_dd == NULL)
return;
TAILQ_INIT(&ncps);
vlp = VP2VNODELOCK(vp);
vlp2 = NULL;
mtx_lock(vlp);
retry:
while (!LIST_EMPTY(&vp->v_cache_src)) {
ncp = LIST_FIRST(&vp->v_cache_src);
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
}
while (!TAILQ_EMPTY(&vp->v_cache_dst)) {
ncp = TAILQ_FIRST(&vp->v_cache_dst);
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
}
ncp = vp->v_cache_dd;
if (ncp != NULL) {
KASSERT(ncp->nc_flag & NCF_ISDOTDOT,
("lost dotdot link"));
if (!cache_zap_locked_vnode_kl2(ncp, vp, &vlp2))
goto retry;
TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
}
KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
mtx_unlock(vlp);
if (vlp2 != NULL)
mtx_unlock(vlp2);
TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
cache_free(ncp);
}
}
/*
* Invalidate all negative entries for a particular directory vnode.
*/
void
cache_purge_negative(struct vnode *vp)
{
TAILQ_HEAD(, namecache) ncps;
struct namecache *ncp, *nnp;
struct mtx *vlp;
CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
if (LIST_EMPTY(&vp->v_cache_src))
return;
TAILQ_INIT(&ncps);
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
LIST_FOREACH_SAFE(ncp, &vp->v_cache_src, nc_src, nnp) {
if (!(ncp->nc_flag & NCF_NEGATIVE))
continue;
cache_zap_negative_locked_vnode_kl(ncp, vp);
TAILQ_INSERT_TAIL(&ncps, ncp, nc_dst);
}
mtx_unlock(vlp);
TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
cache_free(ncp);
}
}
/*
* Flush all entries referencing a particular filesystem.
*/
void
cache_purgevfs(struct mount *mp, bool force)
{
TAILQ_HEAD(, namecache) ncps;
struct mtx *vlp1, *vlp2;
struct rwlock *blp;
struct nchashhead *bucket;
struct namecache *ncp, *nnp;
u_long i, j, n_nchash;
int error;
/* Scan hash tables for applicable entries */
SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
if (!force && mp->mnt_nvnodelistsize <= ncpurgeminvnodes)
return;
TAILQ_INIT(&ncps);
n_nchash = nchash + 1;
vlp1 = vlp2 = NULL;
for (i = 0; i < numbucketlocks; i++) {
blp = (struct rwlock *)&bucketlocks[i];
rw_wlock(blp);
for (j = i; j < n_nchash; j += numbucketlocks) {
retry:
bucket = &nchashtbl[j];
LIST_FOREACH_SAFE(ncp, bucket, nc_hash, nnp) {
cache_assert_bucket_locked(ncp, RA_WLOCKED);
if (ncp->nc_dvp->v_mount != mp)
continue;
error = cache_zap_wlocked_bucket_kl(ncp, blp,
&vlp1, &vlp2);
if (error != 0)
goto retry;
TAILQ_INSERT_HEAD(&ncps, ncp, nc_dst);
}
}
rw_wunlock(blp);
if (vlp1 == NULL && vlp2 == NULL)
cache_maybe_yield();
}
if (vlp1 != NULL)
mtx_unlock(vlp1);
if (vlp2 != NULL)
mtx_unlock(vlp2);
TAILQ_FOREACH_SAFE(ncp, &ncps, nc_dst, nnp) {
cache_free(ncp);
}
}
/*
* Perform canonical checks and cache lookup and pass on to filesystem
* through the vop_cachedlookup only if needed.
*/
int
vfs_cache_lookup(struct vop_lookup_args *ap)
{
struct vnode *dvp;
int error;
struct vnode **vpp = ap->a_vpp;
struct componentname *cnp = ap->a_cnp;
int flags = cnp->cn_flags;
*vpp = NULL;
dvp = ap->a_dvp;
if (dvp->v_type != VDIR)
return (ENOTDIR);
if ((flags & ISLASTCN) && (dvp->v_mount->mnt_flag & MNT_RDONLY) &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME))
return (EROFS);
error = vn_dir_check_exec(dvp, cnp);
if (error != 0)
return (error);
error = cache_lookup(dvp, vpp, cnp, NULL, NULL);
if (error == 0)
return (VOP_CACHEDLOOKUP(dvp, vpp, cnp));
if (error == -1)
return (0);
return (error);
}
/* Implementation of the getcwd syscall. */
int
sys___getcwd(struct thread *td, struct __getcwd_args *uap)
{
char *buf, *retbuf;
size_t buflen;
int error;
buflen = uap->buflen;
if (__predict_false(buflen < 2))
return (EINVAL);
if (buflen > MAXPATHLEN)
buflen = MAXPATHLEN;
buf = malloc(buflen, M_TEMP, M_WAITOK);
error = vn_getcwd(td, buf, &retbuf, &buflen);
if (error == 0)
error = copyout(retbuf, uap->buf, buflen);
free(buf, M_TEMP);
return (error);
}
int
vn_getcwd(struct thread *td, char *buf, char **retbuf, size_t *buflen)
{
- struct filedesc *fdp;
- struct vnode *cdir, *rdir;
+ struct pwd *pwd;
int error;
- fdp = td->td_proc->p_fd;
- FILEDESC_SLOCK(fdp);
- cdir = fdp->fd_cdir;
- vrefact(cdir);
- rdir = fdp->fd_rdir;
- vrefact(rdir);
- FILEDESC_SUNLOCK(fdp);
- error = vn_fullpath_any(td, cdir, rdir, buf, retbuf, buflen);
- vrele(rdir);
- vrele(cdir);
+ pwd = pwd_hold(td);
+ error = vn_fullpath_any(td, pwd->pwd_cdir, pwd->pwd_rdir, buf, retbuf, buflen);
+ pwd_drop(pwd);
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_NAMEI) && error == 0)
ktrnamei(*retbuf);
#endif
return (error);
}
static int
kern___realpathat(struct thread *td, int fd, const char *path, char *buf,
size_t size, int flags, enum uio_seg pathseg)
{
struct nameidata nd;
char *retbuf, *freebuf;
int error;
if (flags != 0)
return (EINVAL);
NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | SAVENAME | WANTPARENT | AUDITVNODE1,
pathseg, path, fd, &cap_fstat_rights, td);
if ((error = namei(&nd)) != 0)
return (error);
error = vn_fullpath_hardlink(td, &nd, &retbuf, &freebuf, &size);
if (error == 0) {
error = copyout(retbuf, buf, size);
free(freebuf, M_TEMP);
}
NDFREE(&nd, 0);
return (error);
}
int
sys___realpathat(struct thread *td, struct __realpathat_args *uap)
{
return (kern___realpathat(td, uap->fd, uap->path, uap->buf, uap->size,
uap->flags, UIO_USERSPACE));
}
/*
* Retrieve the full filesystem path that correspond to a vnode from the name
* cache (if available)
*/
int
vn_fullpath(struct thread *td, struct vnode *vn, char **retbuf, char **freebuf)
{
+ struct pwd *pwd;
char *buf;
- struct filedesc *fdp;
- struct vnode *rdir;
size_t buflen;
int error;
if (__predict_false(vn == NULL))
return (EINVAL);
buflen = MAXPATHLEN;
buf = malloc(buflen, M_TEMP, M_WAITOK);
- fdp = td->td_proc->p_fd;
- FILEDESC_SLOCK(fdp);
- rdir = fdp->fd_rdir;
- vrefact(rdir);
- FILEDESC_SUNLOCK(fdp);
- error = vn_fullpath_any(td, vn, rdir, buf, retbuf, &buflen);
- vrele(rdir);
+ pwd = pwd_hold(td);
+ error = vn_fullpath_any(td, vn, pwd->pwd_rdir, buf, retbuf, &buflen);
+ pwd_drop(pwd);
if (!error)
*freebuf = buf;
else
free(buf, M_TEMP);
return (error);
}
/*
* This function is similar to vn_fullpath, but it attempts to lookup the
* pathname relative to the global root mount point. This is required for the
* auditing sub-system, as audited pathnames must be absolute, relative to the
* global root mount point.
*/
int
vn_fullpath_global(struct thread *td, struct vnode *vn,
char **retbuf, char **freebuf)
{
char *buf;
size_t buflen;
int error;
if (__predict_false(vn == NULL))
return (EINVAL);
buflen = MAXPATHLEN;
buf = malloc(buflen, M_TEMP, M_WAITOK);
error = vn_fullpath_any(td, vn, rootvnode, buf, retbuf, &buflen);
if (!error)
*freebuf = buf;
else
free(buf, M_TEMP);
return (error);
}
int
vn_vptocnp(struct vnode **vp, struct ucred *cred, char *buf, size_t *buflen)
{
struct vnode *dvp;
struct namecache *ncp;
struct mtx *vlp;
int error;
vlp = VP2VNODELOCK(*vp);
mtx_lock(vlp);
TAILQ_FOREACH(ncp, &((*vp)->v_cache_dst), nc_dst) {
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
break;
}
if (ncp != NULL) {
if (*buflen < ncp->nc_nlen) {
mtx_unlock(vlp);
vrele(*vp);
counter_u64_add(numfullpathfail4, 1);
error = ENOMEM;
SDT_PROBE3(vfs, namecache, fullpath, return, error,
vp, NULL);
return (error);
}
*buflen -= ncp->nc_nlen;
memcpy(buf + *buflen, ncp->nc_name, ncp->nc_nlen);
SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
ncp->nc_name, vp);
dvp = *vp;
*vp = ncp->nc_dvp;
vref(*vp);
mtx_unlock(vlp);
vrele(dvp);
return (0);
}
SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
mtx_unlock(vlp);
vn_lock(*vp, LK_SHARED | LK_RETRY);
error = VOP_VPTOCNP(*vp, &dvp, cred, buf, buflen);
vput(*vp);
if (error) {
counter_u64_add(numfullpathfail2, 1);
SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
return (error);
}
*vp = dvp;
if (VN_IS_DOOMED(dvp)) {
/* forced unmount */
vrele(dvp);
error = ENOENT;
SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
return (error);
}
/*
* *vp has its use count incremented still.
*/
return (0);
}
/*
* Resolve a directory to a pathname.
*
* The name of the directory can always be found in the namecache or fetched
* from the filesystem. There is also guaranteed to be only one parent, meaning
* we can just follow vnodes up until we find the root.
*
* The vnode must be referenced.
*/
static int
vn_fullpath_dir(struct thread *td, struct vnode *vp, struct vnode *rdir,
char *buf, char **retbuf, size_t *len, bool slash_prefixed, size_t addend)
{
#ifdef KDTRACE_HOOKS
struct vnode *startvp = vp;
#endif
struct vnode *vp1;
size_t buflen;
int error;
VNPASS(vp->v_type == VDIR || VN_IS_DOOMED(vp), vp);
VNPASS(vp->v_usecount > 0, vp);
buflen = *len;
if (!slash_prefixed) {
MPASS(*len >= 2);
buflen--;
buf[buflen] = '\0';
}
error = 0;
SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
counter_u64_add(numfullpathcalls, 1);
while (vp != rdir && vp != rootvnode) {
/*
* The vp vnode must be already fully constructed,
* since it is either found in namecache or obtained
* from VOP_VPTOCNP(). We may test for VV_ROOT safely
* without obtaining the vnode lock.
*/
if ((vp->v_vflag & VV_ROOT) != 0) {
vn_lock(vp, LK_RETRY | LK_SHARED);
/*
* With the vnode locked, check for races with
* unmount, forced or not. Note that we
* already verified that vp is not equal to
* the root vnode, which means that
* mnt_vnodecovered can be NULL only for the
* case of unmount.
*/
if (VN_IS_DOOMED(vp) ||
(vp1 = vp->v_mount->mnt_vnodecovered) == NULL ||
vp1->v_mountedhere != vp->v_mount) {
vput(vp);
error = ENOENT;
SDT_PROBE3(vfs, namecache, fullpath, return,
error, vp, NULL);
break;
}
vref(vp1);
vput(vp);
vp = vp1;
continue;
}
if (vp->v_type != VDIR) {
vrele(vp);
counter_u64_add(numfullpathfail1, 1);
error = ENOTDIR;
SDT_PROBE3(vfs, namecache, fullpath, return,
error, vp, NULL);
break;
}
error = vn_vptocnp(&vp, td->td_ucred, buf, &buflen);
if (error)
break;
if (buflen == 0) {
vrele(vp);
error = ENOMEM;
SDT_PROBE3(vfs, namecache, fullpath, return, error,
startvp, NULL);
break;
}
buf[--buflen] = '/';
slash_prefixed = true;
}
if (error)
return (error);
if (!slash_prefixed) {
if (buflen == 0) {
vrele(vp);
counter_u64_add(numfullpathfail4, 1);
SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
startvp, NULL);
return (ENOMEM);
}
buf[--buflen] = '/';
}
counter_u64_add(numfullpathfound, 1);
vrele(vp);
*retbuf = buf + buflen;
SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, *retbuf);
*len -= buflen;
*len += addend;
return (0);
}
/*
* Resolve an arbitrary vnode to a pathname.
*
* Note 2 caveats:
* - hardlinks are not tracked, thus if the vnode is not a directory this can
* resolve to a different path than the one used to find it
* - namecache is not mandatory, meaning names are not guaranteed to be added
* (in which case resolving fails)
*/
static int
vn_fullpath_any(struct thread *td, struct vnode *vp, struct vnode *rdir,
char *buf, char **retbuf, size_t *buflen)
{
size_t orig_buflen;
bool slash_prefixed;
int error;
if (*buflen < 2)
return (EINVAL);
orig_buflen = *buflen;
vref(vp);
slash_prefixed = false;
if (vp->v_type != VDIR) {
*buflen -= 1;
buf[*buflen] = '\0';
error = vn_vptocnp(&vp, td->td_ucred, buf, buflen);
if (error)
return (error);
if (*buflen == 0) {
vrele(vp);
return (ENOMEM);
}
*buflen -= 1;
buf[*buflen] = '/';
slash_prefixed = true;
}
return (vn_fullpath_dir(td, vp, rdir, buf, retbuf, buflen, slash_prefixed,
orig_buflen - *buflen));
}
/*
* Resolve an arbitrary vnode to a pathname (taking care of hardlinks).
*
* Since the namecache does not track handlings, the caller is expected to first
* look up the target vnode with SAVENAME | WANTPARENT flags passed to namei.
*
* Then we have 2 cases:
* - if the found vnode is a directory, the path can be constructed just by
* fullowing names up the chain
* - otherwise we populate the buffer with the saved name and start resolving
* from the parent
*/
static int
vn_fullpath_hardlink(struct thread *td, struct nameidata *ndp, char **retbuf,
char **freebuf, size_t *buflen)
{
char *buf, *tmpbuf;
- struct filedesc *fdp;
- struct vnode *rdir;
+ struct pwd *pwd;
struct componentname *cnp;
struct vnode *vp;
size_t addend;
int error;
bool slash_prefixed;
if (*buflen < 2)
return (EINVAL);
if (*buflen > MAXPATHLEN)
*buflen = MAXPATHLEN;
slash_prefixed = false;
buf = malloc(*buflen, M_TEMP, M_WAITOK);
- fdp = td->td_proc->p_fd;
- FILEDESC_SLOCK(fdp);
- rdir = fdp->fd_rdir;
- vrefact(rdir);
- FILEDESC_SUNLOCK(fdp);
+ pwd = pwd_hold(td);
addend = 0;
vp = ndp->ni_vp;
if (vp->v_type != VDIR) {
cnp = &ndp->ni_cnd;
addend = cnp->cn_namelen + 2;
if (*buflen < addend) {
error = ENOMEM;
goto out_bad;
}
*buflen -= addend;
tmpbuf = buf + *buflen;
tmpbuf[0] = '/';
memcpy(&tmpbuf[1], cnp->cn_nameptr, cnp->cn_namelen);
tmpbuf[addend - 1] = '\0';
slash_prefixed = true;
vp = ndp->ni_dvp;
}
vref(vp);
- error = vn_fullpath_dir(td, vp, rdir, buf, retbuf, buflen, slash_prefixed, addend);
+ error = vn_fullpath_dir(td, vp, pwd->pwd_rdir, buf, retbuf, buflen,
+ slash_prefixed, addend);
if (error != 0)
goto out_bad;
- vrele(rdir);
+ pwd_drop(pwd);
*freebuf = buf;
return (0);
out_bad:
- vrele(rdir);
+ pwd_drop(pwd);
free(buf, M_TEMP);
return (error);
}
struct vnode *
vn_dir_dd_ino(struct vnode *vp)
{
struct namecache *ncp;
struct vnode *ddvp;
struct mtx *vlp;
enum vgetstate vs;
ASSERT_VOP_LOCKED(vp, "vn_dir_dd_ino");
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
TAILQ_FOREACH(ncp, &(vp->v_cache_dst), nc_dst) {
if ((ncp->nc_flag & NCF_ISDOTDOT) != 0)
continue;
ddvp = ncp->nc_dvp;
vs = vget_prep(ddvp);
mtx_unlock(vlp);
if (vget_finish(ddvp, LK_SHARED | LK_NOWAIT, vs))
return (NULL);
return (ddvp);
}
mtx_unlock(vlp);
return (NULL);
}
int
vn_commname(struct vnode *vp, char *buf, u_int buflen)
{
struct namecache *ncp;
struct mtx *vlp;
int l;
vlp = VP2VNODELOCK(vp);
mtx_lock(vlp);
TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
break;
if (ncp == NULL) {
mtx_unlock(vlp);
return (ENOENT);
}
l = min(ncp->nc_nlen, buflen - 1);
memcpy(buf, ncp->nc_name, l);
mtx_unlock(vlp);
buf[l] = '\0';
return (0);
}
/*
* This function updates path string to vnode's full global path
* and checks the size of the new path string against the pathlen argument.
*
* Requires a locked, referenced vnode.
* Vnode is re-locked on success or ENODEV, otherwise unlocked.
*
* If vp is a directory, the call to vn_fullpath_global() always succeeds
* because it falls back to the ".." lookup if the namecache lookup fails.
*/
int
vn_path_to_global_path(struct thread *td, struct vnode *vp, char *path,
u_int pathlen)
{
struct nameidata nd;
struct vnode *vp1;
char *rpath, *fbuf;
int error;
ASSERT_VOP_ELOCKED(vp, __func__);
/* Construct global filesystem path from vp. */
VOP_UNLOCK(vp);
error = vn_fullpath_global(td, vp, &rpath, &fbuf);
if (error != 0) {
vrele(vp);
return (error);
}
if (strlen(rpath) >= pathlen) {
vrele(vp);
error = ENAMETOOLONG;
goto out;
}
/*
* Re-lookup the vnode by path to detect a possible rename.
* As a side effect, the vnode is relocked.
* If vnode was renamed, return ENOENT.
*/
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
UIO_SYSSPACE, path, td);
error = namei(&nd);
if (error != 0) {
vrele(vp);
goto out;
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vp1 = nd.ni_vp;
vrele(vp);
if (vp1 == vp)
strcpy(path, rpath);
else {
vput(vp1);
error = ENOENT;
}
out:
free(fbuf, M_TEMP);
return (error);
}
#ifdef DDB
static void
db_print_vpath(struct vnode *vp)
{
while (vp != NULL) {
db_printf("%p: ", vp);
if (vp == rootvnode) {
db_printf("/");
vp = NULL;
} else {
if (vp->v_vflag & VV_ROOT) {
db_printf("<mount point>");
vp = vp->v_mount->mnt_vnodecovered;
} else {
struct namecache *ncp;
char *ncn;
int i;
ncp = TAILQ_FIRST(&vp->v_cache_dst);
if (ncp != NULL) {
ncn = ncp->nc_name;
for (i = 0; i < ncp->nc_nlen; i++)
db_printf("%c", *ncn++);
vp = ncp->nc_dvp;
} else {
vp = NULL;
}
}
}
db_printf("\n");
}
return;
}
DB_SHOW_COMMAND(vpath, db_show_vpath)
{
struct vnode *vp;
if (!have_addr) {
db_printf("usage: show vpath <struct vnode *>\n");
return;
}
vp = (struct vnode *)addr;
db_print_vpath(vp);
}
#endif
Index: head/sys/kern/vfs_lookup.c
===================================================================
--- head/sys/kern/vfs_lookup.c (revision 358502)
+++ head/sys/kern/vfs_lookup.c (revision 358503)
@@ -1,1514 +1,1517 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)vfs_lookup.c 8.4 (Berkeley) 2/16/94
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_ktrace.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/capsicum.h>
#include <sys/fcntl.h>
#include <sys/jail.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/filedesc.h>
#include <sys/proc.h>
#include <sys/sdt.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#ifdef KTRACE
#include <sys/ktrace.h>
#endif
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/uma.h>
#define NAMEI_DIAGNOSTIC 1
#undef NAMEI_DIAGNOSTIC
SDT_PROVIDER_DECLARE(vfs);
SDT_PROBE_DEFINE3(vfs, namei, lookup, entry, "struct vnode *", "char *",
"unsigned long");
SDT_PROBE_DEFINE2(vfs, namei, lookup, return, "int", "struct vnode *");
/* Allocation zone for namei. */
uma_zone_t namei_zone;
/* Placeholder vnode for mp traversal. */
static struct vnode *vp_crossmp;
static int
crossmp_vop_islocked(struct vop_islocked_args *ap)
{
return (LK_SHARED);
}
static int
crossmp_vop_lock1(struct vop_lock1_args *ap)
{
struct vnode *vp;
struct lock *lk __unused;
const char *file __unused;
int flags, line __unused;
vp = ap->a_vp;
lk = vp->v_vnlock;
flags = ap->a_flags;
file = ap->a_file;
line = ap->a_line;
if ((flags & LK_SHARED) == 0)
panic("invalid lock request for crossmp");
WITNESS_CHECKORDER(&lk->lock_object, LOP_NEWORDER, file, line,
flags & LK_INTERLOCK ? &VI_MTX(vp)->lock_object : NULL);
WITNESS_LOCK(&lk->lock_object, 0, file, line);
if ((flags & LK_INTERLOCK) != 0)
VI_UNLOCK(vp);
LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, ap->a_file, line);
return (0);
}
static int
crossmp_vop_unlock(struct vop_unlock_args *ap)
{
struct vnode *vp;
struct lock *lk __unused;
vp = ap->a_vp;
lk = vp->v_vnlock;
WITNESS_UNLOCK(&lk->lock_object, 0, LOCK_FILE, LOCK_LINE);
LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, LOCK_FILE,
LOCK_LINE);
return (0);
}
static struct vop_vector crossmp_vnodeops = {
.vop_default = &default_vnodeops,
.vop_islocked = crossmp_vop_islocked,
.vop_lock1 = crossmp_vop_lock1,
.vop_unlock = crossmp_vop_unlock,
};
/*
* VFS_VOP_VECTOR_REGISTER(crossmp_vnodeops) is not used here since the vnode
* gets allocated early. See nameiinit for the direct call below.
*/
struct nameicap_tracker {
struct vnode *dp;
TAILQ_ENTRY(nameicap_tracker) nm_link;
};
/* Zone for cap mode tracker elements used for dotdot capability checks. */
static uma_zone_t nt_zone;
static void
nameiinit(void *dummy __unused)
{
namei_zone = uma_zcreate("NAMEI", MAXPATHLEN, NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
nt_zone = uma_zcreate("rentr", sizeof(struct nameicap_tracker),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
vfs_vector_op_register(&crossmp_vnodeops);
getnewvnode("crossmp", NULL, &crossmp_vnodeops, &vp_crossmp);
}
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_SECOND, nameiinit, NULL);
static int lookup_cap_dotdot = 1;
SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot, CTLFLAG_RWTUN,
&lookup_cap_dotdot, 0,
"enables \"..\" components in path lookup in capability mode");
static int lookup_cap_dotdot_nonlocal = 1;
SYSCTL_INT(_vfs, OID_AUTO, lookup_cap_dotdot_nonlocal, CTLFLAG_RWTUN,
&lookup_cap_dotdot_nonlocal, 0,
"enables \"..\" components in path lookup in capability mode "
"on non-local mount");
static void
nameicap_tracker_add(struct nameidata *ndp, struct vnode *dp)
{
struct nameicap_tracker *nt;
if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0 || dp->v_type != VDIR)
return;
if ((ndp->ni_lcf & (NI_LCF_BENEATH_ABS | NI_LCF_BENEATH_LATCHED)) ==
NI_LCF_BENEATH_ABS) {
MPASS((ndp->ni_lcf & NI_LCF_LATCH) != 0);
if (dp != ndp->ni_beneath_latch)
return;
ndp->ni_lcf |= NI_LCF_BENEATH_LATCHED;
}
nt = uma_zalloc(nt_zone, M_WAITOK);
vhold(dp);
nt->dp = dp;
TAILQ_INSERT_TAIL(&ndp->ni_cap_tracker, nt, nm_link);
}
static void
nameicap_cleanup(struct nameidata *ndp, bool clean_latch)
{
struct nameicap_tracker *nt, *nt1;
KASSERT(TAILQ_EMPTY(&ndp->ni_cap_tracker) ||
(ndp->ni_lcf & NI_LCF_CAP_DOTDOT) != 0, ("not strictrelative"));
TAILQ_FOREACH_SAFE(nt, &ndp->ni_cap_tracker, nm_link, nt1) {
TAILQ_REMOVE(&ndp->ni_cap_tracker, nt, nm_link);
vdrop(nt->dp);
uma_zfree(nt_zone, nt);
}
if (clean_latch && (ndp->ni_lcf & NI_LCF_LATCH) != 0) {
ndp->ni_lcf &= ~NI_LCF_LATCH;
vrele(ndp->ni_beneath_latch);
}
}
/*
* For dotdot lookups in capability mode, only allow the component
* lookup to succeed if the resulting directory was already traversed
* during the operation. Also fail dotdot lookups for non-local
* filesystems, where external agents might assist local lookups to
* escape the compartment.
*/
static int
nameicap_check_dotdot(struct nameidata *ndp, struct vnode *dp)
{
struct nameicap_tracker *nt;
struct mount *mp;
if ((ndp->ni_lcf & NI_LCF_CAP_DOTDOT) == 0 || dp == NULL ||
dp->v_type != VDIR)
return (0);
mp = dp->v_mount;
if (lookup_cap_dotdot_nonlocal == 0 && mp != NULL &&
(mp->mnt_flag & MNT_LOCAL) == 0)
return (ENOTCAPABLE);
TAILQ_FOREACH_REVERSE(nt, &ndp->ni_cap_tracker, nameicap_tracker_head,
nm_link) {
if (dp == nt->dp)
return (0);
}
if ((ndp->ni_lcf & NI_LCF_BENEATH_ABS) != 0) {
ndp->ni_lcf &= ~NI_LCF_BENEATH_LATCHED;
nameicap_cleanup(ndp, false);
return (0);
}
return (ENOTCAPABLE);
}
static void
namei_cleanup_cnp(struct componentname *cnp)
{
uma_zfree(namei_zone, cnp->cn_pnbuf);
#ifdef DIAGNOSTIC
cnp->cn_pnbuf = NULL;
cnp->cn_nameptr = NULL;
#endif
}
static int
namei_handle_root(struct nameidata *ndp, struct vnode **dpp, u_int n)
{
struct componentname *cnp;
cnp = &ndp->ni_cnd;
if ((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
return (ENOTCAPABLE);
}
if ((cnp->cn_flags & BENEATH) != 0) {
ndp->ni_lcf |= NI_LCF_BENEATH_ABS;
ndp->ni_lcf &= ~NI_LCF_BENEATH_LATCHED;
nameicap_cleanup(ndp, false);
}
while (*(cnp->cn_nameptr) == '/') {
cnp->cn_nameptr++;
ndp->ni_pathlen--;
}
*dpp = ndp->ni_rootdir;
vrefactn(*dpp, n);
return (0);
}
/*
* Convert a pathname into a pointer to a locked vnode.
*
* The FOLLOW flag is set when symbolic links are to be followed
* when they occur at the end of the name translation process.
* Symbolic links are always followed for all other pathname
* components other than the last.
*
* The segflg defines whether the name is to be copied from user
* space or kernel space.
*
* Overall outline of namei:
*
* copy in name
* get starting directory
* while (!done && !error) {
* call lookup to search path.
* if symbolic link, massage name in buffer and continue
* }
*/
int
namei(struct nameidata *ndp)
{
- struct filedesc *fdp; /* pointer to file descriptor state */
char *cp; /* pointer into pathname argument */
struct vnode *dp; /* the directory we are searching */
struct iovec aiov; /* uio for reading symbolic links */
struct componentname *cnp;
struct file *dfp;
struct thread *td;
struct proc *p;
+ struct pwd *pwd;
cap_rights_t rights;
struct filecaps dirfd_caps;
struct uio auio;
int error, linklen, startdir_used;
cnp = &ndp->ni_cnd;
td = cnp->cn_thread;
p = td->td_proc;
ndp->ni_cnd.cn_cred = ndp->ni_cnd.cn_thread->td_ucred;
KASSERT(cnp->cn_cred && p, ("namei: bad cred/proc"));
KASSERT((cnp->cn_nameiop & (~OPMASK)) == 0,
("namei: nameiop contaminated with flags"));
KASSERT((cnp->cn_flags & OPMASK) == 0,
("namei: flags contaminated with nameiops"));
MPASS(ndp->ni_startdir == NULL || ndp->ni_startdir->v_type == VDIR ||
ndp->ni_startdir->v_type == VBAD);
- fdp = p->p_fd;
TAILQ_INIT(&ndp->ni_cap_tracker);
ndp->ni_lcf = 0;
/* We will set this ourselves if we need it. */
cnp->cn_flags &= ~TRAILINGSLASH;
/*
* Get a buffer for the name to be translated, and copy the
* name into the buffer.
*/
if ((cnp->cn_flags & HASBUF) == 0)
cnp->cn_pnbuf = uma_zalloc(namei_zone, M_WAITOK);
if (ndp->ni_segflg == UIO_SYSSPACE)
error = copystr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
&ndp->ni_pathlen);
else
error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf, MAXPATHLEN,
&ndp->ni_pathlen);
/*
* Don't allow empty pathnames.
*/
if (error == 0 && *cnp->cn_pnbuf == '\0')
error = ENOENT;
#ifdef CAPABILITY_MODE
/*
* In capability mode, lookups must be restricted to happen in
* the subtree with the root specified by the file descriptor:
* - The root must be real file descriptor, not the pseudo-descriptor
* AT_FDCWD.
* - The passed path must be relative and not absolute.
* - If lookup_cap_dotdot is disabled, path must not contain the
* '..' components.
* - If lookup_cap_dotdot is enabled, we verify that all '..'
* components lookups result in the directories which were
* previously walked by us, which prevents an escape from
* the relative root.
*/
if (error == 0 && IN_CAPABILITY_MODE(td) &&
(cnp->cn_flags & NOCAPCHECK) == 0) {
ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
if (ndp->ni_dirfd == AT_FDCWD) {
#ifdef KTRACE
if (KTRPOINT(td, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
error = ECAPMODE;
}
}
#endif
if (error != 0) {
namei_cleanup_cnp(cnp);
ndp->ni_vp = NULL;
return (error);
}
ndp->ni_loopcnt = 0;
#ifdef KTRACE
if (KTRPOINT(td, KTR_NAMEI)) {
KASSERT(cnp->cn_thread == curthread,
("namei not using curthread"));
ktrnamei(cnp->cn_pnbuf);
}
#endif
/*
* Get starting point for the translation.
*/
- FILEDESC_SLOCK(fdp);
+ pwd = pwd_hold(td);
/*
* The reference on ni_rootdir is acquired in the block below to avoid
* back-to-back atomics for absolute lookups.
*/
- ndp->ni_rootdir = fdp->fd_rdir;
- ndp->ni_topdir = fdp->fd_jdir;
+ ndp->ni_rootdir = pwd->pwd_rdir;
+ ndp->ni_topdir = pwd->pwd_jdir;
startdir_used = 0;
dp = NULL;
cnp->cn_nameptr = cnp->cn_pnbuf;
if (cnp->cn_pnbuf[0] == '/') {
ndp->ni_resflags |= NIRES_ABS;
error = namei_handle_root(ndp, &dp, 2);
if (error != 0) {
/*
* Simplify error handling, we should almost never be
* here.
*/
vrefact(ndp->ni_rootdir);
}
} else {
if (ndp->ni_startdir != NULL) {
vrefact(ndp->ni_rootdir);
dp = ndp->ni_startdir;
startdir_used = 1;
} else if (ndp->ni_dirfd == AT_FDCWD) {
- dp = fdp->fd_cdir;
+ dp = pwd->pwd_cdir;
if (dp == ndp->ni_rootdir) {
vrefactn(dp, 2);
} else {
vrefact(ndp->ni_rootdir);
vrefact(dp);
}
} else {
vrefact(ndp->ni_rootdir);
rights = ndp->ni_rightsneeded;
cap_rights_set_one(&rights, CAP_LOOKUP);
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_ATFD1(ndp->ni_dirfd);
if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_ATFD2(ndp->ni_dirfd);
/*
* Effectively inlined fgetvp_rights, because we need to
* inspect the file as well as grabbing the vnode.
*/
- error = fget_cap_locked(fdp, ndp->ni_dirfd, &rights,
+ error = fget_cap(td, ndp->ni_dirfd, &rights,
&dfp, &ndp->ni_filecaps);
if (error != 0) {
/*
* Preserve the error; it should either be EBADF
* or capability-related, both of which can be
* safely returned to the caller.
*/
- } else if (dfp->f_ops == &badfileops) {
- error = EBADF;
- } else if (dfp->f_vnode == NULL) {
- error = ENOTDIR;
} else {
- dp = dfp->f_vnode;
- vrefact(dp);
+ if (dfp->f_ops == &badfileops) {
+ error = EBADF;
+ } else if (dfp->f_vnode == NULL) {
+ error = ENOTDIR;
+ } else {
+ dp = dfp->f_vnode;
+ vrefact(dp);
- if ((dfp->f_flag & FSEARCH) != 0)
- cnp->cn_flags |= NOEXECCHECK;
+ if ((dfp->f_flag & FSEARCH) != 0)
+ cnp->cn_flags |= NOEXECCHECK;
+ }
+ fdrop(dfp, td);
}
#ifdef CAPABILITIES
/*
* If file descriptor doesn't have all rights,
* all lookups relative to it must also be
* strictly relative.
*/
CAP_ALL(&rights);
if (!cap_rights_contains(&ndp->ni_filecaps.fc_rights,
&rights) ||
ndp->ni_filecaps.fc_fcntls != CAP_FCNTL_ALL ||
ndp->ni_filecaps.fc_nioctls != -1) {
ndp->ni_lcf |= NI_LCF_STRICTRELATIVE;
}
#endif
}
if (error == 0 && dp->v_type != VDIR)
error = ENOTDIR;
}
if (error == 0 && (cnp->cn_flags & BENEATH) != 0) {
if (ndp->ni_dirfd == AT_FDCWD) {
- ndp->ni_beneath_latch = fdp->fd_cdir;
+ ndp->ni_beneath_latch = pwd->pwd_cdir;
vrefact(ndp->ni_beneath_latch);
} else {
rights = ndp->ni_rightsneeded;
cap_rights_set_one(&rights, CAP_LOOKUP);
error = fgetvp_rights(td, ndp->ni_dirfd, &rights,
&dirfd_caps, &ndp->ni_beneath_latch);
if (error == 0 && dp->v_type != VDIR) {
vrele(ndp->ni_beneath_latch);
error = ENOTDIR;
}
}
if (error == 0)
ndp->ni_lcf |= NI_LCF_LATCH;
}
- FILEDESC_SUNLOCK(fdp);
/*
* If we are auditing the kernel pathname, save the user pathname.
*/
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_UPATH1_VP(td, ndp->ni_rootdir, dp, cnp->cn_pnbuf);
if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_UPATH2_VP(td, ndp->ni_rootdir, dp, cnp->cn_pnbuf);
if (ndp->ni_startdir != NULL && !startdir_used)
vrele(ndp->ni_startdir);
if (error != 0) {
if (dp != NULL)
vrele(dp);
goto out;
}
MPASS((ndp->ni_lcf & (NI_LCF_BENEATH_ABS | NI_LCF_LATCH)) !=
NI_LCF_BENEATH_ABS);
if (((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) != 0 &&
lookup_cap_dotdot != 0) ||
((ndp->ni_lcf & NI_LCF_STRICTRELATIVE) == 0 &&
(cnp->cn_flags & BENEATH) != 0))
ndp->ni_lcf |= NI_LCF_CAP_DOTDOT;
SDT_PROBE3(vfs, namei, lookup, entry, dp, cnp->cn_pnbuf,
cnp->cn_flags);
for (;;) {
ndp->ni_startdir = dp;
error = lookup(ndp);
if (error != 0)
goto out;
/*
* If not a symbolic link, we're done.
*/
if ((cnp->cn_flags & ISSYMLINK) == 0) {
vrele(ndp->ni_rootdir);
if ((cnp->cn_flags & (SAVENAME | SAVESTART)) == 0) {
namei_cleanup_cnp(cnp);
} else
cnp->cn_flags |= HASBUF;
if ((ndp->ni_lcf & (NI_LCF_BENEATH_ABS |
NI_LCF_BENEATH_LATCHED)) == NI_LCF_BENEATH_ABS) {
NDFREE(ndp, 0);
error = ENOTCAPABLE;
}
nameicap_cleanup(ndp, true);
SDT_PROBE2(vfs, namei, lookup, return, error,
(error == 0 ? ndp->ni_vp : NULL));
+ pwd_drop(pwd);
return (error);
}
if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
error = ELOOP;
break;
}
#ifdef MAC
if ((cnp->cn_flags & NOMACCHECK) == 0) {
error = mac_vnode_check_readlink(td->td_ucred,
ndp->ni_vp);
if (error != 0)
break;
}
#endif
if (ndp->ni_pathlen > 1)
cp = uma_zalloc(namei_zone, M_WAITOK);
else
cp = cnp->cn_pnbuf;
aiov.iov_base = cp;
aiov.iov_len = MAXPATHLEN;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = 0;
auio.uio_rw = UIO_READ;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_td = td;
auio.uio_resid = MAXPATHLEN;
error = VOP_READLINK(ndp->ni_vp, &auio, cnp->cn_cred);
if (error != 0) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
break;
}
linklen = MAXPATHLEN - auio.uio_resid;
if (linklen == 0) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
error = ENOENT;
break;
}
if (linklen + ndp->ni_pathlen > MAXPATHLEN) {
if (ndp->ni_pathlen > 1)
uma_zfree(namei_zone, cp);
error = ENAMETOOLONG;
break;
}
if (ndp->ni_pathlen > 1) {
bcopy(ndp->ni_next, cp + linklen, ndp->ni_pathlen);
uma_zfree(namei_zone, cnp->cn_pnbuf);
cnp->cn_pnbuf = cp;
} else
cnp->cn_pnbuf[linklen] = '\0';
ndp->ni_pathlen += linklen;
vput(ndp->ni_vp);
dp = ndp->ni_dvp;
/*
* Check if root directory should replace current directory.
*/
cnp->cn_nameptr = cnp->cn_pnbuf;
if (*(cnp->cn_nameptr) == '/') {
vrele(dp);
error = namei_handle_root(ndp, &dp, 1);
if (error != 0)
goto out;
}
}
vput(ndp->ni_vp);
ndp->ni_vp = NULL;
vrele(ndp->ni_dvp);
out:
vrele(ndp->ni_rootdir);
MPASS(error != 0);
namei_cleanup_cnp(cnp);
nameicap_cleanup(ndp, true);
SDT_PROBE2(vfs, namei, lookup, return, error, NULL);
+ pwd_drop(pwd);
return (error);
}
static int
compute_cn_lkflags(struct mount *mp, int lkflags, int cnflags)
{
if (mp == NULL || ((lkflags & LK_SHARED) &&
(!(mp->mnt_kern_flag & MNTK_LOOKUP_SHARED) ||
((cnflags & ISDOTDOT) &&
(mp->mnt_kern_flag & MNTK_LOOKUP_EXCL_DOTDOT))))) {
lkflags &= ~LK_SHARED;
lkflags |= LK_EXCLUSIVE;
}
lkflags |= LK_NODDLKTREAT;
return (lkflags);
}
static __inline int
needs_exclusive_leaf(struct mount *mp, int flags)
{
/*
* Intermediate nodes can use shared locks, we only need to
* force an exclusive lock for leaf nodes.
*/
if ((flags & (ISLASTCN | LOCKLEAF)) != (ISLASTCN | LOCKLEAF))
return (0);
/* Always use exclusive locks if LOCKSHARED isn't set. */
if (!(flags & LOCKSHARED))
return (1);
/*
* For lookups during open(), if the mount point supports
* extended shared operations, then use a shared lock for the
* leaf node, otherwise use an exclusive lock.
*/
if ((flags & ISOPEN) != 0)
return (!MNT_EXTENDED_SHARED(mp));
/*
* Lookup requests outside of open() that specify LOCKSHARED
* only need a shared lock on the leaf vnode.
*/
return (0);
}
/*
* Search a pathname.
* This is a very central and rather complicated routine.
*
* The pathname is pointed to by ni_ptr and is of length ni_pathlen.
* The starting directory is taken from ni_startdir. The pathname is
* descended until done, or a symbolic link is encountered. The variable
* ni_more is clear if the path is completed; it is set to one if a
* symbolic link needing interpretation is encountered.
*
* The flag argument is LOOKUP, CREATE, RENAME, or DELETE depending on
* whether the name is to be looked up, created, renamed, or deleted.
* When CREATE, RENAME, or DELETE is specified, information usable in
* creating, renaming, or deleting a directory entry may be calculated.
* If flag has LOCKPARENT or'ed into it, the parent directory is returned
* locked. If flag has WANTPARENT or'ed into it, the parent directory is
* returned unlocked. Otherwise the parent directory is not returned. If
* the target of the pathname exists and LOCKLEAF is or'ed into the flag
* the target is returned locked, otherwise it is returned unlocked.
* When creating or renaming and LOCKPARENT is specified, the target may not
* be ".". When deleting and LOCKPARENT is specified, the target may be ".".
*
* Overall outline of lookup:
*
* dirloop:
* identify next component of name at ndp->ni_ptr
* handle degenerate case where name is null string
* if .. and crossing mount points and on mounted filesys, find parent
* call VOP_LOOKUP routine for next component name
* directory vnode returned in ni_dvp, unlocked unless LOCKPARENT set
* component vnode returned in ni_vp (if it exists), locked.
* if result vnode is mounted on and crossing mount points,
* find mounted on vnode
* if more components of name, do next level at dirloop
* return the answer in ni_vp, locked if LOCKLEAF set
* if LOCKPARENT set, return locked parent in ni_dvp
* if WANTPARENT set, return unlocked parent in ni_dvp
*/
int
lookup(struct nameidata *ndp)
{
char *cp; /* pointer into pathname argument */
char *prev_ni_next; /* saved ndp->ni_next */
struct vnode *dp = NULL; /* the directory we are searching */
struct vnode *tdp; /* saved dp */
struct mount *mp; /* mount table entry */
struct prison *pr;
size_t prev_ni_pathlen; /* saved ndp->ni_pathlen */
int docache; /* == 0 do not cache last component */
int wantparent; /* 1 => wantparent or lockparent flag */
int rdonly; /* lookup read-only flag bit */
int error = 0;
int dpunlocked = 0; /* dp has already been unlocked */
int relookup = 0; /* do not consume the path component */
struct componentname *cnp = &ndp->ni_cnd;
int lkflags_save;
int ni_dvp_unlocked;
/*
* Setup: break out flag bits into variables.
*/
ni_dvp_unlocked = 0;
wantparent = cnp->cn_flags & (LOCKPARENT | WANTPARENT);
KASSERT(cnp->cn_nameiop == LOOKUP || wantparent,
("CREATE, DELETE, RENAME require LOCKPARENT or WANTPARENT."));
docache = (cnp->cn_flags & NOCACHE) ^ NOCACHE;
if (cnp->cn_nameiop == DELETE ||
(wantparent && cnp->cn_nameiop != CREATE &&
cnp->cn_nameiop != LOOKUP))
docache = 0;
rdonly = cnp->cn_flags & RDONLY;
cnp->cn_flags &= ~ISSYMLINK;
ndp->ni_dvp = NULL;
/*
* We use shared locks until we hit the parent of the last cn then
* we adjust based on the requesting flags.
*/
cnp->cn_lkflags = LK_SHARED;
dp = ndp->ni_startdir;
ndp->ni_startdir = NULLVP;
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags | LK_RETRY,
cnp->cn_flags));
dirloop:
/*
* Search a new directory.
*
* The last component of the filename is left accessible via
* cnp->cn_nameptr for callers that need the name. Callers needing
* the name set the SAVENAME flag. When done, they assume
* responsibility for freeing the pathname buffer.
*/
for (cp = cnp->cn_nameptr; *cp != 0 && *cp != '/'; cp++)
continue;
cnp->cn_namelen = cp - cnp->cn_nameptr;
if (cnp->cn_namelen > NAME_MAX) {
error = ENAMETOOLONG;
goto bad;
}
#ifdef NAMEI_DIAGNOSTIC
{ char c = *cp;
*cp = '\0';
printf("{%s}: ", cnp->cn_nameptr);
*cp = c; }
#endif
prev_ni_pathlen = ndp->ni_pathlen;
ndp->ni_pathlen -= cnp->cn_namelen;
KASSERT(ndp->ni_pathlen <= PATH_MAX,
("%s: ni_pathlen underflow to %zd\n", __func__, ndp->ni_pathlen));
prev_ni_next = ndp->ni_next;
ndp->ni_next = cp;
/*
* Replace multiple slashes by a single slash and trailing slashes
* by a null. This must be done before VOP_LOOKUP() because some
* fs's don't know about trailing slashes. Remember if there were
* trailing slashes to handle symlinks, existing non-directories
* and non-existing files that won't be directories specially later.
*/
while (*cp == '/' && (cp[1] == '/' || cp[1] == '\0')) {
cp++;
ndp->ni_pathlen--;
if (*cp == '\0') {
*ndp->ni_next = '\0';
cnp->cn_flags |= TRAILINGSLASH;
}
}
ndp->ni_next = cp;
cnp->cn_flags |= MAKEENTRY;
if (*cp == '\0' && docache == 0)
cnp->cn_flags &= ~MAKEENTRY;
if (cnp->cn_namelen == 2 &&
cnp->cn_nameptr[1] == '.' && cnp->cn_nameptr[0] == '.')
cnp->cn_flags |= ISDOTDOT;
else
cnp->cn_flags &= ~ISDOTDOT;
if (*ndp->ni_next == 0)
cnp->cn_flags |= ISLASTCN;
else
cnp->cn_flags &= ~ISLASTCN;
if ((cnp->cn_flags & ISLASTCN) != 0 &&
cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.' &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EINVAL;
goto bad;
}
nameicap_tracker_add(ndp, dp);
/*
* Check for degenerate name (e.g. / or "")
* which is a way of talking about a directory,
* e.g. like "/." or ".".
*/
if (cnp->cn_nameptr[0] == '\0') {
if (dp->v_type != VDIR) {
error = ENOTDIR;
goto bad;
}
if (cnp->cn_nameiop != LOOKUP) {
error = EISDIR;
goto bad;
}
if (wantparent) {
ndp->ni_dvp = dp;
VREF(dp);
}
ndp->ni_vp = dp;
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_VNODE1(dp);
else if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_VNODE2(dp);
if (!(cnp->cn_flags & (LOCKPARENT | LOCKLEAF)))
VOP_UNLOCK(dp);
/* XXX This should probably move to the top of function. */
if (cnp->cn_flags & SAVESTART)
panic("lookup: SAVESTART");
goto success;
}
/*
* Handle "..": five special cases.
* 0. If doing a capability lookup and lookup_cap_dotdot is
* disabled, return ENOTCAPABLE.
* 1. Return an error if this is the last component of
* the name and the operation is DELETE or RENAME.
* 2. If at root directory (e.g. after chroot)
* or at absolute root directory
* then ignore it so can't get out.
* 3. If this vnode is the root of a mounted
* filesystem, then replace it with the
* vnode which was mounted on so we take the
* .. in the other filesystem.
* 4. If the vnode is the top directory of
* the jail or chroot, don't let them out.
* 5. If doing a capability lookup and lookup_cap_dotdot is
* enabled, return ENOTCAPABLE if the lookup would escape
* from the initial file descriptor directory. Checks are
* done by ensuring that namei() already traversed the
* result of dotdot lookup.
*/
if (cnp->cn_flags & ISDOTDOT) {
if ((ndp->ni_lcf & (NI_LCF_STRICTRELATIVE | NI_LCF_CAP_DOTDOT))
== NI_LCF_STRICTRELATIVE) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
error = ENOTCAPABLE;
goto bad;
}
if ((cnp->cn_flags & ISLASTCN) != 0 &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EINVAL;
goto bad;
}
for (;;) {
for (pr = cnp->cn_cred->cr_prison; pr != NULL;
pr = pr->pr_parent)
if (dp == pr->pr_root)
break;
if (dp == ndp->ni_rootdir ||
dp == ndp->ni_topdir ||
dp == rootvnode ||
pr != NULL ||
((dp->v_vflag & VV_ROOT) != 0 &&
(cnp->cn_flags & NOCROSSMOUNT) != 0)) {
ndp->ni_dvp = dp;
ndp->ni_vp = dp;
VREF(dp);
goto nextname;
}
if ((dp->v_vflag & VV_ROOT) == 0)
break;
if (VN_IS_DOOMED(dp)) { /* forced unmount */
error = ENOENT;
goto bad;
}
tdp = dp;
dp = dp->v_mount->mnt_vnodecovered;
VREF(dp);
vput(tdp);
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags |
LK_RETRY, ISDOTDOT));
error = nameicap_check_dotdot(ndp, dp);
if (error != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
goto bad;
}
}
}
/*
* We now have a segment name to search for, and a directory to search.
*/
unionlookup:
#ifdef MAC
error = mac_vnode_check_lookup(cnp->cn_thread->td_ucred, dp, cnp);
if (error)
goto bad;
#endif
ndp->ni_dvp = dp;
ndp->ni_vp = NULL;
ASSERT_VOP_LOCKED(dp, "lookup");
/*
* If we have a shared lock we may need to upgrade the lock for the
* last operation.
*/
if ((cnp->cn_flags & LOCKPARENT) && (cnp->cn_flags & ISLASTCN) &&
dp != vp_crossmp && VOP_ISLOCKED(dp) == LK_SHARED)
vn_lock(dp, LK_UPGRADE|LK_RETRY);
if (VN_IS_DOOMED(dp)) {
error = ENOENT;
goto bad;
}
/*
* If we're looking up the last component and we need an exclusive
* lock, adjust our lkflags.
*/
if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags))
cnp->cn_lkflags = LK_EXCLUSIVE;
#ifdef NAMEI_DIAGNOSTIC
vn_printf(dp, "lookup in ");
#endif
lkflags_save = cnp->cn_lkflags;
cnp->cn_lkflags = compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags,
cnp->cn_flags);
error = VOP_LOOKUP(dp, &ndp->ni_vp, cnp);
cnp->cn_lkflags = lkflags_save;
if (error != 0) {
KASSERT(ndp->ni_vp == NULL, ("leaf should be empty"));
#ifdef NAMEI_DIAGNOSTIC
printf("not found\n");
#endif
if ((error == ENOENT) &&
(dp->v_vflag & VV_ROOT) && (dp->v_mount != NULL) &&
(dp->v_mount->mnt_flag & MNT_UNION)) {
tdp = dp;
dp = dp->v_mount->mnt_vnodecovered;
VREF(dp);
vput(tdp);
vn_lock(dp,
compute_cn_lkflags(dp->v_mount, cnp->cn_lkflags |
LK_RETRY, cnp->cn_flags));
nameicap_tracker_add(ndp, dp);
goto unionlookup;
}
if (error == ERELOOKUP) {
vref(dp);
ndp->ni_vp = dp;
error = 0;
relookup = 1;
goto good;
}
if (error != EJUSTRETURN)
goto bad;
/*
* At this point, we know we're at the end of the
* pathname. If creating / renaming, we can consider
* allowing the file or directory to be created / renamed,
* provided we're not on a read-only filesystem.
*/
if (rdonly) {
error = EROFS;
goto bad;
}
/* trailing slash only allowed for directories */
if ((cnp->cn_flags & TRAILINGSLASH) &&
!(cnp->cn_flags & WILLBEDIR)) {
error = ENOENT;
goto bad;
}
if ((cnp->cn_flags & LOCKPARENT) == 0)
VOP_UNLOCK(dp);
/*
* We return with ni_vp NULL to indicate that the entry
* doesn't currently exist, leaving a pointer to the
* (possibly locked) directory vnode in ndp->ni_dvp.
*/
if (cnp->cn_flags & SAVESTART) {
ndp->ni_startdir = ndp->ni_dvp;
VREF(ndp->ni_startdir);
}
goto success;
}
good:
#ifdef NAMEI_DIAGNOSTIC
printf("found\n");
#endif
dp = ndp->ni_vp;
/*
* Check to see if the vnode has been mounted on;
* if so find the root of the mounted filesystem.
*/
while (dp->v_type == VDIR && (mp = dp->v_mountedhere) &&
(cnp->cn_flags & NOCROSSMOUNT) == 0) {
if (vfs_busy(mp, 0))
continue;
vput(dp);
if (dp != ndp->ni_dvp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
vrefact(vp_crossmp);
ndp->ni_dvp = vp_crossmp;
error = VFS_ROOT(mp, compute_cn_lkflags(mp, cnp->cn_lkflags,
cnp->cn_flags), &tdp);
vfs_unbusy(mp);
if (vn_lock(vp_crossmp, LK_SHARED | LK_NOWAIT))
panic("vp_crossmp exclusively locked or reclaimed");
if (error) {
dpunlocked = 1;
goto bad2;
}
ndp->ni_vp = dp = tdp;
}
/*
* Check for symbolic link
*/
if ((dp->v_type == VLNK) &&
((cnp->cn_flags & FOLLOW) || (cnp->cn_flags & TRAILINGSLASH) ||
*ndp->ni_next == '/')) {
cnp->cn_flags |= ISSYMLINK;
if (VN_IS_DOOMED(dp)) {
/*
* We can't know whether the directory was mounted with
* NOSYMFOLLOW, so we can't follow safely.
*/
error = ENOENT;
goto bad2;
}
if (dp->v_mount->mnt_flag & MNT_NOSYMFOLLOW) {
error = EACCES;
goto bad2;
}
/*
* Symlink code always expects an unlocked dvp.
*/
if (ndp->ni_dvp != ndp->ni_vp) {
VOP_UNLOCK(ndp->ni_dvp);
ni_dvp_unlocked = 1;
}
goto success;
}
nextname:
/*
* Not a symbolic link that we will follow. Continue with the
* next component if there is any; otherwise, we're done.
*/
KASSERT((cnp->cn_flags & ISLASTCN) || *ndp->ni_next == '/',
("lookup: invalid path state."));
if (relookup) {
relookup = 0;
ndp->ni_pathlen = prev_ni_pathlen;
ndp->ni_next = prev_ni_next;
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
goto dirloop;
}
if (cnp->cn_flags & ISDOTDOT) {
error = nameicap_check_dotdot(ndp, ndp->ni_vp);
if (error != 0) {
#ifdef KTRACE
if (KTRPOINT(curthread, KTR_CAPFAIL))
ktrcapfail(CAPFAIL_LOOKUP, NULL, NULL);
#endif
goto bad2;
}
}
if (*ndp->ni_next == '/') {
cnp->cn_nameptr = ndp->ni_next;
while (*cnp->cn_nameptr == '/') {
cnp->cn_nameptr++;
ndp->ni_pathlen--;
}
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
goto dirloop;
}
/*
* If we're processing a path with a trailing slash,
* check that the end result is a directory.
*/
if ((cnp->cn_flags & TRAILINGSLASH) && dp->v_type != VDIR) {
error = ENOTDIR;
goto bad2;
}
/*
* Disallow directory write attempts on read-only filesystems.
*/
if (rdonly &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
error = EROFS;
goto bad2;
}
if (cnp->cn_flags & SAVESTART) {
ndp->ni_startdir = ndp->ni_dvp;
VREF(ndp->ni_startdir);
}
if (!wantparent) {
ni_dvp_unlocked = 2;
if (ndp->ni_dvp != dp)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
} else if ((cnp->cn_flags & LOCKPARENT) == 0 && ndp->ni_dvp != dp) {
VOP_UNLOCK(ndp->ni_dvp);
ni_dvp_unlocked = 1;
}
if (cnp->cn_flags & AUDITVNODE1)
AUDIT_ARG_VNODE1(dp);
else if (cnp->cn_flags & AUDITVNODE2)
AUDIT_ARG_VNODE2(dp);
if ((cnp->cn_flags & LOCKLEAF) == 0)
VOP_UNLOCK(dp);
success:
/*
* Because of shared lookup we may have the vnode shared locked, but
* the caller may want it to be exclusively locked.
*/
if (needs_exclusive_leaf(dp->v_mount, cnp->cn_flags) &&
VOP_ISLOCKED(dp) != LK_EXCLUSIVE) {
vn_lock(dp, LK_UPGRADE | LK_RETRY);
if (VN_IS_DOOMED(dp)) {
error = ENOENT;
goto bad2;
}
}
return (0);
bad2:
if (ni_dvp_unlocked != 2) {
if (dp != ndp->ni_dvp && !ni_dvp_unlocked)
vput(ndp->ni_dvp);
else
vrele(ndp->ni_dvp);
}
bad:
if (!dpunlocked)
vput(dp);
ndp->ni_vp = NULL;
return (error);
}
/*
* relookup - lookup a path name component
* Used by lookup to re-acquire things.
*/
int
relookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp)
{
struct vnode *dp = NULL; /* the directory we are searching */
int wantparent; /* 1 => wantparent or lockparent flag */
int rdonly; /* lookup read-only flag bit */
int error = 0;
KASSERT(cnp->cn_flags & ISLASTCN,
("relookup: Not given last component."));
/*
* Setup: break out flag bits into variables.
*/
wantparent = cnp->cn_flags & (LOCKPARENT|WANTPARENT);
KASSERT(wantparent, ("relookup: parent not wanted."));
rdonly = cnp->cn_flags & RDONLY;
cnp->cn_flags &= ~ISSYMLINK;
dp = dvp;
cnp->cn_lkflags = LK_EXCLUSIVE;
vn_lock(dp, LK_EXCLUSIVE | LK_RETRY);
/*
* Search a new directory.
*
* The last component of the filename is left accessible via
* cnp->cn_nameptr for callers that need the name. Callers needing
* the name set the SAVENAME flag. When done, they assume
* responsibility for freeing the pathname buffer.
*/
#ifdef NAMEI_DIAGNOSTIC
printf("{%s}: ", cnp->cn_nameptr);
#endif
/*
* Check for "" which represents the root directory after slash
* removal.
*/
if (cnp->cn_nameptr[0] == '\0') {
/*
* Support only LOOKUP for "/" because lookup()
* can't succeed for CREATE, DELETE and RENAME.
*/
KASSERT(cnp->cn_nameiop == LOOKUP, ("nameiop must be LOOKUP"));
KASSERT(dp->v_type == VDIR, ("dp is not a directory"));
if (!(cnp->cn_flags & LOCKLEAF))
VOP_UNLOCK(dp);
*vpp = dp;
/* XXX This should probably move to the top of function. */
if (cnp->cn_flags & SAVESTART)
panic("lookup: SAVESTART");
return (0);
}
if (cnp->cn_flags & ISDOTDOT)
panic ("relookup: lookup on dot-dot");
/*
* We now have a segment name to search for, and a directory to search.
*/
#ifdef NAMEI_DIAGNOSTIC
vn_printf(dp, "search in ");
#endif
if ((error = VOP_LOOKUP(dp, vpp, cnp)) != 0) {
KASSERT(*vpp == NULL, ("leaf should be empty"));
if (error != EJUSTRETURN)
goto bad;
/*
* If creating and at end of pathname, then can consider
* allowing file to be created.
*/
if (rdonly) {
error = EROFS;
goto bad;
}
/* ASSERT(dvp == ndp->ni_startdir) */
if (cnp->cn_flags & SAVESTART)
VREF(dvp);
if ((cnp->cn_flags & LOCKPARENT) == 0)
VOP_UNLOCK(dp);
/*
* We return with ni_vp NULL to indicate that the entry
* doesn't currently exist, leaving a pointer to the
* (possibly locked) directory vnode in ndp->ni_dvp.
*/
return (0);
}
dp = *vpp;
/*
* Disallow directory write attempts on read-only filesystems.
*/
if (rdonly &&
(cnp->cn_nameiop == DELETE || cnp->cn_nameiop == RENAME)) {
if (dvp == dp)
vrele(dvp);
else
vput(dvp);
error = EROFS;
goto bad;
}
/*
* Set the parent lock/ref state to the requested state.
*/
if ((cnp->cn_flags & LOCKPARENT) == 0 && dvp != dp) {
if (wantparent)
VOP_UNLOCK(dvp);
else
vput(dvp);
} else if (!wantparent)
vrele(dvp);
/*
* Check for symbolic link
*/
KASSERT(dp->v_type != VLNK || !(cnp->cn_flags & FOLLOW),
("relookup: symlink found.\n"));
/* ASSERT(dvp == ndp->ni_startdir) */
if (cnp->cn_flags & SAVESTART)
VREF(dvp);
if ((cnp->cn_flags & LOCKLEAF) == 0)
VOP_UNLOCK(dp);
return (0);
bad:
vput(dp);
*vpp = NULL;
return (error);
}
void
NDINIT_ALL(struct nameidata *ndp, u_long op, u_long flags, enum uio_seg segflg,
const char *namep, int dirfd, struct vnode *startdir, cap_rights_t *rightsp,
struct thread *td)
{
ndp->ni_cnd.cn_nameiop = op;
ndp->ni_cnd.cn_flags = flags;
ndp->ni_segflg = segflg;
ndp->ni_dirp = namep;
ndp->ni_dirfd = dirfd;
ndp->ni_startdir = startdir;
ndp->ni_resflags = 0;
filecaps_init(&ndp->ni_filecaps);
ndp->ni_cnd.cn_thread = td;
if (rightsp != NULL)
ndp->ni_rightsneeded = *rightsp;
else
cap_rights_init_zero(&ndp->ni_rightsneeded);
}
/*
* Free data allocated by namei(); see namei(9) for details.
*/
void
NDFREE(struct nameidata *ndp, const u_int flags)
{
int unlock_dvp;
int unlock_vp;
unlock_dvp = 0;
unlock_vp = 0;
if (!(flags & NDF_NO_FREE_PNBUF) &&
(ndp->ni_cnd.cn_flags & HASBUF)) {
uma_zfree(namei_zone, ndp->ni_cnd.cn_pnbuf);
ndp->ni_cnd.cn_flags &= ~HASBUF;
}
if (!(flags & NDF_NO_VP_UNLOCK) &&
(ndp->ni_cnd.cn_flags & LOCKLEAF) && ndp->ni_vp)
unlock_vp = 1;
if (!(flags & NDF_NO_DVP_UNLOCK) &&
(ndp->ni_cnd.cn_flags & LOCKPARENT) &&
ndp->ni_dvp != ndp->ni_vp)
unlock_dvp = 1;
if (!(flags & NDF_NO_VP_RELE) && ndp->ni_vp) {
if (unlock_vp) {
vput(ndp->ni_vp);
unlock_vp = 0;
} else
vrele(ndp->ni_vp);
ndp->ni_vp = NULL;
}
if (unlock_vp)
VOP_UNLOCK(ndp->ni_vp);
if (!(flags & NDF_NO_DVP_RELE) &&
(ndp->ni_cnd.cn_flags & (LOCKPARENT|WANTPARENT))) {
if (unlock_dvp) {
vput(ndp->ni_dvp);
unlock_dvp = 0;
} else
vrele(ndp->ni_dvp);
ndp->ni_dvp = NULL;
}
if (unlock_dvp)
VOP_UNLOCK(ndp->ni_dvp);
if (!(flags & NDF_NO_STARTDIR_RELE) &&
(ndp->ni_cnd.cn_flags & SAVESTART)) {
vrele(ndp->ni_startdir);
ndp->ni_startdir = NULL;
}
}
/*
* Determine if there is a suitable alternate filename under the specified
* prefix for the specified path. If the create flag is set, then the
* alternate prefix will be used so long as the parent directory exists.
* This is used by the various compatibility ABIs so that Linux binaries prefer
* files under /compat/linux for example. The chosen path (whether under
* the prefix or under /) is returned in a kernel malloc'd buffer pointed
* to by pathbuf. The caller is responsible for free'ing the buffer from
* the M_TEMP bucket if one is returned.
*/
int
kern_alternate_path(struct thread *td, const char *prefix, const char *path,
enum uio_seg pathseg, char **pathbuf, int create, int dirfd)
{
struct nameidata nd, ndroot;
char *ptr, *buf, *cp;
size_t len, sz;
int error;
buf = (char *) malloc(MAXPATHLEN, M_TEMP, M_WAITOK);
*pathbuf = buf;
/* Copy the prefix into the new pathname as a starting point. */
len = strlcpy(buf, prefix, MAXPATHLEN);
if (len >= MAXPATHLEN) {
*pathbuf = NULL;
free(buf, M_TEMP);
return (EINVAL);
}
sz = MAXPATHLEN - len;
ptr = buf + len;
/* Append the filename to the prefix. */
if (pathseg == UIO_SYSSPACE)
error = copystr(path, ptr, sz, &len);
else
error = copyinstr(path, ptr, sz, &len);
if (error) {
*pathbuf = NULL;
free(buf, M_TEMP);
return (error);
}
/* Only use a prefix with absolute pathnames. */
if (*ptr != '/') {
error = EINVAL;
goto keeporig;
}
if (dirfd != AT_FDCWD) {
/*
* We want the original because the "prefix" is
* included in the already opened dirfd.
*/
bcopy(ptr, buf, len);
return (0);
}
/*
* We know that there is a / somewhere in this pathname.
* Search backwards for it, to find the file's parent dir
* to see if it exists in the alternate tree. If it does,
* and we want to create a file (cflag is set). We don't
* need to worry about the root comparison in this case.
*/
if (create) {
for (cp = &ptr[len] - 1; *cp != '/'; cp--);
*cp = '\0';
NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);
error = namei(&nd);
*cp = '/';
if (error != 0)
goto keeporig;
} else {
NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, buf, td);
error = namei(&nd);
if (error != 0)
goto keeporig;
/*
* We now compare the vnode of the prefix to the one
* vnode asked. If they resolve to be the same, then we
* ignore the match so that the real root gets used.
* This avoids the problem of traversing "../.." to find the
* root directory and never finding it, because "/" resolves
* to the emulation root directory. This is expensive :-(
*/
NDINIT(&ndroot, LOOKUP, FOLLOW, UIO_SYSSPACE, prefix,
td);
/* We shouldn't ever get an error from this namei(). */
error = namei(&ndroot);
if (error == 0) {
if (nd.ni_vp == ndroot.ni_vp)
error = ENOENT;
NDFREE(&ndroot, NDF_ONLY_PNBUF);
vrele(ndroot.ni_vp);
}
}
NDFREE(&nd, NDF_ONLY_PNBUF);
vrele(nd.ni_vp);
keeporig:
/* If there was an error, use the original path name. */
if (error)
bcopy(ptr, buf, len);
return (error);
}
Index: head/sys/kern/vfs_mountroot.c
===================================================================
--- head/sys/kern/vfs_mountroot.c (revision 358502)
+++ head/sys/kern/vfs_mountroot.c (revision 358503)
@@ -1,1167 +1,1153 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2010 Marcel Moolenaar
* Copyright (c) 1999-2004 Poul-Henning Kamp
* Copyright (c) 1999 Michael Smith
* Copyright (c) 1989, 1993
* The Regents of the University of California. All rights reserved.
* (c) UNIX System Laboratories, Inc.
* All or some portions of this file are derived from material licensed
* to the University of California by American Telephone and Telegraph
* Co. or Unix System Laboratories, Inc. and are reproduced herein with
* the permission of UNIX System Laboratories, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "opt_rootdevname.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/eventhandler.h>
#include <sys/fcntl.h>
#include <sys/jail.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mdioctl.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/filedesc.h>
#include <sys/reboot.h>
#include <sys/sbuf.h>
#include <sys/stat.h>
#include <sys/syscallsubr.h>
#include <sys/sysproto.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/systm.h>
#include <sys/vnode.h>
#include <geom/geom.h>
/*
* The root filesystem is detailed in the kernel environment variable
* vfs.root.mountfrom, which is expected to be in the general format
*
* <vfsname>:[<path>][ <vfsname>:[<path>] ...]
* vfsname := the name of a VFS known to the kernel and capable
* of being mounted as root
* path := disk device name or other data used by the filesystem
* to locate its physical store
*
* If the environment variable vfs.root.mountfrom is a space separated list,
* each list element is tried in turn and the root filesystem will be mounted
* from the first one that succeeds.
*
* The environment variable vfs.root.mountfrom.options is a comma delimited
* set of string mount options. These mount options must be parseable
* by nmount() in the kernel.
*/
static int parse_mount(char **);
static struct mntarg *parse_mountroot_options(struct mntarg *, const char *);
static int sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS);
static void vfs_mountroot_wait(void);
static int vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev);
/*
* The vnode of the system's root (/ in the filesystem, without chroot
* active.)
*/
struct vnode *rootvnode;
/*
* Mount of the system's /dev.
*/
struct mount *rootdevmp;
char *rootdevnames[2] = {NULL, NULL};
struct mtx root_holds_mtx;
MTX_SYSINIT(root_holds, &root_holds_mtx, "root_holds", MTX_DEF);
static TAILQ_HEAD(, root_hold_token) root_holds =
TAILQ_HEAD_INITIALIZER(root_holds);
enum action {
A_CONTINUE,
A_PANIC,
A_REBOOT,
A_RETRY
};
enum rh_flags {
RH_FREE,
RH_ALLOC,
RH_ARG,
};
static enum action root_mount_onfail = A_CONTINUE;
static int root_mount_mddev;
static int root_mount_complete;
/* By default wait up to 3 seconds for devices to appear. */
static int root_mount_timeout = 3;
TUNABLE_INT("vfs.mountroot.timeout", &root_mount_timeout);
static int root_mount_always_wait = 0;
SYSCTL_INT(_vfs, OID_AUTO, root_mount_always_wait, CTLFLAG_RDTUN,
&root_mount_always_wait, 0,
"Wait for root mount holds even if the root device already exists");
SYSCTL_PROC(_vfs, OID_AUTO, root_mount_hold,
CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
NULL, 0, sysctl_vfs_root_mount_hold, "A",
"List of root mount hold tokens");
static int
sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS)
{
struct sbuf sb;
struct root_hold_token *h;
int error;
sbuf_new(&sb, NULL, 256, SBUF_AUTOEXTEND | SBUF_INCLUDENUL);
mtx_lock(&root_holds_mtx);
TAILQ_FOREACH(h, &root_holds, list) {
if (h != TAILQ_FIRST(&root_holds))
sbuf_putc(&sb, ' ');
sbuf_printf(&sb, "%s", h->who);
}
mtx_unlock(&root_holds_mtx);
error = sbuf_finish(&sb);
if (error == 0)
error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
sbuf_delete(&sb);
return (error);
}
struct root_hold_token *
root_mount_hold(const char *identifier)
{
struct root_hold_token *h;
h = malloc(sizeof *h, M_DEVBUF, M_ZERO | M_WAITOK);
h->flags = RH_ALLOC;
h->who = identifier;
mtx_lock(&root_holds_mtx);
TSHOLD("root mount");
TAILQ_INSERT_TAIL(&root_holds, h, list);
mtx_unlock(&root_holds_mtx);
return (h);
}
void
root_mount_hold_token(const char *identifier, struct root_hold_token *h)
{
#ifdef INVARIANTS
struct root_hold_token *t;
#endif
h->flags = RH_ARG;
h->who = identifier;
mtx_lock(&root_holds_mtx);
#ifdef INVARIANTS
TAILQ_FOREACH(t, &root_holds, list) {
if (t == h) {
panic("Duplicate mount hold by '%s' on %p",
identifier, h);
}
}
#endif
TSHOLD("root mount");
TAILQ_INSERT_TAIL(&root_holds, h, list);
mtx_unlock(&root_holds_mtx);
}
void
root_mount_rel(struct root_hold_token *h)
{
if (h == NULL || h->flags == RH_FREE)
return;
mtx_lock(&root_holds_mtx);
TAILQ_REMOVE(&root_holds, h, list);
TSRELEASE("root mount");
wakeup(&root_holds);
mtx_unlock(&root_holds_mtx);
if (h->flags == RH_ALLOC) {
free(h, M_DEVBUF);
} else
h->flags = RH_FREE;
}
int
root_mounted(void)
{
/* No mutex is acquired here because int stores are atomic. */
return (root_mount_complete);
}
static void
set_rootvnode(void)
{
- struct proc *p;
if (VFS_ROOT(TAILQ_FIRST(&mountlist), LK_EXCLUSIVE, &rootvnode))
panic("set_rootvnode: Cannot find root vnode");
VOP_UNLOCK(rootvnode);
- p = curthread->td_proc;
- FILEDESC_XLOCK(p->p_fd);
-
- if (p->p_fd->fd_cdir != NULL)
- vrele(p->p_fd->fd_cdir);
- p->p_fd->fd_cdir = rootvnode;
- VREF(rootvnode);
-
- if (p->p_fd->fd_rdir != NULL)
- vrele(p->p_fd->fd_rdir);
- p->p_fd->fd_rdir = rootvnode;
- VREF(rootvnode);
-
- FILEDESC_XUNLOCK(p->p_fd);
+ pwd_ensure_dirs();
}
static int
vfs_mountroot_devfs(struct thread *td, struct mount **mpp)
{
struct vfsoptlist *opts;
struct vfsconf *vfsp;
struct mount *mp;
int error;
*mpp = NULL;
if (rootdevmp != NULL) {
/*
* Already have /dev; this happens during rerooting.
*/
error = vfs_busy(rootdevmp, 0);
if (error != 0)
return (error);
*mpp = rootdevmp;
} else {
vfsp = vfs_byname("devfs");
KASSERT(vfsp != NULL, ("Could not find devfs by name"));
if (vfsp == NULL)
return (ENOENT);
mp = vfs_mount_alloc(NULLVP, vfsp, "/dev", td->td_ucred);
error = VFS_MOUNT(mp);
KASSERT(error == 0, ("VFS_MOUNT(devfs) failed %d", error));
if (error)
return (error);
error = VFS_STATFS(mp, &mp->mnt_stat);
KASSERT(error == 0, ("VFS_STATFS(devfs) failed %d", error));
if (error)
return (error);
opts = malloc(sizeof(struct vfsoptlist), M_MOUNT, M_WAITOK);
TAILQ_INIT(opts);
mp->mnt_opt = opts;
mtx_lock(&mountlist_mtx);
TAILQ_INSERT_HEAD(&mountlist, mp, mnt_list);
mtx_unlock(&mountlist_mtx);
*mpp = mp;
rootdevmp = mp;
vfs_op_exit(mp);
}
set_rootvnode();
error = kern_symlinkat(td, "/", AT_FDCWD, "dev", UIO_SYSSPACE);
if (error)
printf("kern_symlink /dev -> / returns %d\n", error);
return (error);
}
static void
vfs_mountroot_shuffle(struct thread *td, struct mount *mpdevfs)
{
struct nameidata nd;
struct mount *mporoot, *mpnroot;
struct vnode *vp, *vporoot, *vpdevfs;
char *fspath;
int error;
mpnroot = TAILQ_NEXT(mpdevfs, mnt_list);
/* Shuffle the mountlist. */
mtx_lock(&mountlist_mtx);
mporoot = TAILQ_FIRST(&mountlist);
TAILQ_REMOVE(&mountlist, mpdevfs, mnt_list);
if (mporoot != mpdevfs) {
TAILQ_REMOVE(&mountlist, mpnroot, mnt_list);
TAILQ_INSERT_HEAD(&mountlist, mpnroot, mnt_list);
}
TAILQ_INSERT_TAIL(&mountlist, mpdevfs, mnt_list);
mtx_unlock(&mountlist_mtx);
cache_purgevfs(mporoot, true);
if (mporoot != mpdevfs)
cache_purgevfs(mpdevfs, true);
if (VFS_ROOT(mporoot, LK_EXCLUSIVE, &vporoot))
panic("vfs_mountroot_shuffle: Cannot find root vnode");
VI_LOCK(vporoot);
vporoot->v_iflag &= ~VI_MOUNT;
VI_UNLOCK(vporoot);
vporoot->v_mountedhere = NULL;
mporoot->mnt_flag &= ~MNT_ROOTFS;
mporoot->mnt_vnodecovered = NULL;
vput(vporoot);
/* Set up the new rootvnode, and purge the cache */
mpnroot->mnt_vnodecovered = NULL;
set_rootvnode();
cache_purgevfs(rootvnode->v_mount, true);
if (mporoot != mpdevfs) {
/* Remount old root under /.mount or /mnt */
fspath = "/.mount";
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE,
fspath, td);
error = namei(&nd);
if (error) {
NDFREE(&nd, NDF_ONLY_PNBUF);
fspath = "/mnt";
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE,
fspath, td);
error = namei(&nd);
}
if (!error) {
vp = nd.ni_vp;
error = (vp->v_type == VDIR) ? 0 : ENOTDIR;
if (!error)
error = vinvalbuf(vp, V_SAVE, 0, 0);
if (!error) {
cache_purge(vp);
mporoot->mnt_vnodecovered = vp;
vp->v_mountedhere = mporoot;
strlcpy(mporoot->mnt_stat.f_mntonname,
fspath, MNAMELEN);
VOP_UNLOCK(vp);
} else
vput(vp);
}
NDFREE(&nd, NDF_ONLY_PNBUF);
if (error)
printf("mountroot: unable to remount previous root "
"under /.mount or /mnt (error %d)\n", error);
}
/* Remount devfs under /dev */
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, "/dev", td);
error = namei(&nd);
if (!error) {
vp = nd.ni_vp;
error = (vp->v_type == VDIR) ? 0 : ENOTDIR;
if (!error)
error = vinvalbuf(vp, V_SAVE, 0, 0);
if (!error) {
vpdevfs = mpdevfs->mnt_vnodecovered;
if (vpdevfs != NULL) {
cache_purge(vpdevfs);
vpdevfs->v_mountedhere = NULL;
vrele(vpdevfs);
}
mpdevfs->mnt_vnodecovered = vp;
vp->v_mountedhere = mpdevfs;
VOP_UNLOCK(vp);
} else
vput(vp);
}
if (error)
printf("mountroot: unable to remount devfs under /dev "
"(error %d)\n", error);
NDFREE(&nd, NDF_ONLY_PNBUF);
if (mporoot == mpdevfs) {
vfs_unbusy(mpdevfs);
/* Unlink the no longer needed /dev/dev -> / symlink */
error = kern_funlinkat(td, AT_FDCWD, "/dev/dev", FD_NONE,
UIO_SYSSPACE, 0, 0);
if (error)
printf("mountroot: unable to unlink /dev/dev "
"(error %d)\n", error);
}
}
/*
* Configuration parser.
*/
/* Parser character classes. */
#define CC_WHITESPACE -1
#define CC_NONWHITESPACE -2
/* Parse errors. */
#define PE_EOF -1
#define PE_EOL -2
static __inline int
parse_peek(char **conf)
{
return (**conf);
}
static __inline void
parse_poke(char **conf, int c)
{
**conf = c;
}
static __inline void
parse_advance(char **conf)
{
(*conf)++;
}
static int
parse_skipto(char **conf, int mc)
{
int c, match;
while (1) {
c = parse_peek(conf);
if (c == 0)
return (PE_EOF);
switch (mc) {
case CC_WHITESPACE:
match = (c == ' ' || c == '\t' || c == '\n') ? 1 : 0;
break;
case CC_NONWHITESPACE:
if (c == '\n')
return (PE_EOL);
match = (c != ' ' && c != '\t') ? 1 : 0;
break;
default:
match = (c == mc) ? 1 : 0;
break;
}
if (match)
break;
parse_advance(conf);
}
return (0);
}
static int
parse_token(char **conf, char **tok)
{
char *p;
size_t len;
int error;
*tok = NULL;
error = parse_skipto(conf, CC_NONWHITESPACE);
if (error)
return (error);
p = *conf;
error = parse_skipto(conf, CC_WHITESPACE);
len = *conf - p;
*tok = malloc(len + 1, M_TEMP, M_WAITOK | M_ZERO);
bcopy(p, *tok, len);
return (0);
}
static void
parse_dir_ask_printenv(const char *var)
{
char *val;
val = kern_getenv(var);
if (val != NULL) {
printf(" %s=%s\n", var, val);
freeenv(val);
}
}
static int
parse_dir_ask(char **conf)
{
char name[80];
char *mnt;
int error;
vfs_mountroot_wait();
printf("\nLoader variables:\n");
parse_dir_ask_printenv("vfs.root.mountfrom");
parse_dir_ask_printenv("vfs.root.mountfrom.options");
printf("\nManual root filesystem specification:\n");
printf(" <fstype>:<device> [options]\n");
printf(" Mount <device> using filesystem <fstype>\n");
printf(" and with the specified (optional) option list.\n");
printf("\n");
printf(" eg. ufs:/dev/da0s1a\n");
printf(" zfs:zroot/ROOT/default\n");
printf(" cd9660:/dev/cd0 ro\n");
printf(" (which is equivalent to: ");
printf("mount -t cd9660 -o ro /dev/cd0 /)\n");
printf("\n");
printf(" ? List valid disk boot devices\n");
printf(" . Yield 1 second (for background tasks)\n");
printf(" <empty line> Abort manual input\n");
do {
error = EINVAL;
printf("\nmountroot> ");
cngets(name, sizeof(name), GETS_ECHO);
if (name[0] == '\0')
break;
if (name[0] == '?' && name[1] == '\0') {
printf("\nList of GEOM managed disk devices:\n ");
g_dev_print();
continue;
}
if (name[0] == '.' && name[1] == '\0') {
pause("rmask", hz);
continue;
}
mnt = name;
error = parse_mount(&mnt);
if (error == -1)
printf("Invalid file system specification.\n");
} while (error != 0);
return (error);
}
static int
parse_dir_md(char **conf)
{
struct stat sb;
struct thread *td;
struct md_ioctl *mdio;
char *path, *tok;
int error, fd, len;
td = curthread;
error = parse_token(conf, &tok);
if (error)
return (error);
len = strlen(tok);
mdio = malloc(sizeof(*mdio) + len + 1, M_TEMP, M_WAITOK | M_ZERO);
path = (void *)(mdio + 1);
bcopy(tok, path, len);
free(tok, M_TEMP);
/* Get file status. */
error = kern_statat(td, 0, AT_FDCWD, path, UIO_SYSSPACE, &sb, NULL);
if (error)
goto out;
/* Open /dev/mdctl so that we can attach/detach. */
error = kern_openat(td, AT_FDCWD, "/dev/" MDCTL_NAME, UIO_SYSSPACE,
O_RDWR, 0);
if (error)
goto out;
fd = td->td_retval[0];
mdio->md_version = MDIOVERSION;
mdio->md_type = MD_VNODE;
if (root_mount_mddev != -1) {
mdio->md_unit = root_mount_mddev;
(void)kern_ioctl(td, fd, MDIOCDETACH, (void *)mdio);
/* Ignore errors. We don't care. */
root_mount_mddev = -1;
}
mdio->md_file = (void *)(mdio + 1);
mdio->md_options = MD_AUTOUNIT | MD_READONLY;
mdio->md_mediasize = sb.st_size;
mdio->md_unit = 0;
error = kern_ioctl(td, fd, MDIOCATTACH, (void *)mdio);
if (error)
goto out;
if (mdio->md_unit > 9) {
printf("rootmount: too many md units\n");
mdio->md_file = NULL;
mdio->md_options = 0;
mdio->md_mediasize = 0;
error = kern_ioctl(td, fd, MDIOCDETACH, (void *)mdio);
/* Ignore errors. We don't care. */
error = ERANGE;
goto out;
}
root_mount_mddev = mdio->md_unit;
printf(MD_NAME "%u attached to %s\n", root_mount_mddev, mdio->md_file);
error = kern_close(td, fd);
out:
free(mdio, M_TEMP);
return (error);
}
static int
parse_dir_onfail(char **conf)
{
char *action;
int error;
error = parse_token(conf, &action);
if (error)
return (error);
if (!strcmp(action, "continue"))
root_mount_onfail = A_CONTINUE;
else if (!strcmp(action, "panic"))
root_mount_onfail = A_PANIC;
else if (!strcmp(action, "reboot"))
root_mount_onfail = A_REBOOT;
else if (!strcmp(action, "retry"))
root_mount_onfail = A_RETRY;
else {
printf("rootmount: %s: unknown action\n", action);
error = EINVAL;
}
free(action, M_TEMP);
return (0);
}
static int
parse_dir_timeout(char **conf)
{
char *tok, *endtok;
long secs;
int error;
error = parse_token(conf, &tok);
if (error)
return (error);
secs = strtol(tok, &endtok, 0);
error = (secs < 0 || *endtok != '\0') ? EINVAL : 0;
if (!error)
root_mount_timeout = secs;
free(tok, M_TEMP);
return (error);
}
static int
parse_directive(char **conf)
{
char *dir;
int error;
error = parse_token(conf, &dir);
if (error)
return (error);
if (strcmp(dir, ".ask") == 0)
error = parse_dir_ask(conf);
else if (strcmp(dir, ".md") == 0)
error = parse_dir_md(conf);
else if (strcmp(dir, ".onfail") == 0)
error = parse_dir_onfail(conf);
else if (strcmp(dir, ".timeout") == 0)
error = parse_dir_timeout(conf);
else {
printf("mountroot: invalid directive `%s'\n", dir);
/* Ignore the rest of the line. */
(void)parse_skipto(conf, '\n');
error = EINVAL;
}
free(dir, M_TEMP);
return (error);
}
static int
parse_mount_dev_present(const char *dev)
{
struct nameidata nd;
int error;
NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE, dev, curthread);
error = namei(&nd);
if (!error)
vput(nd.ni_vp);
NDFREE(&nd, NDF_ONLY_PNBUF);
return (error != 0) ? 0 : 1;
}
#define ERRMSGL 255
static int
parse_mount(char **conf)
{
char *errmsg;
struct mntarg *ma;
char *dev, *fs, *opts, *tok;
int delay, error, timeout;
error = parse_token(conf, &tok);
if (error)
return (error);
fs = tok;
error = parse_skipto(&tok, ':');
if (error) {
free(fs, M_TEMP);
return (error);
}
parse_poke(&tok, '\0');
parse_advance(&tok);
dev = tok;
if (root_mount_mddev != -1) {
/* Handle substitution for the md unit number. */
tok = strstr(dev, "md#");
if (tok != NULL)
tok[2] = '0' + root_mount_mddev;
}
/* Parse options. */
error = parse_token(conf, &tok);
opts = (error == 0) ? tok : NULL;
printf("Trying to mount root from %s:%s [%s]...\n", fs, dev,
(opts != NULL) ? opts : "");
errmsg = malloc(ERRMSGL, M_TEMP, M_WAITOK | M_ZERO);
if (vfs_byname(fs) == NULL) {
strlcpy(errmsg, "unknown file system", ERRMSGL);
error = ENOENT;
goto out;
}
error = vfs_mountroot_wait_if_neccessary(fs, dev);
if (error != 0)
goto out;
delay = hz / 10;
timeout = root_mount_timeout * hz;
for (;;) {
ma = NULL;
ma = mount_arg(ma, "fstype", fs, -1);
ma = mount_arg(ma, "fspath", "/", -1);
ma = mount_arg(ma, "from", dev, -1);
ma = mount_arg(ma, "errmsg", errmsg, ERRMSGL);
ma = mount_arg(ma, "ro", NULL, 0);
ma = parse_mountroot_options(ma, opts);
error = kernel_mount(ma, MNT_ROOTFS);
if (error == 0 || timeout <= 0)
break;
if (root_mount_timeout * hz == timeout ||
(bootverbose && timeout % hz == 0)) {
printf("Mounting from %s:%s failed with error %d; "
"retrying for %d more second%s\n", fs, dev, error,
timeout / hz, (timeout / hz > 1) ? "s" : "");
}
pause("rmretry", delay);
timeout -= delay;
}
out:
if (error) {
printf("Mounting from %s:%s failed with error %d",
fs, dev, error);
if (errmsg[0] != '\0')
printf(": %s", errmsg);
printf(".\n");
}
free(fs, M_TEMP);
free(errmsg, M_TEMP);
if (opts != NULL)
free(opts, M_TEMP);
/* kernel_mount can return -1 on error. */
return ((error < 0) ? EDOOFUS : error);
}
#undef ERRMSGL
static int
vfs_mountroot_parse(struct sbuf *sb, struct mount *mpdevfs)
{
struct mount *mp;
char *conf;
int error;
root_mount_mddev = -1;
retry:
conf = sbuf_data(sb);
mp = TAILQ_NEXT(mpdevfs, mnt_list);
error = (mp == NULL) ? 0 : EDOOFUS;
root_mount_onfail = A_CONTINUE;
while (mp == NULL) {
error = parse_skipto(&conf, CC_NONWHITESPACE);
if (error == PE_EOL) {
parse_advance(&conf);
continue;
}
if (error < 0)
break;
switch (parse_peek(&conf)) {
case '#':
error = parse_skipto(&conf, '\n');
break;
case '.':
error = parse_directive(&conf);
break;
default:
error = parse_mount(&conf);
if (error == -1) {
printf("mountroot: invalid file system "
"specification.\n");
error = 0;
}
break;
}
if (error < 0)
break;
/* Ignore any trailing garbage on the line. */
if (parse_peek(&conf) != '\n') {
printf("mountroot: advancing to next directive...\n");
(void)parse_skipto(&conf, '\n');
}
mp = TAILQ_NEXT(mpdevfs, mnt_list);
}
if (mp != NULL)
return (0);
/*
* We failed to mount (a new) root.
*/
switch (root_mount_onfail) {
case A_CONTINUE:
break;
case A_PANIC:
panic("mountroot: unable to (re-)mount root.");
/* NOTREACHED */
case A_RETRY:
goto retry;
case A_REBOOT:
kern_reboot(RB_NOSYNC);
/* NOTREACHED */
}
return (error);
}
static void
vfs_mountroot_conf0(struct sbuf *sb)
{
char *s, *tok, *mnt, *opt;
int error;
sbuf_printf(sb, ".onfail panic\n");
sbuf_printf(sb, ".timeout %d\n", root_mount_timeout);
if (boothowto & RB_ASKNAME)
sbuf_printf(sb, ".ask\n");
#ifdef ROOTDEVNAME
if (boothowto & RB_DFLTROOT)
sbuf_printf(sb, "%s\n", ROOTDEVNAME);
#endif
if (boothowto & RB_CDROM) {
sbuf_printf(sb, "cd9660:/dev/cd0 ro\n");
sbuf_printf(sb, ".timeout 0\n");
sbuf_printf(sb, "cd9660:/dev/cd1 ro\n");
sbuf_printf(sb, ".timeout %d\n", root_mount_timeout);
}
s = kern_getenv("vfs.root.mountfrom");
if (s != NULL) {
opt = kern_getenv("vfs.root.mountfrom.options");
tok = s;
error = parse_token(&tok, &mnt);
while (!error) {
sbuf_printf(sb, "%s %s\n", mnt,
(opt != NULL) ? opt : "");
free(mnt, M_TEMP);
error = parse_token(&tok, &mnt);
}
if (opt != NULL)
freeenv(opt);
freeenv(s);
}
if (rootdevnames[0] != NULL)
sbuf_printf(sb, "%s\n", rootdevnames[0]);
if (rootdevnames[1] != NULL)
sbuf_printf(sb, "%s\n", rootdevnames[1]);
#ifdef ROOTDEVNAME
if (!(boothowto & RB_DFLTROOT))
sbuf_printf(sb, "%s\n", ROOTDEVNAME);
#endif
if (!(boothowto & RB_ASKNAME))
sbuf_printf(sb, ".ask\n");
}
static int
vfs_mountroot_readconf(struct thread *td, struct sbuf *sb)
{
static char buf[128];
struct nameidata nd;
off_t ofs;
ssize_t resid;
int error, flags, len;
NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, "/.mount.conf", td);
flags = FREAD;
error = vn_open(&nd, &flags, 0, NULL);
if (error)
return (error);
NDFREE(&nd, NDF_ONLY_PNBUF);
ofs = 0;
len = sizeof(buf) - 1;
while (1) {
error = vn_rdwr(UIO_READ, nd.ni_vp, buf, len, ofs,
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred,
NOCRED, &resid, td);
if (error)
break;
if (resid == len)
break;
buf[len - resid] = 0;
sbuf_printf(sb, "%s", buf);
ofs += len - resid;
}
VOP_UNLOCK(nd.ni_vp);
vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
return (error);
}
static void
vfs_mountroot_wait(void)
{
struct root_hold_token *h;
struct timeval lastfail;
int curfail;
TSENTER();
curfail = 0;
while (1) {
g_waitidle();
mtx_lock(&root_holds_mtx);
if (TAILQ_EMPTY(&root_holds)) {
mtx_unlock(&root_holds_mtx);
break;
}
if (ppsratecheck(&lastfail, &curfail, 1)) {
printf("Root mount waiting for:");
TAILQ_FOREACH(h, &root_holds, list)
printf(" %s", h->who);
printf("\n");
}
TSWAIT("root mount");
msleep(&root_holds, &root_holds_mtx, PZERO | PDROP, "roothold",
hz);
TSUNWAIT("root mount");
}
TSEXIT();
}
static int
vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev)
{
int delay, timeout;
/*
* In case of ZFS and NFS we don't have a way to wait for
* specific device. Also do the wait if the user forced that
* behaviour by setting vfs.root_mount_always_wait=1.
*/
if (strcmp(fs, "zfs") == 0 || strstr(fs, "nfs") != NULL ||
dev[0] == '\0' || root_mount_always_wait != 0) {
vfs_mountroot_wait();
return (0);
}
/*
* Otherwise, no point in waiting if the device is already there.
* Note that we must wait for GEOM to finish reconfiguring itself,
* eg for geom_part(4) to finish tasting.
*/
g_waitidle();
if (parse_mount_dev_present(dev))
return (0);
/*
* No luck. Let's wait. This code looks weird, but it's that way
* to behave exactly as it used to work before.
*/
vfs_mountroot_wait();
printf("mountroot: waiting for device %s...\n", dev);
delay = hz / 10;
timeout = root_mount_timeout * hz;
do {
pause("rmdev", delay);
timeout -= delay;
} while (timeout > 0 && !parse_mount_dev_present(dev));
if (timeout <= 0)
return (ENODEV);
return (0);
}
void
vfs_mountroot(void)
{
struct mount *mp;
struct sbuf *sb;
struct thread *td;
time_t timebase;
int error;
mtx_assert(&Giant, MA_NOTOWNED);
TSENTER();
td = curthread;
sb = sbuf_new_auto();
vfs_mountroot_conf0(sb);
sbuf_finish(sb);
error = vfs_mountroot_devfs(td, &mp);
while (!error) {
error = vfs_mountroot_parse(sb, mp);
if (!error) {
vfs_mountroot_shuffle(td, mp);
sbuf_clear(sb);
error = vfs_mountroot_readconf(td, sb);
sbuf_finish(sb);
}
}
sbuf_delete(sb);
/*
* Iterate over all currently mounted file systems and use
* the time stamp found to check and/or initialize the RTC.
* Call inittodr() only once and pass it the largest of the
* timestamps we encounter.
*/
timebase = 0;
mtx_lock(&mountlist_mtx);
mp = TAILQ_FIRST(&mountlist);
while (mp != NULL) {
if (mp->mnt_time > timebase)
timebase = mp->mnt_time;
mp = TAILQ_NEXT(mp, mnt_list);
}
mtx_unlock(&mountlist_mtx);
inittodr(timebase);
/* Keep prison0's root in sync with the global rootvnode. */
mtx_lock(&prison0.pr_mtx);
prison0.pr_root = rootvnode;
vref(prison0.pr_root);
mtx_unlock(&prison0.pr_mtx);
mtx_lock(&root_holds_mtx);
atomic_store_rel_int(&root_mount_complete, 1);
wakeup(&root_mount_complete);
mtx_unlock(&root_holds_mtx);
EVENTHANDLER_INVOKE(mountroot);
TSEXIT();
}
static struct mntarg *
parse_mountroot_options(struct mntarg *ma, const char *options)
{
char *p;
char *name, *name_arg;
char *val, *val_arg;
char *opts;
if (options == NULL || options[0] == '\0')
return (ma);
p = opts = strdup(options, M_MOUNT);
if (opts == NULL) {
return (ma);
}
while((name = strsep(&p, ",")) != NULL) {
if (name[0] == '\0')
break;
val = strchr(name, '=');
if (val != NULL) {
*val = '\0';
++val;
}
if( strcmp(name, "rw") == 0 ||
strcmp(name, "noro") == 0) {
/*
* The first time we mount the root file system,
* we need to mount 'ro', so We need to ignore
* 'rw' and 'noro' mount options.
*/
continue;
}
name_arg = strdup(name, M_MOUNT);
val_arg = NULL;
if (val != NULL)
val_arg = strdup(val, M_MOUNT);
ma = mount_arg(ma, name_arg, val_arg,
(val_arg != NULL ? -1 : 0));
}
free(opts, M_MOUNT);
return (ma);
}
Index: head/sys/security/audit/audit_bsm_klib.c
===================================================================
--- head/sys/security/audit/audit_bsm_klib.c (revision 358502)
+++ head/sys/security/audit/audit_bsm_klib.c (revision 358503)
@@ -1,530 +1,527 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1999-2009 Apple Inc.
* Copyright (c) 2005, 2016-2017 Robert N. M. Watson
* All rights reserved.
*
* Portions of this software were developed by BAE Systems, the University of
* Cambridge Computer Laboratory, and Memorial University under DARPA/AFRL
* contract FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent
* Computing (TC) research program.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of Apple Inc. ("Apple") nor the names of
* its contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/capsicum.h>
#include <sys/fcntl.h>
#include <sys/filedesc.h>
#include <sys/libkern.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
#include <sys/sem.h>
#include <sys/sbuf.h>
#include <sys/sx.h>
#include <sys/syscall.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/vnode.h>
#include <bsm/audit.h>
#include <bsm/audit_kevents.h>
#include <security/audit/audit.h>
#include <security/audit/audit_private.h>
struct aue_open_event {
int aoe_flags;
au_event_t aoe_event;
};
static const struct aue_open_event aue_open[] = {
{ O_RDONLY, AUE_OPEN_R },
{ (O_RDONLY | O_CREAT), AUE_OPEN_RC },
{ (O_RDONLY | O_CREAT | O_TRUNC), AUE_OPEN_RTC },
{ (O_RDONLY | O_TRUNC), AUE_OPEN_RT },
{ O_RDWR, AUE_OPEN_RW },
{ (O_RDWR | O_CREAT), AUE_OPEN_RWC },
{ (O_RDWR | O_CREAT | O_TRUNC), AUE_OPEN_RWTC },
{ (O_RDWR | O_TRUNC), AUE_OPEN_RWT },
{ O_WRONLY, AUE_OPEN_W },
{ (O_WRONLY | O_CREAT), AUE_OPEN_WC },
{ (O_WRONLY | O_CREAT | O_TRUNC), AUE_OPEN_WTC },
{ (O_WRONLY | O_TRUNC), AUE_OPEN_WT },
};
static const struct aue_open_event aue_openat[] = {
{ O_RDONLY, AUE_OPENAT_R },
{ (O_RDONLY | O_CREAT), AUE_OPENAT_RC },
{ (O_RDONLY | O_CREAT | O_TRUNC), AUE_OPENAT_RTC },
{ (O_RDONLY | O_TRUNC), AUE_OPENAT_RT },
{ O_RDWR, AUE_OPENAT_RW },
{ (O_RDWR | O_CREAT), AUE_OPENAT_RWC },
{ (O_RDWR | O_CREAT | O_TRUNC), AUE_OPENAT_RWTC },
{ (O_RDWR | O_TRUNC), AUE_OPENAT_RWT },
{ O_WRONLY, AUE_OPENAT_W },
{ (O_WRONLY | O_CREAT), AUE_OPENAT_WC },
{ (O_WRONLY | O_CREAT | O_TRUNC), AUE_OPENAT_WTC },
{ (O_WRONLY | O_TRUNC), AUE_OPENAT_WT },
};
static const int aue_msgsys[] = {
/* 0 */ AUE_MSGCTL,
/* 1 */ AUE_MSGGET,
/* 2 */ AUE_MSGSND,
/* 3 */ AUE_MSGRCV,
};
static const int aue_msgsys_count = sizeof(aue_msgsys) / sizeof(int);
static const int aue_semsys[] = {
/* 0 */ AUE_SEMCTL,
/* 1 */ AUE_SEMGET,
/* 2 */ AUE_SEMOP,
};
static const int aue_semsys_count = sizeof(aue_semsys) / sizeof(int);
static const int aue_shmsys[] = {
/* 0 */ AUE_SHMAT,
/* 1 */ AUE_SHMDT,
/* 2 */ AUE_SHMGET,
/* 3 */ AUE_SHMCTL,
};
static const int aue_shmsys_count = sizeof(aue_shmsys) / sizeof(int);
/*
* Check whether an event is auditable by comparing the mask of classes this
* event is part of against the given mask.
*/
int
au_preselect(au_event_t event, au_class_t class, au_mask_t *mask_p, int sorf)
{
au_class_t effmask = 0;
if (mask_p == NULL)
return (-1);
/*
* Perform the actual check of the masks against the event.
*/
if (sorf & AU_PRS_SUCCESS)
effmask |= (mask_p->am_success & class);
if (sorf & AU_PRS_FAILURE)
effmask |= (mask_p->am_failure & class);
if (effmask)
return (1);
else
return (0);
}
/*
* Convert sysctl names and present arguments to events.
*/
au_event_t
audit_ctlname_to_sysctlevent(int name[], uint64_t valid_arg)
{
/* can't parse it - so return the worst case */
if ((valid_arg & (ARG_CTLNAME | ARG_LEN)) != (ARG_CTLNAME | ARG_LEN))
return (AUE_SYSCTL);
switch (name[0]) {
/* non-admin "lookups" treat them special */
case KERN_OSTYPE:
case KERN_OSRELEASE:
case KERN_OSREV:
case KERN_VERSION:
case KERN_ARGMAX:
case KERN_CLOCKRATE:
case KERN_BOOTTIME:
case KERN_POSIX1:
case KERN_NGROUPS:
case KERN_JOB_CONTROL:
case KERN_SAVED_IDS:
case KERN_OSRELDATE:
case KERN_DUMMY:
return (AUE_SYSCTL_NONADMIN);
/* only treat the changeable controls as admin */
case KERN_MAXVNODES:
case KERN_MAXPROC:
case KERN_MAXFILES:
case KERN_MAXPROCPERUID:
case KERN_MAXFILESPERPROC:
case KERN_HOSTID:
case KERN_SECURELVL:
case KERN_HOSTNAME:
case KERN_VNODE:
case KERN_PROC:
case KERN_FILE:
case KERN_PROF:
case KERN_NISDOMAINNAME:
case KERN_UPDATEINTERVAL:
case KERN_NTP_PLL:
case KERN_BOOTFILE:
case KERN_DUMPDEV:
case KERN_IPC:
case KERN_PS_STRINGS:
case KERN_USRSTACK:
case KERN_LOGSIGEXIT:
case KERN_IOV_MAX:
return ((valid_arg & ARG_VALUE) ?
AUE_SYSCTL : AUE_SYSCTL_NONADMIN);
default:
return (AUE_SYSCTL);
}
/* NOTREACHED */
}
/*
* Convert an open flags specifier into a specific type of open event for
* auditing purposes.
*/
au_event_t
audit_flags_and_error_to_openevent(int oflags, int error)
{
int i;
/*
* Need to check only those flags we care about.
*/
oflags = oflags & (O_RDONLY | O_CREAT | O_TRUNC | O_RDWR | O_WRONLY);
for (i = 0; i < nitems(aue_open); i++) {
if (aue_open[i].aoe_flags == oflags)
return (aue_open[i].aoe_event);
}
return (AUE_OPEN);
}
au_event_t
audit_flags_and_error_to_openatevent(int oflags, int error)
{
int i;
/*
* Need to check only those flags we care about.
*/
oflags = oflags & (O_RDONLY | O_CREAT | O_TRUNC | O_RDWR | O_WRONLY);
for (i = 0; i < nitems(aue_openat); i++) {
if (aue_openat[i].aoe_flags == oflags)
return (aue_openat[i].aoe_event);
}
return (AUE_OPENAT);
}
/*
* Convert a MSGCTL command to a specific event.
*/
au_event_t
audit_msgctl_to_event(int cmd)
{
switch (cmd) {
case IPC_RMID:
return (AUE_MSGCTL_RMID);
case IPC_SET:
return (AUE_MSGCTL_SET);
case IPC_STAT:
return (AUE_MSGCTL_STAT);
default:
/* We will audit a bad command. */
return (AUE_MSGCTL);
}
}
/*
* Convert a SEMCTL command to a specific event.
*/
au_event_t
audit_semctl_to_event(int cmd)
{
switch (cmd) {
case GETALL:
return (AUE_SEMCTL_GETALL);
case GETNCNT:
return (AUE_SEMCTL_GETNCNT);
case GETPID:
return (AUE_SEMCTL_GETPID);
case GETVAL:
return (AUE_SEMCTL_GETVAL);
case GETZCNT:
return (AUE_SEMCTL_GETZCNT);
case IPC_RMID:
return (AUE_SEMCTL_RMID);
case IPC_SET:
return (AUE_SEMCTL_SET);
case SETALL:
return (AUE_SEMCTL_SETALL);
case SETVAL:
return (AUE_SEMCTL_SETVAL);
case IPC_STAT:
return (AUE_SEMCTL_STAT);
default:
/* We will audit a bad command. */
return (AUE_SEMCTL);
}
}
/*
* Convert msgsys(2), semsys(2), and shmsys(2) system-call variations into
* audit events, if possible.
*/
au_event_t
audit_msgsys_to_event(int which)
{
if ((which >= 0) && (which < aue_msgsys_count))
return (aue_msgsys[which]);
/* Audit a bad command. */
return (AUE_MSGSYS);
}
au_event_t
audit_semsys_to_event(int which)
{
if ((which >= 0) && (which < aue_semsys_count))
return (aue_semsys[which]);
/* Audit a bad command. */
return (AUE_SEMSYS);
}
au_event_t
audit_shmsys_to_event(int which)
{
if ((which >= 0) && (which < aue_shmsys_count))
return (aue_shmsys[which]);
/* Audit a bad command. */
return (AUE_SHMSYS);
}
/*
* Convert a command for the auditon() system call to a audit event.
*/
au_event_t
auditon_command_event(int cmd)
{
switch(cmd) {
case A_GETPOLICY:
return (AUE_AUDITON_GPOLICY);
case A_SETPOLICY:
return (AUE_AUDITON_SPOLICY);
case A_GETKMASK:
return (AUE_AUDITON_GETKMASK);
case A_SETKMASK:
return (AUE_AUDITON_SETKMASK);
case A_GETQCTRL:
return (AUE_AUDITON_GQCTRL);
case A_SETQCTRL:
return (AUE_AUDITON_SQCTRL);
case A_GETCWD:
return (AUE_AUDITON_GETCWD);
case A_GETCAR:
return (AUE_AUDITON_GETCAR);
case A_GETSTAT:
return (AUE_AUDITON_GETSTAT);
case A_SETSTAT:
return (AUE_AUDITON_SETSTAT);
case A_SETUMASK:
return (AUE_AUDITON_SETUMASK);
case A_SETSMASK:
return (AUE_AUDITON_SETSMASK);
case A_GETCOND:
return (AUE_AUDITON_GETCOND);
case A_SETCOND:
return (AUE_AUDITON_SETCOND);
case A_GETCLASS:
return (AUE_AUDITON_GETCLASS);
case A_SETCLASS:
return (AUE_AUDITON_SETCLASS);
case A_GETPINFO:
case A_SETPMASK:
case A_SETFSIZE:
case A_GETFSIZE:
case A_GETPINFO_ADDR:
case A_GETKAUDIT:
case A_SETKAUDIT:
default:
return (AUE_AUDITON); /* No special record */
}
}
/*
* Create a canonical path from given path by prefixing either the root
* directory, or the current working directory. If the process working
* directory is NULL, we could use 'rootvnode' to obtain the root directory,
* but this results in a volfs name written to the audit log. So we will
* leave the filename starting with '/' in the audit log in this case.
*/
void
audit_canon_path_vp(struct thread *td, struct vnode *rdir, struct vnode *cdir,
char *path, char *cpath)
{
struct vnode *vp;
char *rbuf, *fbuf, *copy;
struct sbuf sbf;
int error;
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: at %s:%d",
__func__, __FILE__, __LINE__);
copy = path;
if (*path == '/')
vp = rdir;
else
vp = cdir;
MPASS(vp != NULL);
/*
* NB: We require that the supplied array be at least MAXPATHLEN bytes
* long. If this is not the case, then we can run into serious trouble.
*/
(void) sbuf_new(&sbf, cpath, MAXPATHLEN, SBUF_FIXEDLEN);
/*
* Strip leading forward slashes.
*
* Note this does nothing to fully canonicalize the path.
*/
while (*copy == '/')
copy++;
/*
* Make sure we handle chroot(2) and prepend the global path to these
* environments.
*
* NB: vn_fullpath(9) on FreeBSD is less reliable than vn_getpath(9)
* on Darwin. As a result, this may need some additional attention
* in the future.
*/
error = vn_fullpath_global(td, vp, &rbuf, &fbuf);
if (error) {
cpath[0] = '\0';
return;
}
(void) sbuf_cat(&sbf, rbuf);
/*
* We are going to concatenate the resolved path with the passed path
* with all slashes removed and we want them glued with a single slash.
* However, if the directory is /, the slash is already there.
*/
if (rbuf[1] != '\0')
(void) sbuf_putc(&sbf, '/');
free(fbuf, M_TEMP);
/*
* Now that we have processed any alternate root and relative path
* names, add the supplied pathname.
*/
(void) sbuf_cat(&sbf, copy);
/*
* One or more of the previous sbuf operations could have resulted in
* the supplied buffer being overflowed. Check to see if this is the
* case.
*/
if (sbuf_error(&sbf) != 0) {
cpath[0] = '\0';
return;
}
sbuf_finish(&sbf);
}
void
audit_canon_path(struct thread *td, int dirfd, char *path, char *cpath)
{
struct vnode *cdir, *rdir;
- struct filedesc *fdp;
+ struct pwd *pwd;
cap_rights_t rights;
int error;
+ bool vrele_cdir;
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: at %s:%d",
__func__, __FILE__, __LINE__);
- rdir = cdir = NULL;
- fdp = td->td_proc->p_fd;
- FILEDESC_SLOCK(fdp);
- if (*path == '/') {
- rdir = fdp->fd_rdir;
- vrefact(rdir);
- } else {
+ pwd = pwd_hold(td);
+ rdir = pwd->pwd_rdir;
+ cdir = NULL;
+ vrele_cdir = false;
+ if (*path != '/') {
if (dirfd == AT_FDCWD) {
- cdir = fdp->fd_cdir;
- vrefact(cdir);
+ cdir = pwd->pwd_cdir;
} else {
error = fgetvp(td, dirfd, cap_rights_init(&rights), &cdir);
if (error != 0) {
- FILEDESC_SUNLOCK(fdp);
cpath[0] = '\0';
+ pwd_drop(pwd);
return;
}
+ vrele_cdir = true;
}
}
- FILEDESC_SUNLOCK(fdp);
audit_canon_path_vp(td, rdir, cdir, path, cpath);
- if (rdir != NULL)
- vrele(rdir);
- if (cdir != NULL)
+ pwd_drop(pwd);
+ if (vrele_cdir)
vrele(cdir);
}
Index: head/sys/sys/filedesc.h
===================================================================
--- head/sys/sys/filedesc.h (revision 358502)
+++ head/sys/sys/filedesc.h (revision 358503)
@@ -1,258 +1,274 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 1990, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)filedesc.h 8.1 (Berkeley) 6/2/93
* $FreeBSD$
*/
#ifndef _SYS_FILEDESC_H_
#define _SYS_FILEDESC_H_
#include <sys/caprights.h>
#include <sys/queue.h>
#include <sys/event.h>
#include <sys/lock.h>
#include <sys/priority.h>
#include <sys/seqc.h>
#include <sys/sx.h>
#include <machine/_limits.h>
struct filecaps {
cap_rights_t fc_rights; /* per-descriptor capability rights */
u_long *fc_ioctls; /* per-descriptor allowed ioctls */
int16_t fc_nioctls; /* fc_ioctls array size */
uint32_t fc_fcntls; /* per-descriptor allowed fcntls */
};
struct filedescent {
struct file *fde_file; /* file structure for open file */
struct filecaps fde_caps; /* per-descriptor rights */
uint8_t fde_flags; /* per-process open file flags */
seqc_t fde_seqc; /* keep file and caps in sync */
};
#define fde_rights fde_caps.fc_rights
#define fde_fcntls fde_caps.fc_fcntls
#define fde_ioctls fde_caps.fc_ioctls
#define fde_nioctls fde_caps.fc_nioctls
#define fde_change_size (offsetof(struct filedescent, fde_seqc))
struct fdescenttbl {
int fdt_nfiles; /* number of open files allocated */
struct filedescent fdt_ofiles[0]; /* open files */
};
#define fd_seqc(fdt, fd) (&(fdt)->fdt_ofiles[(fd)].fde_seqc)
/*
* This structure is used for the management of descriptors. It may be
* shared by multiple processes.
*/
#define NDSLOTTYPE u_long
+struct pwd {
+ volatile u_int pwd_refcount;
+ struct vnode *pwd_cdir; /* current directory */
+ struct vnode *pwd_rdir; /* root directory */
+ struct vnode *pwd_jdir; /* jail root directory */
+};
+
struct filedesc {
struct fdescenttbl *fd_files; /* open files table */
- struct vnode *fd_cdir; /* current directory */
- struct vnode *fd_rdir; /* root directory */
- struct vnode *fd_jdir; /* jail root directory */
+ struct pwd *fd_pwd; /* directories */
NDSLOTTYPE *fd_map; /* bitmap of free fds */
int fd_lastfile; /* high-water mark of fd_ofiles */
int fd_freefile; /* approx. next free file */
u_short fd_cmask; /* mask for file creation */
int fd_refcnt; /* thread reference count */
int fd_holdcnt; /* hold count on structure + mutex */
struct sx fd_sx; /* protects members of this struct */
struct kqlist fd_kqlist; /* list of kqueues on this filedesc */
int fd_holdleaderscount; /* block fdfree() for shared close() */
int fd_holdleaderswakeup; /* fdfree() needs wakeup */
};
/*
* Structure to keep track of (process leader, struct fildedesc) tuples.
* Each process has a pointer to such a structure when detailed tracking
* is needed, e.g., when rfork(RFPROC | RFMEM) causes a file descriptor
* table to be shared by processes having different "p_leader" pointers
* and thus distinct POSIX style locks.
*
* fdl_refcount and fdl_holdcount are protected by struct filedesc mtx.
*/
struct filedesc_to_leader {
int fdl_refcount; /* references from struct proc */
int fdl_holdcount; /* temporary hold during closef */
int fdl_wakeup; /* fdfree() waits on closef() */
struct proc *fdl_leader; /* owner of POSIX locks */
/* Circular list: */
struct filedesc_to_leader *fdl_prev;
struct filedesc_to_leader *fdl_next;
};
#define fd_nfiles fd_files->fdt_nfiles
#define fd_ofiles fd_files->fdt_ofiles
/*
* Per-process open flags.
*/
#define UF_EXCLOSE 0x01 /* auto-close on exec */
#ifdef _KERNEL
/* Lock a file descriptor table. */
#define FILEDESC_LOCK_INIT(fdp) sx_init(&(fdp)->fd_sx, "filedesc structure")
#define FILEDESC_LOCK_DESTROY(fdp) sx_destroy(&(fdp)->fd_sx)
#define FILEDESC_LOCK(fdp) (&(fdp)->fd_sx)
#define FILEDESC_XLOCK(fdp) sx_xlock(&(fdp)->fd_sx)
#define FILEDESC_XUNLOCK(fdp) sx_xunlock(&(fdp)->fd_sx)
#define FILEDESC_SLOCK(fdp) sx_slock(&(fdp)->fd_sx)
#define FILEDESC_SUNLOCK(fdp) sx_sunlock(&(fdp)->fd_sx)
#define FILEDESC_LOCK_ASSERT(fdp) sx_assert(&(fdp)->fd_sx, SX_LOCKED | \
SX_NOTRECURSED)
#define FILEDESC_XLOCK_ASSERT(fdp) sx_assert(&(fdp)->fd_sx, SX_XLOCKED | \
SX_NOTRECURSED)
#define FILEDESC_UNLOCK_ASSERT(fdp) sx_assert(&(fdp)->fd_sx, SX_UNLOCKED)
/* Operation types for kern_dup(). */
enum {
FDDUP_NORMAL, /* dup() behavior. */
FDDUP_FCNTL, /* fcntl()-style errors. */
FDDUP_FIXED, /* Force fixed allocation. */
FDDUP_MUSTREPLACE, /* Target must exist. */
FDDUP_LASTMODE,
};
/* Flags for kern_dup(). */
#define FDDUP_FLAG_CLOEXEC 0x1 /* Atomically set UF_EXCLOSE. */
/* For backward compatibility. */
#define falloc(td, resultfp, resultfd, flags) \
falloc_caps(td, resultfp, resultfd, flags, NULL)
struct thread;
static __inline void
filecaps_init(struct filecaps *fcaps)
{
bzero(fcaps, sizeof(*fcaps));
fcaps->fc_nioctls = -1;
}
bool filecaps_copy(const struct filecaps *src, struct filecaps *dst,
bool locked);
void filecaps_move(struct filecaps *src, struct filecaps *dst);
void filecaps_free(struct filecaps *fcaps);
int closef(struct file *fp, struct thread *td);
int dupfdopen(struct thread *td, struct filedesc *fdp, int dfd, int mode,
int openerror, int *indxp);
int falloc_caps(struct thread *td, struct file **resultfp, int *resultfd,
int flags, struct filecaps *fcaps);
int falloc_noinstall(struct thread *td, struct file **resultfp);
void _finstall(struct filedesc *fdp, struct file *fp, int fd, int flags,
struct filecaps *fcaps);
int finstall(struct thread *td, struct file *fp, int *resultfd, int flags,
struct filecaps *fcaps);
int fdalloc(struct thread *td, int minfd, int *result);
int fdallocn(struct thread *td, int minfd, int *fds, int n);
int fdcheckstd(struct thread *td);
void fdclose(struct thread *td, struct file *fp, int idx);
void fdcloseexec(struct thread *td);
void fdsetugidsafety(struct thread *td);
struct filedesc *fdcopy(struct filedesc *fdp);
int fdcopy_remapped(struct filedesc *fdp, const int *fds, size_t nfds,
struct filedesc **newfdp);
void fdinstall_remapped(struct thread *td, struct filedesc *fdp);
void fdunshare(struct thread *td);
void fdescfree(struct thread *td);
void fdescfree_remapped(struct filedesc *fdp);
struct filedesc *fdinit(struct filedesc *fdp, bool prepfiles);
struct filedesc *fdshare(struct filedesc *fdp);
struct filedesc_to_leader *
filedesc_to_leader_alloc(struct filedesc_to_leader *old,
struct filedesc *fdp, struct proc *leader);
int getvnode(struct thread *td, int fd, cap_rights_t *rightsp,
struct file **fpp);
void mountcheckdirs(struct vnode *olddp, struct vnode *newdp);
int fget_cap_locked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp);
int fget_cap(struct thread *td, int fd, cap_rights_t *needrightsp,
struct file **fpp, struct filecaps *havecapsp);
/* Return a referenced file from an unlocked descriptor. */
int fget_unlocked_seq(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp, seqc_t *seqp);
int fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
struct file **fpp);
/* Requires a FILEDESC_{S,X}LOCK held and returns without a ref. */
static __inline struct file *
fget_locked(struct filedesc *fdp, int fd)
{
FILEDESC_LOCK_ASSERT(fdp);
if (__predict_false((u_int)fd >= fdp->fd_nfiles))
return (NULL);
return (fdp->fd_ofiles[fd].fde_file);
}
static __inline struct filedescent *
fdeget_locked(struct filedesc *fdp, int fd)
{
struct filedescent *fde;
FILEDESC_LOCK_ASSERT(fdp);
if (__predict_false((u_int)fd >= fdp->fd_nfiles))
return (NULL);
fde = &fdp->fd_ofiles[fd];
if (__predict_false(fde->fde_file == NULL))
return (NULL);
return (fde);
}
#ifdef CAPABILITIES
static __inline bool
fd_modified(struct filedesc *fdp, int fd, seqc_t seqc)
{
return (!seqc_consistent(fd_seqc(fdp->fd_files, fd), seqc));
}
#endif
/* cdir/rdir/jdir manipulation functions. */
void pwd_chdir(struct thread *td, struct vnode *vp);
int pwd_chroot(struct thread *td, struct vnode *vp);
void pwd_ensure_dirs(void);
+
+struct pwd *pwd_hold_filedesc(struct filedesc *fdp);
+struct pwd *pwd_hold(struct thread *td);
+void pwd_drop(struct pwd *pwd);
+static inline void
+pwd_set(struct filedesc *fdp, struct pwd *newpwd)
+{
+
+ FILEDESC_XLOCK_ASSERT(fdp);
+ fdp->fd_pwd = newpwd;
+}
#endif /* _KERNEL */
#endif /* !_SYS_FILEDESC_H_ */
Index: head/sys/ufs/ffs/ffs_alloc.c
===================================================================
--- head/sys/ufs/ffs/ffs_alloc.c (revision 358502)
+++ head/sys/ufs/ffs/ffs_alloc.c (revision 358503)
@@ -1,3659 +1,3661 @@
/*-
* SPDX-License-Identifier: (BSD-2-Clause-FreeBSD AND BSD-3-Clause)
*
* Copyright (c) 2002 Networks Associates Technology, Inc.
* All rights reserved.
*
* This software was developed for the FreeBSD Project by Marshall
* Kirk McKusick and Network Associates Laboratories, the Security
* Research Division of Network Associates, Inc. under DARPA/SPAWAR
* contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS
* research program
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* Copyright (c) 1982, 1986, 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* @(#)ffs_alloc.c 8.18 (Berkeley) 5/26/95
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_quota.h"
#include <sys/param.h>
#include <sys/capsicum.h>
#include <sys/gsb_crc32.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/kernel.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/taskqueue.h>
#include <security/audit/audit.h>
#include <geom/geom.h>
#include <geom/geom_vfs.h>
#include <ufs/ufs/dir.h>
#include <ufs/ufs/extattr.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>
#include <ufs/ffs/softdep.h>
typedef ufs2_daddr_t allocfcn_t(struct inode *ip, u_int cg, ufs2_daddr_t bpref,
int size, int rsize);
static ufs2_daddr_t ffs_alloccg(struct inode *, u_int, ufs2_daddr_t, int, int);
static ufs2_daddr_t
ffs_alloccgblk(struct inode *, struct buf *, ufs2_daddr_t, int);
static void ffs_blkfree_cg(struct ufsmount *, struct fs *,
struct vnode *, ufs2_daddr_t, long, ino_t,
struct workhead *);
#ifdef INVARIANTS
static int ffs_checkblk(struct inode *, ufs2_daddr_t, long);
#endif
static ufs2_daddr_t ffs_clusteralloc(struct inode *, u_int, ufs2_daddr_t, int);
static ino_t ffs_dirpref(struct inode *);
static ufs2_daddr_t ffs_fragextend(struct inode *, u_int, ufs2_daddr_t,
int, int);
static ufs2_daddr_t ffs_hashalloc
(struct inode *, u_int, ufs2_daddr_t, int, int, allocfcn_t *);
static ufs2_daddr_t ffs_nodealloccg(struct inode *, u_int, ufs2_daddr_t, int,
int);
static ufs1_daddr_t ffs_mapsearch(struct fs *, struct cg *, ufs2_daddr_t, int);
static int ffs_reallocblks_ufs1(struct vop_reallocblks_args *);
static int ffs_reallocblks_ufs2(struct vop_reallocblks_args *);
static void ffs_ckhash_cg(struct buf *);
/*
* Allocate a block in the filesystem.
*
* The size of the requested block is given, which must be some
* multiple of fs_fsize and <= fs_bsize.
* A preference may be optionally specified. If a preference is given
* the following hierarchy is used to allocate a block:
* 1) allocate the requested block.
* 2) allocate a rotationally optimal block in the same cylinder.
* 3) allocate a block in the same cylinder group.
* 4) quadradically rehash into other cylinder groups, until an
* available block is located.
* If no block preference is given the following hierarchy is used
* to allocate a block:
* 1) allocate a block in the cylinder group that contains the
* inode for the file.
* 2) quadradically rehash into other cylinder groups, until an
* available block is located.
*/
int
ffs_alloc(ip, lbn, bpref, size, flags, cred, bnp)
struct inode *ip;
ufs2_daddr_t lbn, bpref;
int size, flags;
struct ucred *cred;
ufs2_daddr_t *bnp;
{
struct fs *fs;
struct ufsmount *ump;
ufs2_daddr_t bno;
u_int cg, reclaimed;
int64_t delta;
#ifdef QUOTA
int error;
#endif
*bnp = 0;
ump = ITOUMP(ip);
fs = ump->um_fs;
mtx_assert(UFS_MTX(ump), MA_OWNED);
#ifdef INVARIANTS
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) {
printf("dev = %s, bsize = %ld, size = %d, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, size,
fs->fs_fsmnt);
panic("ffs_alloc: bad size");
}
if (cred == NOCRED)
panic("ffs_alloc: missing credential");
#endif /* INVARIANTS */
reclaimed = 0;
retry:
#ifdef QUOTA
UFS_UNLOCK(ump);
error = chkdq(ip, btodb(size), cred, 0);
if (error)
return (error);
UFS_LOCK(ump);
#endif
if (size == fs->fs_bsize && fs->fs_cstotal.cs_nbfree == 0)
goto nospace;
if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE) &&
freespace(fs, fs->fs_minfree) - numfrags(fs, size) < 0)
goto nospace;
if (bpref >= fs->fs_size)
bpref = 0;
if (bpref == 0)
cg = ino_to_cg(fs, ip->i_number);
else
cg = dtog(fs, bpref);
bno = ffs_hashalloc(ip, cg, bpref, size, size, ffs_alloccg);
if (bno > 0) {
delta = btodb(size);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
*bnp = bno;
return (0);
}
nospace:
#ifdef QUOTA
UFS_UNLOCK(ump);
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(size), cred, FORCE);
UFS_LOCK(ump);
#endif
if (reclaimed == 0 && (flags & IO_BUFLOCKED) == 0) {
reclaimed = 1;
softdep_request_cleanup(fs, ITOV(ip), cred, FLUSH_BLOCKS_WAIT);
goto retry;
}
if (reclaimed > 0 &&
ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem is full\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
return (ENOSPC);
}
/*
* Reallocate a fragment to a bigger size
*
* The number and size of the old block is given, and a preference
* and new size is also specified. The allocator attempts to extend
* the original block. Failing that, the regular block allocator is
* invoked to get an appropriate block.
*/
int
ffs_realloccg(ip, lbprev, bprev, bpref, osize, nsize, flags, cred, bpp)
struct inode *ip;
ufs2_daddr_t lbprev;
ufs2_daddr_t bprev;
ufs2_daddr_t bpref;
int osize, nsize, flags;
struct ucred *cred;
struct buf **bpp;
{
struct vnode *vp;
struct fs *fs;
struct buf *bp;
struct ufsmount *ump;
u_int cg, request, reclaimed;
int error, gbflags;
ufs2_daddr_t bno;
int64_t delta;
vp = ITOV(ip);
ump = ITOUMP(ip);
fs = ump->um_fs;
bp = NULL;
gbflags = (flags & BA_UNMAPPED) != 0 ? GB_UNMAPPED : 0;
mtx_assert(UFS_MTX(ump), MA_OWNED);
#ifdef INVARIANTS
if (vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED)
panic("ffs_realloccg: allocation on suspended filesystem");
if ((u_int)osize > fs->fs_bsize || fragoff(fs, osize) != 0 ||
(u_int)nsize > fs->fs_bsize || fragoff(fs, nsize) != 0) {
printf(
"dev = %s, bsize = %ld, osize = %d, nsize = %d, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, osize,
nsize, fs->fs_fsmnt);
panic("ffs_realloccg: bad size");
}
if (cred == NOCRED)
panic("ffs_realloccg: missing credential");
#endif /* INVARIANTS */
reclaimed = 0;
retry:
if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE) &&
freespace(fs, fs->fs_minfree) - numfrags(fs, nsize - osize) < 0) {
goto nospace;
}
if (bprev == 0) {
printf("dev = %s, bsize = %ld, bprev = %jd, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_bsize, (intmax_t)bprev,
fs->fs_fsmnt);
panic("ffs_realloccg: bad bprev");
}
UFS_UNLOCK(ump);
/*
* Allocate the extra space in the buffer.
*/
error = bread_gb(vp, lbprev, osize, NOCRED, gbflags, &bp);
if (error) {
return (error);
}
if (bp->b_blkno == bp->b_lblkno) {
if (lbprev >= UFS_NDADDR)
panic("ffs_realloccg: lbprev out of range");
bp->b_blkno = fsbtodb(fs, bprev);
}
#ifdef QUOTA
error = chkdq(ip, btodb(nsize - osize), cred, 0);
if (error) {
brelse(bp);
return (error);
}
#endif
/*
* Check for extension in the existing location.
*/
*bpp = NULL;
cg = dtog(fs, bprev);
UFS_LOCK(ump);
bno = ffs_fragextend(ip, cg, bprev, osize, nsize);
if (bno) {
if (bp->b_blkno != fsbtodb(fs, bno))
panic("ffs_realloccg: bad blockno");
delta = btodb(nsize - osize);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allocbuf(bp, nsize);
bp->b_flags |= B_DONE;
vfs_bio_bzero_buf(bp, osize, nsize - osize);
if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO)
vfs_bio_set_valid(bp, osize, nsize - osize);
*bpp = bp;
return (0);
}
/*
* Allocate a new disk location.
*/
if (bpref >= fs->fs_size)
bpref = 0;
switch ((int)fs->fs_optim) {
case FS_OPTSPACE:
/*
* Allocate an exact sized fragment. Although this makes
* best use of space, we will waste time relocating it if
* the file continues to grow. If the fragmentation is
* less than half of the minimum free reserve, we choose
* to begin optimizing for time.
*/
request = nsize;
if (fs->fs_minfree <= 5 ||
fs->fs_cstotal.cs_nffree >
(off_t)fs->fs_dsize * fs->fs_minfree / (2 * 100))
break;
log(LOG_NOTICE, "%s: optimization changed from SPACE to TIME\n",
fs->fs_fsmnt);
fs->fs_optim = FS_OPTTIME;
break;
case FS_OPTTIME:
/*
* At this point we have discovered a file that is trying to
* grow a small fragment to a larger fragment. To save time,
* we allocate a full sized block, then free the unused portion.
* If the file continues to grow, the `ffs_fragextend' call
* above will be able to grow it in place without further
* copying. If aberrant programs cause disk fragmentation to
* grow within 2% of the free reserve, we choose to begin
* optimizing for space.
*/
request = fs->fs_bsize;
if (fs->fs_cstotal.cs_nffree <
(off_t)fs->fs_dsize * (fs->fs_minfree - 2) / 100)
break;
log(LOG_NOTICE, "%s: optimization changed from TIME to SPACE\n",
fs->fs_fsmnt);
fs->fs_optim = FS_OPTSPACE;
break;
default:
printf("dev = %s, optim = %ld, fs = %s\n",
devtoname(ump->um_dev), (long)fs->fs_optim, fs->fs_fsmnt);
panic("ffs_realloccg: bad optim");
/* NOTREACHED */
}
bno = ffs_hashalloc(ip, cg, bpref, request, nsize, ffs_alloccg);
if (bno > 0) {
bp->b_blkno = fsbtodb(fs, bno);
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a smaller fragment that
* was just allocated has been replaced with a bigger
* fragment or a full-size block. If it is marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the block was written
* earlier, but very uncommon. If the block has never
* been written, there is no need to send a BIO_DELETE
* for it when it is freed. The gain from avoiding the
* TRIMs for the common case of unwritten blocks far
* exceeds the cost of the write amplification for the
* uncommon case of failing to send a TRIM for a block
* that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp, bprev, (long)osize,
ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
delta = btodb(nsize - osize);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta);
if (flags & IO_EXT)
UFS_INODE_SET_FLAG(ip, IN_CHANGE);
else
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
allocbuf(bp, nsize);
bp->b_flags |= B_DONE;
vfs_bio_bzero_buf(bp, osize, nsize - osize);
if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO)
vfs_bio_set_valid(bp, osize, nsize - osize);
*bpp = bp;
return (0);
}
#ifdef QUOTA
UFS_UNLOCK(ump);
/*
* Restore user's disk quota because allocation failed.
*/
(void) chkdq(ip, -btodb(nsize - osize), cred, FORCE);
UFS_LOCK(ump);
#endif
nospace:
/*
* no space available
*/
if (reclaimed == 0 && (flags & IO_BUFLOCKED) == 0) {
reclaimed = 1;
UFS_UNLOCK(ump);
if (bp) {
brelse(bp);
bp = NULL;
}
UFS_LOCK(ump);
softdep_request_cleanup(fs, vp, cred, FLUSH_BLOCKS_WAIT);
goto retry;
}
if (reclaimed > 0 &&
ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, ip->i_number, "filesystem full");
uprintf("\n%s: write failed, filesystem is full\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
if (bp)
brelse(bp);
return (ENOSPC);
}
/*
* Reallocate a sequence of blocks into a contiguous sequence of blocks.
*
* The vnode and an array of buffer pointers for a range of sequential
* logical blocks to be made contiguous is given. The allocator attempts
* to find a range of sequential blocks starting as close as possible
* from the end of the allocation for the logical block immediately
* preceding the current range. If successful, the physical block numbers
* in the buffer pointers and in the inode are changed to reflect the new
* allocation. If unsuccessful, the allocation is left unchanged. The
* success in doing the reallocation is returned. Note that the error
* return is not reflected back to the user. Rather the previous block
* allocation will be used.
*/
SYSCTL_NODE(_vfs, OID_AUTO, ffs, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"FFS filesystem");
static int doasyncfree = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doasyncfree, CTLFLAG_RW, &doasyncfree, 0,
"do not force synchronous writes when blocks are reallocated");
static int doreallocblks = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doreallocblks, CTLFLAG_RW, &doreallocblks, 0,
"enable block reallocation");
static int dotrimcons = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, dotrimcons, CTLFLAG_RWTUN, &dotrimcons, 0,
"enable BIO_DELETE / TRIM consolidation");
static int maxclustersearch = 10;
SYSCTL_INT(_vfs_ffs, OID_AUTO, maxclustersearch, CTLFLAG_RW, &maxclustersearch,
0, "max number of cylinder group to search for contigous blocks");
#ifdef DIAGNOSTIC
static int prtrealloc = 0;
SYSCTL_INT(_debug, OID_AUTO, ffs_prtrealloc, CTLFLAG_RW, &prtrealloc, 0,
"print out FFS filesystem block reallocation operations");
#endif
int
ffs_reallocblks(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct ufsmount *ump;
/*
* We used to skip reallocating the blocks of a file into a
* contiguous sequence if the underlying flash device requested
* BIO_DELETE notifications, because devices that benefit from
* BIO_DELETE also benefit from not moving the data. However,
* the destination for the data is usually moved before the data
* is written to the initially allocated location, so we rarely
* suffer the penalty of extra writes. With the addition of the
* consolidation of contiguous blocks into single BIO_DELETE
* operations, having fewer but larger contiguous blocks reduces
* the number of (slow and expensive) BIO_DELETE operations. So
* when doing BIO_DELETE consolidation, we do block reallocation.
*
* Skip if reallocblks has been disabled globally.
*/
ump = ap->a_vp->v_mount->mnt_data;
if ((((ump->um_flags) & UM_CANDELETE) != 0 && dotrimcons == 0) ||
doreallocblks == 0)
return (ENOSPC);
/*
* We can't wait in softdep prealloc as it may fsync and recurse
* here. Instead we simply fail to reallocate blocks if this
* rare condition arises.
*/
if (DOINGSOFTDEP(ap->a_vp))
if (softdep_prealloc(ap->a_vp, MNT_NOWAIT) != 0)
return (ENOSPC);
if (ump->um_fstype == UFS1)
return (ffs_reallocblks_ufs1(ap));
return (ffs_reallocblks_ufs2(ap));
}
static int
ffs_reallocblks_ufs1(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct fs *fs;
struct inode *ip;
struct vnode *vp;
struct buf *sbp, *ebp, *bp;
ufs1_daddr_t *bap, *sbap, *ebap;
struct cluster_save *buflist;
struct ufsmount *ump;
ufs_lbn_t start_lbn, end_lbn;
ufs1_daddr_t soff, newblk, blkno;
ufs2_daddr_t pref;
struct indir start_ap[UFS_NIADDR + 1], end_ap[UFS_NIADDR + 1], *idp;
int i, cg, len, start_lvl, end_lvl, ssize;
vp = ap->a_vp;
ip = VTOI(vp);
ump = ITOUMP(ip);
fs = ump->um_fs;
/*
* If we are not tracking block clusters or if we have less than 4%
* free blocks left, then do not attempt to cluster. Running with
* less than 5% free block reserve is not recommended and those that
* choose to do so do not expect to have good file layout.
*/
if (fs->fs_contigsumsize <= 0 || freespace(fs, 4) < 0)
return (ENOSPC);
buflist = ap->a_buflist;
len = buflist->bs_nchildren;
start_lbn = buflist->bs_children[0]->b_lblkno;
end_lbn = start_lbn + len - 1;
#ifdef INVARIANTS
for (i = 0; i < len; i++)
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 1");
for (i = 1; i < len; i++)
if (buflist->bs_children[i]->b_lblkno != start_lbn + i)
panic("ffs_reallocblks: non-logical cluster");
blkno = buflist->bs_children[0]->b_blkno;
ssize = fsbtodb(fs, fs->fs_frag);
for (i = 1; i < len - 1; i++)
if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize))
panic("ffs_reallocblks: non-physical cluster %d", i);
#endif
/*
* If the cluster crosses the boundary for the first indirect
* block, leave space for the indirect block. Indirect blocks
* are initially laid out in a position after the last direct
* block. Block reallocation would usually destroy locality by
* moving the indirect block out of the way to make room for
* data blocks if we didn't compensate here. We should also do
* this for other indirect block boundaries, but it is only
* important for the first one.
*/
if (start_lbn < UFS_NDADDR && end_lbn >= UFS_NDADDR)
return (ENOSPC);
/*
* If the latest allocation is in a new cylinder group, assume that
* the filesystem has decided to move and do not force it back to
* the previous cylinder group.
*/
if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) !=
dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno)))
return (ENOSPC);
if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) ||
ufs_getlbns(vp, end_lbn, end_ap, &end_lvl))
return (ENOSPC);
/*
* Get the starting offset and block map for the first block.
*/
if (start_lvl == 0) {
sbap = &ip->i_din1->di_db[0];
soff = start_lbn;
} else {
idp = &start_ap[start_lvl - 1];
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) {
brelse(sbp);
return (ENOSPC);
}
sbap = (ufs1_daddr_t *)sbp->b_data;
soff = idp->in_off;
}
/*
* If the block range spans two block maps, get the second map.
*/
ebap = NULL;
if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) {
ssize = len;
} else {
#ifdef INVARIANTS
if (start_lvl > 0 &&
start_ap[start_lvl - 1].in_lbn == idp->in_lbn)
panic("ffs_reallocblk: start == end");
#endif
ssize = len - (idp->in_off + 1);
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp))
goto fail;
ebap = (ufs1_daddr_t *)ebp->b_data;
}
/*
* Find the preferred location for the cluster. If we have not
* previously failed at this endeavor, then follow our standard
* preference calculation. If we have failed at it, then pick up
* where we last ended our search.
*/
UFS_LOCK(ump);
if (ip->i_nextclustercg == -1)
pref = ffs_blkpref_ufs1(ip, start_lbn, soff, sbap);
else
pref = cgdata(fs, ip->i_nextclustercg);
/*
* Search the block map looking for an allocation of the desired size.
* To avoid wasting too much time, we limit the number of cylinder
* groups that we will search.
*/
cg = dtog(fs, pref);
for (i = min(maxclustersearch, fs->fs_ncg); i > 0; i--) {
if ((newblk = ffs_clusteralloc(ip, cg, pref, len)) != 0)
break;
cg += 1;
if (cg >= fs->fs_ncg)
cg = 0;
}
/*
* If we have failed in our search, record where we gave up for
* next time. Otherwise, fall back to our usual search citerion.
*/
if (newblk == 0) {
ip->i_nextclustercg = cg;
UFS_UNLOCK(ump);
goto fail;
}
ip->i_nextclustercg = -1;
/*
* We have found a new contiguous block.
*
* First we have to replace the old block pointers with the new
* block pointers in the inode and indirect blocks associated
* with the file.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("realloc: ino %ju, lbns %jd-%jd\n\told:",
(uintmax_t)ip->i_number,
(intmax_t)start_lbn, (intmax_t)end_lbn);
#endif
blkno = newblk;
for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) {
if (i == ssize) {
bap = ebap;
soff = -i;
}
#ifdef INVARIANTS
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 2");
if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap)
panic("ffs_reallocblks: alloc mismatch");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %d,", *bap);
#endif
if (DOINGSOFTDEP(vp)) {
if (sbap == &ip->i_din1->di_db[0] && i < ssize)
softdep_setup_allocdirect(ip, start_lbn + i,
blkno, *bap, fs->fs_bsize, fs->fs_bsize,
buflist->bs_children[i]);
else
softdep_setup_allocindir_page(ip, start_lbn + i,
i < ssize ? sbp : ebp, soff + i, blkno,
*bap, buflist->bs_children[i]);
}
*bap++ = blkno;
}
/*
* Next we must write out the modified inode and indirect blocks.
* For strict correctness, the writes should be synchronous since
* the old block values may have been written to disk. In practise
* they are almost never written, but if we are concerned about
* strict correctness, the `doasyncfree' flag should be set to zero.
*
* The test on `doasyncfree' should be changed to test a flag
* that shows whether the associated buffers and inodes have
* been written. The flag should be set when the cluster is
* started and cleared whenever the buffer or inode is flushed.
* We can then check below to see if it is set, and do the
* synchronous write only when it has been cleared.
*/
if (sbap != &ip->i_din1->di_db[0]) {
if (doasyncfree)
bdwrite(sbp);
else
bwrite(sbp);
} else {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (!doasyncfree)
ffs_update(vp, 1);
}
if (ssize < len) {
if (doasyncfree)
bdwrite(ebp);
else
bwrite(ebp);
}
/*
* Last, free the old blocks and assign the new blocks to the buffers.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("\n\tnew:");
#endif
for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) {
bp = buflist->bs_children[i];
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a set of N-contiguous blocks
* that was just allocated has been replaced with a
* set of N+1-contiguous blocks. If they are marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the blocks were written
* earlier, but very uncommon. If the blocks have never
* been written, there is no need to send a BIO_DELETE
* for them when they are freed. The gain from avoiding
* the TRIMs for the common case of unwritten blocks
* far exceeds the cost of the write amplification for
* the uncommon case of failing to send a TRIM for the
* blocks that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp,
dbtofsb(fs, bp->b_blkno),
fs->fs_bsize, ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
bp->b_blkno = fsbtodb(fs, blkno);
#ifdef INVARIANTS
if (!ffs_checkblk(ip, dbtofsb(fs, bp->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 3");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %d,", blkno);
#endif
}
#ifdef DIAGNOSTIC
if (prtrealloc) {
prtrealloc--;
printf("\n");
}
#endif
return (0);
fail:
if (ssize < len)
brelse(ebp);
if (sbap != &ip->i_din1->di_db[0])
brelse(sbp);
return (ENOSPC);
}
static int
ffs_reallocblks_ufs2(ap)
struct vop_reallocblks_args /* {
struct vnode *a_vp;
struct cluster_save *a_buflist;
} */ *ap;
{
struct fs *fs;
struct inode *ip;
struct vnode *vp;
struct buf *sbp, *ebp, *bp;
ufs2_daddr_t *bap, *sbap, *ebap;
struct cluster_save *buflist;
struct ufsmount *ump;
ufs_lbn_t start_lbn, end_lbn;
ufs2_daddr_t soff, newblk, blkno, pref;
struct indir start_ap[UFS_NIADDR + 1], end_ap[UFS_NIADDR + 1], *idp;
int i, cg, len, start_lvl, end_lvl, ssize;
vp = ap->a_vp;
ip = VTOI(vp);
ump = ITOUMP(ip);
fs = ump->um_fs;
/*
* If we are not tracking block clusters or if we have less than 4%
* free blocks left, then do not attempt to cluster. Running with
* less than 5% free block reserve is not recommended and those that
* choose to do so do not expect to have good file layout.
*/
if (fs->fs_contigsumsize <= 0 || freespace(fs, 4) < 0)
return (ENOSPC);
buflist = ap->a_buflist;
len = buflist->bs_nchildren;
start_lbn = buflist->bs_children[0]->b_lblkno;
end_lbn = start_lbn + len - 1;
#ifdef INVARIANTS
for (i = 0; i < len; i++)
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 1");
for (i = 1; i < len; i++)
if (buflist->bs_children[i]->b_lblkno != start_lbn + i)
panic("ffs_reallocblks: non-logical cluster");
blkno = buflist->bs_children[0]->b_blkno;
ssize = fsbtodb(fs, fs->fs_frag);
for (i = 1; i < len - 1; i++)
if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize))
panic("ffs_reallocblks: non-physical cluster %d", i);
#endif
/*
* If the cluster crosses the boundary for the first indirect
* block, do not move anything in it. Indirect blocks are
* usually initially laid out in a position between the data
* blocks. Block reallocation would usually destroy locality by
* moving the indirect block out of the way to make room for
* data blocks if we didn't compensate here. We should also do
* this for other indirect block boundaries, but it is only
* important for the first one.
*/
if (start_lbn < UFS_NDADDR && end_lbn >= UFS_NDADDR)
return (ENOSPC);
/*
* If the latest allocation is in a new cylinder group, assume that
* the filesystem has decided to move and do not force it back to
* the previous cylinder group.
*/
if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) !=
dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno)))
return (ENOSPC);
if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) ||
ufs_getlbns(vp, end_lbn, end_ap, &end_lvl))
return (ENOSPC);
/*
* Get the starting offset and block map for the first block.
*/
if (start_lvl == 0) {
sbap = &ip->i_din2->di_db[0];
soff = start_lbn;
} else {
idp = &start_ap[start_lvl - 1];
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) {
brelse(sbp);
return (ENOSPC);
}
sbap = (ufs2_daddr_t *)sbp->b_data;
soff = idp->in_off;
}
/*
* If the block range spans two block maps, get the second map.
*/
ebap = NULL;
if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) {
ssize = len;
} else {
#ifdef INVARIANTS
if (start_lvl > 0 &&
start_ap[start_lvl - 1].in_lbn == idp->in_lbn)
panic("ffs_reallocblk: start == end");
#endif
ssize = len - (idp->in_off + 1);
if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp))
goto fail;
ebap = (ufs2_daddr_t *)ebp->b_data;
}
/*
* Find the preferred location for the cluster. If we have not
* previously failed at this endeavor, then follow our standard
* preference calculation. If we have failed at it, then pick up
* where we last ended our search.
*/
UFS_LOCK(ump);
if (ip->i_nextclustercg == -1)
pref = ffs_blkpref_ufs2(ip, start_lbn, soff, sbap);
else
pref = cgdata(fs, ip->i_nextclustercg);
/*
* Search the block map looking for an allocation of the desired size.
* To avoid wasting too much time, we limit the number of cylinder
* groups that we will search.
*/
cg = dtog(fs, pref);
for (i = min(maxclustersearch, fs->fs_ncg); i > 0; i--) {
if ((newblk = ffs_clusteralloc(ip, cg, pref, len)) != 0)
break;
cg += 1;
if (cg >= fs->fs_ncg)
cg = 0;
}
/*
* If we have failed in our search, record where we gave up for
* next time. Otherwise, fall back to our usual search citerion.
*/
if (newblk == 0) {
ip->i_nextclustercg = cg;
UFS_UNLOCK(ump);
goto fail;
}
ip->i_nextclustercg = -1;
/*
* We have found a new contiguous block.
*
* First we have to replace the old block pointers with the new
* block pointers in the inode and indirect blocks associated
* with the file.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("realloc: ino %ju, lbns %jd-%jd\n\told:", (uintmax_t)ip->i_number,
(intmax_t)start_lbn, (intmax_t)end_lbn);
#endif
blkno = newblk;
for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) {
if (i == ssize) {
bap = ebap;
soff = -i;
}
#ifdef INVARIANTS
if (!ffs_checkblk(ip,
dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 2");
if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap)
panic("ffs_reallocblks: alloc mismatch");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %jd,", (intmax_t)*bap);
#endif
if (DOINGSOFTDEP(vp)) {
if (sbap == &ip->i_din2->di_db[0] && i < ssize)
softdep_setup_allocdirect(ip, start_lbn + i,
blkno, *bap, fs->fs_bsize, fs->fs_bsize,
buflist->bs_children[i]);
else
softdep_setup_allocindir_page(ip, start_lbn + i,
i < ssize ? sbp : ebp, soff + i, blkno,
*bap, buflist->bs_children[i]);
}
*bap++ = blkno;
}
/*
* Next we must write out the modified inode and indirect blocks.
* For strict correctness, the writes should be synchronous since
* the old block values may have been written to disk. In practise
* they are almost never written, but if we are concerned about
* strict correctness, the `doasyncfree' flag should be set to zero.
*
* The test on `doasyncfree' should be changed to test a flag
* that shows whether the associated buffers and inodes have
* been written. The flag should be set when the cluster is
* started and cleared whenever the buffer or inode is flushed.
* We can then check below to see if it is set, and do the
* synchronous write only when it has been cleared.
*/
if (sbap != &ip->i_din2->di_db[0]) {
if (doasyncfree)
bdwrite(sbp);
else
bwrite(sbp);
} else {
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
if (!doasyncfree)
ffs_update(vp, 1);
}
if (ssize < len) {
if (doasyncfree)
bdwrite(ebp);
else
bwrite(ebp);
}
/*
* Last, free the old blocks and assign the new blocks to the buffers.
*/
#ifdef DIAGNOSTIC
if (prtrealloc)
printf("\n\tnew:");
#endif
for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) {
bp = buflist->bs_children[i];
if (!DOINGSOFTDEP(vp))
/*
* The usual case is that a set of N-contiguous blocks
* that was just allocated has been replaced with a
* set of N+1-contiguous blocks. If they are marked as
* B_DELWRI, the current contents have not been written
* to disk. It is possible that the blocks were written
* earlier, but very uncommon. If the blocks have never
* been written, there is no need to send a BIO_DELETE
* for them when they are freed. The gain from avoiding
* the TRIMs for the common case of unwritten blocks
* far exceeds the cost of the write amplification for
* the uncommon case of failing to send a TRIM for the
* blocks that had been written.
*/
ffs_blkfree(ump, fs, ump->um_devvp,
dbtofsb(fs, bp->b_blkno),
fs->fs_bsize, ip->i_number, vp->v_type, NULL,
(bp->b_flags & B_DELWRI) != 0 ?
NOTRIM_KEY : SINGLETON_KEY);
bp->b_blkno = fsbtodb(fs, blkno);
#ifdef INVARIANTS
if (!ffs_checkblk(ip, dbtofsb(fs, bp->b_blkno), fs->fs_bsize))
panic("ffs_reallocblks: unallocated block 3");
#endif
#ifdef DIAGNOSTIC
if (prtrealloc)
printf(" %jd,", (intmax_t)blkno);
#endif
}
#ifdef DIAGNOSTIC
if (prtrealloc) {
prtrealloc--;
printf("\n");
}
#endif
return (0);
fail:
if (ssize < len)
brelse(ebp);
if (sbap != &ip->i_din2->di_db[0])
brelse(sbp);
return (ENOSPC);
}
/*
* Allocate an inode in the filesystem.
*
* If allocating a directory, use ffs_dirpref to select the inode.
* If allocating in a directory, the following hierarchy is followed:
* 1) allocate the preferred inode.
* 2) allocate an inode in the same cylinder group.
* 3) quadradically rehash into other cylinder groups, until an
* available inode is located.
* If no inode preference is given the following hierarchy is used
* to allocate an inode:
* 1) allocate an inode in cylinder group 0.
* 2) quadradically rehash into other cylinder groups, until an
* available inode is located.
*/
int
ffs_valloc(pvp, mode, cred, vpp)
struct vnode *pvp;
int mode;
struct ucred *cred;
struct vnode **vpp;
{
struct inode *pip;
struct fs *fs;
struct inode *ip;
struct timespec ts;
struct ufsmount *ump;
ino_t ino, ipref;
u_int cg;
int error, error1, reclaimed;
*vpp = NULL;
pip = VTOI(pvp);
ump = ITOUMP(pip);
fs = ump->um_fs;
UFS_LOCK(ump);
reclaimed = 0;
retry:
if (fs->fs_cstotal.cs_nifree == 0)
goto noinodes;
if ((mode & IFMT) == IFDIR)
ipref = ffs_dirpref(pip);
else
ipref = pip->i_number;
if (ipref >= fs->fs_ncg * fs->fs_ipg)
ipref = 0;
cg = ino_to_cg(fs, ipref);
/*
* Track number of dirs created one after another
* in a same cg without intervening by files.
*/
if ((mode & IFMT) == IFDIR) {
if (fs->fs_contigdirs[cg] < 255)
fs->fs_contigdirs[cg]++;
} else {
if (fs->fs_contigdirs[cg] > 0)
fs->fs_contigdirs[cg]--;
}
ino = (ino_t)ffs_hashalloc(pip, cg, ipref, mode, 0,
(allocfcn_t *)ffs_nodealloccg);
if (ino == 0)
goto noinodes;
/*
* Get rid of the cached old vnode, force allocation of a new vnode
* for this inode.
*/
error = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp, FFSV_REPLACE);
if (error) {
error1 = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp,
FFSV_FORCEINSMQ | FFSV_REPLACE);
ffs_vfree(pvp, ino, mode);
if (error1 == 0) {
ip = VTOI(*vpp);
if (ip->i_mode)
goto dup_alloc;
UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
vput(*vpp);
}
return (error);
}
ip = VTOI(*vpp);
if (ip->i_mode) {
dup_alloc:
printf("mode = 0%o, inum = %ju, fs = %s\n",
ip->i_mode, (uintmax_t)ip->i_number, fs->fs_fsmnt);
panic("ffs_valloc: dup alloc");
}
if (DIP(ip, i_blocks) && (fs->fs_flags & FS_UNCLEAN) == 0) { /* XXX */
printf("free inode %s/%lu had %ld blocks\n",
fs->fs_fsmnt, (u_long)ino, (long)DIP(ip, i_blocks));
DIP_SET(ip, i_blocks, 0);
}
ip->i_flags = 0;
DIP_SET(ip, i_flags, 0);
/*
* Set up a new generation number for this inode.
*/
while (ip->i_gen == 0 || ++ip->i_gen == 0)
ip->i_gen = arc4random();
DIP_SET(ip, i_gen, ip->i_gen);
if (fs->fs_magic == FS_UFS2_MAGIC) {
vfs_timestamp(&ts);
ip->i_din2->di_birthtime = ts.tv_sec;
ip->i_din2->di_birthnsec = ts.tv_nsec;
}
ip->i_flag = 0;
(*vpp)->v_vflag = 0;
(*vpp)->v_type = VNON;
if (fs->fs_magic == FS_UFS2_MAGIC) {
(*vpp)->v_op = &ffs_vnodeops2;
UFS_INODE_SET_FLAG(ip, IN_UFS2);
} else {
(*vpp)->v_op = &ffs_vnodeops1;
}
return (0);
noinodes:
if (reclaimed == 0) {
reclaimed = 1;
softdep_request_cleanup(fs, pvp, cred, FLUSH_INODES_WAIT);
goto retry;
}
if (ppsratecheck(&ump->um_last_fullmsg, &ump->um_secs_fullmsg, 1)) {
UFS_UNLOCK(ump);
ffs_fserr(fs, pip->i_number, "out of inodes");
uprintf("\n%s: create/symlink failed, no inodes free\n",
fs->fs_fsmnt);
} else {
UFS_UNLOCK(ump);
}
return (ENOSPC);
}
/*
* Find a cylinder group to place a directory.
*
* The policy implemented by this algorithm is to allocate a
* directory inode in the same cylinder group as its parent
* directory, but also to reserve space for its files inodes
* and data. Restrict the number of directories which may be
* allocated one after another in the same cylinder group
* without intervening allocation of files.
*
* If we allocate a first level directory then force allocation
* in another cylinder group.
*/
static ino_t
ffs_dirpref(pip)
struct inode *pip;
{
struct fs *fs;
int cg, prefcg, dirsize, cgsize;
u_int avgifree, avgbfree, avgndir, curdirsize;
u_int minifree, minbfree, maxndir;
u_int mincg, minndir;
u_int maxcontigdirs;
mtx_assert(UFS_MTX(ITOUMP(pip)), MA_OWNED);
fs = ITOFS(pip);
avgifree = fs->fs_cstotal.cs_nifree / fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
avgndir = fs->fs_cstotal.cs_ndir / fs->fs_ncg;
/*
* Force allocation in another cg if creating a first level dir.
*/
ASSERT_VOP_LOCKED(ITOV(pip), "ffs_dirpref");
if (ITOV(pip)->v_vflag & VV_ROOT) {
prefcg = arc4random() % fs->fs_ncg;
mincg = prefcg;
minndir = fs->fs_ipg;
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
mincg = cg;
minndir = fs->fs_cs(fs, cg).cs_ndir;
}
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
mincg = cg;
minndir = fs->fs_cs(fs, cg).cs_ndir;
}
return ((ino_t)(fs->fs_ipg * mincg));
}
/*
* Count various limits which used for
* optimal allocation of a directory inode.
*/
maxndir = min(avgndir + fs->fs_ipg / 16, fs->fs_ipg);
minifree = avgifree - avgifree / 4;
if (minifree < 1)
minifree = 1;
minbfree = avgbfree - avgbfree / 4;
if (minbfree < 1)
minbfree = 1;
cgsize = fs->fs_fsize * fs->fs_fpg;
dirsize = fs->fs_avgfilesize * fs->fs_avgfpdir;
curdirsize = avgndir ? (cgsize - avgbfree * fs->fs_bsize) / avgndir : 0;
if (dirsize < curdirsize)
dirsize = curdirsize;
if (dirsize <= 0)
maxcontigdirs = 0; /* dirsize overflowed */
else
maxcontigdirs = min((avgbfree * fs->fs_bsize) / dirsize, 255);
if (fs->fs_avgfpdir > 0)
maxcontigdirs = min(maxcontigdirs,
fs->fs_ipg / fs->fs_avgfpdir);
if (maxcontigdirs == 0)
maxcontigdirs = 1;
/*
* Limit number of dirs in one cg and reserve space for
* regular files, but only if we have no deficit in
* inodes or space.
*
* We are trying to find a suitable cylinder group nearby
* our preferred cylinder group to place a new directory.
* We scan from our preferred cylinder group forward looking
* for a cylinder group that meets our criterion. If we get
* to the final cylinder group and do not find anything,
* we start scanning forwards from the beginning of the
* filesystem. While it might seem sensible to start scanning
* backwards or even to alternate looking forward and backward,
* this approach fails badly when the filesystem is nearly full.
* Specifically, we first search all the areas that have no space
* and finally try the one preceding that. We repeat this on
* every request and in the case of the final block end up
* searching the entire filesystem. By jumping to the front
* of the filesystem, our future forward searches always look
* in new cylinder groups so finds every possible block after
* one pass over the filesystem.
*/
prefcg = ino_to_cg(fs, pip->i_number);
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
fs->fs_cs(fs, cg).cs_nifree >= minifree &&
fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
if (fs->fs_contigdirs[cg] < maxcontigdirs)
return ((ino_t)(fs->fs_ipg * cg));
}
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
fs->fs_cs(fs, cg).cs_nifree >= minifree &&
fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
if (fs->fs_contigdirs[cg] < maxcontigdirs)
return ((ino_t)(fs->fs_ipg * cg));
}
/*
* This is a backstop when we have deficit in space.
*/
for (cg = prefcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
return ((ino_t)(fs->fs_ipg * cg));
for (cg = 0; cg < prefcg; cg++)
if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
break;
return ((ino_t)(fs->fs_ipg * cg));
}
/*
* Select the desired position for the next block in a file. The file is
* logically divided into sections. The first section is composed of the
* direct blocks and the next fs_maxbpg blocks. Each additional section
* contains fs_maxbpg blocks.
*
* If no blocks have been allocated in the first section, the policy is to
* request a block in the same cylinder group as the inode that describes
* the file. The first indirect is allocated immediately following the last
* direct block and the data blocks for the first indirect immediately
* follow it.
*
* If no blocks have been allocated in any other section, the indirect
* block(s) are allocated in the same cylinder group as its inode in an
* area reserved immediately following the inode blocks. The policy for
* the data blocks is to place them in a cylinder group with a greater than
* average number of free blocks. An appropriate cylinder group is found
* by using a rotor that sweeps the cylinder groups. When a new group of
* blocks is needed, the sweep begins in the cylinder group following the
* cylinder group from which the previous allocation was made. The sweep
* continues until a cylinder group with greater than the average number
* of free blocks is found. If the allocation is for the first block in an
* indirect block or the previous block is a hole, then the information on
* the previous allocation is unavailable; here a best guess is made based
* on the logical block number being allocated.
*
* If a section is already partially allocated, the policy is to
* allocate blocks contiguously within the section if possible.
*/
ufs2_daddr_t
ffs_blkpref_ufs1(ip, lbn, indx, bap)
struct inode *ip;
ufs_lbn_t lbn;
int indx;
ufs1_daddr_t *bap;
{
struct fs *fs;
u_int cg, inocg;
u_int avgbfree, startcg;
ufs2_daddr_t pref, prevbn;
KASSERT(indx <= 0 || bap != NULL, ("need non-NULL bap"));
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
fs = ITOFS(ip);
/*
* Allocation of indirect blocks is indicated by passing negative
* values in indx: -1 for single indirect, -2 for double indirect,
* -3 for triple indirect. As noted below, we attempt to allocate
* the first indirect inline with the file data. For all later
* indirect blocks, the data is often allocated in other cylinder
* groups. However to speed random file access and to speed up
* fsck, the filesystem reserves the first fs_metaspace blocks
* (typically half of fs_minfree) of the data area of each cylinder
* group to hold these later indirect blocks.
*/
inocg = ino_to_cg(fs, ip->i_number);
if (indx < 0) {
/*
* Our preference for indirect blocks is the zone at the
* beginning of the inode's cylinder group data area that
* we try to reserve for indirect blocks.
*/
pref = cgmeta(fs, inocg);
/*
* If we are allocating the first indirect block, try to
* place it immediately following the last direct block.
*/
if (indx == -1 && lbn < UFS_NDADDR + NINDIR(fs) &&
ip->i_din1->di_db[UFS_NDADDR - 1] != 0)
pref = ip->i_din1->di_db[UFS_NDADDR - 1] + fs->fs_frag;
return (pref);
}
/*
* If we are allocating the first data block in the first indirect
* block and the indirect has been allocated in the data block area,
* try to place it immediately following the indirect block.
*/
if (lbn == UFS_NDADDR) {
pref = ip->i_din1->di_ib[0];
if (pref != 0 && pref >= cgdata(fs, inocg) &&
pref < cgbase(fs, inocg + 1))
return (pref + fs->fs_frag);
}
/*
* If we are at the beginning of a file, or we have already allocated
* the maximum number of blocks per cylinder group, or we do not
* have a block allocated immediately preceding us, then we need
* to decide where to start allocating new blocks.
*/
if (indx == 0) {
prevbn = 0;
} else {
prevbn = bap[indx - 1];
if (UFS_CHECK_BLKNO(ITOVFS(ip), ip->i_number, prevbn,
fs->fs_bsize) != 0)
prevbn = 0;
}
if (indx % fs->fs_maxbpg == 0 || prevbn == 0) {
/*
* If we are allocating a directory data block, we want
* to place it in the metadata area.
*/
if ((ip->i_mode & IFMT) == IFDIR)
return (cgmeta(fs, inocg));
/*
* Until we fill all the direct and all the first indirect's
* blocks, we try to allocate in the data area of the inode's
* cylinder group.
*/
if (lbn < UFS_NDADDR + NINDIR(fs))
return (cgdata(fs, inocg));
/*
* Find a cylinder with greater than average number of
* unused data blocks.
*/
if (indx == 0 || prevbn == 0)
startcg = inocg + lbn / fs->fs_maxbpg;
else
startcg = dtog(fs, prevbn) + 1;
startcg %= fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
for (cg = startcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
for (cg = 0; cg <= startcg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
return (0);
}
/*
* Otherwise, we just always try to lay things out contiguously.
*/
return (prevbn + fs->fs_frag);
}
/*
* Same as above, but for UFS2
*/
ufs2_daddr_t
ffs_blkpref_ufs2(ip, lbn, indx, bap)
struct inode *ip;
ufs_lbn_t lbn;
int indx;
ufs2_daddr_t *bap;
{
struct fs *fs;
u_int cg, inocg;
u_int avgbfree, startcg;
ufs2_daddr_t pref, prevbn;
KASSERT(indx <= 0 || bap != NULL, ("need non-NULL bap"));
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
fs = ITOFS(ip);
/*
* Allocation of indirect blocks is indicated by passing negative
* values in indx: -1 for single indirect, -2 for double indirect,
* -3 for triple indirect. As noted below, we attempt to allocate
* the first indirect inline with the file data. For all later
* indirect blocks, the data is often allocated in other cylinder
* groups. However to speed random file access and to speed up
* fsck, the filesystem reserves the first fs_metaspace blocks
* (typically half of fs_minfree) of the data area of each cylinder
* group to hold these later indirect blocks.
*/
inocg = ino_to_cg(fs, ip->i_number);
if (indx < 0) {
/*
* Our preference for indirect blocks is the zone at the
* beginning of the inode's cylinder group data area that
* we try to reserve for indirect blocks.
*/
pref = cgmeta(fs, inocg);
/*
* If we are allocating the first indirect block, try to
* place it immediately following the last direct block.
*/
if (indx == -1 && lbn < UFS_NDADDR + NINDIR(fs) &&
ip->i_din2->di_db[UFS_NDADDR - 1] != 0)
pref = ip->i_din2->di_db[UFS_NDADDR - 1] + fs->fs_frag;
return (pref);
}
/*
* If we are allocating the first data block in the first indirect
* block and the indirect has been allocated in the data block area,
* try to place it immediately following the indirect block.
*/
if (lbn == UFS_NDADDR) {
pref = ip->i_din2->di_ib[0];
if (pref != 0 && pref >= cgdata(fs, inocg) &&
pref < cgbase(fs, inocg + 1))
return (pref + fs->fs_frag);
}
/*
* If we are at the beginning of a file, or we have already allocated
* the maximum number of blocks per cylinder group, or we do not
* have a block allocated immediately preceding us, then we need
* to decide where to start allocating new blocks.
*/
if (indx == 0) {
prevbn = 0;
} else {
prevbn = bap[indx - 1];
if (UFS_CHECK_BLKNO(ITOVFS(ip), ip->i_number, prevbn,
fs->fs_bsize) != 0)
prevbn = 0;
}
if (indx % fs->fs_maxbpg == 0 || prevbn == 0) {
/*
* If we are allocating a directory data block, we want
* to place it in the metadata area.
*/
if ((ip->i_mode & IFMT) == IFDIR)
return (cgmeta(fs, inocg));
/*
* Until we fill all the direct and all the first indirect's
* blocks, we try to allocate in the data area of the inode's
* cylinder group.
*/
if (lbn < UFS_NDADDR + NINDIR(fs))
return (cgdata(fs, inocg));
/*
* Find a cylinder with greater than average number of
* unused data blocks.
*/
if (indx == 0 || prevbn == 0)
startcg = inocg + lbn / fs->fs_maxbpg;
else
startcg = dtog(fs, prevbn) + 1;
startcg %= fs->fs_ncg;
avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
for (cg = startcg; cg < fs->fs_ncg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
for (cg = 0; cg <= startcg; cg++)
if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
fs->fs_cgrotor = cg;
return (cgdata(fs, cg));
}
return (0);
}
/*
* Otherwise, we just always try to lay things out contiguously.
*/
return (prevbn + fs->fs_frag);
}
/*
* Implement the cylinder overflow algorithm.
*
* The policy implemented by this algorithm is:
* 1) allocate the block in its requested cylinder group.
* 2) quadradically rehash on the cylinder group number.
* 3) brute force search for a free block.
*
* Must be called with the UFS lock held. Will release the lock on success
* and return with it held on failure.
*/
/*VARARGS5*/
static ufs2_daddr_t
ffs_hashalloc(ip, cg, pref, size, rsize, allocator)
struct inode *ip;
u_int cg;
ufs2_daddr_t pref;
int size; /* Search size for data blocks, mode for inodes */
int rsize; /* Real allocated size. */
allocfcn_t *allocator;
{
struct fs *fs;
ufs2_daddr_t result;
u_int i, icg = cg;
mtx_assert(UFS_MTX(ITOUMP(ip)), MA_OWNED);
#ifdef INVARIANTS
if (ITOV(ip)->v_mount->mnt_kern_flag & MNTK_SUSPENDED)
panic("ffs_hashalloc: allocation on suspended filesystem");
#endif
fs = ITOFS(ip);
/*
* 1: preferred cylinder group
*/
result = (*allocator)(ip, cg, pref, size, rsize);
if (result)
return (result);
/*
* 2: quadratic rehash
*/
for (i = 1; i < fs->fs_ncg; i *= 2) {
cg += i;
if (cg >= fs->fs_ncg)
cg -= fs->fs_ncg;
result = (*allocator)(ip, cg, 0, size, rsize);
if (result)
return (result);
}
/*
* 3: brute force search
* Note that we start at i == 2, since 0 was checked initially,
* and 1 is always checked in the quadratic rehash.
*/
cg = (icg + 2) % fs->fs_ncg;
for (i = 2; i < fs->fs_ncg; i++) {
result = (*allocator)(ip, cg, 0, size, rsize);
if (result)
return (result);
cg++;
if (cg == fs->fs_ncg)
cg = 0;
}
return (0);
}
/*
* Determine whether a fragment can be extended.
*
* Check to see if the necessary fragments are available, and
* if they are, allocate them.
*/
static ufs2_daddr_t
ffs_fragextend(ip, cg, bprev, osize, nsize)
struct inode *ip;
u_int cg;
ufs2_daddr_t bprev;
int osize, nsize;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
int nffree;
long bno;
int frags, bbase;
int i, error;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_cs(fs, cg).cs_nffree < numfrags(fs, nsize - osize))
return (0);
frags = numfrags(fs, nsize);
bbase = fragnum(fs, bprev);
if (bbase > fragnum(fs, (bprev + frags - 1))) {
/* cannot extend across a block boundary */
return (0);
}
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0)
goto fail;
bno = dtogd(fs, bprev);
blksfree = cg_blksfree(cgp);
for (i = numfrags(fs, osize); i < frags; i++)
if (isclr(blksfree, bno + i))
goto fail;
/*
* the current fragment can be extended
* deduct the count on fragment being extended into
* increase the count on the remaining fragment (if any)
* allocate the extended piece
*/
for (i = frags; i < fs->fs_frag - bbase; i++)
if (isclr(blksfree, bno + i))
break;
cgp->cg_frsum[i - numfrags(fs, osize)]--;
if (i != frags)
cgp->cg_frsum[i - frags]++;
for (i = numfrags(fs, osize), nffree = 0; i < frags; i++) {
clrbit(blksfree, bno + i);
cgp->cg_cs.cs_nffree--;
nffree++;
}
UFS_LOCK(ump);
fs->fs_cstotal.cs_nffree -= nffree;
fs->fs_cs(fs, cg).cs_nffree -= nffree;
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), bprev,
frags, numfrags(fs, osize));
bdwrite(bp);
return (bprev);
fail:
brelse(bp);
UFS_LOCK(ump);
return (0);
}
/*
* Determine whether a block can be allocated.
*
* Check to see if a block of the appropriate size is available,
* and if it is, allocate it.
*/
static ufs2_daddr_t
ffs_alloccg(ip, cg, bpref, size, rsize)
struct inode *ip;
u_int cg;
ufs2_daddr_t bpref;
int size;
int rsize;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
ufs1_daddr_t bno;
ufs2_daddr_t blkno;
int i, allocsiz, error, frags;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_cs(fs, cg).cs_nbfree == 0 && size == fs->fs_bsize)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0 ||
(cgp->cg_cs.cs_nbfree == 0 && size == fs->fs_bsize))
goto fail;
if (size == fs->fs_bsize) {
UFS_LOCK(ump);
blkno = ffs_alloccgblk(ip, bp, bpref, rsize);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (blkno);
}
/*
* check to see if any fragments are already available
* allocsiz is the size which will be allocated, hacking
* it down to a smaller size if necessary
*/
blksfree = cg_blksfree(cgp);
frags = numfrags(fs, size);
for (allocsiz = frags; allocsiz < fs->fs_frag; allocsiz++)
if (cgp->cg_frsum[allocsiz] != 0)
break;
if (allocsiz == fs->fs_frag) {
/*
* no fragments were available, so a block will be
* allocated, and hacked up
*/
if (cgp->cg_cs.cs_nbfree == 0)
goto fail;
UFS_LOCK(ump);
blkno = ffs_alloccgblk(ip, bp, bpref, rsize);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (blkno);
}
KASSERT(size == rsize,
("ffs_alloccg: size(%d) != rsize(%d)", size, rsize));
bno = ffs_mapsearch(fs, cgp, bpref, allocsiz);
if (bno < 0)
goto fail;
for (i = 0; i < frags; i++)
clrbit(blksfree, bno + i);
cgp->cg_cs.cs_nffree -= frags;
cgp->cg_frsum[allocsiz]--;
if (frags != allocsiz)
cgp->cg_frsum[allocsiz - frags]++;
UFS_LOCK(ump);
fs->fs_cstotal.cs_nffree -= frags;
fs->fs_cs(fs, cg).cs_nffree -= frags;
fs->fs_fmod = 1;
blkno = cgbase(fs, cg) + bno;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno, frags, 0);
bdwrite(bp);
return (blkno);
fail:
brelse(bp);
UFS_LOCK(ump);
return (0);
}
/*
* Allocate a block in a cylinder group.
*
* This algorithm implements the following policy:
* 1) allocate the requested block.
* 2) allocate a rotationally optimal block in the same cylinder.
* 3) allocate the next available block on the block rotor for the
* specified cylinder group.
* Note that this routine only allocates fs_bsize blocks; these
* blocks may be fragmented by the routine that allocates them.
*/
static ufs2_daddr_t
ffs_alloccgblk(ip, bp, bpref, size)
struct inode *ip;
struct buf *bp;
ufs2_daddr_t bpref;
int size;
{
struct fs *fs;
struct cg *cgp;
struct ufsmount *ump;
ufs1_daddr_t bno;
ufs2_daddr_t blkno;
u_int8_t *blksfree;
int i, cgbpref;
ump = ITOUMP(ip);
fs = ump->um_fs;
mtx_assert(UFS_MTX(ump), MA_OWNED);
cgp = (struct cg *)bp->b_data;
blksfree = cg_blksfree(cgp);
if (bpref == 0) {
bpref = cgbase(fs, cgp->cg_cgx) + cgp->cg_rotor + fs->fs_frag;
} else if ((cgbpref = dtog(fs, bpref)) != cgp->cg_cgx) {
/* map bpref to correct zone in this cg */
if (bpref < cgdata(fs, cgbpref))
bpref = cgmeta(fs, cgp->cg_cgx);
else
bpref = cgdata(fs, cgp->cg_cgx);
}
/*
* if the requested block is available, use it
*/
bno = dtogd(fs, blknum(fs, bpref));
if (ffs_isblock(fs, blksfree, fragstoblks(fs, bno)))
goto gotit;
/*
* Take the next available block in this cylinder group.
*/
bno = ffs_mapsearch(fs, cgp, bpref, (int)fs->fs_frag);
if (bno < 0)
return (0);
/* Update cg_rotor only if allocated from the data zone */
if (bno >= dtogd(fs, cgdata(fs, cgp->cg_cgx)))
cgp->cg_rotor = bno;
gotit:
blkno = fragstoblks(fs, bno);
ffs_clrblock(fs, blksfree, (long)blkno);
ffs_clusteracct(fs, cgp, blkno, -1);
cgp->cg_cs.cs_nbfree--;
fs->fs_cstotal.cs_nbfree--;
fs->fs_cs(fs, cgp->cg_cgx).cs_nbfree--;
fs->fs_fmod = 1;
blkno = cgbase(fs, cgp->cg_cgx) + bno;
/*
* If the caller didn't want the whole block free the frags here.
*/
size = numfrags(fs, size);
if (size != fs->fs_frag) {
bno = dtogd(fs, blkno);
for (i = size; i < fs->fs_frag; i++)
setbit(blksfree, bno + i);
i = fs->fs_frag - size;
cgp->cg_cs.cs_nffree += i;
fs->fs_cstotal.cs_nffree += i;
fs->fs_cs(fs, cgp->cg_cgx).cs_nffree += i;
fs->fs_fmod = 1;
cgp->cg_frsum[i]++;
}
/* XXX Fixme. */
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno, size, 0);
UFS_LOCK(ump);
return (blkno);
}
/*
* Determine whether a cluster can be allocated.
*
* We do not currently check for optimal rotational layout if there
* are multiple choices in the same cylinder group. Instead we just
* take the first one that we find following bpref.
*/
static ufs2_daddr_t
ffs_clusteralloc(ip, cg, bpref, len)
struct inode *ip;
u_int cg;
ufs2_daddr_t bpref;
int len;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
struct ufsmount *ump;
int i, run, bit, map, got, error;
ufs2_daddr_t bno;
u_char *mapp;
int32_t *lp;
u_int8_t *blksfree;
ump = ITOUMP(ip);
fs = ump->um_fs;
if (fs->fs_maxcluster[cg] < len)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0) {
UFS_LOCK(ump);
return (0);
}
/*
* Check to see if a cluster of the needed size (or bigger) is
* available in this cylinder group.
*/
lp = &cg_clustersum(cgp)[len];
for (i = len; i <= fs->fs_contigsumsize; i++)
if (*lp++ > 0)
break;
if (i > fs->fs_contigsumsize) {
/*
* This is the first time looking for a cluster in this
* cylinder group. Update the cluster summary information
* to reflect the true maximum sized cluster so that
* future cluster allocation requests can avoid reading
* the cylinder group map only to find no clusters.
*/
lp = &cg_clustersum(cgp)[len - 1];
for (i = len - 1; i > 0; i--)
if (*lp-- > 0)
break;
UFS_LOCK(ump);
fs->fs_maxcluster[cg] = i;
brelse(bp);
return (0);
}
/*
* Search the cluster map to find a big enough cluster.
* We take the first one that we find, even if it is larger
* than we need as we prefer to get one close to the previous
* block allocation. We do not search before the current
* preference point as we do not want to allocate a block
* that is allocated before the previous one (as we will
* then have to wait for another pass of the elevator
* algorithm before it will be read). We prefer to fail and
* be recalled to try an allocation in the next cylinder group.
*/
if (dtog(fs, bpref) != cg)
bpref = cgdata(fs, cg);
else
bpref = blknum(fs, bpref);
bpref = fragstoblks(fs, dtogd(fs, bpref));
mapp = &cg_clustersfree(cgp)[bpref / NBBY];
map = *mapp++;
bit = 1 << (bpref % NBBY);
for (run = 0, got = bpref; got < cgp->cg_nclusterblks; got++) {
if ((map & bit) == 0) {
run = 0;
} else {
run++;
if (run == len)
break;
}
if ((got & (NBBY - 1)) != (NBBY - 1)) {
bit <<= 1;
} else {
map = *mapp++;
bit = 1;
}
}
if (got >= cgp->cg_nclusterblks) {
UFS_LOCK(ump);
brelse(bp);
return (0);
}
/*
* Allocate the cluster that we have found.
*/
blksfree = cg_blksfree(cgp);
for (i = 1; i <= len; i++)
if (!ffs_isblock(fs, blksfree, got - run + i))
panic("ffs_clusteralloc: map mismatch");
bno = cgbase(fs, cg) + blkstofrags(fs, got - run + 1);
if (dtog(fs, bno) != cg)
panic("ffs_clusteralloc: allocated out of group");
len = blkstofrags(fs, len);
UFS_LOCK(ump);
for (i = 0; i < len; i += fs->fs_frag)
if (ffs_alloccgblk(ip, bp, bno + i, fs->fs_bsize) != bno + i)
panic("ffs_clusteralloc: lost block");
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
bdwrite(bp);
return (bno);
}
static inline struct buf *
getinobuf(struct inode *ip, u_int cg, u_int32_t cginoblk, int gbflags)
{
struct fs *fs;
fs = ITOFS(ip);
return (getblk(ITODEVVP(ip), fsbtodb(fs, ino_to_fsba(fs,
cg * fs->fs_ipg + cginoblk)), (int)fs->fs_bsize, 0, 0,
gbflags));
}
/*
* Synchronous inode initialization is needed only when barrier writes do not
* work as advertised, and will impose a heavy cost on file creation in a newly
* created filesystem.
*/
static int doasyncinodeinit = 1;
SYSCTL_INT(_vfs_ffs, OID_AUTO, doasyncinodeinit, CTLFLAG_RWTUN,
&doasyncinodeinit, 0,
"Perform inode block initialization using asynchronous writes");
/*
* Determine whether an inode can be allocated.
*
* Check to see if an inode is available, and if it is,
* allocate it using the following policy:
* 1) allocate the requested inode.
* 2) allocate the next available inode after the requested
* inode in the specified cylinder group.
*/
static ufs2_daddr_t
ffs_nodealloccg(ip, cg, ipref, mode, unused)
struct inode *ip;
u_int cg;
ufs2_daddr_t ipref;
int mode;
int unused;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp, *ibp;
struct ufsmount *ump;
u_int8_t *inosused, *loc;
struct ufs2_dinode *dp2;
int error, start, len, i;
u_int32_t old_initediblk;
ump = ITOUMP(ip);
fs = ump->um_fs;
check_nifree:
if (fs->fs_cs(fs, cg).cs_nifree == 0)
return (0);
UFS_UNLOCK(ump);
if ((error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp)) != 0) {
UFS_LOCK(ump);
return (0);
}
restart:
if (cgp->cg_cs.cs_nifree == 0) {
brelse(bp);
UFS_LOCK(ump);
return (0);
}
inosused = cg_inosused(cgp);
if (ipref) {
ipref %= fs->fs_ipg;
if (isclr(inosused, ipref))
goto gotit;
}
start = cgp->cg_irotor / NBBY;
len = howmany(fs->fs_ipg - cgp->cg_irotor, NBBY);
loc = memcchr(&inosused[start], 0xff, len);
if (loc == NULL) {
len = start + 1;
start = 0;
loc = memcchr(&inosused[start], 0xff, len);
if (loc == NULL) {
printf("cg = %d, irotor = %ld, fs = %s\n",
cg, (long)cgp->cg_irotor, fs->fs_fsmnt);
panic("ffs_nodealloccg: map corrupted");
/* NOTREACHED */
}
}
ipref = (loc - inosused) * NBBY + ffs(~*loc) - 1;
gotit:
/*
* Check to see if we need to initialize more inodes.
*/
if (fs->fs_magic == FS_UFS2_MAGIC &&
ipref + INOPB(fs) > cgp->cg_initediblk &&
cgp->cg_initediblk < cgp->cg_niblk) {
old_initediblk = cgp->cg_initediblk;
/*
* Free the cylinder group lock before writing the
* initialized inode block. Entering the
* babarrierwrite() with the cylinder group lock
* causes lock order violation between the lock and
* snaplk.
*
* Another thread can decide to initialize the same
* inode block, but whichever thread first gets the
* cylinder group lock after writing the newly
* allocated inode block will update it and the other
* will realize that it has lost and leave the
* cylinder group unchanged.
*/
ibp = getinobuf(ip, cg, old_initediblk, GB_LOCK_NOWAIT);
brelse(bp);
if (ibp == NULL) {
/*
* The inode block buffer is already owned by
* another thread, which must initialize it.
* Wait on the buffer to allow another thread
* to finish the updates, with dropped cg
* buffer lock, then retry.
*/
ibp = getinobuf(ip, cg, old_initediblk, 0);
brelse(ibp);
UFS_LOCK(ump);
goto check_nifree;
}
bzero(ibp->b_data, (int)fs->fs_bsize);
dp2 = (struct ufs2_dinode *)(ibp->b_data);
for (i = 0; i < INOPB(fs); i++) {
while (dp2->di_gen == 0)
dp2->di_gen = arc4random();
dp2++;
}
/*
* Rather than adding a soft updates dependency to ensure
* that the new inode block is written before it is claimed
* by the cylinder group map, we just do a barrier write
* here. The barrier write will ensure that the inode block
* gets written before the updated cylinder group map can be
* written. The barrier write should only slow down bulk
* loading of newly created filesystems.
*/
if (doasyncinodeinit)
babarrierwrite(ibp);
else
bwrite(ibp);
/*
* After the inode block is written, try to update the
* cg initediblk pointer. If another thread beat us
* to it, then leave it unchanged as the other thread
* has already set it correctly.
*/
error = ffs_getcg(fs, ump->um_devvp, cg, 0, &bp, &cgp);
UFS_LOCK(ump);
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (error != 0)
return (error);
if (cgp->cg_initediblk == old_initediblk)
cgp->cg_initediblk += INOPB(fs);
goto restart;
}
cgp->cg_irotor = ipref;
UFS_LOCK(ump);
ACTIVECLEAR(fs, cg);
setbit(inosused, ipref);
cgp->cg_cs.cs_nifree--;
fs->fs_cstotal.cs_nifree--;
fs->fs_cs(fs, cg).cs_nifree--;
fs->fs_fmod = 1;
if ((mode & IFMT) == IFDIR) {
cgp->cg_cs.cs_ndir++;
fs->fs_cstotal.cs_ndir++;
fs->fs_cs(fs, cg).cs_ndir++;
}
UFS_UNLOCK(ump);
if (DOINGSOFTDEP(ITOV(ip)))
softdep_setup_inomapdep(bp, ip, cg * fs->fs_ipg + ipref, mode);
bdwrite(bp);
return ((ino_t)(cg * fs->fs_ipg + ipref));
}
/*
* Free a block or fragment.
*
* The specified block or fragment is placed back in the
* free map. If a fragment is deallocated, a possible
* block reassembly is checked.
*/
static void
ffs_blkfree_cg(ump, fs, devvp, bno, size, inum, dephd)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
struct workhead *dephd;
{
struct mount *mp;
struct cg *cgp;
struct buf *bp;
ufs1_daddr_t fragno, cgbno;
int i, blk, frags, bbase, error;
u_int cg;
u_int8_t *blksfree;
struct cdev *dev;
cg = dtog(fs, bno);
if (devvp->v_type == VREG) {
/* devvp is a snapshot */
MPASS(devvp->v_mount->mnt_data == ump);
dev = ump->um_devvp->v_rdev;
} else if (devvp->v_type == VCHR) {
/* devvp is a normal disk device */
dev = devvp->v_rdev;
ASSERT_VOP_LOCKED(devvp, "ffs_blkfree_cg");
} else
return;
#ifdef INVARIANTS
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0 ||
fragnum(fs, bno) + numfrags(fs, size) > fs->fs_frag) {
printf("dev=%s, bno = %jd, bsize = %ld, size = %ld, fs = %s\n",
devtoname(dev), (intmax_t)bno, (long)fs->fs_bsize,
size, fs->fs_fsmnt);
panic("ffs_blkfree_cg: bad size");
}
#endif
if ((u_int)bno >= fs->fs_size) {
printf("bad block %jd, ino %lu\n", (intmax_t)bno,
(u_long)inum);
ffs_fserr(fs, inum, "bad block");
return;
}
if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
return;
cgbno = dtogd(fs, bno);
blksfree = cg_blksfree(cgp);
UFS_LOCK(ump);
if (size == fs->fs_bsize) {
fragno = fragstoblks(fs, cgbno);
if (!ffs_isfreeblock(fs, blksfree, fragno)) {
if (devvp->v_type == VREG) {
UFS_UNLOCK(ump);
/* devvp is a snapshot */
brelse(bp);
return;
}
printf("dev = %s, block = %jd, fs = %s\n",
devtoname(dev), (intmax_t)bno, fs->fs_fsmnt);
panic("ffs_blkfree_cg: freeing free block");
}
ffs_setblock(fs, blksfree, fragno);
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
fs->fs_cstotal.cs_nbfree++;
fs->fs_cs(fs, cg).cs_nbfree++;
} else {
bbase = cgbno - fragnum(fs, cgbno);
/*
* decrement the counts associated with the old frags
*/
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, -1);
/*
* deallocate the fragment
*/
frags = numfrags(fs, size);
for (i = 0; i < frags; i++) {
if (isset(blksfree, cgbno + i)) {
printf("dev = %s, block = %jd, fs = %s\n",
devtoname(dev), (intmax_t)(bno + i),
fs->fs_fsmnt);
panic("ffs_blkfree_cg: freeing free frag");
}
setbit(blksfree, cgbno + i);
}
cgp->cg_cs.cs_nffree += i;
fs->fs_cstotal.cs_nffree += i;
fs->fs_cs(fs, cg).cs_nffree += i;
/*
* add back in counts associated with the new frags
*/
blk = blkmap(fs, blksfree, bbase);
ffs_fragacct(fs, blk, cgp->cg_frsum, 1);
/*
* if a complete block has been reassembled, account for it
*/
fragno = fragstoblks(fs, bbase);
if (ffs_isblock(fs, blksfree, fragno)) {
cgp->cg_cs.cs_nffree -= fs->fs_frag;
fs->fs_cstotal.cs_nffree -= fs->fs_frag;
fs->fs_cs(fs, cg).cs_nffree -= fs->fs_frag;
ffs_clusteracct(fs, cgp, fragno, 1);
cgp->cg_cs.cs_nbfree++;
fs->fs_cstotal.cs_nbfree++;
fs->fs_cs(fs, cg).cs_nbfree++;
}
}
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
mp = UFSTOVFS(ump);
if (MOUNTEDSOFTDEP(mp) && devvp->v_type == VCHR)
softdep_setup_blkfree(UFSTOVFS(ump), bp, bno,
numfrags(fs, size), dephd);
bdwrite(bp);
}
/*
* Structures and routines associated with trim management.
*
* The following requests are passed to trim_lookup to indicate
* the actions that should be taken.
*/
#define NEW 1 /* if found, error else allocate and hash it */
#define OLD 2 /* if not found, error, else return it */
#define REPLACE 3 /* if not found, error else unhash and reallocate it */
#define DONE 4 /* if not found, error else unhash and return it */
#define SINGLE 5 /* don't look up, just allocate it and don't hash it */
MALLOC_DEFINE(M_TRIM, "ufs_trim", "UFS trim structures");
#define TRIMLIST_HASH(ump, key) \
(&(ump)->um_trimhash[(key) & (ump)->um_trimlisthashsize])
/*
* These structures describe each of the block free requests aggregated
* together to make up a trim request.
*/
struct trim_blkreq {
TAILQ_ENTRY(trim_blkreq) blkreqlist;
ufs2_daddr_t bno;
long size;
struct workhead *pdephd;
struct workhead dephd;
};
/*
* Description of a trim request.
*/
struct ffs_blkfree_trim_params {
TAILQ_HEAD(, trim_blkreq) blklist;
LIST_ENTRY(ffs_blkfree_trim_params) hashlist;
struct task task;
struct ufsmount *ump;
struct vnode *devvp;
ino_t inum;
ufs2_daddr_t bno;
long size;
long key;
};
static void ffs_blkfree_trim_completed(struct buf *);
static void ffs_blkfree_trim_task(void *ctx, int pending __unused);
static struct ffs_blkfree_trim_params *trim_lookup(struct ufsmount *,
struct vnode *, ufs2_daddr_t, long, ino_t, u_long, int);
static void ffs_blkfree_sendtrim(struct ffs_blkfree_trim_params *);
/*
* Called on trim completion to start a task to free the associated block(s).
*/
static void
ffs_blkfree_trim_completed(bp)
struct buf *bp;
{
struct ffs_blkfree_trim_params *tp;
tp = bp->b_fsprivate1;
free(bp, M_TRIM);
TASK_INIT(&tp->task, 0, ffs_blkfree_trim_task, tp);
taskqueue_enqueue(tp->ump->um_trim_tq, &tp->task);
}
/*
* Trim completion task that free associated block(s).
*/
static void
ffs_blkfree_trim_task(ctx, pending)
void *ctx;
int pending;
{
struct ffs_blkfree_trim_params *tp;
struct trim_blkreq *blkelm;
struct ufsmount *ump;
tp = ctx;
ump = tp->ump;
while ((blkelm = TAILQ_FIRST(&tp->blklist)) != NULL) {
ffs_blkfree_cg(ump, ump->um_fs, tp->devvp, blkelm->bno,
blkelm->size, tp->inum, blkelm->pdephd);
TAILQ_REMOVE(&tp->blklist, blkelm, blkreqlist);
free(blkelm, M_TRIM);
}
vn_finished_secondary_write(UFSTOVFS(ump));
UFS_LOCK(ump);
ump->um_trim_inflight -= 1;
ump->um_trim_inflight_blks -= numfrags(ump->um_fs, tp->size);
UFS_UNLOCK(ump);
free(tp, M_TRIM);
}
/*
* Lookup a trim request by inode number.
* Allocate if requested (NEW, REPLACE, SINGLE).
*/
static struct ffs_blkfree_trim_params *
trim_lookup(ump, devvp, bno, size, inum, key, alloctype)
struct ufsmount *ump;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
u_long key;
int alloctype;
{
struct trimlist_hashhead *tphashhead;
struct ffs_blkfree_trim_params *tp, *ntp;
ntp = malloc(sizeof(struct ffs_blkfree_trim_params), M_TRIM, M_WAITOK);
if (alloctype != SINGLE) {
KASSERT(key >= FIRST_VALID_KEY, ("trim_lookup: invalid key"));
UFS_LOCK(ump);
tphashhead = TRIMLIST_HASH(ump, key);
LIST_FOREACH(tp, tphashhead, hashlist)
if (key == tp->key)
break;
}
switch (alloctype) {
case NEW:
KASSERT(tp == NULL, ("trim_lookup: found trim"));
break;
case OLD:
KASSERT(tp != NULL,
("trim_lookup: missing call to ffs_blkrelease_start()"));
UFS_UNLOCK(ump);
free(ntp, M_TRIM);
return (tp);
case REPLACE:
KASSERT(tp != NULL, ("trim_lookup: missing REPLACE trim"));
LIST_REMOVE(tp, hashlist);
/* tp will be freed by caller */
break;
case DONE:
KASSERT(tp != NULL, ("trim_lookup: missing DONE trim"));
LIST_REMOVE(tp, hashlist);
UFS_UNLOCK(ump);
free(ntp, M_TRIM);
return (tp);
}
TAILQ_INIT(&ntp->blklist);
ntp->ump = ump;
ntp->devvp = devvp;
ntp->bno = bno;
ntp->size = size;
ntp->inum = inum;
ntp->key = key;
if (alloctype != SINGLE) {
LIST_INSERT_HEAD(tphashhead, ntp, hashlist);
UFS_UNLOCK(ump);
}
return (ntp);
}
/*
* Dispatch a trim request.
*/
static void
ffs_blkfree_sendtrim(tp)
struct ffs_blkfree_trim_params *tp;
{
struct ufsmount *ump;
struct mount *mp;
struct buf *bp;
/*
* Postpone the set of the free bit in the cg bitmap until the
* BIO_DELETE is completed. Otherwise, due to disk queue
* reordering, TRIM might be issued after we reuse the block
* and write some new data into it.
*/
ump = tp->ump;
bp = malloc(sizeof(*bp), M_TRIM, M_WAITOK | M_ZERO);
bp->b_iocmd = BIO_DELETE;
bp->b_iooffset = dbtob(fsbtodb(ump->um_fs, tp->bno));
bp->b_iodone = ffs_blkfree_trim_completed;
bp->b_bcount = tp->size;
bp->b_fsprivate1 = tp;
UFS_LOCK(ump);
ump->um_trim_total += 1;
ump->um_trim_inflight += 1;
ump->um_trim_inflight_blks += numfrags(ump->um_fs, tp->size);
ump->um_trim_total_blks += numfrags(ump->um_fs, tp->size);
UFS_UNLOCK(ump);
mp = UFSTOVFS(ump);
vn_start_secondary_write(NULL, &mp, 0);
g_vfs_strategy(ump->um_bo, bp);
}
/*
* Allocate a new key to use to identify a range of blocks.
*/
u_long
ffs_blkrelease_start(ump, devvp, inum)
struct ufsmount *ump;
struct vnode *devvp;
ino_t inum;
{
static u_long masterkey;
u_long key;
if (((ump->um_flags & UM_CANDELETE) == 0) || dotrimcons == 0)
return (SINGLETON_KEY);
do {
key = atomic_fetchadd_long(&masterkey, 1);
} while (key < FIRST_VALID_KEY);
(void) trim_lookup(ump, devvp, 0, 0, inum, key, NEW);
return (key);
}
/*
* Deallocate a key that has been used to identify a range of blocks.
*/
void
ffs_blkrelease_finish(ump, key)
struct ufsmount *ump;
u_long key;
{
struct ffs_blkfree_trim_params *tp;
if (((ump->um_flags & UM_CANDELETE) == 0) || dotrimcons == 0)
return;
/*
* If the vfs.ffs.dotrimcons sysctl option is enabled while
* a file deletion is active, specifically after a call
* to ffs_blkrelease_start() but before the call to
* ffs_blkrelease_finish(), ffs_blkrelease_start() will
* have handed out SINGLETON_KEY rather than starting a
* collection sequence. Thus if we get a SINGLETON_KEY
* passed to ffs_blkrelease_finish(), we just return rather
* than trying to finish the nonexistent sequence.
*/
if (key == SINGLETON_KEY) {
#ifdef INVARIANTS
printf("%s: vfs.ffs.dotrimcons enabled on active filesystem\n",
ump->um_mountp->mnt_stat.f_mntonname);
#endif
return;
}
/*
* We are done with sending blocks using this key. Look up the key
* using the DONE alloctype (in tp) to request that it be unhashed
* as we will not be adding to it. If the key has never been used,
* tp->size will be zero, so we can just free tp. Otherwise the call
* to ffs_blkfree_sendtrim(tp) causes the block range described by
* tp to be issued (and then tp to be freed).
*/
tp = trim_lookup(ump, NULL, 0, 0, 0, key, DONE);
if (tp->size == 0)
free(tp, M_TRIM);
else
ffs_blkfree_sendtrim(tp);
}
/*
* Setup to free a block or fragment.
*
* Check for snapshots that might want to claim the block.
* If trims are requested, prepare a trim request. Attempt to
* aggregate consecutive blocks into a single trim request.
*/
void
ffs_blkfree(ump, fs, devvp, bno, size, inum, vtype, dephd, key)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ufs2_daddr_t bno;
long size;
ino_t inum;
enum vtype vtype;
struct workhead *dephd;
u_long key;
{
struct ffs_blkfree_trim_params *tp, *ntp;
struct trim_blkreq *blkelm;
/*
* Check to see if a snapshot wants to claim the block.
* Check that devvp is a normal disk device, not a snapshot,
* it has a snapshot(s) associated with it, and one of the
* snapshots wants to claim the block.
*/
if (devvp->v_type == VCHR &&
(devvp->v_vflag & VV_COPYONWRITE) &&
ffs_snapblkfree(fs, devvp, bno, size, inum, vtype, dephd)) {
return;
}
/*
* Nothing to delay if TRIM is not required for this block or TRIM
* is disabled or the operation is performed on a snapshot.
*/
if (key == NOTRIM_KEY || ((ump->um_flags & UM_CANDELETE) == 0) ||
devvp->v_type == VREG) {
ffs_blkfree_cg(ump, fs, devvp, bno, size, inum, dephd);
return;
}
blkelm = malloc(sizeof(struct trim_blkreq), M_TRIM, M_WAITOK);
blkelm->bno = bno;
blkelm->size = size;
if (dephd == NULL) {
blkelm->pdephd = NULL;
} else {
LIST_INIT(&blkelm->dephd);
LIST_SWAP(dephd, &blkelm->dephd, worklist, wk_list);
blkelm->pdephd = &blkelm->dephd;
}
if (key == SINGLETON_KEY) {
/*
* Just a single non-contiguous piece. Use the SINGLE
* alloctype to return a trim request that will not be
* hashed for future lookup.
*/
tp = trim_lookup(ump, devvp, bno, size, inum, key, SINGLE);
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
ffs_blkfree_sendtrim(tp);
return;
}
/*
* The callers of this function are not tracking whether or not
* the blocks are contiguous. They are just saying that they
* are freeing a set of blocks. It is this code that determines
* the pieces of that range that are actually contiguous.
*
* Calling ffs_blkrelease_start() will have created an entry
* that we will use.
*/
tp = trim_lookup(ump, devvp, bno, size, inum, key, OLD);
if (tp->size == 0) {
/*
* First block of a potential range, set block and size
* for the trim block.
*/
tp->bno = bno;
tp->size = size;
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
return;
}
/*
* If this block is a continuation of the range (either
* follows at the end or preceeds in the front) then we
* add it to the front or back of the list and return.
*
* If it is not a continuation of the trim that we were
* building, using the REPLACE alloctype, we request that
* the old trim request (still in tp) be unhashed and a
* new range started (in ntp). The ffs_blkfree_sendtrim(tp)
* call causes the block range described by tp to be issued
* (and then tp to be freed).
*/
if (bno + numfrags(fs, size) == tp->bno) {
TAILQ_INSERT_HEAD(&tp->blklist, blkelm, blkreqlist);
tp->bno = bno;
tp->size += size;
return;
} else if (bno == tp->bno + numfrags(fs, tp->size)) {
TAILQ_INSERT_TAIL(&tp->blklist, blkelm, blkreqlist);
tp->size += size;
return;
}
ntp = trim_lookup(ump, devvp, bno, size, inum, key, REPLACE);
TAILQ_INSERT_HEAD(&ntp->blklist, blkelm, blkreqlist);
ffs_blkfree_sendtrim(tp);
}
#ifdef INVARIANTS
/*
* Verify allocation of a block or fragment. Returns true if block or
* fragment is allocated, false if it is free.
*/
static int
ffs_checkblk(ip, bno, size)
struct inode *ip;
ufs2_daddr_t bno;
long size;
{
struct fs *fs;
struct cg *cgp;
struct buf *bp;
ufs1_daddr_t cgbno;
int i, error, frags, free;
u_int8_t *blksfree;
fs = ITOFS(ip);
if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) {
printf("bsize = %ld, size = %ld, fs = %s\n",
(long)fs->fs_bsize, size, fs->fs_fsmnt);
panic("ffs_checkblk: bad size");
}
if ((u_int)bno >= fs->fs_size)
panic("ffs_checkblk: bad block %jd", (intmax_t)bno);
error = ffs_getcg(fs, ITODEVVP(ip), dtog(fs, bno), 0, &bp, &cgp);
if (error)
panic("ffs_checkblk: cylinder group read failed");
blksfree = cg_blksfree(cgp);
cgbno = dtogd(fs, bno);
if (size == fs->fs_bsize) {
free = ffs_isblock(fs, blksfree, fragstoblks(fs, cgbno));
} else {
frags = numfrags(fs, size);
for (free = 0, i = 0; i < frags; i++)
if (isset(blksfree, cgbno + i))
free++;
if (free != 0 && free != frags)
panic("ffs_checkblk: partially free fragment");
}
brelse(bp);
return (!free);
}
#endif /* INVARIANTS */
/*
* Free an inode.
*/
int
ffs_vfree(pvp, ino, mode)
struct vnode *pvp;
ino_t ino;
int mode;
{
struct ufsmount *ump;
if (DOINGSOFTDEP(pvp)) {
softdep_freefile(pvp, ino, mode);
return (0);
}
ump = VFSTOUFS(pvp->v_mount);
return (ffs_freefile(ump, ump->um_fs, ump->um_devvp, ino, mode, NULL));
}
/*
* Do the actual free operation.
* The specified inode is placed back in the free map.
*/
int
ffs_freefile(ump, fs, devvp, ino, mode, wkhd)
struct ufsmount *ump;
struct fs *fs;
struct vnode *devvp;
ino_t ino;
int mode;
struct workhead *wkhd;
{
struct cg *cgp;
struct buf *bp;
int error;
u_int cg;
u_int8_t *inosused;
struct cdev *dev;
ino_t cgino;
cg = ino_to_cg(fs, ino);
if (devvp->v_type == VREG) {
/* devvp is a snapshot */
MPASS(devvp->v_mount->mnt_data == ump);
dev = ump->um_devvp->v_rdev;
} else if (devvp->v_type == VCHR) {
/* devvp is a normal disk device */
dev = devvp->v_rdev;
} else {
bp = NULL;
return (0);
}
if (ino >= fs->fs_ipg * fs->fs_ncg)
panic("ffs_freefile: range: dev = %s, ino = %ju, fs = %s",
devtoname(dev), (uintmax_t)ino, fs->fs_fsmnt);
if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
return (error);
inosused = cg_inosused(cgp);
cgino = ino % fs->fs_ipg;
if (isclr(inosused, cgino)) {
printf("dev = %s, ino = %ju, fs = %s\n", devtoname(dev),
(uintmax_t)ino, fs->fs_fsmnt);
if (fs->fs_ronly == 0)
panic("ffs_freefile: freeing free inode");
}
clrbit(inosused, cgino);
if (cgino < cgp->cg_irotor)
cgp->cg_irotor = cgino;
cgp->cg_cs.cs_nifree++;
UFS_LOCK(ump);
fs->fs_cstotal.cs_nifree++;
fs->fs_cs(fs, cg).cs_nifree++;
if ((mode & IFMT) == IFDIR) {
cgp->cg_cs.cs_ndir--;
fs->fs_cstotal.cs_ndir--;
fs->fs_cs(fs, cg).cs_ndir--;
}
fs->fs_fmod = 1;
ACTIVECLEAR(fs, cg);
UFS_UNLOCK(ump);
if (MOUNTEDSOFTDEP(UFSTOVFS(ump)) && devvp->v_type == VCHR)
softdep_setup_inofree(UFSTOVFS(ump), bp, ino, wkhd);
bdwrite(bp);
return (0);
}
/*
* Check to see if a file is free.
* Used to check for allocated files in snapshots.
*/
int
ffs_checkfreefile(fs, devvp, ino)
struct fs *fs;
struct vnode *devvp;
ino_t ino;
{
struct cg *cgp;
struct buf *bp;
int ret, error;
u_int cg;
u_int8_t *inosused;
cg = ino_to_cg(fs, ino);
if ((devvp->v_type != VREG) && (devvp->v_type != VCHR))
return (1);
if (ino >= fs->fs_ipg * fs->fs_ncg)
return (1);
if ((error = ffs_getcg(fs, devvp, cg, 0, &bp, &cgp)) != 0)
return (1);
inosused = cg_inosused(cgp);
ino %= fs->fs_ipg;
ret = isclr(inosused, ino);
brelse(bp);
return (ret);
}
/*
* Find a block of the specified size in the specified cylinder group.
*
* It is a panic if a request is made to find a block if none are
* available.
*/
static ufs1_daddr_t
ffs_mapsearch(fs, cgp, bpref, allocsiz)
struct fs *fs;
struct cg *cgp;
ufs2_daddr_t bpref;
int allocsiz;
{
ufs1_daddr_t bno;
int start, len, loc, i;
int blk, field, subfield, pos;
u_int8_t *blksfree;
/*
* find the fragment by searching through the free block
* map for an appropriate bit pattern
*/
if (bpref)
start = dtogd(fs, bpref) / NBBY;
else
start = cgp->cg_frotor / NBBY;
blksfree = cg_blksfree(cgp);
len = howmany(fs->fs_fpg, NBBY) - start;
loc = scanc((u_int)len, (u_char *)&blksfree[start],
fragtbl[fs->fs_frag],
(u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY))));
if (loc == 0) {
len = start + 1;
start = 0;
loc = scanc((u_int)len, (u_char *)&blksfree[0],
fragtbl[fs->fs_frag],
(u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY))));
if (loc == 0) {
printf("start = %d, len = %d, fs = %s\n",
start, len, fs->fs_fsmnt);
panic("ffs_alloccg: map corrupted");
/* NOTREACHED */
}
}
bno = (start + len - loc) * NBBY;
cgp->cg_frotor = bno;
/*
* found the byte in the map
* sift through the bits to find the selected frag
*/
for (i = bno + NBBY; bno < i; bno += fs->fs_frag) {
blk = blkmap(fs, blksfree, bno);
blk <<= 1;
field = around[allocsiz];
subfield = inside[allocsiz];
for (pos = 0; pos <= fs->fs_frag - allocsiz; pos++) {
if ((blk & field) == subfield)
return (bno + pos);
field <<= 1;
subfield <<= 1;
}
}
printf("bno = %lu, fs = %s\n", (u_long)bno, fs->fs_fsmnt);
panic("ffs_alloccg: block not in map");
return (-1);
}
static const struct statfs *
ffs_getmntstat(struct vnode *devvp)
{
if (devvp->v_type == VCHR)
return (&devvp->v_rdev->si_mountpt->mnt_stat);
return (ffs_getmntstat(VFSTOUFS(devvp->v_mount)->um_devvp));
}
/*
* Fetch and verify a cylinder group.
*/
int
ffs_getcg(fs, devvp, cg, flags, bpp, cgpp)
struct fs *fs;
struct vnode *devvp;
u_int cg;
int flags;
struct buf **bpp;
struct cg **cgpp;
{
struct buf *bp;
struct cg *cgp;
const struct statfs *sfs;
daddr_t blkno;
int error;
*bpp = NULL;
*cgpp = NULL;
if ((fs->fs_metackhash & CK_CYLGRP) != 0)
flags |= GB_CKHASH;
if (devvp->v_type == VREG)
blkno = fragstoblks(fs, cgtod(fs, cg));
else
blkno = fsbtodb(fs, cgtod(fs, cg));
error = breadn_flags(devvp, blkno, blkno, (int)fs->fs_cgsize, NULL,
NULL, 0, NOCRED, flags, ffs_ckhash_cg, &bp);
if (error != 0)
return (error);
cgp = (struct cg *)bp->b_data;
if ((fs->fs_metackhash & CK_CYLGRP) != 0 &&
(bp->b_flags & B_CKHASH) != 0 &&
cgp->cg_ckhash != bp->b_ckhash) {
sfs = ffs_getmntstat(devvp);
printf("UFS %s%s (%s) cylinder checksum failed: cg %u, cgp: "
"0x%x != bp: 0x%jx\n",
devvp->v_type == VCHR ? "" : "snapshot of ",
sfs->f_mntfromname, sfs->f_mntonname,
cg, cgp->cg_ckhash, (uintmax_t)bp->b_ckhash);
bp->b_flags &= ~B_CKHASH;
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
return (EIO);
}
if (!cg_chkmagic(cgp) || cgp->cg_cgx != cg) {
sfs = ffs_getmntstat(devvp);
printf("UFS %s%s (%s)",
devvp->v_type == VCHR ? "" : "snapshot of ",
sfs->f_mntfromname, sfs->f_mntonname);
if (!cg_chkmagic(cgp))
printf(" cg %u: bad magic number 0x%x should be 0x%x\n",
cg, cgp->cg_magic, CG_MAGIC);
else
printf(": wrong cylinder group cg %u != cgx %u\n", cg,
cgp->cg_cgx);
bp->b_flags &= ~B_CKHASH;
bp->b_flags |= B_INVAL | B_NOCACHE;
brelse(bp);
return (EIO);
}
bp->b_flags &= ~B_CKHASH;
bp->b_xflags |= BX_BKGRDWRITE;
/*
* If we are using check hashes on the cylinder group then we want
* to limit changing the cylinder group time to when we are actually
* going to write it to disk so that its check hash remains correct
* in memory. If the CK_CYLGRP flag is set the time is updated in
* ffs_bufwrite() as the buffer is queued for writing. Otherwise we
* update the time here as we have done historically.
*/
if ((fs->fs_metackhash & CK_CYLGRP) != 0)
bp->b_xflags |= BX_CYLGRP;
else
cgp->cg_old_time = cgp->cg_time = time_second;
*bpp = bp;
*cgpp = cgp;
return (0);
}
static void
ffs_ckhash_cg(bp)
struct buf *bp;
{
uint32_t ckhash;
struct cg *cgp;
cgp = (struct cg *)bp->b_data;
ckhash = cgp->cg_ckhash;
cgp->cg_ckhash = 0;
bp->b_ckhash = calculate_crc32c(~0L, bp->b_data, bp->b_bcount);
cgp->cg_ckhash = ckhash;
}
/*
* Fserr prints the name of a filesystem with an error diagnostic.
*
* The form of the error message is:
* fs: error message
*/
void
ffs_fserr(fs, inum, cp)
struct fs *fs;
ino_t inum;
char *cp;
{
struct thread *td = curthread; /* XXX */
struct proc *p = td->td_proc;
log(LOG_ERR, "pid %d (%s), uid %d inumber %ju on %s: %s\n",
p->p_pid, p->p_comm, td->td_ucred->cr_uid, (uintmax_t)inum,
fs->fs_fsmnt, cp);
}
/*
* This function provides the capability for the fsck program to
* update an active filesystem. Fourteen operations are provided:
*
* adjrefcnt(inode, amt) - adjusts the reference count on the
* specified inode by the specified amount. Under normal
* operation the count should always go down. Decrementing
* the count to zero will cause the inode to be freed.
* adjblkcnt(inode, amt) - adjust the number of blocks used by the
* inode by the specified amount.
* adjsize(inode, size) - set the size of the inode to the
* specified size.
* adjndir, adjbfree, adjifree, adjffree, adjnumclusters(amt) -
* adjust the superblock summary.
* freedirs(inode, count) - directory inodes [inode..inode + count - 1]
* are marked as free. Inodes should never have to be marked
* as in use.
* freefiles(inode, count) - file inodes [inode..inode + count - 1]
* are marked as free. Inodes should never have to be marked
* as in use.
* freeblks(blockno, size) - blocks [blockno..blockno + size - 1]
* are marked as free. Blocks should never have to be marked
* as in use.
* setflags(flags, set/clear) - the fs_flags field has the specified
* flags set (second parameter +1) or cleared (second parameter -1).
* setcwd(dirinode) - set the current directory to dirinode in the
* filesystem associated with the snapshot.
* setdotdot(oldvalue, newvalue) - Verify that the inode number for ".."
* in the current directory is oldvalue then change it to newvalue.
* unlink(nameptr, oldvalue) - Verify that the inode number associated
* with nameptr in the current directory is oldvalue then unlink it.
*
* The following functions may only be used on a quiescent filesystem
* by the soft updates journal. They are not safe to be run on an active
* filesystem.
*
* setinode(inode, dip) - the specified disk inode is replaced with the
* contents pointed to by dip.
* setbufoutput(fd, flags) - output associated with the specified file
* descriptor (which must reference the character device supporting
* the filesystem) switches from using physio to running through the
* buffer cache when flags is set to 1. The descriptor reverts to
* physio for output when flags is set to zero.
*/
static int sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vfs_ffs, FFS_ADJ_REFCNT, adjrefcnt,
CTLFLAG_WR | CTLTYPE_STRUCT | CTLFLAG_NEEDGIANT,
0, 0, sysctl_ffs_fsck, "S,fsck",
"Adjust Inode Reference Count");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_BLKCNT, adjblkcnt,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust Inode Used Blocks Count");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_SIZE, setsize,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Set the inode size");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NDIR, adjndir,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of directories");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NBFREE, adjnbfree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free blocks");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NIFREE, adjnifree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NFFREE, adjnffree,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free frags");
static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NUMCLUSTERS, adjnumclusters,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Adjust number of free clusters");
static SYSCTL_NODE(_vfs_ffs, FFS_DIR_FREE, freedirs,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of Directory Inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_FILE_FREE, freefiles,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of File Inodes");
static SYSCTL_NODE(_vfs_ffs, FFS_BLK_FREE, freeblks,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Free Range of Blocks");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_FLAGS, setflags,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Change Filesystem Flags");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_CWD, setcwd,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Set Current Working Directory");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_DOTDOT, setdotdot,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Change Value of .. Entry");
static SYSCTL_NODE(_vfs_ffs, FFS_UNLINK, unlink,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Unlink a Duplicate Name");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_INODE, setinode,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Update an On-Disk Inode");
static SYSCTL_NODE(_vfs_ffs, FFS_SET_BUFOUTPUT, setbufoutput,
CTLFLAG_WR | CTLFLAG_NEEDGIANT, sysctl_ffs_fsck,
"Set Buffered Writing for Descriptor");
#ifdef DIAGNOSTIC
static int fsckcmds = 0;
SYSCTL_INT(_debug, OID_AUTO, ffs_fsckcmds, CTLFLAG_RW, &fsckcmds, 0,
"print out fsck_ffs-based filesystem update commands");
#endif /* DIAGNOSTIC */
static int buffered_write(struct file *, struct uio *, struct ucred *,
int, struct thread *);
static int
sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS)
{
struct thread *td = curthread;
struct fsck_cmd cmd;
struct ufsmount *ump;
struct vnode *vp, *dvp, *fdvp;
struct inode *ip, *dp;
struct mount *mp;
struct fs *fs;
+ struct pwd *pwd;
ufs2_daddr_t blkno;
long blkcnt, blksize;
u_long key;
struct file *fp, *vfp;
cap_rights_t rights;
int filetype, error;
static struct fileops *origops, bufferedops;
if (req->newlen > sizeof cmd)
return (EBADRPC);
if ((error = SYSCTL_IN(req, &cmd, sizeof cmd)) != 0)
return (error);
if (cmd.version != FFS_CMD_VERSION)
return (ERPCMISMATCH);
if ((error = getvnode(td, cmd.handle,
cap_rights_init(&rights, CAP_FSCK), &fp)) != 0)
return (error);
vp = fp->f_data;
if (vp->v_type != VREG && vp->v_type != VDIR) {
fdrop(fp, td);
return (EINVAL);
}
vn_start_write(vp, &mp, V_WAIT);
if (mp == NULL ||
strncmp(mp->mnt_stat.f_fstypename, "ufs", MFSNAMELEN)) {
vn_finished_write(mp);
fdrop(fp, td);
return (EINVAL);
}
ump = VFSTOUFS(mp);
if ((mp->mnt_flag & MNT_RDONLY) &&
ump->um_fsckpid != td->td_proc->p_pid) {
vn_finished_write(mp);
fdrop(fp, td);
return (EROFS);
}
fs = ump->um_fs;
filetype = IFREG;
switch (oidp->oid_number) {
case FFS_SET_FLAGS:
#ifdef DIAGNOSTIC
if (fsckcmds)
printf("%s: %s flags\n", mp->mnt_stat.f_mntonname,
cmd.size > 0 ? "set" : "clear");
#endif /* DIAGNOSTIC */
if (cmd.size > 0)
fs->fs_flags |= (long)cmd.value;
else
fs->fs_flags &= ~(long)cmd.value;
break;
case FFS_ADJ_REFCNT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust inode %jd link count by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
ip->i_nlink += cmd.size;
DIP_SET(ip, i_nlink, ip->i_nlink);
ip->i_effnlink += cmd.size;
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
if (DOINGSOFTDEP(vp))
softdep_change_linkcnt(ip);
vput(vp);
break;
case FFS_ADJ_BLKCNT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust inode %jd block count by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + cmd.size);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
vput(vp);
break;
case FFS_SET_SIZE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: set inode %jd size to %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
ip = VTOI(vp);
DIP_SET(ip, i_size, cmd.size);
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
vput(vp);
break;
case FFS_DIR_FREE:
filetype = IFDIR;
/* fall through */
case FFS_FILE_FREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
if (cmd.size == 1)
printf("%s: free %s inode %ju\n",
mp->mnt_stat.f_mntonname,
filetype == IFDIR ? "directory" : "file",
(uintmax_t)cmd.value);
else
printf("%s: free %s inodes %ju-%ju\n",
mp->mnt_stat.f_mntonname,
filetype == IFDIR ? "directory" : "file",
(uintmax_t)cmd.value,
(uintmax_t)(cmd.value + cmd.size - 1));
}
#endif /* DIAGNOSTIC */
while (cmd.size > 0) {
if ((error = ffs_freefile(ump, fs, ump->um_devvp,
cmd.value, filetype, NULL)))
break;
cmd.size -= 1;
cmd.value += 1;
}
break;
case FFS_BLK_FREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
if (cmd.size == 1)
printf("%s: free block %jd\n",
mp->mnt_stat.f_mntonname,
(intmax_t)cmd.value);
else
printf("%s: free blocks %jd-%jd\n",
mp->mnt_stat.f_mntonname,
(intmax_t)cmd.value,
(intmax_t)cmd.value + cmd.size - 1);
}
#endif /* DIAGNOSTIC */
blkno = cmd.value;
blkcnt = cmd.size;
blksize = fs->fs_frag - (blkno % fs->fs_frag);
key = ffs_blkrelease_start(ump, ump->um_devvp, UFS_ROOTINO);
while (blkcnt > 0) {
if (blkcnt < blksize)
blksize = blkcnt;
ffs_blkfree(ump, fs, ump->um_devvp, blkno,
blksize * fs->fs_fsize, UFS_ROOTINO,
VDIR, NULL, key);
blkno += blksize;
blkcnt -= blksize;
blksize = fs->fs_frag;
}
ffs_blkrelease_finish(ump, key);
break;
/*
* Adjust superblock summaries. fsck(8) is expected to
* submit deltas when necessary.
*/
case FFS_ADJ_NDIR:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of directories by %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_ndir += cmd.value;
break;
case FFS_ADJ_NBFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free blocks by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nbfree += cmd.value;
break;
case FFS_ADJ_NIFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free inodes by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nifree += cmd.value;
break;
case FFS_ADJ_NFFREE:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free frags by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_nffree += cmd.value;
break;
case FFS_ADJ_NUMCLUSTERS:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: adjust number of free clusters by %+jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
fs->fs_cstotal.cs_numclusters += cmd.value;
break;
case FFS_SET_CWD:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: set current directory to inode %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_SHARED, &vp)))
break;
AUDIT_ARG_VNODE1(vp);
if ((error = change_dir(vp, td)) != 0) {
vput(vp);
break;
}
VOP_UNLOCK(vp);
pwd_chdir(td, vp);
break;
case FFS_SET_DOTDOT:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: change .. in cwd from %jd to %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value,
(intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
/*
* First we have to get and lock the parent directory
* to which ".." points.
*/
error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &fdvp);
if (error)
break;
/*
* Now we get and lock the child directory containing "..".
*/
- FILEDESC_SLOCK(td->td_proc->p_fd);
- dvp = td->td_proc->p_fd->fd_cdir;
- FILEDESC_SUNLOCK(td->td_proc->p_fd);
+ pwd = pwd_hold(td);
+ dvp = pwd->pwd_cdir;
if ((error = vget(dvp, LK_EXCLUSIVE, td)) != 0) {
vput(fdvp);
+ pwd_drop(pwd);
break;
}
dp = VTOI(dvp);
dp->i_offset = 12; /* XXX mastertemplate.dot_reclen */
error = ufs_dirrewrite(dp, VTOI(fdvp), (ino_t)cmd.size,
DT_DIR, 0);
cache_purge(fdvp);
cache_purge(dvp);
vput(dvp);
vput(fdvp);
+ pwd_drop(pwd);
break;
case FFS_UNLINK:
#ifdef DIAGNOSTIC
if (fsckcmds) {
char buf[32];
if (copyinstr((char *)(intptr_t)cmd.value, buf,32,NULL))
strncpy(buf, "Name_too_long", 32);
printf("%s: unlink %s (inode %jd)\n",
mp->mnt_stat.f_mntonname, buf, (intmax_t)cmd.size);
}
#endif /* DIAGNOSTIC */
/*
* kern_funlinkat will do its own start/finish writes and
* they do not nest, so drop ours here. Setting mp == NULL
* indicates that vn_finished_write is not needed down below.
*/
vn_finished_write(mp);
mp = NULL;
error = kern_funlinkat(td, AT_FDCWD,
(char *)(intptr_t)cmd.value, FD_NONE, UIO_USERSPACE,
0, (ino_t)cmd.size);
break;
case FFS_SET_INODE:
if (ump->um_fsckpid != td->td_proc->p_pid) {
error = EPERM;
break;
}
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: update inode %jd\n",
mp->mnt_stat.f_mntonname, (intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp)))
break;
AUDIT_ARG_VNODE1(vp);
ip = VTOI(vp);
if (I_IS_UFS1(ip))
error = copyin((void *)(intptr_t)cmd.size, ip->i_din1,
sizeof(struct ufs1_dinode));
else
error = copyin((void *)(intptr_t)cmd.size, ip->i_din2,
sizeof(struct ufs2_dinode));
if (error) {
vput(vp);
break;
}
UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_MODIFIED);
error = ffs_update(vp, 1);
vput(vp);
break;
case FFS_SET_BUFOUTPUT:
if (ump->um_fsckpid != td->td_proc->p_pid) {
error = EPERM;
break;
}
if (ITOUMP(VTOI(vp)) != ump) {
error = EINVAL;
break;
}
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: %s buffered output for descriptor %jd\n",
mp->mnt_stat.f_mntonname,
cmd.size == 1 ? "enable" : "disable",
(intmax_t)cmd.value);
}
#endif /* DIAGNOSTIC */
if ((error = getvnode(td, cmd.value,
cap_rights_init(&rights, CAP_FSCK), &vfp)) != 0)
break;
if (vfp->f_vnode->v_type != VCHR) {
fdrop(vfp, td);
error = EINVAL;
break;
}
if (origops == NULL) {
origops = vfp->f_ops;
bcopy((void *)origops, (void *)&bufferedops,
sizeof(bufferedops));
bufferedops.fo_write = buffered_write;
}
if (cmd.size == 1)
atomic_store_rel_ptr((volatile uintptr_t *)&vfp->f_ops,
(uintptr_t)&bufferedops);
else
atomic_store_rel_ptr((volatile uintptr_t *)&vfp->f_ops,
(uintptr_t)origops);
fdrop(vfp, td);
break;
default:
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("Invalid request %d from fsck\n",
oidp->oid_number);
}
#endif /* DIAGNOSTIC */
error = EINVAL;
break;
}
fdrop(fp, td);
vn_finished_write(mp);
return (error);
}
/*
* Function to switch a descriptor to use the buffer cache to stage
* its I/O. This is needed so that writes to the filesystem device
* will give snapshots a chance to copy modified blocks for which it
* needs to retain copies.
*/
static int
buffered_write(fp, uio, active_cred, flags, td)
struct file *fp;
struct uio *uio;
struct ucred *active_cred;
int flags;
struct thread *td;
{
struct vnode *devvp, *vp;
struct inode *ip;
struct buf *bp;
struct fs *fs;
struct filedesc *fdp;
int error;
daddr_t lbn;
/*
* The devvp is associated with the /dev filesystem. To discover
* the filesystem with which the device is associated, we depend
* on the application setting the current directory to a location
* within the filesystem being written. Yes, this is an ugly hack.
*/
devvp = fp->f_vnode;
if (!vn_isdisk(devvp, NULL))
return (EINVAL);
fdp = td->td_proc->p_fd;
FILEDESC_SLOCK(fdp);
- vp = fdp->fd_cdir;
+ vp = fdp->fd_pwd->pwd_cdir;
vref(vp);
FILEDESC_SUNLOCK(fdp);
vn_lock(vp, LK_SHARED | LK_RETRY);
/*
* Check that the current directory vnode indeed belongs to
* UFS before trying to dereference UFS-specific v_data fields.
*/
if (vp->v_op != &ffs_vnodeops1 && vp->v_op != &ffs_vnodeops2) {
vput(vp);
return (EINVAL);
}
ip = VTOI(vp);
if (ITODEVVP(ip) != devvp) {
vput(vp);
return (EINVAL);
}
fs = ITOFS(ip);
vput(vp);
foffset_lock_uio(fp, uio, flags);
vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
#ifdef DIAGNOSTIC
if (fsckcmds) {
printf("%s: buffered write for block %jd\n",
fs->fs_fsmnt, (intmax_t)btodb(uio->uio_offset));
}
#endif /* DIAGNOSTIC */
/*
* All I/O must be contained within a filesystem block, start on
* a fragment boundary, and be a multiple of fragments in length.
*/
if (uio->uio_resid > fs->fs_bsize - (uio->uio_offset % fs->fs_bsize) ||
fragoff(fs, uio->uio_offset) != 0 ||
fragoff(fs, uio->uio_resid) != 0) {
error = EINVAL;
goto out;
}
lbn = numfrags(fs, uio->uio_offset);
bp = getblk(devvp, lbn, uio->uio_resid, 0, 0, 0);
bp->b_flags |= B_RELBUF;
if ((error = uiomove((char *)bp->b_data, uio->uio_resid, uio)) != 0) {
brelse(bp);
goto out;
}
error = bwrite(bp);
out:
VOP_UNLOCK(devvp);
foffset_unlock_uio(fp, uio, flags | FOF_NEXTOFF);
return (error);
}