Index: head/sys/kern/init_main.c =================================================================== --- head/sys/kern/init_main.c (revision 239327) +++ head/sys/kern/init_main.c (revision 239328) @@ -1,853 +1,853 @@ /*- * Copyright (c) 1995 Terrence R. Lambert * All rights reserved. * * Copyright (c) 1982, 1986, 1989, 1991, 1992, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)init_main.c 8.9 (Berkeley) 1/21/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_init_path.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include void mi_startup(void); /* Should be elsewhere */ /* Components of the first process -- never freed. */ static struct session session0; static struct pgrp pgrp0; struct proc proc0; struct thread thread0 __aligned(16); struct vmspace vmspace0; struct proc *initproc; int boothowto = 0; /* initialized so that it can be patched */ SYSCTL_INT(_debug, OID_AUTO, boothowto, CTLFLAG_RD, &boothowto, 0, "Boot control flags, passed from loader"); int bootverbose; SYSCTL_INT(_debug, OID_AUTO, bootverbose, CTLFLAG_RW, &bootverbose, 0, "Control the output of verbose kernel messages"); /* * This ensures that there is at least one entry so that the sysinit_set * symbol is not undefined. A sybsystem ID of SI_SUB_DUMMY is never * executed. */ SYSINIT(placeholder, SI_SUB_DUMMY, SI_ORDER_ANY, NULL, NULL); /* * The sysinit table itself. Items are checked off as the are run. * If we want to register new sysinit types, add them to newsysinit. */ SET_DECLARE(sysinit_set, struct sysinit); struct sysinit **sysinit, **sysinit_end; struct sysinit **newsysinit, **newsysinit_end; /* * Merge a new sysinit set into the current set, reallocating it if * necessary. This can only be called after malloc is running. */ void sysinit_add(struct sysinit **set, struct sysinit **set_end) { struct sysinit **newset; struct sysinit **sipp; struct sysinit **xipp; int count; count = set_end - set; if (newsysinit) count += newsysinit_end - newsysinit; else count += sysinit_end - sysinit; newset = malloc(count * sizeof(*sipp), M_TEMP, M_NOWAIT); if (newset == NULL) panic("cannot malloc for sysinit"); xipp = newset; if (newsysinit) for (sipp = newsysinit; sipp < newsysinit_end; sipp++) *xipp++ = *sipp; else for (sipp = sysinit; sipp < sysinit_end; sipp++) *xipp++ = *sipp; for (sipp = set; sipp < set_end; sipp++) *xipp++ = *sipp; if (newsysinit) free(newsysinit, M_TEMP); newsysinit = newset; newsysinit_end = newset + count; } #if defined (DDB) && defined(VERBOSE_SYSINIT) static const char * symbol_name(vm_offset_t va, db_strategy_t strategy) { const char *name; c_db_sym_t sym; db_expr_t offset; if (va == 0) return (NULL); sym = db_search_symbol(va, strategy, &offset); if (offset != 0) return (NULL); db_symbol_values(sym, &name, NULL); return (name); } #endif /* * System startup; initialize the world, create process 0, mount root * filesystem, and fork to create init and pagedaemon. Most of the * hard work is done in the lower-level initialization routines including * startup(), which does memory initialization and autoconfiguration. * * This allows simple addition of new kernel subsystems that require * boot time initialization. It also allows substitution of subsystem * (for instance, a scheduler, kernel profiler, or VM system) by object * module. Finally, it allows for optional "kernel threads". */ void mi_startup(void) { register struct sysinit **sipp; /* system initialization*/ register struct sysinit **xipp; /* interior loop of sort*/ register struct sysinit *save; /* bubble*/ #if defined(VERBOSE_SYSINIT) int last; int verbose; #endif if (boothowto & RB_VERBOSE) bootverbose++; if (sysinit == NULL) { sysinit = SET_BEGIN(sysinit_set); sysinit_end = SET_LIMIT(sysinit_set); } restart: /* * Perform a bubble sort of the system initialization objects by * their subsystem (primary key) and order (secondary key). */ for (sipp = sysinit; sipp < sysinit_end; sipp++) { for (xipp = sipp + 1; xipp < sysinit_end; xipp++) { if ((*sipp)->subsystem < (*xipp)->subsystem || ((*sipp)->subsystem == (*xipp)->subsystem && (*sipp)->order <= (*xipp)->order)) continue; /* skip*/ save = *sipp; *sipp = *xipp; *xipp = save; } } #if defined(VERBOSE_SYSINIT) last = SI_SUB_COPYRIGHT; verbose = 0; #if !defined(DDB) printf("VERBOSE_SYSINIT: DDB not enabled, symbol lookups disabled.\n"); #endif #endif /* * Traverse the (now) ordered list of system initialization tasks. * Perform each task, and continue on to the next task. * * The last item on the list is expected to be the scheduler, * which will not return. */ for (sipp = sysinit; sipp < sysinit_end; sipp++) { if ((*sipp)->subsystem == SI_SUB_DUMMY) continue; /* skip dummy task(s)*/ if ((*sipp)->subsystem == SI_SUB_DONE) continue; #if defined(VERBOSE_SYSINIT) if ((*sipp)->subsystem > last) { verbose = 1; last = (*sipp)->subsystem; printf("subsystem %x\n", last); } if (verbose) { #if defined(DDB) const char *func, *data; func = symbol_name((vm_offset_t)(*sipp)->func, DB_STGY_PROC); data = symbol_name((vm_offset_t)(*sipp)->udata, DB_STGY_ANY); if (func != NULL && data != NULL) printf(" %s(&%s)... ", func, data); else if (func != NULL) printf(" %s(%p)... ", func, (*sipp)->udata); else #endif printf(" %p(%p)... ", (*sipp)->func, (*sipp)->udata); } #endif /* Call function */ (*((*sipp)->func))((*sipp)->udata); #if defined(VERBOSE_SYSINIT) if (verbose) printf("done.\n"); #endif /* Check off the one we're just done */ (*sipp)->subsystem = SI_SUB_DONE; /* Check if we've installed more sysinit items via KLD */ if (newsysinit != NULL) { if (sysinit != SET_BEGIN(sysinit_set)) free(sysinit, M_TEMP); sysinit = newsysinit; sysinit_end = newsysinit_end; newsysinit = NULL; newsysinit_end = NULL; goto restart; } } panic("Shouldn't get here!"); /* NOTREACHED*/ } /* *************************************************************************** **** **** The following SYSINIT's belong elsewhere, but have not yet **** been moved. **** *************************************************************************** */ static void print_caddr_t(void *data) { printf("%s", (char *)data); } static void print_version(void *data __unused) { int len; /* Strip a trailing newline from version. */ len = strlen(version); while (len > 0 && version[len - 1] == '\n') len--; printf("%.*s %s\n", len, version, machine); } SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, copyright); SYSINIT(trademark, SI_SUB_COPYRIGHT, SI_ORDER_SECOND, print_caddr_t, trademark); SYSINIT(version, SI_SUB_COPYRIGHT, SI_ORDER_THIRD, print_version, NULL); #ifdef WITNESS static char wit_warn[] = "WARNING: WITNESS option enabled, expect reduced performance.\n"; SYSINIT(witwarn, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1, print_caddr_t, wit_warn); SYSINIT(witwarn2, SI_SUB_RUN_SCHEDULER, SI_ORDER_THIRD + 1, print_caddr_t, wit_warn); #endif #ifdef DIAGNOSTIC static char diag_warn[] = "WARNING: DIAGNOSTIC option enabled, expect reduced performance.\n"; SYSINIT(diagwarn, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 2, print_caddr_t, diag_warn); SYSINIT(diagwarn2, SI_SUB_RUN_SCHEDULER, SI_ORDER_THIRD + 2, print_caddr_t, diag_warn); #endif static int null_fetch_syscall_args(struct thread *td __unused, struct syscall_args *sa __unused) { panic("null_fetch_syscall_args"); } static void null_set_syscall_retval(struct thread *td __unused, int error __unused) { panic("null_set_syscall_retval"); } struct sysentvec null_sysvec = { .sv_size = 0, .sv_table = NULL, .sv_mask = 0, .sv_sigsize = 0, .sv_sigtbl = NULL, .sv_errsize = 0, .sv_errtbl = NULL, .sv_transtrap = NULL, .sv_fixup = NULL, .sv_sendsig = NULL, .sv_sigcode = NULL, .sv_szsigcode = NULL, .sv_prepsyscall = NULL, .sv_name = "null", .sv_coredump = NULL, .sv_imgact_try = NULL, .sv_minsigstksz = 0, .sv_pagesize = PAGE_SIZE, .sv_minuser = VM_MIN_ADDRESS, .sv_maxuser = VM_MAXUSER_ADDRESS, .sv_usrstack = USRSTACK, .sv_psstrings = PS_STRINGS, .sv_stackprot = VM_PROT_ALL, .sv_copyout_strings = NULL, .sv_setregs = NULL, .sv_fixlimit = NULL, .sv_maxssiz = NULL, .sv_flags = 0, .sv_set_syscall_retval = null_set_syscall_retval, .sv_fetch_syscall_args = null_fetch_syscall_args, .sv_syscallnames = NULL, .sv_schedtail = NULL, }; /* *************************************************************************** **** **** The two following SYSINIT's are proc0 specific glue code. I am not **** convinced that they can not be safely combined, but their order of **** operation has been maintained as the same as the original init_main.c **** for right now. **** **** These probably belong in init_proc.c or kern_proc.c, since they **** deal with proc0 (the fork template process). **** *************************************************************************** */ /* ARGSUSED*/ static void proc0_init(void *dummy __unused) { struct proc *p; struct thread *td; vm_paddr_t pageablemem; int i; GIANT_REQUIRED; p = &proc0; td = &thread0; /* * Initialize magic number and osrel. */ p->p_magic = P_MAGIC; p->p_osrel = osreldate; /* * Initialize thread and process structures. */ procinit(); /* set up proc zone */ threadinit(); /* set up UMA zones */ /* * Initialise scheduler resources. * Add scheduler specific parts to proc, thread as needed. */ schedinit(); /* scheduler gets its house in order */ /* * Initialize sleep queue hash table */ sleepinit(); /* * additional VM structures */ vm_init2(); /* * Create process 0 (the swapper). */ LIST_INSERT_HEAD(&allproc, p, p_list); LIST_INSERT_HEAD(PIDHASH(0), p, p_hash); mtx_init(&pgrp0.pg_mtx, "process group", NULL, MTX_DEF | MTX_DUPOK); p->p_pgrp = &pgrp0; LIST_INSERT_HEAD(PGRPHASH(0), &pgrp0, pg_hash); LIST_INIT(&pgrp0.pg_members); LIST_INSERT_HEAD(&pgrp0.pg_members, p, p_pglist); pgrp0.pg_session = &session0; mtx_init(&session0.s_mtx, "session", NULL, MTX_DEF); refcount_init(&session0.s_count, 1); session0.s_leader = p; p->p_sysent = &null_sysvec; p->p_flag = P_SYSTEM | P_INMEM; p->p_state = PRS_NORMAL; knlist_init_mtx(&p->p_klist, &p->p_mtx); STAILQ_INIT(&p->p_ktr); p->p_nice = NZERO; - /* pid_max cannot be greater then PID_MAX */ + /* pid_max cannot be greater than PID_MAX */ td->td_tid = PID_MAX + 1; LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); td->td_state = TDS_RUNNING; td->td_pri_class = PRI_TIMESHARE; td->td_user_pri = PUSER; td->td_base_user_pri = PUSER; td->td_lend_user_pri = PRI_MAX; td->td_priority = PVM; td->td_base_pri = PVM; td->td_oncpu = 0; td->td_flags = TDF_INMEM; td->td_pflags = TDP_KTHREAD; td->td_cpuset = cpuset_thread0(); prison0.pr_cpuset = cpuset_ref(td->td_cpuset); p->p_peers = 0; p->p_leader = p; strncpy(p->p_comm, "kernel", sizeof (p->p_comm)); strncpy(td->td_name, "swapper", sizeof (td->td_name)); callout_init(&p->p_itcallout, CALLOUT_MPSAFE); callout_init_mtx(&p->p_limco, &p->p_mtx, 0); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); /* Create credentials. */ p->p_ucred = crget(); p->p_ucred->cr_ngroups = 1; /* group 0 */ p->p_ucred->cr_uidinfo = uifind(0); p->p_ucred->cr_ruidinfo = uifind(0); p->p_ucred->cr_prison = &prison0; p->p_ucred->cr_loginclass = loginclass_find("default"); #ifdef AUDIT audit_cred_kproc0(p->p_ucred); #endif #ifdef MAC mac_cred_create_swapper(p->p_ucred); #endif td->td_ucred = crhold(p->p_ucred); /* Create sigacts. */ p->p_sigacts = sigacts_alloc(); /* Initialize signal state for process 0. */ siginit(&proc0); /* Create the file descriptor table. */ p->p_fd = fdinit(NULL); p->p_fdtol = NULL; /* Create the limits structures. */ p->p_limit = lim_alloc(); for (i = 0; i < RLIM_NLIMITS; i++) p->p_limit->pl_rlimit[i].rlim_cur = p->p_limit->pl_rlimit[i].rlim_max = RLIM_INFINITY; p->p_limit->pl_rlimit[RLIMIT_NOFILE].rlim_cur = p->p_limit->pl_rlimit[RLIMIT_NOFILE].rlim_max = maxfiles; p->p_limit->pl_rlimit[RLIMIT_NPROC].rlim_cur = p->p_limit->pl_rlimit[RLIMIT_NPROC].rlim_max = maxproc; p->p_limit->pl_rlimit[RLIMIT_DATA].rlim_cur = dfldsiz; p->p_limit->pl_rlimit[RLIMIT_DATA].rlim_max = maxdsiz; p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_cur = dflssiz; p->p_limit->pl_rlimit[RLIMIT_STACK].rlim_max = maxssiz; /* Cast to avoid overflow on i386/PAE. */ pageablemem = ptoa((vm_paddr_t)cnt.v_free_count); p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_cur = p->p_limit->pl_rlimit[RLIMIT_RSS].rlim_max = pageablemem; p->p_limit->pl_rlimit[RLIMIT_MEMLOCK].rlim_cur = pageablemem / 3; p->p_limit->pl_rlimit[RLIMIT_MEMLOCK].rlim_max = pageablemem; p->p_cpulimit = RLIM_INFINITY; /* Initialize resource accounting structures. */ racct_create(&p->p_racct); p->p_stats = pstats_alloc(); /* Allocate a prototype map so we have something to fork. */ pmap_pinit0(vmspace_pmap(&vmspace0)); p->p_vmspace = &vmspace0; vmspace0.vm_refcnt = 1; /* * proc0 is not expected to enter usermode, so there is no special * handling for sv_minuser here, like is done for exec_new_vmspace(). */ vm_map_init(&vmspace0.vm_map, vmspace_pmap(&vmspace0), p->p_sysent->sv_minuser, p->p_sysent->sv_maxuser); /* * Call the init and ctor for the new thread and proc. We wait * to do this until all other structures are fairly sane. */ EVENTHANDLER_INVOKE(process_init, p); EVENTHANDLER_INVOKE(thread_init, td); EVENTHANDLER_INVOKE(process_ctor, p); EVENTHANDLER_INVOKE(thread_ctor, td); /* * Charge root for one process. */ (void)chgproccnt(p->p_ucred->cr_ruidinfo, 1, 0); PROC_LOCK(p); racct_add_force(p, RACCT_NPROC, 1); PROC_UNLOCK(p); } SYSINIT(p0init, SI_SUB_INTRINSIC, SI_ORDER_FIRST, proc0_init, NULL); /* ARGSUSED*/ static void proc0_post(void *dummy __unused) { struct timespec ts; struct proc *p; struct rusage ru; struct thread *td; /* * Now we can look at the time, having had a chance to verify the * time from the filesystem. Pretend that proc0 started now. */ sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { microuptime(&p->p_stats->p_start); PROC_SLOCK(p); rufetch(p, &ru); /* Clears thread stats */ PROC_SUNLOCK(p); p->p_rux.rux_runtime = 0; p->p_rux.rux_uticks = 0; p->p_rux.rux_sticks = 0; p->p_rux.rux_iticks = 0; FOREACH_THREAD_IN_PROC(p, td) { td->td_runtime = 0; } } sx_sunlock(&allproc_lock); PCPU_SET(switchtime, cpu_ticks()); PCPU_SET(switchticks, ticks); /* * Give the ``random'' number generator a thump. */ nanotime(&ts); srandom(ts.tv_sec ^ ts.tv_nsec); } SYSINIT(p0post, SI_SUB_INTRINSIC_POST, SI_ORDER_FIRST, proc0_post, NULL); static void random_init(void *dummy __unused) { /* * After CPU has been started we have some randomness on most * platforms via get_cyclecount(). For platforms that don't * we will reseed random(9) in proc0_post() as well. */ srandom(get_cyclecount()); } SYSINIT(random, SI_SUB_RANDOM, SI_ORDER_FIRST, random_init, NULL); /* *************************************************************************** **** **** The following SYSINIT's and glue code should be moved to the **** respective files on a per subsystem basis. **** *************************************************************************** */ /* *************************************************************************** **** **** The following code probably belongs in another file, like **** kern/init_init.c. **** *************************************************************************** */ /* * List of paths to try when searching for "init". */ static char init_path[MAXPATHLEN] = #ifdef INIT_PATH __XSTRING(INIT_PATH); #else "/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init"; #endif SYSCTL_STRING(_kern, OID_AUTO, init_path, CTLFLAG_RD, init_path, 0, "Path used to search the init process"); /* * Shutdown timeout of init(8). * Unused within kernel, but used to control init(8), hence do not remove. */ #ifndef INIT_SHUTDOWN_TIMEOUT #define INIT_SHUTDOWN_TIMEOUT 120 #endif static int init_shutdown_timeout = INIT_SHUTDOWN_TIMEOUT; SYSCTL_INT(_kern, OID_AUTO, init_shutdown_timeout, CTLFLAG_RW, &init_shutdown_timeout, 0, "Shutdown timeout of init(8). " "Unused within kernel, but used to control init(8)"); /* * Start the initial user process; try exec'ing each pathname in init_path. * The program is invoked with one argument containing the boot flags. */ static void start_init(void *dummy) { vm_offset_t addr; struct execve_args args; int options, error; char *var, *path, *next, *s; char *ucp, **uap, *arg0, *arg1; struct thread *td; struct proc *p; mtx_lock(&Giant); GIANT_REQUIRED; td = curthread; p = td->td_proc; vfs_mountroot(); /* * Need just enough stack to hold the faked-up "execve()" arguments. */ addr = p->p_sysent->sv_usrstack - PAGE_SIZE; if (vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, PAGE_SIZE, FALSE, VM_PROT_ALL, VM_PROT_ALL, 0) != 0) panic("init: couldn't allocate argument space"); p->p_vmspace->vm_maxsaddr = (caddr_t)addr; p->p_vmspace->vm_ssize = 1; if ((var = getenv("init_path")) != NULL) { strlcpy(init_path, var, sizeof(init_path)); freeenv(var); } for (path = init_path; *path != '\0'; path = next) { while (*path == ':') path++; if (*path == '\0') break; for (next = path; *next != '\0' && *next != ':'; next++) /* nothing */ ; if (bootverbose) printf("start_init: trying %.*s\n", (int)(next - path), path); /* * Move out the boot flag argument. */ options = 0; ucp = (char *)p->p_sysent->sv_usrstack; (void)subyte(--ucp, 0); /* trailing zero */ if (boothowto & RB_SINGLE) { (void)subyte(--ucp, 's'); options = 1; } #ifdef notyet if (boothowto & RB_FASTBOOT) { (void)subyte(--ucp, 'f'); options = 1; } #endif #ifdef BOOTCDROM (void)subyte(--ucp, 'C'); options = 1; #endif if (options == 0) (void)subyte(--ucp, '-'); (void)subyte(--ucp, '-'); /* leading hyphen */ arg1 = ucp; /* * Move out the file name (also arg 0). */ (void)subyte(--ucp, 0); for (s = next - 1; s >= path; s--) (void)subyte(--ucp, *s); arg0 = ucp; /* * Move out the arg pointers. */ uap = (char **)((intptr_t)ucp & ~(sizeof(intptr_t)-1)); (void)suword((caddr_t)--uap, (long)0); /* terminator */ (void)suword((caddr_t)--uap, (long)(intptr_t)arg1); (void)suword((caddr_t)--uap, (long)(intptr_t)arg0); /* * Point at the arguments. */ args.fname = arg0; args.argv = uap; args.envv = NULL; /* * Now try to exec the program. If can't for any reason * other than it doesn't exist, complain. * * Otherwise, return via fork_trampoline() all the way * to user mode as init! */ if ((error = sys_execve(td, &args)) == 0) { mtx_unlock(&Giant); return; } if (error != ENOENT) printf("exec %.*s: error %d\n", (int)(next - path), path, error); } printf("init: not found in path %s\n", init_path); panic("no init"); } /* * Like kproc_create(), but runs in it's own address space. * We do this early to reserve pid 1. * * Note special case - do not make it runnable yet. Other work * in progress will change this more. */ static void create_init(const void *udata __unused) { struct ucred *newcred, *oldcred; int error; error = fork1(&thread0, RFFDG | RFPROC | RFSTOPPED, 0, &initproc, NULL, 0); if (error) panic("cannot fork init: %d\n", error); KASSERT(initproc->p_pid == 1, ("create_init: initproc->p_pid != 1")); /* divorce init's credentials from the kernel's */ newcred = crget(); PROC_LOCK(initproc); initproc->p_flag |= P_SYSTEM | P_INMEM; oldcred = initproc->p_ucred; crcopy(newcred, oldcred); #ifdef MAC mac_cred_create_init(newcred); #endif #ifdef AUDIT audit_cred_proc1(newcred); #endif initproc->p_ucred = newcred; PROC_UNLOCK(initproc); crfree(oldcred); cred_update_thread(FIRST_THREAD_IN_PROC(initproc)); cpu_set_fork_handler(FIRST_THREAD_IN_PROC(initproc), start_init, NULL); } SYSINIT(init, SI_SUB_CREATE_INIT, SI_ORDER_FIRST, create_init, NULL); /* * Make it runnable now. */ static void kick_init(const void *udata __unused) { struct thread *td; td = FIRST_THREAD_IN_PROC(initproc); thread_lock(td); TD_SET_CAN_RUN(td); sched_add(td, SRQ_BORING); thread_unlock(td); } SYSINIT(kickinit, SI_SUB_KTHREAD_INIT, SI_ORDER_FIRST, kick_init, NULL); Index: head/sys/kern/kern_thread.c =================================================================== --- head/sys/kern/kern_thread.c (revision 239327) +++ head/sys/kern/kern_thread.c (revision 239328) @@ -1,1044 +1,1044 @@ /*- * Copyright (C) 2001 Julian Elischer . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice(s), this list of conditions and the following disclaimer as * the first lines of this file unmodified other than the possible * addition of one or more copyright notices. * 2. Redistributions in binary form must reproduce the above copyright * notice(s), this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. */ #include "opt_witness.h" #include "opt_kdtrace.h" #include "opt_hwpmc_hooks.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HWPMC_HOOKS #include #endif #include #include #include #include #include SDT_PROVIDER_DECLARE(proc); SDT_PROBE_DEFINE(proc, , , lwp_exit, lwp-exit); /* * thread related storage. */ static uma_zone_t thread_zone; TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); static struct mtx zombie_lock; MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); static void thread_zombie(struct thread *); #define TID_BUFFER_SIZE 1024 struct mtx tid_lock; static struct unrhdr *tid_unrhdr; static lwpid_t tid_buffer[TID_BUFFER_SIZE]; static int tid_head, tid_tail; static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); struct tidhashhead *tidhashtbl; u_long tidhash; struct rwlock tidhash_lock; static lwpid_t tid_alloc(void) { lwpid_t tid; tid = alloc_unr(tid_unrhdr); if (tid != -1) return (tid); mtx_lock(&tid_lock); if (tid_head == tid_tail) { mtx_unlock(&tid_lock); return (-1); } tid = tid_buffer[tid_head++]; tid_head %= TID_BUFFER_SIZE; mtx_unlock(&tid_lock); return (tid); } static void tid_free(lwpid_t tid) { lwpid_t tmp_tid = -1; mtx_lock(&tid_lock); if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { tmp_tid = tid_buffer[tid_head++]; tid_head = (tid_head + 1) % TID_BUFFER_SIZE; } tid_buffer[tid_tail++] = tid; tid_tail %= TID_BUFFER_SIZE; mtx_unlock(&tid_lock); if (tmp_tid != -1) free_unr(tid_unrhdr, tmp_tid); } /* * Prepare a thread for use. */ static int thread_ctor(void *mem, int size, void *arg, int flags) { struct thread *td; td = (struct thread *)mem; td->td_state = TDS_INACTIVE; td->td_oncpu = NOCPU; td->td_tid = tid_alloc(); /* * Note that td_critnest begins life as 1 because the thread is not * running and is thereby implicitly waiting to be on the receiving * end of a context switch. */ td->td_critnest = 1; td->td_lend_user_pri = PRI_MAX; EVENTHANDLER_INVOKE(thread_ctor, td); #ifdef AUDIT audit_thread_alloc(td); #endif umtx_thread_alloc(td); return (0); } /* * Reclaim a thread after use. */ static void thread_dtor(void *mem, int size, void *arg) { struct thread *td; td = (struct thread *)mem; #ifdef INVARIANTS /* Verify that this thread is in a safe state to free. */ switch (td->td_state) { case TDS_INHIBITED: case TDS_RUNNING: case TDS_CAN_RUN: case TDS_RUNQ: /* * We must never unlink a thread that is in one of * these states, because it is currently active. */ panic("bad state for thread unlinking"); /* NOTREACHED */ case TDS_INACTIVE: break; default: panic("bad thread state"); /* NOTREACHED */ } #endif #ifdef AUDIT audit_thread_free(td); #endif /* Free all OSD associated to this thread. */ osd_thread_exit(td); EVENTHANDLER_INVOKE(thread_dtor, td); tid_free(td->td_tid); } /* * Initialize type-stable parts of a thread (when newly created). */ static int thread_init(void *mem, int size, int flags) { struct thread *td; td = (struct thread *)mem; td->td_sleepqueue = sleepq_alloc(); td->td_turnstile = turnstile_alloc(); td->td_rlqe = NULL; EVENTHANDLER_INVOKE(thread_init, td); td->td_sched = (struct td_sched *)&td[1]; umtx_thread_init(td); td->td_kstack = 0; return (0); } /* * Tear down type-stable parts of a thread (just before being discarded). */ static void thread_fini(void *mem, int size) { struct thread *td; td = (struct thread *)mem; EVENTHANDLER_INVOKE(thread_fini, td); rlqentry_free(td->td_rlqe); turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); umtx_thread_fini(td); seltdfini(td); } /* * For a newly created process, * link up all the structures and its initial threads etc. * called from: * {arch}/{arch}/machdep.c ia64_init(), init386() etc. * proc_dtor() (should go away) * proc_init() */ void proc_linkup0(struct proc *p, struct thread *td) { TAILQ_INIT(&p->p_threads); /* all threads in proc */ proc_linkup(p, td); } void proc_linkup(struct proc *p, struct thread *td) { sigqueue_init(&p->p_sigqueue, p); p->p_ksi = ksiginfo_alloc(1); if (p->p_ksi != NULL) { /* XXX p_ksi may be null if ksiginfo zone is not ready */ p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; } LIST_INIT(&p->p_mqnotifier); p->p_numthreads = 0; thread_link(td, p); } /* * Initialize global thread allocation resources. */ void threadinit(void) { mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); /* - * pid_max cannot be greater then PID_MAX. + * pid_max cannot be greater than PID_MAX. * leave one number for thread0. */ tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), thread_ctor, thread_dtor, thread_init, thread_fini, 16 - 1, 0); tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash); rw_init(&tidhash_lock, "tidhash"); } /* * Place an unused thread on the zombie list. * Use the slpq as that must be unused by now. */ void thread_zombie(struct thread *td) { mtx_lock_spin(&zombie_lock); TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); mtx_unlock_spin(&zombie_lock); } /* * Release a thread that has exited after cpu_throw(). */ void thread_stash(struct thread *td) { atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); thread_zombie(td); } /* * Reap zombie resources. */ void thread_reap(void) { struct thread *td_first, *td_next; /* * Don't even bother to lock if none at this instant, * we really don't care about the next instant.. */ if (!TAILQ_EMPTY(&zombie_threads)) { mtx_lock_spin(&zombie_lock); td_first = TAILQ_FIRST(&zombie_threads); if (td_first) TAILQ_INIT(&zombie_threads); mtx_unlock_spin(&zombie_lock); while (td_first) { td_next = TAILQ_NEXT(td_first, td_slpq); if (td_first->td_ucred) crfree(td_first->td_ucred); thread_free(td_first); td_first = td_next; } } } /* * Allocate a thread. */ struct thread * thread_alloc(int pages) { struct thread *td; thread_reap(); /* check if any zombies to get */ td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); if (!vm_thread_new(td, pages)) { uma_zfree(thread_zone, td); return (NULL); } cpu_thread_alloc(td); return (td); } int thread_alloc_stack(struct thread *td, int pages) { KASSERT(td->td_kstack == 0, ("thread_alloc_stack called on a thread with kstack")); if (!vm_thread_new(td, pages)) return (0); cpu_thread_alloc(td); return (1); } /* * Deallocate a thread. */ void thread_free(struct thread *td) { lock_profile_thread_exit(td); if (td->td_cpuset) cpuset_rel(td->td_cpuset); td->td_cpuset = NULL; cpu_thread_free(td); if (td->td_kstack != 0) vm_thread_dispose(td); uma_zfree(thread_zone, td); } /* * Discard the current thread and exit from its context. * Always called with scheduler locked. * * Because we can't free a thread while we're operating under its context, * push the current thread into our CPU's deadthread holder. This means * we needn't worry about someone else grabbing our context before we * do a cpu_throw(). */ void thread_exit(void) { uint64_t runtime, new_switchtime; struct thread *td; struct thread *td2; struct proc *p; int wakeup_swapper; td = curthread; p = td->td_proc; PROC_SLOCK_ASSERT(p, MA_OWNED); mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT(p != NULL, ("thread exiting without a process")); CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, (long)p->p_pid, td->td_name); KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); #ifdef AUDIT AUDIT_SYSCALL_EXIT(0, td); #endif umtx_thread_exit(td); /* * drop FPU & debug register state storage, or any other * architecture specific resources that * would not be on a new untouched process. */ cpu_thread_exit(td); /* XXXSMP */ /* * The last thread is left attached to the process * So that the whole bundle gets recycled. Skip * all this stuff if we never had threads. * EXIT clears all sign of other threads when * it goes to single threading, so the last thread always * takes the short path. */ if (p->p_flag & P_HADTHREADS) { if (p->p_numthreads > 1) { thread_unlink(td); td2 = FIRST_THREAD_IN_PROC(p); sched_exit_thread(td2, td); /* * The test below is NOT true if we are the * sole exiting thread. P_STOPPED_SINGLE is unset * in exit1() after it is the only survivor. */ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { if (p->p_numthreads == p->p_suspcount) { thread_lock(p->p_singlethread); wakeup_swapper = thread_unsuspend_one( p->p_singlethread); thread_unlock(p->p_singlethread); if (wakeup_swapper) kick_proc0(); } } atomic_add_int(&td->td_proc->p_exitthreads, 1); PCPU_SET(deadthread, td); } else { /* * The last thread is exiting.. but not through exit() */ panic ("thread_exit: Last thread exiting on its own"); } } #ifdef HWPMC_HOOKS /* * If this thread is part of a process that is being tracked by hwpmc(4), * inform the module of the thread's impending exit. */ if (PMC_PROC_IS_USING_PMCS(td->td_proc)) PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); #endif PROC_UNLOCK(p); /* Do the same timestamp bookkeeping that mi_switch() would do. */ new_switchtime = cpu_ticks(); runtime = new_switchtime - PCPU_GET(switchtime); td->td_runtime += runtime; td->td_incruntime += runtime; PCPU_SET(switchtime, new_switchtime); PCPU_SET(switchticks, ticks); PCPU_INC(cnt.v_swtch); /* Save our resource usage in our process. */ td->td_ru.ru_nvcsw++; ruxagg(p, td); rucollect(&p->p_ru, &td->td_ru); thread_lock(td); PROC_SUNLOCK(p); td->td_state = TDS_INACTIVE; #ifdef WITNESS witness_thread_exit(td); #endif CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); sched_throw(td); panic("I'm a teapot!"); /* NOTREACHED */ } /* * Do any thread specific cleanups that may be needed in wait() * called with Giant, proc and schedlock not held. */ void thread_wait(struct proc *p) { struct thread *td; mtx_assert(&Giant, MA_NOTOWNED); KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); td = FIRST_THREAD_IN_PROC(p); /* Lock the last thread so we spin until it exits cpu_throw(). */ thread_lock(td); thread_unlock(td); /* Wait for any remaining threads to exit cpu_throw(). */ while (p->p_exitthreads) sched_relinquish(curthread); lock_profile_thread_exit(td); cpuset_rel(td->td_cpuset); td->td_cpuset = NULL; cpu_thread_clean(td); crfree(td->td_ucred); thread_reap(); /* check for zombie threads etc. */ } /* * Link a thread to a process. * set up anything that needs to be initialized for it to * be used by the process. */ void thread_link(struct thread *td, struct proc *p) { /* * XXX This can't be enabled because it's called for proc0 before * its lock has been created. * PROC_LOCK_ASSERT(p, MA_OWNED); */ td->td_state = TDS_INACTIVE; td->td_proc = p; td->td_flags = TDF_INMEM; LIST_INIT(&td->td_contested); LIST_INIT(&td->td_lprof[0]); LIST_INIT(&td->td_lprof[1]); sigqueue_init(&td->td_sigqueue, p); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); p->p_numthreads++; } /* * Convert a process with one thread to an unthreaded process. */ void thread_unthread(struct thread *td) { struct proc *p = td->td_proc; KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); p->p_flag &= ~P_HADTHREADS; } /* * Called from: * thread_exit() */ void thread_unlink(struct thread *td) { struct proc *p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); TAILQ_REMOVE(&p->p_threads, td, td_plist); p->p_numthreads--; /* could clear a few other things here */ /* Must NOT clear links to proc! */ } static int calc_remaining(struct proc *p, int mode) { int remaining; PROC_LOCK_ASSERT(p, MA_OWNED); PROC_SLOCK_ASSERT(p, MA_OWNED); if (mode == SINGLE_EXIT) remaining = p->p_numthreads; else if (mode == SINGLE_BOUNDARY) remaining = p->p_numthreads - p->p_boundary_count; else if (mode == SINGLE_NO_EXIT) remaining = p->p_numthreads - p->p_suspcount; else panic("calc_remaining: wrong mode %d", mode); return (remaining); } /* * Enforce single-threading. * * Returns 1 if the caller must abort (another thread is waiting to * exit the process or similar). Process is locked! * Returns 0 when you are successfully the only thread running. * A process has successfully single threaded in the suspend mode when * There are no threads in user mode. Threads in the kernel must be * allowed to continue until they get to the user boundary. They may even * copy out their return values and data before suspending. They may however be * accelerated in reaching the user boundary as we will wake up * any sleeping threads that are interruptable. (PCATCH). */ int thread_single(int mode) { struct thread *td; struct thread *td2; struct proc *p; int remaining, wakeup_swapper; td = curthread; p = td->td_proc; mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT((td != NULL), ("curthread is NULL")); if ((p->p_flag & P_HADTHREADS) == 0) return (0); /* Is someone already single threading? */ if (p->p_singlethread != NULL && p->p_singlethread != td) return (1); if (mode == SINGLE_EXIT) { p->p_flag |= P_SINGLE_EXIT; p->p_flag &= ~P_SINGLE_BOUNDARY; } else { p->p_flag &= ~P_SINGLE_EXIT; if (mode == SINGLE_BOUNDARY) p->p_flag |= P_SINGLE_BOUNDARY; else p->p_flag &= ~P_SINGLE_BOUNDARY; } p->p_flag |= P_STOPPED_SINGLE; PROC_SLOCK(p); p->p_singlethread = td; remaining = calc_remaining(p, mode); while (remaining != 1) { if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) goto stopme; wakeup_swapper = 0; FOREACH_THREAD_IN_PROC(p, td2) { if (td2 == td) continue; thread_lock(td2); td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; if (TD_IS_INHIBITED(td2)) { switch (mode) { case SINGLE_EXIT: if (TD_IS_SUSPENDED(td2)) wakeup_swapper |= thread_unsuspend_one(td2); if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) wakeup_swapper |= sleepq_abort(td2, EINTR); break; case SINGLE_BOUNDARY: if (TD_IS_SUSPENDED(td2) && !(td2->td_flags & TDF_BOUNDARY)) wakeup_swapper |= thread_unsuspend_one(td2); if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) wakeup_swapper |= sleepq_abort(td2, ERESTART); break; case SINGLE_NO_EXIT: if (TD_IS_SUSPENDED(td2) && !(td2->td_flags & TDF_BOUNDARY)) wakeup_swapper |= thread_unsuspend_one(td2); if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) wakeup_swapper |= sleepq_abort(td2, ERESTART); break; default: break; } } #ifdef SMP else if (TD_IS_RUNNING(td2) && td != td2) { forward_signal(td2); } #endif thread_unlock(td2); } if (wakeup_swapper) kick_proc0(); remaining = calc_remaining(p, mode); /* * Maybe we suspended some threads.. was it enough? */ if (remaining == 1) break; stopme: /* * Wake us up when everyone else has suspended. * In the mean time we suspend as well. */ thread_suspend_switch(td); remaining = calc_remaining(p, mode); } if (mode == SINGLE_EXIT) { /* * We have gotten rid of all the other threads and we * are about to either exit or exec. In either case, * we try our utmost to revert to being a non-threaded * process. */ p->p_singlethread = NULL; p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); thread_unthread(td); } PROC_SUNLOCK(p); return (0); } /* * Called in from locations that can safely check to see * whether we have to suspend or at least throttle for a * single-thread event (e.g. fork). * * Such locations include userret(). * If the "return_instead" argument is non zero, the thread must be able to * accept 0 (caller may continue), or 1 (caller must abort) as a result. * * The 'return_instead' argument tells the function if it may do a * thread_exit() or suspend, or whether the caller must abort and back * out instead. * * If the thread that set the single_threading request has set the * P_SINGLE_EXIT bit in the process flags then this call will never return * if 'return_instead' is false, but will exit. * * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 *---------------+--------------------+--------------------- * 0 | returns 0 | returns 0 or 1 * | when ST ends | immediatly *---------------+--------------------+--------------------- * 1 | thread exits | returns 1 * | | immediatly * 0 = thread_exit() or suspension ok, * other = return error instead of stopping the thread. * * While a full suspension is under effect, even a single threading * thread would be suspended if it made this call (but it shouldn't). * This call should only be made from places where * thread_exit() would be safe as that may be the outcome unless * return_instead is set. */ int thread_suspend_check(int return_instead) { struct thread *td; struct proc *p; int wakeup_swapper; td = curthread; p = td->td_proc; mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); while (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_SUSPEND))) { if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { KASSERT(p->p_singlethread != NULL, ("singlethread not set")); /* * The only suspension in action is a * single-threading. Single threader need not stop. * XXX Should be safe to access unlocked * as it can only be set to be true by us. */ if (p->p_singlethread == td) return (0); /* Exempt from stopping. */ } if ((p->p_flag & P_SINGLE_EXIT) && return_instead) return (EINTR); /* Should we goto user boundary if we didn't come from there? */ if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) return (ERESTART); /* * If the process is waiting for us to exit, * this thread should just suicide. * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. */ if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { PROC_UNLOCK(p); tidhash_remove(td); PROC_LOCK(p); tdsigcleanup(td); PROC_SLOCK(p); thread_stopped(p); thread_exit(); } PROC_SLOCK(p); thread_stopped(p); if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { if (p->p_numthreads == p->p_suspcount + 1) { thread_lock(p->p_singlethread); wakeup_swapper = thread_unsuspend_one(p->p_singlethread); thread_unlock(p->p_singlethread); if (wakeup_swapper) kick_proc0(); } } PROC_UNLOCK(p); thread_lock(td); /* * When a thread suspends, it just * gets taken off all queues. */ thread_suspend_one(td); if (return_instead == 0) { p->p_boundary_count++; td->td_flags |= TDF_BOUNDARY; } PROC_SUNLOCK(p); mi_switch(SW_INVOL | SWT_SUSPEND, NULL); if (return_instead == 0) td->td_flags &= ~TDF_BOUNDARY; thread_unlock(td); PROC_LOCK(p); if (return_instead == 0) { PROC_SLOCK(p); p->p_boundary_count--; PROC_SUNLOCK(p); } } return (0); } void thread_suspend_switch(struct thread *td) { struct proc *p; p = td->td_proc; KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); PROC_LOCK_ASSERT(p, MA_OWNED); PROC_SLOCK_ASSERT(p, MA_OWNED); /* * We implement thread_suspend_one in stages here to avoid * dropping the proc lock while the thread lock is owned. */ thread_stopped(p); p->p_suspcount++; PROC_UNLOCK(p); thread_lock(td); td->td_flags &= ~TDF_NEEDSUSPCHK; TD_SET_SUSPENDED(td); sched_sleep(td, 0); PROC_SUNLOCK(p); DROP_GIANT(); mi_switch(SW_VOL | SWT_SUSPEND, NULL); thread_unlock(td); PICKUP_GIANT(); PROC_LOCK(p); PROC_SLOCK(p); } void thread_suspend_one(struct thread *td) { struct proc *p = td->td_proc; PROC_SLOCK_ASSERT(p, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); p->p_suspcount++; td->td_flags &= ~TDF_NEEDSUSPCHK; TD_SET_SUSPENDED(td); sched_sleep(td, 0); } int thread_unsuspend_one(struct thread *td) { struct proc *p = td->td_proc; PROC_SLOCK_ASSERT(p, MA_OWNED); THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); TD_CLR_SUSPENDED(td); p->p_suspcount--; return (setrunnable(td)); } /* * Allow all threads blocked by single threading to continue running. */ void thread_unsuspend(struct proc *p) { struct thread *td; int wakeup_swapper; PROC_LOCK_ASSERT(p, MA_OWNED); PROC_SLOCK_ASSERT(p, MA_OWNED); wakeup_swapper = 0; if (!P_SHOULDSTOP(p)) { FOREACH_THREAD_IN_PROC(p, td) { thread_lock(td); if (TD_IS_SUSPENDED(td)) { wakeup_swapper |= thread_unsuspend_one(td); } thread_unlock(td); } } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && (p->p_numthreads == p->p_suspcount)) { /* * Stopping everything also did the job for the single * threading request. Now we've downgraded to single-threaded, * let it continue. */ thread_lock(p->p_singlethread); wakeup_swapper = thread_unsuspend_one(p->p_singlethread); thread_unlock(p->p_singlethread); } if (wakeup_swapper) kick_proc0(); } /* * End the single threading mode.. */ void thread_single_end(void) { struct thread *td; struct proc *p; int wakeup_swapper; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); PROC_SLOCK(p); p->p_singlethread = NULL; wakeup_swapper = 0; /* * If there are other threads they may now run, * unless of course there is a blanket 'stop order' * on the process. The single threader must be allowed * to continue however as this is a bad place to stop. */ if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { FOREACH_THREAD_IN_PROC(p, td) { thread_lock(td); if (TD_IS_SUSPENDED(td)) { wakeup_swapper |= thread_unsuspend_one(td); } thread_unlock(td); } } PROC_SUNLOCK(p); if (wakeup_swapper) kick_proc0(); } struct thread * thread_find(struct proc *p, lwpid_t tid) { struct thread *td; PROC_LOCK_ASSERT(p, MA_OWNED); FOREACH_THREAD_IN_PROC(p, td) { if (td->td_tid == tid) break; } return (td); } /* Locate a thread by number; return with proc lock held. */ struct thread * tdfind(lwpid_t tid, pid_t pid) { #define RUN_THRESH 16 struct thread *td; int run = 0; rw_rlock(&tidhash_lock); LIST_FOREACH(td, TIDHASH(tid), td_hash) { if (td->td_tid == tid) { if (pid != -1 && td->td_proc->p_pid != pid) { td = NULL; break; } PROC_LOCK(td->td_proc); if (td->td_proc->p_state == PRS_NEW) { PROC_UNLOCK(td->td_proc); td = NULL; break; } if (run > RUN_THRESH) { if (rw_try_upgrade(&tidhash_lock)) { LIST_REMOVE(td, td_hash); LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); rw_wunlock(&tidhash_lock); return (td); } } break; } run++; } rw_runlock(&tidhash_lock); return (td); } void tidhash_add(struct thread *td) { rw_wlock(&tidhash_lock); LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); rw_wunlock(&tidhash_lock); } void tidhash_remove(struct thread *td) { rw_wlock(&tidhash_lock); LIST_REMOVE(td, td_hash); rw_wunlock(&tidhash_lock); } Index: head/sys/nlm/nlm_advlock.c =================================================================== --- head/sys/nlm/nlm_advlock.c (revision 239327) +++ head/sys/nlm/nlm_advlock.c (revision 239328) @@ -1,1233 +1,1233 @@ /*- * Copyright (c) 2008 Isilon Inc http://www.isilon.com/ * Authors: Doug Rabson * Developed with Red Inc: Alfred Perlstein * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * We need to keep track of the svid values used for F_FLOCK locks. */ struct nlm_file_svid { int ns_refs; /* thread count + 1 if active */ int ns_svid; /* on-the-wire SVID for this file */ struct ucred *ns_ucred; /* creds to use for lock recovery */ void *ns_id; /* local struct file pointer */ bool_t ns_active; /* TRUE if we own a lock */ LIST_ENTRY(nlm_file_svid) ns_link; }; LIST_HEAD(nlm_file_svid_list, nlm_file_svid); #define NLM_SVID_HASH_SIZE 256 struct nlm_file_svid_list nlm_file_svids[NLM_SVID_HASH_SIZE]; struct mtx nlm_svid_lock; static struct unrhdr *nlm_svid_allocator; static volatile u_int nlm_xid = 1; static int nlm_setlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size, bool_t reclaim); static int nlm_clearlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size); static int nlm_getlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size); static int nlm_map_status(nlm4_stats stat); static struct nlm_file_svid *nlm_find_svid(void *id); static void nlm_free_svid(struct nlm_file_svid *nf); static int nlm_init_lock(struct flock *fl, int flags, int svid, rpcvers_t vers, size_t fhlen, void *fh, off_t size, struct nlm4_lock *lock, char oh_space[32]); static void nlm_client_init(void *dummy) { int i; mtx_init(&nlm_svid_lock, "NLM svid lock", NULL, MTX_DEF); - /* pid_max cannot be greater then PID_MAX */ + /* pid_max cannot be greater than PID_MAX */ nlm_svid_allocator = new_unrhdr(PID_MAX + 2, INT_MAX, &nlm_svid_lock); for (i = 0; i < NLM_SVID_HASH_SIZE; i++) LIST_INIT(&nlm_file_svids[i]); } SYSINIT(nlm_client_init, SI_SUB_LOCK, SI_ORDER_FIRST, nlm_client_init, NULL); static int nlm_msg(struct thread *td, const char *server, const char *msg, int error) { struct proc *p; p = td ? td->td_proc : NULL; if (error) { tprintf(p, LOG_INFO, "nfs server %s: %s, error %d\n", server, msg, error); } else { tprintf(p, LOG_INFO, "nfs server %s: %s\n", server, msg); } return (0); } struct nlm_feedback_arg { bool_t nf_printed; struct nfsmount *nf_nmp; }; static void nlm_down(struct nlm_feedback_arg *nf, struct thread *td, const char *msg, int error) { struct nfsmount *nmp = nf->nf_nmp; if (nmp == NULL) return; mtx_lock(&nmp->nm_mtx); if (!(nmp->nm_state & NFSSTA_LOCKTIMEO)) { nmp->nm_state |= NFSSTA_LOCKTIMEO; mtx_unlock(&nmp->nm_mtx); vfs_event_signal(&nmp->nm_mountp->mnt_stat.f_fsid, VQ_NOTRESPLOCK, 0); } else { mtx_unlock(&nmp->nm_mtx); } nf->nf_printed = TRUE; nlm_msg(td, nmp->nm_mountp->mnt_stat.f_mntfromname, msg, error); } static void nlm_up(struct nlm_feedback_arg *nf, struct thread *td, const char *msg) { struct nfsmount *nmp = nf->nf_nmp; if (!nf->nf_printed) return; nlm_msg(td, nmp->nm_mountp->mnt_stat.f_mntfromname, msg, 0); mtx_lock(&nmp->nm_mtx); if (nmp->nm_state & NFSSTA_LOCKTIMEO) { nmp->nm_state &= ~NFSSTA_LOCKTIMEO; mtx_unlock(&nmp->nm_mtx); vfs_event_signal(&nmp->nm_mountp->mnt_stat.f_fsid, VQ_NOTRESPLOCK, 1); } else { mtx_unlock(&nmp->nm_mtx); } } static void nlm_feedback(int type, int proc, void *arg) { struct thread *td = curthread; struct nlm_feedback_arg *nf = (struct nlm_feedback_arg *) arg; switch (type) { case FEEDBACK_REXMIT2: case FEEDBACK_RECONNECT: nlm_down(nf, td, "lockd not responding", 0); break; case FEEDBACK_OK: nlm_up(nf, td, "lockd is alive again"); break; } } /* * nlm_advlock -- * NFS advisory byte-level locks. */ static int nlm_advlock_internal(struct vnode *vp, void *id, int op, struct flock *fl, int flags, bool_t reclaim, bool_t unlock_vp) { struct thread *td = curthread; struct nfsmount *nmp; off_t size; size_t fhlen; union nfsfh fh; struct sockaddr *sa; struct sockaddr_storage ss; char servername[MNAMELEN]; struct timeval timo; int retries; rpcvers_t vers; struct nlm_host *host; struct rpc_callextra ext; struct nlm_feedback_arg nf; AUTH *auth; struct ucred *cred; struct nlm_file_svid *ns; int svid; int error; int is_v3; ASSERT_VOP_LOCKED(vp, "nlm_advlock_1"); nmp = VFSTONFS(vp->v_mount); /* * Push any pending writes to the server and flush our cache * so that if we are contending with another machine for a * file, we get whatever they wrote and vice-versa. */ if (op == F_SETLK || op == F_UNLCK) nmp->nm_vinvalbuf(vp, V_SAVE, td, 1); strcpy(servername, nmp->nm_hostname); nmp->nm_getinfo(vp, fh.fh_bytes, &fhlen, &ss, &is_v3, &size, &timo); sa = (struct sockaddr *) &ss; if (is_v3 != 0) vers = NLM_VERS4; else vers = NLM_VERS; if (nmp->nm_flag & NFSMNT_SOFT) retries = nmp->nm_retry; else retries = INT_MAX; if (unlock_vp) VOP_UNLOCK(vp, 0); /* * We need to switch to mount-point creds so that we can send * packets from a privileged port. */ cred = td->td_ucred; td->td_ucred = vp->v_mount->mnt_cred; host = nlm_find_host_by_name(servername, sa, vers); auth = authunix_create(cred); memset(&ext, 0, sizeof(ext)); nf.nf_printed = FALSE; nf.nf_nmp = nmp; ext.rc_auth = auth; ext.rc_feedback = nlm_feedback; ext.rc_feedback_arg = &nf; ext.rc_timers = NULL; ns = NULL; if (flags & F_FLOCK) { ns = nlm_find_svid(id); KASSERT(fl->l_start == 0 && fl->l_len == 0, ("F_FLOCK lock requests must be whole-file locks")); if (!ns->ns_ucred) { /* * Remember the creds used for locking in case * we need to recover the lock later. */ ns->ns_ucred = crdup(cred); } svid = ns->ns_svid; } else if (flags & F_REMOTE) { /* * If we are recovering after a server restart or * trashing locks on a force unmount, use the same * svid as last time. */ svid = fl->l_pid; } else { svid = ((struct proc *) id)->p_pid; } switch(op) { case F_SETLK: if ((flags & (F_FLOCK|F_WAIT)) == (F_FLOCK|F_WAIT) && fl->l_type == F_WRLCK) { /* * The semantics for flock(2) require that any * shared lock on the file must be released * before an exclusive lock is granted. The * local locking code interprets this by * unlocking the file before sleeping on a * blocked exclusive lock request. We * approximate this by first attempting * non-blocking and if that fails, we unlock * the file and block. */ error = nlm_setlock(host, &ext, vers, &timo, retries, vp, F_SETLK, fl, flags & ~F_WAIT, svid, fhlen, &fh.fh_bytes, size, reclaim); if (error == EAGAIN) { fl->l_type = F_UNLCK; error = nlm_clearlock(host, &ext, vers, &timo, retries, vp, F_UNLCK, fl, flags, svid, fhlen, &fh.fh_bytes, size); fl->l_type = F_WRLCK; if (!error) { mtx_lock(&nlm_svid_lock); if (ns->ns_active) { ns->ns_refs--; ns->ns_active = FALSE; } mtx_unlock(&nlm_svid_lock); flags |= F_WAIT; error = nlm_setlock(host, &ext, vers, &timo, retries, vp, F_SETLK, fl, flags, svid, fhlen, &fh.fh_bytes, size, reclaim); } } } else { error = nlm_setlock(host, &ext, vers, &timo, retries, vp, op, fl, flags, svid, fhlen, &fh.fh_bytes, size, reclaim); } if (!error && ns) { mtx_lock(&nlm_svid_lock); if (!ns->ns_active) { /* * Add one to the reference count to * hold onto the SVID for the lifetime * of the lock. Note that since * F_FLOCK only supports whole-file * locks, there can only be one active * lock for this SVID. */ ns->ns_refs++; ns->ns_active = TRUE; } mtx_unlock(&nlm_svid_lock); } break; case F_UNLCK: error = nlm_clearlock(host, &ext, vers, &timo, retries, vp, op, fl, flags, svid, fhlen, &fh.fh_bytes, size); if (!error && ns) { mtx_lock(&nlm_svid_lock); if (ns->ns_active) { ns->ns_refs--; ns->ns_active = FALSE; } mtx_unlock(&nlm_svid_lock); } break; case F_GETLK: error = nlm_getlock(host, &ext, vers, &timo, retries, vp, op, fl, flags, svid, fhlen, &fh.fh_bytes, size); break; default: error = EINVAL; break; } if (ns) nlm_free_svid(ns); td->td_ucred = cred; AUTH_DESTROY(auth); nlm_host_release(host); return (error); } int nlm_advlock(struct vop_advlock_args *ap) { return (nlm_advlock_internal(ap->a_vp, ap->a_id, ap->a_op, ap->a_fl, ap->a_flags, FALSE, TRUE)); } /* * Set the creds of td to the creds of the given lock's owner. The new * creds reference count will be incremented via crhold. The caller is * responsible for calling crfree and restoring td's original creds. */ static void nlm_set_creds_for_lock(struct thread *td, struct flock *fl) { int i; struct nlm_file_svid *ns; struct proc *p; struct ucred *cred; cred = NULL; if (fl->l_pid > PID_MAX) { /* * If this was originally a F_FLOCK-style lock, we * recorded the creds used when it was originally * locked in the nlm_file_svid structure. */ mtx_lock(&nlm_svid_lock); for (i = 0; i < NLM_SVID_HASH_SIZE; i++) { for (ns = LIST_FIRST(&nlm_file_svids[i]); ns; ns = LIST_NEXT(ns, ns_link)) { if (ns->ns_svid == fl->l_pid) { cred = crhold(ns->ns_ucred); break; } } } mtx_unlock(&nlm_svid_lock); } else { /* * This lock is owned by a process. Get a reference to * the process creds. */ p = pfind(fl->l_pid); if (p) { cred = crhold(p->p_ucred); PROC_UNLOCK(p); } } /* * If we can't find a cred, fall back on the recovery * thread's cred. */ if (!cred) { cred = crhold(td->td_ucred); } td->td_ucred = cred; } static int nlm_reclaim_free_lock(struct vnode *vp, struct flock *fl, void *arg) { struct flock newfl; struct thread *td = curthread; struct ucred *oldcred; int error; newfl = *fl; newfl.l_type = F_UNLCK; oldcred = td->td_ucred; nlm_set_creds_for_lock(td, &newfl); error = nlm_advlock_internal(vp, NULL, F_UNLCK, &newfl, F_REMOTE, FALSE, FALSE); crfree(td->td_ucred); td->td_ucred = oldcred; return (error); } int nlm_reclaim(struct vop_reclaim_args *ap) { nlm_cancel_wait(ap->a_vp); lf_iteratelocks_vnode(ap->a_vp, nlm_reclaim_free_lock, NULL); return (0); } struct nlm_recovery_context { struct nlm_host *nr_host; /* host we are recovering */ int nr_state; /* remote NSM state for recovery */ }; static int nlm_client_recover_lock(struct vnode *vp, struct flock *fl, void *arg) { struct nlm_recovery_context *nr = (struct nlm_recovery_context *) arg; struct thread *td = curthread; struct ucred *oldcred; int state, error; /* * If the remote NSM state changes during recovery, the host * must have rebooted a second time. In that case, we must * restart the recovery. */ state = nlm_host_get_state(nr->nr_host); if (nr->nr_state != state) return (ERESTART); error = vn_lock(vp, LK_SHARED); if (error) return (error); oldcred = td->td_ucred; nlm_set_creds_for_lock(td, fl); error = nlm_advlock_internal(vp, NULL, F_SETLK, fl, F_REMOTE, TRUE, TRUE); crfree(td->td_ucred); td->td_ucred = oldcred; return (error); } void nlm_client_recovery(struct nlm_host *host) { struct nlm_recovery_context nr; int sysid, error; sysid = NLM_SYSID_CLIENT | nlm_host_get_sysid(host); do { nr.nr_host = host; nr.nr_state = nlm_host_get_state(host); error = lf_iteratelocks_sysid(sysid, nlm_client_recover_lock, &nr); } while (error == ERESTART); } static void nlm_convert_to_nlm_lock(struct nlm_lock *dst, struct nlm4_lock *src) { dst->caller_name = src->caller_name; dst->fh = src->fh; dst->oh = src->oh; dst->svid = src->svid; dst->l_offset = src->l_offset; dst->l_len = src->l_len; } static void nlm_convert_to_nlm4_holder(struct nlm4_holder *dst, struct nlm_holder *src) { dst->exclusive = src->exclusive; dst->svid = src->svid; dst->oh = src->oh; dst->l_offset = src->l_offset; dst->l_len = src->l_len; } static void nlm_convert_to_nlm4_res(struct nlm4_res *dst, struct nlm_res *src) { dst->cookie = src->cookie; dst->stat.stat = (enum nlm4_stats) src->stat.stat; } static enum clnt_stat nlm_test_rpc(rpcvers_t vers, nlm4_testargs *args, nlm4_testres *res, CLIENT *client, struct rpc_callextra *ext, struct timeval timo) { if (vers == NLM_VERS4) { return nlm4_test_4(args, res, client, ext, timo); } else { nlm_testargs args1; nlm_testres res1; enum clnt_stat stat; args1.cookie = args->cookie; args1.exclusive = args->exclusive; nlm_convert_to_nlm_lock(&args1.alock, &args->alock); memset(&res1, 0, sizeof(res1)); stat = nlm_test_1(&args1, &res1, client, ext, timo); if (stat == RPC_SUCCESS) { res->cookie = res1.cookie; res->stat.stat = (enum nlm4_stats) res1.stat.stat; if (res1.stat.stat == nlm_denied) nlm_convert_to_nlm4_holder( &res->stat.nlm4_testrply_u.holder, &res1.stat.nlm_testrply_u.holder); } return (stat); } } static enum clnt_stat nlm_lock_rpc(rpcvers_t vers, nlm4_lockargs *args, nlm4_res *res, CLIENT *client, struct rpc_callextra *ext, struct timeval timo) { if (vers == NLM_VERS4) { return nlm4_lock_4(args, res, client, ext, timo); } else { nlm_lockargs args1; nlm_res res1; enum clnt_stat stat; args1.cookie = args->cookie; args1.block = args->block; args1.exclusive = args->exclusive; nlm_convert_to_nlm_lock(&args1.alock, &args->alock); args1.reclaim = args->reclaim; args1.state = args->state; memset(&res1, 0, sizeof(res1)); stat = nlm_lock_1(&args1, &res1, client, ext, timo); if (stat == RPC_SUCCESS) { nlm_convert_to_nlm4_res(res, &res1); } return (stat); } } static enum clnt_stat nlm_cancel_rpc(rpcvers_t vers, nlm4_cancargs *args, nlm4_res *res, CLIENT *client, struct rpc_callextra *ext, struct timeval timo) { if (vers == NLM_VERS4) { return nlm4_cancel_4(args, res, client, ext, timo); } else { nlm_cancargs args1; nlm_res res1; enum clnt_stat stat; args1.cookie = args->cookie; args1.block = args->block; args1.exclusive = args->exclusive; nlm_convert_to_nlm_lock(&args1.alock, &args->alock); memset(&res1, 0, sizeof(res1)); stat = nlm_cancel_1(&args1, &res1, client, ext, timo); if (stat == RPC_SUCCESS) { nlm_convert_to_nlm4_res(res, &res1); } return (stat); } } static enum clnt_stat nlm_unlock_rpc(rpcvers_t vers, nlm4_unlockargs *args, nlm4_res *res, CLIENT *client, struct rpc_callextra *ext, struct timeval timo) { if (vers == NLM_VERS4) { return nlm4_unlock_4(args, res, client, ext, timo); } else { nlm_unlockargs args1; nlm_res res1; enum clnt_stat stat; args1.cookie = args->cookie; nlm_convert_to_nlm_lock(&args1.alock, &args->alock); memset(&res1, 0, sizeof(res1)); stat = nlm_unlock_1(&args1, &res1, client, ext, timo); if (stat == RPC_SUCCESS) { nlm_convert_to_nlm4_res(res, &res1); } return (stat); } } /* * Called after a lock request (set or clear) succeeded. We record the * details in the local lock manager. Note that since the remote * server has granted the lock, we can be sure that it doesn't * conflict with any other locks we have in the local lock manager. * * Since it is possible that host may also make NLM client requests to * our NLM server, we use a different sysid value to record our own * client locks. * * Note that since it is possible for us to receive replies from the * server in a different order than the locks were granted (e.g. if * many local threads are contending for the same lock), we must use a * blocking operation when registering with the local lock manager. * We expect that any actual wait will be rare and short hence we * ignore signals for this. */ static void nlm_record_lock(struct vnode *vp, int op, struct flock *fl, int svid, int sysid, off_t size) { struct vop_advlockasync_args a; struct flock newfl; int error; a.a_vp = vp; a.a_id = NULL; a.a_op = op; a.a_fl = &newfl; a.a_flags = F_REMOTE|F_WAIT|F_NOINTR; a.a_task = NULL; a.a_cookiep = NULL; newfl.l_start = fl->l_start; newfl.l_len = fl->l_len; newfl.l_type = fl->l_type; newfl.l_whence = fl->l_whence; newfl.l_pid = svid; newfl.l_sysid = NLM_SYSID_CLIENT | sysid; error = lf_advlockasync(&a, &vp->v_lockf, size); KASSERT(error == 0 || error == ENOENT, ("Failed to register NFS lock locally - error=%d", error)); } static int nlm_setlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size, bool_t reclaim) { struct nlm4_lockargs args; char oh_space[32]; struct nlm4_res res; u_int xid; CLIENT *client; enum clnt_stat stat; int retry, block, exclusive; void *wait_handle = NULL; int error; memset(&args, 0, sizeof(args)); memset(&res, 0, sizeof(res)); block = (flags & F_WAIT) ? TRUE : FALSE; exclusive = (fl->l_type == F_WRLCK); error = nlm_init_lock(fl, flags, svid, vers, fhlen, fh, size, &args.alock, oh_space); if (error) return (error); args.block = block; args.exclusive = exclusive; args.reclaim = reclaim; args.state = nlm_nsm_state; retry = 5*hz; for (;;) { client = nlm_host_get_rpc(host, FALSE); if (!client) return (ENOLCK); /* XXX retry? */ if (block) wait_handle = nlm_register_wait_lock(&args.alock, vp); xid = atomic_fetchadd_int(&nlm_xid, 1); args.cookie.n_len = sizeof(xid); args.cookie.n_bytes = (char*) &xid; stat = nlm_lock_rpc(vers, &args, &res, client, ext, *timo); CLNT_RELEASE(client); if (stat != RPC_SUCCESS) { if (block) nlm_deregister_wait_lock(wait_handle); if (retries) { retries--; continue; } return (EINVAL); } /* * Free res.cookie. */ xdr_free((xdrproc_t) xdr_nlm4_res, &res); if (block && res.stat.stat != nlm4_blocked) nlm_deregister_wait_lock(wait_handle); if (res.stat.stat == nlm4_denied_grace_period) { /* * The server has recently rebooted and is * giving old clients a change to reclaim * their locks. Wait for a few seconds and try * again. */ error = tsleep(&args, PCATCH, "nlmgrace", retry); if (error && error != EWOULDBLOCK) return (error); retry = 2*retry; if (retry > 30*hz) retry = 30*hz; continue; } if (block && res.stat.stat == nlm4_blocked) { /* * The server should call us back with a * granted message when the lock succeeds. In * order to deal with broken servers, lost * granted messages and server reboots, we * will also re-try every few seconds. */ error = nlm_wait_lock(wait_handle, retry); if (error == EWOULDBLOCK) { retry = 2*retry; if (retry > 30*hz) retry = 30*hz; continue; } if (error) { /* * We need to call the server to * cancel our lock request. */ nlm4_cancargs cancel; memset(&cancel, 0, sizeof(cancel)); xid = atomic_fetchadd_int(&nlm_xid, 1); cancel.cookie.n_len = sizeof(xid); cancel.cookie.n_bytes = (char*) &xid; cancel.block = block; cancel.exclusive = exclusive; cancel.alock = args.alock; do { client = nlm_host_get_rpc(host, FALSE); if (!client) /* XXX retry? */ return (ENOLCK); stat = nlm_cancel_rpc(vers, &cancel, &res, client, ext, *timo); CLNT_RELEASE(client); if (stat != RPC_SUCCESS) { /* * We need to cope * with temporary * network partitions * as well as server * reboots. This means * we have to keep * trying to cancel * until the server * wakes up again. */ pause("nlmcancel", 10*hz); } } while (stat != RPC_SUCCESS); /* * Free res.cookie. */ xdr_free((xdrproc_t) xdr_nlm4_res, &res); switch (res.stat.stat) { case nlm_denied: /* * There was nothing * to cancel. We are * going to go ahead * and assume we got * the lock. */ error = 0; break; case nlm4_denied_grace_period: /* * The server has * recently rebooted - * treat this as a * successful * cancellation. */ break; case nlm4_granted: /* * We managed to * cancel. */ break; default: /* * Broken server * implementation - * can't really do * anything here. */ break; } } } else { error = nlm_map_status(res.stat.stat); } if (!error && !reclaim) { nlm_record_lock(vp, op, fl, args.alock.svid, nlm_host_get_sysid(host), size); nlm_host_monitor(host, 0); } return (error); } } static int nlm_clearlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size) { struct nlm4_unlockargs args; char oh_space[32]; struct nlm4_res res; u_int xid; CLIENT *client; enum clnt_stat stat; int error; memset(&args, 0, sizeof(args)); memset(&res, 0, sizeof(res)); error = nlm_init_lock(fl, flags, svid, vers, fhlen, fh, size, &args.alock, oh_space); if (error) return (error); for (;;) { client = nlm_host_get_rpc(host, FALSE); if (!client) return (ENOLCK); /* XXX retry? */ xid = atomic_fetchadd_int(&nlm_xid, 1); args.cookie.n_len = sizeof(xid); args.cookie.n_bytes = (char*) &xid; stat = nlm_unlock_rpc(vers, &args, &res, client, ext, *timo); CLNT_RELEASE(client); if (stat != RPC_SUCCESS) { if (retries) { retries--; continue; } return (EINVAL); } /* * Free res.cookie. */ xdr_free((xdrproc_t) xdr_nlm4_res, &res); if (res.stat.stat == nlm4_denied_grace_period) { /* * The server has recently rebooted and is * giving old clients a change to reclaim * their locks. Wait for a few seconds and try * again. */ error = tsleep(&args, PCATCH, "nlmgrace", 5*hz); if (error && error != EWOULDBLOCK) return (error); continue; } /* * If we are being called via nlm_reclaim (which will * use the F_REMOTE flag), don't record the lock * operation in the local lock manager since the vnode * is going away. */ if (!(flags & F_REMOTE)) nlm_record_lock(vp, op, fl, args.alock.svid, nlm_host_get_sysid(host), size); return (0); } } static int nlm_getlock(struct nlm_host *host, struct rpc_callextra *ext, rpcvers_t vers, struct timeval *timo, int retries, struct vnode *vp, int op, struct flock *fl, int flags, int svid, size_t fhlen, void *fh, off_t size) { struct nlm4_testargs args; char oh_space[32]; struct nlm4_testres res; u_int xid; CLIENT *client; enum clnt_stat stat; int exclusive; int error; KASSERT(!(flags & F_FLOCK), ("unexpected F_FLOCK for F_GETLK")); memset(&args, 0, sizeof(args)); memset(&res, 0, sizeof(res)); exclusive = (fl->l_type == F_WRLCK); error = nlm_init_lock(fl, flags, svid, vers, fhlen, fh, size, &args.alock, oh_space); if (error) return (error); args.exclusive = exclusive; for (;;) { client = nlm_host_get_rpc(host, FALSE); if (!client) return (ENOLCK); /* XXX retry? */ xid = atomic_fetchadd_int(&nlm_xid, 1); args.cookie.n_len = sizeof(xid); args.cookie.n_bytes = (char*) &xid; stat = nlm_test_rpc(vers, &args, &res, client, ext, *timo); CLNT_RELEASE(client); if (stat != RPC_SUCCESS) { if (retries) { retries--; continue; } return (EINVAL); } if (res.stat.stat == nlm4_denied_grace_period) { /* * The server has recently rebooted and is * giving old clients a change to reclaim * their locks. Wait for a few seconds and try * again. */ xdr_free((xdrproc_t) xdr_nlm4_testres, &res); error = tsleep(&args, PCATCH, "nlmgrace", 5*hz); if (error && error != EWOULDBLOCK) return (error); continue; } if (res.stat.stat == nlm4_denied) { struct nlm4_holder *h = &res.stat.nlm4_testrply_u.holder; fl->l_start = h->l_offset; fl->l_len = h->l_len; fl->l_pid = h->svid; if (h->exclusive) fl->l_type = F_WRLCK; else fl->l_type = F_RDLCK; fl->l_whence = SEEK_SET; fl->l_sysid = 0; } else { fl->l_type = F_UNLCK; } xdr_free((xdrproc_t) xdr_nlm4_testres, &res); return (0); } } static int nlm_map_status(nlm4_stats stat) { switch (stat) { case nlm4_granted: return (0); case nlm4_denied: return (EAGAIN); case nlm4_denied_nolocks: return (ENOLCK); case nlm4_deadlck: return (EDEADLK); case nlm4_rofs: return (EROFS); case nlm4_stale_fh: return (ESTALE); case nlm4_fbig: return (EFBIG); case nlm4_failed: return (EACCES); default: return (EINVAL); } } static struct nlm_file_svid * nlm_find_svid(void *id) { struct nlm_file_svid *ns, *newns; int h; h = (((uintptr_t) id) >> 7) % NLM_SVID_HASH_SIZE; mtx_lock(&nlm_svid_lock); LIST_FOREACH(ns, &nlm_file_svids[h], ns_link) { if (ns->ns_id == id) { ns->ns_refs++; break; } } mtx_unlock(&nlm_svid_lock); if (!ns) { int svid = alloc_unr(nlm_svid_allocator); newns = malloc(sizeof(struct nlm_file_svid), M_NLM, M_WAITOK); newns->ns_refs = 1; newns->ns_id = id; newns->ns_svid = svid; newns->ns_ucred = NULL; newns->ns_active = FALSE; /* * We need to check for a race with some other * thread allocating a svid for this file. */ mtx_lock(&nlm_svid_lock); LIST_FOREACH(ns, &nlm_file_svids[h], ns_link) { if (ns->ns_id == id) { ns->ns_refs++; break; } } if (ns) { mtx_unlock(&nlm_svid_lock); free_unr(nlm_svid_allocator, newns->ns_svid); free(newns, M_NLM); } else { LIST_INSERT_HEAD(&nlm_file_svids[h], newns, ns_link); ns = newns; mtx_unlock(&nlm_svid_lock); } } return (ns); } static void nlm_free_svid(struct nlm_file_svid *ns) { mtx_lock(&nlm_svid_lock); ns->ns_refs--; if (!ns->ns_refs) { KASSERT(!ns->ns_active, ("Freeing active SVID")); LIST_REMOVE(ns, ns_link); mtx_unlock(&nlm_svid_lock); free_unr(nlm_svid_allocator, ns->ns_svid); if (ns->ns_ucred) crfree(ns->ns_ucred); free(ns, M_NLM); } else { mtx_unlock(&nlm_svid_lock); } } static int nlm_init_lock(struct flock *fl, int flags, int svid, rpcvers_t vers, size_t fhlen, void *fh, off_t size, struct nlm4_lock *lock, char oh_space[32]) { size_t oh_len; off_t start, len; if (fl->l_whence == SEEK_END) { if (size > OFF_MAX || (fl->l_start > 0 && size > OFF_MAX - fl->l_start)) return (EOVERFLOW); start = size + fl->l_start; } else if (fl->l_whence == SEEK_SET || fl->l_whence == SEEK_CUR) { start = fl->l_start; } else { return (EINVAL); } if (start < 0) return (EINVAL); if (fl->l_len < 0) { len = -fl->l_len; start -= len; if (start < 0) return (EINVAL); } else { len = fl->l_len; } if (vers == NLM_VERS) { /* * Enforce range limits on V1 locks */ if (start > 0xffffffffLL || len > 0xffffffffLL) return (EOVERFLOW); } snprintf(oh_space, 32, "%d@", svid); oh_len = strlen(oh_space); getcredhostname(NULL, oh_space + oh_len, 32 - oh_len); oh_len = strlen(oh_space); memset(lock, 0, sizeof(*lock)); lock->caller_name = prison0.pr_hostname; lock->fh.n_len = fhlen; lock->fh.n_bytes = fh; lock->oh.n_len = oh_len; lock->oh.n_bytes = oh_space; lock->svid = svid; lock->l_offset = start; lock->l_len = len; return (0); }