Index: share/man/man4/filemon.4
===================================================================
--- share/man/man4/filemon.4
+++ share/man/man4/filemon.4
@@ -31,7 +31,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 28, 2016
+.Dd March 2, 2016
 .Dt FILEMON 4
 .Os
 .Sh NAME
@@ -186,11 +186,4 @@
 device appeared in
 .Fx 9.1 .
 .Sh BUGS
-Loading
-.Nm
-may reduce system performance for the noted syscalls.
-.Pp
-Only children of the set process are logged.
-Processes can escape being traced by double forking.
-This is not seen as a problem as the intended use is build monitoring, which
-does not make sense to have daemons for.
+Unloading the module may panic the system.
Index: sys/dev/filemon/filemon.c
===================================================================
--- sys/dev/filemon/filemon.c
+++ sys/dev/filemon/filemon.c
@@ -46,7 +46,6 @@
 #include <sys/module.h>
 #include <sys/poll.h>
 #include <sys/proc.h>
-#include <sys/queue.h>
 #include <sys/sx.h>
 #include <sys/syscall.h>
 #include <sys/sysent.h>
@@ -68,8 +67,6 @@
 static d_close_t	filemon_close;
 static d_ioctl_t	filemon_ioctl;
 static d_open_t		filemon_open;
-static int		filemon_unload(void);
-static void		filemon_load(void *);
 
 static struct cdevsw filemon_cdevsw = {
 	.d_version	= D_VERSION,
@@ -83,22 +80,77 @@
 MALLOC_DEFINE(M_FILEMON, "filemon", "File access monitor");
 
 struct filemon {
-	TAILQ_ENTRY(filemon) link;	/* Link into the in-use list. */
 	struct sx	lock;		/* Lock mutex for this filemon. */
 	struct file	*fp;		/* Output file pointer. */
-	struct proc     *p;		/* The process being monitored. */
 	char		fname1[MAXPATHLEN]; /* Temporary filename buffer. */
 	char		fname2[MAXPATHLEN]; /* Temporary filename buffer. */
 	char		msgbufr[1024];	/* Output message buffer. */
+	volatile u_int	refcnt;		/* Pointer reference count. */
 };
 
-static TAILQ_HEAD(, filemon) filemons_inuse = TAILQ_HEAD_INITIALIZER(filemons_inuse);
-static TAILQ_HEAD(, filemon) filemons_free = TAILQ_HEAD_INITIALIZER(filemons_free);
-static struct sx access_lock;
-
 static struct cdev *filemon_dev;
 
-#include "filemon_lock.c"
+static void filemon_free(struct filemon *);
+
+static __inline struct filemon *
+filemon_acquire(struct filemon *filemon)
+{
+
+	if (filemon != NULL)
+		refcount_acquire(&filemon->refcnt);
+	return (filemon);
+}
+
+static __inline void
+filemon_release(struct filemon *filemon)
+{
+
+	if (refcount_release(&filemon->refcnt))
+		wakeup(filemon);
+}
+
+static struct filemon *
+filemon_proc_get(struct proc *p)
+{
+	struct filemon *filemon;
+
+	PROC_LOCK(p);
+	filemon = filemon_acquire(p->p_filemon);
+	PROC_UNLOCK(p);
+
+	if (filemon == NULL)
+		return (NULL);
+	sx_xlock(&filemon->lock);
+	return (filemon);
+}
+
+static void
+filemon_proc_drop_locked(struct proc *p)
+{
+
+	PROC_LOCK_ASSERT(p, MA_OWNED);
+
+	filemon_release(p->p_filemon);
+	p->p_filemon = NULL;
+}
+
+static void
+filemon_proc_drop(struct proc *p)
+{
+
+	PROC_LOCK(p);
+	filemon_proc_drop_locked(p);
+	PROC_UNLOCK(p);
+}
+
+static __inline void
+filemon_drop(struct filemon *filemon)
+{
+
+	sx_xunlock(&filemon->lock);
+	filemon_release(filemon);
+}
+
 #include "filemon_wrapper.c"
 
 static void
@@ -118,34 +170,78 @@
 }
 
 static void
-filemon_dtr(void *data)
+filemon_free(struct filemon *filemon)
 {
-	struct filemon *filemon = data;
-
-	if (filemon != NULL) {
-		struct file *fp;
+	size_t len;
+	struct timeval now;
 
-		/* Follow same locking order as filemon_pid_check. */
-		filemon_lock_write();
-		sx_xlock(&filemon->lock);
+	if (filemon->fp != NULL) {
+		getmicrotime(&now);
 
-		/* Remove from the in-use list. */
-		TAILQ_REMOVE(&filemons_inuse, filemon, link);
+		len = snprintf(filemon->msgbufr,
+		    sizeof(filemon->msgbufr),
+		    "# Stop %ju.%06ju\n# Bye bye\n",
+		    (uintmax_t)now.tv_sec, (uintmax_t)now.tv_usec);
 
-		fp = filemon->fp;
-		filemon->fp = NULL;
-		filemon->p = NULL;
+		filemon_output(filemon, filemon->msgbufr, len);
+		fdrop(filemon->fp, curthread);
+	}
 
-		/* Add to the free list. */
-		TAILQ_INSERT_TAIL(&filemons_free, filemon, link);
+	sx_destroy(&filemon->lock);
+	free(filemon, M_FILEMON);
+}
 
-		/* Give up write access. */
-		sx_xunlock(&filemon->lock);
-		filemon_unlock_write();
+/*
+ * Invalidate the passed filemon in all processes.  Uses the filemon.procs
+ * list for efficiency.
+ */
+static void
+filemon_untrack_processes(struct filemon *filemon)
+{
+	struct proc *p;
 
-		if (fp != NULL)
-			fdrop(fp, curthread);
+	sx_assert(&filemon->lock, SA_XLOCKED);
+
+	/* First reference is in the cdevpriv. */
+	if (filemon->refcnt == 1)
+		return;
+
+	/*
+	 * Processes in this list won't go away while here since
+	 * filemon_event_process_exit() will lock on filemon->lock
+	 * which we hold.
+	 */
+	sx_slock(&allproc_lock);
+	FOREACH_PROC_IN_SYSTEM(p) {
+		PROC_LOCK(p);
+		if (p->p_filemon == filemon)
+			filemon_proc_drop_locked(p);
+		PROC_UNLOCK(p);
 	}
+	sx_sunlock(&allproc_lock);
+}
+
+static void
+filemon_drain(struct filemon *filemon)
+{
+
+	if (filemon->refcnt > 0)
+		tsleep(filemon, 0, "filemon", 0);
+}
+
+/* The devfs file is being closed.  Untrace all processes and cleanup. */
+static void
+filemon_dtr(void *data)
+{
+	struct filemon *filemon = data;
+
+	if (filemon == NULL)
+		return;
+
+	filemon_untrack_processes(filemon);
+	filemon_release(filemon);
+	filemon_drain(filemon);
+	filemon_free(filemon);
 }
 
 static int
@@ -178,10 +274,16 @@
 
 	/* Set the monitored process ID. */
 	case FILEMON_SET_PID:
+		/* Invalidate any existing processes already set. */
+		filemon_untrack_processes(filemon);
+
 		error = pget(*((pid_t *)data), PGET_CANDEBUG | PGET_NOTWEXIT,
 		    &p);
 		if (error == 0) {
-			filemon->p = p;
+			if (p->p_filemon != NULL)
+				error = EBUSY;
+			else
+				p->p_filemon = filemon_acquire(filemon);
 			PROC_UNLOCK(p);
 		}
 		break;
@@ -199,35 +301,21 @@
 filemon_open(struct cdev *dev, int oflags __unused, int devtype __unused,
     struct thread *td __unused)
 {
+	int error;
 	struct filemon *filemon;
 
-	/* Get exclusive write access. */
-	filemon_lock_write();
-
-	if ((filemon = TAILQ_FIRST(&filemons_free)) != NULL)
-		TAILQ_REMOVE(&filemons_free, filemon, link);
+	filemon = malloc(sizeof(struct filemon), M_FILEMON,
+	    M_WAITOK | M_ZERO);
+	sx_init(&filemon->lock, "filemon");
+	refcount_init(&filemon->refcnt, 1);
 
-	/* Give up write access. */
-	filemon_unlock_write();
-
-	if (filemon == NULL) {
-		filemon = malloc(sizeof(struct filemon), M_FILEMON,
-		    M_WAITOK | M_ZERO);
-		sx_init(&filemon->lock, "filemon");
+	error = devfs_set_cdevpriv(filemon, filemon_dtr);
+	if (error != 0) {
+		refcount_release(&filemon->refcnt);
+		filemon_free(filemon);
 	}
 
-	devfs_set_cdevpriv(filemon, filemon_dtr);
-
-	/* Get exclusive write access. */
-	filemon_lock_write();
-
-	/* Add to the in-use list. */
-	TAILQ_INSERT_TAIL(&filemons_inuse, filemon, link);
-
-	/* Give up write access. */
-	filemon_unlock_write();
-
-	return (0);
+	return (error);
 }
 
 static int
@@ -241,7 +329,6 @@
 static void
 filemon_load(void *dummy __unused)
 {
-	sx_init(&access_lock, "filemons_inuse");
 
 	/* Install the syscall wrappers. */
 	filemon_wrapper_install();
@@ -253,38 +340,11 @@
 static int
 filemon_unload(void)
 {
- 	struct filemon *filemon;
-	int error = 0;
-
-	/* Get exclusive write access. */
-	filemon_lock_write();
 
-	if (TAILQ_FIRST(&filemons_inuse) != NULL)
-		error = EBUSY;
-	else {
-		destroy_dev(filemon_dev);
-
-		/* Deinstall the syscall wrappers. */
-		filemon_wrapper_deinstall();
-	}
+	destroy_dev(filemon_dev);
+	filemon_wrapper_deinstall();
 
-	/* Give up write access. */
-	filemon_unlock_write();
-
-	if (error == 0) {
-		/* free() filemon structs free list. */
-		filemon_lock_write();
-		while ((filemon = TAILQ_FIRST(&filemons_free)) != NULL) {
-			TAILQ_REMOVE(&filemons_free, filemon, link);
-			sx_destroy(&filemon->lock);
-			free(filemon, M_FILEMON);
-		}
-		filemon_unlock_write();
-
-		sx_destroy(&access_lock);
-	}
-
-	return (error);
+	return (0);
 }
 
 static int
@@ -301,6 +361,13 @@
 		error = filemon_unload();
 		break;
 
+	case MOD_QUIESCE:
+		/*
+		 * The wrapper implementation is unsafe for reliable unload.
+		 * Require forcing an unload.
+		 */
+		error = EBUSY;
+
 	case MOD_SHUTDOWN:
 		break;
 
Index: sys/dev/filemon/filemon_lock.c
===================================================================
--- sys/dev/filemon/filemon_lock.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (c) 2009-2011, Juniper Networks, Inc.
- * Copyright (c) 2015, EMC Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY JUNIPER NETWORKS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL JUNIPER NETWORKS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-static __inline void
-filemon_lock_read(void)
-{
-
-	sx_slock(&access_lock);
-}
-
-static __inline void
-filemon_unlock_read(void)
-{
-
-	sx_sunlock(&access_lock);
-}
-
-static __inline void
-filemon_lock_write(void)
-{
-
-	sx_xlock(&access_lock);
-}
-
-static __inline void
-filemon_unlock_write(void)
-{
-
-	sx_xunlock(&access_lock);
-}
Index: sys/dev/filemon/filemon_wrapper.c
===================================================================
--- sys/dev/filemon/filemon_wrapper.c
+++ sys/dev/filemon/filemon_wrapper.c
@@ -64,33 +64,6 @@
 	fo_write(filemon->fp, &auio, curthread->td_ucred, 0, curthread);
 }
 
-static struct filemon *
-filemon_pid_check(struct proc *p)
-{
-	struct filemon *filemon;
-
-	filemon_lock_read();
-	if (TAILQ_EMPTY(&filemons_inuse)) {
-		filemon_unlock_read();
-		return (NULL);
-	}
-	sx_slock(&proctree_lock);
-	while (p->p_pid != 0) {
-		TAILQ_FOREACH(filemon, &filemons_inuse, link) {
-			if (p == filemon->p) {
-				sx_sunlock(&proctree_lock);
-				sx_xlock(&filemon->lock);
-				filemon_unlock_read();
-				return (filemon);
-			}
-		}
-		p = proc_realparent(p);
-	}
-	sx_sunlock(&proctree_lock);
-	filemon_unlock_read();
-	return (NULL);
-}
-
 static int
 filemon_wrapper_chdir(struct thread *td, struct chdir_args *uap)
 {
@@ -100,7 +73,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_chdir(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -110,7 +83,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -125,7 +98,7 @@
 	char *fullpath, *freepath;
 	size_t len;
 
-	if ((filemon = filemon_pid_check(p)) != NULL) {
+	if ((filemon = filemon_proc_get(p)) != NULL) {
 		fullpath = "<unknown>";
 		freepath = NULL;
 
@@ -138,7 +111,7 @@
 
 		filemon_output(filemon, filemon->msgbufr, len);
 
-		sx_xunlock(&filemon->lock);
+		filemon_drop(filemon);
 
 		free(freepath, M_TEMP);
 	}
@@ -153,7 +126,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_open(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -176,7 +149,7 @@
 			    curproc->p_pid, filemon->fname1);
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -192,7 +165,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_openat(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -228,7 +201,7 @@
 			    curproc->p_pid, filemon->fname2, filemon->fname1);
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -244,7 +217,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_rename(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->from, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 			copyinstr(uap->to, filemon->fname2,
@@ -256,7 +229,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -272,7 +245,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_link(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 			copyinstr(uap->link, filemon->fname2,
@@ -284,7 +257,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -300,7 +273,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_symlink(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 			copyinstr(uap->link, filemon->fname2,
@@ -312,7 +285,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -328,7 +301,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_linkat(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path1, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 			copyinstr(uap->path2, filemon->fname2,
@@ -340,7 +313,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -356,7 +329,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_stat(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -366,7 +339,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -384,7 +357,7 @@
 	struct filemon *filemon;
 
 	if ((ret = freebsd32_stat(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -394,7 +367,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -407,29 +380,16 @@
 {
 	size_t len;
 	struct filemon *filemon;
-	struct timeval now;
-
-	/* Get timestamp before locking. */
-	getmicrotime(&now);
 
-	if ((filemon = filemon_pid_check(p)) != NULL) {
+	if ((filemon = filemon_proc_get(p)) != NULL) {
 		len = snprintf(filemon->msgbufr, sizeof(filemon->msgbufr),
 		    "X %d %d %d\n", p->p_pid, p->p_xexit, p->p_xsig);
 
 		filemon_output(filemon, filemon->msgbufr, len);
 
-		/* Check if the monitored process is about to exit. */
-		if (filemon->p == p) {
-			len = snprintf(filemon->msgbufr,
-			    sizeof(filemon->msgbufr),
-			    "# Stop %ju.%06ju\n# Bye bye\n",
-			    (uintmax_t)now.tv_sec, (uintmax_t)now.tv_usec);
+		filemon_proc_drop(p);
 
-			filemon_output(filemon, filemon->msgbufr, len);
-			filemon->p = NULL;
-		}
-
-		sx_xunlock(&filemon->lock);
+		filemon_drop(filemon);
 	}
 }
 
@@ -442,7 +402,7 @@
 	struct filemon *filemon;
 
 	if ((ret = sys_unlink(td, uap)) == 0) {
-		if ((filemon = filemon_pid_check(curproc)) != NULL) {
+		if ((filemon = filemon_proc_get(curproc)) != NULL) {
 			copyinstr(uap->path, filemon->fname1,
 			    sizeof(filemon->fname1), &done);
 
@@ -452,7 +412,7 @@
 
 			filemon_output(filemon, filemon->msgbufr, len);
 
-			sx_xunlock(&filemon->lock);
+			filemon_drop(filemon);
 		}
 	}
 
@@ -466,14 +426,18 @@
 	size_t len;
 	struct filemon *filemon;
 
-	if ((filemon = filemon_pid_check(p1)) != NULL) {
+	if ((filemon = filemon_proc_get(p1)) != NULL) {
 		len = snprintf(filemon->msgbufr,
 		    sizeof(filemon->msgbufr), "F %d %d\n",
 		    p1->p_pid, p2->p_pid);
 
 		filemon_output(filemon, filemon->msgbufr, len);
 
-		sx_xunlock(&filemon->lock);
+		PROC_LOCK(p2);
+		p2->p_filemon = filemon_acquire(filemon);
+		PROC_UNLOCK(p2);
+
+		filemon_drop(filemon);
 	}
 }
 
Index: sys/sys/proc.h
===================================================================
--- sys/sys/proc.h
+++ sys/sys/proc.h
@@ -162,6 +162,7 @@
  */
 struct cpuset;
 struct filecaps;
+struct filemon;
 struct kaioinfo;
 struct kaudit_record;
 struct kdtrace_proc;
@@ -580,6 +581,7 @@
 	struct procdesc	*p_procdesc;	/* (e) Process descriptor, if any. */
 	u_int		p_treeflag;	/* (e) P_TREE flags */
 	int		p_pendingexits; /* (c) Count of pending thread exits. */
+	struct filemon	*p_filemon;	/* (c) filemon-specific data. */
 /* End area that is zeroed on creation. */
 #define	p_endzero	p_magic