Index: head/contrib/hyperv/tools/hv_vss_daemon.8
===================================================================
--- head/contrib/hyperv/tools/hv_vss_daemon.8
+++ head/contrib/hyperv/tools/hv_vss_daemon.8
@@ -0,0 +1,88 @@
+.\" Copyright (c) 2016 Microsoft Corp.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd October 12, 2016
+.Dt HV_VSS_DAEMON 8
+.Os
+.Sh NAME
+.Nm hv_vss_daemon
+.Nd Hyper-V Volume Shadow Copy Service Daemon
+.Sh SYNOPSIS
+.Nm
+.Op Fl dn
+.Sh DESCRIPTION
+The
+.Nm
+daemon provides the ability to freeze and thaw the file system for
+.Fx
+guest partitions running on Hyper-V.
+.Pp
+Hyper-V allows administrators to backup or restore the 
+.Fx
+guest partition.
+Administrators can
+use Windows Powershell scripts to backup or restore the
+.Fx
+VM.
+.Pp
+The
+.Nm
+accepts file system freeze and thaw requests from the
+.Xr hv_utils 4
+driver and performs the actual file-system operation.
+.Pp
+The file system freeze and thaw functionality is
+useful when the Hyper-V host wants to do live backup of
+.Fx
+guest. Hyper-V host sends file system freezing request to
+.Nm
+which conducts the real operation. After successfully freezing file
+system, Hyper-V host takes a snapshot of the VM. In the future,
+Hyper-V host can restore the
+.Fx
+VM through that snapshot.
+.Pp
+The options are as follows:
+.Bl -tag -width indent
+.It Fl d
+Run as regular process instead of a daemon for debugging purpose.
+.It Fl n
+Generate debugging output.
+.El
+.Sh SEE ALSO
+.Xr hv_vmbus 4 ,
+.Xr hv_utils 4 ,
+.Xr hv_netvsc 4 ,
+.Xr hv_storvsc 4 ,
+.Xr hv_kvp 4
+.Sh HISTORY
+The daemon was introduced in October 2016 and developed by Microsoft Corp.
+.Sh AUTHORS
+.An -nosplit
+.Fx
+support for
+.Nm
+was first added by
+.An Microsoft BSD Integration Services Team Aq Mt bsdic@microsoft.com .
Index: head/contrib/hyperv/tools/hv_vss_daemon.c
===================================================================
--- head/contrib/hyperv/tools/hv_vss_daemon.c
+++ head/contrib/hyperv/tools/hv_vss_daemon.c
@@ -0,0 +1,275 @@
+#include <string.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <sys/param.h>
+#include <sys/ucred.h>
+#include <sys/mount.h>
+#include <sys/types.h>
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <stdint.h>
+#include <syslog.h>
+#include <errno.h>
+#include <err.h>
+#include <fcntl.h>
+#include <ufs/ffs/fs.h>
+#include <paths.h>
+#include <sysexits.h>
+
+#include "hv_snapshot.h"
+
+#define UNDEF_FREEZE_THAW       (0)
+#define FREEZE                  (1)
+#define THAW                    (2)
+
+#define	VSS_LOG(priority, format, args...) do	{				\
+		if (is_debugging == 1) {					\
+			if (is_daemon == 1)					\
+				syslog(priority, format, ## args);		\
+			else							\
+				printf(format, ## args);			\
+		} else {							\
+			if (priority < LOG_DEBUG) {				\
+				if (is_daemon == 1)				\
+					syslog(priority, format, ## args);	\
+				else						\
+					printf(format, ## args);		\
+			}							\
+		}								\
+	} while(0)
+
+static int is_daemon = 1;
+static int is_debugging = 0;
+static int g_ufs_suspend_handle = -1;
+
+static const char *dev = "/dev";
+
+static int
+check(void)
+{
+	struct statfs *mntbuf, *statfsp;
+	int mntsize;
+	int i;
+
+	mntsize = getmntinfo(&mntbuf, MNT_NOWAIT);
+	if (mntsize == 0) {
+		VSS_LOG(LOG_ERR, "There is no mount information\n");
+		return (EINVAL);
+	}
+	for (i = mntsize - 1; i >= 0; --i)
+	{
+		statfsp = &mntbuf[i];
+
+		if (strncmp(statfsp->f_mntonname, dev, strlen(dev)) == 0) {
+			continue; /* skip to freeze '/dev' */
+		} else if (statfsp->f_flags & MNT_RDONLY) {
+			continue; /* skip to freeze RDONLY partition */
+		} else if (strncmp(statfsp->f_fstypename, "ufs", 3) != 0) {
+			return (EPERM); /* only UFS can be freezed */
+		}
+	}
+
+	return (0);
+}
+
+static int
+freeze(void)
+{
+	struct statfs *mntbuf, *statfsp;
+	int mntsize;
+	int error = 0;
+	int i;
+
+	g_ufs_suspend_handle = open(_PATH_UFSSUSPEND, O_RDWR);
+	if (g_ufs_suspend_handle == -1) {
+		VSS_LOG(LOG_ERR, "unable to open %s", _PATH_UFSSUSPEND);
+		return (errno);
+	}
+
+	mntsize = getmntinfo(&mntbuf, MNT_NOWAIT);
+	if (mntsize == 0) {
+		VSS_LOG(LOG_ERR, "There is no mount information\n");
+		return (EINVAL);
+	}
+	for (i = mntsize - 1; i >= 0; --i)
+	{
+		statfsp = &mntbuf[i];
+
+		if (strncmp(statfsp->f_mntonname, dev, strlen(dev)) == 0) {
+			continue; /* skip to freeze '/dev' */
+		} else if (statfsp->f_flags & MNT_RDONLY) {
+			continue; /* skip to freeze RDONLY partition */
+		} else if (strncmp(statfsp->f_fstypename, "ufs", 3) != 0) {
+			continue; /* only UFS can be freezed */
+		}
+		error = ioctl(g_ufs_suspend_handle, UFSSUSPEND, &statfsp->f_fsid);
+		if (error != 0) {
+			VSS_LOG(LOG_ERR, "error: %d\n", errno);
+			error = errno;
+		} else {
+			VSS_LOG(LOG_INFO, "Successfully suspend fs: %s\n",
+			    statfsp->f_mntonname);
+		}
+	}
+
+	return (error);
+}
+
+/**
+ * close the opened handle will thaw the FS.
+ */
+static int
+thaw(void)
+{
+	int error = 0;
+	if (g_ufs_suspend_handle != -1) {
+		error = close(g_ufs_suspend_handle);
+		if (!error) {
+			g_ufs_suspend_handle = -1;
+			VSS_LOG(LOG_INFO, "Successfully thaw the fs\n");
+		} else {
+			error = errno;
+			VSS_LOG(LOG_ERR, "Fail to thaw the fs: "
+			    "%d %s\n", errno, strerror(errno));
+		}
+	} else {
+		VSS_LOG(LOG_INFO, "The fs has already been thawed\n");
+	}
+
+	return (error);
+}
+
+static void
+usage(const char* cmd)
+{
+	fprintf(stderr, "%s: daemon for UFS file system freeze/thaw\n"
+	    " -d : enable debug log printing. Default is disabled.\n"
+	    " -n : run as a regular process instead of a daemon. Default is a daemon.\n"
+	    " -h : print usage.\n", cmd);
+	exit(1);
+}
+
+int
+main(int argc, char* argv[])
+{
+	struct hv_vss_opt_msg  userdata;
+
+	struct pollfd hv_vss_poll_fd[1];
+	uint32_t op;
+	int ch, r, len, error;
+	int hv_vss_dev_fd;
+
+	int freeze_thaw = UNDEF_FREEZE_THAW;
+	while ((ch = getopt(argc, argv, "dnh")) != -1) {
+		switch (ch) {
+		case 'n':
+			/* Run as regular process for debugging purpose. */
+			is_daemon = 0;
+			break;
+		case 'd':
+			/* Generate debugging output */
+			is_debugging = 1;
+			break;
+		case 'h':
+		default:
+			usage(argv[0]);
+			break;
+		}
+	}
+
+	openlog("HV_VSS", 0, LOG_USER);
+
+	/* Become daemon first. */
+	if (is_daemon == 1)
+		daemon(1, 0);
+	else
+		VSS_LOG(LOG_DEBUG, "Run as regular process.\n");
+
+	VSS_LOG(LOG_INFO, "HV_VSS starting; pid is: %d\n", getpid());
+
+	memset(&userdata, 0, sizeof(struct hv_vss_opt_msg));
+	/* register the daemon */
+	hv_vss_dev_fd = open(VSS_DEV(FS_VSS_DEV_NAME), O_RDWR);
+
+	if (hv_vss_dev_fd < 0) {
+		VSS_LOG(LOG_ERR, "Fail to open %s, error: %d %s\n",
+		    VSS_DEV(FS_VSS_DEV_NAME), errno, strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+	hv_vss_poll_fd[0].fd = hv_vss_dev_fd;
+	hv_vss_poll_fd[0].events = POLLIN | POLLRDNORM;
+
+	while (1) {
+		r = poll(hv_vss_poll_fd, 1, INFTIM);
+
+		VSS_LOG(LOG_DEBUG, "poll returned r = %d, revent = 0x%x\n",
+		    r, hv_vss_poll_fd[0].revents);
+
+		if (r == 0 || (r < 0 && errno == EAGAIN) ||
+		    (r < 0 && errno == EINTR)) {
+			/* Nothing to read */
+			continue;
+		}
+
+		if (r < 0) {
+			/*
+			 * For poll return failure other than EAGAIN,
+			 * we want to exit.
+			 */
+			VSS_LOG(LOG_ERR, "Poll failed.\n");
+			perror("poll");
+			exit(EIO);
+		}
+
+		/* Read from character device */
+		error = ioctl(hv_vss_dev_fd, IOCHVVSSREAD, &userdata);
+		if (error < 0) {
+			VSS_LOG(LOG_ERR, "Read failed.\n");
+			perror("pread");
+			exit(EIO);
+		}
+
+		if (userdata.status != 0) {
+			VSS_LOG(LOG_ERR, "data read error\n");
+			continue;
+		}
+
+		/*
+		 * We will use the KVP header information to pass back
+		 * the error from this daemon. So, first save the op
+		 * and pool info to local variables.
+		 */
+
+		op = userdata.opt;
+
+		switch (op) {
+		case HV_VSS_CHECK:
+			error = check();
+			break;
+		case HV_VSS_FREEZE:
+			error = freeze();
+			break;
+		case HV_VSS_THAW:
+			error = thaw();
+			break;
+		default:
+			VSS_LOG(LOG_ERR, "Illegal operation: %d\n", op);
+			error = VSS_FAIL;
+		}
+		if (error)
+			userdata.status = VSS_FAIL;
+		else
+			userdata.status = VSS_SUCCESS;
+		error = ioctl(hv_vss_dev_fd, IOCHVVSSWRITE, &userdata);
+		if (error != 0) {
+			VSS_LOG(LOG_ERR, "Fail to write to device\n");
+			exit(EXIT_FAILURE);
+		} else {
+			VSS_LOG(LOG_INFO, "Send response %d for %s to kernel\n",
+			    userdata.status, op == HV_VSS_FREEZE ? "Freeze" :
+			    (op == HV_VSS_THAW ? "Thaw" : "Check"));
+		}
+	}
+}
Index: head/etc/devd/hyperv.conf
===================================================================
--- head/etc/devd/hyperv.conf
+++ head/etc/devd/hyperv.conf
@@ -17,3 +17,19 @@
 	match "cdev"		"hv_kvp_dev";
 	action "pkill -x hv_kvp_daemon";
 };
+
+notify 11 {
+	match "system"		"DEVFS";
+	match "subsystem"	"CDEV";
+	match "type"		"CREATE";
+	match "cdev"		"hv_fsvss_dev";
+	action "/usr/sbin/hv_vss_daemon";
+};
+
+notify 11 {
+	match "system"		"DEVFS";
+	match "subsystem"	"CDEV";
+	match "type"		"DESTROY";
+	match "cdev"		"hv_fsvss_dev";
+	action "pkill -x hv_vss_daemon";
+};
Index: head/etc/mtree/BSD.include.dist
===================================================================
--- head/etc/mtree/BSD.include.dist
+++ head/etc/mtree/BSD.include.dist
@@ -118,6 +118,8 @@
         ..
         hwpmc
         ..
+        hyperv
+        ..
         ic
         ..
         iicbus
Index: head/include/Makefile
===================================================================
--- head/include/Makefile
+++ head/include/Makefile
@@ -44,7 +44,7 @@
 
 LSUBDIRS=	cam/ata cam/nvme cam/scsi \
 	dev/acpica dev/agp dev/an dev/bktr dev/ciss dev/filemon dev/firewire \
-	dev/hwpmc \
+	dev/hwpmc dev/hyperv \
 	dev/ic dev/iicbus dev/io dev/lmc dev/mfi dev/nvme \
 	dev/ofw dev/pbio dev/pci ${_dev_powermac_nvram} dev/ppbus dev/smbus \
 	dev/speaker dev/utopia dev/vkbd dev/wi \
@@ -155,7 +155,7 @@
 		done; \
 	fi
 .endfor
-.for i in ${LDIRS} ${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/nand:Ndev/pci} ${LSUBSUBDIRS}
+.for i in ${LDIRS} ${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/hyperv:Ndev/nand:Ndev/pci} ${LSUBSUBDIRS}
 	cd ${.CURDIR}/../sys; \
 	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 $i/*.h \
 	    ${DESTDIR}${INCLUDEDIR}/$i
@@ -185,6 +185,9 @@
 	    ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
 	${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 uinput.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/evdev
+	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
+	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hv_snapshot.h \
+	    ${DESTDIR}${INCLUDEDIR}/dev/hyperv
 	cd ${.CURDIR}/../sys/dev/pci; \
 	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 pcireg.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/pci
@@ -257,7 +260,7 @@
 		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../sys/$i/$$h ${DESTDIR}${INCLUDEDIR}/$i; \
 	done
 .endfor
-.for i in ${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/nand:Ndev/pci}
+.for i in ${LSUBDIRS:Ndev/agp:Ndev/acpica:Ndev/bktr:Ndev/evdev:Ndev/hyperv:Ndev/nand:Ndev/pci}
 	cd ${.CURDIR}/../sys/$i; \
 	for h in *.h; do \
 		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/$i/$$h ${DESTDIR}${INCLUDEDIR}/$i; \
@@ -290,6 +293,11 @@
 		ln -fs ../../../../sys/dev/evdev/$$h \
 		    ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
 	done
+	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
+	for h in hv_snapshot.h; do \
+		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/utilities/$$h \
+		    ${DESTDIR}${INCLUDEDIR}/dev/hyperv; \
+	done
 	cd ${.CURDIR}/../sys/dev/pci; \
 	for h in pcireg.h; do \
 		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/pci/$$h \
Index: head/share/man/man4/Makefile
===================================================================
--- head/share/man/man4/Makefile
+++ head/share/man/man4/Makefile
@@ -192,6 +192,7 @@
 	hv_storvsc.4 \
 	hv_utils.4 \
 	hv_vmbus.4 \
+	hv_vss.4 \
 	hwpmc.4 \
 	ichsmb.4 \
 	${_ichwd.4} \
Index: head/share/man/man4/hv_vss.4
===================================================================
--- head/share/man/man4/hv_vss.4
+++ head/share/man/man4/hv_vss.4
@@ -0,0 +1,366 @@
+.\" Copyright (c) 2016 Microsoft Corp.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.Dd October 12, 2016
+.Dt HV_VSS 4
+.Os
+.Sh NAME
+.Nm hv_vss
+.Nd Hyper-V Volume Shadow Copy Service API
+.Sh SYNOPSIS
+.In dev/hyperv/hv_snapshot.h
+.Bd -literal
+#define VSS_SUCCESS		0x00000000
+#define VSS_FAIL		0x00000001
+
+enum hv_vss_op_t {
+	HV_VSS_NONE = 0,
+	HV_VSS_CHECK,
+	HV_VSS_FREEZE,
+	HV_VSS_THAW,
+	HV_VSS_COUNT
+};
+
+struct hv_vss_opt_msg {
+	uint32_t	opt;		/* operation */
+	uint32_t	status;		/* 0 for success, 1 for error */
+	uint64_t	msgid;		/* an ID used to identify the transaction */
+	uint8_t		reserved[48];	/* reserved values are all zeroes */
+};
+.Ed
+.Sh DESCRIPTION
+The freeze or thaw functionality of application is important to guarantee
+the application consistent backup. On windows platform, VSS is defined to do
+live backup. But for VM guest running on Hyper-V, the corresponding VSS is
+not defined yet. For example, a running database server instance, it knows when the
+applications' freeze/thaw should start or finish. But it is not aware of
+the freeze/thaw notification from Hyper-V host. The
+.Nm
+is designed to notify application freeze/thaw request.
+Thus, it plays a role of broker to forward the freeze/thaw command from Hyper-V host
+to userland application if it registered VSS service on
+.Fx
+VM, and sends the result back to Hyper-V host.
+.Pp
+Generally,
+.Xr hv_vss_daemon 8
+takes the responsiblity to freeze/thaw UFS file system,
+and it is automatically launched after system boots. When Hyper-V host wants to
+take a snapshot of the
+.Fx
+VM, it will first send VSS capability check to
+.Fx
+VM. The
+.Nm
+received the request and forward the request to userland application if it is
+registered. Only after
+.Nm
+received the VSS_SUCCESS response from application, the
+.Xr hv_vss_daemon 8
+will be informed to check whether file system freeze/thaw is supported. Any error
+occurs during this period,
+.Nm
+will inform Hyper-V host that VSS is not supported. In addition, there is a default
+timeout limit before sending response to Hyper-V host.
+If the total response time from application and
+.Xr hv_vss_daemon 8
+exceeds this value, timeout
+will occurs and VSS unsupported is responsed to Hyper-V host.
+.Pp
+After Hyper-V host confirmed the
+.Fx
+VM supports VSS, it will send freeze request to VM, and
+.Nm
+will first forward it to application. After application finished freezing, it should
+inform
+.Nm
+and file system level freezing will be triggered by
+.Xr hv_vss_daemon 8 . After all freezing
+on both application and
+.Xr hv_vss_daemon 8
+were finished, the
+.Nm
+will inform Hyper-V host that freezing is done. Of course, there is a timeout limit as
+same as VSS capability is set to make sure freezing on
+.Fx
+VM is not hang. If there is any error occurs or timeout happened, the freezing is failed
+on Hyper-V side.
+.Pp
+Hyper-V host will send thaw request after taking the snapshot, typically, this period is
+very short in order not to block the running application.
+.Nm
+firstly thaw the file system by notifying
+.Xr hv_vss_daemon 8 ,
+then notifies user registered
+application. There is also a timeout check before sending response to Hyper-V host.
+.Pp
+All the default timeout limit used in VSS capability check, freeze or thaw is the same.
+It is 15 seconds currently.
+.Sh NOTES
+.Nm
+only support UFS currently. If any of file system partition is non UFS, the VSS capability
+check will fail. If application does not register VSS,
+.Nm
+only support backup for file system level consistent. The device should be closed before it
+was opened again. If you want to simultaneously open "/dev/hv_appvss_dev" two or more times,
+an error (-1) will be returned, and errno was set.
+.Pp
+If
+.Xr hv_vss_daemon 8
+was killed after system boots, the VSS functionality will not work.
+.Sh EXAMPLES
+The following is a complete example which does nothing except for waiting 2 seconds when
+receiving those notifications from
+.Nm
+.Bd -literal
+#include <string.h>
+#include <stdio.h>
+#include <sys/ioctl.h>
+#include <sys/param.h>
+#include <sys/ucred.h>
+#include <sys/mount.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <poll.h>
+#include <stdint.h>
+#include <syslog.h>
+#include <errno.h>
+#include <err.h>
+#include <fcntl.h>
+#include <ufs/ffs/fs.h>
+#include <paths.h>
+#include <sys/ioccom.h>
+#include <dev/hyperv/hv_snapshot.h>
+
+#define UNDEF_FREEZE_THAW	(0)
+#define FREEZE			(1)
+#define THAW			(2)
+#define CHECK			(3)
+
+#define	VSS_LOG(priority, format, args...) do	{				\\
+		if (is_debugging == 1) {					\\
+			if (is_daemon == 1)					\\
+				syslog(priority, format, ## args);		\\
+			else							\\
+				printf(format, ## args);			\\
+		} else {							\\
+			if (priority < LOG_DEBUG) {				\\
+				if (is_daemon == 1)				\\
+					syslog(priority, format, ## args);	\\
+				else						\\
+					printf(format, ## args);		\\
+			}							\\
+		}								\\
+	} while(0)
+
+#define CHECK_TIMEOUT		1
+#define CHECK_FAIL		2
+#define FREEZE_TIMEOUT		1
+#define FREEZE_FAIL		2
+#define THAW_TIMEOUT		1
+#define THAW_FAIL		2
+
+static int is_daemon        = 1;
+static int is_debugging     = 0;
+static int simu_opt_waiting = 2; // seconds
+
+#define GENERIC_OPT(TIMEOUT, FAIL)						\\
+	do {									\\
+		sleep(simu_opt_waiting);					\\
+		if (opt == CHECK_TIMEOUT) {					\\
+			sleep(simu_opt_waiting * 10);				\\
+			VSS_LOG(LOG_INFO, "%s timeout simulation\\n",		\\
+			    __func__);						\\
+			return (0);						\\
+		} else if (opt == CHECK_FAIL) {					\\
+			VSS_LOG(LOG_INFO, "%s failure simulation\\n",		\\
+			    __func__);						\\
+			return (CHECK_FAIL);					\\
+		} else {							\\
+			VSS_LOG(LOG_INFO, "%s success simulation\\n",		\\
+			    __func__);						\\
+			return (0);						\\
+		}								\\
+	} while (0)
+
+static int
+check(int opt)
+{
+	GENERIC_OPT(CHECK_TIMEOUT, CHECK_FAIL);
+}
+
+static int
+freeze(int opt)
+{
+	GENERIC_OPT(FREEZE_TIMEOUT, FREEZE_FAIL);
+}
+
+static int
+thaw(int opt)
+{
+	GENERIC_OPT(THAW_TIMEOUT, THAW_FAIL);
+}
+
+static void usage(const char* cmd) {
+	fprintf(stderr,
+	    "%s -f <0|1|2>: simulate app freeze."
+	    " 0: successful, 1: freeze timeout, 2: freeze failed\\n"
+	    " -c <0|1|2>: simulate vss feature check"
+	    " -t <0|1|2>: simulate app thaw."
+	    " 0: successful, 1: freeze timeout, 2: freeze failed\\n"
+	    " -d : enable debug mode\\n"
+	    " -n : run this tool under non-daemon mode\\n", cmd);
+}
+
+int
+main(int argc, char* argv[]) {
+	int ch, freezesimuop = 0, thawsimuop = 0, checksimuop = 0, fd, r, error;
+	uint32_t op;
+	struct pollfd app_vss_fd[1];
+	struct hv_vss_opt_msg  userdata;
+
+	while ((ch = getopt(argc, argv, "f:c:t:dnh")) != -1) {
+		switch (ch) {
+		case 'f':
+			/* Run as regular process for debugging purpose. */
+			freezesimuop = (int)strtol(optarg, NULL, 10);
+			break;
+		case 't':
+			thawsimuop = (int)strtol(optarg, NULL, 10);
+			break;
+		case 'c':
+			checksimuop = (int)strtol(optarg, NULL, 10);
+			break;
+		case 'd':
+			is_debugging = 1;
+			break;
+		case 'n':
+			is_daemon = 0;
+			break;
+		case 'h':
+		default:
+			usage(argv[0]);
+			exit(0);
+		}
+	}
+
+	openlog("APPVSS", 0, LOG_USER);
+	/* Become daemon first. */
+	if (is_daemon == 1)
+		daemon(1, 0);
+	else
+		VSS_LOG(LOG_DEBUG, "Run as regular process.\\n");
+
+	VSS_LOG(LOG_INFO, "HV_VSS starting; pid is: %d\\n", getpid());
+
+	fd = open(VSS_DEV(APP_VSS_DEV_NAME), O_RDWR);
+	if (fd < 0) {
+		VSS_LOG(LOG_ERR, "Fail to open %s, error: %d %s\\n",
+		    VSS_DEV(APP_VSS_DEV_NAME), errno, strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+	app_vss_fd[0].fd     = fd;
+	app_vss_fd[0].events = POLLIN | POLLRDNORM;
+
+	while (1) {
+		r = poll(app_vss_fd, 1, INFTIM);
+
+		VSS_LOG(LOG_DEBUG, "poll returned r = %d, revent = 0x%x\\n",
+		    r, app_vss_fd[0].revents);
+
+		if (r == 0 || (r < 0 && errno == EAGAIN) ||
+		    (r < 0 && errno == EINTR)) {
+			/* Nothing to read */
+			continue;
+		}
+
+		if (r < 0) {
+			/*
+			 * For poll return failure other than EAGAIN,
+			 * we want to exit.
+			 */
+			VSS_LOG(LOG_ERR, "Poll failed.\\n");
+			perror("poll");
+			exit(EIO);
+		}
+
+		/* Read from character device */
+		error = ioctl(fd, IOCHVVSSREAD, &userdata);
+		if (error < 0) {
+			VSS_LOG(LOG_ERR, "Read failed.\\n");
+			perror("pread");
+			exit(EIO);
+		}
+
+		if (userdata.status != 0) {
+			VSS_LOG(LOG_ERR, "data read error\\n");
+			continue;
+		}
+
+		op = userdata.opt;
+
+		switch (op) {
+		case HV_VSS_CHECK:
+			error = check(checksimuop);
+			break;
+		case HV_VSS_FREEZE:
+			error = freeze(freezesimuop);
+			break;
+		case HV_VSS_THAW:
+			error = thaw(thawsimuop);
+			break;
+		default:
+			VSS_LOG(LOG_ERR, "Illegal operation: %d\\n", op);
+			error = VSS_FAIL;
+		}
+		if (error)
+			userdata.status = VSS_FAIL;
+		else
+			userdata.status = VSS_SUCCESS;
+		error = ioctl(fd, IOCHVVSSWRITE, &userdata);
+		if (error != 0) {
+			VSS_LOG(LOG_ERR, "Fail to write to device\\n");
+			exit(EXIT_FAILURE);
+		} else {
+			VSS_LOG(LOG_INFO, "Send response %d for %s to kernel\\n",
+			    userdata.status, op == HV_VSS_FREEZE ? "Freeze" :
+			    (op == HV_VSS_THAW ? "Thaw" : "Check"));
+		}
+	}
+	return 0;
+}
+.Sh SEE ALSO
+.Xr hv_vss_daemon 8 ,
+.Xr hv_utils 4
+.Sh HISTORY
+The daemon was introduced in October 2016 and developed by Microsoft Corp.
+.Sh AUTHORS
+.An -nosplit
+.Fx
+support for
+.Nm
+was first added by
+.An Microsoft BSD Integration Services Team Aq Mt bsdic@microsoft.com .
Index: head/sys/conf/files.amd64
===================================================================
--- head/sys/conf/files.amd64
+++ head/sys/conf/files.amd64
@@ -298,6 +298,7 @@
 dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c		optional	hyperv
 dev/hyperv/utilities/hv_heartbeat.c			optional	hyperv
 dev/hyperv/utilities/hv_kvp.c				optional	hyperv
+dev/hyperv/utilities/hv_snapshot.c			optional	hyperv
 dev/hyperv/utilities/hv_shutdown.c			optional	hyperv
 dev/hyperv/utilities/hv_timesync.c			optional	hyperv
 dev/hyperv/utilities/hv_util.c				optional	hyperv
Index: head/sys/conf/files.i386
===================================================================
--- head/sys/conf/files.i386
+++ head/sys/conf/files.i386
@@ -255,6 +255,7 @@
 dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c		optional	hyperv
 dev/hyperv/utilities/hv_heartbeat.c			optional	hyperv
 dev/hyperv/utilities/hv_kvp.c				optional	hyperv
+dev/hyperv/utilities/hv_snapshot.c			optional	hyperv
 dev/hyperv/utilities/hv_shutdown.c			optional	hyperv
 dev/hyperv/utilities/hv_timesync.c			optional	hyperv
 dev/hyperv/utilities/hv_util.c				optional	hyperv
Index: head/sys/dev/hyperv/utilities/hv_snapshot.h
===================================================================
--- head/sys/dev/hyperv/utilities/hv_snapshot.h
+++ head/sys/dev/hyperv/utilities/hv_snapshot.h
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2016 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VSS_H
+#define _VSS_H
+#include <sys/ioccom.h>
+#define FS_VSS_DEV_NAME		"hv_fsvss_dev"
+#define APP_VSS_DEV_NAME	"hv_appvss_dev"
+
+#define VSS_DEV(VSS)		"/dev/"VSS
+
+#define VSS_SUCCESS		0x00000000
+#define VSS_FAIL		0x00000001
+
+enum hv_vss_op_t {
+	HV_VSS_NONE = 0,
+	HV_VSS_CHECK,
+	HV_VSS_FREEZE,
+	HV_VSS_THAW,
+	HV_VSS_COUNT
+};
+
+struct hv_vss_opt_msg {
+	uint32_t	opt;		/* operation */
+	uint32_t	status;		/* 0 for success, 1 for error */
+	uint64_t	msgid;		/* an ID used to identify the transaction */
+	uint8_t		reserved[48];	/* reserved values are all zeroes */
+};
+#define IOCHVVSSREAD		_IOR('v', 2, struct hv_vss_opt_msg)
+#define IOCHVVSSWRITE		_IOW('v', 3, struct hv_vss_opt_msg)
+#endif
Index: head/sys/dev/hyperv/utilities/hv_snapshot.c
===================================================================
--- head/sys/dev/hyperv/utilities/hv_snapshot.c
+++ head/sys/dev/hyperv/utilities/hv_snapshot.c
@@ -0,0 +1,1061 @@
+/*-
+ * Copyright (c) 2016 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/conf.h>
+#include <sys/uio.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/taskqueue.h>
+#include <sys/selinfo.h>
+#include <sys/sysctl.h>
+#include <sys/poll.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
+#include <sys/kthread.h>
+#include <sys/syscallsubr.h>
+#include <sys/sysproto.h>
+#include <sys/un.h>
+#include <sys/endian.h>
+#include <sys/sema.h>
+#include <sys/signal.h>
+#include <sys/syslog.h>
+#include <sys/systm.h>
+#include <sys/mutex.h>
+#include <sys/callout.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/utilities/hv_utilreg.h>
+#include <dev/hyperv/utilities/vmbus_icreg.h>
+
+#include "hv_util.h"
+#include "hv_snapshot.h"
+#include "vmbus_if.h"
+
+#define VSS_MAJOR		5
+#define VSS_MINOR		0
+#define VSS_MSGVER		VMBUS_IC_VERSION(VSS_MAJOR, VSS_MINOR)
+
+#define VSS_FWVER_MAJOR		3
+#define VSS_FWVER		VMBUS_IC_VERSION(VSS_FWVER_MAJOR, 0)
+
+#define TIMEOUT_LIMIT		(15)	// seconds
+enum hv_vss_op {
+	VSS_OP_CREATE = 0,
+	VSS_OP_DELETE,
+	VSS_OP_HOT_BACKUP,
+	VSS_OP_GET_DM_INFO,
+	VSS_OP_BU_COMPLETE,
+	/*
+	 * Following operations are only supported with IC version >= 5.0
+	 */
+	VSS_OP_FREEZE, /* Freeze the file systems in the VM */
+	VSS_OP_THAW, /* Unfreeze the file systems */
+	VSS_OP_AUTO_RECOVER,
+	VSS_OP_COUNT /* Number of operations, must be last */
+};
+
+/*
+ * Header for all VSS messages.
+ */
+struct hv_vss_hdr {
+	struct vmbus_icmsg_hdr	ic_hdr;
+	uint8_t			operation;
+	uint8_t			reserved[7];
+} __packed;
+
+
+/*
+ * Flag values for the hv_vss_check_feature. Here supports only
+ * one value.
+ */
+#define VSS_HBU_NO_AUTO_RECOVERY		0x00000005
+
+struct hv_vss_check_feature {
+	uint32_t flags;
+} __packed;
+
+struct hv_vss_check_dm_info {
+	uint32_t flags;
+} __packed;
+
+struct hv_vss_msg {
+	union {
+		struct hv_vss_hdr vss_hdr;
+	} hdr;
+	union {
+		struct hv_vss_check_feature vss_cf;
+		struct hv_vss_check_dm_info dm_info;
+	} body;
+} __packed;
+
+struct hv_vss_req {
+	struct hv_vss_opt_msg	opt_msg;	/* used to communicate with daemon */
+	struct hv_vss_msg	msg;		/* used to communicate with host */
+} __packed;
+
+/* hv_vss debug control */
+static int hv_vss_log = 0;
+
+#define	hv_vss_log_error(...)	do {				\
+	if (hv_vss_log > 0)					\
+		log(LOG_ERR, "hv_vss: " __VA_ARGS__);		\
+} while (0)
+
+#define	hv_vss_log_info(...) do {				\
+	if (hv_vss_log > 1)					\
+		log(LOG_INFO, "hv_vss: " __VA_ARGS__);		\
+} while (0)
+
+static const struct vmbus_ic_desc vmbus_vss_descs[] = {
+	{
+		.ic_guid = { .hv_guid = {
+		    0x29, 0x2e, 0xfa, 0x35, 0x23, 0xea, 0x36, 0x42,
+		    0x96, 0xae, 0x3a, 0x6e, 0xba, 0xcb, 0xa4,  0x40} },
+		.ic_desc = "Hyper-V VSS"
+	},
+	VMBUS_IC_DESC_END
+};
+
+static const char * vss_opt_name[] = {"None", "VSSCheck", "Freeze", "Thaw"};
+
+/* character device prototypes */
+static d_open_t		hv_vss_dev_open;
+static d_close_t	hv_vss_dev_close;
+static d_poll_t		hv_vss_dev_daemon_poll;
+static d_ioctl_t	hv_vss_dev_daemon_ioctl;
+
+static d_open_t		hv_appvss_dev_open;
+static d_close_t	hv_appvss_dev_close;
+static d_poll_t		hv_appvss_dev_poll;
+static d_ioctl_t	hv_appvss_dev_ioctl;
+
+/* hv_vss character device structure */
+static struct cdevsw hv_vss_cdevsw =
+{
+	.d_version	= D_VERSION,
+	.d_open		= hv_vss_dev_open,
+	.d_close	= hv_vss_dev_close,
+	.d_poll		= hv_vss_dev_daemon_poll,
+	.d_ioctl	= hv_vss_dev_daemon_ioctl,
+	.d_name		= FS_VSS_DEV_NAME,
+};
+
+static struct cdevsw hv_appvss_cdevsw =
+{
+	.d_version	= D_VERSION,
+	.d_open		= hv_appvss_dev_open,
+	.d_close	= hv_appvss_dev_close,
+	.d_poll		= hv_appvss_dev_poll,
+	.d_ioctl	= hv_appvss_dev_ioctl,
+	.d_name		= APP_VSS_DEV_NAME,
+};
+
+struct hv_vss_sc;
+/*
+ * Global state to track cdev
+ */
+struct hv_vss_dev_sc {
+	/*
+	 * msg was transferred from host to notify queue, and
+	 * ack queue. Finally, it was recyled to free list.
+	 */
+	STAILQ_HEAD(, hv_vss_req_internal) 	to_notify_queue;
+	STAILQ_HEAD(, hv_vss_req_internal) 	to_ack_queue;
+	struct hv_vss_sc			*sc;
+	struct proc				*proc_task;
+	struct selinfo				hv_vss_selinfo;
+};
+/*
+ * Global state to track and synchronize the transaction requests from the host.
+ * The VSS allows user to register their function to do freeze/thaw for application.
+ * VSS kernel will notify both vss daemon and user application if it is registered.
+ * The implementation state transition is illustrated by:
+ * https://clovertrail.github.io/assets/vssdot.png
+ */
+typedef struct hv_vss_sc {
+	struct hv_util_sc			util_sc;
+	device_t				dev;
+
+	struct task				task;
+
+	/*
+	 * mutex is used to protect access of list/queue,
+	 * callout in request is also used this mutex.
+	 */
+	struct mtx				pending_mutex;
+	/*
+	 * req_free_list contains all free items
+	 */
+	LIST_HEAD(, hv_vss_req_internal)	req_free_list;
+
+	/* Indicates if daemon registered with driver */
+	boolean_t				register_done;
+
+	boolean_t				app_register_done;
+
+	/* cdev for file system freeze/thaw */
+	struct cdev				*hv_vss_dev;
+	/* cdev for application freeze/thaw */
+	struct cdev				*hv_appvss_dev;
+
+	/* sc for app */
+	struct hv_vss_dev_sc			app_sc;
+	/* sc for deamon */
+	struct hv_vss_dev_sc			daemon_sc;
+} hv_vss_sc;
+
+typedef struct hv_vss_req_internal {
+	LIST_ENTRY(hv_vss_req_internal)		link;
+	STAILQ_ENTRY(hv_vss_req_internal)	slink;
+	struct hv_vss_req			vss_req;
+
+	/* Rcv buffer for communicating with the host*/
+	uint8_t					*rcv_buf;
+	/* Length of host message */
+	uint32_t				host_msg_len;
+	/* Host message id */
+	uint64_t				host_msg_id;
+
+	hv_vss_sc				*sc;
+
+	struct callout				callout;
+} hv_vss_req_internal;
+
+#define SEARCH_REMOVE_REQ_LOCKED(reqp, queue, link, tmp, id)		\
+	do {								\
+		STAILQ_FOREACH_SAFE(reqp, queue, link, tmp) {		\
+			if (reqp->vss_req.opt_msg.msgid == id) {	\
+				STAILQ_REMOVE(queue,			\
+				    reqp, hv_vss_req_internal, link);	\
+				break;					\
+			}						\
+		}							\
+	} while (0)
+
+static bool
+hv_vss_is_daemon_killed_after_launch(hv_vss_sc *sc)
+{
+	return (!sc->register_done && sc->daemon_sc.proc_task);
+}
+
+/*
+ * Callback routine that gets called whenever there is a message from host
+ */
+static void
+hv_vss_callback(struct vmbus_channel *chan __unused, void *context)
+{
+	hv_vss_sc *sc = (hv_vss_sc*)context;
+	if (hv_vss_is_daemon_killed_after_launch(sc))
+		hv_vss_log_info("%s: daemon was killed!\n", __func__);
+	if (sc->register_done || sc->daemon_sc.proc_task) {
+		hv_vss_log_info("%s: Queuing work item\n", __func__);
+		if (hv_vss_is_daemon_killed_after_launch(sc))
+			hv_vss_log_info("%s: daemon was killed!\n", __func__);
+		taskqueue_enqueue(taskqueue_thread, &sc->task);
+	} else {
+		hv_vss_log_info("%s: daemon has never been registered\n", __func__);
+	}
+	hv_vss_log_info("%s: received msg from host\n", __func__);
+}
+/*
+ * Send the response back to the host.
+ */
+static void
+hv_vss_respond_host(uint8_t *rcv_buf, struct vmbus_channel *ch,
+    uint32_t recvlen, uint64_t requestid, uint32_t error)
+{
+	struct vmbus_icmsg_hdr *hv_icmsg_hdrp;
+
+	hv_icmsg_hdrp = (struct vmbus_icmsg_hdr *)rcv_buf;
+
+	hv_icmsg_hdrp->ic_status = error;
+	hv_icmsg_hdrp->ic_flags = HV_ICMSGHDRFLAG_TRANSACTION | HV_ICMSGHDRFLAG_RESPONSE;
+
+	error = vmbus_chan_send(ch, VMBUS_CHANPKT_TYPE_INBAND, 0,
+	    rcv_buf, recvlen, requestid);
+	if (error)
+		hv_vss_log_info("%s: hv_vss_respond_host: sendpacket error:%d\n",
+		    __func__, error);
+}
+
+static void
+hv_vss_notify_host_result_locked(struct hv_vss_req_internal *reqp, uint32_t status)
+{
+	struct hv_vss_msg* msg = (struct hv_vss_msg *)reqp->rcv_buf;
+	hv_vss_sc *sc = reqp->sc;
+	if (reqp->vss_req.opt_msg.opt == HV_VSS_CHECK) {
+		msg->body.vss_cf.flags = VSS_HBU_NO_AUTO_RECOVERY;
+	}
+	hv_vss_log_info("%s, %s response %s to host\n", __func__,
+	    vss_opt_name[reqp->vss_req.opt_msg.opt],
+	    status == HV_S_OK ? "Success" : "Fail");
+	hv_vss_respond_host(reqp->rcv_buf, vmbus_get_channel(reqp->sc->dev),
+	    reqp->host_msg_len, reqp->host_msg_id, status);
+	/* recycle the request */
+	LIST_INSERT_HEAD(&sc->req_free_list, reqp, link);
+}
+
+static void
+hv_vss_notify_host_result(struct hv_vss_req_internal *reqp, uint32_t status)
+{
+	mtx_lock(&reqp->sc->pending_mutex);
+	hv_vss_notify_host_result_locked(reqp, status);
+	mtx_unlock(&reqp->sc->pending_mutex);
+}
+
+static void
+hv_vss_cp_vssreq_to_user(struct hv_vss_req_internal *reqp,
+    struct hv_vss_opt_msg *userdata)
+{
+	struct hv_vss_req *hv_vss_dev_buf;
+	hv_vss_dev_buf = &reqp->vss_req;
+	hv_vss_dev_buf->opt_msg.opt = HV_VSS_NONE;
+	switch (reqp->vss_req.msg.hdr.vss_hdr.operation) {
+	case VSS_OP_FREEZE:
+		hv_vss_dev_buf->opt_msg.opt = HV_VSS_FREEZE;
+		break;
+	case VSS_OP_THAW:
+		hv_vss_dev_buf->opt_msg.opt = HV_VSS_THAW;
+		break;
+	case VSS_OP_HOT_BACKUP:
+		hv_vss_dev_buf->opt_msg.opt = HV_VSS_CHECK;
+		break;
+	}
+	*userdata = hv_vss_dev_buf->opt_msg;
+	hv_vss_log_info("%s, read data from user for "
+	    "%s (%ju) \n", __func__, vss_opt_name[userdata->opt],
+	    (uintmax_t)userdata->msgid);
+}
+
+/**
+ * Remove the request id from app notifiy or ack queue,
+ * and recyle the request by inserting it to free list.
+ *
+ * When app was notified but not yet sending ack, the request
+ * should locate in either notify queue or ack queue.
+ */
+static struct hv_vss_req_internal*
+hv_vss_drain_req_queue_locked(hv_vss_sc *sc, uint64_t req_id)
+{
+	struct hv_vss_req_internal *reqp, *tmp;
+	SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->daemon_sc.to_notify_queue,
+	    slink, tmp, req_id);
+	if (reqp == NULL)
+		SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->daemon_sc.to_ack_queue,
+		    slink, tmp, req_id);
+	if (reqp == NULL)
+		SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->app_sc.to_notify_queue,
+		    slink, tmp, req_id);
+	if (reqp == NULL)
+		SEARCH_REMOVE_REQ_LOCKED(reqp, &sc->app_sc.to_ack_queue, slink,
+		    tmp, req_id);
+	return (reqp);
+}
+/**
+ * Actions for daemon who has been notified.
+ */
+static void
+hv_vss_notified(struct hv_vss_dev_sc *dev_sc, struct hv_vss_opt_msg *userdata)
+{
+	struct hv_vss_req_internal *reqp;
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	if (!STAILQ_EMPTY(&dev_sc->to_notify_queue)) {
+		reqp = STAILQ_FIRST(&dev_sc->to_notify_queue);
+		hv_vss_cp_vssreq_to_user(reqp, userdata);
+		STAILQ_REMOVE_HEAD(&dev_sc->to_notify_queue, slink);
+		/* insert the msg to queue for write */
+		STAILQ_INSERT_TAIL(&dev_sc->to_ack_queue, reqp, slink);
+		userdata->status = VSS_SUCCESS;
+	} else {
+		/* Timeout occur, thus request was removed from queue. */
+		hv_vss_log_info("%s: notify queue is empty!\n", __func__);
+		userdata->status = VSS_FAIL;
+	}
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+}
+
+static void
+hv_vss_notify(struct hv_vss_dev_sc *dev_sc, struct hv_vss_req_internal *reqp)
+{
+	uint32_t opt = reqp->vss_req.opt_msg.opt;
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	STAILQ_INSERT_TAIL(&dev_sc->to_notify_queue, reqp, slink);
+	hv_vss_log_info("%s: issuing query %s (%ju) to %s\n", __func__,
+	    vss_opt_name[opt], (uintmax_t)reqp->vss_req.opt_msg.msgid,
+	    &dev_sc->sc->app_sc == dev_sc ? "app" : "daemon");
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+	selwakeup(&dev_sc->hv_vss_selinfo);
+}
+
+/**
+ * Actions for daemon who has acknowledged.
+ */
+static void
+hv_vss_daemon_acked(struct hv_vss_dev_sc *dev_sc, struct hv_vss_opt_msg *userdata)
+{
+	struct hv_vss_req_internal	*reqp, *tmp;
+	uint64_t			req_id;
+	int				opt;
+	uint32_t			status;
+
+	opt = userdata->opt;
+	req_id = userdata->msgid;
+	status = userdata->status;
+	/* make sure the reserved fields are all zeros. */
+	memset(&userdata->reserved, 0, sizeof(struct hv_vss_opt_msg) -
+	    __offsetof(struct hv_vss_opt_msg, reserved));
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	SEARCH_REMOVE_REQ_LOCKED(reqp, &dev_sc->to_ack_queue, slink, tmp, req_id);
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+	if (reqp == NULL) {
+		hv_vss_log_info("%s Timeout: fail to find daemon ack request\n",
+		    __func__);
+		userdata->status = VSS_FAIL;
+		return;
+	}
+	KASSERT(opt == reqp->vss_req.opt_msg.opt, ("Mismatched VSS operation!"));
+	hv_vss_log_info("%s, get response %d from daemon for %s (%ju) \n", __func__,
+	    status, vss_opt_name[opt], (uintmax_t)req_id);
+	switch (opt) {
+	case HV_VSS_CHECK:
+	case HV_VSS_FREEZE:
+		callout_drain(&reqp->callout);
+		hv_vss_notify_host_result(reqp,
+		    status == VSS_SUCCESS ? HV_S_OK : HV_E_FAIL);
+		break;
+	case HV_VSS_THAW:
+		if (dev_sc->sc->app_register_done) {
+			if (status == VSS_SUCCESS) {
+				hv_vss_notify(&dev_sc->sc->app_sc, reqp);
+			} else {
+				/* handle error */
+				callout_drain(&reqp->callout);
+				hv_vss_notify_host_result(reqp, HV_E_FAIL);
+			}
+		} else {
+			callout_drain(&reqp->callout);
+			hv_vss_notify_host_result(reqp,
+			    status == VSS_SUCCESS ? HV_S_OK : HV_E_FAIL);
+		}
+		break;
+	}
+}
+
+/**
+ * Actions for app who has acknowledged.
+ */
+static void
+hv_vss_app_acked(struct hv_vss_dev_sc *dev_sc, struct hv_vss_opt_msg *userdata)
+{
+	struct hv_vss_req_internal	*reqp, *tmp;
+	uint64_t			req_id;
+	int				opt;
+	uint8_t				status;
+
+	opt = userdata->opt;
+	req_id = userdata->msgid;
+	status = userdata->status;
+	/* make sure the reserved fields are all zeros. */
+	memset(&userdata->reserved, 0, sizeof(struct hv_vss_opt_msg) -
+	    __offsetof(struct hv_vss_opt_msg, reserved));
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	SEARCH_REMOVE_REQ_LOCKED(reqp, &dev_sc->to_ack_queue, slink, tmp, req_id);
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+	if (reqp == NULL) {
+		hv_vss_log_info("%s Timeout: fail to find app ack request\n",
+		    __func__);
+		userdata->status = VSS_FAIL;
+		return;
+	}
+	KASSERT(opt == reqp->vss_req.opt_msg.opt, ("Mismatched VSS operation!"));
+	hv_vss_log_info("%s, get response %d from app for %s (%ju) \n",
+	    __func__, status, vss_opt_name[opt], (uintmax_t)req_id);
+	if (dev_sc->sc->register_done) {
+		switch (opt) {
+		case HV_VSS_CHECK:
+		case HV_VSS_FREEZE:
+			if (status == VSS_SUCCESS) {
+				hv_vss_notify(&dev_sc->sc->daemon_sc, reqp);
+			} else {
+				/* handle error */
+				callout_drain(&reqp->callout);
+				hv_vss_notify_host_result(reqp, HV_E_FAIL);
+			}
+			break;
+		case HV_VSS_THAW:
+			callout_drain(&reqp->callout);
+			hv_vss_notify_host_result(reqp,
+			    status == VSS_SUCCESS ? HV_S_OK : HV_E_FAIL);
+			break;
+		}
+	} else {
+		hv_vss_log_info("%s, Fatal: vss daemon was killed\n", __func__);
+	}
+}
+
+static int
+hv_vss_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
+{
+	struct proc     *td_proc;
+	td_proc = td->td_proc;
+
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+	hv_vss_log_info("%s: %s opens device \"%s\" successfully.\n",
+	    __func__, td_proc->p_comm, FS_VSS_DEV_NAME);
+
+	if (dev_sc->sc->register_done)
+		return (EBUSY);
+
+	dev_sc->sc->register_done = true;
+	hv_vss_callback(vmbus_get_channel(dev_sc->sc->dev), dev_sc->sc);
+
+	dev_sc->proc_task = curproc;
+	return (0);
+}
+
+static int
+hv_vss_dev_close(struct cdev *dev, int fflag __unused, int devtype __unused,
+				 struct thread *td)
+{
+	struct proc     *td_proc;
+	td_proc = td->td_proc;
+
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	hv_vss_log_info("%s: %s closes device \"%s\"\n",
+	    __func__, td_proc->p_comm, FS_VSS_DEV_NAME);
+	dev_sc->sc->register_done = false;
+	return (0);
+}
+
+static int
+hv_vss_dev_daemon_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
+    struct thread *td)
+{
+	struct proc			*td_proc;
+	struct hv_vss_dev_sc		*sc;
+
+	td_proc = td->td_proc;
+	sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	hv_vss_log_info("%s: %s invoked vss ioctl\n", __func__, td_proc->p_comm);
+
+	struct hv_vss_opt_msg* userdata = (struct hv_vss_opt_msg*)data;
+	switch(cmd) {
+	case IOCHVVSSREAD:
+		hv_vss_notified(sc, userdata);
+		break;
+	case IOCHVVSSWRITE:
+		hv_vss_daemon_acked(sc, userdata);
+		break;
+	}
+	return (0);
+}
+
+/*
+ * hv_vss_daemon poll invokes this function to check if data is available
+ * for daemon to read.
+ */
+static int
+hv_vss_dev_daemon_poll(struct cdev *dev, int events, struct thread *td)
+{
+	int revent = 0;
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	/**
+	 * if there is data ready, inform daemon's poll
+	 */
+	if (!STAILQ_EMPTY(&dev_sc->to_notify_queue))
+		revent = POLLIN;
+	if (revent == 0)
+		selrecord(td, &dev_sc->hv_vss_selinfo);
+	hv_vss_log_info("%s return 0x%x\n", __func__, revent);
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+	return (revent);
+}
+
+static int
+hv_appvss_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
+{
+	struct proc     *td_proc;
+	td_proc = td->td_proc;
+
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+	hv_vss_log_info("%s: %s opens device \"%s\" successfully.\n",
+	    __func__, td_proc->p_comm, APP_VSS_DEV_NAME);
+
+	if (dev_sc->sc->app_register_done)
+		return (EBUSY);
+
+	dev_sc->sc->app_register_done = true;
+	dev_sc->proc_task = curproc;
+	return (0);
+}
+
+static int
+hv_appvss_dev_close(struct cdev *dev, int fflag __unused, int devtype __unused,
+				 struct thread *td)
+{
+	struct proc     *td_proc;
+	td_proc = td->td_proc;
+
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	hv_vss_log_info("%s: %s closes device \"%s\".\n",
+	    __func__, td_proc->p_comm, APP_VSS_DEV_NAME);
+	dev_sc->sc->app_register_done = false;
+	return (0);
+}
+
+static int
+hv_appvss_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
+    struct thread *td)
+{
+	struct proc			*td_proc;
+	struct hv_vss_dev_sc		*dev_sc;
+
+	td_proc = td->td_proc;
+	dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	hv_vss_log_info("%s: %s invoked vss ioctl\n", __func__, td_proc->p_comm);
+
+	struct hv_vss_opt_msg* userdata = (struct hv_vss_opt_msg*)data;
+	switch(cmd) {
+	case IOCHVVSSREAD:
+		hv_vss_notified(dev_sc, userdata);
+		break;
+	case IOCHVVSSWRITE:
+		hv_vss_app_acked(dev_sc, userdata);
+		break;
+	}
+	return (0);
+}
+
+/*
+ * hv_vss_daemon poll invokes this function to check if data is available
+ * for daemon to read.
+ */
+static int
+hv_appvss_dev_poll(struct cdev *dev, int events, struct thread *td)
+{
+	int revent = 0;
+	struct hv_vss_dev_sc *dev_sc = (struct hv_vss_dev_sc*)dev->si_drv1;
+
+	mtx_lock(&dev_sc->sc->pending_mutex);
+	/**
+	 * if there is data ready, inform daemon's poll
+	 */
+	if (!STAILQ_EMPTY(&dev_sc->to_notify_queue))
+		revent = POLLIN;
+	if (revent == 0)
+		selrecord(td, &dev_sc->hv_vss_selinfo);
+	hv_vss_log_info("%s return 0x%x\n", __func__, revent);
+	mtx_unlock(&dev_sc->sc->pending_mutex);
+	return (revent);
+}
+
+static void
+hv_vss_timeout(void *arg)
+{
+	hv_vss_req_internal *reqp = arg;
+	hv_vss_req_internal *request;
+	hv_vss_sc* sc = reqp->sc;
+	uint64_t req_id = reqp->vss_req.opt_msg.msgid;
+	/* This thread is locked */
+	KASSERT(mtx_owned(&sc->pending_mutex), ("mutex lock is not owned!"));
+	request = hv_vss_drain_req_queue_locked(sc, req_id);
+	KASSERT(request != NULL, ("timeout but fail to find request"));
+	hv_vss_notify_host_result_locked(reqp, HV_E_FAIL);
+}
+
+/*
+ * This routine is called whenever a message is received from the host
+ */
+static void
+hv_vss_init_req(hv_vss_req_internal *reqp,
+    uint32_t recvlen, uint64_t requestid, uint8_t *vss_buf, hv_vss_sc *sc)
+{
+	struct timespec vm_ts;
+	struct hv_vss_msg* msg = (struct hv_vss_msg *)vss_buf;
+
+	memset(reqp, 0, __offsetof(hv_vss_req_internal, callout));
+	reqp->host_msg_len = recvlen;
+	reqp->host_msg_id = requestid;
+	reqp->rcv_buf = vss_buf;
+	reqp->sc = sc;
+	memcpy(&reqp->vss_req.msg,
+	    (struct hv_vss_msg *)vss_buf, sizeof(struct hv_vss_msg));
+	/* set the opt for users */
+	switch (msg->hdr.vss_hdr.operation) {
+	case VSS_OP_FREEZE:
+		reqp->vss_req.opt_msg.opt = HV_VSS_FREEZE;
+		break;
+	case VSS_OP_THAW:
+		reqp->vss_req.opt_msg.opt = HV_VSS_THAW;
+		break;
+	case VSS_OP_HOT_BACKUP:
+		reqp->vss_req.opt_msg.opt = HV_VSS_CHECK;
+		break;
+	}
+	/* Use a timestamp as msg request ID */
+	nanotime(&vm_ts);
+	reqp->vss_req.opt_msg.msgid = (vm_ts.tv_sec * NANOSEC) + vm_ts.tv_nsec;
+}
+
+static hv_vss_req_internal*
+hv_vss_get_new_req_locked(hv_vss_sc *sc)
+{
+	hv_vss_req_internal *reqp;
+	if (!STAILQ_EMPTY(&sc->daemon_sc.to_notify_queue) ||
+	    !STAILQ_EMPTY(&sc->daemon_sc.to_ack_queue) ||
+	    !STAILQ_EMPTY(&sc->app_sc.to_notify_queue) ||
+	    !STAILQ_EMPTY(&sc->app_sc.to_ack_queue)) {
+		/*
+		 * There is request coming from host before
+		 * finishing previous requests
+		 */
+		hv_vss_log_info("%s: Warning: there is new request "
+		    "coming before finishing previous requests\n", __func__);
+		return (NULL);
+	}
+	if (LIST_EMPTY(&sc->req_free_list)) {
+		/* TODO Error: no buffer */
+		hv_vss_log_info("Error: No buffer\n");
+		return (NULL);
+	}
+	reqp = LIST_FIRST(&sc->req_free_list);
+	LIST_REMOVE(reqp, link);
+	return (reqp);
+}
+
+static void
+hv_vss_start_notify(hv_vss_req_internal *reqp, uint32_t opt)
+{
+	hv_vss_sc *sc = reqp->sc;
+	/*
+	 * Freeze/Check notification sequence: kernel -> app -> daemon(fs)
+	 * Thaw notification sequence:         kernel -> daemon(fs) -> app
+	 *
+	 * We should wake up the daemon, in case it's doing poll().
+	 * The response should be received after 5s, otherwise, trigger timeout.
+	 */
+	switch (opt) {
+	case VSS_OP_FREEZE:
+	case VSS_OP_HOT_BACKUP:
+		if (sc->app_register_done)
+			hv_vss_notify(&sc->app_sc, reqp);
+		else
+			hv_vss_notify(&sc->daemon_sc, reqp);
+		callout_reset(&reqp->callout, TIMEOUT_LIMIT * hz,
+		    hv_vss_timeout, reqp);
+		break;
+	case VSS_OP_THAW:
+		hv_vss_notify(&sc->daemon_sc, reqp);
+		callout_reset(&reqp->callout, TIMEOUT_LIMIT * hz,
+		    hv_vss_timeout, reqp);
+		break;
+	}
+}
+
+/*
+ * Function to read the vss request buffer from host
+ * and interact with daemon
+ */
+static void
+hv_vss_process_request(void *context, int pending __unused)
+{
+	uint8_t *vss_buf;
+	struct vmbus_channel *channel;
+	uint32_t recvlen = 0;
+	uint64_t requestid;
+	struct vmbus_icmsg_hdr *icmsghdrp;
+	int ret = 0;
+	hv_vss_sc *sc;
+	hv_vss_req_internal *reqp;
+
+	hv_vss_log_info("%s: entering hv_vss_process_request\n", __func__);
+
+	sc = (hv_vss_sc*)context;
+	vss_buf = sc->util_sc.receive_buffer;
+	channel = vmbus_get_channel(sc->dev);
+
+	recvlen = sc->util_sc.ic_buflen;
+	ret = vmbus_chan_recv(channel, vss_buf, &recvlen, &requestid);
+	KASSERT(ret != ENOBUFS, ("hvvss recvbuf is not large enough"));
+	/* XXX check recvlen to make sure that it contains enough data */
+
+	while ((ret == 0) && (recvlen > 0)) {
+		icmsghdrp = (struct vmbus_icmsg_hdr *)vss_buf;
+
+		if (icmsghdrp->ic_type == HV_ICMSGTYPE_NEGOTIATE) {
+			ret = vmbus_ic_negomsg(&sc->util_sc, vss_buf,
+			    &recvlen, VSS_FWVER, VSS_MSGVER);
+			hv_vss_respond_host(vss_buf, vmbus_get_channel(sc->dev),
+			    recvlen, requestid, ret);
+			hv_vss_log_info("%s: version negotiated\n", __func__);
+		} else if (!hv_vss_is_daemon_killed_after_launch(sc)) {
+			struct hv_vss_msg* msg = (struct hv_vss_msg *)vss_buf;
+			switch(msg->hdr.vss_hdr.operation) {
+			case VSS_OP_FREEZE:
+			case VSS_OP_THAW:
+			case VSS_OP_HOT_BACKUP:
+				mtx_lock(&sc->pending_mutex);
+				reqp = hv_vss_get_new_req_locked(sc);
+				mtx_unlock(&sc->pending_mutex);
+				if (reqp == NULL) {
+					/* ignore this request from host */
+					break;
+				}
+				hv_vss_init_req(reqp, recvlen, requestid, vss_buf, sc);
+				hv_vss_log_info("%s: receive %s (%ju) from host\n",
+				    __func__,
+				    vss_opt_name[reqp->vss_req.opt_msg.opt],
+				    (uintmax_t)reqp->vss_req.opt_msg.msgid);
+				hv_vss_start_notify(reqp, msg->hdr.vss_hdr.operation);
+				break;
+			case VSS_OP_GET_DM_INFO:
+				hv_vss_log_info("%s: receive GET_DM_INFO from host\n",
+				    __func__);
+				msg->body.dm_info.flags = 0;
+				hv_vss_respond_host(vss_buf, vmbus_get_channel(sc->dev),
+				    recvlen, requestid, HV_S_OK);
+				break;
+			default:
+				device_printf(sc->dev, "Unknown opt from host: %d\n",
+				    msg->hdr.vss_hdr.operation);
+				break;
+			}
+		} else {
+			/* daemon was killed for some reason after it was launched */
+			struct hv_vss_msg* msg = (struct hv_vss_msg *)vss_buf;
+			switch(msg->hdr.vss_hdr.operation) {
+			case VSS_OP_FREEZE:
+				hv_vss_log_info("%s: response fail for FREEZE\n",
+				    __func__);
+				break;
+			case VSS_OP_THAW:
+				hv_vss_log_info("%s: response fail for THAW\n",
+				    __func__);
+				break;
+			case VSS_OP_HOT_BACKUP:
+				hv_vss_log_info("%s: response fail for HOT_BACKUP\n",
+				    __func__);
+				msg->body.vss_cf.flags = VSS_HBU_NO_AUTO_RECOVERY;
+				break;
+			case VSS_OP_GET_DM_INFO:
+				hv_vss_log_info("%s: response fail for GET_DM_INFO\n",
+				    __func__);
+				msg->body.dm_info.flags = 0;
+				break;
+			default:
+				device_printf(sc->dev, "Unknown opt from host: %d\n",
+				    msg->hdr.vss_hdr.operation);
+				break;
+			}
+			hv_vss_respond_host(vss_buf, vmbus_get_channel(sc->dev),
+			    recvlen, requestid, HV_E_FAIL);
+		}
+		/*
+		 * Try reading next buffer
+		 */
+		recvlen = sc->util_sc.ic_buflen;
+		ret = vmbus_chan_recv(channel, vss_buf, &recvlen, &requestid);
+		KASSERT(ret != ENOBUFS, ("hvvss recvbuf is not large enough"));
+		/* XXX check recvlen to make sure that it contains enough data */
+
+		hv_vss_log_info("%s: read: context %p, ret =%d, recvlen=%d\n",
+		    __func__, context, ret, recvlen);
+	}
+}
+
+static int
+hv_vss_probe(device_t dev)
+{
+	return (vmbus_ic_probe(dev, vmbus_vss_descs));
+}
+
+static int
+hv_vss_init_send_receive_queue(device_t dev)
+{
+	hv_vss_sc *sc = (hv_vss_sc*)device_get_softc(dev);
+	int i;
+	const int max_list = 4; /* It is big enough for the list */
+	struct hv_vss_req_internal* reqp;
+
+	LIST_INIT(&sc->req_free_list);
+	STAILQ_INIT(&sc->daemon_sc.to_notify_queue);
+	STAILQ_INIT(&sc->daemon_sc.to_ack_queue);
+	STAILQ_INIT(&sc->app_sc.to_notify_queue);
+	STAILQ_INIT(&sc->app_sc.to_ack_queue);
+
+	for (i = 0; i < max_list; i++) {
+		reqp = malloc(sizeof(struct hv_vss_req_internal),
+		    M_DEVBUF, M_WAITOK|M_ZERO);
+		LIST_INSERT_HEAD(&sc->req_free_list, reqp, link);
+		callout_init_mtx(&reqp->callout, &sc->pending_mutex, 0);
+	}
+	return (0);
+}
+
+static int
+hv_vss_destroy_send_receive_queue(device_t dev)
+{
+	hv_vss_sc *sc = (hv_vss_sc*)device_get_softc(dev);
+	hv_vss_req_internal* reqp;
+
+	while (!LIST_EMPTY(&sc->req_free_list)) {
+		reqp = LIST_FIRST(&sc->req_free_list);
+		LIST_REMOVE(reqp, link);
+		free(reqp, M_DEVBUF);
+	}
+
+	while (!STAILQ_EMPTY(&sc->daemon_sc.to_notify_queue)) {
+		reqp = STAILQ_FIRST(&sc->daemon_sc.to_notify_queue);
+		STAILQ_REMOVE_HEAD(&sc->daemon_sc.to_notify_queue, slink);
+		free(reqp, M_DEVBUF);
+	}
+
+	while (!STAILQ_EMPTY(&sc->daemon_sc.to_ack_queue)) {
+		reqp = STAILQ_FIRST(&sc->daemon_sc.to_ack_queue);
+		STAILQ_REMOVE_HEAD(&sc->daemon_sc.to_ack_queue, slink);
+		free(reqp, M_DEVBUF);
+	}
+
+	while (!STAILQ_EMPTY(&sc->app_sc.to_notify_queue)) {
+		reqp = STAILQ_FIRST(&sc->app_sc.to_notify_queue);
+		STAILQ_REMOVE_HEAD(&sc->app_sc.to_notify_queue, slink);
+		free(reqp, M_DEVBUF);
+	}
+
+	while (!STAILQ_EMPTY(&sc->app_sc.to_ack_queue)) {
+		reqp = STAILQ_FIRST(&sc->app_sc.to_ack_queue);
+		STAILQ_REMOVE_HEAD(&sc->app_sc.to_ack_queue, slink);
+		free(reqp, M_DEVBUF);
+	}
+	return (0);
+}
+
+static int
+hv_vss_attach(device_t dev)
+{
+	int error;
+	struct sysctl_oid_list *child;
+	struct sysctl_ctx_list *ctx;
+
+	hv_vss_sc *sc = (hv_vss_sc*)device_get_softc(dev);
+
+	sc->dev = dev;
+	mtx_init(&sc->pending_mutex, "hv_vss pending mutex", NULL, MTX_DEF);
+
+	ctx = device_get_sysctl_ctx(dev);
+	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
+
+	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "hv_vss_log",
+	    CTLFLAG_RWTUN, &hv_vss_log, 0, "Hyperv VSS service log level");
+
+	TASK_INIT(&sc->task, 0, hv_vss_process_request, sc);
+	hv_vss_init_send_receive_queue(dev);
+	/* create character device for file system freeze/thaw */
+	error = make_dev_p(MAKEDEV_CHECKNAME | MAKEDEV_WAITOK,
+		    &sc->hv_vss_dev,
+		    &hv_vss_cdevsw,
+		    0,
+		    UID_ROOT,
+		    GID_WHEEL,
+		    0640,
+		    FS_VSS_DEV_NAME);
+
+	if (error != 0) {
+		hv_vss_log_info("Fail to create '%s': %d\n", FS_VSS_DEV_NAME, error);
+		return (error);
+	}
+	sc->hv_vss_dev->si_drv1 = &sc->daemon_sc;
+	sc->daemon_sc.sc = sc;
+	/* create character device for application freeze/thaw */
+	error = make_dev_p(MAKEDEV_CHECKNAME | MAKEDEV_WAITOK,
+		    &sc->hv_appvss_dev,
+		    &hv_appvss_cdevsw,
+		    0,
+		    UID_ROOT,
+		    GID_WHEEL,
+		    0640,
+		    APP_VSS_DEV_NAME);
+
+	if (error != 0) {
+		hv_vss_log_info("Fail to create '%s': %d\n", APP_VSS_DEV_NAME, error);
+		return (error);
+	}
+	sc->hv_appvss_dev->si_drv1 = &sc->app_sc;
+	sc->app_sc.sc = sc;
+
+	return hv_util_attach(dev, hv_vss_callback);
+}
+
+static int
+hv_vss_detach(device_t dev)
+{
+	hv_vss_sc *sc = (hv_vss_sc*)device_get_softc(dev);
+	mtx_destroy(&sc->pending_mutex);
+	if (sc->daemon_sc.proc_task != NULL) {
+		PROC_LOCK(sc->daemon_sc.proc_task);
+		kern_psignal(sc->daemon_sc.proc_task, SIGKILL);
+		PROC_UNLOCK(sc->daemon_sc.proc_task);
+	}
+	if (sc->app_sc.proc_task != NULL) {
+		PROC_LOCK(sc->app_sc.proc_task);
+		kern_psignal(sc->app_sc.proc_task, SIGKILL);
+		PROC_UNLOCK(sc->app_sc.proc_task);
+	}
+	hv_vss_destroy_send_receive_queue(dev);
+	destroy_dev(sc->hv_vss_dev);
+	destroy_dev(sc->hv_appvss_dev);
+	return hv_util_detach(dev);
+}
+
+static device_method_t vss_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe, hv_vss_probe),
+	DEVMETHOD(device_attach, hv_vss_attach),
+	DEVMETHOD(device_detach, hv_vss_detach),
+	{ 0, 0 }
+};
+
+static driver_t vss_driver = { "hvvss", vss_methods, sizeof(hv_vss_sc)};
+
+static devclass_t vss_devclass;
+
+DRIVER_MODULE(hv_vss, vmbus, vss_driver, vss_devclass, NULL, NULL);
+MODULE_VERSION(hv_vss, 1);
+MODULE_DEPEND(hv_vss, vmbus, 1, 1, 1);
Index: head/sys/modules/hyperv/utilities/Makefile
===================================================================
--- head/sys/modules/hyperv/utilities/Makefile
+++ head/sys/modules/hyperv/utilities/Makefile
@@ -3,7 +3,7 @@
 .PATH:	${.CURDIR}/../../../dev/hyperv/utilities
 
 KMOD=	hv_utils
-SRCS=	hv_util.c hv_kvp.c hv_timesync.c hv_shutdown.c hv_heartbeat.c
+SRCS=	hv_util.c hv_kvp.c hv_snapshot.c hv_timesync.c hv_shutdown.c hv_heartbeat.c hv_snapshot.c
 SRCS+=	bus_if.h device_if.h vmbus_if.h
 
 CFLAGS+= -I${.CURDIR}/../../../dev/hyperv/include \
Index: head/usr.sbin/hyperv/Makefile
===================================================================
--- head/usr.sbin/hyperv/Makefile
+++ head/usr.sbin/hyperv/Makefile
@@ -2,6 +2,8 @@
 
 .include <bsd.own.mk>
 
-SUBDIR = tools 
+SUBDIR= \
+	tools/kvp \
+	tools/vss
 
 .include <bsd.subdir.mk>
Index: head/usr.sbin/hyperv/tools/Makefile
===================================================================
--- head/usr.sbin/hyperv/tools/Makefile
+++ head/usr.sbin/hyperv/tools/Makefile
@@ -1,13 +0,0 @@
-# $FreeBSD$
-
-.include <bsd.own.mk>
-
-HV_KVP_DAEMON_DISTDIR?= ${.CURDIR}/../../../contrib/hyperv/tools
-.PATH: ${HV_KVP_DAEMON_DISTDIR} 
-
-PROG= 	hv_kvp_daemon
-MAN= 	hv_kvp_daemon.8	
-
-CFLAGS+=	-I${.CURDIR}/../../../sys/dev/hyperv/utilities
-
-.include <bsd.prog.mk>
Index: head/usr.sbin/hyperv/tools/Makefile.depend
===================================================================
--- head/usr.sbin/hyperv/tools/Makefile.depend
+++ head/usr.sbin/hyperv/tools/Makefile.depend
@@ -1,19 +0,0 @@
-# $FreeBSD$
-# Autogenerated - do NOT edit!
-
-DIRDEPS = \
-	gnu/lib/csu \
-	gnu/lib/libgcc \
-	include \
-	include/arpa \
-	include/xlocale \
-	lib/${CSU_DIR} \
-	lib/libc \
-	lib/libcompiler_rt \
-
-
-.include <dirdeps.mk>
-
-.if ${DEP_RELDIR} == ${_DEP_RELDIR}
-# local dependencies - needed for -jN in clean tree
-.endif
Index: head/usr.sbin/hyperv/tools/kvp/Makefile
===================================================================
--- head/usr.sbin/hyperv/tools/kvp/Makefile
+++ head/usr.sbin/hyperv/tools/kvp/Makefile
@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+.include <bsd.own.mk>
+
+HV_KVP_DAEMON_DISTDIR?= ${.CURDIR}/../../../../contrib/hyperv/tools
+.PATH: ${HV_KVP_DAEMON_DISTDIR} 
+
+PROG= 	hv_kvp_daemon
+MAN= 	hv_kvp_daemon.8	
+
+CFLAGS+=	-I${.CURDIR}/../../../../sys/dev/hyperv/utilities
+
+.include <bsd.prog.mk>
Index: head/usr.sbin/hyperv/tools/kvp/Makefile.depend
===================================================================
--- head/usr.sbin/hyperv/tools/kvp/Makefile.depend
+++ head/usr.sbin/hyperv/tools/kvp/Makefile.depend
@@ -0,0 +1,19 @@
+# $FreeBSD$
+# Autogenerated - do NOT edit!
+
+DIRDEPS = \
+	gnu/lib/csu \
+	gnu/lib/libgcc \
+	include \
+	include/arpa \
+	include/xlocale \
+	lib/${CSU_DIR} \
+	lib/libc \
+	lib/libcompiler_rt \
+
+
+.include <dirdeps.mk>
+
+.if ${DEP_RELDIR} == ${_DEP_RELDIR}
+# local dependencies - needed for -jN in clean tree
+.endif
Index: head/usr.sbin/hyperv/tools/vss/Makefile
===================================================================
--- head/usr.sbin/hyperv/tools/vss/Makefile
+++ head/usr.sbin/hyperv/tools/vss/Makefile
@@ -0,0 +1,14 @@
+# $FreeBSD$
+DIRDEPS = lib/libc
+
+.include <bsd.own.mk>
+
+HV_VSS_DAEMON_DISTDIR?= ${.CURDIR}/../../../../contrib/hyperv/tools
+.PATH: ${HV_VSS_DAEMON_DISTDIR} 
+
+PROG= 	hv_vss_daemon
+MAN= 	hv_vss_daemon.8	
+
+CFLAGS+=	-I${.CURDIR}/../../../../sys/dev/hyperv/utilities
+
+.include <bsd.prog.mk>
Index: head/usr.sbin/hyperv/tools/vss/Makefile.depend
===================================================================
--- head/usr.sbin/hyperv/tools/vss/Makefile.depend
+++ head/usr.sbin/hyperv/tools/vss/Makefile.depend
@@ -0,0 +1,18 @@
+# $FreeBSD$
+# Autogenerated - do NOT edit!
+
+DIRDEPS = \
+	gnu/lib/csu \
+	gnu/lib/libgcc \
+	include \
+	include/xlocale \
+	lib/${CSU_DIR} \
+	lib/libc \
+	lib/libcompiler_rt \
+
+
+.include <dirdeps.mk>
+
+.if ${DEP_RELDIR} == ${_DEP_RELDIR}
+# local dependencies - needed for -jN in clean tree
+.endif