Index: lib/libvmmapi/vmmapi.h =================================================================== --- lib/libvmmapi/vmmapi.h +++ lib/libvmmapi/vmmapi.h @@ -43,6 +43,14 @@ */ #define VMMAPI_VERSION 0103 /* 2 digit major followed by 2 digit minor */ +/* + * Note, a bhyve instance uses its VM name in the path for the unix domain socket. + * The socket path must be less than SUNPATHLEN (104). + * The default prefix for BHYVE_RUN_DIR is '/var/run/bhyve/' (15). + * So, 104-15 = 89. This allows for a VM name of 88 characters (+1 for terminator). + */ +#define MAX_VMNAME 89 + struct iovec; struct vmctx; struct vm_snapshot_meta; Index: usr.sbin/bhyve/Makefile =================================================================== --- usr.sbin/bhyve/Makefile +++ usr.sbin/bhyve/Makefile @@ -77,7 +77,7 @@ iov.c .if ${MK_BHYVE_SNAPSHOT} != "no" -SRCS+= snapshot.c +SRCS+= ipc.c snapshot.c .endif CFLAGS.kernemu_dev.c+= -I${SRCTOP}/sys/amd64 Index: usr.sbin/bhyve/bhyverun.c =================================================================== --- usr.sbin/bhyve/bhyverun.c +++ usr.sbin/bhyve/bhyverun.c @@ -102,6 +102,7 @@ #include "pci_lpc.h" #include "smbiostbl.h" #ifdef BHYVE_SNAPSHOT +#include "ipc.h" #include "snapshot.h" #endif #include "xmsr.h" @@ -1543,11 +1544,11 @@ /* initialize mutex/cond variables */ init_snapshot(); - /* - * checkpointing thread for communication with bhyvectl - */ - if (init_checkpoint_thread(ctx) < 0) - printf("Failed to start checkpoint thread!\r\n"); + /* start ipc thread */ + error = init_ipc(ctx); + if (error != 0) + fprintf(stderr, "Unable to start ipc thread for \"%s\": %s\n", + vmname, strerror(error)); if (restore_file != NULL) vm_restore_time(ctx); Index: usr.sbin/bhyve/ipc.h =================================================================== --- /dev/null +++ usr.sbin/bhyve/ipc.h @@ -0,0 +1,63 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Robert Wing + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#ifndef _IPC_H_ +#define _IPC_H_ + +#define BHYVE_RUN_DIR "/var/run/bhyve/" + +#define MAX_SNAPSHOT_FILENAME PATH_MAX + +int init_ipc(struct vmctx *); + +/* Filename that will be used for save/restore */ +struct checkpoint_op { + char snapshot_filename[MAX_SNAPSHOT_FILENAME]; +}; + +/* Messages that a bhyve process understands. */ +enum ipc_opcode { + START_CHECKPOINT = 1, + START_SUSPEND, +}; + +/* + * The type of message and associated data to + * send to a bhyve process. + */ +struct ipc_message { + enum ipc_opcode code; + union { + /* + * message specific structures + */ + struct checkpoint_op checkpoint; + }; +}; + +#endif /* _IPC_H_ */ Index: usr.sbin/bhyve/ipc.c =================================================================== --- /dev/null +++ usr.sbin/bhyve/ipc.c @@ -0,0 +1,147 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2016 Flavius Anton + * Copyright (c) 2016 Mihai Tiganus + * Copyright (c) 2016-2019 Mihai Carabas + * Copyright (c) 2017-2019 Darius Mihai + * Copyright (c) 2017-2019 Elena Mihailescu + * Copyright (c) 2018-2019 Sergiu Weisz + * All rights reserved. + * The bhyve-snapshot feature was developed under sponsorships + * from Matthew Grooms. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "debug.h" +#include "ipc.h" +#include "snapshot.h" + +static int +handle_message(struct ipc_message *imsg, struct vmctx *ctx) +{ + int err; + + switch (imsg->code) { + case START_CHECKPOINT: + err = vm_checkpoint(ctx, imsg->checkpoint.snapshot_filename, false); + break; + case START_SUSPEND: + err = vm_checkpoint(ctx, imsg->checkpoint.snapshot_filename, true); + break; + default: + EPRINTLN("Unrecognized message operation"); + err = -1; + } + + if (err != 0) + EPRINTLN("Unable to perform requested operation"); + + return (err); +} + +static void * +ipc_thread(void *arg) +{ + struct vmctx *ctx; + struct ipc_message imsg; + struct sockaddr_un addr; + ssize_t n; + int socket_fd; + char vmname[MAX_VMNAME]; + + pthread_set_name_np(pthread_self(), "ipc thread"); + ctx = (struct vmctx *)arg; + + if (vm_get_name(ctx, vmname, MAX_VMNAME - 1) != 0) { + EPRINTLN("Failed to get VM name"); + return (NULL); + } + + socket_fd = socket(PF_UNIX, SOCK_DGRAM, 0); + if (socket_fd < 0) { + EPRINTLN("Socket creation failed: %s", strerror(errno)); + return (NULL); + } + + /* + * Note, BHYVE_RUN_DIR + vmname should be < SUNPATHLEN (104) + * which allows for a vmname of 88 characters. + */ + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s", + BHYVE_RUN_DIR, vmname); + addr.sun_len = SUN_LEN(&addr); + unlink(addr.sun_path); + + if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) { + EPRINTLN("Failed to bind socket \"%s\": %s", + addr.sun_path, strerror(errno)); + close(socket_fd); + return (NULL); + } + + for (;;) { + n = recvfrom(socket_fd, &imsg, sizeof(imsg), 0, NULL, 0); + + /* + * slight sanity check: see if there's enough data to at + * least determine the type of message. + */ + if (n >= sizeof(imsg.code)) + handle_message(&imsg, ctx); + else + EPRINTLN("Failed to receive message: %s", + n == -1 ? strerror(errno) : "unknown error"); + } + + /* NOTREACHED */ + return (NULL); +} + +/* + * Start a thread to handle IPC messages. + */ +int +init_ipc(struct vmctx *ctx) +{ + pthread_t ipc_pthread; + + return (pthread_create(&ipc_pthread, NULL, ipc_thread, ctx)); +} Index: usr.sbin/bhyve/snapshot.h =================================================================== --- usr.sbin/bhyve/snapshot.h +++ usr.sbin/bhyve/snapshot.h @@ -42,9 +42,6 @@ #include #include -#define BHYVE_RUN_DIR "/var/run/bhyve/" -#define MAX_SNAPSHOT_FILENAME PATH_MAX - struct vmctx; struct restore_state { @@ -60,36 +57,6 @@ ucl_object_t *meta_root_obj; }; -/* Filename that will be used for save/restore */ -struct checkpoint_op { - char snapshot_filename[MAX_SNAPSHOT_FILENAME]; -}; - -/* Messages that a bhyve process understands. */ -enum ipc_opcode { - START_CHECKPOINT, - START_SUSPEND, -}; - -/* - * The type of message and associated data to - * send to a bhyve process. - */ -struct ipc_message { - enum ipc_opcode code; - union { - /* - * message specific structures - */ - struct checkpoint_op op; - } data; -}; - -struct checkpoint_thread_info { - struct vmctx *ctx; - int socket_fd; -}; - typedef int (*vm_snapshot_dev_cb)(struct vm_snapshot_meta *); typedef int (*vm_pause_dev_cb) (struct vmctx *, const char *); typedef int (*vm_resume_dev_cb) (struct vmctx *, const char *); @@ -124,9 +91,7 @@ int vm_pause_user_devs(struct vmctx *ctx); int vm_resume_user_devs(struct vmctx *ctx); -int get_checkpoint_msg(int conn_fd, struct vmctx *ctx); -void *checkpoint_thread(void *param); -int init_checkpoint_thread(struct vmctx *ctx); +int vm_checkpoint(struct vmctx *ctx, char *checkpoint_file, bool stop_vm); void init_snapshot(void); int load_restore_file(const char *filename, struct restore_state *rstate); Index: usr.sbin/bhyve/snapshot.c =================================================================== --- usr.sbin/bhyve/snapshot.c +++ usr.sbin/bhyve/snapshot.c @@ -116,8 +116,6 @@ #define SNAPSHOT_CHUNK (4 * MB) #define PROG_BUF_SZ (8192) -#define MAX_VMNAME 100 - #define SNAPSHOT_BUFFER_SIZE (20 * MB) #define JSON_STRUCT_ARR_KEY "structs" @@ -1324,7 +1322,7 @@ pthread_cond_broadcast(&vcpus_can_run); } -static int +int vm_checkpoint(struct vmctx *ctx, char *checkpoint_file, bool stop_vm) { int fd_checkpoint = 0, kdata_fd = 0; @@ -1440,59 +1438,6 @@ return (error); } -int -handle_message(struct ipc_message *imsg, struct vmctx *ctx) -{ - int err; - - switch (imsg->code) { - case START_CHECKPOINT: - err = vm_checkpoint(ctx, imsg->data.op.snapshot_filename, false); - break; - case START_SUSPEND: - err = vm_checkpoint(ctx, imsg->data.op.snapshot_filename, true); - break; - default: - EPRINTLN("Unrecognized checkpoint operation\n"); - err = -1; - } - - if (err != 0) - EPRINTLN("Unable to perform the requested operation\n"); - - return (err); -} - -/* - * Listen for commands from bhyvectl - */ -void * -checkpoint_thread(void *param) -{ - struct ipc_message imsg; - struct checkpoint_thread_info *thread_info; - ssize_t n; - - pthread_set_name_np(pthread_self(), "checkpoint thread"); - thread_info = (struct checkpoint_thread_info *)param; - - for (;;) { - n = recvfrom(thread_info->socket_fd, &imsg, sizeof(imsg), 0, NULL, 0); - - /* - * slight sanity check: see if there's enough data to at - * least determine the type of message. - */ - if (n >= sizeof(imsg.code)) - handle_message(&imsg, thread_info->ctx); - else - EPRINTLN("Failed to receive message: %s\n", - n == -1 ? strerror(errno) : "unknown error"); - } - - return (NULL); -} - void init_snapshot(void) { @@ -1509,69 +1454,6 @@ errc(1, err, "checkpoint cv init (vcpus_can_run)"); } -/* - * Create the listening socket for IPC with bhyvectl - */ -int -init_checkpoint_thread(struct vmctx *ctx) -{ - struct checkpoint_thread_info *checkpoint_info = NULL; - struct sockaddr_un addr; - int socket_fd; - pthread_t checkpoint_pthread; - char vmname_buf[MAX_VMNAME]; - int ret, err = 0; - - memset(&addr, 0, sizeof(addr)); - - socket_fd = socket(PF_UNIX, SOCK_DGRAM, 0); - if (socket_fd < 0) { - EPRINTLN("Socket creation failed: %s", strerror(errno)); - err = -1; - goto fail; - } - - addr.sun_family = AF_UNIX; - - err = vm_get_name(ctx, vmname_buf, MAX_VMNAME - 1); - if (err != 0) { - perror("Failed to get VM name"); - goto fail; - } - - snprintf(addr.sun_path, sizeof(addr.sun_path), "%s%s", - BHYVE_RUN_DIR, vmname_buf); - addr.sun_len = SUN_LEN(&addr); - unlink(addr.sun_path); - - if (bind(socket_fd, (struct sockaddr *)&addr, addr.sun_len) != 0) { - EPRINTLN("Failed to bind socket \"%s\": %s\n", - addr.sun_path, strerror(errno)); - err = -1; - goto fail; - } - - checkpoint_info = calloc(1, sizeof(*checkpoint_info)); - checkpoint_info->ctx = ctx; - checkpoint_info->socket_fd = socket_fd; - - ret = pthread_create(&checkpoint_pthread, NULL, checkpoint_thread, - checkpoint_info); - if (ret < 0) { - err = ret; - goto fail; - } - - return (0); -fail: - free(checkpoint_info); - if (socket_fd > 0) - close(socket_fd); - unlink(addr.sun_path); - - return (err); -} - void vm_snapshot_buf_err(const char *bufname, const enum vm_snapshot_op op) { Index: usr.sbin/bhyvectl/Makefile =================================================================== --- usr.sbin/bhyvectl/Makefile +++ usr.sbin/bhyvectl/Makefile @@ -18,9 +18,6 @@ .if ${MK_BHYVE_SNAPSHOT} != "no" CFLAGS+= -DBHYVE_SNAPSHOT - -# usr.sbin/bhyve/snapshot.h needs ucl header -CFLAGS+= -I${SRCTOP}/contrib/libucl/include CFLAGS+= -I${SRCTOP}/usr.sbin/bhyve .endif Index: usr.sbin/bhyvectl/bhyvectl.c =================================================================== --- usr.sbin/bhyvectl/bhyvectl.c +++ usr.sbin/bhyvectl/bhyvectl.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -64,7 +65,7 @@ #include "intel/vmcs.h" #ifdef BHYVE_SNAPSHOT -#include "snapshot.h" +#include "ipc.h" #endif #define MB (1UL << 20) @@ -74,8 +75,6 @@ #define NO_ARG no_argument #define OPT_ARG optional_argument -#define MAX_VMNAME 100 - static const char *progname; static void @@ -1730,9 +1729,9 @@ size_t length; imsg.code = code; - strlcpy(imsg.data.op.snapshot_filename, file, MAX_SNAPSHOT_FILENAME); + strlcpy(imsg.checkpoint.snapshot_filename, file, MAX_SNAPSHOT_FILENAME); - length = offsetof(struct ipc_message, data) + sizeof(imsg.data.op); + length = offsetof(struct ipc_message, checkpoint) + sizeof(imsg.checkpoint); return (send_message(ctx, (void *)&imsg, length)); }