diff --git a/usr.sbin/bhyve/migration.h b/usr.sbin/bhyve/migration.h --- a/usr.sbin/bhyve/migration.h +++ b/usr.sbin/bhyve/migration.h @@ -27,6 +27,8 @@ #define MIGRATION_SPECS_OK 0 #define MIGRATION_SPECS_NOT_OK 1 +#define NO_KERN_STRUCT -1 + enum migration_transfer_req { MIGRATION_SEND_REQ = 0, MIGRATION_RECV_REQ = 1 diff --git a/usr.sbin/bhyve/migration.c b/usr.sbin/bhyve/migration.c --- a/usr.sbin/bhyve/migration.c +++ b/usr.sbin/bhyve/migration.c @@ -38,6 +38,23 @@ #define MB (1024UL * 1024) #define GB (1024UL * MB) +#define ALLOCA_VM_SNAPSHOT_META(DEV_NAME, DEV_REQ, BUFFER, BUFFER_SIZE, OP) \ +({ \ + &(struct vm_snapshot_meta) { \ + .dev_name = DEV_NAME, \ + .dev_req = DEV_REQ, \ + \ + .buffer.buf_start = BUFFER, \ + .buffer.buf_size = BUFFER_SIZE, \ + \ + .buffer.buf = BUFFER, \ + .buffer.buf_rem = BUFFER_SIZE, \ + \ + .op = OP, \ + }; \ + \ +}) + #ifdef BHYVE_DEBUG #define DPRINTF(FMT, ...) \ ({ \ @@ -465,6 +482,112 @@ return (0); } +/** + * The source host saves the state for the kernel structure that will be + * migrated and sends to the destination host a message that contains + * the type of data to be sent (MESSAGE_TYPE_KERN), the size of the structure + * to be received and the index that represents the kernel structure in order to + * be identified by the destination host. Then, the source host transfer the + * state of the kernel structure over the network and the destination host + * restores it. + */ +static inline int +migrate_kern_struct(struct vmctx *ctx, int socket, char *buffer, size_t buf_size, + enum snapshot_req struct_req, enum migration_transfer_req req) +{ + int rc; + struct migration_message_type msg; + struct vm_snapshot_meta *meta; + + if ((req != MIGRATION_SEND_REQ) && (req != MIGRATION_RECV_REQ)) { + EPRINTF("Unknown request"); + return (EINVAL); + } + + memset(&msg, 0, sizeof(msg)); + if (req == MIGRATION_SEND_REQ) { + msg.type = MESSAGE_TYPE_KERN; + + memset(buffer, 0, buf_size); + meta = ALLOCA_VM_SNAPSHOT_META(NULL, struct_req, buffer, buf_size, VM_SNAPSHOT_SAVE); + + rc = vm_snapshot_req(ctx, meta); + if (rc != 0) { + EPRINTF("Could not get struct with req %d", struct_req); + return (rc); + } + + msg.len = vm_get_snapshot_size(meta); + msg.req_type = struct_req; + + } + + rc = migration_transfer_data(socket, &msg, sizeof(msg), req); + if (rc != 0) { + EPRINTF("Could not transfer message type for kern struct %d", struct_req); + return (rc); + } + + if ((req == MIGRATION_RECV_REQ) && (msg.type != MESSAGE_TYPE_KERN)) { + EPRINTF("Receive wrong message type."); + return (EINVAL); + } + + rc = migration_transfer_data(socket, buffer, msg.len, req); + if (rc != 0) { + EPRINTF("Could not transfer struct with req %d", struct_req); + return (rc); + } + + if (req == MIGRATION_RECV_REQ) { + meta = ALLOCA_VM_SNAPSHOT_META(NULL, msg.req_type, buffer, + msg.len, VM_SNAPSHOT_RESTORE); + rc = vm_snapshot_req(ctx, meta); + if (rc != 0) { + EPRINTF("Failed to restore struct %d", msg.req_type); + return (rc); + } + } + + return (0); +} + +static int +migrate_kern_data(struct vmctx *ctx, int socket, enum migration_transfer_req req) +{ + int i, rc, error; + int ndevs; + char *buffer; + enum snapshot_req struct_req = NO_KERN_STRUCT; + const struct vm_snapshot_kern_info *snapshot_kern_structs; + + error = 0; + snapshot_kern_structs = get_snapshot_kern_structs(&ndevs); + + buffer = malloc(SNAPSHOT_BUFFER_SIZE); + if (buffer == NULL) { + EPRINTF("Could not allocate memory."); + return (ENOMEM); + } + + for (i = 0; i < ndevs; i++) { + if (req == MIGRATION_SEND_REQ) + struct_req = snapshot_kern_structs[i].req; + + rc = migrate_kern_struct(ctx, socket, buffer, SNAPSHOT_BUFFER_SIZE, + struct_req, req); + if (rc != 0) { + EPRINTF("Could not transfer struct %s", snapshot_kern_structs[i].struct_name); + error = rc; + break; + } + } + + free(buffer); + + return (error); +} + static inline int migrate_connections(struct migrate_req req, int *socket_fd, enum migration_transfer_req type) @@ -621,6 +744,13 @@ goto unlock_vm_and_exit; } + rc = migrate_kern_data(ctx, s, MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send kern data to destination"); + error = rc; + goto unlock_vm_and_exit; + } + rc = migration_transfer_data(s, &migration_completed, sizeof(migration_completed), MIGRATION_RECV_REQ); if ((rc != 0) || (migration_completed != MIGRATION_SPECS_OK)) { @@ -692,6 +822,12 @@ } } + rc = migrate_kern_data(ctx, s, MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not recv kern data"); + goto done; + } + // fprintf(stdout, "%s: Migration completed\n", __func__); migration_completed = MIGRATION_SPECS_OK; diff --git a/usr.sbin/bhyve/snapshot.h b/usr.sbin/bhyve/snapshot.h --- a/usr.sbin/bhyve/snapshot.h +++ b/usr.sbin/bhyve/snapshot.h @@ -45,6 +45,8 @@ #define BHYVE_RUN_DIR "/var/run/bhyve/" #define MAX_SNAPSHOT_FILENAME PATH_MAX +#define SNAPSHOT_BUFFER_SIZE (20 << 20) + struct vmctx; struct restore_state { @@ -81,6 +83,8 @@ enum snapshot_req req; /* request type */ }; +const struct vm_snapshot_kern_info *get_snapshot_kern_structs(int *ndevs); + void destroy_restore_state(struct restore_state *rstate); const char *lookup_vmname(struct restore_state *rstate); diff --git a/usr.sbin/bhyve/snapshot.c b/usr.sbin/bhyve/snapshot.c --- a/usr.sbin/bhyve/snapshot.c +++ b/usr.sbin/bhyve/snapshot.c @@ -118,8 +118,6 @@ #define SNAPSHOT_CHUNK (4 * MB) #define PROG_BUF_SZ (8192) -#define SNAPSHOT_BUFFER_SIZE (20 * MB) - #define JSON_STRUCT_ARR_KEY "structs" #define JSON_DEV_ARR_KEY "devices" #define JSON_BASIC_METADATA_KEY "basic metadata" @@ -165,6 +163,15 @@ { "vrtc", STRUCT_VRTC }, }; +const struct vm_snapshot_kern_info * +get_snapshot_kern_structs(int *ndevs) +{ + if (ndevs != NULL) + *ndevs = nitems(snapshot_kern_structs); + + return (snapshot_kern_structs); +} + static cpuset_t vcpus_active, vcpus_suspended; static pthread_mutex_t vcpu_lock; static pthread_cond_t vcpus_idle, vcpus_can_run;