diff --git a/usr.sbin/bhyve/migration.h b/usr.sbin/bhyve/migration.h --- a/usr.sbin/bhyve/migration.h +++ b/usr.sbin/bhyve/migration.h @@ -38,6 +38,8 @@ #include #include "snapshot.h" +struct vmctx; + int receive_vm_migration(struct vmctx *ctx, char *migration_data); /* Warm Migration */ @@ -47,6 +49,9 @@ #define MIGRATION_SPECS_OK 0 #define MIGRATION_SPECS_NOT_OK 1 + +#define NO_KERN_STRUCT -1 + enum migration_transfer_req { MIGRATION_SEND_REQ = 0, MIGRATION_RECV_REQ = 1 diff --git a/usr.sbin/bhyve/migration.c b/usr.sbin/bhyve/migration.c --- a/usr.sbin/bhyve/migration.c +++ b/usr.sbin/bhyve/migration.c @@ -66,6 +66,21 @@ #define MB (1024UL * 1024) #define GB (1024UL * MB) + +#define ALLOCA_VM_SNAPSHOT_META(CTX, DEV_NAME, DEV_REQ, BUFFER, BUFFER_SIZE, OP) \ +({ \ + &(struct vm_snapshot_meta) { \ + .ctx = CTX, \ + .dev_name = DEV_NAME, \ + .dev_req = DEV_REQ, \ + \ + .buffer.buf_start = BUFFER, \ + .buffer.buf_size = BUFFER_SIZE, \ + .op = OP, \ + }; \ + \ +}) + #ifdef BHYVE_DEBUG #define DPRINTF(FMT, ...) \ ({ \ @@ -498,6 +513,126 @@ return (0); } + +/** + * The source host saves the state for the kernel structure that will be + * migrated and sends to the destination host a message that contains + * the type of data to be sent (MESSAGE_TYPE_KERN), the size of the structure + * to be received and the index that represents the kernel structure in order to + * be identified by the destination host. Then, the source host transfer the + * state of the kernel structure over the network and the destination host + * restores it. + */ +static inline int +migrate_kern_struct(struct vmctx *ctx, int socket, char *buffer, + enum snapshot_req struct_req, enum migration_transfer_req req) +{ + int rc; + struct migration_message_type msg; + struct vm_snapshot_meta *meta; + + if ((req != MIGRATION_SEND_REQ) && (req != MIGRATION_RECV_REQ)) { + DPRINTF("Unknown request"); + return (-1); + } + + memset(&msg, 0, sizeof(msg)); + if (req == MIGRATION_SEND_REQ) { + msg.type = MESSAGE_TYPE_KERN; + + meta = ALLOCA_VM_SNAPSHOT_META(ctx, NULL, struct_req, buffer, SNAPSHOT_BUFFER_SIZE, VM_SNAPSHOT_SAVE); + memset(meta->buffer.buf_start, 0, meta->buffer.buf_size); + meta->buffer.buf = meta->buffer.buf_start; + meta->buffer.buf_rem = meta->buffer.buf_size; + + rc = vm_snapshot_req(meta); + if (rc < 0) { + DPRINTF("Could not get struct with req %d", struct_req); + return (-1); + } + + msg.len = vm_get_snapshot_size(meta); + msg.req_type = struct_req; + + } + + rc = migration_transfer_data(socket, &msg, sizeof(msg), req); + if (rc < 0) { + DPRINTF("Could not transfer message type for kern struct %d", struct_req); + return (-1); + } + + if ((req == MIGRATION_RECV_REQ) && (msg.type != MESSAGE_TYPE_KERN)) { + DPRINTF("Receive wrong message type."); + return (-1); + } + + rc = migration_transfer_data(socket, buffer, msg.len, req); + if (rc < 0) { + DPRINTF("Could not transfer struct with req %d", struct_req); + return (-1); + } + + if (req == MIGRATION_RECV_REQ) { + meta = ALLOCA_VM_SNAPSHOT_META(ctx, NULL, msg.req_type, buffer, + msg.len, VM_SNAPSHOT_RESTORE); + meta->buffer.buf = meta->buffer.buf_start; + meta->buffer.buf_rem = meta->buffer.buf_size; + + rc = vm_snapshot_req(meta); + if (rc != 0) { + DPRINTF("Failed to restore struct %d", msg.req_type); + return (-1); + } + } + + return (0); +} + +static int +migrate_kern_data(struct vmctx *ctx, int socket, enum migration_transfer_req req) +{ + int i, rc, error; + int ndevs; + char *buffer; + const struct vm_snapshot_kern_info *snapshot_kern_structs; + + error = 0; + snapshot_kern_structs = get_snapshot_kern_structs(&ndevs); + + buffer = malloc(SNAPSHOT_BUFFER_SIZE); + if (buffer == NULL) { + EPRINTF("Could not allocate memory."); + return (-1); + } + + for (i = 0; i < ndevs; i++) { + if (req == MIGRATION_RECV_REQ) { + rc = migrate_kern_struct(ctx, socket, buffer, NO_KERN_STRUCT, MIGRATION_RECV_REQ); + if (rc < 0) { + DPRINTF("Could not restore struct %s", snapshot_kern_structs[i].struct_name); + error = -1; + break; + } + } else if (req == MIGRATION_SEND_REQ) { + rc = migrate_kern_struct(ctx, socket, buffer, + snapshot_kern_structs[i].req, MIGRATION_SEND_REQ); + if (rc < 0) { + DPRINTF("Could not send %s", snapshot_kern_structs[i].struct_name); + error = -1; + break; + } + } else { + DPRINTF("Unknown transfer request"); + error = -1; + break; + } + } + + free(buffer); + + return (error); +} static inline int migrate_connections(struct migrate_req req, int *socket_fd, int *connection_socket_fd, @@ -642,6 +777,14 @@ error = rc; goto unlock_vm_and_exit; } + + rc = migrate_kern_data(ctx, s, MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send kern data to destination"); + error = rc; + goto unlock_vm_and_exit; + } + rc = migration_transfer_data(s, &migration_completed, sizeof(migration_completed), MIGRATION_RECV_REQ); if ((rc < 0) || (migration_completed != MIGRATION_SPECS_OK)) { @@ -697,6 +840,14 @@ return (-1); } + rc = migrate_kern_data(ctx, con_socket, MIGRATION_RECV_REQ); + if (rc < 0) { + EPRINTF("Could not recv kern data"); + close(con_socket); + close(s); + return (-1); + } + fprintf(stdout, "%s: Migration completed\r\n", __func__); migration_completed = MIGRATION_SPECS_OK; rc = migration_transfer_data(con_socket, &migration_completed, diff --git a/usr.sbin/bhyve/snapshot.h b/usr.sbin/bhyve/snapshot.h --- a/usr.sbin/bhyve/snapshot.h +++ b/usr.sbin/bhyve/snapshot.h @@ -48,6 +48,8 @@ #define MAX_SNAPSHOT_FILENAME PATH_MAX #define DEFAULT_MIGRATION_PORT 24983 +#define SNAPSHOT_BUFFER_SIZE (20 << 20) + struct vmctx; struct restore_state { @@ -89,6 +91,9 @@ enum snapshot_req req; /* request type */ }; +const struct vm_snapshot_dev_info *get_snapshot_devs(int *ndevs); +const struct vm_snapshot_kern_info *get_snapshot_kern_structs(int *ndevs); + void destroy_restore_state(struct restore_state *rstate); const char *lookup_vmname(struct restore_state *rstate); diff --git a/usr.sbin/bhyve/snapshot.c b/usr.sbin/bhyve/snapshot.c --- a/usr.sbin/bhyve/snapshot.c +++ b/usr.sbin/bhyve/snapshot.c @@ -119,8 +119,6 @@ #define MAX_VMNAME 100 -#define SNAPSHOT_BUFFER_SIZE (20 * MB) - #define JSON_STRUCT_ARR_KEY "structs" #define JSON_DEV_ARR_KEY "devices" #define JSON_BASIC_METADATA_KEY "basic metadata" @@ -167,6 +165,24 @@ { "vrtc", STRUCT_VRTC }, }; +const struct vm_snapshot_dev_info * +get_snapshot_devs(int *ndevs) +{ + if (ndevs != NULL) + *ndevs = nitems(snapshot_devs); + + return (snapshot_devs); +} + +const struct vm_snapshot_kern_info * +get_snapshot_kern_structs(int *ndevs) +{ + if (ndevs != NULL) + *ndevs = nitems(snapshot_kern_structs); + + return (snapshot_kern_structs); +} + static cpuset_t vcpus_active, vcpus_suspended; static pthread_mutex_t vcpu_lock; static pthread_cond_t vcpus_idle, vcpus_can_run;