diff --git a/usr.sbin/bhyve/migration.c b/usr.sbin/bhyve/migration.c --- a/usr.sbin/bhyve/migration.c +++ b/usr.sbin/bhyve/migration.c @@ -35,6 +35,8 @@ #include "pci_emul.h" #include "snapshot.h" +#define MB (1024UL * 1024) +#define GB (1024UL * MB) #ifdef BHYVE_DEBUG #define DPRINTF(FMT, ...) \ @@ -309,6 +311,160 @@ return (error); } +static int +migrate_check_memsize(size_t local_lowmem_size, size_t local_highmem_size, + size_t remote_lowmem_size, size_t remote_highmem_size) +{ + int rc; + + rc = MIGRATION_SPECS_OK; + + if (local_lowmem_size != remote_lowmem_size){ + rc = MIGRATION_SPECS_NOT_OK; + EPRINTF("Local and remote lowmem size mismatch"); + } + + if (local_highmem_size != remote_highmem_size){ + rc = MIGRATION_SPECS_NOT_OK; + EPRINTF("Local and remote highmem size mismatch"); + } + + return (rc); +} + +static int +migrate_recv_memory(struct vmctx *ctx, int socket) +{ + size_t local_lowmem_size, local_highmem_size; + size_t remote_lowmem_size, remote_highmem_size; + char *baseaddr; + int memsize_ok; + int rc; + + local_lowmem_size = local_highmem_size = 0; + remote_lowmem_size = remote_highmem_size = 0; + rc = 0; + + rc = vm_get_guestmem_from_ctx(ctx, + &baseaddr, &local_lowmem_size, + &local_highmem_size); + if (rc != 0) { + EPRINTF("Could not get guest lowmem size and highmem size"); + return (rc); + } + + rc = migration_transfer_data(socket, &remote_lowmem_size, sizeof(remote_lowmem_size), MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not recv lowmem size"); + return (rc); + } + + rc = migration_transfer_data(socket, &remote_highmem_size, sizeof(remote_highmem_size), MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not recv highmem size"); + return (rc); + } + + memsize_ok = migrate_check_memsize(local_lowmem_size, local_highmem_size, + remote_lowmem_size, remote_highmem_size); + + rc = migration_transfer_data(socket, + &memsize_ok, sizeof(memsize_ok), MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send migration_ok to remote"); + return (rc); + } + + if (memsize_ok != MIGRATION_SPECS_OK) { + EPRINTF("Memory size mismatch with remote host"); + return (EINVAL); + } + + rc = migration_transfer_data(socket, baseaddr, local_lowmem_size, MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not recv chunk lowmem."); + return (rc); + } + + if (local_highmem_size > 0) { + rc = migration_transfer_data(socket, baseaddr + 4 * GB, local_highmem_size, MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not recv highmem"); + return (rc); + } + } + + return (0); +} + +static int +migrate_send_memory(struct vmctx *ctx, int socket) +{ + size_t lowmem_size, highmem_size; + char *mmap_vm_lowmem, *mmap_vm_highmem; + char *baseaddr; + int memsize_ok; + int rc; + + rc = 0; + mmap_vm_lowmem = MAP_FAILED; + mmap_vm_highmem = MAP_FAILED; + + rc = vm_get_guestmem_from_ctx(ctx, &baseaddr, + &lowmem_size, &highmem_size); + if (rc != 0) { + EPRINTF("Could not get guest lowmem size and highmem size"); + return (rc); + } + + /* Send the size of the lowmem segment */ + rc = migration_transfer_data(socket, &lowmem_size, sizeof(lowmem_size), MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send lowmem size"); + return (rc); + } + + /* Send the size of the highmem segment */ + rc = migration_transfer_data(socket, &highmem_size, sizeof(lowmem_size), MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send highmem size"); + return (rc); + } + + /* Wait for answer - params ok (if memory size matches) */ + rc = migration_transfer_data(socket, &memsize_ok, sizeof(memsize_ok), MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not receive response from remote"); + return (rc); + } + + if (memsize_ok != MIGRATION_SPECS_OK) { + EPRINTF("Memory size mismatch with remote host"); + return (EINVAL); + } + + mmap_vm_lowmem = baseaddr; + mmap_vm_highmem = baseaddr + 4 * GB; + + /* Send the lowmem segment */ + rc = migration_transfer_data(socket, mmap_vm_lowmem, lowmem_size, MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send lowmem"); + return (rc); + } + + /* Send the highmem segment */ + if (highmem_size > 0) { + rc = migration_transfer_data(socket, mmap_vm_highmem, highmem_size, MIGRATION_SEND_REQ); + if (rc != 0) { + EPRINTF("Could not send highmem"); + return (rc); + } + } + + return (0); +} + static inline int migrate_connections(struct migrate_req req, int *socket_fd, enum migration_transfer_req type) @@ -440,6 +596,15 @@ goto done; } + if (is_live) { + EPRINTF("Live migration not implemented"); + rc = EOPNOTSUPP; + if (rc != 0) + EPRINTF("Could not live migrate the guest's memory"); + error = rc; + goto done; + } /* else continue the warm migration procedure */ + vm_vcpu_pause(ctx); rc = vm_pause_user_devs(); @@ -449,6 +614,13 @@ goto unlock_vm_and_exit; } + rc = migrate_send_memory(ctx, s); + if (rc != 0) { + EPRINTF("Could not send memory to destination"); + error = rc; + goto unlock_vm_and_exit; + } + rc = migration_transfer_data(s, &migration_completed, sizeof(migration_completed), MIGRATION_RECV_REQ); if ((rc != 0) || (migration_completed != MIGRATION_SPECS_OK)) { @@ -476,7 +648,7 @@ } static int -vm_recv_migrate_req(struct vmctx __unused *ctx, struct migrate_req req) +vm_recv_migrate_req(struct vmctx *ctx, struct migrate_req req) { int s; int rc = 0; @@ -501,6 +673,25 @@ goto done; } + /* For recv, the only difference between warm and live migration is the + * way in which the memory is migrated. + */ + if (is_live) { + EPRINTF("Live migration not implemented"); + rc = EOPNOTSUPP; + if (rc != 0) { + EPRINTF("Could not live migrate the guest's memory"); + goto done; + } + } else { + /* if not live migration, then migrate memory normally. */ + rc = migrate_recv_memory(ctx, s); + if (rc != 0) { + EPRINTF("Could not recv lowmem and highmem"); + goto done; + } + } + // fprintf(stdout, "%s: Migration completed\n", __func__); migration_completed = MIGRATION_SPECS_OK;