diff --git a/usr.sbin/bhyve/migration.h b/usr.sbin/bhyve/migration.h --- a/usr.sbin/bhyve/migration.h +++ b/usr.sbin/bhyve/migration.h @@ -33,7 +33,6 @@ #ifndef _BHYVE_MIGRATION_ #define _BHYVE_MIGRATION_ -#include #include #include #include "snapshot.h" @@ -79,7 +78,7 @@ size_t hw_pagesize; }; -int vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req); +int vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req, bool live); int vm_recv_migrate_req(struct vmctx *ctx, struct migrate_req req); #endif diff --git a/usr.sbin/bhyve/migration.c b/usr.sbin/bhyve/migration.c --- a/usr.sbin/bhyve/migration.c +++ b/usr.sbin/bhyve/migration.c @@ -915,10 +915,10 @@ } int -vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req) +vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req, bool live) { int s; - int rc, error; + int rc, error, is_live_migration; size_t migration_completed; rc = migrate_connections(req, &s, NULL, MIGRATION_SEND_REQ); @@ -935,6 +935,24 @@ goto done; } + is_live_migration = live; + rc = migration_transfer_data(s, &is_live_migration, + sizeof(is_live_migration), MIGRATION_SEND_REQ); + if (rc < 0) { + DPRINTF("Could not send migration type"); + return (-1); + } + + if (live) { + EPRINTF("Live migration not yet implemented"); + rc = -1; + if (rc != 0) { + EPRINTF("Could not live migrate the guest's memory"); + error = rc; + } + goto done; + } // else continue the warm migration procedure + vm_vcpu_pause(ctx); rc = vm_pause_user_devs(ctx); @@ -992,6 +1010,7 @@ { int s, con_socket; int rc; + int is_live_migration; size_t migration_completed; rc = migrate_connections(req, &s, &con_socket, MIGRATION_RECV_REQ); @@ -1008,14 +1027,36 @@ return (rc); } - rc = migrate_recv_memory(ctx, con_socket); + rc = migration_transfer_data(con_socket, &is_live_migration, + sizeof(is_live_migration), MIGRATION_RECV_REQ); if (rc < 0) { - EPRINTF("Could not recv :lowmem and highmem"); - close(con_socket); - close(s); + EPRINTF("Could not recv migration type"); return (-1); } + /* For recv, the only difference between warm and live migration is the + * way in which the memory is migrated. + */ + if (is_live_migration) { + EPRINTF("Live migration not yet implemented"); + rc = -1; + if (rc != 0) { + EPRINTF("Could not live migrate the guest's memory"); + close(con_socket); + close(s); + return (rc); + } + } else { + /* if not live migration, then migrate memory normally. */ + rc = migrate_recv_memory(ctx, con_socket); + if (rc < 0) { + EPRINTF("Could not recv lowmem and highmem"); + close(con_socket); + close(s); + return (-1); + } + } + rc = migrate_kern_data(ctx, con_socket, MIGRATION_RECV_REQ); if (rc < 0) { EPRINTF("Could not recv kern data"); diff --git a/usr.sbin/bhyve/snapshot.c b/usr.sbin/bhyve/snapshot.c --- a/usr.sbin/bhyve/snapshot.c +++ b/usr.sbin/bhyve/snapshot.c @@ -1460,7 +1460,8 @@ static int handle_message(struct vmctx *ctx, nvlist_t *nvl) { - int err; + bool is_live_migration; + int err, memflags; size_t len; const char *cmd; struct migrate_req req; @@ -1471,6 +1472,8 @@ } cmd = nvlist_get_string(nvl, "cmd"); + + is_live_migration = strcmp(cmd, "migrate_live") == 0; if (strcmp(cmd, "checkpoint") == 0) { if (!nvlist_exists_string(nvl, "filename") || !nvlist_exists_bool(nvl, "suspend")) { @@ -1479,12 +1482,30 @@ } err = vm_checkpoint(ctx, nvlist_get_string(nvl, "filename"), nvlist_get_bool(nvl, "suspend")); - } else if (strcmp(cmd, "migrate") == 0) { + } else if (strcmp(cmd, "migrate") == 0 || is_live_migration) { if (!nvlist_exists_string(nvl, "hostname") || !nvlist_exists_number(nvl, "port")) { err = -1; goto done; - } + } + + if (is_live_migration) { + /* Currently, the live migration is implemented only + * for guests that are started using -S (wired + * memory option). + */ + + /* Check memflags. If the VM_MEM_F_WIRED bit is not + * set, then the live migration procedure cannot be + * done. */ + memflags = vm_get_memflags(ctx); + if (!(memflags & VM_MEM_F_WIRED)) { + EPRINTLN("Migration not supported for un-wired guests\r\n"); + err = -1; + goto done; + } + } + memset(&req, 0, sizeof(struct migrate_req)); req.port = nvlist_get_number(nvl, "port"); @@ -1504,7 +1525,7 @@ req.host, req.port); - err = vm_send_migrate_req(ctx, req); + err = vm_send_migrate_req(ctx, req, is_live_migration); } else { EPRINTLN("Unrecognized checkpoint operation\n"); err = -1; diff --git a/usr.sbin/bhyvectl/bhyvectl.8 b/usr.sbin/bhyvectl/bhyvectl.8 --- a/usr.sbin/bhyvectl/bhyvectl.8 +++ b/usr.sbin/bhyvectl/bhyvectl.8 @@ -42,6 +42,7 @@ .Op Fl -checkpoint= Ns Ar .Op Fl -suspend= Ns Ar .Op Fl -migrate= Ns Ar host Ns Op Cm \&, Ns Ar port +.Op Fl -migrate-live= Ns Ar host Ns Op Cm \&, Ns Ar port .Sh DESCRIPTION The .Nm @@ -93,6 +94,17 @@ .Ar port . The default migration port is 24983. The virtual machine will be destroyed after the migration finishes. +.It Fl -migrate-live= Ns Ar host Ns Op Cm \&, Ns Ar port +Live migrate the virtual machine to a +.Ar host +on the specified +.Ar port . +Live migration requires the guest's memory to be wired (using +.Fl S +for +.Xr bhyve 8) . +The default migration port is 24983. +The virtual machine will be destroyed after the migration finishes. .El .Sh EXIT STATUS .Ex -std diff --git a/usr.sbin/bhyvectl/bhyvectl.c b/usr.sbin/bhyvectl/bhyvectl.c --- a/usr.sbin/bhyvectl/bhyvectl.c +++ b/usr.sbin/bhyvectl/bhyvectl.c @@ -91,6 +91,7 @@ " [--checkpoint=]\n" " [--suspend=]\n" " [--migrate=host[,port]]\n" + " [--migrate-live=host[,port]]\n" #endif " [--get-all]\n" " [--get-stats]\n" @@ -305,6 +306,7 @@ static int vm_checkpoint_opt; static int vm_suspend_opt; static int vm_migrate; +static int vm_migrate_live; #endif /* @@ -597,6 +599,7 @@ SET_CHECKPOINT_FILE, SET_SUSPEND_FILE, MIGRATE_VM, + MIGRATE_VM_LIVE, #endif }; @@ -1470,6 +1473,7 @@ { "checkpoint", REQ_ARG, 0, SET_CHECKPOINT_FILE}, { "suspend", REQ_ARG, 0, SET_SUSPEND_FILE}, { "migrate", REQ_ARG, 0, MIGRATE_VM}, + { "migrate-live", REQ_ARG, 0, MIGRATE_VM_LIVE}, #endif }; @@ -1742,7 +1746,7 @@ } static int -send_start_migrate(struct vmctx *ctx, const char *migrate_vm) +send_start_migrate(struct vmctx *ctx, const char *migrate_vm, bool live) { nvlist_t *nvl; char *hostname, *pos; @@ -1768,7 +1772,10 @@ } nvl = nvlist_create(0); - nvlist_add_string(nvl, "cmd", "migrate"); + if (live) + nvlist_add_string(nvl, "cmd", "migrate_live"); + else + nvlist_add_string(nvl, "cmd", "migrate"); nvlist_add_string(nvl, "hostname", hostname); nvlist_add_number(nvl, "port", port); @@ -1779,7 +1786,6 @@ #endif /* BHYVE_SNAPSHOT */ - int main(int argc, char *argv[]) { @@ -1970,6 +1976,10 @@ vm_migrate = 1; migrate_host = optarg; break; + case MIGRATE_VM_LIVE: + vm_migrate_live = 1; + migrate_host = optarg; + break; #endif default: usage(cpu_intel); @@ -2450,7 +2460,10 @@ error = snapshot_request(ctx, suspend_file, true); if (!error && vm_migrate) - error = send_start_migrate(ctx, migrate_host); + error = send_start_migrate(ctx, migrate_host, false); + + if (!error && vm_migrate_live) + error = send_start_migrate(ctx, migrate_host, true); #endif free (opts);