diff --git a/usr.sbin/bhyve/migration.h b/usr.sbin/bhyve/migration.h --- a/usr.sbin/bhyve/migration.h +++ b/usr.sbin/bhyve/migration.h @@ -41,4 +41,43 @@ #include "snapshot.h" int receive_vm_migration(struct vmctx *ctx, char *migration_data); + +/* Warm Migration */ +#define MAX_DEV_NAME_LEN 64 + +#define MAX_IP_LEN 64 +#define MAX_SPEC_LEN 256 + +#define MIGRATION_SPECS_OK 0 +#define MIGRATION_SPECS_NOT_OK 1 +enum migration_transfer_req { + MIGRATION_SEND_REQ = 0, + MIGRATION_RECV_REQ = 1 +}; + +enum message_types { + MESSAGE_TYPE_SPECS = 1, + MESSAGE_TYPE_METADATA = 2, + MESSAGE_TYPE_RAM = 3, + MESSAGE_TYPE_KERN = 4, + MESSAGE_TYPE_DEV = 5, + MESSAGE_TYPE_UNKNOWN = 8, +}; + +struct __attribute__((packed)) migration_message_type { + size_t len; + unsigned int type; /* enum message_type */ + unsigned int req_type; /* enum snapshot_req */ + char name[MAX_DEV_NAME_LEN]; +}; + +struct __attribute__((packed)) migration_system_specs { + char hw_machine[MAX_SPEC_LEN]; + char hw_model[MAX_SPEC_LEN]; + size_t hw_pagesize; +}; + +int vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req); +int vm_recv_migrate_req(struct vmctx *ctx, struct migrate_req req); + #endif diff --git a/usr.sbin/bhyve/migration.c b/usr.sbin/bhyve/migration.c --- a/usr.sbin/bhyve/migration.c +++ b/usr.sbin/bhyve/migration.c @@ -109,7 +109,434 @@ req.port = DEFAULT_MIGRATION_PORT; } + rc = vm_recv_migrate_req(ctx, req); + free(hostname); - EPRINTF("Migration not implemented yet"); + return (rc); +} + +static int +get_system_specs_for_migration(struct migration_system_specs *specs) +{ + int mib[2]; + size_t len_machine, len_model, len_pagesize; + char interm[MAX_SPEC_LEN]; + int rc; + int num; + + mib[0] = CTL_HW; + mib[1] = HW_MACHINE; + memset(interm, 0, MAX_SPEC_LEN); + len_machine = sizeof(interm); + + rc = sysctl(mib, 2, interm, &len_machine, NULL, 0); + if (rc != 0) { + perror("Could not retrieve HW_MACHINE specs"); + return (rc); + } + strlcpy(specs->hw_machine, interm, MAX_SPEC_LEN); + + memset(interm, 0, MAX_SPEC_LEN); + mib[0] = CTL_HW; + mib[1] = HW_MODEL; + len_model = sizeof(interm); + rc = sysctl(mib, 2, interm, &len_model, NULL, 0); + if (rc != 0) { + perror("Could not retrieve HW_MODEL specs"); + return (rc); + } + strlcpy(specs->hw_model, interm, MAX_SPEC_LEN); + + mib[0] = CTL_HW; + mib[1] = HW_PAGESIZE; + len_pagesize = sizeof(num); + rc = sysctl(mib, 2, &num, &len_pagesize, NULL, 0); + if (rc != 0) { + perror("Could not retrieve HW_PAGESIZE specs"); + return (rc); + } + specs->hw_pagesize = num; + + return (0); +} + +static int +migration_transfer_data(int socket, void *msg, size_t len, enum migration_transfer_req req) +{ + uint64_t to_transfer, total_transferred; + int64_t transferred; + + to_transfer = len; + total_transferred = 0; + + while (to_transfer > 0) { + switch (req) { + case MIGRATION_SEND_REQ: + transferred = send(socket, msg + total_transferred, + to_transfer, 0); + break; + case MIGRATION_RECV_REQ: + transferred = recv(socket, msg + total_transferred, + to_transfer, 0); + break; + default: + DPRINTF("Unknown transfer option"); + return (-1); + break; + } + + if (transferred == 0) + break; + if (transferred < 0) { + perror("Error while transfering data"); + return (transferred); + } + + to_transfer -= transferred; + total_transferred += transferred; + } + + return (0); +} + +static int +migration_check_specs(int socket, enum migration_transfer_req req) +{ + struct migration_system_specs local_specs; + struct migration_system_specs remote_specs; + struct migration_system_specs transfer_specs; + struct migration_message_type msg; + enum migration_transfer_req rev_req; + size_t response; + int rc; + + if ((req != MIGRATION_SEND_REQ) && (req != MIGRATION_RECV_REQ)) { + DPRINTF("Unknown option for migration req"); + return (-1); + } + + if (req == MIGRATION_SEND_REQ) + rev_req = MIGRATION_RECV_REQ; + else + rev_req = MIGRATION_SEND_REQ; + + rc = get_system_specs_for_migration(&local_specs); + if (rc != 0) { + EPRINTF("Could not retrieve local specs"); + return (rc); + } + + if (req == MIGRATION_SEND_REQ) { + /* Send message type to server: specs & len */ + msg.type = MESSAGE_TYPE_SPECS; + msg.len = sizeof(local_specs); + } + + rc = migration_transfer_data(socket, &msg, sizeof(msg), req); + if (rc < 0) { + DPRINTF("Could not send message type"); + return (-1); + } + + if ((req == MIGRATION_RECV_REQ) && (msg.type != MESSAGE_TYPE_SPECS)) { + DPRINTF(" Wrong message type received from remote"); + return (-1); + } + + /* For the send req, we send the local specs and for the receive req + * we receive the remote specs. + */ + if (req == MIGRATION_SEND_REQ) + transfer_specs = local_specs; + + rc = migration_transfer_data(socket, &transfer_specs, sizeof(transfer_specs), req); + if (rc < 0) { + DPRINTF("Could not transfer system specs"); + return (-1); + } + + if (req == MIGRATION_RECV_REQ) { + remote_specs = transfer_specs; + + /* Check specs */ + response = MIGRATION_SPECS_OK; + if ((strncmp(local_specs.hw_model, remote_specs.hw_model, MAX_SPEC_LEN) != 0) + || (strncmp(local_specs.hw_machine, remote_specs.hw_machine, MAX_SPEC_LEN) != 0) + || (local_specs.hw_pagesize != remote_specs.hw_pagesize) + ) { + EPRINTF("System specification mismatch"); + DPRINTF("Local specs vs Remote Specs: \r\n" + "\tmachine: %s vs %s\r\n" + "\tmodel: %s vs %s\r\n" + "\tpagesize: %zu vs %zu\r\n", + local_specs.hw_machine, + remote_specs.hw_machine, + local_specs.hw_model, + remote_specs.hw_model, + local_specs.hw_pagesize, + remote_specs.hw_pagesize + ); + response = MIGRATION_SPECS_NOT_OK; + } + } + + /* The source will receive the result of the checkup (i.e. + * whether the migration is possible or the source and destination + * are incompatible for migration) and the destination will send the + * result of the checkup. + */ + rc = migration_transfer_data(socket, &response, sizeof(response), rev_req); + if (rc < 0) { + DPRINTF("Could not transfer response from server"); + return (-1); + } + + if (response == MIGRATION_SPECS_NOT_OK) + return (-1); + + fprintf(stdout, "%s: System specification accepted\r\n", __func__); + + return (0); + +} + +static int +get_migration_host_and_type(const char *hostname, unsigned char *ipv4_addr, + unsigned char *ipv6_addr, int *type) +{ + struct addrinfo hints, *res; + void *addr; + int rc; + + memset(&hints, 0, sizeof(hints)); + + hints.ai_family = AF_UNSPEC; + + rc = getaddrinfo(hostname, NULL, &hints, &res); + + if (rc != 0) { + DPRINTF("Could not get address info"); + return (-1); + } + + *type = res->ai_family; + switch(res->ai_family) { + case AF_INET: + addr = &((struct sockaddr_in *) res->ai_addr)->sin_addr; + inet_ntop(res->ai_family, addr, ipv4_addr, MAX_IP_LEN); + break; + case AF_INET6: + addr = &((struct sockaddr_in6 *) res->ai_addr)->sin6_addr; + inet_ntop(res->ai_family, addr, ipv6_addr, MAX_IP_LEN); + break; + default: + DPRINTF("Unknown address family."); + return (-1); + } + + return (0); +} +static inline int +migrate_connections(struct migrate_req req, int *socket_fd, + int *connection_socket_fd, + enum migration_transfer_req type) +{ + unsigned char ipv4_addr[MAX_IP_LEN]; + unsigned char ipv6_addr[MAX_IP_LEN]; + int addr_type; + int error; + int s, con_socket; + struct sockaddr_in sa, client_sa; + socklen_t client_len; + int rc; + + rc = get_migration_host_and_type(req.host, ipv4_addr, + ipv6_addr, &addr_type); + + if (rc != 0) { + EPRINTF("Invalid address."); + DPRINTF("IP address used for migration: %s;\r\n" + "Port used for migration: %d", + req.host, req.port); + return (rc); + } + + if (addr_type == AF_INET6) { + EPRINTF("IPv6 is not supported yet for migration. " + "Please try again using a IPv4 address."); + + DPRINTF("IP address used for migration: %s;\r\nPort used for migration: %d", + ipv6_addr, req.port); + return (-1); + } + + s = socket(AF_INET, SOCK_STREAM, 0); + + if (s < 0) { + perror("Could not create socket"); + return (-1); + } + + bzero(&sa, sizeof(sa)); + + switch (type) { + case MIGRATION_SEND_REQ: + fprintf(stdout, "%s: Starting connection to %s on %d port...\r\n", + __func__, ipv4_addr, req.port); + + sa.sin_family = AF_INET; + sa.sin_port = htons(req.port); + + rc = inet_pton(AF_INET, ipv4_addr, &sa.sin_addr); + if (rc <= 0) { + DPRINTF("Could not retrive the IPV4 address"); + return (-1); + } + + rc = connect(s, (struct sockaddr *)&sa, sizeof(sa)); + + if (rc < 0) { + perror("Could not connect to the remote host"); + error = rc; + goto done_close_s; + } + *socket_fd = s; + break; + case MIGRATION_RECV_REQ: + fprintf(stdout, "%s: Waiting for connections from %s on %d port...\r\n", + __func__, ipv4_addr, req.port); + + sa.sin_family = AF_INET; + sa.sin_port = htons(req.port); + sa.sin_addr.s_addr = htonl(INADDR_ANY); + + rc = bind(s, (struct sockaddr *)&sa, sizeof(sa)); + + if (rc < 0) { + perror("Could not bind"); + error = rc; + goto done_close_s; + } + + listen(s, 1); + + con_socket = accept(s, (struct sockaddr *)&client_sa, &client_len); + if (con_socket < 0) { + EPRINTF("Could not accept connection"); + error = -1; + goto done_close_s; + } + *socket_fd = s; + *connection_socket_fd = con_socket; + break; + default: + EPRINTF("unknown operation request"); + error = -1; + goto done; + } + + error = 0; + goto done; + +done_close_s: + close(s); +done: + return (error); +} + +int +vm_send_migrate_req(struct vmctx *ctx, struct migrate_req req) +{ + int s; + int rc, error; + size_t migration_completed; + + rc = migrate_connections(req, &s, NULL, MIGRATION_SEND_REQ); + if (rc < 0) { + EPRINTF("Could not create connection"); + return (-1); + } + + rc = migration_check_specs(s, MIGRATION_SEND_REQ); + + if (rc < 0) { + EPRINTF("Error while checking system requirements"); + error = rc; + goto done; + } + + vm_vcpu_pause(ctx); + + rc = vm_pause_user_devs(ctx); + if (rc != 0) { + EPRINTF("Could not pause devices"); + error = rc; + goto unlock_vm_and_exit; + } + + rc = migration_transfer_data(s, &migration_completed, + sizeof(migration_completed), MIGRATION_RECV_REQ); + if ((rc < 0) || (migration_completed != MIGRATION_SPECS_OK)) { + EPRINTF("Could not recv migration completed remote or received error"); + error = -1; + goto unlock_vm_and_exit; + } + + EPRINTF("Migration not yet implemented"); + error = -1; + goto unlock_vm_and_exit; + + vm_destroy(ctx); + exit(0); + +unlock_vm_and_exit: + vm_vcpu_resume(ctx); + + rc = vm_resume_user_devs(ctx); + if (rc != 0) + EPRINTF("Could not resume devices"); +done: + close(s); + return (error); +} + +int +vm_recv_migrate_req(struct vmctx *ctx, struct migrate_req req) +{ + int s, con_socket; + int rc; + size_t migration_completed; + + rc = migrate_connections(req, &s, &con_socket, MIGRATION_RECV_REQ); + if (rc != 0) { + EPRINTF("Could not create connections"); + return (-1); + } + + rc = migration_check_specs(con_socket, MIGRATION_RECV_REQ); + if (rc < 0) { + EPRINTF("Error while checking specs"); + close(con_socket); + close(s); + return (rc); + } + + + fprintf(stdout, "%s: Migration completed\r\n", __func__); + migration_completed = MIGRATION_SPECS_OK; + rc = migration_transfer_data(con_socket, &migration_completed, + sizeof(migration_completed), MIGRATION_SEND_REQ); + if (rc < 0) { + EPRINTF("Could not send migration completed remote"); + close(con_socket); + close(s); + return (-1); + } + + close(con_socket); + close(s); + + EPRINTF("Migration not currently implemented"); return (-1); } + diff --git a/usr.sbin/bhyve/snapshot.h b/usr.sbin/bhyve/snapshot.h --- a/usr.sbin/bhyve/snapshot.h +++ b/usr.sbin/bhyve/snapshot.h @@ -98,6 +98,8 @@ void checkpoint_cpu_add(int vcpu); void checkpoint_cpu_resume(int vcpu); void checkpoint_cpu_suspend(int vcpu); +void vm_vcpu_pause(struct vmctx *ctx); +void vm_vcpu_resume(struct vmctx *ctx); int restore_vm_mem(struct vmctx *ctx, struct restore_state *rstate); int vm_restore_kern_structs(struct vmctx *ctx, struct restore_state *rstate); diff --git a/usr.sbin/bhyve/snapshot.c b/usr.sbin/bhyve/snapshot.c --- a/usr.sbin/bhyve/snapshot.c +++ b/usr.sbin/bhyve/snapshot.c @@ -31,7 +31,7 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. - * * + * * $FreeBSD$ */ @@ -1304,7 +1304,7 @@ pthread_mutex_unlock(&vcpu_lock); } -static void +void vm_vcpu_pause(struct vmctx *ctx) { @@ -1316,7 +1316,7 @@ pthread_mutex_unlock(&vcpu_lock); } -static void +void vm_vcpu_resume(struct vmctx *ctx) { @@ -1476,8 +1476,7 @@ req.host, req.port); - EPRINTLN("Migration operation not implemented yet\n"); - err = -1; + err = vm_send_migrate_req(ctx, req); } } else { EPRINTLN("Unrecognized checkpoint operation\n");