Index: usr.sbin/bhyve/atkbdc.h =================================================================== --- usr.sbin/bhyve/atkbdc.h +++ usr.sbin/bhyve/atkbdc.h @@ -31,13 +31,14 @@ struct atkbdc_softc; struct vm_snapshot_meta; +struct vm_snapshot_dev_info; struct vmctx; void atkbdc_init(struct vmctx *ctx); void atkbdc_event(struct atkbdc_softc *sc, int iskbd); #ifdef BHYVE_SNAPSHOT -int atkbdc_snapshot(struct vm_snapshot_meta *meta); +int atkbdc_snapshot(struct vm_snapshot_meta *meta, struct vm_snapshot_dev_info *dev_info); #endif #endif /* _ATKBDC_H_ */ Index: usr.sbin/bhyve/atkbdc.c =================================================================== --- usr.sbin/bhyve/atkbdc.c +++ usr.sbin/bhyve/atkbdc.c @@ -56,6 +56,10 @@ #include "ps2kbd.h" #include "ps2mouse.h" +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + #define KBD_DATA_PORT 0x60 #define KBD_STS_CTL_PORT 0x64 @@ -138,10 +142,6 @@ struct aux_dev aux; }; -#ifdef BHYVE_SNAPSHOT -static struct atkbdc_softc *atkbdc_sc = NULL; -#endif - static void atkbdc_assert_kbd_intr(struct atkbdc_softc *sc) { @@ -517,6 +517,9 @@ struct inout_port iop; struct atkbdc_softc *sc; int error; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif sc = calloc(1, sizeof(struct atkbdc_softc)); sc->ctx = ctx; @@ -555,16 +558,32 @@ sc->ps2mouse_sc = ps2mouse_init(sc); #ifdef BHYVE_SNAPSHOT - assert(atkbdc_sc == NULL); - atkbdc_sc = sc; + dev_info = calloc(1, sizeof(*dev_info)); + + if (dev_info == NULL) { + error = -1; + fprintf(stderr, "Error allocating space for dev_info"); + } + assert(error == 0); + + dev_info->dev_name = "atkbdc"; + dev_info->was_restored = 0; + dev_info->snapshot_cb = atkbdc_snapshot; + dev_info->meta_data = sc; + + insert_registered_devs(dev_info); #endif } #ifdef BHYVE_SNAPSHOT int -atkbdc_snapshot(struct vm_snapshot_meta *meta) +atkbdc_snapshot(struct vm_snapshot_meta *meta, struct vm_snapshot_dev_info *dev_info) { int ret; + struct atkbdc_softc *atkbdc_sc; + + assert(dev_info->meta_data != NULL); + atkbdc_sc = (struct atkbdc_softc*) dev_info->meta_data; SNAPSHOT_VAR_OR_LEAVE(atkbdc_sc->status, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(atkbdc_sc->outport, meta, ret, done); @@ -584,7 +603,6 @@ SNAPSHOT_VAR_OR_LEAVE(atkbdc_sc->aux.irq_active, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(atkbdc_sc->aux.irq, meta, ret, done); - ret = ps2kbd_snapshot(atkbdc_sc->ps2kbd_sc, meta); if (ret != 0) goto done; Index: usr.sbin/bhyve/bhyverun.c =================================================================== --- usr.sbin/bhyve/bhyverun.c +++ usr.sbin/bhyve/bhyverun.c @@ -1124,12 +1124,12 @@ destroy_on_poweroff = 1; break; case 'p': - if (pincpu_parse(optarg) != 0) { - errx(EX_USAGE, "invalid vcpu pinning " - "configuration '%s'", optarg); - } + if (pincpu_parse(optarg) != 0) { + errx(EX_USAGE, "invalid vcpu pinning " + "configuration '%s'", optarg); + } break; - case 'c': + case 'c': if (topology_parse(optarg) != 0) { errx(EX_USAGE, "invalid cpu topology " "'%s'", optarg); @@ -1442,7 +1442,6 @@ } } #endif - /* * Head off to the main event dispatch loop */ Index: usr.sbin/bhyve/pci_ahci.c =================================================================== --- usr.sbin/bhyve/pci_ahci.c +++ usr.sbin/bhyve/pci_ahci.c @@ -42,6 +42,9 @@ #include #include +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif #include #include @@ -2329,6 +2332,10 @@ size_t block_len; int comma, optpos; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif + ret = 0; #ifdef AHCI_DEBUG @@ -2496,6 +2503,24 @@ pci_lintr_request(pi); +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + goto open_fail; + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->meta_data = pi; + dev_info->snapshot_cb = pci_snapshot; + dev_info->pause_cb = pci_pause; + dev_info->resume_cb = pci_resume; + + insert_registered_devs(dev_info); +#endif + open_fail: if (ret) { for (p = 0; p < sc->ports; p++) { @@ -2684,7 +2709,7 @@ SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(port->rfis, 256, false, meta, ret, done); - SNAPSHOT_VAR_OR_LEAVE(port->ident, meta, ret, done); + SNAPSHOT_VAR_OR_LEAVE(port->ata_ident, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(port->atapi, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(port->reset, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(port->waitforclear, meta, ret, done); Index: usr.sbin/bhyve/pci_e82545.c =================================================================== --- usr.sbin/bhyve/pci_e82545.c +++ usr.sbin/bhyve/pci_e82545.c @@ -48,6 +48,10 @@ #endif #include +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + #include #include #include @@ -2284,6 +2288,10 @@ char *optscopy; char *vtopts; int mac_provided; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif + DPRINTF("Loading with options: %s", opts); @@ -2380,6 +2388,22 @@ /* H/w initiated reset */ e82545_reset(sc, 0); +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + return (-1); + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif + return (0); } Index: usr.sbin/bhyve/pci_emul.h =================================================================== --- usr.sbin/bhyve/pci_emul.h +++ usr.sbin/bhyve/pci_emul.h @@ -46,6 +46,7 @@ struct pci_devinst; struct memory_region; struct vm_snapshot_meta; +struct vm_snapshot_dev_info; struct pci_devemu { char *pe_emu; /* Name of device emulation */ @@ -251,9 +252,9 @@ uint64_t pci_ecfg_base(void); int pci_bus_configured(int bus); #ifdef BHYVE_SNAPSHOT -int pci_snapshot(struct vm_snapshot_meta *meta); -int pci_pause(struct vmctx *ctx, const char *dev_name); -int pci_resume(struct vmctx *ctx, const char *dev_name); +int pci_snapshot(struct vm_snapshot_meta *meta, struct vm_snapshot_dev_info *dev_info); +int pci_pause(struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info); +int pci_resume(struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info); #endif static __inline void Index: usr.sbin/bhyve/pci_emul.c =================================================================== --- usr.sbin/bhyve/pci_emul.c +++ usr.sbin/bhyve/pci_emul.c @@ -58,9 +58,12 @@ #include "pci_irq.h" #include "pci_lpc.h" -#define CONF1_ADDR_PORT 0x0cf8 -#define CONF1_DATA_PORT 0x0cfc +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif +#define CONF1_ADDR_PORT 0x0cf8 +#define CONF1_DATA_PORT 0x0cfc #define CONF1_ENABLE 0x80000000ul #define MAXBUSES (PCI_BUSMAX + 1) @@ -2017,60 +2020,19 @@ return (ret); } -static int -pci_find_slotted_dev(const char *dev_name, struct pci_devemu **pde, - struct pci_devinst **pdi) -{ - struct businfo *bi; - struct slotinfo *si; - struct funcinfo *fi; - int bus, slot, func; - - assert(dev_name != NULL); - assert(pde != NULL); - assert(pdi != NULL); - - for (bus = 0; bus < MAXBUSES; bus++) { - if ((bi = pci_businfo[bus]) == NULL) - continue; - - for (slot = 0; slot < MAXSLOTS; slot++) { - si = &bi->slotinfo[slot]; - for (func = 0; func < MAXFUNCS; func++) { - fi = &si->si_funcs[func]; - if (fi->fi_name == NULL) - continue; - if (strcmp(dev_name, fi->fi_name)) - continue; - - *pde = pci_emul_finddev(fi->fi_name); - assert(*pde != NULL); - - *pdi = fi->fi_devi; - return (0); - } - } - } - - return (EINVAL); -} - int -pci_snapshot(struct vm_snapshot_meta *meta) +pci_snapshot(struct vm_snapshot_meta *meta, struct vm_snapshot_dev_info *dev_info) { struct pci_devemu *pde; struct pci_devinst *pdi; int ret; - assert(meta->dev_name != NULL); - - ret = pci_find_slotted_dev(meta->dev_name, &pde, &pdi); - if (ret != 0) { - fprintf(stderr, "%s: no such name: %s\r\n", - __func__, meta->dev_name); - memset(meta->buffer.buf_start, 0, meta->buffer.buf_size); - return (0); + if (dev_info->meta_data == NULL) { + fprintf(stderr, "%s: device meta data is NULL", __func__); + return (-1); } + pdi = (struct pci_devinst *) dev_info->meta_data; + pde = pdi->pi_d; meta->dev_data = pdi; @@ -2093,28 +2055,22 @@ } int -pci_pause(struct vmctx *ctx, const char *dev_name) +pci_pause(struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info) { struct pci_devemu *pde; struct pci_devinst *pdi; - int ret; - - assert(dev_name != NULL); - - ret = pci_find_slotted_dev(dev_name, &pde, &pdi); - if (ret != 0) { - /* - * It is possible to call this function without - * checking that the device is inserted first. - */ - fprintf(stderr, "%s: no such name: %s\n", __func__, dev_name); - return (0); + + if (dev_info->meta_data == NULL) { + fprintf(stderr, "%s: device meta data is NULL", __func__); + return (-1); } + pdi = (struct pci_devinst *) dev_info->meta_data; + pde = pdi->pi_d; if (pde->pe_pause == NULL) { /* The pause/resume functionality is optional. */ fprintf(stderr, "%s: not implemented for: %s\n", - __func__, dev_name); + __func__, dev_info->dev_name); return (0); } @@ -2122,28 +2078,22 @@ } int -pci_resume(struct vmctx *ctx, const char *dev_name) +pci_resume(struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info) { struct pci_devemu *pde; struct pci_devinst *pdi; - int ret; - assert(dev_name != NULL); - - ret = pci_find_slotted_dev(dev_name, &pde, &pdi); - if (ret != 0) { - /* - * It is possible to call this function without - * checking that the device is inserted first. - */ - fprintf(stderr, "%s: no such name: %s\n", __func__, dev_name); - return (0); + if (dev_info->meta_data == NULL) { + fprintf(stderr, "%s: device meta data is NULL", __func__); + return (-1); } + pdi = (struct pci_devinst *) dev_info->meta_data; + pde = pdi->pi_d; if (pde->pe_resume == NULL) { /* The pause/resume functionality is optional. */ fprintf(stderr, "%s: not implemented for: %s\n", - __func__, dev_name); + __func__, dev_info->dev_name); return (0); } Index: usr.sbin/bhyve/pci_fbuf.c =================================================================== --- usr.sbin/bhyve/pci_fbuf.c +++ usr.sbin/bhyve/pci_fbuf.c @@ -54,6 +54,10 @@ #include "rfb.h" #include "vga.h" +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + /* * bhyve Framebuffer device emulation. * BAR0 points to the current mode information. @@ -355,6 +359,9 @@ { int error, prot; struct pci_fbuf_softc *sc; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif if (fbuf_sc != NULL) { EPRINTLN("Only one frame buffer device is allowed."); @@ -434,6 +441,23 @@ memset((void *)sc->fb_base, 0, FB_SIZE); error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password); + +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + error = -1; + goto done; + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif done: if (error) free(sc); Index: usr.sbin/bhyve/pci_lpc.c =================================================================== --- usr.sbin/bhyve/pci_lpc.c +++ usr.sbin/bhyve/pci_lpc.c @@ -51,6 +51,10 @@ #include "pci_lpc.h" #include "uart_emul.h" +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + #define IO_ICU1 0x20 #define IO_ICU2 0xA0 @@ -395,6 +399,9 @@ static int pci_lpc_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) { +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif /* * Do not allow more than one LPC bridge to be configured. @@ -425,6 +432,22 @@ lpc_bridge = pi; +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + return (-1); + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif + return (0); } Index: usr.sbin/bhyve/pci_virtio_block.c =================================================================== --- usr.sbin/bhyve/pci_virtio_block.c +++ usr.sbin/bhyve/pci_virtio_block.c @@ -62,6 +62,10 @@ #define VTBLK_BSIZE 512 #define VTBLK_RINGSZ 128 +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + _Static_assert(VTBLK_RINGSZ <= BLOCKIF_RING_MAX, "Each ring entry must be able to queue a request"); #define VTBLK_S_OK 0 @@ -444,6 +448,9 @@ struct pci_vtblk_softc *sc; off_t size; int i, sectsz, sts, sto; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif if (opts == NULL) { WPRINTF(("virtio-block: backing device required")); @@ -542,6 +549,23 @@ return (1); } vi_set_io_bar(&sc->vbsc_vs, 0); +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + free(sc); + return (1); + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif + return (0); } Index: usr.sbin/bhyve/pci_virtio_net.c =================================================================== --- usr.sbin/bhyve/pci_virtio_net.c +++ usr.sbin/bhyve/pci_virtio_net.c @@ -39,6 +39,14 @@ #include #include #include /* IFNAMSIZ */ +#ifndef NETMAP_WITH_LIBS +#define NETMAP_WITH_LIBS +#endif +#include + +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif #include #include @@ -550,6 +558,9 @@ int mac_provided; int mtu_provided; unsigned long mtu = ETHERMTU; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif /* * Allocate data structures for further virtio initializations. @@ -691,7 +702,22 @@ snprintf(tname, sizeof(tname), "vtnet-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->tx_tid, tname); +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + free(sc); + return (1); + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif return (0); } @@ -817,7 +843,7 @@ } #endif -static struct pci_devemu pci_de_vnet = { +struct pci_devemu pci_de_vnet = { .pe_emu = "virtio-net", .pe_init = pci_vtnet_init, .pe_barwrite = vi_pci_write, Index: usr.sbin/bhyve/pci_xhci.c =================================================================== --- usr.sbin/bhyve/pci_xhci.c +++ usr.sbin/bhyve/pci_xhci.c @@ -50,6 +50,10 @@ #include +#ifdef BHYVE_SNAPSHOT +#include "snapshot.h" +#endif + #include #include #include @@ -2776,6 +2780,9 @@ { struct pci_xhci_softc *sc; int error; +#ifdef BHYVE_SNAPSHOT + struct vm_snapshot_dev_info *dev_info; +#endif if (xhci_in_use) { WPRINTF(("pci_xhci controller already defined")); @@ -2857,6 +2864,23 @@ pthread_mutex_init(&sc->mtx, NULL); +#ifdef BHYVE_SNAPSHOT + dev_info = calloc(1, sizeof(*dev_info)); + + if (!dev_info) { + fprintf(stderr, "Error allocating space for snapshot struct"); + error = -1; + goto done; + } + + dev_info->dev_name = pi->pi_d->pe_emu; + dev_info->was_restored = 0; + dev_info->snapshot_cb = pci_snapshot; + dev_info->meta_data = pi; + + insert_registered_devs(dev_info); +#endif + done: if (error) { free(sc); Index: usr.sbin/bhyve/snapshot.h =================================================================== --- usr.sbin/bhyve/snapshot.h +++ usr.sbin/bhyve/snapshot.h @@ -43,6 +43,7 @@ #include struct vmctx; +struct vm_snapshot_dev_info; struct restore_state { int kdata_fd; @@ -62,12 +63,14 @@ int socket_fd; }; -typedef int (*vm_snapshot_dev_cb)(struct vm_snapshot_meta *); -typedef int (*vm_pause_dev_cb) (struct vmctx *, const char *); -typedef int (*vm_resume_dev_cb) (struct vmctx *, const char *); +typedef int (*vm_snapshot_dev_cb)(struct vm_snapshot_meta *meta, struct vm_snapshot_dev_info *dev_info); +typedef int (*vm_pause_dev_cb) (struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info); +typedef int (*vm_resume_dev_cb) (struct vmctx *ctx, struct vm_snapshot_dev_info *dev_info); struct vm_snapshot_dev_info { const char *dev_name; /* device name */ + int was_restored; /* flag to check if the device was previously restored*/ + void *meta_data; /* specific meta data for specific devices*/ vm_snapshot_dev_cb snapshot_cb; /* callback for device snapshot */ vm_pause_dev_cb pause_cb; /* callback for device pause */ vm_resume_dev_cb resume_cb; /* callback for device resume */ @@ -78,6 +81,14 @@ enum snapshot_req req; /* request type */ }; +struct vm_snapshot_registered_devs { + struct vm_snapshot_dev_info *dev_info; + // for each device type, the meta should be specific + struct vm_snapshot_registered_devs *next_dev; +}; + +void insert_registered_devs(struct vm_snapshot_dev_info *dev_info); + void destroy_restore_state(struct restore_state *rstate); const char *lookup_vmname(struct restore_state *rstate); @@ -91,7 +102,8 @@ int restore_vm_mem(struct vmctx *ctx, struct restore_state *rstate); int vm_restore_kern_structs(struct vmctx *ctx, struct restore_state *rstate); - +int vm_restore_user_dev(struct vmctx *ctx, struct restore_state *rstate, void *dev_ptr, + size_t dev_size, struct vm_snapshot_dev_info *dev_info); int vm_restore_user_devs(struct vmctx *ctx, struct restore_state *rstate); int vm_pause_user_devs(struct vmctx *ctx); int vm_resume_user_devs(struct vmctx *ctx); Index: usr.sbin/bhyve/snapshot.c =================================================================== --- usr.sbin/bhyve/snapshot.c +++ usr.sbin/bhyve/snapshot.c @@ -109,6 +109,10 @@ static struct winsize winsize; static sig_t old_winch_handler; +#ifdef BHYVE_SNAPSHOT +static struct vm_snapshot_registered_devs *head_registered_devs = NULL; +#endif + #define KB (1024UL) #define MB (1024UL * KB) #define GB (1024UL * MB) @@ -143,19 +147,6 @@ _a < _b ? _a : _b; \ }) -const struct vm_snapshot_dev_info snapshot_devs[] = { - { "atkbdc", atkbdc_snapshot, NULL, NULL }, - { "virtio-net", pci_snapshot, pci_pause, pci_resume }, - { "virtio-blk", pci_snapshot, pci_pause, pci_resume }, - { "lpc", pci_snapshot, NULL, NULL }, - { "fbuf", pci_snapshot, NULL, NULL }, - { "xhci", pci_snapshot, NULL, NULL }, - { "e1000", pci_snapshot, NULL, NULL }, - { "ahci", pci_snapshot, pci_pause, pci_resume }, - { "ahci-hd", pci_snapshot, pci_pause, pci_resume }, - { "ahci-cd", pci_snapshot, NULL, NULL }, -}; - const struct vm_snapshot_kern_info snapshot_kern_structs[] = { { "vhpet", STRUCT_VHPET }, { "vm", STRUCT_VM }, @@ -174,6 +165,18 @@ static pthread_cond_t vcpus_idle, vcpus_can_run; static bool checkpoint_active; +void +insert_registered_devs(struct vm_snapshot_dev_info *dev_info) +{ + struct vm_snapshot_registered_devs *new_node; + new_node = malloc(sizeof(*new_node)); + + new_node->dev_info = dev_info; + new_node->next_dev = head_registered_devs; + + head_registered_devs = new_node; +} + /* * TODO: Harden this function and all of its callers since 'base_str' is a user * provided string. @@ -464,61 +467,78 @@ } static void * -lookup_check_dev(const char *dev_name, struct restore_state *rstate, - const ucl_object_t *obj, size_t *data_size) +lookup_check_dev(struct restore_state *rstate, const ucl_object_t *obj, + size_t *data_size, struct vm_snapshot_dev_info **dev_info) { const char *snapshot_req; + struct vm_snapshot_registered_devs *ptr_registered_devs; int64_t size, file_offset; snapshot_req = NULL; JSON_GET_STRING_OR_RETURN(JSON_SNAPSHOT_REQ_KEY, obj, &snapshot_req, NULL); assert(snapshot_req != NULL); - if (!strcmp(snapshot_req, dev_name)) { - JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj, + + ptr_registered_devs = head_registered_devs; + + while (ptr_registered_devs != NULL) { + if (!strcmp(snapshot_req, ptr_registered_devs->dev_info->dev_name) + && !ptr_registered_devs->dev_info->was_restored) { + *dev_info = ptr_registered_devs->dev_info; + fprintf(stderr, "Found the following device: %s \n", (*dev_info)->dev_name); + JSON_GET_INT_OR_RETURN(JSON_SIZE_KEY, obj, &size, NULL); - assert(size >= 0); + assert(size >= 0); - JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj, - &file_offset, NULL); - assert(file_offset >= 0); - assert(file_offset + size <= rstate->kdata_len); + JSON_GET_INT_OR_RETURN(JSON_FILE_OFFSET_KEY, obj, + &file_offset, NULL); + assert(file_offset >= 0); + assert(file_offset + size <= rstate->kdata_len); + *data_size = (size_t)size; - *data_size = (size_t)size; - return (rstate->kdata_map + file_offset); + return (rstate->kdata_map + file_offset); + } + ptr_registered_devs = ptr_registered_devs->next_dev; } return (NULL); } -static void* -lookup_dev(const char *dev_name, struct restore_state *rstate, - size_t *data_size) +int +lookup_devs(struct vmctx *ctx, struct restore_state *rstate) { const ucl_object_t *devs = NULL, *obj = NULL; ucl_object_iter_t it = NULL; - void *ret; + void *dev_ptr; + size_t dev_size; + struct vm_snapshot_dev_info *dev_info; devs = ucl_object_lookup(rstate->meta_root_obj, JSON_DEV_ARR_KEY); if (devs == NULL) { fprintf(stderr, "Failed to find '%s' object.\n", JSON_DEV_ARR_KEY); - return (NULL); + return (-1); } if (ucl_object_type((ucl_object_t *)devs) != UCL_ARRAY) { fprintf(stderr, "Object '%s' is not an array.\n", JSON_DEV_ARR_KEY); - return (NULL); + return (-1); } + // TODO: create small struct with dev_ptr, dev_size, dev_name while ((obj = ucl_object_iterate(devs, &it, true)) != NULL) { - ret = lookup_check_dev(dev_name, rstate, obj, data_size); - if (ret != NULL) - return (ret); + dev_ptr = lookup_check_dev(rstate, obj, &dev_size, &dev_info); + + if (dev_ptr != NULL) { + int ret_val; + ret_val = vm_restore_user_dev(ctx, rstate, dev_ptr, dev_size, dev_info); + if (ret_val != 0) + return ret_val; + } } - return (NULL); + return (0); } static const ucl_object_t * @@ -930,31 +950,15 @@ } int -vm_restore_user_dev(struct vmctx *ctx, struct restore_state *rstate, - const struct vm_snapshot_dev_info *info) +vm_restore_user_dev(struct vmctx *ctx, struct restore_state *rstate, void *dev_ptr, + size_t dev_size, struct vm_snapshot_dev_info *dev_info) { - void *dev_ptr; - size_t dev_size; int ret; struct vm_snapshot_meta *meta; - dev_ptr = lookup_dev(info->dev_name, rstate, &dev_size); - if (dev_ptr == NULL) { - fprintf(stderr, "Failed to lookup dev: %s\r\n", info->dev_name); - fprintf(stderr, "Continuing the restore/migration process\r\n"); - return (0); - } - - if (dev_size == 0) { - fprintf(stderr, "%s: Device size is 0. " - "Assuming %s is not used\r\n", - __func__, info->dev_name); - return (0); - } - meta = &(struct vm_snapshot_meta) { .ctx = ctx, - .dev_name = info->dev_name, + .dev_name = dev_info->dev_name, .buffer.buf_start = dev_ptr, .buffer.buf_size = dev_size, @@ -965,27 +969,27 @@ .op = VM_SNAPSHOT_RESTORE, }; - ret = (*info->snapshot_cb)(meta); + ret = (*dev_info->snapshot_cb)(meta, dev_info); if (ret != 0) { fprintf(stderr, "Failed to restore dev: %s\r\n", - info->dev_name); + dev_info->dev_name); return (-1); } + dev_info->was_restored = 1; + return (0); } - +//TODO: make this less messier int vm_restore_user_devs(struct vmctx *ctx, struct restore_state *rstate) { int ret; - int i; - for (i = 0; i < nitems(snapshot_devs); i++) { - ret = vm_restore_user_dev(ctx, rstate, &snapshot_devs[i]); - if (ret != 0) - return (ret); + ret = lookup_devs(ctx, rstate); + if (ret != 0) { + return (ret); } return 0; @@ -996,16 +1000,21 @@ { const struct vm_snapshot_dev_info *info; int ret; - int i; + struct vm_snapshot_registered_devs *ptr_registered_devs; + + ptr_registered_devs = head_registered_devs; - for (i = 0; i < nitems(snapshot_devs); i++) { - info = &snapshot_devs[i]; - if (info->pause_cb == NULL) + while (ptr_registered_devs != NULL) { + info = ptr_registered_devs->dev_info; + if (info->pause_cb == NULL) { + ptr_registered_devs = ptr_registered_devs->next_dev; continue; + } - ret = info->pause_cb(ctx, info->dev_name); + ret = info->pause_cb(ctx, ptr_registered_devs->dev_info); if (ret != 0) return (ret); + ptr_registered_devs = ptr_registered_devs->next_dev; } return (0); @@ -1016,16 +1025,20 @@ { const struct vm_snapshot_dev_info *info; int ret; - int i; + struct vm_snapshot_registered_devs *ptr_registered_devs; - for (i = 0; i < nitems(snapshot_devs); i++) { - info = &snapshot_devs[i]; - if (info->resume_cb == NULL) - continue; + ptr_registered_devs = head_registered_devs; - ret = info->resume_cb(ctx, info->dev_name); + while (ptr_registered_devs != NULL) { + info = ptr_registered_devs->dev_info; + if (info->resume_cb == NULL) { + ptr_registered_devs = ptr_registered_devs->next_dev; + continue; + } + ret = info->resume_cb(ctx, ptr_registered_devs->dev_info); if (ret != 0) return (ret); + ptr_registered_devs = ptr_registered_devs->next_dev; } return (0); @@ -1178,13 +1191,15 @@ } static int -vm_snapshot_user_dev(const struct vm_snapshot_dev_info *info, +vm_snapshot_user_dev(struct vm_snapshot_dev_info *info, int data_fd, xo_handle_t *xop, struct vm_snapshot_meta *meta, off_t *offset) { int ret; - ret = (*info->snapshot_cb)(meta); + // TODO: check other devices NOT ONLY PCI + ret = (*info->snapshot_cb)(meta, info); + //ret = (*info->snapshot_cb)(meta); if (ret != 0) { fprintf(stderr, "Failed to snapshot %s; ret=%d\r\n", meta->dev_name, ret); @@ -1202,11 +1217,15 @@ static int vm_snapshot_user_devs(struct vmctx *ctx, int data_fd, xo_handle_t *xop) { - int ret, i; + int ret; off_t offset; void *buffer; size_t buf_size; struct vm_snapshot_meta *meta; + struct vm_snapshot_dev_info *info; + struct vm_snapshot_registered_devs *ptr_registered_devs; + + ptr_registered_devs = head_registered_devs; buf_size = SNAPSHOT_BUFFER_SIZE; @@ -1235,17 +1254,22 @@ xo_open_list_h(xop, JSON_DEV_ARR_KEY); /* Restore other devices that support this feature */ - for (i = 0; i < nitems(snapshot_devs); i++) { - meta->dev_name = snapshot_devs[i].dev_name; + while (ptr_registered_devs != NULL) { + info = ptr_registered_devs->dev_info; + + meta->dev_name = info->dev_name; memset(meta->buffer.buf_start, 0, meta->buffer.buf_size); meta->buffer.buf = meta->buffer.buf_start; meta->buffer.buf_rem = meta->buffer.buf_size; - ret = vm_snapshot_user_dev(&snapshot_devs[i], data_fd, xop, + fprintf(stderr, "Doing the snapshot for: %s \n", info->dev_name); + ret = vm_snapshot_user_dev(info, data_fd, xop, meta, &offset); if (ret != 0) goto snapshot_err; + + ptr_registered_devs = ptr_registered_devs->next_dev; } xo_close_list_h(xop, JSON_DEV_ARR_KEY); @@ -1327,6 +1351,18 @@ pthread_cond_broadcast(&vcpus_can_run); } +void +free_snapshot_array() { + struct vm_snapshot_registered_devs *ptr_registered_devs; + + while(head_registered_devs != NULL) { + ptr_registered_devs = head_registered_devs; + head_registered_devs = head_registered_devs->next_dev; + free(ptr_registered_devs->dev_info); + free(ptr_registered_devs); + } +} + static int vm_checkpoint(struct vmctx *ctx, char *checkpoint_file, bool stop_vm) { @@ -1392,7 +1428,7 @@ perror("Could not write guest memory to file"); error = -1; goto done; - } + } ret = vm_snapshot_basic_metadata(ctx, xop, memsz); if (ret != 0) { @@ -1401,17 +1437,16 @@ goto done; } - - ret = vm_snapshot_kern_structs(ctx, kdata_fd, xop); + ret = vm_snapshot_user_devs(ctx, kdata_fd, xop); if (ret != 0) { - fprintf(stderr, "Failed to snapshot vm kernel data.\n"); + fprintf(stderr, "Failed to snapshot device state.\n"); error = -1; goto done; } - ret = vm_snapshot_user_devs(ctx, kdata_fd, xop); + ret = vm_snapshot_kern_structs(ctx, kdata_fd, xop); if (ret != 0) { - fprintf(stderr, "Failed to snapshot device state.\n"); + fprintf(stderr, "Failed to snapshot vm kernel data.\n"); error = -1; goto done; } @@ -1419,6 +1454,7 @@ xo_finish_h(xop); if (stop_vm) { + free_snapshot_array(); vm_destroy(ctx); exit(0); }