diff --git a/sys/dev/vmware/vmci/vmci_doorbell.c b/sys/dev/vmware/vmci/vmci_doorbell.c index 6bccd0fcc646..746fc91f1507 100644 --- a/sys/dev/vmware/vmci/vmci_doorbell.c +++ b/sys/dev/vmware/vmci/vmci_doorbell.c @@ -1,908 +1,904 @@ /*- * Copyright (c) 2018 VMware, Inc. * * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) */ /* This file implements the VMCI doorbell API. */ #include __FBSDID("$FreeBSD$"); #include #include "vmci_doorbell.h" #include "vmci_driver.h" #include "vmci_kernel_api.h" #include "vmci_kernel_defs.h" #include "vmci_resource.h" #include "vmci_utils.h" #define LGPFX "vmci_doorbell: " #define VMCI_DOORBELL_INDEX_TABLE_SIZE 64 #define VMCI_DOORBELL_HASH(_idx) \ vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE) /* Describes a doorbell notification handle allocated by the host. */ struct vmci_doorbell_entry { struct vmci_resource resource; uint32_t idx; vmci_list_item(vmci_doorbell_entry) idx_list_item; vmci_privilege_flags priv_flags; bool is_doorbell; bool run_delayed; vmci_callback notify_cb; void *client_data; vmci_event destroy_event; volatile int active; }; struct vmci_doorbell_index_table { vmci_lock lock; vmci_list(vmci_doorbell_entry) entries[VMCI_DOORBELL_INDEX_TABLE_SIZE]; }; /* The VMCI index table keeps track of currently registered doorbells. */ static struct vmci_doorbell_index_table vmci_doorbell_it; /* * The max_notify_idx is one larger than the currently known bitmap index in * use, and is used to determine how much of the bitmap needs to be scanned. */ static uint32_t max_notify_idx; /* * The notify_idx_count is used for determining whether there are free entries * within the bitmap (if notify_idx_count + 1 < max_notify_idx). */ static uint32_t notify_idx_count; /* * The last_notify_idx_reserved is used to track the last index handed out - in * the case where multiple handles share a notification index, we hand out * indexes round robin based on last_notify_idx_reserved. */ static uint32_t last_notify_idx_reserved; /* This is a one entry cache used to by the index allocation. */ static uint32_t last_notify_idx_released = PAGE_SIZE; static void vmci_doorbell_free_cb(void *client_data); static int vmci_doorbell_release_cb(void *client_data); static void vmci_doorbell_delayed_dispatch_cb(void *data); /* *------------------------------------------------------------------------------ * * vmci_doorbell_init -- * * General init code. * * Result: * VMCI_SUCCESS on success, lock allocation error otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_doorbell_init(void) { uint32_t bucket; for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries); ++bucket) vmci_list_init(&vmci_doorbell_it.entries[bucket]); return (vmci_init_lock(&vmci_doorbell_it.lock, "VMCI Doorbell index table lock")); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_exit -- * * General exit code. * * Result: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_doorbell_exit(void) { vmci_cleanup_lock(&vmci_doorbell_it.lock); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_free_cb -- * * Callback to free doorbell entry structure when resource is no longer used, * i.e. the reference count reached 0. The entry is freed in * vmci_doorbell_destroy(), which is waiting on the signal that gets fired * here. * * Result: * None. * * Side effects: * Signals VMCI event. * *------------------------------------------------------------------------------ */ static void vmci_doorbell_free_cb(void *client_data) { struct vmci_doorbell_entry *entry; entry = (struct vmci_doorbell_entry *)client_data; ASSERT(entry); vmci_signal_event(&entry->destroy_event); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_release_cb -- * * Callback to release the resource reference. It is called by the * vmci_wait_on_event function before it blocks. * * Result: * Always 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static int vmci_doorbell_release_cb(void *client_data) { struct vmci_doorbell_entry *entry; entry = (struct vmci_doorbell_entry *)client_data; ASSERT(entry); vmci_resource_release(&entry->resource); return (0); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_get_priv_flags -- * * Utility function that retrieves the privilege flags associated with a * given doorbell handle. For guest endpoints, the privileges are determined * by the context ID, but for host endpoints privileges are associated with * the complete handle. Hypervisor endpoints are not yet supported. * * Result: * VMCI_SUCCESS on success, * VMCI_ERROR_NOT_FOUND if handle isn't found, * VMCI_ERROR_INVALID_ARGS if handle is invalid. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_doorbell_get_priv_flags(struct vmci_handle handle, vmci_privilege_flags *priv_flags) { if (priv_flags == NULL || handle.context == VMCI_INVALID_ID) return (VMCI_ERROR_INVALID_ARGS); if (handle.context == VMCI_HOST_CONTEXT_ID) { struct vmci_doorbell_entry *entry; struct vmci_resource *resource; resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL); if (resource == NULL) return (VMCI_ERROR_NOT_FOUND); entry = RESOURCE_CONTAINER( resource, struct vmci_doorbell_entry, resource); *priv_flags = entry->priv_flags; vmci_resource_release(resource); } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) { /* Hypervisor endpoints for notifications are not supported. */ return (VMCI_ERROR_INVALID_ARGS); } else *priv_flags = VMCI_NO_PRIVILEGE_FLAGS; return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_index_table_find -- * * Find doorbell entry by bitmap index. * * Results: * Entry if found, NULL if not. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static struct vmci_doorbell_entry * vmci_doorbell_index_table_find(uint32_t idx) { struct vmci_doorbell_entry *iter; uint32_t bucket; bucket = VMCI_DOORBELL_HASH(idx); vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) { if (idx == iter->idx) return (iter); } return (NULL); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_index_table_add -- * * Add the given entry to the index table. This will hold() the entry's * resource so that the entry is not deleted before it is removed from the * table. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static void vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry) { uint32_t bucket; uint32_t new_notify_idx; ASSERT(entry); vmci_resource_hold(&entry->resource); vmci_grab_lock_bh(&vmci_doorbell_it.lock); /* * Below we try to allocate an index in the notification bitmap with * "not too much" sharing between resources. If we use less that the * full bitmap, we either add to the end if there are no unused flags * within the currently used area, or we search for unused ones. If we * use the full bitmap, we allocate the index round robin. */ if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) { if (last_notify_idx_released < max_notify_idx && !vmci_doorbell_index_table_find(last_notify_idx_released)) { new_notify_idx = last_notify_idx_released; last_notify_idx_released = PAGE_SIZE; } else { bool reused = false; new_notify_idx = last_notify_idx_reserved; if (notify_idx_count + 1 < max_notify_idx) { do { if (!vmci_doorbell_index_table_find( new_notify_idx)) { reused = true; break; } new_notify_idx = (new_notify_idx + 1) % max_notify_idx; } while (new_notify_idx != last_notify_idx_released); } if (!reused) { new_notify_idx = max_notify_idx; max_notify_idx++; } } } else { new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE; } last_notify_idx_reserved = new_notify_idx; notify_idx_count++; entry->idx = new_notify_idx; bucket = VMCI_DOORBELL_HASH(entry->idx); vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry, idx_list_item); vmci_release_lock_bh(&vmci_doorbell_it.lock); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_index_table_remove -- * * Remove the given entry from the index table. This will release() the * entry's resource. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static void vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry) { ASSERT(entry); vmci_grab_lock_bh(&vmci_doorbell_it.lock); vmci_list_remove(entry, idx_list_item); notify_idx_count--; if (entry->idx == max_notify_idx - 1) { /* * If we delete an entry with the maximum known notification * index, we take the opportunity to prune the current max. As * there might be other unused indices immediately below, we * lower the maximum until we hit an index in use */ while (max_notify_idx > 0 && !vmci_doorbell_index_table_find(max_notify_idx - 1)) max_notify_idx--; } last_notify_idx_released = entry->idx; vmci_release_lock_bh(&vmci_doorbell_it.lock); vmci_resource_release(&entry->resource); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_link -- * * Creates a link between the given doorbell handle and the given index in * the bitmap in the device backend. * * Results: * VMCI_SUCCESS if success, appropriate error code otherwise. * * Side effects: * Notification state is created in hypervisor. * *------------------------------------------------------------------------------ */ static int vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell, uint32_t notify_idx) { struct vmci_doorbell_link_msg link_msg; vmci_id resource_id; ASSERT(!VMCI_HANDLE_INVALID(handle)); if (is_doorbell) resource_id = VMCI_DOORBELL_LINK; else { ASSERT(false); return (VMCI_ERROR_UNAVAILABLE); } link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, resource_id); link_msg.hdr.src = VMCI_ANON_SRC_HANDLE; link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE; link_msg.handle = handle; link_msg.notify_idx = notify_idx; return (vmci_send_datagram((struct vmci_datagram *)&link_msg)); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_unlink -- * * Unlinks the given doorbell handle from an index in the bitmap in the * device backend. * * Results: * VMCI_SUCCESS if success, appropriate error code otherwise. * * Side effects: * Notification state is destroyed in hypervisor. * *------------------------------------------------------------------------------ */ static int vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell) { struct vmci_doorbell_unlink_msg unlink_msg; vmci_id resource_id; ASSERT(!VMCI_HANDLE_INVALID(handle)); if (is_doorbell) resource_id = VMCI_DOORBELL_UNLINK; else { ASSERT(false); return (VMCI_ERROR_UNAVAILABLE); } unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, resource_id); unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE; unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE; unlink_msg.handle = handle; return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg)); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_create -- * * Creates a doorbell with the given callback. If the handle is * VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The * callback can be run immediately (potentially with locks held - the * default) or delayed (in a kernel thread) by specifying the flag * VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback * may not be run if the kernel is unable to allocate memory for the delayed * execution (highly unlikely). * * Results: * VMCI_SUCCESS on success, appropriate error code otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags, vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data) { struct vmci_doorbell_entry *entry; struct vmci_handle new_handle; int result; if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB || priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS) return (VMCI_ERROR_INVALID_ARGS); entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL); if (entry == NULL) { VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram " "entry.\n"); return (VMCI_ERROR_NO_MEM); } if (!vmci_can_schedule_delayed_work() && (flags & VMCI_FLAG_DELAYED_CB)) { result = VMCI_ERROR_INVALID_ARGS; goto free_mem; } if (VMCI_HANDLE_INVALID(*handle)) { vmci_id context_id; context_id = vmci_get_context_id(); vmci_id resource_id = vmci_resource_get_id(context_id); if (resource_id == VMCI_INVALID_ID) { result = VMCI_ERROR_NO_HANDLE; goto free_mem; } new_handle = VMCI_MAKE_HANDLE(context_id, resource_id); } else { if (VMCI_INVALID_ID == handle->resource) { VMCI_LOG_DEBUG(LGPFX"Invalid argument " "(handle=0x%x:0x%x).\n", handle->context, handle->resource); result = VMCI_ERROR_INVALID_ARGS; goto free_mem; } new_handle = *handle; } entry->idx = 0; entry->priv_flags = priv_flags; entry->is_doorbell = true; entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false; entry->notify_cb = notify_cb; entry->client_data = client_data; atomic_store_int(&entry->active, 0); vmci_create_event(&entry->destroy_event); result = vmci_resource_add(&entry->resource, VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb, entry); if (result != VMCI_SUCCESS) { VMCI_LOG_WARNING(LGPFX"Failed to add new resource " "(handle=0x%x:0x%x).\n", new_handle.context, new_handle.resource); if (result == VMCI_ERROR_DUPLICATE_ENTRY) result = VMCI_ERROR_ALREADY_EXISTS; goto destroy; } vmci_doorbell_index_table_add(entry); result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx); if (VMCI_SUCCESS != result) goto destroy_resource; atomic_store_int(&entry->active, 1); if (VMCI_HANDLE_INVALID(*handle)) *handle = new_handle; return (result); destroy_resource: vmci_doorbell_index_table_remove(entry); vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL); destroy: vmci_destroy_event(&entry->destroy_event); free_mem: vmci_free_kernel_mem(entry, sizeof(*entry)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_destroy -- * * Destroys a doorbell previously created with vmci_doorbell_create. This * operation may block waiting for a callback to finish. * * Results: * VMCI_SUCCESS on success, appropriate error code otherwise. * * Side effects: * May block. * *------------------------------------------------------------------------------ */ int vmci_doorbell_destroy(struct vmci_handle handle) { struct vmci_doorbell_entry *entry; struct vmci_resource *resource; int result; if (VMCI_HANDLE_INVALID(handle)) return (VMCI_ERROR_INVALID_ARGS); resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL); if (resource == NULL) { VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell " "(handle=0x%x:0x%x).\n", handle.context, handle.resource); return (VMCI_ERROR_NOT_FOUND); } entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry, resource); vmci_doorbell_index_table_remove(entry); result = vmci_doorbell_unlink(handle, entry->is_doorbell); if (VMCI_SUCCESS != result) { /* * The only reason this should fail would be an inconsistency * between guest and hypervisor state, where the guest believes * it has an active registration whereas the hypervisor doesn't. * One case where this may happen is if a doorbell is * unregistered following a hibernation at a time where the * doorbell state hasn't been restored on the hypervisor side * yet. Since the handle has now been removed in the guest, * we just print a warning and return success. */ VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown " "by hypervisor (error=%d).\n", entry->is_doorbell ? "doorbell" : "queuepair", handle.context, handle.resource, result); } /* * Now remove the resource from the table. It might still be in use * after this, in a callback or still on the delayed work queue. */ vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL); /* * We now wait on the destroyEvent and release the reference we got * above. */ vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb, entry); /* * We know that we are now the only reference to the above entry so * can safely free it. */ vmci_destroy_event(&entry->destroy_event); vmci_free_kernel_mem(entry, sizeof(*entry)); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_notify_as_guest -- * * Notify another guest or the host. We send a datagram down to the host * via the hypervisor with the notification info. * * Results: * VMCI_SUCCESS on success, appropriate error code otherwise. * * Side effects: * May do a hypercall. * *------------------------------------------------------------------------------ */ static int vmci_doorbell_notify_as_guest(struct vmci_handle handle, vmci_privilege_flags priv_flags) { struct vmci_doorbell_notify_msg notify_msg; notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, VMCI_DOORBELL_NOTIFY); notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE; notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE; notify_msg.handle = handle; return (vmci_send_datagram((struct vmci_datagram *)¬ify_msg)); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_notify -- * * Generates a notification on the doorbell identified by the handle. For * host side generation of notifications, the caller can specify what the * privilege of the calling side is. * * Results: * VMCI_SUCCESS on success, appropriate error code otherwise. * * Side effects: * May do a hypercall. * *------------------------------------------------------------------------------ */ int vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags) { - struct vmci_handle src; - if (VMCI_HANDLE_INVALID(dst) || (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)) return (VMCI_ERROR_INVALID_ARGS); - src = VMCI_INVALID_HANDLE; - return (vmci_doorbell_notify_as_guest(dst, priv_flags)); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_delayed_dispatch_cb -- * * Calls the specified callback in a delayed context. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static void vmci_doorbell_delayed_dispatch_cb(void *data) { struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data; ASSERT(data); entry->notify_cb(entry->client_data); vmci_resource_release(&entry->resource); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_sync -- * * Use this as a synchronization point when setting globals, for example, * during device shutdown. * * Results: * None. * * Side effects: * None. * *------------------------------------------------------------------------------ */ void vmci_doorbell_sync(void) { vmci_grab_lock_bh(&vmci_doorbell_it.lock); vmci_release_lock_bh(&vmci_doorbell_it.lock); vmci_resource_sync(); } /* *------------------------------------------------------------------------------ * * vmci_register_notification_bitmap -- * * Register the notification bitmap with the host. * * Results: * true if the bitmap is registered successfully with the device, false * otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ bool vmci_register_notification_bitmap(PPN bitmap_ppn) { struct vmci_notify_bitmap_set_msg bitmap_set_msg; int result; /* * Do not ASSERT() on the guest device here. This function can get * called during device initialization, so the ASSERT() will fail even * though the device is (almost) up. */ bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID, VMCI_SET_NOTIFY_BITMAP); bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE; bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE; bitmap_set_msg.bitmap_ppn = bitmap_ppn; result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg); if (result != VMCI_SUCCESS) { VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as " "notification bitmap (error=%d).\n", bitmap_ppn, result); return (false); } return (true); } /* *------------------------------------------------------------------------------ * * vmci_doorbell_fire_entries -- * * Executes or schedules the handlers for a given notify index. * * Result: * Notification hash entry if found. NULL otherwise. * * Side effects: * Whatever the side effects of the handlers are. * *------------------------------------------------------------------------------ */ static void vmci_doorbell_fire_entries(uint32_t notify_idx) { struct vmci_doorbell_entry *iter; uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx); vmci_grab_lock_bh(&vmci_doorbell_it.lock); vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) { if (iter->idx == notify_idx && atomic_load_int(&iter->active) == 1) { ASSERT(iter->notify_cb); if (iter->run_delayed) { int err; vmci_resource_hold(&iter->resource); err = vmci_schedule_delayed_work( vmci_doorbell_delayed_dispatch_cb, iter); if (err != VMCI_SUCCESS) { vmci_resource_release(&iter->resource); goto out; } } else iter->notify_cb(iter->client_data); } } out: vmci_release_lock_bh(&vmci_doorbell_it.lock); } /* *------------------------------------------------------------------------------ * * vmci_scan_notification_bitmap -- * * Scans the notification bitmap, collects pending notifications, resets * the bitmap and invokes appropriate callbacks. * * Results: * None. * * Side effects: * May schedule tasks, allocate memory and run callbacks. * *------------------------------------------------------------------------------ */ void vmci_scan_notification_bitmap(uint8_t *bitmap) { uint32_t idx; ASSERT(bitmap); for (idx = 0; idx < max_notify_idx; idx++) { if (bitmap[idx] & 0x1) { bitmap[idx] &= ~1; vmci_doorbell_fire_entries(idx); } } } diff --git a/sys/dev/vmware/vmci/vmci_qpair.c b/sys/dev/vmware/vmci/vmci_qpair.c index acaef7256f5d..336dc9eb342b 100644 --- a/sys/dev/vmware/vmci/vmci_qpair.c +++ b/sys/dev/vmware/vmci/vmci_qpair.c @@ -1,837 +1,832 @@ /*- * Copyright (c) 2018 VMware, Inc. * * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) */ /* This file implements Queue accessor methods. */ /* * vmci_qpair is an interface that hides the queue pair internals. Rather than * access each queue in a pair directly, operations are performed on the queue * as a whole. This is simpler and less error-prone, and allows for future * queue pair features to be added under the hood with no change to the client * code. */ #include __FBSDID("$FreeBSD$"); #include "vmci_kernel_api.h" #include "vmci_kernel_defs.h" #include "vmci_kernel_if.h" #include "vmci_queue.h" #include "vmci_queue_pair.h" /* This structure is opaque to the clients. */ struct vmci_qpair { struct vmci_handle handle; struct vmci_queue *produce_q; struct vmci_queue *consume_q; uint64_t produce_q_size; uint64_t consume_q_size; vmci_id peer; uint32_t flags; vmci_privilege_flags priv_flags; uint32_t blocked; vmci_event event; }; static void vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair, struct vmci_queue_header **produce_q_header, struct vmci_queue_header **consume_q_header); /* *------------------------------------------------------------------------------ * * vmci_queue_add_producer_tail -- * * Helper routine to increment the Producer Tail. * * Results: * VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot * be found. Otherwise VMCI_SUCCESS. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static inline int vmci_queue_add_producer_tail(struct vmci_queue *queue, size_t add, uint64_t queue_size) { vmci_queue_header_add_producer_tail(queue->q_header, add, queue_size); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_queue_add_consumer_head -- * * Helper routine to increment the Consumer Head. * * Results: * VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot * be found. Otherwise VMCI_SUCCESS. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static inline int vmci_queue_add_consumer_head(struct vmci_queue *queue, size_t add, uint64_t queue_size) { vmci_queue_header_add_consumer_head(queue->q_header, add, queue_size); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_qpair_get_queue_headers -- * * Helper routine that will retrieve the produce and consume headers of a * given queue pair. * * Results: * VMCI_SUCCESS if either current or saved queue headers are found. * Appropriate error code otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ static void vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair, struct vmci_queue_header **produce_q_header, struct vmci_queue_header **consume_q_header) { ASSERT((qpair->produce_q != NULL) && (qpair->consume_q != NULL)); *produce_q_header = qpair->produce_q->q_header; *consume_q_header = qpair->consume_q->q_header; } /* *------------------------------------------------------------------------------ * * vmci_qpair_alloc -- * * This is the client interface for allocating the memory for a vmci_qpair * structure and then attaching to the underlying queue. If an error occurs * allocating the memory for the vmci_qpair structure, no attempt is made to * attach. If an error occurs attaching, then there's the vmci_qpair * structure is freed. * * Results: * An err, if < 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_qpair_alloc(struct vmci_qpair **qpair, struct vmci_handle *handle, uint64_t produce_q_size, uint64_t consume_q_size, vmci_id peer, uint32_t flags, vmci_privilege_flags priv_flags) { struct vmci_qpair *my_qpair; - vmci_event_release_cb wakeup_cb; - void *client_data; int retval; /* * Restrict the size of a queuepair. Though the device enforces a limit * on the total amount of memory that can be allocated to queuepairs for * a guest, we avoid unnecessarily allocating a lot of memory. Also, we * try to allocate this memory before we make the queuepair allocation * hypercall. * * (Note that this doesn't prevent all cases; a user with only this much * physical memory could still get into trouble.) The error used by the * device is NO_RESOURCES, so use that here too. */ if (produce_q_size + consume_q_size < MAX(produce_q_size, consume_q_size) || produce_q_size + consume_q_size > VMCI_MAX_GUEST_QP_MEMORY) return (VMCI_ERROR_NO_RESOURCES); if (flags & VMCI_QPFLAG_NONBLOCK) return (VMCI_ERROR_INVALID_ARGS); my_qpair = vmci_alloc_kernel_mem(sizeof(*my_qpair), VMCI_MEMORY_NORMAL); if (!my_qpair) return (VMCI_ERROR_NO_MEM); my_qpair->produce_q_size = produce_q_size; my_qpair->consume_q_size = consume_q_size; my_qpair->peer = peer; my_qpair->flags = flags; my_qpair->priv_flags = priv_flags; - client_data = NULL; - wakeup_cb = NULL; - retval = vmci_queue_pair_alloc(handle, &my_qpair->produce_q, my_qpair->produce_q_size, &my_qpair->consume_q, my_qpair->consume_q_size, my_qpair->peer, my_qpair->flags, my_qpair->priv_flags); if (retval < VMCI_SUCCESS) { vmci_free_kernel_mem(my_qpair, sizeof(*my_qpair)); return (retval); } *qpair = my_qpair; my_qpair->handle = *handle; return (retval); } /* *------------------------------------------------------------------------------ * * vmci_qpair_detach -- * * This is the client interface for detaching from a vmci_qpair. Note that * this routine will free the memory allocated for the vmci_qpair structure, * too. * * Results: * An error, if < 0. * * Side effects: * Will clear the caller's pointer to the vmci_qpair structure. * *------------------------------------------------------------------------------ */ int vmci_qpair_detach(struct vmci_qpair **qpair) { struct vmci_qpair *old_qpair; int result; if (!qpair || !(*qpair)) return (VMCI_ERROR_INVALID_ARGS); old_qpair = *qpair; result = vmci_queue_pair_detach(old_qpair->handle); /* * The guest can fail to detach for a number of reasons, and if it does * so, it will cleanup the entry (if there is one). We need to release * the qpair struct here; there isn't much the caller can do, and we * don't want to leak. */ if (old_qpair->flags & VMCI_QPFLAG_LOCAL) vmci_destroy_event(&old_qpair->event); vmci_free_kernel_mem(old_qpair, sizeof(*old_qpair)); *qpair = NULL; return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_get_produce_indexes -- * * This is the client interface for getting the current indexes of the * qpair from the point of the view of the caller as the producer. * * Results: * err, if < 0 * Success otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_qpair_get_produce_indexes(const struct vmci_qpair *qpair, uint64_t *producer_tail, uint64_t *consumer_head) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); vmci_queue_header_get_pointers(produce_q_header, consume_q_header, producer_tail, consumer_head); if ((producer_tail && *producer_tail >= qpair->produce_q_size) || (consumer_head && *consumer_head >= qpair->produce_q_size)) return (VMCI_ERROR_INVALID_SIZE); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_qpair_get_consume_indexes -- * * This is the client interface for getting the current indexes of the * QPair from the point of the view of the caller as the consumer. * * Results: * err, if < 0 * Success otherwise. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int vmci_qpair_get_consume_indexes(const struct vmci_qpair *qpair, uint64_t *consumer_tail, uint64_t *producer_head) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); vmci_queue_header_get_pointers(consume_q_header, produce_q_header, consumer_tail, producer_head); if ((consumer_tail && *consumer_tail >= qpair->consume_q_size) || (producer_head && *producer_head >= qpair->consume_q_size)) return (VMCI_ERROR_INVALID_SIZE); return (VMCI_SUCCESS); } /* *------------------------------------------------------------------------------ * * vmci_qpair_produce_free_space -- * * This is the client interface for getting the amount of free space in the * QPair from the point of the view of the caller as the producer which is * the common case. * * Results: * Err, if < 0. * Full queue if = 0. * Number of available bytes into which data can be enqueued if > 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int64_t vmci_qpair_produce_free_space(const struct vmci_qpair *qpair) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; int64_t result; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); result = vmci_queue_header_free_space(produce_q_header, consume_q_header, qpair->produce_q_size); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_consume_free_space -- * * This is the client interface for getting the amount of free space in the * QPair from the point of the view of the caller as the consumer which is * not the common case (see vmci_qpair_Produce_free_space(), above). * * Results: * Err, if < 0. * Full queue if = 0. * Number of available bytes into which data can be enqueued if > 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int64_t vmci_qpair_consume_free_space(const struct vmci_qpair *qpair) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; int64_t result; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); result = vmci_queue_header_free_space(consume_q_header, produce_q_header, qpair->consume_q_size); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_produce_buf_ready -- * * This is the client interface for getting the amount of enqueued data in * the QPair from the point of the view of the caller as the producer which * is not the common case (see vmci_qpair_Consume_buf_ready(), above). * * Results: * Err, if < 0. * Empty queue if = 0. * Number of bytes ready to be dequeued if > 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int64_t vmci_qpair_produce_buf_ready(const struct vmci_qpair *qpair) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; int64_t result; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); result = vmci_queue_header_buf_ready(produce_q_header, consume_q_header, qpair->produce_q_size); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_consume_buf_ready -- * * This is the client interface for getting the amount of enqueued data in * the QPair from the point of the view of the caller as the consumer which * is the normal case. * * Results: * Err, if < 0. * Empty queue if = 0. * Number of bytes ready to be dequeued if > 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ int64_t vmci_qpair_consume_buf_ready(const struct vmci_qpair *qpair) { struct vmci_queue_header *consume_q_header; struct vmci_queue_header *produce_q_header; int64_t result; if (!qpair) return (VMCI_ERROR_INVALID_ARGS); vmci_qpair_get_queue_headers(qpair, &produce_q_header, &consume_q_header); result = vmci_queue_header_buf_ready(consume_q_header, produce_q_header, qpair->consume_q_size); return (result); } /* *------------------------------------------------------------------------------ * * enqueue -- * * Enqueues a given buffer to the produce queue using the provided function. * As many bytes as possible (space available in the queue) are enqueued. * * Results: * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue data. * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue * (as defined by the queue size). * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer. * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't * available. * Otherwise, the number of bytes written to the queue is returned. * * Side effects: * Updates the tail pointer of the produce queue. * *------------------------------------------------------------------------------ */ static ssize_t enqueue(struct vmci_queue *produce_q, struct vmci_queue *consume_q, const uint64_t produce_q_size, const void *buf, size_t buf_size, int buf_type, vmci_memcpy_to_queue_func memcpy_to_queue, bool can_block) { ssize_t result; size_t written; int64_t free_space; uint64_t tail; ASSERT((produce_q != NULL) && (consume_q != NULL)); free_space = vmci_queue_header_free_space(produce_q->q_header, consume_q->q_header, produce_q_size); if (free_space == 0) return (VMCI_ERROR_QUEUEPAIR_NOSPACE); if (free_space < VMCI_SUCCESS) return ((ssize_t)free_space); written = (size_t)(free_space > buf_size ? buf_size : free_space); tail = vmci_queue_header_producer_tail(produce_q->q_header); if (LIKELY(tail + written < produce_q_size)) result = memcpy_to_queue(produce_q, tail, buf, 0, written, buf_type, can_block); else { /* Tail pointer wraps around. */ const size_t tmp = (size_t)(produce_q_size - tail); result = memcpy_to_queue(produce_q, tail, buf, 0, tmp, buf_type, can_block); if (result >= VMCI_SUCCESS) result = memcpy_to_queue(produce_q, 0, buf, tmp, written - tmp, buf_type, can_block); } if (result < VMCI_SUCCESS) return (result); result = vmci_queue_add_producer_tail(produce_q, written, produce_q_size); if (result < VMCI_SUCCESS) return (result); return (written); } /* *------------------------------------------------------------------------------ * * dequeue -- * * Dequeues data (if available) from the given consume queue. Writes data * to the user provided buffer using the provided function. * * Results: * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue. * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue * (as defined by the queue size). * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer. * VMCI_ERROR_NOT_FOUND, if the vmm_world registered with the queue pair * cannot be found. * Otherwise the number of bytes dequeued is returned. * * Side effects: * Updates the head pointer of the consume queue. * *------------------------------------------------------------------------------ */ static ssize_t dequeue(struct vmci_queue *produce_q, struct vmci_queue *consume_q, const uint64_t consume_q_size, void *buf, size_t buf_size, int buf_type, vmci_memcpy_from_queue_func memcpy_from_queue, bool update_consumer, bool can_block) { ssize_t result; size_t read; int64_t buf_ready; uint64_t head; ASSERT((produce_q != NULL) && (consume_q != NULL)); buf_ready = vmci_queue_header_buf_ready(consume_q->q_header, produce_q->q_header, consume_q_size); if (buf_ready == 0) return (VMCI_ERROR_QUEUEPAIR_NODATA); if (buf_ready < VMCI_SUCCESS) return ((ssize_t)buf_ready); read = (size_t)(buf_ready > buf_size ? buf_size : buf_ready); head = vmci_queue_header_consumer_head(produce_q->q_header); if (LIKELY(head + read < consume_q_size)) result = memcpy_from_queue(buf, 0, consume_q, head, read, buf_type, can_block); else { /* Head pointer wraps around. */ const size_t tmp = (size_t)(consume_q_size - head); result = memcpy_from_queue(buf, 0, consume_q, head, tmp, buf_type, can_block); if (result >= VMCI_SUCCESS) result = memcpy_from_queue(buf, tmp, consume_q, 0, read - tmp, buf_type, can_block); } if (result < VMCI_SUCCESS) return (result); if (update_consumer) { result = vmci_queue_add_consumer_head(produce_q, read, consume_q_size); if (result < VMCI_SUCCESS) return (result); } return (read); } /* *------------------------------------------------------------------------------ * * vmci_qpair_enqueue -- * * This is the client interface for enqueueing data into the queue. * * Results: * Err, if < 0. * Number of bytes enqueued if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_enqueue(struct vmci_qpair *qpair, const void *buf, size_t buf_size, int buf_type) { ssize_t result; if (!qpair || !buf) return (VMCI_ERROR_INVALID_ARGS); result = enqueue(qpair->produce_q, qpair->consume_q, qpair->produce_q_size, buf, buf_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_to_queue_local : vmci_memcpy_to_queue, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_dequeue -- * * This is the client interface for dequeueing data from the queue. * * Results: * Err, if < 0. * Number of bytes dequeued if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_dequeue(struct vmci_qpair *qpair, void *buf, size_t buf_size, int buf_type) { ssize_t result; if (!qpair || !buf) return (VMCI_ERROR_INVALID_ARGS); result = dequeue(qpair->produce_q, qpair->consume_q, qpair->consume_q_size, buf, buf_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, true, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_peek -- * * This is the client interface for peeking into a queue. (I.e., copy * data from the queue without updating the head pointer.) * * Results: * Err, if < 0. * Number of bytes peeked, if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_peek(struct vmci_qpair *qpair, void *buf, size_t buf_size, int buf_type) { ssize_t result; if (!qpair || !buf) return (VMCI_ERROR_INVALID_ARGS); result = dequeue(qpair->produce_q, qpair->consume_q, qpair->consume_q_size, buf, buf_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, false, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_enquev -- * * This is the client interface for enqueueing data into the queue. * * Results: * Err, if < 0. * Number of bytes enqueued if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_enquev(struct vmci_qpair *qpair, void *iov, size_t iov_size, int buf_type) { ssize_t result; if (!qpair || !iov) return (VMCI_ERROR_INVALID_ARGS); result = enqueue(qpair->produce_q, qpair->consume_q, qpair->produce_q_size, iov, iov_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_to_queue_v_local : vmci_memcpy_to_queue_v, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_dequev -- * * This is the client interface for dequeueing data from the queue. * * Results: * Err, if < 0. * Number of bytes dequeued if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_dequev(struct vmci_qpair *qpair, void *iov, size_t iov_size, int buf_type) { ssize_t result; if (!qpair || !iov) return (VMCI_ERROR_INVALID_ARGS); result = dequeue(qpair->produce_q, qpair->consume_q, qpair->consume_q_size, iov, iov_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, true, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); } /* *------------------------------------------------------------------------------ * * vmci_qpair_peekv -- * * This is the client interface for peeking into a queue. (I.e., copy * data from the queue without updating the head pointer.) * * Results: * Err, if < 0. * Number of bytes peeked, if >= 0. * * Side effects: * None. * *------------------------------------------------------------------------------ */ ssize_t vmci_qpair_peekv(struct vmci_qpair *qpair, void *iov, size_t iov_size, int buf_type) { ssize_t result; if (!qpair || !iov) return (VMCI_ERROR_INVALID_ARGS); result = dequeue(qpair->produce_q, qpair->consume_q, qpair->consume_q_size, iov, iov_size, buf_type, qpair->flags & VMCI_QPFLAG_LOCAL? vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, false, !(qpair->flags & VMCI_QPFLAG_NONBLOCK)); return (result); }