Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/vmware/vmci/vmci_kernel_if.c
- This file was added.
Property | Old Value | New Value |
---|---|---|
svn:eol-style | null | native \ No newline at end of property |
svn:keywords | null | FreeBSD=%H \ No newline at end of property |
svn:mime-type | null | text/plain \ No newline at end of property |
/*- | |||||
* Copyright (c) 2018 VMware, Inc. All Rights Reserved. | |||||
* | |||||
* SPDX-License-Identifier: (BSD-2-Clause AND GPL-2.0) | |||||
*/ | |||||
/* This file implements defines and helper functions. */ | |||||
#include <sys/malloc.h> | |||||
#include <sys/proc.h> | |||||
#include <sys/uio.h> | |||||
#include <machine/bus.h> | |||||
#include "vmci.h" | |||||
#include "vmci_defs.h" | |||||
#include "vmci_kernel_defs.h" | |||||
#include "vmci_kernel_if.h" | |||||
#include "vmci_queue.h" | |||||
struct vmci_queue_kernel_if { | |||||
size_t num_pages; /* Num pages incl. header. */ | |||||
struct vmci_dma_alloc *dmas; /* For dma alloc. */ | |||||
}; | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_init_lock | |||||
* | |||||
* Initializes the lock. Must be called before use. | |||||
* | |||||
* Results: | |||||
* Always VMCI_SUCCESS. | |||||
* | |||||
* Side effects: | |||||
* Thread can block. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_init_lock(vmci_lock *lock, char *name) | |||||
{ | |||||
mtx_init(lock, name, NULL, MTX_DEF | MTX_NOWITNESS); | |||||
return (VMCI_SUCCESS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_cleanup_lock | |||||
* | |||||
* Cleanup the lock. Must be called before deallocating lock. | |||||
* | |||||
* Results: | |||||
* None | |||||
* | |||||
* Side effects: | |||||
* Deletes kernel lock state | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_cleanup_lock(vmci_lock *lock) | |||||
{ | |||||
mtx_destroy(lock); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_grab_lock | |||||
* | |||||
* Grabs the given lock. | |||||
* | |||||
* Results: | |||||
* None | |||||
* | |||||
* Side effects: | |||||
* Thread can block. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_grab_lock(vmci_lock *lock) | |||||
{ | |||||
mtx_lock(lock); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_release_lock | |||||
* | |||||
* Releases the given lock. | |||||
* | |||||
* Results: | |||||
* None | |||||
* | |||||
* Side effects: | |||||
* A thread blocked on this lock may wake up. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_release_lock(vmci_lock *lock) | |||||
{ | |||||
mtx_unlock(lock); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_grab_lock_bh | |||||
* | |||||
* Grabs the given lock. | |||||
* | |||||
* Results: | |||||
* None | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_grab_lock_bh(vmci_lock *lock) | |||||
{ | |||||
mtx_lock(lock); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_release_lock_bh | |||||
* | |||||
* Releases the given lock. | |||||
* | |||||
* Results: | |||||
* None | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_release_lock_bh(vmci_lock *lock) | |||||
{ | |||||
mtx_unlock(lock); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_alloc_kernel_mem | |||||
* | |||||
* Allocate physically contiguous memory for the VMCI driver. | |||||
* | |||||
* Results: | |||||
* The address allocated or NULL on error. | |||||
* | |||||
* | |||||
* Side effects: | |||||
* Memory may be allocated. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void * | |||||
vmci_alloc_kernel_mem(size_t size, int flags) | |||||
{ | |||||
void *ptr; | |||||
if ((flags & VMCI_MEMORY_ATOMIC) != 0) | |||||
ptr = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, 0xFFFFFFFF, | |||||
8, 1024 * 1024); | |||||
else | |||||
ptr = contigmalloc(size, M_DEVBUF, M_WAITOK, 0, 0xFFFFFFFF, | |||||
8, 1024 * 1024); | |||||
return (ptr); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_free_kernel_mem | |||||
* | |||||
* Free kernel memory allocated for the VMCI driver. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* Memory is freed. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_free_kernel_mem(void *ptr, size_t size) | |||||
{ | |||||
contigfree(ptr, size, M_DEVBUF); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_can_schedule_delayed_work -- | |||||
* | |||||
* Checks to see if the given platform supports delayed work callbacks. | |||||
* | |||||
* Results: | |||||
* true if it does. false otherwise. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
bool | |||||
vmci_can_schedule_delayed_work(void) | |||||
{ | |||||
return (true); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_schedule_delayed_work -- | |||||
* | |||||
* Schedule the specified callback. | |||||
* | |||||
* Results: | |||||
* Zero on success, error code otherwise. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_schedule_delayed_work(vmci_work_fn *work_fn, void *data) | |||||
{ | |||||
return (vmci_schedule_delayed_work_fn(work_fn, data)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_create_event -- | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_create_event(vmci_event *event) | |||||
{ | |||||
sema_init(event, 0, "vmci_event"); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_destroy_event -- | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_destroy_event(vmci_event *event) | |||||
{ | |||||
if (mtx_owned(&event->sema_mtx)) | |||||
sema_destroy(event); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_signal_event -- | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_signal_event(vmci_event *event) | |||||
{ | |||||
sema_post(event); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_wait_on_event -- | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_wait_on_event(vmci_event *event, vmci_event_release_cb release_cb, | |||||
void *client_data) | |||||
{ | |||||
release_cb(client_data); | |||||
sema_wait(event); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_mutex_init -- | |||||
* | |||||
* Initializes the mutex. Must be called before use. | |||||
* | |||||
* Results: | |||||
* Success. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_mutex_init(vmci_mutex *mutex, char *name) | |||||
{ | |||||
mtx_init(mutex, name, NULL, MTX_DEF | MTX_NOWITNESS); | |||||
return (VMCI_SUCCESS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_mutex_destroy -- | |||||
* | |||||
* Destroys the mutex. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_mutex_destroy(vmci_mutex *mutex) | |||||
{ | |||||
mtx_destroy(mutex); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_mutex_acquire -- | |||||
* | |||||
* Acquires the mutex. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* Thread may block. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_mutex_acquire(vmci_mutex *mutex) | |||||
{ | |||||
mtx_lock(mutex); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_mutex_release -- | |||||
* | |||||
* Releases the mutex. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* May wake up the thread blocking on this mutex. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_mutex_release(vmci_mutex *mutex) | |||||
{ | |||||
mtx_unlock(mutex); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_alloc_queue -- | |||||
* | |||||
* Allocates kernel queue pages of specified size with IOMMU mappings, plus | |||||
* space for the queue structure/kernel interface and the queue header. | |||||
* | |||||
* Results: | |||||
* Pointer to the queue on success, NULL otherwise. | |||||
* | |||||
* Side effects: | |||||
* Memory is allocated. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void * | |||||
vmci_alloc_queue(uint64_t size, uint32_t flags) | |||||
{ | |||||
struct vmci_queue *queue; | |||||
size_t i; | |||||
const size_t num_pages = CEILING(size, PAGE_SIZE) + 1; | |||||
const size_t dmas_size = num_pages * sizeof(struct vmci_dma_alloc); | |||||
const size_t queue_size = | |||||
sizeof(*queue) + sizeof(*(queue->kernel_if)) + dmas_size; | |||||
/* Size should be enforced by vmci_qpair_alloc(), double-check here. */ | |||||
if (size > VMCI_MAX_GUEST_QP_MEMORY) { | |||||
ASSERT(false); | |||||
return (NULL); | |||||
} | |||||
queue = malloc(queue_size, M_DEVBUF, M_NOWAIT); | |||||
if (!queue) | |||||
return (NULL); | |||||
queue->q_header = NULL; | |||||
queue->saved_header = NULL; | |||||
queue->kernel_if = (struct vmci_queue_kernel_if *)(queue + 1); | |||||
queue->kernel_if->num_pages = num_pages; | |||||
queue->kernel_if->dmas = (struct vmci_dma_alloc *)(queue->kernel_if + | |||||
1); | |||||
for (i = 0; i < num_pages; i++) { | |||||
vmci_dma_malloc(PAGE_SIZE, 1, &queue->kernel_if->dmas[i]); | |||||
if (!queue->kernel_if->dmas[i].dma_vaddr) { | |||||
/* Size excl. the header. */ | |||||
vmci_free_queue(queue, i * PAGE_SIZE); | |||||
return (NULL); | |||||
} | |||||
} | |||||
/* Queue header is the first page. */ | |||||
queue->q_header = (void *)queue->kernel_if->dmas[0].dma_vaddr; | |||||
return ((void *)queue); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_free_queue -- | |||||
* | |||||
* Frees kernel VA space for a given queue and its queue header, and frees | |||||
* physical data pages. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* Memory is freed. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_free_queue(void *q, uint64_t size) | |||||
{ | |||||
struct vmci_queue *queue = q; | |||||
if (queue) { | |||||
const size_t num_pages = CEILING(size, PAGE_SIZE) + 1; | |||||
uint64_t i; | |||||
/* Given size doesn't include header, so add in a page here. */ | |||||
for (i = 0; i < num_pages; i++) | |||||
vmci_dma_free(&queue->kernel_if->dmas[i]); | |||||
free(queue, M_DEVBUF); | |||||
} | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_alloc_ppn_set -- | |||||
* | |||||
* Allocates two list of PPNs --- one for the pages in the produce queue, | |||||
* and the other for the pages in the consume queue. Intializes the list of | |||||
* PPNs with the page frame numbers of the KVA for the two queues (and the | |||||
* queue headers). | |||||
* | |||||
* Results: | |||||
* Success or failure. | |||||
* | |||||
* Side effects: | |||||
* Memory may be allocated. | |||||
* | |||||
*----------------------------------------------------------------------------- | |||||
*/ | |||||
int | |||||
vmci_alloc_ppn_set(void *prod_q, uint64_t num_produce_pages, void *cons_q, | |||||
uint64_t num_consume_pages, struct ppn_set *ppn_set) | |||||
{ | |||||
struct vmci_queue *consume_q = cons_q; | |||||
struct vmci_queue *produce_q = prod_q; | |||||
vmci_ppn_list consume_ppns; | |||||
vmci_ppn_list produce_ppns; | |||||
uint64_t i; | |||||
if (!produce_q || !num_produce_pages || !consume_q || | |||||
!num_consume_pages || !ppn_set) | |||||
return (VMCI_ERROR_INVALID_ARGS); | |||||
if (ppn_set->initialized) | |||||
return (VMCI_ERROR_ALREADY_EXISTS); | |||||
produce_ppns = | |||||
vmci_alloc_kernel_mem(num_produce_pages * sizeof(*produce_ppns), | |||||
VMCI_MEMORY_NORMAL); | |||||
if (!produce_ppns) | |||||
return (VMCI_ERROR_NO_MEM); | |||||
consume_ppns = | |||||
vmci_alloc_kernel_mem(num_consume_pages * sizeof(*consume_ppns), | |||||
VMCI_MEMORY_NORMAL); | |||||
if (!consume_ppns) { | |||||
vmci_free_kernel_mem(produce_ppns, | |||||
num_produce_pages * sizeof(*produce_ppns)); | |||||
return (VMCI_ERROR_NO_MEM); | |||||
} | |||||
for (i = 0; i < num_produce_pages; i++) { | |||||
unsigned long pfn; | |||||
produce_ppns[i] = | |||||
pfn = produce_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT; | |||||
/* | |||||
* Fail allocation if PFN isn't supported by hypervisor. | |||||
*/ | |||||
if (sizeof(pfn) > | |||||
sizeof(*produce_ppns) && pfn != produce_ppns[i]) | |||||
goto ppn_error; | |||||
} | |||||
for (i = 0; i < num_consume_pages; i++) { | |||||
unsigned long pfn; | |||||
consume_ppns[i] = | |||||
pfn = consume_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT; | |||||
/* | |||||
* Fail allocation if PFN isn't supported by hypervisor. | |||||
*/ | |||||
if (sizeof(pfn) > | |||||
sizeof(*consume_ppns) && pfn != consume_ppns[i]) | |||||
goto ppn_error; | |||||
} | |||||
ppn_set->num_produce_pages = num_produce_pages; | |||||
ppn_set->num_consume_pages = num_consume_pages; | |||||
ppn_set->produce_ppns = produce_ppns; | |||||
ppn_set->consume_ppns = consume_ppns; | |||||
ppn_set->initialized = true; | |||||
return (VMCI_SUCCESS); | |||||
ppn_error: | |||||
vmci_free_kernel_mem(produce_ppns, num_produce_pages * | |||||
sizeof(*produce_ppns)); | |||||
vmci_free_kernel_mem(consume_ppns, num_consume_pages * | |||||
sizeof(*consume_ppns)); | |||||
return (VMCI_ERROR_INVALID_ARGS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_free_ppn_set -- | |||||
* | |||||
* Frees the two list of PPNs for a queue pair. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_free_ppn_set(struct ppn_set *ppn_set) | |||||
{ | |||||
ASSERT(ppn_set); | |||||
if (ppn_set->initialized) { | |||||
/* Do not call these functions on NULL inputs. */ | |||||
ASSERT(ppn_set->produce_ppns && ppn_set->consume_ppns); | |||||
vmci_free_kernel_mem(ppn_set->produce_ppns, | |||||
ppn_set->num_produce_pages * | |||||
sizeof(*ppn_set->produce_ppns)); | |||||
vmci_free_kernel_mem(ppn_set->consume_ppns, | |||||
ppn_set->num_consume_pages * | |||||
sizeof(*ppn_set->consume_ppns)); | |||||
} | |||||
memset(ppn_set, 0, sizeof(*ppn_set)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_populate_ppn_list -- | |||||
* | |||||
* Populates the list of PPNs in the hypercall structure with the PPNS | |||||
* of the produce queue and the consume queue. | |||||
* | |||||
* Results: | |||||
* VMCI_SUCCESS. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_populate_ppn_list(uint8_t *call_buf, const struct ppn_set *ppn_set) | |||||
{ | |||||
ASSERT(call_buf && ppn_set && ppn_set->initialized); | |||||
memcpy(call_buf, ppn_set->produce_ppns, | |||||
ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns)); | |||||
memcpy(call_buf + ppn_set->num_produce_pages * | |||||
sizeof(*ppn_set->produce_ppns), ppn_set->consume_ppns, | |||||
ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns)); | |||||
return (VMCI_SUCCESS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_{to,from}iovec -- | |||||
* | |||||
* These helper routines will copy the specified bytes to/from memory that's | |||||
* specified as a struct iovec. The routines can not verify the correctness | |||||
* of the struct iovec's contents. | |||||
* | |||||
* Results: | |||||
* None. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
static inline void | |||||
vmci_memcpy_toiovec(struct iovec *iov, uint8_t *src, size_t len) | |||||
{ | |||||
while (len > 0) { | |||||
if (iov->iov_len) { | |||||
size_t to_copy = MIN(iov->iov_len, len); | |||||
memcpy(iov->iov_base, src, to_copy); | |||||
src += to_copy; | |||||
len -= to_copy; | |||||
iov->iov_base = (void *)((uintptr_t) iov->iov_base + | |||||
to_copy); | |||||
iov->iov_len -= to_copy; | |||||
} | |||||
iov++; | |||||
} | |||||
} | |||||
static inline void | |||||
vmci_memcpy_fromiovec(uint8_t *dst, struct iovec *iov, size_t len) | |||||
{ | |||||
while (len > 0) { | |||||
if (iov->iov_len) { | |||||
size_t to_copy = MIN(iov->iov_len, len); | |||||
memcpy(dst, iov->iov_base, to_copy); | |||||
dst += to_copy; | |||||
len -= to_copy; | |||||
iov->iov_base = (void *)((uintptr_t) iov->iov_base + | |||||
to_copy); | |||||
iov->iov_len -= to_copy; | |||||
} | |||||
iov++; | |||||
} | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* __vmci_memcpy_to_queue -- | |||||
* | |||||
* Copies from a given buffer or iovector to a VMCI Queue. Assumes that | |||||
* offset + size does not wrap around in the queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
#pragma GCC diagnostic ignored "-Wcast-qual" | |||||
static int | |||||
__vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset, | |||||
const void *src, size_t size, bool is_iovec) | |||||
{ | |||||
struct vmci_queue_kernel_if *kernel_if = queue->kernel_if; | |||||
size_t bytes_copied = 0; | |||||
while (bytes_copied < size) { | |||||
const uint64_t page_index = | |||||
(queue_offset + bytes_copied) / PAGE_SIZE; | |||||
const size_t page_offset = | |||||
(queue_offset + bytes_copied) & (PAGE_SIZE - 1); | |||||
void *va; | |||||
size_t to_copy; | |||||
/* Skip header. */ | |||||
va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr; | |||||
ASSERT(va); | |||||
/* | |||||
* Fill up the page if we have enough payload, or else | |||||
* copy the remaining bytes. | |||||
*/ | |||||
to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied); | |||||
if (is_iovec) { | |||||
struct iovec *iov = (struct iovec *)src; | |||||
/* The iovec will track bytes_copied internally. */ | |||||
vmci_memcpy_fromiovec((uint8_t *)va + page_offset, | |||||
iov, to_copy); | |||||
} else | |||||
memcpy((uint8_t *)va + page_offset, | |||||
(uint8_t *)src + bytes_copied, to_copy); | |||||
bytes_copied += to_copy; | |||||
} | |||||
return (VMCI_SUCCESS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* __vmci_memcpy_from_queue -- | |||||
* | |||||
* Copies to a given buffer or iovector from a VMCI Queue. Assumes that | |||||
* offset + size does not wrap around in the queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
static int | |||||
__vmci_memcpy_from_queue(void *dest, const struct vmci_queue *queue, | |||||
uint64_t queue_offset, size_t size, bool is_iovec) | |||||
{ | |||||
struct vmci_queue_kernel_if *kernel_if = queue->kernel_if; | |||||
size_t bytes_copied = 0; | |||||
while (bytes_copied < size) { | |||||
const uint64_t page_index = | |||||
(queue_offset + bytes_copied) / PAGE_SIZE; | |||||
const size_t page_offset = | |||||
(queue_offset + bytes_copied) & (PAGE_SIZE - 1); | |||||
void *va; | |||||
size_t to_copy; | |||||
/* Skip header. */ | |||||
va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr; | |||||
ASSERT(va); | |||||
/* | |||||
* Fill up the page if we have enough payload, or else | |||||
* copy the remaining bytes. | |||||
*/ | |||||
to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied); | |||||
if (is_iovec) { | |||||
struct iovec *iov = (struct iovec *)dest; | |||||
/* The iovec will track bytesCopied internally. */ | |||||
vmci_memcpy_toiovec(iov, (uint8_t *)va + | |||||
page_offset, to_copy); | |||||
} else | |||||
memcpy((uint8_t *)dest + bytes_copied, | |||||
(uint8_t *)va + page_offset, to_copy); | |||||
bytes_copied += to_copy; | |||||
} | |||||
return (VMCI_SUCCESS); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_to_queue -- | |||||
* | |||||
* Copies from a given buffer to a VMCI Queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset, | |||||
const void *src, size_t src_offset, size_t size, int buf_type, | |||||
bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
return (__vmci_memcpy_to_queue(queue, queue_offset, | |||||
(uint8_t *)src + src_offset, size, false)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_from_queue -- | |||||
* | |||||
* Copies to a given buffer from a VMCI Queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_from_queue(void *dest, size_t dest_offset, | |||||
const struct vmci_queue *queue, uint64_t queue_offset, size_t size, | |||||
int buf_type, bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset, | |||||
queue, queue_offset, size, false)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_to_queue_local -- | |||||
* | |||||
* Copies from a given buffer to a local VMCI queue. This is the | |||||
* same as a regular copy. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_to_queue_local(struct vmci_queue *queue, uint64_t queue_offset, | |||||
const void *src, size_t src_offset, size_t size, int buf_type, | |||||
bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
return (__vmci_memcpy_to_queue(queue, queue_offset, | |||||
(uint8_t *)src + src_offset, size, false)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_from_queue_local -- | |||||
* | |||||
* Copies to a given buffer from a VMCI Queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_from_queue_local(void *dest, size_t dest_offset, | |||||
const struct vmci_queue *queue, uint64_t queue_offset, size_t size, | |||||
int buf_type, bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset, | |||||
queue, queue_offset, size, false)); | |||||
} | |||||
/*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_to_queue_v -- | |||||
* | |||||
* Copies from a given iovec from a VMCI Queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_to_queue_v(struct vmci_queue *queue, uint64_t queue_offset, | |||||
const void *src, size_t src_offset, size_t size, int buf_type, | |||||
bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
/* | |||||
* We ignore src_offset because src is really a struct iovec * and will | |||||
* maintain offset internally. | |||||
*/ | |||||
return (__vmci_memcpy_to_queue(queue, queue_offset, src, size, | |||||
true)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_memcpy_from_queue_v -- | |||||
* | |||||
* Copies to a given iovec from a VMCI Queue. | |||||
* | |||||
* Results: | |||||
* Zero on success, negative error code on failure. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
int | |||||
vmci_memcpy_from_queue_v(void *dest, size_t dest_offset, | |||||
const struct vmci_queue *queue, uint64_t queue_offset, size_t size, | |||||
int buf_type, bool can_block) | |||||
{ | |||||
ASSERT(can_block); | |||||
/* | |||||
* We ignore dest_offset because dest is really a struct iovec * and | |||||
* will maintain offset internally. | |||||
*/ | |||||
return (__vmci_memcpy_from_queue(dest, queue, queue_offset, size, | |||||
true)); | |||||
} | |||||
/* | |||||
*------------------------------------------------------------------------------ | |||||
* | |||||
* vmci_read_port_bytes -- | |||||
* | |||||
* Copy memory from an I/O port to kernel memory. | |||||
* | |||||
* Results: | |||||
* No results. | |||||
* | |||||
* Side effects: | |||||
* None. | |||||
* | |||||
*------------------------------------------------------------------------------ | |||||
*/ | |||||
void | |||||
vmci_read_port_bytes(vmci_io_handle handle, vmci_io_port port, uint8_t *buffer, | |||||
size_t buffer_length) | |||||
{ | |||||
insb(port, buffer, buffer_length); | |||||
} |