Differential D7314 Diff 18785 head/sys/dev/hyperv/vmbus/hv_ring_buffer.c

Changeset View

Standalone View

head/sys/dev/hyperv/vmbus/hv_ring_buffer.c

Show First 20 Lines • Show All 138 Lines • ▼ Show 20 Lines
vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr)		vmbus_rxbr_intr_unmask(struct vmbus_rxbr *rbr)
{		{
rbr->rxbr_imask = 0;		rbr->rxbr_imask = 0;
mb();		mb();

/*		/*
* Now check to see if the ring buffer is still empty.		* Now check to see if the ring buffer is still empty.
* If it is not, we raced and we need to process new		* If it is not, we raced and we need to process new
* incoming messages.		* incoming channel packets.
*/		*/
return vmbus_rxbr_avail(rbr);		return vmbus_rxbr_avail(rbr);
}		}

/*		/*
* When we write to the ring buffer, check if the host needs to		* When we write to the ring buffer, check if the host needs to be
* be signaled. Here is the details of this protocol:		* signaled.
*		*
* 1. The host guarantees that while it is draining the		* The contract:
* ring buffer, it will set the interrupt_mask to		* - The host guarantees that while it is draining the TX bufring,
* indicate it does not need to be interrupted when		* it will set the br_imask to indicate it does not need to be
* new data is placed.		* interrupted when new data are added.
*		* - The host guarantees that it will completely drain the TX bufring
* 2. The host guarantees that it will completely drain		* before exiting the read loop. Further, once the TX bufring is
* the ring buffer before exiting the read loop. Further,		* empty, it will clear the br_imask and re-check to see if new
* once the ring buffer is empty, it will clear the		* data have arrived.
* interrupt_mask and re-check to see if new data has
* arrived.
*/		*/
static boolean_t		static boolean_t
hv_ring_buffer_needsig_on_write(uint32_t old_write_location,		hv_ring_buffer_needsig_on_write(uint32_t old_write_location,
const struct vmbus_txbr *tbr)		const struct vmbus_txbr *tbr)
{		{
mb();		mb();
if (tbr->txbr_imask)		if (tbr->txbr_imask)
return (FALSE);		return (FALSE);

		/* XXX only compiler fence is needed */
/* Read memory barrier */		/* Read memory barrier */
rmb();		rmb();

/*		/*
* This is the only case we need to signal when the		* This is the only case we need to signal when the
* ring transitions from being empty to non-empty.		* ring transitions from being empty to non-empty.
*/		*/
if (old_write_location == tbr->txbr_rindex)		if (old_write_location == tbr->txbr_rindex)
return (TRUE);		return (TRUE);

return (FALSE);		return (FALSE);
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines	vmbus_txbr_write(struct vmbus_txbr *tbr, const struct iovec iov[], int iovlen,
uint32_t byte_avail_to_write;		uint32_t byte_avail_to_write;
uint32_t old_write_location;		uint32_t old_write_location;
uint32_t total_bytes_to_write = 0;		uint32_t total_bytes_to_write = 0;
volatile uint32_t next_write_location;		volatile uint32_t next_write_location;
uint64_t prev_indices = 0;		uint64_t prev_indices = 0;

for (i = 0; i < iovlen; i++)		for (i = 0; i < iovlen; i++)
total_bytes_to_write += iov[i].iov_len;		total_bytes_to_write += iov[i].iov_len;

total_bytes_to_write += sizeof(uint64_t);		total_bytes_to_write += sizeof(uint64_t);

mtx_lock_spin(&tbr->txbr_lock);		mtx_lock_spin(&tbr->txbr_lock);

byte_avail_to_write = vmbus_txbr_avail(tbr);		byte_avail_to_write = vmbus_txbr_avail(tbr);

/*		/*
* If there is only room for the packet, assume it is full.		* NOTE:
* Otherwise, the next time around, we think the ring buffer		* If this write is going to make br_windex same as br_rindex,
* is empty since the read index == write index		* i.e. the available space for write is same as the write size,
		* we can't do it then, since br_windex == br_rindex means that
		* the bufring is empty.
*/		*/
if (byte_avail_to_write <= total_bytes_to_write) {		if (byte_avail_to_write <= total_bytes_to_write) {
mtx_unlock_spin(&tbr->txbr_lock);		mtx_unlock_spin(&tbr->txbr_lock);
return (EAGAIN);		return (EAGAIN);
}		}

/*		/*
* Write to the ring buffer		* Copy the scattered channel packet to the TX bufring.
*/		*/
next_write_location = tbr->txbr_windex;		next_write_location = tbr->txbr_windex;

old_write_location = next_write_location;		old_write_location = next_write_location;

for (i = 0; i < iovlen; i++) {		for (i = 0; i < iovlen; i++) {
next_write_location = copy_to_ring_buffer(tbr,		next_write_location = copy_to_ring_buffer(tbr,
next_write_location, iov[i].iov_base, iov[i].iov_len);		next_write_location, iov[i].iov_base, iov[i].iov_len);
}		}

/*		/*
* Set previous packet start		* Set the offset of the current channel packet.
*/		*/
prev_indices = ((uint64_t)tbr->txbr_windex) << 32;		prev_indices = ((uint64_t)tbr->txbr_windex) << 32;

next_write_location = copy_to_ring_buffer(tbr,		next_write_location = copy_to_ring_buffer(tbr,
next_write_location, (char *)&prev_indices, sizeof(uint64_t));		next_write_location, (char *)&prev_indices, sizeof(uint64_t));

/*		/*
		* XXX only compiler fence is needed.
* Full memory barrier before upding the write index.		* Full memory barrier before upding the write index.
*/		*/
mb();		mb();

/*		/*
* Now, update the write location		* Now, update the write index.
*/		*/
tbr->txbr_windex = next_write_location;		tbr->txbr_windex = next_write_location;

mtx_unlock_spin(&tbr->txbr_lock);		mtx_unlock_spin(&tbr->txbr_lock);

*need_sig = hv_ring_buffer_needsig_on_write(old_write_location, tbr);		*need_sig = hv_ring_buffer_needsig_on_write(old_write_location, tbr);

return (0);		return (0);
}		}

int		int
vmbus_rxbr_peek(struct vmbus_rxbr rbr, void data, int dlen)		vmbus_rxbr_peek(struct vmbus_rxbr rbr, void data, int dlen)
{		{
uint32_t bytesAvailToRead;		uint32_t bytesAvailToRead;
uint32_t nextReadLocation = 0;		uint32_t nextReadLocation = 0;

mtx_lock_spin(&rbr->rxbr_lock);		mtx_lock_spin(&rbr->rxbr_lock);

bytesAvailToRead = vmbus_rxbr_avail(rbr);		bytesAvailToRead = vmbus_rxbr_avail(rbr);

/*
* Make sure there is something to read
*/
if (bytesAvailToRead < dlen) {		if (bytesAvailToRead < dlen) {
mtx_unlock_spin(&rbr->rxbr_lock);		mtx_unlock_spin(&rbr->rxbr_lock);
return (EAGAIN);		return (EAGAIN);
}		}

/*
* Convert to byte offset
*/
nextReadLocation = rbr->rxbr_rindex;		nextReadLocation = rbr->rxbr_rindex;

nextReadLocation = copy_from_ring_buffer(rbr, data, dlen,		nextReadLocation = copy_from_ring_buffer(rbr, data, dlen,
nextReadLocation);		nextReadLocation);

mtx_unlock_spin(&rbr->rxbr_lock);		mtx_unlock_spin(&rbr->rxbr_lock);

return (0);		return (0);
}		}

int		int
vmbus_rxbr_read(struct vmbus_rxbr rbr, void data, int dlen, uint32_t offset)		vmbus_rxbr_read(struct vmbus_rxbr rbr, void data, int dlen, uint32_t offset)
{		{
uint32_t bytes_avail_to_read;		uint32_t bytes_avail_to_read;
uint32_t next_read_location = 0;		uint32_t next_read_location = 0;
uint64_t prev_indices = 0;		uint64_t prev_indices = 0;

KASSERT(dlen > 0, ("invalid dlen %d", dlen));		KASSERT(dlen > 0, ("invalid dlen %d", dlen));

mtx_lock_spin(&rbr->rxbr_lock);		mtx_lock_spin(&rbr->rxbr_lock);

bytes_avail_to_read = vmbus_rxbr_avail(rbr);		bytes_avail_to_read = vmbus_rxbr_avail(rbr);

/*
* Make sure there is something to read
*/
if (bytes_avail_to_read < dlen) {		if (bytes_avail_to_read < dlen) {
mtx_unlock_spin(&rbr->rxbr_lock);		mtx_unlock_spin(&rbr->rxbr_lock);
return (EAGAIN);		return (EAGAIN);
}		}

		/*
		* Copy channel packet from RX bufring.
		*/
next_read_location = (rbr->rxbr_rindex + offset) % rbr->rxbr_dsize;		next_read_location = (rbr->rxbr_rindex + offset) % rbr->rxbr_dsize;

next_read_location = copy_from_ring_buffer(rbr, data, dlen,		next_read_location = copy_from_ring_buffer(rbr, data, dlen,
next_read_location);		next_read_location);

		/*
		* Discard this channel packet's start offset, which is useless
		* for us.
		*/
next_read_location = copy_from_ring_buffer(rbr,		next_read_location = copy_from_ring_buffer(rbr,
(char *)&prev_indices, sizeof(uint64_t), next_read_location);		(char *)&prev_indices, sizeof(uint64_t), next_read_location);

/*		/*
		* XXX only compiler fence is needed.
* Make sure all reads are done before we update the read index since		* Make sure all reads are done before we update the read index since
* the writer may start writing to the read area once the read index		* the writer may start writing to the read area once the read index
* is updated.		* is updated.
*/		*/
wmb();		wmb();

/*		/*
* Update the read index		* Update the read index
*/		*/
rbr->rxbr_rindex = next_read_location;		rbr->rxbr_rindex = next_read_location;

mtx_unlock_spin(&rbr->rxbr_lock);		mtx_unlock_spin(&rbr->rxbr_lock);

return (0);		return (0);
}		}

/**
* @brief Helper routine to copy from source to ring buffer.
*
* Assume there is enough room. Handles wrap-around in dest case only!
*/
static uint32_t		static uint32_t
copy_to_ring_buffer(const struct vmbus_txbr *tbr,		copy_to_ring_buffer(const struct vmbus_txbr *tbr,
uint32_t start_write_offset, const uint8_t *src, uint32_t src_len)		uint32_t start_write_offset, const uint8_t *src, uint32_t src_len)
{		{
char *ring_buffer = tbr->txbr_data;		char *ring_buffer = tbr->txbr_data;
uint32_t ring_buffer_size = tbr->txbr_dsize;		uint32_t ring_buffer_size = tbr->txbr_dsize;
uint32_t fragLen;		uint32_t fragLen;

if (src_len > ring_buffer_size - start_write_offset) {		if (src_len > ring_buffer_size - start_write_offset) {
/* wrap-around detected! */		/* Wrap-around detected! */
fragLen = ring_buffer_size - start_write_offset;		fragLen = ring_buffer_size - start_write_offset;
memcpy(ring_buffer + start_write_offset, src, fragLen);		memcpy(ring_buffer + start_write_offset, src, fragLen);
memcpy(ring_buffer, src + fragLen, src_len - fragLen);		memcpy(ring_buffer, src + fragLen, src_len - fragLen);
} else {		} else {
memcpy(ring_buffer + start_write_offset, src, src_len);		memcpy(ring_buffer + start_write_offset, src, src_len);
}		}

start_write_offset += src_len;		start_write_offset += src_len;
start_write_offset %= ring_buffer_size;		start_write_offset %= ring_buffer_size;

return (start_write_offset);		return (start_write_offset);
}		}

/**
* @brief Helper routine to copy to source from ring buffer.
*
* Assume there is enough room. Handles wrap-around in src case only!
*/
static uint32_t		static uint32_t
copy_from_ring_buffer(const struct vmbus_rxbr rbr, char dest,		copy_from_ring_buffer(const struct vmbus_rxbr rbr, char dest,
uint32_t dest_len, uint32_t start_read_offset)		uint32_t dest_len, uint32_t start_read_offset)
{		{
uint32_t fragLen;		uint32_t fragLen;
char *ring_buffer = rbr->rxbr_data;		char *ring_buffer = rbr->rxbr_data;
uint32_t ring_buffer_size = rbr->rxbr_dsize;		uint32_t ring_buffer_size = rbr->rxbr_dsize;

if (dest_len > ring_buffer_size - start_read_offset) {		if (dest_len > ring_buffer_size - start_read_offset) {
/* wrap-around detected at the src */		/* Wrap-around detected at the src */
fragLen = ring_buffer_size - start_read_offset;		fragLen = ring_buffer_size - start_read_offset;
memcpy(dest, ring_buffer + start_read_offset, fragLen);		memcpy(dest, ring_buffer + start_read_offset, fragLen);
memcpy(dest + fragLen, ring_buffer, dest_len - fragLen);		memcpy(dest + fragLen, ring_buffer, dest_len - fragLen);
} else {		} else {
memcpy(dest, ring_buffer + start_read_offset, dest_len);		memcpy(dest, ring_buffer + start_read_offset, dest_len);
}		}

start_read_offset += dest_len;		start_read_offset += dest_len;
start_read_offset %= ring_buffer_size;		start_read_offset %= ring_buffer_size;

return (start_read_offset);		return (start_read_offset);
}		}