Differential D37878 Diff 165091 sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c

Changeset View

Standalone View

sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c

Show First 20 Lines • Show All 411 Lines • ▼ Show 20 Lines	vchiq_set_conn_state(VCHIQ_STATE_T *state, VCHIQ_CONNSTATE_T newstate)
VCHIQ_CONNSTATE_T oldstate = state->conn_state;		VCHIQ_CONNSTATE_T oldstate = state->conn_state;
vchiq_log_info(vchiq_core_log_level, "%d: %s->%s", state->id,		vchiq_log_info(vchiq_core_log_level, "%d: %s->%s", state->id,
conn_state_names[oldstate],		conn_state_names[oldstate],
conn_state_names[newstate]);		conn_state_names[newstate]);
state->conn_state = newstate;		state->conn_state = newstate;
vchiq_platform_conn_state_changed(state, oldstate, newstate);		vchiq_platform_conn_state_changed(state, oldstate, newstate);
}		}

		#define ACTUAL_EVENT_SEM_ADDR(ref,offset)\
		((struct semaphore *)(((size_t) ref) + ((size_t) offset)))
static inline void		static inline void
remote_event_create(REMOTE_EVENT_T *event)		remote_event_create(VCHIQ_STATE_T ref, REMOTE_EVENT_T event)
{		{
event->armed = 0;		event->armed = 0;
/* Don't clear the 'fired' flag because it may already have been set		/* Don't clear the 'fired' flag because it may already have been set
** by the other side. */		** by the other side. */
_sema_init(event->event, 0);		_sema_init(ACTUAL_EVENT_SEM_ADDR(ref,event->event), 0);
}		}

__unused static inline void		__unused static inline void
remote_event_destroy(REMOTE_EVENT_T *event)		remote_event_destroy(REMOTE_EVENT_T *event)
{		{
(void)event;		(void)event;
}		}

static inline int		static inline int
remote_event_wait(REMOTE_EVENT_T *event)		remote_event_wait(VCHIQ_STATE_T ref, REMOTE_EVENT_T event)
{		{
if (!event->fired) {		if (!event->fired) {
event->armed = 1;		event->armed = 1;
		#if defined(__aarch64__)
		dsb(sy);
		#else
dsb();		dsb();
		#endif

if (!event->fired) {		if (!event->fired) {
if (down_interruptible(event->event) != 0) {		if (down_interruptible(ACTUAL_EVENT_SEM_ADDR(ref,event->event)) != 0) {
event->armed = 0;		event->armed = 0;
return 0;		return 0;
}		}
}		}
event->armed = 0;		event->armed = 0;
wmb();		wmb();
}		}

event->fired = 0;		event->fired = 0;
return 1;		return 1;
}		}

static inline void		static inline void
remote_event_signal_local(REMOTE_EVENT_T *event)		remote_event_signal_local(VCHIQ_STATE_T ref, REMOTE_EVENT_T event)
{		{
		/*
		* Mirror
		* https://github.com/raspberrypi/linux/commit/a50c4c9a65779ca835746b5fd79d3d5278afbdbe
		* for extra safety
		*/
		event->fired = 1;
event->armed = 0;		event->armed = 0;
up(event->event);		up(ACTUAL_EVENT_SEM_ADDR(ref,event->event));
}		}

static inline void		static inline void
remote_event_poll(REMOTE_EVENT_T *event)		remote_event_poll(VCHIQ_STATE_T ref, REMOTE_EVENT_T event)
{		{
if (event->fired && event->armed)		if (event->fired && event->armed)
remote_event_signal_local(event);		remote_event_signal_local(ref,event);
}		}

void		void
remote_event_pollall(VCHIQ_STATE_T *state)		remote_event_pollall(VCHIQ_STATE_T *state)
{		{
remote_event_poll(&state->local->sync_trigger);		remote_event_poll(state , &state->local->sync_trigger);
remote_event_poll(&state->local->sync_release);		remote_event_poll(state , &state->local->sync_release);
remote_event_poll(&state->local->trigger);		remote_event_poll(state , &state->local->trigger);
remote_event_poll(&state->local->recycle);		remote_event_poll(state , &state->local->recycle);
}		}

/* Round up message sizes so that any space at the end of a slot is always big		/* Round up message sizes so that any space at the end of a slot is always big
** enough for a header. This relies on header size being a power of two, which		** enough for a header. This relies on header size being a power of two, which
** has been verified earlier by a static assertion. */		** has been verified earlier by a static assertion. */

static inline unsigned int		static inline unsigned int
calc_stride(unsigned int size)		calc_stride(unsigned int size)
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines	do {
value, value \| (1 << (service->localport & 0x1f)))		value, value \| (1 << (service->localport & 0x1f)))
!= value);		!= value);
}		}

state->poll_needed = 1;		state->poll_needed = 1;
wmb();		wmb();

/* ... and ensure the slot handler runs. */		/* ... and ensure the slot handler runs. */
remote_event_signal_local(&state->local->trigger);		remote_event_signal_local(state, &state->local->trigger);
}		}

/* Called from queue_message, by the slot handler and application threads,		/* Called from queue_message, by the slot handler and application threads,
** with slot_mutex held */		** with slot_mutex held */
static VCHIQ_HEADER_T *		static VCHIQ_HEADER_T *
reserve_space(VCHIQ_STATE_T *state, int space, int is_blocking)		reserve_space(VCHIQ_STATE_T *state, int space, int is_blocking)
{		{
VCHIQ_SHARED_STATE_T *local = state->local;		VCHIQ_SHARED_STATE_T *local = state->local;
▲ Show 20 Lines • Show All 446 Lines • ▼ Show 20 Lines	queue_message_sync(VCHIQ_STATE_T state, VCHIQ_SERVICE_T service,
VCHIQ_HEADER_T *header;		VCHIQ_HEADER_T *header;

local = state->local;		local = state->local;

if ((VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_RESUME) &&		if ((VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_RESUME) &&
(lmutex_lock_interruptible(&state->sync_mutex) != 0))		(lmutex_lock_interruptible(&state->sync_mutex) != 0))
return VCHIQ_RETRY;		return VCHIQ_RETRY;

remote_event_wait(&local->sync_release);		remote_event_wait(state, &local->sync_release);

rmb();		rmb();

header = (VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state,		header = (VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state,
local->slot_sync);		local->slot_sync);

{		{
int oldmsgid = header->msgid;		int oldmsgid = header->msgid;
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines	vchiq_log_trace(vchiq_sync_log_level,
msg_type_str(VCHIQ_MSG_TYPE(msgid)),		msg_type_str(VCHIQ_MSG_TYPE(msgid)),
VCHIQ_MSG_TYPE(msgid),		VCHIQ_MSG_TYPE(msgid),
VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),		VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
VCHIQ_MSG_SRCPORT(msgid),		VCHIQ_MSG_SRCPORT(msgid),
VCHIQ_MSG_DSTPORT(msgid),		VCHIQ_MSG_DSTPORT(msgid),
size);		size);
}		}

/* Make sure the new header is visible to the peer. */
wmb();
adrianUnsubmitted Not Done Inline Actions why are the wmb()s removed here and elsewhere? adrian: why are the wmb()s removed here and elsewhere?
devesas.campos_gmail.comAuthorUnsubmitted Done Inline Actions It's the first thing remote_event_signal does now— if I'm not mistaken all the calls to it were preceded by wmb. This is explained in the commit note:“refactor synchronization around remote_event_signal, forcing a wmb to be on the safe side; thereby make it look more like what linux does [ref:sync] (vchiq_{core,kmod}.c); and make a comment in vchiq_core.c true (wasn't before)” devesas.campos_gmail.com: It's the first thing remote_event_signal does now— if I'm not mistaken all the calls to it were…

remote_event_signal(&state->remote->sync_trigger);		remote_event_signal(&state->remote->sync_trigger);

if (VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_PAUSE)		if (VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_PAUSE)
lmutex_unlock(&state->sync_mutex);		lmutex_unlock(&state->sync_mutex);

return VCHIQ_SUCCESS;		return VCHIQ_SUCCESS;
}		}

▲ Show 20 Lines • Show All 708 Lines • ▼ Show 20 Lines	while (state->rx_pos != tx_pos) {
case VCHIQ_MSG_CONNECT:		case VCHIQ_MSG_CONNECT:
vchiq_log_info(vchiq_core_log_level,		vchiq_log_info(vchiq_core_log_level,
"%d: prs CONNECT@%p",		"%d: prs CONNECT@%p",
state->id, header);		state->id, header);
state->version_common = ((VCHIQ_SLOT_ZERO_T *)		state->version_common = ((VCHIQ_SLOT_ZERO_T *)
state->slot_data)->version;		state->slot_data)->version;
up(&state->connect);		up(&state->connect);
break;		break;
		/*
		* XXXMDC Apparently nothing uses this
		* https://github.com/raspberrypi/linux/commit/14f4d72fb799a9b3170a45ab80d4a3ddad541960
		* but taking out the master bits is a whole new job
		*/
case VCHIQ_MSG_BULK_RX:		case VCHIQ_MSG_BULK_RX:
case VCHIQ_MSG_BULK_TX: {		case VCHIQ_MSG_BULK_TX:
		WARN_ON(1);
		break;
		#if 0
		{
VCHIQ_BULK_QUEUE_T *queue;		VCHIQ_BULK_QUEUE_T *queue;
WARN_ON(!state->is_master);		WARN_ON(!state->is_master);
queue = (type == VCHIQ_MSG_BULK_RX) ?		queue = (type == VCHIQ_MSG_BULK_RX) ?
&service->bulk_tx : &service->bulk_rx;		&service->bulk_tx : &service->bulk_rx;
if ((service->remoteport == remoteport)		if ((service->remoteport == remoteport)
&& (service->srvstate ==		&& (service->srvstate ==
VCHIQ_SRVSTATE_OPEN)) {		VCHIQ_SRVSTATE_OPEN)) {
VCHIQ_BULK_T *bulk;		VCHIQ_BULK_T *bulk;
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines	#if 0
DEBUG_TRACE(PARSE_LINE);		DEBUG_TRACE(PARSE_LINE);
resolved = resolve_bulks(service,		resolved = resolve_bulks(service,
queue);		queue);
}		}

lmutex_unlock(&service->bulk_mutex);		lmutex_unlock(&service->bulk_mutex);
if (resolved)		if (resolved)
notify_bulks(service, queue,		notify_bulks(service, queue,
1/retry_poll/);		1//retry_poll
		);
}		}
} break;		}
		#endif
case VCHIQ_MSG_BULK_RX_DONE:		case VCHIQ_MSG_BULK_RX_DONE:
case VCHIQ_MSG_BULK_TX_DONE:		case VCHIQ_MSG_BULK_TX_DONE:
WARN_ON(state->is_master);		WARN_ON(state->is_master);
if ((service->remoteport == remoteport)		if ((service->remoteport == remoteport)
&& (service->srvstate !=		&& (service->srvstate !=
VCHIQ_SRVSTATE_FREE)) {		VCHIQ_SRVSTATE_FREE)) {
VCHIQ_BULK_QUEUE_T *queue;		VCHIQ_BULK_QUEUE_T *queue;
VCHIQ_BULK_T *bulk;		VCHIQ_BULK_T *bulk;
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines
{		{
VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;		VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;
VCHIQ_SHARED_STATE_T *local = state->local;		VCHIQ_SHARED_STATE_T *local = state->local;
DEBUG_INITIALISE(local)		DEBUG_INITIALISE(local)

while (1) {		while (1) {
DEBUG_COUNT(SLOT_HANDLER_COUNT);		DEBUG_COUNT(SLOT_HANDLER_COUNT);
DEBUG_TRACE(SLOT_HANDLER_LINE);		DEBUG_TRACE(SLOT_HANDLER_LINE);
remote_event_wait(&local->trigger);		remote_event_wait(state, &local->trigger);

rmb();		rmb();

DEBUG_TRACE(SLOT_HANDLER_LINE);		DEBUG_TRACE(SLOT_HANDLER_LINE);
if (state->poll_needed) {		if (state->poll_needed) {
/* Check if we need to suspend - may change our		/* Check if we need to suspend - may change our
* conn_state */		* conn_state */
vchiq_platform_check_suspend(state);		vchiq_platform_check_suspend(state);
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines
int recycle_func(void *v);		int recycle_func(void *v);
int		int
recycle_func(void *v)		recycle_func(void *v)
{		{
VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;		VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;
VCHIQ_SHARED_STATE_T *local = state->local;		VCHIQ_SHARED_STATE_T *local = state->local;

while (1) {		while (1) {
remote_event_wait(&local->recycle);		remote_event_wait(state, &local->recycle);

process_free_queue(state);		process_free_queue(state);
}		}
return 0;		return 0;
}		}


/* Called by the sync thread */		/* Called by the sync thread */
int sync_func(void *v);		int sync_func(void *v);
int		int
sync_func(void *v)		sync_func(void *v)
{		{
VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;		VCHIQ_STATE_T state = (VCHIQ_STATE_T ) v;
VCHIQ_SHARED_STATE_T *local = state->local;		VCHIQ_SHARED_STATE_T *local = state->local;
VCHIQ_HEADER_T header = (VCHIQ_HEADER_T )SLOT_DATA_FROM_INDEX(state,		VCHIQ_HEADER_T header = (VCHIQ_HEADER_T )SLOT_DATA_FROM_INDEX(state,
state->remote->slot_sync);		state->remote->slot_sync);

while (1) {		while (1) {
VCHIQ_SERVICE_T *service;		VCHIQ_SERVICE_T *service;
int msgid, size;		int msgid, size;
int type;		int type;
unsigned int localport, remoteport;		unsigned int localport, remoteport;

remote_event_wait(&local->sync_trigger);		remote_event_wait(state, &local->sync_trigger);

rmb();		rmb();

msgid = header->msgid;		msgid = header->msgid;
size = header->size;		size = header->size;
type = VCHIQ_MSG_TYPE(msgid);		type = VCHIQ_MSG_TYPE(msgid);
localport = VCHIQ_MSG_DSTPORT(msgid);		localport = VCHIQ_MSG_DSTPORT(msgid);
remoteport = VCHIQ_MSG_SRCPORT(msgid);		remoteport = VCHIQ_MSG_SRCPORT(msgid);
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines
{		{
return conn_state_names[conn_state];		return conn_state_names[conn_state];
}		}


VCHIQ_SLOT_ZERO_T *		VCHIQ_SLOT_ZERO_T *
vchiq_init_slots(void *mem_base, int mem_size)		vchiq_init_slots(void *mem_base, int mem_size)
{		{
int mem_align = (VCHIQ_SLOT_SIZE - (int)mem_base) & VCHIQ_SLOT_MASK;		int mem_align = (int)((VCHIQ_SLOT_SIZE - (long)mem_base) & VCHIQ_SLOT_MASK);
VCHIQ_SLOT_ZERO_T *slot_zero =		VCHIQ_SLOT_ZERO_T *slot_zero =
(VCHIQ_SLOT_ZERO_T )((char )mem_base + mem_align);		(VCHIQ_SLOT_ZERO_T )((char )mem_base + mem_align);
int num_slots = (mem_size - mem_align)/VCHIQ_SLOT_SIZE;		int num_slots = (mem_size - mem_align)/VCHIQ_SLOT_SIZE;
int first_data_slot = VCHIQ_SLOT_ZERO_SLOTS;		int first_data_slot = VCHIQ_SLOT_ZERO_SLOTS;

/* Ensure there is enough memory to run an absolutely minimum system */		/* Ensure there is enough memory to run an absolutely minimum system */
num_slots -= first_data_slot;		num_slots -= first_data_slot;

▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines	vchiq_init_state(VCHIQ_STATE_T state, VCHIQ_SLOT_ZERO_T slot_zero,
state->default_message_quota =		state->default_message_quota =
min((unsigned short)(state->default_slot_quota * 256),		min((unsigned short)(state->default_slot_quota * 256),
(unsigned short)~0);		(unsigned short)~0);

state->previous_data_index = -1;		state->previous_data_index = -1;
state->data_use_count = 0;		state->data_use_count = 0;
state->data_quota = state->slot_queue_available - 1;		state->data_quota = state->slot_queue_available - 1;

local->trigger.event = &state->trigger_event;		local->trigger.event = offsetof(VCHIQ_STATE_T, trigger_event);
remote_event_create(&local->trigger);		remote_event_create(state, &local->trigger);
local->tx_pos = 0;		local->tx_pos = 0;

local->recycle.event = &state->recycle_event;		local->recycle.event = offsetof(VCHIQ_STATE_T, recycle_event);
remote_event_create(&local->recycle);		remote_event_create(state, &local->recycle);
local->slot_queue_recycle = state->slot_queue_available;		local->slot_queue_recycle = state->slot_queue_available;

local->sync_trigger.event = &state->sync_trigger_event;		local->sync_trigger.event = offsetof(VCHIQ_STATE_T, sync_trigger_event);
remote_event_create(&local->sync_trigger);		remote_event_create(state, &local->sync_trigger);

local->sync_release.event = &state->sync_release_event;		local->sync_release.event = offsetof(VCHIQ_STATE_T, sync_release_event);
remote_event_create(&local->sync_release);		remote_event_create(state, &local->sync_release);

/* At start-of-day, the slot is empty and available */		/* At start-of-day, the slot is empty and available */
((VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state, local->slot_sync))->msgid		((VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state, local->slot_sync))->msgid
= VCHIQ_MSGID_PADDING;		= VCHIQ_MSGID_PADDING;
remote_event_signal_local(&local->sync_release);		remote_event_signal_local(state, &local->sync_release);

local->debug[DEBUG_ENTRIES] = DEBUG_MAX;		local->debug[DEBUG_ENTRIES] = DEBUG_MAX;

status = vchiq_platform_init_state(state);		status = vchiq_platform_init_state(state);

/*		/*
bring up slot handler thread		bring up slot handler thread
*/		*/
▲ Show 20 Lines • Show All 870 Lines • ▼ Show 20 Lines	vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,

if (state->is_master) {		if (state->is_master) {
queue->local_insert++;		queue->local_insert++;
if (resolve_bulks(service, queue))		if (resolve_bulks(service, queue))
request_poll(state, service,		request_poll(state, service,
(dir == VCHIQ_BULK_TRANSMIT) ?		(dir == VCHIQ_BULK_TRANSMIT) ?
VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY);		VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY);
} else {		} else {
int payload[2] = { (int)bulk->data, bulk->size };		uint32_t payload[2] = { (uint32_t)(uintptr_t)bulk->data, bulk->size };
VCHIQ_ELEMENT_T element = { payload, sizeof(payload) };		VCHIQ_ELEMENT_T element = { payload, sizeof(payload) };

status = queue_message(state, NULL,		status = queue_message(state, NULL,
VCHIQ_MAKE_MSG(dir_msgtype,		VCHIQ_MAKE_MSG(dir_msgtype,
service->localport, service->remoteport),		service->localport, service->remoteport),
&element, 1, sizeof(payload),		&element, 1, sizeof(payload),
QMFLAGS_IS_BLOCKING \|		QMFLAGS_IS_BLOCKING \|
QMFLAGS_NO_MUTEX_LOCK \|		QMFLAGS_NO_MUTEX_LOCK \|
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	vchiq_release_message(VCHIQ_SERVICE_HANDLE_T handle, VCHIQ_HEADER_T *header)

unlock_service(service);		unlock_service(service);
}		}

static void		static void
release_message_sync(VCHIQ_STATE_T state, VCHIQ_HEADER_T header)		release_message_sync(VCHIQ_STATE_T state, VCHIQ_HEADER_T header)
{		{
header->msgid = VCHIQ_MSGID_PADDING;		header->msgid = VCHIQ_MSGID_PADDING;
wmb();
adrianUnsubmitted Not Done Inline Actions and whys this barrier being removed? adrian: and whys this barrier being removed?
remote_event_signal(&state->remote->sync_release);		remote_event_signal(&state->remote->sync_release);
}		}

VCHIQ_STATUS_T		VCHIQ_STATUS_T
vchiq_get_peer_version(VCHIQ_SERVICE_HANDLE_T handle, short *peer_version)		vchiq_get_peer_version(VCHIQ_SERVICE_HANDLE_T handle, short *peer_version)
{		{
VCHIQ_STATUS_T status = VCHIQ_ERROR;		VCHIQ_STATUS_T status = VCHIQ_ERROR;
VCHIQ_SERVICE_T *service = find_service_by_handle(handle);		VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
▲ Show 20 Lines • Show All 416 Lines • Show Last 20 Lines