diff --git a/sys/arm/broadcom/bcm2835/bcm2835_audio.c b/sys/arm/broadcom/bcm2835/bcm2835_audio.c --- a/sys/arm/broadcom/bcm2835/bcm2835_audio.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_audio.c @@ -237,7 +237,7 @@ device_printf(sc->dev, "available_space == %d, count = %d, perr=%d\n", ch->available_space, count, perr); device_printf(sc->dev, - "retrieved_samples = %lld, submitted_samples = %lld\n", + "retrieved_samples = %uld, submitted_samples = %uld\n", ch->retrieved_samples, ch->submitted_samples); } ch->available_space += count; diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_2835_arm.c @@ -65,9 +65,24 @@ #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2) +/* + * XXXMDC + * Do this less ad-hoc-y -- e.g. + * https://github.com/raspberrypi/linux/commit/c683db8860a80562a2bb5b451d77b3e471d24f36 + */ +#if defined(__aarch64__) +int g_cache_line_size = 64; +#else int g_cache_line_size = 32; +#endif static int g_fragment_size; +unsigned int g_long_bulk_space = 0; +#define VM_PAGE_TO_VC_BULK_PAGE(x) (\ + g_long_bulk_space ? VM_PAGE_TO_PHYS(x)\ + : PHYS_TO_VCBUS(VM_PAGE_TO_PHYS(x))\ +) + typedef struct vchiq_2835_state_struct { int inited; VCHIQ_ARM_STATE_T arm_state; @@ -113,6 +128,59 @@ *addr = PHYS_TO_VCBUS(segs[0].ds_addr); } +#if defined(__aarch64__) /* See comment in free_pagelist */ +static int +invalidate_cachelines_in_range_of_ppage( + vm_page_t p, + size_t offset, + size_t count +) +{ + if(offset + count > PAGE_SIZE){ return EINVAL; } + uint8_t *dst = (uint8_t*)pmap_quick_enter_page(p); + if (!dst){ + return ENOMEM; + } + cpu_dcache_inv_range((vm_offset_t)dst + offset, count); + pmap_quick_remove_page((vm_offset_t)dst); + return 0; +} + +/* XXXMDC bulk instead of loading and invalidating single pages? */ +static void +invalidate_cachelines_in_range_of_ppage_seq( + vm_page_t *p, + size_t start, + size_t count +) +{ + if(start >= PAGE_SIZE) goto invalid_input; + +#define _NEXT_AT(x,_m) (((x)+((_m)-1)) & ~((_m)-1)) /* for power of two m */ + size_t offset = _NEXT_AT(start,g_cache_line_size); +#undef _NEXT_AT + count = (offset < start + count) ? count - (offset - start) : 0; + offset = offset & (PAGE_SIZE - 1); + for( + size_t done = 0; + count > done; + p++, done += PAGE_SIZE - offset, offset = 0 + ){ + size_t in_page = PAGE_SIZE - offset; + size_t todo = (count-done > in_page) ? in_page : count-done; + int e = invalidate_cachelines_in_range_of_ppage(*p, offset, todo); + if(e != 0) + goto problem_in_loop; + } + return; + +problem_in_loop: +invalid_input: + WARN_ON(1); + return; +} +#endif + static int copyout_page(vm_page_t p, size_t offset, void *kaddr, size_t size) { @@ -171,7 +239,7 @@ goto failed_load; } - WARN_ON(((int)g_slot_mem & (PAGE_SIZE - 1)) != 0); + WARN_ON(((size_t)g_slot_mem & (PAGE_SIZE - 1)) != 0); vchiq_slot_zero = vchiq_init_slots(g_slot_mem, g_slot_mem_size); if (!vchiq_slot_zero) { @@ -204,8 +272,8 @@ bcm_mbox_write(BCM2835_MBOX_CHAN_VCHIQ, (unsigned int)g_slot_phys); vchiq_log_info(vchiq_arm_log_level, - "vchiq_init - done (slots %x, phys %x)", - (unsigned int)vchiq_slot_zero, g_slot_phys); + "vchiq_init - done (slots %zx, phys %zx)", + (size_t)vchiq_slot_zero, g_slot_phys); vchiq_call_connected_callbacks(); @@ -393,13 +461,14 @@ ** from increased speed as a result. */ + static int create_pagelist(char __user *buf, size_t count, unsigned short type, struct proc *p, BULKINFO_T *bi) { PAGELIST_T *pagelist; vm_page_t* pages; - unsigned long *addrs; + uint32_t *addrs; unsigned int num_pages, i; vm_offset_t offset; int pagelist_size; @@ -436,7 +505,7 @@ err = bus_dmamem_alloc(bi->pagelist_dma_tag, (void **)&pagelist, BUS_DMA_COHERENT | BUS_DMA_WAITOK, &bi->pagelist_dma_map); - if (err) { + if (err || !pagelist) { vchiq_log_error(vchiq_core_log_level, "Unable to allocate pagelist memory"); err = -ENOMEM; goto failed_alloc; @@ -449,16 +518,14 @@ if (err) { vchiq_log_error(vchiq_core_log_level, "cannot load DMA map for pagelist memory"); err = -ENOMEM; + bi->pagelist = pagelist; goto failed_load; } vchiq_log_trace(vchiq_arm_log_level, - "create_pagelist - %x (%d bytes @%p)", (unsigned int)pagelist, count, buf); + "create_pagelist - %zx (%zu bytes @%p)", (size_t)pagelist, count, buf); - if (!pagelist) - return -ENOMEM; - - addrs = pagelist->addrs; + addrs = (uint32_t *) pagelist->addrs; pages = (vm_page_t*)(addrs + num_pages); actual_pages = vm_fault_quick_hold_pages(&p->p_vmspace->vm_map, @@ -467,8 +534,9 @@ if (actual_pages != num_pages) { vm_page_unhold_pages(pages, actual_pages); - free(pagelist, M_VCPAGELIST); - return (-ENOMEM); + err = -ENOMEM; + bi->pagelist = pagelist; + goto failed_hold; } pagelist->length = count; @@ -477,27 +545,28 @@ /* Group the pages into runs of contiguous pages */ - base_addr = (void *)PHYS_TO_VCBUS(VM_PAGE_TO_PHYS(pages[0])); + size_t run_ceil = g_long_bulk_space ? 0x100 : PAGE_SIZE; + unsigned int pg_addr_rshift = g_long_bulk_space ? 4 : 0; + base_addr = (void *) VM_PAGE_TO_VC_BULK_PAGE(pages[0]); next_addr = base_addr + PAGE_SIZE; addridx = 0; run = 0; - +#define _PG_BLOCK(base,run) \ + ((((size_t) (base)) >> pg_addr_rshift) & ~(run_ceil-1)) + (run) for (i = 1; i < num_pages; i++) { - addr = (void *)PHYS_TO_VCBUS(VM_PAGE_TO_PHYS(pages[i])); - if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) { + addr = (void *)VM_PAGE_TO_VC_BULK_PAGE(pages[i]); + if ((addr == next_addr) && (run < run_ceil - 1)) { next_addr += PAGE_SIZE; run++; } else { - addrs[addridx] = (unsigned long)base_addr + run; - addridx++; + addrs[addridx++] = (uint32_t) _PG_BLOCK(base_addr,run); base_addr = addr; next_addr = addr + PAGE_SIZE; run = 0; } } - - addrs[addridx] = (unsigned long)base_addr + run; - addridx++; + addrs[addridx++] = _PG_BLOCK(base_addr, run); +#undef _PG_BLOCK /* Partial cache lines (fragments) require special measures */ if ((type == PAGELIST_READ) && @@ -519,12 +588,24 @@ g_free_fragments = *(char **) g_free_fragments; up(&g_free_fragments_mutex); pagelist->type = - PAGELIST_READ_WITH_FRAGMENTS + - (fragments - g_fragments_base)/g_fragment_size; + PAGELIST_READ_WITH_FRAGMENTS + + (fragments - g_fragments_base)/g_fragment_size; +#if defined(__aarch64__) + bus_dmamap_sync(bcm_slots_dma_tag, bcm_slots_dma_map, BUS_DMASYNC_PREREAD); +#endif } +#if defined(__aarch64__) + if(type == PAGELIST_READ){ + cpu_dcache_wbinv_range((vm_offset_t)buf,count); + }else{ + cpu_dcache_wb_range((vm_offset_t)buf,count); + } + dsb(sy); +#else pa = pmap_extract(PCPU_GET(curpmap), (vm_offset_t)buf); dcache_wbinv_poc((vm_offset_t)buf, pa, count); +#endif bus_dmamap_sync(bi->pagelist_dma_tag, bi->pagelist_dma_map, BUS_DMASYNC_PREWRITE); @@ -532,6 +613,8 @@ return 0; +failed_hold: + bus_dmamap_unload(bi->pagelist_dma_tag,bi->pagelist_dma_map); failed_load: bus_dmamem_free(bi->pagelist_dma_tag, bi->pagelist, bi->pagelist_dma_map); failed_alloc: @@ -550,7 +633,7 @@ pagelist = bi->pagelist; vchiq_log_trace(vchiq_arm_log_level, - "free_pagelist - %x, %d (%lu bytes @%p)", (unsigned int)pagelist, actual, pagelist->length, bi->buf); + "free_pagelist - %zx, %d (%lu bytes @%p)", (size_t)pagelist, actual, pagelist->length, bi->buf); num_pages = (pagelist->length + pagelist->offset + PAGE_SIZE - 1) / @@ -558,6 +641,27 @@ pages = (vm_page_t*)(pagelist->addrs + num_pages); +#if defined(__aarch64__) + /* + * On arm64, even if the user keeps their end of the bargain + * -- do NOT touch the buffers sent to VC -- but reads around the + * pagelist after the invalidation above, the arm might preemptively + * load (and validate) cache lines for areas inside the page list, + * so we must invalidate them again. + * + * The functional test does it and without this it doesn't pass. + * + * XXXMDC might it be enough to invalidate a couple of pages at + * the ends of the page list? + */ + if(pagelist->type >= PAGELIST_READ && actual > 0) + invalidate_cachelines_in_range_of_ppage_seq( + pages, + pagelist->offset, + actual + ); +#endif + /* Deal with any partial cache lines (fragments) */ if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) { char *fragments = g_fragments_base + @@ -594,13 +698,18 @@ up(&g_free_fragments_sema); } - for (i = 0; i < num_pages; i++) { - if (pagelist->type != PAGELIST_WRITE) { + if (pagelist->type != PAGELIST_WRITE) { + for (i = 0; i < num_pages; i++) { vm_page_dirty(pages[i]); pagelist_page_free(pages[i]); } } +#if defined(__aarch64__) + /* XXXMDC necessary? */ + dsb(sy); +#endif + bus_dmamap_unload(bi->pagelist_dma_tag, bi->pagelist_dma_map); bus_dmamem_free(bi->pagelist_dma_tag, bi->pagelist, bi->pagelist_dma_map); bus_dma_tag_destroy(bi->pagelist_dma_tag); diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_arm.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_arm.c --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_arm.c +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_arm.c @@ -442,8 +442,8 @@ #define _IOC_TYPE(x) IOCGROUP(x) vchiq_log_trace(vchiq_arm_log_level, - "vchiq_ioctl - instance %x, cmd %s, arg %p", - (unsigned int)instance, + "vchiq_ioctl - instance %lx, cmd %s, arg %p", + (unsigned long)instance, ((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) && (_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ? ioctl_names[_IOC_NR(cmd)] : "", arg); @@ -745,8 +745,8 @@ break; } vchiq_log_info(vchiq_arm_log_level, - "found bulk_waiter %x for pid %d", - (unsigned int)waiter, current->p_pid); + "found bulk_waiter %lx for pid %d", + (unsigned long)waiter, current->p_pid); args.userdata = &waiter->bulk_waiter; } status = vchiq_bulk_transfer @@ -776,8 +776,8 @@ list_add(&waiter->list, &instance->bulk_waiter_list); lmutex_unlock(&instance->bulk_waiter_list_mutex); vchiq_log_info(vchiq_arm_log_level, - "saved bulk_waiter %x for pid %d", - (unsigned int)waiter, current->p_pid); + "saved bulk_waiter %lx for pid %d", + (unsigned long)waiter, current->p_pid); memcpy((void *) &(((VCHIQ_QUEUE_BULK_TRANSFER_T *) @@ -860,9 +860,9 @@ if (args.msgbufsize < msglen) { vchiq_log_error( vchiq_arm_log_level, - "header %x: msgbufsize" + "header %lx: msgbufsize" " %x < msglen %x", - (unsigned int)header, + (unsigned long)header, args.msgbufsize, msglen); WARN(1, "invalid message " @@ -1031,8 +1031,8 @@ ret = -EFAULT; } else { vchiq_log_error(vchiq_arm_log_level, - "header %x: bufsize %x < size %x", - (unsigned int)header, args.bufsize, + "header %lx: bufsize %x < size %x", + (unsigned long)header, args.bufsize, header->size); WARN(1, "invalid size\n"); ret = -EMSGSIZE; @@ -1093,7 +1093,7 @@ } break; case VCHIQ_IOC_LIB_VERSION: { - unsigned int lib_version = (unsigned int)arg; + unsigned long lib_version = (unsigned long)arg; if (lib_version < VCHIQ_VERSION_MIN) ret = -EINVAL; @@ -1342,9 +1342,9 @@ list); list_del(pos); vchiq_log_info(vchiq_arm_log_level, - "bulk_waiter - cleaned up %x " + "bulk_waiter - cleaned up %lx " "for pid %d", - (unsigned int)waiter, waiter->pid); + (unsigned long)waiter, waiter->pid); _sema_destroy(&waiter->bulk_waiter.event); kfree(waiter); } @@ -1435,9 +1435,9 @@ instance = service->instance; if (instance && !instance->mark) { len = snprintf(buf, sizeof(buf), - "Instance %x: pid %d,%s completions " + "Instance %lx: pid %d,%s completions " "%d/%d", - (unsigned int)instance, instance->pid, + (unsigned long)instance, instance->pid, instance->connected ? " connected, " : "", instance->completion_insert - @@ -1465,8 +1465,8 @@ char buf[80]; int len; - len = snprintf(buf, sizeof(buf), " instance %x", - (unsigned int)service->instance); + len = snprintf(buf, sizeof(buf), " instance %lx", + (unsigned long)service->instance); if ((service->base.callback == service_callback) && user_service->is_vchi) { diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.h b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.h --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.h +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.h @@ -183,6 +183,15 @@ #if VCHIQ_ENABLE_DEBUG +#if defined(__aarch64__) +#define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug; +#define DEBUG_TRACE(d) \ + do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) +#define DEBUG_VALUE(d, v) \ + do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) +#define DEBUG_COUNT(d) \ + do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) +#else #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug; #define DEBUG_TRACE(d) \ do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(); } while (0) @@ -190,7 +199,7 @@ do { debug_ptr[DEBUG_ ## d] = (v); dsb(); } while (0) #define DEBUG_COUNT(d) \ do { debug_ptr[DEBUG_ ## d]++; dsb(); } while (0) - +#endif #else /* VCHIQ_ENABLE_DEBUG */ #define DEBUG_INITIALISE(local) diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c @@ -392,9 +392,9 @@ VCHIQ_HEADER_T *header, void *bulk_userdata) { VCHIQ_STATUS_T status; - vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %x, %x)", + vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %lx, %lx)", service->state->id, service->localport, reason_names[reason], - (unsigned int)header, (unsigned int)bulk_userdata); + (unsigned long)header, (unsigned long)bulk_userdata); status = service->base.callback(reason, header, service->handle, bulk_userdata); if (status == VCHIQ_ERROR) { @@ -437,7 +437,11 @@ { if (!event->fired) { event->armed = 1; - dsb(); +#if defined(__arch64__) + dsb(sy); +#else + dsb(sy); +#endif if (!event->fired) { if (down_interruptible(event->event) != 0) { event->armed = 0; @@ -640,8 +644,8 @@ rmb(); - vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x", - state->id, slot_index, (unsigned int)data, + vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%lx %x %x", + state->id, slot_index, (unsigned long)data, local->slot_queue_recycle, slot_queue_available); /* Initialise the bitmask for services which have used this @@ -675,13 +679,13 @@ vchiq_log_error(vchiq_core_log_level, "service %d " "message_use_count=%d " - "(header %x, msgid %x, " + "(header %lx, msgid %x, " "header->msgid %x, " "header->size %x)", port, service_quota-> message_use_count, - (unsigned int)header, msgid, + (unsigned long)header, msgid, header->msgid, header->size); WARN(1, "invalid message use count\n"); @@ -704,24 +708,24 @@ up(&service_quota->quota_event); vchiq_log_trace( vchiq_core_log_level, - "%d: pfq:%d %x@%x - " + "%d: pfq:%d %x@%lx - " "slot_use->%d", state->id, port, header->size, - (unsigned int)header, + (unsigned long)header, count - 1); } else { vchiq_log_error( vchiq_core_log_level, "service %d " "slot_use_count" - "=%d (header %x" + "=%d (header %lx" ", msgid %x, " "header->msgid" " %x, header->" "size %x)", port, count, - (unsigned int)header, + (unsigned long)header, msgid, header->msgid, header->size); @@ -735,9 +739,9 @@ pos += calc_stride(header->size); if (pos > VCHIQ_SLOT_SIZE) { vchiq_log_error(vchiq_core_log_level, - "pfq - pos %x: header %x, msgid %x, " + "pfq - pos %x: header %lx, msgid %x, " "header->msgid %x, header->size %x", - pos, (unsigned int)header, msgid, + pos, (unsigned long)header, msgid, header->msgid, header->size); WARN(1, "invalid slot position\n"); } @@ -885,10 +889,10 @@ int slot_use_count; vchiq_log_info(vchiq_core_log_level, - "%d: qm %s@%x,%x (%d->%d)", + "%d: qm %s@%lx,%x (%d->%d)", state->id, msg_type_str(VCHIQ_MSG_TYPE(msgid)), - (unsigned int)header, size, + (unsigned long)header, size, VCHIQ_MSG_SRCPORT(msgid), VCHIQ_MSG_DSTPORT(msgid)); @@ -951,9 +955,9 @@ VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size); } else { vchiq_log_info(vchiq_core_log_level, - "%d: qm %s@%x,%x (%d->%d)", state->id, + "%d: qm %s@%lx,%x (%d->%d)", state->id, msg_type_str(VCHIQ_MSG_TYPE(msgid)), - (unsigned int)header, size, + (unsigned long)header, size, VCHIQ_MSG_SRCPORT(msgid), VCHIQ_MSG_DSTPORT(msgid)); if (size != 0) { @@ -1036,9 +1040,9 @@ int i, pos; vchiq_log_info(vchiq_sync_log_level, - "%d: qms %s@%x,%x (%d->%d)", state->id, + "%d: qms %s@%lx,%x (%d->%d)", state->id, msg_type_str(VCHIQ_MSG_TYPE(msgid)), - (unsigned int)header, size, + (unsigned long)header, size, VCHIQ_MSG_SRCPORT(msgid), VCHIQ_MSG_DSTPORT(msgid)); @@ -1065,9 +1069,9 @@ VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size); } else { vchiq_log_info(vchiq_sync_log_level, - "%d: qms %s@%x,%x (%d->%d)", state->id, + "%d: qms %s@%lx,%x (%d->%d)", state->id, msg_type_str(VCHIQ_MSG_TYPE(msgid)), - (unsigned int)header, size, + (unsigned long)header, size, VCHIQ_MSG_SRCPORT(msgid), VCHIQ_MSG_DSTPORT(msgid)); if (size != 0) { @@ -1368,26 +1372,26 @@ "Send Bulk to" : "Recv Bulk from"; if (bulk->actual != VCHIQ_BULK_ACTUAL_ABORTED) vchiq_log_info(SRVTRACE_LEVEL(service), - "%s %c%c%c%c d:%d len:%d %x<->%x", + "%s %c%c%c%c d:%d len:%d %lx<->%lx", header, VCHIQ_FOURCC_AS_4CHARS( service->base.fourcc), service->remoteport, bulk->size, - (unsigned int)bulk->data, - (unsigned int)bulk->remote_data); + (unsigned long)bulk->data, + (unsigned long)bulk->remote_data); else vchiq_log_info(SRVTRACE_LEVEL(service), "%s %c%c%c%c d:%d ABORTED - tx len:%d," - " rx len:%d %x<->%x", + " rx len:%d %lx<->%lx", header, VCHIQ_FOURCC_AS_4CHARS( service->base.fourcc), service->remoteport, bulk->size, bulk->remote_size, - (unsigned int)bulk->data, - (unsigned int)bulk->remote_data); + (unsigned long)bulk->data, + (unsigned long)bulk->remote_data); } vchiq_complete_bulk(bulk); @@ -1522,8 +1526,8 @@ fourcc = payload->fourcc; vchiq_log_info(vchiq_core_log_level, - "%d: prs OPEN@%x (%d->'%c%c%c%c')", - state->id, (unsigned int)header, + "%d: prs OPEN@%lx (%d->'%c%c%c%c')", + state->id, (unsigned long)header, localport, VCHIQ_FOURCC_AS_4CHARS(fourcc)); @@ -1661,7 +1665,7 @@ header = (VCHIQ_HEADER_T *)(state->rx_data + (state->rx_pos & VCHIQ_SLOT_MASK)); - DEBUG_VALUE(PARSE_HEADER, (int)header); + DEBUG_VALUE(PARSE_HEADER, (unsigned long)header); msgid = header->msgid; DEBUG_VALUE(PARSE_MSGID, msgid); size = header->size; @@ -1695,20 +1699,20 @@ remoteport); if (service) vchiq_log_warning(vchiq_core_log_level, - "%d: prs %s@%x (%d->%d) - " + "%d: prs %s@%lx (%d->%d) - " "found connected service %d", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, service->localport); } if (!service) { vchiq_log_error(vchiq_core_log_level, - "%d: prs %s@%x (%d->%d) - " + "%d: prs %s@%lx (%d->%d) - " "invalid/closed service %d", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, localport); goto skip_message; } @@ -1734,12 +1738,12 @@ min(16, size)); } - if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size) + if (((unsigned long) header & VCHIQ_SLOT_MASK) + calc_stride(size) > VCHIQ_SLOT_SIZE) { vchiq_log_error(vchiq_core_log_level, - "header %x (msgid %x) - size %x too big for " + "header %lx (msgid %x) - size %x too big for " "slot", - (unsigned int)header, (unsigned int)msgid, + (unsigned long)header, (unsigned int)msgid, (unsigned int)size); WARN(1, "oversized for slot\n"); } @@ -1758,8 +1762,8 @@ service->peer_version = payload->version; } vchiq_log_info(vchiq_core_log_level, - "%d: prs OPENACK@%x,%x (%d->%d) v:%d", - state->id, (unsigned int)header, size, + "%d: prs OPENACK@%lx,%x (%d->%d) v:%d", + state->id, (unsigned long)header, size, remoteport, localport, service->peer_version); if (service->srvstate == VCHIQ_SRVSTATE_OPENING) { @@ -1776,8 +1780,8 @@ WARN_ON(size != 0); /* There should be no data */ vchiq_log_info(vchiq_core_log_level, - "%d: prs CLOSE@%x (%d->%d)", - state->id, (unsigned int)header, + "%d: prs CLOSE@%lx (%d->%d)", + state->id, (unsigned long)header, remoteport, localport); mark_service_closing_internal(service, 1); @@ -1794,8 +1798,8 @@ break; case VCHIQ_MSG_DATA: vchiq_log_info(vchiq_core_log_level, - "%d: prs DATA@%x,%x (%d->%d)", - state->id, (unsigned int)header, size, + "%d: prs DATA@%lx,%x (%d->%d)", + state->id, (unsigned long)header, size, remoteport, localport); if ((service->remoteport == remoteport) @@ -1819,8 +1823,8 @@ break; case VCHIQ_MSG_CONNECT: vchiq_log_info(vchiq_core_log_level, - "%d: prs CONNECT@%x", - state->id, (unsigned int)header); + "%d: prs CONNECT@%lx", + state->id, (unsigned long)header); state->version_common = ((VCHIQ_SLOT_ZERO_T *) state->slot_data)->version; up(&state->connect); @@ -1849,17 +1853,17 @@ bulk = &queue->bulks[ BULK_INDEX(queue->remote_insert)]; bulk->remote_data = - (void *)((int *)header->data)[0]; + (void *)((long *)header->data)[0]; bulk->remote_size = ((int *)header->data)[1]; wmb(); vchiq_log_info(vchiq_core_log_level, - "%d: prs %s@%x (%d->%d) %x@%x", + "%d: prs %s@%lx (%d->%d) %x@%lx", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, bulk->remote_size, - (unsigned int)bulk->remote_data); + (unsigned long)bulk->remote_data); queue->remote_insert++; @@ -1912,10 +1916,10 @@ if ((int)(queue->remote_insert - queue->local_insert) >= 0) { vchiq_log_error(vchiq_core_log_level, - "%d: prs %s@%x (%d->%d) " + "%d: prs %s@%lx (%d->%d) " "unexpected (ri=%d,li=%d)", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, queue->remote_insert, queue->local_insert); @@ -1932,11 +1936,11 @@ queue->remote_insert++; vchiq_log_info(vchiq_core_log_level, - "%d: prs %s@%x (%d->%d) %x@%x", + "%d: prs %s@%lx (%d->%d) %x@%lx", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, - bulk->actual, (unsigned int)bulk->data); + bulk->actual, (unsigned long)bulk->data); vchiq_log_trace(vchiq_core_log_level, "%d: prs:%d %cx li=%x ri=%x p=%x", @@ -1958,14 +1962,14 @@ break; case VCHIQ_MSG_PADDING: vchiq_log_trace(vchiq_core_log_level, - "%d: prs PADDING@%x,%x", - state->id, (unsigned int)header, size); + "%d: prs PADDING@%lx,%x", + state->id, (unsigned long)header, size); break; case VCHIQ_MSG_PAUSE: /* If initiated, signal the application thread */ vchiq_log_trace(vchiq_core_log_level, - "%d: prs PAUSE@%x,%x", - state->id, (unsigned int)header, size); + "%d: prs PAUSE@%lx,%x", + state->id, (unsigned long)header, size); if (state->conn_state == VCHIQ_CONNSTATE_PAUSED) { vchiq_log_error(vchiq_core_log_level, "%d: PAUSE received in state PAUSED", @@ -1988,8 +1992,8 @@ break; case VCHIQ_MSG_RESUME: vchiq_log_trace(vchiq_core_log_level, - "%d: prs RESUME@%x,%x", - state->id, (unsigned int)header, size); + "%d: prs RESUME@%lx,%x", + state->id, (unsigned long)header, size); /* Release the slot mutex */ lmutex_unlock(&state->slot_mutex); if (state->is_master) @@ -2010,8 +2014,8 @@ default: vchiq_log_error(vchiq_core_log_level, - "%d: prs invalid msgid %x@%x,%x", - state->id, msgid, (unsigned int)header, size); + "%d: prs invalid msgid %x@%lx,%x", + state->id, msgid, (unsigned long)header, size); WARN(1, "invalid message\n"); break; } @@ -2179,10 +2183,10 @@ if (!service) { vchiq_log_error(vchiq_sync_log_level, - "%d: sf %s@%x (%d->%d) - " + "%d: sf %s@%lx (%d->%d) - " "invalid/closed service %d", state->id, msg_type_str(type), - (unsigned int)header, + (unsigned long)header, remoteport, localport, localport); release_message_sync(state, header); continue; @@ -2213,8 +2217,8 @@ service->peer_version = payload->version; } vchiq_log_info(vchiq_sync_log_level, - "%d: sf OPENACK@%x,%x (%d->%d) v:%d", - state->id, (unsigned int)header, size, + "%d: sf OPENACK@%lx,%x (%d->%d) v:%d", + state->id, (unsigned long)header, size, remoteport, localport, service->peer_version); if (service->srvstate == VCHIQ_SRVSTATE_OPENING) { service->remoteport = remoteport; @@ -2228,8 +2232,8 @@ case VCHIQ_MSG_DATA: vchiq_log_trace(vchiq_sync_log_level, - "%d: sf DATA@%x,%x (%d->%d)", - state->id, (unsigned int)header, size, + "%d: sf DATA@%lx,%x (%d->%d)", + state->id, (unsigned long)header, size, remoteport, localport); if ((service->remoteport == remoteport) && @@ -2248,8 +2252,8 @@ default: vchiq_log_error(vchiq_sync_log_level, - "%d: sf unexpected msgid %x@%x,%x", - state->id, msgid, (unsigned int)header, size); + "%d: sf unexpected msgid %x@%lx,%x", + state->id, msgid, (unsigned long)header, size); release_message_sync(state, header); break; } @@ -2282,7 +2286,7 @@ VCHIQ_SLOT_ZERO_T * vchiq_init_slots(void *mem_base, int mem_size) { - int mem_align = (VCHIQ_SLOT_SIZE - (int)mem_base) & VCHIQ_SLOT_MASK; + int mem_align = (VCHIQ_SLOT_SIZE - (long)mem_base) & VCHIQ_SLOT_MASK; VCHIQ_SLOT_ZERO_T *slot_zero = (VCHIQ_SLOT_ZERO_T *)((char *)mem_base + mem_align); int num_slots = (mem_size - mem_align)/VCHIQ_SLOT_SIZE; @@ -2334,8 +2338,8 @@ if (slot_zero->magic != VCHIQ_MAGIC) { vchiq_loud_error_header(); vchiq_loud_error("Invalid VCHIQ magic value found."); - vchiq_loud_error("slot_zero=%x: magic=%x (expected %x)", - (unsigned int)slot_zero, slot_zero->magic, VCHIQ_MAGIC); + vchiq_loud_error("slot_zero=%lx: magic=%x (expected %x)", + (unsigned long)slot_zero, slot_zero->magic, VCHIQ_MAGIC); vchiq_loud_error_footer(); return VCHIQ_ERROR; } @@ -2348,9 +2352,9 @@ if (slot_zero->version < VCHIQ_VERSION_MIN) { vchiq_loud_error_header(); vchiq_loud_error("Incompatible VCHIQ versions found."); - vchiq_loud_error("slot_zero=%x: VideoCore version=%d " + vchiq_loud_error("slot_zero=%lx: VideoCore version=%d " "(minimum %d)", - (unsigned int)slot_zero, slot_zero->version, + (unsigned long)slot_zero, slot_zero->version, VCHIQ_VERSION_MIN); vchiq_loud_error("Restart with a newer VideoCore image."); vchiq_loud_error_footer(); @@ -2360,9 +2364,9 @@ if (VCHIQ_VERSION < slot_zero->version_min) { vchiq_loud_error_header(); vchiq_loud_error("Incompatible VCHIQ versions found."); - vchiq_loud_error("slot_zero=%x: version=%d (VideoCore " + vchiq_loud_error("slot_zero=%lx: version=%d (VideoCore " "minimum %d)", - (unsigned int)slot_zero, VCHIQ_VERSION, + (unsigned long)slot_zero, VCHIQ_VERSION, slot_zero->version_min); vchiq_loud_error("Restart with a newer kernel."); vchiq_loud_error_footer(); @@ -2375,25 +2379,25 @@ (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)) { vchiq_loud_error_header(); if (slot_zero->slot_zero_size != sizeof(VCHIQ_SLOT_ZERO_T)) - vchiq_loud_error("slot_zero=%x: slot_zero_size=%x " + vchiq_loud_error("slot_zero=%lx: slot_zero_size=%x " "(expected %zx)", - (unsigned int)slot_zero, + (unsigned long)slot_zero, slot_zero->slot_zero_size, sizeof(VCHIQ_SLOT_ZERO_T)); if (slot_zero->slot_size != VCHIQ_SLOT_SIZE) - vchiq_loud_error("slot_zero=%x: slot_size=%d " + vchiq_loud_error("slot_zero=%lx: slot_size=%d " "(expected %d", - (unsigned int)slot_zero, slot_zero->slot_size, + (unsigned long)slot_zero, slot_zero->slot_size, VCHIQ_SLOT_SIZE); if (slot_zero->max_slots != VCHIQ_MAX_SLOTS) - vchiq_loud_error("slot_zero=%x: max_slots=%d " + vchiq_loud_error("slot_zero=%lx: max_slots=%d " "(expected %d)", - (unsigned int)slot_zero, slot_zero->max_slots, + (unsigned long)slot_zero, slot_zero->max_slots, VCHIQ_MAX_SLOTS); if (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE) - vchiq_loud_error("slot_zero=%x: max_slots_per_side=%d " + vchiq_loud_error("slot_zero=%lx: max_slots_per_side=%d " "(expected %d)", - (unsigned int)slot_zero, + (unsigned long)slot_zero, slot_zero->max_slots_per_side, VCHIQ_MAX_SLOTS_PER_SIDE); vchiq_loud_error_footer(); @@ -2775,18 +2779,18 @@ if ((port == service->localport) && (msgid & VCHIQ_MSGID_CLAIMED)) { vchiq_log_info(vchiq_core_log_level, - " fsi - hdr %x", - (unsigned int)header); + " fsi - hdr %lx", + (unsigned long)header); release_slot(state, slot_info, header, NULL); } pos += calc_stride(header->size); if (pos > VCHIQ_SLOT_SIZE) { vchiq_log_error(vchiq_core_log_level, - "fsi - pos %x: header %x, " + "fsi - pos %x: header %lx, " "msgid %x, header->msgid %x, " "header->size %x", - pos, (unsigned int)header, + pos, (unsigned long)header, msgid, header->msgid, header->size); WARN(1, "invalid slot position\n"); @@ -3360,10 +3364,10 @@ wmb(); vchiq_log_info(vchiq_core_log_level, - "%d: bt (%d->%d) %cx %x@%x %x", + "%d: bt (%d->%d) %cx %x@%lx %lx", state->id, service->localport, service->remoteport, dir_char, - size, (unsigned int)bulk->data, (unsigned int)userdata); + size, (unsigned long)bulk->data, (unsigned long)userdata); /* The slot mutex must be held when the service is being closed, so claim it here to ensure that isn't happening */ @@ -3382,7 +3386,7 @@ (dir == VCHIQ_BULK_TRANSMIT) ? VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY); } else { - int payload[2] = { (int)bulk->data, bulk->size }; + int payload[2] = { (long)bulk->data, bulk->size }; VCHIQ_ELEMENT_T element = { payload, sizeof(payload) }; status = queue_message(state, NULL, @@ -3710,12 +3714,12 @@ vchiq_dump(dump_context, buf, len + 1); len = snprintf(buf, sizeof(buf), - " tx_pos=%x(@%x), rx_pos=%x(@%x)", + " tx_pos=%x(@%lx), rx_pos=%x(@%lx)", state->local->tx_pos, - (uint32_t)state->tx_data + + (unsigned long)state->tx_data + (state->local_tx_pos & VCHIQ_SLOT_MASK), state->rx_pos, - (uint32_t)state->rx_data + + (unsigned long)state->rx_data + (state->rx_pos & VCHIQ_SLOT_MASK)); vchiq_dump(dump_context, buf, len + 1); @@ -3817,8 +3821,8 @@ vchiq_dump(dump_context, buf, len + 1); len = snprintf(buf, sizeof(buf), - " Ctrl: tx_count=%d, tx_bytes=%llu, " - "rx_count=%d, rx_bytes=%llu", + " Ctrl: tx_count=%d, tx_bytes=%lu, " + "rx_count=%d, rx_bytes=%lu", service->stats.ctrl_tx_count, service->stats.ctrl_tx_bytes, service->stats.ctrl_rx_count, @@ -3826,8 +3830,8 @@ vchiq_dump(dump_context, buf, len + 1); len = snprintf(buf, sizeof(buf), - " Bulk: tx_count=%d, tx_bytes=%llu, " - "rx_count=%d, rx_bytes=%llu", + " Bulk: tx_count=%d, tx_bytes=%lu, " + "rx_count=%d, rx_bytes=%lu", service->stats.bulk_tx_count, service->stats.bulk_tx_bytes, service->stats.bulk_rx_count, diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kern_lib.c @@ -33,6 +33,11 @@ /* ---- Include Files ---------------------------------------------------- */ +#define __STDC_FORMAT_MACROS +#include "/usr/src/include/inttypes.h" +#include +// #include +// #include "/usr/src/include/stdio.h" #include "vchiq_core.h" #include "vchiq_arm.h" #include "vchiq_killable.h" @@ -151,9 +156,9 @@ list); list_del(pos); vchiq_log_info(vchiq_arm_log_level, - "bulk_waiter - cleaned up %x " + "bulk_waiter - cleaned up 0x%" PRIx64 "for pid %d", - (unsigned int)waiter, waiter->pid); + (unsigned long )waiter, waiter->pid); _sema_destroy(&waiter->bulk_waiter.event); kfree(waiter); @@ -454,8 +459,8 @@ list_add(&waiter->list, &instance->bulk_waiter_list); lmutex_unlock(&instance->bulk_waiter_list_mutex); vchiq_log_info(vchiq_arm_log_level, - "saved bulk_waiter %x for pid %d", - (unsigned int)waiter, current->p_pid); + "saved bulk_waiter 0x%" PRIx64 " for pid %d", + (unsigned long) waiter, current->p_pid); } return status; diff --git a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c --- a/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c +++ b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_kmod.c @@ -47,7 +47,11 @@ #include #include +/* XXXMDC Is this necessary at all? */ +#if defined(__aarch64__) +#else #include +#endif #include "vchiq_arm.h" #include "vchiq_2835.h" @@ -78,13 +82,31 @@ static struct bcm_vchiq_softc *bcm_vchiq_sc = NULL; -#define BSD_DTB 1 -#define UPSTREAM_DTB 2 + +#define CONFIG_INVALID 0 +#define CONFIG_VALID 1 << 0 +#define BSD_REG_ADDRS 1 << 1 +#define LONG_BULK_SPACE 1 << 2 + +/* + * Also controls the use of the standard VC address offset for bulk data DMA + * (normal bulks use that offset; bulks for long address spaces use physical + * page addresses) + */ +extern unsigned int g_long_bulk_space; + + +/* + * XXXMDC + * The man page for ofw_bus_is_compatible describes ``features'' + * as ``can be used''. Here we use understand them as ``must be used'' + */ + static struct ofw_compat_data compat_data[] = { - {"broadcom,bcm2835-vchiq", BSD_DTB}, - {"brcm,bcm2835-vchiq", UPSTREAM_DTB}, - {"brcm,bcm2711-vchiq", UPSTREAM_DTB}, - {NULL, 0} + {"broadcom,bcm2835-vchiq", BSD_REG_ADDRS | CONFIG_VALID}, + {"brcm,bcm2835-vchiq", CONFIG_VALID}, + {"brcm,bcm2711-vchiq", LONG_BULK_SPACE | CONFIG_VALID}, + {NULL, CONFIG_INVALID} }; #define vchiq_read_4(reg) \ @@ -119,13 +141,23 @@ void remote_event_signal(REMOTE_EVENT_T *event) { - event->fired = 1; + wmb(); + + event->fired = 1; /* The test on the next line also ensures the write on the previous line has completed */ + /* UPDATE: not on arm64, it would seem... */ +#if defined(__aarch64__) + dsb(sy); +#endif if (event->armed) { /* trigger vc interrupt */ +#if defined(__aarch64__) + dsb(sy); +#else dsb(); +#endif vchiq_write_4(0x48, 0); } } @@ -134,13 +166,17 @@ bcm_vchiq_probe(device_t dev) { - if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + if ((ofw_bus_search_compatible(dev, compat_data)->ocd_data & CONFIG_VALID) == 0) return (ENXIO); device_set_desc(dev, "BCM2835 VCHIQ"); return (BUS_PROBE_DEFAULT); } +/* debug_sysctl */ +extern int vchiq_core_log_level; +extern int vchiq_arm_log_level; + static int bcm_vchiq_attach(device_t dev) { @@ -168,14 +204,36 @@ return (ENXIO); } - if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == UPSTREAM_DTB) + uintptr_t dev_compat_d = ofw_bus_search_compatible(dev, compat_data)->ocd_data; + /* XXXMDC: shouldn't happen (checked for in probe)--but, for symmetry */ + if ((dev_compat_d & CONFIG_VALID) == 0){ + device_printf(dev, "attempting to attach using invalid config.\n"); + bus_release_resource(dev, SYS_RES_IRQ, rid, sc->irq_res); + return (EINVAL); + } + if ((dev_compat_d & BSD_REG_ADDRS) == 0) sc->regs_offset = -0x40; + if(dev_compat_d & LONG_BULK_SPACE) + g_long_bulk_space = 1; node = ofw_bus_get_node(dev); if ((OF_getencprop(node, "cache-line-size", &cell, sizeof(cell))) > 0) g_cache_line_size = cell; vchiq_core_initialize(); + + /* debug_sysctl */ + struct sysctl_ctx_list *ctx_l = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree_node = device_get_sysctl_tree(dev); + struct sysctl_oid_list *tree = SYSCTL_CHILDREN(tree_node); + SYSCTL_ADD_INT( + ctx_l, tree, OID_AUTO, "log", CTLFLAG_RW, + &vchiq_core_log_level, vchiq_core_log_level, "log level" + ); + SYSCTL_ADD_INT( + ctx_l, tree, OID_AUTO, "arm_log", CTLFLAG_RW, + &vchiq_arm_log_level, vchiq_arm_log_level, "arm log level" + ); /* Setup and enable the timer */ if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE,