Differential D53738 Diff 169171 sys/arm64/spe/arm_spe_backend.c

Changeset View

Standalone View

sys/arm64/spe/arm_spe_backend.c

Show First 20 Lines • Show All 85 Lines • ▼ Show 20 Lines
#include <sys/hwt.h>		#include <sys/hwt.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/mman.h>		#include <sys/mman.h>
#include <sys/module.h>		#include <sys/module.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/proc.h>		#include <sys/proc.h>
		#include <sys/queue.h>
#include <sys/rman.h>		#include <sys/rman.h>
#include <sys/rwlock.h>		#include <sys/rwlock.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/systm.h>		#include <sys/systm.h>
		#include <sys/taskqueue.h>

#include <machine/bus.h>		#include <machine/bus.h>

#include <arm64/spe/arm_spe_dev.h>		#include <arm64/spe/arm_spe_dev.h>

#include <dev/hwt/hwt_vm.h>		#include <dev/hwt/hwt_vm.h>
#include <dev/hwt/hwt_backend.h>		#include <dev/hwt/hwt_backend.h>
#include <dev/hwt/hwt_config.h>		#include <dev/hwt/hwt_config.h>
Show All 11 Lines
static device_t spe_dev;		static device_t spe_dev;
static struct hwt_backend_ops spe_ops;		static struct hwt_backend_ops spe_ops;
static struct hwt_backend backend = {		static struct hwt_backend backend = {
.ops = &spe_ops,		.ops = &spe_ops,
.name = "spe",		.name = "spe",
.kva_req = 1,		.kva_req = 1,
};		};

static struct arm_spe_info *spe_info;		/* Pointers to current info structure per CPU. This points to either a per-CPU
		* structure (for CPU mode) or a per-thread structure (for thread mode).
		*/
		static struct arm_spe_info **spe_info;

static int		static struct arm_spe_info *spe_info_cpu;
spe_backend_init_thread(struct hwt_context *ctx)
{
return (ENOTSUP);
}

static void		static void
spe_backend_init_cpu(struct hwt_context *ctx)		spe_backend_init_cpu(struct hwt_context *ctx)
{		{
struct arm_spe_info *info;		struct arm_spe_info *info;
struct arm_spe_softc *sc = device_get_softc(spe_dev);		struct arm_spe_softc *sc = device_get_softc(spe_dev);
char lock_name[32];		char lock_name[32];
char *tmp = "Arm SPE lock/cpu/";		char *tmp = "Arm SPE lock/cpu/";
int cpu_id;		int cpu_id;

spe_info = malloc(sizeof(struct arm_spe_info) * mp_ncpus,		spe_info_cpu = malloc(sizeof(struct arm_spe_info) * mp_ncpus,
M_ARM_SPE, M_WAITOK \| M_ZERO);		M_ARM_SPE, M_WAITOK \| M_ZERO);

sc->spe_info = spe_info;

CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {		CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
info = &spe_info[cpu_id];		info = &spe_info_cpu[cpu_id];
info->sc = sc;		info->sc = sc;
info->ident = cpu_id;		info->ident = cpu_id;
info->buf_info[0].info = info;		info->buf_info[0].info = info;
info->buf_info[0].buf_idx = 0;		info->buf_info[0].buf_idx = 0;
info->buf_info[1].info = info;		info->buf_info[1].info = info;
info->buf_info[1].buf_idx = 1;		info->buf_info[1].buf_idx = 1;
snprintf(lock_name, sizeof(lock_name), "%s%d", tmp, cpu_id);		snprintf(lock_name, sizeof(lock_name), "%s%d", tmp, cpu_id);
mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);		mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);

		spe_info[cpu_id] = info;
}		}
}		}

static int		static int
spe_backend_init(struct hwt_context *ctx)		spe_backend_init(struct hwt_context *ctx)
{		{
struct arm_spe_softc *sc = device_get_softc(spe_dev);		struct arm_spe_softc *sc = device_get_softc(spe_dev);
int error = 0;		int error = 0;
Show All 12 Lines	spe_backend_init(struct hwt_context *ctx)
*/		*/
if (ctx->bufsize < (2 * PAGE_SIZE))		if (ctx->bufsize < (2 * PAGE_SIZE))
return (EINVAL);		return (EINVAL);

sc->ctx = ctx;		sc->ctx = ctx;
sc->kqueue_fd = ctx->kqueue_fd;		sc->kqueue_fd = ctx->kqueue_fd;
sc->hwt_td = ctx->hwt_td;		sc->hwt_td = ctx->hwt_td;

if (ctx->mode == HWT_MODE_THREAD)		spe_info = malloc(sizeof(struct arm_spe_info ) mp_ncpus,
error = spe_backend_init_thread(ctx);		M_ARM_SPE, M_WAITOK \| M_ZERO);
else		sc->spe_info = spe_info;

		if (ctx->mode == HWT_MODE_CPU)
spe_backend_init_cpu(ctx);		spe_backend_init_cpu(ctx);

return (error);		return (error);
}		}

#ifdef ARM_SPE_DEBUG		#ifdef ARM_SPE_DEBUG
static void hex_dump(uint8_t *buf, size_t len)		static void hex_dump(uint8_t *buf, size_t len)
{		{
Show All 16 Lines
}		}
#endif		#endif

static int		static int
spe_backend_deinit(struct hwt_context *ctx)		spe_backend_deinit(struct hwt_context *ctx)
{		{
#ifdef ARM_SPE_DEBUG		#ifdef ARM_SPE_DEBUG
struct arm_spe_info *info;		struct arm_spe_info *info;
		struct hwt_thread *thr;
int cpu_id;		int cpu_id;

		if (ctx->mode == HWT_MODE_CPU) {
CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {		CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
info = &spe_info[cpu_id];		info = &spe_info_cpu[cpu_id];
		printf("CPU %u:\n", cpu_id);
hex_dump((void *)info->kvaddr, 128);		hex_dump((void *)info->kvaddr, 128);
hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);		hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
}		}
		} else {
		TAILQ_FOREACH(thr, &ctx->threads, next) {
		info = (struct arm_spe_info *)thr->private;
		printf("TID %u:\n", thr->thread_id);
		hex_dump((void *)info->kvaddr, 128);
		hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
		}
		}
#endif		#endif

if (ctx->state == CTX_STATE_RUNNING) {
spe_backend_disable_smp(ctx);		spe_backend_disable_smp(ctx);
ctx->state = CTX_STATE_STOPPED;
}

		if (ctx->mode == HWT_MODE_CPU)
		free(spe_info_cpu, M_ARM_SPE);

free(spe_info, M_ARM_SPE);		free(spe_info, M_ARM_SPE);

return (0);		return (0);
}		}

static uint64_t		static uint64_t
arm_spe_min_interval(struct arm_spe_softc *sc)		arm_spe_min_interval(struct arm_spe_softc *sc)
{		{
Show All 31 Lines	arm_spe_set_interval(struct arm_spe_info *info, uint64_t interval)

dprintf("%s %lu\n", __func__, interval);		dprintf("%s %lu\n", __func__, interval);

info->pmsirr &= ~(PMSIRR_INTERVAL_MASK);		info->pmsirr &= ~(PMSIRR_INTERVAL_MASK);
info->pmsirr \|= (interval << PMSIRR_INTERVAL_SHIFT);		info->pmsirr \|= (interval << PMSIRR_INTERVAL_SHIFT);
}		}

static int		static int
spe_backend_configure(struct hwt_context *ctx, int cpu_id, int session_id)		spe_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id)
{		{
struct arm_spe_info *info = &spe_info[cpu_id];		struct arm_spe_info *info = NULL;
struct arm_spe_config *cfg;		struct arm_spe_config *cfg;
		struct hwt_thread *thr = NULL;
int err = 0;		int err = 0;

		if (ctx->mode == HWT_MODE_CPU)
		info = &spe_info_cpu[cpu_id];
		else {
		TAILQ_FOREACH(thr, &ctx->threads, next) {
		if (thr->thread_id != thread_id)
		continue;
		info = (struct arm_spe_info *)thr->private;
		break;
		}
		if (info == NULL)
		return (ENOENT);
		}

mtx_lock_spin(&info->lock);		mtx_lock_spin(&info->lock);
		if (ctx->mode == HWT_MODE_CPU)
info->ident = cpu_id;		info->ident = cpu_id;
		else
		info->ident = thread_id;
/* Set defaults */		/* Set defaults */
info->pmsfcr = 0;		info->pmsfcr = 0;
info->pmsevfr = 0xFFFFFFFFFFFFFFFFUL;		info->pmsevfr = 0xFFFFFFFFFFFFFFFFUL;
info->pmslatfr = 0;		info->pmslatfr = 0;
info->pmsirr =		info->pmsirr =
(arm_spe_min_interval(info->sc) << PMSIRR_INTERVAL_SHIFT)		(arm_spe_min_interval(info->sc) << PMSIRR_INTERVAL_SHIFT)
\| PMSIRR_RND;		\| PMSIRR_RND;
info->pmsicr = 0;		info->pmsicr = 0;
info->pmscr = PMSCR_TS \| PMSCR_PA \| PMSCR_CX \| PMSCR_E1SPE \| PMSCR_E0SPE;		info->pmscr = PMSCR_TS \| PMSCR_PA \| PMSCR_CX \| PMSCR_E1SPE \| PMSCR_E0SPE;

if (ctx->config != NULL &&		if (ctx->config != NULL &&
ctx->config_size == sizeof(struct arm_spe_config) &&		ctx->config_size == sizeof(struct arm_spe_config) &&
ctx->config_version == 1) {		ctx->config_version == 1) {
cfg = (struct arm_spe_config *)ctx->config;		cfg = (struct arm_spe_config *)ctx->config;
if (cfg->interval)		if (cfg->interval)
arm_spe_set_interval(info, cfg->interval);		arm_spe_set_interval(info, cfg->interval);
if (cfg->level == ARM_SPE_KERNEL_ONLY)		if (cfg->level == ARM_SPE_KERNEL_ONLY)
info->pmscr &= ~(PMSCR_E0SPE); /* turn off user */		info->pmscr &= ~(PMSCR_E0SPE); /* turn off user */
if (cfg->level == ARM_SPE_USER_ONLY)		if (cfg->level == ARM_SPE_USER_ONLY)
info->pmscr &= ~(PMSCR_E1SPE); /* turn off kern */		info->pmscr &= ~(PMSCR_E1SPE); /* turn off kern */
if (cfg->ctx_field)		if (cfg->ctx_field)
info->ctx_field = cfg->ctx_field;		info->ctx_field = cfg->ctx_field;
} else		} else
err = (EINVAL);		err = (EINVAL);

		if (ctx->mode == HWT_MODE_THREAD) {
		info->kvaddr = thr->vm->kvaddr;
		info->buf_size = ctx->bufsize;
		}

		spe_info[cpu_id] = info;
mtx_unlock_spin(&info->lock);		mtx_unlock_spin(&info->lock);

return (err);		return (err);
}		}


static void		static void
arm_spe_enable(void *arg __unused)		arm_spe_enable(void *arg __unused)
{		{
struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)];		struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
		struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
		struct hwt_context *ctx = info->sc->ctx;
uint64_t base, limit;		uint64_t base, limit;

dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));		dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));

mtx_lock_spin(&info->lock);		mtx_lock_spin(&info->lock);

		if (info->stopped) {
		mtx_unlock_spin(&info->lock);
		return;
		}

if (info->ctx_field == ARM_SPE_CTX_CPU_ID)		if (info->ctx_field == ARM_SPE_CTX_CPU_ID)
WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, PCPU_GET(cpuid));		WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, PCPU_GET(cpuid));

WRITE_SPECIALREG(PMSFCR_EL1_REG, info->pmsfcr);		WRITE_SPECIALREG(PMSFCR_EL1_REG, info->pmsfcr);
WRITE_SPECIALREG(PMSEVFR_EL1_REG, info->pmsevfr);		WRITE_SPECIALREG(PMSEVFR_EL1_REG, info->pmsevfr);
WRITE_SPECIALREG(PMSLATFR_EL1_REG, info->pmslatfr);		WRITE_SPECIALREG(PMSLATFR_EL1_REG, info->pmslatfr);

/* Set the sampling interval */		/* Set the sampling interval */
WRITE_SPECIALREG(PMSIRR_EL1_REG, info->pmsirr);		WRITE_SPECIALREG(PMSIRR_EL1_REG, info->pmsirr);
isb();		isb();

/* Write 0 here before enabling sampling */		/* Write 0 here before enabling sampling */
WRITE_SPECIALREG(PMSICR_EL1_REG, info->pmsicr);		WRITE_SPECIALREG(PMSICR_EL1_REG, info->pmsicr);
isb();		isb();

base = info->kvaddr;		base = buf_start_addr(info->buf_idx, info);
limit = base + (info->buf_size/2);		limit = base + (info->buf_size/2);
/* Enable the buffer */		/* Enable the buffer */
limit &= PMBLIMITR_LIMIT_MASK; /* Zero lower 12 bits */		limit &= PMBLIMITR_LIMIT_MASK; /* Zero lower 12 bits */
limit \|= PMBLIMITR_E;		limit \|= PMBLIMITR_E;
/* Set the base and limit */		/* Set the base and limit. Restore base pointer if sampling has previously
		* been enabled for this thread.
		*/
		if (buf->pmbptr == 0) {
WRITE_SPECIALREG(PMBPTR_EL1_REG, base);		WRITE_SPECIALREG(PMBPTR_EL1_REG, base);
		} else {
		WRITE_SPECIALREG(PMBPTR_EL1_REG, buf->pmbptr);
		}
WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);		WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);
isb();		isb();

/* Enable sampling */		/* Enable sampling */
WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);		WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
isb();		isb();

info->enabled = true;		info->enabled = true;

		if (ctx->mode == HWT_MODE_THREAD)
		CPU_SET(PCPU_GET(cpuid), &ctx->cpu_map);

mtx_unlock_spin(&info->lock);		mtx_unlock_spin(&info->lock);
}		}

static int		static int
spe_backend_enable_smp(struct hwt_context *ctx)		spe_backend_enable_smp(struct hwt_context *ctx)
{		{
struct arm_spe_info *info;		struct arm_spe_info *info;
struct hwt_vm *vm;		struct hwt_vm *vm;
int cpu_id;		int cpu_id;

		KASSERT(ctx->mode == HWT_MODE_CPU, ("%s: should only be called for CPU mode", __func__));

HWT_CTX_LOCK(ctx);		HWT_CTX_LOCK(ctx);
CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {		CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
vm = hwt_cpu_get(ctx, cpu_id)->vm;		vm = hwt_cpu_get(ctx, cpu_id)->vm;
		KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
		info = &spe_info_cpu[cpu_id];

info = &spe_info[cpu_id];

mtx_lock_spin(&info->lock);		mtx_lock_spin(&info->lock);
info->kvaddr = vm->kvaddr;		info->kvaddr = vm->kvaddr;
info->buf_size = ctx->bufsize;		info->buf_size = ctx->bufsize;
mtx_unlock_spin(&info->lock);		mtx_unlock_spin(&info->lock);
}		}
HWT_CTX_UNLOCK(ctx);		HWT_CTX_UNLOCK(ctx);

cpu_id = CPU_FFS(&ctx->cpu_map) - 1;		cpu_id = CPU_FFS(&ctx->cpu_map) - 1;
info = &spe_info[cpu_id];		KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
		info = spe_info[cpu_id];
if (info->ctx_field == ARM_SPE_CTX_PID)		if (info->ctx_field == ARM_SPE_CTX_PID)
arm64_pid_in_contextidr = true;		arm64_pid_in_contextidr = true;
else		else
arm64_pid_in_contextidr = false;		arm64_pid_in_contextidr = false;

smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,		smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
arm_spe_enable, smp_no_rendezvous_barrier, NULL);		arm_spe_enable, smp_no_rendezvous_barrier, NULL);

return (0);		return (0);
}		}

void		static void
arm_spe_disable(void *arg __unused)		arm_spe_disable_nolock(void)
{		{
struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)];		struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];		struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
		struct hwt_context *ctx = info->sc->ctx;

if (!info->enabled)		if (!info->enabled)
return;		return;

dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));		dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));

/* Disable profiling */		/* Disable profiling */
WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);		WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);
isb();		isb();

/* Drain any remaining tracing data */		/* Drain any remaining tracing data */
psb_csync();		psb_csync();
dsb(nsh);		dsb(nsh);

/* Disable the profiling buffer */		/* Disable the profiling buffer */
WRITE_SPECIALREG(PMBLIMITR_EL1_REG, 0);		WRITE_SPECIALREG(PMBLIMITR_EL1_REG, 0);
isb();		isb();

/* Clear interrupt status reg */		/* Clear interrupt status reg */
WRITE_SPECIALREG(PMBSR_EL1_REG, 0x0);		WRITE_SPECIALREG(PMBSR_EL1_REG, 0x0);

/* Clear PID/CPU_ID from context ID reg */		/* Clear PID/CPU_ID from context ID reg */
WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, 0);		WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, 0);

mtx_lock_spin(&info->lock);
buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);		buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);
info->enabled = false;		info->enabled = false;

		if (ctx->mode == HWT_MODE_THREAD)
		CPU_CLR(PCPU_GET(cpuid), &ctx->cpu_map);
		}

		void
		arm_spe_disable(void *arg __unused)
		{
		struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

		mtx_lock_spin(&info->lock);
		arm_spe_disable_nolock();
mtx_unlock_spin(&info->lock);		mtx_unlock_spin(&info->lock);
}		}

int		int
spe_backend_disable_smp(struct hwt_context *ctx)		spe_backend_disable_smp(struct hwt_context *ctx)
{		{
struct kevent kev;		struct kevent kev;
struct arm_spe_info *info;		struct arm_spe_info *info;
struct arm_spe_buf_info *buf;		struct arm_spe_buf_info *buf;
int cpu_id;		int cpu_id;
int ret;		int ret;

		if (!CPU_EMPTY(&ctx->cpu_map)) {
/* Disable and send out remaining data in bufs */		/* Disable and send out remaining data in bufs */
smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,		smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
arm_spe_disable, smp_no_rendezvous_barrier, NULL);		arm_spe_disable, smp_no_rendezvous_barrier, NULL);

CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {		CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
info = &spe_info[cpu_id];		info = spe_info[cpu_id];
buf = &info->buf_info[info->buf_idx];		buf = &info->buf_info[info->buf_idx];
arm_spe_send_buffer(buf, 0);		arm_spe_send_buffer(buf, 0);
}		}
		}

arm64_pid_in_contextidr = false;		arm64_pid_in_contextidr = false;

/*		/*
* Tracing on all CPUs has been disabled, and we've sent write ptr		* Tracing on all CPUs has been disabled, and we've sent write ptr
* offsets for all bufs - let userspace know it can shutdown		* offsets for all bufs - let userspace know it can shutdown
*/		*/
EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);		EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);		ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);
if (ret)		if (ret)
dprintf("%s kqfd_register ret:%d\n", __func__, ret);		dprintf("%s kqfd_register ret:%d\n", __func__, ret);

return (0);		return (0);
}		}

static void		static void
		spe_backend_enable(struct hwt_context *ctx, int cpu_id)
		{
		struct arm_spe_info *info;

		if (ctx->mode == HWT_MODE_CPU)
		return;
		KASSERT(curcpu == cpu_id, ("%s: attempting to enable SPE on another cpu", __func__));

		info = spe_info[cpu_id];

		KASSERT(info != NULL, ("%s: info=NULL", __func__));

		if (info->ctx_field == ARM_SPE_CTX_PID)
		arm64_pid_in_contextidr = true;
		else
		arm64_pid_in_contextidr = false;

		arm_spe_enable(NULL);
		}

		static void
		spe_backend_disable(struct hwt_context *ctx, int cpu_id)
		{
		struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

		if (ctx->mode == HWT_MODE_CPU)
		return;

		KASSERT(curcpu == cpu_id, ("%s: attempting to disable SPE on another cpu", __func__));

		mtx_lock_spin(&info->lock);

		if (!info->stopped)
		arm_spe_disable_nolock();

		mtx_unlock_spin(&info->lock);
		}

		static void
		arm_spe_flush(void *arg, int pending __unused)
		{
		struct arm_spe_info *info = arg;
		struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];

		arm_spe_send_buffer(buf, 0);
		}

		static void
spe_backend_stop(struct hwt_context *ctx)		spe_backend_stop(struct hwt_context *ctx)
{		{
		struct arm_spe_info *info;
		struct hwt_thread *thr;

		HWT_CTX_LOCK(ctx);

		if (ctx->mode == HWT_MODE_THREAD) {
		ctx->state = CTX_STATE_STOPPED;

		TAILQ_FOREACH(thr, &ctx->threads, next) {
		info = (struct arm_spe_info *)thr->private;

		mtx_lock_spin(&info->lock);

		info->stopped = true;

		if (!info->enabled) {
		/* Not currently tracing. Enqueue buffer for sending */
		TASK_INIT(&info->flush_task, 0, (task_fn_t *)arm_spe_flush, info);
		taskqueue_enqueue(taskqueue_arm_spe, &info->flush_task);
		}
		/* Otherwise tracing currently active. As this thread has been
		* marked as stopped, buffer will be sent on next disable
		*/

		mtx_unlock_spin(&info->lock);
		}

		}

		HWT_CTX_UNLOCK(ctx);

		taskqueue_drain_all(taskqueue_arm_spe);

spe_backend_disable_smp(ctx);		spe_backend_disable_smp(ctx);
}		}

static void		static void
arm_spe_reenable(void *arg __unused)		arm_spe_reenable(void *arg __unused)
{		{
struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)];;		struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];

WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);		WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
isb();		isb();
}		}

static int		static int
spe_backend_svc_buf(struct hwt_context ctx, void data, size_t data_size,		spe_backend_svc_buf(struct hwt_context ctx, void data, size_t data_size,
int data_version)		int data_version)
{		{
struct arm_spe_info *info;		struct arm_spe_info *info = NULL;
struct arm_spe_buf_info *buf;		struct arm_spe_buf_info *buf;
struct arm_spe_svc_buf *s;		struct arm_spe_svc_buf *s;
		struct hwt_thread *thr;
int err = 0;		int err = 0;
cpuset_t cpu_set;		cpuset_t cpu_set;

if (data_size != sizeof(struct arm_spe_svc_buf))		if (data_size != sizeof(struct arm_spe_svc_buf))
return (E2BIG);		return (E2BIG);

if (data_version != 1)		if (data_version != 1)
return (EINVAL);		return (EINVAL);

s = (struct arm_spe_svc_buf *)data;		s = (struct arm_spe_svc_buf *)data;
if (s->buf_idx > 1)		if (s->buf_idx > 1)
return (ENODEV);		return (ENODEV);

		if (ctx->mode == HWT_MODE_CPU) {
if (s->ident >= mp_ncpus)		if (s->ident >= mp_ncpus)
return (EINVAL);		return (EINVAL);

info = &spe_info[s->ident];		info = spe_info[s->ident];
		} else {
		TAILQ_FOREACH(thr, &ctx->threads, next) {
		if (thr->thread_id != s->ident)
		continue;
		info = (struct arm_spe_info *)thr->private;
		break;
		}

		if (info == NULL)
		return (ENOENT);
		}

		andrewUnsubmitted Not Done Inline Actions Is `s->info` a CPU ID or a thread ID in threaded mode? It looks like it's something userspace sets & it shouldn't need to know what CPU a thread is on. andrew: Is `s->info` a CPU ID or a thread ID in threaded mode? It looks like it's something userspace…
		sarah.walker2_arm.comAuthorUnsubmitted Done Inline Actions Do you mean `s->ident`? In threaded mode that's set in `spe_backend_configure()`, from an ID provided by the HWT scheduler hooks. sarah.walker2_arm.com: Do you mean `s->ident`? In threaded mode that's set in `spe_backend_configure()`, from an ID…
		andrewUnsubmitted Not Done Inline Actions Yes it should have been `s->ident`. If it comes from `info->ident` then that is a thread index, while `spe_info` is indexed by CPU ID. You probably need the same `TAILQ_FOREACH` loop as in `spe_backend_configure()` to find the info pointer. andrew: Yes it should have been `s->ident`. If it comes from `info->ident` then that is a thread index…
		sarah.walker2_arm.comAuthorUnsubmitted Done Inline Actions You are correct. Will fix. sarah.walker2_arm.com: You are correct. Will fix.
mtx_lock_spin(&info->lock);		mtx_lock_spin(&info->lock);

buf = &info->buf_info[s->buf_idx];		buf = &info->buf_info[s->buf_idx];

if (!info->enabled) {		if (!info->enabled && ctx->mode == HWT_MODE_CPU) {
err = ENXIO;		err = ENXIO;
goto end;		goto end;
}		}

/* Clear the flag the signals buffer needs servicing */		/* Clear the flag the signals buffer needs servicing */
buf->buf_svc = false;		buf->buf_svc = false;

/* Re-enable profiling if we've been waiting for this notification */		/* Re-enable profiling if we've been waiting for this notification */
if (buf->buf_wait) {		if (buf->buf_wait && !info->stopped) {
CPU_SETOF(s->ident, &cpu_set);		CPU_SETOF(s->ident, &cpu_set);

mtx_unlock_spin(&info->lock);		mtx_unlock_spin(&info->lock);
smp_rendezvous_cpus(cpu_set, smp_no_rendezvous_barrier,		smp_rendezvous_cpus(cpu_set, smp_no_rendezvous_barrier,
arm_spe_reenable, smp_no_rendezvous_barrier, NULL);		arm_spe_reenable, smp_no_rendezvous_barrier, NULL);
mtx_lock_spin(&info->lock);		mtx_lock_spin(&info->lock);

buf->buf_wait = false;		buf->buf_wait = false;
Show All 33 Lines	error:
mtx_unlock_spin(&sc->sc_lock);		mtx_unlock_spin(&sc->sc_lock);
if (error)		if (error)
return (error);		return (error);

free(q, M_ARM_SPE);		free(q, M_ARM_SPE);
return (0);		return (0);
}		}

		static int
		spe_backend_thread_alloc(struct hwt_thread *thr)
		{
		struct arm_spe_softc *sc = device_get_softc(spe_dev);
		char lock_name[32];
		struct arm_spe_info *info;
		andrewUnsubmitted Not Done Inline Actions It looks like this is only used in the `snprintf` & could be moved to the format string. andrew: It looks like this is only used in the `snprintf` & could be moved to the format string.

		info = malloc(sizeof(*info), M_ARM_SPE, M_WAITOK \| M_ZERO);

		andrewUnsubmitted Not Done Inline Actions Is there a reason to use `M_NOWAIT` here? If we can use `M_WAITOK` then the allocation is guaranteed to succeed. andrew: Is there a reason to use `M_NOWAIT` here? If we can use `M_WAITOK` then the allocation is…
		info->sc = sc;
		info->buf_info[0].info = info;
		info->buf_info[0].buf_idx = 0;
		info->buf_info[1].info = info;
		info->buf_info[1].buf_idx = 1;
		snprintf(lock_name, sizeof(lock_name), "Arm SPE lock/thr/%d", thr->thread_id);
		mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);

		thr->private = info;

		return (0);
		}

		static void
		spe_backend_thread_free(struct hwt_thread *thr)
		{
		struct arm_spe_info *info;

		info = (struct arm_spe_info *)thr->private;

		free(info, M_ARM_SPE);
		}

static struct hwt_backend_ops spe_ops = {		static struct hwt_backend_ops spe_ops = {
.hwt_backend_init = spe_backend_init,		.hwt_backend_init = spe_backend_init,
.hwt_backend_deinit = spe_backend_deinit,		.hwt_backend_deinit = spe_backend_deinit,

.hwt_backend_configure = spe_backend_configure,		.hwt_backend_configure = spe_backend_configure,
.hwt_backend_svc_buf = spe_backend_svc_buf,		.hwt_backend_svc_buf = spe_backend_svc_buf,
.hwt_backend_stop = spe_backend_stop,		.hwt_backend_stop = spe_backend_stop,

		.hwt_backend_enable = spe_backend_enable,
		.hwt_backend_disable = spe_backend_disable,

.hwt_backend_enable_smp = spe_backend_enable_smp,		.hwt_backend_enable_smp = spe_backend_enable_smp,
.hwt_backend_disable_smp = spe_backend_disable_smp,		.hwt_backend_disable_smp = spe_backend_disable_smp,

.hwt_backend_read = spe_backend_read,		.hwt_backend_read = spe_backend_read,

		.hwt_backend_thread_alloc = spe_backend_thread_alloc,
		.hwt_backend_thread_free = spe_backend_thread_free,
};		};

int		int
spe_register(device_t dev)		spe_register(device_t dev)
{		{
spe_dev = dev;		spe_dev = dev;

return (hwt_backend_register(&backend));		return (hwt_backend_register(&backend));
}		}