Differential D28340 Diff 84147 sys/arm64/arm64/gicv3_its.c

Changeset View

Standalone View

sys/arm64/arm64/gicv3_its.c

Show All 35 Lines

#include <sys/cdefs.h>		#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");		__FBSDID("$FreeBSD$");

#include <sys/param.h>		#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/bus.h>		#include <sys/bus.h>
#include <sys/cpuset.h>		#include <sys/cpuset.h>
		#include <sys/domainset.h>
#include <sys/endian.h>		#include <sys/endian.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/module.h>		#include <sys/module.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/proc.h>		#include <sys/proc.h>
#include <sys/taskqueue.h>		#include <sys/taskqueue.h>
▲ Show 20 Lines • Show All 189 Lines • ▼ Show 20 Lines
};		};

struct gicv3_its_softc {		struct gicv3_its_softc {
device_t dev;		device_t dev;
struct intr_pic *sc_pic;		struct intr_pic *sc_pic;
struct resource *sc_its_res;		struct resource *sc_its_res;

cpuset_t sc_cpus;		cpuset_t sc_cpus;
		struct domainset *sc_ds;
		markjUnsubmitted Done Inline Actions Most fields here have an sc_ prefix, it'd be nice to try and keep it consistent. markj: Most fields here have an sc_ prefix, it'd be nice to try and keep it consistent.
u_int gic_irq_cpu;		u_int gic_irq_cpu;

struct its_ptable sc_its_ptab[GITS_BASER_NUM];		struct its_ptable sc_its_ptab[GITS_BASER_NUM];
struct its_col sc_its_cols[MAXCPU]; / Per-CPU collections */		struct its_col sc_its_cols[MAXCPU]; / Per-CPU collections */

/*		/*
* TODO: We should get these from the parent as we only want a		* TODO: We should get these from the parent as we only want a
* single copy of each across the interrupt controller.		* single copy of each across the interrupt controller.
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines

static void		static void
gicv3_its_cmdq_init(struct gicv3_its_softc *sc)		gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
{		{
vm_paddr_t cmd_paddr;		vm_paddr_t cmd_paddr;
uint64_t reg, tmp;		uint64_t reg, tmp;

/* Set up the command circular buffer */		/* Set up the command circular buffer */
sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS,		sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
M_WAITOK \| M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0);		sc->sc_ds, M_WAITOK \| M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
		0);
sc->sc_its_cmd_next_idx = 0;		sc->sc_its_cmd_next_idx = 0;

cmd_paddr = vtophys(sc->sc_its_cmd_base);		cmd_paddr = vtophys(sc->sc_its_cmd_base);

/* Set the base of the command buffer */		/* Set the base of the command buffer */
reg = GITS_CBASER_VALID \|		reg = GITS_CBASER_VALID \|
(GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) \|		(GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) \|
cmd_paddr \| (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) \|		cmd_paddr \| (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) \|
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines	case GITS_BASER_TYPE_IC:
its_tbl_size = page_size;		its_tbl_size = page_size;
break;		break;
default:		default:
continue;		continue;
}		}
npages = howmany(its_tbl_size, PAGE_SIZE);		npages = howmany(its_tbl_size, PAGE_SIZE);

/* Allocate the table */		/* Allocate the table */
table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,		table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
M_GICV3_ITS, M_WAITOK \| M_ZERO, 0, (1ul << 48) - 1,		M_GICV3_ITS, sc->sc_ds, M_WAITOK \| M_ZERO, 0,
PAGE_SIZE_64K, 0);		(1ul << 48) - 1, PAGE_SIZE_64K, 0);

sc->sc_its_ptab[i].ptab_vaddr = table;		sc->sc_its_ptab[i].ptab_vaddr = table;
sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;		sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE;

paddr = vtophys(table);		paddr = vtophys(table);

while (1) {		while (1) {
nitspages = howmany(its_tbl_size, page_size);		nitspages = howmany(its_tbl_size, page_size);
▲ Show 20 Lines • Show All 339 Lines • ▼ Show 20 Lines	if (sc->sc_its_res == NULL) {
return (ENXIO);		return (ENXIO);
}		}

phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +		phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
GITS_TRANSLATER, PAGE_SIZE);		GITS_TRANSLATER, PAGE_SIZE);
sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK \| M_ZERO);		sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK \| M_ZERO);
vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);		vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);

		CPU_COPY(&all_cpus, &sc->sc_cpus);
iidr = gic_its_read_4(sc, GITS_IIDR);		iidr = gic_its_read_4(sc, GITS_IIDR);
for (i = 0; i < nitems(its_quirks); i++) {		for (i = 0; i < nitems(its_quirks); i++) {
if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {		if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
if (bootverbose) {		if (bootverbose) {
device_printf(dev, "Applying %s\n",		device_printf(dev, "Applying %s\n",
its_quirks[i].desc);		its_quirks[i].desc);
}		}
its_quirks[i].func(dev);		its_quirks[i].func(dev);
break;		break;
}		}
}		}

		if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
		sc->sc_ds = DOMAINSET_PREF(domain);
		} else {
		sc->sc_ds = DOMAINSET_RR();
		}

/* Allocate the private tables */		/* Allocate the private tables */
err = gicv3_its_table_init(dev, sc);		err = gicv3_its_table_init(dev, sc);
if (err != 0)		if (err != 0)
return (err);		return (err);

/* Protects access to the device list */		/* Protects access to the device list */
mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);		mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);

/* Protects access to the ITS command circular buffer. */		/* Protects access to the ITS command circular buffer. */
mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);		mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);

CPU_ZERO(&sc->sc_cpus);
if (bus_get_domain(dev, &domain) == 0) {
if (domain < MAXMEMDOM)
CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
} else {
CPU_COPY(&all_cpus, &sc->sc_cpus);
}

/* Allocate the command circular buffer */		/* Allocate the command circular buffer */
gicv3_its_cmdq_init(sc);		gicv3_its_cmdq_init(sc);

/* Allocate the per-CPU collections */		/* Allocate the per-CPU collections */
for (int cpu = 0; cpu <= mp_maxid; cpu++)		for (int cpu = 0; cpu <= mp_maxid; cpu++)
if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)		if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
sc->sc_its_cols[cpu] = malloc(		sc->sc_its_cols[cpu] = malloc_domainset(
sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,		sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
		DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
		markjUnsubmitted Done Inline Actions Don't you want to allocate these from the domain local to the CPU? markj: Don't you want to allocate these from the domain local to the CPU?
		scottphAuthorUnsubmitted Done Inline Actions Right you are, I had mistaken this for the hardware's collection table which is actually allocated up on line 492. scottph: Right you are, I had mistaken this for the hardware's collection table which is actually…
M_WAITOK \| M_ZERO);		M_WAITOK \| M_ZERO);
else		else
sc->sc_its_cols[cpu] = NULL;		sc->sc_its_cols[cpu] = NULL;

/* Enable the ITS */		/* Enable the ITS */
gic_its_write_4(sc, GITS_CTLR,		gic_its_write_4(sc, GITS_CTLR,
gic_its_read_4(sc, GITS_CTLR) \| GITS_CTLR_EN);		gic_its_read_4(sc, GITS_CTLR) \| GITS_CTLR_EN);

Show All 35 Lines	gicv3_its_detach(device_t dev)

return (ENXIO);		return (ENXIO);
}		}

static void		static void
its_quirk_cavium_22375(device_t dev)		its_quirk_cavium_22375(device_t dev)
{		{
struct gicv3_its_softc *sc;		struct gicv3_its_softc *sc;
		int domain;

sc = device_get_softc(dev);		sc = device_get_softc(dev);
sc->sc_its_flags \|= ITS_FLAGS_ERRATA_CAVIUM_22375;		sc->sc_its_flags \|= ITS_FLAGS_ERRATA_CAVIUM_22375;

		/*
		* We need to limit which CPUs we send these interrupts to on
		* the original dual socket ThunderX as it is unable to
		* forward them between the two sockets.
		*/
		if (bus_get_domain(dev, &domain) == 0) {
		if (domain < MAXMEMDOM) {
		CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
		} else {
		CPU_ZERO(&sc->sc_cpus);
}		}
		}
		}

static void		static void
gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)		gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
{		{
struct gicv3_its_softc *sc;		struct gicv3_its_softc *sc;
struct gicv3_its_irqsrc *girq;		struct gicv3_its_irqsrc *girq;
uint8_t *conf;		uint8_t *conf;

▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines	its_device_get(device_t dev, device_t child, u_int nvecs)
/* Get ITT entry size */		/* Get ITT entry size */
esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));		esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));

/*		/*
* Allocate ITT for this device.		* Allocate ITT for this device.
* PA has to be 256 B aligned. At least two entries for device.		* PA has to be 256 B aligned. At least two entries for device.
*/		*/
its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);		its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size,		its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
M_GICV3_ITS, M_NOWAIT \| M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR,		M_GICV3_ITS, sc->sc_ds, M_NOWAIT \| M_ZERO, 0,
LPI_INT_TRANS_TAB_ALIGN, 0);		LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
if (its_dev->itt == 0) {		if (its_dev->itt == 0) {
vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);		vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
free(its_dev, M_GICV3_ITS);		free(its_dev, M_GICV3_ITS);
return (NULL);		return (NULL);
}		}

mtx_lock_spin(&sc->sc_its_dev_lock);		mtx_lock_spin(&sc->sc_its_dev_lock);
TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);		TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
▲ Show 20 Lines • Show All 827 Lines • Show Last 20 Lines