D2378.id6978.vs5978.diff
No OneTemporary
Actions

Size

60 KB

Referenced Files

None

Subscribers

None

D2378.id6978.vs5978.diff
View Options

	Index: sys/arm64/arm64/gic_v3.c
	===================================================================
	--- sys/arm64/arm64/gic_v3.c
	+++ sys/arm64/arm64/gic_v3.c
	@@ -236,19 +236,18 @@
	break;

	if (__predict_true((active_irq >= GIC_FIRST_PPI &&
	- active_irq <= GIC_LAST_SPI))) {
	+ active_irq <= GIC_LAST_SPI) \|\| active_irq >= GIC_FIRST_LPI)) {
	arm_dispatch_intr(active_irq, frame);
	continue;
	}

	- if (active_irq <= GIC_LAST_SGI \|\| active_irq >= GIC_FIRST_LPI) {
	+ if (active_irq <= GIC_LAST_SGI) {
	/*
	- * TODO: Implement proper SGI/LPI handling.
	+ * TODO: Implement proper SGI handling.
	* Mask it if such is received for some reason.
	*/
	device_printf(dev,
	- "Received unsupported interrupt type: %s\n",
	- active_irq >= GIC_FIRST_LPI ? "LPI" : "SGI");
	+ "Received unsupported interrupt type: SGI\n");
	PIC_MASK(dev, active_irq);
	}
	}
	@@ -275,6 +274,8 @@
	} else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */
	gic_r_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq));
	gic_v3_wait_for_rwp(sc, DIST);
	+ } else if (irq >= GIC_FIRST_LPI) { /* LPIs */
	+ lpi_mask_irq(dev, irq);
	} else
	panic("%s: Unsupported IRQ number %u", __func__, irq);
	}
	@@ -293,6 +294,8 @@
	} else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */
	gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq));
	gic_v3_wait_for_rwp(sc, DIST);
	+ } else if (irq >= GIC_FIRST_LPI) { /* LPIs */
	+ lpi_unmask_irq(dev, irq);
	} else
	panic("%s: Unsupported IRQ number %u", __func__, irq);
	}
	Index: sys/arm64/arm64/gic_v3_fdt.c
	===================================================================
	--- sys/arm64/arm64/gic_v3_fdt.c
	+++ sys/arm64/arm64/gic_v3_fdt.c
	@@ -35,6 +35,8 @@
	#include <sys/kernel.h>
	#include <sys/module.h>

	+#include <machine/resource.h>
	+
	#include <dev/fdt/fdt_common.h>
	#include <dev/ofw/openfirm.h>
	#include <dev/ofw/ofw_bus.h>
	@@ -51,11 +53,27 @@
	static int gic_v3_fdt_probe(device_t);
	static int gic_v3_fdt_attach(device_t);

	+static struct resource gic_v3_ofw_bus_alloc_res(device_t, device_t, int, int ,
	+ u_long, u_long, u_long, u_int);
	+static const struct ofw_bus_devinfo *gic_v3_ofw_get_devinfo(device_t, device_t);
	+
	static device_method_t gic_v3_fdt_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe, gic_v3_fdt_probe),
	DEVMETHOD(device_attach, gic_v3_fdt_attach),

	+ /* Bus interface */
	+ DEVMETHOD(bus_alloc_resource, gic_v3_ofw_bus_alloc_res),
	+ DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
	+
	+ /* ofw_bus interface */
	+ DEVMETHOD(ofw_bus_get_devinfo, gic_v3_ofw_get_devinfo),
	+ DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat),
	+ DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model),
	+ DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name),
	+ DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node),
	+ DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type),
	+
	/* End */
	DEVMETHOD_END
	};
	@@ -71,6 +89,11 @@
	0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);

	/*
	+ * Helper functions declarations.
	+ */
	+static int gic_v3_ofw_bus_attach(device_t);
	+
	+/*
	* Device interface.
	*/
	static int
	@@ -109,6 +132,17 @@
	err = gic_v3_attach(dev);
	if (err)
	goto error;
	+ /*
	+ * Try to register ITS to this GIC.
	+ * GIC will act as a bus in that case.
	+ * Failure here will not affect main GIC functionality.
	+ */
	+ if (gic_v3_ofw_bus_attach(dev) != 0) {
	+ if (bootverbose) {
	+ device_printf(dev,
	+ "Failed to attach ITS to this GIC\n");
	+ }
	+ }

	return (err);

	@@ -122,3 +156,155 @@

	return (err);
	}
	+
	+/* OFW bus interface */
	+struct gic_v3_ofw_devinfo {
	+ struct ofw_bus_devinfo di_dinfo;
	+ struct resource_list di_rl;
	+};
	+
	+static const struct ofw_bus_devinfo *
	+gic_v3_ofw_get_devinfo(device_t bus __unused, device_t child)
	+{
	+ struct gic_v3_ofw_devinfo *di;
	+
	+ di = device_get_ivars(child);
	+ return (&di->di_dinfo);
	+}
	+
	+static struct resource *
	+gic_v3_ofw_bus_alloc_res(device_t bus, device_t child, int type, int *rid,
	+ u_long start, u_long end, u_long count, u_int flags)
	+{
	+ struct gic_v3_ofw_devinfo *di;
	+ struct resource_list_entry *rle;
	+ int ranges_len;
	+
	+ if ((start == 0UL) && (end == ~0UL)) {
	+ if ((di = device_get_ivars(child)) == NULL)
	+ return (NULL);
	+ if (type != SYS_RES_MEMORY)
	+ return (NULL);
	+
	+ /* Find defaults for this rid */
	+ rle = resource_list_find(&di->di_rl, type, *rid);
	+ if (rle == NULL)
	+ return (NULL);
	+
	+ start = rle->start;
	+ end = rle->end;
	+ count = rle->count;
	+ }
	+ /*
	+ * XXX: No ranges remap!
	+ * Absolute address is expected.
	+ */
	+ if (ofw_bus_has_prop(bus, "ranges")) {
	+ ranges_len = OF_getproplen(ofw_bus_get_node(bus), "ranges");
	+ if (ranges_len != 0) {
	+ if (bootverbose) {
	+ device_printf(child,
	+ "Ranges remap not supported\n");
	+ }
	+ return (NULL);
	+ }
	+ }
	+ return (bus_generic_alloc_resource(bus, child, type, rid, start, end,
	+ count, flags));
	+}
	+
	+/* Helper functions */
	+
	+/*
	+ * Bus capability support for GICv3.
	+ * Collects and configures device informations and finally
	+ * adds ITS device as a child of GICv3 in Newbus hierarchy.
	+ */
	+static int
	+gic_v3_ofw_bus_attach(device_t dev)
	+{
	+ struct gic_v3_ofw_devinfo *di;
	+ device_t child;
	+ phandle_t parent, node;
	+ pcell_t addr_cells, size_cells;
	+
	+ parent = ofw_bus_get_node(dev);
	+ if (parent > 0) {
	+ addr_cells = 2;
	+ OF_getencprop(parent, "#address-cells", &addr_cells,
	+ sizeof(addr_cells));
	+ size_cells = 2;
	+ OF_getencprop(parent, "#size-cells", &size_cells,
	+ sizeof(size_cells));
	+ /* Iterate through all GIC subordinates */
	+ for (node = OF_child(parent); node > 0; node = OF_peer(node)) {
	+ /* Allocate and populate devinfo. */
	+ di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK \| M_ZERO);
	+ if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) {
	+ if (bootverbose) {
	+ device_printf(dev,
	+ "Could not set up devinfo for ITS\n");
	+ }
	+ free(di, M_GIC_V3);
	+ continue;
	+ }
	+
	+ /* Initialize and populate resource list. */
	+ resource_list_init(&di->di_rl);
	+ ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells,
	+ &di->di_rl);
	+
	+ /* Should not have any interrupts, so don't add any */
	+
	+ /* Add newbus device for this FDT node */
	+ child = device_add_child(dev, NULL, -1);
	+ if (!child) {
	+ if (bootverbose) {
	+ device_printf(dev,
	+ "Could not add child: %s\n",
	+ di->di_dinfo.obd_name);
	+ }
	+ resource_list_free(&di->di_rl);
	+ ofw_bus_gen_destroy_devinfo(&di->di_dinfo);
	+ free(di, M_GIC_V3);
	+ continue;
	+ }
	+
	+ device_set_ivars(child, di);
	+ }
	+ }
	+
	+ return (bus_generic_attach(dev));
	+}
	+
	+static int gic_v3_its_fdt_probe(device_t dev);
	+
	+static device_method_t gic_v3_its_fdt_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_probe, gic_v3_its_fdt_probe),
	+
	+ /* End */
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_1(gic_v3_its, gic_v3_its_fdt_driver, gic_v3_its_fdt_methods,
	+ sizeof(struct gic_v3_its_softc), gic_v3_its_driver);
	+
	+static devclass_t gic_v3_its_fdt_devclass;
	+
	+EARLY_DRIVER_MODULE(gic_v3_its, gic_v3, gic_v3_its_fdt_driver,
	+ gic_v3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
	+
	+static int
	+gic_v3_its_fdt_probe(device_t dev)
	+{
	+
	+ if (!ofw_bus_status_okay(dev))
	+ return (ENXIO);
	+
	+ if (!ofw_bus_is_compatible(dev, GIC_V3_ITS_COMPSTR))
	+ return (ENXIO);
	+
	+ device_set_desc(dev, GIC_V3_ITS_DEVSTR);
	+ return (BUS_PROBE_DEFAULT);
	+}
	Index: sys/arm64/arm64/gic_v3_its.c
	===================================================================
	--- /dev/null
	+++ sys/arm64/arm64/gic_v3_its.c
	@@ -0,0 +1,1448 @@
	+/*-
	+ * Copyright (c) 2015 The FreeBSD Foundation
	+ * All rights reserved.
	+ *
	+ * This software was developed by Semihalf under
	+ * the sponsorship of the FreeBSD Foundation.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ */
	+
	+#include <sys/cdefs.h>
	+__FBSDID("$FreeBSD$");
	+
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <sys/bitset.h>
	+#include <sys/bitstring.h>
	+#include <sys/bus.h>
	+#include <sys/endian.h>
	+#include <sys/kernel.h>
	+#include <sys/malloc.h>
	+#include <sys/rman.h>
	+#include <sys/pciio.h>
	+#include <sys/pcpu.h>
	+#include <sys/lock.h>
	+#include <sys/mutex.h>
	+
	+#include <dev/pci/pcivar.h>
	+
	+#include <vm/vm.h>
	+#include <vm/pmap.h>
	+
	+#include <machine/bus.h>
	+#include <machine/cpu.h>
	+#include <machine/cpufunc.h>
	+#include <machine/intr.h>
	+
	+#include "gic_v3_reg.h"
	+#include "gic_v3_var.h"
	+
	+#include "pic_if.h"
	+
	+/* Device and PIC methods */
	+static int gic_v3_its_attach(device_t);
	+
	+static device_method_t gic_v3_its_methods[] = {
	+ /* Device interface */
	+ DEVMETHOD(device_attach, gic_v3_its_attach),
	+ /*
	+ * PIC interface
	+ */
	+ /* MSI-X */
	+ DEVMETHOD(pic_alloc_msix, gic_v3_its_alloc_msix),
	+ DEVMETHOD(pic_map_msix, gic_v3_its_map_msix),
	+ /* MSI */
	+ DEVMETHOD(pic_alloc_msi, gic_v3_its_alloc_msi),
	+ DEVMETHOD(pic_map_msi, gic_v3_its_map_msix),
	+
	+ /* End */
	+ DEVMETHOD_END
	+};
	+
	+DEFINE_CLASS_0(gic_v3_its, gic_v3_its_driver, gic_v3_its_methods,
	+ sizeof(struct gic_v3_its_softc));
	+
	+MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR);
	+
	+static int its_alloc_tables(struct gic_v3_its_softc *);
	+static void its_free_tables(struct gic_v3_its_softc *);
	+static void its_init_commandq(struct gic_v3_its_softc *);
	+static int its_init_cpu(struct gic_v3_its_softc *);
	+static void its_init_cpu_collection(struct gic_v3_its_softc *);
	+
	+static int its_cmd_send(struct gic_v3_its_softc , struct its_cmd_desc );
	+
	+static void its_cmd_mapc(struct gic_v3_its_softc , struct its_col , uint8_t);
	+static void its_cmd_mapvi(struct gic_v3_its_softc , struct its_dev , uint32_t,
	+ uint32_t);
	+static void its_cmd_mapi(struct gic_v3_its_softc , struct its_dev , uint32_t);
	+static void its_cmd_inv(struct gic_v3_its_softc , struct its_dev , uint32_t);
	+static void its_cmd_invall(struct gic_v3_its_softc , struct its_col );
	+
	+static void lpi_init_conftable(struct gic_v3_its_softc *);
	+static void lpi_bitmap_init(struct gic_v3_its_softc *);
	+static void lpi_init_cpu(struct gic_v3_its_softc *);
	+static int lpi_config_cpu(struct gic_v3_its_softc *);
	+
	+const char *its_ptab_cache[] = {
	+ [GITS_BASER_CACHE_NCNB] = "(NC,NB)",
	+ [GITS_BASER_CACHE_NC] = "(NC)",
	+ [GITS_BASER_CACHE_RAWT] = "(RA,WT)",
	+ [GITS_BASER_CACHE_RAWB] = "(RA,WB)",
	+ [GITS_BASER_CACHE_WAWT] = "(WA,WT)",
	+ [GITS_BASER_CACHE_WAWB] = "(WA,WB)",
	+ [GITS_BASER_CACHE_RAWAWT] = "(RAWA,WT)",
	+ [GITS_BASER_CACHE_RAWAWB] = "(RAWA,WB)",
	+};
	+
	+const char *its_ptab_share[] = {
	+ [GITS_BASER_SHARE_NS] = "none",
	+ [GITS_BASER_SHARE_IS] = "inner",
	+ [GITS_BASER_SHARE_OS] = "outer",
	+ [GITS_BASER_SHARE_RES] = "none",
	+};
	+
	+const char *its_ptab_type[] = {
	+ [GITS_BASER_TYPE_UNIMPL] = "Unimplemented",
	+ [GITS_BASER_TYPE_DEV] = "Devices",
	+ [GITS_BASER_TYPE_VP] = "Virtual Processors",
	+ [GITS_BASER_TYPE_PP] = "Physical Processors",
	+ [GITS_BASER_TYPE_IC] = "Interrupt Collections",
	+ [GITS_BASER_TYPE_RES5] = "Reserved (5)",
	+ [GITS_BASER_TYPE_RES6] = "Reserved (6)",
	+ [GITS_BASER_TYPE_RES7] = "Reserved (7)",
	+};
	+
	+static struct gic_v3_its_softc *its_sc;
	+
	+#define gic_its_read(sc, len, reg) \
	+ bus_read_##len(&sc->its_res[0], reg)
	+
	+#define gic_its_write(sc, len, reg, val) \
	+ bus_write_##len(&sc->its_res[0], reg, val)
	+
	+static int
	+gic_v3_its_attach(device_t dev)
	+{
	+ struct gic_v3_its_softc *sc;
	+ uint64_t gits_tmp;
	+ uint32_t gits_pidr2;
	+ int rid;
	+ int ret;
	+
	+ sc = device_get_softc(dev);
	+
	+ /*
	+ * Initialize sleep & spin mutex for ITS
	+ */
	+ /* Protects ITS device list and assigned LPIs bitmaps. */
	+ mtx_init(&sc->its_mtx, "ITS sleep lock", NULL, MTX_DEF);
	+ /* Protects access to ITS command circular buffer. */
	+ mtx_init(&sc->its_spin_mtx, "ITS spin lock", NULL, MTX_SPIN);
	+
	+ rid = 0;
	+ sc->its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
	+ RF_ACTIVE);
	+ if (sc->its_res == NULL) {
	+ device_printf(dev, "Could not allocate memory\n");
	+ return (ENXIO);
	+ }
	+
	+ sc->dev = dev;
	+
	+ gits_pidr2 = gic_its_read(sc, 4, GITS_PIDR2);
	+ switch (gits_pidr2 & GITS_PIDR2_ARCH_MASK) {
	+ case GITS_PIDR2_ARCH_GICv3: /* fall through */
	+ case GITS_PIDR2_ARCH_GICv4:
	+ if (bootverbose) {
	+ device_printf(dev, "ITS found. Architecture rev. %u\n",
	+ (u_int)(gits_pidr2 & GITS_PIDR2_ARCH_MASK) >> 4);
	+ }
	+ break;
	+ default:
	+ device_printf(dev, "No ITS found in the system\n");
	+ gic_v3_its_detach(dev);
	+ return (ENODEV);
	+ }
	+
	+ /* 1. Initialize commands queue */
	+ its_init_commandq(sc);
	+
	+ /* 2. Provide memory for any private ITS tables */
	+ ret = its_alloc_tables(sc);
	+ if (ret != 0) {
	+ gic_v3_its_detach(dev);
	+ return (ret);
	+ }
	+
	+ /* 3. Allocate collections. One per-CPU */
	+ sc->its_cols = malloc(sizeof(sc->its_cols) MAXCPU,
	+ M_GIC_V3_ITS, (M_WAITOK \| M_ZERO));
	+
	+ /* 4. Enable ITS in GITS_CTLR */
	+ gits_tmp = gic_its_read(sc, 4, GITS_CTLR);
	+ gic_its_write(sc, 4, GITS_CTLR, gits_tmp \| GITS_CTLR_EN);
	+
	+ /* 5. Initialize LPIs configuration table */
	+ lpi_init_conftable(sc);
	+
	+ /* 6. LPIs bitmap init */
	+ lpi_bitmap_init(sc);
	+
	+ /* 7. CPU init */
	+ (void)its_init_cpu(sc);
	+
	+ /* 8. Init ITS devices list */
	+ TAILQ_INIT(&sc->its_dev_list);
	+
	+ arm_register_msi_pic(dev);
	+
	+ /*
	+ * XXX ARM64TODO: We need to have ITS software context
	+ * when being called by the interrupt code (mask/unmask).
	+ * This may be used only when one ITS is present in
	+ * the system and eventually should be removed.
	+ */
	+ KASSERT(its_sc == NULL,
	+ ("Trying to assign its_sc that is already set"));
	+ its_sc = sc;
	+
	+ return (0);
	+}
	+
	+/* Will not detach but use it for convenience */
	+int
	+gic_v3_its_detach(device_t dev)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ struct gic_v3_its_softc *sc;
	+ u_int cpuid;
	+ int rid = 0;
	+
	+ sc = device_get_softc(dev);
	+ cpuid = PCPU_GET(cpuid);
	+
	+ /* Release what's possible */
	+
	+ /* Command queue */
	+ if ((void *)sc->its_cmdq_base != NULL) {
	+ contigfree((void *)sc->its_cmdq_base,
	+ ITS_CMDQ_SIZE, M_GIC_V3_ITS);
	+ }
	+ /* ITTs */
	+ its_free_tables(sc);
	+ /* Collections */
	+ free(sc->its_cols, M_GIC_V3_ITS);
	+ /* LPI config table */
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+ if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) {
	+ contigfree((void *)gic_sc->gic_redists.lpis.conf_base,
	+ LPI_CONFTAB_SIZE, M_GIC_V3_ITS);
	+ }
	+ if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) {
	+ contigfree((void *)gic_sc->gic_redists.lpis.pend_base[cpuid],
	+ roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS);
	+ }
	+
	+ /* Resource... */
	+ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res);
	+
	+ /* XXX ARM64TODO: Reset global pointer to ITS software context */
	+ its_sc = NULL;
	+
	+ return (0);
	+}
	+
	+static int
	+its_alloc_tables(struct gic_v3_its_softc *sc)
	+{
	+ uint64_t gits_baser, gits_tmp;
	+ uint64_t type, esize, cache, share, psz;
	+ uint64_t gits_typer;
	+ size_t page_size, npages, nitspages, nidents, tn;
	+ size_t its_tbl_size;
	+ vm_offset_t ptab_vaddr;
	+ vm_paddr_t ptab_paddr;
	+ boolean_t first = TRUE;
	+
	+ page_size = PAGE_SIZE_64K;
	+
	+ /* Read features first */
	+ gits_typer = gic_its_read(sc, 8, GITS_TYPER);
	+
	+ for (tn = 0; tn < GITS_BASER_NUM; tn++) {
	+ gits_baser = gic_its_read(sc, 8, GITS_BASER(tn));
	+ type = GITS_BASER_TYPE(gits_baser);
	+ /* Get the Table Entry size */
	+ esize = GITS_BASER_ESIZE(gits_baser);
	+
	+ switch (type) {
	+ case GITS_BASER_TYPE_UNIMPL: /* fall through */
	+ case GITS_BASER_TYPE_RES5:
	+ case GITS_BASER_TYPE_RES6:
	+ case GITS_BASER_TYPE_RES7:
	+ continue;
	+ case GITS_BASER_TYPE_DEV:
	+ nidents = (1 << GITS_TYPER_DEVB(gits_typer));
	+ its_tbl_size = esize * nidents;
	+ its_tbl_size = roundup2(its_tbl_size, page_size);
	+ npages = howmany(its_tbl_size, PAGE_SIZE);
	+ break;
	+ default:
	+ npages = howmany(page_size, PAGE_SIZE);
	+ break;
	+ }
	+
	+ /* Allocate required space */
	+ ptab_vaddr = (vm_offset_t)contigmalloc(npages * PAGE_SIZE,
	+ M_GIC_V3_ITS, (M_WAITOK \| M_ZERO), 0, ~0UL, PAGE_SIZE, 0);
	+
	+ sc->its_ptabs[tn].ptab_vaddr = ptab_vaddr;
	+ sc->its_ptabs[tn].ptab_pgsz = PAGE_SIZE;
	+ sc->its_ptabs[tn].ptab_npages = npages;
	+
	+ ptab_paddr = vtophys(ptab_vaddr);
	+ KASSERT((ptab_paddr & GITS_BASER_PA_MASK) == ptab_paddr,
	+ ("%s: Unaligned PA for Interrupt Translation Table",
	+ device_get_name(sc->dev)));
	+
	+ /* Set defaults: WAWB, IS */
	+ cache = GITS_BASER_CACHE_WAWB;
	+ share = GITS_BASER_SHARE_IS;
	+
	+ for (;;) {
	+ nitspages = howmany(its_tbl_size, page_size);
	+
	+ switch (page_size) {
	+ case PAGE_SIZE: /* 4KB */
	+ psz = GITS_BASER_PSZ_4K;
	+ break;
	+ case PAGE_SIZE_16K: /* 16KB */
	+ psz = GITS_BASER_PSZ_4K;
	+ break;
	+ case PAGE_SIZE_64K: /* 64KB */
	+ psz = GITS_BASER_PSZ_64K;
	+ break;
	+ default:
	+ device_printf(sc->dev,
	+ "Unsupported page size: %zuKB\n",
	+ (page_size / 1024));
	+ its_free_tables(sc);
	+ return (ENXIO);
	+ }
	+
	+ /* Clear fields under modification first */
	+ gits_baser &= ~(GITS_BASER_VALID \|
	+ GITS_BASER_CACHE_MASK \| GITS_BASER_TYPE_MASK \|
	+ GITS_BASER_ESIZE_MASK \| GITS_BASER_PA_MASK \|
	+ GITS_BASER_SHARE_MASK \| GITS_BASER_PSZ_MASK \|
	+ GITS_BASER_SIZE_MASK);
	+ /* Construct register value */
	+ gits_baser \|=
	+ (type << GITS_BASER_TYPE_SHIFT) \|
	+ ((esize - 1) << GITS_BASER_ESIZE_SHIFT) \|
	+ (cache << GITS_BASER_CACHE_SHIFT) \|
	+ (share << GITS_BASER_SHARE_SHIFT) \|
	+ (psz << GITS_BASER_PSZ_SHIFT) \|
	+ ptab_paddr \| (nitspages - 1) \|
	+ GITS_BASER_VALID;
	+
	+ gic_its_write(sc, 8, GITS_BASER(tn), gits_baser);
	+ /*
	+ * Verify.
	+ * Depending on implementation we may encounter
	+ * shareability and page size mismatch.
	+ */
	+ gits_tmp = gic_its_read(sc, 8, GITS_BASER(tn));
	+ if (((gits_tmp ^ gits_baser) & GITS_BASER_SHARE_MASK) != 0) {
	+ share = gits_tmp & GITS_BASER_SHARE_MASK;
	+ share >>= GITS_BASER_SHARE_SHIFT;
	+ continue;
	+ }
	+
	+ if (((gits_tmp ^ gits_baser) & GITS_BASER_PSZ_MASK) != 0) {
	+ switch (page_size) {
	+ case PAGE_SIZE_16K:
	+ /* Drop to 4KB page */
	+ page_size = PAGE_SIZE;
	+ continue;
	+ case PAGE_SIZE_64K:
	+ /* Drop to 16KB page */
	+ page_size = PAGE_SIZE_16K;
	+ continue;
	+ }
	+ }
	+ /*
	+ * All possible adjustments should
	+ * be applied by now so just break the loop.
	+ */
	+ break;
	+ }
	+ /*
	+ * Do not compare Cacheability field since
	+ * it is implementation defined.
	+ */
	+ gits_tmp &= ~GITS_BASER_CACHE_MASK;
	+ gits_baser &= ~GITS_BASER_CACHE_MASK;
	+
	+ if (gits_tmp != gits_baser) {
	+ device_printf(sc->dev,
	+ "Could not allocate ITS tables\n");
	+ its_free_tables(sc);
	+ return (ENXIO);
	+ }
	+
	+ if (bootverbose) {
	+ if (first) {
	+ device_printf(sc->dev,
	+ "Allocated ITS private tables:\n");
	+ first = FALSE;
	+ }
	+ device_printf(sc->dev,
	+ "\tPTAB%zu for %s: PA 0x%lx,"
	+ " %lu entries,"
	+ " cache policy %s, %s shareable,"
	+ " page size %zuKB\n",
	+ tn, its_ptab_type[type], ptab_paddr,
	+ (page_size * nitspages) / esize,
	+ its_ptab_cache[cache], its_ptab_share[share],
	+ page_size / 1024);
	+ }
	+ }
	+
	+ return (0);
	+}
	+
	+static void
	+its_free_tables(struct gic_v3_its_softc *sc)
	+{
	+ vm_offset_t ptab_vaddr;
	+ size_t size;
	+ size_t tn;
	+
	+ for (tn = 0; tn < GITS_BASER_NUM; tn++) {
	+ ptab_vaddr = sc->its_ptabs[tn].ptab_vaddr;
	+ if (ptab_vaddr == 0)
	+ continue;
	+ size = sc->its_ptabs[tn].ptab_pgsz;
	+ size *= sc->its_ptabs[tn].ptab_npages;
	+
	+ if ((void *)ptab_vaddr != NULL)
	+ contigfree((void *)ptab_vaddr, size, M_GIC_V3_ITS);
	+
	+ /* Clear the table description */
	+ memset(&sc->its_ptabs[tn], 0, sizeof(sc->its_ptabs[tn]));
	+ }
	+}
	+
	+static void
	+its_init_commandq(struct gic_v3_its_softc *sc)
	+{
	+ uint64_t gits_cbaser, gits_tmp;
	+ uint64_t cache, share;
	+ vm_paddr_t cmdq_paddr;
	+ device_t dev;
	+
	+ dev = sc->dev;
	+ /* Allocate memory for command queue */
	+ sc->its_cmdq_base = contigmalloc(ITS_CMDQ_SIZE, M_GIC_V3_ITS,
	+ (M_WAITOK \| M_ZERO), 0, ~0UL, ITS_CMDQ_SIZE, 0);
	+ /* Set command queue write pointer (command queue empty) */
	+ sc->its_cmdq_write = sc->its_cmdq_base;
	+
	+ /* Save command queue pointer and attributes */
	+ cmdq_paddr = vtophys(sc->its_cmdq_base);
	+
	+ /* Set defaults: Normal Inner WAWB, IS */
	+ cache = GITS_CBASER_CACHE_NIWAWB;
	+ share = GITS_CBASER_SHARE_IS;
	+
	+ gits_cbaser = (cmdq_paddr \|
	+ (cache << GITS_CBASER_CACHE_SHIFT) \|
	+ (share << GITS_CBASER_SHARE_SHIFT) \|
	+ /* Number of 4KB pages - 1 */
	+ ((ITS_CMDQ_SIZE / PAGE_SIZE) - 1) \|
	+ /* Valid bit */
	+ GITS_CBASER_VALID);
	+
	+ gic_its_write(sc, 8, GITS_CBASER, gits_cbaser);
	+ gits_tmp = gic_its_read(sc, 8, GITS_CBASER);
	+
	+ if (((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) != 0) {
	+ if (bootverbose) {
	+ device_printf(dev,
	+ "Will use cache flushing for commands queue\n");
	+ }
	+ /* Command queue needs cache flushing */
	+ sc->its_flags \|= ITS_FLAGS_CMDQ_FLUSH;
	+ }
	+
	+ gic_its_write(sc, 8, GITS_CWRITER, 0x0);
	+}
	+
	+static int
	+its_init_cpu(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+
	+ /*
	+ * Check for LPIs support on this Re-Distributor.
	+ */
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+ if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) {
	+ if (bootverbose) {
	+ device_printf(sc->dev,
	+ "LPIs not supported on CPU%u\n", PCPU_GET(cpuid));
	+ }
	+ return (ENXIO);
	+ }
	+
	+ /* Initialize LPIs for this CPU */
	+ lpi_init_cpu(sc);
	+
	+ /* Initialize collections */
	+ its_init_cpu_collection(sc);
	+
	+ return (0);
	+}
	+
	+static void
	+its_init_cpu_collection(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ uint64_t typer;
	+ uint64_t target;
	+ vm_offset_t redist_base;
	+ u_int cpuid;
	+
	+ cpuid = PCPU_GET(cpuid);
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+
	+ typer = gic_its_read(sc, 8, GITS_TYPER);
	+ if ((typer & GITS_TYPER_PTA) != 0) {
	+ redist_base =
	+ rman_get_bushandle(gic_sc->gic_redists.pcpu[cpuid]);
	+ /*
	+ * Target Address correspond to the base physical
	+ * address of Re-Distributors.
	+ */
	+ target = vtophys(redist_base);
	+ } else {
	+ /* Target Address correspond to unique processor numbers */
	+ typer = gic_r_read(gic_sc, 8, GICR_TYPER);
	+ target = GICR_TYPER_CPUNUM(typer);
	+ }
	+
	+ sc->its_cols[cpuid].col_target = target;
	+ sc->its_cols[cpuid].col_id = cpuid;
	+
	+ its_cmd_mapc(sc, &sc->its_cols[cpuid], 1);
	+ its_cmd_invall(sc, &sc->its_cols[cpuid]);
	+}
	+
	+static void
	+lpi_init_conftable(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ vm_offset_t conf_base;
	+ uint8_t prio_default;
	+
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+ /*
	+ * LPI Configuration Table settings.
	+ * Notice that Configuration Table is shared among all
	+ * Re-Distributors, so this is going to be created just once.
	+ */
	+ conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE,
	+ M_GIC_V3_ITS, (M_WAITOK \| M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0);
	+
	+ if (bootverbose) {
	+ device_printf(sc->dev,
	+ "LPI Configuration Table at PA: 0x%lx\n",
	+ vtophys(conf_base));
	+ }
	+
	+ /*
	+ * Let the default priority be aligned with all other
	+ * interrupts assuming that each interrupt is assigned
	+ * MAX priority at startup. MAX priority on the other
	+ * hand cannot be higher than 0xFC for LPIs.
	+ */
	+ prio_default = GIC_PRIORITY_MAX;
	+
	+ /* Write each settings byte to LPI configuration table */
	+ memset((void *)conf_base,
	+ (prio_default & LPI_CONF_PRIO_MASK) \| LPI_CONF_GROUP1,
	+ LPI_CONFTAB_SIZE);
	+
	+ cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE,
	+ PAGE_SIZE_64K));
	+
	+ gic_sc->gic_redists.lpis.conf_base = conf_base;
	+}
	+
	+static void
	+lpi_init_cpu(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ vm_offset_t pend_base;
	+ u_int cpuid;
	+
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+
	+ /*
	+ * LPI Pending Table settings.
	+ * This has to be done for each Re-Distributor, hence for each CPU.
	+ */
	+ cpuid = PCPU_GET(cpuid);
	+
	+ pend_base = (vm_offset_t)contigmalloc(
	+ roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS,
	+ (M_WAITOK \| M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0);
	+
	+ /* Clean D-cache so that ITS can see zeroed pages */
	+ cpu_dcache_wb_range((vm_offset_t)pend_base,
	+ roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K));
	+
	+ if (bootverbose) {
	+ device_printf(sc->dev,
	+ "LPI Pending Table for CPU%u at PA: 0x%lx\n",
	+ cpuid, vtophys(pend_base));
	+ }
	+
	+ gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base;
	+
	+ lpi_config_cpu(sc);
	+}
	+
	+static int
	+lpi_config_cpu(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ vm_offset_t conf_base, pend_base;
	+ uint64_t gicr_xbaser, gicr_temp;
	+ uint64_t cache, share, idbits;
	+ uint32_t gicr_ctlr;
	+ u_int cpuid;
	+
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+ cpuid = PCPU_GET(cpuid);
	+
	+ conf_base = gic_sc->gic_redists.lpis.conf_base;
	+ pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid];
	+
	+ /* Disable LPIs */
	+ gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR);
	+ gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE;
	+ gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr);
	+ /* Perform full system barrier */
	+ dsb(sy);
	+
	+ /*
	+ * Set GICR_PROPBASER
	+ */
	+
	+ /*
	+ * Find out how many bits do we need for LPI identifiers.
	+ * Remark 1.: Even though we have (LPI_CONFTAB_SIZE / 8) LPIs
	+ * the notified LPI ID still starts from 8192
	+ * (GIC_FIRST_LPI).
	+ * Remark 2.: This could be done on compilation time but there
	+ * seems to be no sufficient macro.
	+ */
	+ idbits = flsl(LPI_CONFTAB_SIZE + GIC_FIRST_LPI) - 1;
	+
	+ /* Set defaults: Normal Inner WAWB, IS */
	+ cache = GICR_PROPBASER_CACHE_NIWAWB;
	+ share = GICR_PROPBASER_SHARE_IS;
	+
	+ gicr_xbaser = vtophys(conf_base) \|
	+ ((idbits - 1) & GICR_PROPBASER_IDBITS_MASK) \|
	+ (cache << GICR_PROPBASER_CACHE_SHIFT) \|
	+ (share << GICR_PROPBASER_SHARE_SHIFT);
	+
	+ gic_r_write(gic_sc, 8, GICR_PROPBASER, gicr_xbaser);
	+ gicr_temp = gic_r_read(gic_sc, 8, GICR_PROPBASER);
	+
	+ if (((gicr_xbaser ^ gicr_temp) & GICR_PROPBASER_SHARE_MASK) != 0) {
	+ if (bootverbose) {
	+ device_printf(sc->dev,
	+ "Will use cache flushing for LPI "
	+ "Configuration Table\n");
	+ }
	+ gic_sc->gic_redists.lpis.flags \|= LPI_FLAGS_CONF_FLUSH;
	+ }
	+
	+ /*
	+ * Set GICR_PENDBASER
	+ */
	+
	+ /* Set defaults: Normal Inner WAWB, IS */
	+ cache = GICR_PENDBASER_CACHE_NIWAWB;
	+ share = GICR_PENDBASER_SHARE_IS;
	+
	+ gicr_xbaser = vtophys(pend_base) \|
	+ (cache << GICR_PENDBASER_CACHE_SHIFT) \|
	+ (share << GICR_PENDBASER_SHARE_SHIFT);
	+
	+ gic_r_write(gic_sc, 8, GICR_PENDBASER, gicr_xbaser);
	+
	+ /* Enable LPIs */
	+ gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR);
	+ gicr_ctlr \|= GICR_CTLR_LPI_ENABLE;
	+ gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr);
	+
	+ dsb(sy);
	+
	+ return (0);
	+}
	+
	+static void
	+lpi_bitmap_init(struct gic_v3_its_softc *sc)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ uint32_t lpi_id_num;
	+ size_t lpi_chunks_num;
	+ size_t bits_in_chunk;
	+
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+
	+ lpi_id_num = (1 << gic_sc->gic_idbits) - 1;
	+ /* Substract IDs dedicated for SGIs, PPIs and SPIs */
	+ lpi_id_num -= GIC_FIRST_LPI;
	+
	+ sc->its_lpi_maxid = lpi_id_num;
	+
	+ bits_in_chunk = sizeof(sc->its_lpi_bitmap) NBBY;
	+
	+ /*
	+ * Round up to the number of bits in chunk.
	+ * We will need to take care to avoid using invalid LPI IDs later.
	+ */
	+ lpi_id_num = roundup2(lpi_id_num, bits_in_chunk);
	+ lpi_chunks_num = lpi_id_num / bits_in_chunk;
	+
	+ sc->its_lpi_bitmap =
	+ contigmalloc((lpi_chunks_num * sizeof(*sc->its_lpi_bitmap)),
	+ M_GIC_V3_ITS, (M_WAITOK \| M_ZERO), 0, ~0UL,
	+ sizeof(*sc->its_lpi_bitmap), 0);
	+}
	+
	+static int
	+lpi_alloc_chunk(struct gic_v3_its_softc sc, struct lpi_chunk lpic,
	+ u_int nvecs)
	+{
	+ int fclr; /* First cleared bit */
	+ uint8_t *bitmap;
	+ size_t nb, i;
	+
	+ bitmap = (uint8_t *)sc->its_lpi_bitmap;
	+
	+ fclr = 0;
	+retry:
	+ /* Check other bits - sloooow */
	+ for (i = 0, nb = fclr; i < nvecs; i++, nb++) {
	+ if (nb > sc->its_lpi_maxid)
	+ return (EINVAL);
	+
	+ if (isset(bitmap, nb)) {
	+ /* To little free bits in this area. Move on. */
	+ fclr = nb + 1;
	+ goto retry;
	+ }
	+ }
	+ /* This area is free. Take it. */
	+ bit_nset(bitmap, fclr, fclr + nvecs - 1);
	+ lpic->lpi_base = fclr + GIC_FIRST_LPI;
	+ lpic->lpi_num = nvecs;
	+ lpic->lpi_free = lpic->lpi_num;
	+
	+ return (0);
	+}
	+
	+static void
	+lpi_configure(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint32_t lpinum, boolean_t unmask)
	+{
	+ device_t parent;
	+ struct gic_v3_softc *gic_sc;
	+ uint8_t *conf_byte;
	+
	+ parent = device_get_parent(sc->dev);
	+ gic_sc = device_get_softc(parent);
	+
	+ conf_byte = (uint8_t *)gic_sc->gic_redists.lpis.conf_base;
	+ conf_byte += (lpinum - GIC_FIRST_LPI);
	+
	+ if (unmask)
	+ *conf_byte \|= LPI_CONF_ENABLE;
	+ else
	+ *conf_byte &= ~LPI_CONF_ENABLE;
	+
	+ if ((gic_sc->gic_redists.lpis.flags & LPI_FLAGS_CONF_FLUSH) != 0) {
	+ /* Clean D-cache under configuration byte */
	+ cpu_dcache_wb_range((vm_offset_t)conf_byte, sizeof(*conf_byte));
	+ } else {
	+ /* DSB inner shareable, store */
	+ dsb(ishst);
	+ }
	+
	+ its_cmd_inv(sc, its_dev, lpinum);
	+}
	+
	+static void
	+lpi_map_to_device(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint32_t id, uint32_t pid)
	+{
	+
	+ if ((pid < its_dev->lpis.lpi_base) \|\|
	+ (pid >= (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)))
	+ panic("Trying to map ivalid LPI %u for the device\n", pid);
	+
	+ its_cmd_mapvi(sc, its_dev, id, pid);
	+}
	+
	+static void
	+lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask)
	+{
	+ struct its_dev *its_dev;
	+
	+ TAILQ_FOREACH(its_dev, &its_sc->its_dev_list, entry) {
	+ if (irq >= its_dev->lpis.lpi_base &&
	+ irq < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)) {
	+ lpi_configure(its_sc, its_dev, irq, unmask);
	+ return;
	+ }
	+ }
	+
	+ panic("Trying to %s not existing LPI: %u\n",
	+ (unmask == TRUE) ? "unmask" : "mask", irq);
	+}
	+
	+void
	+lpi_unmask_irq(device_t parent, uint32_t irq)
	+{
	+
	+ lpi_xmask_irq(parent, irq, 1);
	+}
	+
	+void
	+lpi_mask_irq(device_t parent, uint32_t irq)
	+{
	+
	+ lpi_xmask_irq(parent, irq, 0);
	+}
	+
	+/*
	+ * Commands handling.
	+ */
	+
	+static __inline void
	+cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
	+{
	+ /* Command field: DW0 [7:0] */
	+ cmd->cmd_dword[0] &= ~CMD_COMMAND_MASK;
	+ cmd->cmd_dword[0] \|= cmd_type;
	+}
	+
	+static __inline void
	+cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
	+{
	+ /* Device ID field: DW0 [63:32] */
	+ cmd->cmd_dword[0] &= ~CMD_DEVID_MASK;
	+ cmd->cmd_dword[0] \|= ((uint64_t)devid << CMD_DEVID_SHIFT);
	+}
	+
	+static __inline void
	+cmd_format_size(struct its_cmd *cmd, uint16_t size)
	+{
	+ /* Size field: DW1 [4:0] */
	+ cmd->cmd_dword[1] &= ~CMD_SIZE_MASK;
	+ cmd->cmd_dword[1] \|= (size & CMD_SIZE_MASK);
	+}
	+
	+static __inline void
	+cmd_format_id(struct its_cmd *cmd, uint32_t id)
	+{
	+ /* ID field: DW1 [31:0] */
	+ cmd->cmd_dword[1] &= ~CMD_ID_MASK;
	+ cmd->cmd_dword[1] \|= id;
	+}
	+
	+static __inline void
	+cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
	+{
	+ /* Physical ID field: DW1 [63:32] */
	+ cmd->cmd_dword[1] &= ~CMD_PID_MASK;
	+ cmd->cmd_dword[1] \|= ((uint64_t)pid << CMD_PID_SHIFT);
	+}
	+
	+static __inline void
	+cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
	+{
	+ /* Collection field: DW2 [16:0] */
	+ cmd->cmd_dword[2] &= ~CMD_COL_MASK;
	+ cmd->cmd_dword[2] \|= col_id;
	+}
	+
	+static __inline void
	+cmd_format_target(struct its_cmd *cmd, uint64_t target)
	+{
	+ /* Target Address field: DW2 [47:16] */
	+ cmd->cmd_dword[2] &= ~CMD_TARGET_MASK;
	+ cmd->cmd_dword[2] \|= (target & CMD_TARGET_MASK);
	+}
	+
	+static __inline void
	+cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
	+{
	+ /* ITT Address field: DW2 [47:8] */
	+ cmd->cmd_dword[2] &= ~CMD_ITT_MASK;
	+ cmd->cmd_dword[2] \|= (itt & CMD_ITT_MASK);
	+}
	+
	+static __inline void
	+cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
	+{
	+ /* Valid field: DW2 [63] */
	+ cmd->cmd_dword[2] &= ~CMD_VALID_MASK;
	+ cmd->cmd_dword[2] \|= ((uint64_t)valid << CMD_VALID_SHIFT);
	+}
	+
	+static __inline void
	+cmd_fix_endian(struct its_cmd *cmd)
	+{
	+ size_t i;
	+
	+ for (i = 0; i < nitems(cmd->cmd_dword); i++)
	+ cmd->cmd_dword[i] = htole64(cmd->cmd_dword[i]);
	+}
	+
	+static void
	+its_cmd_mapc(struct gic_v3_its_softc sc, struct its_col col, uint8_t valid)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_MAPC;
	+ desc.cmd_desc_mapc.col = col;
	+ /*
	+ * Valid bit set - map the collection.
	+ * Valid bit cleared - unmap the collection.
	+ */
	+ desc.cmd_desc_mapc.valid = valid;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+static void
	+its_cmd_mapvi(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint32_t id, uint32_t pid)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_MAPVI;
	+ desc.cmd_desc_mapvi.its_dev = its_dev;
	+ desc.cmd_desc_mapvi.id = id;
	+ desc.cmd_desc_mapvi.pid = pid;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+static void __unused
	+its_cmd_mapi(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint32_t lpinum)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_MAPI;
	+ desc.cmd_desc_mapi.its_dev = its_dev;
	+ desc.cmd_desc_mapi.lpinum = lpinum;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+static void
	+its_cmd_mapd(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint8_t valid)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_MAPD;
	+ desc.cmd_desc_mapd.its_dev = its_dev;
	+ desc.cmd_desc_mapd.valid = valid;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+static void
	+its_cmd_inv(struct gic_v3_its_softc sc, struct its_dev its_dev,
	+ uint32_t lpinum)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_INV;
	+ desc.cmd_desc_inv.lpinum = lpinum - its_dev->lpis.lpi_base;
	+ desc.cmd_desc_inv.its_dev = its_dev;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+static void
	+its_cmd_invall(struct gic_v3_its_softc sc, struct its_col col)
	+{
	+ struct its_cmd_desc desc;
	+
	+ desc.cmd_type = ITS_CMD_INVALL;
	+ desc.cmd_desc_invall.col = col;
	+
	+ its_cmd_send(sc, &desc);
	+}
	+
	+/*
	+ * Helper routines for commands processing.
	+ */
	+static __inline boolean_t
	+its_cmd_queue_full(struct gic_v3_its_softc *sc)
	+{
	+ size_t read_idx, write_idx;
	+
	+ write_idx = (size_t)(sc->its_cmdq_write - sc->its_cmdq_base);
	+ read_idx = gic_its_read(sc, 4, GITS_CREADR) / sizeof(struct its_cmd);
	+
	+ /*
	+ * The queue is full when the write offset points
	+ * at the command before the current read offset.
	+ */
	+ if (((write_idx + 1) % ITS_CMDQ_NENTRIES) == read_idx)
	+ return (TRUE);
	+
	+ return (FALSE);
	+}
	+
	+static __inline void
	+its_cmd_sync(struct gic_v3_its_softc sc, struct its_cmd cmd)
	+{
	+
	+ if ((sc->its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
	+ /* Clean D-cache under command. */
	+ cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
	+ } else {
	+ /* DSB inner shareable, store */
	+ dsb(ishst);
	+ }
	+
	+}
	+
	+static struct its_cmd *
	+its_cmd_alloc_locked(struct gic_v3_its_softc *sc)
	+{
	+ struct its_cmd *cmd;
	+ size_t us_left;
	+
	+ /*
	+ * XXX ARM64TODO: This is obviously a significant delay.
	+ * The reason for that is that currently the time frames for
	+ * the command to complete (and therefore free the descriptor)
	+ * are not known.
	+ */
	+ us_left = 1000000;
	+
	+ mtx_assert(&sc->its_spin_mtx, MA_OWNED);
	+ while (its_cmd_queue_full(sc)) {
	+ if (us_left-- == 0) {
	+ /* Timeout while waiting for free command */
	+ device_printf(sc->dev,
	+ "Timeout while waiting for free command\n");
	+ return (NULL);
	+ }
	+ DELAY(1);
	+ }
	+
	+ cmd = sc->its_cmdq_write;
	+ sc->its_cmdq_write++;
	+
	+ if (sc->its_cmdq_write == (sc->its_cmdq_base + ITS_CMDQ_NENTRIES)) {
	+ /* Wrap the queue */
	+ sc->its_cmdq_write = sc->its_cmdq_base;
	+ }
	+
	+ return (cmd);
	+}
	+
	+static uint64_t
	+its_cmd_prepare(struct its_cmd cmd, struct its_cmd_desc desc)
	+{
	+ uint64_t target;
	+ uint8_t cmd_type;
	+ u_int size;
	+ boolean_t error;
	+
	+ error = FALSE;
	+ cmd_type = desc->cmd_type;
	+ target = ITS_TARGET_NONE;
	+
	+ switch (cmd_type) {
	+ case ITS_CMD_SYNC: /* Wait for previous commands completion */
	+ target = desc->cmd_desc_sync.col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_SYNC);
	+ cmd_format_target(cmd, target);
	+ break;
	+ case ITS_CMD_MAPD: /* Assign ITT to device */
	+ target = desc->cmd_desc_mapd.its_dev->col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_MAPD);
	+ cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
	+ /*
	+ * Size describes number of bits to encode interrupt IDs
	+ * supported by the device minus one.
	+ * When V (valid) bit is zero, this field should be written
	+ * as zero.
	+ */
	+ if (desc->cmd_desc_mapd.valid != 0) {
	+ size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
	+ size = MAX(1, size) - 1;
	+ } else
	+ size = 0;
	+
	+ cmd_format_size(cmd, size);
	+ cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
	+ cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
	+ break;
	+ case ITS_CMD_MAPC: /* Map collection to Re-Distributor */
	+ target = desc->cmd_desc_mapc.col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_MAPC);
	+ cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
	+ cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
	+ cmd_format_target(cmd, target);
	+ break;
	+ case ITS_CMD_MAPVI:
	+ target = desc->cmd_desc_mapvi.its_dev->col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_MAPVI);
	+ cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
	+ cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
	+ cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
	+ cmd_format_col(cmd, desc->cmd_desc_mapvi.its_dev->col->col_id);
	+ break;
	+ case ITS_CMD_MAPI:
	+ target = desc->cmd_desc_mapi.its_dev->col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_MAPI);
	+ cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
	+ cmd_format_id(cmd, desc->cmd_desc_mapi.lpinum);
	+ cmd_format_col(cmd, desc->cmd_desc_mapi.its_dev->col->col_id);
	+ break;
	+ case ITS_CMD_INV:
	+ target = desc->cmd_desc_inv.its_dev->col->col_target;
	+ cmd_format_command(cmd, ITS_CMD_INV);
	+ cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
	+ cmd_format_id(cmd, desc->cmd_desc_inv.lpinum);
	+ break;
	+ case ITS_CMD_INVALL:
	+ cmd_format_command(cmd, ITS_CMD_INVALL);
	+ cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
	+ break;
	+ default:
	+ error = TRUE;
	+ break;
	+ }
	+
	+ if (!error)
	+ cmd_fix_endian(cmd);
	+
	+ return (target);
	+}
	+
	+static __inline uint64_t
	+its_cmd_cwriter_offset(struct gic_v3_its_softc sc, struct its_cmd cmd)
	+{
	+ uint64_t off;
	+
	+ off = (cmd - sc->its_cmdq_base) * sizeof(*cmd);
	+
	+ return (off);
	+}
	+
	+static void
	+its_cmd_wait_completion(struct gic_v3_its_softc sc, struct its_cmd cmd_first,
	+ struct its_cmd *cmd_last)
	+{
	+ uint64_t first, last, read;
	+ size_t us_left;
	+
	+ /*
	+ * XXX ARM64TODO: This is obviously a significant delay.
	+ * The reason for that is that currently the time frames for
	+ * the command to complete are not known.
	+ */
	+ us_left = 1000000;
	+
	+ first = its_cmd_cwriter_offset(sc, cmd_first);
	+ last = its_cmd_cwriter_offset(sc, cmd_last);
	+
	+ for (;;) {
	+ read = gic_its_read(sc, 8, GITS_CREADR);
	+ if (read < first \|\| read >= last)
	+ break;
	+
	+ if (us_left-- == 0) {
	+ /* This means timeout */
	+ device_printf(sc->dev,
	+ "Timeout while waiting for CMD completion.\n");
	+ return;
	+ }
	+ DELAY(1);
	+ }
	+}
	+
	+static int
	+its_cmd_send(struct gic_v3_its_softc sc, struct its_cmd_desc desc)
	+{
	+ struct its_cmd cmd, cmd_sync;
	+ struct its_col col_sync;
	+ struct its_cmd_desc desc_sync;
	+ uint64_t target, cwriter;
	+
	+ mtx_lock_spin(&sc->its_spin_mtx);
	+ cmd = its_cmd_alloc_locked(sc);
	+ mtx_unlock_spin(&sc->its_spin_mtx);
	+ if (cmd == NULL) {
	+ device_printf(sc->dev, "could not allocate ITS command\n");
	+ return (EBUSY);
	+ }
	+
	+ target = its_cmd_prepare(cmd, desc);
	+ its_cmd_sync(sc, cmd);
	+
	+ if (target != ITS_TARGET_NONE) {
	+ mtx_lock_spin(&sc->its_spin_mtx);
	+ cmd_sync = its_cmd_alloc_locked(sc);
	+ mtx_unlock_spin(&sc->its_spin_mtx);
	+ if (cmd_sync == NULL)
	+ goto end;
	+ desc_sync.cmd_type = ITS_CMD_SYNC;
	+ col_sync.col_target = target;
	+ desc_sync.cmd_desc_sync.col = &col_sync;
	+ its_cmd_prepare(cmd_sync, &desc_sync);
	+ its_cmd_sync(sc, cmd_sync);
	+ }
	+end:
	+ /* Update GITS_CWRITER */
	+ mtx_lock_spin(&sc->its_spin_mtx);
	+ cwriter = its_cmd_cwriter_offset(sc, sc->its_cmdq_write);
	+ gic_its_write(sc, 8, GITS_CWRITER, cwriter);
	+ mtx_unlock_spin(&sc->its_spin_mtx);
	+
	+ its_cmd_wait_completion(sc, cmd, sc->its_cmdq_write);
	+
	+ return (0);
	+}
	+
	+static struct its_dev *
	+its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev)
	+{
	+ struct its_dev *its_dev;
	+
	+ mtx_assert(&sc->its_mtx, MA_OWNED);
	+ /* Find existing device if any */
	+ TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) {
	+ if (its_dev->pci_dev == pci_dev)
	+ return (its_dev);
	+ }
	+
	+ return (NULL);
	+}
	+
	+static struct its_dev *
	+its_device_alloc_locked(struct gic_v3_its_softc *sc, device_t pci_dev,
	+ u_int nvecs)
	+{
	+ struct its_dev *newdev;
	+ uint64_t typer;
	+ uint32_t devid;
	+ u_int cpuid;
	+ size_t esize;
	+
	+ mtx_assert(&sc->its_mtx, MA_OWNED);
	+ /* Find existing device if any */
	+ newdev = its_device_find_locked(sc, pci_dev);
	+ if (newdev != NULL)
	+ return (newdev);
	+
	+ devid = PCI_DEVID(pci_dev);
	+
	+ /* There was no previously created device. Create one now */
	+ newdev = malloc(sizeof(*newdev), M_GIC_V3_ITS, (M_WAITOK \| M_ZERO));
	+ newdev->pci_dev = pci_dev;
	+ newdev->devid = devid;
	+
	+ if (lpi_alloc_chunk(sc, &newdev->lpis, nvecs) != 0) {
	+ free(newdev, M_GIC_V3_ITS);
	+ return (NULL);
	+ }
	+
	+ /* Get ITT entry size */
	+ typer = gic_its_read(sc, 8, GITS_TYPER);
	+ esize = GITS_TYPER_ITTES(typer);
	+ /*
	+ * Allocate ITT for this device.
	+ * PA has to be 256 B aligned. At least two entries for device.
	+ */
	+ newdev->itt = (vm_offset_t)contigmalloc(
	+ roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS,
	+ (M_WAITOK \| M_ZERO), 0, ~0UL, 0x100, 0);
	+
	+ /*
	+ * XXX ARM64TODO: Currently all interrupts are going
	+ * to be bound to the CPU that performs the configuration.
	+ */
	+ cpuid = PCPU_GET(cpuid);
	+ newdev->col = &sc->its_cols[cpuid];
	+
	+ TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry);
	+
	+ /* Map device to its ITT */
	+ its_cmd_mapd(sc, newdev, 1);
	+
	+ return (newdev);
	+}
	+
	+static __inline void
	+its_device_asign_lpi_locked(struct gic_v3_its_softc *sc,
	+ struct its_dev its_dev, u_int irq)
	+{
	+
	+ mtx_assert(&sc->its_mtx, MA_OWNED);
	+ if (its_dev->lpis.lpi_free == 0) {
	+ panic("Requesting more LPIs than allocated for this device. "
	+ "LPI num: %u, free %u", its_dev->lpis.lpi_num,
	+ its_dev->lpis.lpi_free);
	+ }
	+ *irq = its_dev->lpis.lpi_base + (its_dev->lpis.lpi_num -
	+ its_dev->lpis.lpi_free);
	+ its_dev->lpis.lpi_free--;
	+}
	+/*
	+ * Message signalled interrupts handling.
	+ */
	+
	+/*
	+ * XXX ARM64TODO: Watch out for "irq" type.
	+ *
	+ * In theory GIC can handle up to (2^32 - 1) interrupt IDs whereas
	+ * we pass "irq" pointer of type integer. This is obviously wrong but
	+ * is determined by the way as PCI layer wants it to be done.
	+ */
	+int
	+gic_v3_its_alloc_msix(device_t dev, device_t pci_dev, int *irq)
	+{
	+ struct gic_v3_its_softc *sc;
	+ struct its_dev *its_dev;
	+ u_int nvecs;
	+
	+ sc = device_get_softc(dev);
	+
	+ mtx_lock(&sc->its_mtx);
	+ nvecs = PCI_MSIX_NUM(pci_dev);
	+
	+ /*
	+ * Allocate device as seen by ITS if not already available.
	+ * Notice that MSI-X interrupts are allocated on one-by-one basis.
	+ */
	+ its_dev = its_device_alloc_locked(sc, pci_dev, nvecs);
	+ if (its_dev == NULL) {
	+ mtx_unlock(&sc->its_mtx);
	+ return (ENOMEM);
	+ }
	+
	+ its_device_asign_lpi_locked(sc, its_dev, irq);
	+ mtx_unlock(&sc->its_mtx);
	+
	+ return (0);
	+}
	+
	+int
	+gic_v3_its_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs)
	+{
	+ struct gic_v3_its_softc *sc;
	+ struct its_dev *its_dev;
	+
	+ sc = device_get_softc(dev);
	+
	+ /* Allocate device as seen by ITS if not already available. */
	+ mtx_lock(&sc->its_mtx);
	+ its_dev = its_device_alloc_locked(sc, pci_dev, count);
	+ if (its_dev == NULL) {
	+ mtx_unlock(&sc->its_mtx);
	+ return (ENOMEM);
	+ }
	+
	+ for (; count > 0; count--) {
	+ its_device_asign_lpi_locked(sc, its_dev, irqs);
	+ irqs++;
	+ }
	+ mtx_unlock(&sc->its_mtx);
	+
	+ return (0);
	+}
	+
	+int
	+gic_v3_its_map_msix(device_t dev, device_t pci_dev, int irq, uint64_t *addr,
	+ uint32_t *data)
	+{
	+ struct gic_v3_its_softc *sc;
	+ bus_space_handle_t its_bsh;
	+ struct its_dev *its_dev;
	+ uint64_t its_pa;
	+ uint32_t id;
	+
	+ sc = device_get_softc(dev);
	+ /* Verify that this device is allocated and owns this LPI */
	+ mtx_lock(&sc->its_mtx);
	+ its_dev = its_device_find_locked(sc, pci_dev);
	+ mtx_unlock(&sc->its_mtx);
	+ if (its_dev == NULL)
	+ return (EINVAL);
	+
	+ id = irq - its_dev->lpis.lpi_base;
	+ lpi_map_to_device(sc, its_dev, id, irq);
	+
	+ its_bsh = rman_get_bushandle(&sc->its_res[0]);
	+ its_pa = vtophys(its_bsh);
	+
	+ *addr = (its_pa + GITS_TRANSLATER);
	+ *data = id;
	+
	+ return (0);
	+}
	Index: sys/arm64/arm64/gic_v3_reg.h
	===================================================================
	--- sys/arm64/arm64/gic_v3_reg.h
	+++ sys/arm64/arm64/gic_v3_reg.h
	@@ -101,17 +101,100 @@
	#define GICR_PIDR2_ARCH_GICv4 (0x40)

	/* Redistributor registers */
	+#define GICR_CTLR GICD_CTLR
	+#define GICR_CTLR_LPI_ENABLE (1 << 0)
	+
	#define GICR_PIDR2 GICD_PIDR2

	#define GICR_TYPER (0x0008)
	+#define GICR_TYPER_PLPIS (1 << 0)
	#define GICR_TYPER_VLPIS (1 << 1)
	#define GICR_TYPER_LAST (1 << 4)
	+#define GICR_TYPER_CPUNUM_SHIFT (8)
	+#define GICR_TYPER_CPUNUM_MASK (0xFFFUL << GICR_TYPER_CPUNUM_SHIFT)
	+#define GICR_TYPER_CPUNUM(x) \
	+ (((x) & GICR_TYPER_CPUNUM_MASK) >> GICR_TYPER_CPUNUM_SHIFT)
	#define GICR_TYPER_AFF_SHIFT (32)

	#define GICR_WAKER (0x0014)
	#define GICR_WAKER_PS (1 << 1) /* Processor sleep */
	#define GICR_WAKER_CA (1 << 2) /* Children asleep */

	+#define GICR_PROPBASER (0x0070)
	+#define GICR_PROPBASER_IDBITS_MASK 0x1FUL
	+/*
	+ * Cacheability
	+ * 0x0 - Device-nGnRnE
	+ * 0x1 - Normal Inner Non-cacheable
	+ * 0x2 - Normal Inner Read-allocate, Write-through
	+ * 0x3 - Normal Inner Read-allocate, Write-back
	+ * 0x4 - Normal Inner Write-allocate, Write-through
	+ * 0x5 - Normal Inner Write-allocate, Write-back
	+ * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through
	+ * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back
	+ */
	+#define GICR_PROPBASER_CACHE_SHIFT 7
	+#define GICR_PROPBASER_CACHE_DnGnRnE 0x0UL
	+#define GICR_PROPBASER_CACHE_NIN 0x1UL
	+#define GICR_PROPBASER_CACHE_NIRAWT 0x2UL
	+#define GICR_PROPBASER_CACHE_NIRAWB 0x3UL
	+#define GICR_PROPBASER_CACHE_NIWAWT 0x4UL
	+#define GICR_PROPBASER_CACHE_NIWAWB 0x5UL
	+#define GICR_PROPBASER_CACHE_NIRAWAWT 0x6UL
	+#define GICR_PROPBASER_CACHE_NIRAWAWB 0x7UL
	+
	+/*
	+ * Shareability
	+ * 0x0 - Non-shareable
	+ * 0x1 - Inner-shareable
	+ * 0x2 - Outer-shareable
	+ * 0x3 - Reserved. Threated as 0x0
	+ */
	+#define GICR_PROPBASER_SHARE_SHIFT 10
	+#define GICR_PROPBASER_SHARE_NS 0x0UL
	+#define GICR_PROPBASER_SHARE_IS 0x1UL
	+#define GICR_PROPBASER_SHARE_OS 0x2UL
	+#define GICR_PROPBASER_SHARE_RES 0x3UL
	+#define GICR_PROPBASER_SHARE_MASK \
	+ (0x3UL << GICR_PROPBASER_SHARE_SHIFT)
	+
	+#define GICR_PENDBASER (0x0078)
	+/*
	+ * Cacheability
	+ * 0x0 - Device-nGnRnE
	+ * 0x1 - Normal Inner Non-cacheable
	+ * 0x2 - Normal Inner Read-allocate, Write-through
	+ * 0x3 - Normal Inner Read-allocate, Write-back
	+ * 0x4 - Normal Inner Write-allocate, Write-through
	+ * 0x5 - Normal Inner Write-allocate, Write-back
	+ * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through
	+ * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back
	+ */
	+#define GICR_PENDBASER_CACHE_SHIFT 7
	+#define GICR_PENDBASER_CACHE_DnGnRnE 0x0UL
	+#define GICR_PENDBASER_CACHE_NIN 0x1UL
	+#define GICR_PENDBASER_CACHE_NIRAWT 0x2UL
	+#define GICR_PENDBASER_CACHE_NIRAWB 0x3UL
	+#define GICR_PENDBASER_CACHE_NIWAWT 0x4UL
	+#define GICR_PENDBASER_CACHE_NIWAWB 0x5UL
	+#define GICR_PENDBASER_CACHE_NIRAWAWT 0x6UL
	+#define GICR_PENDBASER_CACHE_NIRAWAWB 0x7UL
	+
	+/*
	+ * Shareability
	+ * 0x0 - Non-shareable
	+ * 0x1 - Inner-shareable
	+ * 0x2 - Outer-shareable
	+ * 0x3 - Reserved. Threated as 0x0
	+ */
	+#define GICR_PENDBASER_SHARE_SHIFT 10
	+#define GICR_PENDBASER_SHARE_NS 0x0UL
	+#define GICR_PENDBASER_SHARE_IS 0x1UL
	+#define GICR_PENDBASER_SHARE_OS 0x2UL
	+#define GICR_PENDBASER_SHARE_RES 0x3UL
	+#define GICR_PENDBASER_SHARE_MASK \
	+ (0x3UL << GICR_PENDBASER_SHARE_SHIFT)
	+
	/* Re-distributor registers for SGIs and PPIs */
	#define GICR_RD_BASE_SIZE PAGE_SIZE_64K
	#define GICR_SGI_BASE_SIZE PAGE_SIZE_64K
	@@ -125,6 +208,151 @@

	#define GICR_I_PER_IPRIORITYn (GICD_I_PER_IPRIORITYn)

	+/* ITS registers */
	+#define GITS_PIDR2 GICR_PIDR2
	+#define GITS_PIDR2_ARCH_MASK GICR_PIDR2_ARCH_MASK
	+#define GITS_PIDR2_ARCH_GICv3 GICR_PIDR2_ARCH_GICv3
	+#define GITS_PIDR2_ARCH_GICv4 GICR_PIDR2_ARCH_GICv4
	+
	+#define GITS_CTLR (0x0000)
	+#define GITS_CTLR_EN (1 << 0)
	+
	+#define GITS_CBASER (0x0080)
	+#define GITS_CBASER_VALID (1UL << 63)
	+/*
	+ * Cacheability
	+ * 0x0 - Device-nGnRnE
	+ * 0x1 - Normal Inner Non-cacheable
	+ * 0x2 - Normal Inner Read-allocate, Write-through
	+ * 0x3 - Normal Inner Read-allocate, Write-back
	+ * 0x4 - Normal Inner Write-allocate, Write-through
	+ * 0x5 - Normal Inner Write-allocate, Write-back
	+ * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through
	+ * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back
	+ */
	+#define GITS_CBASER_CACHE_SHIFT 59
	+#define GITS_CBASER_CACHE_DnGnRnE 0x0UL
	+#define GITS_CBASER_CACHE_NIN 0x1UL
	+#define GITS_CBASER_CACHE_NIRAWT 0x2UL
	+#define GITS_CBASER_CACHE_NIRAWB 0x3UL
	+#define GITS_CBASER_CACHE_NIWAWT 0x4UL
	+#define GITS_CBASER_CACHE_NIWAWB 0x5UL
	+#define GITS_CBASER_CACHE_NIRAWAWT 0x6UL
	+#define GITS_CBASER_CACHE_NIRAWAWB 0x7UL
	+#define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_TYPE_SHIFT)
	+/*
	+ * Shareability
	+ * 0x0 - Non-shareable
	+ * 0x1 - Inner-shareable
	+ * 0x2 - Outer-shareable
	+ * 0x3 - Reserved. Threated as 0x0
	+ */
	+#define GITS_CBASER_SHARE_SHIFT 10
	+#define GITS_CBASER_SHARE_NS 0x0UL
	+#define GITS_CBASER_SHARE_IS 0x1UL
	+#define GITS_CBASER_SHARE_OS 0x2UL
	+#define GITS_CBASER_SHARE_RES 0x3UL
	+#define GITS_CBASER_SHARE_MASK \
	+ (0x3UL << GITS_CBASER_SHARE_SHIFT)
	+
	+#define GITS_CBASER_PA_SHIFT 12
	+#define GITS_CBASER_PA_MASK (0xFFFFFFFFFUL << GITS_CBASER_PA_SHIFT)
	+
	+#define GITS_CWRITER (0x0088)
	+#define GITS_CREADR (0x0090)
	+
	+#define GITS_BASER_BASE (0x0100)
	+#define GITS_BASER(x) (GITS_BASER_BASE + (x) * 8)
	+
	+#define GITS_BASER_VALID (1UL << 63)
	+
	+#define GITS_BASER_TYPE_SHIFT 56
	+#define GITS_BASER_TYPE(x) \
	+ (((x) & GITS_BASER_TYPE_MASK) >> GITS_BASER_TYPE_SHIFT)
	+#define GITS_BASER_TYPE_UNIMPL 0x0UL /* Unimplemented */
	+#define GITS_BASER_TYPE_DEV 0x1UL /* Devices */
	+#define GITS_BASER_TYPE_VP 0x2UL /* Virtual Processors */
	+#define GITS_BASER_TYPE_PP 0x3UL /* Physical Processors */
	+#define GITS_BASER_TYPE_IC 0x4UL /* Interrupt Collections */
	+#define GITS_BASER_TYPE_RES5 0x5UL /* Reserved */
	+#define GITS_BASER_TYPE_RES6 0x6UL /* Reserved */
	+#define GITS_BASER_TYPE_RES7 0x7UL /* Reserved */
	+#define GITS_BASER_TYPE_MASK (0x7UL << GITS_BASER_TYPE_SHIFT)
	+/*
	+ * Cacheability
	+ * 0x0 - Non-cacheable, non-bufferable
	+ * 0x1 - Non-cacheable
	+ * 0x2 - Read-allocate, Write-through
	+ * 0x3 - Read-allocate, Write-back
	+ * 0x4 - Write-allocate, Write-through
	+ * 0x5 - Write-allocate, Write-back
	+ * 0x6 - Read-allocate, Write-allocate, Write-through
	+ * 0x7 - Read-allocate, Write-allocate, Write-back
	+ */
	+#define GITS_BASER_CACHE_SHIFT 59
	+#define GITS_BASER_CACHE_NCNB 0x0UL
	+#define GITS_BASER_CACHE_NC 0x1UL
	+#define GITS_BASER_CACHE_RAWT 0x2UL
	+#define GITS_BASER_CACHE_RAWB 0x3UL
	+#define GITS_BASER_CACHE_WAWT 0x4UL
	+#define GITS_BASER_CACHE_WAWB 0x5UL
	+#define GITS_BASER_CACHE_RAWAWT 0x6UL
	+#define GITS_BASER_CACHE_RAWAWB 0x7UL
	+#define GITS_BASER_CACHE_MASK (0x7UL << GITS_BASER_CACHE_SHIFT)
	+
	+#define GITS_BASER_ESIZE_SHIFT 48
	+#define GITS_BASER_ESIZE_MASK (0x1FUL << GITS_BASER_ESIZE_SHIFT)
	+#define GITS_BASER_ESIZE(x) \
	+ ((((x) & GITS_BASER_ESIZE_MASK) >> GITS_BASER_ESIZE_SHIFT) + 1)
	+
	+#define GITS_BASER_PA_SHIFT 12
	+#define GITS_BASER_PA_MASK (0xFFFFFFFFFUL << GITS_BASER_PA_SHIFT)
	+
	+/*
	+ * Shareability
	+ * 0x0 - Non-shareable
	+ * 0x1 - Inner-shareable
	+ * 0x2 - Outer-shareable
	+ * 0x3 - Reserved. Threated as 0x0
	+ */
	+#define GITS_BASER_SHARE_SHIFT 10
	+#define GITS_BASER_SHARE_NS 0x0UL
	+#define GITS_BASER_SHARE_IS 0x1UL
	+#define GITS_BASER_SHARE_OS 0x2UL
	+#define GITS_BASER_SHARE_RES 0x3UL
	+#define GITS_BASER_SHARE_MASK (0x3UL << GITS_BASER_SHARE_SHIFT)
	+
	+#define GITS_BASER_PSZ_SHIFT 8
	+#define GITS_BASER_PSZ_4K 0x0UL
	+#define GITS_BASER_PSZ_16K 0x1UL
	+#define GITS_BASER_PSZ_64K 0x2UL
	+#define GITS_BASER_PSZ_MASK (0x3UL << GITS_BASER_PSZ_SHIFT)
	+
	+#define GITS_BASER_SIZE_MASK 0xFFUL
	+
	+#define GITS_BASER_NUM 8
	+
	+#define GITS_TYPER (0x0008)
	+#define GITS_TYPER_PTA (1UL << 19)
	+#define GITS_TYPER_DEVB_SHIFT 13
	+#define GITS_TYPER_DEVB_MASK (0x1FUL << GITS_TYPER_DEVB_SHIFT)
	+/* Number of device identifiers implemented */
	+#define GITS_TYPER_DEVB(x) \
	+ ((((x) & GITS_TYPER_DEVB_MASK) >> GITS_TYPER_DEVB_SHIFT) + 1)
	+#define GITS_TYPER_ITTES_SHIFT 4
	+#define GITS_TYPER_ITTES_MASK (0xFUL << GITS_TYPER_ITTES_SHIFT)
	+/* Number of bytes per ITT Entry */
	+#define GITS_TYPER_ITTES(x) \
	+ ((((x) & GITS_TYPER_ITTES_MASK) >> GITS_TYPER_ITTES_SHIFT) + 1)
	+
	+#define GITS_TRANSLATER (0x10040)
	+/*
	+ * LPI related
	+ */
	+#define LPI_CONF_PRIO_MASK (0xFC)
	+#define LPI_CONF_GROUP1 (1 << 1)
	+#define LPI_CONF_ENABLE (1 << 0)
	+
	/*
	* CPU interface
	*/
	Index: sys/arm64/arm64/gic_v3_var.h
	===================================================================
	--- sys/arm64/arm64/gic_v3_var.h
	+++ sys/arm64/arm64/gic_v3_var.h
	@@ -36,6 +36,17 @@

	DECLARE_CLASS(gic_v3_driver);

	+#define LPI_FLAGS_CONF_FLUSH (1UL << 0)
	+#define LPI_CONFTAB_SIZE PAGE_SIZE_64K
	+/* 1 bit per LPI + 1 KB more for the obligatory PPI, SGI, SPI stuff */
	+#define LPI_PENDTAB_SIZE ((LPI_CONFTAB_SIZE / 8) + 0x400)
	+
	+struct redist_lpis {
	+ vm_offset_t conf_base;
	+ vm_offset_t pend_base[MAXCPU];
	+ uint64_t flags;
	+};
	+
	struct gic_redists {
	/*
	* Re-Distributor region description.
	@@ -47,6 +58,8 @@
	u_int nregions;
	/* Per-CPU Re-Distributor handler */
	struct resource * pcpu[MAXCPU];
	+ /* LPIs data */
	+ struct redist_lpis lpis;
	};

	struct gic_v3_softc {
	@@ -71,6 +84,167 @@
	int gic_v3_detach(device_t dev);

	/*
	+ * ITS
	+ */
	+#define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service"
	+#define GIC_V3_ITS_COMPSTR "arm,gic-v3-its"
	+
	+DECLARE_CLASS(gic_v3_its_driver);
	+
	+/* LPI chunk owned by ITS device */
	+struct lpi_chunk {
	+ u_int lpi_base;
	+ u_int lpi_num;
	+ u_int lpi_free; /* First free LPI in set */
	+};
	+
	+/* ITS device */
	+struct its_dev {
	+ TAILQ_ENTRY(its_dev) entry;
	+ /* PCI device */
	+ device_t pci_dev;
	+ /* Device ID (i.e. PCI device ID) */
	+ uint32_t devid;
	+ /* List of assigned LPIs */
	+ struct lpi_chunk lpis;
	+ /* Virtual address of ITT */
	+ vm_offset_t itt;
	+ /* Interrupt collection */
	+ struct its_col * col;
	+};
	+TAILQ_HEAD(its_dev_list, its_dev);
	+
	+/* ITS private table description */
	+struct its_ptab {
	+ vm_offset_t ptab_vaddr; /* Virtual Address of table */
	+ size_t ptab_pgsz; /* Page size */
	+ size_t ptab_npages; /* Number of pages */
	+};
	+
	+/* ITS collection description. */
	+struct its_col {
	+ uint64_t col_target; /* Target Re-Distributor */
	+ uint64_t col_id; /* Collection ID */
	+};
	+
	+/* ITS command. Each command is 32 bytes long */
	+struct its_cmd {
	+ uint64_t cmd_dword[4]; /* ITS command double word */
	+};
	+
	+/* ITS commands encoding */
	+#define ITS_CMD_SYNC (0x05)
	+#define ITS_CMD_MAPD (0x08)
	+#define ITS_CMD_MAPC (0x09)
	+#define ITS_CMD_MAPVI (0x0a)
	+#define ITS_CMD_MAPI (0x0b)
	+#define ITS_CMD_INV (0x0c)
	+#define ITS_CMD_INVALL (0x0d)
	+/* Command */
	+#define CMD_COMMAND_MASK (0xFFUL)
	+/* PCI device ID */
	+#define CMD_DEVID_SHIFT (32)
	+#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
	+/* Size of IRQ ID bitfield */
	+#define CMD_SIZE_MASK (0xFFUL)
	+/* Virtual LPI ID */
	+#define CMD_ID_MASK (0xFFFFFFFFUL)
	+/* Physical LPI ID */
	+#define CMD_PID_SHIFT (32)
	+#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT)
	+/* Collection */
	+#define CMD_COL_MASK (0xFFFFUL)
	+/* Target (CPU or Re-Distributor) */
	+#define CMD_TARGET_SHIFT (16)
	+#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
	+/* Interrupt Translation Table address */
	+#define CMD_ITT_MASK (0xFFFFFFFFFF00UL)
	+/* Valid command bit */
	+#define CMD_VALID_SHIFT (63)
	+#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT)
	+
	+/*
	+ * ITS command descriptor.
	+ * Idea for command description passing taken from Linux.
	+ */
	+struct its_cmd_desc {
	+ uint8_t cmd_type;
	+
	+ union {
	+ struct {
	+ struct its_col *col;
	+ } cmd_desc_sync;
	+
	+ struct {
	+ struct its_col *col;
	+ uint8_t valid;
	+ } cmd_desc_mapc;
	+
	+ struct {
	+ struct its_dev *its_dev;
	+ uint32_t pid;
	+ uint32_t id;
	+ } cmd_desc_mapvi;
	+
	+ struct {
	+ struct its_dev *its_dev;
	+ uint32_t lpinum;
	+ } cmd_desc_mapi;
	+
	+ struct {
	+ struct its_dev *its_dev;
	+ uint8_t valid;
	+ } cmd_desc_mapd;
	+
	+ struct {
	+ struct its_dev *its_dev;
	+ uint32_t lpinum;
	+ } cmd_desc_inv;
	+
	+ struct {
	+ struct its_col *col;
	+ } cmd_desc_invall;
	+ };
	+};
	+
	+#define ITS_CMDQ_SIZE PAGE_SIZE_64K
	+#define ITS_CMDQ_NENTRIES (ITS_CMDQ_SIZE / sizeof(struct its_cmd))
	+
	+#define ITS_FLAGS_CMDQ_FLUSH (1UL << 0)
	+
	+#define ITS_TARGET_NONE 0xFBADBEEF
	+
	+struct gic_v3_its_softc {
	+ device_t dev;
	+ struct resource * its_res;
	+
	+ struct its_cmd * its_cmdq_base; /* ITS command queue base */
	+ struct its_cmd * its_cmdq_write; /* ITS command queue write ptr */
	+ struct its_ptab its_ptabs[GITS_BASER_NUM];/* ITS private tables */
	+ struct its_col * its_cols; /* Per-CPU collections */
	+
	+ uint64_t its_flags;
	+
	+ struct its_dev_list its_dev_list;
	+
	+ unsigned long * its_lpi_bitmap;
	+ uint32_t its_lpi_maxid;
	+
	+ struct mtx its_mtx;
	+ struct mtx its_spin_mtx;
	+};
	+
	+extern devclass_t gic_v3_its_devclass;
	+
	+int gic_v3_its_detach(device_t);
	+
	+int gic_v3_its_alloc_msix(device_t, device_t, int *);
	+int gic_v3_its_alloc_msi(device_t, device_t, int, int *);
	+int gic_v3_its_map_msix(device_t, device_t, int, uint64_t , uint32_t );
	+
	+void lpi_unmask_irq(device_t, uint32_t);
	+void lpi_mask_irq(device_t, uint32_t);
	+/*
	* GIC Distributor accessors.
	* Notice that only GIC sofc can be passed.
	*/
	@@ -103,4 +277,28 @@
	reg, val); \
	})

	+#define PCI_DEVID(pci_dev) \
	+({ \
	+ (((pci_get_domain(pci_dev) >> 2) << 19) \| \
	+ ((pci_get_domain(pci_dev) % 4) << 16) \| \
	+ (pci_get_bus(pci_dev) << 8) \| \
	+ (pci_get_slot(pci_dev) << 3) \| \
	+ (pci_get_function(pci_dev) << 0)); \
	+})
	+
	+/*
	+ * Request number of maximum MSI-X vectors for this device.
	+ * Device can ask for less vectors than maximum supported but not more.
	+ */
	+#define PCI_MSIX_NUM(pci_dev) \
	+({ \
	+ struct pci_devinfo *dinfo; \
	+ pcicfgregs *cfg; \
	+ \
	+ dinfo = device_get_ivars(pci_dev); \
	+ cfg = &dinfo->cfg; \
	+ \
	+ cfg->msix.msix_msgnum; \
	+})
	+
	#endif /* _GIC_V3_VAR_H_ */
	Index: sys/arm64/include/param.h
	===================================================================
	--- sys/arm64/include/param.h
	+++ sys/arm64/include/param.h
	@@ -82,6 +82,10 @@
	#define PAGE_SIZE (1 << PAGE_SHIFT) /* Page size */
	#define PAGE_MASK (PAGE_SIZE - 1)

	+#define PAGE_SHIFT_16K 14
	+#define PAGE_SIZE_16K (1 << PAGE_SHIFT_16K)
	+#define PAGE_MASK_16K (PAGE_SIZE_16K - 1)
	+
	#define PAGE_SHIFT_64K 16
	#define PAGE_SIZE_64K (1 << PAGE_SHIFT_64K)
	#define PAGE_MASK_64K (PAGE_SIZE_64K - 1)
	Index: sys/conf/files.arm64
	===================================================================
	--- sys/conf/files.arm64
	+++ sys/conf/files.arm64
	@@ -27,6 +27,7 @@
	arm64/arm64/gic_fdt.c optional fdt
	arm64/arm64/gic_v3.c standard
	arm64/arm64/gic_v3_fdt.c optional fdt
	+arm64/arm64/gic_v3_its.c standard
	arm64/arm64/identcpu.c standard
	arm64/arm64/intr_machdep.c standard
	arm64/arm64/in_cksum.c optional inet \| inet6

File Metadata

Mime Type: text/plain
Expires: Mon, Oct 27, 8:32 AM (7 h, 26 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 24284251
Default Alt Text: D2378.id6978.vs5978.diff (60 KB)

D2378.id6978.vs5978.diffNo OneTemporaryActions

D2378.id6978.vs5978.diffView Options

File Metadata

Event Timeline

D2378.id6978.vs5978.diff
No OneTemporary
Actions

D2378.id6978.vs5978.diff
View Options