Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm64/arm64/gicv3_its.c
Property | Old Value | New Value |
---|---|---|
svn:keywords | null | FreeBSD=%H \ No newline at end of property |
/*- | |||||
* Copyright (c) 2015-2016 The FreeBSD Foundation | |||||
* All rights reserved. | |||||
* | |||||
* This software was developed by Andrew Turner under | |||||
* the sponsorship of the FreeBSD Foundation. | |||||
* | |||||
* This software was developed by Semihalf under | |||||
* the sponsorship of the FreeBSD Foundation. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | |||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | |||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||||
* SUCH DAMAGE. | |||||
*/ | |||||
#include "opt_platform.h" | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD$"); | |||||
#include <sys/param.h> | |||||
#include <sys/systm.h> | |||||
#include <sys/bus.h> | |||||
#include <sys/cpuset.h> | |||||
#include <sys/endian.h> | |||||
#include <sys/kernel.h> | |||||
#include <sys/malloc.h> | |||||
#include <sys/module.h> | |||||
#include <sys/proc.h> | |||||
#include <sys/queue.h> | |||||
#include <sys/rman.h> | |||||
#include <sys/smp.h> | |||||
#include <sys/vmem.h> | |||||
#include <vm/vm.h> | |||||
#include <vm/pmap.h> | |||||
#include <machine/bus.h> | |||||
#include <machine/intr.h> | |||||
#include <arm64/arm64/gic_v3_reg.h> | |||||
#include <arm64/arm64/gic_v3_var.h> | |||||
#ifdef FDT | |||||
#include <dev/ofw/openfirm.h> | |||||
#include <dev/ofw/ofw_bus.h> | |||||
#include <dev/ofw/ofw_bus_subr.h> | |||||
#endif | |||||
#include <dev/pci/pcireg.h> | |||||
#include <dev/pci/pcivar.h> | |||||
#include "pcib_if.h" | |||||
#include "pic_if.h" | |||||
#include "msi_if.h" | |||||
MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", | |||||
"ARM GICv3 Interrupt Translation Service"); | |||||
#define LPI_NIRQS (64 * 1024) | |||||
/* The size and alignment of the command circular buffer */ | |||||
#define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ | |||||
#define ITS_CMDQ_ALIGN (64 * 1024) | |||||
#define LPI_CONFTAB_SIZE LPI_NIRQS | |||||
#define LPI_CONFTAB_ALIGN (64 * 1024) | |||||
#define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ | |||||
/* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ | |||||
#define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) | |||||
#define LPI_PENDTAB_ALIGN (64 * 1024) | |||||
#define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ | |||||
#define LPI_INT_TRANS_TAB_ALIGN 256 | |||||
#define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) | |||||
/* ITS commands encoding */ | |||||
#define ITS_CMD_MOVI (0x01) | |||||
#define ITS_CMD_SYNC (0x05) | |||||
#define ITS_CMD_MAPD (0x08) | |||||
#define ITS_CMD_MAPC (0x09) | |||||
#define ITS_CMD_MAPTI (0x0a) | |||||
#define ITS_CMD_MAPI (0x0b) | |||||
#define ITS_CMD_INV (0x0c) | |||||
#define ITS_CMD_INVALL (0x0d) | |||||
/* Command */ | |||||
#define CMD_COMMAND_MASK (0xFFUL) | |||||
/* PCI device ID */ | |||||
#define CMD_DEVID_SHIFT (32) | |||||
#define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) | |||||
/* Size of IRQ ID bitfield */ | |||||
#define CMD_SIZE_MASK (0xFFUL) | |||||
/* Virtual LPI ID */ | |||||
#define CMD_ID_MASK (0xFFFFFFFFUL) | |||||
/* Physical LPI ID */ | |||||
#define CMD_PID_SHIFT (32) | |||||
#define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) | |||||
/* Collection */ | |||||
#define CMD_COL_MASK (0xFFFFUL) | |||||
/* Target (CPU or Re-Distributor) */ | |||||
#define CMD_TARGET_SHIFT (16) | |||||
#define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) | |||||
/* Interrupt Translation Table address */ | |||||
#define CMD_ITT_MASK (0xFFFFFFFFFF00UL) | |||||
/* Valid command bit */ | |||||
#define CMD_VALID_SHIFT (63) | |||||
#define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) | |||||
/* ITS command. Each command is 32 bytes long */ | |||||
struct its_cmd { | |||||
uint64_t cmd_dword[4]; /* ITS command double word */ | |||||
}; | |||||
/* An ITS private table */ | |||||
struct its_ptable { | |||||
vm_offset_t ptab_vaddr; | |||||
unsigned long ptab_size; | |||||
}; | |||||
/* ITS collection description. */ | |||||
struct its_col { | |||||
uint64_t col_target; /* Target Re-Distributor */ | |||||
uint64_t col_id; /* Collection ID */ | |||||
}; | |||||
struct gicv3_its_irqsrc { | |||||
struct intr_irqsrc gi_isrc; | |||||
u_int gi_irq; | |||||
struct its_dev *gi_its_dev; | |||||
}; | |||||
struct gicv3_its_softc { | |||||
struct intr_pic *sc_pic; | |||||
struct resource *sc_its_res; | |||||
struct its_ptable sc_its_ptab[GITS_BASER_NUM]; | |||||
struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ | |||||
/* | |||||
* TODO: We should get these from the parent as we only want a | |||||
* single copy of each across the interrupt controller. | |||||
*/ | |||||
vm_offset_t sc_conf_base; | |||||
vm_offset_t sc_pend_base[MAXCPU]; | |||||
/* Command handling */ | |||||
struct mtx sc_its_cmd_lock; | |||||
struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ | |||||
size_t sc_its_cmd_next_idx; | |||||
vmem_t *sc_irq_alloc; | |||||
struct gicv3_its_irqsrc *sc_irqs; | |||||
struct mtx sc_its_dev_lock; | |||||
TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; | |||||
#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 | |||||
#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 | |||||
#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 | |||||
u_int sc_its_flags; | |||||
}; | |||||
typedef void (its_quirk_func_t)(device_t); | |||||
static its_quirk_func_t its_quirk_cavium_22375; | |||||
static const struct { | |||||
const char *desc; | |||||
uint32_t iidr; | |||||
uint32_t iidr_mask; | |||||
its_quirk_func_t *func; | |||||
} its_quirks[] = { | |||||
{ | |||||
/* Cavium ThunderX Pass 1.x */ | |||||
.desc = "Cavoum ThunderX errata: 22375, 24313", | |||||
.iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, | |||||
GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), | |||||
.iidr_mask = ~GITS_IIDR_REVISION_MASK, | |||||
.func = its_quirk_cavium_22375, | |||||
}, | |||||
}; | |||||
static u_int gic_irq_cpu; | |||||
#define gic_its_read_4(sc, reg) \ | |||||
bus_read_4((sc)->sc_its_res, (reg)) | |||||
#define gic_its_read_8(sc, reg) \ | |||||
bus_read_8((sc)->sc_its_res, (reg)) | |||||
#define gic_its_write_4(sc, reg, val) \ | |||||
bus_write_4((sc)->sc_its_res, (reg), (val)) | |||||
#define gic_its_write_8(sc, reg, val) \ | |||||
bus_write_8((sc)->sc_its_res, (reg), (val)) | |||||
static device_attach_t gicv3_its_attach; | |||||
static device_detach_t gicv3_its_detach; | |||||
static pic_disable_intr_t gicv3_its_disable_intr; | |||||
static pic_enable_intr_t gicv3_its_enable_intr; | |||||
static pic_map_intr_t gicv3_its_map_intr; | |||||
static pic_setup_intr_t gicv3_its_setup_intr; | |||||
static pic_post_filter_t gicv3_its_post_filter; | |||||
static pic_post_ithread_t gicv3_its_post_ithread; | |||||
static pic_pre_ithread_t gicv3_its_pre_ithread; | |||||
static pic_bind_intr_t gicv3_its_bind_intr; | |||||
#ifdef SMP | |||||
static pic_init_secondary_t gicv3_its_init_secondary; | |||||
#endif | |||||
static msi_alloc_msi_t gicv3_its_alloc_msi; | |||||
static msi_release_msi_t gicv3_its_release_msi; | |||||
static msi_alloc_msix_t gicv3_its_alloc_msix; | |||||
static msi_release_msix_t gicv3_its_release_msix; | |||||
static msi_map_msi_t gicv3_its_map_msi; | |||||
static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); | |||||
static void its_cmd_mapc(device_t, struct its_col *, uint8_t); | |||||
static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); | |||||
static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); | |||||
static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); | |||||
static void its_cmd_invall(device_t, struct its_col *); | |||||
static device_method_t gicv3_its_methods[] = { | |||||
/* Device interface */ | |||||
DEVMETHOD(device_detach, gicv3_its_detach), | |||||
/* Interrupt controller interface */ | |||||
DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), | |||||
DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), | |||||
DEVMETHOD(pic_map_intr, gicv3_its_map_intr), | |||||
DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), | |||||
DEVMETHOD(pic_post_filter, gicv3_its_post_filter), | |||||
DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), | |||||
DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), | |||||
#ifdef SMP | |||||
DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), | |||||
DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), | |||||
#endif | |||||
/* MSI/MSI-X */ | |||||
DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), | |||||
DEVMETHOD(msi_release_msi, gicv3_its_release_msi), | |||||
DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), | |||||
DEVMETHOD(msi_release_msix, gicv3_its_release_msix), | |||||
DEVMETHOD(msi_map_msi, gicv3_its_map_msi), | |||||
/* End */ | |||||
DEVMETHOD_END | |||||
}; | |||||
static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, | |||||
sizeof(struct gicv3_its_softc)); | |||||
static void | |||||
gicv3_its_cmdq_init(struct gicv3_its_softc *sc) | |||||
{ | |||||
vm_paddr_t cmd_paddr; | |||||
uint64_t reg, tmp; | |||||
/* Set up the command circular buffer */ | |||||
sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, | |||||
M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); | |||||
sc->sc_its_cmd_next_idx = 0; | |||||
cmd_paddr = vtophys(sc->sc_its_cmd_base); | |||||
/* Set the base of the command buffer */ | |||||
reg = GITS_CBASER_VALID | | |||||
(GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | | |||||
cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | | |||||
(ITS_CMDQ_SIZE / 4096 - 1); | |||||
gic_its_write_8(sc, GITS_CBASER, reg); | |||||
/* Read back to check for fixed value fields */ | |||||
tmp = gic_its_read_8(sc, GITS_CBASER); | |||||
if ((tmp & GITS_CBASER_SHARE_MASK) != | |||||
(GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { | |||||
/* Check if the hardware reported non-shareable */ | |||||
if ((tmp & GITS_CBASER_SHARE_MASK) == | |||||
(GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { | |||||
/* If so remove the cache attribute */ | |||||
reg &= ~GITS_CBASER_CACHE_MASK; | |||||
reg &= ~GITS_CBASER_SHARE_MASK; | |||||
/* Set to Non-cacheable, Non-shareable */ | |||||
reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; | |||||
reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; | |||||
gic_its_write_8(sc, GITS_CBASER, reg); | |||||
} | |||||
/* The command queue has to be flushed after each command */ | |||||
sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; | |||||
} | |||||
/* Get the next command from the start of the buffer */ | |||||
gic_its_write_8(sc, GITS_CWRITER, 0x0); | |||||
} | |||||
static int | |||||
gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) | |||||
{ | |||||
vm_offset_t table; | |||||
vm_paddr_t paddr; | |||||
uint64_t cache, reg, share, tmp, type; | |||||
size_t esize, its_tbl_size, nidents, nitspages, npages; | |||||
int i, page_size; | |||||
int devbits; | |||||
if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { | |||||
/* | |||||
* GITS_TYPER[17:13] of ThunderX reports that device IDs | |||||
* are to be 21 bits in length. The entry size of the ITS | |||||
* table can be read from GITS_BASERn[52:48] and on ThunderX | |||||
* is supposed to be 8 bytes in length (for device table). | |||||
* Finally the page size that is to be used by ITS to access | |||||
* this table will be set to 64KB. | |||||
* | |||||
* This gives 0x200000 entries of size 0x8 bytes covered by | |||||
* 256 pages each of which 64KB in size. The number of pages | |||||
* (minus 1) should then be written to GITS_BASERn[7:0]. In | |||||
* that case this value would be 0xFF but on ThunderX the | |||||
* maximum value that HW accepts is 0xFD. | |||||
* | |||||
* Set an arbitrary number of device ID bits to 20 in order | |||||
* to limit the number of entries in ITS device table to | |||||
* 0x100000 and the table size to 8MB. | |||||
*/ | |||||
devbits = 20; | |||||
cache = 0; | |||||
} else { | |||||
devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); | |||||
cache = GITS_BASER_CACHE_WAWB; | |||||
} | |||||
share = GITS_BASER_SHARE_IS; | |||||
page_size = PAGE_SIZE_64K; | |||||
for (i = 0; i < GITS_BASER_NUM; i++) { | |||||
reg = gic_its_read_8(sc, GITS_BASER(i)); | |||||
/* The type of table */ | |||||
type = GITS_BASER_TYPE(reg); | |||||
/* The table entry size */ | |||||
esize = GITS_BASER_ESIZE(reg); | |||||
switch(type) { | |||||
case GITS_BASER_TYPE_DEV: | |||||
nidents = (1 << devbits); | |||||
its_tbl_size = esize * nidents; | |||||
its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); | |||||
break; | |||||
case GITS_BASER_TYPE_VP: | |||||
case GITS_BASER_TYPE_PP: /* Undocumented? */ | |||||
case GITS_BASER_TYPE_IC: | |||||
its_tbl_size = page_size; | |||||
break; | |||||
default: | |||||
continue; | |||||
} | |||||
npages = howmany(its_tbl_size, PAGE_SIZE); | |||||
/* Allocate the table */ | |||||
table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, | |||||
M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, | |||||
PAGE_SIZE, 0); | |||||
sc->sc_its_ptab[i].ptab_vaddr = table; | |||||
sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; | |||||
paddr = vtophys(table); | |||||
while (1) { | |||||
nitspages = howmany(its_tbl_size, page_size); | |||||
/* Clear the fields we will be setting */ | |||||
reg &= ~(GITS_BASER_VALID | | |||||
GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | | |||||
GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | | |||||
GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | | |||||
GITS_BASER_SIZE_MASK); | |||||
/* Set the new values */ | |||||
reg |= GITS_BASER_VALID | | |||||
(cache << GITS_BASER_CACHE_SHIFT) | | |||||
(type << GITS_BASER_TYPE_SHIFT) | | |||||
((esize - 1) << GITS_BASER_ESIZE_SHIFT) | | |||||
paddr | (share << GITS_BASER_SHARE_SHIFT) | | |||||
(nitspages - 1); | |||||
switch (page_size) { | |||||
case PAGE_SIZE: /* 4KB */ | |||||
reg |= | |||||
GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; | |||||
break; | |||||
case PAGE_SIZE_16K: /* 16KB */ | |||||
reg |= | |||||
GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; | |||||
break; | |||||
case PAGE_SIZE_64K: /* 64KB */ | |||||
reg |= | |||||
GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; | |||||
break; | |||||
} | |||||
gic_its_write_8(sc, GITS_BASER(i), reg); | |||||
/* Read back to check */ | |||||
tmp = gic_its_read_8(sc, GITS_BASER(i)); | |||||
/* Do the snareability masks line up? */ | |||||
if ((tmp & GITS_BASER_SHARE_MASK) != | |||||
(reg & GITS_BASER_SHARE_MASK)) { | |||||
share = (tmp & GITS_BASER_SHARE_MASK) >> | |||||
GITS_BASER_SHARE_SHIFT; | |||||
continue; | |||||
} | |||||
if ((tmp & GITS_BASER_PSZ_MASK) != | |||||
(reg & GITS_BASER_PSZ_MASK)) { | |||||
switch (page_size) { | |||||
case PAGE_SIZE_16K: | |||||
page_size = PAGE_SIZE; | |||||
continue; | |||||
case PAGE_SIZE_64K: | |||||
page_size = PAGE_SIZE_16K; | |||||
continue; | |||||
} | |||||
} | |||||
if (tmp != reg) { | |||||
device_printf(dev, "GITS_BASER%d: " | |||||
"unable to be updated: %lx != %lx\n", | |||||
i, reg, tmp); | |||||
return (ENXIO); | |||||
} | |||||
/* We should have made all needed changes */ | |||||
break; | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
static void | |||||
gicv3_its_conftable_init(struct gicv3_its_softc *sc) | |||||
{ | |||||
sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, | |||||
M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, | |||||
0); | |||||
/* Set the default configuration */ | |||||
memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, | |||||
LPI_CONFTAB_SIZE); | |||||
/* Flush the table to memory */ | |||||
cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); | |||||
} | |||||
static void | |||||
gicv3_its_pendtables_init(struct gicv3_its_softc *sc) | |||||
{ | |||||
int i; | |||||
for (i = 0; i < mp_ncpus; i++) { | |||||
if (CPU_ISSET(i, &all_cpus) == 0) | |||||
continue; | |||||
sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( | |||||
LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, | |||||
0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); | |||||
/* Flush so the ITS can see the memory */ | |||||
cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base, | |||||
LPI_PENDTAB_SIZE); | |||||
} | |||||
} | |||||
static int | |||||
its_init_cpu(device_t dev, struct gicv3_its_softc *sc) | |||||
{ | |||||
device_t gicv3; | |||||
vm_paddr_t target; | |||||
uint64_t xbaser, tmp; | |||||
uint32_t ctlr; | |||||
u_int cpuid; | |||||
gicv3 = device_get_parent(dev); | |||||
cpuid = PCPU_GET(cpuid); | |||||
/* Check if the ITS is enabled on this CPU */ | |||||
if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { | |||||
return (ENXIO); | |||||
} | |||||
/* Disable LPIs */ | |||||
ctlr = gic_r_read_4(gicv3, GICR_CTLR); | |||||
ctlr &= ~GICR_CTLR_LPI_ENABLE; | |||||
gic_r_write_4(gicv3, GICR_CTLR, ctlr); | |||||
/* Make sure changes are observable my the GIC */ | |||||
dsb(sy); | |||||
/* | |||||
* Set the redistributor base | |||||
*/ | |||||
xbaser = vtophys(sc->sc_conf_base) | | |||||
(GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | | |||||
(GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | | |||||
(flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); | |||||
gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); | |||||
/* Check the cache attributes we set */ | |||||
tmp = gic_r_read_8(gicv3, GICR_PROPBASER); | |||||
if ((tmp & GICR_PROPBASER_SHARE_MASK) != | |||||
(xbaser & GICR_PROPBASER_SHARE_MASK)) { | |||||
if ((tmp & GICR_PROPBASER_SHARE_MASK) == | |||||
(GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { | |||||
/* We need to mark as non-cacheable */ | |||||
xbaser &= ~(GICR_PROPBASER_SHARE_MASK | | |||||
GICR_PROPBASER_CACHE_MASK); | |||||
/* Non-cacheable */ | |||||
xbaser |= GICR_PROPBASER_CACHE_NIN << | |||||
GICR_PROPBASER_CACHE_SHIFT; | |||||
/* Non-sareable */ | |||||
xbaser |= GICR_PROPBASER_SHARE_NS << | |||||
GICR_PROPBASER_SHARE_SHIFT; | |||||
gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); | |||||
} | |||||
sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; | |||||
} | |||||
/* | |||||
* Set the LPI pending table base | |||||
*/ | |||||
xbaser = vtophys(sc->sc_pend_base[cpuid]) | | |||||
(GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | | |||||
(GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); | |||||
gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); | |||||
tmp = gic_r_read_8(gicv3, GICR_PENDBASER); | |||||
if ((tmp & GICR_PENDBASER_SHARE_MASK) == | |||||
(GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { | |||||
/* Clear the cahce and shareability bits */ | |||||
xbaser &= ~(GICR_PENDBASER_CACHE_MASK | | |||||
GICR_PENDBASER_SHARE_MASK); | |||||
/* Mark as non-shareable */ | |||||
xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; | |||||
/* And non-cacheable */ | |||||
xbaser |= GICR_PENDBASER_CACHE_NIN << | |||||
GICR_PENDBASER_CACHE_SHIFT; | |||||
} | |||||
/* Enable LPIs */ | |||||
ctlr = gic_r_read_4(gicv3, GICR_CTLR); | |||||
ctlr |= GICR_CTLR_LPI_ENABLE; | |||||
gic_r_write_4(gicv3, GICR_CTLR, ctlr); | |||||
/* Make sure the GIC has seen everything */ | |||||
dsb(sy); | |||||
if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { | |||||
/* This ITS wants the redistributor physical address */ | |||||
target = vtophys(gicv3_get_redist_vaddr(dev)); | |||||
} else { | |||||
/* This ITS wants the unique processor number */ | |||||
target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); | |||||
} | |||||
sc->sc_its_cols[cpuid]->col_target = target; | |||||
sc->sc_its_cols[cpuid]->col_id = cpuid; | |||||
its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); | |||||
its_cmd_invall(dev, sc->sc_its_cols[cpuid]); | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_attach(device_t dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
const char *name; | |||||
uint32_t iidr; | |||||
int err, i, rid; | |||||
sc = device_get_softc(dev); | |||||
rid = 0; | |||||
sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, | |||||
RF_ACTIVE); | |||||
if (sc->sc_its_res == NULL) { | |||||
device_printf(dev, "Could not allocate memory\n"); | |||||
return (ENXIO); | |||||
} | |||||
iidr = gic_its_read_4(sc, GITS_IIDR); | |||||
for (i = 0; i < nitems(its_quirks); i++) { | |||||
if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { | |||||
if (bootverbose) { | |||||
device_printf(dev, "Applying %s\n", | |||||
its_quirks[i].desc); | |||||
} | |||||
its_quirks[i].func(dev); | |||||
break; | |||||
} | |||||
} | |||||
/* Allocate the private tables */ | |||||
err = gicv3_its_table_init(dev, sc); | |||||
if (err != 0) | |||||
return (err); | |||||
/* Protects access to the device list */ | |||||
mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); | |||||
/* Protects access to the ITS command circular buffer. */ | |||||
mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); | |||||
/* Allocate the command circular buffer */ | |||||
gicv3_its_cmdq_init(sc); | |||||
/* Allocate the per-CPU collections */ | |||||
for (int cpu = 0; cpu < mp_ncpus; cpu++) | |||||
if (CPU_ISSET(cpu, &all_cpus) != 0) | |||||
sc->sc_its_cols[cpu] = malloc( | |||||
sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, | |||||
M_WAITOK | M_ZERO); | |||||
else | |||||
sc->sc_its_cols[cpu] = NULL; | |||||
/* Enable the ITS */ | |||||
gic_its_write_4(sc, GITS_CTLR, | |||||
gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); | |||||
/* Create the LPI configuration table */ | |||||
gicv3_its_conftable_init(sc); | |||||
/* And the pending tebles */ | |||||
gicv3_its_pendtables_init(sc); | |||||
/* Enable LPIs on this CPU */ | |||||
its_init_cpu(dev, sc); | |||||
TAILQ_INIT(&sc->sc_its_dev_list); | |||||
/* | |||||
* Create the vmem object to allocate IRQs from. We try to use all | |||||
* IRQs not already used by the GICv3. | |||||
* XXX: This assumes there are no other interrupt controllers in the | |||||
* system. | |||||
*/ | |||||
sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0, | |||||
NIRQ - gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK); | |||||
sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * LPI_NIRQS, M_GICV3_ITS, | |||||
M_WAITOK | M_ZERO); | |||||
name = device_get_nameunit(dev); | |||||
for (i = 0; i < LPI_NIRQS; i++) { | |||||
sc->sc_irqs[i].gi_irq = i; | |||||
err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, | |||||
"%s,%u", name, i); | |||||
} | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_detach(device_t dev) | |||||
{ | |||||
return (ENXIO); | |||||
} | |||||
static void | |||||
its_quirk_cavium_22375(device_t dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
sc = device_get_softc(dev); | |||||
sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; | |||||
} | |||||
static void | |||||
gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
uint8_t *conf; | |||||
sc = device_get_softc(dev); | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
conf = (uint8_t *)sc->sc_conf_base; | |||||
conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; | |||||
if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { | |||||
/* Clean D-cache under command. */ | |||||
cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); | |||||
} else { | |||||
/* DSB inner shareable, store */ | |||||
dsb(ishst); | |||||
} | |||||
its_cmd_inv(dev, girq->gi_its_dev, girq); | |||||
} | |||||
static void | |||||
gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
uint8_t *conf; | |||||
sc = device_get_softc(dev); | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
conf = (uint8_t *)sc->sc_conf_base; | |||||
conf[girq->gi_irq] |= LPI_CONF_ENABLE; | |||||
if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { | |||||
/* Clean D-cache under command. */ | |||||
cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); | |||||
} else { | |||||
/* DSB inner shareable, store */ | |||||
dsb(ishst); | |||||
} | |||||
its_cmd_inv(dev, girq->gi_its_dev, girq); | |||||
} | |||||
static int | |||||
gicv3_its_intr(void *arg, uintptr_t irq) | |||||
{ | |||||
struct gicv3_its_softc *sc = arg; | |||||
struct gicv3_its_irqsrc *girq; | |||||
struct trapframe *tf; | |||||
irq -= GIC_FIRST_LPI; | |||||
girq = &sc->sc_irqs[irq]; | |||||
if (girq == NULL) | |||||
panic("gicv3_its_intr: Invalid interrupt %ld", | |||||
irq + GIC_FIRST_LPI); | |||||
tf = curthread->td_intr_frame; | |||||
intr_isrc_dispatch(&girq->gi_isrc, tf); | |||||
return (FILTER_HANDLED); | |||||
} | |||||
static void | |||||
gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_irqsrc *girq; | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
gicv3_its_disable_intr(dev, isrc); | |||||
gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); | |||||
} | |||||
static void | |||||
gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
gicv3_its_enable_intr(dev, isrc); | |||||
} | |||||
static void | |||||
gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_irqsrc *girq; | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
gic_icc_write(EOIR1, girq->gi_irq + GIC_FIRST_LPI); | |||||
} | |||||
static int | |||||
gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_irqsrc *girq; | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
if (CPU_EMPTY(&isrc->isrc_cpu)) { | |||||
gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); | |||||
CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); | |||||
} | |||||
its_cmd_movi(dev, girq); | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_map_intr(device_t dev, struct intr_map_data *data, | |||||
struct intr_irqsrc **isrcp) | |||||
{ | |||||
/* | |||||
* This should never happen, we only call this function to map | |||||
* interrupts found before the controller driver is ready. | |||||
*/ | |||||
panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); | |||||
} | |||||
static int | |||||
gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, | |||||
struct resource *res, struct intr_map_data *data) | |||||
{ | |||||
/* Bind the interrupt to a CPU */ | |||||
gicv3_its_bind_intr(dev, isrc); | |||||
return (0); | |||||
} | |||||
#ifdef SMP | |||||
static void | |||||
gicv3_its_init_secondary(device_t dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
sc = device_get_softc(dev); | |||||
/* | |||||
* This is fatal as otherwise we may bind interrupts to this CPU. | |||||
* We need a way to tell the interrupt framework to only bind to a | |||||
* subset of given CPUs when it performs the shuffle. | |||||
*/ | |||||
if (its_init_cpu(dev, sc) != 0) | |||||
panic("gicv3_its_init_secondary: No usable ITS on CPU%d", | |||||
PCPU_GET(cpuid)); | |||||
} | |||||
#endif | |||||
static uint32_t | |||||
its_get_devid(device_t pci_dev) | |||||
{ | |||||
uintptr_t id; | |||||
if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) | |||||
panic("its_get_devid: Unable to get the MSI DeviceID"); | |||||
return (id); | |||||
} | |||||
static struct its_dev * | |||||
its_device_find(device_t dev, device_t child) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_dev *its_dev = NULL; | |||||
sc = device_get_softc(dev); | |||||
mtx_lock_spin(&sc->sc_its_dev_lock); | |||||
TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { | |||||
if (its_dev->pci_dev == child) | |||||
break; | |||||
} | |||||
mtx_unlock_spin(&sc->sc_its_dev_lock); | |||||
return (its_dev); | |||||
} | |||||
static struct its_dev * | |||||
its_device_get(device_t dev, device_t child, u_int nvecs) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_dev *its_dev; | |||||
vmem_addr_t irq_base; | |||||
size_t esize; | |||||
sc = device_get_softc(dev); | |||||
its_dev = its_device_find(dev, child); | |||||
if (its_dev != NULL) | |||||
return (its_dev); | |||||
its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); | |||||
if (its_dev == NULL) | |||||
return (NULL); | |||||
its_dev->pci_dev = child; | |||||
its_dev->devid = its_get_devid(child); | |||||
its_dev->lpis.lpi_busy = 0; | |||||
its_dev->lpis.lpi_num = nvecs; | |||||
its_dev->lpis.lpi_free = nvecs; | |||||
if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, | |||||
&irq_base) != 0) { | |||||
free(its_dev, M_GICV3_ITS); | |||||
return (NULL); | |||||
} | |||||
its_dev->lpis.lpi_base = irq_base; | |||||
/* Get ITT entry size */ | |||||
esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); | |||||
/* | |||||
* Allocate ITT for this device. | |||||
* PA has to be 256 B aligned. At least two entries for device. | |||||
*/ | |||||
its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); | |||||
its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, | |||||
M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, | |||||
LPI_INT_TRANS_TAB_ALIGN, 0); | |||||
if (its_dev->itt == 0) { | |||||
vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); | |||||
free(its_dev, M_GICV3_ITS); | |||||
return (NULL); | |||||
} | |||||
mtx_lock_spin(&sc->sc_its_dev_lock); | |||||
TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); | |||||
mtx_unlock_spin(&sc->sc_its_dev_lock); | |||||
/* Map device to its ITT */ | |||||
its_cmd_mapd(dev, its_dev, 1); | |||||
return (its_dev); | |||||
} | |||||
static void | |||||
its_device_release(device_t dev, struct its_dev *its_dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
KASSERT(its_dev->lpis.lpi_busy == 0, | |||||
("its_device_release: Trying to release an inuse ITS device")); | |||||
/* Unmap device in ITS */ | |||||
its_cmd_mapd(dev, its_dev, 0); | |||||
sc = device_get_softc(dev); | |||||
/* Remove the device from the list of devices */ | |||||
mtx_lock_spin(&sc->sc_its_dev_lock); | |||||
TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); | |||||
mtx_unlock_spin(&sc->sc_its_dev_lock); | |||||
/* Free ITT */ | |||||
KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); | |||||
contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); | |||||
/* Free the IRQ allocation */ | |||||
vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, | |||||
its_dev->lpis.lpi_num); | |||||
free(its_dev, M_GICV3_ITS); | |||||
} | |||||
static int | |||||
gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, | |||||
device_t *pic, struct intr_irqsrc **srcs) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
struct its_dev *its_dev; | |||||
u_int irq; | |||||
int i; | |||||
its_dev = its_device_get(dev, child, count); | |||||
if (its_dev == NULL) | |||||
return (ENXIO); | |||||
KASSERT(its_dev->lpis.lpi_free >= count, | |||||
("gicv3_its_alloc_msi: No free LPIs")); | |||||
sc = device_get_softc(dev); | |||||
irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - | |||||
its_dev->lpis.lpi_free; | |||||
for (i = 0; i < count; i++, irq++) { | |||||
its_dev->lpis.lpi_free--; | |||||
girq = &sc->sc_irqs[irq]; | |||||
girq->gi_its_dev = its_dev; | |||||
srcs[i] = (struct intr_irqsrc *)girq; | |||||
} | |||||
its_dev->lpis.lpi_busy += count; | |||||
*pic = dev; | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_release_msi(device_t dev, device_t child, int count, | |||||
struct intr_irqsrc **isrc) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
struct its_dev *its_dev; | |||||
int i; | |||||
sc = device_get_softc(dev); | |||||
its_dev = its_device_find(dev, child); | |||||
KASSERT(its_dev != NULL, | |||||
("gicv3_its_release_msi: Releasing a MSI interrupt with " | |||||
"no ITS device")); | |||||
KASSERT(its_dev->lpis.lpi_busy >= count, | |||||
("gicv3_its_release_msi: Releasing more interrupts than " | |||||
"were allocated: releasing %d, allocated %d", count, | |||||
its_dev->lpis.lpi_busy)); | |||||
for (i = 0; i < count; i++) { | |||||
girq = (struct gicv3_its_irqsrc *)isrc[i]; | |||||
girq->gi_its_dev = NULL; | |||||
} | |||||
its_dev->lpis.lpi_busy -= count; | |||||
if (its_dev->lpis.lpi_busy == 0) | |||||
its_device_release(dev, its_dev); | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, | |||||
struct intr_irqsrc **isrcp) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
struct its_dev *its_dev; | |||||
u_int nvecs, irq; | |||||
nvecs = pci_msix_count(child); | |||||
its_dev = its_device_get(dev, child, nvecs); | |||||
if (its_dev == NULL) | |||||
return (ENXIO); | |||||
KASSERT(its_dev->lpis.lpi_free > 0, | |||||
("gicv3_its_alloc_msix: No free LPIs")); | |||||
sc = device_get_softc(dev); | |||||
irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - | |||||
its_dev->lpis.lpi_free; | |||||
its_dev->lpis.lpi_free--; | |||||
its_dev->lpis.lpi_busy++; | |||||
girq = &sc->sc_irqs[irq]; | |||||
girq->gi_its_dev = its_dev; | |||||
*pic = dev; | |||||
*isrcp = (struct intr_irqsrc *)girq; | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
struct its_dev *its_dev; | |||||
sc = device_get_softc(dev); | |||||
its_dev = its_device_find(dev, child); | |||||
KASSERT(its_dev != NULL, | |||||
("gicv3_its_release_msix: Releasing a MSI-X interrupt with " | |||||
"no ITS device")); | |||||
KASSERT(its_dev->lpis.lpi_busy > 0, | |||||
("gicv3_its_release_msix: Releasing more interrupts than " | |||||
"were allocated: allocated %d", its_dev->lpis.lpi_busy)); | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
girq->gi_its_dev = NULL; | |||||
its_dev->lpis.lpi_busy--; | |||||
if (its_dev->lpis.lpi_busy == 0) | |||||
its_device_release(dev, its_dev); | |||||
return (0); | |||||
} | |||||
static int | |||||
gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, | |||||
uint64_t *addr, uint32_t *data) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct gicv3_its_irqsrc *girq; | |||||
sc = device_get_softc(dev); | |||||
girq = (struct gicv3_its_irqsrc *)isrc; | |||||
/* Map the message to the given IRQ */ | |||||
its_cmd_mapti(dev, girq); | |||||
*addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; | |||||
*data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; | |||||
return (0); | |||||
} | |||||
/* | |||||
* Commands handling. | |||||
*/ | |||||
static __inline void | |||||
cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) | |||||
{ | |||||
/* Command field: DW0 [7:0] */ | |||||
cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); | |||||
cmd->cmd_dword[0] |= htole64(cmd_type); | |||||
} | |||||
static __inline void | |||||
cmd_format_devid(struct its_cmd *cmd, uint32_t devid) | |||||
{ | |||||
/* Device ID field: DW0 [63:32] */ | |||||
cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); | |||||
cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); | |||||
} | |||||
static __inline void | |||||
cmd_format_size(struct its_cmd *cmd, uint16_t size) | |||||
{ | |||||
/* Size field: DW1 [4:0] */ | |||||
cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); | |||||
cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); | |||||
} | |||||
static __inline void | |||||
cmd_format_id(struct its_cmd *cmd, uint32_t id) | |||||
{ | |||||
/* ID field: DW1 [31:0] */ | |||||
cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); | |||||
cmd->cmd_dword[1] |= htole64(id); | |||||
} | |||||
static __inline void | |||||
cmd_format_pid(struct its_cmd *cmd, uint32_t pid) | |||||
{ | |||||
/* Physical ID field: DW1 [63:32] */ | |||||
cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); | |||||
cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); | |||||
} | |||||
static __inline void | |||||
cmd_format_col(struct its_cmd *cmd, uint16_t col_id) | |||||
{ | |||||
/* Collection field: DW2 [16:0] */ | |||||
cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); | |||||
cmd->cmd_dword[2] |= htole64(col_id); | |||||
} | |||||
static __inline void | |||||
cmd_format_target(struct its_cmd *cmd, uint64_t target) | |||||
{ | |||||
/* Target Address field: DW2 [47:16] */ | |||||
cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); | |||||
cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); | |||||
} | |||||
static __inline void | |||||
cmd_format_itt(struct its_cmd *cmd, uint64_t itt) | |||||
{ | |||||
/* ITT Address field: DW2 [47:8] */ | |||||
cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); | |||||
cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); | |||||
} | |||||
static __inline void | |||||
cmd_format_valid(struct its_cmd *cmd, uint8_t valid) | |||||
{ | |||||
/* Valid field: DW2 [63] */ | |||||
cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); | |||||
cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); | |||||
} | |||||
static inline bool | |||||
its_cmd_queue_full(struct gicv3_its_softc *sc) | |||||
{ | |||||
size_t read_idx, next_write_idx; | |||||
/* Get the index of the next command */ | |||||
next_write_idx = (sc->sc_its_cmd_next_idx + 1) % | |||||
(ITS_CMDQ_SIZE / sizeof(struct its_cmd)); | |||||
/* And the index of the current command being read */ | |||||
read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); | |||||
/* | |||||
* The queue is full when the write offset points | |||||
* at the command before the current read offset. | |||||
*/ | |||||
return (next_write_idx == read_idx); | |||||
} | |||||
static inline void | |||||
its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) | |||||
{ | |||||
if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { | |||||
/* Clean D-cache under command. */ | |||||
cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); | |||||
} else { | |||||
/* DSB inner shareable, store */ | |||||
dsb(ishst); | |||||
} | |||||
} | |||||
static inline uint64_t | |||||
its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) | |||||
{ | |||||
uint64_t off; | |||||
off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); | |||||
return (off); | |||||
} | |||||
static void | |||||
its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, | |||||
struct its_cmd *cmd_last) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
uint64_t first, last, read; | |||||
size_t us_left; | |||||
sc = device_get_softc(dev); | |||||
/* | |||||
* XXX ARM64TODO: This is obviously a significant delay. | |||||
* The reason for that is that currently the time frames for | |||||
* the command to complete are not known. | |||||
*/ | |||||
us_left = 1000000; | |||||
first = its_cmd_cwriter_offset(sc, cmd_first); | |||||
last = its_cmd_cwriter_offset(sc, cmd_last); | |||||
for (;;) { | |||||
read = gic_its_read_8(sc, GITS_CREADR); | |||||
if (first < last) { | |||||
if (read < first || read >= last) | |||||
break; | |||||
} else if (read < first && read >= last) | |||||
break; | |||||
if (us_left-- == 0) { | |||||
/* This means timeout */ | |||||
device_printf(dev, | |||||
"Timeout while waiting for CMD completion.\n"); | |||||
return; | |||||
} | |||||
DELAY(1); | |||||
} | |||||
} | |||||
static struct its_cmd * | |||||
its_cmd_alloc_locked(device_t dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_cmd *cmd; | |||||
size_t us_left; | |||||
sc = device_get_softc(dev); | |||||
/* | |||||
* XXX ARM64TODO: This is obviously a significant delay. | |||||
* The reason for that is that currently the time frames for | |||||
* the command to complete (and therefore free the descriptor) | |||||
* are not known. | |||||
*/ | |||||
us_left = 1000000; | |||||
mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); | |||||
while (its_cmd_queue_full(sc)) { | |||||
if (us_left-- == 0) { | |||||
/* Timeout while waiting for free command */ | |||||
device_printf(dev, | |||||
"Timeout while waiting for free command\n"); | |||||
return (NULL); | |||||
} | |||||
DELAY(1); | |||||
} | |||||
cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; | |||||
sc->sc_its_cmd_next_idx++; | |||||
sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); | |||||
return (cmd); | |||||
} | |||||
static uint64_t | |||||
its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) | |||||
{ | |||||
uint64_t target; | |||||
uint8_t cmd_type; | |||||
u_int size; | |||||
boolean_t error; | |||||
error = FALSE; | |||||
cmd_type = desc->cmd_type; | |||||
target = ITS_TARGET_NONE; | |||||
switch (cmd_type) { | |||||
case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ | |||||
target = desc->cmd_desc_movi.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_MOVI); | |||||
cmd_format_id(cmd, desc->cmd_desc_movi.id); | |||||
cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); | |||||
cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); | |||||
break; | |||||
case ITS_CMD_SYNC: /* Wait for previous commands completion */ | |||||
target = desc->cmd_desc_sync.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_SYNC); | |||||
cmd_format_target(cmd, target); | |||||
break; | |||||
case ITS_CMD_MAPD: /* Assign ITT to device */ | |||||
cmd_format_command(cmd, ITS_CMD_MAPD); | |||||
cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); | |||||
/* | |||||
* Size describes number of bits to encode interrupt IDs | |||||
* supported by the device minus one. | |||||
* When V (valid) bit is zero, this field should be written | |||||
* as zero. | |||||
*/ | |||||
if (desc->cmd_desc_mapd.valid != 0) { | |||||
size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); | |||||
size = MAX(1, size) - 1; | |||||
} else | |||||
size = 0; | |||||
cmd_format_size(cmd, size); | |||||
cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); | |||||
cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); | |||||
break; | |||||
case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ | |||||
target = desc->cmd_desc_mapc.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_MAPC); | |||||
cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); | |||||
cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); | |||||
cmd_format_target(cmd, target); | |||||
break; | |||||
case ITS_CMD_MAPTI: | |||||
target = desc->cmd_desc_mapvi.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_MAPTI); | |||||
cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); | |||||
cmd_format_id(cmd, desc->cmd_desc_mapvi.id); | |||||
cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); | |||||
cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); | |||||
break; | |||||
case ITS_CMD_MAPI: | |||||
target = desc->cmd_desc_mapi.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_MAPI); | |||||
cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); | |||||
cmd_format_id(cmd, desc->cmd_desc_mapi.pid); | |||||
cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); | |||||
break; | |||||
case ITS_CMD_INV: | |||||
target = desc->cmd_desc_inv.col->col_target; | |||||
cmd_format_command(cmd, ITS_CMD_INV); | |||||
cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); | |||||
cmd_format_id(cmd, desc->cmd_desc_inv.pid); | |||||
break; | |||||
case ITS_CMD_INVALL: | |||||
cmd_format_command(cmd, ITS_CMD_INVALL); | |||||
cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); | |||||
break; | |||||
default: | |||||
panic("its_cmd_prepare: Invalid command: %x", cmd_type); | |||||
} | |||||
return (target); | |||||
} | |||||
static int | |||||
its_cmd_send(device_t dev, struct its_cmd_desc *desc) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_cmd *cmd, *cmd_sync, *cmd_write; | |||||
struct its_col col_sync; | |||||
struct its_cmd_desc desc_sync; | |||||
uint64_t target, cwriter; | |||||
sc = device_get_softc(dev); | |||||
mtx_lock_spin(&sc->sc_its_cmd_lock); | |||||
cmd = its_cmd_alloc_locked(dev); | |||||
if (cmd == NULL) { | |||||
device_printf(dev, "could not allocate ITS command\n"); | |||||
mtx_unlock_spin(&sc->sc_its_cmd_lock); | |||||
return (EBUSY); | |||||
} | |||||
target = its_cmd_prepare(cmd, desc); | |||||
its_cmd_sync(sc, cmd); | |||||
if (target != ITS_TARGET_NONE) { | |||||
cmd_sync = its_cmd_alloc_locked(dev); | |||||
if (cmd_sync != NULL) { | |||||
desc_sync.cmd_type = ITS_CMD_SYNC; | |||||
col_sync.col_target = target; | |||||
desc_sync.cmd_desc_sync.col = &col_sync; | |||||
its_cmd_prepare(cmd_sync, &desc_sync); | |||||
its_cmd_sync(sc, cmd_sync); | |||||
} | |||||
} | |||||
/* Update GITS_CWRITER */ | |||||
cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); | |||||
gic_its_write_8(sc, GITS_CWRITER, cwriter); | |||||
cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; | |||||
mtx_unlock_spin(&sc->sc_its_cmd_lock); | |||||
its_cmd_wait_completion(dev, cmd, cmd_write); | |||||
return (0); | |||||
} | |||||
/* Handlers to send commands */ | |||||
static void | |||||
its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_cmd_desc desc; | |||||
struct its_col *col; | |||||
sc = device_get_softc(dev); | |||||
col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; | |||||
desc.cmd_type = ITS_CMD_MOVI; | |||||
desc.cmd_desc_movi.its_dev = girq->gi_its_dev; | |||||
desc.cmd_desc_movi.col = col; | |||||
desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
static void | |||||
its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) | |||||
{ | |||||
struct its_cmd_desc desc; | |||||
desc.cmd_type = ITS_CMD_MAPC; | |||||
desc.cmd_desc_mapc.col = col; | |||||
/* | |||||
* Valid bit set - map the collection. | |||||
* Valid bit cleared - unmap the collection. | |||||
*/ | |||||
desc.cmd_desc_mapc.valid = valid; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
static void | |||||
its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_cmd_desc desc; | |||||
struct its_col *col; | |||||
u_int col_id; | |||||
sc = device_get_softc(dev); | |||||
col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; | |||||
col = sc->sc_its_cols[col_id]; | |||||
desc.cmd_type = ITS_CMD_MAPTI; | |||||
desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; | |||||
desc.cmd_desc_mapvi.col = col; | |||||
/* The EventID sent to the device */ | |||||
desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; | |||||
/* The physical interrupt presented to softeware */ | |||||
desc.cmd_desc_mapvi.pid = girq->gi_irq + GIC_FIRST_LPI; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
static void | |||||
its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) | |||||
{ | |||||
struct its_cmd_desc desc; | |||||
desc.cmd_type = ITS_CMD_MAPD; | |||||
desc.cmd_desc_mapd.its_dev = its_dev; | |||||
desc.cmd_desc_mapd.valid = valid; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
static void | |||||
its_cmd_inv(device_t dev, struct its_dev *its_dev, | |||||
struct gicv3_its_irqsrc *girq) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct its_cmd_desc desc; | |||||
struct its_col *col; | |||||
sc = device_get_softc(dev); | |||||
col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; | |||||
desc.cmd_type = ITS_CMD_INV; | |||||
/* The EventID sent to the device */ | |||||
desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base; | |||||
desc.cmd_desc_inv.its_dev = its_dev; | |||||
desc.cmd_desc_inv.col = col; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
static void | |||||
its_cmd_invall(device_t dev, struct its_col *col) | |||||
{ | |||||
struct its_cmd_desc desc; | |||||
desc.cmd_type = ITS_CMD_INVALL; | |||||
desc.cmd_desc_invall.col = col; | |||||
its_cmd_send(dev, &desc); | |||||
} | |||||
#ifdef FDT | |||||
static device_probe_t gicv3_its_fdt_probe; | |||||
static device_attach_t gicv3_its_fdt_attach; | |||||
static device_method_t gicv3_its_fdt_methods[] = { | |||||
/* Device interface */ | |||||
DEVMETHOD(device_probe, gicv3_its_fdt_probe), | |||||
DEVMETHOD(device_attach, gicv3_its_fdt_attach), | |||||
/* End */ | |||||
DEVMETHOD_END | |||||
}; | |||||
#define its_baseclasses its_fdt_baseclasses | |||||
DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, | |||||
sizeof(struct gicv3_its_softc), gicv3_its_driver); | |||||
#undef its_baseclasses | |||||
static devclass_t gicv3_its_fdt_devclass; | |||||
EARLY_DRIVER_MODULE(its, gic, gicv3_its_fdt_driver, | |||||
gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); | |||||
static int | |||||
gicv3_its_fdt_probe(device_t dev) | |||||
{ | |||||
if (!ofw_bus_status_okay(dev)) | |||||
return (ENXIO); | |||||
if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) | |||||
return (ENXIO); | |||||
device_set_desc(dev, "ARM GIC Interrupt Translation Service"); | |||||
return (BUS_PROBE_DEFAULT); | |||||
} | |||||
static int | |||||
gicv3_its_fdt_attach(device_t dev) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
phandle_t xref; | |||||
int err; | |||||
err = gicv3_its_attach(dev); | |||||
if (err != 0) | |||||
return (err); | |||||
sc = device_get_softc(dev); | |||||
/* Register this device as a interrupt controller */ | |||||
xref = OF_xref_from_node(ofw_bus_get_node(dev)); | |||||
sc->sc_pic = intr_pic_register(dev, xref); | |||||
intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, | |||||
gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS); | |||||
/* Register this device to handle MSI interrupts */ | |||||
intr_msi_register(dev, xref); | |||||
return (0); | |||||
} | |||||
#endif |