Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm64/arm64/gic_v3_its.c
Show All 37 Lines | |||||
#include <sys/endian.h> | #include <sys/endian.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/rman.h> | #include <sys/rman.h> | ||||
#include <sys/pciio.h> | #include <sys/pciio.h> | ||||
#include <sys/pcpu.h> | #include <sys/pcpu.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/smp.h> | |||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
Show All 29 Lines | |||||
DEFINE_CLASS_0(gic_v3_its, gic_v3_its_driver, gic_v3_its_methods, | DEFINE_CLASS_0(gic_v3_its, gic_v3_its_driver, gic_v3_its_methods, | ||||
sizeof(struct gic_v3_its_softc)); | sizeof(struct gic_v3_its_softc)); | ||||
MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR); | MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR); | ||||
static int its_alloc_tables(struct gic_v3_its_softc *); | static int its_alloc_tables(struct gic_v3_its_softc *); | ||||
static void its_free_tables(struct gic_v3_its_softc *); | static void its_free_tables(struct gic_v3_its_softc *); | ||||
static void its_init_commandq(struct gic_v3_its_softc *); | static void its_init_commandq(struct gic_v3_its_softc *); | ||||
static int its_init_cpu(struct gic_v3_its_softc *); | |||||
static void its_init_cpu_collection(struct gic_v3_its_softc *); | static void its_init_cpu_collection(struct gic_v3_its_softc *); | ||||
static uint32_t its_get_devid(device_t); | static uint32_t its_get_devid(device_t); | ||||
static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); | static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); | ||||
static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); | static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); | ||||
static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, | static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, | ||||
uint32_t); | uint32_t); | ||||
static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t); | static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t); | ||||
static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t); | static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t); | ||||
static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *); | static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *); | ||||
static uint32_t its_get_devbits(device_t); | static uint32_t its_get_devbits(device_t); | ||||
static void lpi_init_conftable(struct gic_v3_its_softc *); | static void lpi_init_conftable(struct gic_v3_its_softc *); | ||||
static void lpi_bitmap_init(struct gic_v3_its_softc *); | static void lpi_bitmap_init(struct gic_v3_its_softc *); | ||||
static void lpi_init_cpu(struct gic_v3_its_softc *); | |||||
static int lpi_config_cpu(struct gic_v3_its_softc *); | static int lpi_config_cpu(struct gic_v3_its_softc *); | ||||
static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *); | |||||
const char *its_ptab_cache[] = { | const char *its_ptab_cache[] = { | ||||
[GITS_BASER_CACHE_NCNB] = "(NC,NB)", | [GITS_BASER_CACHE_NCNB] = "(NC,NB)", | ||||
[GITS_BASER_CACHE_NC] = "(NC)", | [GITS_BASER_CACHE_NC] = "(NC)", | ||||
[GITS_BASER_CACHE_RAWT] = "(RA,WT)", | [GITS_BASER_CACHE_RAWT] = "(RA,WT)", | ||||
[GITS_BASER_CACHE_RAWB] = "(RA,WB)", | [GITS_BASER_CACHE_RAWB] = "(RA,WB)", | ||||
[GITS_BASER_CACHE_WAWT] = "(WA,WT)", | [GITS_BASER_CACHE_WAWT] = "(WA,WT)", | ||||
[GITS_BASER_CACHE_WAWB] = "(WA,WB)", | [GITS_BASER_CACHE_WAWB] = "(WA,WB)", | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | gic_v3_its_attach(device_t dev) | ||||
/* 2. Provide memory for any private ITS tables */ | /* 2. Provide memory for any private ITS tables */ | ||||
ret = its_alloc_tables(sc); | ret = its_alloc_tables(sc); | ||||
if (ret != 0) { | if (ret != 0) { | ||||
gic_v3_its_detach(dev); | gic_v3_its_detach(dev); | ||||
return (ret); | return (ret); | ||||
} | } | ||||
/* 3. Allocate collections. One per-CPU */ | /* 3. Allocate collections. One per-CPU */ | ||||
sc->its_cols = malloc(sizeof(*sc->its_cols) * MAXCPU, | for (int cpu = 0; cpu < mp_ncpus; cpu++) | ||||
if (CPU_ISSET(cpu, &all_cpus) != 0) | |||||
sc->its_cols[cpu] = malloc(sizeof(*sc->its_cols[0]), | |||||
M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); | M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); | ||||
else | |||||
sc->its_cols[cpu] = NULL; | |||||
/* 4. Enable ITS in GITS_CTLR */ | /* 4. Enable ITS in GITS_CTLR */ | ||||
gits_tmp = gic_its_read(sc, 4, GITS_CTLR); | gits_tmp = gic_its_read(sc, 4, GITS_CTLR); | ||||
gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN); | gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN); | ||||
/* 5. Initialize LPIs configuration table */ | /* 5. Initialize LPIs configuration table */ | ||||
lpi_init_conftable(sc); | lpi_init_conftable(sc); | ||||
/* 6. LPIs bitmap init */ | /* 6. LPIs bitmap init */ | ||||
lpi_bitmap_init(sc); | lpi_bitmap_init(sc); | ||||
/* 7. CPU init */ | /* 7. Allocate pending tables for all CPUs */ | ||||
lpi_alloc_cpu_pendtables(sc); | |||||
/* 8. CPU init */ | |||||
(void)its_init_cpu(sc); | (void)its_init_cpu(sc); | ||||
/* 8. Init ITS devices list */ | /* 9. Init ITS devices list */ | ||||
TAILQ_INIT(&sc->its_dev_list); | TAILQ_INIT(&sc->its_dev_list); | ||||
arm_register_msi_pic(dev); | arm_register_msi_pic(dev); | ||||
/* | /* | ||||
* XXX ARM64TODO: We need to have ITS software context | * XXX ARM64TODO: We need to have ITS software context | ||||
* when being called by the interrupt code (mask/unmask). | * when being called by the interrupt code (mask/unmask). | ||||
* This may be used only when one ITS is present in | * This may be used only when one ITS is present in | ||||
Show All 24 Lines | gic_v3_its_detach(device_t dev) | ||||
/* Command queue */ | /* Command queue */ | ||||
if ((void *)sc->its_cmdq_base != NULL) { | if ((void *)sc->its_cmdq_base != NULL) { | ||||
contigfree((void *)sc->its_cmdq_base, | contigfree((void *)sc->its_cmdq_base, | ||||
ITS_CMDQ_SIZE, M_GIC_V3_ITS); | ITS_CMDQ_SIZE, M_GIC_V3_ITS); | ||||
} | } | ||||
/* ITTs */ | /* ITTs */ | ||||
its_free_tables(sc); | its_free_tables(sc); | ||||
/* Collections */ | /* Collections */ | ||||
free(sc->its_cols, M_GIC_V3_ITS); | for (cpuid = 0; cpuid < mp_ncpus; cpuid++) | ||||
free(sc->its_cols[cpuid], M_GIC_V3_ITS); | |||||
/* LPI config table */ | /* LPI config table */ | ||||
parent = device_get_parent(sc->dev); | parent = device_get_parent(sc->dev); | ||||
gic_sc = device_get_softc(parent); | gic_sc = device_get_softc(parent); | ||||
if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) { | if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) { | ||||
contigfree((void *)gic_sc->gic_redists.lpis.conf_base, | contigfree((void *)gic_sc->gic_redists.lpis.conf_base, | ||||
LPI_CONFTAB_SIZE, M_GIC_V3_ITS); | LPI_CONFTAB_SIZE, M_GIC_V3_ITS); | ||||
} | } | ||||
for (cpuid = 0; cpuid < mp_ncpus; cpuid++) | |||||
if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { | if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { | ||||
contigfree((void *)gic_sc->gic_redists.lpis.pend_base[cpuid], | contigfree( | ||||
roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS); | (void *)gic_sc->gic_redists.lpis.pend_base[cpuid], | ||||
roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), | |||||
M_GIC_V3_ITS); | |||||
} | } | ||||
/* Resource... */ | /* Resource... */ | ||||
bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); | bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); | ||||
/* XXX ARM64TODO: Reset global pointer to ITS software context */ | /* XXX ARM64TODO: Reset global pointer to ITS software context */ | ||||
its_sc = NULL; | its_sc = NULL; | ||||
return (0); | return (0); | ||||
▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | if (((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) != 0) { | ||||
} | } | ||||
/* Command queue needs cache flushing */ | /* Command queue needs cache flushing */ | ||||
sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH; | sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH; | ||||
} | } | ||||
gic_its_write(sc, 8, GITS_CWRITER, 0x0); | gic_its_write(sc, 8, GITS_CWRITER, 0x0); | ||||
} | } | ||||
static int | int | ||||
its_init_cpu(struct gic_v3_its_softc *sc) | its_init_cpu(struct gic_v3_its_softc *sc) | ||||
{ | { | ||||
device_t parent; | device_t parent; | ||||
struct gic_v3_softc *gic_sc; | struct gic_v3_softc *gic_sc; | ||||
/* | /* | ||||
* NULL in place of the softc pointer means that | |||||
* this function was called during GICv3 secondary initialization. | |||||
*/ | |||||
if (sc == NULL) { | |||||
if (device_is_attached(its_sc->dev)) { | |||||
/* | |||||
* XXX ARM64TODO: This is part of the workaround that | |||||
* saves ITS software context for further use in | |||||
* mask/unmask and here. This should be removed as soon | |||||
* as the upper layer is capable of passing the ITS | |||||
* context to this function. | |||||
*/ | |||||
sc = its_sc; | |||||
} else | |||||
return (ENXIO); | |||||
} | |||||
/* | |||||
* Check for LPIs support on this Re-Distributor. | * Check for LPIs support on this Re-Distributor. | ||||
*/ | */ | ||||
parent = device_get_parent(sc->dev); | parent = device_get_parent(sc->dev); | ||||
gic_sc = device_get_softc(parent); | gic_sc = device_get_softc(parent); | ||||
if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { | if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { | ||||
if (bootverbose) { | if (bootverbose) { | ||||
device_printf(sc->dev, | device_printf(sc->dev, | ||||
"LPIs not supported on CPU%u\n", PCPU_GET(cpuid)); | "LPIs not supported on CPU%u\n", PCPU_GET(cpuid)); | ||||
} | } | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
/* Initialize LPIs for this CPU */ | /* Configure LPIs for this CPU */ | ||||
lpi_init_cpu(sc); | lpi_config_cpu(sc); | ||||
/* Initialize collections */ | /* Initialize collections */ | ||||
its_init_cpu_collection(sc); | its_init_cpu_collection(sc); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
Show All 20 Lines | if ((typer & GITS_TYPER_PTA) != 0) { | ||||
*/ | */ | ||||
target = vtophys(redist_base); | target = vtophys(redist_base); | ||||
} else { | } else { | ||||
/* Target Address correspond to unique processor numbers */ | /* Target Address correspond to unique processor numbers */ | ||||
typer = gic_r_read(gic_sc, 8, GICR_TYPER); | typer = gic_r_read(gic_sc, 8, GICR_TYPER); | ||||
target = GICR_TYPER_CPUNUM(typer); | target = GICR_TYPER_CPUNUM(typer); | ||||
} | } | ||||
sc->its_cols[cpuid].col_target = target; | sc->its_cols[cpuid]->col_target = target; | ||||
sc->its_cols[cpuid].col_id = cpuid; | sc->its_cols[cpuid]->col_id = cpuid; | ||||
its_cmd_mapc(sc, &sc->its_cols[cpuid], 1); | its_cmd_mapc(sc, sc->its_cols[cpuid], 1); | ||||
its_cmd_invall(sc, &sc->its_cols[cpuid]); | its_cmd_invall(sc, sc->its_cols[cpuid]); | ||||
} | } | ||||
static void | static void | ||||
lpi_init_conftable(struct gic_v3_its_softc *sc) | lpi_init_conftable(struct gic_v3_its_softc *sc) | ||||
{ | { | ||||
device_t parent; | device_t parent; | ||||
struct gic_v3_softc *gic_sc; | struct gic_v3_softc *gic_sc; | ||||
vm_offset_t conf_base; | vm_offset_t conf_base; | ||||
Show All 30 Lines | lpi_init_conftable(struct gic_v3_its_softc *sc) | ||||
cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE, | cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE, | ||||
PAGE_SIZE_64K)); | PAGE_SIZE_64K)); | ||||
gic_sc->gic_redists.lpis.conf_base = conf_base; | gic_sc->gic_redists.lpis.conf_base = conf_base; | ||||
} | } | ||||
static void | static void | ||||
lpi_init_cpu(struct gic_v3_its_softc *sc) | lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *sc) | ||||
{ | { | ||||
device_t parent; | device_t parent; | ||||
struct gic_v3_softc *gic_sc; | struct gic_v3_softc *gic_sc; | ||||
vm_offset_t pend_base; | vm_offset_t pend_base; | ||||
u_int cpuid; | u_int cpuid; | ||||
parent = device_get_parent(sc->dev); | parent = device_get_parent(sc->dev); | ||||
gic_sc = device_get_softc(parent); | gic_sc = device_get_softc(parent); | ||||
/* | /* | ||||
* LPI Pending Table settings. | * LPI Pending Table settings. | ||||
* This has to be done for each Re-Distributor, hence for each CPU. | * This has to be done for each Re-Distributor, hence for each CPU. | ||||
*/ | */ | ||||
cpuid = PCPU_GET(cpuid); | for (cpuid = 0; cpuid < mp_ncpus; cpuid++) { | ||||
/* Limit allocation to active CPUs only */ | |||||
if (CPU_ISSET(cpuid, &all_cpus) == 0) | |||||
continue; | |||||
pend_base = (vm_offset_t)contigmalloc( | pend_base = (vm_offset_t)contigmalloc( | ||||
roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, | roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, | ||||
(M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); | (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); | ||||
/* Clean D-cache so that ITS can see zeroed pages */ | /* Clean D-cache so that ITS can see zeroed pages */ | ||||
cpu_dcache_wb_range((vm_offset_t)pend_base, | cpu_dcache_wb_range((vm_offset_t)pend_base, | ||||
roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); | roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); | ||||
if (bootverbose) { | if (bootverbose) { | ||||
device_printf(sc->dev, | device_printf(sc->dev, | ||||
"LPI Pending Table for CPU%u at PA: 0x%lx\n", | "LPI Pending Table for CPU%u at PA: 0x%lx\n", | ||||
cpuid, vtophys(pend_base)); | cpuid, vtophys(pend_base)); | ||||
} | } | ||||
gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; | gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; | ||||
} | |||||
lpi_config_cpu(sc); | /* Ensure visibility of pend_base addresses on other CPUs */ | ||||
wmb(); | |||||
} | } | ||||
static int | static int | ||||
lpi_config_cpu(struct gic_v3_its_softc *sc) | lpi_config_cpu(struct gic_v3_its_softc *sc) | ||||
{ | { | ||||
device_t parent; | device_t parent; | ||||
struct gic_v3_softc *gic_sc; | struct gic_v3_softc *gic_sc; | ||||
vm_offset_t conf_base, pend_base; | vm_offset_t conf_base, pend_base; | ||||
uint64_t gicr_xbaser, gicr_temp; | uint64_t gicr_xbaser, gicr_temp; | ||||
uint64_t cache, share, idbits; | uint64_t cache, share, idbits; | ||||
uint32_t gicr_ctlr; | uint32_t gicr_ctlr; | ||||
u_int cpuid; | u_int cpuid; | ||||
parent = device_get_parent(sc->dev); | parent = device_get_parent(sc->dev); | ||||
gic_sc = device_get_softc(parent); | gic_sc = device_get_softc(parent); | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
/* Ensure data observability on a current CPU */ | |||||
rmb(); | |||||
conf_base = gic_sc->gic_redists.lpis.conf_base; | conf_base = gic_sc->gic_redists.lpis.conf_base; | ||||
pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; | pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; | ||||
/* Disable LPIs */ | /* Disable LPIs */ | ||||
gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); | gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); | ||||
gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE; | gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE; | ||||
gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); | gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); | ||||
/* Perform full system barrier */ | /* Perform full system barrier */ | ||||
▲ Show 20 Lines • Show All 680 Lines • ▼ Show 20 Lines | if (newdev->itt == 0) { | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* XXX ARM64TODO: Currently all interrupts are going | * XXX ARM64TODO: Currently all interrupts are going | ||||
* to be bound to the CPU that performs the configuration. | * to be bound to the CPU that performs the configuration. | ||||
*/ | */ | ||||
cpuid = PCPU_GET(cpuid); | cpuid = PCPU_GET(cpuid); | ||||
newdev->col = &sc->its_cols[cpuid]; | newdev->col = sc->its_cols[cpuid]; | ||||
TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); | TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); | ||||
/* Map device to its ITT */ | /* Map device to its ITT */ | ||||
its_cmd_mapd(sc, newdev, 1); | its_cmd_mapd(sc, newdev, 1); | ||||
return (newdev); | return (newdev); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines |