Changeset View
Changeset View
Standalone View
Standalone View
head/sys/arm64/arm64/gicv3_its.c
Show All 26 Lines | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | ||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | ||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | ||||
* SUCH DAMAGE. | * SUCH DAMAGE. | ||||
*/ | */ | ||||
#include "opt_acpi.h" | #include "opt_acpi.h" | ||||
#include "opt_platform.h" | #include "opt_platform.h" | ||||
#include "opt_iommu.h" | |||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/cpuset.h> | #include <sys/cpuset.h> | ||||
#include <sys/endian.h> | #include <sys/endian.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/module.h> | #include <sys/module.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/taskqueue.h> | |||||
#include <sys/tree.h> | |||||
#include <sys/queue.h> | #include <sys/queue.h> | ||||
#include <sys/rman.h> | #include <sys/rman.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/vmem.h> | #include <sys/vmem.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_page.h> | |||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/intr.h> | #include <machine/intr.h> | ||||
#include <arm/arm/gic_common.h> | #include <arm/arm/gic_common.h> | ||||
#include <arm64/arm64/gic_v3_reg.h> | #include <arm64/arm64/gic_v3_reg.h> | ||||
#include <arm64/arm64/gic_v3_var.h> | #include <arm64/arm64/gic_v3_var.h> | ||||
#ifdef FDT | #ifdef FDT | ||||
#include <dev/ofw/openfirm.h> | #include <dev/ofw/openfirm.h> | ||||
#include <dev/ofw/ofw_bus.h> | #include <dev/ofw/ofw_bus.h> | ||||
#include <dev/ofw/ofw_bus_subr.h> | #include <dev/ofw/ofw_bus_subr.h> | ||||
#endif | #endif | ||||
#include <dev/pci/pcireg.h> | #include <dev/pci/pcireg.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#ifdef IOMMU | |||||
#include <dev/iommu/iommu.h> | |||||
#include <dev/iommu/iommu_gas.h> | |||||
#endif | |||||
#include "pcib_if.h" | #include "pcib_if.h" | ||||
#include "pic_if.h" | #include "pic_if.h" | ||||
#include "msi_if.h" | #include "msi_if.h" | ||||
MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", | MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", | ||||
"ARM GICv3 Interrupt Translation Service"); | "ARM GICv3 Interrupt Translation Service"); | ||||
#define LPI_NIRQS (64 * 1024) | #define LPI_NIRQS (64 * 1024) | ||||
▲ Show 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | struct gicv3_its_softc { | ||||
TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; | TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; | ||||
TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; | TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; | ||||
#define ITS_FLAGS_CMDQ_FLUSH 0x00000001 | #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 | ||||
#define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 | #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 | ||||
#define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 | #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 | ||||
u_int sc_its_flags; | u_int sc_its_flags; | ||||
bool trace_enable; | bool trace_enable; | ||||
vm_page_t ma; /* fake msi page */ | |||||
}; | }; | ||||
static void *conf_base; | static void *conf_base; | ||||
typedef void (its_quirk_func_t)(device_t); | typedef void (its_quirk_func_t)(device_t); | ||||
static its_quirk_func_t its_quirk_cavium_22375; | static its_quirk_func_t its_quirk_cavium_22375; | ||||
static const struct { | static const struct { | ||||
Show All 36 Lines | |||||
#ifdef SMP | #ifdef SMP | ||||
static pic_init_secondary_t gicv3_its_init_secondary; | static pic_init_secondary_t gicv3_its_init_secondary; | ||||
#endif | #endif | ||||
static msi_alloc_msi_t gicv3_its_alloc_msi; | static msi_alloc_msi_t gicv3_its_alloc_msi; | ||||
static msi_release_msi_t gicv3_its_release_msi; | static msi_release_msi_t gicv3_its_release_msi; | ||||
static msi_alloc_msix_t gicv3_its_alloc_msix; | static msi_alloc_msix_t gicv3_its_alloc_msix; | ||||
static msi_release_msix_t gicv3_its_release_msix; | static msi_release_msix_t gicv3_its_release_msix; | ||||
static msi_map_msi_t gicv3_its_map_msi; | static msi_map_msi_t gicv3_its_map_msi; | ||||
#ifdef IOMMU | |||||
static msi_iommu_init_t gicv3_iommu_init; | |||||
static msi_iommu_deinit_t gicv3_iommu_deinit; | |||||
#endif | |||||
static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); | static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); | ||||
static void its_cmd_mapc(device_t, struct its_col *, uint8_t); | static void its_cmd_mapc(device_t, struct its_col *, uint8_t); | ||||
static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); | static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); | ||||
static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); | static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); | ||||
static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); | static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); | ||||
static void its_cmd_invall(device_t, struct its_col *); | static void its_cmd_invall(device_t, struct its_col *); | ||||
Show All 15 Lines | |||||
#endif | #endif | ||||
/* MSI/MSI-X */ | /* MSI/MSI-X */ | ||||
DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), | DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), | ||||
DEVMETHOD(msi_release_msi, gicv3_its_release_msi), | DEVMETHOD(msi_release_msi, gicv3_its_release_msi), | ||||
DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), | DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), | ||||
DEVMETHOD(msi_release_msix, gicv3_its_release_msix), | DEVMETHOD(msi_release_msix, gicv3_its_release_msix), | ||||
DEVMETHOD(msi_map_msi, gicv3_its_map_msi), | DEVMETHOD(msi_map_msi, gicv3_its_map_msi), | ||||
#ifdef IOMMU | |||||
DEVMETHOD(msi_iommu_init, gicv3_iommu_init), | |||||
DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit), | |||||
#endif | |||||
/* End */ | /* End */ | ||||
DEVMETHOD_END | DEVMETHOD_END | ||||
}; | }; | ||||
static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, | static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, | ||||
sizeof(struct gicv3_its_softc)); | sizeof(struct gicv3_its_softc)); | ||||
▲ Show 20 Lines • Show All 435 Lines • ▼ Show 20 Lines | gicv3_its_init_sysctl(struct gicv3_its_softc *sc) | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
gicv3_its_attach(device_t dev) | gicv3_its_attach(device_t dev) | ||||
{ | { | ||||
struct gicv3_its_softc *sc; | struct gicv3_its_softc *sc; | ||||
uint32_t iidr; | |||||
int domain, err, i, rid; | int domain, err, i, rid; | ||||
uint64_t phys; | |||||
uint32_t iidr; | |||||
sc = device_get_softc(dev); | sc = device_get_softc(dev); | ||||
sc->sc_irq_length = gicv3_get_nirqs(dev); | sc->sc_irq_length = gicv3_get_nirqs(dev); | ||||
sc->sc_irq_base = GIC_FIRST_LPI; | sc->sc_irq_base = GIC_FIRST_LPI; | ||||
sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; | sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; | ||||
rid = 0; | rid = 0; | ||||
sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, | sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, | ||||
RF_ACTIVE); | RF_ACTIVE); | ||||
if (sc->sc_its_res == NULL) { | if (sc->sc_its_res == NULL) { | ||||
device_printf(dev, "Could not allocate memory\n"); | device_printf(dev, "Could not allocate memory\n"); | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) + | |||||
GITS_TRANSLATER, PAGE_SIZE); | |||||
sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO); | |||||
vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT); | |||||
iidr = gic_its_read_4(sc, GITS_IIDR); | iidr = gic_its_read_4(sc, GITS_IIDR); | ||||
for (i = 0; i < nitems(its_quirks); i++) { | for (i = 0; i < nitems(its_quirks); i++) { | ||||
if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { | if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { | ||||
if (bootverbose) { | if (bootverbose) { | ||||
device_printf(dev, "Applying %s\n", | device_printf(dev, "Applying %s\n", | ||||
its_quirks[i].desc); | its_quirks[i].desc); | ||||
} | } | ||||
its_quirks[i].func(dev); | its_quirks[i].func(dev); | ||||
▲ Show 20 Lines • Show All 582 Lines • ▼ Show 20 Lines | gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, | ||||
sc = device_get_softc(dev); | sc = device_get_softc(dev); | ||||
girq = (struct gicv3_its_irqsrc *)isrc; | girq = (struct gicv3_its_irqsrc *)isrc; | ||||
*addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; | *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; | ||||
*data = girq->gi_id; | *data = girq->gi_id; | ||||
return (0); | return (0); | ||||
} | } | ||||
#ifdef IOMMU | |||||
static int | |||||
gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain) | |||||
{ | |||||
struct gicv3_its_softc *sc; | |||||
struct iommu_ctx *ctx; | |||||
int error; | |||||
sc = device_get_softc(dev); | |||||
ctx = iommu_get_dev_ctx(child); | |||||
error = iommu_map_msi(ctx, PAGE_SIZE, GITS_TRANSLATER, | |||||
IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma); | |||||
*domain = iommu_get_ctx_domain(ctx); | |||||
return (error); | |||||
} | |||||
static void | |||||
gicv3_iommu_deinit(device_t dev, device_t child) | |||||
{ | |||||
struct iommu_ctx *ctx; | |||||
ctx = iommu_get_dev_ctx(child); | |||||
iommu_unmap_msi(ctx); | |||||
} | |||||
#endif | |||||
/* | /* | ||||
* Commands handling. | * Commands handling. | ||||
*/ | */ | ||||
static __inline void | static __inline void | ||||
cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) | cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 532 Lines • Show Last 20 Lines |