Index: head/sys/arm64/arm64/gic_v3.c =================================================================== --- head/sys/arm64/arm64/gic_v3.c +++ head/sys/arm64/arm64/gic_v3.c @@ -58,6 +58,7 @@ #include "gic_v3_var.h" /* Device and PIC methods */ +static int gic_v3_bind(device_t, u_int, u_int); static void gic_v3_dispatch(device_t, struct trapframe *); static void gic_v3_eoi(device_t, u_int); static void gic_v3_mask_irq(device_t, u_int); @@ -72,6 +73,7 @@ DEVMETHOD(device_detach, gic_v3_detach), /* PIC interface */ + DEVMETHOD(pic_bind, gic_v3_bind), DEVMETHOD(pic_dispatch, gic_v3_dispatch), DEVMETHOD(pic_eoi, gic_v3_eoi), DEVMETHOD(pic_mask, gic_v3_mask_irq), @@ -244,6 +246,28 @@ /* * PIC interface. */ + +static int +gic_v3_bind(device_t dev, u_int irq, u_int cpuid) +{ + uint64_t aff; + struct gic_v3_softc *sc; + + sc = device_get_softc(dev); + + if (irq <= GIC_LAST_PPI) { + /* Can't bind PPI to another CPU but it's not an error */ + return (0); + } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { + aff = CPU_AFFINITY(cpuid); + gic_d_write(sc, 4, GICD_IROUTER(irq), aff); + return (0); + } else if (irq >= GIC_FIRST_LPI) + return (lpi_migrate(dev, irq, cpuid)); + + return (EINVAL); +} + static void gic_v3_dispatch(device_t dev, struct trapframe *frame) { Index: head/sys/arm64/arm64/gic_v3_its.c =================================================================== --- head/sys/arm64/arm64/gic_v3_its.c +++ head/sys/arm64/arm64/gic_v3_its.c @@ -92,9 +92,13 @@ static void its_init_commandq(struct gic_v3_its_softc *); static void its_init_cpu_collection(struct gic_v3_its_softc *); static uint32_t its_get_devid(device_t); +static struct its_dev * its_device_find_locked(struct gic_v3_its_softc *, + device_t, uint32_t); static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); +static void its_cmd_movi(struct gic_v3_its_softc *, struct its_dev *, + struct its_col *, uint32_t); static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, uint32_t); @@ -846,18 +850,28 @@ lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic, u_int nvecs) { + u_int *col_ids; int fclr; /* First cleared bit */ uint8_t *bitmap; size_t nb, i; + col_ids = malloc(sizeof(*col_ids) * nvecs, M_GIC_V3_ITS, + (M_NOWAIT | M_ZERO)); + if (col_ids == NULL) + return (ENOMEM); + + mtx_lock_spin(&sc->its_dev_lock); bitmap = (uint8_t *)sc->its_lpi_bitmap; fclr = 0; retry: /* Check other bits - sloooow */ for (i = 0, nb = fclr; i < nvecs; i++, nb++) { - if (nb > sc->its_lpi_maxid) + if (nb > sc->its_lpi_maxid) { + mtx_unlock_spin(&sc->its_dev_lock); + free(col_ids, M_GIC_V3_ITS); return (EINVAL); + } if (isset(bitmap, nb)) { /* To little free bits in this area. Move on. */ @@ -870,6 +884,15 @@ lpic->lpi_base = fclr + GIC_FIRST_LPI; lpic->lpi_num = nvecs; lpic->lpi_free = lpic->lpi_num; + lpic->lpi_col_ids = col_ids; + for (i = 0; i < lpic->lpi_num; i++) { + /* + * Initially all interrupts go to CPU0 but can be moved + * to another CPU by bus_bind_intr() or interrupts shuffling. + */ + lpic->lpi_col_ids[i] = 0; + } + mtx_unlock_spin(&sc->its_dev_lock); return (0); } @@ -885,6 +908,7 @@ KASSERT((lpic->lpi_free == lpic->lpi_num), ("Trying to free LPI chunk that is still in use.\n")); + mtx_lock_spin(&sc->its_dev_lock); /* First bit of this chunk in a global bitmap */ start = lpic->lpi_base - GIC_FIRST_LPI; /* and last bit of this chunk... */ @@ -892,6 +916,10 @@ /* Finally free this chunk */ bit_nclear(bitmap, start, end); + mtx_unlock_spin(&sc->its_dev_lock); + + free(lpic->lpi_col_ids, M_GIC_V3_ITS); + lpic->lpi_col_ids = NULL; } static void @@ -953,6 +981,32 @@ (unmask == TRUE) ? "unmask" : "mask", irq); } +int +lpi_migrate(device_t parent, uint32_t irq, u_int cpuid) +{ + struct gic_v3_its_softc *sc; + struct its_dev *its_dev; + struct its_col *col; + + sc = its_sc; + mtx_lock_spin(&sc->its_dev_lock); + its_dev = its_device_find_locked(sc, NULL, irq); + mtx_unlock_spin(&sc->its_dev_lock); + if (its_dev == NULL) { + /* Cannot migrate not configured LPI */ + return (ENXIO); + } + + /* Find local device's interrupt identifier */ + irq = irq - its_dev->lpis.lpi_base; + /* Move interrupt to another collection */ + col = sc->its_cols[cpuid]; + its_cmd_movi(sc, its_dev, col, irq); + its_dev->lpis.lpi_col_ids[irq] = cpuid; + + return (0); +} + void lpi_unmask_irq(device_t parent, uint32_t irq) { @@ -1053,6 +1107,20 @@ } static void +its_cmd_movi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, + struct its_col *col, uint32_t id) +{ + struct its_cmd_desc desc; + + desc.cmd_type = ITS_CMD_MOVI; + desc.cmd_desc_movi.its_dev = its_dev; + desc.cmd_desc_movi.col = col; + desc.cmd_desc_movi.id = id; + + its_cmd_send(sc, &desc); +} + +static void its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid) { struct its_cmd_desc desc; @@ -1073,9 +1141,15 @@ uint32_t id, uint32_t pid) { struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + + col_id = its_dev->lpis.lpi_col_ids[id]; + col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_MAPVI; desc.cmd_desc_mapvi.its_dev = its_dev; + desc.cmd_desc_mapvi.col = col; desc.cmd_desc_mapvi.id = id; desc.cmd_desc_mapvi.pid = pid; @@ -1083,14 +1157,23 @@ } static void __unused -its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint32_t lpinum) +its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) { struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + uint32_t id; + + KASSERT(pid >= its_dev->lpis.lpi_base, + ("%s: invalid pid: %d for the ITS device", __func__, pid)); + id = pid - its_dev->lpis.lpi_base; + col_id = its_dev->lpis.lpi_col_ids[id]; + col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_MAPI; desc.cmd_desc_mapi.its_dev = its_dev; - desc.cmd_desc_mapi.lpinum = lpinum; + desc.cmd_desc_mapi.col = col; + desc.cmd_desc_mapi.pid = pid; its_cmd_send(sc, &desc); } @@ -1109,14 +1192,23 @@ } static void -its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint32_t lpinum) +its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) { struct its_cmd_desc desc; + struct its_col *col; + u_int col_id; + uint32_t id; + + KASSERT(pid >= its_dev->lpis.lpi_base, + ("%s: invalid pid: %d for the ITS device", __func__, pid)); + id = pid - its_dev->lpis.lpi_base; + col_id = its_dev->lpis.lpi_col_ids[id]; + col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_INV; - desc.cmd_desc_inv.lpinum = lpinum - its_dev->lpis.lpi_base; + desc.cmd_desc_inv.pid = pid - its_dev->lpis.lpi_base; desc.cmd_desc_inv.its_dev = its_dev; + desc.cmd_desc_inv.col = col; its_cmd_send(sc, &desc); } @@ -1216,13 +1308,19 @@ target = ITS_TARGET_NONE; switch (cmd_type) { + case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ + target = desc->cmd_desc_movi.col->col_target; + cmd_format_command(cmd, ITS_CMD_MOVI); + cmd_format_id(cmd, desc->cmd_desc_movi.id); + cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); + cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); + break; case ITS_CMD_SYNC: /* Wait for previous commands completion */ target = desc->cmd_desc_sync.col->col_target; cmd_format_command(cmd, ITS_CMD_SYNC); cmd_format_target(cmd, target); break; case ITS_CMD_MAPD: /* Assign ITT to device */ - target = desc->cmd_desc_mapd.its_dev->col->col_target; cmd_format_command(cmd, ITS_CMD_MAPD); cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); /* @@ -1249,25 +1347,25 @@ cmd_format_target(cmd, target); break; case ITS_CMD_MAPVI: - target = desc->cmd_desc_mapvi.its_dev->col->col_target; + target = desc->cmd_desc_mapvi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPVI); cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_mapvi.id); cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); - cmd_format_col(cmd, desc->cmd_desc_mapvi.its_dev->col->col_id); + cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); break; case ITS_CMD_MAPI: - target = desc->cmd_desc_mapi.its_dev->col->col_target; + target = desc->cmd_desc_mapi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPI); cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); - cmd_format_id(cmd, desc->cmd_desc_mapi.lpinum); - cmd_format_col(cmd, desc->cmd_desc_mapi.its_dev->col->col_id); + cmd_format_id(cmd, desc->cmd_desc_mapi.pid); + cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); break; case ITS_CMD_INV: - target = desc->cmd_desc_inv.its_dev->col->col_target; + target = desc->cmd_desc_inv.col->col_target; cmd_format_command(cmd, ITS_CMD_INV); cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); - cmd_format_id(cmd, desc->cmd_desc_inv.lpinum); + cmd_format_id(cmd, desc->cmd_desc_inv.pid); break; case ITS_CMD_INVALL: cmd_format_command(cmd, ITS_CMD_INVALL); @@ -1367,16 +1465,28 @@ return (0); } +/* Find ITS device descriptor by pci_dev or irq number */ static struct its_dev * -its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev) +its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev, + uint32_t irq) { struct its_dev *its_dev; + struct lpi_chunk *lpis; mtx_assert(&sc->its_dev_lock, MA_OWNED); + KASSERT((pci_dev == NULL || irq == 0), + ("%s: Can't search by both pci_dev and irq number", __func__)); /* Find existing device if any */ TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) { - if (its_dev->pci_dev == pci_dev) - return (its_dev); + if (pci_dev != NULL) { + if (its_dev->pci_dev == pci_dev) + return (its_dev); + } else if (irq != 0) { + lpis = &its_dev->lpis; + if ((irq >= lpis->lpi_base) && + (irq < (lpis->lpi_base + lpis->lpi_num))) + return (its_dev); + } } return (NULL); @@ -1389,13 +1499,12 @@ struct its_dev *newdev; uint64_t typer; uint32_t devid; - u_int cpuid; size_t esize; int err; mtx_lock_spin(&sc->its_dev_lock); /* Find existing device if any */ - newdev = its_device_find_locked(sc, pci_dev); + newdev = its_device_find_locked(sc, pci_dev, 0); mtx_unlock_spin(&sc->its_dev_lock); if (newdev != NULL) return (newdev); @@ -1410,9 +1519,7 @@ newdev->pci_dev = pci_dev; newdev->devid = devid; - mtx_lock_spin(&sc->its_dev_lock); err = lpi_alloc_chunk(sc, &newdev->lpis, nvecs); - mtx_unlock_spin(&sc->its_dev_lock); if (err != 0) { free(newdev, M_GIC_V3_ITS); return (NULL); @@ -1429,20 +1536,11 @@ roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS, (M_NOWAIT | M_ZERO), 0, ~0UL, 0x100, 0); if (newdev->itt == 0) { - mtx_lock_spin(&sc->its_dev_lock); lpi_free_chunk(sc, &newdev->lpis); - mtx_unlock_spin(&sc->its_dev_lock); free(newdev, M_GIC_V3_ITS); return (NULL); } - /* - * Initially all interrupts go to CPU0 but can be moved - * to another CPU by bus_bind_intr() or interrupts shuffling. - */ - cpuid = 0; - newdev->col = sc->its_cols[cpuid]; - mtx_lock_spin(&sc->its_dev_lock); TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); mtx_unlock_spin(&sc->its_dev_lock); Index: head/sys/arm64/arm64/gic_v3_var.h =================================================================== --- head/sys/arm64/arm64/gic_v3_var.h +++ head/sys/arm64/arm64/gic_v3_var.h @@ -96,6 +96,7 @@ u_int lpi_base; u_int lpi_num; u_int lpi_free; /* First free LPI in set */ + u_int *lpi_col_ids; }; /* ITS device */ @@ -109,8 +110,6 @@ struct lpi_chunk lpis; /* Virtual address of ITT */ vm_offset_t itt; - /* Interrupt collection */ - struct its_col * col; }; TAILQ_HEAD(its_dev_list, its_dev); @@ -133,6 +132,7 @@ }; /* ITS commands encoding */ +#define ITS_CMD_MOVI (0x01) #define ITS_CMD_SYNC (0x05) #define ITS_CMD_MAPD (0x08) #define ITS_CMD_MAPC (0x09) @@ -172,6 +172,12 @@ union { struct { + struct its_dev *its_dev; + struct its_col *col; + uint32_t id; + } cmd_desc_movi; + + struct { struct its_col *col; } cmd_desc_sync; @@ -182,13 +188,15 @@ struct { struct its_dev *its_dev; + struct its_col *col; uint32_t pid; uint32_t id; } cmd_desc_mapvi; struct { struct its_dev *its_dev; - uint32_t lpinum; + struct its_col *col; + uint32_t pid; } cmd_desc_mapi; struct { @@ -198,7 +206,8 @@ struct { struct its_dev *its_dev; - uint32_t lpinum; + struct its_col *col; + uint32_t pid; } cmd_desc_inv; struct { @@ -257,6 +266,7 @@ int its_init_cpu(struct gic_v3_its_softc *); +int lpi_migrate(device_t, uint32_t, u_int); void lpi_unmask_irq(device_t, uint32_t); void lpi_mask_irq(device_t, uint32_t); /*