diff --git a/sys/compat/linuxkpi/common/src/linux_pci.c b/sys/compat/linuxkpi/common/src/linux_pci.c --- a/sys/compat/linuxkpi/common/src/linux_pci.c +++ b/sys/compat/linuxkpi/common/src/linux_pci.c @@ -33,6 +33,7 @@ * (1) PCI * (2) DMA * (3) Backlight + * (4) Native bus attachment glue. */ #include @@ -97,27 +98,12 @@ /* Undef the linux function macro defined in linux/pci.h */ #undef pci_get_class -extern int linuxkpi_debug; +static void lkpi_pcim_iomap_table_release(struct device *, void *); +extern int linuxkpi_debug; SYSCTL_DECL(_compat_linuxkpi); -static counter_u64_t lkpi_pci_nseg1_fail; -SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, - &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); - -static device_probe_t linux_pci_probe; -static device_attach_t linux_pci_attach; -static device_detach_t linux_pci_detach; -static device_suspend_t linux_pci_suspend; -static device_resume_t linux_pci_resume; -static device_shutdown_t linux_pci_shutdown; -static pci_iov_init_t linux_pci_iov_init; -static pci_iov_uninit_t linux_pci_iov_uninit; -static pci_iov_add_vf_t linux_pci_iov_add_vf; -static int linux_backlight_get_status(device_t dev, struct backlight_props *props); -static int linux_backlight_update_status(device_t dev, struct backlight_props *props); -static int linux_backlight_get_info(device_t dev, struct backlight_info *info); -static void lkpi_pcim_iomap_table_release(struct device *, void *); +static device_method_t pci_methods[]; static struct list_head lkpi_pci_drivers; static struct list_head lkpi_pci_devices; @@ -128,27 +114,6 @@ #define LKPI_PCI_LOCK() spin_lock(&lkpi_pci_lock) #define LKPI_PCI_UNLOCK() spin_unlock(&lkpi_pci_lock) -static device_method_t pci_methods[] = { - DEVMETHOD(device_probe, linux_pci_probe), - DEVMETHOD(device_attach, linux_pci_attach), - DEVMETHOD(device_detach, linux_pci_detach), - DEVMETHOD(device_suspend, linux_pci_suspend), - DEVMETHOD(device_resume, linux_pci_resume), - DEVMETHOD(device_shutdown, linux_pci_shutdown), - DEVMETHOD(pci_iov_init, linux_pci_iov_init), - DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit), - DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf), - - /* Bus interface. */ - DEVMETHOD(bus_add_child, bus_generic_add_child), - - /* backlight interface */ - DEVMETHOD(backlight_update_status, linux_backlight_update_status), - DEVMETHOD(backlight_get_status, linux_backlight_get_status), - DEVMETHOD(backlight_get_info, linux_backlight_get_info), - DEVMETHOD_END -}; - const char *pci_power_names[] = { "UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold" }; @@ -176,21 +141,39 @@ #define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock) #define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock) -static void -lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, - void *res) +static struct resource_list_entry * +lkpi_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, + int type, int rid) { - dr->mmio_table[bar] = (void *)rman_get_bushandle(res); - dr->res_table[bar] = res; + device_t dev; + struct resource *res; + + KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, + ("trying to reserve non-BAR type %d", type)); + + dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? + device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; + res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, + 1, 1, 0); + if (res == NULL) + return (NULL); + return (resource_list_find(rl, type, rid)); } -static bool -lkpi_pci_bar_id_valid(int bar) +static struct resource_list_entry * +lkpi_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) { - if (bar < 0 || bar > PCIR_MAX_BAR_0) - return (false); + struct pci_devinfo *dinfo; + struct resource_list *rl; + struct resource_list_entry *rle; - return (true); + dinfo = device_get_ivars(pdev->dev.bsddev); + rl = &dinfo->resources; + rle = resource_list_find(rl, type, rid); + /* Reserve resources for this BAR if needed. */ + if (rle == NULL && reserve_bar) + rle = lkpi_pci_reserve_bar(pdev, rl, type, rid); + return (rle); } static int @@ -362,12 +345,12 @@ } /* - * Shared function releasing the bus resources which can be built independent - * of the way we allocate/initialization the pdev [pci_upstream_bridge(), - * pcie_find_root_port()]. + * Shared helper function releasing the bus resources which can be built + * independent of the way we allocate/initialization the pdev + * [pci_upstream_bridge(), pcie_find_root_port()]. */ static void -lkpi_pci_dev_bus_release(struct pci_dev *pdev) +lkpi_pci_dev_release_common_bus(struct pci_dev *pdev) { if (pdev->bus->self != pdev) { pci_dev_put(pdev->bus->self); @@ -380,9 +363,9 @@ } } -/* Release resources acquired by lkpifill_pci_dev(). */ +/* Release helper function to free resources acquired by lkpi_pci_dev_init_common(). */ static void -lkpi_pci_dev_fill_release(struct device *dev) +lkpi_pci_dev_release_common(struct device *dev) { struct pci_dev *pdev; @@ -416,6 +399,11 @@ kfree(pdev->path_name); } +/* + * This is the .release function for the bus attachments + * [linux_pci_attach_device()], called when the kobj reference drops to 0. + * Note well: there should be no direct calls of this. + */ static void lkpi_pci_dev_release(struct device *dev) { @@ -442,7 +430,7 @@ */ /* Clear the hierarchy recursively to root. */ - lkpi_pci_dev_bus_release(pdev); + lkpi_pci_dev_release_common_bus(pdev); LKPI_PCI_LOCK(); list_del(&pdev->links); @@ -452,9 +440,9 @@ /* irq? */ - /* Undo lkpifill_pci_dev(). */ + /* Undo lkpi_pci_dev_init_common(). */ /* devres is gone already; went at the very top. */ - lkpi_pci_dev_fill_release(dev); + lkpi_pci_dev_release_common(dev); /* * Very last do an internal hack in order to signal @@ -464,8 +452,32 @@ pdev->dev.release = NULL; } +/* + * This is the .release function for lkpinew_pci_dev(), called when the kobj + * reference drops to 0. + * Note well: there should be no direct calls of this. + */ +static void +lkpinew_pci_dev_release(struct device *dev) +{ + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + + /* The root/bus structure could be built up w/o us having asked for it. */ + lkpi_pci_dev_release_common_bus(pdev); + /* Cleanup the lkpi_pci_dev_init_common() resources. */ + lkpi_pci_dev_release_common(dev); + + free(pdev, M_DEVBUF); +} + +/* + * Helper function to iniatlize most of the pdev, dev, .. parts. + * This is share by both pdev creation code paths (bus or function call). + */ static int -lkpifill_pci_dev(device_t dev, struct pci_dev *pdev) +lkpi_pci_dev_init_common(device_t dev, struct pci_dev *pdev) { struct pci_devinfo *dinfo; int error; @@ -508,7 +520,6 @@ } pdev->dev.bsddev = dev; pdev->dev.parent = &linux_root_device; - pdev->dev.release = lkpi_pci_dev_release; if (pci_msi_count(dev) > 0) pdev->msi_desc = malloc(pci_msi_count(dev) * @@ -524,24 +535,8 @@ } /* - * This is the .release function called when the kobj reference drops to 0. - * There should be no direct calls of this. + * LinuxKPI PCI dev (pdev) creation by function call. */ -static void -lkpinew_pci_dev_release(struct device *dev) -{ - struct pci_dev *pdev; - - pdev = to_pci_dev(dev); - - /* The root/bus structure could be built up w/o us having asked for it. */ - lkpi_pci_dev_bus_release(pdev); - /* Cleanup the lkpifill_pci_dev() resources. */ - lkpi_pci_dev_fill_release(dev); - - free(pdev, M_DEVBUF); -} - struct pci_dev * lkpinew_pci_dev(device_t dev) { @@ -549,7 +544,7 @@ int error; pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK|M_ZERO); - error = lkpifill_pci_dev(dev, pdev); + error = lkpi_pci_dev_init_common(dev, pdev); if (error != 0) { free(pdev, M_DEVBUF); return (NULL); @@ -624,76 +619,11 @@ return (pdev); } -static int -linux_pci_probe(device_t dev) -{ - const struct pci_device_id *id; - struct pci_driver *pdrv; - - if ((pdrv = linux_pci_find(dev, &id)) == NULL) - return (ENXIO); - if (device_get_driver(dev) != &pdrv->bsddriver) - return (ENXIO); - device_set_desc(dev, pdrv->name); - - /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ - if (pdrv->bsd_probe_return == 0) - return (BUS_PROBE_DEFAULT); - else - return (pdrv->bsd_probe_return); -} - -static int -linux_pci_attach(device_t dev) -{ - const struct pci_device_id *id; - struct pci_driver *pdrv; - struct pci_dev *pdev; - - pdrv = linux_pci_find(dev, &id); - pdev = device_get_softc(dev); - - MPASS(pdrv != NULL); - MPASS(pdev != NULL); - - return (linux_pci_attach_device(dev, pdrv, id, pdev)); -} - -static struct resource_list_entry * -linux_pci_reserve_bar(struct pci_dev *pdev, struct resource_list *rl, - int type, int rid) -{ - device_t dev; - struct resource *res; - - KASSERT(type == SYS_RES_IOPORT || type == SYS_RES_MEMORY, - ("trying to reserve non-BAR type %d", type)); - - dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ? - device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev; - res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0, - 1, 1, 0); - if (res == NULL) - return (NULL); - return (resource_list_find(rl, type, rid)); -} - -static struct resource_list_entry * -linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar) -{ - struct pci_devinfo *dinfo; - struct resource_list *rl; - struct resource_list_entry *rle; - - dinfo = device_get_ivars(pdev->dev.bsddev); - rl = &dinfo->resources; - rle = resource_list_find(rl, type, rid); - /* Reserve resources for this BAR if needed. */ - if (rle == NULL && reserve_bar) - rle = linux_pci_reserve_bar(pdev, rl, type, rid); - return (rle); -} +/* + * LinuxKPI PCI dev (pdev) initialization/free from the BUS functions. + */ +/* Note well: this function is public and used by, e.g. nvidia-drm. */ int linux_pci_attach_device(device_t dev, struct pci_driver *pdrv, const struct pci_device_id *id, struct pci_dev *pdev) @@ -717,9 +647,10 @@ device_set_ivars(dev, dinfo); } - error = lkpifill_pci_dev(dev, pdev); + error = lkpi_pci_dev_init_common(dev, pdev); if (error != 0) return (error); + pdev->dev.release = lkpi_pci_dev_release; if (isdrm) PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid); @@ -727,7 +658,7 @@ PCI_GET_ID(parent, dev, PCI_ID_RID, &rid); pdev->devfn = rid; pdev->pdrv = pdrv; - rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); + rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 0, false); if (rle != NULL) pdev->dev.irq = rle->start; else @@ -770,7 +701,7 @@ return (0); /* - * If we get here we know that lkpifill_pci_dev() succeeded. + * If we get here we know that lkpi_pci_dev_init_common() succeeded. * That means we have a ref on the dev and a release function set. * We also know the driver probe failed, which means no resources should be * allocated by the driver (unless the driver did not cleanup). @@ -793,7 +724,7 @@ linux_pdev_dma_uninit(pdev); err_dma_init: /* - * Undo lkpifill_pci_dev(); we can ignore some bits like + * Undo lkpi_pci_dev_init_common(); we can ignore some bits like * lists/tailq/devres as they are not filled yet. */ spin_lock_destroy(&pdev->dev.devres_lock); @@ -813,22 +744,7 @@ return (-error); } -static int -linux_pci_detach(device_t dev) -{ - struct pci_dev *pdev; - int error; - - pdev = device_get_softc(dev); - MPASS(pdev != NULL); - - error = linux_pci_detach_device(pdev); - if (error == 0) - device_set_desc(dev, NULL); - - return (error); -} - +/* Note well: this function is public and used by, e.g. nvidia-drm. */ int linux_pci_detach_device(struct pci_dev *pdev) { @@ -839,7 +755,7 @@ * We cannot do much here as almost everything will have * to happen as the last reference to the LinuxKPI device * goes away. That will call the release function, - * which lkpifill_pci_dev() set. That is were most + * which lkpi_pci_dev_init_common() set. That is were most * of the cleanup will happen. But before that give * the driver a chance to cleanup. * The big problem is that the Linux KPI does not @@ -882,6 +798,9 @@ return (EBUSY); } +/* + * Devres resource functions. + */ static int lkpi_pci_disable_dev(struct device *dev) { @@ -1009,6 +928,15 @@ return (dr->mmio_table); } +static bool +lkpi_pci_bar_id_valid(int bar) +{ + if (bar < 0 || bar > PCIR_MAX_BAR_0) + return (false); + + return (true); +} + static struct resource * _lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused) { @@ -1080,6 +1008,14 @@ return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen)); } +static void +lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar, + void *res) +{ + dr->mmio_table[bar] = (void *)rman_get_bushandle(res); + dr->res_table[bar] = res; +} + void * linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen) { @@ -1197,178 +1133,47 @@ } } -static int -linux_pci_suspend(device_t dev) +/* + * Resource functions. + */ +static struct resource_list_entry * +lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) { - const struct dev_pm_ops *pmops; - struct pm_message pm = { }; - struct pci_dev *pdev; - int error; - - error = 0; - linux_set_current(curthread); - pdev = device_get_softc(dev); - pmops = pdev->pdrv->driver.pm; + int type; - if (pdev->pdrv->suspend != NULL) - error = -pdev->pdrv->suspend(pdev, pm); - else if (pmops != NULL && pmops->suspend != NULL) { - error = -pmops->suspend(&pdev->dev); - if (error == 0 && pmops->suspend_late != NULL) - error = -pmops->suspend_late(&pdev->dev); - if (error == 0 && pmops->suspend_noirq != NULL) - error = -pmops->suspend_noirq(&pdev->dev); - } - return (error); + type = pci_resource_type(pdev, bar); + if (type < 0) + return (NULL); + bar = PCIR_BAR(bar); + return (lkpi_pci_get_rle(pdev, type, bar, reserve)); } -static int -linux_pci_resume(device_t dev) +struct device * +lkpi_pci_find_irq_dev(unsigned int irq) { - const struct dev_pm_ops *pmops; struct pci_dev *pdev; - int error; - - error = 0; - linux_set_current(curthread); - pdev = device_get_softc(dev); - pmops = pdev->pdrv->driver.pm; + struct device *found; - if (pdev->pdrv->resume != NULL) - error = -pdev->pdrv->resume(pdev); - else if (pmops != NULL && pmops->resume != NULL) { - if (pmops->resume_early != NULL) - error = -pmops->resume_early(&pdev->dev); - if (error == 0 && pmops->resume != NULL) - error = -pmops->resume(&pdev->dev); + found = NULL; + LKPI_PCI_LOCK(); + list_for_each_entry(pdev, &lkpi_pci_devices, links) { + if (irq == pdev->dev.irq || + (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { + found = &pdev->dev; + break; + } } - return (error); + LKPI_PCI_UNLOCK(); + return (found); } -static int -linux_pci_shutdown(device_t dev) +unsigned long +pci_resource_start(struct pci_dev *pdev, int bar) { - struct pci_dev *pdev; - - linux_set_current(curthread); - pdev = device_get_softc(dev); - if (pdev->pdrv->shutdown != NULL) - pdev->pdrv->shutdown(pdev); - return (0); -} - -static int -linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) -{ - struct pci_dev *pdev; - int error; - - linux_set_current(curthread); - pdev = device_get_softc(dev); - if (pdev->pdrv->bsd_iov_init != NULL) - error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); - else - error = EINVAL; - return (error); -} - -static void -linux_pci_iov_uninit(device_t dev) -{ - struct pci_dev *pdev; - - linux_set_current(curthread); - pdev = device_get_softc(dev); - if (pdev->pdrv->bsd_iov_uninit != NULL) - pdev->pdrv->bsd_iov_uninit(dev); -} - -static int -linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) -{ - struct pci_dev *pdev; - int error; - - linux_set_current(curthread); - pdev = device_get_softc(dev); - if (pdev->pdrv->bsd_iov_add_vf != NULL) - error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); - else - error = EINVAL; - return (error); -} - -static int -_linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) -{ - int error; - - linux_set_current(curthread); - LKPI_PCI_LOCK(); - list_add(&pdrv->node, &lkpi_pci_drivers); - LKPI_PCI_UNLOCK(); - if (pdrv->bsddriver.name == NULL) - pdrv->bsddriver.name = pdrv->name; - pdrv->bsddriver.methods = pci_methods; - pdrv->bsddriver.size = sizeof(struct pci_dev); - - bus_topo_lock(); - error = devclass_add_driver(dc, &pdrv->bsddriver, - BUS_PASS_DEFAULT, &pdrv->bsdclass); - bus_topo_unlock(); - return (-error); -} - -int -linux_pci_register_driver(struct pci_driver *pdrv) -{ - devclass_t dc; - - pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; - dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); - if (dc == NULL) - return (-ENXIO); - return (_linux_pci_register_driver(pdrv, dc)); -} - -static struct resource_list_entry * -lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve) -{ - int type; - - type = pci_resource_type(pdev, bar); - if (type < 0) - return (NULL); - bar = PCIR_BAR(bar); - return (linux_pci_get_rle(pdev, type, bar, reserve)); -} - -struct device * -lkpi_pci_find_irq_dev(unsigned int irq) -{ - struct pci_dev *pdev; - struct device *found; - - found = NULL; - LKPI_PCI_LOCK(); - list_for_each_entry(pdev, &lkpi_pci_devices, links) { - if (irq == pdev->dev.irq || - (irq >= pdev->dev.irq_start && irq < pdev->dev.irq_end)) { - found = &pdev->dev; - break; - } - } - LKPI_PCI_UNLOCK(); - return (found); -} - -unsigned long -pci_resource_start(struct pci_dev *pdev, int bar) -{ - struct resource_list_entry *rle; - rman_res_t newstart; - device_t dev; - int error; + struct resource_list_entry *rle; + rman_res_t newstart; + device_t dev; + int error; if ((rle = lkpi_pci_get_bar(pdev, bar, true)) == NULL) return (0); @@ -1530,51 +1335,9 @@ pci_release_region(pdev, i); } -int -linux_pci_register_drm_driver(struct pci_driver *pdrv) -{ - devclass_t dc; - - dc = devclass_create("vgapci"); - if (dc == NULL) - return (-ENXIO); - pdrv->isdrm = true; - pdrv->name = "drmn"; - return (_linux_pci_register_driver(pdrv, dc)); -} - -void -linux_pci_unregister_driver(struct pci_driver *pdrv) -{ - devclass_t bus; - - bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); - - LKPI_PCI_LOCK(); - list_del(&pdrv->node); - LKPI_PCI_UNLOCK(); - bus_topo_lock(); - if (bus != NULL) - devclass_delete_driver(bus, &pdrv->bsddriver); - bus_topo_unlock(); -} - -void -linux_pci_unregister_drm_driver(struct pci_driver *pdrv) -{ - devclass_t bus; - - bus = devclass_find("vgapci"); - - LKPI_PCI_LOCK(); - list_del(&pdrv->node); - LKPI_PCI_UNLOCK(); - bus_topo_lock(); - if (bus != NULL) - devclass_delete_driver(bus, &pdrv->bsddriver); - bus_topo_unlock(); -} - +/* + * MSI(-X) functions. + */ int linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries, int nreq) @@ -1601,7 +1364,7 @@ pci_release_msi(pdev->dev.bsddev); return avail; } - rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); + rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); pdev->dev.irq_start = rle->start; pdev->dev.irq_end = rle->start + avail; for (i = 0; i < nreq; i++) @@ -1634,7 +1397,7 @@ return (-ENOSPC); } - rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); + rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 1, false); pdev->dev.irq_start = rle->start; pdev->dev.irq_end = rle->start + nvec; pdev->irq = rle->start; @@ -1731,6 +1494,248 @@ return (bus_child_present(dev)); } +/* + * device_* bus functions for PCI. + */ +static int +lkpi_pci_probe(device_t dev) +{ + const struct pci_device_id *id; + struct pci_driver *pdrv; + + if ((pdrv = linux_pci_find(dev, &id)) == NULL) + return (ENXIO); + if (device_get_driver(dev) != &pdrv->bsddriver) + return (ENXIO); + device_set_desc(dev, pdrv->name); + + /* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */ + if (pdrv->bsd_probe_return == 0) + return (BUS_PROBE_DEFAULT); + else + return (pdrv->bsd_probe_return); +} + +static int +lkpi_pci_attach(device_t dev) +{ + const struct pci_device_id *id; + struct pci_driver *pdrv; + struct pci_dev *pdev; + + pdrv = linux_pci_find(dev, &id); + pdev = device_get_softc(dev); + + MPASS(pdrv != NULL); + MPASS(pdev != NULL); + + return (linux_pci_attach_device(dev, pdrv, id, pdev)); +} + +static int +lkpi_pci_detach(device_t dev) +{ + struct pci_dev *pdev; + int error; + + pdev = device_get_softc(dev); + MPASS(pdev != NULL); + + error = linux_pci_detach_device(pdev); + if (error == 0) + device_set_desc(dev, NULL); + + return (error); +} + +static int +lkpi_pci_suspend(device_t dev) +{ + const struct dev_pm_ops *pmops; + struct pm_message pm = { }; + struct pci_dev *pdev; + int error; + + error = 0; + linux_set_current(curthread); + pdev = device_get_softc(dev); + pmops = pdev->pdrv->driver.pm; + + if (pdev->pdrv->suspend != NULL) + error = -pdev->pdrv->suspend(pdev, pm); + else if (pmops != NULL && pmops->suspend != NULL) { + error = -pmops->suspend(&pdev->dev); + if (error == 0 && pmops->suspend_late != NULL) + error = -pmops->suspend_late(&pdev->dev); + if (error == 0 && pmops->suspend_noirq != NULL) + error = -pmops->suspend_noirq(&pdev->dev); + } + return (error); +} + +static int +lkpi_pci_resume(device_t dev) +{ + const struct dev_pm_ops *pmops; + struct pci_dev *pdev; + int error; + + error = 0; + linux_set_current(curthread); + pdev = device_get_softc(dev); + pmops = pdev->pdrv->driver.pm; + + if (pdev->pdrv->resume != NULL) + error = -pdev->pdrv->resume(pdev); + else if (pmops != NULL && pmops->resume != NULL) { + if (pmops->resume_early != NULL) + error = -pmops->resume_early(&pdev->dev); + if (error == 0 && pmops->resume != NULL) + error = -pmops->resume(&pdev->dev); + } + return (error); +} + +static int +lkpi_pci_shutdown(device_t dev) +{ + struct pci_dev *pdev; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->shutdown != NULL) + pdev->pdrv->shutdown(pdev); + return (0); +} + +/* + * PCI IOV bus functions. + */ +static int +lkpi_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config) +{ + struct pci_dev *pdev; + int error; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_init != NULL) + error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config); + else + error = EINVAL; + return (error); +} + +static void +lkpi_pci_iov_uninit(device_t dev) +{ + struct pci_dev *pdev; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_uninit != NULL) + pdev->pdrv->bsd_iov_uninit(dev); +} + +static int +lkpi_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config) +{ + struct pci_dev *pdev; + int error; + + linux_set_current(curthread); + pdev = device_get_softc(dev); + if (pdev->pdrv->bsd_iov_add_vf != NULL) + error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config); + else + error = EINVAL; + return (error); +} + +/* + * LinuxKPI driver (un)registration. + */ +static int +lkpi_pci_register_driver(struct pci_driver *pdrv, devclass_t dc) +{ + int error; + + linux_set_current(curthread); + LKPI_PCI_LOCK(); + list_add(&pdrv->node, &lkpi_pci_drivers); + LKPI_PCI_UNLOCK(); + if (pdrv->bsddriver.name == NULL) + pdrv->bsddriver.name = pdrv->name; + pdrv->bsddriver.methods = pci_methods; + pdrv->bsddriver.size = sizeof(struct pci_dev); + + bus_topo_lock(); + error = devclass_add_driver(dc, &pdrv->bsddriver, + BUS_PASS_DEFAULT, &pdrv->bsdclass); + bus_topo_unlock(); + return (-error); +} + +int +linux_pci_register_driver(struct pci_driver *pdrv) +{ + devclass_t dc; + + pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0; + dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci"); + if (dc == NULL) + return (-ENXIO); + return (lkpi_pci_register_driver(pdrv, dc)); +} + +int +linux_pci_register_drm_driver(struct pci_driver *pdrv) +{ + devclass_t dc; + + dc = devclass_create("vgapci"); + if (dc == NULL) + return (-ENXIO); + pdrv->isdrm = true; + pdrv->name = "drmn"; + return (lkpi_pci_register_driver(pdrv, dc)); +} + +void +linux_pci_unregister_driver(struct pci_driver *pdrv) +{ + devclass_t bus; + + bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci"); + + LKPI_PCI_LOCK(); + list_del(&pdrv->node); + LKPI_PCI_UNLOCK(); + bus_topo_lock(); + if (bus != NULL) + devclass_delete_driver(bus, &pdrv->bsddriver); + bus_topo_unlock(); +} + +void +linux_pci_unregister_drm_driver(struct pci_driver *pdrv) +{ + devclass_t bus; + + bus = devclass_find("vgapci"); + + LKPI_PCI_LOCK(); + list_del(&pdrv->node); + LKPI_PCI_UNLOCK(); + bus_topo_lock(); + if (bus != NULL) + devclass_delete_driver(bus, &pdrv->bsddriver); + bus_topo_unlock(); +} + +/* + * PCI SYSINIT. + */ static void lkpi_pci_init(void *arg __unused) { @@ -1750,57 +1755,61 @@ /* -------------------------------------------------------------------------- */ /* DMA */ +static counter_u64_t lkpi_pci_nseg1_fail; +SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD, + &lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment"); + CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t)); -struct linux_dma_obj { +struct lkpi_dma_obj { void *vaddr; uint64_t dma_addr; bus_dmamap_t dmamap; bus_dma_tag_t dmat; }; -static uma_zone_t linux_dma_trie_zone; -static uma_zone_t linux_dma_obj_zone; +static uma_zone_t lkpi_dma_trie_zone; +static uma_zone_t lkpi_dma_obj_zone; static void -linux_dma_init(void *arg) +lkpi_dma_init(void *arg) { - linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie", + lkpi_dma_trie_zone = uma_zcreate("lkpi_dma_pctrie", pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 0); - linux_dma_obj_zone = uma_zcreate("linux_dma_object", - sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL, + lkpi_dma_obj_zone = uma_zcreate("lkpi_dma_object", + sizeof(struct lkpi_dma_obj), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK); } -SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL); +SYSINIT(lkpi_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, lkpi_dma_init, NULL); static void -linux_dma_uninit(void *arg) +lkpi_dma_uninit(void *arg) { counter_u64_free(lkpi_pci_nseg1_fail); - uma_zdestroy(linux_dma_obj_zone); - uma_zdestroy(linux_dma_trie_zone); + uma_zdestroy(lkpi_dma_obj_zone); + uma_zdestroy(lkpi_dma_trie_zone); } -SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL); +SYSUNINIT(lkpi_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, lkpi_dma_uninit, NULL); static void * linux_dma_trie_alloc(struct pctrie *ptree) { - return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT)); + return (uma_zalloc(lkpi_dma_trie_zone, M_NOWAIT)); } static void linux_dma_trie_free(struct pctrie *ptree, void *node) { - uma_zfree(linux_dma_trie_zone, node); + uma_zfree(lkpi_dma_trie_zone, node); } -PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, +PCTRIE_DEFINE(LINUX_DMA, lkpi_dma_obj, dma_addr, linux_dma_trie_alloc, linux_dma_trie_free); #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) @@ -1809,7 +1818,7 @@ bus_dma_tag_t dmat) { struct linux_dma_priv *priv; - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; int error, nseg; bus_dma_segment_t seg; @@ -1824,7 +1833,7 @@ if (bus_dma_id_mapped(dmat, phys, len)) return (phys); - obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT); + obj = uma_zalloc(lkpi_dma_obj_zone, M_NOWAIT); if (obj == NULL) { return (0); } @@ -1833,7 +1842,7 @@ DMA_PRIV_LOCK(priv); if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) { DMA_PRIV_UNLOCK(priv); - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); return (0); } @@ -1843,7 +1852,7 @@ if (error != 0) { bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); counter_u64_add(lkpi_pci_nseg1_fail, 1); if (linuxkpi_debug) { device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys " @@ -1862,7 +1871,7 @@ bus_dmamap_unload(obj->dmat, obj->dmamap); bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); return (0); } DMA_PRIV_UNLOCK(priv); @@ -1908,7 +1917,7 @@ enum dma_data_direction direction, unsigned long attrs) { struct linux_dma_priv *priv; - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; priv = dev->dma_priv; @@ -1947,7 +1956,7 @@ bus_dmamap_destroy(obj->dmat, obj->dmamap); DMA_PRIV_UNLOCK(priv); - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); } #else void @@ -2081,7 +2090,7 @@ bus_dmasync_op_t op) { struct linux_dma_priv *priv; - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; priv = dev->dma_priv; @@ -2208,7 +2217,7 @@ static inline int dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) { - struct linux_dma_obj *obj = mem; + struct lkpi_dma_obj *obj = mem; struct dma_pool *pool = arg; int error, nseg; bus_dma_segment_t seg; @@ -2231,7 +2240,7 @@ static void dma_pool_obj_dtor(void *mem, int size, void *arg) { - struct linux_dma_obj *obj = mem; + struct lkpi_dma_obj *obj = mem; struct dma_pool *pool = arg; DMA_POOL_LOCK(pool); @@ -2244,18 +2253,18 @@ int flags) { struct dma_pool *pool = arg; - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; int error, i; for (i = 0; i < count; i++) { - obj = uma_zalloc(linux_dma_obj_zone, flags); + obj = uma_zalloc(lkpi_dma_obj_zone, flags); if (obj == NULL) break; error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, BUS_DMA_NOWAIT, &obj->dmamap); if (error!= 0) { - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); break; } @@ -2269,13 +2278,13 @@ dma_pool_obj_release(void *arg, void **store, int count) { struct dma_pool *pool = arg; - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; int i; for (i = 0; i < count; i++) { obj = store[i]; bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); - uma_zfree(linux_dma_obj_zone, obj); + uma_zfree(lkpi_dma_obj_zone, obj); } } @@ -2341,7 +2350,7 @@ linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, dma_addr_t *handle) { - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK); if (obj == NULL) @@ -2362,7 +2371,7 @@ void linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) { - struct linux_dma_obj *obj; + struct lkpi_dma_obj *obj; DMA_POOL_LOCK(pool); obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); @@ -2380,7 +2389,7 @@ /* Backlight. */ static int -linux_backlight_get_status(device_t dev, struct backlight_props *props) +lkpi_backlight_get_status(device_t dev, struct backlight_props *props) { struct pci_dev *pdev; @@ -2395,7 +2404,7 @@ } static int -linux_backlight_get_info(device_t dev, struct backlight_info *info) +lkpi_backlight_get_info(device_t dev, struct backlight_info *info) { struct pci_dev *pdev; @@ -2408,7 +2417,7 @@ } static int -linux_backlight_update_status(device_t dev, struct backlight_props *props) +lkpi_backlight_update_status(device_t dev, struct backlight_props *props) { struct pci_dev *pdev; @@ -2450,3 +2459,28 @@ free(bd->name, M_DEVBUF); free(bd, M_DEVBUF); } + +/* -------------------------------------------------------------------------- */ +/* Native bus attachment glue. */ + +static device_method_t pci_methods[] = { + DEVMETHOD(device_probe, lkpi_pci_probe), + DEVMETHOD(device_attach, lkpi_pci_attach), + DEVMETHOD(device_detach, lkpi_pci_detach), + DEVMETHOD(device_suspend, lkpi_pci_suspend), + DEVMETHOD(device_resume, lkpi_pci_resume), + DEVMETHOD(device_shutdown, lkpi_pci_shutdown), + + DEVMETHOD(pci_iov_init, lkpi_pci_iov_init), + DEVMETHOD(pci_iov_uninit, lkpi_pci_iov_uninit), + DEVMETHOD(pci_iov_add_vf, lkpi_pci_iov_add_vf), + + /* Bus interface. */ + DEVMETHOD(bus_add_child, bus_generic_add_child), + + /* backlight interface */ + DEVMETHOD(backlight_update_status, lkpi_backlight_update_status), + DEVMETHOD(backlight_get_status, lkpi_backlight_get_status), + DEVMETHOD(backlight_get_info, lkpi_backlight_get_info), + DEVMETHOD_END +};