Differential D57433 Diff 179190 sys/compat/linuxkpi/common/src/linux_pci.c

Changeset View

Standalone View

sys/compat/linuxkpi/common/src/linux_pci.c

Show All 27 Lines
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.		* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/		*/

/*		/*
* This file contains multiple parts:		* This file contains multiple parts:
* (1) PCI		* (1) PCI
* (2) DMA		* (2) DMA
* (3) Backlight		* (3) Backlight
		* (4) Native bus attachment glue.
*/		*/

#include <sys/param.h>		#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/bus.h>		#include <sys/bus.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines
* we have no driver (pdev->pdrv).		* we have no driver (pdev->pdrv).
*		*
* Only devices from (1) end up on our LinuxKPI global lkpi_pci_devices list.		* Only devices from (1) end up on our LinuxKPI global lkpi_pci_devices list.
*/		*/

/* Undef the linux function macro defined in linux/pci.h */		/* Undef the linux function macro defined in linux/pci.h */
#undef pci_get_class		#undef pci_get_class

extern int linuxkpi_debug;		static void lkpi_pcim_iomap_table_release(struct device , void );

		extern int linuxkpi_debug;
SYSCTL_DECL(_compat_linuxkpi);		SYSCTL_DECL(_compat_linuxkpi);

static counter_u64_t lkpi_pci_nseg1_fail;		static device_method_t pci_methods[];
SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD,
&lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment");

static device_probe_t linux_pci_probe;
static device_attach_t linux_pci_attach;
static device_detach_t linux_pci_detach;
static device_suspend_t linux_pci_suspend;
static device_resume_t linux_pci_resume;
static device_shutdown_t linux_pci_shutdown;
static pci_iov_init_t linux_pci_iov_init;
static pci_iov_uninit_t linux_pci_iov_uninit;
static pci_iov_add_vf_t linux_pci_iov_add_vf;
static int linux_backlight_get_status(device_t dev, struct backlight_props *props);
static int linux_backlight_update_status(device_t dev, struct backlight_props *props);
static int linux_backlight_get_info(device_t dev, struct backlight_info *info);
static void lkpi_pcim_iomap_table_release(struct device , void );

static struct list_head lkpi_pci_drivers;		static struct list_head lkpi_pci_drivers;
static struct list_head lkpi_pci_devices;		static struct list_head lkpi_pci_devices;
static spinlock_t lkpi_pci_lock;		static spinlock_t lkpi_pci_lock;

#define LKPI_PCI_LOCK_INIT() spin_lock_init(&lkpi_pci_lock)		#define LKPI_PCI_LOCK_INIT() spin_lock_init(&lkpi_pci_lock)
#define LKPI_PCI_LOCK_DESTROY() spin_lock_destroy(&lkpi_pci_lock)		#define LKPI_PCI_LOCK_DESTROY() spin_lock_destroy(&lkpi_pci_lock)
#define LKPI_PCI_LOCK() spin_lock(&lkpi_pci_lock)		#define LKPI_PCI_LOCK() spin_lock(&lkpi_pci_lock)
#define LKPI_PCI_UNLOCK() spin_unlock(&lkpi_pci_lock)		#define LKPI_PCI_UNLOCK() spin_unlock(&lkpi_pci_lock)

static device_method_t pci_methods[] = {
DEVMETHOD(device_probe, linux_pci_probe),
DEVMETHOD(device_attach, linux_pci_attach),
DEVMETHOD(device_detach, linux_pci_detach),
DEVMETHOD(device_suspend, linux_pci_suspend),
DEVMETHOD(device_resume, linux_pci_resume),
DEVMETHOD(device_shutdown, linux_pci_shutdown),
DEVMETHOD(pci_iov_init, linux_pci_iov_init),
DEVMETHOD(pci_iov_uninit, linux_pci_iov_uninit),
DEVMETHOD(pci_iov_add_vf, linux_pci_iov_add_vf),

/* Bus interface. */
DEVMETHOD(bus_add_child, bus_generic_add_child),

/* backlight interface */
DEVMETHOD(backlight_update_status, linux_backlight_update_status),
DEVMETHOD(backlight_get_status, linux_backlight_get_status),
DEVMETHOD(backlight_get_info, linux_backlight_get_info),
DEVMETHOD_END
};

const char *pci_power_names[] = {		const char *pci_power_names[] = {
"UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold"		"UNKNOWN", "D0", "D1", "D2", "D3hot", "D3cold"
};		};

/* We need some meta-struct to keep track of these for devres. */		/* We need some meta-struct to keep track of these for devres. */
struct pci_devres {		struct pci_devres {
bool enable_io;		bool enable_io;
/* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */		/* PCIR_MAX_BAR_0 + 1 = 6 => BIT(0..5). */
Show All 11 Lines	struct linux_dma_priv {
uint64_t dma_coherent_mask;		uint64_t dma_coherent_mask;
bus_dma_tag_t dmat_coherent;		bus_dma_tag_t dmat_coherent;
struct mtx lock;		struct mtx lock;
struct pctrie ptree;		struct pctrie ptree;
};		};
#define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock)		#define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock)
#define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock)		#define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock)

static void		static struct resource_list_entry *
lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar,		lkpi_pci_reserve_bar(struct pci_dev pdev, struct resource_list rl,
void *res)		int type, int rid)
{		{
dr->mmio_table[bar] = (void *)rman_get_bushandle(res);		device_t dev;
dr->res_table[bar] = res;		struct resource *res;

		KASSERT(type == SYS_RES_IOPORT \|\| type == SYS_RES_MEMORY,
		("trying to reserve non-BAR type %d", type));

		dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ?
		device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev;
		res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0,
		1, 1, 0);
		if (res == NULL)
		return (NULL);
		return (resource_list_find(rl, type, rid));
}		}

static bool		static struct resource_list_entry *
lkpi_pci_bar_id_valid(int bar)		lkpi_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar)
{		{
if (bar < 0 \|\| bar > PCIR_MAX_BAR_0)		struct pci_devinfo *dinfo;
return (false);		struct resource_list *rl;
		struct resource_list_entry *rle;

return (true);		dinfo = device_get_ivars(pdev->dev.bsddev);
		rl = &dinfo->resources;
		rle = resource_list_find(rl, type, rid);
		/* Reserve resources for this BAR if needed. */
		if (rle == NULL && reserve_bar)
		rle = lkpi_pci_reserve_bar(pdev, rl, type, rid);
		return (rle);
}		}

static int		static int
linux_pdev_dma_uninit(struct pci_dev *pdev)		linux_pdev_dma_uninit(struct pci_dev *pdev)
{		{
struct linux_dma_priv *priv;		struct linux_dma_priv *priv;

priv = pdev->dev.dma_priv;		priv = pdev->dev.dma_priv;
▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Lines	lkpi_pci_get_device(uint32_t vendor, uint32_t device, struct pci_dev *odev)
pci_dev_get(found);		pci_dev_get(found);
LKPI_PCI_UNLOCK();		LKPI_PCI_UNLOCK();
pci_dev_put(odev0);		pci_dev_put(odev0);

return (found);		return (found);
}		}

/*		/*
* Shared function releasing the bus resources which can be built independent		* Shared helper function releasing the bus resources which can be built
* of the way we allocate/initialization the pdev [pci_upstream_bridge(),		* independent of the way we allocate/initialization the pdev
* pcie_find_root_port()].		* [pci_upstream_bridge(), pcie_find_root_port()].
*/		*/
static void		static void
lkpi_pci_dev_bus_release(struct pci_dev *pdev)		lkpi_pci_dev_release_common_bus(struct pci_dev *pdev)
{		{
if (pdev->bus->self != pdev) {		if (pdev->bus->self != pdev) {
pci_dev_put(pdev->bus->self);		pci_dev_put(pdev->bus->self);
pdev->bus->self = NULL;		pdev->bus->self = NULL;
}		}

if (pdev->root != NULL) {		if (pdev->root != NULL) {
pci_dev_put(pdev->root);		pci_dev_put(pdev->root);
pdev->root = NULL;		pdev->root = NULL;
}		}
}		}

/* Release resources acquired by lkpifill_pci_dev(). */		/* Release helper function to free resources acquired by lkpi_pci_dev_init_common(). */
static void		static void
lkpi_pci_dev_fill_release(struct device *dev)		lkpi_pci_dev_release_common(struct device *dev)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;

pdev = to_pci_dev(dev);		pdev = to_pci_dev(dev);

if (!list_empty_careful(&pdev->dev.irqents)) {		if (!list_empty_careful(&pdev->dev.irqents)) {
dev_warn(&pdev->dev, "%s: driver did not clean up; "		dev_warn(&pdev->dev, "%s: driver did not clean up; "
"leaking IRQs\n", __func__);		"leaking IRQs\n", __func__);
Show All 17 Lines	for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--)
free(pdev->msi_desc[i], M_DEVBUF);		free(pdev->msi_desc[i], M_DEVBUF);
free(pdev->msi_desc, M_DEVBUF);		free(pdev->msi_desc, M_DEVBUF);
}		}

free(pdev->bus, M_DEVBUF);		free(pdev->bus, M_DEVBUF);
kfree(pdev->path_name);		kfree(pdev->path_name);
}		}

		/*
		* This is the .release function for the bus attachments
		* [linux_pci_attach_device()], called when the kobj reference drops to 0.
		* Note well: there should be no direct calls of this.
		*/
static void		static void
lkpi_pci_dev_release(struct device *dev)		lkpi_pci_dev_release(struct device *dev)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;

/*		/*
* The very first we have to free all the dynamic		* The very first we have to free all the dynamic
* resource which are on the devres list.		* resource which are on the devres list.
Show All 10 Lines	lkpi_pci_dev_release(struct device *dev)

/*		/*
* pdrv->remove happens before pci_put_dev() in		* pdrv->remove happens before pci_put_dev() in
* linux_pci_detach_device(), which means the driver should have		* linux_pci_detach_device(), which means the driver should have
* cleaned up before we get here; see irqents and mmio below.		* cleaned up before we get here; see irqents and mmio below.
*/		*/

/* Clear the hierarchy recursively to root. */		/* Clear the hierarchy recursively to root. */
lkpi_pci_dev_bus_release(pdev);		lkpi_pci_dev_release_common_bus(pdev);

LKPI_PCI_LOCK();		LKPI_PCI_LOCK();
list_del(&pdev->links);		list_del(&pdev->links);
LKPI_PCI_UNLOCK();		LKPI_PCI_UNLOCK();

linux_pdev_dma_uninit(pdev);		linux_pdev_dma_uninit(pdev);

/* irq? */		/* irq? */

/* Undo lkpifill_pci_dev(). */		/* Undo lkpi_pci_dev_init_common(). */
/* devres is gone already; went at the very top. */		/* devres is gone already; went at the very top. */
lkpi_pci_dev_fill_release(dev);		lkpi_pci_dev_release_common(dev);

/*		/*
* Very last do an internal hack in order to signal		* Very last do an internal hack in order to signal
* that this was run (device reference fully dropped).		* that this was run (device reference fully dropped).
* See comment in linux_pci_detach_device().		* See comment in linux_pci_detach_device().
*/		*/
pdev->dev.release = NULL;		pdev->dev.release = NULL;
}		}

		/*
		* This is the .release function for lkpinew_pci_dev(), called when the kobj
		* reference drops to 0.
		* Note well: there should be no direct calls of this.
		*/
		static void
		lkpinew_pci_dev_release(struct device *dev)
		{
		struct pci_dev *pdev;

		pdev = to_pci_dev(dev);

		/* The root/bus structure could be built up w/o us having asked for it. */
		lkpi_pci_dev_release_common_bus(pdev);
		/* Cleanup the lkpi_pci_dev_init_common() resources. */
		lkpi_pci_dev_release_common(dev);

		free(pdev, M_DEVBUF);
		}

		/*
		* Helper function to iniatlize most of the pdev, dev, .. parts.
		* This is share by both pdev creation code paths (bus or function call).
		*/
static int		static int
lkpifill_pci_dev(device_t dev, struct pci_dev *pdev)		lkpi_pci_dev_init_common(device_t dev, struct pci_dev *pdev)
{		{
struct pci_devinfo *dinfo;		struct pci_devinfo *dinfo;
int error;		int error;

error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype,		error = kobject_init_and_add(&pdev->dev.kobj, &linux_dev_ktype,
&linux_root_device.kobj, device_get_nameunit(dev));		&linux_root_device.kobj, device_get_nameunit(dev));
if (error != 0) {		if (error != 0) {
printf("%s:%d: kobject_init_and_add returned %d\n",		printf("%s:%d: kobject_init_and_add returned %d\n",
Show All 26 Lines	if (dinfo->cfg.pcie.pcie_location != 0 &&
* This should be the upstream bridge; pci_upstream_bridge()		* This should be the upstream bridge; pci_upstream_bridge()
* handles that case on demand as otherwise we'll shadow the		* handles that case on demand as otherwise we'll shadow the
* entire PCI hierarchy.		* entire PCI hierarchy.
*/		*/
pdev->bus->self = pdev;		pdev->bus->self = pdev;
}		}
pdev->dev.bsddev = dev;		pdev->dev.bsddev = dev;
pdev->dev.parent = &linux_root_device;		pdev->dev.parent = &linux_root_device;
pdev->dev.release = lkpi_pci_dev_release;

if (pci_msi_count(dev) > 0)		if (pci_msi_count(dev) > 0)
pdev->msi_desc = malloc(pci_msi_count(dev) *		pdev->msi_desc = malloc(pci_msi_count(dev) *
sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK \| M_ZERO);		sizeof(*pdev->msi_desc), M_DEVBUF, M_WAITOK \| M_ZERO);

TAILQ_INIT(&pdev->mmio);		TAILQ_INIT(&pdev->mmio);
spin_lock_init(&pdev->pcie_cap_lock);		spin_lock_init(&pdev->pcie_cap_lock);
spin_lock_init(&pdev->dev.devres_lock);		spin_lock_init(&pdev->dev.devres_lock);
INIT_LIST_HEAD(&pdev->dev.devres_head);		INIT_LIST_HEAD(&pdev->dev.devres_head);
INIT_LIST_HEAD(&pdev->dev.irqents);		INIT_LIST_HEAD(&pdev->dev.irqents);

return (0);		return (0);
}		}

/*		/*
* This is the .release function called when the kobj reference drops to 0.		* LinuxKPI PCI dev (pdev) creation by function call.
* There should be no direct calls of this.
*/		*/
static void
lkpinew_pci_dev_release(struct device *dev)
{
struct pci_dev *pdev;

pdev = to_pci_dev(dev);

/* The root/bus structure could be built up w/o us having asked for it. */
lkpi_pci_dev_bus_release(pdev);
/* Cleanup the lkpifill_pci_dev() resources. */
lkpi_pci_dev_fill_release(dev);

free(pdev, M_DEVBUF);
}

struct pci_dev *		struct pci_dev *
lkpinew_pci_dev(device_t dev)		lkpinew_pci_dev(device_t dev)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;
int error;		int error;

pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK\|M_ZERO);		pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK\|M_ZERO);
error = lkpifill_pci_dev(dev, pdev);		error = lkpi_pci_dev_init_common(dev, pdev);
if (error != 0) {		if (error != 0) {
free(pdev, M_DEVBUF);		free(pdev, M_DEVBUF);
return (NULL);		return (NULL);
}		}
pdev->dev.release = lkpinew_pci_dev_release;		pdev->dev.release = lkpinew_pci_dev_release;

return (pdev);		return (pdev);
}		}
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines	lkpi_pci_get_slot(struct pci_bus *pbus, unsigned int devfn)
dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn));		dev = pci_find_bsf(pbus->number, PCI_SLOT(devfn), PCI_FUNC(devfn));
if (dev == NULL)		if (dev == NULL)
return (NULL);		return (NULL);

pdev = lkpinew_pci_dev(dev);		pdev = lkpinew_pci_dev(dev);
return (pdev);		return (pdev);
}		}

static int
linux_pci_probe(device_t dev)
{
const struct pci_device_id *id;
struct pci_driver *pdrv;

if ((pdrv = linux_pci_find(dev, &id)) == NULL)		/*
return (ENXIO);		* LinuxKPI PCI dev (pdev) initialization/free from the BUS functions.
if (device_get_driver(dev) != &pdrv->bsddriver)		*/
return (ENXIO);		/* Note well: this function is public and used by, e.g. nvidia-drm. */
device_set_desc(dev, pdrv->name);

/* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */
if (pdrv->bsd_probe_return == 0)
return (BUS_PROBE_DEFAULT);
else
return (pdrv->bsd_probe_return);
}

static int
linux_pci_attach(device_t dev)
{
const struct pci_device_id *id;
struct pci_driver *pdrv;
struct pci_dev *pdev;

pdrv = linux_pci_find(dev, &id);
pdev = device_get_softc(dev);

MPASS(pdrv != NULL);
MPASS(pdev != NULL);

return (linux_pci_attach_device(dev, pdrv, id, pdev));
}

static struct resource_list_entry *
linux_pci_reserve_bar(struct pci_dev pdev, struct resource_list rl,
int type, int rid)
{
device_t dev;
struct resource *res;

KASSERT(type == SYS_RES_IOPORT \|\| type == SYS_RES_MEMORY,
("trying to reserve non-BAR type %d", type));

dev = pdev->pdrv != NULL && pdev->pdrv->isdrm ?
device_get_parent(pdev->dev.bsddev) : pdev->dev.bsddev;
res = pci_reserve_map(device_get_parent(dev), dev, type, rid, 0, ~0,
1, 1, 0);
if (res == NULL)
return (NULL);
return (resource_list_find(rl, type, rid));
}

static struct resource_list_entry *
linux_pci_get_rle(struct pci_dev *pdev, int type, int rid, bool reserve_bar)
{
struct pci_devinfo *dinfo;
struct resource_list *rl;
struct resource_list_entry *rle;

dinfo = device_get_ivars(pdev->dev.bsddev);
rl = &dinfo->resources;
rle = resource_list_find(rl, type, rid);
/* Reserve resources for this BAR if needed. */
if (rle == NULL && reserve_bar)
rle = linux_pci_reserve_bar(pdev, rl, type, rid);
return (rle);
}

int		int
linux_pci_attach_device(device_t dev, struct pci_driver *pdrv,		linux_pci_attach_device(device_t dev, struct pci_driver *pdrv,
const struct pci_device_id id, struct pci_dev pdev)		const struct pci_device_id id, struct pci_dev pdev)
{		{
struct resource_list_entry *rle;		struct resource_list_entry *rle;
device_t parent;		device_t parent;
struct pci_dev pbus, ppbus;		struct pci_dev pbus, ppbus;
uintptr_t rid;		uintptr_t rid;
int error;		int error;
bool isdrm;		bool isdrm;

linux_set_current(curthread);		linux_set_current(curthread);

parent = device_get_parent(dev);		parent = device_get_parent(dev);
isdrm = pdrv != NULL && pdrv->isdrm;		isdrm = pdrv != NULL && pdrv->isdrm;

if (isdrm) {		if (isdrm) {
struct pci_devinfo *dinfo;		struct pci_devinfo *dinfo;

dinfo = device_get_ivars(parent);		dinfo = device_get_ivars(parent);
device_set_ivars(dev, dinfo);		device_set_ivars(dev, dinfo);
}		}

error = lkpifill_pci_dev(dev, pdev);		error = lkpi_pci_dev_init_common(dev, pdev);
if (error != 0)		if (error != 0)
return (error);		return (error);
		pdev->dev.release = lkpi_pci_dev_release;

if (isdrm)		if (isdrm)
PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid);		PCI_GET_ID(device_get_parent(parent), parent, PCI_ID_RID, &rid);
else		else
PCI_GET_ID(parent, dev, PCI_ID_RID, &rid);		PCI_GET_ID(parent, dev, PCI_ID_RID, &rid);
pdev->devfn = rid;		pdev->devfn = rid;
pdev->pdrv = pdrv;		pdev->pdrv = pdrv;
rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 0, false);		rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 0, false);
if (rle != NULL)		if (rle != NULL)
pdev->dev.irq = rle->start;		pdev->dev.irq = rle->start;
else		else
pdev->dev.irq = LINUX_IRQ_INVALID;		pdev->dev.irq = LINUX_IRQ_INVALID;
pdev->irq = pdev->dev.irq;		pdev->irq = pdev->dev.irq;
error = linux_pdev_dma_init(pdev);		error = linux_pdev_dma_init(pdev);
if (error)		if (error)
goto err_dma_init;		goto err_dma_init;
Show All 26 Lines	linux_pci_attach_device(device_t dev, struct pci_driver *pdrv,
if (pdrv != NULL) {		if (pdrv != NULL) {
error = pdrv->probe(pdev, id);		error = pdrv->probe(pdev, id);
if (error)		if (error)
goto err_probe;		goto err_probe;
}		}
return (0);		return (0);

/*		/*
* If we get here we know that lkpifill_pci_dev() succeeded.		* If we get here we know that lkpi_pci_dev_init_common() succeeded.
* That means we have a ref on the dev and a release function set.		* That means we have a ref on the dev and a release function set.
* We also know the driver probe failed, which means no resources should be		* We also know the driver probe failed, which means no resources should be
* allocated by the driver (unless the driver did not cleanup).		* allocated by the driver (unless the driver did not cleanup).
* That means we could simply call pci_dev_put(pdev) and let the release		* That means we could simply call pci_dev_put(pdev) and let the release
* function do its job.		* function do its job.
*/		*/
err_probe:		err_probe:
if (pdev->bus->self != pdev) {		if (pdev->bus->self != pdev) {
pci_dev_put(pdev->bus->self);		pci_dev_put(pdev->bus->self);
pdev->bus->self = NULL;		pdev->bus->self = NULL;
}		}
if (pdev->root != NULL) {		if (pdev->root != NULL) {
pci_dev_put(pdev->root);		pci_dev_put(pdev->root);
pdev->root = NULL;		pdev->root = NULL;
}		}
err_list_add:		err_list_add:
LKPI_PCI_LOCK();		LKPI_PCI_LOCK();
list_del(&pdev->links);		list_del(&pdev->links);
LKPI_PCI_UNLOCK();		LKPI_PCI_UNLOCK();
linux_pdev_dma_uninit(pdev);		linux_pdev_dma_uninit(pdev);
err_dma_init:		err_dma_init:
/*		/*
* Undo lkpifill_pci_dev(); we can ignore some bits like		* Undo lkpi_pci_dev_init_common(); we can ignore some bits like
* lists/tailq/devres as they are not filled yet.		* lists/tailq/devres as they are not filled yet.
*/		*/
spin_lock_destroy(&pdev->dev.devres_lock);		spin_lock_destroy(&pdev->dev.devres_lock);
spin_lock_destroy(&pdev->pcie_cap_lock);		spin_lock_destroy(&pdev->pcie_cap_lock);
if (pdev->msi_desc != NULL) {		if (pdev->msi_desc != NULL) {
for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--)		for (int i = pci_msi_count(pdev->dev.bsddev) - 1; i >= 0; i--)
free(pdev->msi_desc[i], M_DEVBUF);		free(pdev->msi_desc[i], M_DEVBUF);
free(pdev->msi_desc, M_DEVBUF);		free(pdev->msi_desc, M_DEVBUF);
}		}
free(pdev->bus, M_DEVBUF);		free(pdev->bus, M_DEVBUF);
kfree(pdev->path_name);		kfree(pdev->path_name);

/* Bandaid, just to be sure. */		/* Bandaid, just to be sure. */
pdev->dev.release = NULL;		pdev->dev.release = NULL;
kobject_put(&pdev->dev.kobj);		kobject_put(&pdev->dev.kobj);

return (-error);		return (-error);
}		}

static int		/* Note well: this function is public and used by, e.g. nvidia-drm. */
linux_pci_detach(device_t dev)
{
struct pci_dev *pdev;
int error;

pdev = device_get_softc(dev);
MPASS(pdev != NULL);

error = linux_pci_detach_device(pdev);
if (error == 0)
device_set_desc(dev, NULL);

return (error);
}

int		int
linux_pci_detach_device(struct pci_dev *pdev)		linux_pci_detach_device(struct pci_dev *pdev)
{		{

linux_set_current(curthread);		linux_set_current(curthread);

/*		/*
* We cannot do much here as almost everything will have		* We cannot do much here as almost everything will have
* to happen as the last reference to the LinuxKPI device		* to happen as the last reference to the LinuxKPI device
* goes away. That will call the release function,		* goes away. That will call the release function,
* which lkpifill_pci_dev() set. That is were most		* which lkpi_pci_dev_init_common() set. That is were most
* of the cleanup will happen. But before that give		* of the cleanup will happen. But before that give
* the driver a chance to cleanup.		* the driver a chance to cleanup.
* The big problem is that the Linux KPI does not		* The big problem is that the Linux KPI does not
* report back if it was the last kref (well kref		* report back if it was the last kref (well kref
* does report back but then kobj, dev, pdev do not).		* does report back but then kobj, dev, pdev do not).
* So we have little way of knowing if the release		* So we have little way of knowing if the release
* happened or not. We have to play tricks for that		* happened or not. We have to play tricks for that
* and we can given the softc (pdev) is still valid		* and we can given the softc (pdev) is still valid
Show All 26 Lines	linux_pci_detach_device(struct pci_dev *pdev)
device_printf(pdev->dev.bsddev, "%s failed due to %u other pending "		device_printf(pdev->dev.bsddev, "%s failed due to %u other pending "
"references on the LinuxKPI device.\n", __func__,		"references on the LinuxKPI device.\n", __func__,
kref_read(&pdev->dev.kobj.kref));		kref_read(&pdev->dev.kobj.kref));
pci_dev_get(pdev);		pci_dev_get(pdev);

return (EBUSY);		return (EBUSY);
}		}

		/*
		* Devres resource functions.
		*/
static int		static int
lkpi_pci_disable_dev(struct device *dev)		lkpi_pci_disable_dev(struct device *dev)
{		{

(void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY);		(void) pci_disable_io(dev->bsddev, SYS_RES_MEMORY);
(void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT);		(void) pci_disable_io(dev->bsddev, SYS_RES_IOPORT);
return (0);		return (0);
}		}
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines	linuxkpi_pcim_iomap_table(struct pci_dev *pdev)
*/		*/
if (pdev->want_iomap_res)		if (pdev->want_iomap_res)
return ((void **)dr->res_table);		return ((void **)dr->res_table);

/* This is the Linux default. */		/* This is the Linux default. */
return (dr->mmio_table);		return (dr->mmio_table);
}		}

		static bool
		lkpi_pci_bar_id_valid(int bar)
		{
		if (bar < 0 \|\| bar > PCIR_MAX_BAR_0)
		return (false);

		return (true);
		}

static struct resource *		static struct resource *
_lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused)		_lkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen __unused)
{		{
struct pci_mmio_region mmio, p;		struct pci_mmio_region mmio, p;
int type;		int type;

if (!lkpi_pci_bar_id_valid(bar))		if (!lkpi_pci_bar_id_valid(bar))
return (NULL);		return (NULL);
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines
linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)		linuxkpi_pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
{		{
if (!lkpi_pci_bar_id_valid(bar))		if (!lkpi_pci_bar_id_valid(bar))
return (NULL);		return (NULL);

return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen));		return (linuxkpi_pci_iomap_range(pdev, bar, 0, maxlen));
}		}

		static void
		lkpi_set_pcim_iomap_devres(struct pcim_iomap_devres *dr, int bar,
		void *res)
		{
		dr->mmio_table[bar] = (void *)rman_get_bushandle(res);
		dr->res_table[bar] = res;
		}

void *		void *
linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)		linuxkpi_pcim_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
{		{
struct pcim_iomap_devres *dr;		struct pcim_iomap_devres *dr;
void *res;		void *res;

if (!lkpi_pci_bar_id_valid(bar))		if (!lkpi_pci_bar_id_valid(bar))
return (NULL);		return (NULL);
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines	for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) {

if (dr->mmio_table[bar] == NULL)		if (dr->mmio_table[bar] == NULL)
continue;		continue;

pci_iounmap(pdev, dr->mmio_table[bar]);		pci_iounmap(pdev, dr->mmio_table[bar]);
}		}
}		}

static int		/*
linux_pci_suspend(device_t dev)		* Resource functions.
{		*/
const struct dev_pm_ops *pmops;
struct pm_message pm = { };
struct pci_dev *pdev;
int error;

error = 0;
linux_set_current(curthread);
pdev = device_get_softc(dev);
pmops = pdev->pdrv->driver.pm;

if (pdev->pdrv->suspend != NULL)
error = -pdev->pdrv->suspend(pdev, pm);
else if (pmops != NULL && pmops->suspend != NULL) {
error = -pmops->suspend(&pdev->dev);
if (error == 0 && pmops->suspend_late != NULL)
error = -pmops->suspend_late(&pdev->dev);
if (error == 0 && pmops->suspend_noirq != NULL)
error = -pmops->suspend_noirq(&pdev->dev);
}
return (error);
}

static int
linux_pci_resume(device_t dev)
{
const struct dev_pm_ops *pmops;
struct pci_dev *pdev;
int error;

error = 0;
linux_set_current(curthread);
pdev = device_get_softc(dev);
pmops = pdev->pdrv->driver.pm;

if (pdev->pdrv->resume != NULL)
error = -pdev->pdrv->resume(pdev);
else if (pmops != NULL && pmops->resume != NULL) {
if (pmops->resume_early != NULL)
error = -pmops->resume_early(&pdev->dev);
if (error == 0 && pmops->resume != NULL)
error = -pmops->resume(&pdev->dev);
}
return (error);
}

static int
linux_pci_shutdown(device_t dev)
{
struct pci_dev *pdev;

linux_set_current(curthread);
pdev = device_get_softc(dev);
if (pdev->pdrv->shutdown != NULL)
pdev->pdrv->shutdown(pdev);
return (0);
}

static int
linux_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
{
struct pci_dev *pdev;
int error;

linux_set_current(curthread);
pdev = device_get_softc(dev);
if (pdev->pdrv->bsd_iov_init != NULL)
error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config);
else
error = EINVAL;
return (error);
}

static void
linux_pci_iov_uninit(device_t dev)
{
struct pci_dev *pdev;

linux_set_current(curthread);
pdev = device_get_softc(dev);
if (pdev->pdrv->bsd_iov_uninit != NULL)
pdev->pdrv->bsd_iov_uninit(dev);
}

static int
linux_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
{
struct pci_dev *pdev;
int error;

linux_set_current(curthread);
pdev = device_get_softc(dev);
if (pdev->pdrv->bsd_iov_add_vf != NULL)
error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config);
else
error = EINVAL;
return (error);
}

static int
_linux_pci_register_driver(struct pci_driver *pdrv, devclass_t dc)
{
int error;

linux_set_current(curthread);
LKPI_PCI_LOCK();
list_add(&pdrv->node, &lkpi_pci_drivers);
LKPI_PCI_UNLOCK();
if (pdrv->bsddriver.name == NULL)
pdrv->bsddriver.name = pdrv->name;
pdrv->bsddriver.methods = pci_methods;
pdrv->bsddriver.size = sizeof(struct pci_dev);

bus_topo_lock();
error = devclass_add_driver(dc, &pdrv->bsddriver,
BUS_PASS_DEFAULT, &pdrv->bsdclass);
bus_topo_unlock();
return (-error);
}

int
linux_pci_register_driver(struct pci_driver *pdrv)
{
devclass_t dc;

pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0;
dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci");
if (dc == NULL)
return (-ENXIO);
return (_linux_pci_register_driver(pdrv, dc));
}

static struct resource_list_entry *		static struct resource_list_entry *
lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve)		lkpi_pci_get_bar(struct pci_dev *pdev, int bar, bool reserve)
{		{
int type;		int type;

type = pci_resource_type(pdev, bar);		type = pci_resource_type(pdev, bar);
if (type < 0)		if (type < 0)
return (NULL);		return (NULL);
bar = PCIR_BAR(bar);		bar = PCIR_BAR(bar);
return (linux_pci_get_rle(pdev, type, bar, reserve));		return (lkpi_pci_get_rle(pdev, type, bar, reserve));
}		}

struct device *		struct device *
lkpi_pci_find_irq_dev(unsigned int irq)		lkpi_pci_find_irq_dev(unsigned int irq)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;
struct device *found;		struct device *found;

▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines
linuxkpi_pci_release_regions(struct pci_dev *pdev)		linuxkpi_pci_release_regions(struct pci_dev *pdev)
{		{
int i;		int i;

for (i = 0; i <= PCIR_MAX_BAR_0; i++)		for (i = 0; i <= PCIR_MAX_BAR_0; i++)
pci_release_region(pdev, i);		pci_release_region(pdev, i);
}		}

		/*
		* MSI(-X) functions.
		*/
int		int
linux_pci_register_drm_driver(struct pci_driver *pdrv)
{
devclass_t dc;

dc = devclass_create("vgapci");
if (dc == NULL)
return (-ENXIO);
pdrv->isdrm = true;
pdrv->name = "drmn";
return (_linux_pci_register_driver(pdrv, dc));
}

void
linux_pci_unregister_driver(struct pci_driver *pdrv)
{
devclass_t bus;

bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci");

LKPI_PCI_LOCK();
list_del(&pdrv->node);
LKPI_PCI_UNLOCK();
bus_topo_lock();
if (bus != NULL)
devclass_delete_driver(bus, &pdrv->bsddriver);
bus_topo_unlock();
}

void
linux_pci_unregister_drm_driver(struct pci_driver *pdrv)
{
devclass_t bus;

bus = devclass_find("vgapci");

LKPI_PCI_LOCK();
list_del(&pdrv->node);
LKPI_PCI_UNLOCK();
bus_topo_lock();
if (bus != NULL)
devclass_delete_driver(bus, &pdrv->bsddriver);
bus_topo_unlock();
}

int
linuxkpi_pci_enable_msix(struct pci_dev pdev, struct msix_entry entries,		linuxkpi_pci_enable_msix(struct pci_dev pdev, struct msix_entry entries,
int nreq)		int nreq)
{		{
struct resource_list_entry *rle;		struct resource_list_entry *rle;
int error;		int error;
int avail;		int avail;
int i;		int i;

Show All 9 Lines	linuxkpi_pci_enable_msix(struct pci_dev pdev, struct msix_entry entries,
/*		/*
* Handle case where "pci_alloc_msix()" may allocate less		* Handle case where "pci_alloc_msix()" may allocate less
* interrupts than available and return with no error:		* interrupts than available and return with no error:
*/		*/
if (avail < nreq) {		if (avail < nreq) {
pci_release_msi(pdev->dev.bsddev);		pci_release_msi(pdev->dev.bsddev);
return avail;		return avail;
}		}
rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);		rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);
pdev->dev.irq_start = rle->start;		pdev->dev.irq_start = rle->start;
pdev->dev.irq_end = rle->start + avail;		pdev->dev.irq_end = rle->start + avail;
for (i = 0; i < nreq; i++)		for (i = 0; i < nreq; i++)
entries[i].vector = pdev->dev.irq_start + i;		entries[i].vector = pdev->dev.irq_start + i;
pdev->msix_enabled = true;		pdev->msix_enabled = true;
return (0);		return (0);
}		}

Show All 16 Lines	if ((error = -pci_alloc_msi(pdev->dev.bsddev, &nvec)) != 0)
return error;		return error;

/* Native PCI might only ever ask for 32 vectors. */		/* Native PCI might only ever ask for 32 vectors. */
if (nvec < minvec) {		if (nvec < minvec) {
pci_release_msi(pdev->dev.bsddev);		pci_release_msi(pdev->dev.bsddev);
return (-ENOSPC);		return (-ENOSPC);
}		}

rle = linux_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);		rle = lkpi_pci_get_rle(pdev, SYS_RES_IRQ, 1, false);
pdev->dev.irq_start = rle->start;		pdev->dev.irq_start = rle->start;
pdev->dev.irq_end = rle->start + nvec;		pdev->dev.irq_end = rle->start + nvec;
pdev->irq = rle->start;		pdev->irq = rle->start;
pdev->msi_enabled = true;		pdev->msi_enabled = true;
return (0);		return (0);
}		}

int		int
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines
{		{
device_t dev;		device_t dev;

dev = pdev->dev.bsddev;		dev = pdev->dev.bsddev;

return (bus_child_present(dev));		return (bus_child_present(dev));
}		}

		/*
		* device_* bus functions for PCI.
		*/
		static int
		lkpi_pci_probe(device_t dev)
		{
		const struct pci_device_id *id;
		struct pci_driver *pdrv;

		if ((pdrv = linux_pci_find(dev, &id)) == NULL)
		return (ENXIO);
		if (device_get_driver(dev) != &pdrv->bsddriver)
		return (ENXIO);
		device_set_desc(dev, pdrv->name);

		/* Assume BSS initialized (should never return BUS_PROBE_SPECIFIC). */
		if (pdrv->bsd_probe_return == 0)
		return (BUS_PROBE_DEFAULT);
		else
		return (pdrv->bsd_probe_return);
		}

		static int
		lkpi_pci_attach(device_t dev)
		{
		const struct pci_device_id *id;
		struct pci_driver *pdrv;
		struct pci_dev *pdev;

		pdrv = linux_pci_find(dev, &id);
		pdev = device_get_softc(dev);

		MPASS(pdrv != NULL);
		MPASS(pdev != NULL);

		return (linux_pci_attach_device(dev, pdrv, id, pdev));
		}

		static int
		lkpi_pci_detach(device_t dev)
		{
		struct pci_dev *pdev;
		int error;

		pdev = device_get_softc(dev);
		MPASS(pdev != NULL);

		error = linux_pci_detach_device(pdev);
		if (error == 0)
		device_set_desc(dev, NULL);

		return (error);
		}

		static int
		lkpi_pci_suspend(device_t dev)
		{
		const struct dev_pm_ops *pmops;
		struct pm_message pm = { };
		struct pci_dev *pdev;
		int error;

		error = 0;
		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		pmops = pdev->pdrv->driver.pm;

		if (pdev->pdrv->suspend != NULL)
		error = -pdev->pdrv->suspend(pdev, pm);
		else if (pmops != NULL && pmops->suspend != NULL) {
		error = -pmops->suspend(&pdev->dev);
		if (error == 0 && pmops->suspend_late != NULL)
		error = -pmops->suspend_late(&pdev->dev);
		if (error == 0 && pmops->suspend_noirq != NULL)
		error = -pmops->suspend_noirq(&pdev->dev);
		}
		return (error);
		}

		static int
		lkpi_pci_resume(device_t dev)
		{
		const struct dev_pm_ops *pmops;
		struct pci_dev *pdev;
		int error;

		error = 0;
		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		pmops = pdev->pdrv->driver.pm;

		if (pdev->pdrv->resume != NULL)
		error = -pdev->pdrv->resume(pdev);
		else if (pmops != NULL && pmops->resume != NULL) {
		if (pmops->resume_early != NULL)
		error = -pmops->resume_early(&pdev->dev);
		if (error == 0 && pmops->resume != NULL)
		error = -pmops->resume(&pdev->dev);
		}
		return (error);
		}

		static int
		lkpi_pci_shutdown(device_t dev)
		{
		struct pci_dev *pdev;

		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		if (pdev->pdrv->shutdown != NULL)
		pdev->pdrv->shutdown(pdev);
		return (0);
		}

		/*
		* PCI IOV bus functions.
		*/
		static int
		lkpi_pci_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
		{
		struct pci_dev *pdev;
		int error;

		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		if (pdev->pdrv->bsd_iov_init != NULL)
		error = pdev->pdrv->bsd_iov_init(dev, num_vfs, pf_config);
		else
		error = EINVAL;
		return (error);
		}

static void		static void
		lkpi_pci_iov_uninit(device_t dev)
		{
		struct pci_dev *pdev;

		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		if (pdev->pdrv->bsd_iov_uninit != NULL)
		pdev->pdrv->bsd_iov_uninit(dev);
		}

		static int
		lkpi_pci_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
		{
		struct pci_dev *pdev;
		int error;

		linux_set_current(curthread);
		pdev = device_get_softc(dev);
		if (pdev->pdrv->bsd_iov_add_vf != NULL)
		error = pdev->pdrv->bsd_iov_add_vf(dev, vfnum, vf_config);
		else
		error = EINVAL;
		return (error);
		}

		/*
		* LinuxKPI driver (un)registration.
		*/
		static int
		lkpi_pci_register_driver(struct pci_driver *pdrv, devclass_t dc)
		{
		int error;

		linux_set_current(curthread);
		LKPI_PCI_LOCK();
		list_add(&pdrv->node, &lkpi_pci_drivers);
		LKPI_PCI_UNLOCK();
		if (pdrv->bsddriver.name == NULL)
		pdrv->bsddriver.name = pdrv->name;
		pdrv->bsddriver.methods = pci_methods;
		pdrv->bsddriver.size = sizeof(struct pci_dev);

		bus_topo_lock();
		error = devclass_add_driver(dc, &pdrv->bsddriver,
		BUS_PASS_DEFAULT, &pdrv->bsdclass);
		bus_topo_unlock();
		return (-error);
		}

		int
		linux_pci_register_driver(struct pci_driver *pdrv)
		{
		devclass_t dc;

		pdrv->isdrm = strcmp(pdrv->name, "drmn") == 0;
		dc = pdrv->isdrm ? devclass_create("vgapci") : devclass_find("pci");
		if (dc == NULL)
		return (-ENXIO);
		return (lkpi_pci_register_driver(pdrv, dc));
		}

		int
		linux_pci_register_drm_driver(struct pci_driver *pdrv)
		{
		devclass_t dc;

		dc = devclass_create("vgapci");
		if (dc == NULL)
		return (-ENXIO);
		pdrv->isdrm = true;
		pdrv->name = "drmn";
		return (lkpi_pci_register_driver(pdrv, dc));
		}

		void
		linux_pci_unregister_driver(struct pci_driver *pdrv)
		{
		devclass_t bus;

		bus = devclass_find(pdrv->isdrm ? "vgapci" : "pci");

		LKPI_PCI_LOCK();
		list_del(&pdrv->node);
		LKPI_PCI_UNLOCK();
		bus_topo_lock();
		if (bus != NULL)
		devclass_delete_driver(bus, &pdrv->bsddriver);
		bus_topo_unlock();
		}

		void
		linux_pci_unregister_drm_driver(struct pci_driver *pdrv)
		{
		devclass_t bus;

		bus = devclass_find("vgapci");

		LKPI_PCI_LOCK();
		list_del(&pdrv->node);
		LKPI_PCI_UNLOCK();
		bus_topo_lock();
		if (bus != NULL)
		devclass_delete_driver(bus, &pdrv->bsddriver);
		bus_topo_unlock();
		}

		/*
		* PCI SYSINIT.
		*/
		static void
lkpi_pci_init(void *arg __unused)		lkpi_pci_init(void *arg __unused)
{		{
INIT_LIST_HEAD(&lkpi_pci_drivers);		INIT_LIST_HEAD(&lkpi_pci_drivers);
INIT_LIST_HEAD(&lkpi_pci_devices);		INIT_LIST_HEAD(&lkpi_pci_devices);
LKPI_PCI_LOCK_INIT();		LKPI_PCI_LOCK_INIT();
}		}
SYSINIT(lkpi_pci_init, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_pci_init, NULL);		SYSINIT(lkpi_pci_init, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_pci_init, NULL);

static void		static void
lkpi_pci_uninit(void *arg __unused)		lkpi_pci_uninit(void *arg __unused)
{		{
LKPI_PCI_LOCK_DESTROY();		LKPI_PCI_LOCK_DESTROY();
}		}
SYSUNINIT(lkpi_pci_uninit, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_pci_uninit, NULL);		SYSUNINIT(lkpi_pci_uninit, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_pci_uninit, NULL);

/* -------------------------------------------------------------------------- */		/* -------------------------------------------------------------------------- */
/* DMA */		/* DMA */

		static counter_u64_t lkpi_pci_nseg1_fail;
		SYSCTL_COUNTER_U64(_compat_linuxkpi, OID_AUTO, lkpi_pci_nseg1_fail, CTLFLAG_RD,
		&lkpi_pci_nseg1_fail, "Count of busdma mapping failures of single-segment");

CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t));		CTASSERT(sizeof(dma_addr_t) <= sizeof(uint64_t));

struct linux_dma_obj {		struct lkpi_dma_obj {
void *vaddr;		void *vaddr;
uint64_t dma_addr;		uint64_t dma_addr;
bus_dmamap_t dmamap;		bus_dmamap_t dmamap;
bus_dma_tag_t dmat;		bus_dma_tag_t dmat;
};		};

static uma_zone_t linux_dma_trie_zone;		static uma_zone_t lkpi_dma_trie_zone;
static uma_zone_t linux_dma_obj_zone;		static uma_zone_t lkpi_dma_obj_zone;

static void		static void
linux_dma_init(void *arg)		lkpi_dma_init(void *arg)
{		{

linux_dma_trie_zone = uma_zcreate("linux_dma_pctrie",		lkpi_dma_trie_zone = uma_zcreate("lkpi_dma_pctrie",
pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL,		pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL,
UMA_ALIGN_PTR, 0);		UMA_ALIGN_PTR, 0);
linux_dma_obj_zone = uma_zcreate("linux_dma_object",		lkpi_dma_obj_zone = uma_zcreate("lkpi_dma_object",
sizeof(struct linux_dma_obj), NULL, NULL, NULL, NULL,		sizeof(struct lkpi_dma_obj), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);		UMA_ALIGN_PTR, 0);
lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK);		lkpi_pci_nseg1_fail = counter_u64_alloc(M_WAITOK);
}		}
SYSINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_init, NULL);		SYSINIT(lkpi_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, lkpi_dma_init, NULL);

static void		static void
linux_dma_uninit(void *arg)		lkpi_dma_uninit(void *arg)
{		{

counter_u64_free(lkpi_pci_nseg1_fail);		counter_u64_free(lkpi_pci_nseg1_fail);
uma_zdestroy(linux_dma_obj_zone);		uma_zdestroy(lkpi_dma_obj_zone);
uma_zdestroy(linux_dma_trie_zone);		uma_zdestroy(lkpi_dma_trie_zone);
}		}
SYSUNINIT(linux_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, linux_dma_uninit, NULL);		SYSUNINIT(lkpi_dma, SI_SUB_DRIVERS, SI_ORDER_THIRD, lkpi_dma_uninit, NULL);

static void *		static void *
linux_dma_trie_alloc(struct pctrie *ptree)		linux_dma_trie_alloc(struct pctrie *ptree)
{		{

return (uma_zalloc(linux_dma_trie_zone, M_NOWAIT));		return (uma_zalloc(lkpi_dma_trie_zone, M_NOWAIT));
}		}

static void		static void
linux_dma_trie_free(struct pctrie ptree, void node)		linux_dma_trie_free(struct pctrie ptree, void node)
{		{

uma_zfree(linux_dma_trie_zone, node);		uma_zfree(lkpi_dma_trie_zone, node);
}		}

PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc,		PCTRIE_DEFINE(LINUX_DMA, lkpi_dma_obj, dma_addr, linux_dma_trie_alloc,
linux_dma_trie_free);		linux_dma_trie_free);

#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__aarch64__)		#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__aarch64__)
static dma_addr_t		static dma_addr_t
linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len,		linux_dma_map_phys_common(struct device *dev, vm_paddr_t phys, size_t len,
bus_dma_tag_t dmat)		bus_dma_tag_t dmat)
{		{
struct linux_dma_priv *priv;		struct linux_dma_priv *priv;
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;
int error, nseg;		int error, nseg;
bus_dma_segment_t seg;		bus_dma_segment_t seg;

priv = dev->dma_priv;		priv = dev->dma_priv;

/*		/*
* If the resultant mapping will be entirely 1:1 with the		* If the resultant mapping will be entirely 1:1 with the
* physical address, short-circuit the remainder of the		* physical address, short-circuit the remainder of the
* bus_dma API. This avoids tracking collisions in the pctrie		* bus_dma API. This avoids tracking collisions in the pctrie
* with the additional benefit of reducing overhead.		* with the additional benefit of reducing overhead.
*/		*/
if (bus_dma_id_mapped(dmat, phys, len))		if (bus_dma_id_mapped(dmat, phys, len))
return (phys);		return (phys);

obj = uma_zalloc(linux_dma_obj_zone, M_NOWAIT);		obj = uma_zalloc(lkpi_dma_obj_zone, M_NOWAIT);
if (obj == NULL) {		if (obj == NULL) {
return (0);		return (0);
}		}
obj->dmat = dmat;		obj->dmat = dmat;

DMA_PRIV_LOCK(priv);		DMA_PRIV_LOCK(priv);
if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) {		if (bus_dmamap_create(obj->dmat, 0, &obj->dmamap) != 0) {
DMA_PRIV_UNLOCK(priv);		DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
return (0);		return (0);
}		}

nseg = -1;		nseg = -1;
error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len,		error = _bus_dmamap_load_phys(obj->dmat, obj->dmamap, phys, len,
BUS_DMA_NOWAIT, &seg, &nseg);		BUS_DMA_NOWAIT, &seg, &nseg);
if (error != 0) {		if (error != 0) {
bus_dmamap_destroy(obj->dmat, obj->dmamap);		bus_dmamap_destroy(obj->dmat, obj->dmamap);
DMA_PRIV_UNLOCK(priv);		DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
counter_u64_add(lkpi_pci_nseg1_fail, 1);		counter_u64_add(lkpi_pci_nseg1_fail, 1);
if (linuxkpi_debug) {		if (linuxkpi_debug) {
device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys "		device_printf(dev->bsddev, "%s: _bus_dmamap_load_phys "
"error %d, phys %#018jx len %zu\n", __func__,		"error %d, phys %#018jx len %zu\n", __func__,
error, (uintmax_t)phys, len);		error, (uintmax_t)phys, len);
dump_stack();		dump_stack();
}		}
return (0);		return (0);
}		}

KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));		KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
obj->dma_addr = seg.ds_addr;		obj->dma_addr = seg.ds_addr;

error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);		error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);
if (error != 0) {		if (error != 0) {
bus_dmamap_unload(obj->dmat, obj->dmamap);		bus_dmamap_unload(obj->dmat, obj->dmamap);
bus_dmamap_destroy(obj->dmat, obj->dmamap);		bus_dmamap_destroy(obj->dmat, obj->dmamap);
DMA_PRIV_UNLOCK(priv);		DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
return (0);		return (0);
}		}
DMA_PRIV_UNLOCK(priv);		DMA_PRIV_UNLOCK(priv);
return (obj->dma_addr);		return (obj->dma_addr);
}		}
#else		#else
static dma_addr_t		static dma_addr_t
linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys,		linux_dma_map_phys_common(struct device *dev __unused, vm_paddr_t phys,
Show All 29 Lines
}		}

#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__aarch64__)		#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__aarch64__)
void		void
lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,		lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,
enum dma_data_direction direction, unsigned long attrs)		enum dma_data_direction direction, unsigned long attrs)
{		{
struct linux_dma_priv *priv;		struct linux_dma_priv *priv;
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;

priv = dev->dma_priv;		priv = dev->dma_priv;

if (pctrie_is_empty(&priv->ptree))		if (pctrie_is_empty(&priv->ptree))
return;		return;

DMA_PRIV_LOCK(priv);		DMA_PRIV_LOCK(priv);
obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);		obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
Show All 22 Lines	default:
break;		break;
}		}

skip_sync:		skip_sync:
bus_dmamap_unload(obj->dmat, obj->dmamap);		bus_dmamap_unload(obj->dmat, obj->dmamap);
bus_dmamap_destroy(obj->dmat, obj->dmamap);		bus_dmamap_destroy(obj->dmat, obj->dmamap);
DMA_PRIV_UNLOCK(priv);		DMA_PRIV_UNLOCK(priv);

uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
}		}
#else		#else
void		void
lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,		lkpi_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len,
enum dma_data_direction direction, unsigned long attrs)		enum dma_data_direction direction, unsigned long attrs)
{		{
}		}
#endif		#endif
▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines	linuxkpi_dmam_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
return (dr->mem);		return (dr->mem);
}		}

void		void
linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size,		linuxkpi_dma_sync(struct device *dev, dma_addr_t dma_addr, size_t size,
bus_dmasync_op_t op)		bus_dmasync_op_t op)
{		{
struct linux_dma_priv *priv;		struct linux_dma_priv *priv;
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;

priv = dev->dma_priv;		priv = dev->dma_priv;

if (pctrie_is_empty(&priv->ptree))		if (pctrie_is_empty(&priv->ptree))
return;		return;

DMA_PRIV_LOCK(priv);		DMA_PRIV_LOCK(priv);
obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);		obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines
};		};

#define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock)		#define DMA_POOL_LOCK(pool) mtx_lock(&(pool)->pool_lock)
#define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock)		#define DMA_POOL_UNLOCK(pool) mtx_unlock(&(pool)->pool_lock)

static inline int		static inline int
dma_pool_obj_ctor(void mem, int size, void arg, int flags)		dma_pool_obj_ctor(void mem, int size, void arg, int flags)
{		{
struct linux_dma_obj *obj = mem;		struct lkpi_dma_obj *obj = mem;
struct dma_pool *pool = arg;		struct dma_pool *pool = arg;
int error, nseg;		int error, nseg;
bus_dma_segment_t seg;		bus_dma_segment_t seg;

nseg = -1;		nseg = -1;
DMA_POOL_LOCK(pool);		DMA_POOL_LOCK(pool);
error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap,		error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap,
vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT,		vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT,
&seg, &nseg);		&seg, &nseg);
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);
if (error != 0) {		if (error != 0) {
return (error);		return (error);
}		}
KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));		KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
obj->dma_addr = seg.ds_addr;		obj->dma_addr = seg.ds_addr;

return (0);		return (0);
}		}

static void		static void
dma_pool_obj_dtor(void mem, int size, void arg)		dma_pool_obj_dtor(void mem, int size, void arg)
{		{
struct linux_dma_obj *obj = mem;		struct lkpi_dma_obj *obj = mem;
struct dma_pool *pool = arg;		struct dma_pool *pool = arg;

DMA_POOL_LOCK(pool);		DMA_POOL_LOCK(pool);
bus_dmamap_unload(pool->pool_dmat, obj->dmamap);		bus_dmamap_unload(pool->pool_dmat, obj->dmamap);
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);
}		}

static int		static int
dma_pool_obj_import(void arg, void *store, int count, int domain __unused,		dma_pool_obj_import(void arg, void *store, int count, int domain __unused,
int flags)		int flags)
{		{
struct dma_pool *pool = arg;		struct dma_pool *pool = arg;
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;
int error, i;		int error, i;

for (i = 0; i < count; i++) {		for (i = 0; i < count; i++) {
obj = uma_zalloc(linux_dma_obj_zone, flags);		obj = uma_zalloc(lkpi_dma_obj_zone, flags);
if (obj == NULL)		if (obj == NULL)
break;		break;

error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr,		error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr,
BUS_DMA_NOWAIT, &obj->dmamap);		BUS_DMA_NOWAIT, &obj->dmamap);
if (error!= 0) {		if (error!= 0) {
uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
break;		break;
}		}

store[i] = obj;		store[i] = obj;
}		}

return (i);		return (i);
}		}

static void		static void
dma_pool_obj_release(void arg, void *store, int count)		dma_pool_obj_release(void arg, void *store, int count)
{		{
struct dma_pool *pool = arg;		struct dma_pool *pool = arg;
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;
int i;		int i;

for (i = 0; i < count; i++) {		for (i = 0; i < count; i++) {
obj = store[i];		obj = store[i];
bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap);		bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap);
uma_zfree(linux_dma_obj_zone, obj);		uma_zfree(lkpi_dma_obj_zone, obj);
}		}
}		}

struct dma_pool *		struct dma_pool *
linux_dma_pool_create(char name, struct device dev, size_t size,		linux_dma_pool_create(char name, struct device dev, size_t size,
size_t align, size_t boundary)		size_t align, size_t boundary)
{		{
struct linux_dma_priv *priv;		struct linux_dma_priv *priv;
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines	lkpi_dmam_pool_destroy(struct device dev, void p)
LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree);		LINUX_DMA_PCTRIE_RECLAIM(&pool->pool_ptree);
linux_dma_pool_destroy(pool);		linux_dma_pool_destroy(pool);
}		}

void *		void *
linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,		linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
dma_addr_t *handle)		dma_addr_t *handle)
{		{
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;

obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK);		obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags & GFP_NATIVE_MASK);
if (obj == NULL)		if (obj == NULL)
return (NULL);		return (NULL);

DMA_POOL_LOCK(pool);		DMA_POOL_LOCK(pool);
if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) {		if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) {
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);
uma_zfree_arg(pool->pool_zone, obj, pool);		uma_zfree_arg(pool->pool_zone, obj, pool);
return (NULL);		return (NULL);
}		}
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);

*handle = obj->dma_addr;		*handle = obj->dma_addr;
return (obj->vaddr);		return (obj->vaddr);
}		}

void		void
linux_dma_pool_free(struct dma_pool pool, void vaddr, dma_addr_t dma_addr)		linux_dma_pool_free(struct dma_pool pool, void vaddr, dma_addr_t dma_addr)
{		{
struct linux_dma_obj *obj;		struct lkpi_dma_obj *obj;

DMA_POOL_LOCK(pool);		DMA_POOL_LOCK(pool);
obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr);		obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr);
if (obj == NULL) {		if (obj == NULL) {
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);
return;		return;
}		}
LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr);		LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr);
DMA_POOL_UNLOCK(pool);		DMA_POOL_UNLOCK(pool);

uma_zfree_arg(pool->pool_zone, obj, pool);		uma_zfree_arg(pool->pool_zone, obj, pool);
}		}

/* -------------------------------------------------------------------------- */		/* -------------------------------------------------------------------------- */
/* Backlight. */		/* Backlight. */

static int		static int
linux_backlight_get_status(device_t dev, struct backlight_props *props)		lkpi_backlight_get_status(device_t dev, struct backlight_props *props)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;

linux_set_current(curthread);		linux_set_current(curthread);
pdev = device_get_softc(dev);		pdev = device_get_softc(dev);

props->brightness = pdev->dev.bd->props.brightness;		props->brightness = pdev->dev.bd->props.brightness;
props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness;		props->brightness = props->brightness * 100 / pdev->dev.bd->props.max_brightness;
props->nlevels = 0;		props->nlevels = 0;

return (0);		return (0);
}		}

static int		static int
linux_backlight_get_info(device_t dev, struct backlight_info *info)		lkpi_backlight_get_info(device_t dev, struct backlight_info *info)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;

linux_set_current(curthread);		linux_set_current(curthread);
pdev = device_get_softc(dev);		pdev = device_get_softc(dev);

info->type = BACKLIGHT_TYPE_PANEL;		info->type = BACKLIGHT_TYPE_PANEL;
strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH);		strlcpy(info->name, pdev->dev.bd->name, BACKLIGHTMAXNAMELENGTH);
return (0);		return (0);
}		}

static int		static int
linux_backlight_update_status(device_t dev, struct backlight_props *props)		lkpi_backlight_update_status(device_t dev, struct backlight_props *props)
{		{
struct pci_dev *pdev;		struct pci_dev *pdev;

linux_set_current(curthread);		linux_set_current(curthread);
pdev = device_get_softc(dev);		pdev = device_get_softc(dev);

pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness *		pdev->dev.bd->props.brightness = pdev->dev.bd->props.max_brightness *
props->brightness / 100;		props->brightness / 100;
Show All 25 Lines
void		void
linux_backlight_device_unregister(struct backlight_device *bd)		linux_backlight_device_unregister(struct backlight_device *bd)
{		{

backlight_destroy(bd->dev->backlight_dev);		backlight_destroy(bd->dev->backlight_dev);
free(bd->name, M_DEVBUF);		free(bd->name, M_DEVBUF);
free(bd, M_DEVBUF);		free(bd, M_DEVBUF);
}		}

		/* -------------------------------------------------------------------------- */
		/* Native bus attachment glue. */

		static device_method_t pci_methods[] = {
		DEVMETHOD(device_probe, lkpi_pci_probe),
		DEVMETHOD(device_attach, lkpi_pci_attach),
		DEVMETHOD(device_detach, lkpi_pci_detach),
		DEVMETHOD(device_suspend, lkpi_pci_suspend),
		DEVMETHOD(device_resume, lkpi_pci_resume),
		DEVMETHOD(device_shutdown, lkpi_pci_shutdown),

		DEVMETHOD(pci_iov_init, lkpi_pci_iov_init),
		DEVMETHOD(pci_iov_uninit, lkpi_pci_iov_uninit),
		DEVMETHOD(pci_iov_add_vf, lkpi_pci_iov_add_vf),

		/* Bus interface. */
		DEVMETHOD(bus_add_child, bus_generic_add_child),

		/* backlight interface */
		DEVMETHOD(backlight_update_status, lkpi_backlight_update_status),
		DEVMETHOD(backlight_get_status, lkpi_backlight_get_status),
		DEVMETHOD(backlight_get_info, lkpi_backlight_get_info),
		DEVMETHOD_END
		};