Page MenuHomeFreeBSD
Differential D20097 Diff 56811 sys/compat/linuxkpi/common/src/linux_pci.c

Changeset View

Standalone View

View Options

sys/compat/linuxkpi/common/src/linux_pci.c

Show First 20 Lines • Show All 72 Lines • ▼ Show 20 Linesstatic device_method_t pci_methods[] = {
DEVMETHOD(device_suspend, linux_pci_suspend), DEVMETHOD(device_suspend, linux_pci_suspend),
DEVMETHOD(device_resume, linux_pci_resume), DEVMETHOD(device_resume, linux_pci_resume),
DEVMETHOD(device_shutdown, linux_pci_shutdown), DEVMETHOD(device_shutdown, linux_pci_shutdown),
DEVMETHOD_END DEVMETHOD_END
}; };
struct linux_dma_priv { struct linux_dma_priv {
uint64_t dma_mask; uint64_t dma_mask;
struct mtx dma_lock; struct mtx lock;
bus_dma_tag_t dmat; bus_dma_tag_t dmat;
struct mtx ptree_lock;
struct pctrie ptree; struct pctrie ptree;
}; };
#define DMA_PRIV_LOCK(priv) mtx_lock(&(priv)->lock)
#define DMA_PRIV_UNLOCK(priv) mtx_unlock(&(priv)->lock)
static int static int
linux_pdev_dma_init(struct pci_dev *pdev) linux_pdev_dma_init(struct pci_dev *pdev)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO); priv = malloc(sizeof(*priv), M_DEVBUF, M_WAITOK | M_ZERO);
pdev->dev.dma_priv = priv; pdev->dev.dma_priv = priv;
mtx_init(&priv->dma_lock, "linux_dma", NULL, MTX_DEF); mtx_init(&priv->lock, "lkpi-priv-dma", NULL, MTX_DEF);
mtx_init(&priv->ptree_lock, "linux_dma_ptree", NULL, MTX_DEF);
pctrie_init(&priv->ptree); pctrie_init(&priv->ptree);
return (0); return (0);
} }
static int static int
linux_pdev_dma_uninit(struct pci_dev *pdev) linux_pdev_dma_uninit(struct pci_dev *pdev)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
priv = pdev->dev.dma_priv; priv = pdev->dev.dma_priv;
if (priv->dmat) if (priv->dmat)
bus_dma_tag_destroy(priv->dmat); bus_dma_tag_destroy(priv->dmat);
mtx_destroy(&priv->dma_lock); mtx_destroy(&priv->lock);
mtx_destroy(&priv->ptree_lock);
free(priv, M_DEVBUF); free(priv, M_DEVBUF);
pdev->dev.dma_priv = NULL; pdev->dev.dma_priv = NULL;
return (0); return (0);
} }
int int
linux_dma_tag_init(struct device *dev, u64 dma_mask) linux_dma_tag_init(struct device *dev, u64 dma_mask)
{ {
▲ Show 20 Lines • Show All 328 Lines • ▼ Show 20 Lines
static void static void
linux_dma_trie_free(struct pctrie *ptree, void *node) linux_dma_trie_free(struct pctrie *ptree, void *node)
{ {
uma_zfree(linux_dma_trie_zone, node); uma_zfree(linux_dma_trie_zone, node);
} }
PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc, PCTRIE_DEFINE(LINUX_DMA, linux_dma_obj, dma_addr, linux_dma_trie_alloc,
linux_dma_trie_free); linux_dma_trie_free);
void * void *
linux_dma_alloc_coherent(struct device *dev, size_t size, linux_dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag) dma_addr_t *dma_handle, gfp_t flag)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
Show All 10 Lines if (priv->dma_mask)
high = priv->dma_mask; high = priv->dma_mask;
else if (flag & GFP_DMA32) else if (flag & GFP_DMA32)
high = BUS_SPACE_MAXADDR_32BIT; high = BUS_SPACE_MAXADDR_32BIT;
else else
high = BUS_SPACE_MAXADDR; high = BUS_SPACE_MAXADDR;
align = PAGE_SIZE << get_order(size); align = PAGE_SIZE << get_order(size);
mem = (void *)kmem_alloc_contig(size, flag, 0, high, align, 0, mem = (void *)kmem_alloc_contig(size, flag, 0, high, align, 0,
VM_MEMATTR_DEFAULT); VM_MEMATTR_DEFAULT);
if (mem) if (mem)
kibUnsubmitted
Done
Inline Actions

mem == NULL

kib: mem == NULL
*dma_handle = linux_dma_map_phys(dev, vtophys(mem), size); *dma_handle = linux_dma_map_phys(dev, vtophys(mem), size);
else else
*dma_handle = 0; *dma_handle = 0;
return (mem); return (mem);
} }
dma_addr_t dma_addr_t
linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len) linux_dma_map_phys(struct device *dev, vm_paddr_t phys, size_t len)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
int error, nseg; struct linux_dma_obj *old;
kibUnsubmitted
Done
Inline Actions

Why not write

struct linux_dma_obj *obj, *old;
kib: Why not write ``` struct linux_dma_obj *obj, *old; ```
hselaskyAuthorUnsubmitted
Done
Inline Actions

I prefer one declaration per line.

hselasky: I prefer one declaration per line.
bus_dma_segment_t seg; bus_dma_segment_t seg;
dma_addr_t retval;
int error;
int nseg;
priv = dev->dma_priv; priv = dev->dma_priv;
obj = uma_zalloc(linux_dma_obj_zone, 0); obj = uma_zalloc(linux_dma_obj_zone, 0);
if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0)
uma_zfree(linux_dma_obj_zone, obj); goto out_error;
return (0);
}
nseg = -1; nseg = -1;
mtx_lock(&priv->dma_lock); DMA_PRIV_LOCK(priv);
if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len, error = _bus_dmamap_load_phys(priv->dmat, obj->dmamap, phys, len,
BUS_DMA_NOWAIT, &seg, &nseg) != 0) { BUS_DMA_NOWAIT, &seg, &nseg);
if (error != 0) {
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock); DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj); goto out_error;
return (0);
} }
mtx_unlock(&priv->dma_lock);
KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg)); KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1));
obj->dma_addr = seg.ds_addr; obj->dma_addr = seg.ds_addr;
mtx_lock(&priv->ptree_lock); /* allow new map on same location w/o unmap */
old = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, obj->dma_addr);
kibUnsubmitted
Not Done
Inline Actions

What if new and old requests diff by length ? You should keep around the one with greater size.

Also, wouldn't dma_unmap() from one request kill the pctrie entry for another one ? I believe you need some sort of refcount.

kib: What if new and old requests diff by length ? You should keep around the one with greater size.
slavashUnsubmitted
Not Done
Inline Actions

What if new and old requests diff by length ? You should keep around the one with greater size.

But now if we remove the one with the greater size?

slavash: > What if new and old requests diff by length ? You should keep around the one with greater…
kibUnsubmitted
Not Done
Inline Actions

Compat layer should only keep the largest mapping, refcounted. The mapping only removed when the last reference goes away.

kib: Compat layer should only keep the largest mapping, refcounted. The mapping only removed when…
if (old != NULL) {
kibUnsubmitted
Done
Inline Actions

You still do not compare the length of old and new requests.

You still do not ref-count the duplicated entry.

kib: You still do not compare the length of old and new requests. You still do not ref-count the…
hselaskyAuthorUnsubmitted
Done
Inline Actions

Fixed in next version.

hselasky: Fixed in next version.
LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, obj->dma_addr);
bus_dmamap_unload(priv->dmat, old->dmamap);
kibUnsubmitted
Done
Inline Actions

old->dma_refcount == 0 should be an assert.

kib: old->dma_refcount == 0 should be an assert.
bus_dmamap_destroy(priv->dmat, old->dmamap);
}
error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);
mtx_unlock(&priv->ptree_lock);
if (error != 0) { if (error != 0) {
mtx_lock(&priv->dma_lock);
bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_unload(priv->dmat, obj->dmamap);
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock); retval = 0;
} else {
retval = obj->dma_addr;
}
DMA_PRIV_UNLOCK(priv);
if (old != NULL)
uma_zfree(linux_dma_obj_zone, old);
if (error == 0)
return (retval);
out_error:
uma_zfree(linux_dma_obj_zone, obj); uma_zfree(linux_dma_obj_zone, obj);
return (0); return (0);
} }
return (obj->dma_addr);
}
void void
linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len) linux_dma_unmap(struct device *dev, dma_addr_t dma_addr, size_t len)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
priv = dev->dma_priv; priv = dev->dma_priv;
mtx_lock(&priv->ptree_lock); DMA_PRIV_LOCK(priv);
obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr); obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, dma_addr);
if (obj == NULL) { if (obj != NULL) {
mtx_unlock(&priv->ptree_lock);
return;
}
LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr); LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, dma_addr);
kibUnsubmitted
Done
Inline Actions

Assert that dma_refcount >= 1. Explicitly compare with 0.

kib: Assert that dma_refcount >= 1. Explicitly compare with 0.
hselaskyAuthorUnsubmitted
Done
Inline Actions

Refcounts are unsigned and start at zero, so asserting doesn't make sense.

I've renamed the variable to dma_share_count to make this clearer.

hselasky: Refcounts are unsigned and start at zero, so asserting doesn't make sense. I've renamed the…
mtx_unlock(&priv->ptree_lock);
mtx_lock(&priv->dma_lock);
bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_unload(priv->dmat, obj->dmamap);
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock); }
DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj); uma_zfree(linux_dma_obj_zone, obj);
} }
int int
linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents, linux_dma_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nents,
enum dma_data_direction dir, struct dma_attrs *attrs) enum dma_data_direction dir, struct dma_attrs *attrs)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
struct linux_dma_obj *old;
struct scatterlist *dma_sg, *sg; struct scatterlist *dma_sg, *sg;
int dma_nents, error, nseg; int dma_nents, error, nseg;
size_t seg_len; size_t seg_len;
vm_paddr_t seg_phys, prev_phys_end; vm_paddr_t seg_phys, prev_phys_end;
bus_dma_segment_t seg; bus_dma_segment_t seg;
priv = dev->dma_priv; priv = dev->dma_priv;
obj = uma_zalloc(linux_dma_obj_zone, 0); obj = uma_zalloc(linux_dma_obj_zone, 0);
if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0) { if (bus_dmamap_create(priv->dmat, 0, &obj->dmamap) != 0)
kibUnsubmitted
Done
Inline Actions

I would be happier if we put some canary value for unallocated dma_map, and assert that we only call bus_dmamap_unload/destroy on proper dma_map.

kib: I would be happier if we put some canary value for unallocated dma_map, and assert that we only…
hselaskyAuthorUnsubmitted
Done
Inline Actions

I'm not sure if we can cover all the cases for initializing S/G lists.

hselasky: I'm not sure if we can cover all the cases for initializing S/G lists.
uma_zfree(linux_dma_obj_zone, obj); goto out_error;
return (0);
}
sg = sgl; sg = sgl;
dma_sg = sg; dma_sg = sg;
dma_nents = 0; dma_nents = 0;
DMA_PRIV_LOCK(priv);
while (nents > 0) { while (nents > 0) {
seg_phys = sg_phys(sg); seg_phys = sg_phys(sg);
seg_len = sg->length; seg_len = sg->length;
while (--nents > 0) { while (--nents > 0) {
prev_phys_end = sg_phys(sg) + sg->length; prev_phys_end = sg_phys(sg) + sg->length;
sg = sg_next(sg); sg = sg_next(sg);
if (prev_phys_end != sg_phys(sg)) if (prev_phys_end != sg_phys(sg))
break; break;
seg_len += sg->length; seg_len += sg->length;
} }
nseg = -1; nseg = -1;
mtx_lock(&priv->dma_lock);
if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap, if (_bus_dmamap_load_phys(priv->dmat, obj->dmamap,
seg_phys, seg_len, BUS_DMA_NOWAIT, seg_phys, seg_len, BUS_DMA_NOWAIT,
&seg, &nseg) != 0) { &seg, &nseg) != 0) {
bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_unload(priv->dmat, obj->dmamap);
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock); DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj); goto out_error;
return (0);
} }
mtx_unlock(&priv->dma_lock);
KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1));
sg_dma_address(dma_sg) = seg.ds_addr; sg_dma_address(dma_sg) = seg.ds_addr;
sg_dma_len(dma_sg) = seg.ds_len; sg_dma_len(dma_sg) = seg.ds_len;
dma_sg = sg_next(dma_sg); dma_sg = sg_next(dma_sg);
dma_nents++; dma_nents++;
} }
obj->dma_addr = sg_dma_address(sgl); obj->dma_addr = sg_dma_address(sgl);
mtx_lock(&priv->ptree_lock); old = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, obj->dma_addr);
if (old != NULL) {
LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, obj->dma_addr);
bus_dmamap_unload(priv->dmat, old->dmamap);
bus_dmamap_destroy(priv->dmat, old->dmamap);
}
error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj); error = LINUX_DMA_PCTRIE_INSERT(&priv->ptree, obj);
mtx_unlock(&priv->ptree_lock);
if (error != 0) { if (error != 0) {
mtx_lock(&priv->dma_lock);
bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_unload(priv->dmat, obj->dmamap);
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock);
uma_zfree(linux_dma_obj_zone, obj);
return (0);
} }
DMA_PRIV_UNLOCK(priv);
if (old != NULL)
uma_zfree(linux_dma_obj_zone, old);
if (error == 0)
return (dma_nents); return (dma_nents);
out_error:
uma_zfree(linux_dma_obj_zone, obj);
return (0);
} }
void void
linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, linux_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
int nents, enum dma_data_direction dir, struct dma_attrs *attrs) int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
priv = dev->dma_priv; priv = dev->dma_priv;
mtx_lock(&priv->ptree_lock); DMA_PRIV_LOCK(priv);
obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, sg_dma_address(sgl)); obj = LINUX_DMA_PCTRIE_LOOKUP(&priv->ptree, sg_dma_address(sgl));
if (obj == NULL) { if (obj == NULL) {
mtx_unlock(&priv->ptree_lock); DMA_PRIV_UNLOCK(priv);
return; return;
} }
LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, sg_dma_address(sgl)); LINUX_DMA_PCTRIE_REMOVE(&priv->ptree, sg_dma_address(sgl));
mtx_unlock(&priv->ptree_lock);
mtx_lock(&priv->dma_lock);
bus_dmamap_unload(priv->dmat, obj->dmamap); bus_dmamap_unload(priv->dmat, obj->dmamap);
bus_dmamap_destroy(priv->dmat, obj->dmamap); bus_dmamap_destroy(priv->dmat, obj->dmamap);
mtx_unlock(&priv->dma_lock); DMA_PRIV_UNLOCK(priv);
uma_zfree(linux_dma_obj_zone, obj); uma_zfree(linux_dma_obj_zone, obj);
} }
static inline int static inline int
dma_pool_obj_ctor(void *mem, int size, void *arg, int flags) dma_pool_obj_ctor(void *mem, int size, void *arg, int flags)
{ {
struct linux_dma_obj *obj = mem; struct linux_dma_obj *obj = mem;
struct dma_pool *pool = arg; struct dma_pool *pool = arg;
int error, nseg; int error, nseg;
bus_dma_segment_t seg; bus_dma_segment_t seg;
nseg = -1; nseg = -1;
mtx_lock(&pool->pool_dma_lock); DMA_POOL_LOCK(pool);
error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap, error = _bus_dmamap_load_phys(pool->pool_dmat, obj->dmamap,
vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT, vtophys(obj->vaddr), pool->pool_entry_size, BUS_DMA_NOWAIT,
&seg, &nseg); &seg, &nseg);
mtx_unlock(&pool->pool_dma_lock); DMA_POOL_UNLOCK(pool);
if (error != 0) { if (error != 0) {
return (error); return (error);
} }
KASSERT(++nseg == 1, ("More than one segment (nseg=%d)", nseg));
KASSERT(nseg == 0, ("More than one segment (nseg=%d)", nseg + 1));
obj->dma_addr = seg.ds_addr; obj->dma_addr = seg.ds_addr;
return (0); return (0);
} }
static void static void
dma_pool_obj_dtor(void *mem, int size, void *arg) dma_pool_obj_dtor(void *mem, int size, void *arg)
{ {
struct linux_dma_obj *obj = mem; struct linux_dma_obj *obj = mem;
struct dma_pool *pool = arg; struct dma_pool *pool = arg;
mtx_lock(&pool->pool_dma_lock); DMA_POOL_LOCK(pool);
bus_dmamap_unload(pool->pool_dmat, obj->dmamap); bus_dmamap_unload(pool->pool_dmat, obj->dmamap);
mtx_unlock(&pool->pool_dma_lock); DMA_POOL_UNLOCK(pool);
} }
static int static int
dma_pool_obj_import(void *arg, void **store, int count, int domain __unused, dma_pool_obj_import(void *arg, void **store, int count, int domain __unused,
int flags) int flags)
{ {
struct dma_pool *pool = arg; struct dma_pool *pool = arg;
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
int error, i; int error, i;
priv = pool->pool_pdev->dev.dma_priv; priv = pool->pool_device->dma_priv;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
obj = uma_zalloc(linux_dma_obj_zone, flags); obj = uma_zalloc(linux_dma_obj_zone, flags);
if (obj == NULL) if (obj == NULL)
break; break;
error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr, error = bus_dmamem_alloc(pool->pool_dmat, &obj->vaddr,
BUS_DMA_NOWAIT, &obj->dmamap); BUS_DMA_NOWAIT, &obj->dmamap);
if (error!= 0) { if (error != 0) {
uma_zfree(linux_dma_obj_zone, obj); uma_zfree(linux_dma_obj_zone, obj);
break; break;
} }
store[i] = obj; store[i] = obj;
} }
return (i); return (i);
} }
static void static void
dma_pool_obj_release(void *arg, void **store, int count) dma_pool_obj_release(void *arg, void **store, int count)
{ {
struct dma_pool *pool = arg; struct dma_pool *pool = arg;
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
int i; int i;
priv = pool->pool_pdev->dev.dma_priv; priv = pool->pool_device->dma_priv;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
obj = store[i]; obj = store[i];
bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap); bus_dmamem_free(pool->pool_dmat, obj->vaddr, obj->dmamap);
uma_zfree(linux_dma_obj_zone, obj); uma_zfree(linux_dma_obj_zone, obj);
} }
} }
struct dma_pool * struct dma_pool *
linux_dma_pool_create(char *name, struct device *dev, size_t size, linux_dma_pool_create(char *name, struct device *dev, size_t size,
size_t align, size_t boundary) size_t align, size_t boundary)
{ {
struct linux_dma_priv *priv; struct linux_dma_priv *priv;
struct dma_pool *pool; struct dma_pool *pool;
priv = dev->dma_priv; priv = dev->dma_priv;
pool = kzalloc(sizeof(*pool), GFP_KERNEL); pool = kzalloc(sizeof(*pool), GFP_KERNEL);
pool->pool_pdev = to_pci_dev(dev); pool->pool_device = dev;
pool->pool_entry_size = size; pool->pool_entry_size = size;
if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev), if (bus_dma_tag_create(bus_get_dma_tag(dev->bsddev),
align, boundary, /* alignment, boundary */ align, boundary, /* alignment, boundary */
priv->dma_mask, /* lowaddr */ priv->dma_mask, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filtfunc, filtfuncarg */ NULL, NULL, /* filtfunc, filtfuncarg */
size, /* maxsize */ size, /* maxsize */
1, /* nsegments */ 1, /* nsegments */
size, /* maxsegsz */ size, /* maxsegsz */
0, /* flags */ 0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */ NULL, NULL, /* lockfunc, lockfuncarg */
&pool->pool_dmat)) { &pool->pool_dmat)) {
kfree(pool); kfree(pool);
return (NULL); return (NULL);
} }
pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor, pool->pool_zone = uma_zcache_create(name, -1, dma_pool_obj_ctor,
dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import, dma_pool_obj_dtor, NULL, NULL, dma_pool_obj_import,
dma_pool_obj_release, pool, 0); dma_pool_obj_release, pool, 0);
mtx_init(&pool->pool_dma_lock, "linux_dma_pool", NULL, MTX_DEF); mtx_init(&pool->pool_lock, "lkpi-dma-pool", NULL, MTX_DEF);
mtx_init(&pool->pool_ptree_lock, "linux_dma_pool_ptree", NULL,
MTX_DEF);
pctrie_init(&pool->pool_ptree); pctrie_init(&pool->pool_ptree);
return (pool); return (pool);
} }
void void
linux_dma_pool_destroy(struct dma_pool *pool) linux_dma_pool_destroy(struct dma_pool *pool)
{ {
uma_zdestroy(pool->pool_zone); uma_zdestroy(pool->pool_zone);
bus_dma_tag_destroy(pool->pool_dmat); bus_dma_tag_destroy(pool->pool_dmat);
mtx_destroy(&pool->pool_ptree_lock); mtx_destroy(&pool->pool_lock);
mtx_destroy(&pool->pool_dma_lock);
kfree(pool); kfree(pool);
} }
void * void *
linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, linux_dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
dma_addr_t *handle) dma_addr_t *handle)
{ {
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags); obj = uma_zalloc_arg(pool->pool_zone, pool, mem_flags);
if (obj == NULL) if (obj == NULL)
return (NULL); return (NULL);
mtx_lock(&pool->pool_ptree_lock); DMA_POOL_LOCK(pool);
if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) { if (LINUX_DMA_PCTRIE_INSERT(&pool->pool_ptree, obj) != 0) {
mtx_unlock(&pool->pool_ptree_lock); DMA_POOL_UNLOCK(pool);
uma_zfree_arg(pool->pool_zone, obj, pool); uma_zfree_arg(pool->pool_zone, obj, pool);
return (NULL); return (NULL);
} }
mtx_unlock(&pool->pool_ptree_lock); DMA_POOL_UNLOCK(pool);
*handle = obj->dma_addr; *handle = obj->dma_addr;
return (obj->vaddr); return (obj->vaddr);
} }
void void
linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr) linux_dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma_addr)
{ {
struct linux_dma_obj *obj; struct linux_dma_obj *obj;
mtx_lock(&pool->pool_ptree_lock); DMA_POOL_LOCK(pool);
obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr); obj = LINUX_DMA_PCTRIE_LOOKUP(&pool->pool_ptree, dma_addr);
if (obj == NULL) { if (obj == NULL) {
mtx_unlock(&pool->pool_ptree_lock); DMA_POOL_UNLOCK(pool);
return; return;
} }
LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr); LINUX_DMA_PCTRIE_REMOVE(&pool->pool_ptree, dma_addr);
mtx_unlock(&pool->pool_ptree_lock); DMA_POOL_UNLOCK(pool);
uma_zfree_arg(pool->pool_zone, obj, pool); uma_zfree_arg(pool->pool_zone, obj, pool);
} }