Diff 20186

sys/conf/files

sys/modules/mlx4/Makefile

	# $FreeBSD$			# $FreeBSD$
	.PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4			.PATH: ${.CURDIR}/../../ofed/drivers/net/mlx4

	KMOD= mlx4			KMOD= mlx4
	SRCS= device_if.h bus_if.h vnode_if.h pci_if.h \			SRCS= device_if.h bus_if.h vnode_if.h pci_if.h \
	opt_inet.h opt_inet6.h \			opt_inet.h opt_inet6.h \
	alloc.c catas.c cmd.c cq.c eq.c \			alloc.c catas.c cmd.c cq.c eq.c \
	fw.c icm.c intf.c main.c mcg.c mr.c \			fw.c fw_qos.c icm.c intf.c main.c mcg.c mr.c \
	pd.c port.c profile.c qp.c reset.c sense.c \			pd.c port.c profile.c qp.c reset.c sense.c \
	srq.c resource_tracker.c sys_tune.c			srq.c resource_tracker.c

	CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4			CFLAGS+= -I${.CURDIR}/../../ofed/drivers/net/mlx4
	CFLAGS+= -I${.CURDIR}/../../ofed/include			CFLAGS+= -I${.CURDIR}/../../ofed/include
	CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include			CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include

	.include <bsd.kmod.mk>			.include <bsd.kmod.mk>

	CFLAGS+= -Wno-cast-qual -Wno-pointer-arith			CFLAGS+= -Wno-cast-qual -Wno-pointer-arith

	CWARNFLAGS.mcg.c= -Wno-unused			CWARNFLAGS.mcg.c= -Wno-unused

View Options

sys/ofed/drivers/net/mlx4/alloc.c

Show First 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
u32 obj;		u32 obj;

if (likely(cnt == 1 && align == 1 && !skip_mask))		if (likely(cnt == 1 && align == 1 && !skip_mask))
return mlx4_bitmap_alloc(bitmap);		return mlx4_bitmap_alloc(bitmap);

spin_lock(&bitmap->lock);		spin_lock(&bitmap->lock);

obj = find_aligned_range(bitmap->table, bitmap->last,		obj = find_aligned_range(bitmap->table, bitmap->last,
bitmap->max, cnt, align, skip_mask);		bitmap->max, cnt, align, skip_mask);
if (obj >= bitmap->max) {		if (obj >= bitmap->max) {
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;		& bitmap->mask;
obj = find_aligned_range(bitmap->table, 0, bitmap->max,		obj = find_aligned_range(bitmap->table, 0, bitmap->max,
cnt, align, skip_mask);		cnt, align, skip_mask);
}		}

if (obj < bitmap->max) {		if (obj < bitmap->max) {
bitmap_set(bitmap->table, obj, cnt);		bitmap_set(bitmap->table, obj, cnt);
if (obj == bitmap->last) {		if (obj == bitmap->last) {
bitmap->last = (obj + cnt);		bitmap->last = (obj + cnt);
if (bitmap->last >= bitmap->max)		if (bitmap->last >= bitmap->max)
bitmap->last = 0;		bitmap->last = 0;
Show All 10 Lines	u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
return obj;		return obj;
}		}

u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)		u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)
{		{
return bitmap->avail;		return bitmap->avail;
}		}

		static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj)
		{
		return obj & (bitmap->max + bitmap->reserved_top - 1);
		}

void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,		void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,
int use_rr)		int use_rr)
{		{
obj &= bitmap->max + bitmap->reserved_top - 1;		obj &= bitmap->max + bitmap->reserved_top - 1;

spin_lock(&bitmap->lock);		spin_lock(&bitmap->lock);
if (!use_rr) {		if (!use_rr) {
bitmap->last = min(bitmap->last, obj);		bitmap->last = min(bitmap->last, obj);
bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
& bitmap->mask;		& bitmap->mask;
}		}
bitmap_clear(bitmap->table, obj, cnt);		bitmap_clear(bitmap->table, obj, cnt);
bitmap->avail += cnt;		bitmap->avail += cnt;
spin_unlock(&bitmap->lock);		spin_unlock(&bitmap->lock);
}		}

int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,		int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
u32 reserved_bot, u32 reserved_top)		u32 reserved_bot, u32 reserved_top)
{		{
/* sanity check */
if (num <= (u64)reserved_top + reserved_bot)
return -EINVAL;

/* num must be a power of 2 */		/* num must be a power of 2 */
if (num != roundup_pow_of_two(num))		if (num != roundup_pow_of_two(num))
return -EINVAL;		return -EINVAL;

if (reserved_bot + reserved_top >= num)
return -EINVAL;

bitmap->last = 0;		bitmap->last = 0;
bitmap->top = 0;		bitmap->top = 0;
bitmap->max = num - reserved_top;		bitmap->max = num - reserved_top;
bitmap->mask = mask;		bitmap->mask = mask;
bitmap->reserved_top = reserved_top;		bitmap->reserved_top = reserved_top;
bitmap->avail = num - reserved_top - reserved_bot;		bitmap->avail = num - reserved_top - reserved_bot;
		bitmap->effective_len = bitmap->avail;
spin_lock_init(&bitmap->lock);		spin_lock_init(&bitmap->lock);
bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *		bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
sizeof (long), GFP_KERNEL);		sizeof (long), GFP_KERNEL);
if (!bitmap->table)		if (!bitmap->table)
return -ENOMEM;		return -ENOMEM;

bitmap_set(bitmap->table, 0, reserved_bot);		bitmap_set(bitmap->table, 0, reserved_bot);

return 0;		return 0;
}		}

void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)		void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
{		{
kfree(bitmap->table);		kfree(bitmap->table);
}		}

		struct mlx4_zone_allocator {
		struct list_head entries;
		struct list_head prios;
		u32 last_uid;
		u32 mask;
		/* protect the zone_allocator from concurrent accesses */
		spinlock_t lock;
		enum mlx4_zone_alloc_flags flags;
		};

		struct mlx4_zone_entry {
		struct list_head list;
		struct list_head prio_list;
		u32 uid;
		struct mlx4_zone_allocator *allocator;
		struct mlx4_bitmap *bitmap;
		int use_rr;
		int priority;
		int offset;
		enum mlx4_zone_flags flags;
		};

		struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags)
		{
		struct mlx4_zone_allocator zones = kmalloc(sizeof(zones), GFP_KERNEL);

		if (NULL == zones)
		return NULL;

		INIT_LIST_HEAD(&zones->entries);
		INIT_LIST_HEAD(&zones->prios);
		spin_lock_init(&zones->lock);
		zones->last_uid = 0;
		zones->mask = 0;
		zones->flags = flags;

		return zones;
		}

		int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc,
		struct mlx4_bitmap *bitmap,
		u32 flags,
		int priority,
		int offset,
		u32 *puid)
		{
		u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1);
		struct mlx4_zone_entry *it;
		struct mlx4_zone_entry zone = kmalloc(sizeof(zone), GFP_KERNEL);

		if (NULL == zone)
		return -ENOMEM;

		zone->flags = flags;
		zone->bitmap = bitmap;
		zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0;
		zone->priority = priority;
		zone->offset = offset;

		spin_lock(&zone_alloc->lock);

		zone->uid = zone_alloc->last_uid++;
		zone->allocator = zone_alloc;

		if (zone_alloc->mask < mask)
		zone_alloc->mask = mask;

		list_for_each_entry(it, &zone_alloc->prios, prio_list)
		if (it->priority >= priority)
		break;

		if (&it->prio_list == &zone_alloc->prios \|\| it->priority > priority)
		list_add_tail(&zone->prio_list, &it->prio_list);
		list_add_tail(&zone->list, &it->list);

		spin_unlock(&zone_alloc->lock);

		*puid = zone->uid;

		return 0;
		}

		/* Should be called under a lock */
		static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry)
		{
		struct mlx4_zone_allocator *zone_alloc = entry->allocator;

		if (!list_empty(&entry->prio_list)) {
		/* Check if we need to add an alternative node to the prio list */
		if (!list_is_last(&entry->list, &zone_alloc->entries)) {
		struct mlx4_zone_entry *next = list_first_entry(&entry->list,
		typeof(*next),
		list);

		if (next->priority == entry->priority)
		list_add_tail(&next->prio_list, &entry->prio_list);
		}

		list_del(&entry->prio_list);
		}

		list_del(&entry->list);

		if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) {
		u32 mask = 0;
		struct mlx4_zone_entry *it;

		list_for_each_entry(it, &zone_alloc->prios, prio_list) {
		u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1);

		if (mask < cur_mask)
		mask = cur_mask;
		}
		zone_alloc->mask = mask;
		}

		return 0;
		}

		void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc)
		{
		struct mlx4_zone_entry zone, tmp;

		spin_lock(&zone_alloc->lock);

		list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) {
		list_del(&zone->list);
		list_del(&zone->prio_list);
		kfree(zone);
		}

		spin_unlock(&zone_alloc->lock);
		kfree(zone_alloc);
		}

		/* Should be called under a lock */
		static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count,
		int align, u32 skip_mask, u32 *puid)
		{
		u32 uid = 0;
		u32 res;
		struct mlx4_zone_allocator *zone_alloc = zone->allocator;
		struct mlx4_zone_entry *curr_node;

		res = mlx4_bitmap_alloc_range(zone->bitmap, count,
		align, skip_mask);

		if (res != (u32)-1) {
		res += zone->offset;
		uid = zone->uid;
		goto out;
		}

		list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) {
		if (unlikely(curr_node->priority == zone->priority))
		break;
		}

		if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) {
		struct mlx4_zone_entry *it = curr_node;

		list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) {
		res = mlx4_bitmap_alloc_range(it->bitmap, count,
		align, skip_mask);
		if (res != (u32)-1) {
		res += it->offset;
		uid = it->uid;
		goto out;
		}
		}
		}

		if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) {
		struct mlx4_zone_entry *it = curr_node;

		list_for_each_entry_from(it, &zone_alloc->entries, list) {
		if (unlikely(it == zone))
		continue;

		if (unlikely(it->priority != curr_node->priority))
		break;

		res = mlx4_bitmap_alloc_range(it->bitmap, count,
		align, skip_mask);
		if (res != (u32)-1) {
		res += it->offset;
		uid = it->uid;
		goto out;
		}
		}
		}

		if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) {
		if (list_is_last(&curr_node->prio_list, &zone_alloc->prios))
		goto out;

		curr_node = list_first_entry(&curr_node->prio_list,
		typeof(*curr_node),
		prio_list);

		list_for_each_entry_from(curr_node, &zone_alloc->entries, list) {
		res = mlx4_bitmap_alloc_range(curr_node->bitmap, count,
		align, skip_mask);
		if (res != (u32)-1) {
		res += curr_node->offset;
		uid = curr_node->uid;
		goto out;
		}
		}
		}

		out:
		if (NULL != puid && res != (u32)-1)
		*puid = uid;
		return res;
		}

		/* Should be called under a lock */
		static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj,
		u32 count)
		{
		mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr);
		}

		/* Should be called under a lock */
		static struct mlx4_zone_entry *__mlx4_find_zone_by_uid(
		struct mlx4_zone_allocator *zones, u32 uid)
		{
		struct mlx4_zone_entry *zone;

		list_for_each_entry(zone, &zones->entries, list) {
		if (zone->uid == uid)
		return zone;
		}

		return NULL;
		}

		struct mlx4_bitmap mlx4_zone_get_bitmap(struct mlx4_zone_allocator zones, u32 uid)
		{
		struct mlx4_zone_entry *zone;
		struct mlx4_bitmap *bitmap;

		spin_lock(&zones->lock);

		zone = __mlx4_find_zone_by_uid(zones, uid);

		bitmap = zone == NULL ? NULL : zone->bitmap;

		spin_unlock(&zones->lock);

		return bitmap;
		}

		int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid)
		{
		struct mlx4_zone_entry *zone;
		int res;

		spin_lock(&zones->lock);

		zone = __mlx4_find_zone_by_uid(zones, uid);

		if (NULL == zone) {
		res = -1;
		goto out;
		}

		res = __mlx4_zone_remove_one_entry(zone);

		out:
		spin_unlock(&zones->lock);
		kfree(zone);

		return res;
		}

		/* Should be called under a lock */
		static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique(
		struct mlx4_zone_allocator *zones, u32 obj)
		{
		struct mlx4_zone_entry zone, zone_candidate = NULL;
		u32 dist = (u32)-1;

		/* Search for the smallest zone that this obj could be
		* allocated from. This is done in order to handle
		* situations when small bitmaps are allocated from bigger
		* bitmaps (and the allocated space is marked as reserved in
		* the bigger bitmap.
		*/
		list_for_each_entry(zone, &zones->entries, list) {
		if (obj >= zone->offset) {
		u32 mobj = (obj - zone->offset) & zones->mask;

		if (mobj < zone->bitmap->max) {
		u32 curr_dist = zone->bitmap->effective_len;

		if (curr_dist < dist) {
		dist = curr_dist;
		zone_candidate = zone;
		}
		}
		}
		}

		return zone_candidate;
		}

		u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count,
		int align, u32 skip_mask, u32 *puid)
		{
		struct mlx4_zone_entry *zone;
		int res = -1;

		spin_lock(&zones->lock);

		zone = __mlx4_find_zone_by_uid(zones, uid);

		if (NULL == zone)
		goto out;

		res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid);

		out:
		spin_unlock(&zones->lock);

		return res;
		}

		u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count)
		{
		struct mlx4_zone_entry *zone;
		int res = 0;

		spin_lock(&zones->lock);

		zone = __mlx4_find_zone_by_uid(zones, uid);

		if (NULL == zone) {
		res = -1;
		goto out;
		}

		__mlx4_free_from_zone(zone, obj, count);

		out:
		spin_unlock(&zones->lock);

		return res;
		}

		u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count)
		{
		struct mlx4_zone_entry *zone;
		int res;

		if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP))
		return -EFAULT;

		spin_lock(&zones->lock);

		zone = __mlx4_find_zone_by_uid_unique(zones, obj);

		if (NULL == zone) {
		res = -1;
		goto out;
		}

		__mlx4_free_from_zone(zone, obj, count);
		res = 0;

		out:
		spin_unlock(&zones->lock);

		return res;
		}
/*		/*
* Handling for queue buffers -- we allocate a bunch of memory and		* Handling for queue buffers -- we allocate a bunch of memory and
* register it in a memory region at HCA virtual address 0. If the		* register it in a memory region at HCA virtual address 0. If the
* requested size is > max_direct, we split the allocation into		* requested size is > max_direct, we split the allocation into
* multiple pages, so we don't require too much contiguous memory.		* multiple pages, so we don't require too much contiguous memory.
*/		*/

int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,		int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
struct mlx4_buf *buf)		struct mlx4_buf *buf, gfp_t gfp)
{		{
dma_addr_t t;		dma_addr_t t;

if (size <= max_direct) {		if (size <= max_direct) {
buf->nbufs = 1;		buf->nbufs = 1;
buf->npages = 1;		buf->npages = 1;
buf->page_shift = get_order(size) + PAGE_SHIFT;		buf->page_shift = get_order(size) + PAGE_SHIFT;
buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev,		buf->direct.buf = dma_alloc_coherent(&dev->persist->pdev->dev,
size, &t, GFP_KERNEL);		size, &t, gfp);
if (!buf->direct.buf)		if (!buf->direct.buf)
return -ENOMEM;		return -ENOMEM;

buf->direct.map = t;		buf->direct.map = t;

while (t & ((1 << buf->page_shift) - 1)) {		while (t & ((1 << buf->page_shift) - 1)) {
--buf->page_shift;		--buf->page_shift;
buf->npages *= 2;		buf->npages *= 2;
}		}

memset(buf->direct.buf, 0, size);		memset(buf->direct.buf, 0, size);
} else {		} else {
int i;		int i;

buf->direct.buf = NULL;		buf->direct.buf = NULL;
buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;		buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE;
buf->npages = buf->nbufs;		buf->npages = buf->nbufs;
buf->page_shift = PAGE_SHIFT;		buf->page_shift = PAGE_SHIFT;
buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),		buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
GFP_KERNEL);		gfp);
if (!buf->page_list)		if (!buf->page_list)
return -ENOMEM;		return -ENOMEM;

for (i = 0; i < buf->nbufs; ++i) {		for (i = 0; i < buf->nbufs; ++i) {
buf->page_list[i].buf =		buf->page_list[i].buf =
dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,		dma_alloc_coherent(&dev->persist->pdev->dev,
&t, GFP_KERNEL);		PAGE_SIZE,
		&t, gfp);
if (!buf->page_list[i].buf)		if (!buf->page_list[i].buf)
goto err_free;		goto err_free;

buf->page_list[i].map = t;		buf->page_list[i].map = t;

memset(buf->page_list[i].buf, 0, PAGE_SIZE);		memset(buf->page_list[i].buf, 0, PAGE_SIZE);
}		}

if (BITS_PER_LONG == 64) {		if (BITS_PER_LONG == 64) {
struct page **pages;		struct page **pages;
pages = kmalloc(sizeof pages buf->nbufs, GFP_KERNEL);		pages = kmalloc(sizeof pages buf->nbufs, gfp);
if (!pages)		if (!pages)
goto err_free;		goto err_free;
for (i = 0; i < buf->nbufs; ++i)		for (i = 0; i < buf->nbufs; ++i)
pages[i] = virt_to_page(buf->page_list[i].buf);		pages[i] = virt_to_page(buf->page_list[i].buf);
buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);		buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
kfree(pages);		kfree(pages);
if (!buf->direct.buf)		if (!buf->direct.buf)
goto err_free;		goto err_free;
Show All 9 Lines
}		}
EXPORT_SYMBOL_GPL(mlx4_buf_alloc);		EXPORT_SYMBOL_GPL(mlx4_buf_alloc);

void mlx4_buf_free(struct mlx4_dev dev, int size, struct mlx4_buf buf)		void mlx4_buf_free(struct mlx4_dev dev, int size, struct mlx4_buf buf)
{		{
int i;		int i;

if (buf->nbufs == 1)		if (buf->nbufs == 1)
dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,		dma_free_coherent(&dev->persist->pdev->dev, size,
		buf->direct.buf,
buf->direct.map);		buf->direct.map);
else {		else {
if (BITS_PER_LONG == 64 && buf->direct.buf)		if (BITS_PER_LONG == 64)
vunmap(buf->direct.buf);		vunmap(buf->direct.buf);

for (i = 0; i < buf->nbufs; ++i)		for (i = 0; i < buf->nbufs; ++i)
if (buf->page_list[i].buf)		if (buf->page_list[i].buf)
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,		dma_free_coherent(&dev->persist->pdev->dev,
		PAGE_SIZE,
buf->page_list[i].buf,		buf->page_list[i].buf,
buf->page_list[i].map);		buf->page_list[i].map);
kfree(buf->page_list);		kfree(buf->page_list);
}		}
}		}
EXPORT_SYMBOL_GPL(mlx4_buf_free);		EXPORT_SYMBOL_GPL(mlx4_buf_free);

static struct mlx4_db_pgdir mlx4_alloc_db_pgdir(struct device dma_device)		static struct mlx4_db_pgdir mlx4_alloc_db_pgdir(struct device dma_device,
		gfp_t gfp)
{		{
struct mlx4_db_pgdir *pgdir;		struct mlx4_db_pgdir *pgdir;

pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL);		pgdir = kzalloc(sizeof *pgdir, gfp);
if (!pgdir)		if (!pgdir)
return NULL;		return NULL;

bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);		bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
pgdir->bits[0] = pgdir->order0;		pgdir->bits[0] = pgdir->order0;
pgdir->bits[1] = pgdir->order1;		pgdir->bits[1] = pgdir->order1;
pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,		pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
&pgdir->db_dma, GFP_KERNEL);		&pgdir->db_dma, gfp);
if (!pgdir->db_page) {		if (!pgdir->db_page) {
kfree(pgdir);		kfree(pgdir);
return NULL;		return NULL;
}		}

return pgdir;		return pgdir;
}		}

Show All 23 Lines	found:
db->index = i;		db->index = i;
db->db = pgdir->db_page + db->index;		db->db = pgdir->db_page + db->index;
db->dma = pgdir->db_dma + db->index * 4;		db->dma = pgdir->db_dma + db->index * 4;
db->order = order;		db->order = order;

return 0;		return 0;
}		}

int mlx4_db_alloc(struct mlx4_dev dev, struct mlx4_db db, int order)		int mlx4_db_alloc(struct mlx4_dev dev, struct mlx4_db db, int order, gfp_t gfp)
{		{
struct mlx4_priv *priv = mlx4_priv(dev);		struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_db_pgdir *pgdir;		struct mlx4_db_pgdir *pgdir;
int ret = 0;		int ret = 0;

mutex_lock(&priv->pgdir_mutex);		mutex_lock(&priv->pgdir_mutex);

list_for_each_entry(pgdir, &priv->pgdir_list, list)		list_for_each_entry(pgdir, &priv->pgdir_list, list)
if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))		if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
goto out;		goto out;

pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev));		pgdir = mlx4_alloc_db_pgdir(&dev->persist->pdev->dev, gfp);
if (!pgdir) {		if (!pgdir) {
ret = -ENOMEM;		ret = -ENOMEM;
goto out;		goto out;
}		}

list_add(&pgdir->list, &priv->pgdir_list);		list_add(&pgdir->list, &priv->pgdir_list);

/* This should never fail -- we just allocated an empty page: */		/* This should never fail -- we just allocated an empty page: */
Show All 20 Lines	void mlx4_db_free(struct mlx4_dev dev, struct mlx4_db db)
if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {		if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
clear_bit(i ^ 1, db->u.pgdir->order0);		clear_bit(i ^ 1, db->u.pgdir->order0);
++o;		++o;
}		}
i >>= o;		i >>= o;
set_bit(i, db->u.pgdir->bits[o]);		set_bit(i, db->u.pgdir->bits[o]);

if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {		if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,		dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE,
db->u.pgdir->db_page, db->u.pgdir->db_dma);		db->u.pgdir->db_page, db->u.pgdir->db_dma);
list_del(&db->u.pgdir->list);		list_del(&db->u.pgdir->list);
kfree(db->u.pgdir);		kfree(db->u.pgdir);
}		}

mutex_unlock(&priv->pgdir_mutex);		mutex_unlock(&priv->pgdir_mutex);
}		}
EXPORT_SYMBOL_GPL(mlx4_db_free);		EXPORT_SYMBOL_GPL(mlx4_db_free);

int mlx4_alloc_hwq_res(struct mlx4_dev dev, struct mlx4_hwq_resources wqres,		int mlx4_alloc_hwq_res(struct mlx4_dev dev, struct mlx4_hwq_resources wqres,
int size, int max_direct)		int size, int max_direct)
{		{
int err;		int err;

err = mlx4_db_alloc(dev, &wqres->db, 1);		err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL);
if (err)		if (err)
return err;		return err;

*wqres->db.db = 0;		*wqres->db.db = 0;

err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf);		err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL);
if (err)		if (err)
goto err_db;		goto err_db;

err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,		err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
&wqres->mtt);		&wqres->mtt);
if (err)		if (err)
goto err_buf;		goto err_buf;

err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf);		err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL);
if (err)		if (err)
goto err_mtt;		goto err_mtt;

return 0;		return 0;

err_mtt:		err_mtt:
mlx4_mtt_cleanup(dev, &wqres->mtt);		mlx4_mtt_cleanup(dev, &wqres->mtt);
err_buf:		err_buf:
Show All 16 Lines

View Options

sys/ofed/drivers/net/mlx4/catas.c

	Show All 36 Lines
	#include <linux/module.h>			#include <linux/module.h>

	#include <asm/byteorder.h>			#include <asm/byteorder.h>

	#include "mlx4.h"			#include "mlx4.h"

	#define MLX4_CATAS_POLL_INTERVAL (5 * HZ)			#define MLX4_CATAS_POLL_INTERVAL (5 * HZ)

	static DEFINE_SPINLOCK(catas_lock);

	static LIST_HEAD(catas_list);
	static struct work_struct catas_work;

	static int internal_err_reset = 1;			int mlx4_internal_err_reset = 1;
	module_param(internal_err_reset, int, 0644);			module_param_named(internal_err_reset, mlx4_internal_err_reset, int, 0644);
	MODULE_PARM_DESC(internal_err_reset,			MODULE_PARM_DESC(internal_err_reset,
	"Reset device on internal errors if non-zero"			"Reset device on internal errors if non-zero (default 1)");
	" (default 1, in SRIOV mode default is 0)");

				static int read_vendor_id(struct mlx4_dev *dev)
				{
				u16 vendor_id = 0;
				int ret;

				ret = pci_read_config_word(dev->persist->pdev, 0, &vendor_id);
				if (ret) {
				mlx4_err(dev, "Failed to read vendor ID, ret=%d\n", ret);
				return ret;
				}

				if (vendor_id == 0xffff) {
				mlx4_err(dev, "PCI can't be accessed to read vendor id\n");
				return -EINVAL;
				}

				return 0;
				}

				static int mlx4_reset_master(struct mlx4_dev *dev)
				{
				int err = 0;

				if (mlx4_is_master(dev))
				mlx4_report_internal_err_comm_event(dev);

				if (!pci_channel_offline(dev->persist->pdev)) {
				err = read_vendor_id(dev);
				/* If PCI can't be accessed to read vendor ID we assume that its
				* link was disabled and chip was already reset.
				*/
				if (err)
				return 0;

				err = mlx4_reset(dev);
				if (err)
				mlx4_err(dev, "Fail to reset HCA\n");
				}

				return err;
				}

				static int mlx4_reset_slave(struct mlx4_dev *dev)
				{
				#define COM_CHAN_RST_REQ_OFFSET 0x10
				#define COM_CHAN_RST_ACK_OFFSET 0x08

				u32 comm_flags;
				u32 rst_req;
				u32 rst_ack;
				unsigned long end;
				struct mlx4_priv *priv = mlx4_priv(dev);

				if (pci_channel_offline(dev->persist->pdev))
				return 0;

				comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm +
				MLX4_COMM_CHAN_FLAGS));
				if (comm_flags == 0xffffffff) {
				mlx4_err(dev, "VF reset is not needed\n");
				return 0;
				}

				if (!(dev->caps.vf_caps & MLX4_VF_CAP_FLAG_RESET)) {
				mlx4_err(dev, "VF reset is not supported\n");
				return -EOPNOTSUPP;
				}

				rst_req = (comm_flags & (u32)(1 << COM_CHAN_RST_REQ_OFFSET)) >>
				COM_CHAN_RST_REQ_OFFSET;
				rst_ack = (comm_flags & (u32)(1 << COM_CHAN_RST_ACK_OFFSET)) >>
				COM_CHAN_RST_ACK_OFFSET;
				if (rst_req != rst_ack) {
				mlx4_err(dev, "Communication channel isn't sync, fail to send reset\n");
				return -EIO;
				}

				rst_req ^= 1;
				mlx4_warn(dev, "VF is sending reset request to Firmware\n");
				comm_flags = rst_req << COM_CHAN_RST_REQ_OFFSET;
				__raw_writel((__force u32)cpu_to_be32(comm_flags),
				(__iomem char *)priv->mfunc.comm + MLX4_COMM_CHAN_FLAGS);
				/* Make sure that our comm channel write doesn't
				* get mixed in with writes from another CPU.
				*/
				mmiowb();

				end = msecs_to_jiffies(MLX4_COMM_TIME) + jiffies;
				while (time_before(jiffies, end)) {
				comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm +
				MLX4_COMM_CHAN_FLAGS));
				rst_ack = (comm_flags & (u32)(1 << COM_CHAN_RST_ACK_OFFSET)) >>
				COM_CHAN_RST_ACK_OFFSET;

				/* Reading rst_req again since the communication channel can
				* be reset at any time by the PF and all its bits will be
				* set to zero.
				*/
				rst_req = (comm_flags & (u32)(1 << COM_CHAN_RST_REQ_OFFSET)) >>
				COM_CHAN_RST_REQ_OFFSET;

				if (rst_ack == rst_req) {
				mlx4_warn(dev, "VF Reset succeed\n");
				return 0;
				}
				cond_resched();
				}
				mlx4_err(dev, "Fail to send reset over the communication channel\n");
				return -ETIMEDOUT;
				}

				static int mlx4_comm_internal_err(u32 slave_read)
				{
				return (u32)COMM_CHAN_EVENT_INTERNAL_ERR ==
				(slave_read & (u32)COMM_CHAN_EVENT_INTERNAL_ERR) ? 1 : 0;
				}

				void mlx4_enter_error_state(struct mlx4_dev_persistent *persist)
				{
				int err;
				struct mlx4_dev *dev;

				if (!mlx4_internal_err_reset)
				return;

				mutex_lock(&persist->device_state_mutex);
				if (persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)
				goto out;

				dev = persist->dev;
				mlx4_err(dev, "device is going to be reset\n");
				if (mlx4_is_slave(dev))
				err = mlx4_reset_slave(dev);
				else
				err = mlx4_reset_master(dev);
				BUG_ON(err != 0);

				dev->persist->state \|= MLX4_DEVICE_STATE_INTERNAL_ERROR;
				mlx4_err(dev, "device was reset successfully\n");
				mutex_unlock(&persist->device_state_mutex);

				/* At that step HW was already reset, now notify clients */
				mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0);
				mlx4_cmd_wake_completions(dev);
				return;

				out:
				mutex_unlock(&persist->device_state_mutex);
				}

				static void mlx4_handle_error_state(struct mlx4_dev_persistent *persist)
				{
				int err = 0;

				mlx4_enter_error_state(persist);
				mutex_lock(&persist->interface_state_mutex);
				if (persist->interface_state & MLX4_INTERFACE_STATE_UP &&
				!(persist->interface_state & MLX4_INTERFACE_STATE_DELETION)) {
				err = mlx4_restart_one(persist->pdev);
				mlx4_info(persist->dev, "mlx4_restart_one was ended, ret=%d\n",
				err);
				}
				mutex_unlock(&persist->interface_state_mutex);
				}

	static void dump_err_buf(struct mlx4_dev *dev)			static void dump_err_buf(struct mlx4_dev *dev)
	{			{
	struct mlx4_priv *priv = mlx4_priv(dev);			struct mlx4_priv *priv = mlx4_priv(dev);

	int i;			int i;

	mlx4_err(dev, "Internal error detected:\n");			mlx4_err(dev, "Internal error detected:\n");
	for (i = 0; i < priv->fw.catas_size; ++i)			for (i = 0; i < priv->fw.catas_size; ++i)
	mlx4_err(dev, " buf[%02x]: %08x\n",			mlx4_err(dev, " buf[%02x]: %08x\n",
	i, swab32(readl(priv->catas_err.map + i)));			i, swab32(readl(priv->catas_err.map + i)));
	}			}

	static void poll_catas(unsigned long dev_ptr)			static void poll_catas(unsigned long dev_ptr)
	{			{
	struct mlx4_dev dev = (struct mlx4_dev ) dev_ptr;			struct mlx4_dev dev = (struct mlx4_dev ) dev_ptr;
	struct mlx4_priv *priv = mlx4_priv(dev);			struct mlx4_priv *priv = mlx4_priv(dev);
				u32 slave_read;

	if (readl(priv->catas_err.map)) {			if (mlx4_is_slave(dev)) {
	/* If the device is off-line, we cannot try to recover it */			slave_read = swab32(readl(&priv->mfunc.comm->slave_read));
	if (pci_channel_offline(dev->pdev))			if (mlx4_comm_internal_err(slave_read)) {
	mod_timer(&priv->catas_err.timer,			mlx4_warn(dev, "Internal error detected on the communication channel\n");
	round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL));			goto internal_err;
	else {			}
				} else if (readl(priv->catas_err.map)) {
	dump_err_buf(dev);			dump_err_buf(dev);
	mlx4_dispatch_event(dev, MLX4_DEV_EVENT_CATASTROPHIC_ERROR, 0);			goto internal_err;
				}

	if (internal_err_reset) {			if (dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
	spin_lock(&catas_lock);			mlx4_warn(dev, "Internal error mark was detected on device\n");
	list_add(&priv->catas_err.list, &catas_list);			goto internal_err;
	spin_unlock(&catas_lock);

	queue_work(mlx4_wq, &catas_work);
	}			}
	}
	} else
	mod_timer(&priv->catas_err.timer,			mod_timer(&priv->catas_err.timer,
	round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL));			round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL));
				return;

				internal_err:
				if (mlx4_internal_err_reset)
				queue_work(dev->persist->catas_wq, &dev->persist->catas_work);
	}			}

	static void catas_reset(struct work_struct *work)			static void catas_reset(struct work_struct *work)
	{			{
	struct mlx4_priv priv, tmppriv;			struct mlx4_dev_persistent *persist =
	struct mlx4_dev *dev;			container_of(work, struct mlx4_dev_persistent,
				catas_work);

	LIST_HEAD(tlist);			mlx4_handle_error_state(persist);
	int ret;

	spin_lock_irq(&catas_lock);
	list_splice_init(&catas_list, &tlist);
	spin_unlock_irq(&catas_lock);

	list_for_each_entry_safe(priv, tmppriv, &tlist, catas_err.list) {
	struct pci_dev *pdev = priv->dev.pdev;

	/* If the device is off-line, we cannot reset it */
	if (pci_channel_offline(pdev))
	continue;

	ret = mlx4_restart_one(priv->dev.pdev);
	/* 'priv' now is not valid */
	if (ret)
	pr_err("mlx4 %s: Reset failed (%d)\n",
	pci_name(pdev), ret);
	else {
	dev = pci_get_drvdata(pdev);
	mlx4_dbg(dev, "Reset succeeded\n");
	}			}
	}
	}

	void mlx4_start_catas_poll(struct mlx4_dev *dev)			void mlx4_start_catas_poll(struct mlx4_dev *dev)
	{			{
	struct mlx4_priv *priv = mlx4_priv(dev);			struct mlx4_priv *priv = mlx4_priv(dev);
	phys_addr_t addr;			phys_addr_t addr;

	/If we are in SRIOV the default of the module param must be 0/
	if (mlx4_is_mfunc(dev))
	internal_err_reset = 0;

	INIT_LIST_HEAD(&priv->catas_err.list);			INIT_LIST_HEAD(&priv->catas_err.list);
	init_timer(&priv->catas_err.timer);			init_timer(&priv->catas_err.timer);
	priv->catas_err.map = NULL;			priv->catas_err.map = NULL;

	addr = pci_resource_start(dev->pdev, priv->fw.catas_bar) +			if (!mlx4_is_slave(dev)) {
				addr = pci_resource_start(dev->persist->pdev,
				priv->fw.catas_bar) +
	priv->fw.catas_offset;			priv->fw.catas_offset;

	priv->catas_err.map = ioremap(addr, priv->fw.catas_size * 4);			priv->catas_err.map = ioremap(addr, priv->fw.catas_size * 4);
	if (!priv->catas_err.map) {			if (!priv->catas_err.map) {
	mlx4_warn(dev, "Failed to map internal error buffer at 0x%llx\n",			mlx4_warn(dev, "Failed to map internal error buffer at 0x%llx\n",
	(unsigned long long) addr);			(unsigned long long)addr);
	return;			return;
	}			}
				}

	priv->catas_err.timer.data = (unsigned long) dev;			priv->catas_err.timer.data = (unsigned long) dev;
	priv->catas_err.timer.function = poll_catas;			priv->catas_err.timer.function = poll_catas;
	priv->catas_err.timer.expires =			priv->catas_err.timer.expires =
	round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL);			round_jiffies(jiffies + MLX4_CATAS_POLL_INTERVAL);
	add_timer(&priv->catas_err.timer);			add_timer(&priv->catas_err.timer);
	}			}

	void mlx4_stop_catas_poll(struct mlx4_dev *dev)			void mlx4_stop_catas_poll(struct mlx4_dev *dev)
	{			{
	struct mlx4_priv *priv = mlx4_priv(dev);			struct mlx4_priv *priv = mlx4_priv(dev);

	del_timer_sync(&priv->catas_err.timer);			del_timer_sync(&priv->catas_err.timer);

	if (priv->catas_err.map) {			if (priv->catas_err.map) {
	iounmap(priv->catas_err.map);			iounmap(priv->catas_err.map);
	priv->catas_err.map = NULL;			priv->catas_err.map = NULL;
	}			}

	spin_lock_irq(&catas_lock);			if (dev->persist->interface_state & MLX4_INTERFACE_STATE_DELETION)
	list_del_init(&priv->catas_err.list);			flush_workqueue(dev->persist->catas_wq);
	spin_unlock_irq(&catas_lock);
	}			}

	void __init mlx4_catas_init(void)			int mlx4_catas_init(struct mlx4_dev *dev)
	{			{
	INIT_WORK(&catas_work, catas_reset);			INIT_WORK(&dev->persist->catas_work, catas_reset);
				dev->persist->catas_wq = create_singlethread_workqueue("mlx4_health");
				if (!dev->persist->catas_wq)
				return -ENOMEM;

				return 0;
				}

				void mlx4_catas_end(struct mlx4_dev *dev)
				{
				if (dev->persist->catas_wq) {
				destroy_workqueue(dev->persist->catas_wq);
				dev->persist->catas_wq = NULL;
				}
	}			}