Differential D31149 Diff 94461 sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c

Changeset View

Standalone View

sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c

/*-		/*-
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.		* Copyright (c) 2013-2020, Mellanox Technologies. All rights reserved.
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
* 1. Redistributions of source code must retain the above copyright		* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.		* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright		* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the		* notice, this list of conditions and the following disclaimer in the
Show All 13 Lines
*		*
* $FreeBSD$		* $FreeBSD$
*/		*/

#include <linux/kref.h>		#include <linux/kref.h>
#include <rdma/ib_umem.h>		#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>		#include <rdma/ib_user_verbs.h>
#include <rdma/ib_cache.h>		#include <rdma/ib_cache.h>
		#include <rdma/uverbs_ioctl.h>
#include "mlx5_ib.h"		#include "mlx5_ib.h"

static void mlx5_ib_cq_comp(struct mlx5_core_cq cq, struct mlx5_eqe eqe __unused)		static void mlx5_ib_cq_comp(struct mlx5_core_cq cq, struct mlx5_eqe eqe __unused)
{		{
struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;		struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;

ibcq->comp_handler(ibcq, ibcq->cq_context);		ibcq->comp_handler(ibcq, ibcq->cq_context);
}		}
▲ Show 20 Lines • Show All 474 Lines • ▼ Show 20 Lines	static int mlx5_poll_one(struct mlx5_ib_cq *cq,
struct ib_wc *wc)		struct ib_wc *wc)
{		{
struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);		struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
struct mlx5_err_cqe *err_cqe;		struct mlx5_err_cqe *err_cqe;
struct mlx5_cqe64 *cqe64;		struct mlx5_cqe64 *cqe64;
struct mlx5_core_qp *mqp;		struct mlx5_core_qp *mqp;
struct mlx5_ib_wq *wq;		struct mlx5_ib_wq *wq;
struct mlx5_sig_err_cqe *sig_err_cqe;		struct mlx5_sig_err_cqe *sig_err_cqe;
struct mlx5_core_mr *mmkey;		struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr;		struct mlx5_ib_mr *mr;
unsigned long flags;		unsigned long flags;
uint8_t opcode;		uint8_t opcode;
uint32_t qpn;		uint32_t qpn;
u16 wqe_ctr;		u16 wqe_ctr;
void *cqe;		void *cqe;
int idx;		int idx;

▲ Show 20 Lines • Show All 205 Lines • ▼ Show 20 Lines	static int alloc_cq_buf(struct mlx5_ib_dev dev, struct mlx5_ib_cq_buf buf,

buf->cqe_size = cqe_size;		buf->cqe_size = cqe_size;
buf->nent = nent;		buf->nent = nent;

return 0;		return 0;
}		}

static int create_cq_user(struct mlx5_ib_dev dev, struct ib_udata udata,		static int create_cq_user(struct mlx5_ib_dev dev, struct ib_udata udata,
struct ib_ucontext context, struct mlx5_ib_cq cq,		struct mlx5_ib_cq cq, int entries, u32 *cqb,
int entries, u32 **cqb,
int cqe_size, int index, int *inlen)		int cqe_size, int index, int *inlen)
{		{
struct mlx5_ib_create_cq ucmd;		struct mlx5_ib_create_cq ucmd = {};
size_t ucmdlen;		size_t ucmdlen;
int page_shift;		int page_shift;
__be64 *pas;		__be64 *pas;
int npages;		int npages;
int ncont;		int ncont;
void *cqc;		void *cqc;
int err;		int err;
		struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
		udata, struct mlx5_ib_ucontext, ibucontext);

ucmdlen = min(udata->inlen, sizeof(ucmd));		ucmdlen = min(udata->inlen, sizeof(ucmd));
if (ucmdlen < offsetof(struct mlx5_ib_create_cq, flags))		if (ucmdlen < offsetof(struct mlx5_ib_create_cq, flags))
return -EINVAL;		return -EINVAL;

if (ib_copy_from_udata(&ucmd, udata, ucmdlen))		if (ib_copy_from_udata(&ucmd, udata, ucmdlen))
return -EFAULT;		return -EFAULT;

if ((ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX)))		if ((ucmd.flags & ~(MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX)))
return -EINVAL;		return -EINVAL;

if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128)		if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128)
return -EINVAL;		return -EINVAL;

*cqe_size = ucmd.cqe_size;		*cqe_size = ucmd.cqe_size;

cq->buf.umem = ib_umem_get(context, ucmd.buf_addr,		cq->buf.umem = ib_umem_get(&context->ibucontext, ucmd.buf_addr,
entries * ucmd.cqe_size,		entries * ucmd.cqe_size,
IB_ACCESS_LOCAL_WRITE, 1);		IB_ACCESS_LOCAL_WRITE, 1);
if (IS_ERR(cq->buf.umem)) {		if (IS_ERR(cq->buf.umem)) {
err = PTR_ERR(cq->buf.umem);		err = PTR_ERR(cq->buf.umem);
return err;		return err;
}		}

err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr,		err = mlx5_ib_db_map_user(context, ucmd.db_addr,
&cq->db);		&cq->db);
if (err)		if (err)
goto err_umem;		goto err_umem;

mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, 0, &npages, &page_shift,		mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, 0, &npages, &page_shift,
&ncont, NULL);		&ncont, NULL);
mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",		mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
(long long)ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);		(long long)ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);
Show All 10 Lines	static int create_cq_user(struct mlx5_ib_dev dev, struct ib_udata udata,
mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0);		mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, pas, 0);

cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);		cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);
MLX5_SET(cqc, cqc, log_page_size,		MLX5_SET(cqc, cqc, log_page_size,
page_shift - MLX5_ADAPTER_PAGE_SHIFT);		page_shift - MLX5_ADAPTER_PAGE_SHIFT);

if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {		if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {
*index = ucmd.uar_page_index;		*index = ucmd.uar_page_index;
} else if (to_mucontext(context)->bfregi.lib_uar_dyn) {		} else if (context->bfregi.lib_uar_dyn) {
err = -EINVAL;		err = -EINVAL;
goto err_cqb;		goto err_cqb;
} else {		} else {
*index = to_mucontext(context)->bfregi.sys_pages[0];		*index = context->bfregi.sys_pages[0];
}		}

		MLX5_SET(create_cq_in, *cqb, uid, context->devx_uid);
return 0;		return 0;

err_cqb:		err_cqb:
kvfree(*cqb);		kvfree(*cqb);

err_db:		err_db:
mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);		mlx5_ib_db_unmap_user(context, &cq->db);

err_umem:		err_umem:
ib_umem_release(cq->buf.umem);		ib_umem_release(cq->buf.umem);
return err;		return err;
}		}

static void destroy_cq_user(struct mlx5_ib_cq cq, struct ib_ucontext context)		static void destroy_cq_user(struct mlx5_ib_cq cq, struct ib_udata udata)
{		{
mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);		struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
		udata, struct mlx5_ib_ucontext, ibucontext);

		mlx5_ib_db_unmap_user(context, &cq->db);
ib_umem_release(cq->buf.umem);		ib_umem_release(cq->buf.umem);
}		}

static void init_cq_buf(struct mlx5_ib_cq cq, struct mlx5_ib_cq_buf buf)		static void init_cq_buf(struct mlx5_ib_cq cq, struct mlx5_ib_cq_buf buf)
{		{
int i;		int i;
void *cqe;		void *cqe;
struct mlx5_cqe64 *cqe64;		struct mlx5_cqe64 *cqe64;
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines
static void notify_soft_wc_handler(struct work_struct *work)		static void notify_soft_wc_handler(struct work_struct *work)
{		{
struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,		struct mlx5_ib_cq *cq = container_of(work, struct mlx5_ib_cq,
notify_work);		notify_work);

cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}		}

struct ib_cq mlx5_ib_create_cq(struct ib_device ibdev,		int mlx5_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)		struct ib_udata *udata)
{		{
		struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;		int entries = attr->cqe;
int vector = attr->comp_vector;		int vector = attr->comp_vector;
struct mlx5_ib_dev *dev = to_mdev(ibdev);		struct mlx5_ib_dev *dev = to_mdev(ibdev);
u32 out[MLX5_ST_SZ_DW(create_cq_out)];		u32 out[MLX5_ST_SZ_DW(create_cq_out)];
struct mlx5_ib_cq *cq;		struct mlx5_ib_cq *cq = to_mcq(ibcq);
int uninitialized_var(index);		int uninitialized_var(index);
int uninitialized_var(inlen);		int uninitialized_var(inlen);
u32 *cqb = NULL;		u32 *cqb = NULL;
void *cqc;		void *cqc;
int cqe_size;		int cqe_size;
unsigned int irqn;		unsigned int irqn;
int eqn;		int eqn;
int err;		int err;

if (entries < 0 \|\|		if (entries < 0 \|\|
(entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))		(entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
return ERR_PTR(-EINVAL);		return -EINVAL;

if (check_cq_create_flags(attr->flags))		if (check_cq_create_flags(attr->flags))
return ERR_PTR(-EOPNOTSUPP);		return -EOPNOTSUPP;

entries = roundup_pow_of_two(entries + 1);		entries = roundup_pow_of_two(entries + 1);
if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))		if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
return ERR_PTR(-EINVAL);		return -EINVAL;

cq = kzalloc(sizeof(*cq), GFP_KERNEL);
if (!cq)
return ERR_PTR(-ENOMEM);

cq->ibcq.cqe = entries - 1;		cq->ibcq.cqe = entries - 1;
mutex_init(&cq->resize_mutex);		mutex_init(&cq->resize_mutex);
spin_lock_init(&cq->lock);		spin_lock_init(&cq->lock);
cq->resize_buf = NULL;		cq->resize_buf = NULL;
cq->resize_umem = NULL;		cq->resize_umem = NULL;
cq->create_flags = attr->flags;		cq->create_flags = attr->flags;
INIT_LIST_HEAD(&cq->list_send_qp);		INIT_LIST_HEAD(&cq->list_send_qp);
INIT_LIST_HEAD(&cq->list_recv_qp);		INIT_LIST_HEAD(&cq->list_recv_qp);

if (context) {		if (udata) {
err = create_cq_user(dev, udata, context, cq, entries,		err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
&cqb, &cqe_size, &index, &inlen);		&index, &inlen);
if (err)		if (err)
goto err_create;		return err;
} else {		} else {
cqe_size = cache_line_size() == 128 ? 128 : 64;		cqe_size = cache_line_size() == 128 ? 128 : 64;
err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,		err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
&index, &inlen);		&index, &inlen);
if (err)		if (err)
goto err_create;		return err;

INIT_WORK(&cq->notify_work, notify_soft_wc_handler);		INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
}		}

err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);		err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
if (err)		if (err)
goto err_cqb;		goto err_cqb;

Show All 14 Lines	int mlx5_ib_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,

mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);		mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
cq->mcq.irqn = irqn;		cq->mcq.irqn = irqn;
cq->mcq.comp = mlx5_ib_cq_comp;		cq->mcq.comp = mlx5_ib_cq_comp;
cq->mcq.event = mlx5_ib_cq_event;		cq->mcq.event = mlx5_ib_cq_event;

INIT_LIST_HEAD(&cq->wc_list);		INIT_LIST_HEAD(&cq->wc_list);

if (context)		if (udata)
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT;		err = -EFAULT;
goto err_cmd;		goto err_cmd;
}		}


kvfree(cqb);		kvfree(cqb);
return &cq->ibcq;		return 0;

err_cmd:		err_cmd:
mlx5_core_destroy_cq(dev->mdev, &cq->mcq);		mlx5_core_destroy_cq(dev->mdev, &cq->mcq);

err_cqb:		err_cqb:
kvfree(cqb);		kvfree(cqb);
if (context)		if (udata)
destroy_cq_user(cq, context);		destroy_cq_user(cq, udata);
else		else
destroy_cq_kernel(dev, cq);		destroy_cq_kernel(dev, cq);
		return err;
err_create:
kfree(cq);

return ERR_PTR(err);
}		}

		void mlx5_ib_destroy_cq(struct ib_cq cq, struct ib_udata udata)
int mlx5_ib_destroy_cq(struct ib_cq *cq)
{		{
struct mlx5_ib_dev *dev = to_mdev(cq->device);		struct mlx5_ib_dev *dev = to_mdev(cq->device);
struct mlx5_ib_cq *mcq = to_mcq(cq);		struct mlx5_ib_cq *mcq = to_mcq(cq);
struct ib_ucontext *context = NULL;

if (cq->uobject)
context = cq->uobject->context;

mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);		mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
if (context)		if (udata)
destroy_cq_user(mcq, context);		destroy_cq_user(mcq, udata);
else		else
destroy_cq_kernel(dev, mcq);		destroy_cq_kernel(dev, mcq);

kfree(mcq);

return 0;
}		}

static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)		static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)
{		{
return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);		return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);
}		}

void __mlx5_ib_cq_clean(struct mlx5_ib_cq cq, u32 rsn, struct mlx5_ib_srq srq)		void __mlx5_ib_cq_clean(struct mlx5_ib_cq cq, u32 rsn, struct mlx5_ib_srq srq)
▲ Show 20 Lines • Show All 367 Lines • Show Last 20 Lines