Differential D31149 Diff 94461 sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c

Changeset View

Standalone View

sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c

Show First 20 Lines • Show All 1,033 Lines • ▼ Show 20 Lines	err = mlx4_ib_SET_PORT(mdev, port,
!!(mask & IB_PORT_RESET_QKEY_CNTR),		!!(mask & IB_PORT_RESET_QKEY_CNTR),
cap_mask);		cap_mask);

out:		out:
mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);		mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
return err;		return err;
}		}

static struct ib_ucontext mlx4_ib_alloc_ucontext(struct ib_device ibdev,		static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata)		struct ib_udata *udata)
{		{
		struct ib_device *ibdev = uctx->device;
struct mlx4_ib_dev *dev = to_mdev(ibdev);		struct mlx4_ib_dev *dev = to_mdev(ibdev);
struct mlx4_ib_ucontext *context;		struct mlx4_ib_ucontext *context = to_mucontext(uctx);
struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;		struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
struct mlx4_ib_alloc_ucontext_resp resp;		struct mlx4_ib_alloc_ucontext_resp resp;
int err;		int err;

if (!dev->ib_active)		if (!dev->ib_active)
return ERR_PTR(-EAGAIN);		return -EAGAIN;

if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {		if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
resp_v3.qp_tab_size = dev->dev->caps.num_qps;		resp_v3.qp_tab_size = dev->dev->caps.num_qps;
resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size;		resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size;
resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;		resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
} else {		} else {
resp.dev_caps = dev->dev->caps.userspace_caps;		resp.dev_caps = dev->dev->caps.userspace_caps;
resp.qp_tab_size = dev->dev->caps.num_qps;		resp.qp_tab_size = dev->dev->caps.num_qps;
resp.bf_reg_size = dev->dev->caps.bf_reg_size;		resp.bf_reg_size = dev->dev->caps.bf_reg_size;
resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;		resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
resp.cqe_size = dev->dev->caps.cqe_size;		resp.cqe_size = dev->dev->caps.cqe_size;
}		}

context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context)
return ERR_PTR(-ENOMEM);

err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);		err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);
if (err) {		if (err)
kfree(context);		return err;
return ERR_PTR(err);
}

INIT_LIST_HEAD(&context->db_page_list);		INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex);		mutex_init(&context->db_page_mutex);

if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)		if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));		err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
else		else
err = ib_copy_to_udata(udata, &resp, sizeof(resp));		err = ib_copy_to_udata(udata, &resp, sizeof(resp));

if (err) {		if (err) {
mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);		mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
kfree(context);		return -EFAULT;
return ERR_PTR(-EFAULT);
}		}

return &context->ibucontext;		return err;
}		}

static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)		static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
{		{
struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);		struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);

mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);		mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);
kfree(context);

return 0;
}		}

static void mlx4_ib_vma_open(struct vm_area_struct *area)
{
/* vma_open is called when a new VMA is created on top of our VMA.
* This is done through either mremap flow or split_vma (usually due
* to mlock, madvise, munmap, etc.). We do not support a clone of the
* vma, as this VMA is strongly hardware related. Therefore we set the
* vm_ops of the newly created/cloned VMA to NULL, to prevent it from
* calling us again and trying to do incorrect actions. We assume that
* the original vma size is exactly a single page that there will be no
* "splitting" operations on.
*/
area->vm_ops = NULL;
}

static void mlx4_ib_vma_close(struct vm_area_struct *area)
{
struct mlx4_ib_vma_private_data *mlx4_ib_vma_priv_data;

/* It's guaranteed that all VMAs opened on a FD are closed before the
* file itself is closed, therefore no sync is needed with the regular
* closing flow. (e.g. mlx4_ib_dealloc_ucontext) However need a sync
* with accessing the vma as part of mlx4_ib_disassociate_ucontext.
* The close operation is usually called under mm->mmap_sem except when
* process is exiting. The exiting case is handled explicitly as part
* of mlx4_ib_disassociate_ucontext.
*/
mlx4_ib_vma_priv_data = (struct mlx4_ib_vma_private_data *)
area->vm_private_data;

/* set the vma context pointer to null in the mlx4_ib driver's private
* data to protect against a race condition in mlx4_ib_dissassociate_ucontext().
*/
mlx4_ib_vma_priv_data->vma = NULL;
}

static const struct vm_operations_struct mlx4_ib_vm_ops = {
.open = mlx4_ib_vma_open,
.close = mlx4_ib_vma_close
};

static void mlx4_ib_set_vma_data(struct vm_area_struct *vma,
struct mlx4_ib_vma_private_data *vma_private_data)
{
vma_private_data->vma = vma;
vma->vm_private_data = vma_private_data;
vma->vm_ops = &mlx4_ib_vm_ops;
}

static int mlx4_ib_mmap(struct ib_ucontext context, struct vm_area_struct vma)		static int mlx4_ib_mmap(struct ib_ucontext context, struct vm_area_struct vma)
{		{
struct mlx4_ib_dev *dev = to_mdev(context->device);		struct mlx4_ib_dev *dev = to_mdev(context->device);
struct mlx4_ib_ucontext *mucontext = to_mucontext(context);

if (vma->vm_end - vma->vm_start != PAGE_SIZE)		switch (vma->vm_pgoff) {
return -EINVAL;		case 0:
		return rdma_user_mmap_io(context, vma,
if (vma->vm_pgoff == 0) {
/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_DB].vma)
return -EINVAL;

vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

if (io_remap_pfn_range(vma, vma->vm_start,
to_mucontext(context)->uar.pfn,		to_mucontext(context)->uar.pfn,
PAGE_SIZE, vma->vm_page_prot))		PAGE_SIZE,
return -EAGAIN;		pgprot_noncached(vma->vm_page_prot),
		NULL);

mlx4_ib_set_vma_data(vma, &mucontext->hw_bar_info[HW_BAR_DB]);		case 1:
		if (dev->dev->caps.bf_reg_size == 0)
} else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) {
/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_BF].vma)
return -EINVAL;		return -EINVAL;
		return rdma_user_mmap_io(
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);		context, vma,

if (io_remap_pfn_range(vma, vma->vm_start,
to_mucontext(context)->uar.pfn +		to_mucontext(context)->uar.pfn +
dev->dev->caps.num_uars,		dev->dev->caps.num_uars,
PAGE_SIZE, vma->vm_page_prot))		PAGE_SIZE, pgprot_writecombine(vma->vm_page_prot),
return -EAGAIN;		NULL);

mlx4_ib_set_vma_data(vma, &mucontext->hw_bar_info[HW_BAR_BF]);		case 3: {

} else if (vma->vm_pgoff == 3) {
struct mlx4_clock_params params;		struct mlx4_clock_params params;
int ret;		int ret;

/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_CLOCK].vma)
return -EINVAL;

ret = mlx4_get_internal_clock_params(dev->dev, &params);		ret = mlx4_get_internal_clock_params(dev->dev, &params);

if (ret)		if (ret)
return ret;		return ret;

vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);		return rdma_user_mmap_io(
if (io_remap_pfn_range(vma, vma->vm_start,		context, vma,
(pci_resource_start(dev->dev->persist->pdev,		(pci_resource_start(dev->dev->persist->pdev,
params.bar) +		params.bar) +
params.offset)		params.offset) >>
>> PAGE_SHIFT,		PAGE_SHIFT,
PAGE_SIZE, vma->vm_page_prot))		PAGE_SIZE, pgprot_noncached(vma->vm_page_prot),
return -EAGAIN;		NULL);
		}

mlx4_ib_set_vma_data(vma,		default:
&mucontext->hw_bar_info[HW_BAR_CLOCK]);
} else {
return -EINVAL;		return -EINVAL;
}		}

return 0;
}		}

static struct ib_pd mlx4_ib_alloc_pd(struct ib_device ibdev,		static int mlx4_ib_alloc_pd(struct ib_pd ibpd, struct ib_udata udata)
struct ib_ucontext *context,
struct ib_udata *udata)
{		{
struct mlx4_ib_pd *pd;		struct mlx4_ib_pd *pd = to_mpd(ibpd);
		struct ib_device *ibdev = ibpd->device;
int err;		int err;

pd = kmalloc(sizeof *pd, GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);

err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);		err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);
if (err) {		if (err)
kfree(pd);		return err;
return ERR_PTR(err);
}

if (context)		if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) {
mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);		mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
kfree(pd);		return -EFAULT;
return ERR_PTR(-EFAULT);
}		}
		return 0;
return &pd->ibpd;
}		}

static int mlx4_ib_dealloc_pd(struct ib_pd *pd)		static void mlx4_ib_dealloc_pd(struct ib_pd pd, struct ib_udata udata)
{		{
mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);		mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
kfree(pd);

return 0;
}		}

static struct ib_xrcd mlx4_ib_alloc_xrcd(struct ib_device ibdev,		static struct ib_xrcd mlx4_ib_alloc_xrcd(struct ib_device ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)		struct ib_udata *udata)
{		{
struct mlx4_ib_xrcd *xrcd;		struct mlx4_ib_xrcd *xrcd;
struct ib_cq_init_attr cq_attr = {};		struct ib_cq_init_attr cq_attr = {};
int err;		int err;

if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))		if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
return ERR_PTR(-ENOSYS);		return ERR_PTR(-ENOSYS);
Show All 25 Lines	err3:
ib_dealloc_pd(xrcd->pd);		ib_dealloc_pd(xrcd->pd);
err2:		err2:
mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);		mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
err1:		err1:
kfree(xrcd);		kfree(xrcd);
return ERR_PTR(err);		return ERR_PTR(err);
}		}

static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)		static int mlx4_ib_dealloc_xrcd(struct ib_xrcd xrcd, struct ib_udata udata)
{		{
ib_destroy_cq(to_mxrcd(xrcd)->cq);		ib_destroy_cq(to_mxrcd(xrcd)->cq);
ib_dealloc_pd(to_mxrcd(xrcd)->pd);		ib_dealloc_pd(to_mxrcd(xrcd)->pd);
mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);		mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
kfree(xrcd);		kfree(xrcd);

return 0;		return 0;
}		}
▲ Show 20 Lines • Show All 421 Lines • ▼ Show 20 Lines	if (flow_attr->num_of_specs == 0) {
}		}
}		}

return err;		return err;
}		}

static struct ib_flow mlx4_ib_create_flow(struct ib_qp qp,		static struct ib_flow mlx4_ib_create_flow(struct ib_qp qp,
struct ib_flow_attr *flow_attr,		struct ib_flow_attr *flow_attr,
int domain)		int domain, struct ib_udata *udata)
{		{
int err = 0, i = 0, j = 0;		int err = 0, i = 0, j = 0;
struct mlx4_ib_flow *mflow;		struct mlx4_ib_flow *mflow;
enum mlx4_net_trans_promisc_mode type[2];		enum mlx4_net_trans_promisc_mode type[2];
struct mlx4_dev *dev = (to_mdev(qp->device))->dev;		struct mlx4_dev *dev = (to_mdev(qp->device))->dev;
int is_bonded = mlx4_is_bonded(dev);		int is_bonded = mlx4_is_bonded(dev);

if (flow_attr->port < 1 \|\| flow_attr->port > qp->device->phys_port_cnt)		if (flow_attr->port < 1 \|\| flow_attr->port > qp->device->phys_port_cnt)
return ERR_PTR(-EINVAL);		return ERR_PTR(-EINVAL);

if ((flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&		if ((flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
(flow_attr->type != IB_FLOW_ATTR_NORMAL))		(flow_attr->type != IB_FLOW_ATTR_NORMAL))
return ERR_PTR(-EOPNOTSUPP);		return ERR_PTR(-EOPNOTSUPP);

		if (udata &&
		udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen))
		return ERR_PTR(-EOPNOTSUPP);

memset(type, 0, sizeof(type));		memset(type, 0, sizeof(type));

mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);		mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
if (!mflow) {		if (!mflow) {
err = -ENOMEM;		err = -ENOMEM;
goto err_free;		goto err_free;
}		}

▲ Show 20 Lines • Show All 787 Lines • ▼ Show 20 Lines	ibdev->uar_map = ioremap((phys_addr_t) ibdev->priv_uar.pfn << PAGE_SHIFT,
PAGE_SIZE);		PAGE_SIZE);
if (!ibdev->uar_map)		if (!ibdev->uar_map)
goto err_uar;		goto err_uar;
MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);		MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);

ibdev->dev = dev;		ibdev->dev = dev;
ibdev->bond_next_port = 0;		ibdev->bond_next_port = 0;

		INIT_IB_DEVICE_OPS(&ibdev->ib_dev.ops, mlx4, MLX4);
strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);		strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
ibdev->ib_dev.owner = THIS_MODULE;		ibdev->ib_dev.owner = THIS_MODULE;
ibdev->ib_dev.node_type = RDMA_NODE_IB_CA;		ibdev->ib_dev.node_type = RDMA_NODE_IB_CA;
ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey;		ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey;
ibdev->num_ports = num_ports;		ibdev->num_ports = num_ports;
ibdev->ib_dev.phys_port_cnt = mlx4_is_bonded(dev) ?		ibdev->ib_dev.phys_port_cnt = mlx4_is_bonded(dev) ?
1 : ibdev->num_ports;		1 : ibdev->num_ports;
ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;		ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
▲ Show 20 Lines • Show All 782 Lines • Show Last 20 Lines