[ofa-general] [PATCH 3/3 v3] ib/uverbs: add support for create_qp_expanded in uverbs
Ron Livne
ronli.voltaire at gmail.com
Sun Jul 27 05:20:49 PDT 2008
Jack,
I know there is some duplicated code in ib_uverbs_create_qp
and ib_uverbs_create_qp_expanded.
However, only a small part of this code can be extracted to a common function.
I think it'll be ugly to extract it.
Do you think I should extract it anyway to avoid duplications?
Thank you,
Ron
On Sun, Jul 27, 2008 at 6:23 PM, Ron Livne <ronli at voltaire.com> wrote:
> This patch adds support for create_qp_expanded
> to the uverbs.
> It uses the reserved bitmap in ib_uverbs_create_qp
> to transfer the new creation flags from the user space
> to the kernel.
>
> Changes in v2:
> Minimized code duplication by adding the function
> ib_uverbs_create_qp_common.
>
> LSO now can not be activated through user space.
>
> Changes in v3:
> Added compatibility for old libibverbs.
> Added field __u32 create_flags to struct ib_uverbs_create_qp_expanded.
> Deleted the function ib_uverbs_create_qp_common from v2.
>
> Signed-off-by: Ron Livne <ronli at voltaire.com
>
> diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
> index b55f0d7..ae9f9a8 100644
> --- a/drivers/infiniband/core/uverbs.h
> +++ b/drivers/infiniband/core/uverbs.h
> @@ -214,6 +214,7 @@ IB_UVERBS_DECLARE_CMD(modify_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(query_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(reg_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(unreg_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(create_qp_expanded);
>
>
> #endif /* UVERBS_H */
> diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
> index 4402a07..a9c1485 100644
> --- a/drivers/infiniband/core/uverbs_cmd.c
> +++ b/drivers/infiniband/core/uverbs_cmd.c
> @@ -1030,10 +1030,164 @@ ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
> }
>
> ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> + const char __user *buf, int in_len,
> + int out_len)
> +{
> + struct ib_uverbs_create_qp cmd;
> + struct ib_uverbs_create_qp_resp resp;
> + struct ib_udata udata;
> + struct ib_uqp_object *obj;
> + struct ib_pd *pd;
> + struct ib_cq *scq, *rcq;
> + struct ib_srq *srq;
> + struct ib_qp *qp;
> + struct ib_qp_init_attr attr;
> + struct ib_xrcd *xrcd;
> + struct ib_uobject *xrcd_uobj;
> + int ret;
> +
> + if (out_len < sizeof resp)
> + return -ENOSPC;
> +
> + if (copy_from_user(&cmd, buf, sizeof cmd))
> + return -EFAULT;
> +
> + INIT_UDATA(&udata, buf + sizeof cmd,
> + (unsigned long) cmd.response + sizeof resp,
> + in_len - sizeof cmd, out_len - sizeof resp);
> +
> + obj = kmalloc(sizeof *obj, GFP_KERNEL);
> + if (!obj)
> + return -ENOMEM;
> +
> + init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key);
> + down_write(&obj->uevent.uobject.mutex);
> +
> + srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ?
> + idr_read_srq(cmd.srq_handle, file->ucontext) : NULL;
> + xrcd = cmd.qp_type == IB_QPT_XRC ?
> + idr_read_xrcd(cmd.srq_handle, file->ucontext, &xrcd_uobj) : NULL;
> + pd = idr_read_pd(cmd.pd_handle, file->ucontext);
> + scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
> + rcq = cmd.recv_cq_handle == cmd.send_cq_handle ?
> + scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
> +
> + if (!pd || !scq || !rcq || (cmd.is_srq && !srq) ||
> + (cmd.qp_type == IB_QPT_XRC && !xrcd)) {
> + ret = -EINVAL;
> + goto err_put;
> + }
> +
> + attr.event_handler = ib_uverbs_qp_event_handler;
> + attr.qp_context = file;
> + attr.send_cq = scq;
> + attr.recv_cq = rcq;
> + attr.srq = srq;
> + attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
> + attr.qp_type = cmd.qp_type;
> + attr.xrc_domain = xrcd;
> + attr.create_flags = 0;
> +
> + attr.cap.max_send_wr = cmd.max_send_wr;
> + attr.cap.max_recv_wr = cmd.max_recv_wr;
> + attr.cap.max_send_sge = cmd.max_send_sge;
> + attr.cap.max_recv_sge = cmd.max_recv_sge;
> + attr.cap.max_inline_data = cmd.max_inline_data;
> +
> + obj->uevent.events_reported = 0;
> + INIT_LIST_HEAD(&obj->uevent.event_list);
> + INIT_LIST_HEAD(&obj->mcast_list);
> +
> + qp = pd->device->create_qp(pd, &attr, &udata);
> + if (IS_ERR(qp)) {
> + ret = PTR_ERR(qp);
> + goto err_put;
> + }
> +
> + qp->device = pd->device;
> + qp->pd = pd;
> + qp->send_cq = attr.send_cq;
> + qp->recv_cq = attr.recv_cq;
> + qp->srq = attr.srq;
> + qp->uobject = &obj->uevent.uobject;
> + qp->event_handler = attr.event_handler;
> + qp->qp_context = attr.qp_context;
> + qp->qp_type = attr.qp_type;
> + qp->xrcd = attr.xrc_domain;
> + atomic_inc(&pd->usecnt);
> + atomic_inc(&attr.send_cq->usecnt);
> + atomic_inc(&attr.recv_cq->usecnt);
> + if (attr.srq)
> + atomic_inc(&attr.srq->usecnt);
> + else if (attr.xrc_domain)
> + atomic_inc(&attr.xrc_domain->usecnt);
> +
> + obj->uevent.uobject.object = qp;
> + ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
> + if (ret)
> + goto err_destroy;
> +
> + memset(&resp, 0, sizeof resp);
> + resp.qpn = qp->qp_num;
> + resp.qp_handle = obj->uevent.uobject.id;
> + resp.max_recv_sge = attr.cap.max_recv_sge;
> + resp.max_send_sge = attr.cap.max_send_sge;
> + resp.max_recv_wr = attr.cap.max_recv_wr;
> + resp.max_send_wr = attr.cap.max_send_wr;
> + resp.max_inline_data = attr.cap.max_inline_data;
> +
> + if (copy_to_user((void __user *) (unsigned long) cmd.response,
> + &resp, sizeof resp)) {
> + ret = -EFAULT;
> + goto err_copy;
> + }
> +
> + put_pd_read(pd);
> + put_cq_read(scq);
> + if (rcq != scq)
> + put_cq_read(rcq);
> + if (srq)
> + put_srq_read(srq);
> + if (xrcd)
> + put_xrcd_read(xrcd_uobj);
> +
> + mutex_lock(&file->mutex);
> + list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
> + mutex_unlock(&file->mutex);
> +
> + obj->uevent.uobject.live = 1;
> +
> + up_write(&obj->uevent.uobject.mutex);
> +
> + return in_len;
> +
> +err_copy:
> + idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
> +
> +err_destroy:
> + ib_destroy_qp(qp);
> +
> +err_put:
> + if (pd)
> + put_pd_read(pd);
> + if (scq)
> + put_cq_read(scq);
> + if (rcq && rcq != scq)
> + put_cq_read(rcq);
> + if (srq)
> + put_srq_read(srq);
> + if (xrcd)
> + put_xrcd_read(xrcd_uobj);
> +
> + put_uobj_write(&obj->uevent.uobject);
> + return ret;
> +}
> +
> +ssize_t ib_uverbs_create_qp_expanded(struct ib_uverbs_file *file,
> const char __user *buf, int in_len,
> int out_len)
> {
> - struct ib_uverbs_create_qp cmd;
> + struct ib_uverbs_create_qp_expanded cmd;
> struct ib_uverbs_create_qp_resp resp;
> struct ib_udata udata;
> struct ib_uqp_object *obj;
> @@ -1078,7 +1232,6 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> goto err_put;
> }
>
> - attr.create_flags = 0;
> attr.event_handler = ib_uverbs_qp_event_handler;
> attr.qp_context = file;
> attr.send_cq = scq;
> @@ -1087,7 +1240,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
> attr.qp_type = cmd.qp_type;
> attr.xrc_domain = xrcd;
> - attr.create_flags = 0;
> + attr.create_flags = cmd.create_flags;
>
> attr.cap.max_send_wr = cmd.max_send_wr;
> attr.cap.max_recv_wr = cmd.max_recv_wr;
> diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
> index 1a96c35..cb435be 100644
> --- a/drivers/infiniband/core/uverbs_main.c
> +++ b/drivers/infiniband/core/uverbs_main.c
> @@ -117,6 +117,7 @@ static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
> [IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP] = ib_uverbs_query_xrc_rcv_qp,
> [IB_USER_VERBS_CMD_REG_XRC_RCV_QP] = ib_uverbs_reg_xrc_rcv_qp,
> [IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP] = ib_uverbs_unreg_xrc_rcv_qp,
> + [IB_USER_VERBS_CMD_CREATE_QP_EXPANDED] = ib_uverbs_create_qp_expanded,
> };
>
> static struct vfsmount *uverbs_event_mnt;
> diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
> index 030f696..f954533 100644
> --- a/drivers/infiniband/hw/mlx4/main.c
> +++ b/drivers/infiniband/hw/mlx4/main.c
> @@ -728,7 +728,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
> (1ull << IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP) |
> (1ull << IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP) |
> (1ull << IB_USER_VERBS_CMD_REG_XRC_RCV_QP) |
> - (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP);
> + (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP) |
> + (1ull << IB_USER_VERBS_CMD_CREATE_QP_EXPANDED);
> }
>
> if (init_node_data(ibdev))
> diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
> index 0d3f770..c65f88b 100644
> --- a/drivers/infiniband/hw/mlx4/qp.c
> +++ b/drivers/infiniband/hw/mlx4/qp.c
> @@ -518,9 +518,6 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
> } else {
> qp->sq_no_prefetch = 0;
>
> - if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
> - qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
> -
> if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
> qp->flags |= MLX4_IB_QP_LSO;
>
> @@ -559,6 +556,10 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
> }
> }
>
> + if (init_attr->create_flags &
> + IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
> + qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
> +
> err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);
> if (err)
> goto err_wrid;
> @@ -705,8 +706,11 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
> IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
> return ERR_PTR(-EINVAL);
>
> - if (init_attr->create_flags &&
> - (pd->uobject || init_attr->qp_type != IB_QPT_UD))
> + if (init_attr->create_flags && init_attr->qp_type != IB_QPT_UD)
> + return ERR_PTR(-EINVAL);
> +
> + if ((init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) &&
> + pd->uobject)
> return ERR_PTR(-EINVAL);
>
> switch (init_attr->qp_type) {
> diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h
> index 0df90d8..300474f 100644
> --- a/include/rdma/ib_user_verbs.h
> +++ b/include/rdma/ib_user_verbs.h
> @@ -90,6 +90,7 @@ enum {
> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
> IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
> };
>
> /*
> @@ -411,6 +412,27 @@ struct ib_uverbs_create_qp {
> __u64 driver_data[0];
> };
>
> +struct ib_uverbs_create_qp_expanded {
> + __u64 response;
> + __u64 user_handle;
> + __u32 pd_handle;
> + __u32 send_cq_handle;
> + __u32 recv_cq_handle;
> + __u32 srq_handle;
> + __u32 max_send_wr;
> + __u32 max_recv_wr;
> + __u32 max_send_sge;
> + __u32 max_recv_sge;
> + __u32 max_inline_data;
> + __u8 sq_sig_all;
> + __u8 qp_type;
> + __u8 is_srq;
> + __u8 reserved;
> + __u32 reserved1;
> + __u32 create_flags;
> + __u64 driver_data[0];
> +};
> +
> struct ib_uverbs_create_qp_resp {
> __u32 qp_handle;
> __u32 qpn;
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list