Fwd: [ofa-general] [PATCH 3/3 v3] ib/uverbs: add support for create_qp_expanded in uverbs
Jack Morgenstein
jackm at dev.mellanox.co.il
Sun Aug 3 04:12:50 PDT 2008
Ron,
I have not yet modified the OFED 1.4 patches to what I submitted to the list.
I'm waiting until all the changes settle down and are finalized.
The patches will need to be adapted for OFED 1.4 if you want them in now.
(The OFED 1.4 tree is currently like the 1.4 tree, except for the alignment
bug fixes in the ABI).
- Jack
On Sunday 03 August 2008 11:30, Ron Livne wrote:
> Hi Vlad,
> I'd like this patch to be added to OFED 1.4
> This patch series is based on Jack M XRC patch series.
>
> Can you please tell me what libibvers, libmlx4 and librdmacm version
> are going to be in OFED 1.4, so I can make the proper modifications
> for the patches for these libraries in order to get them in OFED 1.4
>
> Roland,
> Do you have any remarks for those patches?
>
> Thank you,
> Ron
>
>
> ---------- Forwarded message ----------
> From: Ron Livne <ronli at voltaire.com>
> Date: Sun, Jul 27, 2008 at 6:23 PM
> Subject: [ofa-general] [PATCH 3/3 v3] ib/uverbs: add support for
> create_qp_expanded in uverbs
> To: Roland Drier <rolandd at cisco.com>
> Cc: Olga Shern <olgas at voltaire.com>, general list
> <general at lists.openfabrics.org>
>
>
> This patch adds support for create_qp_expanded
> to the uverbs.
> It uses the reserved bitmap in ib_uverbs_create_qp
> to transfer the new creation flags from the user space
> to the kernel.
>
> Changes in v2:
> Minimized code duplication by adding the function
> ib_uverbs_create_qp_common.
>
> LSO now can not be activated through user space.
>
> Changes in v3:
> Added compatibility for old libibverbs.
> Added field __u32 create_flags to struct ib_uverbs_create_qp_expanded.
> Deleted the function ib_uverbs_create_qp_common from v2.
>
> Signed-off-by: Ron Livne <ronli at voltaire.com
>
> diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
> index b55f0d7..ae9f9a8 100644
> --- a/drivers/infiniband/core/uverbs.h
> +++ b/drivers/infiniband/core/uverbs.h
> @@ -214,6 +214,7 @@ IB_UVERBS_DECLARE_CMD(modify_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(query_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(reg_xrc_rcv_qp);
> IB_UVERBS_DECLARE_CMD(unreg_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(create_qp_expanded);
>
>
> #endif /* UVERBS_H */
> diff --git a/drivers/infiniband/core/uverbs_cmd.c
> b/drivers/infiniband/core/uverbs_cmd.c
> index 4402a07..a9c1485 100644
> --- a/drivers/infiniband/core/uverbs_cmd.c
> +++ b/drivers/infiniband/core/uverbs_cmd.c
> @@ -1030,10 +1030,164 @@ ssize_t ib_uverbs_destroy_cq(struct
> ib_uverbs_file *file,
> }
>
> ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> + const char __user *buf, int in_len,
> + int out_len)
> +{
> + struct ib_uverbs_create_qp cmd;
> + struct ib_uverbs_create_qp_resp resp;
> + struct ib_udata udata;
> + struct ib_uqp_object *obj;
> + struct ib_pd *pd;
> + struct ib_cq *scq, *rcq;
> + struct ib_srq *srq;
> + struct ib_qp *qp;
> + struct ib_qp_init_attr attr;
> + struct ib_xrcd *xrcd;
> + struct ib_uobject *xrcd_uobj;
> + int ret;
> +
> + if (out_len < sizeof resp)
> + return -ENOSPC;
> +
> + if (copy_from_user(&cmd, buf, sizeof cmd))
> + return -EFAULT;
> +
> + INIT_UDATA(&udata, buf + sizeof cmd,
> + (unsigned long) cmd.response + sizeof resp,
> + in_len - sizeof cmd, out_len - sizeof resp);
> +
> + obj = kmalloc(sizeof *obj, GFP_KERNEL);
> + if (!obj)
> + return -ENOMEM;
> +
> + init_uobj(&obj->uevent.uobject, cmd.user_handle,
> file->ucontext, &qp_lock_key);
> + down_write(&obj->uevent.uobject.mutex);
> +
> + srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ?
> + idr_read_srq(cmd.srq_handle, file->ucontext) : NULL;
> + xrcd = cmd.qp_type == IB_QPT_XRC ?
> + idr_read_xrcd(cmd.srq_handle, file->ucontext,
> &xrcd_uobj) : NULL;
> + pd = idr_read_pd(cmd.pd_handle, file->ucontext);
> + scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
> + rcq = cmd.recv_cq_handle == cmd.send_cq_handle ?
> + scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
> +
> + if (!pd || !scq || !rcq || (cmd.is_srq && !srq) ||
> + (cmd.qp_type == IB_QPT_XRC && !xrcd)) {
> + ret = -EINVAL;
> + goto err_put;
> + }
> +
> + attr.event_handler = ib_uverbs_qp_event_handler;
> + attr.qp_context = file;
> + attr.send_cq = scq;
> + attr.recv_cq = rcq;
> + attr.srq = srq;
> + attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR :
> IB_SIGNAL_REQ_WR;
> + attr.qp_type = cmd.qp_type;
> + attr.xrc_domain = xrcd;
> + attr.create_flags = 0;
> +
> + attr.cap.max_send_wr = cmd.max_send_wr;
> + attr.cap.max_recv_wr = cmd.max_recv_wr;
> + attr.cap.max_send_sge = cmd.max_send_sge;
> + attr.cap.max_recv_sge = cmd.max_recv_sge;
> + attr.cap.max_inline_data = cmd.max_inline_data;
> +
> + obj->uevent.events_reported = 0;
> + INIT_LIST_HEAD(&obj->uevent.event_list);
> + INIT_LIST_HEAD(&obj->mcast_list);
> +
> + qp = pd->device->create_qp(pd, &attr, &udata);
> + if (IS_ERR(qp)) {
> + ret = PTR_ERR(qp);
> + goto err_put;
> + }
> +
> + qp->device = pd->device;
> + qp->pd = pd;
> + qp->send_cq = attr.send_cq;
> + qp->recv_cq = attr.recv_cq;
> + qp->srq = attr.srq;
> + qp->uobject = &obj->uevent.uobject;
> + qp->event_handler = attr.event_handler;
> + qp->qp_context = attr.qp_context;
> + qp->qp_type = attr.qp_type;
> + qp->xrcd = attr.xrc_domain;
> + atomic_inc(&pd->usecnt);
> + atomic_inc(&attr.send_cq->usecnt);
> + atomic_inc(&attr.recv_cq->usecnt);
> + if (attr.srq)
> + atomic_inc(&attr.srq->usecnt);
> + else if (attr.xrc_domain)
> + atomic_inc(&attr.xrc_domain->usecnt);
> +
> + obj->uevent.uobject.object = qp;
> + ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
> + if (ret)
> + goto err_destroy;
> +
> + memset(&resp, 0, sizeof resp);
> + resp.qpn = qp->qp_num;
> + resp.qp_handle = obj->uevent.uobject.id;
> + resp.max_recv_sge = attr.cap.max_recv_sge;
> + resp.max_send_sge = attr.cap.max_send_sge;
> + resp.max_recv_wr = attr.cap.max_recv_wr;
> + resp.max_send_wr = attr.cap.max_send_wr;
> + resp.max_inline_data = attr.cap.max_inline_data;
> +
> + if (copy_to_user((void __user *) (unsigned long) cmd.response,
> + &resp, sizeof resp)) {
> + ret = -EFAULT;
> + goto err_copy;
> + }
> +
> + put_pd_read(pd);
> + put_cq_read(scq);
> + if (rcq != scq)
> + put_cq_read(rcq);
> + if (srq)
> + put_srq_read(srq);
> + if (xrcd)
> + put_xrcd_read(xrcd_uobj);
> +
> + mutex_lock(&file->mutex);
> + list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
> + mutex_unlock(&file->mutex);
> +
> + obj->uevent.uobject.live = 1;
> +
> + up_write(&obj->uevent.uobject.mutex);
> +
> + return in_len;
> +
> +err_copy:
> + idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
> +
> +err_destroy:
> + ib_destroy_qp(qp);
> +
> +err_put:
> + if (pd)
> + put_pd_read(pd);
> + if (scq)
> + put_cq_read(scq);
> + if (rcq && rcq != scq)
> + put_cq_read(rcq);
> + if (srq)
> + put_srq_read(srq);
> + if (xrcd)
> + put_xrcd_read(xrcd_uobj);
> +
> + put_uobj_write(&obj->uevent.uobject);
> + return ret;
> +}
> +
> +ssize_t ib_uverbs_create_qp_expanded(struct ib_uverbs_file *file,
> const char __user *buf, int in_len,
> int out_len)
> {
> - struct ib_uverbs_create_qp cmd;
> + struct ib_uverbs_create_qp_expanded cmd;
> struct ib_uverbs_create_qp_resp resp;
> struct ib_udata udata;
> struct ib_uqp_object *obj;
> @@ -1078,7 +1232,6 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> goto err_put;
> }
>
> - attr.create_flags = 0;
> attr.event_handler = ib_uverbs_qp_event_handler;
> attr.qp_context = file;
> attr.send_cq = scq;
> @@ -1087,7 +1240,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
> attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR :
> IB_SIGNAL_REQ_WR;
> attr.qp_type = cmd.qp_type;
> attr.xrc_domain = xrcd;
> - attr.create_flags = 0;
> + attr.create_flags = cmd.create_flags;
>
> attr.cap.max_send_wr = cmd.max_send_wr;
> attr.cap.max_recv_wr = cmd.max_recv_wr;
> diff --git a/drivers/infiniband/core/uverbs_main.c
> b/drivers/infiniband/core/uverbs_main.c
> index 1a96c35..cb435be 100644
> --- a/drivers/infiniband/core/uverbs_main.c
> +++ b/drivers/infiniband/core/uverbs_main.c
> @@ -117,6 +117,7 @@ static ssize_t (*uverbs_cmd_table[])(struct
> ib_uverbs_file *file,
> [IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP] = ib_uverbs_query_xrc_rcv_qp,
> [IB_USER_VERBS_CMD_REG_XRC_RCV_QP] = ib_uverbs_reg_xrc_rcv_qp,
> [IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP] = ib_uverbs_unreg_xrc_rcv_qp,
> + [IB_USER_VERBS_CMD_CREATE_QP_EXPANDED] = ib_uverbs_create_qp_expanded,
> };
>
> static struct vfsmount *uverbs_event_mnt;
> diff --git a/drivers/infiniband/hw/mlx4/main.c
> b/drivers/infiniband/hw/mlx4/main.c
> index 030f696..f954533 100644
> --- a/drivers/infiniband/hw/mlx4/main.c
> +++ b/drivers/infiniband/hw/mlx4/main.c
> @@ -728,7 +728,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
> (1ull << IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP) |
> (1ull << IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP) |
> (1ull << IB_USER_VERBS_CMD_REG_XRC_RCV_QP) |
> - (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP);
> + (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP) |
> + (1ull << IB_USER_VERBS_CMD_CREATE_QP_EXPANDED);
> }
>
> if (init_node_data(ibdev))
> diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
> index 0d3f770..c65f88b 100644
> --- a/drivers/infiniband/hw/mlx4/qp.c
> +++ b/drivers/infiniband/hw/mlx4/qp.c
> @@ -518,9 +518,6 @@ static int create_qp_common(struct mlx4_ib_dev
> *dev, struct ib_pd *pd,
> } else {
> qp->sq_no_prefetch = 0;
>
> - if (init_attr->create_flags &
> IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
> - qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
> -
> if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
> qp->flags |= MLX4_IB_QP_LSO;
>
> @@ -559,6 +556,10 @@ static int create_qp_common(struct mlx4_ib_dev
> *dev, struct ib_pd *pd,
> }
> }
>
> + if (init_attr->create_flags &
> + IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
> + qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
> +
> err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);
> if (err)
> goto err_wrid;
> @@ -705,8 +706,11 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
> IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
> return ERR_PTR(-EINVAL);
>
> - if (init_attr->create_flags &&
> - (pd->uobject || init_attr->qp_type != IB_QPT_UD))
> + if (init_attr->create_flags && init_attr->qp_type != IB_QPT_UD)
> + return ERR_PTR(-EINVAL);
> +
> + if ((init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) &&
> + pd->uobject)
> return ERR_PTR(-EINVAL);
>
> switch (init_attr->qp_type) {
> diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h
> index 0df90d8..300474f 100644
> --- a/include/rdma/ib_user_verbs.h
> +++ b/include/rdma/ib_user_verbs.h
> @@ -90,6 +90,7 @@ enum {
> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
> IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
> };
>
> /*
> @@ -411,6 +412,27 @@ struct ib_uverbs_create_qp {
> __u64 driver_data[0];
> };
>
> +struct ib_uverbs_create_qp_expanded {
> + __u64 response;
> + __u64 user_handle;
> + __u32 pd_handle;
> + __u32 send_cq_handle;
> + __u32 recv_cq_handle;
> + __u32 srq_handle;
> + __u32 max_send_wr;
> + __u32 max_recv_wr;
> + __u32 max_send_sge;
> + __u32 max_recv_sge;
> + __u32 max_inline_data;
> + __u8 sq_sig_all;
> + __u8 qp_type;
> + __u8 is_srq;
> + __u8 reserved;
> + __u32 reserved1;
> + __u32 create_flags;
> + __u64 driver_data[0];
> +};
> +
> struct ib_uverbs_create_qp_resp {
> __u32 qp_handle;
> __u32 qpn;
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list