Fwd: [ofa-general] [PATCH 3/3 v3] ib/uverbs: add support for create_qp_expanded in uverbs
Ron Livne
ronli.voltaire at gmail.com
Sun Aug 3 01:30:34 PDT 2008
Hi Vlad,
I'd like this patch to be added to OFED 1.4
This patch series is based on Jack M XRC patch series.
Can you please tell me what libibvers, libmlx4 and librdmacm version
are going to be in OFED 1.4, so I can make the proper modifications
for the patches for these libraries in order to get them in OFED 1.4
Roland,
Do you have any remarks for those patches?
Thank you,
Ron
---------- Forwarded message ----------
From: Ron Livne <ronli at voltaire.com>
Date: Sun, Jul 27, 2008 at 6:23 PM
Subject: [ofa-general] [PATCH 3/3 v3] ib/uverbs: add support for
create_qp_expanded in uverbs
To: Roland Drier <rolandd at cisco.com>
Cc: Olga Shern <olgas at voltaire.com>, general list
<general at lists.openfabrics.org>
This patch adds support for create_qp_expanded
to the uverbs.
It uses the reserved bitmap in ib_uverbs_create_qp
to transfer the new creation flags from the user space
to the kernel.
Changes in v2:
Minimized code duplication by adding the function
ib_uverbs_create_qp_common.
LSO now can not be activated through user space.
Changes in v3:
Added compatibility for old libibverbs.
Added field __u32 create_flags to struct ib_uverbs_create_qp_expanded.
Deleted the function ib_uverbs_create_qp_common from v2.
Signed-off-by: Ron Livne <ronli at voltaire.com
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index b55f0d7..ae9f9a8 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -214,6 +214,7 @@ IB_UVERBS_DECLARE_CMD(modify_xrc_rcv_qp);
IB_UVERBS_DECLARE_CMD(query_xrc_rcv_qp);
IB_UVERBS_DECLARE_CMD(reg_xrc_rcv_qp);
IB_UVERBS_DECLARE_CMD(unreg_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(create_qp_expanded);
#endif /* UVERBS_H */
diff --git a/drivers/infiniband/core/uverbs_cmd.c
b/drivers/infiniband/core/uverbs_cmd.c
index 4402a07..a9c1485 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1030,10 +1030,164 @@ ssize_t ib_uverbs_destroy_cq(struct
ib_uverbs_file *file,
}
ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
+ const char __user *buf, int in_len,
+ int out_len)
+{
+ struct ib_uverbs_create_qp cmd;
+ struct ib_uverbs_create_qp_resp resp;
+ struct ib_udata udata;
+ struct ib_uqp_object *obj;
+ struct ib_pd *pd;
+ struct ib_cq *scq, *rcq;
+ struct ib_srq *srq;
+ struct ib_qp *qp;
+ struct ib_qp_init_attr attr;
+ struct ib_xrcd *xrcd;
+ struct ib_uobject *xrcd_uobj;
+ int ret;
+
+ if (out_len < sizeof resp)
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, buf, sizeof cmd))
+ return -EFAULT;
+
+ INIT_UDATA(&udata, buf + sizeof cmd,
+ (unsigned long) cmd.response + sizeof resp,
+ in_len - sizeof cmd, out_len - sizeof resp);
+
+ obj = kmalloc(sizeof *obj, GFP_KERNEL);
+ if (!obj)
+ return -ENOMEM;
+
+ init_uobj(&obj->uevent.uobject, cmd.user_handle,
file->ucontext, &qp_lock_key);
+ down_write(&obj->uevent.uobject.mutex);
+
+ srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ?
+ idr_read_srq(cmd.srq_handle, file->ucontext) : NULL;
+ xrcd = cmd.qp_type == IB_QPT_XRC ?
+ idr_read_xrcd(cmd.srq_handle, file->ucontext,
&xrcd_uobj) : NULL;
+ pd = idr_read_pd(cmd.pd_handle, file->ucontext);
+ scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
+ rcq = cmd.recv_cq_handle == cmd.send_cq_handle ?
+ scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
+
+ if (!pd || !scq || !rcq || (cmd.is_srq && !srq) ||
+ (cmd.qp_type == IB_QPT_XRC && !xrcd)) {
+ ret = -EINVAL;
+ goto err_put;
+ }
+
+ attr.event_handler = ib_uverbs_qp_event_handler;
+ attr.qp_context = file;
+ attr.send_cq = scq;
+ attr.recv_cq = rcq;
+ attr.srq = srq;
+ attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR :
IB_SIGNAL_REQ_WR;
+ attr.qp_type = cmd.qp_type;
+ attr.xrc_domain = xrcd;
+ attr.create_flags = 0;
+
+ attr.cap.max_send_wr = cmd.max_send_wr;
+ attr.cap.max_recv_wr = cmd.max_recv_wr;
+ attr.cap.max_send_sge = cmd.max_send_sge;
+ attr.cap.max_recv_sge = cmd.max_recv_sge;
+ attr.cap.max_inline_data = cmd.max_inline_data;
+
+ obj->uevent.events_reported = 0;
+ INIT_LIST_HEAD(&obj->uevent.event_list);
+ INIT_LIST_HEAD(&obj->mcast_list);
+
+ qp = pd->device->create_qp(pd, &attr, &udata);
+ if (IS_ERR(qp)) {
+ ret = PTR_ERR(qp);
+ goto err_put;
+ }
+
+ qp->device = pd->device;
+ qp->pd = pd;
+ qp->send_cq = attr.send_cq;
+ qp->recv_cq = attr.recv_cq;
+ qp->srq = attr.srq;
+ qp->uobject = &obj->uevent.uobject;
+ qp->event_handler = attr.event_handler;
+ qp->qp_context = attr.qp_context;
+ qp->qp_type = attr.qp_type;
+ qp->xrcd = attr.xrc_domain;
+ atomic_inc(&pd->usecnt);
+ atomic_inc(&attr.send_cq->usecnt);
+ atomic_inc(&attr.recv_cq->usecnt);
+ if (attr.srq)
+ atomic_inc(&attr.srq->usecnt);
+ else if (attr.xrc_domain)
+ atomic_inc(&attr.xrc_domain->usecnt);
+
+ obj->uevent.uobject.object = qp;
+ ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
+ if (ret)
+ goto err_destroy;
+
+ memset(&resp, 0, sizeof resp);
+ resp.qpn = qp->qp_num;
+ resp.qp_handle = obj->uevent.uobject.id;
+ resp.max_recv_sge = attr.cap.max_recv_sge;
+ resp.max_send_sge = attr.cap.max_send_sge;
+ resp.max_recv_wr = attr.cap.max_recv_wr;
+ resp.max_send_wr = attr.cap.max_send_wr;
+ resp.max_inline_data = attr.cap.max_inline_data;
+
+ if (copy_to_user((void __user *) (unsigned long) cmd.response,
+ &resp, sizeof resp)) {
+ ret = -EFAULT;
+ goto err_copy;
+ }
+
+ put_pd_read(pd);
+ put_cq_read(scq);
+ if (rcq != scq)
+ put_cq_read(rcq);
+ if (srq)
+ put_srq_read(srq);
+ if (xrcd)
+ put_xrcd_read(xrcd_uobj);
+
+ mutex_lock(&file->mutex);
+ list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list);
+ mutex_unlock(&file->mutex);
+
+ obj->uevent.uobject.live = 1;
+
+ up_write(&obj->uevent.uobject.mutex);
+
+ return in_len;
+
+err_copy:
+ idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
+
+err_destroy:
+ ib_destroy_qp(qp);
+
+err_put:
+ if (pd)
+ put_pd_read(pd);
+ if (scq)
+ put_cq_read(scq);
+ if (rcq && rcq != scq)
+ put_cq_read(rcq);
+ if (srq)
+ put_srq_read(srq);
+ if (xrcd)
+ put_xrcd_read(xrcd_uobj);
+
+ put_uobj_write(&obj->uevent.uobject);
+ return ret;
+}
+
+ssize_t ib_uverbs_create_qp_expanded(struct ib_uverbs_file *file,
const char __user *buf, int in_len,
int out_len)
{
- struct ib_uverbs_create_qp cmd;
+ struct ib_uverbs_create_qp_expanded cmd;
struct ib_uverbs_create_qp_resp resp;
struct ib_udata udata;
struct ib_uqp_object *obj;
@@ -1078,7 +1232,6 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
goto err_put;
}
- attr.create_flags = 0;
attr.event_handler = ib_uverbs_qp_event_handler;
attr.qp_context = file;
attr.send_cq = scq;
@@ -1087,7 +1240,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR :
IB_SIGNAL_REQ_WR;
attr.qp_type = cmd.qp_type;
attr.xrc_domain = xrcd;
- attr.create_flags = 0;
+ attr.create_flags = cmd.create_flags;
attr.cap.max_send_wr = cmd.max_send_wr;
attr.cap.max_recv_wr = cmd.max_recv_wr;
diff --git a/drivers/infiniband/core/uverbs_main.c
b/drivers/infiniband/core/uverbs_main.c
index 1a96c35..cb435be 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -117,6 +117,7 @@ static ssize_t (*uverbs_cmd_table[])(struct
ib_uverbs_file *file,
[IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP] = ib_uverbs_query_xrc_rcv_qp,
[IB_USER_VERBS_CMD_REG_XRC_RCV_QP] = ib_uverbs_reg_xrc_rcv_qp,
[IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP] = ib_uverbs_unreg_xrc_rcv_qp,
+ [IB_USER_VERBS_CMD_CREATE_QP_EXPANDED] = ib_uverbs_create_qp_expanded,
};
static struct vfsmount *uverbs_event_mnt;
diff --git a/drivers/infiniband/hw/mlx4/main.c
b/drivers/infiniband/hw/mlx4/main.c
index 030f696..f954533 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -728,7 +728,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
(1ull << IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP) |
(1ull << IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP) |
(1ull << IB_USER_VERBS_CMD_REG_XRC_RCV_QP) |
- (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP);
+ (1ull << IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_QP_EXPANDED);
}
if (init_node_data(ibdev))
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 0d3f770..c65f88b 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -518,9 +518,6 @@ static int create_qp_common(struct mlx4_ib_dev
*dev, struct ib_pd *pd,
} else {
qp->sq_no_prefetch = 0;
- if (init_attr->create_flags &
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
- qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
-
if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
qp->flags |= MLX4_IB_QP_LSO;
@@ -559,6 +556,10 @@ static int create_qp_common(struct mlx4_ib_dev
*dev, struct ib_pd *pd,
}
}
+ if (init_attr->create_flags &
+ IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
+ qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
+
err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);
if (err)
goto err_wrid;
@@ -705,8 +706,11 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
return ERR_PTR(-EINVAL);
- if (init_attr->create_flags &&
- (pd->uobject || init_attr->qp_type != IB_QPT_UD))
+ if (init_attr->create_flags && init_attr->qp_type != IB_QPT_UD)
+ return ERR_PTR(-EINVAL);
+
+ if ((init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) &&
+ pd->uobject)
return ERR_PTR(-EINVAL);
switch (init_attr->qp_type) {
diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h
index 0df90d8..300474f 100644
--- a/include/rdma/ib_user_verbs.h
+++ b/include/rdma/ib_user_verbs.h
@@ -90,6 +90,7 @@ enum {
IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
+ IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
};
/*
@@ -411,6 +412,27 @@ struct ib_uverbs_create_qp {
__u64 driver_data[0];
};
+struct ib_uverbs_create_qp_expanded {
+ __u64 response;
+ __u64 user_handle;
+ __u32 pd_handle;
+ __u32 send_cq_handle;
+ __u32 recv_cq_handle;
+ __u32 srq_handle;
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+ __u8 sq_sig_all;
+ __u8 qp_type;
+ __u8 is_srq;
+ __u8 reserved;
+ __u32 reserved1;
+ __u32 create_flags;
+ __u64 driver_data[0];
+};
+
struct ib_uverbs_create_qp_resp {
__u32 qp_handle;
__u32 qpn;
_______________________________________________
general mailing list
general at lists.openfabrics.org
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
More information about the general
mailing list