[ewg] [PATCH 1/2]libibvers: add create_qp_expanded

Ron Livne ronli at voltaire.com
Mon Aug 11 18:35:54 PDT 2008


1. This patch adds a new capability flags to the enum ibv_device_cap_flags
        IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK.
        which implies that the device is capable of blocking multicast
        loopback packets.

2. This patch also adds a new verb to the libibverbs:
        struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
                               struct ibv_qp_init_attr *qp_init_attr,
                               uint32_t create_flags);
        which works similarly to ibv_create_qp, except for that it now
        takes another argument: uint32_t create_flags
        these creation flags should be aligned with those in the ib_verbs.h
        in the kernel.

3. New creation flags added:
        IBV_QP_CREATE_IPOIB_UD_LSO, IBV_MULTICAST_LOOPBACK_BLOCK

Signed-off-by: Ron Livne <ronli at voltaire.com>

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index f8138ef..ea824c8 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -155,6 +155,11 @@ int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 			   uint32_t xrc_qp_num);
 int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 			     uint32_t xrc_qp_num);
+int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
+			struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
+			uint32_t create_flags,
+			struct ibv_create_qp_expanded *cmd, size_t cmd_size,
+			struct ibv_create_qp_resp *resp, size_t resp_size);

 /*
  * sysfs helper functions
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 8b5cd9a..1d4aa95 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -94,6 +94,7 @@ enum {
 	IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
 	IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
 	IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
+	IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
 };

 /*
@@ -459,6 +460,30 @@ struct ibv_create_qp {
 	__u64 driver_data[0];
 };

+struct ibv_create_qp_expanded {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u64 user_handle;
+	__u32 pd_handle;
+	__u32 send_cq_handle;
+	__u32 recv_cq_handle;
+	__u32 srq_handle;
+	__u32 max_send_wr;
+	__u32 max_recv_wr;
+	__u32 max_send_sge;
+	__u32 max_recv_sge;
+	__u32 max_inline_data;
+	__u8  sq_sig_all;
+	__u8  qp_type;
+	__u8  is_srq;
+	__u8  reserved;
+	__u32 reserved1;
+	__u32 create_flags;
+	__u64 driver_data[0];
+};
+
 struct ibv_create_qp_resp {
 	__u32 qp_handle;
 	__u32 qpn;
@@ -945,6 +970,7 @@ enum {
 	IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
 	IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
 	IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
+	IB_USER_VERBS_CMD_CREATE_QP_EXPANDED_V2 = -1,
 };

 struct ibv_destroy_cq_v1 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 31d1f64..583403a 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -93,7 +93,8 @@ enum ibv_device_cap_flags {
 	IBV_DEVICE_RC_RNR_NAK_GEN	= 1 << 12,
 	IBV_DEVICE_SRQ_RESIZE		= 1 << 13,
 	IBV_DEVICE_N_NOTIFY_CQ		= 1 << 14,
-	IBV_DEVICE_XRC		        = 1 << 20
+	IBV_DEVICE_XRC		        = 1 << 20,
+	IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK	= 1 << 22
 };

 enum ibv_atomic_cap {
@@ -634,6 +635,13 @@ enum {
 	IBV_SYSFS_PATH_MAX	= 256
 };

+/*
+       This enum must be alligned with ib_qp_create_flags
+       in include/rdma/ib_verbs.h
+*/
+enum ibv_qp_create_flags {
+	IBV_QP_CREATE_MULTICAST_LOOPBACK_BLOCK    = 1 << 1
+};
 struct ibv_device {
 	struct ibv_device_ops	ops;
 	enum ibv_node_type	node_type;
@@ -671,6 +679,9 @@ struct ibv_more_ops {
 						  uint32_t xrc_qp_num);
 	int 			(*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
 						    uint32_t xrc_qp_num);
+	struct ibv_qp *         (*create_qp_expanded)(struct ibv_pd *pd,
+						struct ibv_qp_init_attr *attr,
+						uint32_t create_flags);

 };

@@ -1032,6 +1043,13 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
 			     struct ibv_qp_init_attr *qp_init_attr);

 /**
+ * ibv_create_qp_expanded - Create a queue pair with creation flags.
+ */
+struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
+				struct ibv_qp_init_attr *qp_init_attr,
+				uint32_t create_flags);
+
+/**
  * ibv_modify_qp - Modify a queue pair.
  */
 int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/cmd.c b/src/cmd.c
index cedf55e..45f7dd1 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -620,6 +620,45 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
 	return 0;
 }

+static int ibv_cmd_create_qp_common(struct ibv_pd *pd,
+				struct ibv_qp *qp,
+				struct ibv_qp_init_attr *attr,
+				struct ibv_create_qp_resp *resp,
+				size_t resp_size)
+{
+	VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+	qp->handle                = resp->qp_handle;
+	qp->qp_num                = resp->qpn;
+	qp->context               = pd->context;
+
+	if (abi_ver > 3) {
+		attr->cap.max_recv_sge    = resp->max_recv_sge;
+		attr->cap.max_send_sge    = resp->max_send_sge;
+		attr->cap.max_recv_wr     = resp->max_recv_wr;
+		attr->cap.max_send_wr     = resp->max_send_wr;
+		attr->cap.max_inline_data = resp->max_inline_data;
+	}
+
+	if (abi_ver == 4) {
+		struct ibv_create_qp_resp_v4 *resp_v4 =
+		(struct ibv_create_qp_resp_v4 *) resp;
+
+		memmove((void *) resp + sizeof *resp,
+			(void *) resp_v4 + sizeof *resp_v4,
+			resp_size - sizeof *resp);
+	} else if (abi_ver <= 3) {
+		struct ibv_create_qp_resp_v3 *resp_v3 =
+			(struct ibv_create_qp_resp_v3 *) resp;
+
+		memmove((void *) resp + sizeof *resp,
+			(void *) resp_v3 + sizeof *resp_v3,
+			resp_size - sizeof *resp);
+	}
+
+	return 0;
+}
+
 int ibv_cmd_create_qp(struct ibv_pd *pd,
 		      struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
 		      struct ibv_create_qp *cmd, size_t cmd_size,
@@ -627,6 +666,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 {
 	IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);

+	cmd->reserved	     = 0;
 	cmd->user_handle     = (uintptr_t) qp;
 	cmd->pd_handle 	     = pd->handle;
 	cmd->send_cq_handle  = attr->send_cq->handle;
@@ -642,42 +682,42 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 	cmd->srq_handle      = attr->qp_type == IBV_QPT_XRC ?
 		(attr->xrc_domain ? attr->xrc_domain->handle : 0) :
 		(attr->srq ? attr->srq->handle : 0);
-	cmd->reserved	     = 0;

 	if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
 		return errno;

-	VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
-	qp->handle 		  = resp->qp_handle;
-	qp->qp_num 		  = resp->qpn;
-	qp->context		  = pd->context;
-
-	if (abi_ver > 3) {
-		attr->cap.max_recv_sge    = resp->max_recv_sge;
-		attr->cap.max_send_sge    = resp->max_send_sge;
-		attr->cap.max_recv_wr     = resp->max_recv_wr;
-		attr->cap.max_send_wr     = resp->max_send_wr;
-		attr->cap.max_inline_data = resp->max_inline_data;
-	}
+	return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
+}

-	if (abi_ver == 4) {
-		struct ibv_create_qp_resp_v4 *resp_v4 =
-			(struct ibv_create_qp_resp_v4 *) resp;
+int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
+			struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
+			uint32_t create_flags,
+			struct ibv_create_qp_expanded *cmd, size_t cmd_size,
+			struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+	IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP_EXPANDED, resp, resp_size);

-		memmove((void *) resp + sizeof *resp,
-			(void *) resp_v4 + sizeof *resp_v4,
-			resp_size - sizeof *resp);
-	} else if (abi_ver <= 3) {
-		struct ibv_create_qp_resp_v3 *resp_v3 =
-			(struct ibv_create_qp_resp_v3 *) resp;
+	cmd->create_flags    = create_flags;
+	cmd->user_handle     = (uintptr_t) qp;
+	cmd->pd_handle 	     = pd->handle;
+	cmd->send_cq_handle  = attr->send_cq->handle;
+	cmd->recv_cq_handle  = attr->recv_cq->handle;
+	cmd->max_send_wr     = attr->cap.max_send_wr;
+	cmd->max_recv_wr     = attr->cap.max_recv_wr;
+	cmd->max_send_sge    = attr->cap.max_send_sge;
+	cmd->max_recv_sge    = attr->cap.max_recv_sge;
+	cmd->max_inline_data = attr->cap.max_inline_data;
+	cmd->sq_sig_all	     = attr->sq_sig_all;
+	cmd->qp_type 	     = attr->qp_type;
+	cmd->is_srq 	     = !!attr->srq;
+	cmd->srq_handle      = attr->qp_type == IBV_QPT_XRC ?
+		(attr->xrc_domain ? attr->xrc_domain->handle : 0) :
+		(attr->srq ? attr->srq->handle : 0);

-		memmove((void *) resp + sizeof *resp,
-			(void *) resp_v3 + sizeof *resp_v3,
-			resp_size - sizeof *resp);
-	}
+	if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;

-	return 0;
+	return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
 }

 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index fce6965..c032779 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -112,4 +112,6 @@ IBVERBS_1.1 {
 		ibv_port_state_str;
 		ibv_event_type_str;
 		ibv_wc_status_str;
+		ibv_create_qp_expanded;
+		ibv_cmd_create_qp_expanded;
 } IBVERBS_1.0;
diff --git a/src/verbs.c b/src/verbs.c
index 4bfeec2..f185306 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -424,20 +424,19 @@ int __ibv_destroy_srq(struct ibv_srq *srq)
 }
 default_symver(__ibv_destroy_srq, ibv_destroy_srq);

-struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
-			       struct ibv_qp_init_attr *qp_init_attr)
+static struct ibv_qp *create_qp_common(struct ibv_pd *pd,
+				struct ibv_qp_init_attr *qp_init_attr,
+				struct ibv_qp *qp)
 {
-	struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
-
 	if (qp) {
-		qp->context    	     = pd->context;
-		qp->qp_context 	     = qp_init_attr->qp_context;
-		qp->pd         	     = pd;
-		qp->send_cq    	     = qp_init_attr->send_cq;
-		qp->recv_cq    	     = qp_init_attr->recv_cq;
-		qp->srq        	     = qp_init_attr->srq;
+		qp->context          = pd->context;
+		qp->qp_context       = qp_init_attr->qp_context;
+		qp->pd               = pd;
+		qp->send_cq          = qp_init_attr->send_cq;
+		qp->recv_cq          = qp_init_attr->recv_cq;
+		qp->srq              = qp_init_attr->srq;
 		qp->qp_type          = qp_init_attr->qp_type;
-		qp->state	     = IBV_QPS_RESET;
+		qp->state            = IBV_QPS_RESET;
 		qp->events_completed = 0;
 		qp->xrc_domain       = qp_init_attr->qp_type == IBV_QPT_XRC ?
 			qp_init_attr->xrc_domain : NULL;
@@ -447,8 +446,26 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,

 	return qp;
 }
+
+struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
+			       struct ibv_qp_init_attr *qp_init_attr)
+{
+
+	struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
+
+	return create_qp_common(pd, qp_init_attr, qp);
+}
 default_symver(__ibv_create_qp, ibv_create_qp);

+struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
+				struct ibv_qp_init_attr *qp_init_attr,
+				uint32_t create_flags)
+{
+	struct ibv_qp *qp = pd->context->more_ops->create_qp_expanded(pd, qp_init_attr, create_flags);
+
+	return create_qp_common(pd, qp_init_attr, qp);
+}
+
 int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		   enum ibv_qp_attr_mask attr_mask,
 		   struct ibv_qp_init_attr *init_attr)



More information about the ewg mailing list