[ofa-general] [PATCH 1/3 v3] libibverbs : add new verbs: create_qp_expanded
Ron Livne
ronli at voltaire.com
Sun Jul 27 07:34:33 PDT 2008
1. This patch adds a new capability flags to the enum ibv_device_cap_flags
IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK.
which implies that the device is capable of blocking multicast
loopback packets.
2. This patch also adds a new verb to the libibverbs:
struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr,
uint32_t create_flags);
which works similarly to ibv_create_qp, except for that it now
takes another argument: uint32_t create_flags
these creation flags should be aligned with those in the ib_verbs.h
in the kernel.
3. New creation flags added:
IBV_QP_CREATE_IPOIB_UD_LSO, IBV_MULTICAST_LOOPBACK_BLOCK
Changes in v2:
LSO creation flag deleted from user space.
Added struct ibv_create_qp_expanded.
Changes in v3:
Added compatibility for old kernel.
Added __u32 create_flags to ibv_create_qp_expanded.
Signed-off-by: Ron Livne <ronli at voltaire.com>
diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index f8138ef..ea824c8 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -155,6 +155,11 @@ int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
uint32_t xrc_qp_num);
int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
uint32_t xrc_qp_num);
+int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
+ struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
+ uint32_t create_flags,
+ struct ibv_create_qp_expanded *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size);
/*
* sysfs helper functions
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 8b5cd9a..1d4aa95 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -94,6 +94,7 @@ enum {
IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
+ IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
};
/*
@@ -459,6 +460,30 @@ struct ibv_create_qp {
__u64 driver_data[0];
};
+struct ibv_create_qp_expanded {
+ __u32 command;
+ __u16 in_words;
+ __u16 out_words;
+ __u64 response;
+ __u64 user_handle;
+ __u32 pd_handle;
+ __u32 send_cq_handle;
+ __u32 recv_cq_handle;
+ __u32 srq_handle;
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+ __u8 sq_sig_all;
+ __u8 qp_type;
+ __u8 is_srq;
+ __u8 reserved;
+ __u32 reserved1;
+ __u32 create_flags;
+ __u64 driver_data[0];
+};
+
struct ibv_create_qp_resp {
__u32 qp_handle;
__u32 qpn;
@@ -945,6 +970,7 @@ enum {
IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
+ IB_USER_VERBS_CMD_CREATE_QP_EXPANDED_V2 = -1,
};
struct ibv_destroy_cq_v1 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 2865d5c..b88aa7f 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -93,7 +93,8 @@ enum ibv_device_cap_flags {
IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
IBV_DEVICE_SRQ_RESIZE = 1 << 13,
IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
- IBV_DEVICE_XRC = 1 << 20
+ IBV_DEVICE_XRC = 1 << 20,
+ IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK = 1 << 22
};
enum ibv_atomic_cap {
@@ -631,6 +632,13 @@ enum {
IBV_SYSFS_PATH_MAX = 256
};
+/*
+ This enum must be alligned with ib_qp_create_flags
+ in include/rdma/ib_verbs.h
+*/
+enum ibv_qp_create_flags {
+ IBV_QP_CREATE_MULTICAST_LOOPBACK_BLOCK = 1 << 1
+};
struct ibv_device {
struct ibv_device_ops ops;
enum ibv_node_type node_type;
@@ -668,6 +676,9 @@ struct ibv_more_ops {
uint32_t xrc_qp_num);
int (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
uint32_t xrc_qp_num);
+ struct ibv_qp * (*create_qp_expanded)(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr,
+ uint32_t create_flags);
};
@@ -1029,6 +1040,13 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *qp_init_attr);
/**
+ * ibv_create_qp_expanded - Create a queue pair with creation flags.
+ */
+struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr,
+ uint32_t create_flags);
+
+/**
* ibv_modify_qp - Modify a queue pair.
*/
int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/cmd.c b/src/cmd.c
index cedf55e..45f7dd1 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -620,6 +620,45 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
return 0;
}
+static int ibv_cmd_create_qp_common(struct ibv_pd *pd,
+ struct ibv_qp *qp,
+ struct ibv_qp_init_attr *attr,
+ struct ibv_create_qp_resp *resp,
+ size_t resp_size)
+{
+ VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+ qp->handle = resp->qp_handle;
+ qp->qp_num = resp->qpn;
+ qp->context = pd->context;
+
+ if (abi_ver > 3) {
+ attr->cap.max_recv_sge = resp->max_recv_sge;
+ attr->cap.max_send_sge = resp->max_send_sge;
+ attr->cap.max_recv_wr = resp->max_recv_wr;
+ attr->cap.max_send_wr = resp->max_send_wr;
+ attr->cap.max_inline_data = resp->max_inline_data;
+ }
+
+ if (abi_ver == 4) {
+ struct ibv_create_qp_resp_v4 *resp_v4 =
+ (struct ibv_create_qp_resp_v4 *) resp;
+
+ memmove((void *) resp + sizeof *resp,
+ (void *) resp_v4 + sizeof *resp_v4,
+ resp_size - sizeof *resp);
+ } else if (abi_ver <= 3) {
+ struct ibv_create_qp_resp_v3 *resp_v3 =
+ (struct ibv_create_qp_resp_v3 *) resp;
+
+ memmove((void *) resp + sizeof *resp,
+ (void *) resp_v3 + sizeof *resp_v3,
+ resp_size - sizeof *resp);
+ }
+
+ return 0;
+}
+
int ibv_cmd_create_qp(struct ibv_pd *pd,
struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
struct ibv_create_qp *cmd, size_t cmd_size,
@@ -627,6 +666,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
{
IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
+ cmd->reserved = 0;
cmd->user_handle = (uintptr_t) qp;
cmd->pd_handle = pd->handle;
cmd->send_cq_handle = attr->send_cq->handle;
@@ -642,42 +682,42 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
(attr->xrc_domain ? attr->xrc_domain->handle : 0) :
(attr->srq ? attr->srq->handle : 0);
- cmd->reserved = 0;
if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
return errno;
- VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
- qp->handle = resp->qp_handle;
- qp->qp_num = resp->qpn;
- qp->context = pd->context;
-
- if (abi_ver > 3) {
- attr->cap.max_recv_sge = resp->max_recv_sge;
- attr->cap.max_send_sge = resp->max_send_sge;
- attr->cap.max_recv_wr = resp->max_recv_wr;
- attr->cap.max_send_wr = resp->max_send_wr;
- attr->cap.max_inline_data = resp->max_inline_data;
- }
+ return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
+}
- if (abi_ver == 4) {
- struct ibv_create_qp_resp_v4 *resp_v4 =
- (struct ibv_create_qp_resp_v4 *) resp;
+int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
+ struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
+ uint32_t create_flags,
+ struct ibv_create_qp_expanded *cmd, size_t cmd_size,
+ struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+ IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP_EXPANDED, resp, resp_size);
- memmove((void *) resp + sizeof *resp,
- (void *) resp_v4 + sizeof *resp_v4,
- resp_size - sizeof *resp);
- } else if (abi_ver <= 3) {
- struct ibv_create_qp_resp_v3 *resp_v3 =
- (struct ibv_create_qp_resp_v3 *) resp;
+ cmd->create_flags = create_flags;
+ cmd->user_handle = (uintptr_t) qp;
+ cmd->pd_handle = pd->handle;
+ cmd->send_cq_handle = attr->send_cq->handle;
+ cmd->recv_cq_handle = attr->recv_cq->handle;
+ cmd->max_send_wr = attr->cap.max_send_wr;
+ cmd->max_recv_wr = attr->cap.max_recv_wr;
+ cmd->max_send_sge = attr->cap.max_send_sge;
+ cmd->max_recv_sge = attr->cap.max_recv_sge;
+ cmd->max_inline_data = attr->cap.max_inline_data;
+ cmd->sq_sig_all = attr->sq_sig_all;
+ cmd->qp_type = attr->qp_type;
+ cmd->is_srq = !!attr->srq;
+ cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
+ (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
+ (attr->srq ? attr->srq->handle : 0);
- memmove((void *) resp + sizeof *resp,
- (void *) resp_v3 + sizeof *resp_v3,
- resp_size - sizeof *resp);
- }
+ if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
+ return errno;
- return 0;
+ return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
}
int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index fce6965..c032779 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -112,4 +112,6 @@ IBVERBS_1.1 {
ibv_port_state_str;
ibv_event_type_str;
ibv_wc_status_str;
+ ibv_create_qp_expanded;
+ ibv_cmd_create_qp_expanded;
} IBVERBS_1.0;
diff --git a/src/verbs.c b/src/verbs.c
index 4bfeec2..f185306 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -424,20 +424,19 @@ int __ibv_destroy_srq(struct ibv_srq *srq)
}
default_symver(__ibv_destroy_srq, ibv_destroy_srq);
-struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
- struct ibv_qp_init_attr *qp_init_attr)
+static struct ibv_qp *create_qp_common(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr,
+ struct ibv_qp *qp)
{
- struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
-
if (qp) {
- qp->context = pd->context;
- qp->qp_context = qp_init_attr->qp_context;
- qp->pd = pd;
- qp->send_cq = qp_init_attr->send_cq;
- qp->recv_cq = qp_init_attr->recv_cq;
- qp->srq = qp_init_attr->srq;
+ qp->context = pd->context;
+ qp->qp_context = qp_init_attr->qp_context;
+ qp->pd = pd;
+ qp->send_cq = qp_init_attr->send_cq;
+ qp->recv_cq = qp_init_attr->recv_cq;
+ qp->srq = qp_init_attr->srq;
qp->qp_type = qp_init_attr->qp_type;
- qp->state = IBV_QPS_RESET;
+ qp->state = IBV_QPS_RESET;
qp->events_completed = 0;
qp->xrc_domain = qp_init_attr->qp_type == IBV_QPT_XRC ?
qp_init_attr->xrc_domain : NULL;
@@ -447,8 +446,26 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
return qp;
}
+
+struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr)
+{
+
+ struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
+
+ return create_qp_common(pd, qp_init_attr, qp);
+}
default_symver(__ibv_create_qp, ibv_create_qp);
+struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr,
+ uint32_t create_flags)
+{
+ struct ibv_qp *qp = pd->context->more_ops->create_qp_expanded(pd, qp_init_attr, create_flags);
+
+ return create_qp_common(pd, qp_init_attr, qp);
+}
+
int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
enum ibv_qp_attr_mask attr_mask,
struct ibv_qp_init_attr *init_attr)
More information about the general
mailing list