[ewg] [PATCH 1/2]libibvers: add create_qp_expanded
Ralph Campbell
ralph.campbell at qlogic.com
Mon Aug 11 16:35:01 PDT 2008
Is there a man page for ibv_create_qp_expanded()?
It helps to know how to use it, especially since it
isn't part of the IBTA verbs document.
Minor nits inline below.
On Tue, 2008-08-12 at 01:35 +0000, Ron Livne wrote:
> 1. This patch adds a new capability flags to the enum ibv_device_cap_flags
> IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK.
> which implies that the device is capable of blocking multicast
> loopback packets.
>
> 2. This patch also adds a new verb to the libibverbs:
> struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
> struct ibv_qp_init_attr *qp_init_attr,
> uint32_t create_flags);
> which works similarly to ibv_create_qp, except for that it now
> takes another argument: uint32_t create_flags
> these creation flags should be aligned with those in the ib_verbs.h
> in the kernel.
>
> 3. New creation flags added:
> IBV_QP_CREATE_IPOIB_UD_LSO, IBV_MULTICAST_LOOPBACK_BLOCK
>
> Signed-off-by: Ron Livne <ronli at voltaire.com>
>
> diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
> index f8138ef..ea824c8 100644
> --- a/include/infiniband/driver.h
> +++ b/include/infiniband/driver.h
> @@ -155,6 +155,11 @@ int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
> uint32_t xrc_qp_num);
> int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
> uint32_t xrc_qp_num);
> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
> + struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
> + uint32_t create_flags,
> + struct ibv_create_qp_expanded *cmd, size_t cmd_size,
> + struct ibv_create_qp_resp *resp, size_t resp_size);
>
> /*
> * sysfs helper functions
> diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
> index 8b5cd9a..1d4aa95 100644
> --- a/include/infiniband/kern-abi.h
> +++ b/include/infiniband/kern-abi.h
> @@ -94,6 +94,7 @@ enum {
> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
> IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
> };
>
> /*
> @@ -459,6 +460,30 @@ struct ibv_create_qp {
> __u64 driver_data[0];
> };
>
> +struct ibv_create_qp_expanded {
> + __u32 command;
> + __u16 in_words;
> + __u16 out_words;
> + __u64 response;
> + __u64 user_handle;
> + __u32 pd_handle;
> + __u32 send_cq_handle;
> + __u32 recv_cq_handle;
> + __u32 srq_handle;
> + __u32 max_send_wr;
> + __u32 max_recv_wr;
> + __u32 max_send_sge;
> + __u32 max_recv_sge;
> + __u32 max_inline_data;
> + __u8 sq_sig_all;
> + __u8 qp_type;
> + __u8 is_srq;
> + __u8 reserved;
> + __u32 reserved1;
> + __u32 create_flags;
> + __u64 driver_data[0];
> +};
> +
> struct ibv_create_qp_resp {
> __u32 qp_handle;
> __u32 qpn;
> @@ -945,6 +970,7 @@ enum {
> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
> IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED_V2 = -1,
> };
>
> struct ibv_destroy_cq_v1 {
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index 31d1f64..583403a 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -93,7 +93,8 @@ enum ibv_device_cap_flags {
> IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
> IBV_DEVICE_SRQ_RESIZE = 1 << 13,
> IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
> - IBV_DEVICE_XRC = 1 << 20
> + IBV_DEVICE_XRC = 1 << 20,
> + IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK = 1 << 22
> };
What happened to bit 21?
I see some other gaps so perhaps it isn't important.
> enum ibv_atomic_cap {
> @@ -634,6 +635,13 @@ enum {
> IBV_SYSFS_PATH_MAX = 256
> };
>
> +/*
> + This enum must be alligned with ib_qp_create_flags
Spelling: aligned
but better phrased as "This enum must match ib_qp_create_flags"
> + in include/rdma/ib_verbs.h
> +*/
> +enum ibv_qp_create_flags {
> + IBV_QP_CREATE_MULTICAST_LOOPBACK_BLOCK = 1 << 1
> +};
> struct ibv_device {
> struct ibv_device_ops ops;
> enum ibv_node_type node_type;
> @@ -671,6 +679,9 @@ struct ibv_more_ops {
> uint32_t xrc_qp_num);
> int (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
> uint32_t xrc_qp_num);
> + struct ibv_qp * (*create_qp_expanded)(struct ibv_pd *pd,
> + struct ibv_qp_init_attr *attr,
> + uint32_t create_flags);
>
> };
>
> @@ -1032,6 +1043,13 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
> struct ibv_qp_init_attr *qp_init_attr);
>
> /**
> + * ibv_create_qp_expanded - Create a queue pair with creation flags.
> + */
> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
> + struct ibv_qp_init_attr *qp_init_attr,
> + uint32_t create_flags);
> +
> +/**
> * ibv_modify_qp - Modify a queue pair.
> */
> int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
> diff --git a/src/cmd.c b/src/cmd.c
> index cedf55e..45f7dd1 100644
> --- a/src/cmd.c
> +++ b/src/cmd.c
> @@ -620,6 +620,45 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
> return 0;
> }
>
> +static int ibv_cmd_create_qp_common(struct ibv_pd *pd,
> + struct ibv_qp *qp,
> + struct ibv_qp_init_attr *attr,
> + struct ibv_create_qp_resp *resp,
> + size_t resp_size)
> +{
> + VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
> +
> + qp->handle = resp->qp_handle;
> + qp->qp_num = resp->qpn;
> + qp->context = pd->context;
> +
> + if (abi_ver > 3) {
> + attr->cap.max_recv_sge = resp->max_recv_sge;
> + attr->cap.max_send_sge = resp->max_send_sge;
> + attr->cap.max_recv_wr = resp->max_recv_wr;
> + attr->cap.max_send_wr = resp->max_send_wr;
> + attr->cap.max_inline_data = resp->max_inline_data;
> + }
> +
> + if (abi_ver == 4) {
> + struct ibv_create_qp_resp_v4 *resp_v4 =
> + (struct ibv_create_qp_resp_v4 *) resp;
> +
> + memmove((void *) resp + sizeof *resp,
> + (void *) resp_v4 + sizeof *resp_v4,
> + resp_size - sizeof *resp);
> + } else if (abi_ver <= 3) {
> + struct ibv_create_qp_resp_v3 *resp_v3 =
> + (struct ibv_create_qp_resp_v3 *) resp;
> +
> + memmove((void *) resp + sizeof *resp,
> + (void *) resp_v3 + sizeof *resp_v3,
> + resp_size - sizeof *resp);
> + }
> +
> + return 0;
> +}
> +
> int ibv_cmd_create_qp(struct ibv_pd *pd,
> struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
> struct ibv_create_qp *cmd, size_t cmd_size,
> @@ -627,6 +666,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
> {
> IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
>
> + cmd->reserved = 0;
> cmd->user_handle = (uintptr_t) qp;
> cmd->pd_handle = pd->handle;
> cmd->send_cq_handle = attr->send_cq->handle;
> @@ -642,42 +682,42 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
> cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
> (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
> (attr->srq ? attr->srq->handle : 0);
> - cmd->reserved = 0;
>
> if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
> return errno;
>
> - VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
> -
> - qp->handle = resp->qp_handle;
> - qp->qp_num = resp->qpn;
> - qp->context = pd->context;
> -
> - if (abi_ver > 3) {
> - attr->cap.max_recv_sge = resp->max_recv_sge;
> - attr->cap.max_send_sge = resp->max_send_sge;
> - attr->cap.max_recv_wr = resp->max_recv_wr;
> - attr->cap.max_send_wr = resp->max_send_wr;
> - attr->cap.max_inline_data = resp->max_inline_data;
> - }
> + return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
> +}
>
> - if (abi_ver == 4) {
> - struct ibv_create_qp_resp_v4 *resp_v4 =
> - (struct ibv_create_qp_resp_v4 *) resp;
> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
> + struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
> + uint32_t create_flags,
> + struct ibv_create_qp_expanded *cmd, size_t cmd_size,
> + struct ibv_create_qp_resp *resp, size_t resp_size)
> +{
> + IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP_EXPANDED, resp, resp_size);
>
> - memmove((void *) resp + sizeof *resp,
> - (void *) resp_v4 + sizeof *resp_v4,
> - resp_size - sizeof *resp);
> - } else if (abi_ver <= 3) {
> - struct ibv_create_qp_resp_v3 *resp_v3 =
> - (struct ibv_create_qp_resp_v3 *) resp;
> + cmd->create_flags = create_flags;
> + cmd->user_handle = (uintptr_t) qp;
> + cmd->pd_handle = pd->handle;
> + cmd->send_cq_handle = attr->send_cq->handle;
> + cmd->recv_cq_handle = attr->recv_cq->handle;
> + cmd->max_send_wr = attr->cap.max_send_wr;
> + cmd->max_recv_wr = attr->cap.max_recv_wr;
> + cmd->max_send_sge = attr->cap.max_send_sge;
> + cmd->max_recv_sge = attr->cap.max_recv_sge;
> + cmd->max_inline_data = attr->cap.max_inline_data;
> + cmd->sq_sig_all = attr->sq_sig_all;
> + cmd->qp_type = attr->qp_type;
> + cmd->is_srq = !!attr->srq;
> + cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
> + (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
> + (attr->srq ? attr->srq->handle : 0);
>
> - memmove((void *) resp + sizeof *resp,
> - (void *) resp_v3 + sizeof *resp_v3,
> - resp_size - sizeof *resp);
> - }
> + if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
> + return errno;
>
> - return 0;
> + return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
> }
>
> int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
> diff --git a/src/libibverbs.map b/src/libibverbs.map
> index fce6965..c032779 100644
> --- a/src/libibverbs.map
> +++ b/src/libibverbs.map
> @@ -112,4 +112,6 @@ IBVERBS_1.1 {
> ibv_port_state_str;
> ibv_event_type_str;
> ibv_wc_status_str;
> + ibv_create_qp_expanded;
> + ibv_cmd_create_qp_expanded;
> } IBVERBS_1.0;
> diff --git a/src/verbs.c b/src/verbs.c
> index 4bfeec2..f185306 100644
> --- a/src/verbs.c
> +++ b/src/verbs.c
> @@ -424,20 +424,19 @@ int __ibv_destroy_srq(struct ibv_srq *srq)
> }
> default_symver(__ibv_destroy_srq, ibv_destroy_srq);
>
> -struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
> - struct ibv_qp_init_attr *qp_init_attr)
> +static struct ibv_qp *create_qp_common(struct ibv_pd *pd,
> + struct ibv_qp_init_attr *qp_init_attr,
> + struct ibv_qp *qp)
> {
> - struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
> -
> if (qp) {
> - qp->context = pd->context;
> - qp->qp_context = qp_init_attr->qp_context;
> - qp->pd = pd;
> - qp->send_cq = qp_init_attr->send_cq;
> - qp->recv_cq = qp_init_attr->recv_cq;
> - qp->srq = qp_init_attr->srq;
> + qp->context = pd->context;
> + qp->qp_context = qp_init_attr->qp_context;
> + qp->pd = pd;
> + qp->send_cq = qp_init_attr->send_cq;
> + qp->recv_cq = qp_init_attr->recv_cq;
> + qp->srq = qp_init_attr->srq;
> qp->qp_type = qp_init_attr->qp_type;
> - qp->state = IBV_QPS_RESET;
> + qp->state = IBV_QPS_RESET;
> qp->events_completed = 0;
> qp->xrc_domain = qp_init_attr->qp_type == IBV_QPT_XRC ?
> qp_init_attr->xrc_domain : NULL;
> @@ -447,8 +446,26 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>
> return qp;
> }
> +
> +struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
> + struct ibv_qp_init_attr *qp_init_attr)
> +{
> +
> + struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
> +
> + return create_qp_common(pd, qp_init_attr, qp);
> +}
> default_symver(__ibv_create_qp, ibv_create_qp);
>
> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
> + struct ibv_qp_init_attr *qp_init_attr,
> + uint32_t create_flags)
> +{
> + struct ibv_qp *qp = pd->context->more_ops->create_qp_expanded(pd, qp_init_attr, create_flags);
> +
> + return create_qp_common(pd, qp_init_attr, qp);
> +}
> +
> int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
> enum ibv_qp_attr_mask attr_mask,
> struct ibv_qp_init_attr *init_attr)
> _______________________________________________
> ewg mailing list
> ewg at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
More information about the ewg
mailing list