[ewg] [PATCH 1/2]libibvers: add create_qp_expanded

Ron Livne ronli.voltaire at gmail.com
Tue Aug 12 11:42:23 PDT 2008


On Tue, Aug 12, 2008 at 2:35 AM, Ralph Campbell
<ralph.campbell at qlogic.com> wrote:
> Is there a man page for ibv_create_qp_expanded()?
> It helps to know how to use it, especially since it
> isn't part of the IBTA verbs document.

We will add a man page for it.

>> +     IBV_DEVICE_XRC                  = 1 << 20,
>> +     IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK     = 1 << 22
>>  };
>
> What happened to bit 21?
> I see some other gaps so perhaps it isn't important.

Bit 21 is already taken.
This enum must match ib_device_cap_flags in include/rdma/ib_verbs.h in
the kernel.
I'll write a comment about it in the next version of the patch.

>> +/*
>> +       This enum must be alligned with ib_qp_create_flags
>
> Spelling: aligned
> but better phrased as "This enum must match ib_qp_create_flags"
>

Will be corrected.

Ron

>
> Minor nits inline below.
>
> On Tue, 2008-08-12 at 01:35 +0000, Ron Livne wrote:
>> 1. This patch adds a new capability flags to the enum ibv_device_cap_flags
>>         IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK.
>>         which implies that the device is capable of blocking multicast
>>         loopback packets.
>>
>> 2. This patch also adds a new verb to the libibverbs:
>>         struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>>                                struct ibv_qp_init_attr *qp_init_attr,
>>                                uint32_t create_flags);
>>         which works similarly to ibv_create_qp, except for that it now
>>         takes another argument: uint32_t create_flags
>>         these creation flags should be aligned with those in the ib_verbs.h
>>         in the kernel.
>>
>> 3. New creation flags added:
>>         IBV_QP_CREATE_IPOIB_UD_LSO, IBV_MULTICAST_LOOPBACK_BLOCK
>>
>> Signed-off-by: Ron Livne <ronli at voltaire.com>
>>
>> diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
>> index f8138ef..ea824c8 100644
>> --- a/include/infiniband/driver.h
>> +++ b/include/infiniband/driver.h
>> @@ -155,6 +155,11 @@ int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>>                          uint32_t xrc_qp_num);
>>  int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>>                            uint32_t xrc_qp_num);
>> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
>> +                     struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>> +                     uint32_t create_flags,
>> +                     struct ibv_create_qp_expanded *cmd, size_t cmd_size,
>> +                     struct ibv_create_qp_resp *resp, size_t resp_size);
>>
>>  /*
>>   * sysfs helper functions
>> diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
>> index 8b5cd9a..1d4aa95 100644
>> --- a/include/infiniband/kern-abi.h
>> +++ b/include/infiniband/kern-abi.h
>> @@ -94,6 +94,7 @@ enum {
>>       IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
>>       IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
>>       IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
>> +     IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
>>  };
>>
>>  /*
>> @@ -459,6 +460,30 @@ struct ibv_create_qp {
>>       __u64 driver_data[0];
>>  };
>>
>> +struct ibv_create_qp_expanded {
>> +     __u32 command;
>> +     __u16 in_words;
>> +     __u16 out_words;
>> +     __u64 response;
>> +     __u64 user_handle;
>> +     __u32 pd_handle;
>> +     __u32 send_cq_handle;
>> +     __u32 recv_cq_handle;
>> +     __u32 srq_handle;
>> +     __u32 max_send_wr;
>> +     __u32 max_recv_wr;
>> +     __u32 max_send_sge;
>> +     __u32 max_recv_sge;
>> +     __u32 max_inline_data;
>> +     __u8  sq_sig_all;
>> +     __u8  qp_type;
>> +     __u8  is_srq;
>> +     __u8  reserved;
>> +     __u32 reserved1;
>> +     __u32 create_flags;
>> +     __u64 driver_data[0];
>> +};
>> +
>>  struct ibv_create_qp_resp {
>>       __u32 qp_handle;
>>       __u32 qpn;
>> @@ -945,6 +970,7 @@ enum {
>>       IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
>>       IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
>>       IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
>> +     IB_USER_VERBS_CMD_CREATE_QP_EXPANDED_V2 = -1,
>>  };
>>
>>  struct ibv_destroy_cq_v1 {
>> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
>> index 31d1f64..583403a 100644
>> --- a/include/infiniband/verbs.h
>> +++ b/include/infiniband/verbs.h
>> @@ -93,7 +93,8 @@ enum ibv_device_cap_flags {
>>       IBV_DEVICE_RC_RNR_NAK_GEN       = 1 << 12,
>>       IBV_DEVICE_SRQ_RESIZE           = 1 << 13,
>>       IBV_DEVICE_N_NOTIFY_CQ          = 1 << 14,
>> -     IBV_DEVICE_XRC                  = 1 << 20
>> +     IBV_DEVICE_XRC                  = 1 << 20,
>> +     IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK     = 1 << 22
>>  };
>
> What happened to bit 21?
> I see some other gaps so perhaps it isn't important.
>
>>  enum ibv_atomic_cap {
>> @@ -634,6 +635,13 @@ enum {
>>       IBV_SYSFS_PATH_MAX      = 256
>>  };
>>
>> +/*
>> +       This enum must be alligned with ib_qp_create_flags
>
> Spelling: aligned
> but better phrased as "This enum must match ib_qp_create_flags"
>
>> +       in include/rdma/ib_verbs.h
>> +*/
>> +enum ibv_qp_create_flags {
>> +     IBV_QP_CREATE_MULTICAST_LOOPBACK_BLOCK    = 1 << 1
>> +};
>>  struct ibv_device {
>>       struct ibv_device_ops   ops;
>>       enum ibv_node_type      node_type;
>> @@ -671,6 +679,9 @@ struct ibv_more_ops {
>>                                                 uint32_t xrc_qp_num);
>>       int                     (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
>>                                                   uint32_t xrc_qp_num);
>> +     struct ibv_qp *         (*create_qp_expanded)(struct ibv_pd *pd,
>> +                                             struct ibv_qp_init_attr *attr,
>> +                                             uint32_t create_flags);
>>
>>  };
>>
>> @@ -1032,6 +1043,13 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
>>                            struct ibv_qp_init_attr *qp_init_attr);
>>
>>  /**
>> + * ibv_create_qp_expanded - Create a queue pair with creation flags.
>> + */
>> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>> +                             struct ibv_qp_init_attr *qp_init_attr,
>> +                             uint32_t create_flags);
>> +
>> +/**
>>   * ibv_modify_qp - Modify a queue pair.
>>   */
>>  int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>> diff --git a/src/cmd.c b/src/cmd.c
>> index cedf55e..45f7dd1 100644
>> --- a/src/cmd.c
>> +++ b/src/cmd.c
>> @@ -620,6 +620,45 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
>>       return 0;
>>  }
>>
>> +static int ibv_cmd_create_qp_common(struct ibv_pd *pd,
>> +                             struct ibv_qp *qp,
>> +                             struct ibv_qp_init_attr *attr,
>> +                             struct ibv_create_qp_resp *resp,
>> +                             size_t resp_size)
>> +{
>> +     VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
>> +
>> +     qp->handle                = resp->qp_handle;
>> +     qp->qp_num                = resp->qpn;
>> +     qp->context               = pd->context;
>> +
>> +     if (abi_ver > 3) {
>> +             attr->cap.max_recv_sge    = resp->max_recv_sge;
>> +             attr->cap.max_send_sge    = resp->max_send_sge;
>> +             attr->cap.max_recv_wr     = resp->max_recv_wr;
>> +             attr->cap.max_send_wr     = resp->max_send_wr;
>> +             attr->cap.max_inline_data = resp->max_inline_data;
>> +     }
>> +
>> +     if (abi_ver == 4) {
>> +             struct ibv_create_qp_resp_v4 *resp_v4 =
>> +             (struct ibv_create_qp_resp_v4 *) resp;
>> +
>> +             memmove((void *) resp + sizeof *resp,
>> +                     (void *) resp_v4 + sizeof *resp_v4,
>> +                     resp_size - sizeof *resp);
>> +     } else if (abi_ver <= 3) {
>> +             struct ibv_create_qp_resp_v3 *resp_v3 =
>> +                     (struct ibv_create_qp_resp_v3 *) resp;
>> +
>> +             memmove((void *) resp + sizeof *resp,
>> +                     (void *) resp_v3 + sizeof *resp_v3,
>> +                     resp_size - sizeof *resp);
>> +     }
>> +
>> +     return 0;
>> +}
>> +
>>  int ibv_cmd_create_qp(struct ibv_pd *pd,
>>                     struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>>                     struct ibv_create_qp *cmd, size_t cmd_size,
>> @@ -627,6 +666,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
>>  {
>>       IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
>>
>> +     cmd->reserved        = 0;
>>       cmd->user_handle     = (uintptr_t) qp;
>>       cmd->pd_handle       = pd->handle;
>>       cmd->send_cq_handle  = attr->send_cq->handle;
>> @@ -642,42 +682,42 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
>>       cmd->srq_handle      = attr->qp_type == IBV_QPT_XRC ?
>>               (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
>>               (attr->srq ? attr->srq->handle : 0);
>> -     cmd->reserved        = 0;
>>
>>       if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
>>               return errno;
>>
>> -     VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
>> -
>> -     qp->handle                = resp->qp_handle;
>> -     qp->qp_num                = resp->qpn;
>> -     qp->context               = pd->context;
>> -
>> -     if (abi_ver > 3) {
>> -             attr->cap.max_recv_sge    = resp->max_recv_sge;
>> -             attr->cap.max_send_sge    = resp->max_send_sge;
>> -             attr->cap.max_recv_wr     = resp->max_recv_wr;
>> -             attr->cap.max_send_wr     = resp->max_send_wr;
>> -             attr->cap.max_inline_data = resp->max_inline_data;
>> -     }
>> +     return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
>> +}
>>
>> -     if (abi_ver == 4) {
>> -             struct ibv_create_qp_resp_v4 *resp_v4 =
>> -                     (struct ibv_create_qp_resp_v4 *) resp;
>> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
>> +                     struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>> +                     uint32_t create_flags,
>> +                     struct ibv_create_qp_expanded *cmd, size_t cmd_size,
>> +                     struct ibv_create_qp_resp *resp, size_t resp_size)
>> +{
>> +     IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP_EXPANDED, resp, resp_size);
>>
>> -             memmove((void *) resp + sizeof *resp,
>> -                     (void *) resp_v4 + sizeof *resp_v4,
>> -                     resp_size - sizeof *resp);
>> -     } else if (abi_ver <= 3) {
>> -             struct ibv_create_qp_resp_v3 *resp_v3 =
>> -                     (struct ibv_create_qp_resp_v3 *) resp;
>> +     cmd->create_flags    = create_flags;
>> +     cmd->user_handle     = (uintptr_t) qp;
>> +     cmd->pd_handle       = pd->handle;
>> +     cmd->send_cq_handle  = attr->send_cq->handle;
>> +     cmd->recv_cq_handle  = attr->recv_cq->handle;
>> +     cmd->max_send_wr     = attr->cap.max_send_wr;
>> +     cmd->max_recv_wr     = attr->cap.max_recv_wr;
>> +     cmd->max_send_sge    = attr->cap.max_send_sge;
>> +     cmd->max_recv_sge    = attr->cap.max_recv_sge;
>> +     cmd->max_inline_data = attr->cap.max_inline_data;
>> +     cmd->sq_sig_all      = attr->sq_sig_all;
>> +     cmd->qp_type         = attr->qp_type;
>> +     cmd->is_srq          = !!attr->srq;
>> +     cmd->srq_handle      = attr->qp_type == IBV_QPT_XRC ?
>> +             (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
>> +             (attr->srq ? attr->srq->handle : 0);
>>
>> -             memmove((void *) resp + sizeof *resp,
>> -                     (void *) resp_v3 + sizeof *resp_v3,
>> -                     resp_size - sizeof *resp);
>> -     }
>> +     if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
>> +             return errno;
>>
>> -     return 0;
>> +     return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
>>  }
>>
>>  int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>> diff --git a/src/libibverbs.map b/src/libibverbs.map
>> index fce6965..c032779 100644
>> --- a/src/libibverbs.map
>> +++ b/src/libibverbs.map
>> @@ -112,4 +112,6 @@ IBVERBS_1.1 {
>>               ibv_port_state_str;
>>               ibv_event_type_str;
>>               ibv_wc_status_str;
>> +             ibv_create_qp_expanded;
>> +             ibv_cmd_create_qp_expanded;
>>  } IBVERBS_1.0;
>> diff --git a/src/verbs.c b/src/verbs.c
>> index 4bfeec2..f185306 100644
>> --- a/src/verbs.c
>> +++ b/src/verbs.c
>> @@ -424,20 +424,19 @@ int __ibv_destroy_srq(struct ibv_srq *srq)
>>  }
>>  default_symver(__ibv_destroy_srq, ibv_destroy_srq);
>>
>> -struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>> -                            struct ibv_qp_init_attr *qp_init_attr)
>> +static struct ibv_qp *create_qp_common(struct ibv_pd *pd,
>> +                             struct ibv_qp_init_attr *qp_init_attr,
>> +                             struct ibv_qp *qp)
>>  {
>> -     struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
>> -
>>       if (qp) {
>> -             qp->context          = pd->context;
>> -             qp->qp_context       = qp_init_attr->qp_context;
>> -             qp->pd               = pd;
>> -             qp->send_cq          = qp_init_attr->send_cq;
>> -             qp->recv_cq          = qp_init_attr->recv_cq;
>> -             qp->srq              = qp_init_attr->srq;
>> +             qp->context          = pd->context;
>> +             qp->qp_context       = qp_init_attr->qp_context;
>> +             qp->pd               = pd;
>> +             qp->send_cq          = qp_init_attr->send_cq;
>> +             qp->recv_cq          = qp_init_attr->recv_cq;
>> +             qp->srq              = qp_init_attr->srq;
>>               qp->qp_type          = qp_init_attr->qp_type;
>> -             qp->state            = IBV_QPS_RESET;
>> +             qp->state            = IBV_QPS_RESET;
>>               qp->events_completed = 0;
>>               qp->xrc_domain       = qp_init_attr->qp_type == IBV_QPT_XRC ?
>>                       qp_init_attr->xrc_domain : NULL;
>> @@ -447,8 +446,26 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>>
>>       return qp;
>>  }
>> +
>> +struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>> +                            struct ibv_qp_init_attr *qp_init_attr)
>> +{
>> +
>> +     struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
>> +
>> +     return create_qp_common(pd, qp_init_attr, qp);
>> +}
>>  default_symver(__ibv_create_qp, ibv_create_qp);
>>
>> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>> +                             struct ibv_qp_init_attr *qp_init_attr,
>> +                             uint32_t create_flags)
>> +{
>> +     struct ibv_qp *qp = pd->context->more_ops->create_qp_expanded(pd, qp_init_attr, create_flags);
>> +
>> +     return create_qp_common(pd, qp_init_attr, qp);
>> +}
>> +
>>  int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>>                  enum ibv_qp_attr_mask attr_mask,
>>                  struct ibv_qp_init_attr *init_attr)
>> _______________________________________________
>> ewg mailing list
>> ewg at lists.openfabrics.org
>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
>
> _______________________________________________
> ewg mailing list
> ewg at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
>



More information about the ewg mailing list