[ewg] [PATCH 1/2]libibvers: add create_qp_expanded
Ron Livne
ronli.voltaire at gmail.com
Tue Aug 12 11:42:23 PDT 2008
On Tue, Aug 12, 2008 at 2:35 AM, Ralph Campbell
<ralph.campbell at qlogic.com> wrote:
> Is there a man page for ibv_create_qp_expanded()?
> It helps to know how to use it, especially since it
> isn't part of the IBTA verbs document.
We will add a man page for it.
>> + IBV_DEVICE_XRC = 1 << 20,
>> + IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK = 1 << 22
>> };
>
> What happened to bit 21?
> I see some other gaps so perhaps it isn't important.
Bit 21 is already taken.
This enum must match ib_device_cap_flags in include/rdma/ib_verbs.h in
the kernel.
I'll write a comment about it in the next version of the patch.
>> +/*
>> + This enum must be alligned with ib_qp_create_flags
>
> Spelling: aligned
> but better phrased as "This enum must match ib_qp_create_flags"
>
Will be corrected.
Ron
>
> Minor nits inline below.
>
> On Tue, 2008-08-12 at 01:35 +0000, Ron Livne wrote:
>> 1. This patch adds a new capability flags to the enum ibv_device_cap_flags
>> IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK.
>> which implies that the device is capable of blocking multicast
>> loopback packets.
>>
>> 2. This patch also adds a new verb to the libibverbs:
>> struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>> struct ibv_qp_init_attr *qp_init_attr,
>> uint32_t create_flags);
>> which works similarly to ibv_create_qp, except for that it now
>> takes another argument: uint32_t create_flags
>> these creation flags should be aligned with those in the ib_verbs.h
>> in the kernel.
>>
>> 3. New creation flags added:
>> IBV_QP_CREATE_IPOIB_UD_LSO, IBV_MULTICAST_LOOPBACK_BLOCK
>>
>> Signed-off-by: Ron Livne <ronli at voltaire.com>
>>
>> diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
>> index f8138ef..ea824c8 100644
>> --- a/include/infiniband/driver.h
>> +++ b/include/infiniband/driver.h
>> @@ -155,6 +155,11 @@ int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>> uint32_t xrc_qp_num);
>> int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
>> uint32_t xrc_qp_num);
>> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
>> + struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>> + uint32_t create_flags,
>> + struct ibv_create_qp_expanded *cmd, size_t cmd_size,
>> + struct ibv_create_qp_resp *resp, size_t resp_size);
>>
>> /*
>> * sysfs helper functions
>> diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
>> index 8b5cd9a..1d4aa95 100644
>> --- a/include/infiniband/kern-abi.h
>> +++ b/include/infiniband/kern-abi.h
>> @@ -94,6 +94,7 @@ enum {
>> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
>> IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
>> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
>> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED,
>> };
>>
>> /*
>> @@ -459,6 +460,30 @@ struct ibv_create_qp {
>> __u64 driver_data[0];
>> };
>>
>> +struct ibv_create_qp_expanded {
>> + __u32 command;
>> + __u16 in_words;
>> + __u16 out_words;
>> + __u64 response;
>> + __u64 user_handle;
>> + __u32 pd_handle;
>> + __u32 send_cq_handle;
>> + __u32 recv_cq_handle;
>> + __u32 srq_handle;
>> + __u32 max_send_wr;
>> + __u32 max_recv_wr;
>> + __u32 max_send_sge;
>> + __u32 max_recv_sge;
>> + __u32 max_inline_data;
>> + __u8 sq_sig_all;
>> + __u8 qp_type;
>> + __u8 is_srq;
>> + __u8 reserved;
>> + __u32 reserved1;
>> + __u32 create_flags;
>> + __u64 driver_data[0];
>> +};
>> +
>> struct ibv_create_qp_resp {
>> __u32 qp_handle;
>> __u32 qpn;
>> @@ -945,6 +970,7 @@ enum {
>> IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
>> IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
>> IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
>> + IB_USER_VERBS_CMD_CREATE_QP_EXPANDED_V2 = -1,
>> };
>>
>> struct ibv_destroy_cq_v1 {
>> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
>> index 31d1f64..583403a 100644
>> --- a/include/infiniband/verbs.h
>> +++ b/include/infiniband/verbs.h
>> @@ -93,7 +93,8 @@ enum ibv_device_cap_flags {
>> IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
>> IBV_DEVICE_SRQ_RESIZE = 1 << 13,
>> IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
>> - IBV_DEVICE_XRC = 1 << 20
>> + IBV_DEVICE_XRC = 1 << 20,
>> + IBV_DEVICE_BLOCK_MULTICAST_LOOPBACK = 1 << 22
>> };
>
> What happened to bit 21?
> I see some other gaps so perhaps it isn't important.
>
>> enum ibv_atomic_cap {
>> @@ -634,6 +635,13 @@ enum {
>> IBV_SYSFS_PATH_MAX = 256
>> };
>>
>> +/*
>> + This enum must be alligned with ib_qp_create_flags
>
> Spelling: aligned
> but better phrased as "This enum must match ib_qp_create_flags"
>
>> + in include/rdma/ib_verbs.h
>> +*/
>> +enum ibv_qp_create_flags {
>> + IBV_QP_CREATE_MULTICAST_LOOPBACK_BLOCK = 1 << 1
>> +};
>> struct ibv_device {
>> struct ibv_device_ops ops;
>> enum ibv_node_type node_type;
>> @@ -671,6 +679,9 @@ struct ibv_more_ops {
>> uint32_t xrc_qp_num);
>> int (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
>> uint32_t xrc_qp_num);
>> + struct ibv_qp * (*create_qp_expanded)(struct ibv_pd *pd,
>> + struct ibv_qp_init_attr *attr,
>> + uint32_t create_flags);
>>
>> };
>>
>> @@ -1032,6 +1043,13 @@ struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
>> struct ibv_qp_init_attr *qp_init_attr);
>>
>> /**
>> + * ibv_create_qp_expanded - Create a queue pair with creation flags.
>> + */
>> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>> + struct ibv_qp_init_attr *qp_init_attr,
>> + uint32_t create_flags);
>> +
>> +/**
>> * ibv_modify_qp - Modify a queue pair.
>> */
>> int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>> diff --git a/src/cmd.c b/src/cmd.c
>> index cedf55e..45f7dd1 100644
>> --- a/src/cmd.c
>> +++ b/src/cmd.c
>> @@ -620,6 +620,45 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
>> return 0;
>> }
>>
>> +static int ibv_cmd_create_qp_common(struct ibv_pd *pd,
>> + struct ibv_qp *qp,
>> + struct ibv_qp_init_attr *attr,
>> + struct ibv_create_qp_resp *resp,
>> + size_t resp_size)
>> +{
>> + VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
>> +
>> + qp->handle = resp->qp_handle;
>> + qp->qp_num = resp->qpn;
>> + qp->context = pd->context;
>> +
>> + if (abi_ver > 3) {
>> + attr->cap.max_recv_sge = resp->max_recv_sge;
>> + attr->cap.max_send_sge = resp->max_send_sge;
>> + attr->cap.max_recv_wr = resp->max_recv_wr;
>> + attr->cap.max_send_wr = resp->max_send_wr;
>> + attr->cap.max_inline_data = resp->max_inline_data;
>> + }
>> +
>> + if (abi_ver == 4) {
>> + struct ibv_create_qp_resp_v4 *resp_v4 =
>> + (struct ibv_create_qp_resp_v4 *) resp;
>> +
>> + memmove((void *) resp + sizeof *resp,
>> + (void *) resp_v4 + sizeof *resp_v4,
>> + resp_size - sizeof *resp);
>> + } else if (abi_ver <= 3) {
>> + struct ibv_create_qp_resp_v3 *resp_v3 =
>> + (struct ibv_create_qp_resp_v3 *) resp;
>> +
>> + memmove((void *) resp + sizeof *resp,
>> + (void *) resp_v3 + sizeof *resp_v3,
>> + resp_size - sizeof *resp);
>> + }
>> +
>> + return 0;
>> +}
>> +
>> int ibv_cmd_create_qp(struct ibv_pd *pd,
>> struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>> struct ibv_create_qp *cmd, size_t cmd_size,
>> @@ -627,6 +666,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
>> {
>> IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
>>
>> + cmd->reserved = 0;
>> cmd->user_handle = (uintptr_t) qp;
>> cmd->pd_handle = pd->handle;
>> cmd->send_cq_handle = attr->send_cq->handle;
>> @@ -642,42 +682,42 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
>> cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
>> (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
>> (attr->srq ? attr->srq->handle : 0);
>> - cmd->reserved = 0;
>>
>> if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
>> return errno;
>>
>> - VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
>> -
>> - qp->handle = resp->qp_handle;
>> - qp->qp_num = resp->qpn;
>> - qp->context = pd->context;
>> -
>> - if (abi_ver > 3) {
>> - attr->cap.max_recv_sge = resp->max_recv_sge;
>> - attr->cap.max_send_sge = resp->max_send_sge;
>> - attr->cap.max_recv_wr = resp->max_recv_wr;
>> - attr->cap.max_send_wr = resp->max_send_wr;
>> - attr->cap.max_inline_data = resp->max_inline_data;
>> - }
>> + return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
>> +}
>>
>> - if (abi_ver == 4) {
>> - struct ibv_create_qp_resp_v4 *resp_v4 =
>> - (struct ibv_create_qp_resp_v4 *) resp;
>> +int ibv_cmd_create_qp_expanded(struct ibv_pd *pd,
>> + struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
>> + uint32_t create_flags,
>> + struct ibv_create_qp_expanded *cmd, size_t cmd_size,
>> + struct ibv_create_qp_resp *resp, size_t resp_size)
>> +{
>> + IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP_EXPANDED, resp, resp_size);
>>
>> - memmove((void *) resp + sizeof *resp,
>> - (void *) resp_v4 + sizeof *resp_v4,
>> - resp_size - sizeof *resp);
>> - } else if (abi_ver <= 3) {
>> - struct ibv_create_qp_resp_v3 *resp_v3 =
>> - (struct ibv_create_qp_resp_v3 *) resp;
>> + cmd->create_flags = create_flags;
>> + cmd->user_handle = (uintptr_t) qp;
>> + cmd->pd_handle = pd->handle;
>> + cmd->send_cq_handle = attr->send_cq->handle;
>> + cmd->recv_cq_handle = attr->recv_cq->handle;
>> + cmd->max_send_wr = attr->cap.max_send_wr;
>> + cmd->max_recv_wr = attr->cap.max_recv_wr;
>> + cmd->max_send_sge = attr->cap.max_send_sge;
>> + cmd->max_recv_sge = attr->cap.max_recv_sge;
>> + cmd->max_inline_data = attr->cap.max_inline_data;
>> + cmd->sq_sig_all = attr->sq_sig_all;
>> + cmd->qp_type = attr->qp_type;
>> + cmd->is_srq = !!attr->srq;
>> + cmd->srq_handle = attr->qp_type == IBV_QPT_XRC ?
>> + (attr->xrc_domain ? attr->xrc_domain->handle : 0) :
>> + (attr->srq ? attr->srq->handle : 0);
>>
>> - memmove((void *) resp + sizeof *resp,
>> - (void *) resp_v3 + sizeof *resp_v3,
>> - resp_size - sizeof *resp);
>> - }
>> + if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
>> + return errno;
>>
>> - return 0;
>> + return ibv_cmd_create_qp_common(pd, qp, attr, resp, resp_size);
>> }
>>
>> int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>> diff --git a/src/libibverbs.map b/src/libibverbs.map
>> index fce6965..c032779 100644
>> --- a/src/libibverbs.map
>> +++ b/src/libibverbs.map
>> @@ -112,4 +112,6 @@ IBVERBS_1.1 {
>> ibv_port_state_str;
>> ibv_event_type_str;
>> ibv_wc_status_str;
>> + ibv_create_qp_expanded;
>> + ibv_cmd_create_qp_expanded;
>> } IBVERBS_1.0;
>> diff --git a/src/verbs.c b/src/verbs.c
>> index 4bfeec2..f185306 100644
>> --- a/src/verbs.c
>> +++ b/src/verbs.c
>> @@ -424,20 +424,19 @@ int __ibv_destroy_srq(struct ibv_srq *srq)
>> }
>> default_symver(__ibv_destroy_srq, ibv_destroy_srq);
>>
>> -struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>> - struct ibv_qp_init_attr *qp_init_attr)
>> +static struct ibv_qp *create_qp_common(struct ibv_pd *pd,
>> + struct ibv_qp_init_attr *qp_init_attr,
>> + struct ibv_qp *qp)
>> {
>> - struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
>> -
>> if (qp) {
>> - qp->context = pd->context;
>> - qp->qp_context = qp_init_attr->qp_context;
>> - qp->pd = pd;
>> - qp->send_cq = qp_init_attr->send_cq;
>> - qp->recv_cq = qp_init_attr->recv_cq;
>> - qp->srq = qp_init_attr->srq;
>> + qp->context = pd->context;
>> + qp->qp_context = qp_init_attr->qp_context;
>> + qp->pd = pd;
>> + qp->send_cq = qp_init_attr->send_cq;
>> + qp->recv_cq = qp_init_attr->recv_cq;
>> + qp->srq = qp_init_attr->srq;
>> qp->qp_type = qp_init_attr->qp_type;
>> - qp->state = IBV_QPS_RESET;
>> + qp->state = IBV_QPS_RESET;
>> qp->events_completed = 0;
>> qp->xrc_domain = qp_init_attr->qp_type == IBV_QPT_XRC ?
>> qp_init_attr->xrc_domain : NULL;
>> @@ -447,8 +446,26 @@ struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>>
>> return qp;
>> }
>> +
>> +struct ibv_qp *__ibv_create_qp(struct ibv_pd *pd,
>> + struct ibv_qp_init_attr *qp_init_attr)
>> +{
>> +
>> + struct ibv_qp *qp = pd->context->ops.create_qp(pd, qp_init_attr);
>> +
>> + return create_qp_common(pd, qp_init_attr, qp);
>> +}
>> default_symver(__ibv_create_qp, ibv_create_qp);
>>
>> +struct ibv_qp *ibv_create_qp_expanded(struct ibv_pd *pd,
>> + struct ibv_qp_init_attr *qp_init_attr,
>> + uint32_t create_flags)
>> +{
>> + struct ibv_qp *qp = pd->context->more_ops->create_qp_expanded(pd, qp_init_attr, create_flags);
>> +
>> + return create_qp_common(pd, qp_init_attr, qp);
>> +}
>> +
>> int __ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
>> enum ibv_qp_attr_mask attr_mask,
>> struct ibv_qp_init_attr *init_attr)
>> _______________________________________________
>> ewg mailing list
>> ewg at lists.openfabrics.org
>> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
>
> _______________________________________________
> ewg mailing list
> ewg at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg
>
More information about the ewg
mailing list