[ofa-general] [PATCH 7/8 V3] core: Add XRC receive-only qp support

Tang, Changqing changquing.tang at hp.com
Wed Feb 6 11:04:23 PST 2008


Jack:
        Do you have the updated version of srq_pingpong.c.xrc using these new API ?

I always got following error for a ping_pong HP-MPI code, two ranks, one on each node.

pp.x.x: Rank 0:1: MPI_Init: ibv_poll_cq(): bad status 9
pp.x.x: Rank 0:1: MPI_Init: self mpiobl03 peer mpiobl04 (rank: 0)
pp.x.x: Rank 0:1: MPI_Init: error message: remote invalid request error

the returned xrc_srq_num on each rank is zero, is this a correct value ?


--CQ


> -----Original Message-----
> From: general-bounces at lists.openfabrics.org
> [mailto:general-bounces at lists.openfabrics.org] On Behalf Of
> Jack Morgenstein
> Sent: Sunday, February 03, 2008 1:50 AM
> To: Roland Dreier
> Cc: general at lists.openfabrics.org
> Subject: [ofa-general] [PATCH 7/8 V3] core: Add XRC
> receive-only qp support
>
> ib/core:  Implement XRC receive-only QPs for userspace apps.
>
> Added creation of XRC receive-only QPs for userspace, which
> reside in kernel space (user cannot post-to or poll these QPs).
>
> Motivation:  MPI community requires XRC receive QPs which
> will not be destroyed when the creating process terminates.
>
> Solution:  Userspace requests that a QP be created in kernel space.
> Each userspace process using that QP (i.e. receiving packets
> on an XRC SRQ via the qp), registers with that QP (-- the
> creator is also registered, whether or not it is a user of
> the QP). When the last userspace user unregisters with the
> QP, it is destroyed.  Unregistration is also part of
> userspace process cleanup, so there is no leakage.
>
> This patch implements the kernel procedures to implement the following
> (new) libibverbs API:
> ibv_create_xrc_rcv_qp
> ibv_modify_xrc_rcv_qp
> ibv_query_xrc_rcv_qp
> ibv_reg_xrc_rcv_qp
> ibv_unreg_xrc_rcv_qp
>
> In addition, the patch implements the foundation for
> distributing XRC-receive-only QP events to userspace
> processes registered with that QP.
>
> Finally, the patch modifies ib_uverbs_close_xrc_domain() to
> return BUSY if any resources are still in use by the process,
> so that the XRC rcv-only QP cleanup can operate properly.
>
> V2:
> Fixed bug in ib_uverbs_close_xrc_domain.
> We need to allow the process to successfully close its copy
> of the domain, even if it still has undestroyed XRC QPs --
> these will continue to operate, although it will not be
> possible to create new ones (there will be no Oops).
>
> However, we need to check that there are no outstanding
> xrc-qp-registrations:
> the cleanup procedure for this depends on the xrc domain
> still being accessible in this process in order to perform
> all needed un-registrations (and thus prevent resource leakage).
>
> V3:
> Fix thinko in ib_uverbs_reg_xrc_rcv_qp,
> ib_uverbs_unreg_xrc_rcv_qp, and ib_uverbs_modify_xrc_rcv_qp:
> on success, incorrectly returned 0 instead of input length.
>
> Signed-off-by: Jack Morgenstein <jackm at dev.mellanox.co.il>
>
> Index: infiniband/include/rdma/ib_verbs.h
> ===================================================================
> --- infiniband.orig/include/rdma/ib_verbs.h     2008-01-28
> 12:20:55.000000000 +0200
> +++ infiniband/include/rdma/ib_verbs.h  2008-01-28 12:22:09.000000000
> +++ +0200
> @@ -285,6 +285,10 @@ enum ib_event_type {
>         IB_EVENT_CLIENT_REREGISTER
>  };
>
> +enum ib_event_flags {
> +       IB_XRC_QP_EVENT_FLAG = 0x80000000, };
> +
>  struct ib_event {
>         struct ib_device        *device;
>         union {
> @@ -292,6 +296,7 @@ struct ib_event {
>                 struct ib_qp    *qp;
>                 struct ib_srq   *srq;
>                 u8              port_num;
> +               u32             xrc_qp_num;
>         } element;
>         enum ib_event_type      event;
>  };
> @@ -492,6 +497,7 @@ enum ib_qp_type {
>
>  enum qp_create_flags {
>         QP_CREATE_LSO = 1 << 0,
> +       XRC_RCV_QP    = 1 << 1,
>  };
>
>  struct ib_qp_init_attr {
> @@ -723,6 +729,7 @@ struct ib_ucontext {
>         struct list_head        srq_list;
>         struct list_head        ah_list;
>         struct list_head        xrc_domain_list;
> +       struct list_head        xrc_reg_qp_list;
>         int                     closing;
>  };
>
> @@ -744,6 +751,12 @@ struct ib_udata {
>         size_t       outlen;
>  };
>
> +struct ib_uxrc_rcv_object {
> +       struct list_head        list;           /* link to
> context's list */
> +       u32                     qp_num;
> +       u32                     domain_handle;
> +};
> +
>  struct ib_pd {
>         struct ib_device       *device;
>         struct ib_uobject      *uobject;
> @@ -1053,6 +1066,23 @@ struct ib_device {
>                                                  struct
> ib_ucontext *context,
>                                                  struct
> ib_udata *udata);
>         int                        (*dealloc_xrcd)(struct
> ib_xrcd *xrcd);
> +       int
> (*create_xrc_rcv_qp)(struct ib_qp_init_attr *init_attr,
> +                                                       u32* qp_num);
> +       int
> (*modify_xrc_rcv_qp)(struct ib_xrcd *xrcd,
> +                                                       u32 qp_num,
> +
> struct ib_qp_attr *attr,
> +                                                       int
> attr_mask);
> +       int                        (*query_xrc_rcv_qp)(struct
> ib_xrcd *xrcd,
> +                                                      u32 qp_num,
> +                                                      struct
> ib_qp_attr *attr,
> +                                                      int attr_mask,
> +                                                      struct
> ib_qp_init_attr *init_attr);
> +       int                        (*reg_xrc_rcv_qp)(struct
> ib_xrcd *xrcd,
> +                                                    void *context,
> +                                                    u32 qp_num);
> +       int                        (*unreg_xrc_rcv_qp)(struct
> ib_xrcd *xrcd,
> +                                                      void *context,
> +                                                      u32 qp_num);
>
>         struct ib_dma_mapping_ops   *dma_ops;
>
> Index: infiniband/drivers/infiniband/core/uverbs_main.c
> ===================================================================
> --- infiniband.orig/drivers/infiniband/core/uverbs_main.c
>   2008-01-28 12:20:55.000000000 +0200
> +++ infiniband/drivers/infiniband/core/uverbs_main.c
> 2008-01-28 12:20:56.000000000 +0200
> @@ -114,6 +114,11 @@ static ssize_t (*uverbs_cmd_table[])(str
>         [IB_USER_VERBS_CMD_CREATE_XRC_SRQ]      =
> ib_uverbs_create_xrc_srq,
>         [IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN]     =
> ib_uverbs_open_xrc_domain,
>         [IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN]    =
> ib_uverbs_close_xrc_domain,
> +       [IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP]   =
> ib_uverbs_create_xrc_rcv_qp,
> +       [IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP]   =
> ib_uverbs_modify_xrc_rcv_qp,
> +       [IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP]    =
> ib_uverbs_query_xrc_rcv_qp,
> +       [IB_USER_VERBS_CMD_REG_XRC_RCV_QP]      =
> ib_uverbs_reg_xrc_rcv_qp,
> +       [IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP]    =
> ib_uverbs_unreg_xrc_rcv_qp,
>  };
>
>  static struct vfsmount *uverbs_event_mnt; @@ -191,6 +196,7
> @@ static int ib_uverbs_cleanup_ucontext(st
>                                       struct ib_ucontext *context)  {
>         struct ib_uobject *uobj, *tmp;
> +       struct ib_uxrc_rcv_object *xrc_qp_obj, *tmp1;
>
>         if (!context)
>                 return 0;
> @@ -251,6 +257,13 @@ static int ib_uverbs_cleanup_ucontext(st
>                 kfree(uobj);
>         }
>
> +       list_for_each_entry_safe(xrc_qp_obj, tmp1,
> &context->xrc_reg_qp_list, list) {
> +               list_del(&xrc_qp_obj->list);
> +               ib_uverbs_cleanup_xrc_rcv_qp(file,
> xrc_qp_obj->domain_handle,
> +                                            xrc_qp_obj->qp_num);
> +               kfree(xrc_qp_obj);
> +       }
> +
>         mutex_lock(&file->device->ib_dev->xrcd_table_mutex);
>         list_for_each_entry_safe(uobj, tmp,
> &context->xrc_domain_list, list) {
>                 struct ib_xrcd *xrcd = uobj->object; @@
> -506,6 +519,12 @@ void ib_uverbs_event_handler(struct ib_e
>                                 NULL, NULL);  }
>
> +void ib_uverbs_xrc_rcv_qp_event_handler(struct ib_event *event, void
> +*context_ptr) {
> +       ib_uverbs_async_handler(context_ptr,
> event->element.xrc_qp_num,
> +                               event->event, NULL, NULL); }
> +
>  struct file *ib_uverbs_alloc_event_file(struct
> ib_uverbs_file *uverbs_file,
>                                         int is_async, int *fd)  {
> Index: infiniband/drivers/infiniband/core/uverbs_cmd.c
> ===================================================================
> --- infiniband.orig/drivers/infiniband/core/uverbs_cmd.c
>   2008-01-28 12:20:55.000000000 +0200
> +++ infiniband/drivers/infiniband/core/uverbs_cmd.c
> 2008-01-28 12:20:56.000000000 +0200
> @@ -315,6 +315,7 @@ ssize_t ib_uverbs_get_context(struct ib_
>         INIT_LIST_HEAD(&ucontext->srq_list);
>         INIT_LIST_HEAD(&ucontext->ah_list);
>         INIT_LIST_HEAD(&ucontext->xrc_domain_list);
> +       INIT_LIST_HEAD(&ucontext->xrc_reg_qp_list);
>         ucontext->closing = 0;
>
>         resp.num_comp_vectors =
> file->device->num_comp_vectors; @@ -1080,6 +1081,7 @@ ssize_t
> ib_uverbs_create_qp(struct ib_uv
>                 goto err_put;
>         }
>
> +       attr.create_flags  = 0;
>         attr.event_handler = ib_uverbs_qp_event_handler;
>         attr.qp_context    = file;
>         attr.send_cq       = scq;
> @@ -2561,6 +2563,7 @@ ssize_t ib_uverbs_close_xrc_domain(struc
>                                    int out_len)  {
>         struct ib_uverbs_close_xrc_domain cmd;
> +       struct ib_uxrc_rcv_object  *tmp;
>         struct ib_uobject          *uobj;
>         struct ib_xrcd             *xrcd = NULL;
>         struct inode               *inode = NULL;
> @@ -2576,6 +2579,18 @@ ssize_t ib_uverbs_close_xrc_domain(struc
>                 goto err_unlock_mutex;
>         }
>
> +       mutex_lock(&file->mutex);
> +       list_for_each_entry(tmp,
> &file->ucontext->xrc_reg_qp_list, list)
> +               if (cmd.xrcd_handle == tmp->domain_handle) {
> +                       ret = -EBUSY;
> +                       break;
> +               }
> +       mutex_unlock(&file->mutex);
> +       if (ret) {
> +               put_uobj_write(uobj);
> +               goto err_unlock_mutex;
> +       }
> +
>         xrcd = (struct ib_xrcd *) (uobj->object);
>         inode = xrcd->inode;
>
> @@ -2611,7 +2626,7 @@ err_unlock_mutex:
>  }
>
>  void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev,
> -                                  struct ib_xrcd *xrcd)
> +                           struct ib_xrcd *xrcd)
>  {
>         struct inode               *inode = NULL;
>         int                         ret = 0;
> @@ -2625,4 +2640,353 @@ void ib_uverbs_dealloc_xrcd(struct ib_de
>                 xrcd_table_delete(ib_dev, inode);  }
>
> +ssize_t ib_uverbs_create_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                           const char __user *buf, int in_len,
> +                           int out_len) {
> +       struct ib_uverbs_create_xrc_rcv_qp      cmd;
> +       struct ib_uverbs_create_xrc_rcv_qp_resp resp;
> +       struct ib_uxrc_rcv_object      *obj;
> +       struct ib_qp_init_attr          init_attr;
> +       struct ib_xrcd                 *xrcd;
> +       struct ib_uobject              *xrcd_uobj;
> +       u32                             qp_num;
> +       int                             err;
> +
> +       if (out_len < sizeof resp)
> +               return -ENOSPC;
> +
> +       if (copy_from_user(&cmd, buf, sizeof cmd))
> +               return -EFAULT;
> +
> +       obj = kmalloc(sizeof *obj, GFP_KERNEL);
> +       if (!obj)
> +               return -ENOMEM;
> +
> +       xrcd = idr_read_xrcd(cmd.xrc_domain_handle,
> file->ucontext, &xrcd_uobj);
> +       if (!xrcd) {
> +               err = -EINVAL;
> +               goto err_out;
> +       }
> +
> +       memset(&init_attr, 0, sizeof init_attr);
> +       init_attr.event_handler = ib_uverbs_xrc_rcv_qp_event_handler;
> +       init_attr.qp_context    = file;
> +       init_attr.srq           = NULL;
> +       init_attr.sq_sig_type   = cmd.sq_sig_all ?
> IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
> +       init_attr.qp_type       = IB_QPT_XRC;
> +       init_attr.xrc_domain    = xrcd;
> +       init_attr.create_flags  = XRC_RCV_QP;
> +
> +       init_attr.cap.max_send_wr       = 1;
> +       init_attr.cap.max_recv_wr       = 0;
> +       init_attr.cap.max_send_sge      = 1;
> +       init_attr.cap.max_recv_sge      = 0;
> +       init_attr.cap.max_inline_data   = 0;
> +
> +       err = xrcd->device->create_xrc_rcv_qp(&init_attr, &qp_num);
> +       if (err)
> +               goto err_put;
> +
> +       memset(&resp, 0, sizeof resp);
> +       resp.qpn = qp_num;
> +
> +       if (copy_to_user((void __user *) (unsigned long) cmd.response,
> +                        &resp, sizeof resp)) {
> +               err = -EFAULT;
> +               goto err_destroy;
> +       }
> +
> +       atomic_inc(&xrcd->usecnt);
> +       put_xrcd_read(xrcd_uobj);
> +       obj->qp_num = qp_num;
> +       obj->domain_handle = cmd.xrc_domain_handle;
> +       mutex_lock(&file->mutex);
> +       list_add_tail(&obj->list, &file->ucontext->xrc_reg_qp_list);
> +       mutex_unlock(&file->mutex);
> +
> +       return in_len;
> +
> +err_destroy:
> +       xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num);
> +err_put:
> +       put_xrcd_read(xrcd_uobj);
> +err_out:
> +       kfree(obj);
> +       return err;
> +}
> +
> +ssize_t ib_uverbs_modify_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                           const char __user *buf, int in_len,
> +                           int out_len) {
> +       struct ib_uverbs_modify_xrc_rcv_qp      cmd;
> +       struct ib_qp_attr              *attr;
> +       struct ib_xrcd                 *xrcd;
> +       struct ib_uobject              *xrcd_uobj;
> +       int                             err;
> +
> +       if (copy_from_user(&cmd, buf, sizeof cmd))
> +               return -EFAULT;
> +
> +       attr = kmalloc(sizeof *attr, GFP_KERNEL);
> +       if (!attr)
> +               return -ENOMEM;
> +
> +       xrcd = idr_read_xrcd(cmd.xrc_domain_handle,
> file->ucontext, &xrcd_uobj);
> +       if (!xrcd) {
> +               kfree(attr);
> +               return -EINVAL;
> +       }
> +
> +       memset(attr, 0, sizeof *attr);
> +       attr->qp_state            = cmd.qp_state;
> +       attr->cur_qp_state        = cmd.cur_qp_state;
> +       attr->qp_access_flags     = cmd.qp_access_flags;
> +       attr->pkey_index          = cmd.pkey_index;
> +       attr->port_num            = cmd.port_num;
> +       attr->path_mtu            = cmd.path_mtu;
> +       attr->path_mig_state      = cmd.path_mig_state;
> +       attr->qkey                = cmd.qkey;
> +       attr->rq_psn              = cmd.rq_psn;
> +       attr->sq_psn              = cmd.sq_psn;
> +       attr->dest_qp_num         = cmd.dest_qp_num;
> +       attr->alt_pkey_index      = cmd.alt_pkey_index;
> +       attr->en_sqd_async_notify = cmd.en_sqd_async_notify;
> +       attr->max_rd_atomic       = cmd.max_rd_atomic;
> +       attr->max_dest_rd_atomic  = cmd.max_dest_rd_atomic;
> +       attr->min_rnr_timer       = cmd.min_rnr_timer;
> +       attr->port_num            = cmd.port_num;
> +       attr->timeout             = cmd.timeout;
> +       attr->retry_cnt           = cmd.retry_cnt;
> +       attr->rnr_retry           = cmd.rnr_retry;
> +       attr->alt_port_num        = cmd.alt_port_num;
> +       attr->alt_timeout         = cmd.alt_timeout;
> +
> +       memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16);
> +       attr->ah_attr.grh.flow_label        = cmd.dest.flow_label;
> +       attr->ah_attr.grh.sgid_index        = cmd.dest.sgid_index;
> +       attr->ah_attr.grh.hop_limit         = cmd.dest.hop_limit;
> +       attr->ah_attr.grh.traffic_class     = cmd.dest.traffic_class;
> +       attr->ah_attr.dlid                  = cmd.dest.dlid;
> +       attr->ah_attr.sl                    = cmd.dest.sl;
> +       attr->ah_attr.src_path_bits         = cmd.dest.src_path_bits;
> +       attr->ah_attr.static_rate           = cmd.dest.static_rate;
> +       attr->ah_attr.ah_flags              =
> cmd.dest.is_global ? IB_AH_GRH : 0;
> +       attr->ah_attr.port_num              = cmd.dest.port_num;
> +
> +       memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16);
> +       attr->alt_ah_attr.grh.flow_label    = cmd.alt_dest.flow_label;
> +       attr->alt_ah_attr.grh.sgid_index    = cmd.alt_dest.sgid_index;
> +       attr->alt_ah_attr.grh.hop_limit     = cmd.alt_dest.hop_limit;
> +       attr->alt_ah_attr.grh.traffic_class =
> cmd.alt_dest.traffic_class;
> +       attr->alt_ah_attr.dlid              = cmd.alt_dest.dlid;
> +       attr->alt_ah_attr.sl                = cmd.alt_dest.sl;
> +       attr->alt_ah_attr.src_path_bits     =
> cmd.alt_dest.src_path_bits;
> +       attr->alt_ah_attr.static_rate       =
> cmd.alt_dest.static_rate;
> +       attr->alt_ah_attr.ah_flags          =
> cmd.alt_dest.is_global ? IB_AH_GRH : 0;
> +       attr->alt_ah_attr.port_num          = cmd.alt_dest.port_num;
> +
> +       err = xrcd->device->modify_xrc_rcv_qp(xrcd,
> cmd.qp_num, attr, cmd.attr_mask);
> +       put_xrcd_read(xrcd_uobj);
> +       kfree(attr);
> +       return err ? err : in_len;
> +}
> +
> +ssize_t ib_uverbs_query_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                                  const char __user *buf, int in_len,
> +                                  int out_len) {
> +       struct ib_uverbs_query_xrc_rcv_qp cmd;
> +       struct ib_uverbs_query_qp_resp    resp;
> +       struct ib_qp_attr               *attr;
> +       struct ib_qp_init_attr          *init_attr;
> +       struct ib_xrcd                  *xrcd;
> +       struct ib_uobject               *xrcd_uobj;
> +       int                              ret;
> +
> +       if (copy_from_user(&cmd, buf, sizeof cmd))
> +               return -EFAULT;
> +
> +       attr      = kmalloc(sizeof *attr, GFP_KERNEL);
> +       init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
> +       if (!attr || !init_attr) {
> +               ret = -ENOMEM;
> +               goto out;
> +       }
> +
> +       xrcd = idr_read_xrcd(cmd.xrc_domain_handle,
> file->ucontext, &xrcd_uobj);
> +       if (!xrcd) {
> +               ret = -EINVAL;
> +               goto out;
> +       }
> +
> +       ret = xrcd->device->query_xrc_rcv_qp(xrcd, cmd.qp_num, attr,
> +                                            cmd.attr_mask,
> init_attr);
> +
> +       put_xrcd_read(xrcd_uobj);
> +
> +       if (ret)
> +               goto out;
> +
> +       memset(&resp, 0, sizeof resp);
> +       resp.qp_state               = attr->qp_state;
> +       resp.cur_qp_state           = attr->cur_qp_state;
> +       resp.path_mtu               = attr->path_mtu;
> +       resp.path_mig_state         = attr->path_mig_state;
> +       resp.qkey                   = attr->qkey;
> +       resp.rq_psn                 = attr->rq_psn;
> +       resp.sq_psn                 = attr->sq_psn;
> +       resp.dest_qp_num            = attr->dest_qp_num;
> +       resp.qp_access_flags        = attr->qp_access_flags;
> +       resp.pkey_index             = attr->pkey_index;
> +       resp.alt_pkey_index         = attr->alt_pkey_index;
> +       resp.sq_draining            = attr->sq_draining;
> +       resp.max_rd_atomic          = attr->max_rd_atomic;
> +       resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
> +       resp.min_rnr_timer          = attr->min_rnr_timer;
> +       resp.port_num               = attr->port_num;
> +       resp.timeout                = attr->timeout;
> +       resp.retry_cnt              = attr->retry_cnt;
> +       resp.rnr_retry              = attr->rnr_retry;
> +       resp.alt_port_num           = attr->alt_port_num;
> +       resp.alt_timeout            = attr->alt_timeout;
> +
> +       memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
> +       resp.dest.flow_label        = attr->ah_attr.grh.flow_label;
> +       resp.dest.sgid_index        = attr->ah_attr.grh.sgid_index;
> +       resp.dest.hop_limit         = attr->ah_attr.grh.hop_limit;
> +       resp.dest.traffic_class     = attr->ah_attr.grh.traffic_class;
> +       resp.dest.dlid              = attr->ah_attr.dlid;
> +       resp.dest.sl                = attr->ah_attr.sl;
> +       resp.dest.src_path_bits     = attr->ah_attr.src_path_bits;
> +       resp.dest.static_rate       = attr->ah_attr.static_rate;
> +       resp.dest.is_global         =
> !!(attr->ah_attr.ah_flags & IB_AH_GRH);
> +       resp.dest.port_num          = attr->ah_attr.port_num;
> +
> +       memcpy(resp.alt_dest.dgid,
> attr->alt_ah_attr.grh.dgid.raw, 16);
> +       resp.alt_dest.flow_label    =
> attr->alt_ah_attr.grh.flow_label;
> +       resp.alt_dest.sgid_index    =
> attr->alt_ah_attr.grh.sgid_index;
> +       resp.alt_dest.hop_limit     = attr->alt_ah_attr.grh.hop_limit;
> +       resp.alt_dest.traffic_class =
> attr->alt_ah_attr.grh.traffic_class;
> +       resp.alt_dest.dlid          = attr->alt_ah_attr.dlid;
> +       resp.alt_dest.sl            = attr->alt_ah_attr.sl;
> +       resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
> +       resp.alt_dest.static_rate   = attr->alt_ah_attr.static_rate;
> +       resp.alt_dest.is_global     =
> !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH);
> +       resp.alt_dest.port_num      = attr->alt_ah_attr.port_num;
> +
> +       resp.max_send_wr            = init_attr->cap.max_send_wr;
> +       resp.max_recv_wr            = init_attr->cap.max_recv_wr;
> +       resp.max_send_sge           = init_attr->cap.max_send_sge;
> +       resp.max_recv_sge           = init_attr->cap.max_recv_sge;
> +       resp.max_inline_data        = init_attr->cap.max_inline_data;
> +       resp.sq_sig_all             = init_attr->sq_sig_type
> == IB_SIGNAL_ALL_WR;
> +
> +       if (copy_to_user((void __user *) (unsigned long) cmd.response,
> +                        &resp, sizeof resp))
> +               ret = -EFAULT;
> +
> +out:
> +       kfree(attr);
> +       kfree(init_attr);
> +
> +       return ret ? ret : in_len;
> +}
> +
> +ssize_t ib_uverbs_reg_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                           const char __user *buf, int in_len,
> +                           int out_len) {
> +       struct ib_uverbs_reg_xrc_rcv_qp  cmd;
> +       struct ib_uxrc_rcv_object       *qp_obj, *tmp;
> +       struct ib_xrcd                  *xrcd;
> +       struct ib_uobject               *xrcd_uobj;
> +       int                              ret;
> +
> +       if (copy_from_user(&cmd, buf, sizeof cmd))
> +               return -EFAULT;
> +
> +       qp_obj = kmalloc(sizeof *qp_obj, GFP_KERNEL);
> +       if (!qp_obj)
> +               return -ENOMEM;
> +
> +       xrcd = idr_read_xrcd(cmd.xrc_domain_handle,
> file->ucontext, &xrcd_uobj);
> +       if (!xrcd) {
> +               ret = -EINVAL;
> +               goto err_out;
> +       }
> +
> +       ret = xrcd->device->reg_xrc_rcv_qp(xrcd, file, cmd.qp_num);
> +       if (ret)
> +               goto err_put;
> +
> +       atomic_inc(&xrcd->usecnt);
> +       put_xrcd_read(xrcd_uobj);
> +       mutex_lock(&file->mutex);
> +       list_for_each_entry(tmp,
> &file->ucontext->xrc_reg_qp_list, list)
> +               if (cmd.qp_num == tmp->qp_num) {
> +                       kfree(qp_obj);
> +                       mutex_unlock(&file->mutex);
> +                       put_xrcd_read(xrcd_uobj);
> +                       return 0;
> +               }
> +       qp_obj->qp_num = cmd.qp_num;
> +       qp_obj->domain_handle = cmd.xrc_domain_handle;
> +       list_add_tail(&qp_obj->list,
> &file->ucontext->xrc_reg_qp_list);
> +       mutex_unlock(&file->mutex);
> +       return in_len;
> +
> +err_put:
> +       put_xrcd_read(xrcd_uobj);
> +err_out:
> +
> +       kfree(qp_obj);
> +       return ret;
> +}
> +
> +int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                                u32 domain_handle, u32 qp_num) {
> +       struct ib_xrcd *xrcd;
> +       struct ib_uobject *xrcd_uobj;
> +       int err;
> +
> +       xrcd = idr_read_xrcd(domain_handle, file->ucontext,
> &xrcd_uobj);
> +       if (!xrcd)
> +               return -EINVAL;
>
> +       err = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num);
> +
> +       if (!err)
> +               atomic_dec(&xrcd->usecnt);
> +       put_xrcd_read(xrcd_uobj);
> +       return err;
> +}
> +
> +ssize_t ib_uverbs_unreg_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                           const char __user *buf, int in_len,
> +                           int out_len) {
> +       struct ib_uverbs_unreg_xrc_rcv_qp cmd;
> +       struct ib_uxrc_rcv_object *qp_obj, *tmp;
> +       int ret;
> +
> +       if (copy_from_user(&cmd, buf, sizeof cmd))
> +               return -EFAULT;
> +
> +       ret = ib_uverbs_cleanup_xrc_rcv_qp(file,
> cmd.xrc_domain_handle, cmd.qp_num);
> +       if (ret)
> +               return ret;
> +
> +       mutex_lock(&file->mutex);
> +       list_for_each_entry_safe(qp_obj, tmp,
> &file->ucontext->xrc_reg_qp_list, list)
> +               if (cmd.qp_num == qp_obj->qp_num) {
> +                       list_del(&qp_obj->list);
> +                       kfree(qp_obj);
> +                       break;
> +               }
> +       mutex_unlock(&file->mutex);
> +       return in_len;
> +
> +}
> Index: infiniband/include/rdma/ib_user_verbs.h
> ===================================================================
> --- infiniband.orig/include/rdma/ib_user_verbs.h
> 2008-01-28 12:20:54.000000000 +0200
> +++ infiniband/include/rdma/ib_user_verbs.h     2008-01-28
> 12:20:56.000000000 +0200
> @@ -86,7 +86,12 @@ enum {
>         IB_USER_VERBS_CMD_POST_SRQ_RECV,
>         IB_USER_VERBS_CMD_CREATE_XRC_SRQ,
>         IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN,
> -       IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN
> +       IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN,
> +       IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP,
> +       IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP,
> +       IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
> +       IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
> +       IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
>  };
>
>  /*
> @@ -714,6 +719,76 @@ struct ib_uverbs_close_xrc_domain {
>         __u64 driver_data[0];
>  };
>
> +struct ib_uverbs_create_xrc_rcv_qp {
> +       __u64 response;
> +       __u64 user_handle;
> +       __u32 xrc_domain_handle;
> +       __u32 max_send_wr;
> +       __u32 max_recv_wr;
> +       __u32 max_send_sge;
> +       __u32 max_recv_sge;
> +       __u32 max_inline_data;
> +       __u8  sq_sig_all;
> +       __u8  qp_type;
> +       __u8  reserved[2];
> +       __u64 driver_data[0];
> +};
> +
> +struct ib_uverbs_create_xrc_rcv_qp_resp {
> +       __u32 qpn;
> +       __u32 reserved;
> +};
> +
> +struct ib_uverbs_modify_xrc_rcv_qp {
> +       __u32 xrc_domain_handle;
> +       __u32 qp_num;
> +       struct ib_uverbs_qp_dest dest;
> +       struct ib_uverbs_qp_dest alt_dest;
> +       __u32 attr_mask;
> +       __u32 qkey;
> +       __u32 rq_psn;
> +       __u32 sq_psn;
> +       __u32 dest_qp_num;
> +       __u32 qp_access_flags;
> +       __u16 pkey_index;
> +       __u16 alt_pkey_index;
> +       __u8  qp_state;
> +       __u8  cur_qp_state;
> +       __u8  path_mtu;
> +       __u8  path_mig_state;
> +       __u8  en_sqd_async_notify;
> +       __u8  max_rd_atomic;
> +       __u8  max_dest_rd_atomic;
> +       __u8  min_rnr_timer;
> +       __u8  port_num;
> +       __u8  timeout;
> +       __u8  retry_cnt;
> +       __u8  rnr_retry;
> +       __u8  alt_port_num;
> +       __u8  alt_timeout;
> +       __u8  reserved[2];
> +       __u64 driver_data[0];
> +};
> +
> +struct ib_uverbs_query_xrc_rcv_qp {
> +       __u64 response;
> +       __u32 xrc_domain_handle;
> +       __u32 qp_num;
> +       __u32 attr_mask;
> +       __u64 driver_data[0];
> +};
> +
> +struct ib_uverbs_reg_xrc_rcv_qp {
> +       __u32 xrc_domain_handle;
> +       __u32 qp_num;
> +       __u64 driver_data[0];
> +};
> +
> +struct ib_uverbs_unreg_xrc_rcv_qp {
> +       __u32 xrc_domain_handle;
> +       __u32 qp_num;
> +       __u64 driver_data[0];
> +};
>
>
>  #endif /* IB_USER_VERBS_H */
> Index: infiniband/drivers/infiniband/core/uverbs.h
> ===================================================================
> --- infiniband.orig/drivers/infiniband/core/uverbs.h
> 2008-01-28 12:20:55.000000000 +0200
> +++ infiniband/drivers/infiniband/core/uverbs.h 2008-01-28
> +++ 12:20:56.000000000 +0200
> @@ -163,8 +163,12 @@ void ib_uverbs_qp_event_handler(struct i
>  void ib_uverbs_srq_event_handler(struct ib_event *event,
> void *context_ptr);  void ib_uverbs_event_handler(struct
> ib_event_handler *handler,
>                              struct ib_event *event);
> +void ib_uverbs_xrc_rcv_qp_event_handler(struct ib_event *event,
> +                                       void *context_ptr);
>  void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev,
>                             struct ib_xrcd *xrcd);
> +int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file,
> +                                u32 domain_handle, u32 qp_num);
>
>  #define IB_UVERBS_DECLARE_CMD(name)
>           \
>         ssize_t ib_uverbs_##name(struct ib_uverbs_file *file,
>           \
> @@ -202,6 +206,11 @@ IB_UVERBS_DECLARE_CMD(destroy_srq);
>  IB_UVERBS_DECLARE_CMD(create_xrc_srq);
>  IB_UVERBS_DECLARE_CMD(open_xrc_domain);
>  IB_UVERBS_DECLARE_CMD(close_xrc_domain);
> +IB_UVERBS_DECLARE_CMD(create_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(modify_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(query_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(reg_xrc_rcv_qp);
> +IB_UVERBS_DECLARE_CMD(unreg_xrc_rcv_qp);
>
>
>  #endif /* UVERBS_H */
>
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit
> http://openib.org/mailman/listinfo/openib-general
>



More information about the general mailing list