[openib-general] PATCH creation of ib_verbs.c file

Sean Hefty mshefty at ichips.intel.com
Fri Aug 27 15:28:07 PDT 2004


Below is a patch to create a new file, ib_verbs.c.  Several non-performance critical calls were relocated into the file and updated based on changes in Roland's branch.

Roland, here's a list of differences between this patch and your branch:

ib_create_qp / ib_destroy_qp - include support for srq

optional calls are not yet checked - I'll start a separate discussion on this.

ib_create_cq / ib_resize_cq - I was thinking that that device driver would set the struct cq.cqe value directly, rather than returning a changed &cqe value and requiring the access layer to set it.

ib_rereg_phys_mr - checks for bound mw's and pd changes

I only moved calls from ib_verbs.h to ib_verbs.c that were more than simple pass-through calls.

- Sean

Index: ib_verbs.h
===================================================================
--- ib_verbs.h	(revision 691)
+++ ib_verbs.h	(working copy)
@@ -72,7 +72,6 @@
 struct ib_ah {
 	struct ib_device	*device;
 	struct ib_pd		*pd;
-	atomic_t		usecnt;
 };
 
 struct ib_cq {
@@ -95,9 +94,9 @@
 	struct ib_pd		*pd;
 	struct ib_cq		*send_cq;
 	struct ib_cq		*recv_cq;
+	struct ib_srq		*srq;
 	void			*qp_context;
 	u32			qp_num;
-	atomic_t		usecnt;
 };
 
 struct ib_mr {
@@ -112,7 +111,6 @@
 	struct ib_device	*device;
 	struct ib_pd		*pd;
 	u32			rkey;
-	atomic_t		usecnt;
 };
 
 struct ib_fmr {
@@ -782,22 +780,6 @@
 				   port_modify);
 }
 
-static inline struct ib_pd *ib_alloc_pd(struct ib_device *device)
-{
-	return device->alloc_pd(device);
-}
-
-static inline int ib_dealloc_pd(struct ib_pd *pd)
-{
-	return pd->device->dealloc_pd(pd);
-}
-
-static inline struct ib_ah *ib_create_ah(struct ib_pd *pd,
-					 struct ib_ah_attr *ah_attr)
-{
-	return pd->device->create_ah(pd, ah_attr);
-}
-
 static inline int ib_modify_ah(struct ib_ah *ah,
 			       struct ib_ah_attr *ah_attr)
 {
@@ -810,18 +792,6 @@
 	return ah->device->query_ah(ah, ah_attr);
 }
 
-static inline int ib_destroy_ah(struct ib_ah *ah)
-{
-	return ah->device->destroy_ah(ah);
-}
-
-static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
-					 struct ib_qp_init_attr *qp_init_attr,
-					 struct ib_qp_cap *qp_cap)
-{
-	return pd->device->create_qp(pd, qp_init_attr, qp_cap);
-}
-
 static inline int ib_modify_qp(struct ib_qp *qp,
 			       struct ib_qp_attr *qp_attr,
 			       int qp_attr_mask,
@@ -838,18 +808,6 @@
 	return qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr);
 }
 
-static inline int ib_destroy_qp(struct ib_qp *qp)
-{
-	return qp->device->destroy_qp(qp);
-}
-
-static inline struct ib_srq *ib_create_srq(struct ib_pd *pd,
-					   void *srq_context,
-					   struct ib_srq_attr *srq_attr)
-{
-	return pd->device->create_srq(pd, srq_context, srq_attr);
-}
-
 static inline int ib_query_srq(struct ib_srq *srq,
 			       struct ib_srq_attr *srq_attr)
 {
@@ -864,11 +822,6 @@
 	return srq->device->modify_srq(srq, pd, srq_attr, srq_attr_mask);
 }
 
-static inline int ib_destroy_srq(struct ib_srq *srq)
-{
-	return srq->device->destroy_srq(srq);
-}
-
 static inline int ib_post_srq(struct ib_srq *srq,
 			      struct ib_recv_wr *recv_wr,
 			      struct ib_recv_wr **bad_recv_wr)
@@ -876,65 +829,18 @@
 	return srq->device->post_srq(srq, recv_wr, bad_recv_wr);
 }
 
-static inline struct ib_cq *ib_create_cq(struct ib_device *device,
-					 ib_comp_handler comp_handler,
-					 void *cq_context,
-					 int cqe)
-{
-	return device->create_cq(device, comp_handler, cq_context, cqe);
-}
-
 static inline int ib_resize_cq(struct ib_cq *cq,
 			       int cqe)
 {
 	return cq->device->resize_cq(cq, cqe);
 }
 
-static inline int ib_destroy_cq(struct ib_cq *cq)
-{
-	return cq->device->destroy_cq(cq);
-}
-
-/* in functions below iova_start is in/out parameter */
-static inline struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
-					   struct ib_phys_buf *phys_buf_array,
-					   int num_phys_buf,
-					   int mr_access_flags,
-					   u64 *iova_start)
-{
-	return pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
-				       mr_access_flags, iova_start);
-}
-
 static inline int ib_query_mr(struct ib_mr *mr,
 			      struct ib_mr_attr *mr_attr)
 {
 	return mr->device->query_mr(mr, mr_attr);
 }
 
-static inline int ib_dereg_mr(struct ib_mr *mr)
-{
-	return mr->device->dereg_mr(mr);
-}
-
-static inline int ib_rereg_phys_mr(struct ib_mr *mr,
-				   int mr_rereg_mask,
-				   struct ib_pd *pd,
-				   struct ib_phys_buf *phys_buf_array,
-				   int num_phys_buf,
-				   int mr_access_flags,
-				   u64 *iova_start)
-{
-	return mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd, phys_buf_array,
-					 num_phys_buf, mr_access_flags,
-					 iova_start);
-}
-
-static inline struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
-{
-	return pd->device->alloc_mw(pd);
-}
-
 static inline int ib_bind_mw(struct ib_qp *qp,
 			     struct ib_mw *mw,
 			     struct ib_mw_bind *mw_bind)
@@ -942,18 +848,6 @@
 	return mw->device->bind_mw(qp, mw, mw_bind);
 }
 
-static inline int ib_dealloc_mw(struct ib_mw *mw)
-{
-	return mw->device->dealloc_mw(mw);
-}
-
-static inline struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
-					  int mr_access_flags,
-					  struct ib_fmr_attr *fmr_attr)
-{
-	return pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
-}
-
 static inline int ib_map_fmr(struct ib_fmr *fmr,
 			     void *addr,
 			     u64 size)
@@ -961,13 +855,6 @@
 	return fmr->device->map_fmr(fmr, addr, size);
 }
 
-static inline int ib_map_phys_fmr(struct ib_fmr *fmr,
-				  struct ib_phys_buf *phys_buf_array,
-				  int num_phys_buf)
-{
-	return fmr->device->map_phys_fmr(fmr, phys_buf_array, num_phys_buf);
-}
-
 static inline int ib_unmap_fmr(struct ib_fmr **fmr_array,
 			       int fmr_cnt)
 {
@@ -975,11 +862,6 @@
 	return fmr_array[0]->device->unmap_fmr(fmr_array, fmr_cnt);
 }
 
-static inline int ib_free_fmr(struct ib_fmr *fmr)
-{
-	return fmr->device->free_fmr(fmr);
-}
-
 static inline int ib_attach_mcast(struct ib_qp *qp,
 				  union ib_gid *gid,
 				  u16 lid)
Index: TODO
===================================================================
--- TODO	(revision 691)
+++ TODO	(working copy)
@@ -1,11 +1,10 @@
 Verbs TODOs:
 
+	- Hey there!  Howz about a makefile?
 	- Ensure ib_mod_qp can change all QP parameters - check resize.
 	- Determine proper value for static_rate - match CM or inter-packet
 	  delay.  Can an abstracted value be easier to use?
-	- Need to define struct ib_mw_bind.
 	- Optional calls need checks before invoking device driver.
-	- Migrate non-speed path routines into .c file.
 	- Add comments for API.
 	- Should ib_unmap_fmr take fmr_array as input, or just fmr?
 	  What should the restriction on the fmr_array be?  All from same
Index: ib_verbs.c
===================================================================
--- ib_verbs.c	(revision 0)
+++ ib_verbs.c	(revision 0)
@@ -0,0 +1,338 @@
+/*
+  This software is available to you under a choice of one of two
+  licenses.  You may choose to be licensed under the terms of the GNU
+  General Public License (GPL) Version 2, available at
+  <http://www.fsf.org/copyleft/gpl.html>, or the OpenIB.org BSD
+  license, available in the LICENSE.TXT file accompanying this
+  software.  These details are also available at
+  <http://openib.org/license.html>.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+  BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+  ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+
+  Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
+  Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
+  Copyright (c) 2004 Intel Corporation.  All rights reserved.
+  Copyright (c) 2004 Topspin Corporation.  All rights reserved.
+  Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
+*/
+
+#include <linux/errno.h>
+#include <ib_verbs.h>
+
+struct ib_pd *ib_alloc_pd(struct ib_device *device)
+{
+	struct ib_pd *pd;
+
+	pd = device->alloc_pd(device);
+
+	if (!IS_ERR(pd)) {
+		pd->device = device;
+		atomic_set(&pd->usecnt, 0);
+	}
+
+	return pd;
+}
+EXPORT_SYMBOL(ib_alloc_pd);
+
+int ib_dealloc_pd(struct ib_pd *pd)
+{
+	if (atomic_read(&pd->usecnt))
+		return -EBUSY;
+
+	return pd->device->dealloc_pd(pd);
+}
+EXPORT_SYMBOL(ib_dealloc_pd);
+
+struct ib_ah *ib_create_ah(struct ib_pd *pd,
+			   struct ib_ah_attr *ah_attr)
+{
+	struct ib_ah *ah;
+
+	ah = pd->device->create_ah(pd, ah_attr);
+
+	if (!IS_ERR(ah)) {
+		ah->device = pd->device;
+		ah->pd     = pd;
+		atomic_inc(&pd->usecnt);
+	}
+
+	return ah;
+}
+EXPORT_SYMBOL(ib_create_ah);
+
+int ib_destroy_ah(struct ib_ah *ah)
+{
+	struct ib_pd *pd;
+	int ret;
+
+	pd = ah->pd;
+
+	ret = ah->device->destroy_ah(ah);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_destroy_ah);
+
+struct ib_qp *ib_create_qp(struct ib_pd *pd,
+			   struct ib_qp_init_attr *qp_init_attr,
+			   struct ib_qp_cap *qp_cap)
+{
+	struct ib_qp *qp;
+
+	qp = pd->device->create_qp(pd, qp_init_attr, qp_cap);
+
+	if (!IS_ERR(qp)) {
+		qp->device	= pd->device;
+		qp->pd		= pd;
+		qp->send_cq	= qp_init_attr->send_cq;
+		qp->recv_cq	= qp_init_attr->recv_cq;
+		qp->srq		= qp_init_attr->srq;
+		qp->qp_context	= qp_init_attr->qp_context;
+
+		atomic_inc(&pd->usecnt);
+		atomic_inc(&qp_init_attr->send_cq->usecnt);
+		atomic_inc(&qp_init_attr->recv_cq->usecnt);
+		if (qp_init_attr->srq)
+			atomic_inc(&qp_init_attr->srq->usecnt);
+	}
+
+	return qp;
+}
+EXPORT_SYMBOL(ib_create_qp);
+
+int ib_destroy_qp(struct ib_qp *qp)
+{
+	struct ib_pd *pd;
+	struct ib_cq *send_cq, *recv_cq;
+	struct ib_srq *srq;
+	int ret;
+
+	pd      = qp->pd;
+	send_cq = qp->send_cq;
+	recv_cq = qp->recv_cq;
+	srq     = qp->srq;
+
+	ret = qp->device->destroy_qp(qp);
+	if (!ret) {
+		atomic_dec(&pd->usecnt);
+		atomic_dec(&send_cq->usecnt);
+		atomic_dec(&recv_cq->usecnt);
+		if (srq)
+			atomic_dec(&srq->usecnt);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_destroy_qp);
+
+struct ib_srq *ib_create_srq(struct ib_pd *pd,
+			     void *srq_context,
+			     struct ib_srq_attr *srq_attr)
+{
+	struct ib_srq *srq;
+
+	if (!pd->device->create_srq)
+		return -ENOSYS;
+
+	srq = pd->device->create_srq(pd, srq_context, srq_attr);
+
+	if (!IS_ERR(srq)) {
+		srq->device	 = pd->device;
+		srq->pd		 = pd;
+		srq->srq_context = srq_context;
+		atomic_inc(&pd->usecnt);
+		atomic_set(&srq->usecnt, 0);
+	}
+
+	return srq;
+}
+EXPORT_SYMBOL(ib_create_srq);
+
+int ib_destroy_srq(struct ib_srq *srq)
+{
+	struct ib_pd *pd;
+	int ret;
+
+	if (atomic_read(&srq->usecnt))
+		return -EBUSY;
+
+	pd = srq->pd;
+
+	ret = srq->device->destroy_srq(srq);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_destroy_srq);
+
+struct ib_cq *ib_create_cq(struct ib_device *device,
+			   ib_comp_handler comp_handler,
+			   void *cq_context,
+			   int cqe)
+{
+	struct ib_cq *cq;
+
+	cq = device->create_cq(device, comp_handler, cq_context, cqe);
+
+	if (!IS_ERR(cq)) {
+		cq->device	 = device;
+		cq->comp_handler = comp_handler;
+		cq->cq_context	 = cq_context;
+		atomic_set(&cq->usecnt, 0);
+	}
+
+	return cq;
+}
+EXPORT_SYMBOL(ib_create_cq);
+
+int ib_destroy_cq(struct ib_cq *cq)
+{
+	if (atomic_read(&cq->usecnt))
+		return -EBUSY;
+
+	return cq->device->destroy_cq(cq);
+}
+EXPORT_SYMBOL(ib_destroy_cq);
+
+struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
+			     struct ib_phys_buf *phys_buf_array,
+			     int num_phys_buf,
+			     int mr_access_flags,
+			     u64 *iova_start)
+{
+	struct ib_mr *mr;
+
+	mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
+				     mr_access_flags, iova_start);
+
+	if (!IS_ERR(mr)) {
+		mr->device = pd->device;
+		mr->pd     = pd;
+		atomic_inc(&pd->usecnt);
+		atomic_set(&mr->usecnt, 0);
+	}
+
+	return mr;
+}
+EXPORT_SYMBOL(ib_reg_phys_mr);
+
+int ib_dereg_mr(struct ib_mr *mr)
+{
+	struct ib_pd *pd;
+	int ret;
+
+	if (atomic_read(&mr->usecnt))
+		return -EBUSY;
+
+	pd = mr->pd;
+
+	ret = mr->device->dereg_mr(mr);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_dereg_mr);
+
+int ib_rereg_phys_mr(struct ib_mr *mr,
+		     int mr_rereg_mask,
+		     struct ib_pd *pd,
+		     struct ib_phys_buf *phys_buf_array,
+		     int num_phys_buf,
+		     int mr_access_flags,
+		     u64 *iova_start)
+{
+	struct ib_pd *old_pd;
+	int ret;
+
+	if (atomic_read(&mr->usecnt))
+		return -EBUSY;
+
+	old_pd = mr->pd;
+
+	ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
+					phys_buf_array, num_phys_buf,
+					mr_access_flags, iova_start);
+
+	if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
+		atomic_dec(&old_pd->usecnt);
+		atomic_inc(&pd->usecnt);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_rereg_phys_mr);
+
+struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
+{
+	struct ib_mw *mw;
+
+	mw = pd->device->alloc_mw(pd);
+
+	if (!IS_ERR(mw)) {
+		mw->device = pd->device;
+		mw->pd     = pd;
+		atomic_inc(&pd->usecnt);
+	}
+
+	return mw;
+}
+EXPORT_SYMBOL(ib_alloc_mw);
+
+int ib_dealloc_mw(struct ib_mw *mw)
+{
+	struct ib_pd *pd;
+	int ret;
+
+	pd = mw->pd;
+
+	ret = mw->device->dealloc_mw(mw);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_dealloc_mw);
+
+struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
+			    int mr_access_flags,
+			    struct ib_fmr_attr *fmr_attr)
+{
+	struct ib_fmr *fmr;
+
+	fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
+
+	if (!IS_ERR(fmr)) {
+		fmr->device = pd->device;
+		fmr->pd     = pd;
+		atomic_inc(&pd->usecnt);
+	}
+
+	return fmr;
+}
+EXPORT_SYMBOL(ib_alloc_fmr);
+
+int ib_free_fmr(struct ib_fmr *fmr)
+{
+	struct ib_pd *pd;
+	int ret;
+
+	pd = fmr->pd;
+
+	ret = fmr->device->free_fmr(fmr);
+	if (!ret)
+		atomic_dec(&pd->usecnt);
+
+	return ret;
+}
+EXPORT_SYMBOL(ib_free_fmr);

-- 



More information about the general mailing list