[openib-general] [PATCH] libmthca: fix wqe post (was Re: strange mem-free bug)

[Michael S. Tsirkin]

Quoting r. Roland Dreier <rolandd at cisco.com>:
> Subject: strange mem-free bug (was: [openib-general] completion Q overflow error/panic)
> 
> While looking at Viswa's example, I've found what seems to be a
> problem using lots of QPs on mem-free HCAs.

Hi, Roland!
This seems to be a bug in libmthca. Patch below.

We probably need a similiar fix for kernel mthca - let me know if
you plan to work on that, otherwise I'll look into it tomorrow.
And its probably something we want fixed for 2.6.14, right?
Let me know.

With regard to the test code that you posted - I also have some small
comments. If you plan to use it in the future, you can stick it
in svn somewhere and I'll send patches.

---

Fix posting of the first work request for memfree hardware.
Simplify code for tavor mode hardware.

Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>

Index: userspace/libmthca/src/qp.c
===================================================================
--- userspace.orig/libmthca/src/qp.c	2005-09-13 17:17:58.000000000 +0300
+++ userspace/libmthca/src/qp.c	2005-09-13 17:26:23.000000000 +0300
@@ -259,15 +259,13 @@ int mthca_tavor_post_send(struct ibv_qp 
 			goto out;
 		}
 
-		if (prev_wqe) {
-			((struct mthca_next_seg *) prev_wqe)->nda_op =
-				htonl(((ind << qp->sq.wqe_shift) +
-				       qp->send_wqe_offset) |
-				      mthca_opcode[wr->opcode]);
+		((struct mthca_next_seg *) prev_wqe)->nda_op =
+			htonl(((ind << qp->sq.wqe_shift) +
+			       qp->send_wqe_offset) |
+			      mthca_opcode[wr->opcode]);
 
-			((struct mthca_next_seg *) prev_wqe)->ee_nds =
-				htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
-		}
+		((struct mthca_next_seg *) prev_wqe)->ee_nds =
+			htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size);
 
 		if (!size0) {
 			size0 = size;
@@ -353,12 +351,10 @@ int mthca_tavor_post_recv(struct ibv_qp 
 
 		qp->wrid[ind] = wr->wr_id;
 
-		if (prev_wqe) {
-			((struct mthca_next_seg *) prev_wqe)->nda_op =
-				htonl((ind << qp->rq.wqe_shift) | 1);
-			((struct mthca_next_seg *) prev_wqe)->ee_nds =
-				htonl(MTHCA_NEXT_DBD | size);
-		}
+		((struct mthca_next_seg *) prev_wqe)->nda_op =
+			htonl((ind << qp->rq.wqe_shift) | 1);
+		((struct mthca_next_seg *) prev_wqe)->ee_nds =
+			htonl(MTHCA_NEXT_DBD | size);
 
 		if (!size0)
 			size0 = size;
@@ -562,15 +558,13 @@ int mthca_arbel_post_send(struct ibv_qp 
 			goto out;
 		}
 
-		if (prev_wqe) {
-			((struct mthca_next_seg *) prev_wqe)->nda_op =
-				htonl(((ind << qp->sq.wqe_shift) +
-				       qp->send_wqe_offset) |
-				      mthca_opcode[wr->opcode]);
-			mb();
-			((struct mthca_next_seg *) prev_wqe)->ee_nds =
-				htonl(MTHCA_NEXT_DBD | size);
-		}
+		((struct mthca_next_seg *) prev_wqe)->nda_op =
+			htonl(((ind << qp->sq.wqe_shift) +
+			       qp->send_wqe_offset) |
+			      mthca_opcode[wr->opcode]);
+		mb();
+		((struct mthca_next_seg *) prev_wqe)->ee_nds =
+			htonl(MTHCA_NEXT_DBD | size);
 
 		if (!size0) {
 			size0 = size;
@@ -767,6 +761,8 @@ int mthca_alloc_qp_buf(struct ibv_pd *pd
 		}
 	}
 
+	qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
+	qp->rq.last = get_recv_wqe(qp, qp->sq.max - 1);
 	return 0;
 }
 
Index: userspace/libmthca/src/srq.c
===================================================================
--- userspace.orig/libmthca/src/srq.c	2005-09-13 17:25:41.000000000 +0300
+++ userspace/libmthca/src/srq.c	2005-09-13 17:25:51.000000000 +0300
@@ -142,13 +142,11 @@ int mthca_tavor_post_srq_recv(struct ibv
 			((struct mthca_data_seg *) wqe)->addr = 0;
 		}
 
-		if (prev_wqe) {
-			((struct mthca_next_seg *) prev_wqe)->nda_op =
-				htonl((ind << srq->wqe_shift) | 1);
-			mb();
-			((struct mthca_next_seg *) prev_wqe)->ee_nds =
-				htonl(MTHCA_NEXT_DBD);
-		}
+		((struct mthca_next_seg *) prev_wqe)->nda_op =
+			htonl((ind << srq->wqe_shift) | 1);
+		mb();
+		((struct mthca_next_seg *) prev_wqe)->ee_nds =
+			htonl(MTHCA_NEXT_DBD);
 
 		srq->wrid[ind]  = wr->wr_id;
 		srq->first_free = next_ind;
@@ -294,6 +292,7 @@ int mthca_alloc_srq_buf(struct ibv_pd *p
 
 	srq->first_free = 0;
 	srq->last_free  = srq->max - 1;
+	srq->last = get_wqe(srq, srq->max - 1);
 
 	return 0;
 }
Index: userspace/libmthca/src/verbs.c
===================================================================
--- userspace.orig/libmthca/src/verbs.c	2005-08-23 14:03:12.000000000 +0300
+++ userspace/libmthca/src/verbs.c	2005-09-13 17:25:14.000000000 +0300
@@ -306,7 +306,6 @@ struct ibv_srq *mthca_create_srq(struct 
 
 	srq->max     = align_queue_size(pd->context, attr->attr.max_wr, 1);
 	srq->max_gs  = attr->attr.max_sge;
-	srq->last    = NULL;
 	srq->counter = 0;
 
 	if (mthca_alloc_srq_buf(pd, &attr->attr, srq))
@@ -413,14 +412,12 @@ struct ibv_qp *mthca_create_qp(struct ib
 	qp->sq.last_comp = qp->sq.max - 1;
 	qp->sq.head    	 = 0;
 	qp->sq.tail    	 = 0;
-	qp->sq.last      = NULL;
 
 	qp->rq.max       = align_queue_size(pd->context, attr->cap.max_recv_wr, 0);
 	qp->rq.next_ind	 = 0;
 	qp->rq.last_comp = qp->rq.max - 1;
 	qp->rq.head    	 = 0;
 	qp->rq.tail    	 = 0;
-	qp->rq.last      = NULL;
 
 	if (mthca_alloc_qp_buf(pd, &attr->cap, qp))
 		goto err;


-- 
MST