[openib-general] srq_pingpong with many QPs and events may never ends

[Dotan Barak]

here is a patch to solve this problem:



Empty the CQ on each event (there can be more than 1 QP connected to a CQ in
this test) before arm. Make sure there is a receive work request per each QP.

Signed-off-by: Dotan Barak <dotanb at mellanox.co.il>

Index: last_stable/src/userspace/libibverbs/examples/srq_pingpong.c
===================================================================
--- last_stable.orig/src/userspace/libibverbs/examples/srq_pingpong.c
+++ last_stable/src/userspace/libibverbs/examples/srq_pingpong.c
@@ -514,9 +514,11 @@ int main(int argc, char *argv[])
 	struct pingpong_context *ctx;
 	struct pingpong_dest     my_dest[MAX_QP];
 	struct pingpong_dest    *rem_dest;
+	struct ibv_wc 		*wc_arr;
 	struct timeval           start, end;
 	char                    *ib_devname = NULL;
 	char                    *servername = NULL;
+	int			 num_of_wc;
 	int                      port = 18515;
 	int                      ib_port = 1;
 	int                      size = 4096;
@@ -603,6 +605,13 @@ int main(int argc, char *argv[])
 		return 1;
 	}
 
+	num_of_wc = num_qp + rx_depth;
+	wc_arr = malloc(num_of_wc * sizeof *wc_arr);
+	if (!wc_arr) {
+		fprintf(stderr, "Failed to allocate memory\n");
+		return 1;
+	}
+
 	page_size = sysconf(_SC_PAGESIZE);
 
 	dev_list = ibv_get_device_list(NULL);
@@ -714,11 +723,10 @@ int main(int argc, char *argv[])
 		}
 
 		{
-			struct ibv_wc wc[2];
 			int ne, qp_ind;
 
 			do {
-				ne = ibv_poll_cq(ctx->cq, 2, wc);
+				ne = ibv_poll_cq(ctx->cq, num_of_wc, wc_arr);
 			} while (!use_event && ne < 1);
 
 			if (ne < 0) {
@@ -727,26 +735,26 @@ int main(int argc, char *argv[])
 			}
 
 			for (i = 0; i < ne; ++i) {
-				if (wc[i].status != IBV_WC_SUCCESS) {
+				if (wc_arr[i].status != IBV_WC_SUCCESS) {
 					fprintf(stderr, "Failed status %d for wr_id %d\n",
-						wc[i].status, (int) wc[i].wr_id);
+						wc_arr[i].status, (int) wc_arr[i].wr_id);
 					return 1;
 				}
 
-				qp_ind = find_qp(wc[i].qp_num, ctx, num_qp);
+				qp_ind = find_qp(wc_arr[i].qp_num, ctx, num_qp);
 				if (qp_ind < 0) {
 					fprintf(stderr, "Couldn't find QPN %06x\n",
-						wc[i].qp_num);
+						wc_arr[i].qp_num);
 					return 1;
 				}
 
-				switch ((int) wc[i].wr_id) {
+				switch ((int) wc_arr[i].wr_id) {
 				case PINGPONG_SEND_WRID:
 					++scnt;
 					break;
 
 				case PINGPONG_RECV_WRID:
-					if (--routs <= 1) {
+					if (--routs <= num_qp) {
 						routs += pp_post_recv(ctx, ctx->rx_depth - routs);
 						if (routs < ctx->rx_depth) {
 							fprintf(stderr,
@@ -761,11 +769,11 @@ int main(int argc, char *argv[])
 
 				default:
 					fprintf(stderr, "Completion for unknown wr_id %d\n",
-						(int) wc[i].wr_id);
+						(int) wc_arr[i].wr_id);
 					return 1;
 				}
 
-				ctx->pending[qp_ind] &= ~(int) wc[i].wr_id;
+				ctx->pending[qp_ind] &= ~(int) wc_arr[i].wr_id;
 				if (scnt < iters && !ctx->pending[qp_ind]) {
 					if (pp_post_send(ctx, qp_ind)) {
 						fprintf(stderr, "Couldn't post send\n");




Dotan Barak
Software Verification Engineer
Mellanox Technologies LTD
Tel: +972-4-9097200 Ext: 231 Fax: +972-4-9593245
P.O. Box 86 Yokneam 20692 ISRAEL.
Home: +972-77-8841095  Cell: 052-4222383

[ May the fork be with you ]

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/general/attachments/20051228/eea5f71c/attachment.html>