[openib-general] [PATCH] race in pingpong -e

[Ralph Campbell]

The example pingpong programs have a race when using events
where the client sends the first packet but the server hasn't
yet armed the CQ by calling ibv_req_notify_cq() thus waiting
forever in ibv_get_cq_event().  The fix is to move the call
to ibv_req_notify_cq() before signaling the client to "start".

Signed-off-by: Ralph Campbell <ralphc at pathscale.com>

Index: libibverbs/examples/rc_pingpong.c
===================================================================
--- libibverbs/examples/rc_pingpong.c	(revision 5031)
+++ libibverbs/examples/rc_pingpong.c	(working copy)
@@ -568,6 +568,12 @@
 		return 1;
 	}
 
+	if (use_event)
+		if (ibv_req_notify_cq(ctx->cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return 1;
+		}
+
 	my_dest.lid = pp_get_local_lid(ctx, ib_port);
 	my_dest.qpn = ctx->qp->qp_num;
 	my_dest.psn = lrand48() & 0xffffff;
@@ -594,12 +600,6 @@
 		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
 			return 1;
 
-	if (use_event)
-		if (ibv_req_notify_cq(ctx->cq, 0)) {
-			fprintf(stderr, "Couldn't request CQ notification\n");
-			return 1;
-		}
-
 	ctx->pending = PINGPONG_RECV_WRID;
 
 	if (servername) {
Index: libibverbs/examples/uc_pingpong.c
===================================================================
--- libibverbs/examples/uc_pingpong.c	(revision 5031)
+++ libibverbs/examples/uc_pingpong.c	(working copy)
@@ -556,6 +556,12 @@
 		return 1;
 	}
 
+	if (use_event)
+		if (ibv_req_notify_cq(ctx->cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return 1;
+		}
+
 	my_dest.lid = pp_get_local_lid(ctx, ib_port);
 	my_dest.qpn = ctx->qp->qp_num;
 	my_dest.psn = lrand48() & 0xffffff;
@@ -582,12 +588,6 @@
 		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
 			return 1;
 
-	if (use_event)
-		if (ibv_req_notify_cq(ctx->cq, 0)) {
-			fprintf(stderr, "Couldn't request CQ notification\n");
-			return 1;
-		}
-
 	ctx->pending = PINGPONG_RECV_WRID;
 
 	if (servername) {
Index: libibverbs/examples/ud_pingpong.c
===================================================================
--- libibverbs/examples/ud_pingpong.c	(revision 5031)
+++ libibverbs/examples/ud_pingpong.c	(working copy)
@@ -564,6 +564,12 @@
 		return 1;
 	}
 
+	if (use_event)
+		if (ibv_req_notify_cq(ctx->cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return 1;
+		}
+
 	my_dest.lid = pp_get_local_lid(ctx, ib_port);
 	my_dest.qpn = ctx->qp->qp_num;
 	my_dest.psn = lrand48() & 0xffffff;
@@ -590,12 +596,6 @@
 		if (pp_connect_ctx(ctx, ib_port, my_dest.psn, rem_dest))
 			return 1;
 
-	if (use_event)
-		if (ibv_req_notify_cq(ctx->cq, 0)) {
-			fprintf(stderr, "Couldn't request CQ notification\n");
-			return 1;
-		}
-
 	ctx->pending = PINGPONG_RECV_WRID;
 
 	if (servername) {
Index: libibverbs/examples/srq_pingpong.c
===================================================================
--- libibverbs/examples/srq_pingpong.c	(revision 5031)
+++ libibverbs/examples/srq_pingpong.c	(working copy)
@@ -649,6 +649,12 @@
 		return 1;
 	}
 
+	if (use_event)
+		if (ibv_req_notify_cq(ctx->cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return 1;
+		}
+
 	memset(my_dest, 0, sizeof my_dest);
 
 	for (i = 0; i < num_qp; ++i) {
@@ -680,12 +686,6 @@
 		if (pp_connect_ctx(ctx, ib_port, my_dest, rem_dest))
 			return 1;
 
-	if (use_event)
-		if (ibv_req_notify_cq(ctx->cq, 0)) {
-			fprintf(stderr, "Couldn't request CQ notification\n");
-			return 1;
-		}
-
 	if (servername)
 		for (i = 0; i < num_qp; ++i) {
 			if (pp_post_send(ctx, i)) {

-- 
Ralph Campbell <ralphc at pathscale.com>