[ofa-general] [PATCH] uDAPL v2: dapltest: replace small malloc's with stack allocation, add completion vector threshold

Davis, Arlin R arlin.r.davis at intel.com
Thu Jul 16 14:38:28 PDT 2009


 
Implement a malloc() threshold for the completion reaping byte vector allocation in functions:
  DT_handle_send_op, DT_handle_rdma_op & DT_handle_recv_op.

When allocation size is under the threshold, use a stack local allocation instead of malloc/free.
Move redundant bzero() to be called only in the case of using local stack allocation as DT_Mdep_malloc() already does a bzero().
Consolidate error handling return and free()check to a single point by using goto.

Signed-off-by: Stan Smith <stan.smith at intel.com>

diff --git a/test/dapltest/test/dapl_transaction_util.c b/test/dapltest/test/dapl_transaction_util.c
index ffe5d7d..14a14dd 100644
--- a/test/dapltest/test/dapl_transaction_util.c
+++ b/test/dapltest/test/dapl_transaction_util.c
@@ -30,6 +30,8 @@
 
 #include "dapl_proto.h"
 
+#define DT_LOCAL_COMPLETION_VECTOR_SIZE 32
+
 /* -----------------------------------------------------------
  * Post a recv buffer on each of this thread's EPs.
  */
@@ -85,11 +87,19 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 {
 	unsigned int i, j;
 	unsigned char *completion_reaped;
+	unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
+	bool rc = false;
 
-	completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
-
-	if (!completion_reaped) {
-		return false;
+	if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) {
+		completion_reaped = lcomp;
+		bzero((void *)completion_reaped,
+			sizeof(unsigned char) * num_eps);
+	}
+	else {
+		completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
+		if (!completion_reaped) {
+			return false;
+		}
 	}
 
 	for (i = 0; i < num_eps; i++) {
@@ -120,8 +130,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 					  "Test Error: dat_ep_post_send failed: %s\n",
 					  DT_RetToString(ret));
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return false;
+			goto xit;
 		}
 	}
 
@@ -130,13 +139,11 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 
 		if (op->reap_send_on_recv && !op->server_initiated) {
 			/* we will reap the send on the recv (Client SR) */
-			DT_Mdep_Free(completion_reaped);
-			return true;
+			rc = true;
+			goto xit;
 		}
 	}
 
-	bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps);
-
 	/* reap the send completion */
 	for (i = 0; i < num_eps; i++) {
 		Transaction_Test_Op_t *op;
@@ -146,8 +153,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 
 		if (!DT_dto_event_reap
 		    (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) {
-			DT_Mdep_Free(completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		epnum = dto_stat.user_cookie.as_64 >> 32;
@@ -160,8 +166,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 					  dto_stat.user_cookie.as_64,
 					  dto_stat.transfered_length);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		op = &ep_context[epnum].op[op_indx];
@@ -176,8 +181,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 				  ep_context[epnum].ep_handle,
 				  op->num_segs * op->seg_size,
 				  dto_cookie, "Send")) {
-			DT_Mdep_Free(completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		if (completion_reaped[epnum]) {
@@ -185,8 +189,7 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 					  "Test Error: Send: Secondary completion seen for endpoint 0x%p (%d)\n",
 					  ep_context[epnum].ep_handle, epnum);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto xit;
 		}
 		completion_reaped[epnum] = 1;
 	}
@@ -197,13 +200,16 @@ DT_handle_send_op(DT_Tdep_Print_Head * phead,
 					  "Test Error: Send: No completion seen for endpoint 0x%p (#%d)\n",
 					  ep_context[i].ep_handle, i);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto xit;
 		}
 	}
 
-	DT_Mdep_Free(completion_reaped);
-	return true;
+	rc = true;
+
+xit:
+	if (completion_reaped != lcomp)
+		DT_Mdep_Free(completion_reaped);
+	return rc;
 }
 
 /* -----------------------------------------------------------
@@ -220,21 +226,32 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 	unsigned int i;
 	unsigned char *recv_completion_reaped;
 	unsigned char *send_completion_reaped;
-
-	recv_completion_reaped = DT_Mdep_Malloc(num_eps);
-	if (recv_completion_reaped == NULL) {
-		return false;
+	unsigned char rcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
+	unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
+	bool rc = false;
+
+	if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE ) {
+		recv_completion_reaped = rcomp;
+		send_completion_reaped = lcomp;
+		bzero((void *)recv_completion_reaped,
+			sizeof(unsigned char) * num_eps);
+		bzero((void *)send_completion_reaped,
+			sizeof(unsigned char) * num_eps);
 	}
+	else {
+		recv_completion_reaped = DT_Mdep_Malloc(num_eps);
+		if (recv_completion_reaped == NULL) {
+			return false;
+		}
 
-	send_completion_reaped = DT_Mdep_Malloc(num_eps);
-	if (send_completion_reaped == NULL) {
-		DT_Mdep_Free(recv_completion_reaped);
-		return false;
+		send_completion_reaped = DT_Mdep_Malloc(num_eps);
+		if (send_completion_reaped == NULL) {
+			DT_Mdep_Free(recv_completion_reaped);
+			return false;
+		}
 	}
 
 	/* Foreach EP, reap */
-	bzero((void *)recv_completion_reaped, sizeof(unsigned char) * num_eps);
-	bzero((void *)send_completion_reaped, sizeof(unsigned char) * num_eps);
 	for (i = 0; i < num_eps; i++) {
 		Transaction_Test_Op_t *op;
 		DAT_DTO_COMPLETION_EVENT_DATA dto_stat;
@@ -244,9 +261,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 		/* First reap the recv DTO event */
 		if (!DT_dto_event_reap
 		    (phead, ep_context[i].recv_evd_hdl, poll, &dto_stat)) {
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		epnum = dto_stat.user_cookie.as_64 >> 32;
@@ -259,9 +274,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 					  dto_stat.user_cookie.as_64,
 					  dto_stat.transfered_length);
 			DT_Test_Error();
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		op = &ep_context[epnum].op[op_indx];
@@ -278,9 +291,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 			DT_Tdep_PT_Printf(phead,
 					  "Test Error: recv DTO problem\n");
 			DT_Test_Error();
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return false;
+			goto xit;
 		}
 
 		if (recv_completion_reaped[epnum]) {
@@ -288,9 +299,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 					  "Test Error: Receive: Secondary completion seen for endpoint 0x%p (%d)\n",
 					  ep_context[epnum].ep_handle, epnum);
 			DT_Test_Error();
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return (false);
+			goto xit;
 		}
 		recv_completion_reaped[epnum] = 1;
 
@@ -306,17 +315,13 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 						  "Internal Error: reap_send_on_recv"
 						  " but current op == #%d\n",
 						  op_indx);
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return false;
+				goto xit;
 			}
 
 			if (!DT_dto_event_reap
 			    (phead, ep_context[i].reqt_evd_hdl, poll,
 			     &dto_stat)) {
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return false;
+				goto xit;
 			}
 
 			epnum = dto_stat.user_cookie.as_64 >> 32;
@@ -329,9 +334,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 						  dto_stat.user_cookie.as_64,
 						  dto_stat.transfered_length);
 				DT_Test_Error();
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return false;
+				goto xit;
 			}
 
 			/*
@@ -359,9 +362,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 				DT_Tdep_PT_Printf(phead,
 						  "Test Error: send DTO problem\n");
 				DT_Test_Error();
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return false;
+				goto xit;
 			}
 
 			if (send_completion_reaped[epnum]) {
@@ -370,9 +371,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 						  ep_context[epnum].ep_handle,
 						  epnum);
 				DT_Test_Error();
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return (false);
+				goto xit;
 			}
 			send_completion_reaped[epnum] = 1;
 		}
@@ -384,9 +383,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 					  "Test Error: Receive: No completion seen for endpoint 0x%p (#%d)\n",
 					  ep_context[i].ep_handle, i);
 			DT_Test_Error();
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return (false);
+			goto xit;
 		}
 	}
 
@@ -398,9 +395,7 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 						  "Test Error: Send (ror): No completion seen for endpoint 0x%p (#%d)\n",
 						  ep_context[i].ep_handle, i);
 				DT_Test_Error();
-				DT_Mdep_Free(recv_completion_reaped);
-				DT_Mdep_Free(send_completion_reaped);
-				return (false);
+				goto xit;
 			}
 		}
 	}
@@ -412,15 +407,16 @@ DT_handle_recv_op(DT_Tdep_Print_Head * phead,
 			DT_Tdep_PT_Printf(phead,
 					  "Test Error: recv re-post problem\n");
 			DT_Test_Error();
-			DT_Mdep_Free(recv_completion_reaped);
-			DT_Mdep_Free(send_completion_reaped);
-			return false;
+			goto xit;
 		}
 	}
-
-	DT_Mdep_Free(recv_completion_reaped);
-	DT_Mdep_Free(send_completion_reaped);
-	return true;
+	rc = true;
+xit:
+	if (send_completion_reaped != lcomp) {
+		DT_Mdep_Free(recv_completion_reaped);
+		DT_Mdep_Free(send_completion_reaped);
+	}
+	return rc;
 }
 
 /* -----------------------------------------------------------
@@ -435,11 +431,18 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 	unsigned int i, j;
 	DAT_RETURN ret;
 	unsigned char *completion_reaped;
+	unsigned char lcomp[DT_LOCAL_COMPLETION_VECTOR_SIZE];
+	bool rc = false;
 
-	completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
-
-	if (!completion_reaped) {
-		return false;
+	if (num_eps <= DT_LOCAL_COMPLETION_VECTOR_SIZE) {
+		completion_reaped = lcomp;
+		bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps);
+	}
+	else {
+		completion_reaped = DT_Mdep_Malloc(num_eps * sizeof(unsigned char));
+		if (!completion_reaped) {
+			return false;
+		}
 	}
 
 	/* Initiate the operation */
@@ -500,8 +503,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 					   RDMA_WRITE ? "write" : "read"),
 					  DT_RetToString(ret));
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto err;
 		} else {
 			DT_Tdep_PT_Debug(3, (phead,
 					     "Done dat_ep_post_rdma_%s %s\n",
@@ -511,7 +513,6 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 		}
 	}
 
-	bzero((void *)completion_reaped, sizeof(unsigned char) * num_eps);
 	/* Wait for it to happen */
 	for (i = 0; i < num_eps; i++) {
 		Transaction_Test_Op_t *op;
@@ -521,8 +522,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 
 		if (!DT_dto_event_reap
 		    (phead, ep_context[i].reqt_evd_hdl, poll, &dto_stat)) {
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto err;
 		}
 
 		epnum = dto_stat.user_cookie.as_64 >> 32;
@@ -537,8 +537,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 					  dto_stat.user_cookie.as_64,
 					  dto_stat.transfered_length);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return false;
+			goto err;
 		}
 		op = &ep_context[epnum].op[op_indx];
 
@@ -554,8 +553,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 				  dto_cookie,
 				  (opcode ==
 				   RDMA_WRITE ? "RDMA/WR" : "RDMA/RD"))) {
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto err;
 		}
 
 		if (completion_reaped[epnum]) {
@@ -565,8 +563,7 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 					  RDMA_WRITE ? "RDMA/WR" : "RDMA/RD",
 					  ep_context[epnum].ep_handle, epnum);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto err;
 		}
 		completion_reaped[epnum] = 1;
 
@@ -584,14 +581,17 @@ DT_handle_rdma_op(DT_Tdep_Print_Head * phead,
 					  RDMA_WRITE ? "RDMA/WR" : "RDMA/RD",
 					  ep_context[i].ep_handle, i);
 			DT_Test_Error();
-			DT_Mdep_Free(completion_reaped);
-			return (false);
+			goto err;
 		}
 	}
 
-	DT_Mdep_Free(completion_reaped);
+	rc = true;
+
+err:
+	if (completion_reaped != lcomp)
+		DT_Mdep_Free(completion_reaped);
 
-	return (true);
+	return rc;
 }
 
 /* -----------------------------------------------------------






More information about the general mailing list