[ofa-general] [PATCH] uDAPL v2: fix dtest to handle CNO events properly
Davis, Arlin R
arlin.r.davis at intel.com
Tue Aug 4 22:32:06 PDT 2009
modify dtest.c to cleanup CNO wait code and consolidate into
collect_event() call. After waking up from CNO wait the
consumer must check all EVD's. The EVD's under the CNO
could be dropped if already triggered or could come in any order.
DT_RetToString changed to DT_RetToStr and DT_EventToSTr
changed to DT_EventToStr for consistency.
diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index 739ccca..d868490 100755
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -104,6 +104,7 @@
/* definitions */
#define SERVER_CONN_QUAL 45248
#define DTO_TIMEOUT (1000*1000*5)
+#define CNO_TIMEOUT (1000*1000*1)
#define DTO_FLUSH_TIMEOUT (1000*1000*2)
#define CONN_TIMEOUT (1000*1000*10)
#define SERVER_TIMEOUT DAT_TIMEOUT_INFINITE
@@ -208,8 +209,8 @@ static int burst_msg_posted = 0;
static int burst_msg_index = 0;
/* forward prototypes */
-const char *DT_RetToString(DAT_RETURN ret_value);
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code);
+const char *DT_RetToStr(DAT_RETURN ret_value);
+const char *DT_EventToStr(DAT_EVENT_NUMBER event_code);
void print_usage(void);
double get_time(void);
void init_data(void);
@@ -262,6 +263,51 @@ void flush_evds(void)
}
}
+
+static inline DAT_RETURN
+collect_event(DAT_EVD_HANDLE dto_evd,
+ DAT_EVENT *event,
+ DAT_TIMEOUT timeout,
+ int *counter)
+{
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ DAT_COUNT nmore;
+ DAT_RETURN ret = DAT_SUCCESS;
+
+ if (use_cno) {
+retry:
+ /* CNO wait could return EVD's in any order and
+ * may drop some EVD notification's if already
+ * triggered. Once woken, simply dequeue the
+ * Evd the caller wants to collect and return.
+ * If notification without EVD, retry.
+ */
+ ret = dat_cno_wait(h_dto_cno, CNO_TIMEOUT, &evd);
+ if (dat_evd_dequeue(dto_evd, event) != DAT_SUCCESS) {
+ if (ret == DAT_SUCCESS)
+ printf(" WARNING: CNO notification:"
+ " without EVD?\n");
+ goto retry;
+ }
+ ret = DAT_SUCCESS; /* cno timed out, but EVD dequeued */
+
+ } else if (!polling) {
+
+ /* use wait to dequeue */
+ ret = dat_evd_wait(dto_evd, timeout, 1, event, &nmore);
+ if (ret != DAT_SUCCESS)
+ fprintf(stderr,
+ "Error waiting on h_dto_evd %p: %s\n",
+ dto_evd, DT_RetToStr(ret));
+
+ } else {
+ while (dat_evd_dequeue(dto_evd, event) == DAT_QUEUE_EMPTY)
+ if (counter)
+ (*counter)++;
+ }
+ return (ret);
+}
+
int main(int argc, char **argv)
{
int i, c;
@@ -355,7 +401,7 @@ int main(int argc, char **argv)
time.open += ((stop - start) * 1.0e6);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: Error Adaptor open: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
exit(1);
} else
LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
@@ -368,7 +414,7 @@ int main(int argc, char **argv)
time.pzc += ((stop - start) * 1.0e6);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error creating Protection Zone: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
exit(1);
} else
LOGPRINTF("%d Created Protection Zone\n", getpid());
@@ -378,7 +424,7 @@ int main(int argc, char **argv)
ret = register_rdma_memory();
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error registering RDMA memory: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d Register RDMA memory done\n", getpid());
@@ -387,7 +433,7 @@ int main(int argc, char **argv)
ret = create_events();
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error creating events: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else {
LOGPRINTF("%d Create events done\n", getpid());
@@ -419,7 +465,7 @@ int main(int argc, char **argv)
time.total += time.epc;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_ep_create: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
@@ -431,7 +477,7 @@ int main(int argc, char **argv)
ret = connect_ep(hostname, SERVER_CONN_QUAL);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error connect_ep: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d connect_ep complete\n", getpid());
@@ -440,7 +486,7 @@ int main(int argc, char **argv)
ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_ep_query: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
@@ -483,7 +529,7 @@ int main(int argc, char **argv)
ret = do_rdma_write_with_msg();
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
@@ -492,7 +538,7 @@ int main(int argc, char **argv)
ret = do_rdma_read_with_msg();
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else
LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
@@ -501,7 +547,7 @@ int main(int argc, char **argv)
ret = do_ping_pong_msg();
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
goto cleanup;
} else {
LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
@@ -528,7 +574,7 @@ complete:
time.total += time.epf;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing EP: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d Freed EP\n", getpid());
h_ep = DAT_HANDLE_NULL;
@@ -540,7 +586,7 @@ complete:
ret = destroy_events();
if (ret != DAT_SUCCESS)
fprintf(stderr, "%d Error destroy_events: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
else
LOGPRINTF("%d destroy events done\n", getpid());
@@ -548,7 +594,7 @@ complete:
LOGPRINTF("%d unregister_rdma_memory \n", getpid());
if (ret != DAT_SUCCESS)
fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
else
LOGPRINTF("%d unregister_rdma_memory done\n", getpid());
@@ -560,7 +606,7 @@ complete:
time.pzf += ((stop - start) * 1.0e6);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing PZ: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d Freed pz\n", getpid());
h_pz = NULL;
@@ -574,7 +620,7 @@ complete:
time.close += ((stop - start) * 1.0e6);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: Error Adaptor close: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else
LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
@@ -652,7 +698,6 @@ send_msg(void *data,
{
DAT_LMR_TRIPLET iov;
DAT_EVENT event;
- DAT_COUNT nmore;
DAT_RETURN ret;
iov.lmr_context = context;
@@ -669,47 +714,23 @@ send_msg(void *data,
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return ret;
}
if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
- if (polling) {
- printf("%d Polling post send completion...\n",
- getpid());
- while (dat_evd_dequeue(h_dto_req_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- LOGPRINTF("%d waiting for post_send completion event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_req_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return ret;
- }
- }
+
+ if (collect_event(h_dto_req_evd,
+ &event,
+ DTO_TIMEOUT,
+ &poll_count) != DAT_SUCCESS)
+ return (DAT_ABORT);
/* validate event number, len, cookie, and status */
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(),
+ DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
@@ -730,7 +751,7 @@ send_msg(void *data,
if (event.event_data.dto_completion_event_data.status !=
DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (DAT_ABORT);
}
}
@@ -772,7 +793,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error registering send msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d Registered send Message Buffer %p \n",
@@ -796,7 +817,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
®istered_addr_recv_msg);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d Registered Receive Message Buffer %p\n",
@@ -823,7 +844,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error registering recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d Registered Receive Message Buffer %p\n",
@@ -846,7 +867,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
h_cr_evd, DAT_PSP_CONSUMER_FLAG, &h_psp);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_psp_create: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d dat_psp_created for server listen\n",
@@ -858,7 +879,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
ret = dat_evd_wait(h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d dat_evd_wait for cr_evd completed\n",
@@ -866,7 +887,8 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
fprintf(stderr, "%d Error unexpected cr event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(),
+ DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
if ((event.event_data.cr_arrival_event_data.conn_qual !=
@@ -874,7 +896,8 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
|| (event.event_data.cr_arrival_event_data.sp_handle.
psp_handle != h_psp)) {
fprintf(stderr, "%d Error wrong cr event data : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(),
+ DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
@@ -922,7 +945,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_cr_accept: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d dat_cr_accept completed\n", getpid());
@@ -966,7 +989,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
0, DAT_CONNECT_DEFAULT_FLAG);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_ep_connect: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d dat_ep_connect completed\n", getpid());
@@ -990,7 +1013,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
#ifdef TEST_REJECT_WITH_PRIVATE_DATA
if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
fprintf(stderr, "%d expected conn reject event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
/* get the reject private data and validate */
@@ -1013,7 +1036,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
fprintf(stderr, "%d Error unexpected conn event : 0x%x %s\n",
getpid(), event.event_number,
- DT_EventToSTr(event.event_number));
+ DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
@@ -1064,7 +1087,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else
LOGPRINTF("%d send_msg completed\n", getpid());
@@ -1072,42 +1095,17 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
/*
* Wait for remote RMR information for RDMA
*/
- if (polling) {
- printf("%d Polling for remote to send RMR data\n", getpid());
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- printf("%d Waiting for remote to send RMR data\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d Error waiting on h_dto_rcv_evd: %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- } else {
- LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n",
- getpid());
- }
- }
-
+ if (collect_event(h_dto_rcv_evd,
+ &event,
+ DTO_TIMEOUT,
+ &poll_count) != DAT_SUCCESS)
+ return (DAT_ABORT);
+
printf("%d remote RMR data arrived!\n", getpid());
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
if ((event.event_data.dto_completion_event_data.transfered_length !=
@@ -1162,7 +1160,7 @@ void disconnect_ep(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error dat_ep_disconnect: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d dat_ep_disconnect completed\n",
getpid());
@@ -1177,7 +1175,7 @@ void disconnect_ep(void)
&nmore);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_wait: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
getpid());
@@ -1189,7 +1187,7 @@ void disconnect_ep(void)
ret = dat_psp_free(h_psp);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_psp_free: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d dat_psp_free completed\n", getpid());
}
@@ -1203,7 +1201,7 @@ void disconnect_ep(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error deregistering send msg mr: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d Unregistered send message Buffer\n",
getpid());
@@ -1219,7 +1217,7 @@ void disconnect_ep(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error deregistering recv msg mr: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
} else {
LOGPRINTF("%d Unregistered recv message Buffer\n",
getpid());
@@ -1232,7 +1230,6 @@ void disconnect_ep(void)
DAT_RETURN do_rdma_write_with_msg(void)
{
DAT_EVENT event;
- DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
DAT_RMR_TRIPLET r_iov;
DAT_DTO_COOKIE cookie;
@@ -1277,7 +1274,7 @@ DAT_RETURN do_rdma_write_with_msg(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d: ERROR: dat_ep_post_rdma_write() %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (DAT_ABORT);
}
LOGPRINTF("%d rdma_write # %d completed\n", getpid(), i + 1);
@@ -1296,41 +1293,19 @@ DAT_RETURN do_rdma_write_with_msg(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d send_msg completed\n", getpid());
}
- /*
- * Collect first event, write completion or the inbound recv
- */
- if (polling) {
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY)
- rdma_wr_poll_count++;
- } else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- }
+ /* inbound recv event, send completion's suppressed */
+ if (collect_event(h_dto_rcv_evd,
+ &event,
+ DTO_TIMEOUT,
+ &rdma_wr_poll_count) != DAT_SUCCESS)
+ return (DAT_ABORT);
+
stop = get_time();
time.rdma_wr = ((stop - start) * 1.0e6);
@@ -1338,7 +1313,7 @@ DAT_RETURN do_rdma_write_with_msg(void)
printf("%d inbound rdma_write; send message arrived!\n", getpid());
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
@@ -1386,7 +1361,6 @@ DAT_RETURN do_rdma_write_with_msg(void)
DAT_RETURN do_rdma_read_with_msg(void)
{
DAT_EVENT event;
- DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov;
DAT_RMR_TRIPLET r_iov;
DAT_DTO_COOKIE cookie;
@@ -1425,44 +1399,21 @@ DAT_RETURN do_rdma_read_with_msg(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d: ERROR: dat_ep_post_rdma_read() %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (DAT_ABORT);
}
- if (polling) {
- while (dat_evd_dequeue(h_dto_req_evd, &event) ==
- DAT_QUEUE_EMPTY)
- rdma_rd_poll_count[i]++;
- } else {
- LOGPRINTF("%d waiting for rdma_read completion event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_req_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return ret;
- }
- }
+ /* RDMA read completion event */
+ if (collect_event(h_dto_req_evd,
+ &event,
+ DTO_TIMEOUT,
+ &rdma_rd_poll_count[i]) != DAT_SUCCESS)
+ return (DAT_ABORT);
+
/* validate event number, len, cookie, and status */
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d: ERROR: DTO event number %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
if ((event.event_data.dto_completion_event_data.
@@ -1481,7 +1432,7 @@ DAT_RETURN do_rdma_read_with_msg(void)
if (event.event_data.dto_completion_event_data.status !=
DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO event status %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (DAT_ABORT);
}
stop = get_time();
@@ -1513,48 +1464,25 @@ DAT_RETURN do_rdma_read_with_msg(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d send_msg completed\n", getpid());
}
- /*
- * Collect first event, write completion or the inbound recv with immed
- */
printf("%d Waiting for inbound message....\n", getpid());
- if (polling) {
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY) ;
- } else {
- LOGPRINTF("%d waiting for message receive event\n", getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-
- ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- }
+
+ if (collect_event(h_dto_rcv_evd,
+ &event,
+ DTO_TIMEOUT,
+ &poll_count) != DAT_SUCCESS)
+ return (DAT_ABORT);
/* validate event number and status */
printf("%d inbound rdma_read; send message arrived!\n", getpid());
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
@@ -1603,7 +1531,6 @@ DAT_RETURN do_rdma_read_with_msg(void)
DAT_RETURN do_ping_pong_msg()
{
DAT_EVENT event;
- DAT_COUNT nmore;
DAT_DTO_COOKIE cookie;
DAT_LMR_TRIPLET l_iov;
DAT_RETURN ret;
@@ -1635,7 +1562,7 @@ DAT_RETURN do_ping_pong_msg()
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error posting recv msg buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Posted Receive Message Buffer %p\n",
@@ -1673,47 +1600,21 @@ DAT_RETURN do_ping_pong_msg()
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d send_msg completed\n", getpid());
}
}
- /* Wait for recv message */
- if (polling) {
- poll_count = 0;
- LOGPRINTF("%d Polling for message receive event\n",
- getpid());
- while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
- DAT_QUEUE_EMPTY)
- poll_count++;
- } else {
- LOGPRINTF("%d waiting for message receive event\n",
- getpid());
- if (use_cno) {
- DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
- ret =
- dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
- LOGPRINTF("%d cno wait return evd_handle=%p\n",
- getpid(), evd);
- if (evd != h_dto_rcv_evd) {
- /* CNO timeout, already on EVD */
- if (evd != NULL)
- return (ret);
- }
- }
- /* use wait to dequeue */
- ret =
- dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event,
- &nmore);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr,
- "%d: ERROR: DTO dat_evd_wait() %s\n",
- getpid(), DT_RetToString(ret));
- return (ret);
- }
- }
+ /* recv message, send completions suppressed */
+ if (collect_event(h_dto_rcv_evd,
+ &event,
+ DTO_TIMEOUT,
+ &poll_count) != DAT_SUCCESS)
+ return (DAT_ABORT);
+
+
/* start timer after first message arrives on server */
if (i == 0) {
start = get_time();
@@ -1722,7 +1623,7 @@ DAT_RETURN do_ping_pong_msg()
LOGPRINTF("%d inbound message; message arrived!\n", getpid());
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
fprintf(stderr, "%d Error unexpected DTO event : %s\n",
- getpid(), DT_EventToSTr(event.event_number));
+ getpid(), DT_EventToStr(event.event_number));
return (DAT_ABORT);
}
if ((event.event_data.dto_completion_event_data.
@@ -1762,7 +1663,7 @@ DAT_RETURN do_ping_pong_msg()
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error send_msg: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d send_msg completed\n", getpid());
@@ -1805,7 +1706,7 @@ DAT_RETURN register_rdma_memory(void)
if (ret != DAT_SUCCESS) {
fprintf(stderr,
"%d Error registering Receive RDMA buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
@@ -1827,7 +1728,7 @@ DAT_RETURN register_rdma_memory(void)
®istered_size_send, ®istered_addr_send);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
@@ -1854,7 +1755,7 @@ DAT_RETURN unregister_rdma_memory(void)
time.total += time.unreg;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error deregistering recv mr: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Unregistered Recv Buffer\n", getpid());
@@ -1868,7 +1769,7 @@ DAT_RETURN unregister_rdma_memory(void)
ret = dat_lmr_free(h_lmr_send);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error deregistering send mr: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Unregistered send Buffer\n", getpid());
@@ -1904,7 +1805,7 @@ DAT_RETURN create_events(void)
time.total += time.cnoc;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_cno_create: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d cr_evd created, %p\n", getpid(),
@@ -1922,7 +1823,7 @@ DAT_RETURN create_events(void)
time.total += time.evdc;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d cr_evd created %p\n", getpid(), h_cr_evd);
@@ -1935,7 +1836,7 @@ DAT_RETURN create_events(void)
DAT_EVD_CONNECTION_FLAG, &h_conn_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_create: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d con_evd created %p\n", getpid(), h_conn_evd);
@@ -1947,7 +1848,7 @@ DAT_RETURN create_events(void)
h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_req_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d dto_req_evd created %p\n", getpid(),
@@ -1960,7 +1861,7 @@ DAT_RETURN create_events(void)
h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_rcv_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(),
@@ -1971,7 +1872,7 @@ DAT_RETURN create_events(void)
ret = dat_evd_query(h_dto_req_evd, DAT_EVD_FIELD_EVD_QLEN, ¶m);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else if (param.evd_qlen < (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2) {
fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
@@ -2001,7 +1902,7 @@ DAT_RETURN destroy_events(void)
ret = dat_evd_free(h_cr_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing cr EVD: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Freed cr EVD\n", getpid());
@@ -2015,7 +1916,7 @@ DAT_RETURN destroy_events(void)
ret = dat_evd_free(h_conn_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing conn EVD: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Freed conn EVD\n", getpid());
@@ -2033,7 +1934,7 @@ DAT_RETURN destroy_events(void)
time.total += time.evdf;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Freed dto EVD\n", getpid());
@@ -2047,7 +1948,7 @@ DAT_RETURN destroy_events(void)
ret = dat_evd_free(h_dto_req_evd);
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing dto EVD: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Freed dto EVD\n", getpid());
@@ -2065,7 +1966,7 @@ DAT_RETURN destroy_events(void)
time.total += time.cnof;
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d Error freeing dto CNO: %s\n",
- getpid(), DT_RetToString(ret));
+ getpid(), DT_RetToStr(ret));
return (ret);
} else {
LOGPRINTF("%d Freed dto CNO\n", getpid());
@@ -2080,7 +1981,7 @@ DAT_RETURN destroy_events(void)
* but don't assume the values are zero-based or contiguous.
*/
char errmsg[512] = { 0 };
-const char *DT_RetToString(DAT_RETURN ret_value)
+const char *DT_RetToStr(DAT_RETURN ret_value)
{
const char *major_msg, *minor_msg;
@@ -2096,7 +1997,7 @@ const char *DT_RetToString(DAT_RETURN ret_value)
/*
* Map DAT_EVENT_CODE values to readable strings
*/
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code)
+const char *DT_EventToStr(DAT_EVENT_NUMBER event_code)
{
unsigned int i;
static struct {
More information about the general
mailing list