[ofa-general] [PATCH] uDAPL v2: fix dtest to handle CNO events properly

Davis, Arlin R arlin.r.davis at intel.com
Tue Aug 4 22:32:06 PDT 2009


modify dtest.c to cleanup CNO wait code and consolidate into
collect_event() call. After waking up from CNO wait the
consumer must check all EVD's. The EVD's under the CNO
could be dropped if already triggered or could come in any order.
DT_RetToString changed to DT_RetToStr and DT_EventToSTr
changed to DT_EventToStr for consistency.

diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index 739ccca..d868490 100755
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -104,6 +104,7 @@
 /* definitions */
 #define SERVER_CONN_QUAL  45248
 #define DTO_TIMEOUT       (1000*1000*5)
+#define CNO_TIMEOUT       (1000*1000*1)
 #define DTO_FLUSH_TIMEOUT (1000*1000*2)
 #define CONN_TIMEOUT      (1000*1000*10)
 #define SERVER_TIMEOUT    DAT_TIMEOUT_INFINITE
@@ -208,8 +209,8 @@ static int burst_msg_posted = 0;
 static int burst_msg_index = 0;

 /* forward prototypes */
-const char *DT_RetToString(DAT_RETURN ret_value);
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code);
+const char *DT_RetToStr(DAT_RETURN ret_value);
+const char *DT_EventToStr(DAT_EVENT_NUMBER event_code);
 void print_usage(void);
 double get_time(void);
 void init_data(void);
@@ -262,6 +263,51 @@ void flush_evds(void)
        }
 }

+
+static inline DAT_RETURN
+collect_event(DAT_EVD_HANDLE dto_evd,
+             DAT_EVENT *event,
+             DAT_TIMEOUT timeout,
+             int *counter)
+{
+       DAT_EVD_HANDLE  evd = DAT_HANDLE_NULL;
+       DAT_COUNT       nmore;
+       DAT_RETURN      ret = DAT_SUCCESS;
+
+       if (use_cno) {
+retry:
+               /* CNO wait could return EVD's in any order and
+                * may drop some EVD notification's if already
+                * triggered. Once woken, simply dequeue the
+                * Evd the caller wants to collect and return.
+                * If notification without EVD, retry.
+                */
+               ret = dat_cno_wait(h_dto_cno, CNO_TIMEOUT, &evd);
+               if (dat_evd_dequeue(dto_evd, event) != DAT_SUCCESS) {
+                       if (ret == DAT_SUCCESS)
+                               printf(" WARNING: CNO notification:"
+                                      " without EVD?\n");
+                       goto retry;
+               }
+               ret = DAT_SUCCESS; /* cno timed out, but EVD dequeued */
+
+       } else if (!polling) {
+
+               /* use wait to dequeue */
+               ret = dat_evd_wait(dto_evd, timeout, 1, event, &nmore);
+               if (ret != DAT_SUCCESS)
+                       fprintf(stderr,
+                               "Error waiting on h_dto_evd %p: %s\n",
+                               dto_evd, DT_RetToStr(ret));
+
+       } else {
+               while (dat_evd_dequeue(dto_evd, event) == DAT_QUEUE_EMPTY)
+                       if (counter)
+                               (*counter)++;
+       }
+       return (ret);
+}
+
 int main(int argc, char **argv)
 {
        int i, c;
@@ -355,7 +401,7 @@ int main(int argc, char **argv)
        time.open += ((stop - start) * 1.0e6);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d: Error Adaptor open: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                exit(1);
        } else
                LOGPRINTF("%d Opened Interface Adaptor\n", getpid());
@@ -368,7 +414,7 @@ int main(int argc, char **argv)
        time.pzc += ((stop - start) * 1.0e6);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error creating Protection Zone: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                exit(1);
        } else
                LOGPRINTF("%d Created Protection Zone\n", getpid());
@@ -378,7 +424,7 @@ int main(int argc, char **argv)
        ret = register_rdma_memory();
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error registering RDMA memory: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d Register RDMA memory done\n", getpid());
@@ -387,7 +433,7 @@ int main(int argc, char **argv)
        ret = create_events();
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error creating events: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else {
                LOGPRINTF("%d Create events done\n", getpid());
@@ -419,7 +465,7 @@ int main(int argc, char **argv)
        time.total += time.epc;
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_ep_create: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d EP created %p \n", getpid(), h_ep);
@@ -431,7 +477,7 @@ int main(int argc, char **argv)
        ret = connect_ep(hostname, SERVER_CONN_QUAL);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error connect_ep: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d connect_ep complete\n", getpid());
@@ -440,7 +486,7 @@ int main(int argc, char **argv)
        ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_ep_query: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);
@@ -483,7 +529,7 @@ int main(int argc, char **argv)
        ret = do_rdma_write_with_msg();
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error do_rdma_write_with_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());
@@ -492,7 +538,7 @@ int main(int argc, char **argv)
        ret = do_rdma_read_with_msg();
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error do_rdma_read_with_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else
                LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());
@@ -501,7 +547,7 @@ int main(int argc, char **argv)
        ret = do_ping_pong_msg();
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                goto cleanup;
        } else {
                LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());
@@ -528,7 +574,7 @@ complete:
                time.total += time.epf;
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing EP: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                } else {
                        LOGPRINTF("%d Freed EP\n", getpid());
                        h_ep = DAT_HANDLE_NULL;
@@ -540,7 +586,7 @@ complete:
        ret = destroy_events();
        if (ret != DAT_SUCCESS)
                fprintf(stderr, "%d Error destroy_events: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
        else
                LOGPRINTF("%d destroy events done\n", getpid());

@@ -548,7 +594,7 @@ complete:
        LOGPRINTF("%d unregister_rdma_memory \n", getpid());
        if (ret != DAT_SUCCESS)
                fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
        else
                LOGPRINTF("%d unregister_rdma_memory done\n", getpid());

@@ -560,7 +606,7 @@ complete:
        time.pzf += ((stop - start) * 1.0e6);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error freeing PZ: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
        } else {
                LOGPRINTF("%d Freed pz\n", getpid());
                h_pz = NULL;
@@ -574,7 +620,7 @@ complete:
        time.close += ((stop - start) * 1.0e6);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d: Error Adaptor close: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
        } else
                LOGPRINTF("%d Closed Interface Adaptor\n", getpid());

@@ -652,7 +698,6 @@ send_msg(void *data,
 {
        DAT_LMR_TRIPLET iov;
        DAT_EVENT event;
-       DAT_COUNT nmore;
        DAT_RETURN ret;

        iov.lmr_context = context;
@@ -669,47 +714,23 @@ send_msg(void *data,

        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d: ERROR: dat_ep_post_send() %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return ret;
        }

        if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
-               if (polling) {
-                       printf("%d Polling post send completion...\n",
-                              getpid());
-                       while (dat_evd_dequeue(h_dto_req_evd, &event) ==
-                              DAT_QUEUE_EMPTY) ;
-               } else {
-                       LOGPRINTF("%d waiting for post_send completion event\n",
-                                 getpid());
-                       if (use_cno) {
-                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                               ret =
-                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                         getpid(), evd);
-                               if (evd != h_dto_req_evd) {
-                                       /* CNO timeout, already on EVD */
-                                       if (evd != NULL)
-                                               return (ret);
-                               }
-                       }
-                       /* use wait to dequeue */
-                       ret =
-                           dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
-                                        &nmore);
-                       if (ret != DAT_SUCCESS) {
-                               fprintf(stderr,
-                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                       getpid(), DT_RetToString(ret));
-                               return ret;
-                       }
-               }
+
+               if (collect_event(h_dto_req_evd,
+                                 &event,
+                                 DTO_TIMEOUT,
+                                 &poll_count) != DAT_SUCCESS)
+                       return (DAT_ABORT);

                /* validate event number, len, cookie, and status */
                if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                        fprintf(stderr, "%d: ERROR: DTO event number %s\n",
-                               getpid(), DT_EventToSTr(event.event_number));
+                               getpid(),
+                               DT_EventToStr(event.event_number));
                        return (DAT_ABORT);
                }

@@ -730,7 +751,7 @@ send_msg(void *data,
                if (event.event_data.dto_completion_event_data.status !=
                    DAT_SUCCESS) {
                        fprintf(stderr, "%d: ERROR: DTO event status %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (DAT_ABORT);
                }
        }
@@ -772,7 +793,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)

        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error registering send msg buffer: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else
                LOGPRINTF("%d Registered send Message Buffer %p \n",
@@ -796,7 +817,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                             &registered_addr_recv_msg);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error registering recv msg buffer: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else
                LOGPRINTF("%d Registered Receive Message Buffer %p\n",
@@ -823,7 +844,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d Error registering recv msg buffer: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else
                        LOGPRINTF("%d Registered Receive Message Buffer %p\n",
@@ -846,7 +867,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                                     h_cr_evd, DAT_PSP_CONSUMER_FLAG, &h_psp);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_psp_create: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else
                        LOGPRINTF("%d dat_psp_created for server listen\n",
@@ -858,7 +879,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                ret = dat_evd_wait(h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_evd_wait: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else
                        LOGPRINTF("%d dat_evd_wait for cr_evd completed\n",
@@ -866,7 +887,8 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)

                if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
                        fprintf(stderr, "%d Error unexpected cr event : %s\n",
-                               getpid(), DT_EventToSTr(event.event_number));
+                               getpid(),
+                               DT_EventToStr(event.event_number));
                        return (DAT_ABORT);
                }
                if ((event.event_data.cr_arrival_event_data.conn_qual !=
@@ -874,7 +896,8 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                    || (event.event_data.cr_arrival_event_data.sp_handle.
                        psp_handle != h_psp)) {
                        fprintf(stderr, "%d Error wrong cr event data : %s\n",
-                               getpid(), DT_EventToSTr(event.event_number));
+                               getpid(),
+                               DT_EventToStr(event.event_number));
                        return (DAT_ABORT);
                }

@@ -922,7 +945,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)

                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_cr_accept: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else
                        LOGPRINTF("%d dat_cr_accept completed\n", getpid());
@@ -966,7 +989,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
                                     0, DAT_CONNECT_DEFAULT_FLAG);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_ep_connect: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else
                        LOGPRINTF("%d dat_ep_connect completed\n", getpid());
@@ -990,7 +1013,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
 #ifdef TEST_REJECT_WITH_PRIVATE_DATA
        if (event.event_number != DAT_CONNECTION_EVENT_PEER_REJECTED) {
                fprintf(stderr, "%d expected conn reject event : %s\n",
-                       getpid(), DT_EventToSTr(event.event_number));
+                       getpid(), DT_EventToStr(event.event_number));
                return (DAT_ABORT);
        }
        /* get the reject private data and validate */
@@ -1013,7 +1036,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
        if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
                fprintf(stderr, "%d Error unexpected conn event : 0x%x %s\n",
                        getpid(), event.event_number,
-                       DT_EventToSTr(event.event_number));
+                       DT_EventToStr(event.event_number));
                return (DAT_ABORT);
        }

@@ -1064,7 +1087,7 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)

        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error send_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else
                LOGPRINTF("%d send_msg completed\n", getpid());
@@ -1072,42 +1095,17 @@ DAT_RETURN connect_ep(char *hostname, DAT_CONN_QUAL conn_id)
        /*
         *  Wait for remote RMR information for RDMA
         */
-       if (polling) {
-               printf("%d Polling for remote to send RMR data\n", getpid());
-               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
-                      DAT_QUEUE_EMPTY) ;
-       } else {
-               printf("%d Waiting for remote to send RMR data\n", getpid());
-               if (use_cno) {
-                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                 getpid(), evd);
-                       if (evd != h_dto_rcv_evd) {
-                               /* CNO timeout, already on EVD */
-                               if (evd != NULL)
-                                       return (ret);
-                       }
-               }
-               /* use wait to dequeue */
-               ret =
-                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
-               if (ret != DAT_SUCCESS) {
-                       fprintf(stderr,
-                               "%d Error waiting on h_dto_rcv_evd: %s\n",
-                               getpid(), DT_RetToString(ret));
-                       return (ret);
-               } else {
-                       LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n",
-                                 getpid());
-               }
-       }
-
+       if (collect_event(h_dto_rcv_evd,
+                         &event,
+                         DTO_TIMEOUT,
+                         &poll_count) != DAT_SUCCESS)
+               return (DAT_ABORT);
+
        printf("%d remote RMR data arrived!\n", getpid());

        if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                       getpid(), DT_EventToSTr(event.event_number));
+                       getpid(), DT_EventToStr(event.event_number));
                return (DAT_ABORT);
        }
        if ((event.event_data.dto_completion_event_data.transfered_length !=
@@ -1162,7 +1160,7 @@ void disconnect_ep(void)
                        if (ret != DAT_SUCCESS) {
                                fprintf(stderr,
                                        "%d Error dat_ep_disconnect: %s\n",
-                                       getpid(), DT_RetToString(ret));
+                                       getpid(), DT_RetToStr(ret));
                        } else {
                                LOGPRINTF("%d dat_ep_disconnect completed\n",
                                          getpid());
@@ -1177,7 +1175,7 @@ void disconnect_ep(void)
                                 &nmore);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_evd_wait: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                } else {
                        LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n",
                                  getpid());
@@ -1189,7 +1187,7 @@ void disconnect_ep(void)
                ret = dat_psp_free(h_psp);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_psp_free: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                } else {
                        LOGPRINTF("%d dat_psp_free completed\n", getpid());
                }
@@ -1203,7 +1201,7 @@ void disconnect_ep(void)
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d Error deregistering send msg mr: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                } else {
                        LOGPRINTF("%d Unregistered send message Buffer\n",
                                  getpid());
@@ -1219,7 +1217,7 @@ void disconnect_ep(void)
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d Error deregistering recv msg mr: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                } else {
                        LOGPRINTF("%d Unregistered recv message Buffer\n",
                                  getpid());
@@ -1232,7 +1230,6 @@ void disconnect_ep(void)
 DAT_RETURN do_rdma_write_with_msg(void)
 {
        DAT_EVENT event;
-       DAT_COUNT nmore;
        DAT_LMR_TRIPLET l_iov[MSG_IOV_COUNT];
        DAT_RMR_TRIPLET r_iov;
        DAT_DTO_COOKIE cookie;
@@ -1277,7 +1274,7 @@ DAT_RETURN do_rdma_write_with_msg(void)
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d: ERROR: dat_ep_post_rdma_write() %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (DAT_ABORT);
                }
                LOGPRINTF("%d rdma_write # %d completed\n", getpid(), i + 1);
@@ -1296,41 +1293,19 @@ DAT_RETURN do_rdma_write_with_msg(void)

        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error send_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d send_msg completed\n", getpid());
        }

-       /*
-        *  Collect first event, write completion or the inbound recv
-        */
-       if (polling) {
-               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
-                      DAT_QUEUE_EMPTY)
-                       rdma_wr_poll_count++;
-       } else {
-               LOGPRINTF("%d waiting for message receive event\n", getpid());
-               if (use_cno) {
-                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                 getpid(), evd);
-                       if (evd != h_dto_rcv_evd) {
-                               /* CNO timeout, already on EVD */
-                               if (evd != NULL)
-                                       return (ret);
-                       }
-               }
-               /* use wait to dequeue */
-               ret =
-                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
-               if (ret != DAT_SUCCESS) {
-                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                               getpid(), DT_RetToString(ret));
-                       return (ret);
-               }
-       }
+       /* inbound recv event, send completion's suppressed */
+       if (collect_event(h_dto_rcv_evd,
+                         &event,
+                         DTO_TIMEOUT,
+                         &rdma_wr_poll_count) != DAT_SUCCESS)
+               return (DAT_ABORT);
+
        stop = get_time();
        time.rdma_wr = ((stop - start) * 1.0e6);

@@ -1338,7 +1313,7 @@ DAT_RETURN do_rdma_write_with_msg(void)
        printf("%d inbound rdma_write; send message arrived!\n", getpid());
        if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                       getpid(), DT_EventToSTr(event.event_number));
+                       getpid(), DT_EventToStr(event.event_number));
                return (DAT_ABORT);
        }

@@ -1386,7 +1361,6 @@ DAT_RETURN do_rdma_write_with_msg(void)
 DAT_RETURN do_rdma_read_with_msg(void)
 {
        DAT_EVENT event;
-       DAT_COUNT nmore;
        DAT_LMR_TRIPLET l_iov;
        DAT_RMR_TRIPLET r_iov;
        DAT_DTO_COOKIE cookie;
@@ -1425,44 +1399,21 @@ DAT_RETURN do_rdma_read_with_msg(void)
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d: ERROR: dat_ep_post_rdma_read() %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (DAT_ABORT);
                }

-               if (polling) {
-                       while (dat_evd_dequeue(h_dto_req_evd, &event) ==
-                              DAT_QUEUE_EMPTY)
-                               rdma_rd_poll_count[i]++;
-               } else {
-                       LOGPRINTF("%d waiting for rdma_read completion event\n",
-                                 getpid());
-                       if (use_cno) {
-                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                               ret =
-                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                         getpid(), evd);
-                               if (evd != h_dto_req_evd) {
-                                       /* CNO timeout, already on EVD */
-                                       if (evd != NULL)
-                                               return (ret);
-                               }
-                       }
-                       /* use wait to dequeue */
-                       ret =
-                           dat_evd_wait(h_dto_req_evd, DTO_TIMEOUT, 1, &event,
-                                        &nmore);
-                       if (ret != DAT_SUCCESS) {
-                               fprintf(stderr,
-                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                       getpid(), DT_RetToString(ret));
-                               return ret;
-                       }
-               }
+               /* RDMA read completion event */
+               if (collect_event(h_dto_req_evd,
+                                 &event,
+                                 DTO_TIMEOUT,
+                                 &rdma_rd_poll_count[i]) != DAT_SUCCESS)
+                       return (DAT_ABORT);
+
                /* validate event number, len, cookie, and status */
                if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                        fprintf(stderr, "%d: ERROR: DTO event number %s\n",
-                               getpid(), DT_EventToSTr(event.event_number));
+                               getpid(), DT_EventToStr(event.event_number));
                        return (DAT_ABORT);
                }
                if ((event.event_data.dto_completion_event_data.
@@ -1481,7 +1432,7 @@ DAT_RETURN do_rdma_read_with_msg(void)
                if (event.event_data.dto_completion_event_data.status !=
                    DAT_SUCCESS) {
                        fprintf(stderr, "%d: ERROR: DTO event status %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (DAT_ABORT);
                }
                stop = get_time();
@@ -1513,48 +1464,25 @@ DAT_RETURN do_rdma_read_with_msg(void)

        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error send_msg: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d send_msg completed\n", getpid());
        }

-       /*
-        *  Collect first event, write completion or the inbound recv with immed
-        */
        printf("%d Waiting for inbound message....\n", getpid());
-       if (polling) {
-               while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
-                      DAT_QUEUE_EMPTY) ;
-       } else {
-               LOGPRINTF("%d waiting for message receive event\n", getpid());
-               if (use_cno) {
-                       DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-
-                       ret = dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                       LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                 getpid(), evd);
-                       if (evd != h_dto_rcv_evd) {
-                               /* CNO timeout, already on EVD */
-                               if (evd != NULL)
-                                       return (ret);
-                       }
-               }
-               /* use wait to dequeue */
-               ret =
-                   dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);
-               if (ret != DAT_SUCCESS) {
-                       fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
-                               getpid(), DT_RetToString(ret));
-                       return (ret);
-               }
-       }
+
+       if (collect_event(h_dto_rcv_evd,
+                         &event,
+                         DTO_TIMEOUT,
+                         &poll_count) != DAT_SUCCESS)
+               return (DAT_ABORT);

        /* validate event number and status */
        printf("%d inbound rdma_read; send message arrived!\n", getpid());
        if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                       getpid(), DT_EventToSTr(event.event_number));
+                       getpid(), DT_EventToStr(event.event_number));
                return (DAT_ABORT);
        }

@@ -1603,7 +1531,6 @@ DAT_RETURN do_rdma_read_with_msg(void)
 DAT_RETURN do_ping_pong_msg()
 {
        DAT_EVENT event;
-       DAT_COUNT nmore;
        DAT_DTO_COOKIE cookie;
        DAT_LMR_TRIPLET l_iov;
        DAT_RETURN ret;
@@ -1635,7 +1562,7 @@ DAT_RETURN do_ping_pong_msg()
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr,
                                "%d Error posting recv msg buffer: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Posted Receive Message Buffer %p\n",
@@ -1673,47 +1600,21 @@ DAT_RETURN do_ping_pong_msg()

                        if (ret != DAT_SUCCESS) {
                                fprintf(stderr, "%d Error send_msg: %s\n",
-                                       getpid(), DT_RetToString(ret));
+                                       getpid(), DT_RetToStr(ret));
                                return (ret);
                        } else {
                                LOGPRINTF("%d send_msg completed\n", getpid());
                        }
                }

-               /* Wait for recv message */
-               if (polling) {
-                       poll_count = 0;
-                       LOGPRINTF("%d Polling for message receive event\n",
-                                 getpid());
-                       while (dat_evd_dequeue(h_dto_rcv_evd, &event) ==
-                              DAT_QUEUE_EMPTY)
-                               poll_count++;
-               } else {
-                       LOGPRINTF("%d waiting for message receive event\n",
-                                 getpid());
-                       if (use_cno) {
-                               DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
-                               ret =
-                                   dat_cno_wait(h_dto_cno, DTO_TIMEOUT, &evd);
-                               LOGPRINTF("%d cno wait return evd_handle=%p\n",
-                                         getpid(), evd);
-                               if (evd != h_dto_rcv_evd) {
-                                       /* CNO timeout, already on EVD */
-                                       if (evd != NULL)
-                                               return (ret);
-                               }
-                       }
-                       /* use wait to dequeue */
-                       ret =
-                           dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event,
-                                        &nmore);
-                       if (ret != DAT_SUCCESS) {
-                               fprintf(stderr,
-                                       "%d: ERROR: DTO dat_evd_wait() %s\n",
-                                       getpid(), DT_RetToString(ret));
-                               return (ret);
-                       }
-               }
+               /* recv message, send completions suppressed */
+               if (collect_event(h_dto_rcv_evd,
+                                 &event,
+                                 DTO_TIMEOUT,
+                                 &poll_count) != DAT_SUCCESS)
+                       return (DAT_ABORT);
+
+
                /* start timer after first message arrives on server */
                if (i == 0) {
                        start = get_time();
@@ -1722,7 +1623,7 @@ DAT_RETURN do_ping_pong_msg()
                LOGPRINTF("%d inbound message; message arrived!\n", getpid());
                if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
                        fprintf(stderr, "%d Error unexpected DTO event : %s\n",
-                               getpid(), DT_EventToSTr(event.event_number));
+                               getpid(), DT_EventToStr(event.event_number));
                        return (DAT_ABORT);
                }
                if ((event.event_data.dto_completion_event_data.
@@ -1762,7 +1663,7 @@ DAT_RETURN do_ping_pong_msg()

                        if (ret != DAT_SUCCESS) {
                                fprintf(stderr, "%d Error send_msg: %s\n",
-                                       getpid(), DT_RetToString(ret));
+                                       getpid(), DT_RetToStr(ret));
                                return (ret);
                        } else {
                                LOGPRINTF("%d send_msg completed\n", getpid());
@@ -1805,7 +1706,7 @@ DAT_RETURN register_rdma_memory(void)
        if (ret != DAT_SUCCESS) {
                fprintf(stderr,
                        "%d Error registering Receive RDMA buffer: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",
@@ -1827,7 +1728,7 @@ DAT_RETURN register_rdma_memory(void)
                             &registered_size_send, &registered_addr_send);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error registering send RDMA buffer: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d Registered Send RDMA Buffer %p\n",
@@ -1854,7 +1755,7 @@ DAT_RETURN unregister_rdma_memory(void)
                time.total += time.unreg;
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error deregistering recv mr: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Unregistered Recv Buffer\n", getpid());
@@ -1868,7 +1769,7 @@ DAT_RETURN unregister_rdma_memory(void)
                ret = dat_lmr_free(h_lmr_send);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error deregistering send mr: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Unregistered send Buffer\n", getpid());
@@ -1904,7 +1805,7 @@ DAT_RETURN create_events(void)
                time.total += time.cnoc;
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error dat_cno_create: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d cr_evd created, %p\n", getpid(),
@@ -1922,7 +1823,7 @@ DAT_RETURN create_events(void)
        time.total += time.evdc;
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_evd_create: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d cr_evd created %p\n", getpid(), h_cr_evd);
@@ -1935,7 +1836,7 @@ DAT_RETURN create_events(void)
                             DAT_EVD_CONNECTION_FLAG, &h_conn_evd);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_evd_create: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d con_evd created %p\n", getpid(), h_conn_evd);
@@ -1947,7 +1848,7 @@ DAT_RETURN create_events(void)
                             h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_req_evd);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d dto_req_evd created %p\n", getpid(),
@@ -1960,7 +1861,7 @@ DAT_RETURN create_events(void)
                             h_dto_cno, DAT_EVD_DTO_FLAG, &h_dto_rcv_evd);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else {
                LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(),
@@ -1971,7 +1872,7 @@ DAT_RETURN create_events(void)
        ret = dat_evd_query(h_dto_req_evd, DAT_EVD_FIELD_EVD_QLEN, &param);
        if (ret != DAT_SUCCESS) {
                fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",
-                       getpid(), DT_RetToString(ret));
+                       getpid(), DT_RetToStr(ret));
                return (ret);
        } else if (param.evd_qlen < (MSG_BUF_COUNT + MAX_RDMA_RD + burst) * 2) {
                fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",
@@ -2001,7 +1902,7 @@ DAT_RETURN destroy_events(void)
                ret = dat_evd_free(h_cr_evd);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing cr EVD: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Freed cr EVD\n", getpid());
@@ -2015,7 +1916,7 @@ DAT_RETURN destroy_events(void)
                ret = dat_evd_free(h_conn_evd);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing conn EVD: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Freed conn EVD\n", getpid());
@@ -2033,7 +1934,7 @@ DAT_RETURN destroy_events(void)
                time.total += time.evdf;
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing dto EVD: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Freed dto EVD\n", getpid());
@@ -2047,7 +1948,7 @@ DAT_RETURN destroy_events(void)
                ret = dat_evd_free(h_dto_req_evd);
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing dto EVD: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Freed dto EVD\n", getpid());
@@ -2065,7 +1966,7 @@ DAT_RETURN destroy_events(void)
                time.total += time.cnof;
                if (ret != DAT_SUCCESS) {
                        fprintf(stderr, "%d Error freeing dto CNO: %s\n",
-                               getpid(), DT_RetToString(ret));
+                               getpid(), DT_RetToStr(ret));
                        return (ret);
                } else {
                        LOGPRINTF("%d Freed dto CNO\n", getpid());
@@ -2080,7 +1981,7 @@ DAT_RETURN destroy_events(void)
  * but don't assume the values are zero-based or contiguous.
  */
 char errmsg[512] = { 0 };
-const char *DT_RetToString(DAT_RETURN ret_value)
+const char *DT_RetToStr(DAT_RETURN ret_value)
 {
        const char *major_msg, *minor_msg;

@@ -2096,7 +1997,7 @@ const char *DT_RetToString(DAT_RETURN ret_value)
 /*
  * Map DAT_EVENT_CODE values to readable strings
  */
-const char *DT_EventToSTr(DAT_EVENT_NUMBER event_code)
+const char *DT_EventToStr(DAT_EVENT_NUMBER event_code)
 {
        unsigned int i;
        static struct {



More information about the general mailing list