[openib-general] [PATCH] uDAPL dtest changes to measure RDMA reads
Arlin Davis
arlin.r.davis at intel.com
Wed Sep 21 10:24:06 PDT 2005
James,
Here is a patch to improve dtest and measure RDMA reads. Attachment included.
-arlin
Signed-off by: Arlin Davis <ardavis at ichips.intel.com>
Index: test/dtest/dtest.c
===================================================================
--- test/dtest/dtest.c (revision 3459)
+++ test/dtest/dtest.c (working copy)
@@ -42,10 +42,12 @@
#include <getopt.h>
#ifndef DAPL_PROVIDER
-#define DAPL_PROVIDER "OpenIB1_2"
+#define DAPL_PROVIDER "OpenIB-ib0"
#endif
#define MAX_POLLING_CNT 50000
+#define MAX_RDMA_RD 4
+#define MAX_PROCS 1000
/* Header files needed for DAT/uDAPL */
#include "dat/udat.h"
@@ -66,7 +68,8 @@
static DAT_CR_HANDLE h_cr = DAT_HANDLE_NULL;
static DAT_EVD_HANDLE h_async_evd = DAT_HANDLE_NULL;
-static DAT_EVD_HANDLE h_dto_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_req_evd = DAT_HANDLE_NULL;
+static DAT_EVD_HANDLE h_dto_rcv_evd = DAT_HANDLE_NULL;
static DAT_EVD_HANDLE h_cr_evd = DAT_HANDLE_NULL;
static DAT_EVD_HANDLE h_conn_evd = DAT_HANDLE_NULL;
static DAT_CNO_HANDLE h_dto_cno = DAT_HANDLE_NULL;
@@ -123,7 +126,8 @@
double epc;
double epf;
double rdma_wr;
- double rdma_rd;
+ double rdma_rd[MAX_RDMA_RD];
+ double rdma_rd_total;
double rtt;
double close;
} time;
@@ -137,7 +141,7 @@
static int polling=1;
static int poll_count=0;
static int rdma_wr_poll_count=0;
-static int rdma_rd_poll_count=0;
+static int rdma_rd_poll_count[MAX_RDMA_RD]={0};
static int pin_memory=0;
static int delay=0;
static int buf_len=RDMA_BUFFER_SIZE;
@@ -147,9 +151,6 @@
static int burst_msg_posted=0;
static int burst_msg_index=0;
-#define MAX_RDMA_RD 4
-#define MAX_PROCS 1000
-
static pid_t child[MAX_PROCS+1];
/* forward prototypes */
@@ -181,7 +182,7 @@
main(int argc, char **argv)
{
- int c;
+ int i,c;
DAT_RETURN ret;
/* parse arguments */
@@ -191,7 +192,6 @@
{
case 's':
server = 1;
- printf("%d Running as server\n",getpid());
fflush(stdout);
break;
case 'c':
@@ -310,7 +310,8 @@
ep_attr.ep_provider_specific = NULL;
start = get_time();
- ret = dat_ep_create( h_ia, h_pz, h_dto_evd, h_dto_evd, h_conn_evd, &ep_attr, &h_ep );
+ ret = dat_ep_create( h_ia, h_pz, h_dto_rcv_evd,
+ h_dto_req_evd, h_conn_evd, &ep_attr, &h_ep );
stop = get_time();
time.epc += ((stop - start)*1.0e6);
time.total += time.epc;
@@ -432,13 +433,16 @@
LOGPRINTF("%d Closed Interface Adaptor\n",getpid());
printf( "\n%d: DAPL Test Complete.\n\n",getpid());
- printf( "%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
- getpid(), time.rdma_wr, burst, time.rdma_wr/burst, rdma_wr_poll_count );
- printf( "%d: RDMA read: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
- getpid(), time.rdma_rd, MAX_RDMA_RD, time.rdma_rd/MAX_RDMA_RD, rdma_rd_poll_count );
- printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n\n",
- getpid(), time.rtt, burst, time.rtt/burst, poll_count );
-
+ printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
+ getpid(), time.rtt, burst, time.rtt/burst, poll_count );
+ printf( "%d: RDMA write: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
+ getpid(), time.rdma_wr, burst,
+ time.rdma_wr/burst, rdma_wr_poll_count );
+ for(i=0;i<MAX_RDMA_RD;i++) {
+ printf( "%d: RDMA read: Total=%10.2lf usec, %d bursts, itime=%10.2lf usec, pc=%d\n",
+ getpid(),time.rdma_rd_total,MAX_RDMA_RD,
+ time.rdma_rd[i],rdma_rd_poll_count[i] );
+ }
printf( "%d: open: %10.2lf usec\n", getpid(), time.open );
printf( "%d: close: %10.2lf usec\n", getpid(), time.close );
printf( "%d: PZ create: %10.2lf usec\n", getpid(), time.pzc );
@@ -509,7 +513,7 @@
if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
if ( polling ) {
printf("%d Polling post send completion...\n",getpid());
- while ( dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY );
+ while ( dat_evd_dequeue( h_dto_req_evd, &event ) == DAT_QUEUE_EMPTY );
}
else {
LOGPRINTF("%d waiting for post_send completion event\n", getpid());
@@ -517,15 +521,15 @@
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_evd ) {
+ if ( evd != h_dto_req_evd ) {
fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
getpid());
return( DAT_ABORT );
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
getpid(),DT_RetToString(ret));
@@ -790,7 +794,7 @@
*/
if ( polling ) {
printf("%d Polling for remote to send RMR data\n",getpid());
- while ( dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY );
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
}
else {
printf("%d Waiting for remote to send RMR data\n",getpid());
@@ -799,22 +803,22 @@
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_evd ) {
+ if ( evd != h_dto_rcv_evd ) {
fprintf(stderr,
- "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
getpid());
return( DAT_ABORT );
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error waiting on h_dto_evd: %s\n",
+ fprintf(stderr, "%d Error waiting on h_dto_rcv_evd: %s\n",
getpid(),DT_RetToString(ret));
return(ret);
}
else {
- LOGPRINTF("%d dat_evd_wait h_dto_evd completed\n", getpid());
+ LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());
}
}
@@ -988,7 +992,7 @@
* Collect first event, write completion or the inbound recv with immed
*/
if ( polling ) {
- while ( dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
rdma_wr_poll_count++;
}
else {
@@ -997,14 +1001,15 @@
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_evd ) {
- fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
- getpid());
+ if ( evd != h_dto_rcv_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
return( ret );
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
getpid(),DT_RetToString(ret));
@@ -1067,8 +1072,6 @@
printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());
- cookie.as_64 = 0x5555;
-
if ( recv_msg_index >= MSG_BUF_COUNT )
return( DAT_ABORT );
@@ -1087,22 +1090,71 @@
l_iov.pad = 0;
l_iov.virtual_address = (DAT_VADDR)(unsigned long)sbuf;
l_iov.segment_length = buf_len;
-
- start = get_time();
+
for (i=0;i<MAX_RDMA_RD;i++) {
- cookie.as_64 = 0x9999;
- ret = dat_ep_post_rdma_read( h_ep, // ep_handle
- 1, // num_segments
- &l_iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- DAT_COMPLETION_SUPPRESS_FLAG );
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n",
- getpid(),DT_RetToString(ret));
- return( DAT_ABORT );
- }
- LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
+ cookie.as_64 = 0x9999;
+ start = get_time();
+ ret = dat_ep_post_rdma_read( h_ep, // ep_handle
+ 1, // num_segments
+ &l_iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ DAT_COMPLETION_DEFAULT_FLAG );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n",
+ getpid(),DT_RetToString(ret));
+ return( DAT_ABORT );
+ }
+
+ if (polling) {
+ while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY)
+ rdma_rd_poll_count[i]++;
+ }
+ else {
+ LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());
+ if (use_cno) {
+ DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
+ ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
+ if ( evd != h_dto_req_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",
+ getpid());
+ return( DAT_ABORT );
+ }
+ }
+ /* use wait to dequeue */
+ ret = dat_evd_wait( h_dto_req_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
+ getpid(),DT_RetToString(ret));
+ return ret;
+ }
+ }
+ /* validate event number, len, cookie, and status */
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ fprintf(stderr, "%d: ERROR: DTO event number %s\n",
+ getpid(),DT_EventToSTr(event.event_number));
+ return( DAT_ABORT );
+ }
+ if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {
+ fprintf(stderr, "%d: ERROR: DTO len %d or cookie %x\n",
+ getpid(),
+ event.event_data.dto_completion_event_data.transfered_length,
+ event.event_data.dto_completion_event_data.user_cookie.as_64 );
+ return( DAT_ABORT );
+ }
+ if (event.event_data.dto_completion_event_data.status != DAT_SUCCESS) {
+ fprintf(stderr, "%d: ERROR: DTO event status %s\n",
+ getpid(),DT_RetToString(ret));
+ return( DAT_ABORT );
+ }
+ stop = get_time();
+ time.rdma_rd[i] = ((stop - start)*1.0e6);
+ time.rdma_rd_total += time.rdma_rd[i];
+
+ LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);
}
/*
@@ -1134,8 +1186,7 @@
*/
printf("%d Waiting for inbound message....\n",getpid());
if ( polling ) {
- while ( dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
- rdma_rd_poll_count++;
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );
}
else {
LOGPRINTF("%d waiting for message receive event\n", getpid());
@@ -1143,22 +1194,21 @@
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_evd ) {
- fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
- getpid());
+ if ( evd != h_dto_rcv_evd ) {
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
return( ret );
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
getpid(),DT_RetToString(ret));
return( ret );
}
}
- stop = get_time();
- time.rdma_rd = ((stop - start)*1.0e6);
/* validate event number and status */
printf("%d inbound rdma_read; send message arrived!\n",getpid());
@@ -1283,7 +1333,7 @@
if ( polling ) {
poll_count=0;
LOGPRINTF("%d Polling for message receive event\n", getpid());
- while ( dat_evd_dequeue( h_dto_evd, &event ) == DAT_QUEUE_EMPTY )
+ while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )
poll_count++;
}
else {
@@ -1292,15 +1342,16 @@
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );
LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);
- if ( evd != h_dto_evd )
+ if ( evd != h_dto_rcv_evd )
{
- fprintf(stderr, "%d Error waiting on h_dto_cno: evd != h_dto_evd\n",
- getpid());
+ fprintf(stderr,
+ "%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",
+ getpid());
return( ret );
}
}
/* use wait to dequeue */
- ret = dat_evd_wait( h_dto_evd, DTO_TIMEOUT, 1, &event, &nmore );
+ ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );
if (ret != DAT_SUCCESS) {
fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",
getpid(),DT_RetToString(ret));
@@ -1511,6 +1562,7 @@
else {
LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);
}
+
/* create conn EVD */
ret = dat_evd_create( h_ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &h_conn_evd );
if(ret != DAT_SUCCESS) {
@@ -1522,19 +1574,34 @@
LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);
}
- /* create dto EVD, with CNO if use_cno was set */
+ /* create dto SND EVD, with CNO if use_cno was set */
ret = dat_evd_create( h_ia,
- (MSG_BUF_COUNT*2)+burst*2,
+ MSG_BUF_COUNT+MAX_RDMA_RD+burst*2,
h_dto_cno,
DAT_EVD_DTO_FLAG,
- &h_dto_evd );
+ &h_dto_req_evd );
if(ret != DAT_SUCCESS) {
- fprintf(stderr, "%d Error dat_evd_create: %s\n",
+ fprintf(stderr, "%d Error dat_evd_create REQ: %s\n",
+ getpid(),DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );
+ }
+
+ /* create dto RCV EVD, with CNO if use_cno was set */
+ ret = dat_evd_create( h_ia,
+ MSG_BUF_COUNT,
+ h_dto_cno,
+ DAT_EVD_DTO_FLAG,
+ &h_dto_rcv_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error dat_evd_create RCV: %s\n",
getpid(),DT_RetToString(ret));
return (ret);
}
else {
- LOGPRINTF("%d dto_evd created %p\n", getpid(), h_dto_evd );
+ LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );
}
return DAT_SUCCESS;
@@ -1577,11 +1644,11 @@
}
}
- /* free dto EVD */
- if ( h_dto_evd != DAT_HANDLE_NULL ) {
- LOGPRINTF("%d Free dto EVD %p \n",getpid(),h_dto_evd);
+ /* free RCV dto EVD */
+ if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);
start = get_time();
- ret = dat_evd_free( h_dto_evd );
+ ret = dat_evd_free( h_dto_rcv_evd );
stop = get_time();
time.evdf += ((stop - start)*1.0e6);
time.total += time.evdf;
@@ -1592,7 +1659,22 @@
}
else {
LOGPRINTF("%d Freed dto EVD\n",getpid());
- h_dto_evd = DAT_HANDLE_NULL;
+ h_dto_rcv_evd = DAT_HANDLE_NULL;
+ }
+ }
+
+ /* free REQ dto EVD */
+ if ( h_dto_req_evd != DAT_HANDLE_NULL ) {
+ LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);
+ ret = dat_evd_free( h_dto_req_evd );
+ if(ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d Error freeing dto EVD: %s\n",
+ getpid(), DT_RetToString(ret));
+ return (ret);
+ }
+ else {
+ LOGPRINTF("%d Freed dto EVD\n",getpid());
+ h_dto_req_evd = DAT_HANDLE_NULL;
}
}
Index: test/dtest/makefile
===================================================================
--- test/dtest/makefile (revision 3459)
+++ test/dtest/makefile (working copy)
@@ -11,6 +11,6 @@
dtest: ./dtest.c
$(CC) $(CFLAGS) ./dtest.c -o dtest \
- -DDAPL_PROVIDER='"IB1"' \
+ -DDAPL_PROVIDER='"OpenIB-ib0"' \
-I $(DAT_INC) -L $(DAT_LIB) -ldat
-------------- next part --------------
A non-text attachment was scrubbed...
Name: dtest.patch
Type: application/octet-stream
Size: 18589 bytes
Desc: not available
URL: <http://lists.openfabrics.org/pipermail/general/attachments/20050921/afd17772/attachment.obj>
More information about the general
mailing list