[ofa-general] [PATCH 3/5][v2.0] dtestx: Add new options to test UD.
Arlin Davis
arlin.r.davis at intel.com
Wed Sep 24 13:44:03 PDT 2008
- many to one/many EP remote AH resolution, data flow
- bi-directional EP remote AH resolution, data flow
Signed-off by: Arlin Davis ardavis at ichips.intel.com
---
test/dtest/dtestx.c | 650 ++++++++++++++++++++++++++++++++++-----------------
1 files changed, 441 insertions(+), 209 deletions(-)
diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c
index fb89364..cfe00cd 100755
--- a/test/dtest/dtestx.c
+++ b/test/dtest/dtestx.c
@@ -75,6 +75,8 @@ int disconnect_ep(void);
dat_strerror(status, &maj_msg, &min_msg);\
fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\
exit(1);\
+ } else if (verbose) {\
+ printf("dtestx: %s success\n",str);\
}\
}
@@ -85,6 +87,8 @@ int disconnect_ep(void);
dat_strerror(status, &maj_msg, &min_msg);\
fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\
exit(1);\
+ } else if (verbose) {\
+ printf("dtestx: %s\n",str);\
}\
}
@@ -119,10 +123,14 @@ int disconnect_ep(void);
#define ntoh64(x) (x)
#endif /* __BYTE_ORDER == __BIG_ENDIAN */
+#define MIN(a, b) ((a < b) ? (a) : (b))
+#define MAX(a, b) ((a > b) ? (a) : (b))
+
#define DTO_TIMEOUT (1000*1000*5)
-#define CONN_TIMEOUT (1000*1000*10)
-#define SERVER_TIMEOUT (1000*1000*120)
-#define SERVER_CONN_QUAL 31111
+#define CONN_TIMEOUT (1000*1000*30)
+#define SERVER_TIMEOUT (DAT_TIMEOUT_INFINITE)
+#define CLIENT_ID 31111
+#define SERVER_ID 31112
#define BUF_SIZE 256
#define BUF_SIZE_ATOMIC 8
#define REG_MEM_COUNT 10
@@ -130,6 +138,7 @@ int disconnect_ep(void);
#define RCV_RDMA_BUF_INDEX 1
#define SEND_BUF_INDEX 2
#define RECV_BUF_INDEX 3
+#define MAX_EP_COUNT 8
DAT_VADDR *atomic_buf;
DAT_LMR_HANDLE lmr_atomic;
@@ -137,14 +146,14 @@ DAT_LMR_CONTEXT lmr_atomic_context;
DAT_RMR_CONTEXT rmr_atomic_context;
DAT_VLEN reg_atomic_size;
DAT_VADDR reg_atomic_addr;
-DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
-DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
-DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
-DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
-DAT_VLEN reg_size[ REG_MEM_COUNT ];
-DAT_VADDR reg_addr[ REG_MEM_COUNT ];
-DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
-DAT_EP_HANDLE ep;
+DAT_LMR_HANDLE lmr[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_VLEN reg_size[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_VADDR reg_addr[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT*MAX_EP_COUNT ];
+DAT_EP_HANDLE ep[MAX_EP_COUNT];
DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
@@ -152,21 +161,30 @@ DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
-DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
int server = 1;
int ud_test = 0;
+int multi_eps = 0;
int buf_size = BUF_SIZE;
int msg_size = sizeof(DAT_RMR_TRIPLET);
char provider[64] = DAPL_PROVIDER;
char hostname[256] = { 0 };
-DAT_IB_ADDR_HANDLE remote_ah;
+DAT_IB_ADDR_HANDLE remote_ah[MAX_EP_COUNT];
+int eps = 1;
+int verbose = 0;
+
+#define LOGPRINTF if (verbose) printf
void print_usage(void)
{
printf("\n dtestx usage \n\n");
+ printf("v: verbose\n");
printf("u unreliable datagram test\n");
+ printf("U: unreliable datagram test, UD endpoint count\n");
+ printf("m unreliable datagram test, multiple Server endpoints\n");
printf("b: buf length to allocate\n");
- printf("h: hostname/address of server, specified on client\n");
+ printf("h: hostname/address of Server, specified on Client\n");
+ printf("c: Client\n");
+ printf("s: Server, default\n");
printf("P: provider name (default = ofa-v2-ib0)\n");
printf("\n");
}
@@ -193,6 +211,7 @@ send_msg(
DAT_EVENT event;
DAT_COUNT nmore;
DAT_RETURN status;
+ int i,ep_idx=0,ah_idx=0;
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
@@ -200,26 +219,171 @@ send_msg(
iov.virtual_address = (DAT_VADDR)data;
iov.segment_length = (DAT_VLEN)size;
- if (ud_test)
- status = dat_ib_post_send_ud(ep, 1, &iov,
- &remote_ah, cookie, flags);
- else
- status = dat_ep_post_send(ep, 1, &iov, cookie, flags);
+ for (i=0;i<eps;i++) {
+ if (ud_test) {
+ /*
+ * Client and Server: ep[0] and ah[0] on single
+ * and ep[i] on multiple (-m) endpoint options.
+ */
+ if (multi_eps) {
+ ep_idx=i;
+ ah_idx=server?0:i;
+ }
+ printf("%s sending on ep=%p to remote_ah: ah=%p"
+ " qpn=0x%x addr=%s\n",
+ server?"Server":"Client", ep[ep_idx],
+ remote_ah[ah_idx].ah,
+ remote_ah[ah_idx].qpn,
+ inet_ntoa(((struct sockaddr_in *)
+ &remote_ah[ah_idx].ia_addr)->sin_addr));
+
+ /* client expects all data in on first EP */
+ status = dat_ib_post_send_ud(ep[ep_idx],
+ 1,
+ &iov,
+ &remote_ah[ah_idx],
+ cookie,
+ flags);
+
+ } else {
+ status = dat_ep_post_send(ep[0], 1, &iov,
+ cookie, flags);
+ }
+ _OK(status, "dat_ep_post_send");
+
+ if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT,
+ 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT &&
+ ud_test && event.event_number !=
+ DAT_IB_DTO_EVENT) {
+ printf("unexpected event waiting post_send "
+ "completion - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+ _OK(dto_event->status, "event status for post_send");
+ }
+ }
+}
- _OK(status, "dat_ep_post_send");
+/* RC - Server only, UD - Server and Client, one per EP */
+void process_cr(int idx)
+{
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN status;
+ int pdata;
+ DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
+ DAT_CONN_QUAL exp_qual = server?SERVER_ID:CLIENT_ID;
+ DAT_CR_PARAM cr_param;
+ DAT_CR_ARRIVAL_EVENT_DATA *cr_event =
+ &event.event_data.cr_arrival_event_data;
+
+ LOGPRINTF("%s waiting for connect[%d] request\n",
+ server?"Server":"Client",idx);
- if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT,
- 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ep_post_send");
+ status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "CR dat_evd_wait");
- if ((event.event_number != DAT_DTO_COMPLETION_EVENT) &&
- (ud_test && event.event_number != DAT_IB_DTO_EVENT)) {
- printf("unexpected event waiting for post_send "
- "completion - 0x%x\n", event.event_number);
- exit(1);
- }
- _OK(dto_event->status, "event status for post_send");
+ if (event.event_number != DAT_CONNECTION_REQUEST_EVENT &&
+ (ud_test && event.event_number !=
+ DAT_IB_UD_CONNECTION_REQUEST_EVENT)) {
+ printf("unexpected event,!conn req: 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ if ((cr_event->conn_qual != exp_qual) ||
+ (cr_event->sp_handle.psp_handle != psp)) {
+ printf("wrong cr event data\n");
+ exit(1);
+ }
+
+ cr = cr_event->cr_handle;
+ status = dat_cr_query(cr, DAT_CSP_FIELD_ALL, &cr_param);
+ _OK(status, "dat_cr_query");
+
+ /* use private data to select EP */
+ pdata = ntoh32(*((int*)cr_param.private_data));
+
+ LOGPRINTF("%s recvd pdata=0x%x, send pdata=0x%x\n",
+ server?"Server":"Client",pdata,
+ *(int*)cr_param.private_data);
+
+ status = dat_cr_accept(cr, ep[pdata], 4, cr_param.private_data);
+ _OK(status, "dat_cr_accept");
+
+ printf("%s accepted CR on EP[%d]=%p\n",
+ server?"Server":"Client",
+ pdata, ep[pdata]);
+}
+
+/* RC - Client and Server: 1, UD - Client: 1 per EP, Server: 2 per EP's */
+void process_conn(int idx)
+{
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN status;
+ int pdata;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)
+ &event.event_extension_data[0];
+ DAT_CONNECTION_EVENT_DATA *conn_event =
+ &event.event_data.connect_event_data;
+
+ LOGPRINTF("%s waiting for connect[%d] establishment\n",
+ server?"Server":"Client",idx);
+
+ status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "CONN dat_evd_wait");
+
+ LOGPRINTF("%s got connect[%d] event, pdata %p sz=%d\n",
+ server?"Server":"Client",idx,
+ conn_event->private_data,
+ conn_event->private_data_size);
+
+ /* Waiting on CR's or CONN_EST */
+ if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED &&
+ (ud_test && event.event_number !=
+ DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) {
+ printf("unexpected event, !conn established: 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ /* RC or PASSIVE CONN_EST we are done */
+ if (!ud_test)
+ return;
+
+ /* store each remote_ah according to remote EP index */
+ pdata = ntoh32(*((int*)conn_event->private_data));
+ LOGPRINTF(" Client got private data=0x%x\n", pdata);
+
+ /* UD, get AH for sends.
+ * NOTE: bi-directional AH resolution results in a CONN_EST
+ * for both outbound connect and inbound CR.
+ * Use Active CONN_EST which includes server's CR
+ * pdata for remote_ah idx to send on and ignore PASSIVE CONN_EST.
+ *
+ * DAT_IB_UD_PASSIVE_REMOTE_AH == passive side CONN_EST
+ * DAT_IB_UD_REMOTE_AH == active side CONN_EST
+ */
+ if (ext_event->type == DAT_IB_UD_REMOTE_AH) {
+ remote_ah[pdata] = ext_event->remote_ah;
+ printf("remote_ah[%d]: ah=%p, qpn=0x%x "
+ "addr=%s\n",
+ pdata, remote_ah[pdata].ah,
+ remote_ah[pdata].qpn,
+ inet_ntoa(((struct sockaddr_in*)
+ &remote_ah[pdata].ia_addr)->sin_addr));
+
+ } else if (ext_event->type != DAT_IB_UD_PASSIVE_REMOTE_AH) {
+ printf("unexpected UD ext_event type: 0x%x\n",
+ ext_event->type);
+ exit(1);
}
}
@@ -235,28 +399,37 @@ connect_ep(char *hostname)
DAT_LMR_TRIPLET iov;
DAT_RMR_TRIPLET *r_iov;
DAT_DTO_COOKIE cookie;
- int i;
- DAT_CR_ARRIVAL_EVENT_DATA *cr_event =
- &event.event_data.cr_arrival_event_data;
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ DAT_CONN_QUAL conn_qual;
+ int i,ii,pdata,ctx;
+ DAT_PROVIDER_ATTR prov_attrs;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
- DAT_IB_EXTENSION_EVENT_DATA *ext_event =
- (DAT_IB_EXTENSION_EVENT_DATA *)
- &event.event_extension_data[0];
-
+
status = dat_ia_open(provider, 8, &async_evd, &ia);
_OK(status, "dat_ia_open");
-
+
+ memset(&prov_attrs, 0, sizeof(prov_attrs));
+ status = dat_ia_query(ia, NULL, 0, NULL,
+ DAT_PROVIDER_FIELD_ALL, &prov_attrs);
+ _OK(status, "dat_ia_query");
+
+ /* Print provider specific attributes */
+ for (i=0;i<prov_attrs.num_provider_specific_attr;i++) {
+ LOGPRINTF(" Provider Specific Attribute[%d] %s=%s\n",
+ i, prov_attrs.provider_specific_attr[i].name,
+ prov_attrs.provider_specific_attr[i].value);
+ }
+
status = dat_pz_create(ia, &pz);
_OK(status, "dat_pz_create");
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
+ status = dat_evd_create(ia, eps*2, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG,
&cr_evd );
_OK(status, "dat_evd_create CR");
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL,
+ status = dat_evd_create(ia, eps*2, DAT_HANDLE_NULL,
DAT_EVD_CONNECTION_FLAG, &con_evd );
_OK(status, "dat_evd_create CR");
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG,
+ status = dat_evd_create(ia, eps*10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG,
&dto_evd );
_OK(status, "dat_evd_create DTO");
@@ -275,8 +448,8 @@ connect_ep(char *hostname)
}
ep_attr.qos = 0;
ep_attr.recv_completion_flags = 0;
- ep_attr.max_recv_dtos = 10;
- ep_attr.max_request_dtos = 10;
+ ep_attr.max_recv_dtos = eps*10;
+ ep_attr.max_request_dtos = eps*10;
ep_attr.max_recv_iov = 1;
ep_attr.max_request_iov = 1;
ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
@@ -285,13 +458,16 @@ connect_ep(char *hostname)
ep_attr.ep_provider_specific_count = 0;
ep_attr.ep_provider_specific = NULL;
- status = dat_ep_create(ia, pz, dto_evd, dto_evd,
- con_evd, &ep_attr, &ep);
- _OK(status, "dat_ep_create");
+ for (i=0;i<eps;i++) {
+ status = dat_ep_create(ia, pz, dto_evd, dto_evd,
+ con_evd, &ep_attr, &ep[i]);
+ _OK(status, "dat_ep_create");
+ LOGPRINTF(" create_ep[%d]=%p\n",i,ep[i]);
+ }
- for (i = 0; i < REG_MEM_COUNT; i++) {
- buf[ i ] = (DAT_RMR_TRIPLET*)malloc(buf_size);
- region.for_va = buf[ i ];
+ for (i=0;i<REG_MEM_COUNT*eps;i++) {
+ buf[i] = (DAT_RMR_TRIPLET*)malloc(buf_size);
+ region.for_va = buf[i];
status = dat_lmr_create(ia,
DAT_MEM_TYPE_VIRTUAL,
region,
@@ -300,11 +476,11 @@ connect_ep(char *hostname)
DAT_MEM_PRIV_ALL_FLAG|
DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
- &lmr[ i ],
- &lmr_context[ i ],
- &rmr_context[ i ],
- ®_size[ i ],
- ®_addr[ i ]);
+ &lmr[i],
+ &lmr_context[i],
+ &rmr_context[i],
+ ®_size[i],
+ ®_addr[i]);
_OK(status, "dat_lmr_create");
}
@@ -326,56 +502,56 @@ connect_ep(char *hostname)
®_atomic_addr);
_OK(status, "dat_lmr_create atomic");
- for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
- cookie.as_64 = i;
- iov.lmr_context = lmr_context[ i ];
- iov.virtual_address = (DAT_VADDR) buf[ i ];
- iov.segment_length = buf_size;
-
- status = dat_ep_post_recv(ep,
- 1,
- &iov,
- cookie,
- DAT_COMPLETION_DEFAULT_FLAG);
- _OK(status, "dat_ep_post_recv");
+ for (ii=0;ii<eps;ii++) {
+ for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
+ int ep_idx = 0;
+ cookie.as_64 = (ii*REG_MEM_COUNT)+i;
+ iov.lmr_context = lmr_context[(ii*REG_MEM_COUNT)+i];
+ iov.virtual_address =
+ (DAT_VADDR)buf[(ii*REG_MEM_COUNT)+i];
+ iov.segment_length = buf_size;
+ LOGPRINTF(" post_recv (%p) on ep[%d]=%p\n",
+ buf[(ii*REG_MEM_COUNT)+i],ii,ep[ii]);
+ /* ep[0], unless testing Server and multi EP's */
+ if (server && multi_eps) {
+ ep_idx = ii;
+ cookie.as_64 = i;
+ }
+ status = dat_ep_post_recv(ep[ep_idx],
+ 1,
+ &iov,
+ cookie,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ _OK(status, "dat_ep_post_recv");
+ }
}
-
/* setup receive buffer to initial string to be overwritten */
strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
- if (server) {
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ],"server written data");
+ /* ud can resolve_ah and connect both ways, same EP */
+ if (server || (!server && ud_test)) {
+ if (server) {
+ conn_qual = SERVER_ID;
+ strcpy((char*)buf[SND_RDMA_BUF_INDEX],"Server data");
+ } else {
+ conn_qual = CLIENT_ID;
+ strcpy((char*)buf[SND_RDMA_BUF_INDEX],"Client data");
+ }
status = dat_psp_create(ia,
- SERVER_CONN_QUAL,
+ conn_qual,
cr_evd,
DAT_PSP_CONSUMER_FLAG,
&psp);
_OK(status, "dat_psp_create");
- printf("Server waiting for connect request\n");
- status = dat_evd_wait(cr_evd, SERVER_TIMEOUT,
- 1, &event, &nmore);
- _OK(status, "listen dat_evd_wait");
-
- if (event.event_number != DAT_CONNECTION_REQUEST_EVENT &&
- (ud_test && event.event_number !=
- DAT_IB_UD_CONNECTION_REQUEST_EVENT)) {
- printf("unexpected event, !conn request: 0x%x\n",
- event.event_number);
- exit(1);
- }
+ /* Server always waits for first CR from Client */
+ if (server)
+ process_cr(0);
- if ((cr_event->conn_qual != SERVER_CONN_QUAL) ||
- (cr_event->sp_handle.psp_handle != psp)) {
- printf("wrong cr event data\n");
- exit(1);
- }
-
- cr = cr_event->cr_handle;
- status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
- printf("Server waiting for accept response\n");
-
- } else {
+ }
+
+ /* ud can resolve_ah and connect both ways */
+ if (!server || (server && ud_test)) {
struct addrinfo *target;
if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
@@ -383,52 +559,58 @@ connect_ep(char *hostname)
exit(1);
}
- printf ("Server Name: %s \n", hostname);
- printf ("Server Net Address: %s\n", inet_ntoa(
- ((struct sockaddr_in *)target->ai_addr)->sin_addr));
+ printf ("Remote %s Name: %s \n",
+ server?"Client":"Server", hostname);
+ printf ("Remote %s Net Address: %s\n",
+ server?"Client":"Server",
+ inet_ntoa(((struct sockaddr_in *)
+ target->ai_addr)->sin_addr));
remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
freeaddrinfo(target);
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ],"client written data");
- status = dat_ep_connect(ep,
- &remote_addr,
- SERVER_CONN_QUAL,
- CONN_TIMEOUT,
- 0,
- (DAT_PVOID)0,
- 0,
- DAT_CONNECT_DEFAULT_FLAG );
- _OK(status, "dat_psp_create");
- printf("Client waiting for connect response\n");
- }
-
- status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
- _OK(status, "connect dat_evd_wait");
-
- if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED &&
- (ud_test && event.event_number !=
- DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) {
- printf("unexpected event, !conn established: 0x%x\n",
- event.event_number);
- exit(1);
+ strcpy((char*)buf[SND_RDMA_BUF_INDEX],"Client written data");
+
+ /* one Client EP, multiple Server EPs, same conn_qual
+ * use private data to select EP on Server
+ */
+ for (i=0;i<eps;i++) {
+ /* pdata selects Server EP,
+ * support both muliple Server and single EP's
+ */
+ if (multi_eps)
+ pdata = hton32(i);
+ else
+ pdata = 0; /* just use first EP */
+
+ status = dat_ep_connect(ep[0],
+ &remote_addr,
+ (server?CLIENT_ID:SERVER_ID),
+ CONN_TIMEOUT,
+ 4,
+ (DAT_PVOID)&pdata,
+ 0,
+ DAT_CONNECT_DEFAULT_FLAG );
+ _OK(status, "dat_ep_connect");
+ }
}
- /* if UD get the AH for the sends */
+ /* UD: process CR's starting with 2nd on server, 1st for client */
if (ud_test) {
- if (ext_event->type == DAT_IB_UD_REMOTE_AH) {
- remote_ah = ext_event->remote_ah;
- printf(" remote_ah: ah=%p, qpn=0x%x addr=%s\n",
- remote_ah.ah, remote_ah.qpn,
- inet_ntoa(((struct sockaddr_in *)
- &remote_ah.ia_addr)->sin_addr));
- } else {
- printf("unexpected UD ext_event type: 0x%x\n",
- ext_event->type);
- exit(1);
- }
+ for (i=(server?1:0);i<eps;i++)
+ process_cr(i);
}
- printf("Connected!\n");
+ /* RC and UD: process CONN EST events */
+ for (i=0;i<eps;i++)
+ process_conn(i);
+
+ /* UD: CONN EST events for CONN's and CR's */
+ if(ud_test) {
+ for (i=0;i<eps;i++)
+ process_conn(i);
+ }
+
+ printf("Connected! %d endpoints\n", eps);
/*
* Setup our remote memory and tell the other side about it
@@ -439,9 +621,10 @@ connect_ep(char *hostname)
r_iov->virtual_address = hton64((DAT_VADDR)buf[RCV_RDMA_BUF_INDEX]);
r_iov->segment_length = hton32(buf_size);
- printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va="F64x",len=0x%x\n",
- getpid(), hton32(r_iov->rmr_context),
- hton64(r_iov->virtual_address), hton32(r_iov->segment_length));
+ printf("Send RMR message: r_key_ctx=0x%x,va="F64x",len=0x%x\n",
+ hton32(r_iov->rmr_context),
+ hton64(r_iov->virtual_address),
+ hton32(r_iov->segment_length));
send_msg(buf[SEND_BUF_INDEX],
sizeof(DAT_RMR_TRIPLET),
@@ -452,44 +635,69 @@ connect_ep(char *hostname)
/*
* Wait for their RMR
*/
- printf("Waiting for remote to send RMR data\n");
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ep_post_send");
+ for (i=0,ctx=0;i<eps;i++,ctx++) {
+ /* expected cookie, recv buf idx in every mem pool */
+ ctx = (ctx%REG_MEM_COUNT)?ctx:ctx+RECV_BUF_INDEX;
+ LOGPRINTF("Waiting for remote to send RMR data\n");
- if ((event.event_number != DAT_DTO_COMPLETION_EVENT) &&
- (ud_test && event.event_number != DAT_IB_DTO_EVENT)) {
- printf("unexpected event waiting for RMR context "
- "- 0x%x\n", event.event_number);
- exit(1);
- }
- _OK(dto_event->status, "event status for post_recv");
-
- if (dto_event->transfered_length != msg_size ||
- dto_event->user_cookie.as_64 != RECV_BUF_INDEX) {
- printf("unexpected event data for receive: len=%d cookie=%d "
- "expected %d/%d\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64,
- msg_size, RECV_BUF_INDEX);
- exit(1);
- }
-
- /* swap RMR,address info to host order */
- if (ud_test)
- r_iov = (DAT_RMR_TRIPLET*)((char*)buf[RECV_BUF_INDEX]+40);
- else
- r_iov = (DAT_RMR_TRIPLET*)buf[RECV_BUF_INDEX];
-
- r_iov->rmr_context = ntoh32(r_iov->rmr_context);
- r_iov->virtual_address = ntoh64(r_iov->virtual_address);
- r_iov->segment_length = ntoh32(r_iov->segment_length);
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
- printf("%d Received RMR from remote: "
- "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",
- getpid(), r_iov->rmr_context,
- r_iov->virtual_address,
- r_iov->segment_length);
+ if ((event.event_number != DAT_DTO_COMPLETION_EVENT) &&
+ (ud_test && event.event_number != DAT_IB_DTO_EVENT)) {
+ printf("unexpected event waiting for RMR context "
+ "- 0x%x\n", event.event_number);
+ exit(1);
+ }
+ _OK(dto_event->status, "event status for post_recv");
+
+ /* careful when checking cookies:
+ * Client - receiving multi messages on a single EP
+ * Server - not receiving on multiple EP's
+ */
+ if (!server || (server && !multi_eps)) {
+ if (dto_event->transfered_length != msg_size ||
+ dto_event->user_cookie.as_64 != ctx) {
+ printf("unexpected event data on recv: len=%d"
+ " cookie="F64x" expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ dto_event->user_cookie.as_64,
+ msg_size, ctx);
+ exit(1);
+ }
+ /* Server - receiving one message each across many EP's */
+ } else {
+ if (dto_event->transfered_length != msg_size ||
+ dto_event->user_cookie.as_64 != RECV_BUF_INDEX) {
+ printf("unexpected event data on recv: len=%d"
+ "cookie="F64x" expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ dto_event->user_cookie.as_64,
+ msg_size, RECV_BUF_INDEX);
+ exit(1);
+ }
+ }
+ /* swap RMR,address info to host order */
+ if (!server || (server && !multi_eps))
+ r_iov = (DAT_RMR_TRIPLET*)buf[ctx];
+ else
+ r_iov = (DAT_RMR_TRIPLET*)buf[(i*REG_MEM_COUNT)+RECV_BUF_INDEX];
+
+ if (ud_test)
+ r_iov = (DAT_RMR_TRIPLET*)((char*)r_iov + 40);
+
+ r_iov->rmr_context = ntoh32(r_iov->rmr_context);
+ r_iov->virtual_address = ntoh64(r_iov->virtual_address);
+ r_iov->segment_length = ntoh32(r_iov->segment_length);
+
+ printf("Recv RMR message: r_iov(%p):"
+ " r_key_ctx=%x,va="F64x",len=0x%x on EP=%p\n",
+ r_iov, r_iov->rmr_context,
+ r_iov->virtual_address,
+ r_iov->segment_length,
+ dto_event->ep_handle);
+ }
return(0);
}
@@ -497,42 +705,43 @@ int
disconnect_ep(void)
{
DAT_RETURN status;
- int i;
DAT_EVENT event;
DAT_COUNT nmore;
+ int i;
- status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
- _OK2(status, "dat_ep_disconnect");
-
- status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1,
- &event, &nmore);
- _OK(status, "dat_evd_wait");
-
- if (server) {
+ if (!ud_test) {
+ status = dat_ep_disconnect(ep[0], DAT_CLOSE_DEFAULT);
+ _OK2(status, "dat_ep_disconnect");
+
+ status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1,
+ &event, &nmore);
+ _OK(status, "dat_evd_wait");
+ }
+ if (psp) {
status = dat_psp_free(psp);
_OK2(status, "dat_psp_free");
}
-
- for (i = 0; i < REG_MEM_COUNT; i++) {
+ for (i = 0; i < REG_MEM_COUNT*eps; i++) {
status = dat_lmr_free(lmr[ i ]);
_OK2(status, "dat_lmr_free");
}
-
if (lmr_atomic) {
status = dat_lmr_free(lmr_atomic);
_OK2(status, "dat_lmr_free_atomic");
}
-
- status = dat_ep_free(ep);
- _OK2(status, "dat_ep_free");
-
+ for (i=0;i<eps;i++) {
+ status = dat_ep_free(ep[i]);
+ _OK2(status, "dat_ep_free");
+ }
status = dat_evd_free(dto_evd);
_OK2(status, "dat_evd_free DTO");
+
status = dat_evd_free(con_evd);
_OK2(status, "dat_evd_free CON");
+
status = dat_evd_free(cr_evd);
_OK2(status, "dat_evd_free CR");
-
+
status = dat_pz_free(pz);
_OK2(status, "dat_pz_free");
@@ -577,7 +786,7 @@ do_immediate()
cookie.as_64 = 0x9999;
- status = dat_ib_post_rdma_write_immed(ep, // ep_handle
+ status = dat_ib_post_rdma_write_immed(ep[0], // ep_handle
1, // segments
&iov, // LMR
cookie, // user_cookie
@@ -718,7 +927,7 @@ do_cmp_swap()
DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov;
DAT_RMR_TRIPLET r_iov;
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
DAT_IB_EXTENSION_EVENT_DATA *ext_event =
@@ -738,10 +947,10 @@ do_cmp_swap()
if (server) {
*target = 0x12345;
sleep(1);
- /* server does not compare and should not swap */
+ /* Server does not compare and should not swap */
printf("dtx svr - starting cmp_swap\n");
status = dat_ib_post_cmp_and_swap(
- ep,
+ ep[0],
(DAT_UINT64)0x654321,
(DAT_UINT64)0x6789A,
&l_iov,
@@ -753,9 +962,9 @@ do_cmp_swap()
*target = 0x54321;
sleep(1);
printf("dtx cli - starting cmp_swap\n");
- /* client does compare and should swap */
+ /* Client does compare and should swap */
status = dat_ib_post_cmp_and_swap(
- ep,
+ ep[0],
(DAT_UINT64)0x12345,
(DAT_UINT64)0x98765,
&l_iov,
@@ -788,7 +997,7 @@ do_cmp_swap()
if (server) {
printf("Server got original data = "F64x", expected "
"0x54321\n", *atomic_buf);
- printf("Client final result (on server) = "F64x", expected "
+ printf("Client final result (on Server) = "F64x", expected "
"0x98765\n", *target);
if (*atomic_buf != 0x54321 || *target != 0x98765) {
@@ -798,7 +1007,7 @@ do_cmp_swap()
} else {
printf("Client got original data = "F64x", expected "
"0x12345\n",*atomic_buf);
- printf("Server final result (on client) = 0x"F64x", expected "
+ printf("Server final result (on Client) = 0x"F64x", expected "
"0x54321\n", *target);
if (*atomic_buf != 0x12345 || *target != 0x54321) {
@@ -819,7 +1028,7 @@ do_fetch_add()
DAT_COUNT nmore;
DAT_LMR_TRIPLET l_iov;
DAT_RMR_TRIPLET r_iov;
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];
DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
&event.event_data.dto_completion_event_data;
DAT_IB_EXTENSION_EVENT_DATA *ext_event =
@@ -836,24 +1045,24 @@ do_fetch_add()
cookie.as_64 = 0x7777;
if (server) {
- /* Wait for client to finish cmp_swap */
+ /* Wait for Client to finish cmp_swap */
while (*target != 0x98765)
sleep(1);
*target = 0x10;
sleep(1);
status = dat_ib_post_fetch_and_add(
- ep,
+ ep[0],
(DAT_UINT64)0x100,
&l_iov,
cookie,
&r_iov,
DAT_COMPLETION_DEFAULT_FLAG);
} else {
- /* Wait for server, no swap so nothing to check */
+ /* Wait for Server, no swap so nothing to check */
*target = 0x100;
sleep(1);
status = dat_ib_post_fetch_and_add(
- ep,
+ ep[0],
(DAT_UINT64)0x10,
&l_iov,
cookie,
@@ -880,10 +1089,10 @@ do_fetch_add()
}
if (server) {
- printf("Client original data (on server) = "F64x", expected "
+ printf("Client original data (on Server) = "F64x", expected "
"0x100\n", *atomic_buf);
} else {
- printf("Server original data (on client) = "F64x", expected "
+ printf("Server original data (on Client) = "F64x", expected "
"0x10\n", *atomic_buf);
}
@@ -891,7 +1100,7 @@ do_fetch_add()
if (server) {
status = dat_ib_post_fetch_and_add(
- ep,
+ ep[0],
(DAT_UINT64)0x100,
&l_iov,
cookie,
@@ -899,7 +1108,7 @@ do_fetch_add()
DAT_COMPLETION_DEFAULT_FLAG);
} else {
status = dat_ib_post_fetch_and_add(
- ep,
+ ep[0],
(DAT_UINT64)0x10,
&l_iov,
cookie,
@@ -930,7 +1139,7 @@ do_fetch_add()
if (server) {
printf("Server got original data = "F64x", expected "
"0x200\n", *atomic_buf);
- printf("Client final result (on server) = "F64x", expected "
+ printf("Client final result (on Server) = "F64x", expected "
"0x30\n", *target);
if (*atomic_buf != 0x200 || *target != 0x30) {
@@ -940,7 +1149,7 @@ do_fetch_add()
} else {
printf("Server side original data = "F64x", expected "
"0x20\n", *atomic_buf);
- printf("Server final result (on client) = "F64x", expected "
+ printf("Server final result (on Client) = "F64x", expected "
"0x300\n", *target);
if (*atomic_buf != 0x20 || *target != 0x300) {
@@ -959,13 +1168,22 @@ main(int argc, char **argv)
int rc;
/* parse arguments */
- while ((rc = getopt(argc, argv, "uh:b:P:")) != -1)
+ while ((rc = getopt(argc, argv, "csvumU:h:b:P:")) != -1)
{
switch(rc)
{
case 'u':
ud_test = 1;
- fflush(stdout);
+ eps = MAX_EP_COUNT/2;
+ break;
+ case 'm':
+ multi_eps = 1;
+ break;
+ case 'c':
+ server = 0;
+ break;
+ case 's':
+ server = 1;
break;
case 'h':
server = 0;
@@ -974,9 +1192,16 @@ main(int argc, char **argv)
case 'b':
buf_size = atoi(optarg);
break;
+ case 'U':
+ ud_test = 1;
+ eps = MIN(atoi(optarg),MAX_EP_COUNT);
+ break;
case 'P':
strcpy (provider, optarg);
break;
+ case 'v':
+ verbose = 1;
+ break;
default:
print_usage();
exit(-12);
@@ -998,6 +1223,13 @@ main(int argc, char **argv)
}
#endif
+ if (!server) {
+ printf("\nRunning as Client - %s %s %d endpoint(s)\n",
+ provider, ud_test?"UD test":"", eps);
+ } else {
+ printf("\nRunning as Server - %s %s %d endpoint(s)\n",
+ provider, ud_test?"UD test":"", eps);
+ }
/*
* connect
*/
--
1.5.2.5
More information about the general
mailing list