[ofa-general] [PATCH] uDAPL 2.0 mods to co-exist with uDAPL 1.2
Arlin Davis
arlin.r.davis at intel.com
Thu Sep 20 11:50:56 PDT 2007
James,
Please review patches to allow coexistence of 2.0 and 1.2 libraries. I updated the dat.conf to
provide configuration to both 1.2 and 2.0 providers. In addition, the development package (headers)
is not targeted to include/dat2 instead of include/dat. A patch for 1.2 will follow shortly.
Modifications to DAT 2.0 package to coexist with 1.2 libraries
- cleanup CR-LF in dtestx
- fix RPM specfile, 2.0.1 package
- move devel to include/dat2
- change test examples to use new 2.0 provider names.
Signed-off by: Arlin Davis ardavis at ichips.intel.com
diff --git a/Makefile.am b/Makefile.am
index b3a0149..f473aaa 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -66,7 +66,7 @@ dat_udat_libdat_la_SOURCES = dat/udat/udat.c \
dat/common/dat_init.c \
dat/common/dat_dr.c \
dat/common/dat_sr.c
-
+# version-info current:revision:age
dat_udat_libdat_la_LDFLAGS = -version-info 2:0:0 $(dat_version_script) -ldl
#
@@ -178,11 +178,12 @@ dapl_udapl_libdaplcma_la_SOURCES = dapl/udapl/dapl_init.c \
dapl/openib_cma/dapl_ib_cm.c \
dapl/openib_cma/dapl_ib_mem.c $(XPROGRAMS)
+# version-info current:revision:age
dapl_udapl_libdaplcma_la_LDFLAGS = -version-info 2:0:0 $(daplcma_version_script) \
-Wl,-init,dapl_init -Wl,-fini,dapl_fini \
-lpthread -libverbs -lrdmacm
-libdatincludedir = $(includedir)/dat
+libdatincludedir = $(includedir)/dat2
libdatinclude_HEADERS = dat/include/dat/dat.h \
dat/include/dat/dat_error.h \
@@ -244,7 +245,7 @@ EXTRA_DIST = dat/common/dat_dictionary.h \
dat/udat/libdat.map \
doc/dat.conf \
dapl/udapl/libdaplcma.map \
- libdat.spec.in \
+ libdat2.spec.in \
$(man_MANS) \
test/dapltest/include/dapl_bpool.h \
test/dapltest/include/dapl_client_info.h \
@@ -274,7 +275,7 @@ EXTRA_DIST = dat/common/dat_dictionary.h \
test/dapltest/include/dapl_version.h \
test/dapltest/mdep/linux/dapl_mdep_user.h
-dist-hook: libdat.spec
- cp libdat.spec $(distdir)
+dist-hook: libdat2.spec
+ cp libdat2.spec $(distdir)
SUBDIRS = . test/dtest test/dapltest
diff --git a/README b/README
index 437c1f7..1fc55a2 100644
--- a/README
+++ b/README
@@ -17,16 +17,18 @@ Building debug version:
./configure --enable-debug
make
-Build example with OFED prefix (x86_64)
------------------------------------------
+Build example with OFED 1.2+ prefix (x86_64)
+---------------------------------------------
./autogen.sh
-./configure --prefix /usr/local/ofed --libdir /usr/local/ofed/lib64 LDFLAGS=-L/usr/local/ofed/lib64
CPPFLAGS="-I/usr/local/ofed/include"
+./configure --prefix /usr --sysconf=/etc --libdir /usr/lib64 LDFLAGS=-L/usr/lib64
CPPFLAGS="-I/usr/include"
make
Installing:
----------
make install
+Note: The development package installs DAT 2.0 include files under /usr/include/dat2 to co-exist
with DAT 1.2 /usr/include/dat
+
NOTE: to link these libraries you must either use libtool and
specify the full pathname of the library, or use the `-LLIBDIR'
flag during linking and do at least one of the following:
@@ -47,19 +49,32 @@ more information, such as the ld(1) and ld.so(8) manual pages.
sample /etc/dat.conf
#
-# DAT 1.2 configuration file, sample OFED
+# DAT 1.2 and 2.0 configuration file
#
# Each entry should have the following fields:
#
# <ia_name> <api_version> <threadsafety> <default> <lib_path> \
# <provider_version> <ia_params> <platform_params>
#
-# For openib-cma provider you can specify <ia_params> as either:
-# network address, network hostname, or netdev name and 0 for port
+# For the uDAPL cma provder, specify <ia_params> as one of the following:
+# network address, network hostname, or netdev name and 0 for port
+#
+# Simple (OpenIB-cma) default with netdev name provided first on list
+# to enable use of same dat.conf version on all nodes
#
-# This example shows netdev name, enabling administrator to use same copy across cluster
+# Add examples for multiple interfaces and IPoIB HA fail over, and bonding
#
-OpenIB-cma u1.2 nonthreadsafe default /usr/local/ofed/lib64/libdapl-cma.so mv_dapl.1.2 "ib0 0" ""
+OpenIB-cma u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib0 0" ""
+OpenIB-cma-1 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib1 0" ""
+OpenIB-cma-2 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib2 0" ""
+OpenIB-cma-3 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib3 0" ""
+OpenIB-bond u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "bond0 0" ""
+OpenIB-2-cma u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib0 0" ""
+OpenIB-2-cma-1 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib1 0" ""
+OpenIB-2-cma-2 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib2 0" ""
+OpenIB-2-cma-3 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib3 0" ""
+OpenIB-2-bond u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "bond0 0" ""
+
=============================
3.0 Bugs/Known issues
diff --git a/configure.in b/configure.in
index 7608e64..4eda85f 100644
--- a/configure.in
+++ b/configure.in
@@ -1,11 +1,11 @@
dnl Process this file with autoconf to produce a configure script.
AC_PREREQ(2.57)
-AC_INIT(dapl, 2.0.0, openib-general at openib.org)
+AC_INIT(dapl, 2.0.1, general at lists.openfabrics.org)
AC_CONFIG_SRCDIR([dat/udat/udat.c])
AC_CONFIG_AUX_DIR(config)
AM_CONFIG_HEADER(config.h)
-AM_INIT_AUTOMAKE(dapl, 2.0.0)
+AM_INIT_AUTOMAKE(dapl, 2.0.1)
AM_PROG_LIBTOOL
@@ -86,6 +86,6 @@ AC_CACHE_CHECK(Check for RHEL5 system, ac_cv_rhel5,
fi)
AM_CONDITIONAL(OS_RHEL5, test "$ac_cv_rhel5" = "yes")
-AC_CONFIG_FILES([Makefile test/dtest/Makefile test/dapltest/Makefile libdat.spec])
+AC_CONFIG_FILES([Makefile test/dtest/Makefile test/dapltest/Makefile libdat2.spec])
AC_OUTPUT
diff --git a/doc/dat.conf b/doc/dat.conf
index 2651673..005f9ee 100755
--- a/doc/dat.conf
+++ b/doc/dat.conf
@@ -1,5 +1,5 @@
#
-# DAT 2.0 configuration file
+# DAT 1.2 and 2.0 configuration file
#
# Each entry should have the following fields:
#
@@ -9,10 +9,18 @@
# For the uDAPL cma provder, specify <ia_params> as one of the following:
# network address, network hostname, or netdev name and 0 for port
#
-# Simple (OpenIB-cma) default configuration with netdev name provided first on list
-# to enable use of same dat.conf version on all nodes. Assumes x86_64 installation.
+# Simple (OpenIB-cma) default with netdev name provided first on list
+# to enable use of same dat.conf version on all nodes
#
-OpenIB-cma u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
-OpenIB-cma-1 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
-OpenIB-cma-2 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
-OpenIB-cma-3 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
+# Add examples for multiple interfaces and IPoIB HA fail over, and bonding
+#
+OpenIB-cma u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib0 0" ""
+OpenIB-cma-1 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib1 0" ""
+OpenIB-cma-2 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib2 0" ""
+OpenIB-cma-3 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib3 0" ""
+OpenIB-bond u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "bond0 0" ""
+OpenIB-2-cma u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib0 0" ""
+OpenIB-2-cma-1 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib1 0" ""
+OpenIB-2-cma-2 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib2 0" ""
+OpenIB-2-cma-3 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib3 0" ""
+OpenIB-2-bond u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "bond0 0" ""
diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
index 07b40ec..ba12a58 100644
--- a/test/dtest/dtest.c
+++ b/test/dtest/dtest.c
@@ -44,7 +44,7 @@
#include <inttypes.h>
#ifndef DAPL_PROVIDER
-#define DAPL_PROVIDER "OpenIB-cma"
+#define DAPL_PROVIDER "OpenIB-2-cma"
#endif
#define MAX_POLLING_CNT 50000
diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c
index 153ce76..04a0d5d 100755
--- a/test/dtest/dtestx.c
+++ b/test/dtest/dtestx.c
@@ -30,785 +30,785 @@
* SOFTWARE.
*
* $Id: $
- */
-#include <stdio.h>
-#include <stdlib.h>
-#include <netdb.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <endian.h>
-#include <byteswap.h>
-
-#include "dat/udat.h"
-#include "dat/dat_ib_extensions.h"
-
-#define _OK(status, str) \
-{ \
- const char *maj_msg, *min_msg; \
- if (status != DAT_SUCCESS) { \
- dat_strerror(status, &maj_msg, &min_msg); \
- fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \
- exit(1); \
- } \
-}
-
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include "dat/udat.h"
+#include "dat/dat_ib_extensions.h"
+
+#define _OK(status, str) \
+{ \
+ const char *maj_msg, *min_msg; \
+ if (status != DAT_SUCCESS) { \
+ dat_strerror(status, &maj_msg, &min_msg); \
+ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \
+ exit(1); \
+ } \
+}
+
#define DTO_TIMEOUT (1000*1000*5)
#define CONN_TIMEOUT (1000*1000*10)
-#define SERVER_TIMEOUT (1000*1000*120)
-#define SERVER_CONN_QUAL 31111
-#define BUF_SIZE 256
-#define BUF_SIZE_ATOMIC 8
-#define REG_MEM_COUNT 10
-#define SND_RDMA_BUF_INDEX 0
-#define RCV_RDMA_BUF_INDEX 1
-#define SEND_BUF_INDEX 2
-#define RECV_BUF_INDEX 3
-
-u_int64_t *atomic_buf;
-DAT_LMR_HANDLE lmr_atomic;
-DAT_LMR_CONTEXT lmr_atomic_context;
-DAT_RMR_CONTEXT rmr_atomic_context;
-DAT_VLEN reg_atomic_size;
-DAT_VADDR reg_atomic_addr;
-DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
-DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
-DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
-DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
-DAT_VLEN reg_size[ REG_MEM_COUNT ];
-DAT_VADDR reg_addr[ REG_MEM_COUNT ];
-DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
-DAT_EP_HANDLE ep;
-DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
-DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
-DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
-DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
-DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
-DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
-DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
-DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
-int server;
-
-char *usage = "-s | hostname (default == -s)\n";
-
-void
-send_msg(
- void *data,
- DAT_COUNT size,
- DAT_LMR_CONTEXT context,
- DAT_DTO_COOKIE cookie,
- DAT_COMPLETION_FLAGS flags)
-{
- DAT_LMR_TRIPLET iov;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_RETURN status;
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
- &event.event_data.dto_completion_event_data;
-
- iov.lmr_context = context;
- iov.virtual_address = (DAT_VADDR)(unsigned long)data;
- iov.segment_length = (DAT_VLEN)size;
-
- status = dat_ep_post_send(ep,
- 1,
- &iov,
- cookie,
- flags);
- _OK(status, "dat_ep_post_send");
-
- if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ep_post_send");
-
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- printf("unexpected event waiting for post_send completion - 0x%x\n",
event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status for post_send");
- }
-}
-
-int
-connect_ep(char *hostname)
-{
- DAT_SOCK_ADDR remote_addr;
- DAT_EP_ATTR ep_attr;
- DAT_RETURN status;
- DAT_REGION_DESCRIPTION region;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- int i;
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
- &event.event_data.dto_completion_event_data;
-
- status = dat_ia_open("OpenIB-cma", 8, &async_evd, &ia);
- _OK(status, "dat_ia_open");
-
- status = dat_pz_create(ia, &pz);
- _OK(status, "dat_pz_create");
-
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );
- _OK(status, "dat_evd_create CR");
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );
- _OK(status, "dat_evd_create CR");
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );
- _OK(status, "dat_evd_create DTO");
-
- memset(&ep_attr, 0, sizeof(ep_attr));
- ep_attr.service_type = DAT_SERVICE_TYPE_RC;
- ep_attr.max_rdma_size = 0x10000;
- ep_attr.qos = 0;
- ep_attr.recv_completion_flags = 0;
- ep_attr.max_recv_dtos = 10;
- ep_attr.max_request_dtos = 10;
- ep_attr.max_recv_iov = 1;
- ep_attr.max_request_iov = 1;
- ep_attr.max_rdma_read_in = 4;
- ep_attr.max_rdma_read_out = 4;
- ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
- ep_attr.ep_transport_specific_count = 0;
- ep_attr.ep_transport_specific = NULL;
- ep_attr.ep_provider_specific_count = 0;
- ep_attr.ep_provider_specific = NULL;
-
- status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);
- _OK(status, "dat_ep_create");
-
- for (i = 0; i < REG_MEM_COUNT; i++) {
- buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);
- region.for_va = buf[ i ];
- status = dat_lmr_create(ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- BUF_SIZE,
- pz,
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
+#define SERVER_TIMEOUT (1000*1000*120)
+#define SERVER_CONN_QUAL 31111
+#define BUF_SIZE 256
+#define BUF_SIZE_ATOMIC 8
+#define REG_MEM_COUNT 10
+#define SND_RDMA_BUF_INDEX 0
+#define RCV_RDMA_BUF_INDEX 1
+#define SEND_BUF_INDEX 2
+#define RECV_BUF_INDEX 3
+
+u_int64_t *atomic_buf;
+DAT_LMR_HANDLE lmr_atomic;
+DAT_LMR_CONTEXT lmr_atomic_context;
+DAT_RMR_CONTEXT rmr_atomic_context;
+DAT_VLEN reg_atomic_size;
+DAT_VADDR reg_atomic_addr;
+DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
+DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
+DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
+DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
+DAT_VLEN reg_size[ REG_MEM_COUNT ];
+DAT_VADDR reg_addr[ REG_MEM_COUNT ];
+DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
+DAT_EP_HANDLE ep;
+DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
+DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
+DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
+DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
+DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
+int server;
+
+char *usage = "-s | hostname (default == -s)\n";
+
+void
+send_msg(
+ void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie,
+ DAT_COMPLETION_FLAGS flags)
+{
+ DAT_LMR_TRIPLET iov;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN status;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+
+ iov.lmr_context = context;
+ iov.virtual_address = (DAT_VADDR)(unsigned long)data;
+ iov.segment_length = (DAT_VLEN)size;
+
+ status = dat_ep_post_send(ep,
+ 1,
+ &iov,
+ cookie,
+ flags);
+ _OK(status, "dat_ep_post_send");
+
+ if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ printf("unexpected event waiting for post_send completion - 0x%x\n",
event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for post_send");
+ }
+}
+
+int
+connect_ep(char *hostname)
+{
+ DAT_SOCK_ADDR remote_addr;
+ DAT_EP_ATTR ep_attr;
+ DAT_RETURN status;
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ int i;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+
+ status = dat_ia_open("OpenIB-2-cma", 8, &async_evd, &ia);
+ _OK(status, "dat_ia_open");
+
+ status = dat_pz_create(ia, &pz);
+ _OK(status, "dat_pz_create");
+
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );
+ _OK(status, "dat_evd_create CR");
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );
+ _OK(status, "dat_evd_create CR");
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );
+ _OK(status, "dat_evd_create DTO");
+
+ memset(&ep_attr, 0, sizeof(ep_attr));
+ ep_attr.service_type = DAT_SERVICE_TYPE_RC;
+ ep_attr.max_rdma_size = 0x10000;
+ ep_attr.qos = 0;
+ ep_attr.recv_completion_flags = 0;
+ ep_attr.max_recv_dtos = 10;
+ ep_attr.max_request_dtos = 10;
+ ep_attr.max_recv_iov = 1;
+ ep_attr.max_request_iov = 1;
+ ep_attr.max_rdma_read_in = 4;
+ ep_attr.max_rdma_read_out = 4;
+ ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
+ ep_attr.ep_transport_specific_count = 0;
+ ep_attr.ep_transport_specific = NULL;
+ ep_attr.ep_provider_specific_count = 0;
+ ep_attr.ep_provider_specific = NULL;
+
+ status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);
+ _OK(status, "dat_ep_create");
+
+ for (i = 0; i < REG_MEM_COUNT; i++) {
+ buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);
+ region.for_va = buf[ i ];
+ status = dat_lmr_create(ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ BUF_SIZE,
+ pz,
+ DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
- &lmr[ i ],
- &lmr_context[ i ],
- &rmr_context[ i ],
- ®_size[ i ],
- ®_addr[ i ]);
- _OK(status, "dat_lmr_create");
- }
-
- /* register atomic return buffer for original data */
- atomic_buf = (u_int64_t*)malloc(BUF_SIZE);
- region.for_va = atomic_buf;
- status = dat_lmr_create(ia,
- DAT_MEM_TYPE_VIRTUAL,
- region,
- BUF_SIZE_ATOMIC,
- pz,
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
+ &lmr[ i ],
+ &lmr_context[ i ],
+ &rmr_context[ i ],
+ ®_size[ i ],
+ ®_addr[ i ]);
+ _OK(status, "dat_lmr_create");
+ }
+
+ /* register atomic return buffer for original data */
+ atomic_buf = (u_int64_t*)malloc(BUF_SIZE);
+ region.for_va = atomic_buf;
+ status = dat_lmr_create(ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ BUF_SIZE_ATOMIC,
+ pz,
+ DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
- &lmr_atomic,
- &lmr_atomic_context,
- &rmr_atomic_context,
- ®_atomic_size,
- ®_atomic_addr);
- _OK(status, "dat_lmr_create atomic");
-
- for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
- cookie.as_64 = i;
- iov.lmr_context = lmr_context[ i ];
- iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];
- iov.segment_length = BUF_SIZE;
-
- status = dat_ep_post_recv(ep,
- 1,
- &iov,
- cookie,
- DAT_COMPLETION_DEFAULT_FLAG);
- _OK(status, "dat_ep_post_recv");
- }
-
- /* setup receive buffer to initial string to be overwritten */
- strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
-
- if (server) {
-
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");
-
- status = dat_psp_create(ia,
- SERVER_CONN_QUAL,
- cr_evd,
- DAT_PSP_CONSUMER_FLAG,
- &psp);
- _OK(status, "dat_psp_create");
-
- printf("Server waiting for connect request\n");
- status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
- _OK(status, "listen dat_evd_wait");
-
- if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
- printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number);
- exit(1);
- }
-
- if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
- (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {
-
- printf("wrong cr event data\n");
- exit(1);
- }
-
- cr = event.event_data.cr_arrival_event_data.cr_handle;
- status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
-
- } else {
- struct addrinfo *target;
- int rval;
-
- if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
- printf("Error getting remote address.\n");
- exit(1);
- }
-
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
- printf ("Server Name: %s \n", hostname);
- printf ("Server Net Address: %d.%d.%d.%d\n",
- (rval >> 0) & 0xff,
- (rval >> 8) & 0xff,
- (rval >> 16) & 0xff,
- (rval >> 24) & 0xff);
-
- remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
-
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");
-
- status = dat_ep_connect(ep,
- &remote_addr,
- SERVER_CONN_QUAL,
- CONN_TIMEOUT,
- 0,
- (DAT_PVOID)0,
- 0,
- DAT_CONNECT_DEFAULT_FLAG );
- _OK(status, "dat_psp_create");
- }
-
- printf("Client waiting for connect response\n");
- status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
- _OK(status, "connect dat_evd_wait");
-
- if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
- printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number);
- exit(1);
- }
-
- printf("Connected!\n");
-
- /*
- * Setup our remote memory and tell the other side about it
- */
- printf("Sending RMR data to remote\n");
- r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];
- r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);
- r_iov.segment_length = BUF_SIZE;
-
- *buf[ SEND_BUF_INDEX ] = r_iov;
-
- send_msg( buf[ SEND_BUF_INDEX ],
- sizeof(DAT_RMR_TRIPLET),
- lmr_context[ SEND_BUF_INDEX ],
- cookie,
- DAT_COMPLETION_SUPPRESS_FLAG);
-
- /*
- * Wait for their RMR
- */
- printf("Waiting for remote to send RMR data\n");
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ep_post_send");
-
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
- printf("unexpected event waiting for RMR context - 0x%x\n",
- event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status for post_send");
- if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {
- printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64,
- sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);
- exit(1);
- }
-
- r_iov = *buf[ RECV_BUF_INDEX ];
-
- printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",
- r_iov.rmr_context,
- (void*)(unsigned long)r_iov.virtual_address,
- r_iov.segment_length);
-
- return(0);
-}
-
-int
-disconnect_ep()
-{
- DAT_RETURN status;
- int i;
- DAT_EVENT event;
- DAT_COUNT nmore;
-
- status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
- _OK(status, "dat_ep_disconnect");
-
+ &lmr_atomic,
+ &lmr_atomic_context,
+ &rmr_atomic_context,
+ ®_atomic_size,
+ ®_atomic_addr);
+ _OK(status, "dat_lmr_create atomic");
+
+ for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
+ cookie.as_64 = i;
+ iov.lmr_context = lmr_context[ i ];
+ iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];
+ iov.segment_length = BUF_SIZE;
+
+ status = dat_ep_post_recv(ep,
+ 1,
+ &iov,
+ cookie,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ _OK(status, "dat_ep_post_recv");
+ }
+
+ /* setup receive buffer to initial string to be overwritten */
+ strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
+
+ if (server) {
+
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");
+
+ status = dat_psp_create(ia,
+ SERVER_CONN_QUAL,
+ cr_evd,
+ DAT_PSP_CONSUMER_FLAG,
+ &psp);
+ _OK(status, "dat_psp_create");
+
+ printf("Server waiting for connect request\n");
+ status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "listen dat_evd_wait");
+
+ if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
+ printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
+ (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {
+
+ printf("wrong cr event data\n");
+ exit(1);
+ }
+
+ cr = event.event_data.cr_arrival_event_data.cr_handle;
+ status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
+
+ } else {
+ struct addrinfo *target;
+ int rval;
+
+ if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
+ printf("Error getting remote address.\n");
+ exit(1);
+ }
+
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ printf ("Server Name: %s \n", hostname);
+ printf ("Server Net Address: %d.%d.%d.%d\n",
+ (rval >> 0) & 0xff,
+ (rval >> 8) & 0xff,
+ (rval >> 16) & 0xff,
+ (rval >> 24) & 0xff);
+
+ remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
+
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");
+
+ status = dat_ep_connect(ep,
+ &remote_addr,
+ SERVER_CONN_QUAL,
+ CONN_TIMEOUT,
+ 0,
+ (DAT_PVOID)0,
+ 0,
+ DAT_CONNECT_DEFAULT_FLAG );
+ _OK(status, "dat_psp_create");
+ }
+
+ printf("Client waiting for connect response\n");
+ status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "connect dat_evd_wait");
+
+ if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
+ printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ printf("Connected!\n");
+
+ /*
+ * Setup our remote memory and tell the other side about it
+ */
+ printf("Sending RMR data to remote\n");
+ r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];
+ r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);
+ r_iov.segment_length = BUF_SIZE;
+
+ *buf[ SEND_BUF_INDEX ] = r_iov;
+
+ send_msg( buf[ SEND_BUF_INDEX ],
+ sizeof(DAT_RMR_TRIPLET),
+ lmr_context[ SEND_BUF_INDEX ],
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG);
+
+ /*
+ * Wait for their RMR
+ */
+ printf("Waiting for remote to send RMR data\n");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ printf("unexpected event waiting for RMR context - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for post_send");
+ if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {
+ printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);
+ exit(1);
+ }
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",
+ r_iov.rmr_context,
+ (void*)(unsigned long)r_iov.virtual_address,
+ r_iov.segment_length);
+
+ return(0);
+}
+
+int
+disconnect_ep()
+{
+ DAT_RETURN status;
+ int i;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+
+ status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
+ _OK(status, "dat_ep_disconnect");
+
status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
_OK(status, "dat_ep_disconnect");
-
- if (server) {
- status = dat_psp_free(psp);
- _OK(status, "dat_psp_free");
- }
-
- for (i = 0; i < REG_MEM_COUNT; i++) {
- status = dat_lmr_free(lmr[ i ]);
- _OK(status, "dat_lmr_free");
- }
-
- status = dat_lmr_free(lmr_atomic);
- _OK(status, "dat_lmr_free_atomic");
-
- status = dat_ep_free(ep);
- _OK(status, "dat_ep_free");
-
- status = dat_evd_free(dto_evd);
- _OK(status, "dat_evd_free DTO");
- status = dat_evd_free(con_evd);
- _OK(status, "dat_evd_free CON");
- status = dat_evd_free(cr_evd);
- _OK(status, "dat_evd_free CR");
-
- status = dat_pz_free(pz);
- _OK(status, "dat_pz_free");
-
- status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
- _OK(status, "dat_ia_close");
-
- return(0);
-}
-
-int
-do_immediate()
-{
- DAT_REGION_DESCRIPTION region;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET iov;
- DAT_RMR_TRIPLET r_iov;
- DAT_DTO_COOKIE cookie;
- DAT_RMR_CONTEXT their_context;
- DAT_RETURN status;
- DAT_UINT32 immed_data;
- DAT_UINT32 immed_data_recv;
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
- &event.event_data.dto_completion_event_data;
- DAT_IB_EXTENSION_EVENT_DATA *ext_event =
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
-
- printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
-
- if (server) {
- immed_data = 0x1111;
- } else {
- immed_data = 0x7777;
- }
-
- cookie.as_64 = 0x5555;
-
- r_iov = *buf[ RECV_BUF_INDEX ];
-
- iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];
- iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];
- iov.segment_length = BUF_SIZE;
-
- cookie.as_64 = 0x9999;
-
- status = dat_ib_post_rdma_write_immed(ep, // ep_handle
- 1, // num_segments
- &iov, // LMR
- cookie, // user_cookie
- &r_iov, // RMR
- immed_data,
- DAT_COMPLETION_DEFAULT_FLAG);
- _OK(status, "dat_ib_post_rdma_write_immed");
-
- /*
- * Collect first event, write completion or the inbound recv with immed
- */
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
- if (event.event_number != DAT_IB_DTO_EVENT)
- {
- printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
- event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status");
- if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
- {
- if ((dto_event->transfered_length != BUF_SIZE) ||
- (dto_event->user_cookie.as_64 != 0x9999))
- {
- printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64);
- exit(1);
- }
- }
- else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
- {
- if ((dto_event->transfered_length != BUF_SIZE) ||
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
- {
- printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64,
- sizeof(int), RECV_BUF_INDEX+1);
- exit(1);
- }
-
- /* get immediate data from event */
- immed_data_recv = ext_event->val.immed.data;
- }
- else
- {
- printf("unexpected extension type for event - 0x%x, 0x%x\n",
- event.event_number, ext_event->type);
- exit(1);
- }
-
-
- /*
- * Collect second event, write completion or the inbound recv with immed
- */
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
- if (event.event_number != DAT_IB_DTO_EVENT)
- {
- printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
- event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status");
- if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
- {
- if ((dto_event->transfered_length != BUF_SIZE) ||
- (dto_event->user_cookie.as_64 != 0x9999))
- {
- printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64);
- exit(1);
- }
- }
- else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
- {
- if ((dto_event->transfered_length != BUF_SIZE) ||
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
- {
- printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
- (int)dto_event->transfered_length,
- (int)dto_event->user_cookie.as_64,
- sizeof(int), RECV_BUF_INDEX+1);
- exit(1);
- }
-
- /* get immediate data from event */
- immed_data_recv = ext_event->val.immed.data;
- }
- else
- {
- printf("unexpected extension type for event - 0x%x, 0x%x\n",
- event.event_number, ext_event->type);
- exit(1);
- }
-
- if ((server) && (immed_data_recv != 0x7777))
- {
- printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n",
- 0x7777, immed_data_recv);
- exit(1);
- }
- else if ((!server) && (immed_data_recv != 0x1111))
- {
- printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n",
- 0x1111, immed_data_recv);
- exit(1);
- }
-
- if (server)
- printf("Server received immed_data=0x%x\n", immed_data_recv);
- else
- printf("Client received immed_data=0x%x\n", immed_data_recv);
-
- printf("rdma buffer %p contains: %s\n",
- buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);
-
- printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
- return (0);
-}
-
-int
-do_cmp_swap()
-{
- DAT_DTO_COOKIE cookie;
- DAT_RETURN status;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
- &event.event_data.dto_completion_event_data;
- DAT_IB_EXTENSION_EVENT_DATA *ext_event =
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
-
- printf("\nDoing CMP and SWAP\n");
-
- r_iov = *buf[ RECV_BUF_INDEX ];
-
- l_iov.lmr_context = lmr_atomic_context;
- l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
- l_iov.segment_length = BUF_SIZE_ATOMIC;
-
- cookie.as_64 = 3333;
- if (server) {
- *target = 0x12345;
- sleep(1);
- /* server does not compare and should not swap */
- status = dat_ib_post_cmp_and_swap( ep,
- (DAT_UINT64)0x654321,
- (DAT_UINT64)0x6789A,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- } else {
- *target = 0x54321;
- sleep(1);
- /* client does compare and should swap */
- status = dat_ib_post_cmp_and_swap( ep,
- (DAT_UINT64)0x12345,
- (DAT_UINT64)0x98765,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- }
- _OK(status, "dat_ib_post_cmp_and_swap");
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait for compare and swap");
- if (event.event_number != DAT_IB_DTO_EVENT) {
- printf("unexpected event after post_cmp_and_swap: 0x%x\n",
- event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status for CMP and SWAP");
- if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
- printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",
- (int)ext_event->type,
- (int)dto_event->user_cookie.as_64,
- *atomic_buf);
- exit(1);
- }
- sleep(1); /* wait for other side to complete swap */
- if (server) {
- printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);
- printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);
-
- if (*atomic_buf != 0x54321 || *target != 0x98765) {
- printf("ERROR: Server CMP_SWAP\n");
- exit(1);
- }
- } else {
- printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);
- printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);
-
- if (*atomic_buf != 0x12345 || *target != 0x54321) {
- printf("ERROR: Client CMP_SWAP\n");
- exit(1);
- }
- }
- printf("\n CMP_SWAP test - PASSED\n");
- return(0);
-}
-
-int
-do_fetch_add()
-{
- DAT_DTO_COOKIE cookie;
- DAT_RETURN status;
- DAT_EVENT event;
- DAT_COUNT nmore;
- DAT_LMR_TRIPLET l_iov;
- DAT_RMR_TRIPLET r_iov;
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
- &event.event_data.dto_completion_event_data;
- DAT_IB_EXTENSION_EVENT_DATA *ext_event =
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
-
- printf("\nDoing FETCH and ADD\n");
-
- r_iov = *buf[ RECV_BUF_INDEX ];
-
- l_iov.lmr_context = lmr_atomic_context;
- l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
- l_iov.segment_length = BUF_SIZE_ATOMIC;
-
- cookie.as_64 = 0x7777;
- if (server) {
- /* Wait for client to finish cmp_swap */
- while (*target != 0x98765)
- sleep(1);
- *target = 0x10;
- sleep(1);
- status = dat_ib_post_fetch_and_add( ep,
- (DAT_UINT64)0x100,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- } else {
- /* Wait for server, no swap so nothing to check */
- *target = 0x100;
- sleep(1);
- status = dat_ib_post_fetch_and_add( ep,
- (DAT_UINT64)0x10,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- }
- _OK(status, "dat_ib_post_fetch_and_add");
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait for fetch and add");
- if (event.event_number != DAT_IB_DTO_EVENT) {
- printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status for FETCH and ADD");
- if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
- printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",
- (int)ext_event->type,
- (int)dto_event->user_cookie.as_64,
- (int)*atomic_buf);
- exit(1);
- }
-
- if (server) {
- printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);
- } else {
- printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);
- }
-
- sleep(1);
-
- if (server) {
- status = dat_ib_post_fetch_and_add( ep,
- (DAT_UINT64)0x100,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- } else {
- status = dat_ib_post_fetch_and_add( ep,
- (DAT_UINT64)0x10,
- &l_iov,
- cookie,
- &r_iov,
- DAT_COMPLETION_DEFAULT_FLAG);
- }
-
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
- _OK(status, "dat_evd_wait for second fetch and add");
- if (event.event_number != DAT_IB_DTO_EVENT) {
- printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number);
- exit(1);
- }
-
- _OK(dto_event->status, "event status for second FETCH and ADD");
- if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
- printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",
- (int)ext_event->type,
- (int)dto_event->user_cookie.as_64,
- (long)atomic_buf);
- exit(1);
- }
-
- sleep(1); /* wait for other side to complete fetch_add */
-
- if (server) {
- printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);
- printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);
-
- if (*atomic_buf != 0x200 || *target != 0x30) {
- printf("ERROR: Server FETCH_ADD\n");
- exit(1);
- }
- } else {
- printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);
- printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);
-
- if (*atomic_buf != 0x20 || *target != 0x300) {
- printf("ERROR: Server FETCH_ADD\n");
- exit(1);
- }
- }
- printf("\n FETCH_ADD test - PASSED\n");
- return(0);
-}
-
-int
-main(int argc, char **argv)
-{
- char *hostname;
-
- if (argc > 2) {
- printf(usage);
- exit(1);
- }
-
- if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)
- {
- server = 1;
- } else {
- server = 0;
- hostname = argv[ 1 ];
- }
-
-
- /*
- * connect
- */
- if (connect_ep(hostname)) {
- exit(1);
- }
- if (do_immediate()) {
- exit(1);
- }
- if (do_cmp_swap()) {
- exit(1);
- }
- if (do_fetch_add()) {
- exit(1);
- }
- return (disconnect_ep());
-}
+
+ if (server) {
+ status = dat_psp_free(psp);
+ _OK(status, "dat_psp_free");
+ }
+
+ for (i = 0; i < REG_MEM_COUNT; i++) {
+ status = dat_lmr_free(lmr[ i ]);
+ _OK(status, "dat_lmr_free");
+ }
+
+ status = dat_lmr_free(lmr_atomic);
+ _OK(status, "dat_lmr_free_atomic");
+
+ status = dat_ep_free(ep);
+ _OK(status, "dat_ep_free");
+
+ status = dat_evd_free(dto_evd);
+ _OK(status, "dat_evd_free DTO");
+ status = dat_evd_free(con_evd);
+ _OK(status, "dat_evd_free CON");
+ status = dat_evd_free(cr_evd);
+ _OK(status, "dat_evd_free CR");
+
+ status = dat_pz_free(pz);
+ _OK(status, "dat_pz_free");
+
+ status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
+ _OK(status, "dat_ia_close");
+
+ return(0);
+}
+
+int
+do_immediate()
+{
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RMR_CONTEXT their_context;
+ DAT_RETURN status;
+ DAT_UINT32 immed_data;
+ DAT_UINT32 immed_data_recv;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
+
+ if (server) {
+ immed_data = 0x1111;
+ } else {
+ immed_data = 0x7777;
+ }
+
+ cookie.as_64 = 0x5555;
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];
+ iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];
+ iov.segment_length = BUF_SIZE;
+
+ cookie.as_64 = 0x9999;
+
+ status = dat_ib_post_rdma_write_immed(ep, // ep_handle
+ 1, // num_segments
+ &iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ immed_data,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ _OK(status, "dat_ib_post_rdma_write_immed");
+
+ /*
+ * Collect first event, write completion or the inbound recv with immed
+ */
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
+ if (event.event_number != DAT_IB_DTO_EVENT)
+ {
+ printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status");
+ if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != 0x9999))
+ {
+ printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64);
+ exit(1);
+ }
+ }
+ else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
+ {
+ printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(int), RECV_BUF_INDEX+1);
+ exit(1);
+ }
+
+ /* get immediate data from event */
+ immed_data_recv = ext_event->val.immed.data;
+ }
+ else
+ {
+ printf("unexpected extension type for event - 0x%x, 0x%x\n",
+ event.event_number, ext_event->type);
+ exit(1);
+ }
+
+
+ /*
+ * Collect second event, write completion or the inbound recv with immed
+ */
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
+ if (event.event_number != DAT_IB_DTO_EVENT)
+ {
+ printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status");
+ if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != 0x9999))
+ {
+ printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64);
+ exit(1);
+ }
+ }
+ else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
+ {
+ printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(int), RECV_BUF_INDEX+1);
+ exit(1);
+ }
+
+ /* get immediate data from event */
+ immed_data_recv = ext_event->val.immed.data;
+ }
+ else
+ {
+ printf("unexpected extension type for event - 0x%x, 0x%x\n",
+ event.event_number, ext_event->type);
+ exit(1);
+ }
+
+ if ((server) && (immed_data_recv != 0x7777))
+ {
+ printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n",
+ 0x7777, immed_data_recv);
+ exit(1);
+ }
+ else if ((!server) && (immed_data_recv != 0x1111))
+ {
+ printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n",
+ 0x1111, immed_data_recv);
+ exit(1);
+ }
+
+ if (server)
+ printf("Server received immed_data=0x%x\n", immed_data_recv);
+ else
+ printf("Client received immed_data=0x%x\n", immed_data_recv);
+
+ printf("rdma buffer %p contains: %s\n",
+ buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);
+
+ printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
+ return (0);
+}
+
+int
+do_cmp_swap()
+{
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN status;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing CMP and SWAP\n");
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ l_iov.lmr_context = lmr_atomic_context;
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
+ l_iov.segment_length = BUF_SIZE_ATOMIC;
+
+ cookie.as_64 = 3333;
+ if (server) {
+ *target = 0x12345;
+ sleep(1);
+ /* server does not compare and should not swap */
+ status = dat_ib_post_cmp_and_swap( ep,
+ (DAT_UINT64)0x654321,
+ (DAT_UINT64)0x6789A,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ *target = 0x54321;
+ sleep(1);
+ /* client does compare and should swap */
+ status = dat_ib_post_cmp_and_swap( ep,
+ (DAT_UINT64)0x12345,
+ (DAT_UINT64)0x98765,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+ _OK(status, "dat_ib_post_cmp_and_swap");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for compare and swap");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after post_cmp_and_swap: 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for CMP and SWAP");
+ if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
+ printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ *atomic_buf);
+ exit(1);
+ }
+ sleep(1); /* wait for other side to complete swap */
+ if (server) {
+ printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);
+ printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);
+
+ if (*atomic_buf != 0x54321 || *target != 0x98765) {
+ printf("ERROR: Server CMP_SWAP\n");
+ exit(1);
+ }
+ } else {
+ printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);
+ printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);
+
+ if (*atomic_buf != 0x12345 || *target != 0x54321) {
+ printf("ERROR: Client CMP_SWAP\n");
+ exit(1);
+ }
+ }
+ printf("\n CMP_SWAP test - PASSED\n");
+ return(0);
+}
+
+int
+do_fetch_add()
+{
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN status;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing FETCH and ADD\n");
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ l_iov.lmr_context = lmr_atomic_context;
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
+ l_iov.segment_length = BUF_SIZE_ATOMIC;
+
+ cookie.as_64 = 0x7777;
+ if (server) {
+ /* Wait for client to finish cmp_swap */
+ while (*target != 0x98765)
+ sleep(1);
+ *target = 0x10;
+ sleep(1);
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x100,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ /* Wait for server, no swap so nothing to check */
+ *target = 0x100;
+ sleep(1);
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x10,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+ _OK(status, "dat_ib_post_fetch_and_add");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for fetch and add");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for FETCH and ADD");
+ if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
+ printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ (int)*atomic_buf);
+ exit(1);
+ }
+
+ if (server) {
+ printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);
+ } else {
+ printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);
+ }
+
+ sleep(1);
+
+ if (server) {
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x100,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x10,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for second fetch and add");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for second FETCH and ADD");
+ if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
+ printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ (long)atomic_buf);
+ exit(1);
+ }
+
+ sleep(1); /* wait for other side to complete fetch_add */
+
+ if (server) {
+ printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);
+ printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);
+
+ if (*atomic_buf != 0x200 || *target != 0x30) {
+ printf("ERROR: Server FETCH_ADD\n");
+ exit(1);
+ }
+ } else {
+ printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);
+ printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);
+
+ if (*atomic_buf != 0x20 || *target != 0x300) {
+ printf("ERROR: Server FETCH_ADD\n");
+ exit(1);
+ }
+ }
+ printf("\n FETCH_ADD test - PASSED\n");
+ return(0);
+}
+
+int
+main(int argc, char **argv)
+{
+ char *hostname;
+
+ if (argc > 2) {
+ printf(usage);
+ exit(1);
+ }
+
+ if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)
+ {
+ server = 1;
+ } else {
+ server = 0;
+ hostname = argv[ 1 ];
+ }
+
+
+ /*
+ * connect
+ */
+ if (connect_ep(hostname)) {
+ exit(1);
+ }
+ if (do_immediate()) {
+ exit(1);
+ }
+ if (do_cmp_swap()) {
+ exit(1);
+ }
+ if (do_fetch_add()) {
+ exit(1);
+ }
+ return (disconnect_ep());
+}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/general/attachments/20070920/4928742f/attachment.html>
More information about the general
mailing list