[ofa-general] Re: [PATCH] uDAPL 2.0 mods to co-exist with uDAPL 1.2
James Lentini
jlentini at netapp.com
Fri Sep 21 12:40:06 PDT 2007
Comments below:
On Thu, 20 Sep 2007, Arlin Davis wrote:
> James,
>
>
>
> Please review patches to allow coexistence of 2.0 and 1.2 libraries. I updated the dat.conf to
> provide configuration to both 1.2 and 2.0 providers. In addition, the development package (headers)
> is not targeted to include/dat2 instead of include/dat. A patch for 1.2 will follow shortly.
>
>
>
> Modifications to DAT 2.0 package to coexist with 1.2 libraries
>
> - cleanup CR-LF in dtestx
>
> - fix RPM specfile, 2.0.1 package
>
> - move devel to include/dat2
>
> - change test examples to use new 2.0 provider names.
>
>
>
> Signed-off by: Arlin Davis ardavis at ichips.intel.com
>
>
>
> diff --git a/Makefile.am b/Makefile.am
> index b3a0149..f473aaa 100755
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -66,7 +66,7 @@ dat_udat_libdat_la_SOURCES = dat/udat/udat.c \
> dat/common/dat_init.c \
> dat/common/dat_dr.c \
> dat/common/dat_sr.c
> -
> +# version-info current:revision:age
What does this comment do?
> dat_udat_libdat_la_LDFLAGS = -version-info 2:0:0 $(dat_version_script) -ldl
>
> #
> @@ -178,11 +178,12 @@ dapl_udapl_libdaplcma_la_SOURCES = dapl/udapl/dapl_init.c \
> dapl/openib_cma/dapl_ib_cm.c \
> dapl/openib_cma/dapl_ib_mem.c $(XPROGRAMS)
>
> +# version-info current:revision:age
ditto
> dapl_udapl_libdaplcma_la_LDFLAGS = -version-info 2:0:0 $(daplcma_version_script) \
> -Wl,-init,dapl_init -Wl,-fini,dapl_fini \
> -lpthread -libverbs -lrdmacm
>
> -libdatincludedir = $(includedir)/dat
> +libdatincludedir = $(includedir)/dat2
>
> libdatinclude_HEADERS = dat/include/dat/dat.h \
> dat/include/dat/dat_error.h \
> @@ -244,7 +245,7 @@ EXTRA_DIST = dat/common/dat_dictionary.h \
> dat/udat/libdat.map \
> doc/dat.conf \
> dapl/udapl/libdaplcma.map \
> - libdat.spec.in \
> + libdat2.spec.in \
> $(man_MANS) \
> test/dapltest/include/dapl_bpool.h \
> test/dapltest/include/dapl_client_info.h \
> @@ -274,7 +275,7 @@ EXTRA_DIST = dat/common/dat_dictionary.h \
> test/dapltest/include/dapl_version.h \
> test/dapltest/mdep/linux/dapl_mdep_user.h
>
> -dist-hook: libdat.spec
> - cp libdat.spec $(distdir)
> +dist-hook: libdat2.spec
> + cp libdat2.spec $(distdir)
>
> SUBDIRS = . test/dtest test/dapltest
> diff --git a/README b/README
> index 437c1f7..1fc55a2 100644
> --- a/README
> +++ b/README
> @@ -17,16 +17,18 @@ Building debug version:
> ./configure --enable-debug
> make
>
> -Build example with OFED prefix (x86_64)
> ------------------------------------------
> +Build example with OFED 1.2+ prefix (x86_64)
> +---------------------------------------------
> ./autogen.sh
> -./configure --prefix /usr/local/ofed --libdir /usr/local/ofed/lib64 LDFLAGS=-L/usr/local/ofed/lib64
> CPPFLAGS="-I/usr/local/ofed/include"
> +./configure --prefix /usr --sysconf=/etc --libdir /usr/lib64 LDFLAGS=-L/usr/lib64
> CPPFLAGS="-I/usr/include"
It looks like there was some line wrap here and in other places below.
No big deal.
> make
>
> Installing:
> ----------
> make install
>
> +Note: The development package installs DAT 2.0 include files under /usr/include/dat2 to co-exist
> with DAT 1.2 /usr/include/dat
> +
> NOTE: to link these libraries you must either use libtool and
> specify the full pathname of the library, or use the `-LLIBDIR'
> flag during linking and do at least one of the following:
> @@ -47,19 +49,32 @@ more information, such as the ld(1) and ld.so(8) manual pages.
> sample /etc/dat.conf
>
> #
> -# DAT 1.2 configuration file, sample OFED
> +# DAT 1.2 and 2.0 configuration file
> #
> # Each entry should have the following fields:
> #
> # <ia_name> <api_version> <threadsafety> <default> <lib_path> \
> # <provider_version> <ia_params> <platform_params>
> #
> -# For openib-cma provider you can specify <ia_params> as either:
> -# network address, network hostname, or netdev name and 0 for port
> +# For the uDAPL cma provder, specify <ia_params> as one of the following:
> +# network address, network hostname, or netdev name and 0 for port
> +#
> +# Simple (OpenIB-cma) default with netdev name provided first on list
> +# to enable use of same dat.conf version on all nodes
> #
> -# This example shows netdev name, enabling administrator to use same copy across cluster
> +# Add examples for multiple interfaces and IPoIB HA fail over, and bonding
The previous line is TODO, right? I'd suggest annotating it with that
text to make it clear to users.
> #
> -OpenIB-cma u1.2 nonthreadsafe default /usr/local/ofed/lib64/libdapl-cma.so mv_dapl.1.2 "ib0 0" ""
> +OpenIB-cma u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib0 0" ""
> +OpenIB-cma-1 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib1 0" ""
> +OpenIB-cma-2 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib2 0" ""
> +OpenIB-cma-3 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib3 0" ""
> +OpenIB-bond u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "bond0 0" ""
> +OpenIB-2-cma u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib0 0" ""
> +OpenIB-2-cma-1 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib1 0" ""
> +OpenIB-2-cma-2 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib2 0" ""
> +OpenIB-2-cma-3 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib3 0" ""
> +OpenIB-2-bond u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "bond0 0" ""
> +
>
> =============================
> 3.0 Bugs/Known issues
> diff --git a/configure.in b/configure.in
> index 7608e64..4eda85f 100644
> --- a/configure.in
> +++ b/configure.in
> @@ -1,11 +1,11 @@
> dnl Process this file with autoconf to produce a configure script.
>
> AC_PREREQ(2.57)
> -AC_INIT(dapl, 2.0.0, openib-general at openib.org)
> +AC_INIT(dapl, 2.0.1, general at lists.openfabrics.org)
> AC_CONFIG_SRCDIR([dat/udat/udat.c])
> AC_CONFIG_AUX_DIR(config)
> AM_CONFIG_HEADER(config.h)
> -AM_INIT_AUTOMAKE(dapl, 2.0.0)
> +AM_INIT_AUTOMAKE(dapl, 2.0.1)
>
> AM_PROG_LIBTOOL
>
> @@ -86,6 +86,6 @@ AC_CACHE_CHECK(Check for RHEL5 system, ac_cv_rhel5,
> fi)
> AM_CONDITIONAL(OS_RHEL5, test "$ac_cv_rhel5" = "yes")
>
> -AC_CONFIG_FILES([Makefile test/dtest/Makefile test/dapltest/Makefile libdat.spec])
> +AC_CONFIG_FILES([Makefile test/dtest/Makefile test/dapltest/Makefile libdat2.spec])
>
> AC_OUTPUT
> diff --git a/doc/dat.conf b/doc/dat.conf
> index 2651673..005f9ee 100755
> --- a/doc/dat.conf
> +++ b/doc/dat.conf
> @@ -1,5 +1,5 @@
> #
> -# DAT 2.0 configuration file
> +# DAT 1.2 and 2.0 configuration file
> #
> # Each entry should have the following fields:
> #
> @@ -9,10 +9,18 @@
> # For the uDAPL cma provder, specify <ia_params> as one of the following:
> # network address, network hostname, or netdev name and 0 for port
> #
> -# Simple (OpenIB-cma) default configuration with netdev name provided first on list
> -# to enable use of same dat.conf version on all nodes. Assumes x86_64 installation.
> +# Simple (OpenIB-cma) default with netdev name provided first on list
> +# to enable use of same dat.conf version on all nodes
> #
> -OpenIB-cma u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
> -OpenIB-cma-1 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
> -OpenIB-cma-2 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
> -OpenIB-cma-3 u2.0 nonthreadsafe default /usr/lib64/libdaplcma.so mv_dapl.2.0 "ib0 0" ""
> +# Add examples for multiple interfaces and IPoIB HA fail over, and bonding
> +#
Again, the previous line is a TODO?
> +OpenIB-cma u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib0 0" ""
> +OpenIB-cma-1 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib1 0" ""
> +OpenIB-cma-2 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib2 0" ""
> +OpenIB-cma-3 u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "ib3 0" ""
> +OpenIB-bond u1.2 nonthreadsafe default libdaplcma.so.1 dapl.1.2 "bond0 0" ""
> +OpenIB-2-cma u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib0 0" ""
> +OpenIB-2-cma-1 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib1 0" ""
> +OpenIB-2-cma-2 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib2 0" ""
> +OpenIB-2-cma-3 u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "ib3 0" ""
> +OpenIB-2-bond u2.0 nonthreadsafe default libdaplcma.so.2 dapl.2.0 "bond0 0" ""
> diff --git a/test/dtest/dtest.c b/test/dtest/dtest.c
> index 07b40ec..ba12a58 100644
> --- a/test/dtest/dtest.c
> +++ b/test/dtest/dtest.c
> @@ -44,7 +44,7 @@
> #include <inttypes.h>
>
> #ifndef DAPL_PROVIDER
> -#define DAPL_PROVIDER "OpenIB-cma"
> +#define DAPL_PROVIDER "OpenIB-2-cma"
Should we update OpenIB to ofa? Obviously, this isn't necessary as
part of this change
> #endif
>
> #define MAX_POLLING_CNT 50000
> diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c
> index 153ce76..04a0d5d 100755
> --- a/test/dtest/dtestx.c
> +++ b/test/dtest/dtestx.c
> @@ -30,785 +30,785 @@
> * SOFTWARE.
> *
> * $Id: $
> - */
>
The formating seems strange below here. There appears to be an extra
space after each "-" line. Ignoring that, I'm in complete agreement
with switching the examples over to use the 2.0 APIs.
> -#include <stdio.h>
>
> -#include <stdlib.h>
>
> -#include <netdb.h>
>
> -#include <sys/socket.h>
>
> -#include <netinet/in.h>
>
> -#include <netinet/tcp.h>
>
> -#include <arpa/inet.h>
>
> -#include <endian.h>
>
> -#include <byteswap.h>
>
> -
>
> -#include "dat/udat.h"
>
> -#include "dat/dat_ib_extensions.h"
>
> -
>
> -#define _OK(status, str) \
>
> -{ \
>
> - const char *maj_msg, *min_msg; \
>
> - if (status != DAT_SUCCESS) { \
>
> - dat_strerror(status, &maj_msg, &min_msg); \
>
> - fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \
>
> - exit(1); \
>
> - } \
>
> -}
>
> -
>
> + */
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <netdb.h>
> +#include <sys/socket.h>
> +#include <netinet/in.h>
> +#include <netinet/tcp.h>
> +#include <arpa/inet.h>
> +#include <endian.h>
> +#include <byteswap.h>
> +
> +#include "dat/udat.h"
> +#include "dat/dat_ib_extensions.h"
> +
> +#define _OK(status, str) \
> +{ \
> + const char *maj_msg, *min_msg; \
> + if (status != DAT_SUCCESS) { \
> + dat_strerror(status, &maj_msg, &min_msg); \
> + fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \
> + exit(1); \
> + } \
> +}
> +
> #define DTO_TIMEOUT (1000*1000*5)
> #define CONN_TIMEOUT (1000*1000*10)
> -#define SERVER_TIMEOUT (1000*1000*120)
>
> -#define SERVER_CONN_QUAL 31111
>
> -#define BUF_SIZE 256
>
> -#define BUF_SIZE_ATOMIC 8
>
> -#define REG_MEM_COUNT 10
>
> -#define SND_RDMA_BUF_INDEX 0
>
> -#define RCV_RDMA_BUF_INDEX 1
>
> -#define SEND_BUF_INDEX 2
>
> -#define RECV_BUF_INDEX 3
>
> -
>
> -u_int64_t *atomic_buf;
>
> -DAT_LMR_HANDLE lmr_atomic;
>
> -DAT_LMR_CONTEXT lmr_atomic_context;
>
> -DAT_RMR_CONTEXT rmr_atomic_context;
>
> -DAT_VLEN reg_atomic_size;
>
> -DAT_VADDR reg_atomic_addr;
>
> -DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
>
> -DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
>
> -DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
>
> -DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
>
> -DAT_VLEN reg_size[ REG_MEM_COUNT ];
>
> -DAT_VADDR reg_addr[ REG_MEM_COUNT ];
>
> -DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
>
> -DAT_EP_HANDLE ep;
>
> -DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
>
> -DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
>
> -DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
>
> -DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
>
> -DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
>
> -DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
>
> -DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
>
> -DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
>
> -int server;
>
> -
>
> -char *usage = "-s | hostname (default == -s)\n";
>
> -
>
> -void
>
> -send_msg(
>
> - void *data,
>
> - DAT_COUNT size,
>
> - DAT_LMR_CONTEXT context,
>
> - DAT_DTO_COOKIE cookie,
>
> - DAT_COMPLETION_FLAGS flags)
>
> -{
>
> - DAT_LMR_TRIPLET iov;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> - DAT_RETURN status;
>
> - DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
>
> - &event.event_data.dto_completion_event_data;
>
> -
>
> - iov.lmr_context = context;
>
> - iov.virtual_address = (DAT_VADDR)(unsigned long)data;
>
> - iov.segment_length = (DAT_VLEN)size;
>
> -
>
> - status = dat_ep_post_send(ep,
>
> - 1,
>
> - &iov,
>
> - cookie,
>
> - flags);
>
> - _OK(status, "dat_ep_post_send");
>
> -
>
> - if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait after dat_ep_post_send");
>
> -
>
> - if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
>
> - printf("unexpected event waiting for post_send completion - 0x%x\n",
> event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status for post_send");
>
> - }
>
> -}
>
> -
>
> -int
>
> -connect_ep(char *hostname)
>
> -{
>
> - DAT_SOCK_ADDR remote_addr;
>
> - DAT_EP_ATTR ep_attr;
>
> - DAT_RETURN status;
>
> - DAT_REGION_DESCRIPTION region;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> - DAT_LMR_TRIPLET iov;
>
> - DAT_RMR_TRIPLET r_iov;
>
> - DAT_DTO_COOKIE cookie;
>
> - int i;
>
> - DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
>
> - &event.event_data.dto_completion_event_data;
>
> -
>
> - status = dat_ia_open("OpenIB-cma", 8, &async_evd, &ia);
>
> - _OK(status, "dat_ia_open");
>
> -
>
> - status = dat_pz_create(ia, &pz);
>
> - _OK(status, "dat_pz_create");
>
> -
>
> - status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );
>
> - _OK(status, "dat_evd_create CR");
>
> - status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );
>
> - _OK(status, "dat_evd_create CR");
>
> - status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );
>
> - _OK(status, "dat_evd_create DTO");
>
> -
>
> - memset(&ep_attr, 0, sizeof(ep_attr));
>
> - ep_attr.service_type = DAT_SERVICE_TYPE_RC;
>
> - ep_attr.max_rdma_size = 0x10000;
>
> - ep_attr.qos = 0;
>
> - ep_attr.recv_completion_flags = 0;
>
> - ep_attr.max_recv_dtos = 10;
>
> - ep_attr.max_request_dtos = 10;
>
> - ep_attr.max_recv_iov = 1;
>
> - ep_attr.max_request_iov = 1;
>
> - ep_attr.max_rdma_read_in = 4;
>
> - ep_attr.max_rdma_read_out = 4;
>
> - ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
>
> - ep_attr.ep_transport_specific_count = 0;
>
> - ep_attr.ep_transport_specific = NULL;
>
> - ep_attr.ep_provider_specific_count = 0;
>
> - ep_attr.ep_provider_specific = NULL;
>
> -
>
> - status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);
>
> - _OK(status, "dat_ep_create");
>
> -
>
> - for (i = 0; i < REG_MEM_COUNT; i++) {
>
> - buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);
>
> - region.for_va = buf[ i ];
>
> - status = dat_lmr_create(ia,
>
> - DAT_MEM_TYPE_VIRTUAL,
>
> - region,
>
> - BUF_SIZE,
>
> - pz,
>
> - DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
>
> +#define SERVER_TIMEOUT (1000*1000*120)
> +#define SERVER_CONN_QUAL 31111
> +#define BUF_SIZE 256
> +#define BUF_SIZE_ATOMIC 8
> +#define REG_MEM_COUNT 10
> +#define SND_RDMA_BUF_INDEX 0
> +#define RCV_RDMA_BUF_INDEX 1
> +#define SEND_BUF_INDEX 2
> +#define RECV_BUF_INDEX 3
> +
> +u_int64_t *atomic_buf;
> +DAT_LMR_HANDLE lmr_atomic;
> +DAT_LMR_CONTEXT lmr_atomic_context;
> +DAT_RMR_CONTEXT rmr_atomic_context;
> +DAT_VLEN reg_atomic_size;
> +DAT_VADDR reg_atomic_addr;
> +DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
> +DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
> +DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
> +DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
> +DAT_VLEN reg_size[ REG_MEM_COUNT ];
> +DAT_VADDR reg_addr[ REG_MEM_COUNT ];
> +DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
> +DAT_EP_HANDLE ep;
> +DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
> +DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
> +DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
> +DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
> +DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
> +DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
> +DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
> +DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
> +int server;
> +
> +char *usage = "-s | hostname (default == -s)\n";
> +
> +void
> +send_msg(
> + void *data,
> + DAT_COUNT size,
> + DAT_LMR_CONTEXT context,
> + DAT_DTO_COOKIE cookie,
> + DAT_COMPLETION_FLAGS flags)
> +{
> + DAT_LMR_TRIPLET iov;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> + DAT_RETURN status;
> + DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
> + &event.event_data.dto_completion_event_data;
> +
> + iov.lmr_context = context;
> + iov.virtual_address = (DAT_VADDR)(unsigned long)data;
> + iov.segment_length = (DAT_VLEN)size;
> +
> + status = dat_ep_post_send(ep,
> + 1,
> + &iov,
> + cookie,
> + flags);
> + _OK(status, "dat_ep_post_send");
> +
> + if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait after dat_ep_post_send");
> +
> + if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
> + printf("unexpected event waiting for post_send completion - 0x%x\n",
> event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status for post_send");
> + }
> +}
> +
> +int
> +connect_ep(char *hostname)
> +{
> + DAT_SOCK_ADDR remote_addr;
> + DAT_EP_ATTR ep_attr;
> + DAT_RETURN status;
> + DAT_REGION_DESCRIPTION region;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> + DAT_LMR_TRIPLET iov;
> + DAT_RMR_TRIPLET r_iov;
> + DAT_DTO_COOKIE cookie;
> + int i;
> + DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
> + &event.event_data.dto_completion_event_data;
> +
> + status = dat_ia_open("OpenIB-2-cma", 8, &async_evd, &ia);
> + _OK(status, "dat_ia_open");
> +
> + status = dat_pz_create(ia, &pz);
> + _OK(status, "dat_pz_create");
> +
> + status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );
> + _OK(status, "dat_evd_create CR");
> + status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );
> + _OK(status, "dat_evd_create CR");
> + status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );
> + _OK(status, "dat_evd_create DTO");
> +
> + memset(&ep_attr, 0, sizeof(ep_attr));
> + ep_attr.service_type = DAT_SERVICE_TYPE_RC;
> + ep_attr.max_rdma_size = 0x10000;
> + ep_attr.qos = 0;
> + ep_attr.recv_completion_flags = 0;
> + ep_attr.max_recv_dtos = 10;
> + ep_attr.max_request_dtos = 10;
> + ep_attr.max_recv_iov = 1;
> + ep_attr.max_request_iov = 1;
> + ep_attr.max_rdma_read_in = 4;
> + ep_attr.max_rdma_read_out = 4;
> + ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
> + ep_attr.ep_transport_specific_count = 0;
> + ep_attr.ep_transport_specific = NULL;
> + ep_attr.ep_provider_specific_count = 0;
> + ep_attr.ep_provider_specific = NULL;
> +
> + status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);
> + _OK(status, "dat_ep_create");
> +
> + for (i = 0; i < REG_MEM_COUNT; i++) {
> + buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);
> + region.for_va = buf[ i ];
> + status = dat_lmr_create(ia,
> + DAT_MEM_TYPE_VIRTUAL,
> + region,
> + BUF_SIZE,
> + pz,
> + DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
> DAT_VA_TYPE_VA,
> - &lmr[ i ],
>
> - &lmr_context[ i ],
>
> - &rmr_context[ i ],
>
> - ®_size[ i ],
>
> - ®_addr[ i ]);
>
> - _OK(status, "dat_lmr_create");
>
> - }
>
> -
>
> - /* register atomic return buffer for original data */
>
> - atomic_buf = (u_int64_t*)malloc(BUF_SIZE);
>
> - region.for_va = atomic_buf;
>
> - status = dat_lmr_create(ia,
>
> - DAT_MEM_TYPE_VIRTUAL,
>
> - region,
>
> - BUF_SIZE_ATOMIC,
>
> - pz,
>
> - DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
>
> + &lmr[ i ],
> + &lmr_context[ i ],
> + &rmr_context[ i ],
> + ®_size[ i ],
> + ®_addr[ i ]);
> + _OK(status, "dat_lmr_create");
> + }
> +
> + /* register atomic return buffer for original data */
> + atomic_buf = (u_int64_t*)malloc(BUF_SIZE);
> + region.for_va = atomic_buf;
> + status = dat_lmr_create(ia,
> + DAT_MEM_TYPE_VIRTUAL,
> + region,
> + BUF_SIZE_ATOMIC,
> + pz,
> + DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
> DAT_VA_TYPE_VA,
> - &lmr_atomic,
>
> - &lmr_atomic_context,
>
> - &rmr_atomic_context,
>
> - ®_atomic_size,
>
> - ®_atomic_addr);
>
> - _OK(status, "dat_lmr_create atomic");
>
> -
>
> - for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
>
> - cookie.as_64 = i;
>
> - iov.lmr_context = lmr_context[ i ];
>
> - iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];
>
> - iov.segment_length = BUF_SIZE;
>
> -
>
> - status = dat_ep_post_recv(ep,
>
> - 1,
>
> - &iov,
>
> - cookie,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - _OK(status, "dat_ep_post_recv");
>
> - }
>
> -
>
> - /* setup receive buffer to initial string to be overwritten */
>
> - strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
>
> -
>
> - if (server) {
>
> -
>
> - strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");
>
> -
>
> - status = dat_psp_create(ia,
>
> - SERVER_CONN_QUAL,
>
> - cr_evd,
>
> - DAT_PSP_CONSUMER_FLAG,
>
> - &psp);
>
> - _OK(status, "dat_psp_create");
>
> -
>
> - printf("Server waiting for connect request\n");
>
> - status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "listen dat_evd_wait");
>
> -
>
> - if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
>
> - printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
>
> - (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {
>
> -
>
> - printf("wrong cr event data\n");
>
> - exit(1);
>
> - }
>
> -
>
> - cr = event.event_data.cr_arrival_event_data.cr_handle;
>
> - status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
>
> -
>
> - } else {
>
> - struct addrinfo *target;
>
> - int rval;
>
> -
>
> - if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
>
> - printf("Error getting remote address.\n");
>
> - exit(1);
>
> - }
>
> -
>
> - rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
>
> - printf ("Server Name: %s \n", hostname);
>
> - printf ("Server Net Address: %d.%d.%d.%d\n",
>
> - (rval >> 0) & 0xff,
>
> - (rval >> 8) & 0xff,
>
> - (rval >> 16) & 0xff,
>
> - (rval >> 24) & 0xff);
>
> -
>
> - remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
>
> -
>
> - strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");
>
> -
>
> - status = dat_ep_connect(ep,
>
> - &remote_addr,
>
> - SERVER_CONN_QUAL,
>
> - CONN_TIMEOUT,
>
> - 0,
>
> - (DAT_PVOID)0,
>
> - 0,
>
> - DAT_CONNECT_DEFAULT_FLAG );
>
> - _OK(status, "dat_psp_create");
>
> - }
>
> -
>
> - printf("Client waiting for connect response\n");
>
> - status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "connect dat_evd_wait");
>
> -
>
> - if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
>
> - printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - printf("Connected!\n");
>
> -
>
> - /*
>
> - * Setup our remote memory and tell the other side about it
>
> - */
>
> - printf("Sending RMR data to remote\n");
>
> - r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];
>
> - r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);
>
> - r_iov.segment_length = BUF_SIZE;
>
> -
>
> - *buf[ SEND_BUF_INDEX ] = r_iov;
>
> -
>
> - send_msg( buf[ SEND_BUF_INDEX ],
>
> - sizeof(DAT_RMR_TRIPLET),
>
> - lmr_context[ SEND_BUF_INDEX ],
>
> - cookie,
>
> - DAT_COMPLETION_SUPPRESS_FLAG);
>
> -
>
> - /*
>
> - * Wait for their RMR
>
> - */
>
> - printf("Waiting for remote to send RMR data\n");
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait after dat_ep_post_send");
>
> -
>
> - if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
>
> - printf("unexpected event waiting for RMR context - 0x%x\n",
>
> - event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status for post_send");
>
> - if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||
>
> - (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {
>
> - printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",
>
> - (int)dto_event->transfered_length,
>
> - (int)dto_event->user_cookie.as_64,
>
> - sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);
>
> - exit(1);
>
> - }
>
> -
>
> - r_iov = *buf[ RECV_BUF_INDEX ];
>
> -
>
> - printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",
>
> - r_iov.rmr_context,
>
> - (void*)(unsigned long)r_iov.virtual_address,
>
> - r_iov.segment_length);
>
> -
>
> - return(0);
>
> -}
>
> -
>
> -int
>
> -disconnect_ep()
>
> -{
>
> - DAT_RETURN status;
>
> - int i;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> -
>
> - status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
>
> - _OK(status, "dat_ep_disconnect");
>
> -
>
> + &lmr_atomic,
> + &lmr_atomic_context,
> + &rmr_atomic_context,
> + ®_atomic_size,
> + ®_atomic_addr);
> + _OK(status, "dat_lmr_create atomic");
> +
> + for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
> + cookie.as_64 = i;
> + iov.lmr_context = lmr_context[ i ];
> + iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];
> + iov.segment_length = BUF_SIZE;
> +
> + status = dat_ep_post_recv(ep,
> + 1,
> + &iov,
> + cookie,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + _OK(status, "dat_ep_post_recv");
> + }
> +
> + /* setup receive buffer to initial string to be overwritten */
> + strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
> +
> + if (server) {
> +
> + strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");
> +
> + status = dat_psp_create(ia,
> + SERVER_CONN_QUAL,
> + cr_evd,
> + DAT_PSP_CONSUMER_FLAG,
> + &psp);
> + _OK(status, "dat_psp_create");
> +
> + printf("Server waiting for connect request\n");
> + status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "listen dat_evd_wait");
> +
> + if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
> + printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number);
> + exit(1);
> + }
> +
> + if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
> + (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {
> +
> + printf("wrong cr event data\n");
> + exit(1);
> + }
> +
> + cr = event.event_data.cr_arrival_event_data.cr_handle;
> + status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
> +
> + } else {
> + struct addrinfo *target;
> + int rval;
> +
> + if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
> + printf("Error getting remote address.\n");
> + exit(1);
> + }
> +
> + rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
> + printf ("Server Name: %s \n", hostname);
> + printf ("Server Net Address: %d.%d.%d.%d\n",
> + (rval >> 0) & 0xff,
> + (rval >> 8) & 0xff,
> + (rval >> 16) & 0xff,
> + (rval >> 24) & 0xff);
> +
> + remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
> +
> + strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");
> +
> + status = dat_ep_connect(ep,
> + &remote_addr,
> + SERVER_CONN_QUAL,
> + CONN_TIMEOUT,
> + 0,
> + (DAT_PVOID)0,
> + 0,
> + DAT_CONNECT_DEFAULT_FLAG );
> + _OK(status, "dat_psp_create");
> + }
> +
> + printf("Client waiting for connect response\n");
> + status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "connect dat_evd_wait");
> +
> + if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
> + printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number);
> + exit(1);
> + }
> +
> + printf("Connected!\n");
> +
> + /*
> + * Setup our remote memory and tell the other side about it
> + */
> + printf("Sending RMR data to remote\n");
> + r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];
> + r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);
> + r_iov.segment_length = BUF_SIZE;
> +
> + *buf[ SEND_BUF_INDEX ] = r_iov;
> +
> + send_msg( buf[ SEND_BUF_INDEX ],
> + sizeof(DAT_RMR_TRIPLET),
> + lmr_context[ SEND_BUF_INDEX ],
> + cookie,
> + DAT_COMPLETION_SUPPRESS_FLAG);
> +
> + /*
> + * Wait for their RMR
> + */
> + printf("Waiting for remote to send RMR data\n");
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait after dat_ep_post_send");
> +
> + if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
> + printf("unexpected event waiting for RMR context - 0x%x\n",
> + event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status for post_send");
> + if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||
> + (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {
> + printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",
> + (int)dto_event->transfered_length,
> + (int)dto_event->user_cookie.as_64,
> + sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);
> + exit(1);
> + }
> +
> + r_iov = *buf[ RECV_BUF_INDEX ];
> +
> + printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",
> + r_iov.rmr_context,
> + (void*)(unsigned long)r_iov.virtual_address,
> + r_iov.segment_length);
> +
> + return(0);
> +}
> +
> +int
> +disconnect_ep()
> +{
> + DAT_RETURN status;
> + int i;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> +
> + status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
> + _OK(status, "dat_ep_disconnect");
> +
> status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
> _OK(status, "dat_ep_disconnect");
> -
>
> - if (server) {
>
> - status = dat_psp_free(psp);
>
> - _OK(status, "dat_psp_free");
>
> - }
>
> -
>
> - for (i = 0; i < REG_MEM_COUNT; i++) {
>
> - status = dat_lmr_free(lmr[ i ]);
>
> - _OK(status, "dat_lmr_free");
>
> - }
>
> -
>
> - status = dat_lmr_free(lmr_atomic);
>
> - _OK(status, "dat_lmr_free_atomic");
>
> -
>
> - status = dat_ep_free(ep);
>
> - _OK(status, "dat_ep_free");
>
> -
>
> - status = dat_evd_free(dto_evd);
>
> - _OK(status, "dat_evd_free DTO");
>
> - status = dat_evd_free(con_evd);
>
> - _OK(status, "dat_evd_free CON");
>
> - status = dat_evd_free(cr_evd);
>
> - _OK(status, "dat_evd_free CR");
>
> -
>
> - status = dat_pz_free(pz);
>
> - _OK(status, "dat_pz_free");
>
> -
>
> - status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
>
> - _OK(status, "dat_ia_close");
>
> -
>
> - return(0);
>
> -}
>
> -
>
> -int
>
> -do_immediate()
>
> -{
>
> - DAT_REGION_DESCRIPTION region;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> - DAT_LMR_TRIPLET iov;
>
> - DAT_RMR_TRIPLET r_iov;
>
> - DAT_DTO_COOKIE cookie;
>
> - DAT_RMR_CONTEXT their_context;
>
> - DAT_RETURN status;
>
> - DAT_UINT32 immed_data;
>
> - DAT_UINT32 immed_data_recv;
>
> - DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
>
> - &event.event_data.dto_completion_event_data;
>
> - DAT_IB_EXTENSION_EVENT_DATA *ext_event =
>
> - (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
>
> -
>
> - printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
>
> -
>
> - if (server) {
>
> - immed_data = 0x1111;
>
> - } else {
>
> - immed_data = 0x7777;
>
> - }
>
> -
>
> - cookie.as_64 = 0x5555;
>
> -
>
> - r_iov = *buf[ RECV_BUF_INDEX ];
>
> -
>
> - iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];
>
> - iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];
>
> - iov.segment_length = BUF_SIZE;
>
> -
>
> - cookie.as_64 = 0x9999;
>
> -
>
> - status = dat_ib_post_rdma_write_immed(ep, // ep_handle
>
> - 1, // num_segments
>
> - &iov, // LMR
>
> - cookie, // user_cookie
>
> - &r_iov, // RMR
>
> - immed_data,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - _OK(status, "dat_ib_post_rdma_write_immed");
>
> -
>
> - /*
>
> - * Collect first event, write completion or the inbound recv with immed
>
> - */
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
>
> - if (event.event_number != DAT_IB_DTO_EVENT)
>
> - {
>
> - printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
>
> - event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status");
>
> - if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
>
> - {
>
> - if ((dto_event->transfered_length != BUF_SIZE) ||
>
> - (dto_event->user_cookie.as_64 != 0x9999))
>
> - {
>
> - printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
>
> - (int)dto_event->transfered_length,
>
> - (int)dto_event->user_cookie.as_64);
>
> - exit(1);
>
> - }
>
> - }
>
> - else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
>
> - {
>
> - if ((dto_event->transfered_length != BUF_SIZE) ||
>
> - (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
>
> - {
>
> - printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
>
> - (int)dto_event->transfered_length,
>
> - (int)dto_event->user_cookie.as_64,
>
> - sizeof(int), RECV_BUF_INDEX+1);
>
> - exit(1);
>
> - }
>
> -
>
> - /* get immediate data from event */
>
> - immed_data_recv = ext_event->val.immed.data;
>
> - }
>
> - else
>
> - {
>
> - printf("unexpected extension type for event - 0x%x, 0x%x\n",
>
> - event.event_number, ext_event->type);
>
> - exit(1);
>
> - }
>
> -
>
> -
>
> - /*
>
> - * Collect second event, write completion or the inbound recv with immed
>
> - */
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
>
> - if (event.event_number != DAT_IB_DTO_EVENT)
>
> - {
>
> - printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
>
> - event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status");
>
> - if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
>
> - {
>
> - if ((dto_event->transfered_length != BUF_SIZE) ||
>
> - (dto_event->user_cookie.as_64 != 0x9999))
>
> - {
>
> - printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
>
> - (int)dto_event->transfered_length,
>
> - (int)dto_event->user_cookie.as_64);
>
> - exit(1);
>
> - }
>
> - }
>
> - else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
>
> - {
>
> - if ((dto_event->transfered_length != BUF_SIZE) ||
>
> - (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
>
> - {
>
> - printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
>
> - (int)dto_event->transfered_length,
>
> - (int)dto_event->user_cookie.as_64,
>
> - sizeof(int), RECV_BUF_INDEX+1);
>
> - exit(1);
>
> - }
>
> -
>
> - /* get immediate data from event */
>
> - immed_data_recv = ext_event->val.immed.data;
>
> - }
>
> - else
>
> - {
>
> - printf("unexpected extension type for event - 0x%x, 0x%x\n",
>
> - event.event_number, ext_event->type);
>
> - exit(1);
>
> - }
>
> -
>
> - if ((server) && (immed_data_recv != 0x7777))
>
> - {
>
> - printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n",
>
> - 0x7777, immed_data_recv);
>
> - exit(1);
>
> - }
>
> - else if ((!server) && (immed_data_recv != 0x1111))
>
> - {
>
> - printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n",
>
> - 0x1111, immed_data_recv);
>
> - exit(1);
>
> - }
>
> -
>
> - if (server)
>
> - printf("Server received immed_data=0x%x\n", immed_data_recv);
>
> - else
>
> - printf("Client received immed_data=0x%x\n", immed_data_recv);
>
> -
>
> - printf("rdma buffer %p contains: %s\n",
>
> - buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);
>
> -
>
> - printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
>
> - return (0);
>
> -}
>
> -
>
> -int
>
> -do_cmp_swap()
>
> -{
>
> - DAT_DTO_COOKIE cookie;
>
> - DAT_RETURN status;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> - DAT_LMR_TRIPLET l_iov;
>
> - DAT_RMR_TRIPLET r_iov;
>
> - volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
>
> - DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
>
> - &event.event_data.dto_completion_event_data;
>
> - DAT_IB_EXTENSION_EVENT_DATA *ext_event =
>
> - (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
>
> -
>
> - printf("\nDoing CMP and SWAP\n");
>
> -
>
> - r_iov = *buf[ RECV_BUF_INDEX ];
>
> -
>
> - l_iov.lmr_context = lmr_atomic_context;
>
> - l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
>
> - l_iov.segment_length = BUF_SIZE_ATOMIC;
>
> -
>
> - cookie.as_64 = 3333;
>
> - if (server) {
>
> - *target = 0x12345;
>
> - sleep(1);
>
> - /* server does not compare and should not swap */
>
> - status = dat_ib_post_cmp_and_swap( ep,
>
> - (DAT_UINT64)0x654321,
>
> - (DAT_UINT64)0x6789A,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - } else {
>
> - *target = 0x54321;
>
> - sleep(1);
>
> - /* client does compare and should swap */
>
> - status = dat_ib_post_cmp_and_swap( ep,
>
> - (DAT_UINT64)0x12345,
>
> - (DAT_UINT64)0x98765,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - }
>
> - _OK(status, "dat_ib_post_cmp_and_swap");
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait for compare and swap");
>
> - if (event.event_number != DAT_IB_DTO_EVENT) {
>
> - printf("unexpected event after post_cmp_and_swap: 0x%x\n",
>
> - event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status for CMP and SWAP");
>
> - if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
>
> - printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",
>
> - (int)ext_event->type,
>
> - (int)dto_event->user_cookie.as_64,
>
> - *atomic_buf);
>
> - exit(1);
>
> - }
>
> - sleep(1); /* wait for other side to complete swap */
>
> - if (server) {
>
> - printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);
>
> - printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);
>
> -
>
> - if (*atomic_buf != 0x54321 || *target != 0x98765) {
>
> - printf("ERROR: Server CMP_SWAP\n");
>
> - exit(1);
>
> - }
>
> - } else {
>
> - printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);
>
> - printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);
>
> -
>
> - if (*atomic_buf != 0x12345 || *target != 0x54321) {
>
> - printf("ERROR: Client CMP_SWAP\n");
>
> - exit(1);
>
> - }
>
> - }
>
> - printf("\n CMP_SWAP test - PASSED\n");
>
> - return(0);
>
> -}
>
> -
>
> -int
>
> -do_fetch_add()
>
> -{
>
> - DAT_DTO_COOKIE cookie;
>
> - DAT_RETURN status;
>
> - DAT_EVENT event;
>
> - DAT_COUNT nmore;
>
> - DAT_LMR_TRIPLET l_iov;
>
> - DAT_RMR_TRIPLET r_iov;
>
> - volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
>
> - DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
>
> - &event.event_data.dto_completion_event_data;
>
> - DAT_IB_EXTENSION_EVENT_DATA *ext_event =
>
> - (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
>
> -
>
> - printf("\nDoing FETCH and ADD\n");
>
> -
>
> - r_iov = *buf[ RECV_BUF_INDEX ];
>
> -
>
> - l_iov.lmr_context = lmr_atomic_context;
>
> - l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
>
> - l_iov.segment_length = BUF_SIZE_ATOMIC;
>
> -
>
> - cookie.as_64 = 0x7777;
>
> - if (server) {
>
> - /* Wait for client to finish cmp_swap */
>
> - while (*target != 0x98765)
>
> - sleep(1);
>
> - *target = 0x10;
>
> - sleep(1);
>
> - status = dat_ib_post_fetch_and_add( ep,
>
> - (DAT_UINT64)0x100,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - } else {
>
> - /* Wait for server, no swap so nothing to check */
>
> - *target = 0x100;
>
> - sleep(1);
>
> - status = dat_ib_post_fetch_and_add( ep,
>
> - (DAT_UINT64)0x10,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - }
>
> - _OK(status, "dat_ib_post_fetch_and_add");
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait for fetch and add");
>
> - if (event.event_number != DAT_IB_DTO_EVENT) {
>
> - printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status for FETCH and ADD");
>
> - if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
>
> - printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",
>
> - (int)ext_event->type,
>
> - (int)dto_event->user_cookie.as_64,
>
> - (int)*atomic_buf);
>
> - exit(1);
>
> - }
>
> -
>
> - if (server) {
>
> - printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);
>
> - } else {
>
> - printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);
>
> - }
>
> -
>
> - sleep(1);
>
> -
>
> - if (server) {
>
> - status = dat_ib_post_fetch_and_add( ep,
>
> - (DAT_UINT64)0x100,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - } else {
>
> - status = dat_ib_post_fetch_and_add( ep,
>
> - (DAT_UINT64)0x10,
>
> - &l_iov,
>
> - cookie,
>
> - &r_iov,
>
> - DAT_COMPLETION_DEFAULT_FLAG);
>
> - }
>
> -
>
> - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
>
> - _OK(status, "dat_evd_wait for second fetch and add");
>
> - if (event.event_number != DAT_IB_DTO_EVENT) {
>
> - printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number);
>
> - exit(1);
>
> - }
>
> -
>
> - _OK(dto_event->status, "event status for second FETCH and ADD");
>
> - if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
>
> - printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",
>
> - (int)ext_event->type,
>
> - (int)dto_event->user_cookie.as_64,
>
> - (long)atomic_buf);
>
> - exit(1);
>
> - }
>
> -
>
> - sleep(1); /* wait for other side to complete fetch_add */
>
> -
>
> - if (server) {
>
> - printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);
>
> - printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);
>
> -
>
> - if (*atomic_buf != 0x200 || *target != 0x30) {
>
> - printf("ERROR: Server FETCH_ADD\n");
>
> - exit(1);
>
> - }
>
> - } else {
>
> - printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);
>
> - printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);
>
> -
>
> - if (*atomic_buf != 0x20 || *target != 0x300) {
>
> - printf("ERROR: Server FETCH_ADD\n");
>
> - exit(1);
>
> - }
>
> - }
>
> - printf("\n FETCH_ADD test - PASSED\n");
>
> - return(0);
>
> -}
>
> -
>
> -int
>
> -main(int argc, char **argv)
>
> -{
>
> - char *hostname;
>
> -
>
> - if (argc > 2) {
>
> - printf(usage);
>
> - exit(1);
>
> - }
>
> -
>
> - if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)
>
> - {
>
> - server = 1;
>
> - } else {
>
> - server = 0;
>
> - hostname = argv[ 1 ];
>
> - }
>
> -
>
> -
>
> - /*
>
> - * connect
>
> - */
>
> - if (connect_ep(hostname)) {
>
> - exit(1);
>
> - }
>
> - if (do_immediate()) {
>
> - exit(1);
>
> - }
>
> - if (do_cmp_swap()) {
>
> - exit(1);
>
> - }
>
> - if (do_fetch_add()) {
>
> - exit(1);
>
> - }
>
> - return (disconnect_ep());
>
> -}
>
> +
> + if (server) {
> + status = dat_psp_free(psp);
> + _OK(status, "dat_psp_free");
> + }
> +
> + for (i = 0; i < REG_MEM_COUNT; i++) {
> + status = dat_lmr_free(lmr[ i ]);
> + _OK(status, "dat_lmr_free");
> + }
> +
> + status = dat_lmr_free(lmr_atomic);
> + _OK(status, "dat_lmr_free_atomic");
> +
> + status = dat_ep_free(ep);
> + _OK(status, "dat_ep_free");
> +
> + status = dat_evd_free(dto_evd);
> + _OK(status, "dat_evd_free DTO");
> + status = dat_evd_free(con_evd);
> + _OK(status, "dat_evd_free CON");
> + status = dat_evd_free(cr_evd);
> + _OK(status, "dat_evd_free CR");
> +
> + status = dat_pz_free(pz);
> + _OK(status, "dat_pz_free");
> +
> + status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
> + _OK(status, "dat_ia_close");
> +
> + return(0);
> +}
> +
> +int
> +do_immediate()
> +{
> + DAT_REGION_DESCRIPTION region;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> + DAT_LMR_TRIPLET iov;
> + DAT_RMR_TRIPLET r_iov;
> + DAT_DTO_COOKIE cookie;
> + DAT_RMR_CONTEXT their_context;
> + DAT_RETURN status;
> + DAT_UINT32 immed_data;
> + DAT_UINT32 immed_data_recv;
> + DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
> + &event.event_data.dto_completion_event_data;
> + DAT_IB_EXTENSION_EVENT_DATA *ext_event =
> + (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
> +
> + printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
> +
> + if (server) {
> + immed_data = 0x1111;
> + } else {
> + immed_data = 0x7777;
> + }
> +
> + cookie.as_64 = 0x5555;
> +
> + r_iov = *buf[ RECV_BUF_INDEX ];
> +
> + iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];
> + iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];
> + iov.segment_length = BUF_SIZE;
> +
> + cookie.as_64 = 0x9999;
> +
> + status = dat_ib_post_rdma_write_immed(ep, // ep_handle
> + 1, // num_segments
> + &iov, // LMR
> + cookie, // user_cookie
> + &r_iov, // RMR
> + immed_data,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + _OK(status, "dat_ib_post_rdma_write_immed");
> +
> + /*
> + * Collect first event, write completion or the inbound recv with immed
> + */
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
> + if (event.event_number != DAT_IB_DTO_EVENT)
> + {
> + printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
> + event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status");
> + if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
> + {
> + if ((dto_event->transfered_length != BUF_SIZE) ||
> + (dto_event->user_cookie.as_64 != 0x9999))
> + {
> + printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
> + (int)dto_event->transfered_length,
> + (int)dto_event->user_cookie.as_64);
> + exit(1);
> + }
> + }
> + else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
> + {
> + if ((dto_event->transfered_length != BUF_SIZE) ||
> + (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
> + {
> + printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
> + (int)dto_event->transfered_length,
> + (int)dto_event->user_cookie.as_64,
> + sizeof(int), RECV_BUF_INDEX+1);
> + exit(1);
> + }
> +
> + /* get immediate data from event */
> + immed_data_recv = ext_event->val.immed.data;
> + }
> + else
> + {
> + printf("unexpected extension type for event - 0x%x, 0x%x\n",
> + event.event_number, ext_event->type);
> + exit(1);
> + }
> +
> +
> + /*
> + * Collect second event, write completion or the inbound recv with immed
> + */
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
> + if (event.event_number != DAT_IB_DTO_EVENT)
> + {
> + printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
> + event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status");
> + if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
> + {
> + if ((dto_event->transfered_length != BUF_SIZE) ||
> + (dto_event->user_cookie.as_64 != 0x9999))
> + {
> + printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
> + (int)dto_event->transfered_length,
> + (int)dto_event->user_cookie.as_64);
> + exit(1);
> + }
> + }
> + else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
> + {
> + if ((dto_event->transfered_length != BUF_SIZE) ||
> + (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
> + {
> + printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
> + (int)dto_event->transfered_length,
> + (int)dto_event->user_cookie.as_64,
> + sizeof(int), RECV_BUF_INDEX+1);
> + exit(1);
> + }
> +
> + /* get immediate data from event */
> + immed_data_recv = ext_event->val.immed.data;
> + }
> + else
> + {
> + printf("unexpected extension type for event - 0x%x, 0x%x\n",
> + event.event_number, ext_event->type);
> + exit(1);
> + }
> +
> + if ((server) && (immed_data_recv != 0x7777))
> + {
> + printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n",
> + 0x7777, immed_data_recv);
> + exit(1);
> + }
> + else if ((!server) && (immed_data_recv != 0x1111))
> + {
> + printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n",
> + 0x1111, immed_data_recv);
> + exit(1);
> + }
> +
> + if (server)
> + printf("Server received immed_data=0x%x\n", immed_data_recv);
> + else
> + printf("Client received immed_data=0x%x\n", immed_data_recv);
> +
> + printf("rdma buffer %p contains: %s\n",
> + buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);
> +
> + printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
> + return (0);
> +}
> +
> +int
> +do_cmp_swap()
> +{
> + DAT_DTO_COOKIE cookie;
> + DAT_RETURN status;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> + DAT_LMR_TRIPLET l_iov;
> + DAT_RMR_TRIPLET r_iov;
> + volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
> + DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
> + &event.event_data.dto_completion_event_data;
> + DAT_IB_EXTENSION_EVENT_DATA *ext_event =
> + (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
> +
> + printf("\nDoing CMP and SWAP\n");
> +
> + r_iov = *buf[ RECV_BUF_INDEX ];
> +
> + l_iov.lmr_context = lmr_atomic_context;
> + l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
> + l_iov.segment_length = BUF_SIZE_ATOMIC;
> +
> + cookie.as_64 = 3333;
> + if (server) {
> + *target = 0x12345;
> + sleep(1);
> + /* server does not compare and should not swap */
> + status = dat_ib_post_cmp_and_swap( ep,
> + (DAT_UINT64)0x654321,
> + (DAT_UINT64)0x6789A,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + } else {
> + *target = 0x54321;
> + sleep(1);
> + /* client does compare and should swap */
> + status = dat_ib_post_cmp_and_swap( ep,
> + (DAT_UINT64)0x12345,
> + (DAT_UINT64)0x98765,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + }
> + _OK(status, "dat_ib_post_cmp_and_swap");
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait for compare and swap");
> + if (event.event_number != DAT_IB_DTO_EVENT) {
> + printf("unexpected event after post_cmp_and_swap: 0x%x\n",
> + event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status for CMP and SWAP");
> + if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
> + printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",
> + (int)ext_event->type,
> + (int)dto_event->user_cookie.as_64,
> + *atomic_buf);
> + exit(1);
> + }
> + sleep(1); /* wait for other side to complete swap */
> + if (server) {
> + printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);
> + printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);
> +
> + if (*atomic_buf != 0x54321 || *target != 0x98765) {
> + printf("ERROR: Server CMP_SWAP\n");
> + exit(1);
> + }
> + } else {
> + printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);
> + printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);
> +
> + if (*atomic_buf != 0x12345 || *target != 0x54321) {
> + printf("ERROR: Client CMP_SWAP\n");
> + exit(1);
> + }
> + }
> + printf("\n CMP_SWAP test - PASSED\n");
> + return(0);
> +}
> +
> +int
> +do_fetch_add()
> +{
> + DAT_DTO_COOKIE cookie;
> + DAT_RETURN status;
> + DAT_EVENT event;
> + DAT_COUNT nmore;
> + DAT_LMR_TRIPLET l_iov;
> + DAT_RMR_TRIPLET r_iov;
> + volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
> + DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
> + &event.event_data.dto_completion_event_data;
> + DAT_IB_EXTENSION_EVENT_DATA *ext_event =
> + (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
> +
> + printf("\nDoing FETCH and ADD\n");
> +
> + r_iov = *buf[ RECV_BUF_INDEX ];
> +
> + l_iov.lmr_context = lmr_atomic_context;
> + l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
> + l_iov.segment_length = BUF_SIZE_ATOMIC;
> +
> + cookie.as_64 = 0x7777;
> + if (server) {
> + /* Wait for client to finish cmp_swap */
> + while (*target != 0x98765)
> + sleep(1);
> + *target = 0x10;
> + sleep(1);
> + status = dat_ib_post_fetch_and_add( ep,
> + (DAT_UINT64)0x100,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + } else {
> + /* Wait for server, no swap so nothing to check */
> + *target = 0x100;
> + sleep(1);
> + status = dat_ib_post_fetch_and_add( ep,
> + (DAT_UINT64)0x10,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + }
> + _OK(status, "dat_ib_post_fetch_and_add");
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait for fetch and add");
> + if (event.event_number != DAT_IB_DTO_EVENT) {
> + printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status for FETCH and ADD");
> + if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
> + printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",
> + (int)ext_event->type,
> + (int)dto_event->user_cookie.as_64,
> + (int)*atomic_buf);
> + exit(1);
> + }
> +
> + if (server) {
> + printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);
> + } else {
> + printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);
> + }
> +
> + sleep(1);
> +
> + if (server) {
> + status = dat_ib_post_fetch_and_add( ep,
> + (DAT_UINT64)0x100,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + } else {
> + status = dat_ib_post_fetch_and_add( ep,
> + (DAT_UINT64)0x10,
> + &l_iov,
> + cookie,
> + &r_iov,
> + DAT_COMPLETION_DEFAULT_FLAG);
> + }
> +
> + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
> + _OK(status, "dat_evd_wait for second fetch and add");
> + if (event.event_number != DAT_IB_DTO_EVENT) {
> + printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number);
> + exit(1);
> + }
> +
> + _OK(dto_event->status, "event status for second FETCH and ADD");
> + if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
> + printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",
> + (int)ext_event->type,
> + (int)dto_event->user_cookie.as_64,
> + (long)atomic_buf);
> + exit(1);
> + }
> +
> + sleep(1); /* wait for other side to complete fetch_add */
> +
> + if (server) {
> + printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);
> + printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);
> +
> + if (*atomic_buf != 0x200 || *target != 0x30) {
> + printf("ERROR: Server FETCH_ADD\n");
> + exit(1);
> + }
> + } else {
> + printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);
> + printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);
> +
> + if (*atomic_buf != 0x20 || *target != 0x300) {
> + printf("ERROR: Server FETCH_ADD\n");
> + exit(1);
> + }
> + }
> + printf("\n FETCH_ADD test - PASSED\n");
> + return(0);
> +}
> +
> +int
> +main(int argc, char **argv)
> +{
> + char *hostname;
> +
> + if (argc > 2) {
> + printf(usage);
> + exit(1);
> + }
> +
> + if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)
> + {
> + server = 1;
> + } else {
> + server = 0;
> + hostname = argv[ 1 ];
> + }
> +
> +
> + /*
> + * connect
> + */
> + if (connect_ep(hostname)) {
> + exit(1);
> + }
> + if (do_immediate()) {
> + exit(1);
> + }
> + if (do_cmp_swap()) {
> + exit(1);
> + }
> + if (do_fetch_add()) {
> + exit(1);
> + }
> + return (disconnect_ep());
> +}
>
>
>
>
>
>
>
>
>
More information about the general
mailing list