[ofa-general] [PATCH] [DAPL] scm: add support for WinOF

Davis, Arlin R arlin.r.davis at intel.com
Tue Feb 17 09:06:18 PST 2009


Thanks, applied.

Since this was in the CM code I had to do some regression testing before accepting.


>-----Original Message-----
>From: general-bounces at lists.openfabrics.org
>[mailto:general-bounces at lists.openfabrics.org] On Behalf Of Hefty, Sean
>Sent: Friday, February 13, 2009 2:55 PM
>To: Davis, Arlin R; general at lists.openfabrics.org;
>ofw at lists.openfabrics.org
>Subject: [ofa-general] [PATCH] [DAPL] scm: add support for WinOF
>
>Modify the openib_scm provider to support both OFED and WinOF releases.
>This takes advantage of having a libibverbs compatibility library.*
>
>Signed-off-by: Sean Hefty <sean.hefty at intel.com>
>---
>* If only there were a sockets compatility layer... gurgle
>This is only build tested for windows, but does run on Linux.
>
>diff --git a/Makefile.am b/Makefile.am
>index bfc93f7..5044e36 100755
>--- a/Makefile.am
>+++ b/Makefile.am
>@@ -49,7 +49,8 @@ dapl_udapl_libdaploscm_la_CFLAGS =
>$(AM_CFLAGS) -D_GNU_SOURCE $(OSFLAGS) $(XFLAG
>                                 -DOPENIB -DCQ_WAIT_OBJECT \
>                                 -I$(srcdir)/dat/include/
>-I$(srcdir)/dapl/include/ \
>                                 -I$(srcdir)/dapl/common
>-I$(srcdir)/dapl/udapl/linux \
>-                                -I$(srcdir)/dapl/openib_scm
>+                                -I$(srcdir)/dapl/openib_scm \
>+                              -I$(srcdir)/dapl/openib_scm/linux
>
> if HAVE_LD_VERSION_SCRIPT
>     dat_version_script =
>-Wl,--version-script=$(srcdir)/dat/udat/libdat2.map
>diff --git a/dapl/openib_scm/README b/dapl/openib_scm/README
>deleted file mode 100644
>index 239dfe6..0000000
>--- a/dapl/openib_scm/README
>+++ /dev/null
>@@ -1,40 +0,0 @@
>-
>-OpenIB uDAPL provider using socket-based CM, in leiu of
>uCM/uAT, to setup QP/channels.
>-
>-to build:
>-
>-cd dapl/udapl
>-make VERBS=openib_scm clean
>-make VERBS=openib_scm
>-
>-
>-Modifications to common code:
>-
>-- added dapl/openib_scm directory
>-
>-      dapl/udapl/Makefile
>-
>-New files for openib_scm provider
>-
>-      dapl/openib/dapl_ib_cq.c
>-      dapl/openib/dapl_ib_dto.h
>-      dapl/openib/dapl_ib_mem.c
>-      dapl/openib/dapl_ib_qp.c
>-      dapl/openib/dapl_ib_util.c
>-      dapl/openib/dapl_ib_util.h
>-      dapl/openib/dapl_ib_cm.c
>-
>-A simple dapl test just for openib_scm testing...
>-
>-      test/dtest/dtest.c
>-      test/dtest/makefile
>-
>-      server: dtest -s
>-      client: dtest -h hostname
>-
>-known issues:
>-
>-      no memory windows support in ibverbs, dat_create_rmr fails.
>-
>-
>-
>diff --git a/dapl/openib_scm/dapl_ib_cm.c
>b/dapl/openib_scm/dapl_ib_cm.c
>index 80a7d5e..9a15e42 100644
>--- a/dapl/openib_scm/dapl_ib_cm.c
>+++ b/dapl/openib_scm/dapl_ib_cm.c
>@@ -52,26 +52,169 @@
> #include "dapl_cr_util.h"
> #include "dapl_name_service.h"
> #include "dapl_ib_util.h"
>-
>-#include <stdio.h>
>-#include <unistd.h>
>-#include <fcntl.h>
>-#include <netinet/tcp.h>
>-#include <byteswap.h>
>-#include <poll.h>
>-
>-#include <sys/socket.h>
>-#include <netinet/in.h>
>-#include <arpa/inet.h>
>-
>-#if __BYTE_ORDER == __LITTLE_ENDIAN
>-static inline uint64_t cpu_to_be64(uint64_t x) {return bswap_64(x);}
>-#elif __BYTE_ORDER == __BIG_ENDIAN
>-static inline uint64_t cpu_to_be64(uint64_t x) {return x;}
>-#endif
>+#include "dapl_osd.h"
>
> extern int g_scm_pipe[2];
>
>+#if defined(_WIN32) || defined(_WIN64)
>+enum DAPL_FD_EVENTS {
>+      DAPL_FD_READ    = 0x1,
>+      DAPL_FD_WRITE   = 0x2,
>+      DAPL_FD_ERROR   = 0x4
>+};
>+
>+static int dapl_config_socket(DAPL_SOCKET s)
>+{
>+      unsigned long nonblocking = 1;
>+      return ioctlsocket(s, FIONBIO, &nonblocking);
>+}
>+
>+static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr *addr,
>+                             int addrlen)
>+{
>+      int err;
>+
>+      connect(s, addr, addrlen);
>+      err = WSAGetLastError();
>+      return (err == WSAEWOULDBLOCK) ? EAGAIN : err;
>+}
>+
>+struct dapl_fd_set {
>+      struct fd_set set[3];
>+};
>+
>+static struct dapl_fd_set *dapl_alloc_fd_set(void)
>+{
>+      return dapl_os_alloc(sizeof(struct dapl_fd_set));
>+}
>+
>+static void dapl_fd_zero(struct dapl_fd_set *set)
>+{
>+      FD_ZERO(&set->set[0]);
>+      FD_ZERO(&set->set[1]);
>+      FD_ZERO(&set->set[2]);
>+}
>+
>+static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set,
>+                      enum DAPL_FD_EVENTS event)
>+{
>+      FD_SET(s, &set->set[(event == DAPL_FD_READ) ? 0 : 1]);
>+      FD_SET(s, &set->set[2]);
>+      return 0;
>+}
>+
>+static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum
>DAPL_FD_EVENTS event)
>+{
>+      struct fd_set rw_fds;
>+      struct fd_set err_fds;
>+      struct timeval tv;
>+      int ret;
>+
>+      FD_ZERO(&rw_fds);
>+      FD_ZERO(&err_fds);
>+      FD_SET(s, &rw_fds);
>+      FD_SET(s, &err_fds);
>+
>+      tv.tv_sec = 0;
>+      tv.tv_usec = 0;
>+
>+      if (event == DAPL_FD_READ)
>+              ret = select(1, &rw_fds, NULL, &err_fds, &tv);
>+      else
>+              ret = select(1, NULL, &rw_fds, &err_fds, &tv);
>+
>+      if (ret == 0)
>+              return 0;
>+      else if (FD_ISSET(s, &rw_fds))
>+              return event;
>+      else if (FD_ISSET(s, &err_fds))
>+              return DAPL_FD_ERROR;
>+      else
>+              return WSAGetLastError();
>+}
>+
>+static int dapl_select(struct dapl_fd_set *set)
>+{
>+      return select(0, &set->set[0], &set->set[1],
>&set->set[2], NULL);
>+}
>+#else // _WIN32 || _WIN64
>+enum DAPL_FD_EVENTS {
>+      DAPL_FD_READ    = POLLIN,
>+      DAPL_FD_WRITE   = POLLOUT,
>+      DAPL_FD_ERROR   = POLLERR
>+};
>+
>+static int dapl_config_socket(DAPL_SOCKET s)
>+{
>+      int ret;
>+
>+      ret = fcntl(s, F_GETFL);
>+      if (ret >= 0)
>+              ret = fcntl(s, F_SETFL, ret | O_NONBLOCK);
>+      return ret;
>+}
>+
>+static int dapl_connect_socket(DAPL_SOCKET s, struct sockaddr
>*addr, int addrlen)
>+{
>+      int ret;
>+
>+      ret = connect(s, addr, addrlen);
>+
>+      return (errno == EINPROGRESS) ? EAGAIN : ret;
>+}
>+
>+struct dapl_fd_set {
>+      int index;
>+      struct pollfd set[DAPL_FD_SETSIZE];
>+};
>+
>+static struct dapl_fd_set *dapl_alloc_fd_set(void)
>+{
>+      return dapl_os_alloc(sizeof(struct dapl_fd_set));
>+}
>+
>+static void dapl_fd_zero(struct dapl_fd_set *set)
>+{
>+      set->index = 0;
>+}
>+
>+static int dapl_fd_set(DAPL_SOCKET s, struct dapl_fd_set *set,
>+                      enum DAPL_FD_EVENTS event)
>+{
>+      if (set->index == DAPL_FD_SETSIZE - 1) {
>+              dapl_log(DAPL_DBG_TYPE_ERR,
>+                       "SCM ERR: cm_thread exceeded FD_SETSIZE %d\n",
>+                       set->index + 1);
>+              return -1;
>+      }
>+
>+      set->set[set->index].fd = s;
>+      set->set[set->index].revents = 0;
>+      set->set[set->index++].events = event;
>+      return 0;
>+}
>+
>+static enum DAPL_FD_EVENTS dapl_poll(DAPL_SOCKET s, enum
>DAPL_FD_EVENTS event)
>+{
>+      struct pollfd fds;
>+      int ret;
>+
>+      fds.fd = s;
>+      fds.events = event;
>+      fds.revents = 0;
>+      ret = poll(&fds, 1, 0);
>+      if (ret <= 0)
>+              return ret;
>+
>+      return fds.revents;
>+}
>+
>+static int dapl_select(struct dapl_fd_set *set)
>+{
>+      return poll(set->set, set->index, -1);
>+}
>+#endif
>+
> static struct ib_cm_handle *dapli_cm_create(void)
> {
>       struct ib_cm_handle *cm_ptr;
>@@ -85,7 +228,7 @@ static struct ib_cm_handle *dapli_cm_create(void)
>
>       (void)dapl_os_memzero(cm_ptr, sizeof(*cm_ptr));
>       cm_ptr->dst.ver = htons(DSCM_VER);
>-      cm_ptr->socket = -1;
>+      cm_ptr->socket = DAPL_INVALID_SOCKET;
>       return cm_ptr;
> bail:
>       dapl_os_free(cm_ptr, sizeof(*cm_ptr));
>@@ -100,8 +243,8 @@ static void dapli_cm_destroy(struct
>ib_cm_handle *cm_ptr)
>
>       /* cleanup, never made it to work queue */
>       if (cm_ptr->state == SCM_INIT) {
>-              if (cm_ptr->socket >= 0)
>-                      close(cm_ptr->socket);
>+              if (cm_ptr->socket != DAPL_INVALID_SOCKET)
>+                      closesocket(cm_ptr->socket);
>               dapl_os_free(cm_ptr, sizeof(*cm_ptr));
>               return;
>       }
>@@ -112,9 +255,9 @@ static void dapli_cm_destroy(struct
>ib_cm_handle *cm_ptr)
>               cm_ptr->ep->cm_handle = IB_INVALID_HANDLE;
>
>       /* close socket if still active */
>-      if (cm_ptr->socket >= 0) {
>-              close(cm_ptr->socket);
>-              cm_ptr->socket = -1;
>+      if (cm_ptr->socket != DAPL_INVALID_SOCKET) {
>+              closesocket(cm_ptr->socket);
>+              cm_ptr->socket = DAPL_INVALID_SOCKET;
>       }
>       dapl_os_unlock(&cm_ptr->lock);
>
>@@ -172,14 +315,14 @@
>dapli_socket_disconnect(dp_ib_cm_handle_t      cm_ptr)
>               return DAT_SUCCESS;
>       } else {
>               /* send disc date, close socket, schedule destroy */
>-              if (cm_ptr->socket >= 0) {
>-                      if (write(cm_ptr->socket,
>-                                &disc_data, sizeof(disc_data)) == -1)
>+              if (cm_ptr->socket != DAPL_INVALID_SOCKET) {
>+                      if (send(cm_ptr->socket, (char *) &disc_data,
>+                                      sizeof(disc_data), 0) == -1)
>                               dapl_log(DAPL_DBG_TYPE_WARN,
>                                        " cm_disc: write error
>= %s\n",
>                                        strerror(errno));
>-                      close(cm_ptr->socket);
>-                      cm_ptr->socket = -1;
>+                      closesocket(cm_ptr->socket);
>+                      cm_ptr->socket = DAPL_INVALID_SOCKET;
>               }
>               cm_ptr->state = SCM_DISCONNECTED;
>       }
>@@ -211,7 +354,7 @@ void
> dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
> {
>       int             len, opt = 1;
>-      struct iovec    iovec[2];
>+      struct iovec iov[2];
>       struct dapl_ep  *ep_ptr = cm_ptr->ep;
>
>       if (err) {
>@@ -226,18 +369,21 @@ dapli_socket_connected(dp_ib_cm_handle_t
>cm_ptr, int err)
>                    " socket connected, write QP and private data\n");
>
>       /* no delay for small packets */
>-
>setsockopt(cm_ptr->socket,IPPROTO_TCP,TCP_NODELAY,&opt,sizeof(opt));
>+      setsockopt(cm_ptr->socket, IPPROTO_TCP, TCP_NODELAY,
>+              (char *) &opt, sizeof(opt));
>
>       /* send qp info and pdata to remote peer */
>-      iovec[0].iov_base = &cm_ptr->dst;
>-      iovec[0].iov_len  = sizeof(ib_qp_cm_t);
>+      iov[0].iov_base = (void *) &cm_ptr->dst;
>+      iov[0].iov_len = sizeof(ib_qp_cm_t);
>       if (cm_ptr->dst.p_size) {
>-              iovec[1].iov_base = cm_ptr->p_data;
>-              iovec[1].iov_len  = ntohl(cm_ptr->dst.p_size);
>+              iov[1].iov_base = cm_ptr->p_data;
>+              iov[1].iov_len = ntohl(cm_ptr->dst.p_size);
>+              len = writev(cm_ptr->socket, iov, 2);
>+      } else {
>+              len = writev(cm_ptr->socket, iov, 1);
>       }
>
>-      len = writev(cm_ptr->socket, iovec, (cm_ptr->dst.p_size ? 2:1));
>-      if (len != (ntohl(cm_ptr->dst.p_size) + sizeof(ib_qp_cm_t))) {
>+      if (len != (ntohl(cm_ptr->dst.p_size) + sizeof(ib_qp_cm_t))) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " CONN_PENDING write: ERR %s, wcnt=%d -> %s\n",
>                        strerror(errno), len,
>@@ -253,9 +399,9 @@ dapli_socket_connected(dp_ib_cm_handle_t
>cm_ptr, int err)
>         dapl_dbg_log(DAPL_DBG_TYPE_CM,
>                      " connected: sending SRC GID subnet
>%016llx id %016llx\n",
>                      (unsigned long long)
>-
>cpu_to_be64(cm_ptr->dst.gid.global.subnet_prefix),
>+                      htonll(cm_ptr->dst.gid.global.subnet_prefix),
>                      (unsigned long long)
>-
>cpu_to_be64(cm_ptr->dst.gid.global.interface_id));
>+                      htonll(cm_ptr->dst.gid.global.interface_id));
>
>       /* queue up to work thread to avoid blocking consumer */
>       cm_ptr->state = SCM_RTU_PENDING;
>@@ -290,25 +436,23 @@ dapli_socket_connect(DAPL_EP             *ep_ptr,
>               return DAT_INSUFFICIENT_RESOURCES;
>
>       /* create, connect, sockopt, and exchange QP information */
>-      if ((cm_ptr->socket = socket(AF_INET,SOCK_STREAM,0)) < 0 ) {
>+      if ((cm_ptr->socket = socket(AF_INET,SOCK_STREAM,0)) ==
>DAPL_INVALID_SOCKET) {
>               dapl_os_free( cm_ptr, sizeof( *cm_ptr ) );
>               return DAT_INSUFFICIENT_RESOURCES;
>       }
>
>-      /* non-blocking */
>-      ret = fcntl(cm_ptr->socket, F_GETFL);
>-        if (ret < 0 || fcntl(cm_ptr->socket,
>-                              F_SETFL, ret | O_NONBLOCK) < 0) {
>-                dapl_log(DAPL_DBG_TYPE_ERR,
>-                         " socket connect: fcntl on socket %d
>ERR %d %s\n",
>-                         cm_ptr->socket, ret,
>-                         strerror(errno));
>-                goto bail;
>-        }
>+      ret = dapl_config_socket(cm_ptr->socket);
>+      if (ret < 0) {
>+              dapl_log(DAPL_DBG_TYPE_ERR,
>+                      " socket connect: config socket %d ERR %d %s\n",
>+                      cm_ptr->socket, ret, strerror(errno));
>+              goto bail;
>+      }
>
>       ((struct sockaddr_in*)r_addr)->sin_port = htons(r_qual);
>-      ret = connect(cm_ptr->socket, r_addr, sizeof(*r_addr));
>-      if (ret && errno != EINPROGRESS) {
>+      ret = dapl_connect_socket(cm_ptr->socket, (struct
>sockaddr *) r_addr,
>+                              sizeof(*r_addr));
>+      if (ret && ret != EAGAIN) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " socket connect ERROR: %s -> %s r_qual %d\n",
>                        strerror(errno),
>@@ -391,16 +535,13 @@
>dapli_socket_connect_rtu(dp_ib_cm_handle_t     cm_ptr)
> {
>       DAPL_EP         *ep_ptr = cm_ptr->ep;
>       int             len;
>-      struct iovec    iovec[2];
>       short           rtu_data = htons(0x0E0F);
>       ib_cm_events_t  event = IB_CME_DESTINATION_REJECT;
>
>       /* read DST information into cm_ptr, overwrite SRC info */
>       dapl_dbg_log(DAPL_DBG_TYPE_EP," connect_rtu: recv peer
>QP data\n");
>
>-      iovec[0].iov_base = &cm_ptr->dst;
>-      iovec[0].iov_len  = sizeof(ib_qp_cm_t);
>-      len = readv(cm_ptr->socket, iovec, 1);
>+      len = recv(cm_ptr->socket, (char *) &cm_ptr->dst,
>sizeof(ib_qp_cm_t), 0);
>       if (len != sizeof(ib_qp_cm_t) || ntohs(cm_ptr->dst.ver)
>!= DSCM_VER) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                    " CONN_RTU read: ERR %s, rcnt=%d, ver=%d -> %s\n",
>@@ -456,9 +597,7 @@ dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
>       /* read private data into cm_handle if any present */
>       dapl_dbg_log(DAPL_DBG_TYPE_EP," socket connected, read
>private data\n");
>       if (cm_ptr->dst.p_size) {
>-              iovec[0].iov_base = cm_ptr->p_data;
>-              iovec[0].iov_len  = cm_ptr->dst.p_size;
>-              len = readv(cm_ptr->socket, iovec, 1);
>+              len = recv(cm_ptr->socket, cm_ptr->p_data,
>cm_ptr->dst.p_size, 0);
>               if (len != cm_ptr->dst.p_size) {
>                       dapl_log(DAPL_DBG_TYPE_ERR,
>                           " CONN_RTU read pdata: ERR %s,
>rcnt=%d -> %s\n",
>@@ -495,7 +634,7 @@ dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
>       dapl_dbg_log(DAPL_DBG_TYPE_EP," connect_rtu: send RTU\n");
>
>       /* complete handshake after final QP state change */
>-      if (write(cm_ptr->socket, &rtu_data, sizeof(rtu_data)) == -1) {
>+      if (send(cm_ptr->socket, (char *) &rtu_data,
>sizeof(rtu_data), 0) == -1) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " CONN_RTU: write error = %s\n",
>strerror(errno));
>               goto bail;
>@@ -564,7 +703,7 @@ dapli_socket_listen(DAPL_IA                *ia_ptr,
>       cm_ptr->hca = ia_ptr->hca_ptr;
>
>       /* bind, listen, set sockopt, accept, exchange data */
>-      if ((cm_ptr->socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
>+      if ((cm_ptr->socket = socket(AF_INET, SOCK_STREAM, 0))
>== DAPL_INVALID_SOCKET) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " ERR: listen socket create: %s\n",
>                        strerror(errno));
>@@ -572,7 +711,8 @@ dapli_socket_listen(DAPL_IA                *ia_ptr,
>               goto bail;
>       }
>
>-
>setsockopt(cm_ptr->socket,SOL_SOCKET,SO_REUSEADDR,&opt,sizeof(opt));
>+      setsockopt(cm_ptr->socket, SOL_SOCKET, SO_REUSEADDR,
>+              (char *) &opt, sizeof(opt));
>       addr.sin_port        = htons(serviceID);
>       addr.sin_family      = AF_INET;
>       addr.sin_addr.s_addr = INADDR_ANY;
>@@ -625,7 +765,7 @@ dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
>
>       (void) dapl_os_memzero(acm_ptr, sizeof(*acm_ptr));
>
>-      acm_ptr->socket = -1;
>+      acm_ptr->socket = DAPL_INVALID_SOCKET;
>       acm_ptr->sp = cm_ptr->sp;
>       acm_ptr->hca = cm_ptr->hca;
>
>@@ -633,7 +773,7 @@ dapli_socket_accept(ib_cm_srvc_handle_t cm_ptr)
>       acm_ptr->socket = accept(cm_ptr->socket,
>                               (struct
>sockaddr*)&acm_ptr->dst.ia_address,
>                               (socklen_t*)&len);
>-      if (acm_ptr->socket < 0) {
>+      if (acm_ptr->socket == DAPL_INVALID_SOCKET) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                       " accept: ERR %s on FD %d l_cr %p\n",
>                       strerror(errno),cm_ptr->socket,cm_ptr);
>@@ -664,7 +804,7 @@
>dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr)
>       dapl_dbg_log(DAPL_DBG_TYPE_EP," socket accepted, read
>QP data\n");
>
>       /* read in DST QP info, IA address. check for private data */
>-      len = read(acm_ptr->socket, &acm_ptr->dst, sizeof(ib_qp_cm_t));
>+      len = recv(acm_ptr->socket, (char *) &acm_ptr->dst,
>sizeof(ib_qp_cm_t), 0);
>       if (len != sizeof(ib_qp_cm_t) ||
>           ntohs(acm_ptr->dst.ver) != DSCM_VER) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>@@ -700,8 +840,7 @@
>dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr)
>
>       /* read private data into cm_handle if any present */
>       if (acm_ptr->dst.p_size) {
>-              len = read( acm_ptr->socket,
>-                          acm_ptr->p_data, acm_ptr->dst.p_size);
>+              len = recv(acm_ptr->socket, acm_ptr->p_data,
>acm_ptr->dst.p_size, 0);
>               if (len != acm_ptr->dst.p_size) {
>                       dapl_log(DAPL_DBG_TYPE_ERR,
>                                    " accept read pdata: ERR
>%s, rcnt=%d\n",
>@@ -757,14 +896,14 @@ dapli_socket_accept_usr(DAPL_EP          *ep_ptr,
>       DAPL_IA         *ia_ptr = ep_ptr->header.owner_ia;
>       dp_ib_cm_handle_t  cm_ptr = cr_ptr->ib_cm_handle;
>       ib_qp_cm_t      local;
>-      struct iovec    iovec[2];
>+      struct iovec    iov[2];
>       int             len;
>
>       if (p_size > IB_MAX_REP_PDATA_SIZE)
>               return DAT_LENGTH_ERROR;
>
>       /* must have a accepted socket */
>-      if (cm_ptr->socket < 0)
>+      if (cm_ptr->socket == DAPL_INVALID_SOCKET)
>               return DAT_INTERNAL_ERROR;
>
>       dapl_dbg_log(DAPL_DBG_TYPE_EP,
>@@ -844,14 +983,17 @@ dapli_socket_accept_usr(DAPL_EP          *ep_ptr,
>
>       local.ia_address = ia_ptr->hca_ptr->hca_address;
>       local.p_size = htonl(p_size);
>-      iovec[0].iov_base = &local;
>-      iovec[0].iov_len  = sizeof(ib_qp_cm_t);
>+      iov[0].iov_base = (void *) &local;
>+      iov[0].iov_len = sizeof(ib_qp_cm_t);
>       if (p_size) {
>-              iovec[1].iov_base = p_data;
>-              iovec[1].iov_len  = p_size;
>+              iov[1].iov_base = p_data;
>+              iov[1].iov_len = p_size;
>+              len = writev(cm_ptr->socket, iov, 2);
>+      } else {
>+              len = writev(cm_ptr->socket, iov, 1);
>       }
>-      len = writev(cm_ptr->socket, iovec, (p_size ? 2:1));
>-      if (len != (p_size + sizeof(ib_qp_cm_t))) {
>+
>+      if (len != (p_size + sizeof(ib_qp_cm_t))) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " ACCEPT_USR: ERR %s, wcnt=%d -> %s\n",
>                        strerror(errno), len,
>@@ -859,6 +1001,7 @@ dapli_socket_accept_usr(DAPL_EP           *ep_ptr,
>                            &cm_ptr->dst.ia_address)->sin_addr));
>               goto bail;
>       }
>+
>       dapl_dbg_log(DAPL_DBG_TYPE_CM,
>                    " ACCEPT_USR: local port=0x%x lid=0x%x"
>                    " qpn=0x%x psize=%d\n",
>@@ -867,9 +1010,9 @@ dapli_socket_accept_usr(DAPL_EP           *ep_ptr,
>         dapl_dbg_log(DAPL_DBG_TYPE_CM,
>                      " ACCEPT_USR SRC GID subnet %016llx id
>%016llx\n",
>                      (unsigned long long)
>-                      cpu_to_be64(local.gid.global.subnet_prefix),
>+                      htonll(local.gid.global.subnet_prefix),
>                      (unsigned long long)
>-                      cpu_to_be64(local.gid.global.interface_id));
>+                      htonll(local.gid.global.interface_id));
>
>       /* save state and reference to EP, queue for RTU data */
>       cm_ptr->ep = ep_ptr;
>@@ -894,7 +1037,7 @@ dapli_socket_accept_rtu(dp_ib_cm_handle_t cm_ptr)
>       short           rtu_data = 0;
>
>       /* complete handshake after final QP state change */
>-      len = read(cm_ptr->socket, &rtu_data, sizeof(rtu_data));
>+      len = recv(cm_ptr->socket, (char *) &rtu_data,
>sizeof(rtu_data), 0);
>       if (len != sizeof(rtu_data) || ntohs(rtu_data) != 0x0e0f) {
>               dapl_log(DAPL_DBG_TYPE_ERR,
>                        " ACCEPT_RTU: ERR %s, rcnt=%d rdata=%x\n",
>@@ -1108,9 +1251,9 @@ dapls_ib_remove_conn_listener (
>
>       /* close accepted socket, free cm_srvc_handle and return */
>       if (cm_ptr != NULL) {
>-              if (cm_ptr->socket >= 0) {
>-                      close(cm_ptr->socket );
>-                      cm_ptr->socket = -1;
>+              if (cm_ptr->socket != DAPL_INVALID_SOCKET) {
>+                      closesocket(cm_ptr->socket);
>+                      cm_ptr->socket = DAPL_INVALID_SOCKET;
>               }
>               /* cr_thread will free */
>               cm_ptr->state = SCM_DESTROY;
>@@ -1195,27 +1338,29 @@ dapls_ib_reject_connection(
>       IN DAT_COUNT psize,
>       IN const DAT_PVOID pdata)
> {
>-      struct iovec iovec[2];
>+      struct iovec iov[2];
>
>       dapl_dbg_log (DAPL_DBG_TYPE_EP,
>                     " reject(cm %p reason %x, pdata %p, psize %d)\n",
>                     cm_ptr, reason, pdata, psize);
>
>       /* write reject data to indicate reject */
>-      if (cm_ptr->socket >= 0) {
>+      if (cm_ptr->socket != DAPL_INVALID_SOCKET) {
>               cm_ptr->dst.rej = (uint16_t)reason;
>               cm_ptr->dst.rej = htons(cm_ptr->dst.rej);
>-              iovec[0].iov_base = &cm_ptr->dst;
>-              iovec[0].iov_len  = sizeof(ib_qp_cm_t);
>+
>+              iov[0].iov_base = (void *) &cm_ptr->dst;
>+              iov[0].iov_len = sizeof(ib_qp_cm_t);
>               if (psize) {
>-                      iovec[1].iov_base = pdata;
>-                      iovec[2].iov_len = psize;
>-                      writev(cm_ptr->socket, &iovec[0], 2);
>-              } else
>-                      writev(cm_ptr->socket, &iovec[0], 1);
>-
>-              close(cm_ptr->socket);
>-              cm_ptr->socket = -1;
>+                      iov[1].iov_base = pdata;
>+                      iov[1].iov_len = psize;
>+                      writev(cm_ptr->socket, iov, 2);
>+              } else {
>+                      writev(cm_ptr->socket, iov, 1);
>+              }
>+
>+              closesocket(cm_ptr->socket);
>+              cm_ptr->socket = DAPL_INVALID_SOCKET;
>       }
>
>       /* cr_thread will destroy CR */
>@@ -1444,138 +1589,141 @@ dapls_ib_get_cm_event (
> }
>
> /* outbound/inbound CR processing thread to avoid blocking
>applications */
>-#define SCM_MAX_CONN 8192
> void cr_thread(void *arg)
> {
>-    struct dapl_hca   *hca_ptr = arg;
>-    dp_ib_cm_handle_t cr, next_cr;
>-    int               opt,ret,idx;
>-    socklen_t         opt_len;
>-    char              rbuf[2];
>-    struct pollfd     ufds[SCM_MAX_CONN];
>-
>-    dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread: ENTER hca
>%p\n",hca_ptr);
>-
>-    dapl_os_lock( &hca_ptr->ib_trans.lock );
>-    hca_ptr->ib_trans.cr_state = IB_THREAD_RUN;
>-    while (hca_ptr->ib_trans.cr_state == IB_THREAD_RUN) {
>-      idx=0;
>-      ufds[idx].fd = g_scm_pipe[0]; /* wakeup and process work */
>-        ufds[idx].events = POLLIN;
>-      ufds[idx].revents = 0;
>-
>-      if (!dapl_llist_is_empty(&hca_ptr->ib_trans.list))
>-            next_cr = dapl_llist_peek_head (&hca_ptr->ib_trans.list);
>-      else
>-          next_cr = NULL;
>-
>-      while (next_cr) {
>-          cr = next_cr;
>-          if ((cr->socket == -1 && cr->state == SCM_DESTROY) ||
>-              hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
>-
>-              dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread: Free
>%p\n", cr);
>-              next_cr = dapl_llist_next_entry(&hca_ptr->ib_trans.list,
>-
>(DAPL_LLIST_ENTRY*)&cr->entry );
>-              dapl_llist_remove_entry(&hca_ptr->ib_trans.list,
>-                                      (DAPL_LLIST_ENTRY*)&cr->entry);
>-              dapl_os_free(cr, sizeof(*cr));
>-              continue;
>-          }
>-
>-          if (idx==SCM_MAX_CONN-1) {
>-              dapl_dbg_log(DAPL_DBG_TYPE_ERR,
>-                           "SCM ERR: cm_thread exceeded
>FD_SETSIZE %d\n",idx+1);
>-              continue;
>-          }
>-
>-          /* Add to ufds for poll, check for immediate work */
>-          ufds[++idx].fd = cr->socket; /* add listen or cr */
>-          ufds[idx].revents = 0;
>-          if (cr->state == SCM_CONN_PENDING)
>-              ufds[idx].events = POLLOUT;
>-          else
>-              ufds[idx].events = POLLIN;
>-
>-          /* check socket for event, accept in or connect out */
>-          dapl_dbg_log(DAPL_DBG_TYPE_CM," poll cr=%p, fd=%d,%d\n",
>-                              cr, cr->socket, ufds[idx].fd);
>-          dapl_os_unlock(&hca_ptr->ib_trans.lock);
>-          ret = poll(&ufds[idx],1,0);
>-          dapl_dbg_log(DAPL_DBG_TYPE_CM,
>-                       " poll wakeup ret=%d cr->st=%d"
>-                       " ev=0x%x fd=%d\n",
>-                       ret,cr->state,ufds[idx].revents,ufds[idx].fd);
>-
>-          /* data on listen, qp exchange, and on disconnect request */
>-          if ((ret == 1) && ufds[idx].revents == POLLIN) {
>-              if (cr->socket > 0) {
>-                      if (cr->state == SCM_LISTEN)
>-                              dapli_socket_accept(cr);
>-                      else if (cr->state == SCM_ACCEPTING)
>-                              dapli_socket_accept_data(cr);
>-                      else if (cr->state == SCM_ACCEPTED)
>-                              dapli_socket_accept_rtu(cr);
>-                      else if (cr->state == SCM_RTU_PENDING)
>-                              dapli_socket_connect_rtu(cr);
>-                      else if (cr->state == SCM_CONNECTED)
>-                              dapli_socket_disconnect(cr);
>+      struct dapl_hca *hca_ptr = arg;
>+      dp_ib_cm_handle_t cr, next_cr;
>+      int opt, ret;
>+      socklen_t opt_len;
>+      char rbuf[2];
>+      struct dapl_fd_set *set;
>+      enum DAPL_FD_EVENTS event;
>+
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread: ENTER hca
>%p\n", hca_ptr);
>+      set = dapl_alloc_fd_set();
>+      if (!set)
>+              goto out;
>+
>+      dapl_os_lock(&hca_ptr->ib_trans.lock);
>+      hca_ptr->ib_trans.cr_state = IB_THREAD_RUN;
>+
>+      while (hca_ptr->ib_trans.cr_state == IB_THREAD_RUN) {
>+              dapl_fd_zero(set);
>+              dapl_fd_set(g_scm_pipe[0], set, DAPL_FD_READ);
>+
>+              if (!dapl_llist_is_empty(&hca_ptr->ib_trans.list))
>+                      next_cr =
>dapl_llist_peek_head(&hca_ptr->ib_trans.list);
>+              else
>+                      next_cr = NULL;
>+
>+              while (next_cr) {
>+                      cr = next_cr;
>+                      if ((cr->socket == DAPL_INVALID_SOCKET
>&& cr->state == SCM_DESTROY) ||
>+                              hca_ptr->ib_trans.cr_state !=
>IB_THREAD_RUN) {
>+                              next_cr =
>dapl_llist_next_entry(&hca_ptr->ib_trans.list,
>+
>(DAPL_LLIST_ENTRY*)&cr->entry);
>+
>dapl_llist_remove_entry(&hca_ptr->ib_trans.list,
>+
>(DAPL_LLIST_ENTRY*)&cr->entry);
>+                              dapl_os_free(cr, sizeof(*cr));
>+                              continue;
>+                      }
>+
>+                      event = (cr->state == SCM_CONN_PENDING) ?
>+                              DAPL_FD_WRITE : DAPL_FD_READ;
>+                      if (dapl_fd_set(cr->socket, set, event)) {
>+                              dapl_log(DAPL_DBG_TYPE_ERR,
>+                                       " cr_thread: DESTROY
>CR st=%d fd %d"
>+                                       " -> %s\n", cr->state,
>cr->socket,
>+                                       inet_ntoa(((struct
>sockaddr_in*)
>+
>&cr->dst.ia_address)->sin_addr));
>+                              dapli_cm_destroy(cr);
>+                              continue;
>+                      }
>+
>+                      dapl_dbg_log(DAPL_DBG_TYPE_CM, " poll
>cr=%p, fd=%d\n",
>+                              cr, cr->socket);
>+                      dapl_os_unlock(&hca_ptr->ib_trans.lock);
>+
>+                      ret = dapl_poll(cr->socket, event);
>+
>+                      dapl_dbg_log(DAPL_DBG_TYPE_CM,
>+                              " poll wakeup ret=%d cr->st=%d fd=%d\n",
>+                              ret, cr->state, cr->socket);
>+
>+                      /* data on listen, qp exchange, and on
>disconnect request */
>+                      if (ret == DAPL_FD_READ) {
>+                              if (cr->socket != DAPL_INVALID_SOCKET) {
>+                                      switch (cr->state) {
>+                                      case SCM_LISTEN:
>+                                              dapli_socket_accept(cr);
>+                                              break;
>+                                      case SCM_ACCEPTING:
>+
>dapli_socket_accept_data(cr);
>+                                              break;
>+                                      case SCM_ACCEPTED:
>+
>dapli_socket_accept_rtu(cr);
>+                                              break;
>+                                      case SCM_RTU_PENDING:
>+
>dapli_socket_connect_rtu(cr);
>+                                              break;
>+                                      case SCM_CONNECTED:
>+
>dapli_socket_disconnect(cr);
>+                                              break;
>+                                      default:
>+                                              break;
>+                                      }
>+                              }
>+                      /* connect socket is writable, check status */
>+                      } else if (ret == DAPL_FD_WRITE || ret
>== DAPL_FD_ERROR) {
>+                              if (cr->state == SCM_CONN_PENDING) {
>+                                      opt = 0;
>+                                      ret =
>getsockopt(cr->socket, SOL_SOCKET,
>+                                              SO_ERROR, (char
>*) &opt, &opt_len);
>+                                      if (!ret)
>+
>dapli_socket_connected(cr, opt);
>+                                      else
>+
>dapli_socket_connected(cr, errno);
>+                              } else {
>+                                      dapl_log(DAPL_DBG_TYPE_CM,
>+                                              " CM poll ERR,
>wrong state(%d) -> %s SKIP\n", cr->state,
>+
>inet_ntoa(((struct sockaddr_in*)&cr->dst.ia_address)->sin_addr));
>+                              }
>+                      } else if (ret != 0) {
>+                              dapl_log(DAPL_DBG_TYPE_CM,
>+                                      " CM poll warning %s,
>ret=%d st=%d -> %s\n",
>+                                      strerror(errno), ret, cr->state,
>+                                      inet_ntoa(((struct sockaddr_in*)
>+
>&cr->dst.ia_address)->sin_addr));
>+
>+                              /* POLLUP, NVAL, or poll error,
>issue event if connected */
>+                              if (cr->state == SCM_CONNECTED)
>+                                      dapli_socket_disconnect(cr);
>+                      }
>+
>+                      dapl_os_lock(&hca_ptr->ib_trans.lock);
>+                      next_cr =
>dapl_llist_next_entry(&hca_ptr->ib_trans.list,
>+                              (DAPL_LLIST_ENTRY*)&cr->entry);
>               }
>-          /* connect socket is writable, check status */
>-          } else if ((ret == 1) &&
>-                      (ufds[idx].revents & POLLOUT ||
>-                       ufds[idx].revents & POLLERR)) {
>-              if (cr->state == SCM_CONN_PENDING) {
>-                      opt = 0;
>-                      ret = getsockopt(cr->socket, SOL_SOCKET,
>-                                       SO_ERROR, &opt, &opt_len);
>-                      if (!ret)
>-                              dapli_socket_connected(cr,opt);
>-                      else
>-                              dapli_socket_connected(cr,errno);
>-              } else {
>-                      dapl_log(DAPL_DBG_TYPE_CM,
>-                               " CM poll ERR, wrong state(%d)
>-> %s SKIP\n",
>-                               cr->state,
>-                               inet_ntoa(((struct sockaddr_in*)
>-
>&cr->dst.ia_address)->sin_addr));
>+
>+              dapl_os_unlock(&hca_ptr->ib_trans.lock);
>+              dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread:
>sleep, fds=%d\n",
>+                           set->index+1);
>+              dapl_select(set);
>+              dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread: wakeup\n");
>+
>+              /* if pipe used to wakeup, consume */
>+              if (dapl_poll(g_scm_pipe[0], DAPL_FD_READ) ==
>DAPL_FD_READ) {
>+                      if (read(g_scm_pipe[0], rbuf, 2) == -1)
>+                              dapl_log(DAPL_DBG_TYPE_CM,
>+                                       " cr_thread: read pipe
>error = %s\n",
>+                                       strerror(errno));
>               }
>-          } else if (ret != 0) {
>-              dapl_log(DAPL_DBG_TYPE_CM,
>-                       " CM poll warning %s, ret=%d revnt=%x
>st=%d -> %s\n",
>-                       strerror(errno), ret,
>ufds[idx].revents, cr->state,
>-                       inet_ntoa(((struct sockaddr_in*)
>-                              &cr->dst.ia_address)->sin_addr));
>-
>-              /* POLLUP, NVAL, or poll error, issue event if
>connected */
>-              if (cr->state == SCM_CONNECTED)
>-                      dapli_socket_disconnect(cr);
>-          }
>-          dapl_os_lock(&hca_ptr->ib_trans.lock);
>-          next_cr =  dapl_llist_next_entry(&hca_ptr->ib_trans.list,
>-
>(DAPL_LLIST_ENTRY*)&cr->entry);
>+              dapl_os_lock(&hca_ptr->ib_trans.lock);
>       }
>+
>       dapl_os_unlock(&hca_ptr->ib_trans.lock);
>-      dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread: sleep, %d\n", idx+1);
>-      poll(ufds,idx+1,-1); /* infinite, all sockets and pipe */
>-      /* if pipe used to wakeup, consume */
>-      if (ufds[0].revents == POLLIN)
>-              if (read(g_scm_pipe[0], rbuf, 2) == -1)
>-                      dapl_log(DAPL_DBG_TYPE_CM,
>-                               " cr_thread: read pipe error = %s\n",
>-                               strerror(errno));
>-      dapl_dbg_log(DAPL_DBG_TYPE_CM," cr_thread: wakeup\n");
>-      dapl_os_lock(&hca_ptr->ib_trans.lock);
>-    }
>-    dapl_os_unlock(&hca_ptr->ib_trans.lock);
>-    hca_ptr->ib_trans.cr_state = IB_THREAD_EXIT;
>-    dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread(hca %p)
>exit\n",hca_ptr);
>+      free(set);
>+out:
>+      hca_ptr->ib_trans.cr_state = IB_THREAD_EXIT;
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cr_thread(hca %p)
>exit\n",hca_ptr);
> }
>-
>-/*
>- * Local variables:
>- *  c-indent-level: 4
>- *  c-basic-offset: 4
>- *  tab-width: 8
>- * End:
>- */
>diff --git a/dapl/openib_scm/dapl_ib_cq.c
>b/dapl/openib_scm/dapl_ib_cq.c
>index 7d6bd4f..59fff11 100644
>--- a/dapl/openib_scm/dapl_ib_cq.c
>+++ b/dapl/openib_scm/dapl_ib_cq.c
>@@ -46,97 +46,111 @@
>  *
>
>***************************************************************
>***********/
>
>+#include "openib_osd.h"
> #include "dapl.h"
> #include "dapl_adapter_util.h"
> #include "dapl_lmr_util.h"
> #include "dapl_evd_util.h"
> #include "dapl_ring_buffer_util.h"
>-#include <sys/poll.h>
>-#include <signal.h>
>
>-int dapli_cq_thread_init(struct dapl_hca *hca_ptr)
>+#if defined(_WIN64) || defined(_WIN32)
>+void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr)
> {
>-        DAT_RETURN dat_status;
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_destroy(%p)\n", hca_ptr);
>
>-        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_init(%p)\n", hca_ptr);
>+      if (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN)
>+              return;
>
>-        /* create thread to process inbound connect request */
>-      hca_ptr->ib_trans.cq_state = IB_THREAD_INIT;
>-        dat_status = dapl_os_thread_create(cq_thread,
>(void*)hca_ptr, &hca_ptr->ib_trans.cq_thread);
>-        if (dat_status != DAT_SUCCESS)
>-        {
>-                dapl_dbg_log(DAPL_DBG_TYPE_ERR,
>-                             " cq_thread_init: failed to
>create thread\n");
>-                return 1;
>-        }
>+      /* destroy cr_thread and lock */
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_CANCEL;
>+      SetEvent(hca_ptr->ib_trans.ib_cq->event);
>+      dapl_dbg_log(DAPL_DBG_TYPE_CM," cq_thread_destroy(%p)
>cancel\n",hca_ptr);
>+      while (hca_ptr->ib_trans.cq_state != IB_THREAD_EXIT) {
>+              dapl_os_sleep_usec(20000);
>+      }
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_destroy(%d)
>exit\n",dapl_os_getpid());
>+}
>+
>+static void cq_thread(void *arg)
>+{
>+      struct dapl_hca *hca_ptr = arg;
>+      struct dapl_evd *evd_ptr;
>+      struct ibv_cq   *ibv_cq = NULL;
>+
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_RUN;
>+
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: ENTER hca
>%p\n",hca_ptr);
>
>-      /* wait for thread to start */
>-      while (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN) {
>-                struct timespec sleep, remain;
>-                sleep.tv_sec = 0;
>-                sleep.tv_nsec = 20000000; /* 20 ms */
>-                dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>-                             " cq_thread_init: waiting for
>cq_thread\n");
>-                nanosleep (&sleep, &remain);
>-        }
>-      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_init(%d)
>exit\n",getpid());
>-        return 0;
>+      /* wait on DTO event, or signal to abort */
>+      while (hca_ptr->ib_trans.cq_state == IB_THREAD_RUN) {
>+              if (!ibv_get_cq_event(hca_ptr->ib_trans.ib_cq,
>&ibv_cq, (void*)&evd_ptr)) {
>+
>+                      if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) {
>+                              ibv_ack_cq_events(ibv_cq, 1);
>+                              return;
>+                      }
>+
>+                      /* process DTO event via callback */
>+
>dapl_evd_dto_callback(hca_ptr->ib_hca_handle, evd_ptr->ib_cq_handle,
>+                              (void*)evd_ptr );
>+
>+                      ibv_ack_cq_events(ibv_cq, 1);
>+              }
>+      }
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_EXIT;
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: EXIT: hca
>%p \n", hca_ptr);
> }
>
>+#else // _WIN32 || _WIN64
>+
> void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr)
> {
>-        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_destroy(%p)\n", hca_ptr);
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_destroy(%p)\n", hca_ptr);
>
>       if (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN)
>               return;
>
>-        /* destroy cr_thread and lock */
>-        hca_ptr->ib_trans.cq_state = IB_THREAD_CANCEL;
>-        pthread_kill(hca_ptr->ib_trans.cq_thread, SIGUSR1);
>-        dapl_dbg_log(DAPL_DBG_TYPE_CM," cq_thread_destroy(%p)
>cancel\n",hca_ptr);
>-        while (hca_ptr->ib_trans.cq_state != IB_THREAD_EXIT) {
>-                struct timespec sleep, remain;
>-                sleep.tv_sec = 0;
>-                sleep.tv_nsec = 2000000; /* 2 ms */
>-                dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>-                             " cq_thread_destroy: waiting for
>cq_thread\n");
>-                nanosleep (&sleep, &remain);
>-        }
>-        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_destroy(%d) exit\n",getpid());
>+      /* destroy cr_thread and lock */
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_CANCEL;
>+      pthread_kill(hca_ptr->ib_trans.cq_thread, SIGUSR1);
>+      dapl_dbg_log(DAPL_DBG_TYPE_CM," cq_thread_destroy(%p)
>cancel\n",hca_ptr);
>+      while (hca_ptr->ib_trans.cq_state != IB_THREAD_EXIT) {
>+              dapl_os_sleep_usec(20000);
>+      }
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_destroy(%d)
>exit\n",dapl_os_getpid());
> }
>
> /* catch the signal */
> static void ib_cq_handler(int signum)
> {
>-        return;
>+      return;
> }
>
>-void cq_thread( void *arg )
>+static void cq_thread(void *arg)
> {
>-        struct dapl_hca *hca_ptr = arg;
>-        struct dapl_evd *evd_ptr;
>-        struct ibv_cq   *ibv_cq = NULL;
>+      struct dapl_hca *hca_ptr = arg;
>+      struct dapl_evd *evd_ptr;
>+      struct ibv_cq   *ibv_cq = NULL;
>       sigset_t        sigset;
>
>       sigemptyset(&sigset);
>-        sigaddset(&sigset,SIGUSR1);
>-        pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
>-        signal(SIGUSR1, ib_cq_handler);
>+      sigaddset(&sigset,SIGUSR1);
>+      pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
>+      signal(SIGUSR1, ib_cq_handler);
>
>       hca_ptr->ib_trans.cq_state = IB_THREAD_RUN;
>-
>+
>       dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: ENTER hca
>%p\n",hca_ptr);
>
>-        /* wait on DTO event, or signal to abort */
>-        while (hca_ptr->ib_trans.cq_state == IB_THREAD_RUN) {
>-                struct pollfd cq_fd = {
>-                        .fd      = hca_ptr->ib_trans.ib_cq->fd,
>-                        .events  = POLLIN,
>-                        .revents = 0
>-                };
>+      /* wait on DTO event, or signal to abort */
>+      while (hca_ptr->ib_trans.cq_state == IB_THREAD_RUN) {
>+              struct pollfd cq_fd = {
>+                      .fd      = hca_ptr->ib_trans.ib_cq->fd,
>+                      .events  = POLLIN,
>+                      .revents = 0
>+              };
>               if ((poll(&cq_fd, 1, -1) == 1) &&
>-                      (!ibv_get_cq_event(hca_ptr->ib_trans.ib_cq,
>-                                 &ibv_cq, (void*)&evd_ptr))) {
>+
>(!ibv_get_cq_event(hca_ptr->ib_trans.ib_cq, &ibv_cq,
>(void*)&evd_ptr))) {
>
>                       if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) {
>                               ibv_ack_cq_events(ibv_cq, 1);
>@@ -144,15 +158,40 @@ void cq_thread( void *arg )
>                       }
>
>                       /* process DTO event via callback */
>-                      dapl_evd_dto_callback ( hca_ptr->ib_hca_handle,
>-                                              evd_ptr->ib_cq_handle,
>-                                              (void*)evd_ptr );
>+                      dapl_evd_dto_callback(hca_ptr->ib_hca_handle,
>+                              evd_ptr->ib_cq_handle, (void*)evd_ptr );
>
>                       ibv_ack_cq_events(ibv_cq, 1);
>               }
>-        }
>-        hca_ptr->ib_trans.cq_state = IB_THREAD_EXIT;
>-        dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: EXIT:
>hca %p \n", hca_ptr);
>+      }
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_EXIT;
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread: EXIT: hca
>%p \n", hca_ptr);
>+}
>+
>+#endif // _WIN32 || _WIN64
>+
>+
>+int dapli_cq_thread_init(struct dapl_hca *hca_ptr)
>+{
>+      DAT_RETURN dat_status;
>+
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,"
>cq_thread_init(%p)\n", hca_ptr);
>+
>+      /* create thread to process inbound connect request */
>+      hca_ptr->ib_trans.cq_state = IB_THREAD_INIT;
>+      dat_status = dapl_os_thread_create(cq_thread,
>(void*)hca_ptr, &hca_ptr->ib_trans.cq_thread);
>+      if (dat_status != DAT_SUCCESS) {
>+              dapl_dbg_log(DAPL_DBG_TYPE_ERR,
>+                      " cq_thread_init: failed to create thread\n");
>+              return 1;
>+      }
>+
>+      /* wait for thread to start */
>+      while (hca_ptr->ib_trans.cq_state != IB_THREAD_RUN) {
>+              dapl_os_sleep_usec(20000);
>+      }
>+      dapl_dbg_log(DAPL_DBG_TYPE_UTIL," cq_thread_init(%d)
>exit\n",dapl_os_getpid());
>+      return 0;
> }
>
>
>@@ -308,11 +347,11 @@ dapls_ib_cq_alloc (
>       IN  DAPL_EVD            *evd_ptr,
>       IN  DAT_COUNT           *cqlen )
> {
>+      struct ibv_comp_channel *channel =
>ia_ptr->hca_ptr->ib_trans.ib_cq;
>+
>       dapl_dbg_log ( DAPL_DBG_TYPE_UTIL,
>               "dapls_ib_cq_alloc: evd %p cqlen=%d \n",
>evd_ptr, *cqlen );
>
>-      struct ibv_comp_channel *channel =
>ia_ptr->hca_ptr->ib_trans.ib_cq;
>-
> #ifdef CQ_WAIT_OBJECT
>       if (evd_ptr->cq_wait_obj_handle)
>               channel = evd_ptr->cq_wait_obj_handle;
>diff --git a/dapl/openib_scm/dapl_ib_dto.h
>b/dapl/openib_scm/dapl_ib_dto.h
>index 45000b9..fa19d01 100644
>--- a/dapl/openib_scm/dapl_ib_dto.h
>+++ b/dapl/openib_scm/dapl_ib_dto.h
>@@ -147,12 +147,6 @@ dapls_ib_post_send (
>       IN  const DAT_RMR_TRIPLET       *remote_iov,
>       IN  DAT_COMPLETION_FLAGS        completion_flags)
> {
>-      dapl_dbg_log(DAPL_DBG_TYPE_EP,
>-                   " post_snd: ep %p op %d ck %p sgs",
>-                   "%d l_iov %p r_iov %p f %d\n",
>-                   ep_ptr, op_type, cookie, segments, local_iov,
>-                   remote_iov, completion_flags);
>-
>       ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
>       ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
>       struct ibv_send_wr wr;
>@@ -163,6 +157,12 @@ dapls_ib_post_send (
>       int ret;
>
>       dapl_dbg_log(DAPL_DBG_TYPE_EP,
>+                   " post_snd: ep %p op %d ck %p sgs",
>+                   "%d l_iov %p r_iov %p f %d\n",
>+                   ep_ptr, op_type, cookie, segments, local_iov,
>+                   remote_iov, completion_flags);
>+
>+      dapl_dbg_log(DAPL_DBG_TYPE_EP,
>                    " post_snd: ep %p cookie %p segs %d l_iov %p\n",
>                    ep_ptr, cookie, segments, local_iov);
>
>@@ -317,12 +317,6 @@ dapls_ib_post_ext_send (
>       IN  DAT_COMPLETION_FLAGS        completion_flags,
>       IN  DAT_IB_ADDR_HANDLE          *remote_ah)
> {
>-      dapl_dbg_log(DAPL_DBG_TYPE_EP,
>-                   " post_ext_snd: ep %p op %d ck %p sgs",
>-                   "%d l_iov %p r_iov %p f %d\n",
>-                   ep_ptr, op_type, cookie, segments, local_iov,
>-                   remote_iov, completion_flags, remote_ah);
>-
>       ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
>       ib_data_segment_t *ds_array_p, *ds_array_start_p = NULL;
>       struct ibv_send_wr wr;
>@@ -331,6 +325,12 @@ dapls_ib_post_ext_send (
>       int ret;
>
>       dapl_dbg_log(DAPL_DBG_TYPE_EP,
>+                   " post_ext_snd: ep %p op %d ck %p sgs",
>+                   "%d l_iov %p r_iov %p f %d\n",
>+                   ep_ptr, op_type, cookie, segments, local_iov,
>+                   remote_iov, completion_flags, remote_ah);
>+
>+      dapl_dbg_log(DAPL_DBG_TYPE_EP,
>                    " post_snd: ep %p cookie %p segs %d l_iov %p\n",
>                    ep_ptr, cookie, segments, local_iov);
>
>diff --git a/dapl/openib_scm/dapl_ib_mem.c
>b/dapl/openib_scm/dapl_ib_mem.c
>index 54340ed..9a97e5e 100644
>--- a/dapl/openib_scm/dapl_ib_mem.c
>+++ b/dapl/openib_scm/dapl_ib_mem.c
>@@ -1,4 +1,4 @@
>-/*
>+      /*
>  * Copyright (c) 2005-2007 Intel Corporation.  All rights reserved.
>  *
>  * This Software is licensed under one of the following licenses:
>@@ -35,13 +35,6 @@
>  *
>
>**********************************************************************/
>
>-#include <sys/ioctl.h>  /* for IOCTL's */
>-#include <sys/types.h>  /* for socket(2) and related bits and
>pieces */
>-#include <sys/socket.h> /* for socket(2) */
>-#include <net/if.h>     /* for struct ifreq */
>-#include <net/if_arp.h> /* for ARPHRD_ETHER */
>-#include <unistd.h>           /* for _SC_CLK_TCK */
>-
> #include "dapl.h"
> #include "dapl_adapter_util.h"
> #include "dapl_lmr_util.h"
>@@ -215,10 +208,9 @@ dapls_ib_mr_register(IN  DAPL_IA *ia_ptr,
>       lmr->param.registered_address = (DAT_VADDR)(uintptr_t)virt_addr;
>
>       dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>-                   " mr_register: mr=%p addr=%p h %x pd %p ctx %p "
>+                   " mr_register: mr=%p addr=%p pd %p ctx %p "
>                    "lkey=0x%x rkey=0x%x priv=%x\n",
>                    lmr->mr_handle, lmr->mr_handle->addr,
>-                   lmr->mr_handle->handle,
>                    lmr->mr_handle->pd, lmr->mr_handle->context,
>                    lmr->mr_handle->lkey, lmr->mr_handle->rkey,
>                    length, dapls_convert_privileges(privileges));
>diff --git a/dapl/openib_scm/dapl_ib_util.c
>b/dapl/openib_scm/dapl_ib_util.c
>index 92b45d5..d82d3f5 100644
>--- a/dapl/openib_scm/dapl_ib_util.c
>+++ b/dapl/openib_scm/dapl_ib_util.c
>@@ -49,17 +49,13 @@
> static const char rcsid[] = "$Id:  $";
> #endif
>
>+#include "openib_osd.h"
> #include "dapl.h"
> #include "dapl_adapter_util.h"
> #include "dapl_ib_util.h"
>+#include "dapl_osd.h"
>
> #include <stdlib.h>
>-#include <netinet/tcp.h>
>-#include <sys/utsname.h>
>-#include <sys/socket.h>
>-#include <arpa/inet.h>
>-#include <unistd.h>
>-#include <fcntl.h>
>
> int g_dapl_loopback_connection = 0;
> int g_scm_pipe[2];
>@@ -88,52 +84,43 @@ char *dapl_ib_mtu_str(enum ibv_mtu mtu)
>       }
> }
>
>-/* just get IP address for hostname */
>-DAT_RETURN getipaddr( char *addr, int addr_len)
>+static DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR *addr, int addr_len)
> {
>-      struct sockaddr_in      *ipv4_addr = (struct sockaddr_in*)addr;
>-      struct hostent          *h_ptr;
>-      struct utsname          ourname;
>+      struct sockaddr_in *sin;
>+      struct addrinfo *res, hint, *ai;
>+      int ret;
>+      char hostname[256];
>
>-      if (uname(&ourname) < 0)  {
>-               dapl_log(DAPL_DBG_TYPE_ERR,
>-                        " open_hca: uname err=%s\n", strerror(errno));
>+      if (addr_len < sizeof(*sin)) {
>               return DAT_INTERNAL_ERROR;
>       }
>
>-      h_ptr = gethostbyname(ourname.nodename);
>-      if (h_ptr == NULL) {
>-               dapl_log(DAPL_DBG_TYPE_ERR,
>-                        " open_hca: gethostbyname err=%s\n",
>-                        strerror(errno));
>-              return DAT_INTERNAL_ERROR;
>+      ret = gethostname(hostname,256);
>+      if (ret)
>+              return ret;
>+
>+      memset(&hint, 0, sizeof hint);
>+      hint.ai_flags = AI_PASSIVE;
>+      hint.ai_family = AF_INET;
>+      hint.ai_socktype = SOCK_STREAM;
>+      hint.ai_protocol = IPPROTO_TCP;
>+
>+      ret = getaddrinfo(hostname, NULL, &hint, &res);
>+      if (ret)
>+              return ret;
>+
>+      ret = DAT_INVALID_ADDRESS;
>+      for (ai = res; ai; ai = ai->ai_next) {
>+              sin = (struct sockaddr_in *) ai->ai_addr;
>+              if (*((uint32_t *) &sin->sin_addr) !=
>htonl(0x7f000001)) {
>+                      *((struct sockaddr_in *) addr) = *sin;
>+                      ret = DAT_SUCCESS;
>+                      break;
>+              }
>       }
>
>-      if (h_ptr->h_addrtype == AF_INET) {
>-              int i;
>-              struct in_addr  **alist =
>-                      (struct in_addr **)h_ptr->h_addr_list;
>-
>-              *(uint32_t*)&ipv4_addr->sin_addr = 0;
>-              ipv4_addr->sin_family = AF_INET;
>-
>-              /* Walk the list of addresses for host */
>-              for (i=0; alist[i] != NULL; i++) {
>-                     /* first non-loopback address */
>-                     if (*(uint32_t*)alist[i] != htonl(0x7f000001)) {
>-                               dapl_os_memcpy(&ipv4_addr->sin_addr,
>-                                              h_ptr->h_addr_list[i],
>-                                              4);
>-                               break;
>-                       }
>-               }
>-               /* if no acceptable address found */
>-               if (*(uint32_t*)&ipv4_addr->sin_addr == 0)
>-                      return DAT_INVALID_ADDRESS;
>-      } else
>-              return DAT_INVALID_ADDRESS;
>-
>-      return DAT_SUCCESS;
>+      freeaddrinfo(res);
>+      return ret;
> }
>
> /*
>@@ -165,6 +152,28 @@ int32_t dapls_ib_release (void)
>       return 0;
> }
>
>+#if defined(_WIN64) || defined(_WIN32)
>+int dapls_config_comp_channel(struct ibv_comp_channel *channel)
>+{
>+      return 0;
>+}
>+#else // _WIN64 || WIN32
>+int dapls_config_comp_channel(struct ibv_comp_channel *channel)
>+{
>+      int opts;
>+
>+      opts = fcntl(channel->fd, F_GETFL); /* uCQ */
>+      if (opts < 0 || fcntl(channel->fd, F_SETFL, opts |
>O_NONBLOCK) < 0) {
>+              dapl_log(DAPL_DBG_TYPE_ERR,
>+                       " dapls_create_comp_channel: fcntl on
>ib_cq->fd %d ERR %d %s\n",
>+                       channel->fd, opts, strerror(errno));
>+              return errno;
>+      }
>+
>+      return 0;
>+}
>+#endif
>+
> /*
>  * dapls_ib_open_hca
>  *
>@@ -187,7 +196,6 @@ DAT_RETURN dapls_ib_open_hca (
>         IN   DAPL_HCA         *hca_ptr)
> {
>       struct ibv_device **dev_list;
>-      int             opts;
>       int             i;
>       DAT_RETURN      dat_status = DAT_SUCCESS;
>
>@@ -219,7 +227,7 @@ found:
>       dapl_dbg_log(DAPL_DBG_TYPE_UTIL," open_hca: Found dev
>%s %016llx\n",
>                    ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
>                    (unsigned long long)
>-
>bswap_64(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev)));
>+
>ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev)));
>
>       hca_ptr->ib_hca_handle =
>ibv_open_device(hca_ptr->ib_trans.ib_dev);
>       if (!hca_ptr->ib_hca_handle) {
>@@ -268,13 +276,7 @@ found:
>               goto bail;
>       }
>
>-      opts = fcntl(hca_ptr->ib_trans.ib_cq->fd, F_GETFL); /* uCQ */
>-      if (opts < 0 || fcntl(hca_ptr->ib_trans.ib_cq->fd,
>-                            F_SETFL, opts | O_NONBLOCK) < 0) {
>-              dapl_log(DAPL_DBG_TYPE_ERR,
>-                       " open_hca: fcntl on ib_cq->fd %d ERR
>%d %s\n",
>-                       hca_ptr->ib_trans.ib_cq->fd, opts,
>-                       strerror(errno));
>+      if (dapls_config_comp_channel(hca_ptr->ib_trans.ib_cq)) {
>               goto bail;
>       }
>
>@@ -309,16 +311,11 @@ found:
>
>       /* wait for thread */
>       while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
>-              struct timespec sleep, remain;
>-              sleep.tv_sec = 0;
>-              sleep.tv_nsec = 2000000; /* 2 ms */
>-              dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>-                           " open_hca: waiting for cr_thread\n");
>-              nanosleep (&sleep, &remain);
>+              dapl_os_sleep_usec(20000);
>       }
>
>       /* get the IP address of the device */
>-      dat_status = getipaddr((char*)&hca_ptr->hca_address,
>+      dat_status = getlocalipaddr((DAT_SOCK_ADDR*)
>&hca_ptr->hca_address,
>                               sizeof(DAT_SOCK_ADDR6));
>
>       dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>@@ -376,16 +373,13 @@ DAT_RETURN dapls_ib_close_hca (  IN
>DAPL_HCA       *hca_ptr )
>                        " thread_destroy: thread wakeup err = %s\n",
>                        strerror(errno));
>       while (hca_ptr->ib_trans.cr_state != IB_THREAD_EXIT) {
>-              struct timespec sleep, remain;
>-              sleep.tv_sec = 0;
>-              sleep.tv_nsec = 2000000; /* 2 ms */
>               dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
>                            " close_hca: waiting for cr_thread\n");
>               if (write(g_scm_pipe[1], "w", sizeof "w") == -1)
>                       dapl_log(DAPL_DBG_TYPE_UTIL,
>                                " thread_destroy: thread
>wakeup err = %s\n",
>                                strerror(errno));
>-              nanosleep (&sleep, &remain);
>+              dapl_os_sleep_usec(20000);
>       }
>       dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);
>
>diff --git a/dapl/openib_scm/dapl_ib_util.h
>b/dapl/openib_scm/dapl_ib_util.h
>index 863da2b..fd1c24e 100644
>--- a/dapl/openib_scm/dapl_ib_util.h
>+++ b/dapl/openib_scm/dapl_ib_util.h
>@@ -49,8 +49,8 @@
> #ifndef _DAPL_IB_UTIL_H_
> #define _DAPL_IB_UTIL_H_
>
>+#include "openib_osd.h"
> #include <infiniband/verbs.h>
>-#include <byteswap.h>
>
> #ifdef DAT_EXTENSIONS
> #include <dat2/dat_ib_extensions.h>
>@@ -73,8 +73,6 @@ typedef      struct ibv_wc
>ib_work_completion_t;
> typedef       struct ibv_context      *ib_hca_handle_t;
> typedef ib_hca_handle_t               dapl_ibal_ca_t;
>
>-/* CM mappings, user CM not complete use SOCKETS */
>-
> /* destination info to exchange, define wire protocol version */
> #define DSCM_VER 3
> typedef struct _ib_qp_cm
>@@ -86,7 +84,7 @@ typedef struct _ib_qp_cm
>       uint32_t                qpn;
>       uint32_t                p_size;
>       DAT_SOCK_ADDR6          ia_address;
>-        union ibv_gid         gid;
>+      union ibv_gid           gid;
>       uint16_t                qp_type;
> } ib_qp_cm_t;
>
>@@ -110,20 +108,18 @@ struct ib_cm_handle
>       struct dapl_llist_entry entry;
>       DAPL_OS_LOCK            lock;
>       SCM_STATE               state;
>-      int                     socket;
>+      DAPL_SOCKET             socket;
>       struct dapl_hca         *hca;
>       struct dapl_sp          *sp;
>-      struct dapl_ep          *ep;
>+      struct dapl_ep          *ep;
>       ib_qp_cm_t              dst;
>-      unsigned char           p_data[256];
>+      unsigned char           p_data[256];    /* must follow
>ib_qp_cm_t */
>       struct ibv_ah           *ah;
> };
>
> typedef struct ib_cm_handle   *dp_ib_cm_handle_t;
> typedef dp_ib_cm_handle_t     ib_cm_srvc_handle_t;
>
>-DAT_RETURN getipaddr(char *addr, int addr_len);
>-
> /* CM events */
> typedef enum
> {
>@@ -141,9 +137,6 @@ typedef enum
>
> } ib_cm_events_t;
>
>-/* prototype for cm thread */
>-void cr_thread (void *arg);
>-
> /* Operation and state mappings */
> typedef enum  ibv_send_flags  ib_send_op_type_t;
> typedef       struct  ibv_sge         ib_data_segment_t;
>@@ -289,7 +282,7 @@ typedef struct _ib_hca_transport
>       DAPL_OS_LOCK            cq_lock;
>       int                     max_inline_send;
>       ib_thread_state_t       cq_state;
>-      DAPL_OS_THREAD          cq_thread;
>+      DAPL_OS_THREAD                  cq_thread;
>       struct ibv_comp_channel *ib_cq;
>       int                     cr_state;
>       DAPL_OS_THREAD          thread;
>@@ -317,7 +310,6 @@ typedef uint32_t ib_shm_transport_t;
> /* prototypes */
> int32_t       dapls_ib_init (void);
> int32_t       dapls_ib_release (void);
>-void cq_thread (void *arg);
> void cr_thread(void *arg);
> int dapli_cq_thread_init(struct dapl_hca *hca_ptr);
> void dapli_cq_thread_destroy(struct dapl_hca *hca_ptr);
>@@ -349,7 +341,7 @@ dapl_convert_errno( IN int err, IN const
>char *str )
>     if (!err) return DAT_SUCCESS;
>
> #if DAPL_DBG
>-    if ((err != EAGAIN) && (err != ETIME) && (err != ETIMEDOUT))
>+    if ((err != EAGAIN) && (err != ETIMEDOUT))
>       dapl_dbg_log (DAPL_DBG_TYPE_ERR," %s %s\n", str, strerror(err));
> #endif
>
>@@ -357,24 +349,15 @@ dapl_convert_errno( IN int err, IN const
>char *str )
>     {
>       case EOVERFLOW  : return DAT_LENGTH_ERROR;
>       case EACCES     : return DAT_PRIVILEGES_VIOLATION;
>-      case ENXIO      :
>-      case ERANGE     :
>       case EPERM      : return DAT_PROTECTION_VIOLATION;
>
>-      case EINVAL     :
>-        case EBADF    :
>-      case ENOENT     :
>-      case ENOTSOCK   : return DAT_INVALID_HANDLE;
>+      case EINVAL     : return DAT_INVALID_HANDLE;
>       case EISCONN    : return DAT_INVALID_STATE |
>DAT_INVALID_STATE_EP_CONNECTED;
>       case ECONNREFUSED : return DAT_INVALID_STATE |
>DAT_INVALID_STATE_EP_NOTREADY;
>-      case ETIME      :
>       case ETIMEDOUT  : return DAT_TIMEOUT_EXPIRED;
>       case ENETUNREACH: return DAT_INVALID_ADDRESS |
>DAT_INVALID_ADDRESS_UNREACHABLE;
>       case EADDRINUSE : return DAT_CONN_QUAL_IN_USE;
>       case EALREADY   : return DAT_INVALID_STATE |
>DAT_INVALID_STATE_EP_ACTCONNPENDING;
>-        case ENOSPC   :
>-      case ENOMEM     :
>-        case E2BIG    :
>-        case EDQUOT   : return DAT_INSUFFICIENT_RESOURCES;
>+      case ENOMEM     : return DAT_INSUFFICIENT_RESOURCES;
>         case EAGAIN   : return DAT_QUEUE_EMPTY;
>       case EINTR      : return DAT_INTERRUPTED_CALL;
>       case EAFNOSUPPORT : return DAT_INVALID_ADDRESS |
>DAT_INVALID_ADDRESS_MALFORMED;
>diff --git a/dapl/openib_scm/linux/openib_osd.h
>b/dapl/openib_scm/linux/openib_osd.h
>new file mode 100644
>index 0000000..235a82e
>--- /dev/null
>+++ b/dapl/openib_scm/linux/openib_osd.h
>@@ -0,0 +1,21 @@
>+#ifndef OPENIB_OSD_H
>+#define OPENIB_OSD_H
>+
>+#include <endian.h>
>+#include <netinet/in.h>
>+
>+#if __BYTE_ORDER == __BIG_ENDIAN
>+#define htonll(x) (x)
>+#define ntohll(x) (x)
>+#elif __BYTE_ORDER == __LITTLE_ENDIAN
>+#define htonll(x)  bswap_64(x)
>+#define ntohll(x)  bswap_64(x)
>+#endif
>+
>+#define DAPL_SOCKET int
>+#define DAPL_INVALID_SOCKET -1
>+#define DAPL_FD_SETSIZE 8192
>+
>+#define closesocket close
>+
>+#endif // OPENIB_OSD_H
>diff --git a/dapl/openib_scm/windows/openib_osd.h
>b/dapl/openib_scm/windows/openib_osd.h
>new file mode 100644
>index 0000000..67c70ec
>--- /dev/null
>+++ b/dapl/openib_scm/windows/openib_osd.h
>@@ -0,0 +1,39 @@
>+#ifndef OPENIB_OSD_H
>+#define OPENIB_OSD_H
>+
>+#ifndef FD_SETSIZE
>+#define FD_SETSIZE 1024 /* Set before including winsock2 -
>see select help */
>+#define DAPL_FD_SETSIZE FD_SETSIZE
>+#endif
>+
>+#include <winsock2.h>
>+#include <ws2tcpip.h>
>+#include <io.h>
>+#include <fcntl.h>
>+
>+#define ntohll _byteswap_uint64
>+#define htonll _byteswap_uint64
>+
>+#define pipe(x) _pipe(x, 4096, _O_TEXT)
>+#define read _read
>+#define write _write
>+#define DAPL_SOCKET SOCKET
>+#define DAPL_INVALID_SOCKET INVALID_SOCKET
>+
>+/* allow casting to WSABUF */
>+struct iovec
>+{
>+       u_long iov_len;
>+       char FAR* iov_base;
>+};
>+
>+static int writev(DAPL_SOCKET s, struct iovec *vector, int count)
>+{
>+       int len, ret;
>+
>+       ret = WSASend(s, (WSABUF *) vector, count, &len, 0,
>NULL, NULL);
>+       return ret ? ret : len;
>+}
>+
>+#endif // OPENIB_OSD_H
>+
>diff --git a/dapl/udapl/linux/dapl_osd.h b/dapl/udapl/linux/dapl_osd.h
>index 6fef9af..ae02944 100644
>--- a/dapl/udapl/linux/dapl_osd.h
>+++ b/dapl/udapl/linux/dapl_osd.h
>@@ -302,6 +302,15 @@ dapl_os_thread_create (
>       IN  void                        *data,
>       OUT DAPL_OS_THREAD              *thread_id );
>
>+STATIC _INLINE_ void
>+dapl_os_sleep_usec(int usec)
>+{
>+      struct timespec sleep, remain;
>+
>+      sleep.tv_sec = 0;
>+      sleep.tv_nsec = usec * 1000;
>+      nanosleep(&sleep, &remain);
>+}
>
> /*
>  * Lock Functions
>
>
>
>_______________________________________________
>general mailing list
>general at lists.openfabrics.org
>http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
>To unsubscribe, please visit
>http://openib.org/mailman/listinfo/openib-general
>



More information about the general mailing list