[openib-general] [PATCH][uDAPL] new provider with uCMA (librdmacm) support
Arlin Davis
arlin.r.davis at intel.com
Wed Nov 30 15:39:20 PST 2005
James,
Here is a provider for the latest uCMA that is tested
with dapltest, dtest, and Intel MPI. I also added a top
level README with instructions. Default build is
set for uCMA.
See README for build and configuration details.
Modifications:
test/dtest/makefile
dapl/udapl/Makefile
doc/dat.conf
New files:
README
dapl/openib_cma/README
dapl/openib_cma/dapl_ib_dto.h
dapl/openib_cma/dapl_ib_util.c
dapl/openib_cma/dapl_ib_mem.c
dapl/openib_cma/dapl_ib_cm.c
dapl/openib_cma/dapl_ib_qp.c
dapl/openib_cma/dapl_ib_util.h
dapl/openib_cma/dapl_ib_cq.c
Thanks,
-arlin
Signed-off by: Arlin Davis <ardavis at ichips.intel.com>
Index: test/dtest/makefile
===================================================================
--- test/dtest/makefile (revision 4178)
+++ test/dtest/makefile (working copy)
@@ -2,7 +2,7 @@ CC = gcc
CFLAGS = -O2 -g
DAT_INC = ../../dat/include
-DAT_LIB = /usr/lib64
+DAT_LIB = /usr/local/lib
all: dtest
@@ -11,6 +11,6 @@ clean:
dtest: ./dtest.c
$(CC) $(CFLAGS) ./dtest.c -o dtest \
- -DDAPL_PROVIDER='"OpenIB-ib0"' \
+ -DDAPL_PROVIDER='"OpenIB-cma-ip"' \
-I $(DAT_INC) -L $(DAT_LIB) -ldat
Index: dapl/udapl/Makefile
===================================================================
--- dapl/udapl/Makefile (revision 4178)
+++ dapl/udapl/Makefile (working copy)
@@ -53,7 +53,7 @@ OSRELEASE=$(shell expr `uname -r | cut -
# Set up the default provider
#
ifndef $VERBS
-VERBS=openib
+VERBS=openib_cma
endif
#
@@ -149,6 +149,16 @@ CFLAGS += -I/usr/local/include/infinib
endif
#
+# OpenIB provider with IB CMA
+#
+ifeq ($(VERBS),openib_cma)
+PROVIDER = $(TOPDIR)/../openib_cma
+CFLAGS += -DOPENIB
+CFLAGS += -DCQ_WAIT_OBJECT
+CFLAGS += -I/usr/local/include/infiniband
+endif
+
+#
# If an implementation supports CM and DTO completions on the same EVD
# then DAPL_MERGE_CM_DTO should be set
# CFLAGS += -DDAPL_MERGE_CM_DTO=1
@@ -268,6 +278,13 @@ PROVIDER_SRCS = dapl_ib_util.c dapl_ib_
dapl_ib_cm.c dapl_ib_mem.c
endif
+ifeq ($(VERBS),openib_cma)
+LDFLAGS += -libverbs -lrdmacm
+LDFLAGS += -rpath /usr/local/lib -L /usr/local/lib
+PROVIDER_SRCS = dapl_ib_util.c dapl_ib_cq.c dapl_ib_qp.c \
+ dapl_ib_cm.c dapl_ib_mem.c
+endif
+
UDAPL_SRCS = dapl_init.c \
dapl_evd_create.c \
dapl_evd_query.c \
Index: dapl/openib_cma/dapl_ib_dto.h
===================================================================
--- dapl/openib_cma/dapl_ib_dto.h (revision 0)
+++ dapl/openib_cma/dapl_ib_dto.h (revision 0)
@@ -0,0 +1,266 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/***************************************************************************
+ *
+ * Module: uDAPL
+ *
+ * Filename: dapl_ib_dto.h
+ *
+ * Author: Arlin Davis
+ *
+ * Created: 3/10/2005
+ *
+ * Description:
+ *
+ * The uDAPL openib provider - DTO operations and CQE macros
+ *
+ ****************************************************************************
+ * Source Control System Information
+ *
+ * $Id: $
+ *
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ **************************************************************************/
+#ifndef _DAPL_IB_DTO_H_
+#define _DAPL_IB_DTO_H_
+
+#include "dapl_ib_util.h"
+
+#define DEFAULT_DS_ENTRIES 8
+
+STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p);
+
+/*
+ * dapls_ib_post_recv
+ *
+ * Provider specific Post RECV function
+ */
+STATIC _INLINE_ DAT_RETURN
+dapls_ib_post_recv (
+ IN DAPL_EP *ep_ptr,
+ IN DAPL_COOKIE *cookie,
+ IN DAT_COUNT segments,
+ IN DAT_LMR_TRIPLET *local_iov )
+{
+ ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
+ ib_data_segment_t *ds_array_p;
+ struct ibv_recv_wr wr;
+ struct ibv_recv_wr *bad_wr;
+ DAT_COUNT i, total_len;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_rcv: ep %p cookie %p segs %d l_iov %p\n",
+ ep_ptr, cookie, segments, local_iov);
+
+ if (segments <= DEFAULT_DS_ENTRIES)
+ ds_array_p = ds_array;
+ else
+ ds_array_p =
+ dapl_os_alloc(segments * sizeof(ib_data_segment_t));
+
+ if (NULL == ds_array_p)
+ return (DAT_INSUFFICIENT_RESOURCES);
+
+ /* setup work request */
+ total_len = 0;
+ wr.next = 0;
+ wr.num_sge = 0;
+ wr.wr_id = (uint64_t)(uintptr_t)cookie;
+ wr.sg_list = ds_array_p;
+
+ for (i = 0; i < segments; i++) {
+ if (!local_iov[i].segment_length)
+ continue;
+
+ ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
+ ds_array_p->length = local_iov[i].segment_length;
+ ds_array_p->lkey = local_iov[i].lmr_context;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_rcv: l_key 0x%x va %p len %d\n",
+ ds_array_p->lkey, ds_array_p->addr,
+ ds_array_p->length );
+
+ total_len += ds_array_p->length;
+ wr.num_sge++;
+ ds_array_p++;
+ }
+
+ if (cookie != NULL)
+ cookie->val.dto.size = total_len;
+
+ if (ibv_post_recv(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr))
+ return( dapl_convert_errno(EFAULT,"ibv_recv") );
+
+ return DAT_SUCCESS;
+}
+
+
+/*
+ * dapls_ib_post_send
+ *
+ * Provider specific Post SEND function
+ */
+STATIC _INLINE_ DAT_RETURN
+dapls_ib_post_send (
+ IN DAPL_EP *ep_ptr,
+ IN ib_send_op_type_t op_type,
+ IN DAPL_COOKIE *cookie,
+ IN DAT_COUNT segments,
+ IN DAT_LMR_TRIPLET *local_iov,
+ IN const DAT_RMR_TRIPLET *remote_iov,
+ IN DAT_COMPLETION_FLAGS completion_flags)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: ep %p op %d ck %p sgs",
+ "%d l_iov %p r_iov %p f %d\n",
+ ep_ptr, op_type, cookie, segments, local_iov,
+ remote_iov, completion_flags);
+
+ ib_data_segment_t ds_array[DEFAULT_DS_ENTRIES];
+ ib_data_segment_t *ds_array_p;
+ struct ibv_send_wr wr;
+ struct ibv_send_wr *bad_wr;
+ ib_hca_transport_t *ibt_ptr =
+ &ep_ptr->header.owner_ia->hca_ptr->ib_trans;
+ DAT_COUNT i, total_len;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: ep %p cookie %p segs %d l_iov %p\n",
+ ep_ptr, cookie, segments, local_iov);
+
+ if(segments <= DEFAULT_DS_ENTRIES)
+ ds_array_p = ds_array;
+ else
+ ds_array_p =
+ dapl_os_alloc(segments * sizeof(ib_data_segment_t));
+
+ if (NULL == ds_array_p)
+ return (DAT_INSUFFICIENT_RESOURCES);
+
+ /* setup the work request */
+ wr.next = 0;
+ wr.opcode = op_type;
+ wr.num_sge = 0;
+ wr.send_flags = 0;
+ wr.wr_id = (uint64_t)(uintptr_t)cookie;
+ wr.sg_list = ds_array_p;
+ total_len = 0;
+
+ for (i = 0; i < segments; i++ ) {
+ if ( !local_iov[i].segment_length )
+ continue;
+
+ ds_array_p->addr = (uint64_t) local_iov[i].virtual_address;
+ ds_array_p->length = local_iov[i].segment_length;
+ ds_array_p->lkey = local_iov[i].lmr_context;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: lkey 0x%x va %p len %d\n",
+ ds_array_p->lkey, ds_array_p->addr,
+ ds_array_p->length );
+
+ total_len += ds_array_p->length;
+ wr.num_sge++;
+ ds_array_p++;
+ }
+
+ if (cookie != NULL)
+ cookie->val.dto.size = total_len;
+
+ if ((op_type == OP_RDMA_WRITE) || (op_type == OP_RDMA_READ)) {
+ wr.wr.rdma.remote_addr = remote_iov->target_address;
+ wr.wr.rdma.rkey = remote_iov->rmr_context;
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd_rdma: rkey 0x%x va %#016Lx\n",
+ wr.wr.rdma.rkey, wr.wr.rdma.remote_addr);
+ }
+
+ /* inline data for send or write ops */
+ if ((total_len <= ibt_ptr->max_inline_send) &&
+ ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE)))
+ wr.send_flags |= IBV_SEND_INLINE;
+
+ /* set completion flags in work request */
+ wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
+ completion_flags) ? 0 : IBV_SEND_SIGNALED;
+ wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG &
+ completion_flags) ? IBV_SEND_FENCE : 0;
+ wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG &
+ completion_flags) ? IBV_SEND_SOLICITED : 0;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " post_snd: op 0x%x flags 0x%x sglist %p, %d\n",
+ wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge);
+
+ if (ibv_post_send(ep_ptr->qp_handle->cm_id->qp, &wr, &bad_wr))
+ return( dapl_convert_errno(EFAULT,"ibv_recv") );
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
+ return DAT_SUCCESS;
+}
+
+STATIC _INLINE_ DAT_RETURN
+dapls_ib_optional_prv_dat(
+ IN DAPL_CR *cr_ptr,
+ IN const void *event_data,
+ OUT DAPL_CR **cr_pp)
+{
+ return DAT_SUCCESS;
+}
+
+STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
+{
+ switch (cqe_p->opcode) {
+ case IBV_WC_SEND:
+ return (OP_SEND);
+ case IBV_WC_RDMA_WRITE:
+ return (OP_RDMA_WRITE);
+ case IBV_WC_RDMA_READ:
+ return (OP_RDMA_READ);
+ case IBV_WC_COMP_SWAP:
+ return (OP_COMP_AND_SWAP);
+ case IBV_WC_FETCH_ADD:
+ return (OP_FETCH_AND_ADD);
+ case IBV_WC_BIND_MW:
+ return (OP_BIND_MW);
+ case IBV_WC_RECV:
+ return (OP_RECEIVE);
+ case IBV_WC_RECV_RDMA_WITH_IMM:
+ return (OP_RECEIVE_IMM);
+ default:
+ return (OP_INVALID);
+ }
+}
+
+#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p)
+#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id
+#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status
+#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len
+#define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data
+
+#endif /* _DAPL_IB_DTO_H_ */
Index: dapl/openib_cma/dapl_ib_util.c
===================================================================
--- dapl/openib_cma/dapl_ib_util.c (revision 0)
+++ dapl/openib_cma/dapl_ib_util.c (revision 0)
@@ -0,0 +1,791 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/***************************************************************************
+ *
+ * Module: uDAPL
+ *
+ * Filename: dapl_ib_util.c
+ *
+ * Author: Arlin Davis
+ *
+ * Created: 3/10/2005
+ *
+ * Description:
+ *
+ * The uDAPL openib provider - init, open, close, utilities, work thread
+ *
+ ****************************************************************************
+ * Source Control System Information
+ *
+ * $Id: $
+ *
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ **************************************************************************/
+#ifdef RCSID
+static const char rcsid[] = "$Id: $";
+#endif
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+#include "dapl_ib_util.h"
+
+#include <stdlib.h>
+#include <netinet/tcp.h>
+#include <sys/poll.h>
+#include <fcntl.h>
+
+int g_dapl_loopback_connection = 0;
+int g_ib_pipe[2];
+ib_thread_state_t g_ib_thread_state = 0;
+DAPL_OS_THREAD g_ib_thread;
+DAPL_OS_LOCK g_hca_lock;
+struct dapl_llist_entry *g_hca_list;
+
+/* Get IP address */
+static int getipaddr(char *name, char *addr, int len)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(name, NULL, NULL, &res);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " getipaddr: invalid name or address (%s)\n",
+ name);
+ return ret;
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " getipaddr: family %d port %d addr %d.%d.%d.%d\n",
+ ((struct sockaddr_in *)res->ai_addr)->sin_family,
+ ((struct sockaddr_in *)res->ai_addr)->sin_port,
+ ((struct sockaddr_in *)
+ res->ai_addr)->sin_addr.s_addr >> 0 & 0xff,
+ ((struct sockaddr_in *)
+ res->ai_addr)->sin_addr.s_addr >> 8 & 0xff,
+ ((struct sockaddr_in *)
+ res->ai_addr)->sin_addr.s_addr >> 16 & 0xff,
+ ((struct sockaddr_in *)
+ res->ai_addr)->sin_addr.s_addr >> 24 & 0xff );
+
+ if (len >= res->ai_addrlen)
+ memcpy(addr, res->ai_addr, res->ai_addrlen);
+ else
+ return EINVAL;
+
+ freeaddrinfo(res);
+
+ return 0;
+}
+
+/*
+ * dapls_ib_init, dapls_ib_release
+ *
+ * Initialize Verb related items for device open
+ *
+ * Input:
+ * none
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * 0 success, -1 error
+ *
+ */
+int32_t dapls_ib_init(void)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_init: \n" );
+
+ /* initialize hca_list lock */
+ dapl_os_lock_init(&g_hca_lock);
+
+ /* initialize hca list for CQ events */
+ dapl_llist_init_head(&g_hca_list);
+
+ /* create pipe for waking up work thread */
+ if (pipe(g_ib_pipe))
+ return 1;
+
+ return 0;
+}
+
+int32_t dapls_ib_release(void)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n");
+ dapli_ib_thread_destroy();
+ return 0;
+}
+
+/*
+ * dapls_ib_open_hca
+ *
+ * Open HCA
+ *
+ * Input:
+ * *hca_name pointer to provider device name
+ * *ib_hca_handle_p pointer to provide HCA handle
+ *
+ * Output:
+ * none
+ *
+ * Return:
+ * DAT_SUCCESS
+ * dapl_convert_errno
+ *
+ */
+DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA *hca_ptr)
+{
+ long opts;
+ struct rdma_cm_id *cm_id;
+ union ibv_gid *gid;
+ int ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " open_hca: %s - %p\n", hca_name, hca_ptr);
+
+ if (dapli_ib_thread_init())
+ return DAT_INTERNAL_ERROR;
+
+
+ /* HCA name will be hostname or IP address */
+ if (getipaddr((char*)hca_name,
+ (char*)&hca_ptr->hca_address,
+ sizeof(DAT_SOCK_ADDR6)))
+ return DAT_INVALID_ADDRESS;
+
+
+ /* cm_id will bind local device/GID based on IP address */
+ if (rdma_create_id(&cm_id, (void*)hca_ptr))
+ return DAT_INTERNAL_ERROR;
+
+ ret = rdma_bind_addr(cm_id,
+ (struct sockaddr *)&hca_ptr->hca_address);
+ if (ret) {
+ rdma_destroy_id(cm_id);
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " open_hca: ERR bind (%d) %s \n",
+ ret, strerror(-ret));
+ return DAT_INVALID_ADDRESS;
+ }
+
+ /* keep reference to IB device and cm_id */
+ hca_ptr->ib_trans.cm_id = cm_id;
+ hca_ptr->ib_hca_handle = cm_id->verbs;
+ hca_ptr->port_num = cm_id->port_num;
+ gid = &cm_id->route.addr.addr.ibaddr.sgid;
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_UTIL,
+ " open_hca: ctx=%p port=%d GID subnet %016llx id %016llx\n",
+ cm_id->verbs,cm_id->port_num,
+ (unsigned long long)bswap_64(gid->global.subnet_prefix),
+ (unsigned long long)bswap_64(gid->global.interface_id));
+
+ /* set inline max with env or default, get local lid and gid 0 */
+ hca_ptr->ib_trans.max_inline_send =
+ dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
+
+ /* EVD events without direct CQ channels, non-blocking */
+ hca_ptr->ib_trans.ib_cq =
+ ibv_create_comp_channel(hca_ptr->ib_hca_handle);
+ opts = fcntl(hca_ptr->ib_trans.ib_cq->fd, F_GETFL); /* uCQ */
+ if (opts < 0 || fcntl(hca_ptr->ib_trans.ib_cq->fd,
+ F_SETFL, opts | O_NONBLOCK) < 0) {
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+ " open_hca: ERR with CQ FD\n" );
+ goto bail;
+ }
+
+ /*
+ * Put new hca_transport on list for async and CQ event processing
+ * Wakeup work thread to add to polling list
+ */
+ dapl_llist_init_entry((DAPL_LLIST_ENTRY*)&hca_ptr->ib_trans.entry);
+ dapl_os_lock( &g_hca_lock );
+ dapl_llist_add_tail(&g_hca_list,
+ (DAPL_LLIST_ENTRY*)&hca_ptr->ib_trans.entry,
+ &hca_ptr->ib_trans.entry);
+ write(g_ib_pipe[1], "w", sizeof "w");
+ dapl_os_unlock(&g_hca_lock);
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_UTIL,
+ " open_hca: %s, %s %d.%d.%d.%d INLINE_MAX=%d\n", hca_name,
+ ((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_family == AF_INET ?
+ "AF_INET":"AF_INET6",
+ ((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_addr.s_addr >> 0 & 0xff,
+ ((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_addr.s_addr >> 8 & 0xff,
+ ((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_addr.s_addr >> 16 & 0xff,
+ ((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_addr.s_addr >> 24 & 0xff,
+ hca_ptr->ib_trans.max_inline_send );
+
+ hca_ptr->ib_trans.d_hca = hca_ptr;
+ return DAT_SUCCESS;
+bail:
+ rdma_destroy_id(hca_ptr->ib_trans.cm_id);
+ hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
+ return DAT_INTERNAL_ERROR;
+}
+
+
+/*
+ * dapls_ib_close_hca
+ *
+ * Open HCA
+ *
+ * Input:
+ * DAPL_HCA provide CA handle
+ *
+ * Output:
+ * none
+ *
+ * Return:
+ * DAT_SUCCESS
+ * dapl_convert_errno
+ *
+ */
+DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA *hca_ptr)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL," close_hca: %p->%p\n",
+ hca_ptr,hca_ptr->ib_hca_handle);
+
+ if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {
+ if (rdma_destroy_id(hca_ptr->ib_trans.cm_id))
+ return(dapl_convert_errno(errno,"ib_close_device"));
+ hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
+ }
+
+ /*
+ * Remove hca from async and CQ event processing list
+ * Wakeup work thread to remove from polling list
+ */
+ hca_ptr->ib_trans.destroy = 1;
+ write(g_ib_pipe[1], "w", sizeof "w");
+
+ /* wait for thread to remove HCA references */
+ while (hca_ptr->ib_trans.destroy != 2) {
+ struct timespec sleep, remain;
+ sleep.tv_sec = 0;
+ sleep.tv_nsec = 10000000; /* 10 ms */
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_destroy: wait on hca %p destroy\n");
+ nanosleep (&sleep, &remain);
+ }
+ return (DAT_SUCCESS);
+}
+
+/*
+ * dapls_ib_query_hca
+ *
+ * Query the hca attribute
+ *
+ * Input:
+ * hca_handl hca handle
+ * ia_attr attribute of the ia
+ * ep_attr attribute of the ep
+ * ip_addr ip address of DET NIC
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_HANDLE
+ */
+
+DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
+ OUT DAT_IA_ATTR *ia_attr,
+ OUT DAT_EP_ATTR *ep_attr,
+ OUT DAT_SOCK_ADDR6 *ip_addr)
+{
+ struct ibv_device_attr dev_attr;
+ struct ibv_port_attr port_attr;
+
+ if (hca_ptr->ib_hca_handle == NULL) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR," query_hca: BAD handle\n");
+ return (DAT_INVALID_HANDLE);
+ }
+
+ /* local IP address of device, set during ia_open */
+ if (ip_addr != NULL)
+ memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6));
+
+ if (ia_attr == NULL && ep_attr == NULL)
+ return DAT_SUCCESS;
+
+ /* query verbs for this device and port attributes */
+ if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
+ ibv_query_port(hca_ptr->ib_hca_handle,
+ hca_ptr->port_num, &port_attr))
+ return(dapl_convert_errno(errno,"ib_query_hca"));
+
+ if (ia_attr != NULL) {
+ ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
+ ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
+ ia_attr->ia_address_ptr =
+ (DAT_IA_ADDRESS_PTR)&hca_ptr->hca_address;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " query_hca: %s %s %d.%d.%d.%d\n", hca_ptr->name,
+ ((struct sockaddr_in *)
+ ia_attr->ia_address_ptr)->sin_family == AF_INET ?
+ "AF_INET":"AF_INET6",
+ ((struct sockaddr_in *)
+ ia_attr->ia_address_ptr)->sin_addr.s_addr >> 0 & 0xff,
+ ((struct sockaddr_in *)
+ ia_attr->ia_address_ptr)->sin_addr.s_addr >> 8 & 0xff,
+ ((struct sockaddr_in *)
+ ia_attr->ia_address_ptr)->sin_addr.s_addr >> 16 & 0xff,
+ ((struct sockaddr_in *)
+ ia_attr->ia_address_ptr)->sin_addr.s_addr >> 24 & 0xff);
+
+ ia_attr->hardware_version_major = dev_attr.hw_ver;
+ ia_attr->max_eps = dev_attr.max_qp;
+ ia_attr->max_dto_per_ep = dev_attr.max_qp_wr;
+ ia_attr->max_rdma_read_per_ep = dev_attr.max_qp_rd_atom;
+ ia_attr->max_evds = dev_attr.max_cq;
+ ia_attr->max_evd_qlen = dev_attr.max_cqe;
+ ia_attr->max_iov_segments_per_dto = dev_attr.max_sge;
+ ia_attr->max_lmrs = dev_attr.max_mr;
+ ia_attr->max_lmr_block_size = dev_attr.max_mr_size;
+ ia_attr->max_rmrs = dev_attr.max_mw;
+ ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size;
+ ia_attr->max_rmr_target_address = dev_attr.max_mr_size;
+ ia_attr->max_pzs = dev_attr.max_pd;
+ ia_attr->max_mtu_size = port_attr.max_msg_sz;
+ ia_attr->max_rdma_size = port_attr.max_msg_sz;
+ ia_attr->num_transport_attr = 0;
+ ia_attr->transport_attr = NULL;
+ ia_attr->num_vendor_attr = 0;
+ ia_attr->vendor_attr = NULL;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " query_hca: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n",
+ ia_attr->hardware_version_major,
+ ia_attr->max_eps, ia_attr->max_dto_per_ep,
+ ia_attr->max_evds, ia_attr->max_evd_qlen );
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d\n",
+ ia_attr->max_mtu_size, ia_attr->max_rdma_size,
+ ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs,
+ ia_attr->max_rmrs );
+ }
+
+ if (ep_attr != NULL) {
+ ep_attr->max_mtu_size = port_attr.max_msg_sz;
+ ep_attr->max_rdma_size = port_attr.max_msg_sz;
+ ep_attr->max_recv_dtos = dev_attr.max_qp_wr;
+ ep_attr->max_request_dtos = dev_attr.max_qp_wr;
+ ep_attr->max_recv_iov = dev_attr.max_sge;
+ ep_attr->max_request_iov = dev_attr.max_sge;
+ ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom;
+ ep_attr->max_rdma_read_out= dev_attr.max_qp_rd_atom;
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " query_hca: MAX msg %llu dto %d iov %d rdma i%d,o%d\n",
+ ep_attr->max_mtu_size,
+ ep_attr->max_recv_dtos, ep_attr->max_recv_iov,
+ ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out);
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_setup_async_callback
+ *
+ * Set up an asynchronous callbacks of various kinds
+ *
+ * Input:
+ * ia_handle IA handle
+ * handler_type type of handler to set up
+ * callback_handle handle param for completion callbacks
+ * callback callback routine pointer
+ * context argument for callback routine
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INVALID_PARAMETER
+ *
+ */
+DAT_RETURN dapls_ib_setup_async_callback(IN DAPL_IA *ia_ptr,
+ IN DAPL_ASYNC_HANDLER_TYPE type,
+ IN DAPL_EVD *evd_ptr,
+ IN ib_async_handler_t callback,
+ IN void *context)
+
+{
+ ib_hca_transport_t *hca_ptr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " setup_async_cb: ia %p type %d hdl %p cb %p ctx %p\n",
+ ia_ptr, type, evd_ptr, callback, context);
+
+ hca_ptr = &ia_ptr->hca_ptr->ib_trans;
+ switch(type)
+ {
+ case DAPL_ASYNC_UNAFILIATED:
+ hca_ptr->async_unafiliated =
+ (ib_async_handler_t)callback;
+ hca_ptr->async_un_ctx = context;
+ break;
+ case DAPL_ASYNC_CQ_ERROR:
+ hca_ptr->async_cq_error =
+ (ib_async_cq_handler_t)callback;
+ break;
+ case DAPL_ASYNC_CQ_COMPLETION:
+ hca_ptr->async_cq =
+ (ib_async_dto_handler_t)callback;
+ break;
+ case DAPL_ASYNC_QP_ERROR:
+ hca_ptr->async_qp_error =
+ (ib_async_qp_handler_t)callback;
+ break;
+ default:
+ break;
+ }
+ return DAT_SUCCESS;
+}
+
+int dapli_ib_thread_init(void)
+{
+ long opts;
+ DAT_RETURN ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_init(%d)\n", getpid());
+
+ dapl_os_lock(&g_hca_lock);
+ if (g_ib_thread_state != IB_THREAD_INIT) {
+ dapl_os_unlock(&g_hca_lock);
+ return 0;
+ }
+
+ /* uCMA events non-blocking */
+ opts = fcntl(rdma_get_fd(), F_GETFL); /* uCMA */
+ if (opts < 0 || fcntl(rdma_get_fd(),
+ F_SETFL, opts | O_NONBLOCK) < 0) {
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR,
+ " dapl_ib_init: ERR with uCMA FD\n" );
+ dapl_os_unlock(&g_hca_lock);
+ return 1;
+ }
+
+ g_ib_thread_state = IB_THREAD_CREATE;
+ dapl_os_unlock(&g_hca_lock);
+
+ /* create thread to process inbound connect request */
+ ret = dapl_os_thread_create(dapli_thread, NULL, &g_ib_thread);
+ if (ret != DAT_SUCCESS)
+ {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " ib_thread_init: failed to create thread\n");
+ return 1;
+ }
+
+ /* wait for thread to start */
+ dapl_os_lock(&g_hca_lock);
+ while (g_ib_thread_state != IB_THREAD_RUN) {
+ struct timespec sleep, remain;
+ sleep.tv_sec = 0;
+ sleep.tv_nsec = 20000000; /* 20 ms */
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_init: waiting for ib_thread\n");
+ dapl_os_unlock(&g_hca_lock);
+ nanosleep (&sleep, &remain);
+ dapl_os_lock(&g_hca_lock);
+ }
+ dapl_os_unlock(&g_hca_lock);
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_init(%d) exit\n",getpid());
+ return 0;
+}
+
+void dapli_ib_thread_destroy(void)
+{
+ int retries = 10;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_destroy(%d)\n", getpid());
+ /*
+ * wait for async thread to terminate.
+ * pthread_join would be the correct method
+ * but some applications have some issues
+ */
+
+ /* destroy ib_thread, wait for termination, if not already */
+ dapl_os_lock(&g_hca_lock);
+ if (g_ib_thread_state != IB_THREAD_RUN)
+ goto bail;
+
+ g_ib_thread_state = IB_THREAD_CANCEL;
+ write(g_ib_pipe[1], "w", sizeof "w");
+ while ((g_ib_thread_state != IB_THREAD_EXIT) && (retries--)) {
+ struct timespec sleep, remain;
+ sleep.tv_sec = 0;
+ sleep.tv_nsec = 20000000; /* 20 ms */
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_destroy: waiting for ib_thread\n");
+ write(g_ib_pipe[1], "w", sizeof "w");
+ dapl_os_unlock( &g_hca_lock );
+ nanosleep(&sleep, &remain);
+ dapl_os_lock( &g_hca_lock );
+ }
+
+bail:
+ dapl_os_unlock( &g_hca_lock );
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " ib_thread_destroy(%d) exit\n",getpid());
+}
+
+void dapli_async_event_cb(struct _ib_hca_transport *hca)
+{
+ struct ibv_async_event event;
+ struct pollfd async_fd = {
+ .fd = hca->cm_id->verbs->async_fd,
+ .events = POLLIN,
+ .revents = 0
+ };
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " async_event(%p)\n",hca);
+
+ if (hca->destroy)
+ return;
+
+ if ((poll(&async_fd, 1, 0)==1) &&
+ (!ibv_get_async_event(hca->cm_id->verbs, &event))) {
+
+ switch (event.event_type) {
+ case IBV_EVENT_CQ_ERR:
+ {
+ struct dapl_ep *evd_ptr =
+ event.element.cq->cq_context;
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_WARN,
+ " async_event CQ (%p) ERR %d\n",
+ evd_ptr, event.event_type);
+
+ /* report up if async callback still setup */
+ if (hca->async_cq_error)
+ hca->async_cq_error(hca->cm_id->verbs,
+ event.element.cq,
+ &event,
+ (void*)evd_ptr);
+ break;
+ }
+ case IBV_EVENT_COMM_EST:
+ {
+ /* Received msgs on connected QP before RTU */
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_UTIL,
+ " async_event COMM_EST(%p) rdata beat RTU\n",
+ event.element.qp);
+
+ break;
+ }
+ case IBV_EVENT_QP_FATAL:
+ case IBV_EVENT_QP_REQ_ERR:
+ case IBV_EVENT_QP_ACCESS_ERR:
+ case IBV_EVENT_QP_LAST_WQE_REACHED:
+ case IBV_EVENT_SRQ_ERR:
+ case IBV_EVENT_SRQ_LIMIT_REACHED:
+ case IBV_EVENT_SQ_DRAINED:
+ {
+ struct dapl_ep *ep_ptr =
+ event.element.qp->qp_context;
+
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_WARN,
+ " async_event QP (%p) ERR %d\n",
+ ep_ptr, event.event_type);
+
+ /* report up if async callback still setup */
+ if (hca->async_qp_error)
+ hca->async_qp_error(hca->cm_id->verbs,
+ ep_ptr->qp_handle,
+ &event,
+ (void*)ep_ptr);
+ break;
+ }
+ case IBV_EVENT_PATH_MIG:
+ case IBV_EVENT_PATH_MIG_ERR:
+ case IBV_EVENT_DEVICE_FATAL:
+ case IBV_EVENT_PORT_ACTIVE:
+ case IBV_EVENT_PORT_ERR:
+ case IBV_EVENT_LID_CHANGE:
+ case IBV_EVENT_PKEY_CHANGE:
+ case IBV_EVENT_SM_CHANGE:
+ {
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " async_event: DEV ERR %d\n",
+ event.event_type);
+
+ /* report up if async callback still setup */
+ if (hca->async_unafiliated)
+ hca->async_unafiliated(
+ hca->cm_id->verbs,
+ &event,
+ hca->async_un_ctx);
+ break;
+ }
+ default:
+ dapl_dbg_log (DAPL_DBG_TYPE_WARN,
+ " async_event: UNKNOWN\n");
+ break;
+
+ }
+ ibv_ack_async_event(&event);
+ }
+}
+
+/* work thread for uAT, uCM, CQ, and async events */
+void dapli_thread(void *arg)
+{
+ struct pollfd ufds[__FD_SETSIZE];
+ struct _ib_hca_transport *uhca[__FD_SETSIZE]={NULL};
+ struct _ib_hca_transport *hca;
+ int ret,idx,fds;
+ char rbuf[2];
+
+ dapl_dbg_log (DAPL_DBG_TYPE_CM,
+ " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n",
+ getpid(), g_ib_thread, g_ib_pipe[0], rdma_get_fd());
+
+ /* Poll across pipe, CM, AT never changes */
+ dapl_os_lock( &g_hca_lock );
+ g_ib_thread_state = IB_THREAD_RUN;
+
+ ufds[0].fd = g_ib_pipe[0]; /* pipe */
+ ufds[0].events = POLLIN;
+ ufds[1].fd = rdma_get_fd(); /* uCMA */
+ ufds[1].events = POLLIN;
+
+ while (g_ib_thread_state == IB_THREAD_RUN) {
+
+ /* build ufds after pipe and uCMA events */
+ ufds[0].revents = 0;
+ ufds[1].revents = 0;
+ idx=1;
+
+ /* Walk HCA list and setup async and CQ events */
+ if (!dapl_llist_is_empty(&g_hca_list))
+ hca = dapl_llist_peek_head(&g_hca_list);
+ else
+ hca = NULL;
+
+ while(hca) {
+
+ /* uASYNC events */
+ ufds[++idx].fd = hca->cm_id->verbs->async_fd;
+ ufds[idx].events = POLLIN;
+ ufds[idx].revents = 0;
+ uhca[idx] = hca;
+
+ /* uCQ, non-direct events */
+ ufds[++idx].fd = hca->ib_cq->fd;
+ ufds[idx].events = POLLIN;
+ ufds[idx].revents = 0;
+ uhca[idx] = hca;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " ib_thread(%d) poll_fd: hca[%d]=%p, async=%d"
+ " pipe=%d cm=%d cq=d\n",
+ getpid(), hca, ufds[idx-1].fd,
+ ufds[0].fd, ufds[1].fd, ufds[idx].fd);
+
+ hca = dapl_llist_next_entry(
+ &g_hca_list,
+ (DAPL_LLIST_ENTRY*)&hca->entry);
+ }
+
+ /* unlock, and setup poll */
+ fds = idx+1;
+ dapl_os_unlock(&g_hca_lock);
+ ret = poll(ufds, fds, -1);
+ if (ret <= 0) {
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " ib_thread(%d): ERR %s poll\n",
+ getpid(),strerror(errno));
+ dapl_os_lock(&g_hca_lock);
+ continue;
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " ib_thread(%d) poll_event: "
+ " async=0x%x pipe=0x%x cm=0x%x cq=0x%x\n",
+ getpid(), ufds[idx-1].revents, ufds[0].revents,
+ ufds[1].revents, ufds[idx].revents);
+
+ /* uCMA events */
+ if (ufds[1].revents == POLLIN)
+ dapli_cma_event_cb();
+
+ /* check and process CQ and ASYNC events, per device */
+ for(idx=2;idx<fds;idx++) {
+ if (ufds[idx].revents == POLLIN) {
+ dapli_cq_event_cb(uhca[idx]);
+ dapli_async_event_cb(uhca[idx]);
+ }
+ }
+
+ /* check and process user events, PIPE */
+ if (ufds[0].revents == POLLIN) {
+
+ read(g_ib_pipe[0], rbuf, 2);
+
+ /* cleanup any device on list marked for destroy */
+ for(idx=3;idx<fds;idx++) {
+ if(uhca[idx] && uhca[idx]->destroy == 1) {
+ dapl_os_lock(&g_hca_lock);
+ dapl_llist_remove_entry(
+ &g_hca_list,
+ (DAPL_LLIST_ENTRY*)
+ &uhca[idx]->entry);
+ dapl_os_unlock(&g_hca_lock);
+ uhca[idx]->destroy = 2;
+ }
+ }
+ }
+ dapl_os_lock(&g_hca_lock);
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL," ib_thread(%d) EXIT\n",getpid());
+ g_ib_thread_state = IB_THREAD_EXIT;
+ dapl_os_unlock(&g_hca_lock);
+}
+
Index: dapl/openib_cma/dapl_ib_mem.c
===================================================================
--- dapl/openib_cma/dapl_ib_mem.c (revision 0)
+++ dapl/openib_cma/dapl_ib_mem.c (revision 0)
@@ -0,0 +1,391 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/**********************************************************************
+ *
+ * MODULE: dapl_det_mem.c
+ *
+ * PURPOSE: Intel DET APIs: Memory windows, registration,
+ * and protection domain
+ *
+ * $Id: $
+ *
+ **********************************************************************/
+
+#include <sys/ioctl.h> /* for IOCTL's */
+#include <sys/types.h> /* for socket(2) and related bits and pieces */
+#include <sys/socket.h> /* for socket(2) */
+#include <net/if.h> /* for struct ifreq */
+#include <net/if_arp.h> /* for ARPHRD_ETHER */
+#include <unistd.h> /* for _SC_CLK_TCK */
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+#include "dapl_lmr_util.h"
+
+/*
+ * dapls_convert_privileges
+ *
+ * Convert LMR privileges to provider
+ *
+ * Input:
+ * DAT_MEM_PRIV_FLAGS
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * ibv_access_flags
+ *
+ */
+STATIC _INLINE_ int
+dapls_convert_privileges(IN DAT_MEM_PRIV_FLAGS privileges)
+{
+ int access = 0;
+
+ /*
+ * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & privileges) do nothing
+ */
+ if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & privileges)
+ access |= IBV_ACCESS_LOCAL_WRITE;
+ if (DAT_MEM_PRIV_REMOTE_WRITE_FLAG & privileges)
+ access |= IBV_ACCESS_REMOTE_WRITE;
+ if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges)
+ access |= IBV_ACCESS_REMOTE_READ;
+ if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges)
+ access |= IBV_ACCESS_REMOTE_READ;
+ if (DAT_MEM_PRIV_REMOTE_READ_FLAG & privileges)
+ access |= IBV_ACCESS_REMOTE_READ;
+
+ return access;
+}
+
+/*
+ * dapl_ib_pd_alloc
+ *
+ * Alloc a PD
+ *
+ * Input:
+ * ia_handle IA handle
+ * pz pointer to PZ struct
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_pd_alloc(IN DAPL_IA *ia_ptr, IN DAPL_PZ *pz)
+{
+ /* get a protection domain */
+ pz->pd_handle = ibv_alloc_pd(ia_ptr->hca_ptr->ib_hca_handle);
+ if (!pz->pd_handle)
+ return(dapl_convert_errno(ENOMEM,"alloc_pd"));
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " pd_alloc: pd_handle=%p\n",
+ pz->pd_handle );
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapl_ib_pd_free
+ *
+ * Free a PD
+ *
+ * Input:
+ * ia_handle IA handle
+ * PZ_ptr pointer to PZ struct
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_STATE
+ *
+ */
+DAT_RETURN
+dapls_ib_pd_free(IN DAPL_PZ *pz )
+{
+ if (pz->pd_handle != IB_INVALID_HANDLE) {
+ if (ibv_dealloc_pd(pz->pd_handle))
+ return(dapl_convert_errno(errno,"dealloc_pd"));
+ pz->pd_handle = IB_INVALID_HANDLE;
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapl_ib_mr_register
+ *
+ * Register a virtual memory region
+ *
+ * Input:
+ * ia_handle IA handle
+ * lmr pointer to dapl_lmr struct
+ * virt_addr virtual address of beginning of mem region
+ * length length of memory region
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_mr_register(IN DAPL_IA *ia_ptr,
+ IN DAPL_LMR *lmr,
+ IN DAT_PVOID virt_addr,
+ IN DAT_VLEN length,
+ IN DAT_MEM_PRIV_FLAGS privileges)
+{
+ ib_pd_handle_t ib_pd_handle;
+
+ ib_pd_handle = ((DAPL_PZ *)lmr->param.pz_handle)->pd_handle;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " mr_register: ia=%p, lmr=%p va=%p ln=%d pv=0x%x\n",
+ ia_ptr, lmr, virt_addr, length, privileges );
+
+ /* TODO: shared memory */
+ if (lmr->param.mem_type == DAT_MEM_TYPE_SHARED_VIRTUAL) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mr_register_shared: NOT IMPLEMENTED\n");
+ return DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+ }
+
+ /* local read is default on IB */
+ lmr->mr_handle =
+ ibv_reg_mr(((DAPL_PZ *)lmr->param.pz_handle)->pd_handle,
+ virt_addr,
+ length,
+ dapls_convert_privileges(privileges));
+
+ if (!lmr->mr_handle)
+ return(dapl_convert_errno(ENOMEM,"reg_mr"));
+
+ lmr->param.lmr_context = lmr->mr_handle->lkey;
+ lmr->param.rmr_context = lmr->mr_handle->rkey;
+ lmr->param.registered_size = length;
+ lmr->param.registered_address = (DAT_VADDR)(uintptr_t)virt_addr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " mr_register: mr=%p h %x pd %p ctx %p "
+ "lkey=0x%x rkey=0x%x priv=%x\n",
+ lmr->mr_handle, lmr->mr_handle->handle,
+ lmr->mr_handle->pd, lmr->mr_handle->context,
+ lmr->mr_handle->lkey, lmr->mr_handle->rkey,
+ length, dapls_convert_privileges(privileges));
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapl_ib_mr_deregister
+ *
+ * Free a memory region
+ *
+ * Input:
+ * lmr pointer to dapl_lmr struct
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_STATE
+ *
+ */
+DAT_RETURN
+dapls_ib_mr_deregister(IN DAPL_LMR *lmr)
+{
+ if (lmr->mr_handle != IB_INVALID_HANDLE) {
+ if (ibv_dereg_mr(lmr->mr_handle))
+ return(dapl_convert_errno(errno,"dereg_pd"));
+ lmr->mr_handle = IB_INVALID_HANDLE;
+ }
+ return DAT_SUCCESS;
+}
+
+
+/*
+ * dapl_ib_mr_register_shared
+ *
+ * Register a virtual memory region
+ *
+ * Input:
+ * ia_ptr IA handle
+ * lmr pointer to dapl_lmr struct
+ * virt_addr virtual address of beginning of mem region
+ * length length of memory region
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_mr_register_shared(IN DAPL_IA *ia_ptr,
+ IN DAPL_LMR *lmr,
+ IN DAT_MEM_PRIV_FLAGS privileges)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mr_register_shared: NOT IMPLEMENTED\n");
+
+ return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+}
+
+/*
+ * dapls_ib_mw_alloc
+ *
+ * Bind a protection domain to a memory window
+ *
+ * Input:
+ * rmr Initialized rmr to hold binding handles
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_mw_alloc (IN DAPL_RMR *rmr)
+{
+
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mw_alloc: NOT IMPLEMENTED\n");
+
+ return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+}
+
+/*
+ * dapls_ib_mw_free
+ *
+ * Release bindings of a protection domain to a memory window
+ *
+ * Input:
+ * rmr Initialized rmr to hold binding handles
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_STATE
+ *
+ */
+DAT_RETURN
+dapls_ib_mw_free(IN DAPL_RMR *rmr)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mw_free: NOT IMPLEMENTED\n");
+
+ return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+}
+
+/*
+ * dapls_ib_mw_bind
+ *
+ * Bind a protection domain to a memory window
+ *
+ * Input:
+ * rmr Initialized rmr to hold binding handles
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_PARAMETER;
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_mw_bind(IN DAPL_RMR *rmr,
+ IN DAPL_LMR *lmr,
+ IN DAPL_EP *ep,
+ IN DAPL_COOKIE *cookie,
+ IN DAT_VADDR virtual_address,
+ IN DAT_VLEN length,
+ IN DAT_MEM_PRIV_FLAGS mem_priv,
+ IN DAT_BOOLEAN is_signaled)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mw_bind: NOT IMPLEMENTED\n");
+
+ return DAT_ERROR (DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+}
+
+/*
+ * dapls_ib_mw_unbind
+ *
+ * Unbind a protection domain from a memory window
+ *
+ * Input:
+ * rmr Initialized rmr to hold binding handles
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_PARAMETER;
+ * DAT_INVALID_STATE;
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_mw_unbind(IN DAPL_RMR *rmr,
+ IN DAPL_EP *ep,
+ IN DAPL_COOKIE *cookie,
+ IN DAT_BOOLEAN is_signaled )
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " mw_unbind: NOT IMPLEMENTED\n");
+
+ return DAT_ERROR(DAT_NOT_IMPLEMENTED, DAT_NO_SUBTYPE);
+}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * tab-width: 8
+ * End:
+ */
+
Index: dapl/openib_cma/dapl_ib_cm.c
===================================================================
--- dapl/openib_cma/dapl_ib_cm.c (revision 0)
+++ dapl/openib_cma/dapl_ib_cm.c (revision 0)
@@ -0,0 +1,1107 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/***************************************************************************
+ *
+ * Module: uDAPL
+ *
+ * Filename: dapl_ib_cm.c
+ *
+ * Author: Arlin Davis
+ *
+ * Created: 3/10/2005
+ *
+ * Description:
+ *
+ * The uDAPL openib provider - connection management
+ *
+ ****************************************************************************
+ * Source Control System Information
+ *
+ * $Id: $
+ *
+ * Copyright (c) 2005 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004-2005, Mellanox Technologies, Inc. All rights reserved.
+ * Copyright (c) 2003 Topspin Corporation. All rights reserved.
+ * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ *
+ **************************************************************************/
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+#include "dapl_evd_util.h"
+#include "dapl_cr_util.h"
+#include "dapl_name_service.h"
+#include "dapl_ib_util.h"
+#include <sys/poll.h>
+#include <signal.h>
+
+/* local prototypes */
+static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event);
+static int dapli_cm_active_cb(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event);
+static int dapli_cm_passive_cb(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event);
+static void dapli_addr_resolve(struct dapl_cm_id *conn);
+static void dapli_route_resolve(struct dapl_cm_id *conn);
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+#elif __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+#endif
+
+/* cma requires 16 bit SID */
+#define IB_PORT_MOD 32001
+#define IB_PORT_BASE (65535 - IB_PORT_MOD)
+#define MAKE_PORT(SID) \
+ (SID > 0xffff ? \
+ (unsigned short)((SID % IB_PORT_MOD) + IB_PORT_BASE) :\
+ (unsigned short)SID)
+
+
+static void dapli_addr_resolve(struct dapl_cm_id *conn)
+{
+ int ret;
+ struct rdma_addr *ipaddr = &conn->cm_id->route.addr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " addr_resolve: cm_id %p SRC %x DST %x\n",
+ conn->cm_id,
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_addr.s_addr),
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_addr.s_addr));
+
+ ret = rdma_resolve_route(conn->cm_id, 2000);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " rdma_connect failed: %s\n",strerror(errno));
+
+ dapl_evd_connection_callback(conn,
+ IB_CME_LOCAL_FAILURE,
+ NULL, conn->ep);
+ }
+}
+
+static void dapli_route_resolve(struct dapl_cm_id *conn)
+{
+ int ret;
+ struct rdma_cm_id *cm_id = conn->cm_id;
+ struct rdma_addr *ipaddr = &cm_id->route.addr;
+ struct ib_addr *ibaddr = &cm_id->route.addr.addr.ibaddr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " route_resolve: cm_id %p SRC %x DST %x PORT %d\n",
+ conn->cm_id,
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_addr.s_addr),
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_port) );
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " route_resolve: SRC GID subnet %016llx id %016llx\n",
+ (unsigned long long)
+ cpu_to_be64(ibaddr->sgid.global.subnet_prefix),
+ (unsigned long long)
+ cpu_to_be64(ibaddr->sgid.global.interface_id));
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " route_resolve: DST GID subnet %016llx id %016llx\n",
+ (unsigned long long)
+ cpu_to_be64(ibaddr->dgid.global.subnet_prefix),
+ (unsigned long long)
+ cpu_to_be64(ibaddr->dgid.global.interface_id));
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " rdma_connect: cm_id %p pdata %p plen %d rr %d ind %d\n",
+ conn->cm_id,
+ conn->params.private_data,
+ conn->params.private_data_len,
+ conn->params.responder_resources,
+ conn->params.initiator_depth );
+
+ ret = rdma_connect(conn->cm_id, &conn->params);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " rdma_connect failed: %s\n",
+ strerror(errno));
+ goto bail;
+ }
+ return;
+
+bail:
+ dapl_evd_connection_callback(conn,
+ IB_CME_LOCAL_FAILURE,
+ NULL, conn->ep);
+}
+
+void dapli_destroy_conn(struct dapl_cm_id *conn)
+{
+ int in_callback;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " destroy_conn: conn %p id %d\n",
+ conn,conn->cm_id);
+
+ dapl_os_lock(&conn->lock);
+ conn->destroy = 1;
+ in_callback = conn->in_callback;
+ dapl_os_unlock(&conn->lock);
+
+ if (!in_callback) {
+ if (conn->ep)
+ conn->ep->cm_handle = IB_INVALID_HANDLE;
+ if (conn->cm_id) {
+ if (conn->cm_id->qp)
+ rdma_destroy_qp(conn->cm_id);
+ rdma_destroy_id(conn->cm_id);
+ }
+
+ conn->cm_id = NULL;
+ dapl_os_free(conn, sizeof(*conn));
+ }
+}
+
+static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event)
+{
+ struct dapl_cm_id *new_conn;
+ struct rdma_addr *ipaddr = &event->id->route.addr;
+
+ if (conn->sp == NULL) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " dapli_rep_recv: on invalid listen "
+ "handle\n");
+ return NULL;
+ }
+
+ /* allocate new cm_id and merge listen parameters */
+ new_conn = dapl_os_alloc(sizeof(*new_conn));
+ if (new_conn) {
+ (void)dapl_os_memzero(new_conn, sizeof(*new_conn));
+ new_conn->cm_id = event->id; /* provided by uCMA */
+ event->id->context = new_conn; /* update CM_ID context */
+ new_conn->sp = conn->sp;
+ new_conn->hca = conn->hca;
+
+ /* save private data */
+ if (event->private_data_len) {
+ dapl_os_memcpy(new_conn->p_data,
+ event->private_data,
+ event->private_data_len);
+ new_conn->params.private_data = new_conn->p_data;
+ new_conn->params.private_data_len =
+ event->private_data_len;
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: "
+ "REQ: SP %p PORT %d LID %d "
+ "NEW CONN %p ID %p pD %p,%d\n",
+ new_conn->sp,
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_port),
+ event->listen_id, new_conn, event->id,
+ event->private_data, event->private_data_len);
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: "
+ "REQ: IP SRC %x PORT %d DST %x PORT %d\n",
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_port),
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_port));
+ }
+ return new_conn;
+}
+
+static int dapli_cm_active_cb(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event)
+{
+ int destroy;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " active_cb: conn %p id %d event %d\n",
+ conn, conn->cm_id, event->event );
+
+ dapl_os_lock(&conn->lock);
+ if (conn->destroy) {
+ dapl_os_unlock(&conn->lock);
+ return 0;
+ }
+ conn->in_callback = 1;
+ dapl_os_unlock(&conn->lock);
+
+ switch (event->event) {
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ dapl_evd_connection_callback(conn,
+ IB_CME_DESTINATION_UNREACHABLE,
+ NULL, conn->ep);
+ break;
+ case RDMA_CM_EVENT_REJECTED:
+ dapl_evd_connection_callback(conn, IB_CME_DESTINATION_REJECT,
+ NULL, conn->ep);
+ break;
+
+ case RDMA_CM_EVENT_ESTABLISHED:
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " active_cb: cm_id %d PORT %d CONNECTED to 0x%x!\n",
+ conn->cm_id,
+ ntohs(((struct sockaddr_in *)
+ &conn->cm_id->route.addr.dst_addr)->sin_port),
+ ntohl(((struct sockaddr_in *)
+ &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
+
+ dapl_evd_connection_callback(conn, IB_CME_CONNECTED,
+ event->private_data, conn->ep);
+ break;
+
+ case RDMA_CM_EVENT_DISCONNECTED:
+ break;
+ default:
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_ERR,
+ " dapli_cm_active_cb_handler: Unexpected CM "
+ "event %d on ID 0x%p\n", event->event, conn->cm_id);
+ break;
+ }
+
+ dapl_os_lock(&conn->lock);
+ destroy = conn->destroy;
+ conn->in_callback = conn->destroy;
+ dapl_os_unlock(&conn->lock);
+ if (destroy) {
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " active_cb: DESTROY conn %p id %d \n",
+ conn, conn->cm_id );
+ if (conn->ep)
+ conn->ep->cm_handle = IB_INVALID_HANDLE;
+
+ dapl_os_free(conn, sizeof(*conn));
+ }
+ return(destroy);
+}
+
+static int dapli_cm_passive_cb(struct dapl_cm_id *conn,
+ struct rdma_cm_event *event)
+{
+ int destroy;
+ struct dapl_cm_id *new_conn;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " passive_cb: conn %p id %d event %d\n",
+ conn, event->id, event->event);
+
+ dapl_os_lock(&conn->lock);
+ if (conn->destroy) {
+ dapl_os_unlock(&conn->lock);
+ return 0;
+ }
+ conn->in_callback = 1;
+ dapl_os_unlock(&conn->lock);
+
+ switch (event->event) {
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ /* create new conn object with new conn_id from event */
+ new_conn = dapli_req_recv(conn,event);
+
+ if (new_conn)
+ dapls_cr_callback(new_conn,
+ IB_CME_CONNECTION_REQUEST_PENDING,
+ event->private_data, new_conn->sp);
+ break;
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ dapls_cr_callback(conn, IB_CME_DESTINATION_UNREACHABLE,
+ NULL, conn->sp);
+ break;
+ case RDMA_CM_EVENT_REJECTED:
+ dapls_cr_callback(conn, IB_CME_DESTINATION_REJECT, NULL,
+ conn->sp);
+ break;
+ case RDMA_CM_EVENT_ESTABLISHED:
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " passive_cb: cm_id %p PORT %d CONNECTED from 0x%x!\n",
+ conn->cm_id,
+ ntohs(((struct sockaddr_in *)
+ &conn->cm_id->route.addr.src_addr)->sin_port),
+ ntohl(((struct sockaddr_in *)
+ &conn->cm_id->route.addr.dst_addr)->sin_addr.s_addr));
+
+ dapls_cr_callback(conn, IB_CME_CONNECTED,
+ NULL, conn->sp);
+
+ break;
+ case RDMA_CM_EVENT_DISCONNECTED:
+ break;
+ default:
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " passive_cb: "
+ "Unexpected CM event %d on ID 0x%p\n",
+ event->event, conn->cm_id);
+ break;
+ }
+
+ dapl_os_lock(&conn->lock);
+ destroy = conn->destroy;
+ conn->in_callback = conn->destroy;
+ dapl_os_unlock(&conn->lock);
+ if (destroy) {
+ if (conn->ep)
+ conn->ep->cm_handle = IB_INVALID_HANDLE;
+
+ dapl_os_free(conn, sizeof(*conn));
+ }
+ return(destroy);
+}
+
+
+/************************ DAPL provider entry points **********************/
+
+/*
+ * dapls_ib_connect
+ *
+ * Initiate a connection with the passive listener on another node
+ *
+ * Input:
+ * ep_handle,
+ * remote_ia_address,
+ * remote_conn_qual,
+ * prd_size size of private data and structure
+ * prd_prt pointer to private data structure
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INVALID_PARAMETER
+ *
+ */
+DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
+ IN DAT_IA_ADDRESS_PTR r_addr,
+ IN DAT_CONN_QUAL r_qual,
+ IN DAT_COUNT p_size,
+ IN void *p_data)
+{
+ struct dapl_ep *ep_ptr = ep_handle;
+
+ /* Sanity check */
+ if (NULL == ep_ptr)
+ return DAT_SUCCESS;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, " connect: rSID %d, pdata %p, ln %d\n",
+ r_qual,p_data,p_size);
+
+ /* rdma conn and cm_id pre-bound; reference via qp_handle */
+ ep_ptr->cm_handle = ep_ptr->qp_handle;
+
+ /* Setup QP/CM parameters and private data in cm_id */
+ (void)dapl_os_memzero(&ep_ptr->cm_handle->params,
+ sizeof(ep_ptr->cm_handle->params));
+ ep_ptr->cm_handle->params.responder_resources = IB_TARGET_MAX;
+ ep_ptr->cm_handle->params.initiator_depth = IB_INITIATOR_DEPTH;
+ ep_ptr->cm_handle->params.flow_control = 1;
+ ep_ptr->cm_handle->params.rnr_retry_count = IB_RNR_RETRY_COUNT;
+ ep_ptr->cm_handle->params.retry_count = IB_RC_RETRY_COUNT;
+ if (p_size) {
+ dapl_os_memcpy(ep_ptr->cm_handle->p_data, p_data, p_size);
+ ep_ptr->cm_handle->params.private_data =
+ ep_ptr->cm_handle->p_data;
+ ep_ptr->cm_handle->params.private_data_len = p_size;
+ }
+
+ /* Resolve remote address, src already bound during QP create */
+ ((struct sockaddr_in*)r_addr)->sin_port = htons(MAKE_PORT(r_qual));
+ if (rdma_resolve_addr(ep_ptr->cm_handle->cm_id,
+ NULL, (struct sockaddr *)r_addr, 2000))
+ return dapl_convert_errno(errno,"ib_connect");
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " connect: resolve_addr: cm_id %p SRC %x DST %x port %d\n",
+ ep_ptr->cm_handle->cm_id,
+ ntohl(((struct sockaddr_in *)
+ &ep_ptr->cm_handle->hca->hca_address)->sin_addr.s_addr),
+ ntohl(((struct sockaddr_in *)r_addr)->sin_addr.s_addr),
+ MAKE_PORT(r_qual) );
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_disconnect
+ *
+ * Disconnect an EP
+ *
+ * Input:
+ * ep_handle,
+ * disconnect_flags
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ *
+ */
+DAT_RETURN
+dapls_ib_disconnect(IN DAPL_EP *ep_ptr,
+ IN DAT_CLOSE_FLAGS close_flags)
+{
+ ib_cm_handle_t conn = ep_ptr->cm_handle;
+ int ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " disconnect(ep %p, conn %p, id %d flags %x)\n",
+ ep_ptr,conn, (conn?conn->cm_id:0),close_flags);
+
+ if (conn == IB_INVALID_HANDLE)
+ return DAT_SUCCESS;
+
+ /* no graceful half-pipe disconnect option */
+ ret = rdma_disconnect(conn->cm_id);
+ if (ret)
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " disconnect: ID %p ret %d\n",
+ ep_ptr->cm_handle, ret);
+
+ /*
+ * uDAPL does NOT expect disconnect callback from provider
+ * with abrupt close. uDAPL will callback with DISC event when
+ * from provider returns. So, if callback is expected from
+ * rdma_cma then block and don't post the event during callback.
+ */
+ if (close_flags != DAT_CLOSE_ABRUPT_FLAG)
+ {
+ if (ep_ptr->cr_ptr)
+ dapls_cr_callback(conn, IB_CME_DISCONNECTED, NULL,
+ ((DAPL_CR *)ep_ptr->cr_ptr)->sp_ptr);
+ else
+ dapl_evd_connection_callback(conn, IB_CME_DISCONNECTED,
+ NULL, ep_ptr);
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_disconnect_clean
+ *
+ * Clean up outstanding connection data. This routine is invoked
+ * after the final disconnect callback has occurred. Only on the
+ * ACTIVE side of a connection.
+ *
+ * Input:
+ * ep_ptr DAPL_EP
+ * active Indicates active side of connection
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * void
+ *
+ */
+void
+dapls_ib_disconnect_clean(IN DAPL_EP *ep_ptr,
+ IN DAT_BOOLEAN active,
+ IN const ib_cm_events_t ib_cm_event)
+{
+ /*
+ * Clean up outstanding connection state
+ */
+ dapls_ib_disconnect(ep_ptr, DAT_CLOSE_ABRUPT_FLAG);
+
+}
+
+/*
+ * dapl_ib_setup_conn_listener
+ *
+ * Have the CM set up a connection listener.
+ *
+ * Input:
+ * ibm_hca_handle HCA handle
+ * qp_handle QP handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INTERNAL_ERROR
+ * DAT_CONN_QUAL_UNAVAILBLE
+ * DAT_CONN_QUAL_IN_USE
+ *
+ */
+DAT_RETURN
+dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr,
+ IN DAT_UINT64 ServiceID,
+ IN DAPL_SP *sp_ptr )
+{
+ DAT_RETURN dat_status = DAT_SUCCESS;
+ int status;
+ ib_cm_srvc_handle_t conn;
+
+ /* Allocate CM and initialize lock */
+ if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL)
+ return DAT_INSUFFICIENT_RESOURCES;
+
+ dapl_os_memzero(conn, sizeof(*conn));
+ dapl_os_lock_init(&conn->lock);
+
+ /* create CM_ID, bind to local device, create QP */
+ if (rdma_create_id(&conn->cm_id, (void*)conn)) {
+ dapl_os_free(conn, sizeof(*conn));
+ return(dapl_convert_errno(errno,"setup_listener"));
+ }
+
+ /* open identifies the local device; per DAT specification */
+ ((struct sockaddr_in *)&ia_ptr->hca_ptr->hca_address)->sin_port =
+ htons(MAKE_PORT(ServiceID));
+
+ if (rdma_bind_addr(conn->cm_id,
+ (struct sockaddr *)&ia_ptr->hca_ptr->hca_address)) {
+ dat_status = dapl_convert_errno(errno,"setup_listener");
+ goto bail;
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " listen(ia_ptr %p SID %d sp %p conn %p id %d)\n",
+ ia_ptr, MAKE_PORT(ServiceID),
+ sp_ptr, conn, conn->cm_id);
+
+ sp_ptr->cm_srvc_handle = conn;
+ conn->sp = sp_ptr;
+ conn->hca = ia_ptr->hca_ptr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " listen(conn=%p cm_id=%d)\n",
+ sp_ptr->cm_srvc_handle,conn->cm_id);
+
+ status = rdma_listen(conn->cm_id,64); /* backlog to 64 */
+
+ if (status) {
+ if (status == -EBUSY)
+ dat_status = DAT_CONN_QUAL_IN_USE;
+ else
+ dat_status =
+ dapl_convert_errno(errno,"setup_listener");
+
+ goto bail;
+ }
+
+ /* success */
+ return DAT_SUCCESS;
+
+bail:
+ rdma_destroy_id(conn->cm_id);
+ dapl_os_free(conn, sizeof(*conn));
+ return dat_status;
+}
+
+
+/*
+ * dapl_ib_remove_conn_listener
+ *
+ * Have the CM remove a connection listener.
+ *
+ * Input:
+ * ia_handle IA handle
+ * ServiceID IB Channel Service ID
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_STATE
+ *
+ */
+DAT_RETURN
+dapls_ib_remove_conn_listener(IN DAPL_IA *ia_ptr, IN DAPL_SP *sp_ptr)
+{
+ ib_cm_srvc_handle_t conn = sp_ptr->cm_srvc_handle;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " remove_listen(ia_ptr %p sp_ptr %p cm_ptr %p)\n",
+ ia_ptr, sp_ptr, conn );
+
+ if (conn != IB_INVALID_HANDLE) {
+ sp_ptr->cm_srvc_handle = NULL;
+ dapli_destroy_conn(conn);
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_accept_connection
+ *
+ * Perform necessary steps to accept a connection
+ *
+ * Input:
+ * cr_handle
+ * ep_handle
+ * private_data_size
+ * private_data
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INTERNAL_ERROR
+ *
+ */
+DAT_RETURN
+dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
+ IN DAT_EP_HANDLE ep_handle,
+ IN DAT_COUNT p_size,
+ IN const DAT_PVOID p_data)
+{
+ DAPL_CR *cr_ptr = (DAPL_CR *)cr_handle;
+ DAPL_EP *ep_ptr = (DAPL_EP *)ep_handle;
+ DAPL_IA *ia_ptr = ep_ptr->header.owner_ia;
+ struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle;
+ int ret;
+ DAT_RETURN dat_status;
+ struct rdma_conn_param conn_params;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n",
+ cr_ptr, cr_conn, cr_conn->cm_id, p_data, p_size );
+
+ /* Obtain size of private data structure & contents */
+ if (p_size > IB_MAX_REP_PDATA_SIZE) {
+ dat_status = DAT_ERROR(DAT_LENGTH_ERROR, DAT_NO_SUBTYPE);
+ goto bail;
+ }
+
+ if (ep_ptr->qp_state == DAPL_QP_STATE_UNATTACHED) {
+ /*
+ * If we are lazy attaching the QP then we may need to
+ * hook it up here. Typically, we run this code only for
+ * DAT_PSP_PROVIDER_FLAG
+ */
+ dat_status = dapls_ib_qp_alloc(ia_ptr, ep_ptr, NULL);
+ if (dat_status != DAT_SUCCESS) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " accept: ib_qp_alloc failed: %d\n",
+ dat_status);
+ goto bail;
+ }
+ }
+
+ /*
+ * Validate device and port in EP cm_id against inbound
+ * CR cm_id. The pre-allocated EP cm_id is already bound to
+ * a local device (cm_id and QP) when created. Move the QP
+ * to the new cm_id only if device and port numbers match.
+ */
+ if (ep_ptr->qp_handle->cm_id->verbs == cr_conn->cm_id->verbs &&
+ ep_ptr->qp_handle->cm_id->port_num == cr_conn->cm_id->port_num) {
+ /* move QP to new cr_conn, remove QP ref in EP cm_id */
+ cr_conn->cm_id->qp = ep_ptr->qp_handle->cm_id->qp;
+ ep_ptr->qp_handle->cm_id->qp = NULL;
+ dapli_destroy_conn(ep_ptr->qp_handle);
+ } else {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " accept: ERR dev(%p!=%p) or port mismatch(%d!=%d)\n",
+ ep_ptr->qp_handle->cm_id->verbs,cr_conn->cm_id->verbs,
+ ep_ptr->qp_handle->cm_id->port_num,
+ cr_conn->cm_id->port_num );
+ dat_status = DAT_INTERNAL_ERROR;
+ goto bail;
+ }
+
+ cr_ptr->param.local_ep_handle = ep_handle;
+ ep_ptr->qp_handle = cr_conn;
+ ep_ptr->cm_handle = cr_conn;
+ cr_conn->ep = ep_ptr;
+
+ memset(&conn_params, 0, sizeof(conn_params));
+ conn_params.private_data = p_data;
+ conn_params.private_data_len = p_size;
+ conn_params.responder_resources = IB_TARGET_MAX;
+ conn_params.initiator_depth = IB_INITIATOR_DEPTH;
+ conn_params.flow_control = 1;
+ conn_params.rnr_retry_count = IB_RNR_RETRY_COUNT;
+
+ ret = rdma_accept(cr_conn->cm_id, &conn_params);
+ if (ret) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR," accept: ERROR %d\n", ret);
+ dat_status = dapl_convert_errno(ret, "accept");
+ goto bail;
+ }
+
+ return DAT_SUCCESS;
+bail:
+ rdma_reject(cr_conn->cm_id, NULL, 0);
+ dapli_destroy_conn(cr_conn);
+ return dat_status;
+}
+
+
+/*
+ * dapls_ib_reject_connection
+ *
+ * Reject a connection
+ *
+ * Input:
+ * cr_handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INTERNAL_ERROR
+ *
+ */
+DAT_RETURN
+dapls_ib_reject_connection(IN ib_cm_handle_t cm_handle, IN int reason)
+{
+ int ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " reject(cm_handle %p reason %x)\n",
+ cm_handle, reason );
+
+ if (cm_handle == IB_INVALID_HANDLE) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ " reject: invalid handle: reason %d\n",
+ reason);
+ return DAT_SUCCESS;
+ }
+
+ ret = rdma_reject(cm_handle->cm_id, NULL, 0);
+
+ dapli_destroy_conn(cm_handle);
+ return dapl_convert_errno(ret, "reject");
+}
+
+/*
+ * dapls_ib_cm_remote_addr
+ *
+ * Obtain the remote IP address given a connection
+ *
+ * Input:
+ * cr_handle
+ *
+ * Output:
+ * remote_ia_address: where to place the remote address
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_HANDLE
+ *
+ */
+DAT_RETURN
+dapls_ib_cm_remote_addr(IN DAT_HANDLE dat_handle, OUT DAT_SOCK_ADDR6 *raddr)
+{
+ DAPL_HEADER *header;
+ ib_cm_handle_t ib_cm_handle;
+ struct rdma_addr *ipaddr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " remote_addr(cm_handle=%p, r_addr=%p)\n",
+ dat_handle, raddr);
+
+ header = (DAPL_HEADER *)dat_handle;
+
+ if (header->magic == DAPL_MAGIC_EP)
+ ib_cm_handle = ((DAPL_EP *)dat_handle)->cm_handle;
+ else if (header->magic == DAPL_MAGIC_CR)
+ ib_cm_handle = ((DAPL_CR *)dat_handle)->ib_cm_handle;
+ else
+ return DAT_INVALID_HANDLE;
+
+ /* get remote IP address from cm_id route */
+ ipaddr = &ib_cm_handle->cm_id->route.addr;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " remote_addr: conn %p id %p SRC %x DST %x PORT %d\n",
+ ib_cm_handle, ib_cm_handle->cm_id,
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->src_addr)->sin_addr.s_addr),
+ ntohl(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_addr.s_addr),
+ ntohs(((struct sockaddr_in *)
+ &ipaddr->dst_addr)->sin_port));
+
+ dapl_os_memcpy(raddr,&ipaddr->dst_addr,sizeof(DAT_SOCK_ADDR));
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_private_data_size
+ *
+ * Return the size of private data given a connection op type
+ *
+ * Input:
+ * prd_ptr private data pointer
+ * conn_op connection operation type
+ *
+ * If prd_ptr is NULL, this is a query for the max size supported by
+ * the provider, otherwise it is the actual size of the private data
+ * contained in prd_ptr.
+ *
+ *
+ * Output:
+ * None
+ *
+ * Returns:
+ * length of private data
+ *
+ */
+int dapls_ib_private_data_size(IN DAPL_PRIVATE *prd_ptr,
+ IN DAPL_PDATA_OP conn_op)
+{
+ int size;
+
+ switch(conn_op) {
+
+ case DAPL_PDATA_CONN_REQ:
+ size = IB_MAX_REQ_PDATA_SIZE;
+ break;
+ case DAPL_PDATA_CONN_REP:
+ size = IB_MAX_REP_PDATA_SIZE;
+ break;
+ case DAPL_PDATA_CONN_REJ:
+ size = IB_MAX_REJ_PDATA_SIZE;
+ break;
+ case DAPL_PDATA_CONN_DREQ:
+ size = IB_MAX_DREQ_PDATA_SIZE;
+ break;
+ case DAPL_PDATA_CONN_DREP:
+ size = IB_MAX_DREP_PDATA_SIZE;
+ break;
+ default:
+ size = 0;
+
+ } /* end case */
+
+ return size;
+}
+
+/*
+ * Map all socket CM event codes to the DAT equivelent.
+ */
+#define DAPL_IB_EVENT_CNT 12
+
+static struct ib_cm_event_map
+{
+ const ib_cm_events_t ib_cm_event;
+ DAT_EVENT_NUMBER dat_event_num;
+ } ib_cm_event_map[DAPL_IB_EVENT_CNT] = {
+ /* 00 */ { IB_CME_CONNECTED,
+ DAT_CONNECTION_EVENT_ESTABLISHED},
+ /* 01 */ { IB_CME_DISCONNECTED,
+ DAT_CONNECTION_EVENT_DISCONNECTED},
+ /* 02 */ { IB_CME_DISCONNECTED_ON_LINK_DOWN,
+ DAT_CONNECTION_EVENT_DISCONNECTED},
+ /* 03 */ { IB_CME_CONNECTION_REQUEST_PENDING,
+ DAT_CONNECTION_REQUEST_EVENT},
+ /* 04 */ { IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA,
+ DAT_CONNECTION_REQUEST_EVENT},
+ /* 05 */ { IB_CME_CONNECTION_REQUEST_ACKED,
+ DAT_CONNECTION_REQUEST_EVENT},
+ /* 06 */ { IB_CME_DESTINATION_REJECT,
+ DAT_CONNECTION_EVENT_NON_PEER_REJECTED},
+ /* 07 */ { IB_CME_DESTINATION_REJECT_PRIVATE_DATA,
+ DAT_CONNECTION_EVENT_PEER_REJECTED},
+ /* 08 */ { IB_CME_DESTINATION_UNREACHABLE,
+ DAT_CONNECTION_EVENT_UNREACHABLE},
+ /* 09 */ { IB_CME_TOO_MANY_CONNECTION_REQUESTS,
+ DAT_CONNECTION_EVENT_NON_PEER_REJECTED},
+ /* 10 */ { IB_CME_LOCAL_FAILURE,
+ DAT_CONNECTION_EVENT_BROKEN},
+ /* 11 */ { IB_CME_BROKEN,
+ DAT_CONNECTION_EVENT_BROKEN}
+};
+
+/*
+ * dapls_ib_get_cm_event
+ *
+ * Return a DAT connection event given a provider CM event.
+ *
+ * Input:
+ * dat_event_num DAT event we need an equivelent CM event for
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * ib_cm_event of translated DAPL value
+ */
+DAT_EVENT_NUMBER
+dapls_ib_get_dat_event(IN const ib_cm_events_t ib_cm_event,
+ IN DAT_BOOLEAN active)
+{
+ DAT_EVENT_NUMBER dat_event_num;
+ int i;
+
+ active = active;
+
+ if (ib_cm_event > IB_CME_BROKEN)
+ return (DAT_EVENT_NUMBER) 0;
+
+ dat_event_num = 0;
+ for(i = 0; i < DAPL_IB_EVENT_CNT; i++) {
+ if (ib_cm_event == ib_cm_event_map[i].ib_cm_event) {
+ dat_event_num = ib_cm_event_map[i].dat_event_num;
+ break;
+ }
+ }
+ dapl_dbg_log(DAPL_DBG_TYPE_CALLBACK,
+ "dapls_ib_get_dat_event: event(%s) ib=0x%x dat=0x%x\n",
+ active ? "active" : "passive", ib_cm_event, dat_event_num);
+
+ return dat_event_num;
+}
+
+
+/*
+ * dapls_ib_get_dat_event
+ *
+ * Return a DAT connection event given a provider CM event.
+ *
+ * Input:
+ * ib_cm_event event provided to the dapl callback routine
+ * active switch indicating active or passive connection
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_EVENT_NUMBER of translated provider value
+ */
+ib_cm_events_t
+dapls_ib_get_cm_event(IN DAT_EVENT_NUMBER dat_event_num)
+{
+ ib_cm_events_t ib_cm_event;
+ int i;
+
+ ib_cm_event = 0;
+ for(i = 0; i < DAPL_IB_EVENT_CNT; i++) {
+ if (dat_event_num == ib_cm_event_map[i].dat_event_num) {
+ ib_cm_event = ib_cm_event_map[i].ib_cm_event;
+ break;
+ }
+ }
+ return ib_cm_event;
+}
+
+
+void dapli_cma_event_cb(void)
+{
+ struct rdma_cm_event *event;
+ int ret;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n");
+
+ ret = rdma_get_cm_event(&event);
+
+ /* process one CM event, fairness */
+ if(!ret) {
+ struct dapl_cm_id *conn;
+ int ret;
+
+ /* set proper conn from cm_id context*/
+ if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST)
+ conn = (struct dapl_cm_id *)event->listen_id->context;
+ else
+ conn = (struct dapl_cm_id *)event->id->context;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " cm_event: EVENT=%d ID=%p LID=%p CTX=%p\n",
+ event->event, event->id, event->listen_id, conn);
+
+ switch (event->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ dapli_addr_resolve(conn);
+ break;
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ dapli_route_resolve(conn);
+ break;
+ case RDMA_CM_EVENT_ADDR_ERROR:
+ case RDMA_CM_EVENT_ROUTE_ERROR:
+ dapl_evd_connection_callback(conn,
+ IB_CME_LOCAL_FAILURE,
+ NULL, conn->ep);
+ break;
+ case RDMA_CM_EVENT_DEVICE_REMOVAL:
+ dapl_evd_connection_callback(conn,
+ IB_CME_LOCAL_FAILURE,
+ NULL, conn->ep);
+ break;
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_REJECTED:
+ case RDMA_CM_EVENT_ESTABLISHED:
+ case RDMA_CM_EVENT_DISCONNECTED:
+ /* passive or active */
+ if (conn->sp)
+ ret = dapli_cm_passive_cb(conn,event);
+ else
+ ret = dapli_cm_active_cb(conn,event);
+
+ if (ret)
+ rdma_destroy_id(conn->cm_id);
+
+ break;
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ default:
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " cm_event: UNEXPECTED EVENT=%p ID=%p CTX=%p\n",
+ event->event, event->id,
+ event->id->context);
+ break;
+ }
+ rdma_ack_cm_event(event);
+ } else {
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " cm_event: ERROR: rdma_get_cm_event() %d %d %s\n",
+ ret, errno, strerror(errno));
+ }
+}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * tab-width: 8
+ * End:
+ */
Index: dapl/openib_cma/dapl_ib_qp.c
===================================================================
--- dapl/openib_cma/dapl_ib_qp.c (revision 0)
+++ dapl/openib_cma/dapl_ib_qp.c (revision 0)
@@ -0,0 +1,305 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/**********************************************************************
+ *
+ * MODULE: dapl_det_qp.c
+ *
+ * PURPOSE: QP routines for access to DET Verbs
+ *
+ * $Id: $
+ **********************************************************************/
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+
+/*
+ * dapl_ib_qp_alloc
+ *
+ * Alloc a QP
+ *
+ * Input:
+ * *ep_ptr pointer to EP INFO
+ * ib_hca_handle provider HCA handle
+ * ib_pd_handle provider protection domain handle
+ * cq_recv provider recv CQ handle
+ * cq_send provider send CQ handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INTERNAL_ERROR
+ *
+ */
+DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA *ia_ptr,
+ IN DAPL_EP *ep_ptr,
+ IN DAPL_EP *ep_ctx_ptr)
+{
+ DAT_EP_ATTR *attr;
+ DAPL_EVD *rcv_evd, *req_evd;
+ ib_cq_handle_t rcv_cq, req_cq;
+ ib_pd_handle_t ib_pd_handle;
+ struct ibv_qp_init_attr qp_create;
+ ib_cm_handle_t conn;
+ struct rdma_cm_id *cm_id;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " qp_alloc: ia_ptr %p ep_ptr %p ep_ctx_ptr %p\n",
+ ia_ptr, ep_ptr, ep_ctx_ptr);
+
+ attr = &ep_ptr->param.ep_attr;
+ ib_pd_handle = ((DAPL_PZ *)ep_ptr->param.pz_handle)->pd_handle;
+ rcv_evd = (DAPL_EVD *) ep_ptr->param.recv_evd_handle;
+ req_evd = (DAPL_EVD *) ep_ptr->param.request_evd_handle;
+
+ /*
+ * DAT allows usage model of EP's with no EVD's but IB does not.
+ * Create a CQ with zero entries under the covers to support and
+ * catch any invalid posting.
+ */
+ if (rcv_evd != DAT_HANDLE_NULL)
+ rcv_cq = rcv_evd->ib_cq_handle;
+ else if (!ia_ptr->hca_ptr->ib_trans.ib_cq_empty)
+ rcv_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty;
+ else {
+ struct ibv_comp_channel *channel =
+ ia_ptr->hca_ptr->ib_trans.ib_cq;
+#ifdef CQ_WAIT_OBJECT
+ if (rcv_evd->cq_wait_obj_handle)
+ channel = rcv_evd->cq_wait_obj_handle;
+#endif
+ /* Call IB verbs to create CQ */
+ rcv_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle,
+ 0, NULL, channel, 0);
+
+ if (rcv_cq == IB_INVALID_HANDLE)
+ return(dapl_convert_errno(ENOMEM, "create_cq"));
+
+ ia_ptr->hca_ptr->ib_trans.ib_cq_empty = rcv_cq;
+ }
+ if (req_evd != DAT_HANDLE_NULL)
+ req_cq = req_evd->ib_cq_handle;
+ else
+ req_cq = ia_ptr->hca_ptr->ib_trans.ib_cq_empty;
+
+ /*
+ * IMPLEMENTATION NOTE:
+ * uDAPL allows consumers to post buffers on the EP after creation
+ * and before a connect request (outbound and inbound). This forces
+ * a binding to a device during the hca_open call and requires the
+ * consumer to predetermine which device to listen on or connect from.
+ * This restriction eliminates any option of listening or connecting
+ * over multiple devices. uDAPL should add API's to resolve addresses
+ * and bind to the device at the approriate time (before connect
+ * and after CR arrives). Discovery should happen at connection time
+ * based on addressing and not on static configuration during open.
+ */
+
+ /* Allocate CM and initialize lock */
+ if ((conn = dapl_os_alloc(sizeof(*conn))) == NULL)
+ return(dapl_convert_errno(ENOMEM, "create_cq"));
+
+ dapl_os_memzero(conn, sizeof(*conn));
+ dapl_os_lock_init(&conn->lock);
+
+ /* create CM_ID, bind to local device, create QP */
+ if (rdma_create_id(&cm_id, (void*)conn)) {
+ dapl_os_free(conn, sizeof(*conn));
+ return(dapl_convert_errno(errno, "create_qp"));
+ }
+
+ /* open identifies the local device; per DAT specification */
+ if (rdma_bind_addr(cm_id,
+ (struct sockaddr *)&ia_ptr->hca_ptr->hca_address))
+ goto bail;
+
+ /* Setup attributes and create qp */
+ dapl_os_memzero((void*)&qp_create, sizeof(qp_create));
+ qp_create.cap.max_send_wr = attr->max_request_dtos;
+ qp_create.cap.max_recv_wr = attr->max_recv_dtos;
+ qp_create.cap.max_send_sge = attr->max_request_iov;
+ qp_create.cap.max_recv_sge = attr->max_recv_iov;
+ qp_create.cap.max_inline_data =
+ ia_ptr->hca_ptr->ib_trans.max_inline_send;
+ qp_create.send_cq = req_cq;
+ qp_create.recv_cq = rcv_cq;
+ qp_create.qp_type = IBV_QPT_RC;
+ qp_create.qp_context = (void*)ep_ptr;
+
+ /* Let uCMA transition QP states */
+ if (rdma_create_qp(cm_id, ib_pd_handle, &qp_create))
+ goto bail;
+
+ conn->cm_id = cm_id;
+ conn->ep = ep_ptr;
+ conn->hca = ia_ptr->hca_ptr;
+ ep_ptr->qp_handle = conn;
+ ep_ptr->qp_state = IB_QP_STATE_INIT;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ " qp_alloc: qpn %p sq %d,%d rq %d,%d\n",
+ ep_ptr->qp_handle->cm_id->qp->qp_num,
+ qp_create.cap.max_send_wr,qp_create.cap.max_send_sge,
+ qp_create.cap.max_recv_wr,qp_create.cap.max_recv_sge);
+
+ return DAT_SUCCESS;
+bail:
+ rdma_destroy_id(cm_id);
+ dapl_os_free(conn, sizeof(*conn));
+ return(dapl_convert_errno(errno, "create_qp"));
+}
+
+/*
+ * dapl_ib_qp_free
+ *
+ * Free a QP
+ *
+ * Input:
+ * ia_handle IA handle
+ * *ep_ptr pointer to EP INFO
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * dapl_convert_errno
+ *
+ */
+DAT_RETURN dapls_ib_qp_free(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_EP, " qp_free: ep_ptr %p qp %p\n",
+ ep_ptr, ep_ptr->qp_handle);
+
+ if (ep_ptr->qp_handle != IB_INVALID_HANDLE) {
+ /* qp_handle is conn object with reference to cm_id and qp */
+ dapli_destroy_conn(ep_ptr->qp_handle);
+ ep_ptr->qp_handle = IB_INVALID_HANDLE;
+ ep_ptr->qp_state = IB_QP_STATE_ERROR;
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapl_ib_qp_modify
+ *
+ * Set the QP to the parameters specified in an EP_PARAM
+ *
+ * The EP_PARAM structure that is provided has been
+ * sanitized such that only non-zero values are valid.
+ *
+ * Input:
+ * ib_hca_handle HCA handle
+ * qp_handle QP handle
+ * ep_attr Sanitized EP Params
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INVALID_PARAMETER
+ *
+ */
+DAT_RETURN dapls_ib_qp_modify(IN DAPL_IA *ia_ptr,
+ IN DAPL_EP *ep_ptr,
+ IN DAT_EP_ATTR *attr)
+{
+ struct ibv_qp_attr qp_attr;
+
+ if (ep_ptr->qp_handle == IB_INVALID_HANDLE)
+ return DAT_INVALID_PARAMETER;
+
+ /*
+ * Check if we have the right qp_state to modify attributes
+ */
+ if ((ep_ptr->qp_handle->cm_id->qp->state != IBV_QPS_RTR) &&
+ (ep_ptr->qp_handle->cm_id->qp->state != IBV_QPS_RTS))
+ return DAT_INVALID_STATE;
+
+ /* Adjust to current EP attributes */
+ dapl_os_memzero((void*)&qp_attr, sizeof(qp_attr));
+ qp_attr.cap.max_send_wr = attr->max_request_dtos;
+ qp_attr.cap.max_recv_wr = attr->max_recv_dtos;
+ qp_attr.cap.max_send_sge = attr->max_request_iov;
+ qp_attr.cap.max_recv_sge = attr->max_recv_iov;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_EP,
+ "modify_qp: qp %p sq %d,%d, rq %d,%d\n",
+ ep_ptr->qp_handle->cm_id->qp,
+ qp_attr.cap.max_send_wr, qp_attr.cap.max_send_sge,
+ qp_attr.cap.max_recv_wr, qp_attr.cap.max_recv_sge);
+
+ if (ibv_modify_qp(ep_ptr->qp_handle->cm_id->qp, &qp_attr, IBV_QP_CAP)) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ "modify_qp: modify ep %p qp %p failed\n",
+ ep_ptr, ep_ptr->qp_handle->cm_id->qp);
+ return(dapl_convert_errno(errno,"modify_qp_state"));
+ }
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_reinit_ep
+ *
+ * Move the QP to INIT state again.
+ *
+ * Input:
+ * ep_ptr DAPL_EP
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * void
+ *
+ */
+void dapls_ib_reinit_ep(IN DAPL_EP *ep_ptr)
+{
+ /* uCMA does not allow reuse of CM_ID, destroy and create new one */
+ if (ep_ptr->qp_handle != IB_INVALID_HANDLE) {
+
+ /* destroy */
+ dapli_destroy_conn(ep_ptr->qp_handle);
+
+ /* create new CM_ID and QP */
+ ep_ptr->qp_handle = IB_INVALID_HANDLE;
+ dapls_ib_qp_alloc(ep_ptr->header.owner_ia, ep_ptr, ep_ptr);
+ }
+}
+
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * tab-width: 8
+ * End:
+ */
Index: dapl/openib_cma/README
===================================================================
--- dapl/openib_cma/README (revision 0)
+++ dapl/openib_cma/README (revision 0)
@@ -0,0 +1,40 @@
+
+OpenIB uDAPL provider using rdma cma and openib verbs interfaces
+
+to build:
+
+cd dapl/udapl
+make VERBS=openib_cma clean
+make VERBS=openib_cma
+
+
+Modifications to common code:
+
+- added dapl/openib_cma directory
+
+ dapl/udapl/Makefile
+
+New files for openib_scm provider
+
+ dapl/openib_cma/dapl_ib_cq.c
+ dapl/openib_cma/dapl_ib_dto.h
+ dapl/openib_cma/dapl_ib_mem.c
+ dapl/openib_cma/dapl_ib_qp.c
+ dapl/openib_cma/dapl_ib_util.c
+ dapl/openib_cma/dapl_ib_util.h
+ dapl/openib_cma/dapl_ib_cm.c
+
+A simple dapl test just for openib_scm testing...
+
+ test/dtest/dtest.c
+ test/dtest/makefile
+
+ server: dtest -s
+ client: dtest -h hostname
+
+known issues:
+
+ no memory windows support in ibverbs, dat_create_rmr fails.
+
+
+
Index: dapl/openib_cma/dapl_ib_util.h
===================================================================
--- dapl/openib_cma/dapl_ib_util.h (revision 0)
+++ dapl/openib_cma/dapl_ib_util.h (revision 0)
@@ -0,0 +1,333 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/***************************************************************************
+ *
+ * Module: uDAPL
+ *
+ * Filename: dapl_ib_util.h
+ *
+ * Author: Arlin Davis
+ *
+ * Created: 3/10/2005
+ *
+ * Description:
+ *
+ * The uDAPL openib provider - definitions, prototypes,
+ *
+ ****************************************************************************
+ * Source Control System Information
+ *
+ * $Id: $
+ *
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ **************************************************************************/
+
+#ifndef _DAPL_IB_UTIL_H_
+#define _DAPL_IB_UTIL_H_
+
+#include "verbs.h"
+#include <byteswap.h>
+#include <rdma/rdma_cma.h>
+
+/* Typedefs to map common DAPL provider types to IB verbs */
+typedef struct dapl_cm_id *ib_qp_handle_t;
+typedef struct ibv_cq *ib_cq_handle_t;
+typedef struct ibv_pd *ib_pd_handle_t;
+typedef struct ibv_mr *ib_mr_handle_t;
+typedef struct ibv_mw *ib_mw_handle_t;
+typedef struct ibv_wc ib_work_completion_t;
+
+/* HCA context type maps to IB verbs */
+typedef struct ibv_context *ib_hca_handle_t;
+typedef ib_hca_handle_t dapl_ibal_ca_t;
+
+#define IB_RC_RETRY_COUNT 7
+#define IB_RNR_RETRY_COUNT 7
+#define IB_CM_RESPONSE_TIMEOUT 18 /* 1 sec */
+#define IB_MAX_CM_RETRIES 7
+#define IB_REQ_MRA_TIMEOUT 27 /* a little over 9 minutes */
+#define IB_MAX_AT_RETRY 3
+#define IB_TARGET_MAX 4 /* max_qp_ous_rd_atom */
+#define IB_INITIATOR_DEPTH 4 /* max_qp_init_rd_atom */
+
+typedef enum {
+ IB_CME_CONNECTED,
+ IB_CME_DISCONNECTED,
+ IB_CME_DISCONNECTED_ON_LINK_DOWN,
+ IB_CME_CONNECTION_REQUEST_PENDING,
+ IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA,
+ IB_CME_CONNECTION_REQUEST_ACKED,
+ IB_CME_DESTINATION_REJECT,
+ IB_CME_DESTINATION_REJECT_PRIVATE_DATA,
+ IB_CME_DESTINATION_UNREACHABLE,
+ IB_CME_TOO_MANY_CONNECTION_REQUESTS,
+ IB_CME_LOCAL_FAILURE,
+ IB_CME_BROKEN
+} ib_cm_events_t;
+
+/* CQ notifications */
+typedef enum
+{
+ IB_NOTIFY_ON_NEXT_COMP,
+ IB_NOTIFY_ON_SOLIC_COMP
+
+} ib_notification_type_t;
+
+/* other mappings */
+typedef int ib_bool_t;
+typedef union ibv_gid GID;
+typedef char *IB_HCA_NAME;
+typedef uint16_t ib_hca_port_t;
+typedef uint32_t ib_comp_handle_t;
+
+#ifdef CQ_WAIT_OBJECT
+typedef struct ibv_comp_channel *ib_wait_obj_handle_t;
+#endif
+
+/* Definitions */
+#define IB_INVALID_HANDLE NULL
+
+/* inline send rdma threshold */
+#define INLINE_SEND_DEFAULT 128
+
+/* CM private data areas */
+#define IB_MAX_REQ_PDATA_SIZE 48
+#define IB_MAX_REP_PDATA_SIZE 196
+#define IB_MAX_REJ_PDATA_SIZE 148
+#define IB_MAX_DREQ_PDATA_SIZE 220
+#define IB_MAX_DREP_PDATA_SIZE 224
+
+/* DTO OPs, ordered for DAPL ENUM definitions */
+#define OP_RDMA_WRITE IBV_WR_RDMA_WRITE
+#define OP_RDMA_WRITE_IMM IBV_WR_RDMA_WRITE_WITH_IMM
+#define OP_SEND IBV_WR_SEND
+#define OP_SEND_IMM IBV_WR_SEND_WITH_IMM
+#define OP_RDMA_READ IBV_WR_RDMA_READ
+#define OP_COMP_AND_SWAP IBV_WR_ATOMIC_CMP_AND_SWP
+#define OP_FETCH_AND_ADD IBV_WR_ATOMIC_FETCH_AND_ADD
+#define OP_RECEIVE 7 /* internal op */
+#define OP_RECEIVE_IMM 8 /* internel op */
+#define OP_BIND_MW 9 /* internal op */
+#define OP_INVALID 0xff
+
+/* Definitions to map QP state */
+#define IB_QP_STATE_RESET IBV_QPS_RESET
+#define IB_QP_STATE_INIT IBV_QPS_INIT
+#define IB_QP_STATE_RTR IBV_QPS_RTR
+#define IB_QP_STATE_RTS IBV_QPS_RTS
+#define IB_QP_STATE_SQD IBV_QPS_SQD
+#define IB_QP_STATE_SQE IBV_QPS_SQE
+#define IB_QP_STATE_ERROR IBV_QPS_ERR
+
+typedef enum
+{
+ IB_THREAD_INIT,
+ IB_THREAD_CREATE,
+ IB_THREAD_RUN,
+ IB_THREAD_CANCEL,
+ IB_THREAD_EXIT
+
+} ib_thread_state_t;
+
+/*
+ * dapl_llist_entry in dapl.h but dapl.h depends on provider
+ * typedef's in this file first. move dapl_llist_entry out of dapl.h
+ */
+struct ib_llist_entry
+{
+ struct dapl_llist_entry *flink;
+ struct dapl_llist_entry *blink;
+ void *data;
+ struct dapl_llist_entry *list_head;
+};
+
+struct dapl_cm_id {
+ DAPL_OS_LOCK lock;
+ int destroy;
+ int in_callback;
+ struct rdma_cm_id *cm_id;
+ struct dapl_hca *hca;
+ struct dapl_sp *sp;
+ struct dapl_ep *ep;
+ struct rdma_conn_param params;
+ int p_len;
+ unsigned char p_data[IB_MAX_DREP_PDATA_SIZE];
+};
+
+typedef struct dapl_cm_id *ib_cm_handle_t;
+typedef struct dapl_cm_id *ib_cm_srvc_handle_t;
+
+/* Operation and state mappings */
+typedef enum ibv_send_flags ib_send_op_type_t;
+typedef struct ibv_sge ib_data_segment_t;
+typedef enum ibv_qp_state ib_qp_state_t;
+typedef enum ibv_event_type ib_async_event_type;
+typedef struct ibv_async_event ib_error_record_t;
+
+/* Definitions for ibverbs/mthca return codes, should be defined in verbs.h */
+/* some are errno and some are -n values */
+
+/**
+ * ibv_get_device_name - Return kernel device name
+ * ibv_get_device_guid - Return device's node GUID
+ * ibv_open_device - Return ibv_context or NULL
+ * ibv_close_device - Return 0, (errno?)
+ * ibv_get_async_event - Return 0, -1
+ * ibv_alloc_pd - Return ibv_pd, NULL
+ * ibv_dealloc_pd - Return 0, errno
+ * ibv_reg_mr - Return ibv_mr, NULL
+ * ibv_dereg_mr - Return 0, errno
+ * ibv_create_cq - Return ibv_cq, NULL
+ * ibv_destroy_cq - Return 0, errno
+ * ibv_get_cq_event - Return 0 & ibv_cq/context, int
+ * ibv_poll_cq - Return n & ibv_wc, 0 ok, -1 empty, -2 error
+ * ibv_req_notify_cq - Return 0 (void?)
+ * ibv_create_qp - Return ibv_qp, NULL
+ * ibv_modify_qp - Return 0, errno
+ * ibv_destroy_qp - Return 0, errno
+ * ibv_post_send - Return 0, -1 & bad_wr
+ * ibv_post_recv - Return 0, -1 & bad_wr
+ */
+
+/* async handlers for DTO, CQ, QP, and unafiliated */
+typedef void (*ib_async_dto_handler_t)(
+ IN ib_hca_handle_t ib_hca_handle,
+ IN ib_error_record_t *err_code,
+ IN void *context);
+
+typedef void (*ib_async_cq_handler_t)(
+ IN ib_hca_handle_t ib_hca_handle,
+ IN ib_cq_handle_t ib_cq_handle,
+ IN ib_error_record_t *err_code,
+ IN void *context);
+
+typedef void (*ib_async_qp_handler_t)(
+ IN ib_hca_handle_t ib_hca_handle,
+ IN ib_qp_handle_t ib_qp_handle,
+ IN ib_error_record_t *err_code,
+ IN void *context);
+
+typedef void (*ib_async_handler_t)(
+ IN ib_hca_handle_t ib_hca_handle,
+ IN ib_error_record_t *err_code,
+ IN void *context);
+
+
+/* ib_hca_transport_t, specific to this implementation */
+typedef struct _ib_hca_transport
+{
+ struct ib_llist_entry entry;
+ int destroy;
+ struct dapl_hca *d_hca;
+ struct rdma_cm_id *cm_id;
+ struct ibv_comp_channel *ib_cq;
+ ib_cq_handle_t ib_cq_empty;
+ int max_inline_send;
+ ib_async_handler_t async_unafiliated;
+ void *async_un_ctx;
+ ib_async_cq_handler_t async_cq_error;
+ ib_async_dto_handler_t async_cq;
+ ib_async_qp_handler_t async_qp_error;
+
+} ib_hca_transport_t;
+
+/* provider specfic fields for shared memory support */
+typedef uint32_t ib_shm_transport_t;
+
+/* prototypes */
+int32_t dapls_ib_init (void);
+int32_t dapls_ib_release (void);
+void dapli_thread(void *arg);
+int dapli_ib_thread_init(void);
+void dapli_ib_thread_destroy(void);
+void dapli_cma_event_cb(void);
+void dapli_cq_event_cb(struct _ib_hca_transport *hca);
+void dapli_async_event_cb(struct _ib_hca_transport *hca);
+void dapli_destroy_conn(struct dapl_cm_id *conn);
+
+DAT_RETURN
+dapls_modify_qp_state ( IN ib_qp_handle_t qp_handle,
+ IN ib_qp_state_t qp_state,
+ IN struct dapl_cm_id *conn );
+
+/* inline functions */
+STATIC _INLINE_ IB_HCA_NAME dapl_ib_convert_name (IN char *name)
+{
+ /* use ascii; name of local device */
+ return dapl_os_strdup(name);
+}
+
+STATIC _INLINE_ void dapl_ib_release_name (IN IB_HCA_NAME name)
+{
+ return;
+}
+
+/*
+ * Convert errno to DAT_RETURN values
+ */
+STATIC _INLINE_ DAT_RETURN
+dapl_convert_errno( IN int err, IN const char *str )
+{
+ if (!err) return DAT_SUCCESS;
+
+#if DAPL_DBG
+ if ((err != EAGAIN) && (err != ETIME) && (err != ETIMEDOUT))
+ dapl_dbg_log (DAPL_DBG_TYPE_ERR," %s %s\n", str, strerror(err));
+#endif
+
+ switch( err )
+ {
+ case EOVERFLOW : return DAT_LENGTH_ERROR;
+ case EACCES : return DAT_PRIVILEGES_VIOLATION;
+ case ENXIO :
+ case ERANGE :
+ case EPERM : return DAT_PROTECTION_VIOLATION;
+ case EINVAL :
+ case EBADF :
+ case ENOENT :
+ case ENOTSOCK : return DAT_INVALID_HANDLE;
+ case EISCONN : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_CONNECTED;
+ case ECONNREFUSED : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_NOTREADY;
+ case ETIME :
+ case ETIMEDOUT : return DAT_TIMEOUT_EXPIRED;
+ case ENETUNREACH: return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_UNREACHABLE;
+ case EBUSY : return DAT_PROVIDER_IN_USE;
+ case EADDRINUSE : return DAT_CONN_QUAL_IN_USE;
+ case EALREADY : return DAT_INVALID_STATE | DAT_INVALID_STATE_EP_ACTCONNPENDING;
+ case ENOSPC :
+ case ENOMEM :
+ case E2BIG :
+ case EDQUOT : return DAT_INSUFFICIENT_RESOURCES;
+ case EAGAIN : return DAT_QUEUE_EMPTY;
+ case EINTR : return DAT_INTERRUPTED_CALL;
+ case EAFNOSUPPORT : return DAT_INVALID_ADDRESS | DAT_INVALID_ADDRESS_MALFORMED;
+ case EFAULT :
+ default : return DAT_INTERNAL_ERROR;
+ }
+ }
+
+#endif /* _DAPL_IB_UTIL_H_ */
Index: dapl/openib_cma/dapl_ib_cq.c
===================================================================
--- dapl/openib_cma/dapl_ib_cq.c (revision 0)
+++ dapl/openib_cma/dapl_ib_cq.c (revision 0)
@@ -0,0 +1,531 @@
+/*
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ */
+
+/***************************************************************************
+ *
+ * Module: uDAPL
+ *
+ * Filename: dapl_ib_cq.c
+ *
+ * Author: Arlin Davis
+ *
+ * Created: 3/10/2005
+ *
+ * Description:
+ *
+ * The uDAPL openib provider - completion queue
+ *
+ ****************************************************************************
+ * Source Control System Information
+ *
+ * $Id: $
+ *
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ **************************************************************************/
+
+#include "dapl.h"
+#include "dapl_adapter_util.h"
+#include "dapl_lmr_util.h"
+#include "dapl_evd_util.h"
+#include "dapl_ring_buffer_util.h"
+#include <sys/poll.h>
+
+/* One CQ event channel per HCA */
+void dapli_cq_event_cb(struct _ib_hca_transport *hca)
+{
+ /* check all comp events on this device */
+ struct dapl_evd *evd_ptr = NULL;
+ struct ibv_cq *ibv_cq = NULL;
+ struct pollfd cq_fd = {
+ .fd = hca->ib_cq->fd,
+ .events = POLLIN,
+ .revents = 0
+ };
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL," dapli_cq_event_cb(%p)\n", hca);
+
+ if ((poll(&cq_fd, 1, 0) == 1) &&
+ (!ibv_get_cq_event(hca->ib_cq,
+ &ibv_cq, (void*)&evd_ptr))) {
+
+ if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD)) {
+ ibv_ack_cq_events(ibv_cq, 1);
+ return;
+ }
+
+ /* process DTO event via callback */
+ dapl_evd_dto_callback ( hca->cm_id->verbs,
+ evd_ptr->ib_cq_handle,
+ (void*)evd_ptr );
+
+ ibv_ack_cq_events(ibv_cq, 1);
+ }
+}
+
+/*
+ * Map all verbs DTO completion codes to the DAT equivelent.
+ *
+ * Not returned by verbs: DAT_DTO_ERR_PARTIAL_PACKET
+ */
+static struct ib_status_map
+{
+ int ib_status;
+ DAT_DTO_COMPLETION_STATUS dat_status;
+} ib_status_map[] = {
+/* 00 */ { IBV_WC_SUCCESS, DAT_DTO_SUCCESS},
+/* 01 */ { IBV_WC_LOC_LEN_ERR, DAT_DTO_ERR_LOCAL_LENGTH},
+/* 02 */ { IBV_WC_LOC_QP_OP_ERR, DAT_DTO_ERR_LOCAL_EP},
+/* 03 */ { IBV_WC_LOC_EEC_OP_ERR, DAT_DTO_ERR_TRANSPORT},
+/* 04 */ { IBV_WC_LOC_PROT_ERR, DAT_DTO_ERR_LOCAL_PROTECTION},
+/* 05 */ { IBV_WC_WR_FLUSH_ERR, DAT_DTO_ERR_FLUSHED},
+/* 06 */ { IBV_WC_MW_BIND_ERR, DAT_RMR_OPERATION_FAILED},
+/* 07 */ { IBV_WC_BAD_RESP_ERR, DAT_DTO_ERR_BAD_RESPONSE},
+/* 08 */ { IBV_WC_LOC_ACCESS_ERR, DAT_DTO_ERR_LOCAL_PROTECTION},
+/* 09 */ { IBV_WC_REM_INV_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER},
+/* 10 */ { IBV_WC_REM_ACCESS_ERR, DAT_DTO_ERR_REMOTE_ACCESS},
+/* 11 */ { IBV_WC_REM_OP_ERR, DAT_DTO_ERR_REMOTE_RESPONDER},
+/* 12 */ { IBV_WC_RETRY_EXC_ERR, DAT_DTO_ERR_TRANSPORT},
+/* 13 */ { IBV_WC_RNR_RETRY_EXC_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY},
+/* 14 */ { IBV_WC_LOC_RDD_VIOL_ERR, DAT_DTO_ERR_LOCAL_PROTECTION},
+/* 15 */ { IBV_WC_REM_INV_RD_REQ_ERR, DAT_DTO_ERR_REMOTE_RESPONDER},
+/* 16 */ { IBV_WC_REM_ABORT_ERR, DAT_DTO_ERR_REMOTE_RESPONDER},
+/* 17 */ { IBV_WC_INV_EECN_ERR, DAT_DTO_ERR_TRANSPORT},
+/* 18 */ { IBV_WC_INV_EEC_STATE_ERR, DAT_DTO_ERR_TRANSPORT},
+/* 19 */ { IBV_WC_FATAL_ERR, DAT_DTO_ERR_TRANSPORT},
+/* 20 */ { IBV_WC_RESP_TIMEOUT_ERR, DAT_DTO_ERR_RECEIVER_NOT_READY},
+/* 21 */ { IBV_WC_GENERAL_ERR, DAT_DTO_ERR_TRANSPORT},
+};
+
+/*
+ * dapls_ib_get_dto_status
+ *
+ * Return the DAT status of a DTO operation
+ *
+ * Input:
+ * cqe_ptr pointer to completion queue entry
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * Value from ib_status_map table above
+ */
+
+DAT_DTO_COMPLETION_STATUS
+dapls_ib_get_dto_status (IN ib_work_completion_t *cqe_ptr)
+{
+ uint32_t ib_status;
+ int i;
+
+ ib_status = DAPL_GET_CQE_STATUS (cqe_ptr);
+
+ /*
+ * Due to the implementation of verbs completion code, we need to
+ * search the table for the correct value rather than assuming
+ * linear distribution.
+ */
+ for (i=0; i <= IBV_WC_GENERAL_ERR; i++) {
+ if (ib_status == ib_status_map[i].ib_status) {
+ if (ib_status != IBV_WC_SUCCESS) {
+ dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
+ " DTO completion ERROR: %d: op %#x\n",
+ ib_status,
+ DAPL_GET_CQE_OPTYPE (cqe_ptr));
+ }
+ return ib_status_map[i].dat_status;
+ }
+ }
+
+ dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
+ " DTO completion ERROR: %d: op %#x\n",
+ ib_status,
+ DAPL_GET_CQE_OPTYPE (cqe_ptr));
+
+ return DAT_DTO_FAILURE;
+}
+
+DAT_RETURN dapls_ib_get_async_event(IN ib_error_record_t *err_record,
+ OUT DAT_EVENT_NUMBER *async_event)
+{
+ DAT_RETURN dat_status = DAT_SUCCESS;
+ int err_code = err_record->event_type;
+
+ switch (err_code) {
+ /* OVERFLOW error */
+ case IBV_EVENT_CQ_ERR:
+ *async_event = DAT_ASYNC_ERROR_EVD_OVERFLOW;
+ break;
+ /* INTERNAL errors */
+ case IBV_EVENT_DEVICE_FATAL:
+ *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR;
+ break;
+ /* CATASTROPHIC errors */
+ case IBV_EVENT_PORT_ERR:
+ *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC;
+ break;
+ /* BROKEN QP error */
+ case IBV_EVENT_SQ_DRAINED:
+ case IBV_EVENT_QP_FATAL:
+ case IBV_EVENT_QP_REQ_ERR:
+ case IBV_EVENT_QP_ACCESS_ERR:
+ *async_event = DAT_ASYNC_ERROR_EP_BROKEN;
+ break;
+ /* connection completion */
+ case IBV_EVENT_COMM_EST:
+ *async_event = DAT_CONNECTION_EVENT_ESTABLISHED;
+ break;
+ /* TODO: process HW state changes */
+ case IBV_EVENT_PATH_MIG:
+ case IBV_EVENT_PATH_MIG_ERR:
+ case IBV_EVENT_PORT_ACTIVE:
+ case IBV_EVENT_LID_CHANGE:
+ case IBV_EVENT_PKEY_CHANGE:
+ case IBV_EVENT_SM_CHANGE:
+ default:
+ dat_status = DAT_ERROR (DAT_NOT_IMPLEMENTED, 0);
+ }
+ return dat_status;
+}
+
+/*
+ * dapl_ib_cq_alloc
+ *
+ * Alloc a CQ
+ *
+ * Input:
+ * ia_handle IA handle
+ * evd_ptr pointer to EVD struct
+ * cqlen minimum QLen
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ *
+ */
+DAT_RETURN
+dapls_ib_cq_alloc(IN DAPL_IA *ia_ptr,
+ IN DAPL_EVD *evd_ptr,
+ IN DAT_COUNT *cqlen )
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ "dapls_ib_cq_alloc: evd %p cqlen=%d \n", evd_ptr, *cqlen);
+
+ struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq;
+
+#ifdef CQ_WAIT_OBJECT
+ if (evd_ptr->cq_wait_obj_handle)
+ channel = evd_ptr->cq_wait_obj_handle;
+#endif
+
+ /* Call IB verbs to create CQ */
+ evd_ptr->ib_cq_handle = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle,
+ *cqlen,
+ evd_ptr,
+ channel, 0);
+
+ if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE)
+ return DAT_INSUFFICIENT_RESOURCES;
+
+ /* arm cq for events */
+ dapls_set_cq_notify(ia_ptr, evd_ptr);
+
+ /* update with returned cq entry size */
+ *cqlen = evd_ptr->ib_cq_handle->cqe;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ "dapls_ib_cq_alloc: new_cq %p cqlen=%d \n",
+ evd_ptr->ib_cq_handle, *cqlen );
+
+ return DAT_SUCCESS;
+}
+
+
+/*
+ * dapl_ib_cq_resize
+ *
+ * Alloc a CQ
+ *
+ * Input:
+ * ia_handle IA handle
+ * evd_ptr pointer to EVD struct
+ * cqlen minimum QLen
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_PARAMETER
+ *
+ */
+DAT_RETURN
+dapls_ib_cq_resize(IN DAPL_IA *ia_ptr,
+ IN DAPL_EVD *evd_ptr,
+ IN DAT_COUNT *cqlen )
+{
+ ib_cq_handle_t new_cq;
+ struct ibv_comp_channel *channel = ia_ptr->hca_ptr->ib_trans.ib_cq;
+
+ /* IB verbs doe not support resize. Try to re-create CQ
+ * with new size. Can only be done if QP is not attached.
+ * destroy EBUSY == QP still attached.
+ */
+
+#ifdef CQ_WAIT_OBJECT
+ if (evd_ptr->cq_wait_obj_handle)
+ channel = evd_ptr->cq_wait_obj_handle;
+#endif
+
+ /* Call IB verbs to create CQ */
+ new_cq = ibv_create_cq(ia_ptr->hca_ptr->ib_hca_handle, *cqlen,
+ evd_ptr, channel, 0);
+
+ if (new_cq == IB_INVALID_HANDLE)
+ return DAT_INSUFFICIENT_RESOURCES;
+
+ /* destroy the original and replace if successful */
+ if (ibv_destroy_cq(evd_ptr->ib_cq_handle)) {
+ ibv_destroy_cq(new_cq);
+ return(dapl_convert_errno(errno,"resize_cq"));
+ }
+
+ /* update EVD with new cq handle and size */
+ evd_ptr->ib_cq_handle = new_cq;
+ *cqlen = new_cq->cqe;
+
+ /* arm cq for events */
+ dapls_set_cq_notify (ia_ptr, evd_ptr);
+
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_cq_free
+ *
+ * destroy a CQ
+ *
+ * Input:
+ * ia_handle IA handle
+ * evd_ptr pointer to EVD struct
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_INVALID_PARAMETER
+ *
+ */
+DAT_RETURN dapls_ib_cq_free(IN DAPL_IA *ia_ptr, IN DAPL_EVD *evd_ptr)
+{
+ if (evd_ptr->ib_cq_handle != IB_INVALID_HANDLE) {
+ /* copy all entries on CQ to EVD before destroying */
+ dapls_evd_copy_cq(evd_ptr);
+ if (ibv_destroy_cq(evd_ptr->ib_cq_handle))
+ return(dapl_convert_errno(errno,"destroy_cq"));
+ evd_ptr->ib_cq_handle = IB_INVALID_HANDLE;
+ }
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_set_cq_notify
+ *
+ * Set the CQ notification for next
+ *
+ * Input:
+ * hca_handl hca handle
+ * DAPL_EVD evd handle
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * dapl_convert_errno
+ */
+DAT_RETURN dapls_set_cq_notify(IN DAPL_IA *ia_ptr, IN DAPL_EVD *evd_ptr)
+{
+ if (ibv_req_notify_cq(evd_ptr->ib_cq_handle, 0))
+ return(dapl_convert_errno(errno,"notify_cq"));
+ else
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_completion_notify
+ *
+ * Set the CQ notification type
+ *
+ * Input:
+ * hca_handl hca handle
+ * evd_ptr evd handle
+ * type notification type
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * dapl_convert_errno
+ */
+DAT_RETURN dapls_ib_completion_notify(IN ib_hca_handle_t hca_handle,
+ IN DAPL_EVD *evd_ptr,
+ IN ib_notification_type_t type)
+{
+ if (ibv_req_notify_cq( evd_ptr->ib_cq_handle, type ))
+ return(dapl_convert_errno(errno,"notify_cq_type"));
+ else
+ return DAT_SUCCESS;
+}
+
+/*
+ * dapls_ib_completion_poll
+ *
+ * CQ poll for completions
+ *
+ * Input:
+ * hca_handl hca handle
+ * evd_ptr evd handle
+ * wc_ptr work completion
+ *
+ * Output:
+ * none
+ *
+ * Returns:
+ * DAT_SUCCESS
+ * DAT_QUEUE_EMPTY
+ *
+ */
+DAT_RETURN dapls_ib_completion_poll(IN DAPL_HCA *hca_ptr,
+ IN DAPL_EVD *evd_ptr,
+ IN ib_work_completion_t *wc_ptr)
+{
+ if (ibv_poll_cq(evd_ptr->ib_cq_handle, 1, wc_ptr) == 1)
+ return DAT_SUCCESS;
+
+ return DAT_QUEUE_EMPTY;
+}
+
+#ifdef CQ_WAIT_OBJECT
+
+/* NEW common wait objects for providers with direct CQ wait objects */
+DAT_RETURN
+dapls_ib_wait_object_create(IN DAPL_EVD *evd_ptr,
+ IN ib_wait_obj_handle_t *p_cq_wait_obj_handle)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " cq_object_create: (%p,%p)\n",
+ evd_ptr, p_cq_wait_obj_handle );
+
+ /* set cq_wait object to evd_ptr */
+ *p_cq_wait_obj_handle =
+ ibv_create_comp_channel(
+ evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle);
+
+ return DAT_SUCCESS;
+}
+
+DAT_RETURN
+dapls_ib_wait_object_destroy(IN ib_wait_obj_handle_t p_cq_wait_obj_handle)
+{
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " cq_object_destroy: wait_obj=%p\n",
+ p_cq_wait_obj_handle );
+
+ ibv_destroy_comp_channel(p_cq_wait_obj_handle);
+
+ return DAT_SUCCESS;
+}
+
+DAT_RETURN
+dapls_ib_wait_object_wakeup (IN ib_wait_obj_handle_t p_cq_wait_obj_handle)
+{
+ dapl_dbg_log ( DAPL_DBG_TYPE_UTIL,
+ " cq_object_wakeup: wait_obj=%p\n",
+ p_cq_wait_obj_handle );
+
+ /* no wake up mechanism */
+ return DAT_SUCCESS;
+}
+
+DAT_RETURN
+dapls_ib_wait_object_wait(IN ib_wait_obj_handle_t p_cq_wait_obj_handle,
+ IN u_int32_t timeout)
+{
+ struct dapl_evd *evd_ptr;
+ struct ibv_cq *ibv_cq = NULL;
+ void *ibv_ctx = NULL;
+ int status = 0;
+ int timeout_ms = -1;
+ struct pollfd cq_fd = {
+ .fd = p_cq_wait_obj_handle->fd,
+ .events = POLLIN,
+ .revents = 0
+ };
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " cq_object_wait: CQ channel %p time %d\n",
+ p_cq_wait_obj_handle, timeout );
+
+ /* uDAPL timeout values in usecs */
+ if (timeout != DAT_TIMEOUT_INFINITE)
+ timeout_ms = timeout/1000;
+
+ status = poll(&cq_fd, 1, timeout_ms);
+
+ /* returned event */
+ if (status > 0) {
+ if (!ibv_get_cq_event(p_cq_wait_obj_handle,
+ &ibv_cq, (void*)&evd_ptr)) {
+ ibv_ack_cq_events(ibv_cq, 1);
+ }
+ status = 0;
+
+ /* timeout */
+ } else if (status == 0)
+ status = ETIMEDOUT;
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " cq_object_wait: RET evd %p ibv_cq %p ibv_ctx %p %s\n",
+ evd_ptr, ibv_cq,ibv_ctx,strerror(errno));
+
+ return(dapl_convert_errno(status,"cq_wait_object_wait"));
+
+}
+#endif
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * tab-width: 8
+ * End:
+ */
+
Index: doc/dat.conf
===================================================================
--- doc/dat.conf (revision 4178)
+++ doc/dat.conf (working copy)
@@ -1,15 +1,17 @@
#
-# DAT 1.1 configuration file
+# DAT 1.2 configuration file
#
# Each entry should have the following fields:
#
-# <ia_name> <api_version> <threadsafety> <default> <lib_path> \
+# <ia_name> <api_version> <threadsafety> <default> <lib_path> \
# <provider_version> <ia_params> <platform_params>
#
-
-ia0 u1.1 nonthreadsafe default /usr/lib/libdapl.so ri.1.1 "ia_params" "pd_params"
-
-# Example for openib using the first Mellanox adapter, port 1 and port 2
-OpenIB1 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 1" ""
-OpenIB2 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 2" ""
-
+# Example for openib_cma and openib_scm
+#
+# For scm version you specify <ia_params> as actual device name and port
+# For cma version you specify <ia_params> as the ib device network address or network hostname and
0 for port
+#
+OpenIB-scm1 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 1" ""
+OpenIB-scm2 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 2" ""
+OpenIB-cma-ip u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"192.168.0.22 0" ""
+OpenIB-cma-name u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so
mv_dapl.1.2 "svr1-ib0 0" ""
Index: README
===================================================================
--- README (revision 0)
+++ README (revision 0)
@@ -0,0 +1,94 @@
+There are now 3 uDAPL providers for openib (openib,openib_scm,openib_cma).
+
+
+==========
+1.0 BUILD:
+==========
+
+The default build is now set to openib_cma. This version requires librdmacm
+installation, IPoIB installation, and IPoIB configuration with an IP address.
+
+Building DAT library:
+--------------------
+cd dat/udat
+make clean
+make
+
+the dat library target directory is dat/udat/$(ARCH)/Target
+
+Building default DAPL library:
+----------------------------
+
+cd dapl/udapl
+make clean
+make
+
+the dapl library target directory is dapl/udapl/Target
+
+NOTE: to link these libraries you must either use libtool and
+specify the full pathname of the library, or use the `-LLIBDIR'
+flag during linking and do at least one of the following:
+ - add LIBDIR to the `LD_LIBRARY_PATH' environment variable
+ during execution
+ - add LIBDIR to the `LD_RUN_PATH' environment variable
+ during linking
+ - use the `-Wl,--rpath -Wl,LIBDIR' linker flag
+ - have your system administrator add LIBDIR to `/etc/ld.so.conf'
+
+See any operating system documentation about shared libraries for
+more information, such as the ld(1) and ld.so(8) manual pages.
+
+Building specific verbs provider set VERBS to provider name:
+-----------------------------------------------------------
+example for socket cm version (openib_scm):
+
+cd dapl/udapl
+make VERBS=openib_scm clean
+make VERBS=openib_scm
+
+===================
+2.0 CONFIGURATION:
+===================
+
+sample /etc/dat.conf
+
+#
+# DAT 1.2 configuration file
+#
+# Each entry should have the following fields:
+#
+# <ia_name> <api_version> <threadsafety> <default> <lib_path> \
+# <provider_version> <ia_params> <platform_params>
+#
+# Example for openib_cma and openib_scm
+#
+# For scm version you specify <ia_params> as actual device name and port
+# For cma version you specify <ia_params> as the ib device network address or network hostname and
0 for port
+#
+OpenIB-scm1 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 1" ""
+OpenIB-scm2 u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"mthca0 2" ""
+OpenIB-cma-ip u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so mv_dapl.1.2
"192.168.0.22 0" ""
+OpenIB-cma-name u1.2 nonthreadsafe default /usr/local/openib_dapl/udapl/Target/libdapl.so
mv_dapl.1.2 "svr1-ib0 0" ""
+
+=============================
+3.0 SAMPLE uDAPL APPLICATION:
+=============================
+
+sample makefile for simple application (dapl/test/dtest.c) base on prevous dat.conf settings:
+
+CC = gcc
+CFLAGS = -O2 -g
+
+DAT_INC = ../../dat/include
+DAT_LIB = /usr/local/lib
+
+all: dtest
+
+clean:
+ rm -f *.o;touch *.c;rm -f dtest
+
+dtest: ./dtest.c
+ $(CC) $(CFLAGS) ./dtest.c -o dtest \
+ -DDAPL_PROVIDER='"OpenIB_cma_name"' \
+ -I $(DAT_INC) -L $(DAT_LIB) -ldat
+
More information about the general
mailing list