[openib-general] [PATCH] update uDAPL openib_cma provider to work with new uCMA event channels

[Arlin Davis]

James,

Update the uDAPL openib_cma provider to work with the new uCMA event channel interface. I ran a full
set of Intel-MPI test suites with these latest changes and it looks fine. Sync up with Sean on
commits.

Signed-off by: Arlin Davis <ardavis at ichips.intel.com>


Index: dapl/openib_cma/dapl_ib_util.c
===================================================================
--- dapl/openib_cma/dapl_ib_util.c	(revision 6942)
+++ dapl/openib_cma/dapl_ib_util.c	(working copy)
@@ -67,6 +67,7 @@ static const char rcsid[] = "$Id:  $";
 
 int g_dapl_loopback_connection = 0;
 int g_ib_pipe[2];
+struct rdma_event_channel *g_cm_events = NULL;
 ib_thread_state_t g_ib_thread_state = 0;
 DAPL_OS_THREAD g_ib_thread;
 DAPL_OS_LOCK g_hca_lock;
@@ -184,6 +185,7 @@ int32_t dapls_ib_release(void)
 {
 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " dapl_ib_release: \n");
 	dapli_ib_thread_destroy();
+	rdma_destroy_event_channel(g_cm_events);
 	return 0;
 }
 
@@ -214,9 +216,17 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N
 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
 		     " open_hca: %s - %p\n", hca_name, hca_ptr);
 
+	/* Setup the global cm event channel */
+	dapl_os_lock(&g_hca_lock);
+	if (g_cm_events == NULL) {
+		g_cm_events = rdma_create_event_channel();
+		if (g_cm_events == NULL)
+			return DAT_INTERNAL_ERROR;
+	}
+	dapl_os_unlock(&g_hca_lock);
+
 	if (dapli_ib_thread_init()) 
 		return DAT_INTERNAL_ERROR;
-	
 
 	/* HCA name will be hostname or IP address */
 	if (getipaddr((char*)hca_name,
@@ -224,14 +234,13 @@ DAT_RETURN dapls_ib_open_hca(IN IB_HCA_N
 		      sizeof(DAT_SOCK_ADDR6)))
 		return DAT_INVALID_ADDRESS;
 
-
 	/* cm_id will bind local device/GID based on IP address */
-	if (rdma_create_id(&cm_id, (void*)hca_ptr))
+	if (rdma_create_id(g_cm_events, &cm_id, (void*)hca_ptr))
 		return DAT_INTERNAL_ERROR;
 
 	ret = rdma_bind_addr(cm_id,
 			     (struct sockaddr *)&hca_ptr->hca_address);
-	if (ret) {
+	if ((ret) || (cm_id->verbs == NULL)) {
                 rdma_destroy_id(cm_id); 
 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
 			     " open_hca: ERR bind (%d) %s \n", 
@@ -551,8 +560,8 @@ int dapli_ib_thread_init(void)
 	}
 		
 	/* uCMA events non-blocking */
-	opts = fcntl(rdma_get_fd(), F_GETFL); /* uCMA */
-	if (opts < 0 || fcntl(rdma_get_fd(), 
+	opts = fcntl(g_cm_events->fd, F_GETFL); /* uCMA */
+	if (opts < 0 || fcntl(g_cm_events->fd, 
 			      F_SETFL, opts | O_NONBLOCK) < 0) {
 		dapl_dbg_log (DAPL_DBG_TYPE_ERR, 
 			      " dapl_ib_init: ERR with uCMA FD\n" );
@@ -741,7 +750,7 @@ void dapli_thread(void *arg) 
 	
 	dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
 		      " ib_thread(%d,0x%x): ENTER: pipe %d ucma %d\n",
-		      getpid(), g_ib_thread, g_ib_pipe[0], rdma_get_fd());
+		      getpid(), g_ib_thread, g_ib_pipe[0], g_cm_events->fd);
 
  	/* Poll across pipe, CM, AT never changes */
 	dapl_os_lock( &g_hca_lock );
@@ -749,7 +758,7 @@ void dapli_thread(void *arg) 
 		
 	ufds[0].fd = g_ib_pipe[0];	/* pipe */
 	ufds[0].events = POLLIN;
-	ufds[1].fd = rdma_get_fd();	/* uCMA */
+	ufds[1].fd = g_cm_events->fd;	/* uCMA */
 	ufds[1].events = POLLIN;
 	
 	while (g_ib_thread_state == IB_THREAD_RUN) {
Index: dapl/openib_cma/dapl_ib_cm.c
===================================================================
--- dapl/openib_cma/dapl_ib_cm.c	(revision 6942)
+++ dapl/openib_cma/dapl_ib_cm.c	(working copy)
@@ -59,6 +59,8 @@
 #include <sys/poll.h>
 #include <signal.h>
 
+extern struct rdma_event_channel *g_cm_events;
+
 /* local prototypes */
 static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, 
 					  struct rdma_cm_event *event);
@@ -614,7 +616,7 @@ dapls_ib_setup_conn_listener(IN DAPL_IA 
 	dapl_os_lock_init(&conn->lock);
 		 
 	/* create CM_ID, bind to local device, create QP */
-	if (rdma_create_id(&conn->cm_id, (void*)conn)) {
+	if (rdma_create_id(g_cm_events, &conn->cm_id, (void*)conn)) {
 		dapl_os_free(conn, sizeof(*conn));
 		return(dapl_convert_errno(errno,"setup_listener"));
 	}
@@ -1067,7 +1069,7 @@ void dapli_cma_event_cb(void)
 	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " cm_event()\n");
 
 	/* process one CM event, fairness */
-	if(!rdma_get_cm_event(&event)) {
+	if(!rdma_get_cm_event(g_cm_events, &event)) {
 		struct dapl_cm_id *conn;
 				
 		/* set proper conn from cm_id context*/
Index: dapl/openib_cma/dapl_ib_qp.c
===================================================================
--- dapl/openib_cma/dapl_ib_qp.c	(revision 6942)
+++ dapl/openib_cma/dapl_ib_qp.c	(working copy)
@@ -35,6 +35,8 @@
 #include "dapl.h"
 #include "dapl_adapter_util.h"
 
+extern struct rdma_event_channel *g_cm_events;
+
 /*
  * dapl_ib_qp_alloc
  *
@@ -128,7 +130,7 @@ DAT_RETURN dapls_ib_qp_alloc(IN DAPL_IA 
 	dapl_os_lock_init(&conn->lock);
 
 	/* create CM_ID, bind to local device, create QP */
-	if (rdma_create_id(&cm_id, (void*)conn)) {
+	if (rdma_create_id(g_cm_events, &cm_id, (void*)conn)) {
 		dapl_os_free(conn, sizeof(*conn));
 		return(dapl_convert_errno(errno, "create_qp"));
 	}