[openib-general] [PATCH] uDAPL openib_cma disconnect processing fix

[Arlin Davis]

James,

Here is a patch to fix up the disconnect event processing and a change to dtest to validate. Tested
with dtest and dapltest.

-arlin


Signed-off-by: Arlin Davis ardavis at ichips.intel.com

Index: test/dtest/dtest.c
===================================================================
--- test/dtest/dtest.c	(revision 4759)
+++ test/dtest/dtest.c	(working copy)
@@ -862,15 +862,31 @@ disconnect_ep()
 
     if (connected) {
 
-       LOGPRINTF("%d dat_ep_disconnect\n", getpid());
-       ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );
-       if(ret != DAT_SUCCESS)  {
-               fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",
-                               getpid(),DT_RetToString(ret));
-       }
-       else {
+       /* 
+        * Only the client needs to call disconnect. The server _should_ be able to
+        * just wait on the EVD associated with connection events for a disconnect
+        * request and exit then.
+        */
+       if ( !server ) {
+           LOGPRINTF("%d dat_ep_disconnect\n", getpid());
+           ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );
+           if(ret != DAT_SUCCESS)  {
+                   fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",
+                                   getpid(),DT_RetToString(ret));
+           }
+           else {
                LOGPRINTF("%d dat_ep_disconnect completed\n", getpid());
+           }
        }
+
+    	ret = dat_evd_wait( h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore );
+    	if(ret != DAT_SUCCESS) {
+           	fprintf(stderr, "%d Error dat_evd_wait: %s\n",
+                           	getpid(),DT_RetToString(ret));
+    	}
+    	else {
+           	LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());
+    	}
     }
 
     /* destroy service point */
Index: dapl/openib_cma/dapl_ib_cm.c
===================================================================
--- dapl/openib_cma/dapl_ib_cm.c	(revision 4759)
+++ dapl/openib_cma/dapl_ib_cm.c	(working copy)
@@ -35,7 +35,7 @@
  *
  *   Description: 
  *
- *   The uDAPL openib provider - connection management
+ *   The OpenIB uCMA provider - uCMA connection management
  *
  ****************************************************************************
  *		   Source Control System Information
@@ -287,6 +287,12 @@ static int dapli_cm_active_cb(struct dap
 		break;
 
 	case RDMA_CM_EVENT_DISCONNECTED:
+		/* validate EP handle */
+		if (!DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP)) 
+			dapl_evd_connection_callback(conn, 
+						     IB_CME_DISCONNECTED,
+						     NULL, 
+						     conn->ep);
 		break;
 	default:
 		dapl_dbg_log(
@@ -364,6 +370,13 @@ static int dapli_cm_passive_cb(struct da
 		
 		break;
 	case RDMA_CM_EVENT_DISCONNECTED:
+		/* validate SP handle context */
+		if (!DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_PSP) || 
+		    !DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_RSP))
+			dapls_cr_callback(conn, 
+					  IB_CME_DISCONNECTED, 
+					  NULL, 
+					  conn->sp);
 		break;
 	default:
 		dapl_dbg_log(DAPL_DBG_TYPE_ERR, " passive_cb: "
@@ -496,21 +509,10 @@ dapls_ib_disconnect(IN DAPL_EP *ep_ptr,
 			     " disconnect: ID %p ret %d\n", 
 			     ep_ptr->cm_handle, ret);
 
-	/*
-	 * uDAPL does NOT expect disconnect callback from provider
-	 * with abrupt close. uDAPL will callback with DISC event when 
-	 * from provider returns. So, if callback is expected from 
-	 * rdma_cma then block and don't post the event during callback.
+	/* 
+	 * DAT event notification occurs from the callback
+	 * Note: will fire even if DREQ goes unanswered on timeout 
 	 */
-	if (close_flags != DAT_CLOSE_ABRUPT_FLAG)
-	{
-		if (ep_ptr->cr_ptr) 
-			dapls_cr_callback(conn, IB_CME_DISCONNECTED, NULL,
-					  ((DAPL_CR *)ep_ptr->cr_ptr)->sp_ptr);
-		else 
-			dapl_evd_connection_callback(conn, IB_CME_DISCONNECTED,
-						     NULL, ep_ptr);
-	}
 	return DAT_SUCCESS;
 }
 
@@ -537,11 +539,8 @@ dapls_ib_disconnect_clean(IN DAPL_EP *ep
 			  IN DAT_BOOLEAN active,
 			  IN const ib_cm_events_t ib_cm_event)
 {
-	/*
-	 * Clean up outstanding connection state
-	 */
-	dapls_ib_disconnect(ep_ptr, DAT_CLOSE_ABRUPT_FLAG);
-        
+	/* nothing to do */
+	return;
 }
 
 /*
@@ -592,7 +591,11 @@ dapls_ib_setup_conn_listener(IN DAPL_IA 
 
 	if (rdma_bind_addr(conn->cm_id,
 			   (struct sockaddr *)&ia_ptr->hca_ptr->hca_address)) {
-		dat_status = dapl_convert_errno(errno,"setup_listener");
+		if (errno == -EBUSY)
+			dat_status = DAT_CONN_QUAL_IN_USE;
+		else
+			dat_status = 
+				dapl_convert_errno(errno,"setup_listener");
 		goto bail;
 	}