[openib-general] [PATCH] [kDAPL} Fix locking problem in some CM callback functions

[Hal Rosenstock]

Fix locking problem in some CM callback functions
Since modify QP can sleep, dapl_ib_disconnect must be called without
holding spinlock.

Signed-off-by: Itamar Rabenstein <itamar at mellanox.co.il>
Signed-off-by: Hal Rosenstock <halr at voltaire.com>

Index: dapl_openib_util.h
===================================================================
--- dapl_openib_util.h	(revision 2681)
+++ dapl_openib_util.h	(working copy)
@@ -125,7 +125,7 @@
 
 void dapl_ib_reinit_ep(struct dapl_ep *ep);
 
-void dapl_ib_disconnect_clean(struct dapl_ep *ep, boolean_t passive);
+void dapl_ib_disconnect_clean(struct dapl_ep *ep);
 
 u32 dapl_ib_get_async_event(struct ib_event *cause,
 			    enum dat_event_number *async_event);
Index: dapl_cr.c
===================================================================
--- dapl_cr.c	(revision 2681)
+++ dapl_cr.c	(working copy)
@@ -397,7 +397,7 @@
 	struct dapl_ep *ep;
 	struct dapl_evd *evd;
 	u32 status;
-
+	
 	dapl_dbg_log(DAPL_DBG_TYPE_CM | DAPL_DBG_TYPE_CALLBACK,
 		     "--> dapl_cr_callback! cm_handle: %p event: %x sp: %p\n",
 		     cm_ctx, event, sp);
@@ -431,7 +431,6 @@
 	}
 
 	status = DAT_INTERNAL_ERROR;	/* init to ERR */
-
 	switch (event) {
 	case DAT_CONNECTION_REQUEST_EVENT:
 		/*
@@ -479,8 +478,7 @@
 			/* If someone pulled the plug on the EP or connection,
 			 * just exit
 			 */
-			spin_unlock_irqrestore(&ep->common.lock, 
-					       ep->common.flags);
+			spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
 			status = DAT_SUCCESS;
 			/* Set evd = NULL so we don't generate an event below */
 			evd = NULL;
@@ -504,36 +502,22 @@
 			/* The disconnect has already occurred, we are now
 			 * cleaned up and ready to exit
 			 */
-			spin_unlock_irqrestore(&ep->common.lock, 
-					       ep->common.flags);
+			spin_unlock_irqrestore(&ep->common.lock, ep->common.flags); 
 			return;
 		}
 		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, FALSE);
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
+		dapl_ib_disconnect_clean(ep);
 		break;
 	case DAT_CONNECTION_EVENT_NON_PEER_REJECTED:
 	case DAT_CONNECTION_EVENT_PEER_REJECTED:
 	case DAT_CONNECTION_EVENT_UNREACHABLE:
-		/*
-		 * After posting an accept the requesting node has
-		 * stopped talking.
-		 */
+	case DAT_CONNECTION_EVENT_BROKEN:
 		spin_lock_irqsave(&ep->common.lock, ep->common.flags);
 		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, FALSE);
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
+		dapl_ib_disconnect_clean(ep);
 		break;
-	case DAT_CONNECTION_EVENT_BROKEN:
-		spin_lock_irqsave(&ep->common.lock, ep->common.flags);
-		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, FALSE);
-		spin_unlock_irqrestore(&ep->common.lock,
-				       ep->common.flags);
-
-		break;
 	default:
 		evd = NULL;
 		dapl_os_assert(0);	/* shouldn't happen */
Index: dapl_evd.c
===================================================================
--- dapl_evd.c	(revision 2681)
+++ dapl_evd.c	(working copy)
@@ -757,10 +757,8 @@
 	 * when necessary
 	 */
 	spin_lock_irqsave(&ep->common.lock, ep->common.flags);
-
 	switch (event) {
 	case DAT_CONNECTION_EVENT_ESTABLISHED:
-	{
 		/* 
 		 * If we don't have an EP at this point we are very screwed up 
 		 */
@@ -784,65 +782,28 @@
 				      private_data_size);
 		}
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
 		break;
-	}
 	case DAT_CONNECTION_EVENT_DISCONNECTED:
-	{
-		/*
-		 * EP is now fully disconnected; initiate any post processing
-		 * to reset the underlying QP and get the EP ready for
-		 * another connection
-		 */
 		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, TRUE);
-		spin_unlock_irqrestore(&ep->common.lock,
-				       ep->common.flags);
-
-		break;
-	}
-	case DAT_CONNECTION_EVENT_PEER_REJECTED:
-	{
-		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, TRUE);
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
+		dapl_ib_disconnect_clean(ep);
 		break;
-	}
+	case DAT_CONNECTION_EVENT_PEER_REJECTED:
 	case DAT_CONNECTION_EVENT_UNREACHABLE:
-	{
-		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, TRUE);
-		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
-		break;
-	}
 	case DAT_CONNECTION_EVENT_NON_PEER_REJECTED:
-	{
-		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, TRUE);
-		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
-		break;
-	}
 	case DAT_CONNECTION_EVENT_BROKEN:
-	{
 		ep->param.ep_state = DAT_EP_STATE_DISCONNECTED;
-		dapl_ib_disconnect_clean(ep, FALSE);
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
-
+		dapl_ib_disconnect_clean(ep);
 		break;
-	}
 	case DAT_CONNECTION_REQUEST_EVENT:
 	default:
-	{
 		spin_unlock_irqrestore(&ep->common.lock, ep->common.flags);
 		evd = NULL;
 
 		dapl_os_assert(0);	/* shouldn't happen */
 		break;
 	}
-	}
 
 	/*
 	 * Post the event
Index: dapl_openib_cm.c
===================================================================
--- dapl_openib_cm.c	(revision 2681)
+++ dapl_openib_cm.c	(working copy)
@@ -553,7 +553,6 @@
  *
  * Input:
  *      ep_ptr          pointer to dapl_ep struct
- *      active          Indicates active side of connection
  *
  * Output:
  *      none
@@ -562,13 +561,12 @@
  *      void
  *
  */
-void dapl_ib_disconnect_clean(struct dapl_ep *ep_ptr, boolean_t active)
+void dapl_ib_disconnect_clean(struct dapl_ep *ep_ptr)
 {
 	int status;
 
 	dapl_dbg_log(DAPL_DBG_TYPE_CM,
-		     "  >>> dapl_ib_disconnect_clean: EP: %p active %d\n",
-		     ep_ptr, active);
+		     "  >>> dapl_ib_disconnect_clean: EP: %p\n", ep_ptr);
 
 	/*
 	 * Clean up outstanding connection state