[openib-general] [PATCH] CM: associate sends with current CEP state

[Sean Hefty]

Committing patch that will associate sends with the current
communication identifier state.  Fixes a race condition completing
a send in failure with receiving a response.  Also added calls to
cancel sends once response is received.

- Sean

signed-off-by: Sean Hefty <sean.hefty at intel.com>

Index: core/cm.c
===================================================================
--- core/cm.c	(revision 1567)
+++ core/cm.c	(working copy)
@@ -81,6 +81,7 @@
 	struct cm_id_private *cm_id_priv;
 	struct ib_send_wr send_wr;
 	struct ib_sge sge;
+	enum ib_cm_state sent_state;
 	u8 retry;
 	DECLARE_PCI_UNMAP_ADDR(mapping)
 	struct ib_mad mad;
@@ -768,6 +769,7 @@
 		goto out;
 	}
 	cm_id->state = IB_CM_REQ_SENT;
+	msg->sent_state = IB_CM_REQ_SENT;
 	cm_id_priv->msg = msg;
 
 	if (param->peer_to_peer) {
@@ -904,8 +906,8 @@
 		cm_insert_remote_id(cur_cm_id_priv);
 		spin_unlock_irqrestore(&cm.lock, flags);
 
-		ib_cancel_mad(cur_cm_id_priv->port->mad_agent,
-			      (unsigned long)cur_cm_id_priv->msg);
+		ib_cancel_mad(port->mad_agent,
+			      (unsigned long) cur_cm_id_priv->msg);
 		ib_destroy_cm_id(&cm_id_priv->id);
 		cm_id_priv = cur_cm_id_priv;
 	}
@@ -1029,6 +1031,7 @@
 	}
 
 	cm_id->state = IB_CM_REP_SENT;
+	msg->sent_state = IB_CM_REP_SENT;
 	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
@@ -1145,6 +1148,8 @@
 	cm_id_priv->id.state = IB_CM_REP_RCVD;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
+	ib_cancel_mad(port->mad_agent, (unsigned long) cm_id_priv->msg);
+
 	cm_id_priv->id.remote_id = rep_msg->local_comm_id;
 	cm_id_priv->remote_ca_guid = rep_msg->local_ca_guid;
 	cm_id_priv->remote_qpn = cm_rep_get_local_qpn(rep_msg);
@@ -1201,6 +1206,8 @@
 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
+	ib_cancel_mad(port->mad_agent, (unsigned long) cm_id_priv->msg);
+
 	cm_event->event = IB_CM_RTU_RECEIVED;
 	cm_event->private_data = &rtu_msg->private_data;
 	cm_id_priv->id.cm_handler(&cm_id_priv->id, cm_event);
@@ -1287,6 +1294,7 @@
 	}
 
 	cm_id->state = IB_CM_DREQ_SENT;
+	msg->sent_state = IB_CM_DREQ_SENT;
 	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
@@ -1418,6 +1426,8 @@
 	cm_id_priv->id.state = IB_CM_TIMEWAIT;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
+	ib_cancel_mad(port->mad_agent, (unsigned long) cm_id_priv->msg);
+
 	cm_event->event = IB_CM_DREP_RECEIVED;
 	cm_event->private_data = &drep_msg->private_data;
 	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, cm_event);
@@ -1514,7 +1524,6 @@
 	}
 
 	cm_id->state = IB_CM_IDLE;
-	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
 	return ret;
@@ -1621,7 +1630,6 @@
 	default:
 		break;
 	}
-	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
 	return ret;
@@ -1704,6 +1712,7 @@
 	}
 
 	cm_id->lap_state = IB_CM_LAP_SENT;
+	msg->sent_state = IB_CM_ESTABLISHED;
 	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
@@ -1858,6 +1867,7 @@
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_apr_msg *apr_msg;
+	struct cm_msg *msg;
 	struct ib_cm_event *cm_event;
 	unsigned long flags;
 
@@ -1878,10 +1888,11 @@
 		goto out;
 	}
 	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
+	msg = cm_id_priv->msg;
+	cm_id_priv->msg = NULL;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ib_cancel_mad(cm_id_priv->port->mad_agent,
-		      (unsigned long)cm_id_priv->msg);
+	ib_cancel_mad(port->mad_agent, (unsigned long) msg);
 
 	cm_event->event = IB_CM_APR_RECEIVED;
 	cm_event->param.apr_rcvd.ap_status = apr_msg->ap_status;
@@ -1959,6 +1970,7 @@
 		goto out;
 	}
 	cm_id->state = IB_CM_SIDR_REQ_SENT;
+	msg->sent_state = IB_CM_SIDR_REQ_SENT;
 	cm_id_priv->msg = msg;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 out:
@@ -2127,8 +2139,7 @@
 	cm_id_priv->id.state = IB_CM_IDLE;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ib_cancel_mad(cm_id_priv->port->mad_agent,
-		      (unsigned long)cm_id_priv->msg);
+	ib_cancel_mad(port->mad_agent, (unsigned long) cm_id_priv->msg);
 
 	cm_event->event = IB_CM_SIDR_REP_RECEIVED;
 	cm_event->param.sidr_rep_rcvd.status = sidr_rep_msg->status;
@@ -2162,7 +2173,7 @@
 
 	/* Discard old sends or ones without a response. */
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
-	if (msg != cm_id_priv->msg)
+	if (msg != cm_id_priv->msg || msg->sent_state != cm_id_priv->id.state)
 		goto discard;
 
 	switch (cm_id_priv->id.state) {
@@ -2207,7 +2218,7 @@
 
 	/* Discard old sends. */
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
-	if (msg != cm_id_priv->msg) {
+	if (msg != cm_id_priv->msg || msg->sent_state != cm_id_priv->id.state) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		cm_free_msg(msg);
 		return;