[ofa-general] [PATCH] [RFC] ib/cm: modify interface to send MRAs in response to duplicate messages

[Sean Hefty]

The IB CM provides a message received acknowledged (MRA) message that
can be sent to indicate that a REQ or REP message has been received, but
will require more time to process than the timeout specified by those
messages.  In many cases, the application may not know how long it will
take to respond to a CM message, but the majority of the time, it will
usually respond before a retry has been sent.  Rather than sending an MRA
in response to all messages just to handle the case where a longer timeout
is needed, it is more efficient to queue the MRA for sending in case a
duplicate message is received.

This avoids sending an MRA when it is not needed, but limits the number
of times that a REQ or REP will be resent.  It also provides for a
simpler implementation than generating the MRA based on a timer event.
(That is, trying to send the MRA after receiving the first REQ or REP
if a response has not been generated, so that it is received at the
remote side before a duplicate REQ or REP has been received.)

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
This patch was created in response to ideas posted in this thread:

http://lists.openfabrics.org/pipermail/general/2007-July/037763.html

If this approach looks okay, I would like this patch considered for 2.6.23.
I have a follow on patch to the rdma_cm that uses it.

 drivers/infiniband/core/cm.c |   51 +++++++++++++++++++-----------------------
 include/rdma/ib_cm.h         |    7 ++++--
 2 files changed, 28 insertions(+), 30 deletions(-)

diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 40c004a..7a0b214 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -2203,6 +2203,9 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
+	enum ib_cm_state cm_state;
+	enum ib_cm_lap_state lap_state;
+	enum cm_msg_response msg_response;
 	void *data;
 	unsigned long flags;
 	int ret;
@@ -2219,48 +2222,40 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	switch(cm_id_priv->id.state) {
 	case IB_CM_REQ_RCVD:
-		ret = cm_alloc_msg(cm_id_priv, &msg);
-		if (ret)
-			goto error1;
-
-		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
-			      CM_MSG_RESPONSE_REQ, service_timeout,
-			      private_data, private_data_len);
-		ret = ib_post_send_mad(msg, NULL);
-		if (ret)
-			goto error2;
-		cm_id->state = IB_CM_MRA_REQ_SENT;
+		cm_state = IB_CM_MRA_REQ_SENT;
+		lap_state = cm_id->lap_state;
+		msg_response = CM_MSG_RESPONSE_REQ;
 		break;
 	case IB_CM_REP_RCVD:
-		ret = cm_alloc_msg(cm_id_priv, &msg);
-		if (ret)
-			goto error1;
-
-		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
-			      CM_MSG_RESPONSE_REP, service_timeout,
-			      private_data, private_data_len);
-		ret = ib_post_send_mad(msg, NULL);
-		if (ret)
-			goto error2;
-		cm_id->state = IB_CM_MRA_REP_SENT;
+		cm_state = IB_CM_MRA_REP_SENT;
+		lap_state = cm_id->lap_state;
+		msg_response = CM_MSG_RESPONSE_REP;
 		break;
 	case IB_CM_ESTABLISHED:
+		cm_state = cm_id->state;
+		lap_state = IB_CM_MRA_LAP_SENT;
+		msg_response = CM_MSG_RESPONSE_OTHER;
+		break;
+	default:
+		ret = -EINVAL;
+		goto error1;
+	}
+
+	if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
 		ret = cm_alloc_msg(cm_id_priv, &msg);
 		if (ret)
 			goto error1;
 
 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
-			      CM_MSG_RESPONSE_OTHER, service_timeout,
+			      msg_response, service_timeout,
 			      private_data, private_data_len);
 		ret = ib_post_send_mad(msg, NULL);
 		if (ret)
 			goto error2;
-		cm_id->lap_state = IB_CM_MRA_LAP_SENT;
-		break;
-	default:
-		ret = -EINVAL;
-		goto error1;
 	}
+
+	cm_id->state = cm_state;
+	cm_id->lap_state = lap_state;
 	cm_id_priv->service_timeout = service_timeout;
 	cm_set_private_data(cm_id_priv, data, private_data_len);
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h
index 5c07017..9d769c5 100644
--- a/include/rdma/ib_cm.h
+++ b/include/rdma/ib_cm.h
@@ -478,12 +478,15 @@ int ib_send_cm_rej(struct ib_cm_id *cm_id,
 		   const void *private_data,
 		   u8 private_data_len);
 
+#define IB_CM_MRA_FLAG_DELAY 0x80  /* Send MRA only after a duplicate msg */
+
 /**
  * ib_send_cm_mra - Sends a message receipt acknowledgement to a connection
  *   message.
  * @cm_id: Connection identifier associated with the connection message.
- * @service_timeout: The maximum time required for the sender to reply to
- *   to the connection message.
+ * @service_timeout: The lower 5-bits specify the maximum time required for
+ *   the sender to reply to the connection message.  The upper 3-bits
+ *   specify additional control flags.
  * @private_data: Optional user-defined private data sent with the
  *   message receipt acknowledgement.
  * @private_data_len: Size of the private data buffer, in bytes.