[ofa-general] [PATCH] [2.6.25] rdma/cm: override default responder_resources with user value

[Sean Hefty]

By default, the responder_resources parameter is set to that received
in a connection request.  The passive side may override this value
when accepting the connection.  Use the value provided by the passive
side when transitioning the QP to RTR state, rather than the value
given in the connect request.  Without this change, the RTR transition
may fail if the passive side supports fewer responder_resources than
that in the request.

For code consistency and to protect against QP destruction, restructure
overriding initiator_depth to match how responder_resources is set.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
This change is based off of problems seen with uDAPL.  Although the
fix for uDAPL is entirely in userspace (where the modify QP is done),
the kernel rdma_cm has a similar problem.  The problem only occurs if
the passive side CAs supports a smaller number of outstanding RDMA
operations than that requested by the active side.

This should be okay to queue for 2.6.25.  I don't see anything upstream
that is affected by this change.

 drivers/infiniband/core/cma.c |   39 ++++++++++++++++++---------------------
 1 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 656d6df..b3917fe 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -488,7 +488,8 @@ void rdma_destroy_qp(struct rdma_cm_id *id)
 }
 EXPORT_SYMBOL(rdma_destroy_qp);
 
-static int cma_modify_qp_rtr(struct rdma_id_private *id_priv)
+static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
+			     struct rdma_conn_param *conn_param)
 {
 	struct ib_qp_attr qp_attr;
 	int qp_attr_mask, ret;
@@ -514,13 +515,16 @@ static int cma_modify_qp_rtr(struct rdma_id_private *id_priv)
 	if (ret)
 		goto out;
 
+	if (conn_param)
+		qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 out:
 	mutex_unlock(&id_priv->qp_mutex);
 	return ret;
 }
 
-static int cma_modify_qp_rts(struct rdma_id_private *id_priv)
+static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
+			     struct rdma_conn_param *conn_param)
 {
 	struct ib_qp_attr qp_attr;
 	int qp_attr_mask, ret;
@@ -536,6 +540,8 @@ static int cma_modify_qp_rts(struct rdma_id_private *id_priv)
 	if (ret)
 		goto out;
 
+	if (conn_param)
+		qp_attr.max_rd_atomic = conn_param->initiator_depth;
 	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 out:
 	mutex_unlock(&id_priv->qp_mutex);
@@ -866,11 +872,11 @@ static int cma_rep_recv(struct rdma_id_private *id_priv)
 {
 	int ret;
 
-	ret = cma_modify_qp_rtr(id_priv);
+	ret = cma_modify_qp_rtr(id_priv, NULL);
 	if (ret)
 		goto reject;
 
-	ret = cma_modify_qp_rts(id_priv);
+	ret = cma_modify_qp_rts(id_priv, NULL);
 	if (ret)
 		goto reject;
 
@@ -2274,7 +2280,7 @@ static int cma_connect_iw(struct rdma_id_private *id_priv,
 	sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
 	cm_id->remote_addr = *sin;
 
-	ret = cma_modify_qp_rtr(id_priv);
+	ret = cma_modify_qp_rtr(id_priv, conn_param);
 	if (ret)
 		goto out;
 
@@ -2340,22 +2346,13 @@ static int cma_accept_ib(struct rdma_id_private *id_priv,
 	struct ib_qp_attr qp_attr;
 	int qp_attr_mask, ret;
 
-	if (id_priv->id.qp) {
-		ret = cma_modify_qp_rtr(id_priv);
-		if (ret)
-			goto out;
-
-		qp_attr.qp_state = IB_QPS_RTS;
-		ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, &qp_attr,
-					 &qp_attr_mask);
-		if (ret)
-			goto out;
+	ret = cma_modify_qp_rtr(id_priv, conn_param);
+	if (ret)
+		goto out;
 
-		qp_attr.max_rd_atomic = conn_param->initiator_depth;
-		ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
-		if (ret)
-			goto out;
-	}
+	ret = cma_modify_qp_rts(id_priv, conn_param);
+	if (ret)
+		goto out;
 
 	memset(&rep, 0, sizeof rep);
 	rep.qp_num = id_priv->qp_num;
@@ -2380,7 +2377,7 @@ static int cma_accept_iw(struct rdma_id_private *id_priv,
 	struct iw_cm_conn_param iw_param;
 	int ret;
 
-	ret = cma_modify_qp_rtr(id_priv);
+	ret = cma_modify_qp_rtr(id_priv, conn_param);
 	if (ret)
 		return ret;