[ofw] [PATCH] dapl-2.0: ucm, scm: exchange max_qp_rd_atom and limit outstanding requests

Davis, Arlin R arlin.r.davis at intel.com
Thu Jan 20 18:52:06 PST 2011


exchange and add proper checking to limit outstanding
rdma reads and atomics. Use one of the reserve bytes
in CM message protocol to exchange limits and reset
EP attribute rdma_out and set QP RTS attribute properly.

Signed-off-by: Arlin Davis <arlin.r.davis at intel.com>
---
 dapl/openib_common/dapl_ib_common.h |    5 +++--
 dapl/openib_common/qp.c             |    5 +++--
 dapl/openib_scm/cm.c                |   11 +++++++++++
 dapl/openib_ucm/cm.c                |   13 +++++++++++++
 4 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/dapl/openib_common/dapl_ib_common.h b/dapl/openib_common/dapl_ib_common.h
index 8b0e0ba..78f71d2 100644
--- a/dapl/openib_common/dapl_ib_common.h
+++ b/dapl/openib_common/dapl_ib_common.h
@@ -58,7 +58,7 @@ typedef	struct ibv_context	*ib_hca_handle_t;
 typedef ib_hca_handle_t		dapl_ibal_ca_t;
 
 /* QP info to exchange, wire protocol version for these CM's */
-#define DCM_VER 6
+#define DCM_VER 7
 
 /* CM private data areas, same for all operations */
 #define        DCM_MAX_PDATA_SIZE      118
@@ -98,7 +98,8 @@ typedef struct _ib_cm_msg
 	uint16_t		p_size;
 	uint32_t		s_id;  /* src pid */
 	uint32_t		d_id;  /* dst pid */
-	uint8_t			resv[6];
+	uint8_t			rd_in; /* atomic_rd_in */
+	uint8_t			resv[5];
 	union dcm_addr		saddr;
 	union dcm_addr		daddr;
 	union dcm_addr		saddr_alt;
diff --git a/dapl/openib_common/qp.c b/dapl/openib_common/qp.c
index 1826362..fb13576 100644
--- a/dapl/openib_common/qp.c
+++ b/dapl/openib_common/qp.c
@@ -442,11 +442,12 @@ dapls_modify_qp_state(IN ib_qp_handle_t		qp_handle,
 	case IBV_QPS_RTR:
 		dapl_dbg_log(DAPL_DBG_TYPE_EP,
 				" QPS_RTR: type %d qpn 0x%x gid %p (%d) lid 0x%x"
-				" port %d ep %p qp_state %d \n",
+				" port %d ep %p qp_state %d rd_atomic %d\n",
 				qp_handle->qp_type, ntohl(qpn), gid, 
 				ia_ptr->hca_ptr->ib_trans.global,
 				ntohs(lid), ia_ptr->hca_ptr->port_num,
-				ep_ptr, ep_ptr->qp_state);
+				ep_ptr, ep_ptr->qp_state,
+				ep_ptr->param.ep_attr.max_rdma_read_in);
 
 		mask |= IBV_QP_AV |
 			IBV_QP_PATH_MTU |
diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c
index c638663..b0fbadf 100644
--- a/dapl/openib_scm/cm.c
+++ b/dapl/openib_scm/cm.c
@@ -530,6 +530,9 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err)
 	cm_ptr->state = DCM_REP_PENDING;
 	dapl_os_unlock(&cm_ptr->lock);
 
+	/* set max rdma inbound requests */
+	cm_ptr->msg.rd_in = ep_ptr->param.ep_attr.max_rdma_read_in;
+
 	/* send qp info and pdata to remote peer */
 	exp = sizeof(ib_cm_msg_t) - DCM_MAX_PDATA_SIZE;
 	iov[0].iov_base = (void *)&cm_ptr->msg;
@@ -798,6 +801,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr)
 		goto bail;
 	}
 
+	/* rdma_out, initiator, cannot exceed remote rdma_in max */
+	ep_ptr->param.ep_attr.max_rdma_read_out = 
+		DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, cm_ptr->msg.rd_in);
+
 	/* modify QP to RTR and then to RTS with remote info */
 	dapl_os_lock(&ep_ptr->header.lock);
 	if (dapls_modify_qp_state(ep_ptr->qp_handle,
@@ -1196,6 +1203,9 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
 		goto bail;
 	}
 #endif
+	/* rdma_out, initiator, cannot exceed remote rdma_in max */
+	ep_ptr->param.ep_attr.max_rdma_read_out = 
+		DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, cm_ptr->msg.rd_in);
 
 	/* modify QP to RTR and then to RTS with remote info already read */
 	dapl_os_lock(&ep_ptr->header.lock);
@@ -1235,6 +1245,7 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr,
 	/* send our QP info, IA address, pdata. Don't overwrite dst data */
 	local.ver = htons(DCM_VER);
 	local.op = htons(DCM_REP);
+	local.rd_in = ep_ptr->param.ep_attr.max_rdma_read_in;
 	local.saddr.ib.qpn = htonl(ep_ptr->qp_handle->qp_num);
 	local.saddr.ib.qp_type = ep_ptr->qp_handle->qp_type;
 	local.saddr.ib.lid = ia_ptr->hca_ptr->ib_trans.lid;
diff --git a/dapl/openib_ucm/cm.c b/dapl/openib_ucm/cm.c
index 5b157fd..c5ddf04 100644
--- a/dapl/openib_ucm/cm.c
+++ b/dapl/openib_ucm/cm.c
@@ -1080,6 +1080,10 @@ static void ucm_connect_rtu(dp_ib_cm_handle_t cm, ib_cm_msg_t *msg)
 	}
 	dapl_os_unlock(&cm->lock);
 
+        /* rdma_out, initiator, cannot exceed remote rdma_in max */
+        cm->ep->param.ep_attr.max_rdma_read_out =
+                DAPL_MIN(cm->ep->param.ep_attr.max_rdma_read_out, cm->msg.rd_in);
+
 	/* modify QP to RTR and then to RTS with remote info */
 	dapl_os_lock(&cm->ep->header.lock);
 	if (dapls_modify_qp_state(cm->ep->qp_handle,
@@ -1232,6 +1236,7 @@ static void ucm_accept(ib_cm_srvc_handle_t cm, ib_cm_msg_t *msg)
 	acm->msg.sqpn = cm->msg.sqpn;
 	acm->msg.p_size = msg->p_size;
 	acm->msg.d_id = msg->s_id;
+	acm->msg.rd_in = msg->rd_in;
 
 	/* CR saddr is CM daddr info, need EP for local saddr */
 	dapl_os_memcpy(&acm->msg.daddr, &msg->saddr, sizeof(union dcm_addr));
@@ -1510,6 +1515,10 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
 	}
 #endif
 
+        /* rdma_out, initiator, cannot exceed remote rdma_in max */
+        ep->param.ep_attr.max_rdma_read_out =
+                DAPL_MIN(ep->param.ep_attr.max_rdma_read_out, cm->msg.rd_in);
+
 	/* modify QP to RTR and then to RTS with remote info already read */
 	dapl_os_lock(&ep->header.lock);
 	if (dapls_modify_qp_state(ep->qp_handle,
@@ -1544,6 +1553,7 @@ dapli_accept_usr(DAPL_EP *ep, DAPL_CR *cr, DAT_COUNT p_size, DAT_PVOID p_data)
 
 	/* setup local QP info and type from EP, copy pdata, for reply */
 	cm->msg.op = htons(DCM_REP);
+	cm->msg.rd_in = ep->param.ep_attr.max_rdma_read_in;
 	cm->msg.saddr.ib.qpn = htonl(ep->qp_handle->qp_num);
 	cm->msg.saddr.ib.qp_type = ep->qp_handle->qp_type;
 	cm->msg.saddr.ib.lid = cm->hca->ib_trans.addr.ib.lid; 
@@ -1626,6 +1636,9 @@ dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
 	cm->msg.dqpn = cm->msg.daddr.ib.qpn;
 	cm->msg.daddr.ib.qpn = 0; /* don't have a remote qpn until reply */
 	
+        /* set max rdma inbound requests */
+        cm->msg.rd_in = ep->param.ep_attr.max_rdma_read_in;
+
 	if (p_size) {
 		cm->msg.p_size = htons(p_size);
 		dapl_os_memcpy(&cm->msg.p_data, p_data, p_size);
-- 
1.7.3






More information about the ofw mailing list