[openib-general] [PATCH] uDAPL - rdma_ucm branch: add changes to support rr/init exchange

[Arlin Davis]

Some uDAPL changes to support exchanging and validation of the device responder_resources and the
initiator_depth during connection establishment.

Signed-off by: Arlin Davis ardavis at ichips.intel.com


diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c
old mode 100644
new mode 100755
index 0f24244..8bdd0eb
--- a/dapl/openib_cma/dapl_ib_cm.c
+++ b/dapl/openib_cma/dapl_ib_cm.c
@@ -259,6 +259,18 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
 		new_conn->sp = conn->sp;
 		new_conn->hca = conn->hca;
 		
+		/* Get requesters connect data, setup for accept */
+		new_conn->params.responder_resources = 
+			DAPL_MIN(event->param.conn.initiator_depth,
+				 conn->hca->ib_trans.max_rdma_rd_in);
+		new_conn->params.initiator_depth = 
+			DAPL_MIN(event->param.conn.responder_resources,
+				 conn->hca->ib_trans.max_rdma_rd_out);
+
+		new_conn->params.flow_control = event->param.conn.flow_control;
+		new_conn->params.rnr_retry_count = event->param.conn.rnr_retry_count;
+		new_conn->params.retry_count = event->param.conn.retry_count;
+
 		/* save private data */
 		if (event->param.conn.private_data_len) {
 			dapl_os_memcpy(new_conn->p_data, 
@@ -279,7 +291,8 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
 			     event->param.conn.private_data, event->param.conn.private_data_len);
 		
 		dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: "
-			     "REQ: IP SRC %x PORT %d DST %x PORT %d\n", 
+			     "REQ: IP SRC %x PORT %d DST %x PORT %d "
+			     "rr %d init %d\n", 
 			     ntohl(((struct sockaddr_in *)
 				&ipaddr->src_addr)->sin_addr.s_addr),
 			     ntohs(((struct sockaddr_in *)
@@ -287,7 +300,9 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn,
 			     ntohl(((struct sockaddr_in *)
 				&ipaddr->dst_addr)->sin_addr.s_addr),
 			     ntohs(((struct sockaddr_in *)
-				&ipaddr->dst_addr)->sin_port));
+				&ipaddr->dst_addr)->sin_port),
+			     new_conn->params.responder_resources,
+			     new_conn->params.initiator_depth);
 	}
 	return new_conn;
 }
@@ -556,8 +571,8 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle,
 
 	/* Setup QP/CM parameters and private data in cm_id */
 	(void)dapl_os_memzero(&conn->params, sizeof(conn->params));
-	conn->params.responder_resources = IB_TARGET_MAX;
-	conn->params.initiator_depth = IB_INITIATOR_DEPTH;
+	conn->params.responder_resources = conn->hca->ib_trans.max_rdma_rd_in;
+	conn->params.initiator_depth = conn->hca->ib_trans.max_rdma_rd_out;
 	conn->params.flow_control = 1;
 	conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT;
 	conn->params.retry_count = IB_RC_RETRY_COUNT;
@@ -814,7 +829,6 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
 	struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle;
 	int ret;
 	DAT_RETURN dat_status;
-	struct rdma_conn_param conn_params;
 	
 	dapl_dbg_log(DAPL_DBG_TYPE_CM,
 		     " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n",
@@ -867,16 +881,10 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle,
 	ep_ptr->qp_handle = cr_conn;
 	ep_ptr->cm_handle = cr_conn;
 	cr_conn->ep = ep_ptr;
+	cr_conn->params.private_data = p_data;
+	cr_conn->params.private_data_len = p_size;
 
-	memset(&conn_params, 0, sizeof(conn_params));
-	conn_params.private_data = p_data;
-	conn_params.private_data_len = p_size;
-	conn_params.responder_resources = IB_TARGET_MAX;
-	conn_params.initiator_depth = IB_INITIATOR_DEPTH;
-	conn_params.flow_control = 1;
-	conn_params.rnr_retry_count = IB_RNR_RETRY_COUNT;
-
-	ret = rdma_accept(cr_conn->cm_id, &conn_params);
+	ret = rdma_accept(cr_conn->cm_id, &cr_conn->params);
 	if (ret) {
 		dapl_dbg_log(DAPL_DBG_TYPE_ERR," accept: ERROR %d\n", ret);
 		dat_status = dapl_convert_errno(ret, "accept");
diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c
old mode 100644
new mode 100755
index 6bb35f6..0606312
--- a/dapl/openib_cma/dapl_ib_util.c
+++ b/dapl/openib_cma/dapl_ib_util.c
@@ -469,6 +469,9 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr,
 		ia_attr->num_vendor_attr          = 0;
 		ia_attr->vendor_attr              = NULL;
 		ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
+		/* save rd_atom for peer validation during connect requests */
+		hca_ptr->ib_trans.max_rdma_rd_in  = dev_attr.max_qp_rd_atom;
+		hca_ptr->ib_trans.max_rdma_rd_out = dev_attr.max_qp_rd_atom;
 
 		dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 
 			" query_hca: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n", 
diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h
old mode 100644
new mode 100755
index 1fadb08..457d26b
--- a/dapl/openib_cma/dapl_ib_util.h
+++ b/dapl/openib_cma/dapl_ib_util.h
@@ -73,10 +73,7 @@ typedef ib_hca_handle_t		dapl_ibal_ca_t;
 #define IB_ARP_RETRY_COUNT	15	/* 60 sec total */
 #define IB_ROUTE_TIMEOUT	4000	/* 4 sec */
 #define IB_ROUTE_RETRY_COUNT	15	/* 60 sec total */
-#define IB_REQ_MRA_TIMEOUT	27	/* a little over 9 minutes */
 #define IB_MAX_AT_RETRY		3
-#define IB_TARGET_MAX		4	/* max_qp_ous_rd_atom */
-#define IB_INITIATOR_DEPTH	4	/* max_qp_init_rd_atom */
 
 typedef enum {
 	IB_CME_CONNECTED,
@@ -272,6 +269,9 @@ typedef struct _ib_hca_transport
 	ib_async_qp_handler_t	async_qp_error;
 	uint8_t			max_cm_timeout;
 	uint8_t			max_cm_retries;
+	/* device attributes */
+	int			max_rdma_rd_in;
+	int			max_rdma_rd_out;
 
 } ib_hca_transport_t;