[ofa-general] [PATCH 5/7][v1.2] dapl scm: support global routing and set mtu based on active_mtu

Davis, Arlin R arlin.r.davis at intel.com
Fri Jun 20 11:51:13 PDT 2008


Signed-off by: Arlin Davis ardavis at ichips.intel.com
---
 dapl/openib_scm/dapl_ib_qp.c   |   42
+++++++++++++++++++++------------------
 dapl/openib_scm/dapl_ib_util.c |    7 ++++++
 dapl/openib_scm/dapl_ib_util.h |   11 +++++++++-
 3 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/dapl/openib_scm/dapl_ib_qp.c b/dapl/openib_scm/dapl_ib_qp.c
index 1eba2bd..8577131 100644
--- a/dapl/openib_scm/dapl_ib_qp.c
+++ b/dapl/openib_scm/dapl_ib_qp.c
@@ -25,9 +25,9 @@
 
 /**********************************************************************
  *
- * MODULE: dapl_det_qp.c
+ * MODULE: dapl_ib_qp.c
  *
- * PURPOSE: QP routines for access to DET Verbs
+ * PURPOSE: QP routines for access to ofa rdma verbs 
  *
  * $Id: $
 
**********************************************************************/
@@ -285,17 +285,12 @@ void
 dapls_ib_reinit_ep (
 	IN  DAPL_EP	*ep_ptr)
 {
-	
 	if ( ep_ptr->qp_handle != IB_INVALID_HANDLE ) {
 		/* move to RESET state and then to INIT */
 		dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_RESET,
0);
 		dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_INIT,
0);
 		ep_ptr->qp_state = IB_QP_STATE_INIT;
 	}
-
-	/* TODO: When IB-CM is implement then handle timewait before 
-	 * allowing re-use of this QP
-	 */
 }
 
 /* 
@@ -325,19 +320,28 @@ dapls_modify_qp_state ( IN ib_qp_handle_t
qp_handle,
 				IBV_QP_MAX_DEST_RD_ATOMIC |
 				IBV_QP_MIN_RNR_TIMER;
 
-			qp_attr.qp_state 		= IBV_QPS_RTR;
-			qp_attr.path_mtu 		= IBV_MTU_2048;
-			qp_attr.dest_qp_num 		= qp_cm->qpn;
-			qp_attr.rq_psn 			= 1;
-			qp_attr.max_dest_rd_atomic	= 
+			qp_attr.qp_state = IBV_QPS_RTR;
+			qp_attr.dest_qp_num = qp_cm->qpn;
+			qp_attr.rq_psn = 1;
+			qp_attr.path_mtu = 
+				ia_ptr->hca_ptr->ib_trans.mtu;
+			qp_attr.max_dest_rd_atomic = 
 				ep_ptr->param.ep_attr.max_rdma_read_out;
-			qp_attr.min_rnr_timer		=
ia_ptr->hca_ptr->ib_trans.rnr_timer;
-			qp_attr.ah_attr.is_global	= 0;
-			qp_attr.ah_attr.grh.dgid        = qp_cm->gid;
-			qp_attr.ah_attr.dlid		= qp_cm->lid;
-			qp_attr.ah_attr.sl		= 0;
-			qp_attr.ah_attr.src_path_bits	= 0;
-			qp_attr.ah_attr.port_num	= qp_cm->port;
+			qp_attr.min_rnr_timer =
+				ia_ptr->hca_ptr->ib_trans.rnr_timer;
+			qp_attr.ah_attr.dlid = qp_cm->lid;
+			/* global routing */
+			if (ia_ptr->hca_ptr->ib_trans.global) {
+				qp_attr.ah_attr.is_global = 1;
+				qp_attr.ah_attr.grh.dgid = qp_cm->gid;
+				qp_attr.ah_attr.grh.hop_limit = 
+
ia_ptr->hca_ptr->ib_trans.hop_limit;
+				qp_attr.ah_attr.grh.traffic_class = 
+
ia_ptr->hca_ptr->ib_trans.tclass;
+			}
+			qp_attr.ah_attr.sl = 0;
+			qp_attr.ah_attr.src_path_bits = 0;
+			qp_attr.ah_attr.port_num = qp_cm->port;
 			
 			dapl_dbg_log (DAPL_DBG_TYPE_EP,
 			      " modify_qp_rtr: qpn %x lid %x "
diff --git a/dapl/openib_scm/dapl_ib_util.c
b/dapl/openib_scm/dapl_ib_util.c
index a9941f5..ecf6f6a 100644
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -210,6 +210,12 @@ found:
 		dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY);
 	hca_ptr->ib_trans.rnr_timer = 
 		dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER);
+	hca_ptr->ib_trans.global = 
+		dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL);
+	hca_ptr->ib_trans.hop_limit = 
+		dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
+	hca_ptr->ib_trans.tclass = 
+		dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
 
 	/* initialize cq_lock */
 	dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
@@ -433,6 +439,7 @@ DAT_RETURN dapls_ib_query_hca (
 		ia_attr->vendor_attr              = NULL;
 		hca_ptr->ib_trans.ack_timer	  =
DAPL_MAX(dev_attr.local_ca_ack_delay,
 
hca_ptr->ib_trans.ack_timer);
+		hca_ptr->ib_trans.mtu		  =
port_attr.active_mtu;
 		dapl_dbg_log (DAPL_DBG_TYPE_UTIL, 
 			" query_hca: (%x.%x) ep %d ep_q %d evd %d evd_q
%d\n", 
 			ia_attr->hardware_version_major,
diff --git a/dapl/openib_scm/dapl_ib_util.h
b/dapl/openib_scm/dapl_ib_util.h
index 81a1752..37c5dbb 100644
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -186,6 +186,11 @@ typedef struct ibv_comp_channel
*ib_wait_obj_handle_t;
 #define SCM_RNR_TIMER 28	/* 5 bits, 28 == 163ms, 31 == 491ms */
 #define SCM_RNR_RETRY 7		/* 3 bits, 7 == infinite */
 
+/* Global routing defaults */
+#define SCM_GLOBAL	0	/* global routing is disabled */
+#define SCM_HOP_LIMIT	0xff
+#define SCM_TCLASS	0
+
 /* CM private data areas */
 #define	IB_MAX_REQ_PDATA_SIZE	92
 #define	IB_MAX_REP_PDATA_SIZE	196
@@ -193,7 +198,7 @@ typedef struct ibv_comp_channel
*ib_wait_obj_handle_t;
 #define	IB_MAX_DREQ_PDATA_SIZE	220
 #define	IB_MAX_DREP_PDATA_SIZE	224
 
-/* DTO OPs, ordered for DAPL ENUM definitions ???*/
+/* DTO OPs, ordered for DAPL ENUM definitions */
 #define OP_RDMA_WRITE           IBV_WR_RDMA_WRITE
 #define OP_RDMA_WRITE_IMM       IBV_WR_RDMA_WRITE_WITH_IMM
 #define OP_SEND                 IBV_WR_SEND
@@ -295,6 +300,10 @@ typedef struct _ib_hca_transport
 	uint8_t			ack_retry;
 	uint8_t			rnr_timer;
 	uint8_t			rnr_retry;
+	uint8_t			global;
+	uint8_t			hop_limit;
+	uint8_t			tclass;
+	uint8_t			mtu;
 } ib_hca_transport_t;
 
 /* provider specfic fields for shared memory support */
-- 
1.5.2.5




More information about the general mailing list