[ofa-general] [PATCH 5/7][v1.2] dapl scm: support global routing and set mtu based on active_mtu
Davis, Arlin R
arlin.r.davis at intel.com
Fri Jun 20 11:51:13 PDT 2008
Signed-off by: Arlin Davis ardavis at ichips.intel.com
---
dapl/openib_scm/dapl_ib_qp.c | 42
+++++++++++++++++++++------------------
dapl/openib_scm/dapl_ib_util.c | 7 ++++++
dapl/openib_scm/dapl_ib_util.h | 11 +++++++++-
3 files changed, 40 insertions(+), 20 deletions(-)
diff --git a/dapl/openib_scm/dapl_ib_qp.c b/dapl/openib_scm/dapl_ib_qp.c
index 1eba2bd..8577131 100644
--- a/dapl/openib_scm/dapl_ib_qp.c
+++ b/dapl/openib_scm/dapl_ib_qp.c
@@ -25,9 +25,9 @@
/**********************************************************************
*
- * MODULE: dapl_det_qp.c
+ * MODULE: dapl_ib_qp.c
*
- * PURPOSE: QP routines for access to DET Verbs
+ * PURPOSE: QP routines for access to ofa rdma verbs
*
* $Id: $
**********************************************************************/
@@ -285,17 +285,12 @@ void
dapls_ib_reinit_ep (
IN DAPL_EP *ep_ptr)
{
-
if ( ep_ptr->qp_handle != IB_INVALID_HANDLE ) {
/* move to RESET state and then to INIT */
dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_RESET,
0);
dapls_modify_qp_state(ep_ptr->qp_handle, IBV_QPS_INIT,
0);
ep_ptr->qp_state = IB_QP_STATE_INIT;
}
-
- /* TODO: When IB-CM is implement then handle timewait before
- * allowing re-use of this QP
- */
}
/*
@@ -325,19 +320,28 @@ dapls_modify_qp_state ( IN ib_qp_handle_t
qp_handle,
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER;
- qp_attr.qp_state = IBV_QPS_RTR;
- qp_attr.path_mtu = IBV_MTU_2048;
- qp_attr.dest_qp_num = qp_cm->qpn;
- qp_attr.rq_psn = 1;
- qp_attr.max_dest_rd_atomic =
+ qp_attr.qp_state = IBV_QPS_RTR;
+ qp_attr.dest_qp_num = qp_cm->qpn;
+ qp_attr.rq_psn = 1;
+ qp_attr.path_mtu =
+ ia_ptr->hca_ptr->ib_trans.mtu;
+ qp_attr.max_dest_rd_atomic =
ep_ptr->param.ep_attr.max_rdma_read_out;
- qp_attr.min_rnr_timer =
ia_ptr->hca_ptr->ib_trans.rnr_timer;
- qp_attr.ah_attr.is_global = 0;
- qp_attr.ah_attr.grh.dgid = qp_cm->gid;
- qp_attr.ah_attr.dlid = qp_cm->lid;
- qp_attr.ah_attr.sl = 0;
- qp_attr.ah_attr.src_path_bits = 0;
- qp_attr.ah_attr.port_num = qp_cm->port;
+ qp_attr.min_rnr_timer =
+ ia_ptr->hca_ptr->ib_trans.rnr_timer;
+ qp_attr.ah_attr.dlid = qp_cm->lid;
+ /* global routing */
+ if (ia_ptr->hca_ptr->ib_trans.global) {
+ qp_attr.ah_attr.is_global = 1;
+ qp_attr.ah_attr.grh.dgid = qp_cm->gid;
+ qp_attr.ah_attr.grh.hop_limit =
+
ia_ptr->hca_ptr->ib_trans.hop_limit;
+ qp_attr.ah_attr.grh.traffic_class =
+
ia_ptr->hca_ptr->ib_trans.tclass;
+ }
+ qp_attr.ah_attr.sl = 0;
+ qp_attr.ah_attr.src_path_bits = 0;
+ qp_attr.ah_attr.port_num = qp_cm->port;
dapl_dbg_log (DAPL_DBG_TYPE_EP,
" modify_qp_rtr: qpn %x lid %x "
diff --git a/dapl/openib_scm/dapl_ib_util.c
b/dapl/openib_scm/dapl_ib_util.c
index a9941f5..ecf6f6a 100644
--- a/dapl/openib_scm/dapl_ib_util.c
+++ b/dapl/openib_scm/dapl_ib_util.c
@@ -210,6 +210,12 @@ found:
dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY);
hca_ptr->ib_trans.rnr_timer =
dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER);
+ hca_ptr->ib_trans.global =
+ dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL);
+ hca_ptr->ib_trans.hop_limit =
+ dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
+ hca_ptr->ib_trans.tclass =
+ dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
/* initialize cq_lock */
dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
@@ -433,6 +439,7 @@ DAT_RETURN dapls_ib_query_hca (
ia_attr->vendor_attr = NULL;
hca_ptr->ib_trans.ack_timer =
DAPL_MAX(dev_attr.local_ca_ack_delay,
hca_ptr->ib_trans.ack_timer);
+ hca_ptr->ib_trans.mtu =
port_attr.active_mtu;
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
" query_hca: (%x.%x) ep %d ep_q %d evd %d evd_q
%d\n",
ia_attr->hardware_version_major,
diff --git a/dapl/openib_scm/dapl_ib_util.h
b/dapl/openib_scm/dapl_ib_util.h
index 81a1752..37c5dbb 100644
--- a/dapl/openib_scm/dapl_ib_util.h
+++ b/dapl/openib_scm/dapl_ib_util.h
@@ -186,6 +186,11 @@ typedef struct ibv_comp_channel
*ib_wait_obj_handle_t;
#define SCM_RNR_TIMER 28 /* 5 bits, 28 == 163ms, 31 == 491ms */
#define SCM_RNR_RETRY 7 /* 3 bits, 7 == infinite */
+/* Global routing defaults */
+#define SCM_GLOBAL 0 /* global routing is disabled */
+#define SCM_HOP_LIMIT 0xff
+#define SCM_TCLASS 0
+
/* CM private data areas */
#define IB_MAX_REQ_PDATA_SIZE 92
#define IB_MAX_REP_PDATA_SIZE 196
@@ -193,7 +198,7 @@ typedef struct ibv_comp_channel
*ib_wait_obj_handle_t;
#define IB_MAX_DREQ_PDATA_SIZE 220
#define IB_MAX_DREP_PDATA_SIZE 224
-/* DTO OPs, ordered for DAPL ENUM definitions ???*/
+/* DTO OPs, ordered for DAPL ENUM definitions */
#define OP_RDMA_WRITE IBV_WR_RDMA_WRITE
#define OP_RDMA_WRITE_IMM IBV_WR_RDMA_WRITE_WITH_IMM
#define OP_SEND IBV_WR_SEND
@@ -295,6 +300,10 @@ typedef struct _ib_hca_transport
uint8_t ack_retry;
uint8_t rnr_timer;
uint8_t rnr_retry;
+ uint8_t global;
+ uint8_t hop_limit;
+ uint8_t tclass;
+ uint8_t mtu;
} ib_hca_transport_t;
/* provider specfic fields for shared memory support */
--
1.5.2.5
More information about the general
mailing list