[openib-general] [PATCH 4/4] osm: building LID matrices in fat-tree routing
Yevgeny Kliteynik
kliteyn at dev.mellanox.co.il
Mon Jan 1 05:31:35 PST 2007
Building LID matrices in fat-tree routing
Lid matrices are now built while setting routing
w/o any additional complexity, and removed the
temporary ftree dump function (that differed from
the ucast mgr dump function only by not dumping
anything that is related to lid matrices).
Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
osm/opensm/osm_ucast_ftree.c | 173 ++++++++++++++++++++++++++++--------------
1 files changed, 115 insertions(+), 58 deletions(-)
diff --git a/osm/opensm/osm_ucast_ftree.c b/osm/opensm/osm_ucast_ftree.c
index 9a1ac69..6f72791 100644
--- a/osm/opensm/osm_ucast_ftree.c
+++ b/osm/opensm/osm_ucast_ftree.c
@@ -770,6 +770,26 @@ __osm_ftree_sw_get_fwd_table_block(
return p_sw->lft_buf[lid_ho];
}
+/***************************************************/
+
+static inline cl_status_t
+__osm_ftree_sw_set_hops(
+ IN ftree_sw_t * p_sw,
+ IN uint16_t max_lid_ho,
+ IN uint16_t lid_ho,
+ IN uint8_t port_num,
+ IN uint8_t hops)
+{
+ /* make sure the lid matrix has enough room */
+ osm_switch_set_min_lid_size(p_sw->p_osm_sw, max_lid_ho);
+
+ /* set local min hop table(LID) */
+ return osm_switch_set_hops(p_sw->p_osm_sw,
+ lid_ho,
+ port_num,
+ hops);
+}
+
/***************************************************
**
** ftree_hca_t functions
@@ -1747,8 +1767,10 @@ __osm_ftree_fabric_route_upgoing_by_goin
IN ftree_sw_t * p_sw,
IN ftree_sw_t * p_prev_sw,
IN ib_net16_t target_lid,
+ IN uint8_t target_rank,
IN boolean_t is_real_lid,
- IN boolean_t is_main_path)
+ IN boolean_t is_main_path,
+ IN uint8_t highest_rank_in_route)
{
ftree_sw_t * p_remote_sw;
uint16_t ports_num;
@@ -1813,6 +1835,7 @@ __osm_ftree_fabric_route_upgoing_by_goin
* - going DOWN(TRUE,TRUE) through ALL the groups
* + promoting port counter
* + setting path in remote switch fwd tbl
+ * + setting hops in remote switch on all the ports of each group
*
* 2. is_real_lid == TRUE && is_main_path == FALSE:
* - going DOWN(TRUE,FALSE) through ALL the groups but only if
@@ -1820,11 +1843,14 @@ __osm_ftree_fabric_route_upgoing_by_goin
* for this target LID
* + NOT promoting port counter
* + setting path in remote switch fwd tbl if it hasn't been set yet
+ * + setting hops in remote switch on all the ports of each group
+ * if it hasn't been set yet
*
* 3. is_real_lid == FALSE && is_main_path == TRUE:
* - going DOWN(FALSE,TRUE) through ALL the groups
* + promoting port counter
* + NOT setting path in remote switch fwd tbl
+ * + NOT setting hops in remote switch
*
* 4. is_real_lid == FALSE && is_main_path == FALSE:
* - illegal state - we shouldn't get here
@@ -1849,6 +1875,23 @@ __osm_ftree_fabric_route_upgoing_by_goin
__osm_ftree_tuple_to_str(p_remote_sw->tuple),
cl_ntoh16(target_lid),
p_min_port->remote_port_num);
+
+ /* On the remote switch that is pointed by the p_group,
+ set hops for ALL the ports in the remote group. */
+
+ for (j = 0; j < ports_num; j++)
+ {
+ cl_ptr_vector_at(&p_group->ports, j, (void **)&p_port);
+
+ __osm_ftree_sw_set_hops(p_remote_sw,
+ p_ftree->lft_max_lid_ho,
+ cl_ntoh16(target_lid),
+ p_port->remote_port_num,
+ ( (target_rank - highest_rank_in_route) +
+ (p_remote_sw->rank - highest_rank_in_route) ));
+ }
+
+
}
/* The number of upgoing routes is tracked in the
@@ -1869,8 +1912,10 @@ __osm_ftree_fabric_route_upgoing_by_goin
p_remote_sw, /* remote switch - used as a route-upgoing alg. start point */
NULL, /* prev. position - NULL to mark that we went down and not up */
target_lid, /* LID that we're routing to */
+ target_rank, /* rank of the LID that we're routing to */
is_real_lid, /* whether the target LID is real or dummy */
- is_main_path); /* whether this is path to HCA that should by tracked by counters */
+ is_main_path, /* whether this is path to HCA that should by tracked by counters */
+ highest_rank_in_route); /* highest visited point in the tree before going down */
}
}
/* done scanning all the down-going port groups */
@@ -1896,6 +1941,7 @@ __osm_ftree_fabric_route_downgoing_by_go
IN ftree_sw_t * p_sw,
IN ftree_sw_t * p_prev_sw,
IN ib_net16_t target_lid,
+ IN uint8_t target_rank,
IN boolean_t is_real_lid,
IN boolean_t is_main_path)
{
@@ -1920,8 +1966,10 @@ __osm_ftree_fabric_route_downgoing_by_go
p_sw, /* local switch - used as a route-upgoing alg. start point */
p_prev_sw, /* switch that we went up from (NULL means that we went down) */
target_lid, /* LID that we're routing to */
+ target_rank, /* rank of the LID that we're routing to */
is_real_lid, /* whether this target LID is real or dummy */
- is_main_path); /* whether this path to HCA should by tracked by counters */
+ is_main_path, /* whether this path to HCA should by tracked by counters */
+ p_sw->rank); /* the highest visited point in the tree before going down */
}
/* recursion stop condition - if it's a root switch, */
@@ -1971,9 +2019,12 @@ __osm_ftree_fabric_route_downgoing_by_go
* - going UP(TRUE,TRUE) on selected min_group and min_port
* + promoting port counter
* + setting path in remote switch fwd tbl
+ * + setting hops in remote switch on all the ports of selected group
* - going UP(TRUE,FALSE) on rest of the groups, each time on port 0
* + NOT promoting port counter
* + setting path in remote switch fwd tbl if it hasn't been set yet
+ * + setting hops in remote switch on all the ports of each group
+ * if it hasn't been set yet
*
* 2. is_real_lid == TRUE && is_main_path == FALSE:
* - going UP(TRUE,FALSE) on ALL the groups, each time on port 0,
@@ -1981,11 +2032,14 @@ __osm_ftree_fabric_route_downgoing_by_go
* configured for this target LID
* + NOT promoting port counter
* + setting path in remote switch fwd tbl if it hasn't been set yet
+ * + setting hops in remote switch on all the ports of each group
+ * if it hasn't been set yet
*
* 3. is_real_lid == FALSE && is_main_path == TRUE:
* - going UP(FALSE,TRUE) ONLY on selected min_group and min_port
* + promoting port counter
* + NOT setting path in remote switch fwd tbl
+ * + NOT setting hops in remote switch
*
* 4. is_real_lid == FALSE && is_main_path == FALSE:
* - illegal state - we shouldn't get here
@@ -2013,12 +2067,25 @@ __osm_ftree_fabric_route_downgoing_by_go
__osm_ftree_sw_set_fwd_table_block(p_remote_sw,
cl_ntoh16(target_lid),
p_min_port->remote_port_num);
- p_remote_sw->lft_buf[cl_ntoh16(target_lid)] = p_min_port->remote_port_num;
osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
"__osm_ftree_fabric_route_downgoing_by_going_up: "
"Switch %s: set path to HCA LID 0x%x through port %u\n",
__osm_ftree_tuple_to_str(p_remote_sw->tuple),
cl_ntoh16(target_lid),p_min_port->remote_port_num);
+
+ /* On the remote switch that is pointed by the min_group,
+ set hops for ALL the ports in the remote group. */
+
+ ports_num = (uint16_t)cl_ptr_vector_get_size(&p_min_group->ports);
+ for (j = 0; j < ports_num; j++)
+ {
+ cl_ptr_vector_at(&p_min_group->ports, j, (void **)&p_port);
+ __osm_ftree_sw_set_hops(p_remote_sw,
+ p_ftree->lft_max_lid_ho,
+ cl_ntoh16(target_lid),
+ p_port->remote_port_num,
+ target_rank - p_remote_sw->rank);
+ }
}
/* Recursion step:
@@ -2028,6 +2095,7 @@ __osm_ftree_fabric_route_downgoing_by_go
p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */
p_sw, /* this switch - prev. position switch for the function */
target_lid, /* LID that we're routing to */
+ target_rank, /* rank of the LID that we're routing to */
is_real_lid, /* whether this target LID is real or dummy */
is_main_path); /* whether this is path to HCA that should by tracked by counters */
}
@@ -2044,6 +2112,8 @@ __osm_ftree_fabric_route_downgoing_by_go
* configured for this target LID
* + NOT promoting port counter
* + setting path in remote switch fwd tbl if it hasn't been set yet
+ * + setting hops in remote switch on all the ports of each group
+ * if it hasn't been set yet
*
* 2. is_real_lid == TRUE && is_main_path == FALSE:
* - going UP(TRUE,FALSE) on ALL the groups, each time on port 0,
@@ -2051,6 +2121,8 @@ __osm_ftree_fabric_route_downgoing_by_go
* configured for this target LID
* + NOT promoting port counter
* + setting path in remote switch fwd tbl if it hasn't been set yet
+ * + setting hops in remote switch on all the ports of each group
+ * if it hasn't been set yet
*
* These two rules can be rephrased this way:
* - foreach UP port group
@@ -2060,6 +2132,7 @@ __osm_ftree_fabric_route_downgoing_by_go
* - select port 0
* - do NOT promote port counter
* - set path in remote switch fwd tbl
+ * - set hops in remote switch on all the ports of this group
* - go UP(TRUE,FALSE) to the remote switch
*/
@@ -2087,6 +2160,22 @@ __osm_ftree_fabric_route_downgoing_by_go
__osm_ftree_sw_set_fwd_table_block(p_remote_sw,
cl_ntoh16(target_lid),
p_port->remote_port_num);
+
+ /* On the remote switch that is pointed by the p_group,
+ set hops for ALL the ports in the remote group. */
+
+ ports_num = (uint16_t)cl_ptr_vector_get_size(&p_group->ports);
+ for (j = 0; j < ports_num; j++)
+ {
+ cl_ptr_vector_at(&p_group->ports, j, (void **)&p_port);
+
+ __osm_ftree_sw_set_hops(p_remote_sw,
+ p_ftree->lft_max_lid_ho,
+ cl_ntoh16(target_lid),
+ p_port->remote_port_num,
+ target_rank - p_remote_sw->rank);
+ }
+
/* Recursion step:
Assign downgoing ports by stepping up, starting on REMOTE switch. */
__osm_ftree_fabric_route_downgoing_by_going_up(
@@ -2094,6 +2183,7 @@ __osm_ftree_fabric_route_downgoing_by_go
p_remote_sw, /* remote switch - used as a route-downgoing alg. next step point */
p_sw, /* this switch - prev. position switch for the function */
target_lid, /* LID that we're routing to */
+ target_rank, /* rank of the LID that we're routing to */
TRUE, /* whether the target LID is real or dummy */
FALSE); /* whether this is path to HCA that should by tracked by counters */
}
@@ -2150,17 +2240,26 @@ __osm_ftree_fabric_route_to_hcas(
cl_ntoh16(remote_lid),
p_port->port_num);
+ /* set local min hop table(LID) to route to the CA */
+ __osm_ftree_sw_set_hops(p_sw,
+ p_ftree->lft_max_lid_ho,
+ cl_ntoh16(remote_lid),
+ p_port->port_num,
+ 1);
+
/* assign downgoing ports by stepping up */
__osm_ftree_fabric_route_downgoing_by_going_up(
p_ftree,
p_sw, /* local switch - used as a route-downgoing alg. start point */
NULL, /* prev. position switch */
remote_lid, /* LID that we're routing to */
+ __osm_ftree_fabric_get_rank(p_ftree), /* rank of the LID that we're routing to */
TRUE, /* whether this HCA LID is real or dummy */
TRUE); /* whether this path to HCA should by tracked by counters */
}
- /* We're done with the real HCAs. Now route the dummy HCAs that are missing.*/
+ /* We're done with the real HCAs. Now route the dummy HCAs that are missing.
+ When routing to dummy HCAs we don't fill lid matrices. */
if (p_ftree->max_hcas_per_leaf > p_sw->down_port_groups_num)
{
@@ -2177,6 +2276,7 @@ __osm_ftree_fabric_route_to_hcas(
p_sw, /* local switch - used as a route-downgoing alg. start point */
NULL, /* prev. position switch */
0, /* LID that we're routing to - ignored for dummy HCA */
+ 0, /* rank of the LID that we're routing to - ignored for dummy HCA */
FALSE, /* whether this HCA LID is real or dummy */
TRUE); /* whether this path to HCA should by tracked by counters */
}
@@ -2226,11 +2326,19 @@ __osm_ftree_fabric_route_to_switches(
__osm_ftree_tuple_to_str(p_sw->tuple),
cl_ntoh16(p_sw->base_lid));
+ /* set min hop table of the switch to itself */
+ __osm_ftree_sw_set_hops(p_sw,
+ p_ftree->lft_max_lid_ho,
+ cl_ntoh16(p_sw->base_lid),
+ 0, /* port_num */
+ 0);/* hops */
+
__osm_ftree_fabric_route_downgoing_by_going_up(
p_ftree,
p_sw, /* local switch - used as a route-downgoing alg. start point */
NULL, /* prev. position switch */
p_sw->base_lid, /* LID that we're routing to */
+ p_sw->rank, /* rank of the LID that we're routing to */
TRUE, /* whether the target LID is a real or dummy */
FALSE); /* whether this path should by tracked by counters */
}
@@ -2971,48 +3079,6 @@ __osm_ftree_do_routing(
/***************************************************
***************************************************/
-static int
-__osm_ftree_routing(
- IN void * context)
-{
- int status = __osm_ftree_construct_fabric(context);
- if (status != 0)
- return status;
-
- __osm_ftree_do_routing(context);
- return 0;
-}
-
-/***************************************************
- ***************************************************/
-
-void
-ucast_mgr_dump_to_file(
- IN osm_ucast_mgr_t *p_mgr,
- IN const char *file_name,
- IN void (*func)(cl_map_item_t *, void *));
-
-void
-ucast_mgr_dump_lfts(
- IN cl_map_item_t *p_map_item,
- void *cxt);
-
-static void
-__osm_ftree_dump_tables(
- IN void * context)
-{
- ftree_fabric_t * p_ftree = context;
- if (!p_ftree)
- return;
-
- ucast_mgr_dump_to_file(&p_ftree->p_osm->sm.ucast_mgr,
- "opensm-lfts.dump",
- ucast_mgr_dump_lfts);
-}
-
-/***************************************************
- ***************************************************/
-
static void
__osm_ftree_delete(
IN void * context)
@@ -3034,17 +3100,8 @@ int osm_ucast_ftree_setup(osm_opensm_t *
p_ftree->p_osm = p_osm;
p_osm->routing_engine.context = (void *)p_ftree;
- p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_routing;
- /* ToDo: Resolve multicast routing.
- * Until then lid matrices are built, despite the
- * fact that FatTree routing doesn't need them.
- * When the multicast routing will be resolved,
- * __osm_ftree_routing() function should be removed,
- * and here's how the FatTree routing will be set:
- * p_osm->routing_engine.build_lid_matrices = __osm_ftree_construct_fabric;
- * p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
- */
- p_osm->routing_engine.ucast_dump_tables = __osm_ftree_dump_tables;
+ p_osm->routing_engine.build_lid_matrices = __osm_ftree_construct_fabric;
+ p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
p_osm->routing_engine.delete = __osm_ftree_delete;
return 0;
}
--
1.4.4.1.GIT
More information about the general
mailing list