[openib-general] [PATCHv2] osm: improving FatTree routing engine

Yevgeny Kliteynik kliteyn at dev.mellanox.co.il
Wed Dec 20 00:54:50 PST 2006


Hi Hal

[V2 of the patch - not using max_lid_ho setter]

FatTree routing engine improvemets:
1. Improved building of LFTs
2. Setting max lid on osm switches
3. Using ucast manager LFT dump function
4. Stoped using global variable 'osm'
5. Improved logging
6. Some cosmetics

Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
 osm/opensm/osm_ucast_ftree.c |  439 +++++++++++++++++++++++++++---------------
 1 files changed, 281 insertions(+), 158 deletions(-)

diff --git a/osm/opensm/osm_ucast_ftree.c b/osm/opensm/osm_ucast_ftree.c
index 15e4cd0..0d7188a 100644
--- a/osm/opensm/osm_ucast_ftree.c
+++ b/osm/opensm/osm_ucast_ftree.c
@@ -57,9 +57,6 @@
 #include <opensm/osm_opensm.h>
 #include <opensm/osm_switch.h>
 
-/* This var is predefined and initialized */
-extern osm_opensm_t osm;
-
 /*
  * FatTree rank is bounded between 2 and 8:
  *  - Tree of rank 1 has only trivial routing pathes,
@@ -211,14 +208,16 @@ typedef struct ftree_hca_t_ {
 
 typedef struct ftree_fabric_t_ 
 {
-   cl_qmap_t     hca_tbl;
-   cl_qmap_t     sw_tbl;
-   cl_qmap_t     sw_by_tuple_tbl;
-   uint32_t      tree_rank;
-   ftree_sw_t ** leaf_switches;
-   uint32_t      leaf_switches_num;
-   uint16_t      max_hcas_per_leaf;
-   cl_pool_t     sw_fwd_tbl_pool;
+   osm_opensm_t  * p_osm;
+   cl_qmap_t       hca_tbl;
+   cl_qmap_t       sw_tbl;
+   cl_qmap_t       sw_by_tuple_tbl;
+   uint32_t        tree_rank;
+   ftree_sw_t   ** leaf_switches;
+   uint32_t        leaf_switches_num;
+   uint16_t        max_hcas_per_leaf;
+   cl_pool_t       sw_fwd_tbl_pool;
+   uint16_t        lft_max_lid_ho;
 } ftree_fabric_t;
 
 /***************************************************
@@ -506,6 +505,7 @@ __osm_ftree_port_group_destroy(
 
 static void 
 __osm_ftree_port_group_dump(
+   IN  ftree_fabric_t *p_ftree,
    IN  ftree_port_group_t * p_group,
    IN  ftree_direction_t direction)
 {
@@ -517,7 +517,7 @@ __osm_ftree_port_group_dump(
    if (!p_group)
       return;
 
-   if (!osm_log_is_active(&osm.log,OSM_LOG_DEBUG))
+   if (!osm_log_is_active(&p_ftree->p_osm->log, OSM_LOG_DEBUG))
       return;
 
    size = cl_ptr_vector_get_size(&p_group->ports);
@@ -533,7 +533,7 @@ __osm_ftree_port_group_dump(
       sprintf(buff + strlen(buff), "%u", p_port->port_num);
    }
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
            "__osm_ftree_port_group_dump:"
            "    Port Group of size %u, port(s): %s, direction: %s\n" 
            "                  Local <--> Remote GUID (LID):"
@@ -648,16 +648,17 @@ __osm_ftree_sw_destroy(
 
 static void 
 __osm_ftree_sw_dump(
+   IN  ftree_fabric_t * p_ftree,
    IN  ftree_sw_t * p_sw)
 {
    uint32_t i;
    if (!p_sw)
       return;
 
-   if (!osm_log_is_active(&osm.log,OSM_LOG_DEBUG))
+   if (!osm_log_is_active(&p_ftree->p_osm->log, OSM_LOG_DEBUG))
       return;
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
            "__osm_ftree_sw_dump: "
            "Switch index: %s, GUID: 0x%016" PRIx64 ", Ports: %u DOWN, %u UP\n",
           __osm_ftree_tuple_to_str(p_sw->tuple),
@@ -665,10 +666,14 @@ __osm_ftree_sw_dump(
           p_sw->down_port_groups_num, 
           p_sw->up_port_groups_num);
 
-   for( i = 0; i < p_sw->down_port_groups_num; i++ ) 
-      __osm_ftree_port_group_dump(p_sw->down_port_groups[i], FTREE_DIRECTION_DOWN);
-   for( i = 0; i < p_sw->up_port_groups_num; i++ ) 
-      __osm_ftree_port_group_dump(p_sw->up_port_groups[i], FTREE_DIRECTION_UP);
+   for( i = 0; i < p_sw->down_port_groups_num; i++ )
+      __osm_ftree_port_group_dump(p_ftree,
+                                  p_sw->down_port_groups[i],
+                                  FTREE_DIRECTION_DOWN);
+   for( i = 0; i < p_sw->up_port_groups_num; i++ )
+      __osm_ftree_port_group_dump(p_ftree,
+                                  p_sw->up_port_groups[i],
+                                  FTREE_DIRECTION_UP);
 
 } /* __osm_ftree_sw_dump() */
 
@@ -823,23 +828,26 @@ __osm_ftree_hca_destroy(
 
 static void 
 __osm_ftree_hca_dump(
+   IN  ftree_fabric_t * p_ftree,
    IN  ftree_hca_t * p_hca)
 {
    uint32_t i;
    if (!p_hca)
       return;
 
-   if (!osm_log_is_active(&osm.log,OSM_LOG_DEBUG))
+   if (!osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
       return;
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
            "__osm_ftree_hca_dump: "
            "HCA GUID: 0x%016" PRIx64 ", Ports: %u UP\n",
           cl_ntoh64(osm_node_get_node_guid(p_hca->p_osm_node)), 
           p_hca->up_port_groups_num);
 
    for( i = 0; i < p_hca->up_port_groups_num; i++ ) 
-      __osm_ftree_port_group_dump(p_hca->up_port_groups[i],FTREE_DIRECTION_UP);
+      __osm_ftree_port_group_dump(p_ftree,
+                                  p_hca->up_port_groups[i],
+                                  FTREE_DIRECTION_UP);
 }
 
 /***************************************************/
@@ -1050,6 +1058,10 @@ __osm_ftree_fabric_add_sw(ftree_fabric_t
    cl_qmap_insert(&p_ftree->sw_tbl,
                   p_osm_sw->p_node->node_info.node_guid,
                   &p_sw->map_item);
+
+   /* track the max lid (in host order) that exists in the fabric */
+   if (cl_ntoh16(p_sw->base_lid) > p_ftree->lft_max_lid_ho)
+      p_ftree->lft_max_lid_ho = cl_ntoh16(p_sw->base_lid);
 }
 
 /***************************************************/
@@ -1096,38 +1108,38 @@ __osm_ftree_fabric_dump(ftree_fabric_t *
    ftree_hca_t * p_hca;
    ftree_sw_t * p_sw;
 
-   if (!osm_log_is_active(&osm.log,OSM_LOG_DEBUG))
+   if (!osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
       return;
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,"__osm_ftree_fabric_dump: \n"
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,"__osm_ftree_fabric_dump: \n"
            "                       |-------------------------------|\n"
            "                       |-  Full fabric topology dump  -|\n"
            "                       |-------------------------------|\n\n");
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
            "__osm_ftree_fabric_dump: -- HCAs:\n");
 
    for ( p_hca = (ftree_hca_t *)cl_qmap_head(&p_ftree->hca_tbl);
          p_hca != (ftree_hca_t *)cl_qmap_end(&p_ftree->hca_tbl);
          p_hca = (ftree_hca_t *)cl_qmap_next(&p_hca->map_item) )
    {
-      __osm_ftree_hca_dump(p_hca);
+      __osm_ftree_hca_dump(p_ftree, p_hca);
    }
 
    for (i = 0; i < __osm_ftree_fabric_get_rank(p_ftree); i++)
    {
-      osm_log(&osm.log, OSM_LOG_DEBUG,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
               "__osm_ftree_fabric_dump: -- Rank %u switches\n", i);
       for ( p_sw = (ftree_sw_t *)cl_qmap_head(&p_ftree->sw_tbl);
             p_sw != (ftree_sw_t *)cl_qmap_end(&p_ftree->sw_tbl);
             p_sw = (ftree_sw_t *)cl_qmap_next(&p_sw->map_item) )
       {
          if (p_sw->rank == i)
-            __osm_ftree_sw_dump(p_sw);
+            __osm_ftree_sw_dump(p_ftree, p_sw);
       }
    }
 
-   osm_log(&osm.log, OSM_LOG_DEBUG,"__osm_ftree_fabric_dump: \n"
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,"__osm_ftree_fabric_dump: \n"
            "                       |---------------------------------------|\n"
            "                       |- Full fabric topology dump completed -|\n"
            "                       |---------------------------------------|\n\n");
@@ -1143,16 +1155,18 @@ __osm_ftree_fabric_dump_general_info(
    ftree_sw_t * p_sw;
    char * addition_str;
 
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info:\n");
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,
+           "__osm_ftree_fabric_dump_general_info: "
            "General fabric topology info\n");
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
            "============================\n");
 
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,
+           "__osm_ftree_fabric_dump_general_info: "
            "  - FatTree rank (switches only): %u\n",
           p_ftree->tree_rank);
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,
+           "__osm_ftree_fabric_dump_general_info: "
            "  - Fabric has %u HCAs, %u switches\n",
           cl_qmap_count(&p_ftree->hca_tbl),
           cl_qmap_count(&p_ftree->sw_tbl));
@@ -1174,13 +1188,15 @@ __osm_ftree_fabric_dump_general_info(
             addition_str = " (leaf) ";
          else
             addition_str = " ";
-         osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_dump_general_info: "
-                 "  - Fabric has %u rank %u%sswitches\n",j,i,addition_str);
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,
+                 "__osm_ftree_fabric_dump_general_info: "
+                 "  - Fabric has %u rank %u%sswitches\n",
+                 j,i,addition_str);
    }
 
-   if (osm_log_is_active(&osm.log,OSM_LOG_VERBOSE))
+   if (osm_log_is_active(&p_ftree->p_osm->log, OSM_LOG_VERBOSE))
    {
-      osm_log(&osm.log, OSM_LOG_VERBOSE,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
               "__osm_ftree_fabric_dump_general_info: "
               "  - Root switches:\n");
       for ( p_sw = (ftree_sw_t *)cl_qmap_head(&p_ftree->sw_tbl);
@@ -1188,7 +1204,7 @@ __osm_ftree_fabric_dump_general_info(
             p_sw = (ftree_sw_t *)cl_qmap_next(&p_sw->map_item) )
       {
          if (p_sw->rank == 0)
-               osm_log(&osm.log, OSM_LOG_VERBOSE,
+               osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
                        "__osm_ftree_fabric_dump_general_info: "
                        "      GUID: 0x%016" PRIx64 ", LID: 0x%x, Index %s\n",
                        cl_ntoh64(osm_node_get_node_guid(osm_switch_get_node_ptr(p_sw->p_osm_sw))),
@@ -1196,15 +1212,17 @@ __osm_ftree_fabric_dump_general_info(
                        __osm_ftree_tuple_to_str(p_sw->tuple));
       }
 
-      osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_fabric_dump_general_info: "
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+              "__osm_ftree_fabric_dump_general_info: "
               "  - Leaf switches (sorted by index):\n");
       for (i = 0; i < p_ftree->leaf_switches_num; i++)
       {
-            osm_log(&osm.log, OSM_LOG_VERBOSE,
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
                     "__osm_ftree_fabric_dump_general_info: "
                     "      GUID: 0x%016" PRIx64 ", LID: 0x%x, Index %s\n",
                     cl_ntoh64(osm_node_get_node_guid(
-                                 osm_switch_get_node_ptr(p_ftree->leaf_switches[i]->p_osm_sw))),
+                                 osm_switch_get_node_ptr(
+                                    p_ftree->leaf_switches[i]->p_osm_sw))),
                     cl_ntoh16(p_ftree->leaf_switches[i]->base_lid),
                     __osm_ftree_tuple_to_str(p_ftree->leaf_switches[i]->tuple));
       }
@@ -1229,15 +1247,15 @@ __osm_ftree_fabric_dump_hca_ordering(
    char * filename = "osm-ftree-ca-order.dump";
 
    snprintf(path, sizeof(path), "%s/%s", 
-            osm.subn.opt.dump_files_dir, filename);
+            p_ftree->p_osm->subn.opt.dump_files_dir, filename);
    p_hca_ordering_file = fopen(path, "w");
    if (!p_hca_ordering_file) 
    {
-      osm_log(&osm.log, OSM_LOG_ERROR,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
               "__osm_ftree_fabric_dump_hca_ordering: ERR AB01: "
               "cannot open file \'%s\': %s\n",
                filename, strerror(errno));
-      OSM_LOG_EXIT(&(osm.log));
+      OSM_LOG_EXIT(&p_ftree->p_osm->log);
       return;
    }
    
@@ -1383,9 +1401,9 @@ __osm_ftree_fabric_make_indexing(
    cl_list_t            bfs_list;
    ftree_sw_tbl_element_t * p_sw_tbl_element;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_make_indexing);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_make_indexing);
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_fabric_make_indexing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_fabric_make_indexing: "
            "Starting FatTree indexing\n");
 
    /* create array of leaf switches */
@@ -1411,8 +1429,8 @@ __osm_ftree_fabric_make_indexing(
       This fuction also adds the switch it into the switch_by_tuple table. */
    __osm_ftree_fabric_assign_first_tuple(p_ftree,p_sw);
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_fabric_make_indexing: "
-           "Indexing starting point:\n"
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_fabric_make_indexing: Indexing starting point:\n"
            "                                            - Switch rank  : %u\n"
            "                                            - Switch index : %s\n"
            "                                            - Node LID     : 0x%x\n"
@@ -1537,7 +1555,7 @@ __osm_ftree_fabric_make_indexing(
          sizeof(ftree_sw_t *),       /* size of each element */
          __osm_ftree_compare_switches_by_index); /* comparator */
 
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
 } /* __osm_ftree_fabric_make_indexing() */
 
 /***************************************************/
@@ -1555,15 +1573,17 @@ __osm_ftree_fabric_validate_topology(
    boolean_t            res = TRUE;
    uint8_t              i;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_validate_topology);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_validate_topology);
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE, "__osm_ftree_fabric_validate_topology: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_fabric_validate_topology: "
            "Validating fabric topology\n");
 
    reference_sw_arr = (ftree_sw_t **)malloc(tree_rank * sizeof(ftree_sw_t *));
    if ( reference_sw_arr == NULL )
    {
-      osm_log(&osm.log, OSM_LOG_SYS,"Fat-tree routing: Memory allocation failed\n");
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
+              "Fat-tree routing: Memory allocation failed\n");
       return FALSE;
    }
    memset(reference_sw_arr, 0, tree_rank * sizeof(ftree_sw_t *));
@@ -1587,7 +1607,8 @@ __osm_ftree_fabric_validate_topology(
 
          if ( reference_sw_arr[p_sw->rank]->up_port_groups_num != p_sw->up_port_groups_num )
          {
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_validate_topology: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_validate_topology: "
                     "ERR AB09: Different number of upward port groups on switches:\n"
                     "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u groups\n"
                     "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u groups\n",
@@ -1607,7 +1628,8 @@ __osm_ftree_fabric_validate_topology(
               reference_sw_arr[p_sw->rank]->down_port_groups_num != p_sw->down_port_groups_num )
          {
             /* we're allowing some hca's to be missing */
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_validate_topology: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_validate_topology: "
                     "ERR AB0A: Different number of downward port groups on switches:\n"
                     "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u port groups\n"
                     "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u port groups\n",
@@ -1631,7 +1653,8 @@ __osm_ftree_fabric_validate_topology(
                 p_group = p_sw->up_port_groups[i];
                 if (cl_ptr_vector_get_size(&p_ref_group->ports) != cl_ptr_vector_get_size(&p_group->ports))
                 {
-                   osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_validate_topology: "
+                   osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                           "__osm_ftree_fabric_validate_topology: "
                            "ERR AB0B: Different number of ports in an upward port group on switches:\n"
                            "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u ports\n"
                            "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u ports\n",
@@ -1658,7 +1681,8 @@ __osm_ftree_fabric_validate_topology(
                 p_group = p_sw->down_port_groups[0];
                 if (cl_ptr_vector_get_size(&p_ref_group->ports) != cl_ptr_vector_get_size(&p_group->ports))
                 {
-                   osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_validate_topology: "
+                   osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                           "__osm_ftree_fabric_validate_topology: "
                            "ERR AB0C: Different number of ports in an downward port group on switches:\n"
                            "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u ports\n"
                            "       GUID 0x%016" PRIx64 ", LID 0x%x, Index %s - %u ports\n",
@@ -1679,14 +1703,16 @@ __osm_ftree_fabric_validate_topology(
    } /* end of while */
 
    if (res == TRUE)
-      osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_fabric_validate_topology: "
-                    "Fabric topology has been identified as FatTree\n");
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+              "__osm_ftree_fabric_validate_topology: "
+              "Fabric topology has been identified as FatTree\n");
    else
-      osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_validate_topology: "
-                    "ERR AB0D: Fabric topology hasn't been identified as FatTree\n");
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+              "__osm_ftree_fabric_validate_topology: "
+              "ERR AB0D: Fabric topology hasn't been identified as FatTree\n");
 
    free(reference_sw_arr);
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return res;
 } /* __osm_ftree_fabric_validate_topology() */
 
@@ -1699,8 +1725,17 @@ __osm_ftree_set_sw_fwd_table(
    IN  void *context)
 {
    ftree_sw_t * p_sw = (ftree_sw_t * const) p_map_item;
-   memcpy(osm.sm.ucast_mgr.lft_buf, p_sw->lft_buf, FTREE_FWD_TBL_LEN);
-   osm_ucast_mgr_set_fwd_table(&osm.sm.ucast_mgr,p_sw->p_osm_sw);
+   ftree_fabric_t * p_ftree = (ftree_fabric_t *)context;
+
+   /* calculate lft length rounded up to a multiple of 64 (block length) */ 
+   uint16_t lft_len = 64 * ((p_ftree->lft_max_lid_ho + 1 + 63) / 64);
+
+   p_sw->p_osm_sw->max_lid_ho = p_ftree->lft_max_lid_ho;
+
+   memcpy(p_ftree->p_osm->sm.ucast_mgr.lft_buf, 
+          p_sw->lft_buf, 
+          lft_len);
+   osm_ucast_mgr_set_fwd_table(&p_ftree->p_osm->sm.ucast_mgr, p_sw->p_osm_sw);
 }
 
 /***************************************************
@@ -1746,8 +1781,6 @@ __osm_ftree_fabric_route_upgoing_by_goin
    if (p_sw->down_port_groups_num == 0) 
        return;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_route_upgoing_by_going_down);
-
    /* foreach down-going port group (in indexing order) */
    for (i = 0; i < p_sw->down_port_groups_num; i++)
    {
@@ -1823,7 +1856,7 @@ __osm_ftree_fabric_route_upgoing_by_goin
          __osm_ftree_sw_set_fwd_table_block(p_remote_sw,
                                             cl_ntoh16(target_lid),
                                             p_min_port->remote_port_num);
-         osm_log(&osm.log, OSM_LOG_DEBUG,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_upgoing_by_going_down: "
                  "Switch %s: set path to HCA LID 0x%x through port %u\n",
                  __osm_ftree_tuple_to_str(p_remote_sw->tuple),
@@ -1855,7 +1888,6 @@ __osm_ftree_fabric_route_upgoing_by_goin
    }
    /* done scanning all the down-going port groups */
 
-   OSM_LOG_EXIT(&(osm.log));
 } /* __osm_ftree_fabric_route_upgoing_by_going_down() */
 
 /***************************************************/
@@ -1892,8 +1924,6 @@ __osm_ftree_fabric_route_downgoing_by_go
    /* we shouldn't enter here if both real_lid and main_path are false */
    CL_ASSERT(is_real_lid || is_main_path);
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_route_downgoing_by_going_up);
-
    /* If this switch isn't a leaf switch:
       Assign upgoing ports by stepping down, starting on THIS switch. */
    if (p_sw->rank != (__osm_ftree_fabric_get_rank(p_ftree) - 1))
@@ -1909,10 +1939,7 @@ __osm_ftree_fabric_route_downgoing_by_go
 
    /* recursion stop condition - if it's a root switch, */
    if (p_sw->rank == 0)
-   {
-      OSM_LOG_EXIT(&(osm.log));
       return;
-   }
 
    /* Find the least loaded port of all the upgoing port groups
       (in indexing order of the remote switches). */
@@ -1982,7 +2009,7 @@ __osm_ftree_fabric_route_downgoing_by_go
    {
       if (p_sw->rank == (__osm_ftree_fabric_get_rank(p_ftree) - 1))
       {
-         osm_log(&osm.log, OSM_LOG_DEBUG,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_downgoing_by_going_up: "
                  " - Routing MAIN path for %s HCA LID 0x%x: %s --> %s\n",
                  (is_real_lid)? "real" : "DUMMY",
@@ -2000,7 +2027,7 @@ __osm_ftree_fabric_route_downgoing_by_go
                                             cl_ntoh16(target_lid),
                                             p_min_port->remote_port_num);
          p_remote_sw->lft_buf[cl_ntoh16(target_lid)] = p_min_port->remote_port_num;
-         osm_log(&osm.log, OSM_LOG_DEBUG,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_downgoing_by_going_up: "
                  "Switch %s: set path to HCA LID 0x%x through port %u\n",
                  __osm_ftree_tuple_to_str(p_remote_sw->tuple),
@@ -2020,10 +2047,7 @@ __osm_ftree_fabric_route_downgoing_by_go
 
    /* we're done for the third case */
    if (!is_real_lid)
-   {
-      OSM_LOG_EXIT(&(osm.log));
       return;
-   }
 
    /* What's left to do at this point:
     *
@@ -2064,7 +2088,7 @@ __osm_ftree_fabric_route_downgoing_by_go
 
       if (p_sw->rank == (__osm_ftree_fabric_get_rank(p_ftree) - 1))
       {
-         osm_log(&osm.log, OSM_LOG_DEBUG,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_downgoing_by_going_up: "
                  " - Routing SECONDARY path for LID 0x%x: %s --> %s\n",
                 cl_ntoh16(target_lid),
@@ -2087,7 +2111,6 @@ __osm_ftree_fabric_route_downgoing_by_go
             FALSE);      /* whether this is path to HCA that should by tracked by counters */
    }
 
-   OSM_LOG_EXIT(&(osm.log));
 } /* ftree_fabric_route_downgoing_by_going_up() */
 
 /***************************************************/
@@ -2114,7 +2137,7 @@ __osm_ftree_fabric_route_to_hcas(
    uint32_t             j;
    ib_net16_t           remote_lid;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_route_to_hcas);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_route_to_hcas);
 
    /* for each leaf switch (in indexing order) */
    for(i = 0; i < p_ftree->leaf_switches_num; i++)
@@ -2133,7 +2156,7 @@ __osm_ftree_fabric_route_to_hcas(
          __osm_ftree_sw_set_fwd_table_block(p_sw,
                                             cl_ntoh16(remote_lid),
                                             p_port->port_num);
-         osm_log(&osm.log, OSM_LOG_DEBUG,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
                  "__osm_ftree_fabric_route_to_hcas: "
                  "Switch %s: set path to HCA LID 0x%x through port %u\n",
                  __osm_ftree_tuple_to_str(p_sw->tuple),
@@ -2154,7 +2177,7 @@ __osm_ftree_fabric_route_to_hcas(
 
       if (p_ftree->max_hcas_per_leaf > p_sw->down_port_groups_num)
       {
-         osm_log(&osm.log, OSM_LOG_DEBUG,"__osm_ftree_fabric_route_to_hcas: "
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,"__osm_ftree_fabric_route_to_hcas: "
                  "Routing %u dummy HCAs\n",
                  p_ftree->max_hcas_per_leaf - p_sw->down_port_groups_num);
          for (j = 0; j < (p_ftree->max_hcas_per_leaf - p_sw->down_port_groups_num); j++)
@@ -2171,7 +2194,7 @@ __osm_ftree_fabric_route_to_hcas(
       }
    }
    /* done going through all the leaf switches */
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
 } /* __osm_ftree_fabric_route_to_hcas() */
 
 /***************************************************/
@@ -2195,7 +2218,7 @@ __osm_ftree_fabric_route_to_switches(
    ftree_sw_t         * p_sw;
    ftree_sw_t         * p_next_sw;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_route_to_switches);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_route_to_switches);
 
    p_next_sw = (ftree_sw_t *)cl_qmap_head(&p_ftree->sw_tbl);
    while( p_next_sw != (ftree_sw_t *)cl_qmap_end(&p_ftree->sw_tbl) )
@@ -2208,7 +2231,8 @@ __osm_ftree_fabric_route_to_switches(
                                          cl_ntoh16(p_sw->base_lid),
                                          0);
 
-      osm_log(&osm.log, OSM_LOG_DEBUG,"__osm_ftree_fabric_route_to_switches: "
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
+              "__osm_ftree_fabric_route_to_switches: "
               "Switch %s (LID 0x%x): routing switch-to-switch pathes\n",
               __osm_ftree_tuple_to_str(p_sw->tuple),
               cl_ntoh16(p_sw->base_lid));
@@ -2222,7 +2246,7 @@ __osm_ftree_fabric_route_to_switches(
             FALSE);         /* whether this path should by tracked by counters */
    }
 
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
 } /* __osm_ftree_fabric_route_to_switches() */
 
 /***************************************************
@@ -2234,18 +2258,17 @@ __osm_ftree_fabric_populate_switches(
 {
    osm_switch_t * p_osm_sw;
    osm_switch_t * p_next_osm_sw;
-   osm_opensm_t * p_osm = &osm;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_populate_switches);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_populate_switches);
 
-   p_next_osm_sw = (osm_switch_t *)cl_qmap_head(&p_osm->subn.sw_guid_tbl);
-   while( p_next_osm_sw != (osm_switch_t *)cl_qmap_end(&p_osm->subn.sw_guid_tbl) )
+   p_next_osm_sw = (osm_switch_t *)cl_qmap_head(&p_ftree->p_osm->subn.sw_guid_tbl);
+   while( p_next_osm_sw != (osm_switch_t *)cl_qmap_end(&p_ftree->p_osm->subn.sw_guid_tbl) )
    {
       p_osm_sw = p_next_osm_sw;
       p_next_osm_sw = (osm_switch_t *)cl_qmap_next(&p_osm_sw->map_item );
       __osm_ftree_fabric_add_sw(p_ftree,p_osm_sw);
    }
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return 0;
 } /* __osm_ftree_fabric_populate_switches() */
 
@@ -2258,12 +2281,11 @@ __osm_ftree_fabric_populate_hcas(
 {
    osm_node_t   * p_osm_node;
    osm_node_t   * p_next_osm_node;
-   osm_opensm_t * p_osm = &osm;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_populate_hcas);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_populate_hcas);
 
-   p_next_osm_node = (osm_node_t *)cl_qmap_head(&p_osm->subn.node_guid_tbl);
-   while( p_next_osm_node != (osm_node_t *)cl_qmap_end(&p_osm->subn.node_guid_tbl) )
+   p_next_osm_node = (osm_node_t *)cl_qmap_head(&p_ftree->p_osm->subn.node_guid_tbl);
+   while( p_next_osm_node != (osm_node_t *)cl_qmap_end(&p_ftree->p_osm->subn.node_guid_tbl) )
    {
       p_osm_node = p_next_osm_node;
       p_next_osm_node = (osm_node_t *)cl_qmap_next(&p_osm_node->map_item);
@@ -2278,16 +2300,17 @@ __osm_ftree_fabric_populate_hcas(
             /* all the switches added separately */
             break;
          default:
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_populate_hcas: ERR AB0E: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_populate_hcas: ERR AB0E: "
                     "Node GUID 0x%016" PRIx64 " - Unknown node type: %s\n",
                     cl_ntoh64(osm_node_get_node_guid(p_osm_node)),
                     ib_get_node_type_str(osm_node_get_type(p_osm_node)));
-            OSM_LOG_EXIT(&(osm.log));
+            OSM_LOG_EXIT(&p_ftree->p_osm->log);
             return -1;
       }
    }
 
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return 0;
 } /* __osm_ftree_fabric_populate_hcas() */
 
@@ -2372,7 +2395,7 @@ __osm_ftree_rank_switches_from_hca(
    static uint16_t i = 0;
    int res = 0;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_rank_switches_from_hca);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_rank_switches_from_hca);
 
    for (i = 0; i < osm_node_get_num_physp(p_osm_node); i++)
    {
@@ -2388,7 +2411,8 @@ __osm_ftree_rank_switches_from_hca(
       {
          case IB_NODE_TYPE_CA:
             /* HCA connected directly to another HCA - not FatTree */
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_rank_switches_from_hca: ERR AB0F: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_rank_switches_from_hca: ERR AB0F: "
                     "HCA conected directly to another HCA: "
                     "0x%016" PRIx64 " <---> 0x%016" PRIx64 "\n",
                     cl_ntoh64(osm_node_get_node_guid(p_hca->p_osm_node)),
@@ -2405,7 +2429,8 @@ __osm_ftree_rank_switches_from_hca(
             break;
 
          default:
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_rank_switches_from_hca: ERR AB10: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_rank_switches_from_hca: ERR AB10: "
                     "Node GUID 0x%016" PRIx64 " - Unknown node type: %s\n",
                     cl_ntoh64(osm_node_get_node_guid(p_remote_osm_node)),
                     ib_get_node_type_str(osm_node_get_type(p_remote_osm_node)));
@@ -2423,7 +2448,8 @@ __osm_ftree_rank_switches_from_hca(
       if (__osm_ftree_sw_ranked(p_sw) && p_sw->rank == 0)
          continue;
 
-      osm_log(&osm.log, OSM_LOG_DEBUG,"__osm_ftree_rank_switches_from_hca: "
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_DEBUG,
+              "__osm_ftree_rank_switches_from_hca: "
               "Marking rank of switch that is directly connected to HCA:\n"
               "                                            - HCA guid   : 0x%016" PRIx64 "\n"
               "                                            - Switch guid: 0x%016" PRIx64 "\n"
@@ -2435,7 +2461,7 @@ __osm_ftree_rank_switches_from_hca(
    }
 
  Exit:
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return res;
 } /* __osm_ftree_rank_switches_from_hca() */
 
@@ -2495,7 +2521,8 @@ __osm_ftree_fabric_construct_hca_ports(
 
          case IB_NODE_TYPE_CA:
             /* HCA connected directly to another HCA - not FatTree */
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_construct_hca_ports: ERR AB11: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_construct_hca_ports: ERR AB11: "
                     "HCA conected directly to another HCA: "
                     "0x%016" PRIx64 " <---> 0x%016" PRIx64 "\n",
                     cl_ntoh64(osm_node_get_node_guid(p_node)),
@@ -2508,7 +2535,8 @@ __osm_ftree_fabric_construct_hca_ports(
             break;
 
          default:
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_construct_hca_ports: ERR AB12: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_construct_hca_ports: ERR AB12: "
                     "Node GUID 0x%016" PRIx64 " - Unknown node type: %s\n",
                     cl_ntoh64(remote_node_guid),
                     ib_get_node_type_str(remote_node_type));
@@ -2625,7 +2653,8 @@ __osm_ftree_fabric_construct_sw_ports(
             break;
 
          default:
-            osm_log(&osm.log, OSM_LOG_ERROR,"__osm_ftree_fabric_construct_sw_ports: ERR AB13: "
+            osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                    "__osm_ftree_fabric_construct_sw_ports: ERR AB13: "
                     "Node GUID 0x%016" PRIx64 " - Unknown node type: %s\n",
                     cl_ntoh64(remote_node_guid),
                     ib_get_node_type_str(remote_node_type));
@@ -2646,6 +2675,10 @@ __osm_ftree_fabric_construct_sw_ports(
             remote_node_type,                           /* remote node type */           
             p_remote_hca_or_sw,                         /* remote ftree_hca/sw object */ 
             direction);                                 /* port direction (up or down) */
+
+      /* Track the max lid (in host order) that exists in the fabric */
+      if (cl_ntoh16(remote_base_lid) > p_ftree->lft_max_lid_ho)
+         p_ftree->lft_max_lid_ho = cl_ntoh16(remote_base_lid);
    }
 
  Exit:
@@ -2665,7 +2698,7 @@ __osm_ftree_fabric_perform_ranking(
    ftree_hca_t * p_next_hca;
    int res = 0;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_perform_ranking);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_perform_ranking);
 
    /* Mark REVERSED rank of all the switches in the subnet. 
       Start from switches that are connected to hca's, and 
@@ -2678,7 +2711,8 @@ __osm_ftree_fabric_perform_ranking(
       if (__osm_ftree_rank_switches_from_hca(p_ftree,p_hca) != 0)
       {
          res = -1;
-         osm_log(&osm.log, OSM_LOG_ERROR, "__osm_ftree_fabric_perform_ranking: ERR AB14: "
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR,
+                 "__osm_ftree_fabric_perform_ranking: ERR AB14: "
                  "Subnet ranking failed - subnet is not FatTree");
          goto Exit;
       }
@@ -2686,7 +2720,8 @@ __osm_ftree_fabric_perform_ranking(
 
    /* calculate and set FatTree rank */
    __osm_ftree_fabric_calculate_rank(p_ftree);
-   osm_log(&osm.log, OSM_LOG_INFO,"__osm_ftree_fabric_perform_ranking: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_INFO,
+           "__osm_ftree_fabric_perform_ranking: "
            "FatTree rank is %u\n", __osm_ftree_fabric_get_rank(p_ftree));
    
    /* fix ranking of the switches by reversing the ranking direction */
@@ -2695,7 +2730,8 @@ __osm_ftree_fabric_perform_ranking(
    if ( __osm_ftree_fabric_get_rank(p_ftree) > FAT_TREE_MAX_RANK ||
         __osm_ftree_fabric_get_rank(p_ftree) < FAT_TREE_MIN_RANK )
    {
-      osm_log(&osm.log, OSM_LOG_ERROR, "__osm_ftree_fabric_perform_ranking: ERR AB15: "
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_ERROR, 
+              "__osm_ftree_fabric_perform_ranking: ERR AB15: "
               "Tree rank is %u (should be between %u and %u)\n",
               __osm_ftree_fabric_get_rank(p_ftree),
               FAT_TREE_MIN_RANK,
@@ -2705,7 +2741,7 @@ __osm_ftree_fabric_perform_ranking(
    }
 
   Exit:
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return res;
 } /* __osm_ftree_fabric_perform_ranking() */
 
@@ -2722,7 +2758,7 @@ __osm_ftree_fabric_populate_ports(
    ftree_sw_t * p_next_sw;
    int res = 0;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_fabric_populate_ports);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_populate_ports);
 
    p_next_hca = (ftree_hca_t *)cl_qmap_head(&p_ftree->hca_tbl);
    while( p_next_hca != (ftree_hca_t *)cl_qmap_end( &p_ftree->hca_tbl ) )
@@ -2748,7 +2784,7 @@ __osm_ftree_fabric_populate_ports(
       }
    }
  Exit:
-   OSM_LOG_EXIT(&(osm.log));
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
    return res;
 } /* __osm_ftree_fabric_populate_ports() */
 
@@ -2756,58 +2792,61 @@ __osm_ftree_fabric_populate_ports(
  ***************************************************/
 
 static int 
-__osm_ftree_do_routing(void *context)
+__osm_ftree_construct_fabric(
+   IN  void * context)
 {
    ftree_fabric_t * p_ftree = context;
    int status = 0;
 
-   OSM_LOG_ENTER(&(osm.log), __osm_ftree_do_routing);
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_construct_fabric);
 
-   if ( cl_qmap_count(&osm.subn.sw_guid_tbl) < 2 )
+   if ( cl_qmap_count(&p_ftree->p_osm->subn.sw_guid_tbl) < 2 )
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric has %u switches - topology is not fat-tree.\n"
               "Falling back to default routing.\n",
-              cl_qmap_count(&osm.subn.sw_guid_tbl));
+              cl_qmap_count(&p_ftree->p_osm->subn.sw_guid_tbl));
       status = -1;
       goto Exit;
    }
 
-   if ( (cl_qmap_count(&osm.subn.node_guid_tbl) - 
-         cl_qmap_count(&osm.subn.sw_guid_tbl)) < 2)
+   if ( (cl_qmap_count(&p_ftree->p_osm->subn.node_guid_tbl) - 
+         cl_qmap_count(&p_ftree->p_osm->subn.sw_guid_tbl)) < 2)
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric has %u nodes (%u switches) - topology is not fat-tree.\n"
               "Falling back to default routing.\n",
-              cl_qmap_count(&osm.subn.node_guid_tbl),
-              cl_qmap_count(&osm.subn.sw_guid_tbl));
+              cl_qmap_count(&p_ftree->p_osm->subn.node_guid_tbl),
+              cl_qmap_count(&p_ftree->p_osm->subn.sw_guid_tbl));
       status = -1;
       goto Exit;
    }
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: \n"
-           "                       |------------------------------|\n"
-           "                       |-  Starting FatTree Routing  -|\n"
-           "                       |------------------------------|\n\n");
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_construct_fabric: \n"
+           "                       |----------------------------------------|\n"
+           "                       |- Starting FatTree fabric construction -|\n"
+           "                       |----------------------------------------|\n\n");
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: "
            "Populating FatTree switch table\n");
    /* ToDo: now that the pointer from node to switch exists,  
       no need to fill the switch table in a separate loop */
    if (__osm_ftree_fabric_populate_switches(p_ftree) != 0)
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric topology is not fat-tree - "
               "falling back to default routing\n");
       status = -1;
       goto Exit;
    }
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: "
            "Populating FatTree HCA table\n");
    if (__osm_ftree_fabric_populate_hcas(p_ftree) != 0)
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric topology is not fat-tree - "
               "falling back to default routing\n");
       status = -1;
@@ -2816,7 +2855,7 @@ __osm_ftree_do_routing(void *context)
 
    if (cl_qmap_count(&p_ftree->hca_tbl) < 2)
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric has %u HCAa - topology is not fat-tree.\n"
               "Falling back to default routing.\n",
               cl_qmap_count(&p_ftree->hca_tbl));
@@ -2825,12 +2864,13 @@ __osm_ftree_do_routing(void *context)
    }
 
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
-           "Ranking FatTree\n");
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: Ranking FatTree\n");
+
    if (__osm_ftree_fabric_perform_ranking(p_ftree) != 0)
    {
       if (__osm_ftree_fabric_get_rank(p_ftree) > FAT_TREE_MAX_RANK)
-         osm_log(&osm.log, OSM_LOG_SYS,
+         osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
                  "Fabric rank is %u (>%u) - "
                  "fat-tree routing falls back to default routing\n",
                  __osm_ftree_fabric_get_rank(p_ftree), FAT_TREE_MAX_RANK);
@@ -2841,11 +2881,12 @@ __osm_ftree_do_routing(void *context)
    /* For each hca and switch, construct array of ports.
       This is done after the whole FatTree data structure is ready, because
       we want the ports to have pointers to ftree_{sw,hca}_t objects.*/
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: "
            "Populating HCA & switch ports\n");
    if (__osm_ftree_fabric_populate_ports(p_ftree) != 0)
    {
-      osm_log(&osm.log, OSM_LOG_SYS,
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
               "Fabric topology is not a fat-tree - "
               "routing falls back to default routing\n");
       status = -1;
@@ -2863,7 +2904,7 @@ __osm_ftree_do_routing(void *context)
    __osm_ftree_fabric_dump_general_info(p_ftree);
 
    /* dump full tree topology */
-   if (osm_log_is_active(&osm.log, OSM_LOG_DEBUG))
+   if (osm_log_is_active(&p_ftree->p_osm->log, OSM_LOG_DEBUG))
        __osm_ftree_fabric_dump(p_ftree);
 
    if (! __osm_ftree_fabric_validate_topology(p_ftree))
@@ -2872,46 +2913,118 @@ __osm_ftree_do_routing(void *context)
       goto Exit;
    }
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: "
+           "Max LID in switch LFTs (in host order): 0x%x\n",
+           p_ftree->lft_max_lid_ho);
+
+ Exit:
+   if (status != 0)
+   {
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+              "__osm_ftree_construct_fabric: "
+             "Clearing FatTree Fabric data structures\n");
+     __osm_ftree_fabric_clear(p_ftree);
+   }
+
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: \n"
+           "                       |--------------------------------------------------|\n"
+           "                       |- Done constructing FatTree fabric (status = %d) -|\n"
+           "                       |--------------------------------------------------|\n\n",
+           status);
+
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
+   return status;
+}
+
+/***************************************************
+ ***************************************************/
+
+static int 
+__osm_ftree_do_routing(
+   IN  void * context)
+{
+   ftree_fabric_t * p_ftree = context;
+
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_do_routing);
+
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+           "Starting FatTree routing\n");
+
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
            "Filling switch forwarding tables for routes to HCAs\n");
    __osm_ftree_fabric_route_to_hcas(p_ftree);
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
            "Filling switch forwarding tables for switch-to-switch pathes\n");
    __osm_ftree_fabric_route_to_switches(p_ftree);
 
    /* for each switch, set its fwd table */
-   cl_qmap_apply_func(&p_ftree->sw_tbl, __osm_ftree_set_sw_fwd_table, NULL);
+   cl_qmap_apply_func(&p_ftree->sw_tbl, __osm_ftree_set_sw_fwd_table, (void *)p_ftree);
 
    /* write out hca ordering file */
    __osm_ftree_fabric_dump_hca_ordering(p_ftree);
 
- Exit:
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
-           "Clearing FatTree Fabric data structures\n");
-   __osm_ftree_fabric_clear(p_ftree);
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: "
+           "FatTree routing is done\n");
 
-   osm_log(&osm.log, OSM_LOG_VERBOSE,"__osm_ftree_do_routing: \n"
-           "                       |---------------------------------------|\n"
-           "                       |-  Done FatTree Routing (status = %d)  -|\n"
-           "                       |---------------------------------------|\n\n", status);
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
+   return 0;
+}
 
-   OSM_LOG_EXIT(&(osm.log));
-   return status;
+/***************************************************
+ ***************************************************/
+
+static int 
+__osm_ftree_routing(
+   IN  void * context)
+{
+   int status = __osm_ftree_construct_fabric(context);
+   if (status != 0)
+      return status;
+
+   __osm_ftree_do_routing(context);
+   return 0;
 }
 
 /***************************************************
  ***************************************************/
 
+void
+ucast_mgr_dump_to_file(
+   IN  osm_ucast_mgr_t *p_mgr,
+   IN  const char *file_name,
+   IN  void (*func)(cl_map_item_t *, void *));
+
+void
+ucast_mgr_dump_lfts(
+   IN  cl_map_item_t *p_map_item,
+   void *cxt);
+
 static void 
-__osm_ftree_delete(void * context)
+__osm_ftree_dump_tables(
+   IN  void * context)
 {
-   ftree_fabric_t * p_ftree = (ftree_fabric_t *)context;
+   ftree_fabric_t * p_ftree = context;
    if (!p_ftree)
       return;
 
-   __osm_ftree_fabric_destroy(p_ftree);
+   ucast_mgr_dump_to_file(&p_ftree->p_osm->sm.ucast_mgr,
+                          "opensm-lfts.dump",
+                          ucast_mgr_dump_lfts);
+}
 
+/***************************************************
+ ***************************************************/
+
+static void 
+__osm_ftree_delete(
+   IN  void * context)
+{
+   if (!context)
+      return;
+   __osm_ftree_fabric_destroy((ftree_fabric_t *)context);
 }
 
 /***************************************************
@@ -2923,11 +3036,21 @@ int osm_ucast_ftree_setup(osm_opensm_t *
    if (!p_ftree)
       return -1;
 
+   p_ftree->p_osm = p_osm;
+
    p_osm->routing_engine.context = (void *)p_ftree;
-   p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
+   p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_routing;
+   /* ToDo: Resolve multicast routing. 
+    *       Until then lid matrices are built, despite the
+    *       fact that FatTree routing doesn't need them.
+    *       When the multicast routing will be resolved,
+    *       __osm_ftree_routing() function should be removed,
+    *       and here's how the FatTree routing will be set:
+    *  p_osm->routing_engine.build_lid_matrices = __osm_ftree_construct_fabric;
+    *  p_osm->routing_engine.ucast_build_fwd_tables = __osm_ftree_do_routing;
+    */
+   p_osm->routing_engine.ucast_dump_tables = __osm_ftree_dump_tables;
    p_osm->routing_engine.delete = __osm_ftree_delete;
-   /* ToDo: fat-tree routing doesn't use min_hop tables, so we
-      shouldn't fill them (p_osm->routing_engine.build_lid_matrices) */
    return 0;
 }
 
-- 
1.4.4.1.GIT





More information about the general mailing list