[ofa-general] PATCH [2/3] osm: adding root and compute node guid files options for fat-tree

Yevgeny Kliteynik kliteyn at dev.mellanox.co.il
Thu Jun 14 01:20:13 PDT 2007


Hi Hal.

Fat-tree routing reads root guid file and compute node guid file,
and creates map of roots and compute nodes (CNs) to be used later.

--Yevgeny

Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
 opensm/opensm/osm_ucast_ftree.c |  232 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 232 insertions(+), 0 deletions(-)

diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c
index 1730ef2..b1ee0ca 100644
--- a/opensm/opensm/osm_ucast_ftree.c
+++ b/opensm/opensm/osm_ucast_ftree.c
@@ -119,6 +119,17 @@ typedef struct {
 
 /***************************************************
  **
+ **  ftree_guid_tbl_element_t definition
+ **
+ ***************************************************/
+
+typedef struct {
+   cl_map_item_t map_item;
+   uint64_t guid;
+} ftree_guid_tbl_element_t;
+
+/***************************************************
+ **
  **  ftree_fwd_tbl_t definition
  **
  ***************************************************/
@@ -182,6 +193,7 @@ typedef struct ftree_sw_t_
    ftree_port_group_t  ** up_port_groups;
    uint8_t                up_port_groups_num;
    ftree_fwd_tbl_t        lft_buf;
+   boolean_t              is_root;
 } ftree_sw_t;
 
 /***************************************************
@@ -195,6 +207,7 @@ typedef struct ftree_hca_t_ {
    osm_node_t           * p_osm_node;
    ftree_port_group_t  ** up_port_groups;
    uint16_t               up_port_groups_num;
+   boolean_t              is_cn;
 } ftree_hca_t;
 
 /***************************************************
@@ -209,6 +222,8 @@ typedef struct ftree_fabric_t_
    cl_qmap_t       hca_tbl;
    cl_qmap_t       sw_tbl;
    cl_qmap_t       sw_by_tuple_tbl;
+   cl_qmap_t       cn_guids_tbl;
+   cl_qmap_t       root_guids_tbl;
    uint8_t         tree_rank;
    ftree_sw_t   ** leaf_switches;
    uint32_t        leaf_switches_num;
@@ -393,6 +408,36 @@ __osm_ftree_sw_tbl_element_destroy(
 
 /***************************************************
  **
+ ** ftree_guid_tbl_element_t functions
+ **
+ ***************************************************/
+
+static ftree_guid_tbl_element_t *
+__osm_ftree_guid_tbl_element_create(
+   IN  uint64_t guid)
+{
+   ftree_guid_tbl_element_t * p_element = 
+      (ftree_guid_tbl_element_t *) malloc(sizeof(ftree_guid_tbl_element_t));
+   if (!p_element)
+       return NULL;
+
+   memset(p_element, 0,sizeof(ftree_guid_tbl_element_t));
+   p_element->guid = guid;
+   return p_element;
+}
+
+/***************************************************/
+
+static void
+__osm_ftree_guid_tbl_element_destroy(
+   IN  ftree_guid_tbl_element_t * p_element)
+{
+   if (p_element)
+      free(p_element);
+}
+
+/***************************************************
+ **
  ** ftree_port_t functions
  **
  ***************************************************/
@@ -607,6 +652,9 @@ __osm_ftree_sw_create(
    p_sw->lft_buf = (ftree_fwd_tbl_t)cl_pool_get(&p_ftree->sw_fwd_tbl_pool);
    memset(p_sw->lft_buf, OSM_NO_PATH, FTREE_FWD_TBL_LEN);
 
+   /* by default the switch is not root */
+   p_sw->is_root = FALSE;
+
    return p_sw;
 } /* __osm_ftree_sw_create() */
 
@@ -810,6 +858,10 @@ __osm_ftree_hca_create(
    if (!p_hca->up_port_groups)
       return NULL;
    p_hca->up_port_groups_num = 0;
+
+   /* by default every CA is treated as compute node */
+   p_hca->is_cn = TRUE;
+
    return p_hca;
 }
 
@@ -934,6 +986,9 @@ __osm_ftree_fabric_create()
    cl_qmap_init(&p_ftree->sw_tbl);
    cl_qmap_init(&p_ftree->sw_by_tuple_tbl);
 
+   cl_qmap_init(&p_ftree->cn_guids_tbl);
+   cl_qmap_init(&p_ftree->root_guids_tbl);
+
    status = cl_pool_init( &p_ftree->sw_fwd_tbl_pool,
                           8,                 /* min pool size */
                           0,                 /* max pool size - unlimited */
@@ -960,6 +1015,8 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
    ftree_sw_t * p_next_sw;
    ftree_sw_tbl_element_t * p_element;
    ftree_sw_tbl_element_t * p_next_element;
+   ftree_guid_tbl_element_t * p_guid_element;
+   ftree_guid_tbl_element_t * p_next_guid_element;
 
    if (!p_ftree)
       return;
@@ -1000,6 +1057,28 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
    }
    cl_qmap_remove_all(&p_ftree->sw_by_tuple_tbl);
 
+   /* remove all the elements of root_guids_tbl */
+
+   p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->root_guids_tbl);
+   while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->root_guids_tbl) )
+   {
+      p_guid_element = p_next_guid_element;
+      p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+      __osm_ftree_guid_tbl_element_destroy(p_guid_element);
+   }
+   cl_qmap_remove_all(&p_ftree->root_guids_tbl);
+
+   /* remove all the elements of cn_guids_tbl */
+
+   p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->cn_guids_tbl);
+   while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
+   {
+      p_guid_element = p_next_guid_element;
+      p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+      __osm_ftree_guid_tbl_element_destroy(p_guid_element);
+   }
+   cl_qmap_remove_all(&p_ftree->cn_guids_tbl);
+
    /* free the leaf switches array */
    if ((p_ftree->leaf_switches_num > 0) && (p_ftree->leaf_switches))
       free(p_ftree->leaf_switches);
@@ -1048,6 +1127,16 @@ __osm_ftree_fabric_add_hca(ftree_fabric_t * p_ftree, osm_node_t * p_osm_node)
 
    CL_ASSERT(osm_node_get_type(p_osm_node) == IB_NODE_TYPE_CA);
 
+   /* if a user has supplied CN guids list, and this CA's guid 
+      is not there, then the CA should be marked as non-CN */
+   if ( (!cl_is_qmap_empty(&p_ftree->cn_guids_tbl)) && 
+        (cl_qmap_get(&p_ftree->cn_guids_tbl,
+                    cl_ntoh64(osm_node_get_node_guid(p_hca->p_osm_node))) ==
+                        cl_qmap_end(&p_ftree->cn_guids_tbl)) )
+   {
+      p_hca->is_cn = FALSE;
+   }
+
    cl_qmap_insert(&p_ftree->hca_tbl,
                   p_osm_node->node_info.node_guid,
                   &p_hca->map_item);
@@ -1062,6 +1151,16 @@ __osm_ftree_fabric_add_sw(ftree_fabric_t * p_ftree, osm_switch_t * p_osm_sw)
 
    CL_ASSERT(osm_node_get_type(p_osm_sw->p_node) == IB_NODE_TYPE_SWITCH);
 
+   /* if a user has supplied root guids list, and this switch's guid 
+      *is* there, then the switch should be marked as root */
+   if ( (!cl_is_qmap_empty(&p_ftree->root_guids_tbl)) && 
+        (cl_qmap_get(&p_ftree->root_guids_tbl,
+                    cl_ntoh64(osm_node_get_node_guid(p_sw->p_osm_sw->p_node))) !=
+                        cl_qmap_end(&p_ftree->root_guids_tbl)) )
+   {
+      p_sw->is_root = TRUE;
+   }
+
    cl_qmap_insert(&p_ftree->sw_tbl,
                   p_osm_sw->p_node->node_info.node_guid,
                   &p_sw->map_item);
@@ -2907,6 +3006,127 @@ __osm_ftree_fabric_populate_ports(
 /***************************************************
  ***************************************************/
 
+static int
+__osm_ftree_convert_list2qmap(
+   cl_list_t * p_guid_list,
+   cl_qmap_t * p_map )
+{
+   uint64_t * p_guid;
+
+   if ( !p_map )
+      return -1;
+
+   if ( !p_guid_list || !cl_list_count(p_guid_list) )
+      return 0;
+
+   while ( (p_guid = (uint64_t*)cl_list_remove_head(p_guid_list)) )
+   {
+      cl_qmap_insert( p_map, 
+                      *p_guid,
+                      &(__osm_ftree_guid_tbl_element_create(*p_guid)->map_item) );
+      free(p_guid);
+   }
+
+   CL_ASSERT(cl_is_list_empty(p_guid_list));
+
+   return 0;
+} /* __osm_ftree_convert_list2qmap() */
+
+/***************************************************
+ ***************************************************/
+
+static int
+__osm_ftree_fabric_read_guid_files(
+   IN  ftree_fabric_t * p_ftree)
+{
+   cl_list_t guid_list;
+   ftree_guid_tbl_element_t * p_guid_element;
+   ftree_guid_tbl_element_t * p_next_guid_element;
+   int status = 0;
+
+   OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_read_guid_files);
+
+   cl_list_construct( &guid_list );
+   cl_list_init( &guid_list, 10 );
+
+   p_ftree->p_osm->subn.opt.ftree_root_guid_file    = "/tmp/ftree.root.guids";
+   p_ftree->p_osm->subn.opt.ftree_cn_guid_file      = "/tmp/ftree.cn.guids";
+
+   if (p_ftree->p_osm->subn.opt.ftree_root_guid_file)
+   {
+      osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+               "__osm_ftree_read_guid_files: "
+               "Fetching root nodes from file %s\n",
+               p_ftree->p_osm->subn.opt.ftree_root_guid_file );
+
+      if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
+                                         p_ftree->p_osm->subn.opt.ftree_root_guid_file,
+                                         &guid_list ) ||
+           __osm_ftree_convert_list2qmap( &guid_list,
+                                          &p_ftree->root_guids_tbl ) )
+      {
+         status = -1;
+         goto Exit;
+      }
+
+      if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
+      {
+         p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->root_guids_tbl);
+         while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->root_guids_tbl) )
+         {
+            p_guid_element = p_next_guid_element;
+            p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+            osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+                     "__osm_ftree_fabric_read_guid_files:   "
+                     "root guid 0x%016" PRIx64 "\n",
+                     p_guid_element->guid );
+         }
+      }
+   }
+   CL_ASSERT(cl_is_list_empty(&guid_list));
+
+   if (p_ftree->p_osm->subn.opt.ftree_cn_guid_file)
+   {
+      osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+               "__osm_ftree_read_guid_files: "
+               "Fetching compute nodes from file %s\n",
+               p_ftree->p_osm->subn.opt.ftree_cn_guid_file );
+
+      if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
+                                         p_ftree->p_osm->subn.opt.ftree_cn_guid_file,
+                                         &guid_list ) ||
+           __osm_ftree_convert_list2qmap( &guid_list,
+                                          &p_ftree->cn_guids_tbl ) )
+      {
+         status = -1;
+         goto Exit;
+      }
+
+      if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
+      {
+         p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->cn_guids_tbl);
+         while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
+         {
+            p_guid_element = p_next_guid_element;
+            p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+            osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+                     "__osm_ftree_fabric_read_guid_files:   "
+                     "compute node guid 0x%016" PRIx64 "\n",
+                     p_guid_element->guid );
+         }
+      }
+   }
+   CL_ASSERT(cl_is_list_empty(&guid_list));
+
+  Exit:
+   OSM_LOG_EXIT(&p_ftree->p_osm->log);
+   cl_list_destroy(&guid_list);
+   return status;
+} /*__osm_ftree_fabric_read_guid_files() */
+
+/***************************************************
+ ***************************************************/
+
 static int 
 __osm_ftree_construct_fabric(
    IN  void * context)
@@ -2947,6 +3167,18 @@ __osm_ftree_construct_fabric(
       goto Exit;
    }
 
+   osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+           "__osm_ftree_construct_fabric: "
+           "Reading guid files provided by user\n");
+   if (__osm_ftree_fabric_read_guid_files(p_ftree) != 0)
+   {
+      osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
+              "Failed reading guid files - "
+              "falling back to default routing\n");
+      status = -1;
+      goto Exit;
+   }
+
    osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_construct_fabric: \n"
            "                       |----------------------------------------|\n"
            "                       |- Starting FatTree fabric construction -|\n"
-- 
1.5.1.4




More information about the general mailing list