[ofa-general] PATCH [2/3] osm: adding root and compute node guid files options for fat-tree
Yevgeny Kliteynik
kliteyn at dev.mellanox.co.il
Thu Jun 14 01:20:13 PDT 2007
Hi Hal.
Fat-tree routing reads root guid file and compute node guid file,
and creates map of roots and compute nodes (CNs) to be used later.
--Yevgeny
Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
opensm/opensm/osm_ucast_ftree.c | 232 +++++++++++++++++++++++++++++++++++++++
1 files changed, 232 insertions(+), 0 deletions(-)
diff --git a/opensm/opensm/osm_ucast_ftree.c b/opensm/opensm/osm_ucast_ftree.c
index 1730ef2..b1ee0ca 100644
--- a/opensm/opensm/osm_ucast_ftree.c
+++ b/opensm/opensm/osm_ucast_ftree.c
@@ -119,6 +119,17 @@ typedef struct {
/***************************************************
**
+ ** ftree_guid_tbl_element_t definition
+ **
+ ***************************************************/
+
+typedef struct {
+ cl_map_item_t map_item;
+ uint64_t guid;
+} ftree_guid_tbl_element_t;
+
+/***************************************************
+ **
** ftree_fwd_tbl_t definition
**
***************************************************/
@@ -182,6 +193,7 @@ typedef struct ftree_sw_t_
ftree_port_group_t ** up_port_groups;
uint8_t up_port_groups_num;
ftree_fwd_tbl_t lft_buf;
+ boolean_t is_root;
} ftree_sw_t;
/***************************************************
@@ -195,6 +207,7 @@ typedef struct ftree_hca_t_ {
osm_node_t * p_osm_node;
ftree_port_group_t ** up_port_groups;
uint16_t up_port_groups_num;
+ boolean_t is_cn;
} ftree_hca_t;
/***************************************************
@@ -209,6 +222,8 @@ typedef struct ftree_fabric_t_
cl_qmap_t hca_tbl;
cl_qmap_t sw_tbl;
cl_qmap_t sw_by_tuple_tbl;
+ cl_qmap_t cn_guids_tbl;
+ cl_qmap_t root_guids_tbl;
uint8_t tree_rank;
ftree_sw_t ** leaf_switches;
uint32_t leaf_switches_num;
@@ -393,6 +408,36 @@ __osm_ftree_sw_tbl_element_destroy(
/***************************************************
**
+ ** ftree_guid_tbl_element_t functions
+ **
+ ***************************************************/
+
+static ftree_guid_tbl_element_t *
+__osm_ftree_guid_tbl_element_create(
+ IN uint64_t guid)
+{
+ ftree_guid_tbl_element_t * p_element =
+ (ftree_guid_tbl_element_t *) malloc(sizeof(ftree_guid_tbl_element_t));
+ if (!p_element)
+ return NULL;
+
+ memset(p_element, 0,sizeof(ftree_guid_tbl_element_t));
+ p_element->guid = guid;
+ return p_element;
+}
+
+/***************************************************/
+
+static void
+__osm_ftree_guid_tbl_element_destroy(
+ IN ftree_guid_tbl_element_t * p_element)
+{
+ if (p_element)
+ free(p_element);
+}
+
+/***************************************************
+ **
** ftree_port_t functions
**
***************************************************/
@@ -607,6 +652,9 @@ __osm_ftree_sw_create(
p_sw->lft_buf = (ftree_fwd_tbl_t)cl_pool_get(&p_ftree->sw_fwd_tbl_pool);
memset(p_sw->lft_buf, OSM_NO_PATH, FTREE_FWD_TBL_LEN);
+ /* by default the switch is not root */
+ p_sw->is_root = FALSE;
+
return p_sw;
} /* __osm_ftree_sw_create() */
@@ -810,6 +858,10 @@ __osm_ftree_hca_create(
if (!p_hca->up_port_groups)
return NULL;
p_hca->up_port_groups_num = 0;
+
+ /* by default every CA is treated as compute node */
+ p_hca->is_cn = TRUE;
+
return p_hca;
}
@@ -934,6 +986,9 @@ __osm_ftree_fabric_create()
cl_qmap_init(&p_ftree->sw_tbl);
cl_qmap_init(&p_ftree->sw_by_tuple_tbl);
+ cl_qmap_init(&p_ftree->cn_guids_tbl);
+ cl_qmap_init(&p_ftree->root_guids_tbl);
+
status = cl_pool_init( &p_ftree->sw_fwd_tbl_pool,
8, /* min pool size */
0, /* max pool size - unlimited */
@@ -960,6 +1015,8 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
ftree_sw_t * p_next_sw;
ftree_sw_tbl_element_t * p_element;
ftree_sw_tbl_element_t * p_next_element;
+ ftree_guid_tbl_element_t * p_guid_element;
+ ftree_guid_tbl_element_t * p_next_guid_element;
if (!p_ftree)
return;
@@ -1000,6 +1057,28 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
}
cl_qmap_remove_all(&p_ftree->sw_by_tuple_tbl);
+ /* remove all the elements of root_guids_tbl */
+
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->root_guids_tbl);
+ while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->root_guids_tbl) )
+ {
+ p_guid_element = p_next_guid_element;
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+ __osm_ftree_guid_tbl_element_destroy(p_guid_element);
+ }
+ cl_qmap_remove_all(&p_ftree->root_guids_tbl);
+
+ /* remove all the elements of cn_guids_tbl */
+
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->cn_guids_tbl);
+ while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
+ {
+ p_guid_element = p_next_guid_element;
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+ __osm_ftree_guid_tbl_element_destroy(p_guid_element);
+ }
+ cl_qmap_remove_all(&p_ftree->cn_guids_tbl);
+
/* free the leaf switches array */
if ((p_ftree->leaf_switches_num > 0) && (p_ftree->leaf_switches))
free(p_ftree->leaf_switches);
@@ -1048,6 +1127,16 @@ __osm_ftree_fabric_add_hca(ftree_fabric_t * p_ftree, osm_node_t * p_osm_node)
CL_ASSERT(osm_node_get_type(p_osm_node) == IB_NODE_TYPE_CA);
+ /* if a user has supplied CN guids list, and this CA's guid
+ is not there, then the CA should be marked as non-CN */
+ if ( (!cl_is_qmap_empty(&p_ftree->cn_guids_tbl)) &&
+ (cl_qmap_get(&p_ftree->cn_guids_tbl,
+ cl_ntoh64(osm_node_get_node_guid(p_hca->p_osm_node))) ==
+ cl_qmap_end(&p_ftree->cn_guids_tbl)) )
+ {
+ p_hca->is_cn = FALSE;
+ }
+
cl_qmap_insert(&p_ftree->hca_tbl,
p_osm_node->node_info.node_guid,
&p_hca->map_item);
@@ -1062,6 +1151,16 @@ __osm_ftree_fabric_add_sw(ftree_fabric_t * p_ftree, osm_switch_t * p_osm_sw)
CL_ASSERT(osm_node_get_type(p_osm_sw->p_node) == IB_NODE_TYPE_SWITCH);
+ /* if a user has supplied root guids list, and this switch's guid
+ *is* there, then the switch should be marked as root */
+ if ( (!cl_is_qmap_empty(&p_ftree->root_guids_tbl)) &&
+ (cl_qmap_get(&p_ftree->root_guids_tbl,
+ cl_ntoh64(osm_node_get_node_guid(p_sw->p_osm_sw->p_node))) !=
+ cl_qmap_end(&p_ftree->root_guids_tbl)) )
+ {
+ p_sw->is_root = TRUE;
+ }
+
cl_qmap_insert(&p_ftree->sw_tbl,
p_osm_sw->p_node->node_info.node_guid,
&p_sw->map_item);
@@ -2907,6 +3006,127 @@ __osm_ftree_fabric_populate_ports(
/***************************************************
***************************************************/
+static int
+__osm_ftree_convert_list2qmap(
+ cl_list_t * p_guid_list,
+ cl_qmap_t * p_map )
+{
+ uint64_t * p_guid;
+
+ if ( !p_map )
+ return -1;
+
+ if ( !p_guid_list || !cl_list_count(p_guid_list) )
+ return 0;
+
+ while ( (p_guid = (uint64_t*)cl_list_remove_head(p_guid_list)) )
+ {
+ cl_qmap_insert( p_map,
+ *p_guid,
+ &(__osm_ftree_guid_tbl_element_create(*p_guid)->map_item) );
+ free(p_guid);
+ }
+
+ CL_ASSERT(cl_is_list_empty(p_guid_list));
+
+ return 0;
+} /* __osm_ftree_convert_list2qmap() */
+
+/***************************************************
+ ***************************************************/
+
+static int
+__osm_ftree_fabric_read_guid_files(
+ IN ftree_fabric_t * p_ftree)
+{
+ cl_list_t guid_list;
+ ftree_guid_tbl_element_t * p_guid_element;
+ ftree_guid_tbl_element_t * p_next_guid_element;
+ int status = 0;
+
+ OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_read_guid_files);
+
+ cl_list_construct( &guid_list );
+ cl_list_init( &guid_list, 10 );
+
+ p_ftree->p_osm->subn.opt.ftree_root_guid_file = "/tmp/ftree.root.guids";
+ p_ftree->p_osm->subn.opt.ftree_cn_guid_file = "/tmp/ftree.cn.guids";
+
+ if (p_ftree->p_osm->subn.opt.ftree_root_guid_file)
+ {
+ osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+ "__osm_ftree_read_guid_files: "
+ "Fetching root nodes from file %s\n",
+ p_ftree->p_osm->subn.opt.ftree_root_guid_file );
+
+ if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
+ p_ftree->p_osm->subn.opt.ftree_root_guid_file,
+ &guid_list ) ||
+ __osm_ftree_convert_list2qmap( &guid_list,
+ &p_ftree->root_guids_tbl ) )
+ {
+ status = -1;
+ goto Exit;
+ }
+
+ if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
+ {
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->root_guids_tbl);
+ while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->root_guids_tbl) )
+ {
+ p_guid_element = p_next_guid_element;
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+ osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+ "__osm_ftree_fabric_read_guid_files: "
+ "root guid 0x%016" PRIx64 "\n",
+ p_guid_element->guid );
+ }
+ }
+ }
+ CL_ASSERT(cl_is_list_empty(&guid_list));
+
+ if (p_ftree->p_osm->subn.opt.ftree_cn_guid_file)
+ {
+ osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+ "__osm_ftree_read_guid_files: "
+ "Fetching compute nodes from file %s\n",
+ p_ftree->p_osm->subn.opt.ftree_cn_guid_file );
+
+ if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
+ p_ftree->p_osm->subn.opt.ftree_cn_guid_file,
+ &guid_list ) ||
+ __osm_ftree_convert_list2qmap( &guid_list,
+ &p_ftree->cn_guids_tbl ) )
+ {
+ status = -1;
+ goto Exit;
+ }
+
+ if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
+ {
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_head(&p_ftree->cn_guids_tbl);
+ while( p_next_guid_element != (ftree_guid_tbl_element_t *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
+ {
+ p_guid_element = p_next_guid_element;
+ p_next_guid_element = (ftree_guid_tbl_element_t *)cl_qmap_next(&p_guid_element->map_item );
+ osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
+ "__osm_ftree_fabric_read_guid_files: "
+ "compute node guid 0x%016" PRIx64 "\n",
+ p_guid_element->guid );
+ }
+ }
+ }
+ CL_ASSERT(cl_is_list_empty(&guid_list));
+
+ Exit:
+ OSM_LOG_EXIT(&p_ftree->p_osm->log);
+ cl_list_destroy(&guid_list);
+ return status;
+} /*__osm_ftree_fabric_read_guid_files() */
+
+/***************************************************
+ ***************************************************/
+
static int
__osm_ftree_construct_fabric(
IN void * context)
@@ -2947,6 +3167,18 @@ __osm_ftree_construct_fabric(
goto Exit;
}
+ osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
+ "__osm_ftree_construct_fabric: "
+ "Reading guid files provided by user\n");
+ if (__osm_ftree_fabric_read_guid_files(p_ftree) != 0)
+ {
+ osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
+ "Failed reading guid files - "
+ "falling back to default routing\n");
+ status = -1;
+ goto Exit;
+ }
+
osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,"__osm_ftree_construct_fabric: \n"
" |----------------------------------------|\n"
" |- Starting FatTree fabric construction -|\n"
--
1.5.1.4
More information about the general
mailing list