[ofa-general] PATCH [2/3] osm: adding root and compute node guid files options for fat-tree
Yevgeny Kliteynik
kliteyn at dev.mellanox.co.il
Thu Jun 14 06:54:35 PDT 2007
Hi Hal,
Yevgeny Kliteynik wrote:
> Hi Hal.
>
> Fat-tree routing reads root guid file and compute node guid file,
> and creates map of roots and compute nodes (CNs) to be used later.
>
> --Yevgeny
>
> Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
> ---
> opensm/opensm/osm_ucast_ftree.c | 232
> +++++++++++++++++++++++++++++++++++++++
> 1 files changed, 232 insertions(+), 0 deletions(-)
>
> diff --git a/opensm/opensm/osm_ucast_ftree.c
> b/opensm/opensm/osm_ucast_ftree.c
> index 1730ef2..b1ee0ca 100644
> --- a/opensm/opensm/osm_ucast_ftree.c
> +++ b/opensm/opensm/osm_ucast_ftree.c
> @@ -119,6 +119,17 @@ typedef struct {
>
> /***************************************************
> **
> + ** ftree_guid_tbl_element_t definition
> + **
> + ***************************************************/
> +
> +typedef struct {
> + cl_map_item_t map_item;
> + uint64_t guid;
> +} ftree_guid_tbl_element_t;
> +
> +/***************************************************
> + **
> ** ftree_fwd_tbl_t definition
> **
> ***************************************************/
> @@ -182,6 +193,7 @@ typedef struct ftree_sw_t_
> ftree_port_group_t ** up_port_groups;
> uint8_t up_port_groups_num;
> ftree_fwd_tbl_t lft_buf;
> + boolean_t is_root;
> } ftree_sw_t;
>
> /***************************************************
> @@ -195,6 +207,7 @@ typedef struct ftree_hca_t_ {
> osm_node_t * p_osm_node;
> ftree_port_group_t ** up_port_groups;
> uint16_t up_port_groups_num;
> + boolean_t is_cn;
> } ftree_hca_t;
>
> /***************************************************
> @@ -209,6 +222,8 @@ typedef struct ftree_fabric_t_
> cl_qmap_t hca_tbl;
> cl_qmap_t sw_tbl;
> cl_qmap_t sw_by_tuple_tbl;
> + cl_qmap_t cn_guids_tbl;
> + cl_qmap_t root_guids_tbl;
> uint8_t tree_rank;
> ftree_sw_t ** leaf_switches;
> uint32_t leaf_switches_num;
> @@ -393,6 +408,36 @@ __osm_ftree_sw_tbl_element_destroy(
>
> /***************************************************
> **
> + ** ftree_guid_tbl_element_t functions
> + **
> + ***************************************************/
> +
> +static ftree_guid_tbl_element_t *
> +__osm_ftree_guid_tbl_element_create(
> + IN uint64_t guid)
> +{
> + ftree_guid_tbl_element_t * p_element = +
> (ftree_guid_tbl_element_t *) malloc(sizeof(ftree_guid_tbl_element_t));
> + if (!p_element)
> + return NULL;
> +
> + memset(p_element, 0,sizeof(ftree_guid_tbl_element_t));
> + p_element->guid = guid;
> + return p_element;
> +}
> +
> +/***************************************************/
> +
> +static void
> +__osm_ftree_guid_tbl_element_destroy(
> + IN ftree_guid_tbl_element_t * p_element)
> +{
> + if (p_element)
> + free(p_element);
> +}
> +
> +/***************************************************
> + **
> ** ftree_port_t functions
> **
> ***************************************************/
> @@ -607,6 +652,9 @@ __osm_ftree_sw_create(
> p_sw->lft_buf = (ftree_fwd_tbl_t)cl_pool_get(&p_ftree->sw_fwd_tbl_pool);
> memset(p_sw->lft_buf, OSM_NO_PATH, FTREE_FWD_TBL_LEN);
>
> + /* by default the switch is not root */
> + p_sw->is_root = FALSE;
> +
> return p_sw;
> } /* __osm_ftree_sw_create() */
>
> @@ -810,6 +858,10 @@ __osm_ftree_hca_create(
> if (!p_hca->up_port_groups)
> return NULL;
> p_hca->up_port_groups_num = 0;
> +
> + /* by default every CA is treated as compute node */
> + p_hca->is_cn = TRUE;
> +
> return p_hca;
> }
>
> @@ -934,6 +986,9 @@ __osm_ftree_fabric_create()
> cl_qmap_init(&p_ftree->sw_tbl);
> cl_qmap_init(&p_ftree->sw_by_tuple_tbl);
>
> + cl_qmap_init(&p_ftree->cn_guids_tbl);
> + cl_qmap_init(&p_ftree->root_guids_tbl);
> +
> status = cl_pool_init( &p_ftree->sw_fwd_tbl_pool,
> 8, /* min pool size */
> 0, /* max pool size -
> unlimited */
> @@ -960,6 +1015,8 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
> ftree_sw_t * p_next_sw;
> ftree_sw_tbl_element_t * p_element;
> ftree_sw_tbl_element_t * p_next_element;
> + ftree_guid_tbl_element_t * p_guid_element;
> + ftree_guid_tbl_element_t * p_next_guid_element;
>
> if (!p_ftree)
> return;
> @@ -1000,6 +1057,28 @@ __osm_ftree_fabric_clear(ftree_fabric_t * p_ftree)
> }
> cl_qmap_remove_all(&p_ftree->sw_by_tuple_tbl);
>
> + /* remove all the elements of root_guids_tbl */
> +
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_head(&p_ftree->root_guids_tbl);
> + while( p_next_guid_element != (ftree_guid_tbl_element_t
> *)cl_qmap_end(&p_ftree->root_guids_tbl) )
> + {
> + p_guid_element = p_next_guid_element;
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_next(&p_guid_element->map_item );
> + __osm_ftree_guid_tbl_element_destroy(p_guid_element);
> + }
> + cl_qmap_remove_all(&p_ftree->root_guids_tbl);
> +
> + /* remove all the elements of cn_guids_tbl */
> +
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_head(&p_ftree->cn_guids_tbl);
> + while( p_next_guid_element != (ftree_guid_tbl_element_t
> *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
> + {
> + p_guid_element = p_next_guid_element;
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_next(&p_guid_element->map_item );
> + __osm_ftree_guid_tbl_element_destroy(p_guid_element);
> + }
> + cl_qmap_remove_all(&p_ftree->cn_guids_tbl);
> +
> /* free the leaf switches array */
> if ((p_ftree->leaf_switches_num > 0) && (p_ftree->leaf_switches))
> free(p_ftree->leaf_switches);
> @@ -1048,6 +1127,16 @@ __osm_ftree_fabric_add_hca(ftree_fabric_t *
> p_ftree, osm_node_t * p_osm_node)
>
> CL_ASSERT(osm_node_get_type(p_osm_node) == IB_NODE_TYPE_CA);
>
> + /* if a user has supplied CN guids list, and this CA's guid +
> is not there, then the CA should be marked as non-CN */
> + if ( (!cl_is_qmap_empty(&p_ftree->cn_guids_tbl)) && +
> (cl_qmap_get(&p_ftree->cn_guids_tbl,
> +
> cl_ntoh64(osm_node_get_node_guid(p_hca->p_osm_node))) ==
> + cl_qmap_end(&p_ftree->cn_guids_tbl)) )
> + {
> + p_hca->is_cn = FALSE;
> + }
> +
> cl_qmap_insert(&p_ftree->hca_tbl,
> p_osm_node->node_info.node_guid,
> &p_hca->map_item);
> @@ -1062,6 +1151,16 @@ __osm_ftree_fabric_add_sw(ftree_fabric_t *
> p_ftree, osm_switch_t * p_osm_sw)
>
> CL_ASSERT(osm_node_get_type(p_osm_sw->p_node) == IB_NODE_TYPE_SWITCH);
>
> + /* if a user has supplied root guids list, and this switch's guid
> + *is* there, then the switch should be marked as root */
> + if ( (!cl_is_qmap_empty(&p_ftree->root_guids_tbl)) && +
> (cl_qmap_get(&p_ftree->root_guids_tbl,
> +
> cl_ntoh64(osm_node_get_node_guid(p_sw->p_osm_sw->p_node))) !=
> + cl_qmap_end(&p_ftree->root_guids_tbl)) )
> + {
> + p_sw->is_root = TRUE;
> + }
> +
> cl_qmap_insert(&p_ftree->sw_tbl,
> p_osm_sw->p_node->node_info.node_guid,
> &p_sw->map_item);
> @@ -2907,6 +3006,127 @@ __osm_ftree_fabric_populate_ports(
> /***************************************************
> ***************************************************/
>
> +static int
> +__osm_ftree_convert_list2qmap(
> + cl_list_t * p_guid_list,
> + cl_qmap_t * p_map )
> +{
> + uint64_t * p_guid;
> +
> + if ( !p_map )
> + return -1;
> +
> + if ( !p_guid_list || !cl_list_count(p_guid_list) )
> + return 0;
> +
> + while ( (p_guid = (uint64_t*)cl_list_remove_head(p_guid_list)) )
> + {
> + cl_qmap_insert( p_map, + *p_guid,
> +
> &(__osm_ftree_guid_tbl_element_create(*p_guid)->map_item) );
> + free(p_guid);
> + }
> +
> + CL_ASSERT(cl_is_list_empty(p_guid_list));
> +
> + return 0;
> +} /* __osm_ftree_convert_list2qmap() */
> +
> +/***************************************************
> + ***************************************************/
> +
> +static int
> +__osm_ftree_fabric_read_guid_files(
> + IN ftree_fabric_t * p_ftree)
> +{
> + cl_list_t guid_list;
> + ftree_guid_tbl_element_t * p_guid_element;
> + ftree_guid_tbl_element_t * p_next_guid_element;
> + int status = 0;
> +
> + OSM_LOG_ENTER(&p_ftree->p_osm->log, __osm_ftree_fabric_read_guid_files);
> +
> + cl_list_construct( &guid_list );
> + cl_list_init( &guid_list, 10 );
> +
> + p_ftree->p_osm->subn.opt.ftree_root_guid_file = "/tmp/ftree.root.guids";
> + p_ftree->p_osm->subn.opt.ftree_cn_guid_file = "/tmp/ftree.cn.guids";
These two lines are, of course, a mistake :)
-- Yevgeny
> +
> + if (p_ftree->p_osm->subn.opt.ftree_root_guid_file)
> + {
> + osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
> + "__osm_ftree_read_guid_files: "
> + "Fetching root nodes from file %s\n",
> + p_ftree->p_osm->subn.opt.ftree_root_guid_file );
> +
> + if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
> +
> p_ftree->p_osm->subn.opt.ftree_root_guid_file,
> + &guid_list ) ||
> + __osm_ftree_convert_list2qmap( &guid_list,
> + &p_ftree->root_guids_tbl ) )
> + {
> + status = -1;
> + goto Exit;
> + }
> +
> + if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
> + {
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_head(&p_ftree->root_guids_tbl);
> + while( p_next_guid_element != (ftree_guid_tbl_element_t
> *)cl_qmap_end(&p_ftree->root_guids_tbl) )
> + {
> + p_guid_element = p_next_guid_element;
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_next(&p_guid_element->map_item );
> + osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
> + "__osm_ftree_fabric_read_guid_files: "
> + "root guid 0x%016" PRIx64 "\n",
> + p_guid_element->guid );
> + }
> + }
> + }
> + CL_ASSERT(cl_is_list_empty(&guid_list));
> +
> + if (p_ftree->p_osm->subn.opt.ftree_cn_guid_file)
> + {
> + osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
> + "__osm_ftree_read_guid_files: "
> + "Fetching compute nodes from file %s\n",
> + p_ftree->p_osm->subn.opt.ftree_cn_guid_file );
> +
> + if ( osm_ucast_mgr_read_guid_file( &p_ftree->p_osm->sm.ucast_mgr,
> +
> p_ftree->p_osm->subn.opt.ftree_cn_guid_file,
> + &guid_list ) ||
> + __osm_ftree_convert_list2qmap( &guid_list,
> + &p_ftree->cn_guids_tbl ) )
> + {
> + status = -1;
> + goto Exit;
> + }
> +
> + if (osm_log_is_active(&p_ftree->p_osm->log,OSM_LOG_DEBUG))
> + {
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_head(&p_ftree->cn_guids_tbl);
> + while( p_next_guid_element != (ftree_guid_tbl_element_t
> *)cl_qmap_end(&p_ftree->cn_guids_tbl) )
> + {
> + p_guid_element = p_next_guid_element;
> + p_next_guid_element = (ftree_guid_tbl_element_t
> *)cl_qmap_next(&p_guid_element->map_item );
> + osm_log( &p_ftree->p_osm->log, OSM_LOG_DEBUG,
> + "__osm_ftree_fabric_read_guid_files: "
> + "compute node guid 0x%016" PRIx64 "\n",
> + p_guid_element->guid );
> + }
> + }
> + }
> + CL_ASSERT(cl_is_list_empty(&guid_list));
> +
> + Exit:
> + OSM_LOG_EXIT(&p_ftree->p_osm->log);
> + cl_list_destroy(&guid_list);
> + return status;
> +} /*__osm_ftree_fabric_read_guid_files() */
> +
> +/***************************************************
> + ***************************************************/
> +
> static int __osm_ftree_construct_fabric(
> IN void * context)
> @@ -2947,6 +3167,18 @@ __osm_ftree_construct_fabric(
> goto Exit;
> }
>
> + osm_log(&p_ftree->p_osm->log, OSM_LOG_VERBOSE,
> + "__osm_ftree_construct_fabric: "
> + "Reading guid files provided by user\n");
> + if (__osm_ftree_fabric_read_guid_files(p_ftree) != 0)
> + {
> + osm_log(&p_ftree->p_osm->log, OSM_LOG_SYS,
> + "Failed reading guid files - "
> + "falling back to default routing\n");
> + status = -1;
> + goto Exit;
> + }
> +
> osm_log(&p_ftree->p_osm->log,
> OSM_LOG_VERBOSE,"__osm_ftree_construct_fabric: \n"
> "
> |----------------------------------------|\n"
> " |- Starting FatTree fabric
> construction -|\n"
More information about the general
mailing list