[ofa-general] [PATCH 6/6] opensm/updn: --ids_guid_file - node guids to ids map
Sasha Khapyorsky
sashak at voltaire.com
Wed Mar 26 17:27:17 PDT 2008
With Up/Down if switches have equal rank (distance from root node), up
or down direction is determined by additional switch unique ID
comparison. Currently in OpenSM node GUIDs is used as such IDs.
Doing some experiments with cluster simulation I found that with some
topologies using IDs other than random node GUID numbers can optimize
Up/Down routing and cluster performance dramatically.
It was resulted in this patch where such IDs can be defined in text file
which has simple "<guid> <id>" format and can be passed as command line
or configuration option to OpenSM.
Signed-off-by: Sasha Khapyorsky <sashak at voltaire.com>
---
opensm/include/opensm/osm_subnet.h | 5 +++
opensm/opensm/main.c | 14 +++++++-
opensm/opensm/osm_subnet.c | 10 ++++++
opensm/opensm/osm_ucast_updn.c | 63 +++++++++++++++++++++++++++++-------
4 files changed, 79 insertions(+), 13 deletions(-)
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 1c8b6d6..1b24381 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -260,6 +260,7 @@ typedef struct _osm_subn_opt {
char *ucast_dump_file;
char *root_guid_file;
char *cn_guid_file;
+ char *ids_guid_file;
char *sa_db_file;
boolean_t exit_on_fatal;
boolean_t honor_guid2lid_file;
@@ -456,6 +457,10 @@ typedef struct _osm_subn_opt {
* Name of the file that contains list of compute node guids that
* will be used by fat-tree routing (provided by User)
*
+* ids_guid_file
+* Name of the file that contains list of ids which should be
+* used by Up/Down algorithm instead of node GUIDs
+*
* sa_db_file
* Name of the SA database file.
*
diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
index 5be0271..fb41d50 100644
--- a/opensm/opensm/main.c
+++ b/opensm/opensm/main.c
@@ -206,6 +206,11 @@ static void show_usage(void)
" Set the compute nodes for the Fat-Tree routing algorithm\n"
" to the guids provided in the given file (one to a line)\n"
"\n");
+ printf("-m\n"
+ "--ids_guid_file <path to file>\n"
+ " Name of the map file with set of the IDs which will be used\n"
+ " by Up/Down routing algorithm instead of node GUIDs\n"
+ " (format: <guid> <id> per line)\n");
printf("-o\n"
"--once\n"
" This option causes OpenSM to configure the subnet\n"
@@ -594,7 +599,7 @@ int main(int argc, char *argv[])
char *ignore_guids_file_name = NULL;
uint32_t val;
const char *const short_option =
- "i:f:ed:g:l:L:s:t:a:u:R:zM:U:S:P:Y:NBIQvVhorcyxp:n:q:k:C:";
+ "i:f:ed:g:l:L:s:t:a:u:m:R:zM:U:S:P:Y:NBIQvVhorcyxp:n:q:k:C:";
/*
In the array below, the 2nd parameter specifies the number
@@ -634,6 +639,7 @@ int main(int argc, char *argv[])
{"sadb_file", 1, NULL, 'S'},
{"root_guid_file", 1, NULL, 'a'},
{"cn_guid_file", 1, NULL, 'u'},
+ {"ids_guid_file", 1, NULL, 'm'},
{"cache-options", 0, NULL, 'c'},
{"stay_on_fatal", 0, NULL, 'y'},
{"honor_guid2lid", 0, NULL, 'x'},
@@ -917,6 +923,12 @@ int main(int argc, char *argv[])
opt.cn_guid_file);
break;
+ case 'm':
+ /* Specifies ids guid file */
+ opt.ids_guid_file = optarg;
+ printf(" IDs Guid File: %s\n", opt.ids_guid_file);
+ break;
+
case 'c':
cache_options = TRUE;
printf(" Caching command line options\n");
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index b65cd74..47d735f 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -465,6 +465,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt)
p_opt->ucast_dump_file = NULL;
p_opt->root_guid_file = NULL;
p_opt->cn_guid_file = NULL;
+ p_opt->ids_guid_file = NULL;
p_opt->sa_db_file = NULL;
p_opt->exit_on_fatal = TRUE;
p_opt->enable_quirks = FALSE;
@@ -1324,6 +1325,9 @@ ib_api_status_t osm_subn_parse_conf_file(IN osm_subn_opt_t * const p_opts)
opts_unpack_charp("cn_guid_file",
p_key, p_val, &p_opts->cn_guid_file);
+ opts_unpack_charp("ids_guid_file",
+ p_key, p_val, &p_opts->ids_guid_file);
+
opts_unpack_charp("sa_db_file",
p_key, p_val, &p_opts->sa_db_file);
@@ -1558,6 +1562,12 @@ ib_api_status_t osm_subn_write_conf_file(IN osm_subn_opt_t * const p_opts)
"# The file holding the fat-tree compute node guids\n"
"# One guid in each line\n"
"cn_guid_file %s\n\n", p_opts->cn_guid_file);
+ if (p_opts->ids_guid_file)
+ fprintf(opts_file,
+ "# The file holding the node ids which will be used by"
+ " Up/Down algorithm instead\n# of GUIDs (one guid and"
+ " id in each line)\n"
+ "ids_guid_file %s\n\n", p_opts->ids_guid_file);
if (p_opts->sa_db_file)
fprintf(opts_file,
"# SA database file name\n"
diff --git a/opensm/opensm/osm_ucast_updn.c b/opensm/opensm/osm_ucast_updn.c
index 3623a69..5e778c2 100644
--- a/opensm/opensm/osm_ucast_updn.c
+++ b/opensm/opensm/osm_ucast_updn.c
@@ -74,6 +74,7 @@ typedef struct _updn {
struct updn_node {
cl_list_item_t list;
osm_switch_t *sw;
+ uint64_t id;
updn_switch_dir_t dir;
unsigned rank;
unsigned visited;
@@ -90,7 +91,7 @@ static void __osm_updn_find_root_nodes_by_min_hop(OUT updn_t * p_updn);
remote ports */
static updn_switch_dir_t
__updn_get_dir(IN unsigned cur_rank,
- IN unsigned rem_rank, IN uint64_t cur_guid, IN uint64_t rem_guid)
+ IN unsigned rem_rank, IN uint64_t cur_id, IN uint64_t rem_id)
{
/* HACK: comes to solve root nodes connection, in a classic subnet root nodes do not connect
directly, but in case they are we assign to root node an UP direction to allow UPDN to discover
@@ -104,8 +105,8 @@ __updn_get_dir(IN unsigned cur_rank,
else if (cur_rank > rem_rank)
return UP;
else {
- /* Equal rank, decide by guid number, bigger == UP direction */
- if (cur_guid > rem_guid)
+ /* Equal rank, decide by id number, bigger == UP direction */
+ if (cur_id > rem_id)
return UP;
else
return DOWN;
@@ -145,12 +146,9 @@ __updn_bfs_by_node(IN osm_log_t * p_log,
/* BFS the list till no next element */
while (!cl_is_qlist_empty(&list)) {
- ib_net64_t remote_guid, current_guid;
-
u = (struct updn_node *)cl_qlist_remove_head(&list);
u->visited = 0; /* cleanup */
current_dir = u->dir;
- current_guid = osm_node_get_node_guid(u->sw->p_node);
/* Go over all ports of the switch and find unvisited remote nodes */
for (pn = 1; pn < u->sw->num_ports; pn++) {
osm_node_t *p_remote_node;
@@ -167,13 +165,11 @@ __updn_bfs_by_node(IN osm_log_t * p_log,
if (!p_remote_node || !p_remote_node->sw)
continue;
/* Fetch remote guid only after validation of remote node */
- remote_guid = osm_node_get_node_guid(p_remote_node);
p_remote_sw = p_remote_node->sw;
rem_u = p_remote_sw->priv;
/* Decide which direction to mark it (UP/DOWN) */
next_dir = __updn_get_dir(u->rank, rem_u->rank,
- cl_ntoh64(current_guid),
- cl_ntoh64(remote_guid));
+ u->id, rem_u->id);
/* Check if this is a legal step : the only illegal step is going
from DOWN to UP */
@@ -181,8 +177,8 @@ __updn_bfs_by_node(IN osm_log_t * p_log,
OSM_LOG(p_log, OSM_LOG_DEBUG,
"Avoiding move from 0x%016" PRIx64
" to 0x%016" PRIx64 "\n",
- cl_ntoh64(current_guid),
- cl_ntoh64(remote_guid));
+ cl_ntoh64(osm_node_get_node_guid(u->sw->p_node)),
+ cl_ntoh64(osm_node_get_node_guid(p_remote_node)));
/* Illegal step */
continue;
}
@@ -414,6 +410,7 @@ static struct updn_node *create_updn_node(osm_switch_t * sw)
return NULL;
memset(u, 0, sizeof(*u));
u->sw = sw;
+ u->id = cl_ntoh64(osm_node_get_node_guid(sw->p_node));
u->rank = 0xffffffff;
return u;
}
@@ -434,6 +431,35 @@ static void dump_roots(cl_map_item_t *item, FILE *file, void *cxt)
cl_ntoh64(osm_node_get_node_guid(sw->p_node)));
}
+static int update_id(void *cxt, uint64_t guid, char *p)
+{
+ osm_opensm_t *osm = cxt;
+ osm_switch_t *sw;
+ uint64_t id;
+ char *e;
+
+ sw = osm_get_switch_by_guid(&osm->subn, cl_hton64(guid));
+ if (!sw) {
+ OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
+ "switch with guid 0x%" PRIx64 " is not found\n", guid);
+ return 0;
+ }
+
+ id = strtoull(p, &e, 0);
+ if (*e) {
+ OSM_LOG(&osm->log, OSM_LOG_ERROR,
+ "ERR: cannot parse node id \'%s\'", p);
+ return -1;
+ }
+
+ OSM_LOG(&osm->log, OSM_LOG_DEBUG,
+ "update node 0x%" PRIx64 " id to 0x%" PRIx64 "\n", guid, id);
+
+ ((struct updn_node *)sw->priv)->id = id;
+
+ return 0;
+}
+
static int rank_root_node(void *cxt, uint64_t guid, char *p)
{
updn_t *updn = cxt;
@@ -483,7 +509,7 @@ static int __osm_updn_call(void *ctx)
if (p_updn->p_osm->subn.opt.root_guid_file) {
OSM_LOG(&p_updn->p_osm->log, OSM_LOG_DEBUG,
- "UPDN - Fetching root nodes from file %s\n",
+ "UPDN - Fetching root nodes from file \'%s\'\n",
p_updn->p_osm->subn.opt.root_guid_file);
ret = parse_node_map(p_updn->p_osm->subn.opt.root_guid_file,
@@ -500,6 +526,19 @@ static int __osm_updn_call(void *ctx)
__osm_updn_find_root_nodes_by_min_hop(p_updn);
}
+ if (p_updn->p_osm->subn.opt.ids_guid_file) {
+ OSM_LOG(&p_updn->p_osm->log, OSM_LOG_DEBUG,
+ "UPDN - update node ids from file \'%s\'\n",
+ p_updn->p_osm->subn.opt.ids_guid_file);
+
+ ret = parse_node_map(p_updn->p_osm->subn.opt.ids_guid_file,
+ update_id, p_updn->p_osm);
+ if (ret)
+ OSM_LOG(&p_updn->p_osm->log, OSM_LOG_ERROR, "ERR : "
+ "cannot parse node ids file \'%s\'\n",
+ p_updn->p_osm->subn.opt.ids_guid_file);
+ }
+
/* Only if there are assigned root nodes do the algorithm, otherwise perform do nothing */
if (p_updn->num_roots) {
OSM_LOG(&p_updn->p_osm->log, OSM_LOG_DEBUG,
--
1.5.4.1.122.gaa8d
More information about the general
mailing list