[ofa-general] [PATCHv4] opensm/lash: Set minimum VL for LASH to use
Hal Rosenstock
hnrose at comcast.net
Tue Jul 21 13:51:04 PDT 2009
rather than starting from VL 0
Signed-off-by: Robert Pearson <rpearson at systemfabricworks.com>
Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
---
Changes since v3:
Reduce vl_min by start_vl and preserve most code flows
Only need to update routing_table.lane
Simplification pointed out by Sasha
Changes since v2:
Restored malloc use (rather than calloc) as pointed out by Sasha
Changes since v1:
Fixed comparisons with maximum VL
Better lash_start_vl option handling
Both as pointed out by Sasha
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 59a32ad..da8cc5e 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -221,6 +222,7 @@ typedef struct osm_subn_opt {
char *prefix_routes_file;
boolean_t consolidate_ipv6_snm_req;
struct osm_subn_opt *file_opts; /* used for update */
+ uint8_t lash_start_vl; /* starting vl to use in lash */
} osm_subn_opt_t;
/*
* FIELDS
diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in
index dbf4b73..c71a79d 100644
--- a/opensm/man/opensm.8.in
+++ b/opensm/man/opensm.8.in
@@ -1,4 +1,4 @@
-.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management"
+.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management"
.SH NAME
opensm \- InfiniBand subnet manager and administration (SM/SA)
@@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA)
[\-r(eassign_lids)]
[\-R <engine name(s)> | \-\-routing_engine <engine name(s)>]
[\-\-do_mesh_analysis]
+[\-\-lash_start_vl <vl number>]
[\-A | \-\-ucast_cache]
[\-z | \-\-connect_roots]
[\-M <file name> | \-\-lid_matrix_file <file name>]
@@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to
precondition switch port assignments in regular cartesian meshes which
may reduce the number of SLs required to give a deadlock free routing.
.TP
+\fB\-\-lash_start_vl\fR <vl number>
+This option sets the starting VL to use for the lash routing algorithm.
+Defaults to 0.
+.TP
\fB\-A\fR, \fB\-\-ucast_cache\fR
This option enables unicast routing cache and prevents routing
recalculation (which is a heavy task in a large cluster) when
diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
index 296d5d5..d682ff5 100644
--- a/opensm/opensm/main.c
+++ b/opensm/opensm/main.c
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -181,6 +182,9 @@ static void show_usage(void)
" routing engine to precondition switch port assignments\n"
" in regular cartesian meshes which may reduce the number\n"
" of SLs required to give a deadlock free routing\n\n");
+ printf("--lash_start_vl <vl number>\n"
+ " Sets the starting VL to use for the lash routing algorithm.\n"
+ " Defaults to 0.\n");
printf("--connect_roots, -z\n"
" This option enforces a routing engine (currently\n"
" up/down only) to make connectivity between root switches\n"
@@ -601,6 +605,7 @@ int main(int argc, char *argv[])
{"prefix_routes_file", 1, NULL, 3},
{"consolidate_ipv6_snm_req", 0, NULL, 4},
{"do_mesh_analysis", 0, NULL, 5},
+ {"lash_start_vl", 1, NULL, 6},
{NULL, 0, NULL, 0} /* Required at the end of the array */
};
@@ -951,6 +956,15 @@ int main(int argc, char *argv[])
case 5:
opt.do_mesh_analysis = TRUE;
break;
+ case 6:
+ temp = strtol(optarg, NULL, 0);
+ if (temp < 0 || temp >= IB_MAX_NUM_VLS) {
+ fprintf(stderr,
+ "ERROR: starting lash vl must be between 0 and 15\n");
+ return (-1);
+ }
+ opt.lash_start_vl = (uint8_t) temp;
+ break;
case 'h':
case '?':
case ':':
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index ec15f8a..fda2eb0 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = {
{ "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 },
{ "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 },
{ "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 },
+ { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 },
{0}
};
@@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt)
p_opt->no_clients_rereg = FALSE;
p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE);
p_opt->consolidate_ipv6_snm_req = FALSE;
+ p_opt->lash_start_vl = 0;
subn_init_qos_options(&p_opt->qos_options, NULL);
subn_init_qos_options(&p_opt->qos_ca_options, NULL);
subn_init_qos_options(&p_opt->qos_sw0_options, NULL);
@@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts)
p_opts->do_mesh_analysis ? "TRUE" : "FALSE");
fprintf(out,
+ "# Starting VL for LASH algorithm\n"
+ "lash_start_vl %d\n\n",
+ p_opts->lash_start_vl);
+
+ fprintf(out,
"# SA database file name\nsa_db_file %s\n\n",
p_opts->sa_db_file ? p_opts->sa_db_file : null_str);
diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c
index 12b5e34..b785551 100644
--- a/opensm/opensm/osm_ucast_lash.c
+++ b/opensm/opensm/osm_ucast_lash.c
@@ -486,6 +486,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
int next_switch2, output_link2;
int stop = 0, cycle_found;
int cycle_found2;
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
max_filled_lane = 0;
min_filled_lane = lanes_needed - 1;
@@ -572,8 +573,8 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
virtual_location[dest][src][max_filled_lane] = 0;
virtual_location[src][dest][min_filled_lane] = 1;
virtual_location[dest][src][min_filled_lane] = 1;
- p_lash->switches[src]->routing_table[dest].lane = min_filled_lane;
- p_lash->switches[dest]->routing_table[src].lane = min_filled_lane;
+ p_lash->switches[src]->routing_table[dest].lane = min_filled_lane + start_vl;
+ p_lash->switches[dest]->routing_table[src].lane = min_filled_lane + start_vl;
}
if (trials == 0)
@@ -804,6 +805,7 @@ static int lash_core(lash_t * p_lash)
int cycle_found2 = 0;
int status = 0;
int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
OSM_LOG_ENTER(p_log);
@@ -838,7 +840,9 @@ static int lash_core(lash_t * p_lash)
}
for (i = 0; i < num_switches; i++) {
- for (dest_switch = 0; dest_switch < num_switches; dest_switch++)
+ for (dest_switch = 0; dest_switch < num_switches; dest_switch++) {
+ if (lanes_needed > p_lash->vl_min)
+ goto Error_Not_Enough_Lanes;
if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) {
v_lane = 0;
stop = 0;
@@ -902,8 +906,8 @@ static int lash_core(lash_t * p_lash)
}
}
- switches[i]->routing_table[dest_switch].lane = v_lane;
- switches[dest_switch]->routing_table[i].lane = v_lane;
+ switches[i]->routing_table[dest_switch].lane = v_lane + start_vl;
+ switches[dest_switch]->routing_table[i].lane = v_lane + start_vl;
if (cycle_found == 1 || cycle_found2 == 1) {
if (++lanes_needed > p_lash->vl_min)
@@ -926,16 +930,20 @@ static int lash_core(lash_t * p_lash)
switch_bitmap[i * num_switches + dest_switch] = 1;
switch_bitmap[dest_switch * num_switches + i] = 1;
}
+ }
}
- OSM_LOG(p_log, OSM_LOG_INFO,
- "Lanes needed: %d, Balancing\n", lanes_needed);
+ if (lanes_needed > p_lash->vl_min)
+ goto Error_Not_Enough_Lanes;
for (i = 0; i < lanes_needed; i++) {
OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n",
i, p_lash->num_mst_in_lane[i]);
}
+ OSM_LOG(p_log, OSM_LOG_INFO,
+ "Lanes needed: %d, Balancing\n", lanes_needed);
+
balance_virtual_lanes(p_lash, lanes_needed);
for (i = 0; i < lanes_needed; i++) {
@@ -948,8 +956,9 @@ static int lash_core(lash_t * p_lash)
Error_Not_Enough_Lanes:
status = -1;
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: "
- "Lane requirements (%d) exceed available lanes (%d)\n",
- lanes_needed, p_lash->vl_min);
+ "Lane requirements (%d) exceed available lanes (%d)"
+ " with starting lane (%d)\n",
+ lanes_needed, p_lash->vl_min, start_vl);
Exit:
if (switch_bitmap)
free(switch_bitmap);
@@ -1177,10 +1186,11 @@ static int discover_network_properties(lash_t * p_lash)
if (vl_min > 15)
vl_min = 15;
- p_lash->vl_min = vl_min;
+ p_lash->vl_min = vl_min - p_lash->p_osm->subn.opt.lash_start_vl;
OSM_LOG(p_log, OSM_LOG_INFO,
- "min operational vl(%d) max_switches(%d)\n", p_lash->vl_min,
+ "min operational vl(%d) start vl(%d) max_switches(%d)\n",
+ p_lash->vl_min, p_lash->p_osm->subn.opt.lash_start_vl,
p_lash->num_switches);
return 0;
}
@@ -1286,7 +1296,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port,
src_id = get_lash_id(p_sw);
if (src_id == dst_id)
- return OSM_DEFAULT_SL;
+ return p_osm->subn.opt.lash_start_vl;
return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane;
}
More information about the general
mailing list