[ofa-general] [PATCHv2] opensm/lash: Set minimum VL for LASH to use
Hal Rosenstock
hnrose at comcast.net
Mon Jul 6 08:46:43 PDT 2009
rather than starting from VL 0
Signed-off-by: Robert Pearson <rpearson at systemfabricworks.com>
Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
---
Changes since v1:
Fixed comparisons with maximum VL
Better lash_start_vl option handling
Both as pointed out by Sasha
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 59a32ad..da8cc5e 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -221,6 +222,7 @@ typedef struct osm_subn_opt {
char *prefix_routes_file;
boolean_t consolidate_ipv6_snm_req;
struct osm_subn_opt *file_opts; /* used for update */
+ uint8_t lash_start_vl; /* starting vl to use in lash */
} osm_subn_opt_t;
/*
* FIELDS
diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in
index 66d2fe6..e8801c9 100644
--- a/opensm/man/opensm.8.in
+++ b/opensm/man/opensm.8.in
@@ -1,4 +1,4 @@
-.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management"
+.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management"
.SH NAME
opensm \- InfiniBand subnet manager and administration (SM/SA)
@@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA)
[\-r(eassign_lids)]
[\-R <engine name(s)> | \-\-routing_engine <engine name(s)>]
[\-\-do_mesh_analysis]
+[\-\-lash_start_vl <vl number>]
[\-A | \-\-ucast_cache]
[\-z | \-\-connect_roots]
[\-M <file name> | \-\-lid_matrix_file <file name>]
@@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to
precondition switch port assignments in regular cartesian meshes which
may reduce the number of SLs required to give a deadlock free routing.
.TP
+\fB\-\-lash_start_vl\fR <vl number>
+This option sets the starting VL to use for the lash routing algorithm.
+Defaults to 0.
+.TP
\fB\-A\fR, \fB\-\-ucast_cache\fR
This option enables unicast routing cache and prevents routing
recalculation (which is a heavy task in a large cluster) when
diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
index 296d5d5..d682ff5 100644
--- a/opensm/opensm/main.c
+++ b/opensm/opensm/main.c
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -181,6 +182,9 @@ static void show_usage(void)
" routing engine to precondition switch port assignments\n"
" in regular cartesian meshes which may reduce the number\n"
" of SLs required to give a deadlock free routing\n\n");
+ printf("--lash_start_vl <vl number>\n"
+ " Sets the starting VL to use for the lash routing algorithm.\n"
+ " Defaults to 0.\n");
printf("--connect_roots, -z\n"
" This option enforces a routing engine (currently\n"
" up/down only) to make connectivity between root switches\n"
@@ -601,6 +605,7 @@ int main(int argc, char *argv[])
{"prefix_routes_file", 1, NULL, 3},
{"consolidate_ipv6_snm_req", 0, NULL, 4},
{"do_mesh_analysis", 0, NULL, 5},
+ {"lash_start_vl", 1, NULL, 6},
{NULL, 0, NULL, 0} /* Required at the end of the array */
};
@@ -951,6 +956,15 @@ int main(int argc, char *argv[])
case 5:
opt.do_mesh_analysis = TRUE;
break;
+ case 6:
+ temp = strtol(optarg, NULL, 0);
+ if (temp < 0 || temp >= IB_MAX_NUM_VLS) {
+ fprintf(stderr,
+ "ERROR: starting lash vl must be between 0 and 15\n");
+ return (-1);
+ }
+ opt.lash_start_vl = (uint8_t) temp;
+ break;
case 'h':
case '?':
case ':':
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index ec15f8a..fda2eb0 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -3,6 +3,7 @@
* Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = {
{ "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 },
{ "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 },
{ "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 },
+ { "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 },
{0}
};
@@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt)
p_opt->no_clients_rereg = FALSE;
p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE);
p_opt->consolidate_ipv6_snm_req = FALSE;
+ p_opt->lash_start_vl = 0;
subn_init_qos_options(&p_opt->qos_options, NULL);
subn_init_qos_options(&p_opt->qos_ca_options, NULL);
subn_init_qos_options(&p_opt->qos_sw0_options, NULL);
@@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts)
p_opts->do_mesh_analysis ? "TRUE" : "FALSE");
fprintf(out,
+ "# Starting VL for LASH algorithm\n"
+ "lash_start_vl %d\n\n",
+ p_opts->lash_start_vl);
+
+ fprintf(out,
"# SA database file name\nsa_db_file %s\n\n",
p_opts->sa_db_file ? p_opts->sa_db_file : null_str);
diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c
index 12b5e34..3db75a0 100644
--- a/opensm/opensm/osm_ucast_lash.c
+++ b/opensm/opensm/osm_ucast_lash.c
@@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
int *num_mst_in_lane = p_lash->num_mst_in_lane;
int ***virtual_location = p_lash->virtual_location;
- int min_filled_lane, max_filled_lane, trials;
+ int min_filled_lane, max_filled_lane, trials, max_vl;
int old_min_filled_lane, old_max_filled_lane, new_num_min_lane,
new_num_max_lane;
unsigned int i, j;
@@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
int next_switch2, output_link2;
int stop = 0, cycle_found;
int cycle_found2;
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
- max_filled_lane = 0;
- min_filled_lane = lanes_needed - 1;
+ max_filled_lane = start_vl;
+ max_vl = lanes_needed + start_vl;
+ min_filled_lane = max_vl - 1;
trials = num_mst_in_lane[max_filled_lane];
if (lanes_needed == 1)
@@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
new_num_min_lane = MAX_INT;
new_num_max_lane = 0;
- for (i = 0; i < lanes_needed; i++) {
+ for (i = start_vl; i < max_vl; i++) {
if (num_mst_in_lane[i] < new_num_min_lane) {
new_num_min_lane = num_mst_in_lane[i];
@@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash)
unsigned int i, j, k;
unsigned num_switches = p_lash->num_switches;
osm_log_t *p_log = &p_lash->p_osm->log;
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
OSM_LOG_ENTER(p_log);
/* free cdg_vertex_matrix */
- for (i = 0; i < p_lash->vl_min; i++) {
+ for (i = start_vl; i < p_lash->vl_min; i++) {
for (j = 0; j < num_switches; j++) {
for (k = 0; k < num_switches; k++)
if (p_lash->cdg_vertex_matrix[i][j][k])
@@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash)
osm_log_t *p_log = &p_lash->p_osm->log;
int status = 0;
unsigned int i, j, k;
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
OSM_LOG_ENTER(p_log);
/* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */
p_lash->cdg_vertex_matrix =
- (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****));
- for (i = 0; i < vl_min; i++) {
+ (cdg_vertex_t ****) calloc(vl_min, sizeof(cdg_vertex_t ****));
+ for (i = start_vl; i < vl_min; i++) {
p_lash->cdg_vertex_matrix[i] =
(cdg_vertex_t ***) malloc(num_switches *
sizeof(cdg_vertex_t ***));
@@ -730,7 +734,7 @@ static int init_lash_structures(lash_t * p_lash)
goto Exit_Mem_Error;
}
- for (i = 0; i < vl_min; i++) {
+ for (i = start_vl; i < vl_min; i++) {
for (j = 0; j < num_switches; j++) {
p_lash->cdg_vertex_matrix[i][j] =
(cdg_vertex_t **) malloc(num_switches *
@@ -763,7 +767,7 @@ static int init_lash_structures(lash_t * p_lash)
for (i = 0; i < num_switches; i++) {
for (j = 0; j < num_switches; j++) {
p_lash->virtual_location[i][j] =
- (int *)malloc(vl_min * sizeof(int *));
+ (int *)calloc(vl_min, sizeof(int *));
if (p_lash->virtual_location[i][j] == NULL)
goto Exit_Mem_Error;
for (k = 0; k < vl_min; k++) {
@@ -804,6 +808,8 @@ static int lash_core(lash_t * p_lash)
int cycle_found2 = 0;
int status = 0;
int *switch_bitmap = NULL; /* Bitmap to check if we have processed this pair */
+ int max_vl;
+ unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
OSM_LOG_ENTER(p_log);
@@ -838,11 +844,14 @@ static int lash_core(lash_t * p_lash)
}
for (i = 0; i < num_switches; i++) {
- for (dest_switch = 0; dest_switch < num_switches; dest_switch++)
+ for (dest_switch = 0; dest_switch < num_switches; dest_switch++) {
+ max_vl = lanes_needed + start_vl;
+ if (max_vl > p_lash->vl_min)
+ goto Error_Not_Enough_Lanes;
if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) {
- v_lane = 0;
+ v_lane = start_vl;
stop = 0;
- while (v_lane < lanes_needed && stop == 0) {
+ while (v_lane < max_vl && stop == 0) {
generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
generate_cdg_for_sp(p_lash, dest_switch, i, v_lane);
@@ -906,7 +915,8 @@ static int lash_core(lash_t * p_lash)
switches[dest_switch]->routing_table[i].lane = v_lane;
if (cycle_found == 1 || cycle_found2 == 1) {
- if (++lanes_needed > p_lash->vl_min)
+ lanes_needed++;
+ if (start_vl + lanes_needed > p_lash->vl_min)
goto Error_Not_Enough_Lanes;
generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
@@ -926,19 +936,24 @@ static int lash_core(lash_t * p_lash)
switch_bitmap[i * num_switches + dest_switch] = 1;
switch_bitmap[dest_switch * num_switches + i] = 1;
}
+ }
}
- OSM_LOG(p_log, OSM_LOG_INFO,
- "Lanes needed: %d, Balancing\n", lanes_needed);
+ max_vl = lanes_needed + start_vl;
+ if (max_vl > p_lash->vl_min)
+ goto Error_Not_Enough_Lanes;
- for (i = 0; i < lanes_needed; i++) {
+ for (i = start_vl; i < max_vl; i++) {
OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n",
i, p_lash->num_mst_in_lane[i]);
}
+ OSM_LOG(p_log, OSM_LOG_INFO,
+ "Lanes needed: %d, Balancing\n", lanes_needed);
+
balance_virtual_lanes(p_lash, lanes_needed);
- for (i = 0; i < lanes_needed; i++) {
+ for (i = start_vl; i < max_vl; i++) {
OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n",
i, p_lash->num_mst_in_lane[i]);
}
@@ -948,8 +963,9 @@ static int lash_core(lash_t * p_lash)
Error_Not_Enough_Lanes:
status = -1;
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: "
- "Lane requirements (%d) exceed available lanes (%d)\n",
- lanes_needed, p_lash->vl_min);
+ "Lane requirements (%d) exceed available lanes (%d)"
+ " with starting lane (%d)\n",
+ lanes_needed, p_lash->vl_min, start_vl);
Exit:
if (switch_bitmap)
free(switch_bitmap);
@@ -1286,7 +1302,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port,
src_id = get_lash_id(p_sw);
if (src_id == dst_id)
- return OSM_DEFAULT_SL;
+ return p_osm->subn.opt.lash_start_vl;
return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane;
}
More information about the general
mailing list