[ofa-general] [PATCHv2] opensm/lash: Set minimum VL for LASH to use

Hal Rosenstock hnrose at comcast.net
Mon Jul 6 08:46:43 PDT 2009


rather than starting from VL 0

Signed-off-by: Robert Pearson <rpearson at systemfabricworks.com>
Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
---
Changes since v1:
Fixed comparisons with maximum VL
Better lash_start_vl option handling
Both as pointed out by Sasha

diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index 59a32ad..da8cc5e 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -3,6 +3,7 @@
  * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
  * Copyright (c) 2008 Xsigo Systems Inc.  All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -221,6 +222,7 @@ typedef struct osm_subn_opt {
 	char *prefix_routes_file;
 	boolean_t consolidate_ipv6_snm_req;
 	struct osm_subn_opt *file_opts; /* used for update */
+	uint8_t lash_start_vl;			/* starting vl to use in lash */
 } osm_subn_opt_t;
 /*
 * FIELDS
diff --git a/opensm/man/opensm.8.in b/opensm/man/opensm.8.in
index 66d2fe6..e8801c9 100644
--- a/opensm/man/opensm.8.in
+++ b/opensm/man/opensm.8.in
@@ -1,4 +1,4 @@
-.TH OPENSM 8 "April 22, 2009" "OpenIB" "OpenIB Management"
+.TH OPENSM 8 "May 28, 2009" "OpenIB" "OpenIB Management"
 
 .SH NAME
 opensm \- InfiniBand subnet manager and administration (SM/SA)
@@ -15,6 +15,7 @@ opensm \- InfiniBand subnet manager and administration (SM/SA)
 [\-r(eassign_lids)]
 [\-R <engine name(s)> | \-\-routing_engine <engine name(s)>]
 [\-\-do_mesh_analysis]
+[\-\-lash_start_vl <vl number>]
 [\-A | \-\-ucast_cache]
 [\-z | \-\-connect_roots]
 [\-M <file name> | \-\-lid_matrix_file <file name>]
@@ -147,6 +148,10 @@ This option enables additional analysis for the lash routing engine to
 precondition switch port assignments in regular cartesian meshes which
 may reduce the number of SLs required to give a deadlock free routing.
 .TP
+\fB\-\-lash_start_vl\fR <vl number>
+This option sets the starting VL to use for the lash routing algorithm.
+Defaults to 0.
+.TP
 \fB\-A\fR, \fB\-\-ucast_cache\fR
 This option enables unicast routing cache and prevents routing
 recalculation (which is a heavy task in a large cluster) when
diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
index 296d5d5..d682ff5 100644
--- a/opensm/opensm/main.c
+++ b/opensm/opensm/main.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
  * Copyright (c) 2009 HNR Consulting. All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -181,6 +182,9 @@ static void show_usage(void)
 	       "          routing engine to precondition switch port assignments\n"
 	       "          in regular cartesian meshes which may reduce the number\n"
 	       "          of SLs required to give a deadlock free routing\n\n");
+	printf("--lash_start_vl <vl number>\n"
+		   "          Sets the starting VL to use for the lash routing algorithm.\n"
+		   "          Defaults to 0.\n");
 	printf("--connect_roots, -z\n"
 	       "          This option enforces a routing engine (currently\n"
 	       "          up/down only) to make connectivity between root switches\n"
@@ -601,6 +605,7 @@ int main(int argc, char *argv[])
 		{"prefix_routes_file", 1, NULL, 3},
 		{"consolidate_ipv6_snm_req", 0, NULL, 4},
 		{"do_mesh_analysis", 0, NULL, 5},
+		{"lash_start_vl", 1, NULL, 6},
 		{NULL, 0, NULL, 0}	/* Required at the end of the array */
 	};
 
@@ -951,6 +956,15 @@ int main(int argc, char *argv[])
 		case 5:
 			opt.do_mesh_analysis = TRUE;
 			break;
+		case 6:
+			temp = strtol(optarg, NULL, 0);
+			if (temp < 0 || temp >= IB_MAX_NUM_VLS) {
+				fprintf(stderr,
+					"ERROR: starting lash vl must be between 0 and 15\n");
+				return (-1);
+			}
+			opt.lash_start_vl = (uint8_t) temp;
+			break;
 		case 'h':
 		case '?':
 		case ':':
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index ec15f8a..fda2eb0 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
  * Copyright (c) 2008 Xsigo Systems Inc.  All rights reserved.
+ * Copyright (c) 2009 System Fabric Works, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -389,6 +390,7 @@ static const opt_rec_t opt_tbl[] = {
 	{ "no_clients_rereg", OPT_OFFSET(no_clients_rereg), opts_parse_boolean, NULL, 1 },
 	{ "prefix_routes_file", OPT_OFFSET(prefix_routes_file), opts_parse_charp, NULL, 0 },
 	{ "consolidate_ipv6_snm_req", OPT_OFFSET(consolidate_ipv6_snm_req), opts_parse_boolean, NULL, 1 },
+	{ "lash_start_vl", OPT_OFFSET(lash_start_vl), opts_parse_uint8, NULL, 1 },
 	{0}
 };
 
@@ -749,6 +751,7 @@ void osm_subn_set_default_opt(IN osm_subn_opt_t * const p_opt)
 	p_opt->no_clients_rereg = FALSE;
 	p_opt->prefix_routes_file = strdup(OSM_DEFAULT_PREFIX_ROUTES_FILE);
 	p_opt->consolidate_ipv6_snm_req = FALSE;
+	p_opt->lash_start_vl = 0;
 	subn_init_qos_options(&p_opt->qos_options, NULL);
 	subn_init_qos_options(&p_opt->qos_ca_options, NULL);
 	subn_init_qos_options(&p_opt->qos_sw0_options, NULL);
@@ -1432,6 +1435,11 @@ int osm_subn_output_conf(FILE *out, IN osm_subn_opt_t *const p_opts)
 		p_opts->do_mesh_analysis ? "TRUE" : "FALSE");
 
 	fprintf(out,
+		"# Starting VL for LASH algorithm\n"
+		"lash_start_vl %d\n\n",
+		p_opts->lash_start_vl);
+
+	fprintf(out,
 		"# SA database file name\nsa_db_file %s\n\n",
 		p_opts->sa_db_file ? p_opts->sa_db_file : null_str);
 
diff --git a/opensm/opensm/osm_ucast_lash.c b/opensm/opensm/osm_ucast_lash.c
index 12b5e34..3db75a0 100644
--- a/opensm/opensm/osm_ucast_lash.c
+++ b/opensm/opensm/osm_ucast_lash.c
@@ -478,7 +478,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
 	cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
 	int *num_mst_in_lane = p_lash->num_mst_in_lane;
 	int ***virtual_location = p_lash->virtual_location;
-	int min_filled_lane, max_filled_lane, trials;
+	int min_filled_lane, max_filled_lane, trials, max_vl;
 	int old_min_filled_lane, old_max_filled_lane, new_num_min_lane,
 	    new_num_max_lane;
 	unsigned int i, j;
@@ -486,9 +486,11 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
 	int next_switch2, output_link2;
 	int stop = 0, cycle_found;
 	int cycle_found2;
+	unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
 
-	max_filled_lane = 0;
-	min_filled_lane = lanes_needed - 1;
+	max_filled_lane = start_vl;
+	max_vl = lanes_needed + start_vl;
+	min_filled_lane = max_vl - 1;
 
 	trials = num_mst_in_lane[max_filled_lane];
 	if (lanes_needed == 1)
@@ -590,7 +592,7 @@ static void balance_virtual_lanes(lash_t * p_lash, unsigned lanes_needed)
 		new_num_min_lane = MAX_INT;
 		new_num_max_lane = 0;
 
-		for (i = 0; i < lanes_needed; i++) {
+		for (i = start_vl; i < max_vl; i++) {
 
 			if (num_mst_in_lane[i] < new_num_min_lane) {
 				new_num_min_lane = num_mst_in_lane[i];
@@ -674,11 +676,12 @@ static void free_lash_structures(lash_t * p_lash)
 	unsigned int i, j, k;
 	unsigned num_switches = p_lash->num_switches;
 	osm_log_t *p_log = &p_lash->p_osm->log;
+	unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
 
 	OSM_LOG_ENTER(p_log);
 
 	/* free cdg_vertex_matrix */
-	for (i = 0; i < p_lash->vl_min; i++) {
+	for (i = start_vl; i < p_lash->vl_min; i++) {
 		for (j = 0; j < num_switches; j++) {
 			for (k = 0; k < num_switches; k++)
 				if (p_lash->cdg_vertex_matrix[i][j][k])
@@ -715,13 +718,14 @@ static int init_lash_structures(lash_t * p_lash)
 	osm_log_t *p_log = &p_lash->p_osm->log;
 	int status = 0;
 	unsigned int i, j, k;
+	unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
 
 	OSM_LOG_ENTER(p_log);
 
 	/* initialise cdg_vertex_matrix[num_switches][num_switches][num_switches] */
 	p_lash->cdg_vertex_matrix =
-	    (cdg_vertex_t ****) malloc(vl_min * sizeof(cdg_vertex_t ****));
-	for (i = 0; i < vl_min; i++) {
+	    (cdg_vertex_t ****) calloc(vl_min, sizeof(cdg_vertex_t ****));
+	for (i = start_vl; i < vl_min; i++) {
 		p_lash->cdg_vertex_matrix[i] =
 		    (cdg_vertex_t ***) malloc(num_switches *
 					      sizeof(cdg_vertex_t ***));
@@ -730,7 +734,7 @@ static int init_lash_structures(lash_t * p_lash)
 			goto Exit_Mem_Error;
 	}
 
-	for (i = 0; i < vl_min; i++) {
+	for (i = start_vl; i < vl_min; i++) {
 		for (j = 0; j < num_switches; j++) {
 			p_lash->cdg_vertex_matrix[i][j] =
 			    (cdg_vertex_t **) malloc(num_switches *
@@ -763,7 +767,7 @@ static int init_lash_structures(lash_t * p_lash)
 	for (i = 0; i < num_switches; i++) {
 		for (j = 0; j < num_switches; j++) {
 			p_lash->virtual_location[i][j] =
-			    (int *)malloc(vl_min * sizeof(int *));
+			    (int *)calloc(vl_min, sizeof(int *));
 			if (p_lash->virtual_location[i][j] == NULL)
 				goto Exit_Mem_Error;
 			for (k = 0; k < vl_min; k++) {
@@ -804,6 +808,8 @@ static int lash_core(lash_t * p_lash)
 	int cycle_found2 = 0;
 	int status = 0;
 	int *switch_bitmap = NULL;	/* Bitmap to check if we have processed this pair */
+	int max_vl;
+	unsigned start_vl = p_lash->p_osm->subn.opt.lash_start_vl;
 
 	OSM_LOG_ENTER(p_log);
 
@@ -838,11 +844,14 @@ static int lash_core(lash_t * p_lash)
 	}
 
 	for (i = 0; i < num_switches; i++) {
-		for (dest_switch = 0; dest_switch < num_switches; dest_switch++)
+		for (dest_switch = 0; dest_switch < num_switches; dest_switch++) {
+			max_vl = lanes_needed + start_vl;
+			if (max_vl > p_lash->vl_min)
+				goto Error_Not_Enough_Lanes;
 			if (dest_switch != i && switch_bitmap[i * num_switches + dest_switch] == 0) {
-				v_lane = 0;
+				v_lane = start_vl;
 				stop = 0;
-				while (v_lane < lanes_needed && stop == 0) {
+				while (v_lane < max_vl && stop == 0) {
 					generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
 					generate_cdg_for_sp(p_lash, dest_switch, i, v_lane);
 
@@ -906,7 +915,8 @@ static int lash_core(lash_t * p_lash)
 				switches[dest_switch]->routing_table[i].lane = v_lane;
 
 				if (cycle_found == 1 || cycle_found2 == 1) {
-					if (++lanes_needed > p_lash->vl_min)
+					lanes_needed++;
+					if (start_vl + lanes_needed > p_lash->vl_min)
 						goto Error_Not_Enough_Lanes;
 
 					generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
@@ -926,19 +936,24 @@ static int lash_core(lash_t * p_lash)
 				switch_bitmap[i * num_switches + dest_switch] = 1;
 				switch_bitmap[dest_switch * num_switches + i] = 1;
 			}
+		}
 	}
 
-	OSM_LOG(p_log, OSM_LOG_INFO,
-		"Lanes needed: %d, Balancing\n", lanes_needed);
+	max_vl = lanes_needed + start_vl;
+	if (max_vl > p_lash->vl_min)
+		goto Error_Not_Enough_Lanes;
 
-	for (i = 0; i < lanes_needed; i++) {
+	for (i = start_vl; i < max_vl; i++) {
 		OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n",
 			i, p_lash->num_mst_in_lane[i]);
 	}
 
+	OSM_LOG(p_log, OSM_LOG_INFO,
+		"Lanes needed: %d, Balancing\n", lanes_needed);
+
 	balance_virtual_lanes(p_lash, lanes_needed);
 
-	for (i = 0; i < lanes_needed; i++) {
+	for (i = start_vl; i < max_vl; i++) {
 		OSM_LOG(p_log, OSM_LOG_INFO, "Lanes in layer %d: %d\n",
 			i, p_lash->num_mst_in_lane[i]);
 	}
@@ -948,8 +963,9 @@ static int lash_core(lash_t * p_lash)
 Error_Not_Enough_Lanes:
 	status = -1;
 	OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4D02: "
-		"Lane requirements (%d) exceed available lanes (%d)\n",
-		lanes_needed, p_lash->vl_min);
+		"Lane requirements (%d) exceed available lanes (%d)"
+		" with starting lane (%d)\n",
+		lanes_needed, p_lash->vl_min, start_vl);
 Exit:
 	if (switch_bitmap)
 		free(switch_bitmap);
@@ -1286,7 +1302,7 @@ uint8_t osm_get_lash_sl(osm_opensm_t * p_osm, const osm_port_t * p_src_port,
 
 	src_id = get_lash_id(p_sw);
 	if (src_id == dst_id)
-		return OSM_DEFAULT_SL;
+		return p_osm->subn.opt.lash_start_vl;
 
 	return (uint8_t) ((switch_t *) p_sw->priv)->routing_table[dst_id].lane;
 }



More information about the general mailing list