[openib-general] [PATCH] OpenSM: Experimental code for LASH routing
Hal Rosenstock
halr at voltaire.com
Mon Jan 29 15:32:41 PST 2007
OpenSM: Experimental code for LASH routing
LASH is an acronym for LAyered SHortest Path Routing. This algorithm
was developed by Simula Research Lab. LASH uses VL layers to create
deadlock free paths.
LASH routing is experimental status in OpenSM and currently limited to
small topologies. It is low risk as there are almost no changes to the
mainline code paths in OpenSM.
Signed-off-by: Thomas Sødring <tsodring at simula.no>
---
osm/include/opensm/osm_switch.h | 1 +
osm/opensm/Makefile.am | 2 +-
osm/opensm/osm_opensm.c | 2 +
osm/opensm/osm_sa_path_record.c | 12 +-
osm/opensm/osm_ucast_lash.c | 1553 +++++++++++++++++++++++++++++++++++++++
5 files changed, 1568 insertions(+), 2 deletions(-)
create mode 100644 osm/opensm/osm_ucast_lash.c
diff --git a/osm/include/opensm/osm_switch.h b/osm/include/opensm/osm_switch.h
index b2bf0db..1c66028 100644
--- a/osm/include/opensm/osm_switch.h
+++ b/osm/include/opensm/osm_switch.h
@@ -111,6 +111,7 @@ typedef struct _osm_switch
osm_port_profile_t *p_prof;
osm_mcast_tbl_t mcast_tbl;
uint32_t discovery_count;
+ void *priv;
} osm_switch_t;
/*
* FIELDS
diff --git a/osm/opensm/Makefile.am b/osm/opensm/Makefile.am
index b1028d8..15af336 100644
--- a/osm/opensm/Makefile.am
+++ b/osm/opensm/Makefile.am
@@ -54,7 +54,7 @@ opensm_SOURCES = main.c osm_console.c osm_db_files.c \
osm_sweep_fail_ctrl.c osm_sw_info_rcv.c osm_switch.c \
osm_prtn.c osm_prtn_config.c osm_qos.c osm_router.c \
osm_trap_rcv.c osm_ucast_mgr.c osm_ucast_updn.c \
- osm_ucast_file.c osm_ucast_ftree.c \
+ osm_ucast_lash.c osm_ucast_file.c osm_ucast_ftree.c \
osm_vl15intf.c osm_vl_arb_rcv.c \
st.c
if OSMV_OPENIB
diff --git a/osm/opensm/osm_opensm.c b/osm/opensm/osm_opensm.c
index 1c17979..337130f 100644
--- a/osm/opensm/osm_opensm.c
+++ b/osm/opensm/osm_opensm.c
@@ -72,6 +72,7 @@ struct routing_engine_module {
extern int osm_ucast_updn_setup(osm_opensm_t *p_osm);
extern int osm_ucast_file_setup(osm_opensm_t *p_osm);
extern int osm_ucast_ftree_setup(osm_opensm_t *p_osm);
+extern int osm_ucast_lash_setup(osm_opensm_t * p_osm);
static int osm_ucast_null_setup(osm_opensm_t *p_osm);
@@ -80,6 +81,7 @@ const static struct routing_engine_module routing_modules[] = {
{ "updn", osm_ucast_updn_setup },
{ "file", osm_ucast_file_setup },
{ "ftree", osm_ucast_ftree_setup },
+ { "lash", osm_ucast_lash_setup },
{ NULL, NULL }
};
diff --git a/osm/opensm/osm_sa_path_record.c b/osm/opensm/osm_sa_path_record.c
index a0dbb07..71cadda 100644
--- a/osm/opensm/osm_sa_path_record.c
+++ b/osm/opensm/osm_sa_path_record.c
@@ -66,6 +66,7 @@
#include <opensm/osm_pkey.h>
#include <opensm/osm_multicast.h>
#include <opensm/osm_partition.h>
+#include <opensm/osm_opensm.h>
#ifdef ROUTER_EXP
#include <opensm/osm_router.h>
#include <opensm/osm_sa_mcmember_record.h>
@@ -74,6 +75,10 @@
#define OSM_PR_RCV_POOL_MIN_SIZE 64
#define OSM_PR_RCV_POOL_GROW_SIZE 64
+extern uint8_t osm_get_lash_sl(osm_opensm_t *p_osm,
+ const osm_port_t *p_src_port,
+ const osm_port_t *p_dst_port);
+
typedef struct _osm_pr_item
{
cl_pool_item_t pool_item;
@@ -674,7 +679,12 @@ __osm_pr_rcv_get_path_parms(
}
}
- sl = OSM_DEFAULT_SL;
+ if (p_rcv->p_subn->opt.routing_engine_name &&
+ strcmp(p_rcv->p_subn->opt.routing_engine_name, "lash") == 0)
+ // slid and dest_lid are stored in network in lash
+ sl = osm_get_lash_sl(p_rcv->p_subn->p_osm, p_src_port, p_dest_port);
+ else
+ sl = OSM_DEFAULT_SL;
if (pkey) {
p_prtn = (osm_prtn_t *)cl_qmap_get(&p_rcv->p_subn->prtn_pkey_tbl,
diff --git a/osm/opensm/osm_ucast_lash.c b/osm/opensm/osm_ucast_lash.c
new file mode 100644
index 0000000..c716855
--- /dev/null
+++ b/osm/opensm/osm_ucast_lash.c
@@ -0,0 +1,1553 @@
+/*
+ * Copyright (c) 2004-2006 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
+ * Copyright (c) 2007 Simula Research Laboratory. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: osm_ucast_updn.c 10057 2006-11-03 16:35:23Z halr $
+ */
+
+
+/*
+ * Abstract:
+ * Implementation of LASH algorithm Calculation functions
+ *
+ * Environment:
+ * Linux User Mode
+ *
+ * $Revision: 1.0 $
+ */
+
+
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <complib/cl_debug.h>
+#include <complib/cl_qmap.h>
+#include <opensm/osm_switch.h>
+#include <opensm/osm_opensm.h>
+#include <opensm/osm_log.h>
+
+#include <opensm/osm_fwd_tbl.h>
+
+/* //////////////////////////// */
+/* Local types */
+/* //////////////////////////// */
+
+enum {
+ MAX_INT = 9999,
+ NONE = MAX_INT
+};
+
+
+typedef struct _cdg_vertex {
+ int switch_size;
+ int num_dependencies;
+ struct _cdg_vertex ** dependency;
+ int from;
+ int to;
+ int seen;
+ int temp;
+ int visiting_number;
+ struct _cdg_vertex * next;
+ int num_temp_depend;
+ int num_using_vertex;
+ int *num_using_this_depend;
+} cdg_vertex_t;
+
+typedef struct _reachable_dest {
+ int switch_id;
+ struct _reachable_dest * next;
+} reachable_dest_t;
+
+typedef struct _q_item {
+ int sw;
+ struct _q_item * next;
+} q_item_t;
+
+typedef struct _switch {
+ osm_switch_t *p_sw;
+ struct _switch ** dij_channels;
+ int id;
+ int used_channels;
+ int mst_member;
+ int q_member;
+ int dist;
+ int prev;
+ struct routing_table {
+ unsigned out_link;
+ unsigned lane;
+ } *routing_table;
+ unsigned int num_connections;
+#if 0
+ struct connections {
+ unsigned sw;
+ unsigned port;
+ } *connections;
+#else
+ int *virtual_physical_port_table;
+ int *phys_connections;
+#endif
+} switch_t;
+
+
+typedef struct _lash
+{
+ osm_opensm_t *p_osm;
+ int num_switches;
+ uint8_t vl_min;
+ int balance_limit;
+ switch_t ** switches;
+ int q_count;
+ q_item_t * q_head;
+ cdg_vertex_t **** cdg_vertex_matrix;
+ int *num_mst_in_lane;
+ int **adj_matrix;
+ int ***virtual_location;
+} lash_t;
+
+
+
+static cdg_vertex_t* create_cdg_vertex(unsigned num_switches)
+{
+ cdg_vertex_t* cdg_vertex = (cdg_vertex_t*) malloc(sizeof(cdg_vertex_t));
+
+ cdg_vertex->dependency = malloc((num_switches-1)*sizeof(cdg_vertex_t*));
+ cdg_vertex->num_using_this_depend =(int*)malloc((num_switches-1)*sizeof(int));
+ return cdg_vertex;
+}
+
+
+/*
+cl_map_t map_switch_guid_to_lash_id;
+
+0x200001 -> 0;
+0x200002 -> 1;
+0x200003 -> 2;
+0x200004 -> 3;
+0x200005 -> 4;
+0x200006 -> 5;
+*/
+
+static int connect_switches(lash_t *p_lash, int sw1, int sw2, int physical_port_1, int physical_port_2)
+{
+ osm_log_t *p_log = &p_lash->p_osm->log;
+ unsigned num = p_lash->switches[sw1]->num_connections;
+
+ p_lash->switches[sw1]->phys_connections[num] = sw2;
+ p_lash->switches[sw1]->virtual_physical_port_table[num] = physical_port_1;
+ p_lash->switches[sw1]->num_connections++;
+
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "connect_switches: "
+ "LASH connect: %d, %d, %d, %d \n",
+ sw1, sw2, physical_port_1, physical_port_1);
+
+ return p_lash->adj_matrix[sw1][sw2] = 1;
+}
+
+
+static uint64_t osm_lash_get_switch_guid(IN const osm_switch_t *p_sw) {
+
+ uint64_t switch_guid = -1;
+ osm_physp_t* p_physp = osm_node_get_physp_ptr(p_sw->p_node, 0);
+
+ if (p_physp && osm_physp_is_valid (p_physp)) {
+ switch_guid = osm_physp_get_port_guid(p_physp);
+ }
+
+ return switch_guid;
+}
+
+
+static osm_switch_t *get_osm_switch_from_port(osm_port_t *port)
+{
+ osm_physp_t *p = osm_port_get_default_phys_ptr(port);
+ if (p->p_node->sw)
+ return p->p_node->sw;
+ else if (p->p_remote_physp->p_node->sw)
+ return p->p_remote_physp->p_node->sw;
+ return NULL;
+}
+
+static osm_switch_t *get_osm_switch_from_lid(osm_opensm_t *osm, uint16_t lid)
+{
+ osm_port_t *port = cl_ptr_vector_get(&osm->subn.port_lid_tbl, lid);
+ if (!port)
+ return NULL;
+ return get_osm_switch_from_port(port);
+}
+
+// This is a time consuming way to find a port from a lid
+// we will come up with a better way later
+static uint8_t find_port_from_lid(IN const ib_net16_t lid_no,
+ IN const osm_switch_t *p_sw) {
+
+ uint8_t port_count = 0;
+ uint8_t i=0;
+ osm_physp_t *p_current_physp, *p_remote_physp = NULL;
+
+ uint8_t egress_port = 255;
+
+ if (p_sw->p_node) {
+ port_count = osm_node_get_num_physp (p_sw->p_node);
+ }
+
+ // process management port first
+ p_current_physp = osm_node_get_physp_ptr(p_sw->p_node, 0);
+
+ ib_port_info_t *port_info = &p_current_physp->port_info;
+ ib_net16_t port_lid = port_info->base_lid;
+ if (port_lid == lid_no) {
+ egress_port = 0;
+ goto Exit;
+ }
+ // process each port on this switch
+ for (i=1; i<port_count; i++) {
+
+ p_current_physp = osm_node_get_physp_ptr(p_sw->p_node, i);
+
+ if (p_current_physp && osm_physp_is_valid (p_current_physp)) {
+
+ p_remote_physp = p_current_physp->p_remote_physp;
+
+ if (p_remote_physp && osm_physp_is_valid ( p_remote_physp )) {
+ osm_node_t *p_opposite_node = osm_physp_get_node_ptr(p_remote_physp);
+
+ if (osm_node_get_type( p_opposite_node ) == IB_NODE_TYPE_CA) {
+ ib_port_info_t *port_info = &p_remote_physp->port_info;
+ ib_net16_t remote_port_lid = port_info->base_lid;
+ if (remote_port_lid == lid_no) {
+ egress_port = i;
+ goto Exit;
+ }
+ }
+ }
+ }
+ }// for
+
+ Exit:
+ return egress_port;
+}
+
+
+static int randint ( int high )
+{
+ int r;
+
+ if (high == 0) return 0;
+ r = rand();
+ high++;
+ return (r%high);
+}
+
+
+/************************************
+
+ CYCLE EXISTS
+
+************************************/
+
+static int cycle_exists(cdg_vertex_t * start, cdg_vertex_t * current,
+ cdg_vertex_t * prev, int visit_num) {
+
+ cdg_vertex_t * h;
+ int i, new_visit_num;
+ int cycle_found = 0;
+
+ if(current!= NULL && current->visiting_number > 0) {
+ if(visit_num > current->visiting_number && current->seen == 0) {
+ h = start;
+ cycle_found = 1;
+ }
+ } else {
+ if(current == NULL) {
+ current = start;
+ assert(prev == NULL);
+ }
+
+ current->visiting_number = visit_num;
+
+ if(prev != NULL) {
+ prev->next = current;
+ assert(prev->to == current->from);
+ assert(prev->visiting_number > 0);
+ }
+
+ new_visit_num = visit_num + 1;
+
+ for(i=0; i<current->num_dependencies; i++) {
+ cycle_found = cycle_exists(start, current->dependency[i], current,
+ new_visit_num);
+ if(cycle_found == 1)
+ i = current->num_dependencies;
+ }
+
+ current->seen = 1;
+ if(prev != NULL)
+ prev->next = NULL;
+ }
+
+ return cycle_found;
+
+
+
+}
+
+
+
+/************************************
+
+ REMOVE SEMIPERMANENTDEPEND FOR SP
+
+************************************/
+
+static void remove_semipermanent_depend_for_sp(lash_t *p_lash, int sw, int dest_switch, int lane)
+{
+ switch_t **switches = p_lash->switches;
+ cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
+ int i_next_switch, output_link, i, next_link, i_next_next_switch, depend =0;
+ cdg_vertex_t * v;
+ int found;
+
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+ i_next_switch = switches[sw]->phys_connections[output_link];
+
+ while(sw != dest_switch){
+ v = cdg_vertex_matrix[lane][sw][i_next_switch];
+ assert(v != NULL);
+
+ if(v->num_using_vertex == 1) {
+
+ cdg_vertex_matrix[lane][sw][i_next_switch] = NULL;
+
+ free(v);
+ } else {
+ v->num_using_vertex--;
+ if(i_next_switch != dest_switch) {
+ next_link = switches[i_next_switch]->routing_table[dest_switch].out_link;
+ i_next_next_switch = switches[i_next_switch]->phys_connections[next_link];
+ found = 0;
+
+ for(i=0; i<v->num_dependencies; i++)
+ if(v->dependency[i] == cdg_vertex_matrix[lane][i_next_switch][i_next_next_switch]) {
+ found = 1;
+ depend = i;
+ }
+
+ assert(found);
+
+ if(v->num_using_this_depend[depend] == 1) {
+ for(i=depend; i<v->num_dependencies-1; i++) {
+ v->dependency[i] = v->dependency[i+1];
+ v->num_using_this_depend[i] = v->num_using_this_depend[i+1];
+ }
+
+ v->num_dependencies--;
+ } else
+ v->num_using_this_depend[depend]--;
+ }
+ }
+
+ sw = i_next_switch;
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+
+ if(sw != dest_switch)
+ i_next_switch = switches[sw]->phys_connections[output_link];
+ }
+}
+
+
+
+
+/************************************
+
+ ENQUEUE
+
+************************************/
+
+
+static void enqueue(lash_t *p_lash, int sw, int dist, int prev)
+{
+ switch_t **switches = p_lash->switches;
+ q_item_t *q_head;
+
+ assert(switches[sw]->q_member == 0);
+ switches[sw]->q_member = 1;
+ switches[sw]->dist = dist;
+ switches[sw]->prev = prev;
+
+ q_head = (q_item_t*) malloc(sizeof(q_item_t));
+ q_head->sw = sw;
+ q_head->next = p_lash->q_head;
+ p_lash->q_head = q_head;
+ p_lash->q_count++;
+}
+
+
+/************************************
+
+ DEQUEUE
+
+************************************/
+
+static void dequeue(lash_t *p_lash, int * sw, int * dist, int * prev)
+{
+ switch_t **switches = p_lash->switches;
+ q_item_t * q_h, * q_min = NULL, * q_prev;
+ int min_dist = MAX_INT;
+
+ q_h = p_lash->q_head;
+
+ while(!(q_h == NULL)) {
+ if (switches[q_h->sw]->dist < min_dist) {
+ min_dist = switches[q_h->sw]->dist;
+ q_min = q_h;
+ }
+
+ q_h = q_h->next;
+ }
+
+ if(q_min == p_lash->q_head)
+ p_lash->q_head = p_lash->q_head->next;
+ else {
+ q_prev = p_lash->q_head;
+ while(!(q_prev->next == q_min))
+ q_prev = q_prev->next;
+
+ q_prev->next = q_min->next;
+ }
+ p_lash->q_count--;
+
+ *sw = q_min->sw;
+ *dist = switches[q_min->sw]->dist;
+ *prev = switches[q_min->sw]->prev;
+
+ assert(switches[q_min->sw]->q_member == 1 && !switches[q_min->sw]->mst_member);
+ switches[q_min->sw]->q_member = 0;
+ free(q_min);
+}
+
+
+/************************************
+
+ GET PHYS CONNECTION
+
+************************************/
+
+static int get_phys_connection(switch_t **switches, int switch_from, int switch_to)
+{
+ int i = 0;
+
+ for (i = 0; i < switches[switch_from]->num_connections; i++)
+ if(switches[switch_from]->phys_connections[i] == switch_to)
+ return i;
+ assert(1==1);
+ return i;
+}
+
+
+/************************************
+
+ SHORTEST PATH
+
+************************************/
+
+static void shortest_path(lash_t *p_lash, int ir, int num_switches)
+{
+ switch_t **switches = p_lash->switches;
+ int sw, dist, prev, i, channel;
+
+ p_lash->q_head = NULL;
+ p_lash->q_count = 0;
+
+ enqueue(p_lash, ir,0,NONE);
+
+ while(p_lash->q_count > 0) {
+ dequeue(p_lash, &sw, &dist, &prev);
+ switches[sw]->mst_member = 1;
+
+ if(prev != NONE) {
+ channel = switches[prev]->used_channels;
+ switches[prev]->dij_channels[channel] = switches[sw];
+ switches[prev]->used_channels++;
+ }
+
+ for(i=0; i<num_switches; i++) {
+ if(p_lash->adj_matrix[sw][i]==1) {
+ if(!switches[i]->mst_member) {
+ if(switches[i]->q_member) {
+ if(dist+1 == switches[i]->dist) {
+ channel = switches[sw]->used_channels;
+ switches[sw]->dij_channels[channel] = switches[i];
+ switches[sw]->used_channels++;
+ } else if(dist+1 < switches[i]->dist) {
+ switches[i]->dist = dist+1;
+ switches[i]->prev = sw;
+ }
+ } else {
+ enqueue(p_lash, i,dist+1,sw);
+ }
+ }
+ }
+ }
+ }
+}
+
+
+
+/************************************
+
+ GENERATE ROUTING FUNC FOR MST
+
+************************************/
+
+static void generate_routing_func_for_mst(lash_t *p_lash, int sw, reachable_dest_t ** destinations)
+{
+ int i, next_switch;
+ switch_t **switches = p_lash->switches;
+ int num_channels = switches[sw]->used_channels;
+ reachable_dest_t * dest, * i_dest, * concat_dest = NULL, * prev;
+
+ for(i=0; i<num_channels; i++) {
+ next_switch = switches[sw]->dij_channels[i]->id;
+ generate_routing_func_for_mst(p_lash, next_switch, &dest);
+
+ i_dest = dest;
+ prev = i_dest;
+
+ while(i_dest != NULL) {
+ if(switches[sw]->routing_table[i_dest->switch_id].out_link == NONE) {
+ switches[sw]->routing_table[i_dest->switch_id].out_link =
+ get_phys_connection(switches, sw, next_switch);
+ }
+
+ prev = i_dest;
+ i_dest = i_dest->next;
+ }
+
+ assert(prev->next == NULL);
+ prev->next = concat_dest;
+ concat_dest = dest;
+ }
+
+ i_dest = (reachable_dest_t*) malloc(sizeof(reachable_dest_t));
+ i_dest->switch_id = sw;
+ i_dest->next = concat_dest;
+ *destinations = i_dest;
+}
+
+
+
+/************************************
+
+ GENERATE CDG FOR SP
+
+************************************/
+
+static void generate_cdg_for_sp(lash_t*p_lash, int sw, int dest_switch, int lane)
+{
+ unsigned num_switches = p_lash->num_switches;
+ switch_t **switches = p_lash->switches;
+ cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
+ int next_switch, output_link, j, exists;
+ cdg_vertex_t * v, * prev = NULL;
+
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+ next_switch = switches[sw]->phys_connections[output_link];
+
+ while(sw != dest_switch) {
+
+ if(cdg_vertex_matrix[lane][sw][next_switch] == NULL) {
+ v = create_cdg_vertex(num_switches);
+
+ int i;
+
+ for(i=0; i<num_switches-1; i++) {
+ v->dependency[i] = NULL;
+ v->num_using_this_depend[i] = 0;
+ }
+
+ v->num_using_vertex = 0;
+ v->num_dependencies = 0;
+ v->from = sw;
+ v->to = next_switch;
+ v->seen = 0;
+ v->visiting_number = 0;
+ v->next = NULL;
+ v->temp = 1;
+ v->num_temp_depend = 0;
+
+ cdg_vertex_matrix[lane][sw][next_switch] = v;
+ } else {
+ v = cdg_vertex_matrix[lane][sw][next_switch];
+ }
+
+ v->num_using_vertex++;
+
+ if(prev!=NULL) {
+ exists = 0;
+
+ for(j=0; j<prev->num_dependencies; j++)
+ if(prev->dependency[j] == v) {
+ exists = 1;
+ prev->num_using_this_depend[j]++;
+ }
+
+ if(exists == 0) {
+ prev->dependency[prev->num_dependencies] = v;
+ prev->num_using_this_depend[prev->num_dependencies]++;
+ prev->num_dependencies++;
+
+ assert(prev->num_dependencies < num_switches);
+
+ if(prev->temp==0)
+ prev->num_temp_depend++;
+
+ }
+ }
+
+ sw = next_switch;
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+
+ if(sw != dest_switch) {
+ assert(output_link != NONE);
+ next_switch = switches[sw]->phys_connections[output_link];
+ }
+
+ prev = v;
+ }
+}
+
+
+
+/************************************
+
+ SET TEMP DEPEND TO PERMANENT FOR SP
+
+************************************/
+
+static void set_temp_depend_to_permanent_for_sp(lash_t *p_lash, int sw, int dest_switch, int lane)
+{
+ switch_t **switches = p_lash->switches;
+ cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
+ int next_switch, output_link;
+ cdg_vertex_t * v;
+
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+ next_switch = switches[sw]->phys_connections[output_link];
+
+ while(sw != dest_switch) {
+ v = cdg_vertex_matrix[lane][sw][next_switch];
+ assert(v != NULL);
+
+ if(v->temp == 1) {
+ v->temp = 0;
+ } else {
+ v->num_temp_depend = 0;
+ }
+
+ sw = next_switch;
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+
+ if(sw != dest_switch)
+ next_switch = switches[sw]->phys_connections[output_link];
+ }
+
+}
+
+
+/************************************
+
+ REMOVE TEMP DEPEND FOR SP
+
+************************************/
+
+static void remove_temp_depend_for_sp(lash_t *p_lash, int sw, int dest_switch, int lane)
+{
+ switch_t **switches =p_lash->switches;
+ cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
+ int next_switch, output_link, i;
+ cdg_vertex_t * v;
+
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+ next_switch = switches[sw]->phys_connections[output_link];
+
+ while(sw != dest_switch) {
+ v = cdg_vertex_matrix[lane][sw][next_switch];
+ assert(v != NULL);
+
+ if(v->temp==1) {
+ cdg_vertex_matrix[lane][sw][next_switch] = NULL;
+ free(v);
+ } else {
+ assert(v->num_temp_depend <= v->num_dependencies);
+ v->num_dependencies = v->num_dependencies - v->num_temp_depend;
+ v->num_temp_depend = 0;
+ v->num_using_vertex--;
+
+ for(i = v->num_dependencies; i<p_lash->num_switches-1; i++)
+ v->num_using_this_depend[i] = 0;
+ }
+
+ sw = next_switch;
+ output_link = switches[sw]->routing_table[dest_switch].out_link;
+
+ if(sw != dest_switch)
+ next_switch = switches[sw]->phys_connections[output_link];
+
+ }
+}
+
+
+/************************************
+
+ BALANCE VIRTUAL LANES
+
+************************************/
+
+static void balance_virtual_lanes(lash_t *p_lash, unsigned lanes_needed)
+{
+ unsigned num_switches = p_lash->num_switches;
+ cdg_vertex_t ****cdg_vertex_matrix = p_lash->cdg_vertex_matrix;
+ int *num_mst_in_lane = p_lash->num_mst_in_lane;
+ int ***virtual_location = p_lash->virtual_location;
+ int min_filled_lane, max_filled_lane, medium_filled_lane, trials;
+ int old_min_filled_lane, old_max_filled_lane, i, j, new_num_min_lane, new_num_max_lane;
+ int src, dest, start, next_switch, output_link;
+ int stop = 0, cycle_found;
+
+ max_filled_lane = 0;
+ min_filled_lane = lanes_needed-1;
+
+ if(max_filled_lane > 1)
+ medium_filled_lane = max_filled_lane-1;
+
+ trials = num_mst_in_lane[max_filled_lane];
+ if(lanes_needed == 1)
+ stop = 1;
+
+ while(stop == 0) {
+ src = abs(rand())%(num_switches);
+ dest = abs(rand())%(num_switches);
+
+ while(virtual_location[src][dest][max_filled_lane] != 1) {
+ start = dest;
+ if(dest == num_switches-1)
+ dest = 0;
+ else
+ dest++;
+
+ while(dest != start && virtual_location[src][dest][max_filled_lane] != 1) {
+ if(dest == num_switches-1)
+ dest = 0;
+ else
+ dest++;
+ }
+
+ if(virtual_location[src][dest][max_filled_lane] != 1) {
+ if(src == num_switches-1)
+ src = 0;
+ else
+ src++;
+ }
+ }
+
+ generate_cdg_for_sp(p_lash, src, dest, min_filled_lane);
+ output_link = p_lash->switches[src]->routing_table[dest].out_link;
+ next_switch = p_lash->switches[src]->phys_connections[output_link];
+
+ assert(cdg_vertex_matrix[min_filled_lane][src][next_switch] != NULL);
+ cycle_found = cycle_exists(cdg_vertex_matrix[min_filled_lane][src][next_switch], NULL, NULL, 1);
+
+ for(i=0; i<num_switches; i++)
+ for(j=0; j<num_switches; j++)
+ if(cdg_vertex_matrix[min_filled_lane][i][j] != NULL) {
+ cdg_vertex_matrix[min_filled_lane][i][j]->visiting_number = 0;
+ cdg_vertex_matrix[min_filled_lane][i][j]->seen = 0;
+ }
+
+ if(cycle_found == 1) {
+ remove_temp_depend_for_sp(p_lash, src, dest, min_filled_lane);
+ virtual_location[src][dest][max_filled_lane] = 2;
+ trials--;
+ } else {
+ set_temp_depend_to_permanent_for_sp(p_lash, src, dest, min_filled_lane);
+ num_mst_in_lane[max_filled_lane]--;
+ num_mst_in_lane[min_filled_lane]++;
+
+ remove_semipermanent_depend_for_sp(p_lash, src, dest,max_filled_lane);
+ virtual_location[src][dest][max_filled_lane] = 0;
+ virtual_location[src][dest][min_filled_lane] = 1;
+ p_lash->switches[src]->routing_table[dest].lane = min_filled_lane;
+ }
+
+ if(trials==0)
+ stop = 1;
+ else {
+ if(num_mst_in_lane[max_filled_lane]-num_mst_in_lane[min_filled_lane] < p_lash->balance_limit)
+ stop = 1;
+ }
+
+ old_min_filled_lane = min_filled_lane;
+ old_max_filled_lane = max_filled_lane;
+
+ new_num_min_lane = MAX_INT;
+ new_num_max_lane = 0;
+
+ for(i=0; i<lanes_needed; i++) {
+
+ if(num_mst_in_lane[i] < new_num_min_lane) {
+ new_num_min_lane = num_mst_in_lane[i];
+ min_filled_lane = i;
+ }
+
+ if(num_mst_in_lane[i] > new_num_max_lane) {
+ new_num_max_lane = num_mst_in_lane[i];
+ max_filled_lane = i;
+ }
+ }
+
+ if(old_min_filled_lane != min_filled_lane) {
+ trials = num_mst_in_lane[max_filled_lane];
+ for(i=0; i<num_switches; i++)
+ for(j=0; j<num_switches; j++)
+ if(virtual_location[i][j][max_filled_lane] == 2)
+ virtual_location[i][j][max_filled_lane] = 1;
+ }
+
+ if(old_max_filled_lane != max_filled_lane) {
+ trials = num_mst_in_lane[max_filled_lane];
+ for(i=0; i<num_switches; i++)
+ for(j=0; j<num_switches; j++)
+ if(virtual_location[i][j][old_max_filled_lane] == 2)
+ virtual_location[i][j][old_max_filled_lane] = 1;
+ }
+ }
+}
+
+
+
+static switch_t *switch_create(lash_t *p_lash, unsigned id, osm_switch_t *p_sw)
+{
+ unsigned num_switches = p_lash->num_switches;
+ switch_t *sw;
+ int i;
+
+ sw = malloc(sizeof(*sw));
+ if (!sw)
+ return NULL;
+
+ memset(sw, 0, sizeof(*sw));
+
+ sw->id = id;
+ sw->dist = MAX_INT;
+
+ sw->dij_channels = malloc((num_switches)*sizeof(switch_t*));
+ if (!sw->dij_channels) {
+ free(sw);
+ return NULL;
+ }
+ memset(sw->dij_channels, 0, (num_switches)*sizeof(switch_t*));
+
+ sw->virtual_physical_port_table = malloc(num_switches*sizeof(int));
+ if (!sw->virtual_physical_port_table) {
+ free(sw->dij_channels);
+ free(sw);
+ return NULL;
+ }
+
+ sw->phys_connections = malloc(num_switches*sizeof(int));
+ if (!sw->phys_connections)
+ return NULL;
+
+ sw->routing_table = malloc(num_switches*sizeof(sw->routing_table[0]));
+ if (!sw->routing_table)
+ return NULL;
+
+ for (i = 0; i < num_switches; i++) {
+ sw->routing_table[i].out_link = NONE;
+ sw->routing_table[i].lane = NONE;
+ sw->virtual_physical_port_table[i] = -1;
+ if(i < num_switches-1)
+ sw->phys_connections[i] = NONE;
+ }
+
+ sw->p_sw = p_sw;
+ if (p_sw)
+ p_sw->priv = sw;
+
+ return sw;
+}
+
+static void switch_delete(switch_t *sw)
+{
+ if (sw->dij_channels)
+ free(sw->dij_channels);
+ if (sw->virtual_physical_port_table)
+ free(sw->virtual_physical_port_table);
+ if (sw->phys_connections)
+ free(sw->phys_connections);
+ if (sw->routing_table)
+ free(sw->routing_table);
+ free(sw);
+}
+
+static void free_lash_structures(lash_t *p_lash)
+{
+ int i,j,k;
+ unsigned num_switches = p_lash->num_switches;
+ osm_log_t *p_log = &p_lash->p_osm->log;
+
+ OSM_LOG_ENTER( p_log, free_lash_structures);
+
+ // free cdg_vertex_matrix
+ for (i = 0; i < p_lash->vl_min; i++) {
+ for (j = 0; j < num_switches; j++) {
+ for (k = 0; k < num_switches; k++) {
+ if (p_lash->cdg_vertex_matrix[i][j][k]) {
+
+ if (p_lash->cdg_vertex_matrix[i][j][k]->dependency)
+ free(p_lash->cdg_vertex_matrix[i][j][k]->dependency);
+
+ if (p_lash->cdg_vertex_matrix[i][j][k]->num_using_this_depend)
+ free(p_lash->cdg_vertex_matrix[i][j][k]->num_using_this_depend);
+
+ free(p_lash->cdg_vertex_matrix[i][j][k]);
+ }
+ }
+ if (p_lash->cdg_vertex_matrix[i][j])
+ free(p_lash->cdg_vertex_matrix[i][j]);
+ }
+ if (p_lash->cdg_vertex_matrix[i])
+ free(p_lash->cdg_vertex_matrix[i]);
+ }
+
+ if (p_lash->cdg_vertex_matrix)
+ free(p_lash->cdg_vertex_matrix);
+
+
+ // free virtual_location
+ for (i = 0; i < num_switches; i++) {
+ for (j = 0; j < num_switches; j++) {
+ if (p_lash->virtual_location[i][j])
+ free(p_lash->virtual_location[i][j]);
+ }
+ if (p_lash->virtual_location[i])
+ free(p_lash->virtual_location[i]);
+ }
+ if (p_lash->virtual_location)
+ free(p_lash->virtual_location);
+
+
+ for (i = 0; i < num_switches; i++) {
+ if (p_lash->adj_matrix[i]) {
+ free(p_lash->adj_matrix[i]);
+ }
+ }
+ free(p_lash->adj_matrix);
+
+ if(p_lash->num_mst_in_lane)
+ free(p_lash->num_mst_in_lane);
+}
+
+
+static int init_lash_structures(lash_t *p_lash)
+{
+ unsigned vl_min = p_lash->vl_min;
+ unsigned num_switches = p_lash->num_switches;
+ osm_log_t *p_log = &p_lash->p_osm->log;
+
+ OSM_LOG_ENTER( p_log, init_lash_structures);
+
+ int status = IB_SUCCESS;
+ int i, j, k;
+
+ // initialise cdg_vertex_matrix[num_switches][num_switches][num_switches]
+ p_lash->cdg_vertex_matrix = (cdg_vertex_t****)malloc(vl_min * sizeof(cdg_vertex_t ****));
+ for (i = 0; i < vl_min; i++) {
+ p_lash->cdg_vertex_matrix[i] =(cdg_vertex_t***) malloc(num_switches * sizeof(cdg_vertex_t ***));
+
+ if (p_lash->cdg_vertex_matrix[i] == NULL)
+ goto Exit_Mem_Error;
+ }
+
+ for (i = 0; i < vl_min; i++) {
+ for (j = 0; j < num_switches; j++) {
+ p_lash->cdg_vertex_matrix[i][j] = (cdg_vertex_t**)malloc(num_switches * sizeof(cdg_vertex_t**));
+ if (p_lash->cdg_vertex_matrix[i][j] == NULL)
+ goto Exit_Mem_Error;
+
+ for (k = 0; k < num_switches; k++) {
+ p_lash->cdg_vertex_matrix[i][j][k] = NULL;
+ }
+ }
+ }
+
+ // initialise virtual_location[num_switches][num_switches][num_layers],
+ // default value = 0
+ p_lash->virtual_location = (int***)malloc(num_switches * sizeof(int ***));
+ if (p_lash->virtual_location == NULL)
+ goto Exit_Mem_Error;
+
+ for (i = 0; i < num_switches; i++) {
+ p_lash->virtual_location[i] =(int**) malloc(num_switches * sizeof(int **));
+ if (p_lash->virtual_location[i] == NULL)
+ goto Exit_Mem_Error;
+ }
+
+ for (i = 0; i < num_switches; i++) {
+ for (j = 0; j < num_switches; j++) {
+ p_lash->virtual_location[i][j] = (int*)malloc(vl_min * sizeof(int*));
+ if (p_lash->virtual_location[i][j] == NULL)
+ goto Exit_Mem_Error;
+ for (k = 0; k < vl_min; k++) {
+ p_lash->virtual_location[i][j][k] = 0;
+ }
+ }
+ }
+
+ // initialise adj_matrix[num_switches][num_switches], default value
+ // = 0
+ p_lash->adj_matrix = (int**)malloc(num_switches * sizeof(int **));
+ if (p_lash->adj_matrix == NULL)
+ goto Exit_Mem_Error;
+
+ for (i = 0; i < num_switches; i++) {
+ p_lash->adj_matrix[i] =(int*) malloc(num_switches * sizeof(int));
+ if (p_lash->adj_matrix[i] == NULL)
+ goto Exit_Mem_Error;
+
+ for (j = 0; j < num_switches; j++)
+ p_lash->adj_matrix[i][j] = 0;
+ }
+
+ // initialise num_mst_in_lane[num_switches], default 0
+ p_lash->num_mst_in_lane = (int*) malloc(num_switches * sizeof(int));;
+ if (p_lash->num_mst_in_lane == NULL)
+ goto Exit_Mem_Error;
+ memset(p_lash->num_mst_in_lane, 0,
+ num_switches*sizeof(p_lash->num_mst_in_lane[0]));
+
+ goto Exit;
+
+ Exit_Mem_Error:
+ status = IB_ERROR;
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash_init_structures (ERROR) "
+ "Could not allocate required memeory for LASH errno = : %d, errno for lack of memory = %d\n",
+ errno, ENOMEM);
+
+ Exit:
+ OSM_LOG_EXIT( p_log );
+ return status;
+}
+
+
+
+
+
+static int lash_core(lash_t *p_lash)
+{
+ osm_log_t *p_log = &p_lash->p_osm->log;
+ unsigned num_switches = p_lash->num_switches;
+ switch_t **switches = p_lash->switches;
+ unsigned lanes_needed = 1;
+ int i, j, k, dest_switch = 0;
+ reachable_dest_t * dests, * idest;
+ int cycle_found = 0;
+ int v_lane, stop = 0, output_link, i_next_switch;
+ int status = IB_SUCCESS;
+
+ OSM_LOG_ENTER( p_log, lash_core);
+
+
+ for(i=0; i<num_switches; i++) {
+ shortest_path(p_lash, i, num_switches);
+ generate_routing_func_for_mst(p_lash, i, &dests);
+
+ idest = dests;
+ while(idest != NULL) {
+ dests = dests->next;
+ free(idest);
+ idest = dests;
+ }
+
+ for(dest_switch=0; dest_switch<num_switches; dest_switch++)
+ if(dest_switch != i) {
+ v_lane = 0;
+ stop = 0;
+ while(v_lane < lanes_needed && stop == 0) {
+ generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
+ output_link = switches[i]->routing_table[dest_switch].out_link;
+ i_next_switch = switches[i]->phys_connections[output_link];
+
+ assert(p_lash->cdg_vertex_matrix[v_lane][i][i_next_switch] != NULL);
+ cycle_found = cycle_exists(p_lash->cdg_vertex_matrix[v_lane][i][i_next_switch], NULL, NULL, 1);
+
+ for(j=0; j<num_switches; j++)
+ for(k=0; k<num_switches; k++)
+ if(p_lash->cdg_vertex_matrix[v_lane][j][k] != NULL) {
+ p_lash->cdg_vertex_matrix[v_lane][j][k]->visiting_number = 0;
+ p_lash->cdg_vertex_matrix[v_lane][j][k]->seen = 0;
+ }
+
+ if(cycle_found == 1) {
+ remove_temp_depend_for_sp(p_lash, i, dest_switch, v_lane);
+ v_lane++;
+ } else {
+ set_temp_depend_to_permanent_for_sp(p_lash, i, dest_switch, v_lane);
+ stop = 1;
+ p_lash->num_mst_in_lane[v_lane]++;
+ }
+ }
+
+ switches[i]->routing_table[dest_switch].lane = v_lane;
+
+ if(cycle_found == 1) {
+ generate_cdg_for_sp(p_lash, i, dest_switch, v_lane);
+ set_temp_depend_to_permanent_for_sp(p_lash, i, dest_switch, v_lane);
+
+ if (lanes_needed + 1 > p_lash->vl_min){
+ lanes_needed++;
+ goto Error_Not_Enough_Lanes;
+ }
+ else
+ lanes_needed++;
+
+ // goto error exit with message
+ p_lash->num_mst_in_lane[v_lane]++;
+ }
+ p_lash->virtual_location[i][dest_switch][v_lane] = 1;
+ }
+
+ for(j=0; j<num_switches; j++) {
+ for(k=0; k<num_switches; k++)
+ switches[j]->dij_channels[k] = NULL;
+ switches[j]->used_channels = 0;
+ switches[j]->mst_member = switches[j]->q_member = 0;
+ switches[j]->dist = MAX_INT;
+ }
+ }
+
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash_core: "
+ "Lanes needed: %d, Balancing\n", lanes_needed);
+
+ for(i = 0; i<lanes_needed; i++) {
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash_core: "
+ "Lanes in layer %d: %d\n",i, p_lash->num_mst_in_lane[i]);
+ }
+
+ balance_virtual_lanes(p_lash, lanes_needed);
+
+
+ for(i = 0; i<lanes_needed; i++) {
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash_core: "
+ "Lanes in layer %d: %d\n",i, p_lash->num_mst_in_lane[i]);
+ }
+
+ goto Exit;
+
+ Error_Not_Enough_Lanes:
+ status = IB_ERROR;
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash_core (ERROR): "
+ "Lane requirements (%d) exceeds available lanes (%d)\n",
+ p_lash->vl_min, lanes_needed);
+ Exit:
+ OSM_LOG_EXIT( p_log );
+ return status;
+}
+
+
+static unsigned get_lash_id(osm_switch_t *p_sw)
+{
+ return ((switch_t *)p_sw->priv)->id;
+}
+
+
+static void populate_fwd_tbls(lash_t *p_lash)
+{
+ osm_log_t *p_log = &p_lash->p_osm->log;
+ osm_subn_t *p_subn = &p_lash->p_osm->subn;
+ osm_opensm_t *p_osm = p_lash->p_osm;
+ osm_switch_t *p_sw, *p_next_sw, *p_dst_sw;
+ uint16_t max_lid_ho, lid = 0;
+
+ OSM_LOG_ENTER( p_log, populate_fwd_tbls );
+
+ p_next_sw = (osm_switch_t*)cl_qmap_head( &p_subn->sw_guid_tbl );
+
+ // Go through each swtich individually
+ while(p_next_sw != (osm_switch_t*)cl_qmap_end( &p_subn->sw_guid_tbl )) {
+ p_sw = p_next_sw;
+ p_next_sw = (osm_switch_t*)cl_qmap_next( &p_sw->map_item );
+
+ max_lid_ho = osm_switch_get_max_lid_ho(p_sw);
+ uint64_t current_guid = p_sw->p_node->node_info.port_guid;
+ switch_t *sw = p_sw->priv;
+
+ memset(p_osm->sm.ucast_mgr.lft_buf, 0xff, IB_LID_UCAST_END_HO + 1);
+
+ for (lid = 1; lid <= max_lid_ho; lid++) {
+ p_dst_sw = get_osm_switch_from_lid(p_lash->p_osm, lid);
+
+ if (p_dst_sw == NULL) {
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "populate_fwd_tbls: ERROR "
+ "LASH fwd NULL Cannot find GUID 0x%016" PRIx64
+ " src lash id (%d), src lid no(0x%04X)\n",
+ cl_ntoh64(current_guid), sw->id, lid);
+ } else if (p_dst_sw == p_sw) {
+ uint8_t egress_port = find_port_from_lid(cl_hton16(lid), p_sw);
+ p_osm->sm.ucast_mgr.lft_buf[lid] = egress_port;
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "populate_fwd_tbls: "
+ "LASH fwd MY SRC SRC GUID 0x%016" PRIx64
+ " src lash id (%d), src lid no(0x%04X) src lash port (%d) "
+ "DST GUID 0x%016" PRIx64 " src lash id (%d), src lash port (%d)\n",
+ cl_ntoh64(current_guid), -1, lid, egress_port,
+ cl_ntoh64(current_guid), -1, egress_port);
+ } else {
+ unsigned dst_lash_switch_id = get_lash_id(p_dst_sw);
+ uint8_t lash_egress_port = sw->routing_table[dst_lash_switch_id].out_link;
+ uint8_t physical_egress_port = sw->virtual_physical_port_table[lash_egress_port];
+
+ p_osm->sm.ucast_mgr.lft_buf[lid] = physical_egress_port;
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "populate_fwd_tbls: "
+ "LASH fwd SRC GUID 0x%016" PRIx64 " src lash id (%d), "
+ "src lid no( 0x%04X ) src lash port (%d) "
+ "DST GUID 0x%016" PRIx64 " src lash id (%d), src lash port (%d)\n",
+ cl_ntoh64(current_guid), sw->id, lid, lash_egress_port,
+ cl_ntoh64(p_dst_sw->p_node->node_info.port_guid),
+ dst_lash_switch_id, physical_egress_port);
+ }
+ } // for
+ osm_ucast_mgr_set_fwd_table(&p_osm->sm.ucast_mgr, p_sw);
+ }
+ OSM_LOG_EXIT( p_log );
+}
+
+
+
+static void print_fwd_table(IN const osm_switch_t *p_sw)
+{
+ uint16_t max_lid_ho, lid_ho;
+ uint64_t switch_guid = osm_lash_get_switch_guid(p_sw);
+
+ max_lid_ho = osm_switch_get_max_lid_ho(p_sw);
+ printf("FWDTBL: 0x%016" PRIx64 " max LID 0x%04X\n", cl_ntoh64(switch_guid), max_lid_ho);
+
+ // starting at 1, not 0. Assuming no LID with an ID of 0
+ for (lid_ho = 1; lid_ho <= max_lid_ho; lid_ho++) {
+ uint8_t port_num = osm_switch_get_port_by_lid( p_sw, lid_ho );
+
+ if (port_num == OSM_NO_PATH)
+ printf("0x%04X : UNREACHABLE\n", lid_ho);
+ else
+ printf("0x%04X : %d \n", lid_ho, port_num);
+ }
+ printf("\n");
+}
+
+
+
+
+static void print_fwd_tables(lash_t *p_lash)
+{
+ osm_subn_t *p_subn = &p_lash->p_osm->subn;
+ osm_switch_t *p_next_sw, *p_sw;
+
+ p_next_sw = (osm_switch_t*)cl_qmap_head( &p_subn->sw_guid_tbl );
+ while(p_next_sw != (osm_switch_t*)cl_qmap_end( &p_subn->sw_guid_tbl ) ) {
+ p_sw = p_next_sw;
+ p_next_sw = (osm_switch_t*)cl_qmap_next( &p_sw->map_item );
+
+ if (p_sw && p_sw->p_node) {
+ print_fwd_table(p_sw);
+ }
+ }
+}
+
+
+static void osm_lash_process_switch(lash_t *p_lash, osm_switch_t *p_sw)
+{
+ osm_log_t *p_log = &p_lash->p_osm->log;
+ int i, port_count;
+ osm_physp_t *p_current_physp, *p_remote_physp;
+ unsigned switch_a_lash_id, switch_b_lash_id;
+
+ OSM_LOG_ENTER(p_log, _osm_lash_process_switch);
+
+ switch_a_lash_id = get_lash_id(p_sw);
+ port_count = osm_node_get_num_physp(p_sw->p_node);
+
+ // starting at port 1, ignoring management port on switch
+ for (i=1; i<port_count; i++) {
+
+ p_current_physp = osm_node_get_physp_ptr(p_sw->p_node, i);
+
+ if (osm_physp_is_valid (p_current_physp)) {
+ p_remote_physp = p_current_physp->p_remote_physp;
+
+ if (p_remote_physp && osm_physp_is_valid ( p_remote_physp ) &&
+ p_remote_physp->p_node->sw) {
+ int physical_port_a_num = osm_physp_get_port_num(p_current_physp);
+ int physical_port_b_num = osm_physp_get_port_num(p_remote_physp);
+ switch_b_lash_id = get_lash_id(p_remote_physp->p_node->sw);
+
+ if(connect_switches(p_lash, switch_a_lash_id, switch_b_lash_id,
+ physical_port_a_num, physical_port_b_num) == TRUE) {
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "osm_lash_process_switch: "
+ "LASH SUCSESS connected G 0x%016" PRIx64
+ " , lash_id(%u), P(%u) "
+ " to G 0x%016" PRIx64 " , lash_id(%u) , P(%u)\n",
+ cl_ntoh64(osm_physp_get_port_guid(p_current_physp)),
+ switch_a_lash_id, physical_port_a_num,
+ cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
+ switch_b_lash_id, physical_port_b_num);
+ }
+ }
+ }
+ }
+
+ OSM_LOG_EXIT(p_log);
+}
+
+
+static void lash_cleanup(lash_t *p_lash)
+{
+ osm_subn_t *p_subn = &p_lash->p_osm->subn;
+ osm_switch_t *p_next_sw, *p_sw;
+
+ /* drop any existing references to old lash switches */
+ p_next_sw = (osm_switch_t*)cl_qmap_head( &p_subn->sw_guid_tbl );
+ while (p_next_sw != (osm_switch_t*)cl_qmap_end(&p_subn->sw_guid_tbl)) {
+ p_sw = p_next_sw;
+ p_next_sw = (osm_switch_t*)cl_qmap_next(&p_sw->map_item);
+ p_sw->priv = NULL;
+ }
+
+ if (p_lash->switches) {
+ unsigned id;
+ for (id = 0; id < p_lash->num_switches ; id++)
+ if (p_lash->switches[id])
+ switch_delete(p_lash->switches[id]);
+ free(p_lash->switches);
+ }
+ p_lash->switches = NULL;
+}
+
+/*
+ static int discover_network_properties()
+ Traverse the topology of the network in order to determine
+ - the maximum number of switches,
+ - the minimum number of virtual layers
+*/
+
+static int discover_network_properties(lash_t *p_lash)
+{
+ int i = 0, id = 0;
+ uint8_t vl_min;
+ osm_subn_t *p_subn = &p_lash->p_osm->subn;
+ osm_switch_t *p_next_sw, *p_sw;
+ osm_log_t *p_log = &p_lash->p_osm->log;
+
+ p_lash->num_switches = cl_qmap_count(&p_subn->sw_guid_tbl);
+
+ p_lash->switches = malloc(p_lash->num_switches*sizeof(switch_t *));
+ if (!p_lash->switches)
+ return -1;
+ memset(p_lash->switches, 0, p_lash->num_switches*sizeof(switch_t *));
+
+ vl_min = 128; // set to a high value
+
+ p_next_sw = (osm_switch_t*)cl_qmap_head( &p_subn->sw_guid_tbl );
+ while(p_next_sw != (osm_switch_t*)cl_qmap_end( &p_subn->sw_guid_tbl ) ) {
+ p_sw = p_next_sw;
+ p_next_sw = (osm_switch_t*)cl_qmap_next( &p_sw->map_item );
+
+ p_lash->switches[id] = switch_create(p_lash, id, p_sw);
+ if (!p_lash->switches[id])
+ return -1;
+ id++;
+
+ uint16_t port_count = osm_node_get_num_physp (p_sw->p_node);
+
+ /// Note, ignoring port 0. management port
+ for (i=1; i<port_count; i++) {
+ osm_physp_t *p_current_physp = osm_node_get_physp_ptr(p_sw->p_node, i);
+
+ if (p_current_physp && osm_physp_is_valid (p_current_physp) &&
+ p_current_physp->p_remote_physp) {
+
+ ib_port_info_t *p_port_info = &p_current_physp->port_info;
+ int port_vl_min = ib_port_info_get_op_vls(p_port_info);
+ if (port_vl_min && port_vl_min < vl_min)
+ vl_min = port_vl_min;
+ }
+ } // for
+ } // while
+
+ vl_min = 1 << (vl_min - 1);
+ if (vl_min > 15) vl_min = 15;
+
+ p_lash->vl_min = vl_min;
+
+ osm_log(p_log, OSM_LOG_DEBUG,
+ "lash discover_network_properties: "
+ "min operatioanl vl(%d) max_switches(%d)\n",
+ p_lash->vl_min, p_lash->num_switches);
+ return 0;
+}
+
+
+static void process_switches(lash_t *p_lash)
+{
+ osm_switch_t *p_sw, *p_next_sw;
+ osm_subn_t *p_subn = &p_lash->p_osm->subn;
+
+ /* Go through each swithc and process it. i.e build the connection
+ strucure required by LASH */
+ p_next_sw = (osm_switch_t*)cl_qmap_head( &p_subn->sw_guid_tbl );
+ while(p_next_sw != (osm_switch_t*)cl_qmap_end( &p_subn->sw_guid_tbl ) ) {
+ p_sw = p_next_sw;
+ p_next_sw = (osm_switch_t*)cl_qmap_next( &p_sw->map_item );
+
+ osm_lash_process_switch(p_lash, p_sw);
+ }
+}
+
+
+static int lash_process(void *context)
+{
+ lash_t *p_lash = context;
+ osm_log_t *p_log = &p_lash->p_osm->log;
+ int return_status = IB_SUCCESS;
+
+ OSM_LOG_ENTER(p_log, lash_process);
+ p_lash->balance_limit = 3;
+
+ // everything starts here
+ lash_cleanup(p_lash);
+
+ discover_network_properties(p_lash);
+
+ return_status = init_lash_structures(p_lash);
+ if (return_status != IB_SUCCESS)
+ goto Exit;
+
+ process_switches(p_lash);
+
+ return_status = lash_core(p_lash);
+ if (return_status != IB_SUCCESS)
+ goto Exit;
+
+ populate_fwd_tbls(p_lash);
+ print_fwd_tables(p_lash);
+
+ Exit:
+ free_lash_structures(p_lash);
+ OSM_LOG_EXIT(p_log);
+
+ return return_status;
+}
+
+static lash_t* lash_create(osm_opensm_t *p_osm)
+{
+ lash_t* p_lash;
+
+ p_lash = malloc(sizeof(lash_t));
+ if (!p_lash)
+ return NULL;
+
+ memset(p_lash, 0, sizeof(lash_t));
+ p_lash->p_osm = p_osm;
+
+ return(p_lash);
+}
+
+static void lash_delete(void *context)
+{
+ lash_t *p_lash = context;
+ if (p_lash->switches) {
+ unsigned id;
+ for (id = 0; id < p_lash->num_switches ; id++)
+ if (p_lash->switches[id])
+ switch_delete(p_lash->switches[id]);
+ free(p_lash->switches);
+ }
+ free(p_lash);
+}
+
+uint8_t osm_get_lash_sl(osm_opensm_t *p_osm,
+ osm_port_t *p_src_port, osm_port_t *p_dst_port)
+{
+ unsigned dst_id;
+ osm_switch_t *p_sw;
+
+ if (p_osm->routing_engine.ucast_build_fwd_tables != lash_process)
+ return OSM_DEFAULT_SL;
+
+ p_sw = get_osm_switch_from_port(p_dst_port);
+ if (!p_sw)
+ return OSM_DEFAULT_SL;
+ dst_id = get_lash_id(p_sw);
+
+ p_sw = get_osm_switch_from_port(p_src_port);
+ if (!p_sw || !p_sw->priv)
+ return OSM_DEFAULT_SL;
+
+ return ((switch_t *)p_sw->priv)->routing_table[dst_id].lane;
+}
+
+int osm_ucast_lash_setup(osm_opensm_t * p_osm)
+{
+ lash_t *p_lash = lash_create(p_osm);
+ if (!p_lash)
+ return -1;
+
+ p_osm->routing_engine.context = p_lash;
+ p_osm->routing_engine.ucast_build_fwd_tables = lash_process;
+ p_osm->routing_engine.delete = lash_delete;
+
+ return 0;
+}
--
1.5.0.rc2.g11a3
More information about the general
mailing list