[openib-general] [PATCH] opensm: faster min hops
Sasha Khapyorsky
sashak at voltaire.com
Sun Feb 25 13:48:45 PST 2007
After gprof output analyzing, I noticed that current lmx (switch's lid
matrix) implementation is extremely slow. This simple hops matrix
reimplementation makes lid matrices build process two times faster.
Signed-off-by: Sasha Khapyorsky <sashak at voltaire.com>
---
osm/include/opensm/osm_port_profile.h | 1 -
osm/include/opensm/osm_router.h | 1 -
osm/include/opensm/osm_switch.h | 182 ++++++++-------------------------
osm/opensm/osm_switch.c | 115 ++++++++++++++-------
osm/opensm/osm_ucast_ftree.c | 3 -
osm/opensm/osm_ucast_mgr.c | 16 +--
osm/opensm/osm_ucast_updn.c | 2 +-
7 files changed, 124 insertions(+), 196 deletions(-)
diff --git a/osm/include/opensm/osm_port_profile.h b/osm/include/opensm/osm_port_profile.h
index 952393d..a07b057 100644
--- a/osm/include/opensm/osm_port_profile.h
+++ b/osm/include/opensm/osm_port_profile.h
@@ -55,7 +55,6 @@
#include <opensm/osm_subnet.h>
#include <opensm/osm_node.h>
#include <opensm/osm_port.h>
-#include <opensm/osm_matrix.h>
#include <opensm/osm_fwd_tbl.h>
#include <opensm/osm_mcast_tbl.h>
diff --git a/osm/include/opensm/osm_router.h b/osm/include/opensm/osm_router.h
index 168ce77..63c7566 100644
--- a/osm/include/opensm/osm_router.h
+++ b/osm/include/opensm/osm_router.h
@@ -52,7 +52,6 @@
#include <opensm/osm_madw.h>
#include <opensm/osm_node.h>
#include <opensm/osm_port.h>
-#include <opensm/osm_matrix.h>
#include <opensm/osm_fwd_tbl.h>
#include <opensm/osm_mcast_tbl.h>
#include <opensm/osm_port_profile.h>
diff --git a/osm/include/opensm/osm_switch.h b/osm/include/opensm/osm_switch.h
index 053b18a..19381f8 100644
--- a/osm/include/opensm/osm_switch.h
+++ b/osm/include/opensm/osm_switch.h
@@ -53,7 +53,6 @@
#include <opensm/osm_madw.h>
#include <opensm/osm_node.h>
#include <opensm/osm_port.h>
-#include <opensm/osm_matrix.h>
#include <opensm/osm_fwd_tbl.h>
#include <opensm/osm_mcast_tbl.h>
#include <opensm/osm_port_profile.h>
@@ -105,10 +104,12 @@ typedef struct _osm_switch
cl_map_item_t map_item;
osm_node_t *p_node;
ib_switch_info_t switch_info;
- osm_fwd_tbl_t fwd_tbl;
- osm_lid_matrix_t lmx;
uint16_t max_lid_ho;
+ unsigned num_ports;
+ unsigned num_hops;
+ uint8_t **hops;
osm_port_profile_t *p_prof;
+ osm_fwd_tbl_t fwd_tbl;
osm_mcast_tbl_t mcast_tbl;
uint32_t discovery_count;
void *priv;
@@ -124,19 +125,25 @@ typedef struct _osm_switch
* switch_info
* IBA defined SwitchInfo structure for this switch.
*
-* fwd_tbl
-* This switch's forwarding table.
+* max_lid_ho
+* Max LID that is accessible from this switch.
+*
+* num_ports
+* Number of ports for this switch.
*
-* lmx
+* num_hops
+* Size of hops table for this switch.
+*
+* hops
* LID Matrix for this switch containing the hop count
* to every LID from every port.
*
-* max_lid_ho
-* Max LID that is accessible from this switch.
-*
-* p_pro
+* p_prof
* Pointer to array of Port Profile objects for this switch.
*
+* fwd_tbl
+* This switch's forwarding table.
+*
* mcast_tbl
* Multicast forwarding table for this switch.
*
@@ -149,70 +156,9 @@ typedef struct _osm_switch
* Switch object
*********/
-/****f* OpenSM: Switch/osm_switch_construct
+/****f* OpenSM: Switch/osm_switch_delete
* NAME
-* osm_switch_construct
-*
-* DESCRIPTION
-* This function constructs a Switch object.
-*
-* SYNOPSIS
-*/
-void
-osm_switch_construct(
- IN osm_switch_t* const p_sw );
-/*
-* PARAMETERS
-* p_sw
-* [in] Pointer to a Switch object to construct.
-*
-* RETURN VALUE
-* This function does not return a value.
-*
-* NOTES
-* Allows calling osm_switch_init, and osm_switch_destroy.
-*
-* Calling osm_switch_construct is a prerequisite to calling any other
-* method except osm_switch_init.
-*
-* SEE ALSO
-* Switch object, osm_switch_init, osm_switch_destroy
-*********/
-
-/****f* OpenSM: Switch/osm_switch_destroy
-* NAME
-* osm_switch_destroy
-*
-* DESCRIPTION
-* The osm_switch_destroy function destroys the object, releasing
-* all resources.
-*
-* SYNOPSIS
-*/
-void
-osm_switch_destroy(
- IN osm_switch_t* const p_sw );
-/*
-* PARAMETERS
-* p_sw
-* [in] Pointer to the object to destroy.
-*
-* RETURN VALUE
-* None.
-*
-* NOTES
-* Performs any necessary cleanup of the specified object.
-* Further operations should not be attempted on the destroyed object.
-* This function should only be called after a call to osm_switch_construct
-* or osm_switch_init.
-*
-* SEE ALSO
-* Switch object, osm_switch_construct, osm_switch_init
-*********/
-
-/****f* OpenSM: Switch/osm_switch_destroy
-* NAME
-* osm_switch_destroy
+* osm_switch_delete
*
* DESCRIPTION
* Destroys and deallocates the object.
@@ -236,42 +182,6 @@ osm_switch_delete(
* Switch object, osm_switch_construct, osm_switch_init
*********/
-/****f* OpenSM: Switch/osm_switch_init
-* NAME
-* osm_switch_init
-*
-* DESCRIPTION
-* The osm_switch_init function initializes a Switch object for use.
-*
-* SYNOPSIS
-*/
-ib_api_status_t
-osm_switch_init(
- IN osm_switch_t* const p_sw,
- IN osm_node_t* const p_node,
- IN const osm_madw_t* const p_madw );
-/*
-* PARAMETERS
-* p_sw
-* [in] Pointer to an osm_switch_t object to initialize.
-*
-* p_node
-* [in] Pointer to the node object of this switch
-*
-* p_madw
-* [in] Pointer to the MAD Wrapper containing the switch's
-* SwitchInfo attribute.
-*
-* RETURN VALUES
-* IB_SUCCESS if the Switch object was initialized successfully.
-*
-* NOTES
-* Allows calling other node methods.
-*
-* SEE ALSO
-* Switch object, osm_switch_construct, osm_switch_destroy
-*********/
-
/****f* OpenSM: Switch/osm_switch_new
* NAME
* osm_switch_new
@@ -317,8 +227,9 @@ static inline boolean_t
osm_switch_is_leaf_lid(
IN const osm_switch_t* const p_sw,
IN const uint16_t lid_ho )
-{
- return( osm_lid_matrix_get_least_hops( &p_sw->lmx, lid_ho ) <= 1 );
+{
+ return (lid_ho > p_sw->max_lid_ho || !p_sw->hops[lid_ho]) ? FALSE :
+ (p_sw->hops[lid_ho][0] <= 1);
}
/*
* PARAMETERS
@@ -353,7 +264,8 @@ osm_switch_get_hop_count(
IN const uint16_t lid_ho,
IN const uint8_t port_num )
{
- return( osm_lid_matrix_get( &p_sw->lmx, lid_ho, port_num ) );
+ return (lid_ho > p_sw->max_lid_ho || !p_sw->hops[lid_ho]) ?
+ OSM_NO_PATH : p_sw->hops[lid_ho][port_num];
}
/*
* PARAMETERS
@@ -411,15 +323,12 @@ osm_switch_get_fwd_tbl_ptr(
*
* SYNOPSIS
*/
-static inline cl_status_t
+cl_status_t
osm_switch_set_hops(
IN osm_switch_t* const p_sw,
IN const uint16_t lid_ho,
IN const uint8_t port_num,
- IN const uint8_t num_hops )
-{
- return( osm_lid_matrix_set( &p_sw->lmx, lid_ho, port_num, num_hops ) );
-}
+ IN const uint8_t num_hops );
/*
* PARAMETERS
* p_sw
@@ -442,35 +351,23 @@ osm_switch_set_hops(
* SEE ALSO
*********/
-/****f* OpenSM: Switch/osm_switch_set_min_lid_size
+/****f* OpenSM: Switch/osm_switch_hops_clear
* NAME
-* osm_switch_set_min_lid_size
+* osm_switch_hops_clear
*
* DESCRIPTION
-* Sets the size of the switch's routing table to at least accomodate the
-* specified LID value (host ordered)
+* Cleanup existing hops tables (lid matrix)
*
* SYNOPSIS
*/
-static inline cl_status_t
-osm_switch_set_min_lid_size(
- IN osm_switch_t* const p_sw,
- IN const uint16_t lid_ho )
-{
- return( osm_lid_matrix_set_min_lid_size( &p_sw->lmx, lid_ho ) );
-}
+void
+osm_switch_hops_clear(
+ IN osm_switch_t *p_sw );
/*
* PARAMETERS
* p_sw
* [in] Pointer to a Switch object.
*
-* lid_ho
-* [in] LID value (host order) for which to set the count.
-*
-* RETURN VALUES
-* Sets the size of the switch's routing table to at least accomodate the
-* specified LID value (host ordered)
-*
* NOTES
*
* SEE ALSO
@@ -491,7 +388,8 @@ osm_switch_get_least_hops(
IN const osm_switch_t* const p_sw,
IN const uint16_t lid_ho )
{
- return( osm_lid_matrix_get_least_hops( &p_sw->lmx, lid_ho ) );
+ return (lid_ho > p_sw->max_lid_ho || !p_sw->hops[lid_ho]) ?
+ OSM_NO_PATH : p_sw->hops[lid_ho][0];
}
/*
* PARAMETERS
@@ -768,9 +666,7 @@ static inline uint16_t
osm_switch_get_max_lid_ho(
IN const osm_switch_t* const p_sw )
{
- if (p_sw->max_lid_ho != 0)
- return p_sw->max_lid_ho;
- return( osm_lid_matrix_get_max_lid_ho( &p_sw->lmx ) );
+ return p_sw->max_lid_ho;
}
/*
* PARAMETERS
@@ -799,7 +695,7 @@ static inline uint8_t
osm_switch_get_num_ports(
IN const osm_switch_t* const p_sw )
{
- return( osm_lid_matrix_get_num_ports( &p_sw->lmx ) );
+ return p_sw->num_ports;
}
/*
* PARAMETERS
@@ -1348,12 +1244,16 @@ osm_switch_path_count_get(
*/
void
osm_switch_prepare_path_rebuild(
- IN osm_switch_t* const p_sw );
+ IN osm_switch_t* p_sw,
+ IN uint16_t max_lids );
/*
* PARAMETERS
* p_sw
* [in] Pointer to the Switch object.
*
+* max_lids
+* [in] Max number of lids in the subnet.
+*
* RETURN VALUE
* None.
*
diff --git a/osm/opensm/osm_switch.c b/osm/opensm/osm_switch.c
index 8e7728b..7c57398 100644
--- a/osm/opensm/osm_switch.c
+++ b/osm/opensm/osm_switch.c
@@ -55,20 +55,34 @@
#include <iba/ib_types.h>
#include <opensm/osm_switch.h>
+cl_status_t
/**********************************************************************
**********************************************************************/
-void
-osm_switch_construct(
- IN osm_switch_t* const p_sw )
+osm_switch_set_hops(
+ IN osm_switch_t* const p_sw,
+ IN const uint16_t lid_ho,
+ IN const uint8_t port_num,
+ IN const uint8_t num_hops )
{
- CL_ASSERT( p_sw );
- memset( p_sw, 0, sizeof(*p_sw) );
- osm_lid_matrix_construct( &p_sw->lmx );
+ if (lid_ho > p_sw->max_lid_ho)
+ return -1;
+ if (!p_sw->hops[lid_ho]) {
+ p_sw->hops[lid_ho] = malloc(p_sw->num_ports);
+ if (!p_sw->hops[lid_ho])
+ return -1;
+ memset(p_sw->hops[lid_ho], 0xff, p_sw->num_ports);
+ }
+
+ p_sw->hops[lid_ho][port_num] = num_hops;
+ if (p_sw->hops[lid_ho][0] > num_hops)
+ p_sw->hops[lid_ho][0] = num_hops;
+
+ return 0;
}
/**********************************************************************
**********************************************************************/
-ib_api_status_t
+static ib_api_status_t
osm_switch_init(
IN osm_switch_t* const p_sw,
IN osm_node_t* const p_node,
@@ -80,12 +94,6 @@ osm_switch_init(
uint8_t num_ports;
uint32_t port_num;
- CL_ASSERT( p_sw );
- CL_ASSERT( p_madw );
- CL_ASSERT( p_node );
-
- osm_switch_construct( p_sw );
-
p_smp = osm_madw_get_smp_ptr( p_madw );
p_si = (ib_switch_info_t*)ib_smp_get_payload_ptr( p_smp );
num_ports = osm_node_get_num_physp( p_node );
@@ -94,10 +102,7 @@ osm_switch_init(
p_sw->p_node = p_node;
p_sw->switch_info = *p_si;
-
- status = osm_lid_matrix_init( &p_sw->lmx, num_ports );
- if( status != IB_SUCCESS )
- goto Exit;
+ p_sw->num_ports = num_ports;
status = osm_fwd_tbl_init( &p_sw->fwd_tbl, p_si );
if( status != IB_SUCCESS )
@@ -127,23 +132,20 @@ osm_switch_init(
/**********************************************************************
**********************************************************************/
void
-osm_switch_destroy(
- IN osm_switch_t* const p_sw )
+osm_switch_delete(
+ IN OUT osm_switch_t** const pp_sw )
{
- /* free memory to avoid leaks */
+ osm_switch_t *p_sw = *pp_sw;
+ unsigned i;
osm_mcast_tbl_destroy( &p_sw->mcast_tbl );
free( p_sw->p_prof );
osm_fwd_tbl_destroy( &p_sw->fwd_tbl );
- osm_lid_matrix_destroy( &p_sw->lmx );
-}
-
-/**********************************************************************
- **********************************************************************/
-void
-osm_switch_delete(
- IN OUT osm_switch_t** const pp_sw )
-{
- osm_switch_destroy( *pp_sw );
+ if (p_sw->hops) {
+ for (i = 0 ; i < p_sw->num_hops ; i++)
+ if (p_sw->hops[i])
+ free(p_sw->hops[i]);
+ free(p_sw->hops);
+ }
free( *pp_sw );
*pp_sw = NULL;
}
@@ -158,6 +160,9 @@ osm_switch_new(
ib_api_status_t status;
osm_switch_t *p_sw;
+ CL_ASSERT( p_madw );
+ CL_ASSERT( p_node );
+
p_sw = (osm_switch_t*)malloc( sizeof(*p_sw) );
if( p_sw )
{
@@ -322,6 +327,9 @@ osm_switch_recommend_path(
}
}
+ if (osm_node_get_base_lid(p_sw->p_node, 0) == cl_hton16(lid_ho))
+ return 0;
+
/*
This algorithm selects a port based on a static load balanced
selection across equal hop-count ports.
@@ -337,7 +345,7 @@ osm_switch_recommend_path(
*/
/* port number starts with zero and num_ports is 1 + num phys ports */
- for ( port_num = 0; port_num < num_ports; port_num++ )
+ for ( port_num = 1; port_num < num_ports; port_num++ )
{
if ( osm_switch_get_hop_count( p_sw, lid_ho, port_num ) == least_hops)
{
@@ -466,16 +474,45 @@ osm_switch_recommend_path(
/**********************************************************************
**********************************************************************/
void
-osm_switch_prepare_path_rebuild(
- IN osm_switch_t* const p_sw )
+osm_switch_hops_clear(
+ IN osm_switch_t *p_sw )
{
- uint8_t port_num;
- uint8_t num_ports;
+ unsigned i;
+ for (i = 0 ; i < p_sw->num_hops ; i++)
+ if (p_sw->hops[i])
+ memset(p_sw->hops[i], 0xff, p_sw->num_ports);
+}
- num_ports = osm_switch_get_num_ports( p_sw );
- osm_lid_matrix_clear( &p_sw->lmx );
- for( port_num = 0; port_num < num_ports; port_num++ )
- osm_port_prof_construct( &p_sw->p_prof[port_num] );
+/**********************************************************************
+ **********************************************************************/
+void
+osm_switch_prepare_path_rebuild(
+ IN osm_switch_t* p_sw,
+ IN uint16_t max_lids )
+{
+ unsigned i;
+
+ for( i = 0; i < p_sw->num_ports; i++ )
+ osm_port_prof_construct( &p_sw->p_prof[i] );
+ if (!p_sw->hops) {
+ p_sw->hops = malloc((max_lids + 1)*sizeof(p_sw->hops[0]));
+ if (!p_sw->hops)
+ return;
+ memset(p_sw->hops, 0, (max_lids + 1)*sizeof(p_sw->hops[0]));
+ p_sw->num_hops = max_lids + 1;
+ }
+ else if (max_lids + 1 > p_sw->num_hops) {
+ uint8_t **old_hops = p_sw->hops;
+ p_sw->hops = malloc((max_lids + 1)*sizeof(p_sw->hops[0]));
+ if (!p_sw->hops)
+ return;
+ memcpy(p_sw->hops, old_hops, p_sw->num_hops*sizeof(p_sw->hops[0]));
+ memset(p_sw->hops + p_sw->num_hops, 0,
+ (max_lids + 1 - p_sw->num_hops)*sizeof(p_sw->hops[0]));
+ p_sw->num_hops = max_lids + 1;
+ free(old_hops);
+ }
+ p_sw->max_lid_ho = max_lids;
}
/**********************************************************************
diff --git a/osm/opensm/osm_ucast_ftree.c b/osm/opensm/osm_ucast_ftree.c
index 21aa4a8..61db1d7 100644
--- a/osm/opensm/osm_ucast_ftree.c
+++ b/osm/opensm/osm_ucast_ftree.c
@@ -782,9 +782,6 @@ __osm_ftree_sw_set_hops(
IN uint8_t port_num,
IN uint8_t hops)
{
- /* make sure the lid matrix has enough room */
- osm_switch_set_min_lid_size(p_sw->p_osm_sw, max_lid_ho);
-
/* set local min hop table(LID) */
return osm_switch_set_hops(p_sw->p_osm_sw,
lid_ho,
diff --git a/osm/opensm/osm_ucast_mgr.c b/osm/opensm/osm_ucast_mgr.c
index 306c795..93cafae 100644
--- a/osm/opensm/osm_ucast_mgr.c
+++ b/osm/opensm/osm_ucast_mgr.c
@@ -407,11 +407,13 @@ static void __osm_ucast_mgr_dump_tables(osm_ucast_mgr_t *p_mgr)
Starting a rebuild, so notify the switch so it can clear tables, etc...
**********************************************************************/
static void
-__osm_ucast_mgr_clean_switch(
+__osm_ucast_mgr_setup_switch(
IN cl_map_item_t* const p_map_item,
- IN void* context )
+ IN void* cxt )
{
- osm_switch_prepare_path_rebuild((osm_switch_t *)p_map_item);
+ uint16_t lids = cl_ptr_vector_get_size(&((osm_subn_t *)cxt)->port_lid_tbl);
+ osm_switch_prepare_path_rebuild((osm_switch_t *)p_map_item,
+ lids ? lids - 1 : 0);
}
/**********************************************************************
@@ -519,12 +521,6 @@ __osm_ucast_mgr_process_neighbor(
*/
max_lid_ho = osm_switch_get_max_lid_ho( p_remote_sw );
- /*
- Make sure the local lid matrix has enough room to hold
- all the LID info coming from the remote LID matrix.
- */
- osm_switch_set_min_lid_size( p_sw, max_lid_ho );
-
hops = OSM_NO_PATH;
for( lid_ho = 1; lid_ho <= max_lid_ho; lid_ho++ )
{
@@ -1221,7 +1217,7 @@ osm_ucast_mgr_process(
goto Exit;
p_mgr->any_change = FALSE;
- cl_qmap_apply_func(p_sw_guid_tbl, __osm_ucast_mgr_clean_switch, NULL);
+ cl_qmap_apply_func(p_sw_guid_tbl, __osm_ucast_mgr_setup_switch, p_mgr->p_subn);
if (!p_routing_eng->build_lid_matrices ||
p_routing_eng->build_lid_matrices(p_routing_eng->context) != 0)
diff --git a/osm/opensm/osm_ucast_updn.c b/osm/opensm/osm_ucast_updn.c
index e8282f4..950bcb4 100644
--- a/osm/opensm/osm_ucast_updn.c
+++ b/osm/opensm/osm_ucast_updn.c
@@ -627,7 +627,7 @@ __osm_subn_set_up_down_min_hop_table(
p_sw = p_next_sw;
p_next_sw = (osm_switch_t*)cl_qmap_next( &p_sw->map_item );
/* Clear Min Hop Table */
- osm_lid_matrix_clear(&(p_sw->lmx));
+ osm_switch_hops_clear(p_sw);
}
osm_log( p_log, OSM_LOG_VERBOSE,
--
1.5.0.1.26.gf5a92
More information about the general
mailing list