[ofa-general] [PATCH 5/6 v2] opensm/Unicast Routing Cache: integrate cache into opensm
Yevgeny Kliteynik
kliteyn at dev.mellanox.co.il
Wed Oct 15 15:19:43 PDT 2008
Integrating unicast cache into the discovery and ucast manager.
Unicast cache is integrated as part of osm_ucast_mgt_t object.
The cache is marked valid after the first routing engine
execution.
During the discovery or in the drop mgr, whenever some node/port
goes down/disconnected, it is cached in the unicast cache.
Whenever new node or port were spotted, unicast cache checks
whether this is a new node/port or is it something that was
previously cached.
As long as fabric changes do not require full routing recalculation,
unicast cache will prevent osm_ucast_mgr_process() execution (which
also prevents routing engine execution).
When the topology change is not something that can be tolerated
by the unicast cache, cache is invalidated, all its data structures
are freed, new routing is calculated by the routing engine, and the
cache is marked as valid again.
Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
---
opensm/include/opensm/osm_ucast_mgr.h | 9 +++++++++
opensm/opensm/osm_drop_mgr.c | 10 +++++++++-
opensm/opensm/osm_node_info_rcv.c | 9 ++++++++-
opensm/opensm/osm_port_info_rcv.c | 8 +++++++-
opensm/opensm/osm_state_mgr.c | 12 ++++++++++--
opensm/opensm/osm_ucast_mgr.c | 10 ++++++++++
6 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/opensm/include/opensm/osm_ucast_mgr.h b/opensm/include/opensm/osm_ucast_mgr.h
index 27e89e9..13b17b8 100644
--- a/opensm/include/opensm/osm_ucast_mgr.h
+++ b/opensm/include/opensm/osm_ucast_mgr.h
@@ -49,6 +49,7 @@
#include <opensm/osm_subnet.h>
#include <opensm/osm_switch.h>
#include <opensm/osm_log.h>
+#include <opensm/osm_ucast_cache.h>
#ifdef __cplusplus
# define BEGIN_C_DECLS extern "C" {
@@ -97,6 +98,8 @@ typedef struct osm_ucast_mgr {
cl_qlist_t port_order_list;
boolean_t is_dor;
boolean_t some_hop_count_set;
+ cl_qmap_t cache_sw_tbl;
+ boolean_t cache_valid;
} osm_ucast_mgr_t;
/*
* FIELDS
@@ -128,6 +131,12 @@ typedef struct osm_ucast_mgr {
* tables calculation iteration cycle, set to TRUE to indicate
* that some hop count changes were done.
*
+* cache_sw_tbl
+* Cached switches table.
+*
+* cache_valid
+* TRUE if the unicast cache is valid.
+*
* SEE ALSO
* Unicast Manager object
*********/
diff --git a/opensm/opensm/osm_drop_mgr.c b/opensm/opensm/osm_drop_mgr.c
index 8c6e7fb..45fc670 100644
--- a/opensm/opensm/osm_drop_mgr.c
+++ b/opensm/opensm/osm_drop_mgr.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
*
@@ -61,6 +61,7 @@
#include <opensm/osm_multicast.h>
#include <opensm/osm_remote_sm.h>
#include <opensm/osm_inform.h>
+#include <opensm/osm_ucast_mgr.h>
/**********************************************************************
**********************************************************************/
@@ -134,6 +135,10 @@ static void drop_mgr_clean_physp(osm_sm_t * sm, IN osm_physp_t * p_physp)
(p_remote_physp->p_node)),
p_remote_physp->port_num);
+ if (sm->ucast_mgr.cache_valid)
+ osm_ucast_cache_add_link(&sm->ucast_mgr,
+ p_physp, p_remote_physp);
+
osm_physp_unlink(p_physp, p_remote_physp);
}
@@ -308,6 +313,9 @@ __osm_drop_mgr_process_node(osm_sm_t * sm, IN osm_node_t * p_node)
"Unreachable node 0x%016" PRIx64 "\n",
cl_ntoh64(osm_node_get_node_guid(p_node)));
+ if (sm->ucast_mgr.cache_valid)
+ osm_ucast_cache_add_node(&sm->ucast_mgr, p_node);
+
/*
Delete all the logical and physical port objects
associated with this node.
diff --git a/opensm/opensm/osm_node_info_rcv.c b/opensm/opensm/osm_node_info_rcv.c
index a37ce0a..ee2f8e8 100644
--- a/opensm/opensm/osm_node_info_rcv.c
+++ b/opensm/opensm/osm_node_info_rcv.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -59,6 +59,7 @@
#include <opensm/osm_helper.h>
#include <opensm/osm_msgdef.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_ucast_mgr.h>
static void
report_duplicated_guid(IN osm_sm_t * sm,
@@ -240,6 +241,12 @@ __osm_ni_rcv_set_links(IN osm_sm_t * sm,
cl_ntoh64(osm_node_get_node_guid(p_node)), port_num,
cl_ntoh64(p_ni_context->node_guid), p_ni_context->port_num);
+ if (sm->ucast_mgr.cache_valid)
+ osm_ucast_cache_check_new_link(&sm->ucast_mgr,
+ p_node, port_num,
+ p_neighbor_node,
+ p_ni_context->port_num);
+
osm_node_link(p_node, port_num, p_neighbor_node,
p_ni_context->port_num);
diff --git a/opensm/opensm/osm_port_info_rcv.c b/opensm/opensm/osm_port_info_rcv.c
index 73afd8e..efb8830 100644
--- a/opensm/opensm/osm_port_info_rcv.c
+++ b/opensm/opensm/osm_port_info_rcv.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -60,6 +60,7 @@
#include <opensm/osm_pkey.h>
#include <opensm/osm_remote_sm.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_ucast_mgr.h>
/**********************************************************************
**********************************************************************/
@@ -244,6 +245,11 @@ __osm_pi_rcv_process_switch_port(IN osm_sm_t * sm,
(p_remote_node)),
remote_port_num);
+ if (sm->ucast_mgr.cache_valid)
+ osm_ucast_cache_add_link(&sm->ucast_mgr,
+ p_physp,
+ p_remote_physp);
+
osm_node_unlink(p_node, (uint8_t) port_num,
p_remote_node,
(uint8_t) remote_port_num);
diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c
index b4eb87b..ce010cb 100644
--- a/opensm/opensm/osm_state_mgr.c
+++ b/opensm/opensm/osm_state_mgr.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -1075,6 +1075,10 @@ static void do_sweep(osm_sm_t * sm)
/* Re-program the switches fully */
sm->p_subn->ignore_existing_lfts = TRUE;
+ /* we want to re-route, so cache should be invalidated */
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_invalidate(&sm->ucast_mgr);
+
osm_ucast_mgr_process(&sm->ucast_mgr);
/* Reset flag */
@@ -1229,7 +1233,11 @@ _repeat_discovery:
/*
* Proceed with unicast forwarding table configuration.
*/
- osm_ucast_mgr_process(&sm->ucast_mgr);
+
+ if (!sm->ucast_mgr.cache_valid ||
+ osm_ucast_cache_process(&sm->ucast_mgr))
+ osm_ucast_mgr_process(&sm->ucast_mgr);
+
if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
return;
diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c
index 2dc5dd4..3bc3912 100644
--- a/opensm/opensm/osm_ucast_mgr.c
+++ b/opensm/opensm/osm_ucast_mgr.c
@@ -73,6 +73,10 @@ void osm_ucast_mgr_destroy(IN osm_ucast_mgr_t * const p_mgr)
CL_ASSERT(p_mgr);
OSM_LOG_ENTER(p_mgr->p_log);
+
+ if (p_mgr->cache_valid)
+ osm_ucast_cache_invalidate(p_mgr);
+
OSM_LOG_EXIT(p_mgr->p_log);
}
@@ -92,6 +96,9 @@ osm_ucast_mgr_init(IN osm_ucast_mgr_t * const p_mgr, IN osm_sm_t * sm)
p_mgr->p_subn = sm->p_subn;
p_mgr->p_lock = sm->p_lock;
+ if (sm->p_subn->opt.use_ucast_cache)
+ cl_qmap_init(&p_mgr->cache_sw_tbl);
+
OSM_LOG_EXIT(p_mgr->p_log);
return (status);
}
@@ -840,6 +847,9 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * const p_mgr)
"%s tables configured on all switches\n",
osm_routing_engine_type_str(p_osm->routing_engine_used));
+ if (p_mgr->p_subn->opt.use_ucast_cache)
+ p_mgr->cache_valid = TRUE;
+
Exit:
CL_PLOCK_RELEASE(p_mgr->p_lock);
OSM_LOG_EXIT(p_mgr->p_log);
--
1.5.1.4
More information about the general
mailing list