[ofa-general] Re: [PATCH 5/6] opensm/Unicast Routing Cache: integrate cache into opensm
Sasha Khapyorsky
sashak at voltaire.com
Thu Oct 9 10:10:00 PDT 2008
Hi Yevgeny,
On 03:29 Mon 06 Oct , Yevgeny Kliteynik wrote:
[snip...]
> @@ -818,27 +826,37 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * const p_mgr)
> /*
> If there are no switches in the subnet, we are done.
> */
> - if (cl_qmap_count(p_sw_guid_tbl) == 0 ||
> - ucast_mgr_setup_all_switches(p_mgr->p_subn) < 0)
> + if (cl_qmap_count(p_sw_guid_tbl) == 0)
> goto Exit;
>
> p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_NONE;
> - while (p_routing_eng) {
> - if (!ucast_mgr_route(p_routing_eng, p_osm))
> - break;
> - p_routing_eng = p_routing_eng->next;
> - }
> + if (p_mgr->p_subn->opt.use_ucast_cache &&
> + osm_ucast_cache_is_valid(p_mgr->p_cache)) {
> + OSM_LOG(p_mgr->p_log, OSM_LOG_INFO,
> + "Configuring switch tables using cached routing\n");
> + osm_ucast_cache_apply(p_mgr->p_cache);
>
> - if (p_osm->routing_engine_used == OSM_ROUTING_ENGINE_TYPE_NONE) {
> - /* If configured routing algorithm failed, use default MinHop */
> - osm_ucast_mgr_build_lid_matrices(p_mgr);
> - ucast_mgr_build_lfts(p_mgr);
> - p_osm->routing_engine_used = OSM_ROUTING_ENGINE_TYPE_MINHOP;
> - }
> + } else {
I think this will break some routing engines (such LASH) and logging
because p_osm->routing_engine_used is leaved as
OSM_ROUTING_ENGINE_TYPE_NONE.
And since we have cache validation calls in do_sweep() anyway:
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_validate(sm->ucast_mgr.p_cache);
Isn't it would be better to not touch osm_ucast_mgr_process() at all and
instead to replace lines above by:
if (!sm->p_subn->opt.use_ucast_cache ||
osm_ucast_cache_process(sm->ucast_mgr.p_cache))
This also saves couple of public calls in ucast cache. I think then the
patch could look like below. Agreed?
Sasha
>From a8db7582785d9103c74be1cd9224a8825ab0963a Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
Date: Mon, 6 Oct 2008 03:29:34 +0200
Subject: [PATCH] opensm/Unicast Routing Cache: integrate cache into opensm
Integrating unicast cache into the discovery and ucast manager.
Signed-off-by: Yevgeny Kliteynik <kliteyn at dev.mellanox.co.il>
Signed-off-by: Sasha Khapyorsky <sashak at voltaire.com>
---
opensm/include/opensm/osm_ucast_cache.h | 68 ++----------------------------
opensm/include/opensm/osm_ucast_mgr.h | 6 +++
opensm/opensm/osm_drop_mgr.c | 13 +++++-
opensm/opensm/osm_node_info_rcv.c | 9 ++++-
opensm/opensm/osm_port_info_rcv.c | 9 ++++-
opensm/opensm/osm_state_mgr.c | 12 +++++-
opensm/opensm/osm_ucast_cache.c | 42 ++++++++-----------
opensm/opensm/osm_ucast_mgr.c | 11 +++++
8 files changed, 77 insertions(+), 93 deletions(-)
diff --git a/opensm/include/opensm/osm_ucast_cache.h b/opensm/include/opensm/osm_ucast_cache.h
index 2dc1c4e..7f01876 100644
--- a/opensm/include/opensm/osm_ucast_cache.h
+++ b/opensm/include/opensm/osm_ucast_cache.h
@@ -198,38 +198,6 @@ osm_ucast_cache_invalidate(osm_ucast_cache_t * p_cache);
* Unicast Cache object
*********/
-/****f* OpenSM: Unicast Cache/osm_ucast_cache_validate
-* NAME
-* osm_ucast_cache_validate
-*
-* DESCRIPTION
-* The osm_ucast_cache_validate function checks
-* whether or not the cached routing can be applied
-* to the current subnet switches.
-*
-* SYNOPSIS
-*/
-void
-osm_ucast_cache_validate(osm_ucast_cache_t * p_cache);
-/*
-* PARAMETERS
-* p_cache
-* [in] Pointer to the object to check.
-*
-* RETURN VALUE
-* This function does not return any value.
-*
-* NOTES
-* This function checks the current subnet and the
-* cached links, and decides whether or not there
-* is a need to re-run unicast routing engine.
-* If the cached routing can't be applied to the
-* current subnet switches as is, cache is invalidated.
-*
-* SEE ALSO
-* Unicast Cache object
-*********/
-
/****f* OpenSM: Unicast Cache/osm_ucast_cache_mark_valid
* NAME
* osm_ucast_cache_mark_valid
@@ -256,31 +224,6 @@ osm_ucast_cache_mark_valid(osm_ucast_cache_t * p_cache);
* Unicast Cache object
*********/
-/****f* OpenSM: Unicast Cache/osm_ucast_cache_is_valid
-* NAME
-* osm_ucast_cache_is_valid
-*
-* DESCRIPTION
-* Check whether the unicast cache is valid.
-*
-* SYNOPSIS
-*/
-boolean_t
-osm_ucast_cache_is_valid(osm_ucast_cache_t * p_cache);
-/*
-* PARAMETERS
-* p_cache
-* [in] Pointer to the object to check.
-*
-* RETURN VALUE
-* TRUE if the cache is valid, FALSE otherwise.
-*
-* NOTES
-*
-* SEE ALSO
-* Unicast Cache object
-*********/
-
/****f* OpenSM: Unicast Cache/osm_ucast_cache_check_new_link
* NAME
* osm_ucast_cache_check_new_link
@@ -406,25 +349,24 @@ osm_ucast_cache_add_node(osm_ucast_cache_t * p_cache,
* Unicast Cache object
*********/
-/****f* OpenSM: Unicast Cache/osm_ucast_cache_apply
+/****f* OpenSM: Unicast Cache/osm_ucast_cache_process
* NAME
-* osm_ucast_cache_apply
+* osm_ucast_cache_process
*
* DESCRIPTION
-* The osm_ucast_cache_apply function writes the
+* The osm_ucast_cache_process function writes the
* cached unicast routing on the subnet switches.
*
* SYNOPSIS
*/
-void
-osm_ucast_cache_apply(osm_ucast_cache_t * p_cache);
+int osm_ucast_cache_process(osm_ucast_cache_t * p_cache);
/*
* PARAMETERS
* p_cache
* [in] Pointer to the cache object to be used.
*
* RETURN VALUE
-* This function does not return any value.
+* This function returns zero on sucess and non-zero value otherwise.
*
* NOTES
* Iterates through all the subnet switches and writes
diff --git a/opensm/include/opensm/osm_ucast_mgr.h b/opensm/include/opensm/osm_ucast_mgr.h
index 27e89e9..e4006bb 100644
--- a/opensm/include/opensm/osm_ucast_mgr.h
+++ b/opensm/include/opensm/osm_ucast_mgr.h
@@ -49,6 +49,7 @@
#include <opensm/osm_subnet.h>
#include <opensm/osm_switch.h>
#include <opensm/osm_log.h>
+#include <opensm/osm_ucast_cache.h>
#ifdef __cplusplus
# define BEGIN_C_DECLS extern "C" {
@@ -77,6 +78,7 @@ BEGIN_C_DECLS
*
*********/
struct osm_sm;
+struct _osm_ucast_cache;
/****s* OpenSM: Unicast Manager/osm_ucast_mgr_t
* NAME
* osm_ucast_mgr_t
@@ -97,6 +99,7 @@ typedef struct osm_ucast_mgr {
cl_qlist_t port_order_list;
boolean_t is_dor;
boolean_t some_hop_count_set;
+ struct _osm_ucast_cache *p_cache;
} osm_ucast_mgr_t;
/*
* FIELDS
@@ -128,6 +131,9 @@ typedef struct osm_ucast_mgr {
* tables calculation iteration cycle, set to TRUE to indicate
* that some hop count changes were done.
*
+* p_cache
+* Pointer to the Unicast Cache object.
+*
* SEE ALSO
* Unicast Manager object
*********/
diff --git a/opensm/opensm/osm_drop_mgr.c b/opensm/opensm/osm_drop_mgr.c
index 8c6e7fb..5fc46a9 100644
--- a/opensm/opensm/osm_drop_mgr.c
+++ b/opensm/opensm/osm_drop_mgr.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
* Copyright (c) 2008 Xsigo Systems Inc. All rights reserved.
*
@@ -61,6 +61,7 @@
#include <opensm/osm_multicast.h>
#include <opensm/osm_remote_sm.h>
#include <opensm/osm_inform.h>
+#include <opensm/osm_ucast_mgr.h>
/**********************************************************************
**********************************************************************/
@@ -134,6 +135,13 @@ static void drop_mgr_clean_physp(osm_sm_t * sm, IN osm_physp_t * p_physp)
(p_remote_physp->p_node)),
p_remote_physp->port_num);
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_add_link(sm->ucast_mgr.p_cache,
+ p_physp->p_node,
+ p_physp->port_num,
+ p_remote_physp->p_node,
+ p_remote_physp->port_num);
+
osm_physp_unlink(p_physp, p_remote_physp);
}
@@ -308,6 +316,9 @@ __osm_drop_mgr_process_node(osm_sm_t * sm, IN osm_node_t * p_node)
"Unreachable node 0x%016" PRIx64 "\n",
cl_ntoh64(osm_node_get_node_guid(p_node)));
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_add_node(sm->ucast_mgr.p_cache, p_node);
+
/*
Delete all the logical and physical port objects
associated with this node.
diff --git a/opensm/opensm/osm_node_info_rcv.c b/opensm/opensm/osm_node_info_rcv.c
index a37ce0a..94903b7 100644
--- a/opensm/opensm/osm_node_info_rcv.c
+++ b/opensm/opensm/osm_node_info_rcv.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -59,6 +59,7 @@
#include <opensm/osm_helper.h>
#include <opensm/osm_msgdef.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_ucast_mgr.h>
static void
report_duplicated_guid(IN osm_sm_t * sm,
@@ -240,6 +241,12 @@ __osm_ni_rcv_set_links(IN osm_sm_t * sm,
cl_ntoh64(osm_node_get_node_guid(p_node)), port_num,
cl_ntoh64(p_ni_context->node_guid), p_ni_context->port_num);
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_check_new_link(sm->ucast_mgr.p_cache,
+ p_node, port_num,
+ p_neighbor_node,
+ p_ni_context->port_num);
+
osm_node_link(p_node, port_num, p_neighbor_node,
p_ni_context->port_num);
diff --git a/opensm/opensm/osm_port_info_rcv.c b/opensm/opensm/osm_port_info_rcv.c
index 73afd8e..d8d2021 100644
--- a/opensm/opensm/osm_port_info_rcv.c
+++ b/opensm/opensm/osm_port_info_rcv.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -60,6 +60,7 @@
#include <opensm/osm_pkey.h>
#include <opensm/osm_remote_sm.h>
#include <opensm/osm_opensm.h>
+#include <opensm/osm_ucast_mgr.h>
/**********************************************************************
**********************************************************************/
@@ -244,6 +245,12 @@ __osm_pi_rcv_process_switch_port(IN osm_sm_t * sm,
(p_remote_node)),
remote_port_num);
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_add_link(sm->ucast_mgr.p_cache,
+ p_node, port_num,
+ p_remote_node,
+ remote_port_num);
+
osm_node_unlink(p_node, (uint8_t) port_num,
p_remote_node,
(uint8_t) remote_port_num);
diff --git a/opensm/opensm/osm_state_mgr.c b/opensm/opensm/osm_state_mgr.c
index b4eb87b..530a705 100644
--- a/opensm/opensm/osm_state_mgr.c
+++ b/opensm/opensm/osm_state_mgr.c
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
- * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
* Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
@@ -1075,6 +1075,10 @@ static void do_sweep(osm_sm_t * sm)
/* Re-program the switches fully */
sm->p_subn->ignore_existing_lfts = TRUE;
+ /* we want to re-route, so cache should be invalidated */
+ if (sm->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_invalidate(sm->ucast_mgr.p_cache);
+
osm_ucast_mgr_process(&sm->ucast_mgr);
/* Reset flag */
@@ -1229,7 +1233,11 @@ _repeat_discovery:
/*
* Proceed with unicast forwarding table configuration.
*/
- osm_ucast_mgr_process(&sm->ucast_mgr);
+
+ if (!sm->p_subn->opt.use_ucast_cache ||
+ osm_ucast_cache_process(sm->ucast_mgr.p_cache))
+ osm_ucast_mgr_process(&sm->ucast_mgr);
+
if (wait_for_pending_transactions(&sm->p_subn->p_osm->stats))
return;
diff --git a/opensm/opensm/osm_ucast_cache.c b/opensm/opensm/osm_ucast_cache.c
index 3c32d35..57dc0a0 100644
--- a/opensm/opensm/osm_ucast_cache.c
+++ b/opensm/opensm/osm_ucast_cache.c
@@ -479,16 +479,6 @@ osm_ucast_cache_mark_valid(osm_ucast_cache_t * p_cache)
/**********************************************************************
**********************************************************************/
-boolean_t
-osm_ucast_cache_is_valid(osm_ucast_cache_t * p_cache)
-{
- CL_ASSERT(p_cache && p_cache->p_ucast_mgr);
- return p_cache->valid;
-}
-
-/**********************************************************************
- **********************************************************************/
-
void
osm_ucast_cache_invalidate(osm_ucast_cache_t * p_cache)
{
@@ -520,8 +510,7 @@ Exit:
/**********************************************************************
**********************************************************************/
-void
-osm_ucast_cache_validate(osm_ucast_cache_t * p_cache)
+static void ucast_cache_validate(osm_ucast_cache_t * p_cache)
{
cache_switch_t * p_cache_sw;
cache_switch_t * p_remote_cache_sw;
@@ -1150,24 +1139,27 @@ Exit:
/**********************************************************************
**********************************************************************/
-
-void
-osm_ucast_cache_apply(osm_ucast_cache_t * p_cache)
+int osm_ucast_cache_process(osm_ucast_cache_t * p_cache)
{
- osm_subn_t * p_subn = p_cache->p_ucast_mgr->p_subn;
- osm_switch_t *p_sw;
+ cl_qmap_t *tbl = &p_cache->p_ucast_mgr->p_subn->sw_guid_tbl;
+ cl_map_item_t *item;
+
+ if (!p_cache->p_ucast_mgr->p_subn->opt.use_ucast_cache)
+ return 1;
+
+ ucast_cache_validate(p_cache);
+ if (!p_cache->valid)
+ return 1;
- OSM_LOG_ENTER(p_cache->p_ucast_mgr->p_log);
OSM_LOG(p_cache->p_ucast_mgr->p_log, OSM_LOG_INFO,
- "Applying unicast cache\n");
- CL_ASSERT(p_cache && p_cache->p_ucast_mgr && p_cache->valid);
+ "Configuring switch tables using cached routing\n");
- for (p_sw = (osm_switch_t *) cl_qmap_head(&p_subn->sw_guid_tbl);
- p_sw != (osm_switch_t *) cl_qmap_end(&p_subn->sw_guid_tbl);
- p_sw = (osm_switch_t *) cl_qmap_next(&p_sw->map_item))
- osm_ucast_mgr_set_fwd_table(p_cache->p_ucast_mgr, p_sw);
+ for (item = cl_qmap_head(tbl); item != cl_qmap_end(tbl);
+ item = cl_qmap_next(item))
+ osm_ucast_mgr_set_fwd_table(p_cache->p_ucast_mgr,
+ (osm_switch_t *)item);
- OSM_LOG_EXIT(p_cache->p_ucast_mgr->p_log);
+ return 0;
}
/**********************************************************************
diff --git a/opensm/opensm/osm_ucast_mgr.c b/opensm/opensm/osm_ucast_mgr.c
index 2dc5dd4..34eddd0 100644
--- a/opensm/opensm/osm_ucast_mgr.c
+++ b/opensm/opensm/osm_ucast_mgr.c
@@ -73,6 +73,8 @@ void osm_ucast_mgr_destroy(IN osm_ucast_mgr_t * const p_mgr)
CL_ASSERT(p_mgr);
OSM_LOG_ENTER(p_mgr->p_log);
+ if (p_mgr->p_cache)
+ osm_ucast_cache_destroy(p_mgr->p_cache);
OSM_LOG_EXIT(p_mgr->p_log);
}
@@ -92,6 +94,12 @@ osm_ucast_mgr_init(IN osm_ucast_mgr_t * const p_mgr, IN osm_sm_t * sm)
p_mgr->p_subn = sm->p_subn;
p_mgr->p_lock = sm->p_lock;
+ if (sm->p_subn->opt.use_ucast_cache){
+ p_mgr->p_cache = osm_ucast_cache_construct(p_mgr);
+ if (!p_mgr->p_cache)
+ status = IB_INSUFFICIENT_MEMORY;
+ }
+
OSM_LOG_EXIT(p_mgr->p_log);
return (status);
}
@@ -840,6 +848,9 @@ int osm_ucast_mgr_process(IN osm_ucast_mgr_t * const p_mgr)
"%s tables configured on all switches\n",
osm_routing_engine_type_str(p_osm->routing_engine_used));
+ if (p_mgr->p_subn->opt.use_ucast_cache)
+ osm_ucast_cache_mark_valid(p_mgr->p_cache);
+
Exit:
CL_PLOCK_RELEASE(p_mgr->p_lock);
OSM_LOG_EXIT(p_mgr->p_log);
--
1.6.0.1.196.g01914
More information about the general
mailing list