[openib-general] [PATCH] OpenSM: Support LMC > 0 on enhanced switch port 0
Hal Rosenstock
halr at voltaire.com
Tue Jul 4 06:04:02 PDT 2006
OpenSM: Support LMC > 0 on enhanced switch port 0
Allow enhanced switch port 0 (ESP0) to have a non zero LMC. Use the
configured subnet wide LMC for this. Modifications were necessary to the
LID assignment and routing to support this.
Also, add an option to the configuration to use LMC configured for
subnet for enhanced switch port 0 or set it to 0 even if a non zero LMC
is configured for the subnet. The default is currently the latter
option.
Signed-off-by: Hal Rosenstock <halr at voltaire.com>
Index: include/opensm/osm_subnet.h
===================================================================
--- include/opensm/osm_subnet.h (revision 8357)
+++ include/opensm/osm_subnet.h (working copy)
@@ -237,6 +237,7 @@ typedef struct _osm_subn_opt
uint32_t transaction_timeout;
uint8_t sm_priority;
uint8_t lmc;
+ boolean_t lmc_esp0;
uint8_t max_op_vls;
boolean_t reassign_lids;
boolean_t reassign_lfts;
@@ -314,12 +315,17 @@ typedef struct _osm_subn_opt
* lmc
* The LMC value used on this subnet.
*
+* lmc_esp0
+* Whether LMC value used on subnet should be used for
+* enhanced switch port 0 or not. If TRUE, it is used.
+* Otherwise (the default), LMC is set to 0 for ESP0.
+*
* max_op_vls
* Limit the maximal operational VLs. default is 1.
*
* reassign_lids
* If TRUE cause all lids to be re-assigend.
-* Otherwise (the default)
+* Otherwise (the default),
* OpenSM always tries to preserve as LIDs as much as possible.
*
* reassign_lfts
Index: opensm/osm_lid_mgr.c
===================================================================
--- opensm/osm_lid_mgr.c (revision 8357)
+++ opensm/osm_lid_mgr.c (working copy)
@@ -94,6 +94,7 @@
#include <opensm/osm_lid_mgr.h>
#include <opensm/osm_log.h>
#include <opensm/osm_node.h>
+#include <opensm/osm_switch.h>
#include <opensm/osm_helper.h>
#include <opensm/osm_msgdef.h>
#include <vendor/osm_vendor_api.h>
@@ -351,6 +352,7 @@ __osm_lid_mgr_init_sweep(
osm_lid_mgr_range_t *p_range = NULL;
osm_port_t *p_port;
cl_qmap_t *p_port_guid_tbl;
+ osm_switch_t *p_sw;
uint8_t lmc_num_lids = (uint8_t)(1 << p_mgr->p_subn->opt.lmc);
uint16_t lmc_mask;
uint16_t req_lid, num_lids;
@@ -436,7 +438,19 @@ __osm_lid_mgr_init_sweep(
IB_NODE_TYPE_SWITCH )
num_lids = lmc_num_lids;
else
- num_lids = 1;
+ {
+ /* Determine if enhanced switch port 0 */
+ p_sw = osm_get_switch_by_guid(p_mgr->p_subn,
+ osm_node_get_node_guid(osm_port_get_parent_node(p_port)));
+ if (p_mgr->p_subn->opt.lmc_esp0 && osm_switch_is_sp0_enhanced(p_sw))
+ {
+ num_lids = lmc_num_lids;
+ }
+ else
+ {
+ num_lids = 1;
+ }
+ }
if ((num_lids != 1) &&
(((db_min_lid & lmc_mask) != db_min_lid) ||
@@ -539,7 +553,18 @@ __osm_lid_mgr_init_sweep(
}
else
{
- num_lids = 1;
+ /* Determine if enhanced switch port 0 */
+ p_sw = osm_get_switch_by_guid(p_mgr->p_subn,
+ osm_node_get_node_guid(osm_port_get_parent_node(p_port)));
+ if (p_mgr->p_subn->opt.lmc_esp0 && osm_switch_is_sp0_enhanced(p_sw))
+ {
+ disc_max_lid = disc_min_lid + lmc_num_lids - 1;
+ num_lids = lmc_num_lids;
+ }
+ else
+ {
+ num_lids = 1;
+ }
}
/* Make sure the lid is aligned */
@@ -798,6 +823,7 @@ __osm_lid_mgr_get_port_lid(
uint8_t num_lids = (1 << p_mgr->p_subn->opt.lmc);
int lid_changed = 0;
uint16_t lmc_mask;
+ osm_switch_t *p_sw;
OSM_LOG_ENTER( p_mgr->p_log, __osm_lid_mgr_get_port_lid );
@@ -809,10 +835,18 @@ __osm_lid_mgr_get_port_lid(
/* get the lid from the guid2lid */
guid = cl_ntoh64( osm_port_get_guid( p_port ) );
- /* if the port is a switch then we only need one lid */
+ /* if the port is a base switch port 0 then we only need one lid */
if( osm_node_get_type( osm_port_get_parent_node( p_port ) ) ==
IB_NODE_TYPE_SWITCH )
- num_lids = 1;
+ {
+ /* Determine if base switch port 0 */
+ p_sw = osm_get_switch_by_guid(p_mgr->p_subn,
+ osm_node_get_node_guid(osm_port_get_parent_node(p_port)));
+ if (!osm_switch_is_sp0_enhanced(p_sw))
+ {
+ num_lids = 1;
+ }
+ }
/* if the port matches the guid2lid */
if (!osm_db_guid2lid_get( p_mgr->p_g2l, guid, &min_lid, &max_lid))
@@ -954,6 +988,7 @@ __osm_lid_mgr_set_physp_pi(
const ib_port_info_t* p_old_pi;
osm_madw_context_t context;
osm_node_t* p_node;
+ osm_switch_t* p_sw;
ib_api_status_t status;
uint8_t mtu;
uint8_t op_vls;
@@ -1182,6 +1217,18 @@ __osm_lid_mgr_set_physp_pi(
"__osm_lid_mgr_set_physp_pi: "
"Updating neighbor_mtu on switch port 0 to:%u\n",
ib_port_info_get_neighbor_mtu( p_pi ) );
+
+ /* Determine if enhanced switch port 0 and if so set LMC */
+ p_sw = osm_get_switch_by_guid( p_mgr->p_subn, p_node->node_info.node_guid );
+ if (osm_switch_is_sp0_enhanced(p_sw))
+ {
+ /* M_KeyProtectBits are always zero */
+ p_pi->mkey_lmc = p_mgr->p_subn->opt.lmc;
+ /* Check to see if the value we are setting is different than
+ the value in the port_info. If it is, turn on send_set flag */
+ if (memcmp( &p_pi->mkey_lmc, &p_old_pi->mkey_lmc, sizeof(p_pi->mkey_lmc) ))
+ send_set = TRUE;
+ }
}
context.pi_context.node_guid = osm_node_get_node_guid( p_node );
Index: opensm/osm_link_mgr.c
===================================================================
--- opensm/osm_link_mgr.c (revision 8357)
+++ opensm/osm_link_mgr.c (working copy)
@@ -126,6 +126,7 @@ __osm_link_mgr_set_physp_pi(
uint8_t port_num;
uint8_t mtu;
uint8_t op_vls;
+ boolean_t esp0 = FALSE;
boolean_t send_set = FALSE;
osm_physp_t *p_remote_physp;
@@ -172,6 +173,7 @@ __osm_link_mgr_set_physp_pi(
}
goto Exit;
}
+ esp0 = TRUE;
}
/*
@@ -236,7 +238,15 @@ __osm_link_mgr_set_physp_pi(
sizeof(p_pi->m_key_lease_period) ))
send_set = TRUE;
- p_pi->mkey_lmc = p_mgr->p_subn->opt.lmc;
+ if (esp0 == FALSE)
+ p_pi->mkey_lmc = p_mgr->p_subn->opt.lmc;
+ else
+ {
+ if (p_mgr->p_subn->opt.lmc_esp0)
+ p_pi->mkey_lmc = p_mgr->p_subn->opt.lmc;
+ else
+ p_pi->mkey_lmc = 0;
+ }
if (memcmp( &p_pi->mkey_lmc, &p_old_pi->mkey_lmc, sizeof(p_pi->mkey_lmc) ))
send_set = TRUE;
Index: opensm/osm_subnet.c
===================================================================
--- opensm/osm_subnet.c (revision 8357)
+++ opensm/osm_subnet.c (working copy)
@@ -449,6 +449,7 @@ osm_subn_set_default_opt(
p_opt->max_msg_fifo_timeout = 50*OSM_DEFAULT_TRANS_TIMEOUT_MILLISEC;
p_opt->sm_priority = OSM_DEFAULT_SM_PRIORITY;
p_opt->lmc = OSM_DEFAULT_LMC;
+ p_opt->lmc_esp0 = FALSE;
p_opt->max_op_vls = OSM_DEFAULT_MAX_OP_VLS;
p_opt->reassign_lids = FALSE;
p_opt->reassign_lfts = TRUE;
@@ -823,6 +824,10 @@ osm_subn_parse_conf_file(
"lmc",
p_key, p_val, &p_opts->lmc);
+ __osm_subn_opts_unpack_boolean(
+ "lmc_esp0",
+ p_key, p_val, &p_opts->lmc_esp0);
+
__osm_subn_opts_unpack_uint8(
"max_op_vls",
p_key, p_val, &p_opts->max_op_vls);
@@ -1009,6 +1014,10 @@ osm_subn_write_conf_file(
"subnet_prefix 0x%016" PRIx64 "\n\n"
"# The LMC value used on this subnet\n"
"lmc %u\n\n"
+ "# lmc_esp0 determines whether LMC value used on subnet is used for\n"
+ "#enhanced switch port 0. If TRUE, LMC value for subnet is used for\n"
+ "#ESP0. Otherwise, LMC value for ESP0s is 0.\n"
+ "lmc_esp0 %s\n\n"
"# The code of maximal time a packet can live in a switch\n"
"# The actual time is 4.096usec * 2^<packet_life_time>\n"
"# The value 0x14 disables this mechanism\n"
@@ -1045,6 +1054,7 @@ osm_subn_write_conf_file(
cl_ntoh64(p_opts->sm_key),
cl_ntoh64(p_opts->subnet_prefix),
p_opts->lmc,
+ p_opts->lmc_esp0 ? "TRUE" : "FALSE",
p_opts->packet_life_time,
p_opts->vl_stall_count,
p_opts->leaf_vl_stall_count,
Index: opensm/osm_ucast_mgr.c
===================================================================
--- opensm/osm_ucast_mgr.c (revision 8357)
+++ opensm/osm_ucast_mgr.c (working copy)
@@ -351,7 +351,7 @@ static void __osm_ucast_mgr_dump_tables(
}
/**********************************************************************
- Add each switch's own LID to its LID matrix.
+ Add each switch's own LID(s) to its LID matrix.
**********************************************************************/
static void
__osm_ucast_mgr_process_hop_0(
@@ -361,8 +361,9 @@ __osm_ucast_mgr_process_hop_0(
osm_switch_t* const p_sw = (osm_switch_t*)p_map_item;
osm_ucast_mgr_t* const p_mgr = (osm_ucast_mgr_t*)context;
osm_node_t *p_node;
- uint16_t lid_ho;
+ uint16_t lid_ho, base_lid_ho, max_lid_ho;
cl_status_t status;
+ uint8_t lmc;
OSM_LOG_ENTER( p_mgr->p_log, __osm_ucast_mgr_process_hop_0 );
@@ -377,27 +378,35 @@ __osm_ucast_mgr_process_hop_0(
*/
osm_switch_prepare_path_rebuild( p_sw );
- lid_ho = cl_ntoh16( osm_node_get_base_lid( p_node, 0 ) );
+ base_lid_ho = cl_ntoh16( osm_node_get_base_lid( p_node, 0 ) );
+ if (osm_switch_is_sp0_enhanced( p_sw ))
+ lmc = osm_node_get_lmc( p_node, 0 );
+ else
+ lmc = 0;
+ max_lid_ho = (uint16_t)( base_lid_ho + (1 << lmc) - 1 );
- if( osm_log_is_active( p_mgr->p_log, OSM_LOG_DEBUG ) )
+ for (lid_ho = base_lid_ho; lid_ho <= max_lid_ho; lid_ho++)
{
- osm_log( p_mgr->p_log, OSM_LOG_DEBUG,
- "__osm_ucast_mgr_process_hop_0: "
- "Processing switch GUID 0x%" PRIx64 ", LID 0x%X\n",
- cl_ntoh64( osm_node_get_node_guid( p_node ) ),
- lid_ho );
- }
+ if( osm_log_is_active( p_mgr->p_log, OSM_LOG_DEBUG ) )
+ {
+ osm_log( p_mgr->p_log, OSM_LOG_DEBUG,
+ "__osm_ucast_mgr_process_hop_0: "
+ "Processing switch GUID 0x%" PRIx64 ", LID 0x%X\n",
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ lid_ho );
+ }
- status = osm_switch_set_hops( p_sw, lid_ho, 0, 0 );
- if( status != CL_SUCCESS )
- {
- osm_log( p_mgr->p_log, OSM_LOG_ERROR,
- "__osm_ucast_mgr_process_hop_0: ERR 3A02: "
- "Setting hop count failed (%s) for "
- "switch GUID 0x%" PRIx64 ", LID 0x%X\n",
- CL_STATUS_MSG( status ),
- cl_ntoh64( osm_node_get_node_guid( p_node ) ),
- lid_ho );
+ status = osm_switch_set_hops( p_sw, lid_ho, 0, 0 );
+ if( status != CL_SUCCESS )
+ {
+ osm_log( p_mgr->p_log, OSM_LOG_ERROR,
+ "__osm_ucast_mgr_process_hop_0: ERR 3A02: "
+ "Setting hop count failed (%s) for "
+ "switch GUID 0x%" PRIx64 ", LID 0x%X\n",
+ CL_STATUS_MSG( status ),
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ lid_ho );
+ }
}
OSM_LOG_EXIT( p_mgr->p_log );
@@ -1095,7 +1104,7 @@ osm_ucast_mgr_process(
"Starting switches Min Hop Table Assignment\n");
/*
- Set the switch matrices for each switch's own port 0 LID,
+ Set the switch matrices for each switch's own port 0 LID(s)
then set the lid matrices for the each switch's leaf nodes.
*/
cl_qmap_apply_func( p_sw_guid_tbl,
More information about the general
mailing list