[openib-general] [PATCHv3] osm: partition manager force policy
Eitan Zahavi
eitan at mellanox.co.il
Sun Jun 18 04:46:17 PDT 2006
Hi Hal
This is a third take after incorporating Sasha's comments to the
partition manager patch I have previously provided.
The main difference is that the manager does not touch the current
set of pkey tables but only sends Set(PKeyTable).
Another one is the handling of switch limited partition cap by
clearing the switch enforcement bit (on the specific port).
Also modified interface of SMDB access functions from 0/1 to
IB_SUCCESS/IB_ERROR/IB_NOT_FOUND appropriately.
~100 Tests passed both dedicated pkey enforcement (pkey.*) and
stress test (osmStress.*). The pkey.* test was enhanced to verify
correct pkey index is used by the manager (it should keep the original).
BTW: the patch intentionally uses tabs and not spaces as I did not know
what we have decided to use. To modify back simply replace every tab with 3
spaces.
Eitan
Signed-off-by: Eitan Zahavi <eitan at mellanox.co.il>
Index: include/opensm/osm_port.h
===================================================================
--- include/opensm/osm_port.h (revision 8100)
+++ include/opensm/osm_port.h (working copy)
@@ -591,6 +591,39 @@ osm_physp_get_pkey_tbl( IN const osm_phy
* Port, Physical Port
*********/
+/****f* OpenSM: Physical Port/osm_physp_get_mod_pkey_tbl
+* NAME
+* osm_physp_get_mod_pkey_tbl
+*
+* DESCRIPTION
+* Returns a NON CONST pointer to the P_Key table object of the Physical Port object.
+*
+* SYNOPSIS
+*/
+static inline osm_pkey_tbl_t *
+osm_physp_get_mod_pkey_tbl( IN osm_physp_t* const p_physp )
+{
+ CL_ASSERT( osm_physp_is_valid( p_physp ) );
+ /*
+ (14.2.5.7) - the block number valid values are 0-2047, and are further
+ limited by the size of the P_Key table specified by the PartitionCap on the node.
+ */
+ return( &p_physp->pkeys );
+};
+/*
+* PARAMETERS
+* p_physp
+* [in] Pointer to an osm_physp_t object.
+*
+* RETURN VALUES
+* The pointer to the P_Key table object.
+*
+* NOTES
+*
+* SEE ALSO
+* Port, Physical Port
+*********/
+
/****f* OpenSM: Physical Port/osm_physp_set_slvl_tbl
* NAME
* osm_physp_set_slvl_tbl
Index: include/opensm/osm_pkey.h
===================================================================
--- include/opensm/osm_pkey.h (revision 8100)
+++ include/opensm/osm_pkey.h (working copy)
@@ -92,6 +92,9 @@ typedef struct _osm_pkey_tbl
cl_ptr_vector_t blocks;
cl_ptr_vector_t new_blocks;
cl_map_t keys;
+ cl_qlist_t pending;
+ uint16_t used_blocks;
+ uint16_t max_blocks;
} osm_pkey_tbl_t;
/*
* FIELDS
@@ -104,6 +107,18 @@ typedef struct _osm_pkey_tbl
* keys
* A set holding all keys
*
+* pending
+* A list osm_pending_pkey structs that is temporarily set by the
+* pkey mgr and used during pkey mgr algorithm only
+*
+* used_blocks
+* Tracks the number of blocks having non-zero pkeys
+*
+* max_blocks
+* The maximal number of blocks this partition table might hold
+* this value is based on node_info (for port 0 or CA) or switch_info
+* updated on receiving the node_info or switch_info GetResp
+*
* NOTES
* 'blocks' vector should be used to store pkey values obtained from
* the port and SM pkey manager should not change it directly, for this
@@ -114,6 +129,39 @@ typedef struct _osm_pkey_tbl
*
*********/
+/****s* OpenSM: osm_pending_pkey_t
+* NAME
+* osm_pending_pkey_t
+*
+* DESCRIPTION
+* This objects stores temporary information on pkeys their target block and index
+* during the pkey manager operation
+*
+* SYNOPSIS
+*/
+typedef struct _osm_pending_pkey {
+ cl_list_item_t list_item;
+ uint16_t pkey;
+ uint32_t block;
+ uint8_t index;
+ boolean_t is_new;
+} osm_pending_pkey_t;
+/*
+* FIELDS
+* pkey
+* The actual P_Key
+*
+* block
+* The block index based on the previous table extracted from the device
+*
+* index
+* The index of the pky within the block
+*
+* is_new
+* TRUE for new P_Keys such that the block and index are invalid in that case
+*
+*********/
+
/****f* OpenSM: osm_pkey_tbl_construct
* NAME
* osm_pkey_tbl_construct
@@ -142,7 +190,8 @@ void osm_pkey_tbl_construct(
*
* SYNOPSIS
*/
-int osm_pkey_tbl_init(
+ib_api_status_t
+osm_pkey_tbl_init(
IN osm_pkey_tbl_t *p_pkey_tbl);
/*
* p_pkey_tbl
@@ -209,8 +258,8 @@ osm_pkey_tbl_get_num_blocks(
static inline ib_pkey_table_t *osm_pkey_tbl_block_get(
const osm_pkey_tbl_t *p_pkey_tbl, uint16_t block)
{
- CL_ASSERT(block < cl_ptr_vector_get_size(&p_pkey_tbl->blocks));
- return(cl_ptr_vector_get(&p_pkey_tbl->blocks, block));
+ return( (block < cl_ptr_vector_get_size(&p_pkey_tbl->blocks)) ?
+ cl_ptr_vector_get(&p_pkey_tbl->blocks, block) : NULL);
};
/*
* p_pkey_tbl
@@ -244,16 +293,117 @@ static inline ib_pkey_table_t *osm_pkey_
/*
*********/
-/****f* OpenSM: osm_pkey_tbl_sync_new_blocks
+
+/****f* OpenSM: osm_pkey_tbl_make_block_pair
+* NAME
+* osm_pkey_tbl_make_block_pair
+*
+* DESCRIPTION
+* Find or create a pair of "old" and "new" blocks for the
+* given block index
+*
+* SYNOPSIS
+*/
+ib_api_status_t
+osm_pkey_tbl_make_block_pair(
+ osm_pkey_tbl_t *p_pkey_tbl,
+ uint16_t block_idx,
+ ib_pkey_table_t **pp_old_block,
+ ib_pkey_table_t **pp_new_block);
+/*
+* p_pkey_tbl
+* [in] Pointer to the PKey table
+*
+* block_idx
+* [in] The block index to use
+*
+* pp_old_block
+* [out] Pointer to the old block pointer arg
+*
+* pp_new_block
+* [out] Pointer to the new block pointer arg
+*
+* RETURN VALUES
+* IB_SUCCESS if OK IB_ERROR if failed
+*
+*********/
+
+/****f* OpenSM: osm_pkey_tbl_set_new_entry
* NAME
-* osm_pkey_tbl_sync_new_blocks
+* osm_pkey_tbl_set_new_entry
*
* DESCRIPTION
-* Syncs new_blocks vector content with current pkey table blocks
+* stores the given pkey in the "new" blocks array and update
+* the "map" to show that on the "old" blocks
*
* SYNOPSIS
*/
-void osm_pkey_tbl_sync_new_blocks(
+ib_api_status_t
+osm_pkey_tbl_set_new_entry(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ IN uint16_t block_idx,
+ IN uint8_t pkey_idx,
+ IN uint16_t pkey);
+/*
+* p_pkey_tbl
+* [in] Pointer to the PKey table
+*
+* block_idx
+* [in] The block index to use
+*
+* pkey_idx
+* [in] The index within the block
+*
+* pkey
+* [in] PKey to store
+*
+* RETURN VALUES
+* IB_SUCCESS if OK IB_ERROR if failed
+*
+*********/
+
+/****f* OpenSM: osm_pkey_find_next_free_entry
+* NAME
+* osm_pkey_find_next_free_entry
+*
+* DESCRIPTION
+* Find the next free entry in the PKey table. Starting at the given
+* index and block number. The user should increment pkey_idx before
+* next call
+* Inspect the "new" blocks array for empty space.
+*
+* SYNOPSIS
+*/
+boolean_t
+osm_pkey_find_next_free_entry(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ OUT uint16_t *p_block_idx,
+ OUT uint8_t *p_pkey_idx);
+/*
+* p_pkey_tbl
+* [in] Pointer to the PKey table
+*
+* p_block_idx
+* [out] The block index to use
+*
+* p_pkey_idx
+* [out] The index within the block to use
+*
+* RETURN VALUES
+* TRUE if found FALSE if did not find
+*
+*********/
+
+/****f* OpenSM: osm_pkey_tbl_init_new_blocks
+* NAME
+* osm_pkey_tbl_init_new_blocks
+*
+* DESCRIPTION
+* Initializes new_blocks vector content (clear and allocate)
+*
+* SYNOPSIS
+*/
+void osm_pkey_tbl_init_new_blocks(
const osm_pkey_tbl_t *p_pkey_tbl);
/*
* p_pkey_tbl
@@ -263,6 +413,41 @@ void osm_pkey_tbl_sync_new_blocks(
*
*********/
+/****f* OpenSM: osm_pkey_tbl_get_block_and_idx
+* NAME
+* osm_pkey_tbl_get_block_and_idx
+*
+* DESCRIPTION
+* set the block index and pkey index the given
+* pkey is found in. return IB_NOT_FOUND if cound not find
+* it, IB_SUCCESS if OK
+*
+* SYNOPSIS
+*/
+ib_api_status_t
+osm_pkey_tbl_get_block_and_idx(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ IN uint16_t *p_pkey,
+ OUT uint32_t *block_idx,
+ OUT uint8_t *pkey_index);
+/*
+* p_pkey_tbl
+* [in] Pointer to osm_pkey_tbl_t object.
+*
+* p_pkey
+* [in] Pointer to the P_Key entry searched
+*
+* p_block_idx
+* [out] Pointer to the block index to be updated
+*
+* p_pkey_idx
+* [out] Pointer to the pkey index (in the block) to be updated
+*
+*
+* NOTES
+*
+*********/
+
/****f* OpenSM: osm_pkey_tbl_set
* NAME
* osm_pkey_tbl_set
@@ -272,7 +457,8 @@ void osm_pkey_tbl_sync_new_blocks(
*
* SYNOPSIS
*/
-int osm_pkey_tbl_set(
+ib_api_status_t
+osm_pkey_tbl_set(
IN osm_pkey_tbl_t *p_pkey_tbl,
IN uint16_t block,
IN ib_pkey_table_t *p_tbl);
Index: opensm/osm_prtn.c
===================================================================
--- opensm/osm_prtn.c (revision 8100)
+++ opensm/osm_prtn.c (working copy)
@@ -140,6 +140,12 @@ ib_api_status_t osm_prtn_add_port(osm_lo
p_tbl = (full == TRUE) ? &p->full_guid_tbl : &p->part_guid_tbl ;
+ osm_log(p_log, OSM_LOG_VERBOSE, "osm_prtn_add_port: "
+ "Added port 0x%" PRIx64 " to "
+ "partition \'%s\' (0x%04x) As %s member\n",
+ cl_ntoh64(guid), p->name, cl_ntoh16(p->pkey),
+ full ? "full" : "partial" );
+
if (cl_map_insert(p_tbl, guid, p_physp) == NULL)
return IB_INSUFFICIENT_MEMORY;
Index: opensm/osm_pkey.c
===================================================================
--- opensm/osm_pkey.c (revision 8100)
+++ opensm/osm_pkey.c (working copy)
@@ -94,18 +94,22 @@ void osm_pkey_tbl_destroy(
/**********************************************************************
**********************************************************************/
-int osm_pkey_tbl_init(
+ib_api_status_t
+osm_pkey_tbl_init(
IN osm_pkey_tbl_t *p_pkey_tbl)
{
cl_ptr_vector_init( &p_pkey_tbl->blocks, 0, 1);
cl_ptr_vector_init( &p_pkey_tbl->new_blocks, 0, 1);
cl_map_init( &p_pkey_tbl->keys, 1 );
+ cl_qlist_init( &p_pkey_tbl->pending );
+ p_pkey_tbl->used_blocks = 0;
+ p_pkey_tbl->max_blocks = 0;
return(IB_SUCCESS);
}
/**********************************************************************
**********************************************************************/
-void osm_pkey_tbl_sync_new_blocks(
+void osm_pkey_tbl_init_new_blocks(
IN const osm_pkey_tbl_t *p_pkey_tbl)
{
ib_pkey_table_t *p_block, *p_new_block;
@@ -123,16 +127,31 @@ void osm_pkey_tbl_sync_new_blocks(
p_new_block = (ib_pkey_table_t *)malloc(sizeof(*p_new_block));
if (!p_new_block)
break;
+ cl_ptr_vector_set(&((osm_pkey_tbl_t *)p_pkey_tbl)->new_blocks,
+ b, p_new_block);
+ }
+
memset(p_new_block, 0, sizeof(*p_new_block));
- cl_ptr_vector_set(&((osm_pkey_tbl_t *)p_pkey_tbl)->new_blocks, b, p_new_block);
}
- memcpy(p_new_block, p_block, sizeof(*p_new_block));
+}
+
+/**********************************************************************
+ **********************************************************************/
+void osm_pkey_tbl_cleanup_pending(
+ IN osm_pkey_tbl_t *p_pkey_tbl)
+{
+ cl_list_item_t *p_item;
+ p_item = cl_qlist_remove_head( &p_pkey_tbl->pending );
+ while (p_item != cl_qlist_end( &p_pkey_tbl->pending ) )
+ {
+ free( (osm_pending_pkey_t *)p_item );
}
}
/**********************************************************************
**********************************************************************/
-int osm_pkey_tbl_set(
+ib_api_status_t
+osm_pkey_tbl_set(
IN osm_pkey_tbl_t *p_pkey_tbl,
IN uint16_t block,
IN ib_pkey_table_t *p_tbl)
@@ -203,7 +222,138 @@ int osm_pkey_tbl_set(
/**********************************************************************
**********************************************************************/
-static boolean_t __osm_match_pkey (
+ib_api_status_t
+osm_pkey_tbl_make_block_pair(
+ osm_pkey_tbl_t *p_pkey_tbl,
+ uint16_t block_idx,
+ ib_pkey_table_t **pp_old_block,
+ ib_pkey_table_t **pp_new_block)
+{
+ if (block_idx >= p_pkey_tbl->max_blocks) return(IB_ERROR);
+
+ if (pp_old_block)
+ {
+ *pp_old_block = osm_pkey_tbl_block_get( p_pkey_tbl, block_idx );
+ if (! *pp_old_block)
+ {
+ *pp_old_block = (ib_pkey_table_t *)malloc(sizeof(ib_pkey_table_t));
+ if (!*pp_old_block) return(IB_ERROR);
+ memset(*pp_old_block, 0, sizeof(ib_pkey_table_t));
+ cl_ptr_vector_set(&p_pkey_tbl->blocks, block_idx, *pp_old_block);
+ }
+ }
+
+ if (pp_new_block)
+ {
+ *pp_new_block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_idx );
+ if (! *pp_new_block)
+ {
+ *pp_new_block = (ib_pkey_table_t *)malloc(sizeof(ib_pkey_table_t));
+ if (!*pp_new_block) return(IB_ERROR);
+ memset(*pp_new_block, 0, sizeof(ib_pkey_table_t));
+ cl_ptr_vector_set(&p_pkey_tbl->new_blocks, block_idx, *pp_new_block);
+ }
+ }
+ return( IB_SUCCESS );
+}
+
+/**********************************************************************
+ **********************************************************************/
+/*
+ store the given pkey in the "new" blocks array
+ also makes sure the regular block exists.
+*/
+ib_api_status_t
+osm_pkey_tbl_set_new_entry(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ IN uint16_t block_idx,
+ IN uint8_t pkey_idx,
+ IN uint16_t pkey)
+{
+ ib_pkey_table_t *p_old_block;
+ ib_pkey_table_t *p_new_block;
+
+ if (osm_pkey_tbl_make_block_pair(
+ p_pkey_tbl, block_idx, &p_old_block, &p_new_block))
+ return( IB_ERROR );
+
+ p_new_block->pkey_entry[pkey_idx] = pkey;
+ if (p_pkey_tbl->used_blocks < block_idx)
+ p_pkey_tbl->used_blocks = block_idx;
+
+ return( IB_SUCCESS );
+}
+
+/**********************************************************************
+ **********************************************************************/
+boolean_t
+osm_pkey_find_next_free_entry(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ OUT uint16_t *p_block_idx,
+ OUT uint8_t *p_pkey_idx)
+{
+ ib_pkey_table_t *p_new_block;
+
+ CL_ASSERT(p_block_idx);
+ CL_ASSERT(p_pkey_idx);
+
+ while ( *p_block_idx < p_pkey_tbl->max_blocks)
+ {
+ if (*p_pkey_idx > IB_NUM_PKEY_ELEMENTS_IN_BLOCK - 1)
+ {
+ *p_pkey_idx = 0;
+ (*p_block_idx)++;
+ if (*p_block_idx >= p_pkey_tbl->max_blocks)
+ return FALSE;
+ }
+
+ p_new_block = osm_pkey_tbl_new_block_get( p_pkey_tbl, *p_block_idx);
+
+ if ( !p_new_block ||
+ ib_pkey_is_invalid(p_new_block->pkey_entry[*p_pkey_idx]))
+ return TRUE;
+ else
+ (*p_pkey_idx)++;
+ }
+ return FALSE;
+}
+
+/**********************************************************************
+ **********************************************************************/
+ib_api_status_t
+osm_pkey_tbl_get_block_and_idx(
+ IN osm_pkey_tbl_t *p_pkey_tbl,
+ IN uint16_t *p_pkey,
+ OUT uint32_t *p_block_idx,
+ OUT uint8_t *p_pkey_index)
+{
+ uint32_t num_of_blocks;
+ uint32_t block_index;
+ ib_pkey_table_t *block;
+
+ CL_ASSERT( p_pkey_tbl );
+ CL_ASSERT( p_block_idx != NULL );
+ CL_ASSERT( p_pkey_idx != NULL );
+
+ num_of_blocks = cl_ptr_vector_get_size( &p_pkey_tbl->blocks);
+ for ( block_index = 0; block_index < num_of_blocks; block_index++ )
+ {
+ block = osm_pkey_tbl_block_get( p_pkey_tbl, block_index );
+ if ( ( block->pkey_entry <= p_pkey ) &&
+ ( p_pkey < block->pkey_entry + IB_NUM_PKEY_ELEMENTS_IN_BLOCK))
+ {
+ *p_block_idx = block_index;
+ *p_pkey_index = p_pkey - block->pkey_entry;
+ return( IB_SUCCESS );
+ }
+ }
+ return( IB_NOT_FOUND );
+}
+
+/**********************************************************************
+ **********************************************************************/
+static boolean_t
+__osm_match_pkey (
IN const ib_net16_t *pkey1,
IN const ib_net16_t *pkey2 ) {
@@ -306,7 +456,8 @@ osm_physp_share_pkey(
if (cl_is_map_empty(&pkey_tbl1->keys) || cl_is_map_empty(&pkey_tbl2->keys))
return TRUE;
- return !ib_pkey_is_invalid(osm_physp_find_common_pkey(p_physp_1, p_physp_2));
+ return
+ !ib_pkey_is_invalid(osm_physp_find_common_pkey(p_physp_1, p_physp_2));
}
/**********************************************************************
@@ -322,7 +473,8 @@ osm_port_share_pkey(
OSM_LOG_ENTER( p_log, osm_port_share_pkey );
- if (!p_port_1 || !p_port_2) {
+ if (!p_port_1 || !p_port_2)
+ {
ret = FALSE;
goto Exit;
}
@@ -330,7 +482,8 @@ osm_port_share_pkey(
p_physp1 = osm_port_get_default_phys_ptr(p_port_1);
p_physp2 = osm_port_get_default_phys_ptr(p_port_2);
- if (!p_physp1 || !p_physp2) {
+ if (!p_physp1 || !p_physp2)
+ {
ret = FALSE;
goto Exit;
}
Index: opensm/osm_pkey_mgr.c
===================================================================
--- opensm/osm_pkey_mgr.c (revision 8100)
+++ opensm/osm_pkey_mgr.c (working copy)
@@ -62,6 +62,131 @@
/**********************************************************************
**********************************************************************/
+/*
+ the max number of pkey blocks for a physical port is located in
+ different place for switch external ports (SwitchInfo) and the
+ rest of the ports (NodeInfo)
+*/
+static int
+pkey_mgr_get_physp_max_blocks(
+ IN const osm_subn_t *p_subn,
+ IN const osm_physp_t *p_physp)
+{
+ osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
+ osm_switch_t *p_sw;
+ uint16_t num_pkeys = 0;
+
+ if ( (osm_node_get_type(p_node) != IB_NODE_TYPE_SWITCH) ||
+ (osm_physp_get_port_num( p_physp ) == 0))
+ num_pkeys = cl_ntoh16( p_node->node_info.partition_cap );
+ else
+ {
+ p_sw = osm_get_switch_by_guid(p_subn, p_node->node_info.node_guid);
+ if (p_sw)
+ num_pkeys = cl_ntoh16( p_sw->switch_info.enforce_cap );
+ }
+ return( (num_pkeys + 31) / 32 );
+}
+
+/**********************************************************************
+ **********************************************************************/
+/*
+ * Insert the new pending pkey entry to the specific port pkey table
+ * pending pkeys. new entries are inserted at the back.
+ */
+static void
+pkey_mgr_process_physical_port(
+ IN osm_log_t *p_log,
+ IN const osm_req_t *p_req,
+ IN const ib_net16_t pkey,
+ IN osm_physp_t *p_physp )
+{
+ osm_node_t *p_node = osm_physp_get_node_ptr( p_physp );
+ osm_pkey_tbl_t *p_pkey_tbl;
+ ib_net16_t *p_orig_pkey;
+ char *stat = NULL;
+ osm_pending_pkey_t *p_pending;
+
+ p_pkey_tbl = osm_physp_get_mod_pkey_tbl( p_physp );
+ p_pending = (osm_pending_pkey_t *)malloc(sizeof(osm_pending_pkey_t));
+ if (! p_pending)
+ {
+ osm_log( p_log, OSM_LOG_ERROR,
+ "pkey_mgr_process_physical_port: ERR 0502: "
+ "Fail to allocate new pending pkey entry for node "
+ "0x%016" PRIx64 " port %u\n",
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( p_physp ) );
+ return;
+ }
+ p_pending->pkey = pkey;
+ p_orig_pkey = cl_map_get( &p_pkey_tbl->keys, ib_pkey_get_base( pkey ) );
+ if ( !p_orig_pkey )
+ {
+ p_pending->is_new = TRUE;
+ cl_qlist_insert_tail(&p_pkey_tbl->pending, (cl_list_item_t*)p_pending);
+ stat = "inserted";
+ }
+ else
+ {
+ CL_ASSERT( ib_pkey_get_base(*p_orig_pkey) == ib_pkey_get_base(pkey) );
+ p_pending->is_new = FALSE;
+ if (osm_pkey_tbl_get_block_and_idx(
+ p_pkey_tbl, p_orig_pkey,
+ &p_pending->block, &p_pending->index) != IB_SUCCESS)
+ {
+ osm_log( p_log, OSM_LOG_ERROR,
+ "pkey_mgr_process_physical_port: ERR 0503: "
+ "Fail to obtain P_Key 0x%04x block and index for node "
+ "0x%016" PRIx64 " port %u\n",
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( p_physp ) );
+ return;
+ }
+ cl_qlist_insert_head(&p_pkey_tbl->pending, (cl_list_item_t*)p_pending);
+ stat = "updated";
+ }
+
+ osm_log( p_log, OSM_LOG_DEBUG,
+ "pkey_mgr_process_physical_port: "
+ "pkey 0x%04x was %s for node 0x%016" PRIx64
+ " port %u\n",
+ cl_ntoh16( pkey ), stat,
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( p_physp ) );
+}
+
+/**********************************************************************
+ **********************************************************************/
+static void
+pkey_mgr_process_partition_table(
+ osm_log_t *p_log,
+ const osm_req_t *p_req,
+ const osm_prtn_t *p_prtn,
+ const boolean_t full )
+{
+ const cl_map_t *p_tbl =
+ full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
+ cl_map_iterator_t i, i_next;
+ ib_net16_t pkey = p_prtn->pkey;
+ osm_physp_t *p_physp;
+
+ if ( full )
+ pkey = cl_hton16( cl_ntoh16( pkey ) | 0x8000 );
+
+ i_next = cl_map_head( p_tbl );
+ while ( i_next != cl_map_end( p_tbl ) )
+ {
+ i = i_next;
+ i_next = cl_map_next( i );
+ p_physp = cl_map_obj( i );
+ if ( p_physp && osm_physp_is_valid( p_physp ) )
+ pkey_mgr_process_physical_port( p_log, p_req, pkey, p_physp );
+ }
+}
+
+/**********************************************************************
+ **********************************************************************/
static ib_api_status_t
pkey_mgr_update_pkey_entry(
IN const osm_req_t *p_req,
@@ -114,7 +239,8 @@ pkey_mgr_enforce_partition(
p_pi->state_info2 = 0;
ib_port_info_set_port_state( p_pi, IB_LINK_NO_CHANGE );
- context.pi_context.node_guid = osm_node_get_node_guid( osm_physp_get_node_ptr( p_physp ) );
+ context.pi_context.node_guid =
+ osm_node_get_node_guid( osm_physp_get_node_ptr( p_physp ) );
context.pi_context.port_guid = osm_physp_get_port_guid( p_physp );
context.pi_context.set_method = TRUE;
context.pi_context.update_master_sm_base_lid = FALSE;
@@ -131,80 +257,131 @@ pkey_mgr_enforce_partition(
/**********************************************************************
**********************************************************************/
-/*
- * Prepare a new entry for the pkey table for this port when this pkey
- * does not exist. Update existed entry when membership was changed.
- */
-static void pkey_mgr_process_physical_port(
- IN osm_log_t *p_log,
- IN const osm_req_t *p_req,
- IN const ib_net16_t pkey,
- IN osm_physp_t *p_physp )
+static boolean_t pkey_mgr_update_port(
+ osm_log_t *p_log,
+ osm_req_t *p_req,
+ const osm_port_t * const p_port )
{
- osm_node_t *p_node = osm_physp_get_node_ptr( p_physp );
- ib_pkey_table_t *block;
+ osm_physp_t *p_physp;
+ osm_node_t *p_node;
+ ib_pkey_table_t *block, *new_block;
+ osm_pkey_tbl_t *p_pkey_tbl;
uint16_t block_index;
+ uint8_t pkey_index;
+ uint16_t last_free_block_index = 0;
+ uint8_t last_free_pkey_index = 0;
uint16_t num_of_blocks;
- const osm_pkey_tbl_t *p_pkey_tbl;
- ib_net16_t *p_orig_pkey;
- char *stat = NULL;
- uint32_t i;
+ uint16_t max_num_of_blocks;
- p_pkey_tbl = osm_physp_get_pkey_tbl( p_physp );
- num_of_blocks = osm_pkey_tbl_get_num_blocks( p_pkey_tbl );
+ ib_api_status_t status;
+ boolean_t ret_val = FALSE;
+ osm_pending_pkey_t *p_pending;
+ boolean_t found;
- p_orig_pkey = cl_map_get( &p_pkey_tbl->keys, ib_pkey_get_base( pkey ) );
+ p_physp = osm_port_get_default_phys_ptr( p_port );
+ if ( !osm_physp_is_valid( p_physp ) )
+ return FALSE;
- if ( !p_orig_pkey )
- {
- for ( block_index = 0; block_index < num_of_blocks; block_index++ )
+ p_pkey_tbl = osm_physp_get_mod_pkey_tbl( p_physp );
+ num_of_blocks = osm_pkey_tbl_get_num_blocks( p_pkey_tbl );
+ max_num_of_blocks = pkey_mgr_get_physp_max_blocks( p_req->p_subn, p_physp );
+ if ( p_pkey_tbl->max_blocks > max_num_of_blocks )
{
- block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_index );
- for ( i = 0; i < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; i++ )
+ osm_log( p_log, OSM_LOG_INFO,
+ "pkey_mgr_update_port: "
+ "Max number of blocks reduced from %u to %u "
+ "for node 0x%016" PRIx64 " port %u\n",
+ p_pkey_tbl->max_blocks, max_num_of_blocks,
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( p_physp ) );
+ }
+ p_pkey_tbl->max_blocks = max_num_of_blocks;
+
+ osm_pkey_tbl_init_new_blocks( p_pkey_tbl );
+ p_pkey_tbl->used_blocks = 0;
+
+ /*
+ process every pending pkey in order -
+ first must be "updated" last are "new"
+ */
+ p_pending =
+ (osm_pending_pkey_t *)cl_qlist_remove_head( &p_pkey_tbl->pending );
+ while (p_pending !=
+ (osm_pending_pkey_t *)cl_qlist_end( &p_pkey_tbl->pending ) )
+ {
+ if (p_pending->is_new == FALSE)
+ {
+ block_index = p_pending->block;
+ pkey_index = p_pending->index;
+ found = TRUE;
+ }
+ else
{
- if ( ib_pkey_is_invalid( block->pkey_entry[i] ) )
+ found = osm_pkey_find_next_free_entry(p_pkey_tbl,
+ &last_free_block_index,
+ &last_free_pkey_index);
+ if ( !found )
{
- block->pkey_entry[i] = pkey;
- stat = "inserted";
- goto _done;
+ osm_log( p_log, OSM_LOG_ERROR,
+ "pkey_mgr_update_port: ERR 0504: "
+ "failed to find empty space for new pkey 0x%04x "
+ "of node 0x%016" PRIx64 " port %u\n",
+ cl_ntoh16(p_pending->pkey),
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( p_physp ) );
}
+ else
+ {
+ block_index = last_free_block_index;
+ pkey_index = last_free_pkey_index++;
}
}
+
+ if (found)
+ {
+ if ( IB_SUCCESS != osm_pkey_tbl_set_new_entry(
+ p_pkey_tbl, block_index, pkey_index, p_pending->pkey) )
+ {
osm_log( p_log, OSM_LOG_ERROR,
- "pkey_mgr_process_physical_port: ERR 0501: "
- "No empty pkey entry was found to insert 0x%04x for node "
- "0x%016" PRIx64 " port %u\n",
- cl_ntoh16( pkey ),
+ "pkey_mgr_update_port: ERR 0505: "
+ "failed to set PKey 0x%04x in block %u idx %u "
+ "of node 0x%016" PRIx64 " port %u\n",
+ p_pending->pkey, block_index, pkey_index,
cl_ntoh64( osm_node_get_node_guid( p_node ) ),
osm_physp_get_port_num( p_physp ) );
}
- else if ( *p_orig_pkey != pkey )
- {
+ }
+
+ free( p_pending );
+ p_pending =
+ (osm_pending_pkey_t *)cl_qlist_remove_head( &p_pkey_tbl->pending );
+ }
+
+ /* now look for changes and store */
for ( block_index = 0; block_index < num_of_blocks; block_index++ )
{
- /* we need real block (not just new_block) in order
- * to resolve block/pkey indices */
block = osm_pkey_tbl_block_get( p_pkey_tbl, block_index );
- i = p_orig_pkey - block->pkey_entry;
- if (i < IB_NUM_PKEY_ELEMENTS_IN_BLOCK) {
- block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_index );
- block->pkey_entry[i] = pkey;
- stat = "updated";
- goto _done;
- }
- }
- }
+ new_block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_index );
- _done:
- if (stat) {
- osm_log( p_log, OSM_LOG_VERBOSE,
- "pkey_mgr_process_physical_port: "
- "pkey 0x%04x was %s for node 0x%016" PRIx64
- " port %u\n",
- cl_ntoh16( pkey ), stat,
+ if (block &&
+ (!new_block || !memcmp( new_block, block, sizeof( *block ) )) )
+ continue;
+
+ status = pkey_mgr_update_pkey_entry(
+ p_req, p_physp , new_block, block_index );
+ if (status == IB_SUCCESS)
+ ret_val = TRUE;
+ else
+ osm_log( p_log, OSM_LOG_ERROR,
+ "pkey_mgr_update_port: ERR 0506: "
+ "pkey_mgr_update_pkey_entry() failed to update "
+ "pkey table block %d for node 0x%016" PRIx64 " port %u\n",
+ block_index,
cl_ntoh64( osm_node_get_node_guid( p_node ) ),
osm_physp_get_port_num( p_physp ) );
}
+
+ return ret_val;
}
/**********************************************************************
@@ -217,21 +394,23 @@ pkey_mgr_update_peer_port(
const osm_port_t * const p_port,
boolean_t enforce )
{
- osm_physp_t *p, *peer;
+ osm_physp_t *p_physp, *peer;
osm_node_t *p_node;
ib_pkey_table_t *block, *peer_block;
- const osm_pkey_tbl_t *p_pkey_tbl, *p_peer_pkey_tbl;
+ const osm_pkey_tbl_t *p_pkey_tbl;
+ osm_pkey_tbl_t *p_peer_pkey_tbl;
osm_switch_t *p_sw;
ib_switch_info_t *p_si;
uint16_t block_index;
uint16_t num_of_blocks;
+ uint16_t peer_max_blocks;
ib_api_status_t status = IB_SUCCESS;
boolean_t ret_val = FALSE;
- p = osm_port_get_default_phys_ptr( p_port );
- if ( !osm_physp_is_valid( p ) )
+ p_physp = osm_port_get_default_phys_ptr( p_port );
+ if ( !osm_physp_is_valid( p_physp ) )
return FALSE;
- peer = osm_physp_get_remote( p );
+ peer = osm_physp_get_remote( p_physp );
if ( !peer || !osm_physp_is_valid( peer ) )
return FALSE;
p_node = osm_physp_get_node_ptr( peer );
@@ -242,10 +421,26 @@ pkey_mgr_update_peer_port(
if (!p_sw || !(p_si = osm_switch_get_si_ptr( p_sw )) || !p_si->enforce_cap)
return FALSE;
+ p_pkey_tbl = osm_physp_get_pkey_tbl( p_physp );
+ p_peer_pkey_tbl = osm_physp_get_mod_pkey_tbl( peer );
+ num_of_blocks = osm_pkey_tbl_get_num_blocks( p_pkey_tbl );
+ peer_max_blocks = pkey_mgr_get_physp_max_blocks( p_subn, peer );
+ if (peer_max_blocks < p_pkey_tbl->used_blocks)
+ {
+ osm_log( p_log, OSM_LOG_ERROR,
+ "pkey_mgr_update_peer_port: ERR 0508: "
+ "not enough entries (%u < %u) on switch 0x%016" PRIx64
+ " port %u. Clearing Enforcement bit.\n",
+ peer_max_blocks, num_of_blocks,
+ cl_ntoh64( osm_node_get_node_guid( p_node ) ),
+ osm_physp_get_port_num( peer ) );
+ enforce = FALSE;
+ }
+
if (pkey_mgr_enforce_partition( p_req, peer, enforce ) != IB_SUCCESS)
{
osm_log( p_log, OSM_LOG_ERROR,
- "pkey_mgr_update_peer_port: ERR 0502: "
+ "pkey_mgr_update_peer_port: ERR 0507: "
"pkey_mgr_enforce_partition() failed to update "
"node 0x%016" PRIx64 " port %u\n",
cl_ntoh64( osm_node_get_node_guid( p_node ) ),
@@ -255,13 +450,8 @@ pkey_mgr_update_peer_port(
if (enforce == FALSE)
return FALSE;
- p_pkey_tbl = osm_physp_get_pkey_tbl( p );
- p_peer_pkey_tbl = osm_physp_get_pkey_tbl( peer );
- num_of_blocks = osm_pkey_tbl_get_num_blocks( p_pkey_tbl );
- if ( num_of_blocks > osm_pkey_tbl_get_num_blocks( p_peer_pkey_tbl ) )
- num_of_blocks = osm_pkey_tbl_get_num_blocks( p_peer_pkey_tbl );
-
- for ( block_index = 0; block_index < num_of_blocks; block_index++ )
+ p_peer_pkey_tbl->used_blocks = p_pkey_tbl->used_blocks;
+ for ( block_index = 0; block_index < p_pkey_tbl->used_blocks; block_index++)
{
block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_index );
peer_block = osm_pkey_tbl_block_get( p_peer_pkey_tbl, block_index );
@@ -272,7 +462,7 @@ pkey_mgr_update_peer_port(
ret_val = TRUE;
else
osm_log( p_log, OSM_LOG_ERROR,
- "pkey_mgr_update_peer_port: ERR 0503: "
+ "pkey_mgr_update_peer_port: ERR 0509: "
"pkey_mgr_update_pkey_entry() failed to update "
"pkey table block %d for node 0x%016" PRIx64
" port %u\n",
@@ -282,10 +472,10 @@ pkey_mgr_update_peer_port(
}
}
- if ( ret_val == TRUE &&
- osm_log_is_active( p_log, OSM_LOG_VERBOSE ) )
+ if ( (ret_val == TRUE) &&
+ osm_log_is_active( p_log, OSM_LOG_DEBUG ) )
{
- osm_log( p_log, OSM_LOG_VERBOSE,
+ osm_log( p_log, OSM_LOG_DEBUG,
"pkey_mgr_update_peer_port: "
"pkey table was updated for node 0x%016" PRIx64
" port %u\n",
@@ -298,82 +488,6 @@ pkey_mgr_update_peer_port(
/**********************************************************************
**********************************************************************/
-static boolean_t pkey_mgr_update_port(
- osm_log_t *p_log,
- osm_req_t *p_req,
- const osm_port_t * const p_port )
-{
- osm_physp_t *p;
- osm_node_t *p_node;
- ib_pkey_table_t *block, *new_block;
- const osm_pkey_tbl_t *p_pkey_tbl;
- uint16_t block_index;
- uint16_t num_of_blocks;
- ib_api_status_t status;
- boolean_t ret_val = FALSE;
-
- p = osm_port_get_default_phys_ptr( p_port );
- if ( !osm_physp_is_valid( p ) )
- return FALSE;
-
- p_pkey_tbl = osm_physp_get_pkey_tbl(p);
- num_of_blocks = osm_pkey_tbl_get_num_blocks( p_pkey_tbl );
-
- for ( block_index = 0; block_index < num_of_blocks; block_index++ )
- {
- block = osm_pkey_tbl_block_get( p_pkey_tbl, block_index );
- new_block = osm_pkey_tbl_new_block_get( p_pkey_tbl, block_index );
-
- if (!new_block || !memcmp( new_block, block, sizeof( *block ) ) )
- continue;
-
- status = pkey_mgr_update_pkey_entry( p_req, p, new_block, block_index );
- if (status == IB_SUCCESS)
- ret_val = TRUE;
- else
- osm_log( p_log, OSM_LOG_ERROR,
- "pkey_mgr_update_port: ERR 0504: "
- "pkey_mgr_update_pkey_entry() failed to update "
- "pkey table block %d for node 0x%016" PRIx64 " port %u\n",
- block_index,
- cl_ntoh64( osm_node_get_node_guid( p_node ) ),
- osm_physp_get_port_num( p ) );
- }
-
- return ret_val;
-}
-
-/**********************************************************************
- **********************************************************************/
-static void
-pkey_mgr_process_partition_table(
- osm_log_t *p_log,
- const osm_req_t *p_req,
- const osm_prtn_t *p_prtn,
- const boolean_t full )
-{
- const cl_map_t *p_tbl = full ?
- &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
- cl_map_iterator_t i, i_next;
- ib_net16_t pkey = p_prtn->pkey;
- osm_physp_t *p_physp;
-
- if ( full )
- pkey = cl_hton16( cl_ntoh16( pkey ) | 0x8000 );
-
- i_next = cl_map_head( p_tbl );
- while ( i_next != cl_map_end( p_tbl ) )
- {
- i = i_next;
- i_next = cl_map_next( i );
- p_physp = cl_map_obj( i );
- if ( p_physp && osm_physp_is_valid( p_physp ) )
- pkey_mgr_process_physical_port( p_log, p_req, pkey, p_physp );
- }
-}
-
-/**********************************************************************
- **********************************************************************/
osm_signal_t
osm_pkey_mgr_process(
IN osm_opensm_t *p_osm )
@@ -383,8 +497,7 @@ osm_pkey_mgr_process(
osm_prtn_t *p_prtn;
osm_port_t *p_port;
osm_signal_t signal = OSM_SIGNAL_DONE;
- osm_physp_t *p_physp;
-
+ osm_node_t *p_node;
CL_ASSERT( p_osm );
OSM_LOG_ENTER( &p_osm->log, osm_pkey_mgr_process );
@@ -394,32 +507,25 @@ osm_pkey_mgr_process(
if ( osm_prtn_make_partitions( &p_osm->log, &p_osm->subn ) != IB_SUCCESS )
{
osm_log( &p_osm->log, OSM_LOG_ERROR,
- "osm_pkey_mgr_process: ERR 0505: "
+ "osm_pkey_mgr_process: ERR 0510: "
"osm_prtn_make_partitions() failed\n" );
goto _err;
}
- p_tbl = &p_osm->subn.port_guid_tbl;
- p_next = cl_qmap_head( p_tbl );
- while ( p_next != cl_qmap_end( p_tbl ) )
- {
- p_port = ( osm_port_t * ) p_next;
- p_next = cl_qmap_next( p_next );
- p_physp = osm_port_get_default_phys_ptr( p_port );
- if ( osm_physp_is_valid( p_physp ) )
- osm_pkey_tbl_sync_new_blocks( osm_physp_get_pkey_tbl( p_physp ) );
- }
-
+ /* populate the pending pkey entries by scanning all partitions */
p_tbl = &p_osm->subn.prtn_pkey_tbl;
p_next = cl_qmap_head( p_tbl );
while ( p_next != cl_qmap_end( p_tbl ) )
{
p_prtn = ( osm_prtn_t * ) p_next;
p_next = cl_qmap_next( p_next );
- pkey_mgr_process_partition_table( &p_osm->log, &p_osm->sm.req, p_prtn, FALSE );
- pkey_mgr_process_partition_table( &p_osm->log, &p_osm->sm.req, p_prtn, TRUE );
+ pkey_mgr_process_partition_table(
+ &p_osm->log, &p_osm->sm.req, p_prtn, FALSE );
+ pkey_mgr_process_partition_table(
+ &p_osm->log, &p_osm->sm.req, p_prtn, TRUE );
}
+ /* calculate new pkey tables and set */
p_tbl = &p_osm->subn.port_guid_tbl;
p_next = cl_qmap_head( p_tbl );
while ( p_next != cl_qmap_end( p_tbl ) )
@@ -428,8 +534,10 @@ osm_pkey_mgr_process(
p_next = cl_qmap_next( p_next );
if ( pkey_mgr_update_port( &p_osm->log, &p_osm->sm.req, p_port ) )
signal = OSM_SIGNAL_DONE_PENDING;
- if ( osm_node_get_type( osm_port_get_parent_node( p_port ) ) != IB_NODE_TYPE_SWITCH &&
- pkey_mgr_update_peer_port( &p_osm->log, &p_osm->sm.req,
+ p_node = osm_port_get_parent_node( p_port );
+ if ( ( osm_node_get_type( p_node ) != IB_NODE_TYPE_SWITCH ) &&
+ pkey_mgr_update_peer_port(
+ &p_osm->log, &p_osm->sm.req,
&p_osm->subn, p_port,
!p_osm->subn.opt.no_partition_enforcement ) )
signal = OSM_SIGNAL_DONE_PENDING;
More information about the general
mailing list