[ofa-general] [PATCH 2/2 v3] opensm: Compression of multicast group according to pkey
Slava Strebkov
slavas at Voltaire.COM
Wed Aug 5 06:48:35 PDT 2009
Subject: [PATCH 2/2] Compression of multicast group according to pkey
Additional data structure added:
1. Map of all partition keys opened in the fabric.
2. Map of all multicast group holders shared same pkey.
MLID assignment for multicast groups works in a usual
manner, allocating free entry for newly created group.
Proposed compression algorithm starts working when there
are no more free entries in the mlid array. List of MLIDs
for new multicast group will be chosen from the pkey
indexed map according to the requested pkey. MLID which
shares minimum number of ports will be given to newly
created multicast group.
Signed-off-by: Slava Strebkov <slavas at voltaire.com>
---
opensm/include/opensm/osm_multicast.h | 133 ++++++++++++++++++++++++++++++++
opensm/include/opensm/osm_subnet.h | 36 +++++++++
opensm/opensm/osm_mcast_mgr.c | 4 +
opensm/opensm/osm_multicast.c | 109 +++++++++++++++++++++++++-
opensm/opensm/osm_sa_mcmember_record.c | 38 +++++----
opensm/opensm/osm_subnet.c | 8 ++
6 files changed, 308 insertions(+), 20 deletions(-)
diff --git a/opensm/include/opensm/osm_multicast.h b/opensm/include/opensm/osm_multicast.h
index 61d1ba6..7bd2f81 100644
--- a/opensm/include/opensm/osm_multicast.h
+++ b/opensm/include/opensm/osm_multicast.h
@@ -128,6 +128,7 @@ typedef struct osm_mgrp_holder {
boolean_t to_be_deleted;
uint32_t last_tree_id;
uint32_t last_change_id;
+ cl_map_item_t mlid_item;
} osm_mgrp_holder_t;
/*
@@ -156,6 +157,9 @@ typedef struct osm_mgrp_holder {
*
* last_tree_id
* the last change id used for building the current tree.
+*
+* mlid_item
+* list item in list of holders shared same pkey.
*/
/****s* OpenSM: Multicast group Port /osm_mgrp_port _t
* NAME
@@ -775,5 +779,134 @@ static inline boolean_t osm_mgrp_holder_is_empty(IN const osm_mgrp_holder_t *
return (cl_qmap_count(&p_mgrp_holder->mgrp_port_map) == 0);
}
+/****f* OpenSM: Subnet/osm_mlid_pkey_delete
+* NAME
+* osm_mlid_pkey_delete
+*
+* DESCRIPTION
+* Frees the objects.
+*
+* SYNOPSIS
+*/
+void osm_mlid_pkey_delete(osm_mlid_pkey_t *p_mlid_pkey);
+/*
+* PARAMETERS
+* p_mlid_pkey
+* [in] Pointer to an osm_mlid_pkey_t object
+*
+* RETURN VALUES
+* None.
+*
+*
+* SEE ALSO
+* osm_mlid_pkey_new
+*********/
+
+/****f* OpenSM: Subnet/osm_mlid_pkey_new
+* NAME
+* osm_mlid_pkey_new
+*
+* DESCRIPTION
+* Creates new object of osm_mlid_pkey_t.
+*
+* SYNOPSIS
+*/
+osm_mlid_pkey_t *osm_mlid_pkey_new(IN ib_net16_t pkey);
+/*
+* PARAMETERS
+* pkey
+* [in] Partition key for the object
+*
+* RETURN VALUES
+* Pointer to osm_mlid_pkey_t, or NULL.
+*
+* SEE ALSO
+* osm_mlid_pkey_delete
+*********/
+
+/****f* OpenSM: Subnet/osm_mlid_pkey_add_holder
+* NAME
+* osm_mlid_pkey_add_holder
+*
+* DESCRIPTION
+* Adds osm_mlid_pkey_t object to map
+*
+* SYNOPSIS
+*/
+void osm_mlid_pkey_add_holder(osm_mgrp_holder_t *p_mgrp_holder,
+ ib_net16_t pkey, osm_subn_t *p_subn);
+/*
+* PARAMETERS
+* p_mgrp_holder
+* [in] Pointer to osm_mgrp_holder_t
+*
+* pkey
+* [in] Partition key for the object
+*
+* p_subn
+* [in] Pointer to an osm_subn_t object
+*
+* RETURN VALUES
+* None.
+*
+* SEE ALSO
+* osm_mlid_pkey_remove_holder
+*********/
+
+/****f* OpenSM: Subnet/osm_mlid_pkey_remove_holder
+* NAME
+* osm_mlid_pkey_remove_holder
+*
+* DESCRIPTION
+* removes osm_mlid_pkey_t object from map
+*
+* SYNOPSIS
+*/
+void osm_mlid_pkey_remove_holder(osm_mgrp_holder_t *p_mgrp_holder,
+ ib_net16_t pkey, osm_subn_t *p_subn);
+/*
+* PARAMETERS
+* p_mgrp_holder
+* [in] Pointer to osm_mgrp_holder_t
+*
+* pkey
+* [in] Partition key for the object
+*
+* p_subn
+* [in] Pointer to an osm_subn_t object
+*
+* RETURN VALUES
+* None.
+*
+* SEE ALSO
+* osm_mlid_pkey_add_holder
+*********/
+
+/****f* OpenSM: Subnet/osm_mlid_pkey_get_existed_mlid
+* NAME
+* osm_mlid_pkey_get_existed_mlid
+*
+* DESCRIPTION
+* return used mlid with miminum ports, matched by pkey
+*
+* SYNOPSIS
+*/
+ib_net16_t osm_mlid_pkey_get_existed_mlid(IN osm_subn_t *p_subn, IN ib_net16_t pkey);
+/*
+* PARAMETERS
+*
+* p_subn
+* [in] Pointer to an osm_subn_t object
+*
+* pkey
+* [in] Partition key for the object
+*
+* RETURN VALUES
+* matched mlid or 0 if not found
+*
+* SEE ALSO
+* osm_mlid_pkey_add_holder
+*********/
+
END_C_DECLS
#endif /* _OSM_MULTICAST_H_ */
diff --git a/opensm/include/opensm/osm_subnet.h b/opensm/include/opensm/osm_subnet.h
index fad8780..aea6c45 100644
--- a/opensm/include/opensm/osm_subnet.h
+++ b/opensm/include/opensm/osm_subnet.h
@@ -469,6 +469,37 @@ typedef struct osm_subn_opt {
* Subnet object
*********/
+/****s* OpenSM: Subnet/osm_mlid_pkey_t
+* NAME
+* osm_mlid_pkey_t
+*
+* DESCRIPTION
+* Structure combines all MLIDs opened on same pkey value.
+* Used for mgid to mlid compresion
+*
+* SYNOPSIS
+*/
+typedef struct osm_mlid_pkey {
+ cl_map_item_t pkey_item;
+ ib_net16_t pkey;
+ cl_qmap_t mlid_holder_map;
+} osm_mlid_pkey_t;
+/*
+* FIELDS
+* pkey_item
+* Map Item for qmap linkage. Must be first element!!
+* Indexed by pkey.
+*
+* pkey
+* Partition key (P_Key) for multicast group(s).
+*
+* mlid_holder_map
+* Map of osm_mgrp_holder_t objects. Indexed by mlid
+*
+* SEE ALSO
+* osm_mgrp_holder_t
+*********/
+
/****s* OpenSM: Subnet/osm_subn_t
* NAME
* osm_subn_t
@@ -514,6 +545,7 @@ typedef struct osm_subn {
unsigned need_update;
cl_fmap_t mgrp_mgid_tbl;
void *mgroup_holders[IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO + 1];
+ cl_qmap_t mlid_pkey_tbl;
} osm_subn_t;
/*
* FIELDS
@@ -638,6 +670,10 @@ typedef struct osm_subn {
* Array of pointers to all Multicast Group Holder objects in the subnet.
* Indexed by MLID offset from base MLID.
*
+* mlid_pkey_tbl;
+* Map of osm_pkey_mlid_t objects. Arranged by mgrp pkey value.
+* Contains MLIDs for mgroups with same pkey.
+*
* SEE ALSO
* Subnet object
*********/
diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c
index f506393..ec3dec6 100644
--- a/opensm/opensm/osm_mcast_mgr.c
+++ b/opensm/opensm/osm_mcast_mgr.c
@@ -1075,6 +1075,10 @@ static ib_api_status_t mcast_mgr_process_mgrp(osm_sm_t * sm,
gid_str, sizeof(gid_str)),
cl_ntoh16(p_mgrp->mlid));
osm_mgrp_holder_delete_mgrp(p_mgrp_holder, p_mgrp);
+ if (p_mgrp_holder->to_be_deleted) {
+ osm_mlid_pkey_remove_holder(p_mgrp_holder,
+ p_mgrp->mcmember_rec.pkey,sm->p_subn);
+ }
p_mcm_port = (osm_mcm_port_t *) cl_qmap_head(&p_mgrp->mcm_port_tbl);
while (p_mcm_port !=
(osm_mcm_port_t *) cl_qmap_end(&p_mgrp->mcm_port_tbl)) {
diff --git a/opensm/opensm/osm_multicast.c b/opensm/opensm/osm_multicast.c
index 072b591..4724bd3 100644
--- a/opensm/opensm/osm_multicast.c
+++ b/opensm/opensm/osm_multicast.c
@@ -366,10 +366,9 @@ void osm_mgrp_holder_remove_port(osm_subn_t * subn, osm_log_t * p_log,
char gid_str[INET6_ADDRSTRLEN];
OSM_LOG(p_log, OSM_LOG_DEBUG,
"port 0x%" PRIx64 " removed from mlid 0x%X\n",
- port_guid, cl_ntoh16(p_mgrp_holder->mlid));
- while ((p_item =
- cl_qlist_remove_head(&p_mgrp_port->mgroups)) !=
- cl_qlist_end(&p_mgrp_port->mgroups)) {
+ cl_ntoh64(port_guid), cl_ntoh16(p_mgrp_holder->mlid));
+ while (!cl_is_qlist_empty(&p_mgrp_port->mgroups)) {
+ p_item = cl_qlist_remove_head(&p_mgrp_port->mgroups);
p_mgrp = (osm_mgrp_t *)
PARENT_STRUCT(p_item, osm_mgrp_t,port_item);
OSM_LOG(p_log, OSM_LOG_DEBUG,
@@ -460,3 +459,105 @@ void osm_mgrp_holder_port_delete_mgrp(osm_mgrp_holder_t * p_mgrp_holder,
p_mgrp_holder->last_change_id++;
}
}
+
+/**********************************************************************
+ **********************************************************************/
+void osm_mlid_pkey_delete(osm_mlid_pkey_t *p_mlid_pkey)
+{
+ cl_qmap_remove_all(&p_mlid_pkey->mlid_holder_map);
+ free(p_mlid_pkey);
+}
+
+/**********************************************************************
+ **********************************************************************/
+osm_mlid_pkey_t *osm_mlid_pkey_new(ib_net16_t pkey)
+{
+ osm_mlid_pkey_t *p_mlid_pkey = malloc(sizeof(osm_mlid_pkey_t));
+ if (!p_mlid_pkey) {
+ return NULL;
+ }
+ memset(p_mlid_pkey, 0, sizeof(*p_mlid_pkey));
+ cl_qmap_init(&p_mlid_pkey->mlid_holder_map);
+ p_mlid_pkey->pkey = pkey;
+ return p_mlid_pkey;
+}
+
+/**********************************************************************
+ **********************************************************************/
+void osm_mlid_pkey_add_holder(osm_mgrp_holder_t *p_mgrp_holder,
+ ib_net16_t pkey, osm_subn_t *p_subn)
+{
+ osm_mlid_pkey_t *p_mlid_pkey = (osm_mlid_pkey_t*)cl_qmap_get(&p_subn->mlid_pkey_tbl,
+ 0x7fff & pkey);
+ if (p_mlid_pkey != (osm_mlid_pkey_t*)cl_qmap_end(&p_subn->mlid_pkey_tbl)) {
+ cl_qmap_insert(&p_mlid_pkey->mlid_holder_map, p_mgrp_holder->mlid,&p_mgrp_holder->mlid_item);
+ }
+ else {
+ p_mlid_pkey = osm_mlid_pkey_new(pkey);
+ if (p_mlid_pkey) {
+ cl_qmap_insert(&p_mlid_pkey->mlid_holder_map, p_mgrp_holder->mlid,
+ &p_mgrp_holder->mlid_item);
+ cl_qmap_insert(&p_subn->mlid_pkey_tbl, 0x7fff & pkey,&p_mlid_pkey->pkey_item);
+ }
+ }
+}
+
+/**********************************************************************
+ **********************************************************************/
+void osm_mlid_pkey_remove_holder(osm_mgrp_holder_t *p_mgrp_holder,
+ ib_net16_t pkey, osm_subn_t *p_subn)
+{
+ osm_mlid_pkey_t *p_mlid_pkey = (osm_mlid_pkey_t*)
+ cl_qmap_get(&p_subn->mlid_pkey_tbl, 0x7fff & pkey);
+ if (p_mlid_pkey != (osm_mlid_pkey_t*)cl_qmap_end(&p_subn->mlid_pkey_tbl)) {
+ cl_qmap_remove_item(&p_mlid_pkey->mlid_holder_map, &p_mgrp_holder->mlid_item);
+ if (!cl_qmap_count(&p_mlid_pkey->mlid_holder_map)) {
+ /* no more groups with given pkey exist */
+ osm_mlid_pkey_delete(p_mlid_pkey);
+ }
+ }
+}
+
+/**********************************************************************
+ **********************************************************************/
+static ib_net16_t osm_mlid_pkey_get_mlid(IN osm_mlid_pkey_t *p_mlid_pkey)
+{
+ cl_map_item_t *p_item;
+ osm_mgrp_holder_t *p_mgrp_holder;
+ osm_mgrp_holder_t *p_matched_holder = NULL;
+ size_t port_count = 0;
+ for (p_item = cl_qmap_head(&p_mlid_pkey->mlid_holder_map);
+ p_item != cl_qmap_end(&p_mlid_pkey->mlid_holder_map);
+ p_item = cl_qmap_next(p_item)) {
+ p_mgrp_holder = (osm_mgrp_holder_t*)
+ PARENT_STRUCT(p_item, osm_mgrp_holder_t,mlid_item);
+ if (!port_count) {
+ /* init p_matched_holder and count */
+ port_count = cl_qmap_count(&p_mgrp_holder->mgrp_port_map);
+ p_matched_holder = p_mgrp_holder;
+ }
+ else {
+ if (port_count > cl_qmap_count(&p_mgrp_holder->mgrp_port_map)) {
+ port_count = cl_qmap_count(&p_mgrp_holder->mgrp_port_map);
+ p_matched_holder = p_mgrp_holder;
+ }
+ }
+ }
+ if (p_matched_holder) {
+ return p_matched_holder->mlid;
+ }
+ return 0;
+}
+
+/**********************************************************************
+ **********************************************************************/
+ib_net16_t osm_mlid_pkey_get_existed_mlid(IN osm_subn_t *p_subn, IN ib_net16_t pkey)
+{
+ osm_mlid_pkey_t *p_mlid_pkey =
+ (osm_mlid_pkey_t*)cl_qmap_get(&p_subn->mlid_pkey_tbl, 0x7fff & pkey);
+ if (p_mlid_pkey != (osm_mlid_pkey_t*)cl_qmap_end(&p_subn->mlid_pkey_tbl)) {
+ /* found obect with mgroups matched requested pkey */
+ return osm_mlid_pkey_get_mlid(p_mlid_pkey);
+ }
+ return 0;
+}
diff --git a/opensm/opensm/osm_sa_mcmember_record.c b/opensm/opensm/osm_sa_mcmember_record.c
index 3838a08..3c34592 100644
--- a/opensm/opensm/osm_sa_mcmember_record.c
+++ b/opensm/opensm/osm_sa_mcmember_record.c
@@ -152,6 +152,10 @@ static void cleanup_mgrp(IN osm_sa_t * sa, osm_mgrp_t * mgrp)
osm_mgrp_holder_t *p_mgrp_holder =
osm_get_mgrp_holder_by_mlid(sa->p_subn, mgrp->mlid);
osm_mgrp_holder_delete_mgrp(p_mgrp_holder, mgrp);
+ if (p_mgrp_holder->to_be_deleted) {
+ osm_mlid_pkey_remove_holder(p_mgrp_holder,
+ mgrp->mcmember_rec.pkey,sa->p_subn);
+ }
cl_fmap_remove_item(&sa->p_subn->mgrp_mgid_tbl,
&mgrp->map_item);
osm_mgrp_delete(mgrp);
@@ -812,13 +816,14 @@ ib_api_status_t osm_mcmr_rcv_create_new_mgrp(IN osm_sa_t * sa,
IN const osm_physp_t * p_physp,
OUT osm_mgrp_t ** pp_mgrp)
{
- ib_net16_t mlid, existed_mlid;
+ ib_net16_t mlid;
unsigned zero_mgid, i;
uint8_t scope;
ib_gid_t *p_mgid;
ib_api_status_t status = IB_SUCCESS;
ib_member_rec_t mcm_rec = *p_recvd_mcmember_rec; /* copy for modifications */
osm_mgrp_holder_t * p_mgrp_holder;
+ boolean_t new_mlid = TRUE;
OSM_LOG_ENTER(sa->p_log);
@@ -836,15 +841,22 @@ ib_api_status_t osm_mcmr_rcv_create_new_mgrp(IN osm_sa_t * sa,
*/
mlid = get_new_mlid(sa, mcm_rec.mlid);
if (mlid == 0) {
- OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1B19: "
- "get_new_mlid failed request mlid 0x%04x\n",
- cl_ntoh16(mcm_rec.mlid));
- status = IB_SA_MAD_STATUS_NO_RESOURCES;
- goto Exit;
+ /* try to add mcgroup to existed mlid */
+ mlid = osm_mlid_pkey_get_existed_mlid(sa->p_subn, mcm_rec.pkey);
+ if (mlid == 0) {
+ OSM_LOG(sa->p_log, OSM_LOG_ERROR, "ERR 1B19: "
+ "get_new_mlid failed request mlid 0x%04x\n",
+ cl_ntoh16(mcm_rec.mlid));
+ status = IB_SA_MAD_STATUS_NO_RESOURCES;
+ goto Exit;
+ }
+ new_mlid = FALSE;
+ OSM_LOG(sa->p_log, OSM_LOG_DEBUG,
+ "Found existed mlid 0x%X\n", cl_ntoh16(mlid));
}
OSM_LOG(sa->p_log, OSM_LOG_DEBUG,
- "Obtained new mlid 0x%X\n", cl_ntoh16(mlid));
+ "Obtained mlid 0x%X\n", cl_ntoh16(mlid));
/* we need to create the new MGID if it was not defined */
if (zero_mgid) {
@@ -894,15 +906,6 @@ ib_api_status_t osm_mcmr_rcv_create_new_mgrp(IN osm_sa_t * sa,
goto Exit;
}
- if (0 != (existed_mlid = osm_mgrp_holder_get_mlid_by_mgid(sa->p_subn, p_mgid))) {
- char gid_str[INET6_ADDRSTRLEN];
- mlid = existed_mlid;
- OSM_LOG(sa->p_log, OSM_LOG_DEBUG,
- "found existed mlid 0x%04x for mgid %s\n",
- cl_ntoh16(mlid), inet_ntop(AF_INET6, p_mgid->raw,
- gid_str, sizeof gid_str));
- }
-
/* create a new MC Group */
*pp_mgrp = osm_mgrp_new(mlid);
if (*pp_mgrp == NULL) {
@@ -947,6 +950,9 @@ ib_api_status_t osm_mcmr_rcv_create_new_mgrp(IN osm_sa_t * sa,
&(*pp_mgrp)->mcmember_rec.mgid, &(*pp_mgrp)->map_item);
osm_mgrp_holder_add_mgrp(p_mgrp_holder, *pp_mgrp, sa->p_log);
+ if (new_mlid)
+ osm_mlid_pkey_add_holder(p_mgrp_holder,
+ (*pp_mgrp)->mcmember_rec.pkey, sa->p_subn);
Exit:
OSM_LOG_EXIT(sa->p_log);
diff --git a/opensm/opensm/osm_subnet.c b/opensm/opensm/osm_subnet.c
index 6ed95d4..1826219 100644
--- a/opensm/opensm/osm_subnet.c
+++ b/opensm/opensm/osm_subnet.c
@@ -416,6 +416,7 @@ void osm_subn_construct(IN osm_subn_t * const p_subn)
cl_qmap_init(&p_subn->rtr_guid_tbl);
cl_qmap_init(&p_subn->prtn_pkey_tbl);
cl_fmap_init(&p_subn->mgrp_mgid_tbl, compar_mgids);
+ cl_qmap_init(&p_subn->mlid_pkey_tbl);
}
/**********************************************************************
@@ -431,6 +432,7 @@ void osm_subn_destroy(IN osm_subn_t * const p_subn)
osm_mgrp_holder_t *p_mgrp_holder;
osm_infr_t *p_infr, *p_next_infr;
osm_mgrp_t *p_mgrp;
+ osm_mlid_pkey_t *p_mlid_pkey;
/* it might be a good idea to de-allocate all known objects */
p_next_node = (osm_node_t *) cl_qmap_head(&p_subn->node_guid_tbl);
@@ -472,6 +474,12 @@ void osm_subn_destroy(IN osm_subn_t * const p_subn)
osm_prtn_delete(&p_prtn);
}
+ p_mlid_pkey = (osm_mlid_pkey_t*)cl_qmap_head(&p_subn->mlid_pkey_tbl);
+ while (p_mlid_pkey != (osm_mlid_pkey_t*)cl_qmap_end(&p_subn->mlid_pkey_tbl)) {
+ cl_qmap_remove_item(&p_subn->mlid_pkey_tbl, (cl_map_item_t*)p_mlid_pkey);
+ osm_mlid_pkey_delete(p_mlid_pkey);
+ p_mlid_pkey = (osm_mlid_pkey_t*)cl_qmap_head(&p_subn->mlid_pkey_tbl);
+ }
for (i = 0; i <= p_subn->max_mcast_lid_ho - IB_LID_MCAST_START_HO;
i++) {
--
1.6.3.3
More information about the general
mailing list