[openib-general] [PATCH] opensm: per-partition ipoib MC Group precreation
Sasha Khapyorsky
sashak at voltaire.com
Tue Mar 14 16:47:20 PST 2006
Hello,
There are changes in IPoIB MC Groups pre-creation - now it is on
per-partition basis.
This patch is for trunk.
Sasha.
Changes in IPoIB MC Groups pre-creation:
- IPoIB MC Group precreation is done on per-partition basis.
- IPoIB MC Group precreation is moved from osm_sa.c to osm_ptrn.c.
- TS workaround is applied for all IPoIB MCGs.
- No IPoIB MC group is created for partitions which is not marked for IPoIB
support (yet).
Signed-off-by: Sasha Khapyorsky <sashak at voltaire.com>
---
osm/include/opensm/osm_sa.h | 83 ---------------------------------
osm/opensm/osm_opensm.c | 14 -----
osm/opensm/osm_prtn.c | 56 ++++++++++++++++++----
osm/opensm/osm_prtn_config.c | 5 +-
osm/opensm/osm_sa.c | 107 ------------------------------------------
5 files changed, 50 insertions(+), 215 deletions(-)
diff --git a/osm/include/opensm/osm_sa.h b/osm/include/opensm/osm_sa.h
index 704fc2d..183b98f 100644
--- a/osm/include/opensm/osm_sa.h
+++ b/osm/include/opensm/osm_sa.h
@@ -442,89 +442,6 @@ osm_sa_bind(
* SEE ALSO
*********/
-/****f* OpenSM: SA/osm_sa_add_well_known_mc_record
-* NAME
-* osm_sa_add_well_known_mc_record
-*
-* DESCRIPTION
-* Adds the well known Multicast group to the SA database. This
-* should be called by SM before programming the switches and after
-* SA has been initialized
-*
-* SYNOPSIS
-*/
-
-void
-osm_sa_add_well_known_mc_record(
- osm_mcmr_recv_t* const p_ctrl,
- const ib_member_rec_t * const p_well_know_mc_rec);
-/*
-* PARAMETERS
-* p_ctrl
-* [in] Pointer to an osm_mcmr_recv_t object.
-*
-* p_well_know_mc_rec
-* [in] pointer to ib_member_rec_t structure.
-*
-*
-* RETURN VALUES
-* None
-*
-* NOTES
-* Called by SM after SA has been initialized and before programming the switches
-*
-* SEE ALSO
-*********/
-
-
-/****f* OpenSM: SA/osm_sa_create_template_record_ipoib
-* NAME
-* osm_sa_create_template_record_ipoib
-*
-* DESCRIPTION
-* Creates the well known MC record and calls osm_sa_add_well_known_mc_record. This
-* should be called by SM before programming the switches and after
-* SA has been initialized
-*
-* SYNOPSIS
-*/
-void
-osm_sa_create_template_record_ipoib(
- IN osm_sa_t* const p_sa,
- IN const osm_subn_opt_t* const p_opt );
-
-/*
-* PARAMETERS
-* p_sa
-* [in] Pointer to an osm_sa_t object.
-*
-* p_opt
-* [in] pointer to cmd line option structure.
-*
-*
-* RETURN VALUES
-* None
-*
-* NOTES
-* Called by SM after SA has been initialized and before programming the switches
-*
-* SEE ALSO
-*********/
-
-/****g* OpenSM: SA/osm_ipoib_mgid
-* NAME
-* osm_ipoib_mgid
-*
-* DESCRIPTION
-* The MGID of the IPoIB Multicast Group
-*
-* SYNOPSIS
-*/
-extern ib_gid_t osm_ipoib_mgid;
-/*
-* SEE ALSO
-*********/
-
END_C_DECLS
#endif /* _OSM_SA_H_ */
diff --git a/osm/opensm/osm_opensm.c b/osm/opensm/osm_opensm.c
index 06a04e5..9a64eb2 100644
--- a/osm/opensm/osm_opensm.c
+++ b/osm/opensm/osm_opensm.c
@@ -137,18 +137,6 @@ osm_opensm_destroy(
/**********************************************************************
**********************************************************************/
-static void
-osm_opensm_create_mcgroups(
- IN osm_opensm_t * const p_osm,
- IN const osm_subn_opt_t * const p_opt )
-{
- OSM_LOG_ENTER( &p_osm->log, osm_opensm_create_mcgroups );
- osm_sa_create_template_record_ipoib( &p_osm->sa, p_opt );
- OSM_LOG_EXIT( &p_osm->log );
-}
-
-/**********************************************************************
- **********************************************************************/
ib_api_status_t
osm_opensm_init(
IN osm_opensm_t * const p_osm,
@@ -266,8 +254,6 @@ osm_opensm_init(
if( status != IB_SUCCESS )
goto Exit;
- osm_opensm_create_mcgroups( p_osm, p_opt );
-
/* HACK - the UpDown manager should have been a part of the osm_sm_t */
/* Init updn struct */
p_osm->p_updn_ucast_routing = updn_construct( );
diff --git a/osm/opensm/osm_prtn.c b/osm/opensm/osm_prtn.c
index 8ffb584..2d006f4 100644
--- a/osm/opensm/osm_prtn.c
+++ b/osm/opensm/osm_prtn.c
@@ -168,6 +168,34 @@ ib_api_status_t osm_prtn_add_all(osm_log
return status;
}
+
+static const ib_gid_t osm_ipoib_mgid = {
+ {
+ 0xff, /* multicast field */
+ 0x12, /* non-permanent bit,scope */
+ 0x40, 0x1b, /* IPv4 signature */
+ 0xff, 0xff, /* 16 bits of P_Key (to be filled in) */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 48 bits of zeros */
+ 0xff, 0xff, 0xff, 0xff, /* 32 bit IPv4 broadcast address */
+ },
+};
+
+/*
+ * HACK: Until TS resolves their noncompliant join compmask
+ * we have to pre-define the MGID
+ */
+static const ib_gid_t osm_ts_ipoib_mgid = {
+ {
+ 0xff, /* multicast field */
+ 0x12, /* non-permanent bit,scope */
+ 0x40, 0x1b, /* IPv4 signature */
+ 0xff, 0xff, /* 16 bits of P_Key (to be filled in) */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 48 bits of zeros */
+ 0x00, 0x00, 0x00, 0x01, /* 32 bit IPv4 broadcast address */
+ },
+};
+
+
ib_api_status_t osm_prtn_add_mcgroup(osm_log_t *p_log,
osm_subn_t *p_subn, osm_prtn_t *p,
unsigned is_ipoib, uint8_t rate, uint8_t mtu)
@@ -179,7 +207,7 @@ ib_api_status_t osm_prtn_add_mcgroup(osm
osm_sa_t *p_sa = &p_subn->p_osm->sa;
ib_api_status_t status = IB_SUCCESS;
- pkey = is_ipoib ? cl_hton16(cl_ntoh16(p->pkey) | 0x8000) : p->pkey;
+ pkey = cl_hton16(cl_ntoh16(p->pkey) | 0x8000);
cl_memclr(&mc_rec, sizeof(mc_rec));
@@ -201,12 +229,20 @@ ib_api_status_t osm_prtn_add_mcgroup(osm
status = osm_mcmr_rcv_find_or_create_new_mgrp(&p_sa->mcmr_rcv,
comp_mask, &mc_rec, &p_mgrp);
-
if (!p_mgrp || status != IB_SUCCESS)
osm_log( p_log, OSM_LOG_ERROR,
"osm_prtn_add_mcgroup: "
"Failed to create MC group with pkey %04x\n",
cl_ntoh16(pkey));
+ p_mgrp->well_known = TRUE;
+
+ /* workaround for TS */ /* FIXME: remove this upon TS fixes */
+ mc_rec.mgid = osm_ts_ipoib_mgid;
+ cl_memcpy(&mc_rec.mgid.raw[4], &pkey, sizeof(pkey));
+ status = osm_mcmr_rcv_find_or_create_new_mgrp(&p_sa->mcmr_rcv,
+ comp_mask, &mc_rec, &p_mgrp);
+ if (p_mgrp)
+ p_mgrp->well_known = TRUE;
return status;
}
@@ -267,7 +303,7 @@ osm_prtn_t *osm_prtn_make_new(osm_log_t
static ib_api_status_t osm_prtn_make_default(osm_log_t * const p_log,
osm_subn_t * const p_subn,
- boolean_t is_config)
+ boolean_t no_config)
{
ib_api_status_t status = IB_UNKNOWN_ERROR;
osm_prtn_t *p;
@@ -275,13 +311,15 @@ static ib_api_status_t osm_prtn_make_def
p = osm_prtn_make_new(p_log, p_subn, "Default", IB_DEFAULT_PARTIAL_PKEY);
if (!p)
goto _err;
- status = osm_prtn_add_all(p_log, p_subn, p,
- (is_config == TRUE) ? FALSE : TRUE);
+ status = osm_prtn_add_all(p_log, p_subn, p, no_config);
if (status != IB_SUCCESS)
goto _err;
cl_map_remove(&p->part_guid_tbl, p_subn->sm_port_guid);
status = osm_prtn_add_port(p_log, p_subn, p, p_subn->sm_port_guid, TRUE);
+ if (no_config)
+ osm_prtn_add_mcgroup(p_log, p_subn, p, 1, 0, 0);
+
_err:
return status;
}
@@ -289,11 +327,11 @@ static ib_api_status_t osm_prtn_make_def
ib_api_status_t osm_prtn_make_partitions(osm_log_t * const p_log,
osm_subn_t * const p_subn)
{
+ struct stat statbuf;
const char *file_name;
+ boolean_t is_config = TRUE;
ib_api_status_t status = IB_SUCCESS;
osm_prtn_t *p, *p_next;
- boolean_t is_config = TRUE;
- struct stat statbuf;
file_name = p_subn->opt.partition_config_file ?
p_subn->opt.partition_config_file :
@@ -313,11 +351,11 @@ ib_api_status_t osm_prtn_make_partitions
global_pkey_counter = 0;
- status = osm_prtn_make_default(p_log, p_subn, is_config);
+ status = osm_prtn_make_default(p_log, p_subn, !is_config);
if (status != IB_SUCCESS)
goto _err;
- if (osm_prtn_config_parse_file(p_log, p_subn, file_name)) {
+ if (is_config && osm_prtn_config_parse_file(p_log, p_subn, file_name)) {
osm_log(p_log, OSM_LOG_VERBOSE,
"osm_prtn_make_partitions: Partition configuration "
"was not fully processed\n");
diff --git a/osm/opensm/osm_prtn_config.c b/osm/opensm/osm_prtn_config.c
index 2ba427b..97f3022 100644
--- a/osm/opensm/osm_prtn_config.c
+++ b/osm/opensm/osm_prtn_config.c
@@ -123,8 +123,9 @@ static int partition_create(unsigned lin
name, cl_hton16(pkey));
if (!conf->p_prtn)
return -1;
-
- osm_prtn_add_mcgroup(conf->p_log, conf->p_subn, conf->p_prtn,
+
+ if (conf->is_ipoib)
+ osm_prtn_add_mcgroup(conf->p_log, conf->p_subn, conf->p_prtn,
conf->is_ipoib, conf->rate, conf->mtu);
return 0;
diff --git a/osm/opensm/osm_sa.c b/osm/opensm/osm_sa.c
index 5bbe6f0..e186162 100644
--- a/osm/opensm/osm_sa.c
+++ b/osm/opensm/osm_sa.c
@@ -503,110 +503,3 @@ osm_sa_bind(
OSM_LOG_EXIT( p_sa->p_log );
return( status );
}
-
-/**********************************************************************
- **********************************************************************/
-
-ib_gid_t osm_ipoib_mgid = {
- {
- 0xff, /* multicast field */
- 0x12, /* non-permanent bit,scope */
- 0x40, 0x1b, /* IPv4 signature */
- 0xff, 0xff, /* 16 bits of P_Key (to be filled in) */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 48 bits of zeros */
- 0xff, 0xff, 0xff, 0xff, /* 32 bit IPv4 broadcast address */
- },
-};
-
-/*
- * HACK: Until TS resolves their noncompliant join compmask
- * we have to pre-define the MGID
- */
-ib_gid_t osm_ts_ipoib_mgid = {
- {
- 0xff, /* multicast field */
- 0x12, /* non-permanent bit,scope */
- 0x40, 0x1b, /* IPv4 signature */
- 0xff, 0xff, /* 16 bits of P_Key (to be filled in) */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 48 bits of zeros */
- 0x00, 0x00, 0x00, 0x01, /* 32 bit IPv4 broadcast address */
- },
-};
-
-void
-osm_sa_create_template_record_ipoib(
- IN osm_sa_t* const p_sa,
- IN const osm_subn_opt_t* const p_opt )
-{
- ib_member_rec_t mc_rec;
-
- osm_log( p_sa->p_log, OSM_LOG_FUNCS,
- "osm_sa_create_template_record_ipoib: [\n" );
-
- UNUSED_PARAM( p_opt );
- cl_memclr(&mc_rec, sizeof(mc_rec));
-
- /*
- * Fill in the default MC Member record
- */
- mc_rec.mgid = osm_ipoib_mgid;
- mc_rec.mtu = 4; /* 2048 Bytes */
- mc_rec.qkey = CL_HTON32(0x0b1b); /* This value is pushed into the mad, and thus needs
- to be in network order */
- mc_rec.pkey = IB_DEFAULT_PKEY;
- mc_rec.rate = 0x3; /* 10Gb/sec */
- mc_rec.pkt_life = OSM_DEFAULT_SUBNET_TIMEOUT;
- mc_rec.sl_flow_hop = OSM_DEFAULT_SL << 28;
- /* Note: scope needs to be consistent with MGID */
- mc_rec.scope_state = 0x21;
-
- osm_sa_add_well_known_mc_record(&p_sa->mcmr_rcv, &mc_rec);
-
- /*
- * HACK: Until TS resolves their noncompliant join compmask
- * we have to predefine the MGID
- */
- mc_rec.mgid = osm_ts_ipoib_mgid;
- osm_sa_add_well_known_mc_record(&p_sa->mcmr_rcv, &mc_rec);
-
- osm_log( p_sa->p_log, OSM_LOG_FUNCS,
- "osm_sa_create_template_record_ipoib: ]\n" );
-
-}
-
-/**********************************************************************
- **********************************************************************/
-void
-osm_sa_add_well_known_mc_record(
- osm_mcmr_recv_t* const p_mcmr,
- const ib_member_rec_t * const p_well_know_mc_rec)
-{
-
- osm_mgrp_t * p_mgrp = NULL;
- ib_api_status_t status;
- ib_net64_t comp_mask;
-
- comp_mask = IB_MCR_COMPMASK_MTU | IB_MCR_COMPMASK_RATE;
- status = osm_mcmr_rcv_create_new_mgrp(
- p_mcmr,
- comp_mask,
- p_well_know_mc_rec,
- NULL,
- &p_mgrp);
- if(p_mgrp)
- {
- p_mgrp->well_known = TRUE;
- }
- else
- {
- osm_log( p_mcmr->p_log, OSM_LOG_DEBUG,
- "osm_sa_add_well_known_mc_record:"
- "Failed to create a well known group\n");
-
- }
-
-}
-
-/**********************************************************************
- **********************************************************************/
-
More information about the general
mailing list