<br><br>
<div class="gmail_quote">On Sun, Sep 6, 2009 at 1:39 PM, Sasha Khapyorsky <span dir="ltr"><<a href="mailto:sashak@voltaire.com" target="_blank">sashak@voltaire.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="PADDING-LEFT: 1ex; MARGIN: 0px 0px 0px 0.8ex; BORDER-LEFT: #ccc 1px solid"><br>When we have two or more changes in a same multicast group multiple<br>multicast rerouting requests will be created and processed. To prevent<br>
this we will use array of requests indexed by mlid value minus<br>IB_LID_MCAST_START_HO and for each multicast group change we will just<br>mark that specific mlid requires re-routing and "duplicated" requests<br>
will be merged there.<br><br>Also in this way we will be able to process multicast group routing<br>entries deletion for already removed groups by just knowing its MLID<br>and not using its content - this will let us to not delay mutlicast<br>
groups deletion ('to_be_deleted' flag) and will simplify many multicast<br>related code flows.<br></blockquote>
<div> </div>
<div>While the delay adds complexity, it is a feature. Delayed deletion (and join) is allowed by IBA and is needed in a fast changing subnet when there are a lot of groups changing. This was seen quite a while ago and was how OpenSM evolved based on field experience and other testing. Eitan is the expert here. IMO support for this needs to be added (back in).</div>
<div> </div>
<div>-- Hal</div>
<div> </div>
<div> </div>
<blockquote class="gmail_quote" style="PADDING-LEFT: 1ex; MARGIN: 0px 0px 0px 0.8ex; BORDER-LEFT: #ccc 1px solid"><span></span><br>Signed-off-by: Sasha Khapyorsky <<a href="mailto:sashak@voltaire.com" target="_blank">sashak@voltaire.com</a>><br>
---<br> opensm/include/opensm/osm_sm.h | 4 +-<br> opensm/opensm/osm_mcast_mgr.c | 27 +++++++++------------<br> opensm/opensm/osm_sm.c | 49 +++++++++++++---------------------------<br> 3 files changed, 30 insertions(+), 50 deletions(-)<br>
<br>diff --git a/opensm/include/opensm/osm_sm.h b/opensm/include/opensm/osm_sm.h<br>index 0914a95..986143a 100644<br>--- a/opensm/include/opensm/osm_sm.h<br>+++ b/opensm/include/opensm/osm_sm.h<br>@@ -126,8 +126,8 @@ typedef struct osm_sm {<br>
cl_dispatcher_t *p_disp;<br> cl_plock_t *p_lock;<br> atomic32_t sm_trans_id;<br>- cl_spinlock_t mgrp_lock;<br>- cl_qlist_t mgrp_list;<br>+ unsigned mlids_req_max;<br>+ uint8_t *mlids_req;<br>
osm_sm_mad_ctrl_t mad_ctrl;<br> osm_lid_mgr_t lid_mgr;<br> osm_ucast_mgr_t ucast_mgr;<br>diff --git a/opensm/opensm/osm_mcast_mgr.c b/opensm/opensm/osm_mcast_mgr.c<br>index d7c5ce1..dd504ef 100644<br>--- a/opensm/opensm/osm_mcast_mgr.c<br>
+++ b/opensm/opensm/osm_mcast_mgr.c<br>@@ -1116,7 +1116,6 @@ static int mcast_mgr_set_mftables(osm_sm_t * sm)<br> int osm_mcast_mgr_process(osm_sm_t * sm)<br> {<br> cl_qmap_t *p_sw_tbl;<br>- cl_qlist_t *p_list = &sm->mgrp_list;<br>
osm_mgrp_t *p_mgrp;<br> int i, ret = 0;<br><br>@@ -1150,16 +1149,14 @@ int osm_mcast_mgr_process(osm_sm_t * sm)<br> mcast_mgr_process_mgrp(sm, p_mgrp);<br> }<br><br>+ memset(sm->mlids_req, 0, sm->mlids_req_max);<br>
+ sm->mlids_req_max = 0;<br>+<br> /*<br> Walk the switches and download the tables for each.<br> */<br> ret = mcast_mgr_set_mftables(sm);<br><br>- while (!cl_is_qlist_empty(p_list)) {<br>
- cl_list_item_t *p = cl_qlist_remove_head(p_list);<br>- free(p);<br>- }<br>-<br> exit:<br> CL_PLOCK_RELEASE(sm->p_lock);<br><br>@@ -1174,11 +1171,10 @@ exit:<br> **********************************************************************/<br>
int osm_mcast_mgr_process_mgroups(osm_sm_t * sm)<br> {<br>- cl_qlist_t *p_list = &sm->mgrp_list;<br> osm_mgrp_t *p_mgrp;<br> ib_net16_t mlid;<br>- osm_mcast_mgr_ctxt_t *ctx;<br> int ret = 0;<br>
+ unsigned i;<br><br> OSM_LOG_ENTER(sm->p_log);<br><br>@@ -1192,14 +1188,12 @@ int osm_mcast_mgr_process_mgroups(osm_sm_t * sm)<br> goto exit;<br> }<br><br>- while (!cl_is_qlist_empty(p_list)) {<br>
- ctx = (osm_mcast_mgr_ctxt_t *) cl_qlist_remove_head(p_list);<br>-<br>- /* nice copy no warning on size diff */<br>- memcpy(&mlid, &ctx->mlid, sizeof(mlid));<br>+ for (i = 0; i <= sm->mlids_req_max; i++) {<br>
+ if (!sm->mlids_req[i])<br>+ continue;<br>+ sm->mlids_req[i] = 0;<br><br>- /* we can destroy the context now */<br>- free(ctx);<br>+ mlid = cl_hton16(i + IB_LID_MCAST_START_HO);<br>
<br> /* since we delayed the execution we prefer to pass the<br> mlid as the mgrp identifier and then find it or abort */<br>@@ -1223,6 +1217,9 @@ int osm_mcast_mgr_process_mgroups(osm_sm_t * sm)<br>
mcast_mgr_process_mgrp(sm, p_mgrp);<br> }<br><br>+ memset(sm->mlids_req, 0, sm->mlids_req_max);<br>+ sm->mlids_req_max = 0;<br>+<br> /*<br> Walk the switches and download the tables for each.<br>
*/<br>diff --git a/opensm/opensm/osm_sm.c b/opensm/opensm/osm_sm.c<br>index 50aee91..e446c9d 100644<br>--- a/opensm/opensm/osm_sm.c<br>+++ b/opensm/opensm/osm_sm.c<br>@@ -166,7 +166,6 @@ void osm_sm_construct(IN osm_sm_t * p_sm)<br>
cl_event_construct(&p_sm->subnet_up_event);<br> cl_event_wheel_construct(&p_sm->trap_aging_tracker);<br> cl_thread_construct(&p_sm->sweeper);<br>- cl_spinlock_construct(&p_sm->mgrp_lock);<br>
osm_sm_mad_ctrl_construct(&p_sm->mad_ctrl);<br> osm_lid_mgr_construct(&p_sm->lid_mgr);<br> osm_ucast_mgr_construct(&p_sm->ucast_mgr);<br>@@ -234,8 +233,8 @@ void osm_sm_destroy(IN osm_sm_t * p_sm)<br>
cl_event_destroy(&p_sm->signal_event);<br> cl_event_destroy(&p_sm->subnet_up_event);<br> cl_spinlock_destroy(&p_sm->signal_lock);<br>- cl_spinlock_destroy(&p_sm->mgrp_lock);<br>
cl_spinlock_destroy(&p_sm->state_lock);<br>+ free(p_sm->mlids_req);<br><br> osm_log(p_sm->p_log, OSM_LOG_SYS, "Exiting SM\n"); /* Format Waived */<br> OSM_LOG_EXIT(p_sm->p_log);<br>
@@ -288,11 +287,14 @@ ib_api_status_t osm_sm_init(IN osm_sm_t * p_sm, IN osm_subn_t * p_subn,<br> if (status != CL_SUCCESS)<br> goto Exit;<br><br>- cl_qlist_init(&p_sm->mgrp_list);<br>-<br>
- status = cl_spinlock_init(&p_sm->mgrp_lock);<br>- if (status != CL_SUCCESS)<br>+ p_sm->mlids_req_max = 0;<br>+ p_sm->mlids_req = malloc((IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO +<br>
+ 1) * sizeof(p_sm->mlids_req[0]));<br>+ if (!p_sm->mlids_req)<br> goto Exit;<br>+ memset(p_sm->mlids_req, 0,<br>+ (IB_LID_MCAST_END_HO - IB_LID_MCAST_START_HO +<br>
+ 1) * sizeof(p_sm->mlids_req[0]));<br><br> status = osm_sm_mad_ctrl_init(&p_sm->mad_ctrl, p_sm->p_subn,<br> p_sm->p_mad_pool, p_sm->p_vl15,<br>@@ -441,32 +443,15 @@ Exit:<br>
<br> /**********************************************************************<br> **********************************************************************/<br>-static ib_api_status_t sm_mgrp_process(IN osm_sm_t * p_sm,<br>- IN osm_mgrp_t * p_mgrp)<br>
+static void request_mlid(osm_sm_t * sm, uint16_t mlid)<br> {<br>- osm_mcast_mgr_ctxt_t *ctx;<br>-<br>- /*<br>- * 'Schedule' all the QP0 traffic for when the state manager<br>- * isn't busy trying to do something else.<br>
- */<br>- ctx = malloc(sizeof(*ctx));<br>- if (!ctx)<br>- return IB_ERROR;<br>- memset(ctx, 0, sizeof(*ctx));<br>- ctx->mlid = p_mgrp->mlid;<br>-<br>- cl_spinlock_acquire(&p_sm->mgrp_lock);<br>
- cl_qlist_insert_tail(&p_sm->mgrp_list, &ctx->list_item);<br>- cl_spinlock_release(&p_sm->mgrp_lock);<br>-<br>- osm_sm_signal(p_sm, OSM_SIGNAL_IDLE_TIME_PROCESS_REQUEST);<br>-<br>- return IB_SUCCESS;<br>
+ mlid -= IB_LID_MCAST_START_HO;<br>+ sm->mlids_req[mlid] = 1;<br>+ if (sm->mlids_req_max < mlid)<br>+ sm->mlids_req_max = mlid;<br>+ osm_sm_signal(sm, OSM_SIGNAL_IDLE_TIME_PROCESS_REQUEST);<br>
}<br><br>-/**********************************************************************<br>- **********************************************************************/<br> ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN osm_mgrp_t *mgrp,<br>
IN const ib_net64_t port_guid)<br> {<br>@@ -519,7 +504,7 @@ ib_api_status_t osm_sm_mcgrp_join(IN osm_sm_t * p_sm, IN osm_mgrp_t *mgrp,<br> goto Exit;<br> }<br><br>- status = sm_mgrp_process(p_sm, mgrp);<br>
+ request_mlid(p_sm, cl_ntoh16(mgrp->mlid));<br> Exit:<br> CL_PLOCK_RELEASE(p_sm->p_lock);<br> OSM_LOG_EXIT(p_sm->p_log);<br>@@ -527,8 +512,6 @@ Exit:<br> return status;<br> }<br><br>-/**********************************************************************<br>
- **********************************************************************/<br> ib_api_status_t osm_sm_mcgrp_leave(IN osm_sm_t * p_sm, IN osm_mgrp_t *mgrp,<br> IN const ib_net64_t port_guid)<br>
{<br>@@ -557,7 +540,7 @@ ib_api_status_t osm_sm_mcgrp_leave(IN osm_sm_t * p_sm, IN osm_mgrp_t *mgrp,<br><br> osm_port_remove_mgrp(p_port, mgrp);<br><br>- status = sm_mgrp_process(p_sm, mgrp);<br>+ request_mlid(p_sm, cl_hton16(mgrp->mlid));<br>
Exit:<br> CL_PLOCK_RELEASE(p_sm->p_lock);<br> OSM_LOG_EXIT(p_sm->p_log);<br>--<br>1.6.4.2<br><br>_______________________________________________<br>general mailing list<br><a href="mailto:general@lists.openfabrics.org" target="_blank">general@lists.openfabrics.org</a><br>
<a href="http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general" target="_blank">http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general</a><br><br>To unsubscribe, please visit <a href="http://openib.org/mailman/listinfo/openib-general" target="_blank">http://openib.org/mailman/listinfo/openib-general</a><br>
</blockquote></div><br>