[Openib-windows] [PATCH] CM: Handle stale connections
Fab Tillier
ftillier at silverstorm.com
Wed Apr 5 16:45:31 PDT 2006
Hi Tzachi,
Here's my patch for the CEP manager for handling the stale connection when
receiving a REP. I implemented generating a local REJ event for the local
endpoint too, which had been left as TODOs.
The patch looks like more changes than it is. I took __process_rej and split it
into two - __rej_handler and __process_rej.
__rej_handler is what the receive callback invokes, and performs the lookup for
a matching CEP. If found, it calls __process_rej to generate the local REJ
event. Note that with this rename, all the other __process_<CM message>
functions were renamed __<CM message>_handler, e.g. __req_handler).
__process_rej now just does the REJ processing, assuming a valid CEP as input.
There is then a new function, __process_stale, which allocates a MAD, formats
the REJ as if it had been received from the remote side, and then calls
__process_rej to generate the REJ event. __process_stale is called when a stale
connection is detected during REQ matching (on the passive side), as well as
when a stale connection is detected during REP matching (on the active side).
I've also added warning messages to the debug output should a stale connection
be encountered.
Could you test this and see if it works for you with SDP? If it does I'll
commit it.
Thanks,
- Fab
Index: core/al/kernel/al_cm_cep.c
===================================================================
--- core/al/kernel/al_cm_cep.c (revision 276)
+++ core/al/kernel/al_cm_cep.c (working copy)
@@ -457,6 +457,13 @@
IN kcep_t* const
p_cep );
static ib_api_status_t
+__cep_get_mad(
+ IN kcep_t* const
p_cep,
+ IN net16_t
attr_id,
+ OUT cep_agent_t** const
pp_port_cep,
+ OUT ib_mad_element_t** const pp_mad
);
+
+static ib_api_status_t
__cep_send_mad(
IN cep_agent_t* const
p_port_cep,
IN ib_mad_element_t* const p_mad );
@@ -863,14 +870,155 @@
}
+static ib_api_status_t
+__process_rej(
+ IN kcep_t* const
p_cep,
+ IN ib_mad_element_t* const p_mad )
+{
+ ib_api_status_t status;
+ mad_cm_rej_t *p_rej;
+
+ AL_ENTER( AL_DBG_CM );
+
+ ASSERT( p_cep );
+ ASSERT( p_mad );
+ ASSERT( p_mad->p_mad_buf );
+
+ p_rej = (mad_cm_rej_t*)p_mad->p_mad_buf;
+
+ switch( p_cep->state )
+ {
+ case CEP_STATE_REQ_SENT:
+ /*
+ * Ignore rejects with the status set to IB_REJ_INVALID_SID. We
will
+ * continue to retry (up to max_cm_retries) to connect to the
remote
+ * side. This is required to support peer-to-peer connections
and
+ * clients that try to connect before the server comes up.
+ */
+ if( p_rej->reason == IB_REJ_INVALID_SID )
+ {
+ AL_TRACE( AL_DBG_CM,
+ ("Request rejected (invalid SID) - retrying.\n")
);
+ goto err1;
+ }
+
+ /* Fall through */
+ case CEP_STATE_REP_SENT:
+ case CEP_STATE_REQ_MRA_RCVD:
+ case CEP_STATE_REP_MRA_RCVD:
+ /* Cancel any outstanding MAD. */
+ if( p_cep->p_send_mad )
+ {
+ ib_cancel_mad( p_cep->h_mad_svc, p_cep->p_send_mad );
+ p_cep->p_send_mad = NULL;
+ }
+
+ /* Fall through */
+ case CEP_STATE_REQ_RCVD:
+ case CEP_STATE_REP_RCVD:
+ case CEP_STATE_REQ_MRA_SENT:
+ case CEP_STATE_REP_MRA_SENT:
+ case CEP_STATE_PRE_REP:
+ case CEP_STATE_PRE_REP_MRA_SENT:
+ if( p_cep->state & CEP_STATE_PREP )
+ {
+ CL_ASSERT( p_cep->p_mad );
+ ib_put_mad( p_cep->p_mad );
+ p_cep->p_mad = NULL;
+ }
+ /* Abort connection establishment. No transition to timewait. */
+ __remove_cep( p_cep );
+ p_cep->state = CEP_STATE_IDLE;
+ break;
+
+ case CEP_STATE_ESTABLISHED:
+ case CEP_STATE_LAP_RCVD:
+ case CEP_STATE_LAP_SENT:
+ case CEP_STATE_LAP_MRA_RCVD:
+ case CEP_STATE_LAP_MRA_SENT:
+ case CEP_STATE_PRE_APR:
+ case CEP_STATE_PRE_APR_MRA_SENT:
+ if( p_cep->state & CEP_STATE_PREP )
+ {
+ CL_ASSERT( p_cep->p_mad );
+ ib_put_mad( p_cep->p_mad );
+ p_cep->p_mad = NULL;
+ }
+ p_cep->state = CEP_STATE_TIMEWAIT;
+ __insert_timewait( p_cep );
+ break;
+
+ default:
+ /* Ignore the REJ. */
+ AL_TRACE( AL_DBG_CM, ("REJ received in invalid state.\n") );
+err1:
+ ib_put_mad( p_mad );
+ AL_EXIT( AL_DBG_CM );
+ return IB_NO_MATCH;
+ }
+
+ status = __cep_queue_mad( p_cep, p_mad );
+
+ AL_EXIT( AL_DBG_CM );
+ return status;
+}
+
+
+static ib_api_status_t
+__process_stale(
+ IN kcep_t* const
p_cep )
+{
+ ib_api_status_t status;
+ cep_agent_t *p_port_cep;
+ ib_mad_element_t *p_mad;
+ mad_cm_rej_t *p_rej;
+
+ status = __cep_get_mad( p_cep, CM_REJ_ATTR_ID, &p_port_cep, &p_mad );
+ if( status != IB_SUCCESS )
+ return status;
+
+ p_rej = ib_get_mad_buf( p_mad );
+
+ conn_rej_set_ari( NULL, 0, p_rej );
+ conn_rej_set_pdata( NULL, 0, p_rej );
+
+ p_rej->local_comm_id = p_cep->remote_comm_id;
+ p_rej->remote_comm_id = p_cep->local_comm_id;
+ p_rej->reason = IB_REJ_STALE_CONN;
+
+ switch( p_cep->state )
+ {
+ case CEP_STATE_REQ_RCVD:
+ case CEP_STATE_REQ_MRA_SENT:
+ case CEP_STATE_PRE_REP:
+ case CEP_STATE_PRE_REP_MRA_SENT:
+ conn_rej_set_msg_rejected( 0, p_rej );
+ break;
+
+ case CEP_STATE_REQ_SENT:
+ case CEP_STATE_REP_RCVD:
+ case CEP_STATE_REP_MRA_SENT:
+ conn_rej_set_msg_rejected( 1, p_rej );
+ break;
+
+ default:
+ conn_rej_set_msg_rejected( 2, p_rej );
+ break;
+ }
+ conn_rej_clr_rsvd_fields( p_rej );
+
+ return __process_rej( p_cep, p_mad );
+}
+
+
static void
-__process_req(
+__req_handler(
IN cep_agent_t* const
p_port_cep,
IN ib_mad_element_t* const p_mad )
{
- ib_api_status_t status;
+ ib_api_status_t status = IB_SUCCESS;
mad_cm_req_t *p_req;
- kcep_t *p_cep, *p_new_cep, *p_stale_cep;
+ kcep_t *p_cep, *p_new_cep, *p_stale_cep = NULL;
KLOCK_QUEUE_HANDLE hdl;
ib_rej_status_t reason;
@@ -958,8 +1106,9 @@
if( p_stale_cep != p_new_cep )
{
/* Duplicate - must be a stale connection. */
- /* TODO: Fail the CEP in p_stale_cep */
reason = IB_REJ_STALE_CONN;
+ /* Fail the local stale CEP. */
+ status = __process_stale( p_stale_cep );
goto unbind;
}
@@ -1040,6 +1189,10 @@
__reject_req( p_port_cep, p_mad, reason );
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
+
+ if( reason == IB_REJ_STALE_CONN && status == IB_SUCCESS )
+ __process_cep( p_stale_cep );
+
AL_EXIT( AL_DBG_CM );
}
@@ -1087,7 +1240,7 @@
static void
-__process_mra(
+__mra_handler(
IN ib_mad_element_t* const p_mad )
{
ib_api_status_t status;
@@ -1119,13 +1272,13 @@
goto err;
}
}
+
/*
* Note that we don't update the CEP's remote comm ID - it messes up REP
* processing since a non-zero RCID implies the connection is in the
RCID
* map. Adding it here requires checking there and conditionally adding
* it. Ignoring it is a valid thing to do.
*/
-
if( !(p_cep->state & CEP_STATE_SENT) ||
(1 << conn_mra_get_msg_mraed( p_mra ) !=
(p_cep->state & CEP_MSG_MASK)) )
@@ -1152,7 +1305,6 @@
p_cep->state |= CEP_STATE_MRA;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
@@ -1170,7 +1322,7 @@
static void
-__process_rej(
+__rej_handler(
IN ib_mad_element_t* const p_mad )
{
ib_api_status_t status;
@@ -1210,98 +1362,26 @@
if( p_cep->remote_comm_id &&
p_cep->remote_comm_id != p_rej->local_comm_id )
{
- goto err2;
+ err2:
+ KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
+ err1:
+ ib_put_mad( p_mad );
+ AL_EXIT( AL_DBG_CM );
}
- switch( p_cep->state )
- {
- case CEP_STATE_REQ_SENT:
- /*
- * Ignore rejects with the status set to IB_REJ_INVALID_SID. We
will
- * continue to retry (up to max_cm_retries) to connect to the
remote
- * side. This is required to support peer-to-peer connections
and
- * clients that try to connect before the server comes up.
- */
- if( p_rej->reason == IB_REJ_INVALID_SID )
- {
- AL_TRACE( AL_DBG_CM,
- ("Request rejected (invalid SID) - retrying.\n")
);
- goto err2;
- }
+ status = __process_rej( p_cep, p_mad );
- /* Fall through */
- case CEP_STATE_REP_SENT:
- case CEP_STATE_REQ_MRA_RCVD:
- case CEP_STATE_REP_MRA_RCVD:
- /* Cancel any outstanding MAD. */
- if( p_cep->p_send_mad )
- {
- ib_cancel_mad( p_cep->h_mad_svc, p_cep->p_send_mad );
- p_cep->p_send_mad = NULL;
- }
-
- /* Fall through */
- case CEP_STATE_REQ_RCVD:
- case CEP_STATE_REP_RCVD:
- case CEP_STATE_REQ_MRA_SENT:
- case CEP_STATE_REP_MRA_SENT:
- case CEP_STATE_PRE_REP:
- case CEP_STATE_PRE_REP_MRA_SENT:
- if( p_cep->state & CEP_STATE_PREP )
- {
- CL_ASSERT( p_cep->p_mad );
- ib_put_mad( p_cep->p_mad );
- p_cep->p_mad = NULL;
- }
- /* Abort connection establishment. No transition to timewait. */
- __remove_cep( p_cep );
- p_cep->state = CEP_STATE_IDLE;
- break;
-
- case CEP_STATE_ESTABLISHED:
- case CEP_STATE_LAP_RCVD:
- case CEP_STATE_LAP_SENT:
- case CEP_STATE_LAP_MRA_RCVD:
- case CEP_STATE_LAP_MRA_SENT:
- case CEP_STATE_PRE_APR:
- case CEP_STATE_PRE_APR_MRA_SENT:
- if( p_cep->state & CEP_STATE_PREP )
- {
- CL_ASSERT( p_cep->p_mad );
- ib_put_mad( p_cep->p_mad );
- p_cep->p_mad = NULL;
- }
- p_cep->state = CEP_STATE_TIMEWAIT;
- __insert_timewait( p_cep );
- break;
-
- default:
- /* Ignore the REJ. */
- AL_TRACE( AL_DBG_CM, ("REJ received in invalid state.\n") );
- goto err2;
- }
-
- status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
-
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
if( status == IB_SUCCESS )
__process_cep( p_cep );
AL_EXIT( AL_DBG_CM );
- return;
-
-err2:
- KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
-err1:
- ib_put_mad( p_mad );
- AL_EXIT( AL_DBG_CM );
}
static void
-__process_rep(
+__rep_handler(
IN cep_agent_t* const
p_port_cep,
IN ib_mad_element_t* const p_mad )
{
@@ -1342,26 +1422,26 @@
if( __insert_cep( p_cep ) != p_cep )
{
/* Roll back the state change. */
+ __reject_mad( p_port_cep, p_cep, p_mad,
IB_REJ_STALE_CONN );
p_cep->state = old_state;
- __reject_mad( p_port_cep, p_cep, p_mad,
IB_REJ_STALE_CONN );
- /* TODO: Handle stale connection. */
- break;
+ status = __process_stale( p_cep );
}
+ else
+ {
+ /*
+ * Cancel any outstanding send. Note that we do this
only after
+ * inserting the CEP - if we failed, then the send will
timeout
+ * and we'll finish our way through the state machine.
+ */
+ if( p_cep->p_send_mad )
+ {
+ ib_cancel_mad( p_cep->h_mad_svc,
p_cep->p_send_mad );
+ p_cep->p_send_mad = NULL;
+ }
- /*
- * Cancel any outstanding send. Note that we do this only after
- * inserting the CEP - if we failed, then we the send will
timeout
- * and we'll finish our way through the state machine.
- */
- if( p_cep->p_send_mad )
- {
- ib_cancel_mad( p_cep->h_mad_svc, p_cep->p_send_mad );
- p_cep->p_send_mad = NULL;
+ status = __cep_queue_mad( p_cep, p_mad );
}
- status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
-
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
if( status == IB_SUCCESS )
@@ -1393,7 +1473,7 @@
static void
-__process_rtu(
+__rtu_handler(
IN ib_mad_element_t* const p_mad )
{
ib_api_status_t status;
@@ -1433,7 +1513,6 @@
p_cep->state = CEP_STATE_ESTABLISHED;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
/* Update timewait time. */
__calc_timewait( p_cep );
@@ -1459,7 +1538,7 @@
static void
-__process_dreq(
+__dreq_handler(
IN cep_agent_t* const
p_port_cep,
IN ib_mad_element_t* const p_mad )
{
@@ -1513,7 +1592,6 @@
p_cep->state = CEP_STATE_DREQ_RCVD;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
/* Store the TID for use in the reply DREP. */
p_cep->tid = p_dreq->hdr.trans_id;
@@ -1544,7 +1622,7 @@
static void
-__process_drep(
+__drep_handler(
IN ib_mad_element_t* const p_mad )
{
ib_api_status_t status;
@@ -1593,7 +1671,6 @@
p_cep->state = CEP_STATE_TIMEWAIT;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
}
else
{
@@ -1684,7 +1761,7 @@
static void
-__process_lap(
+__lap_handler(
IN cep_agent_t* const
p_port_cep,
IN ib_mad_element_t* const p_mad )
{
@@ -1748,7 +1825,6 @@
p_cep->state = CEP_STATE_LAP_RCVD;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
@@ -1774,7 +1850,7 @@
static void
-__process_apr(
+__apr_handler(
IN ib_mad_element_t* const p_mad )
{
ib_api_status_t status;
@@ -1819,7 +1895,6 @@
p_cep->state = CEP_STATE_ESTABLISHED;
status = __cep_queue_mad( p_cep, p_mad );
- CL_ASSERT( status != IB_INVALID_STATE );
KeReleaseInStackQueuedSpinLockFromDpcLevel( &hdl );
@@ -1869,39 +1944,39 @@
switch( p_hdr->attr_id )
{
case CM_REQ_ATTR_ID:
- __process_req( p_port_cep, p_mad );
+ __req_handler( p_port_cep, p_mad );
break;
case CM_MRA_ATTR_ID:
- __process_mra( p_mad );
+ __mra_handler( p_mad );
break;
case CM_REJ_ATTR_ID:
- __process_rej( p_mad );
+ __rej_handler( p_mad );
break;
case CM_REP_ATTR_ID:
- __process_rep( p_port_cep, p_mad );
+ __rep_handler( p_port_cep, p_mad );
break;
case CM_RTU_ATTR_ID:
- __process_rtu( p_mad );
+ __rtu_handler( p_mad );
break;
case CM_DREQ_ATTR_ID:
- __process_dreq( p_port_cep, p_mad );
+ __dreq_handler( p_port_cep, p_mad );
break;
case CM_DREP_ATTR_ID:
- __process_drep( p_mad );
+ __drep_handler( p_mad );
break;
case CM_LAP_ATTR_ID:
- __process_lap( p_port_cep, p_mad );
+ __lap_handler( p_port_cep, p_mad );
break;
case CM_APR_ATTR_ID:
- __process_apr( p_mad );
+ __apr_handler( p_mad );
break;
case CM_SIDR_REQ_ATTR_ID:
@@ -2642,7 +2717,11 @@
else if( p_new_cep->remote_ca_guid > p_cep->remote_ca_guid )
p_item = cl_rbmap_right( p_item ), left = FALSE;
else
+ {
+ AL_TRACE( AL_DBG_CM | AL_DBG_WARN,
+ ("WARNING: Duplicate remote CID and CA GUID.\n")
);
goto done;
+ }
}
cl_rbmap_insert(
@@ -2681,7 +2760,11 @@
else if( p_new_cep->remote_ca_guid > p_cep->remote_ca_guid )
p_item = cl_rbmap_right( p_item ), left = FALSE;
else
+ {
+ AL_TRACE( AL_DBG_CM | AL_DBG_WARN,
+ ("WARNING: Duplicate remote QPN and CA GUID.\n")
);
goto done;
+ }
}
cl_rbmap_insert(
-------------- next part --------------
A non-text attachment was scrubbed...
Name: cm_rej_stale.patch
Type: application/octet-stream
Size: 12996 bytes
Desc: not available
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20060405/173575f6/attachment.obj>
More information about the ofw
mailing list