[ofw] [IPoIB CM]patch for endpoint connection recovery.
Alex Estrin
alex.estrin at qlogic.com
Thu Mar 19 13:09:06 PDT 2009
Hello,
These changes will refine endpoint disconnection and recovery:
- Eliminated unnecessary endpoint destruction and preserve neighbors discovered lists. That restored original datagram mode endpoints cache handling.
- Removed dedicated map for connected endpoints, since it is not in use and basically duplicates existing lid map.
- Introduced idle connection expiration timeout to reset stale connection. With this timer after RTU mad was sent/received, if no data comes in through CM QP (for 60 seconds now), connection drops. For example
consider a scenario with Host A connected to host B(both cabled to a switch) and no data is going through CM QP.
Host B got re-cabled, but host A is not aware of that fact. Host B sends ARP REQ advertizing it's CM capabilities, for host A connection seems fine and it sends ARP REP back (through UD) not trying to reconnect.
Note: after connection reset on host A, next ARP exchange will reestablish A-B connection.
Note: earlier issue with cable disconnect/reconnect for back-to-back configuration is fixed as well.
Sorry for sending such a big changes in one chunk, but I really couldn't find best way to split it for easier digesting.
Thanks,
Alex.
Sign-off by: Alex Estrin <alex.estrin at qlogic.com>
---
Index: kernel/ipoib_cm.c
===================================================================
--- kernel/ipoib_cm.c (revision 2004)
+++ kernel/ipoib_cm.c (working copy)
@@ -201,21 +201,36 @@
void
endpt_cm_destroy_conn(
IN ipoib_port_t* const p_port,
- IN ipoib_endpt_t* const p_endpt )
+ IN ipoib_endpt_t* const p_endpt,
+ IN cm_state_t req_state )
{
- cm_state_t cm_state;
+ cm_state_t old_state;
+
IPOIB_ENTER( IPOIB_DBG_ENDPT );
- cm_state = endpt_cm_get_state( p_endpt );
- if( cm_state == IPOIB_CM_DESTROY )
+ old_state = endpt_cm_get_state( p_endpt );
+
+ if( old_state == IPOIB_CM_DESTROY ||
+ old_state == IPOIB_CM_DISCONNECT )
return;
- if( cm_state == IPOIB_CM_DISCONNECTED )
+
+ CL_ASSERT( req_state == IPOIB_CM_DESTROY ||
+ req_state == IPOIB_CM_DISCONNECT );
+
+ endpt_cm_set_state( p_endpt, req_state );
+ if( old_state == IPOIB_CM_CONNECTED )
+ p_port->endpt_mgr.conn_endpts--;
+ if( old_state == IPOIB_CM_DISCONNECTED )
{
- cl_obj_destroy( &p_endpt->obj );
+ if( req_state == IPOIB_CM_DESTROY )
+ {
+ cl_obj_destroy( &p_endpt->obj );
+ }
return;
}
+
NdisInterlockedInsertTailList( &p_port->endpt_mgr.remove_conns,
- &p_endpt->list_item, &p_port->endpt_mgr.remove_lock );
+ &p_endpt->list_entry, &p_port->endpt_mgr.remove_lock );
cl_event_signal( &p_port->endpt_mgr.event );
IPOIB_EXIT( IPOIB_DBG_ENDPT );
@@ -354,8 +369,6 @@
goto listen_done;
}
- endpt_cm_set_state( p_port->p_local_endpt , IPOIB_CM_LISTEN );
-
cl_memclr( &cm_listen, sizeof( ib_cm_listen_t ) );
ipoib_addr_set_sid( &cm_listen.svc_id, p_port->ib_mgr.qpn );
@@ -381,8 +394,8 @@
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,
("\n\tPort[%d] CREATED LISTEN CEP. SID: %#I64x\n",
- p_port->port_num,
- p_port->p_local_endpt->conn.service_id ) );
+ p_port->port_num,
+ cl_ntoh64( p_port->p_local_endpt->conn.service_id ) ) );
listen_done:
if( ib_status != IB_SUCCESS )
{
@@ -481,6 +494,8 @@
goto conn_exit;
}
+ InterlockedCompareExchange( (volatile LONG *)&p_endpt->conn.state,
+ IPOIB_CM_CONNECT, IPOIB_CM_DISCONNECTED );
ib_status =
p_port->p_adapter->p_ifc->rearm_cq( p_endpt->conn.h_send_cq, FALSE );
if( ib_status != IB_SUCCESS )
@@ -611,16 +626,17 @@
("Endpoint [%p] Connect failed status %#x\n", p_endpt, ib_status ) );
goto done;
}
+
+ cl_obj_lock( &p_port->obj );
+
+ p_endpt->conn.time_stamp = KeQueryInterruptTime();
InterlockedCompareExchangePointer(
(void *)&p_endpt->conn.h_work_qp,
p_endpt->conn.h_send_qp, (void *)NULL );
-
- cl_obj_lock( &p_port->obj );
- if( endpt_cm_set_state( p_endpt, IPOIB_CM_CONNECTED ) != IPOIB_CM_CONNECTED )
+ if( InterlockedCompareExchange( (volatile LONG *)&p_endpt->conn.state,
+ IPOIB_CM_CONNECTED, IPOIB_CM_CONNECT )== IPOIB_CM_CONNECT )
{
- cl_fmap_insert( &p_port->endpt_mgr.conn_endpts,
- &p_endpt->dgid,
- &p_endpt->conn_item );
+ p_port->endpt_mgr.conn_endpts++;
}
cl_obj_unlock( &p_port->obj );
@@ -631,7 +647,7 @@
if( ib_status != IB_SUCCESS )
{
__conn_reject( p_port, p_cm_rep->h_cm_rep, IB_REJ_INSUF_RESOURCES );
- ipoib_port_remove_endpt( p_port, p_endpt->mac );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DISCONNECT );
}
ipoib_port_resume( p_port );
@@ -666,19 +682,19 @@
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
("Endpoint [ %p ] CONNECT ACCEPTED\n", p_endpt ) );
+ cl_obj_lock( &p_port->obj );
+
+ p_endpt->conn.time_stamp = KeQueryInterruptTime();
InterlockedExchangePointer(
(void *)&p_endpt->conn.h_work_qp,
p_endpt->conn.h_recv_qp );
-
- cl_obj_lock( &p_port->obj );
- if( endpt_cm_set_state( p_endpt, IPOIB_CM_CONNECTED ) != IPOIB_CM_CONNECTED )
+ if( InterlockedCompareExchange( (volatile LONG *)&p_endpt->conn.state,
+ IPOIB_CM_CONNECTED, IPOIB_CM_CONNECT )
+ == IPOIB_CM_CONNECT )
{
- cl_fmap_insert( &p_port->endpt_mgr.conn_endpts,
- &p_endpt->dgid,
- &p_endpt->conn_item );
+ p_port->endpt_mgr.conn_endpts++;
}
cl_obj_unlock( &p_port->obj );
-
cl_spinlock_acquire( &p_port->send_lock );
if( cl_qlist_count( &p_port->send_mgr.pending_list ) )
{
@@ -721,11 +737,8 @@
IPOIB_PRINT_EXIT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
("Connect REQ Rejected Status: %d\n", cl_ntoh16( p_rej_rec->rej_status ) ) );
}
- /*endpt not connected, but conn QP was created so let's offload destroy QP */
- InterlockedCompareExchange( (volatile LONG *)&p_endpt->conn.state,
- IPOIB_CM_REJ_RECVD, IPOIB_CM_DISCONNECTED );
- ipoib_port_remove_endpt( p_port, p_endpt->mac );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DISCONNECT );
ipoib_port_resume( p_port );
@@ -737,8 +750,6 @@
__conn_dreq_cb(
IN ib_cm_dreq_rec_t *p_dreq_rec )
{
- ib_api_status_t ib_status = IB_SUCCESS;
- ib_cm_drep_t cm_drep;
ipoib_endpt_t* p_endpt ;
ipoib_port_t* p_port;
cm_state_t cm_state;
@@ -752,31 +763,15 @@
if( !p_endpt )
return;
- cm_state = ( cm_state_t )InterlockedCompareExchange(
- (volatile LONG *)&p_endpt->conn.state,
- IPOIB_CM_DREP_SENT, IPOIB_CM_CONNECTED );
-
+ cm_state = endpt_cm_get_state( p_endpt );
IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ERROR,
("RECEIVED DREQ for ENDPT [%p] CONN Status: %d\n", p_endpt, cm_state ) );
if( cm_state != IPOIB_CM_CONNECTED )
return;
-
p_port = ipoib_endpt_parent( p_endpt );
- cm_drep.drep_length = 0;
- cm_drep.p_drep_pdata = NULL;
-/*
- ib_status = p_port->p_adapter->p_ifc->cm_drep(
- p_dreq_rec->h_cm_dreq,
- &cm_drep );
-*/
- if ( ib_status != IB_SUCCESS )
- {
- IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
- ("dreq_cb failed status %#x\n", ib_status ) );
- }
- ipoib_port_remove_endpt( p_port, p_endpt->mac );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DISCONNECT );
IPOIB_EXIT( IPOIB_DBG_ENDPT );
return;
@@ -891,7 +886,7 @@
}
else
{
- ipoib_port_remove_endpt( p_port, p_endpt->mac );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DISCONNECT );
}
ipoib_port_resume( p_port );
@@ -1041,9 +1036,7 @@
IPOIB_ENTER( IPOIB_DBG_ENDPT );
- if( InterlockedCompareExchange(
- (volatile LONG *)&p_endpt->conn.state,
- IPOIB_CM_DREQ_SENT, IPOIB_CM_CONNECTED ) != IPOIB_CM_CONNECTED )
+ if( endpt_cm_get_state( p_endpt )!= IPOIB_CM_CONNECTED )
return;
cm_dreq.h_qp = p_endpt->conn.h_send_qp;
Index: kernel/ipoib_driver.h
===================================================================
--- kernel/ipoib_driver.h (revision 2004)
+++ kernel/ipoib_driver.h (working copy)
@@ -75,6 +75,10 @@
#define IPOIB_CM_FLAG_UC (0x40)
#define IPOIB_CM_FLAG_SVCID (0x10) // OFED set IETF bit this way ( open OFED PR 1121 )
+/* idle connection expiration timeout. in seconds */
+#define ENDPT_CONN_TIMEOUT (60)
+
+/* max Scatter/Gather entries per work request */
#define MAX_SEND_SGE (30)
/* Amount of physical memory to register. */
Index: kernel/ipoib_endpoint.c
===================================================================
--- kernel/ipoib_endpoint.c (revision 2004)
+++ kernel/ipoib_endpoint.c (working copy)
@@ -292,7 +292,8 @@
else if( p_port->p_adapter->params.cm_enabled )
{
p_endpt->cm_flag = 0;
- CL_ASSERT( endpt_cm_get_state( p_endpt ) == IPOIB_CM_DISCONNECTED );
+ CL_ASSERT( endpt_cm_get_state( p_endpt ) == IPOIB_CM_DISCONNECTED ||
+ endpt_cm_get_state( p_endpt ) == IPOIB_CM_DESTROY );
}
cl_obj_unlock( p_obj );
@@ -861,6 +862,8 @@
CL_ASSERT( p_endpt );
+ p_endpt->conn.h_work_qp = NULL;
+
if( p_endpt->conn.h_recv_qp )
{
cl_memclr( &mod_attr, sizeof( mod_attr ) );
@@ -969,6 +972,7 @@
/* Successful completion
Setup the ethernet/ip/arp header and queue descriptor for report. */
ib_status = IB_SUCCESS;
+ p_endpt->conn.time_stamp = KeQueryInterruptTime();
p_ipoib = (ipoib_pkt_t *)((uint8_t*)p_desc->p_buf );
p_eth = (eth_pkt_t *)((uint8_t*)p_desc->p_buf - DATA_OFFSET );
Index: kernel/ipoib_endpoint.h
===================================================================
--- kernel/ipoib_endpoint.h (revision 2004)
+++ kernel/ipoib_endpoint.h (working copy)
@@ -66,14 +66,10 @@
typedef enum _cm_state
{
- IPOIB_CM_DISCONNECTED,
- IPOIB_CM_INIT,
+ IPOIB_CM_DISCONNECTED = 0,
IPOIB_CM_CONNECT,
IPOIB_CM_CONNECTED,
- IPOIB_CM_LISTEN,
- IPOIB_CM_DREP_SENT,
- IPOIB_CM_DREQ_SENT,
- IPOIB_CM_REJ_RECVD,
+ IPOIB_CM_DISCONNECT,
IPOIB_CM_DESTROY
} cm_state_t;
@@ -93,6 +89,7 @@
ib_cq_handle_t h_send_cq;
ib_cq_handle_t h_recv_cq;
ib_listen_handle_t h_cm_listen;
+ uint64_t time_stamp;
cm_state_t state;
} endpt_conn_t;
@@ -104,8 +101,8 @@
cl_map_item_t mac_item;
cl_fmap_item_t gid_item;
cl_map_item_t lid_item;
- cl_fmap_item_t conn_item;
- LIST_ENTRY list_item;
+ cl_list_item_t conn_item;
+ LIST_ENTRY list_entry;
ib_query_handle_t h_query;
ib_mcast_handle_t h_mcast;
mac_addr_t mac;
Index: kernel/ipoib_port.c
===================================================================
--- kernel/ipoib_port.c (revision 2004)
+++ kernel/ipoib_port.c (working copy)
@@ -2165,7 +2165,7 @@
(*pp_src )->mac.addr[0], (*pp_src )->mac.addr[1],
(*pp_src )->mac.addr[2], (*pp_src )->mac.addr[3],
(*pp_src )->mac.addr[4], (*pp_src )->mac.addr[5]) );
-// (*pp_src)->qpn = p_wc->recv.ud.remote_qp;
+ (*pp_src)->qpn = p_wc->recv.ud.remote_qp;
}
if( *pp_src && *pp_dst )
@@ -2630,7 +2630,6 @@
ib_gid_t gid;
mac_addr_t mac;
ipoib_hw_addr_t null_hw = {0};
- uint8_t cm_capable = 0;
IPOIB_ENTER( IPOIB_DBG_RECV );
@@ -2665,8 +2664,6 @@
return IB_INVALID_SETTING;
}
- cm_capable = ipoib_addr_get_flags( &p_ib_arp->src_hw );
-
/*
* If we don't have a source, lookup the endpoint specified in the payload.
*/
@@ -2764,44 +2761,28 @@
cl_obj_unlock( &p_port->obj );
}
- (*pp_src)->cm_flag = cm_capable;
-
CL_ASSERT( !cl_memcmp(
&(*pp_src)->dgid, &p_ib_arp->src_hw.gid, sizeof(ib_gid_t) ) );
CL_ASSERT( ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) ||
(*pp_src)->qpn == ipoib_addr_get_qpn( &p_ib_arp->src_hw ) );
+ (*pp_src)->cm_flag =
+ ( ipoib_addr_get_flags( &p_ib_arp->src_hw )& IPOIB_CM_FLAG_RC );
+
if( p_port->p_adapter->params.cm_enabled &&
- p_ib_arp->op == ARP_OP_REQ &&
- cm_capable == IPOIB_CM_FLAG_RC )
+ (*pp_src)->cm_flag == IPOIB_CM_FLAG_RC )
{
/* if we've got ARP request and RC flag is set,
save SID for connect REQ to be sent in ARP reply
when requestor's path get resolved */
if( endpt_cm_get_state( (*pp_src) ) == IPOIB_CM_DISCONNECTED )
{
- (*pp_src)->cm_flag = cm_capable;
ipoib_addr_set_sid(
&(*pp_src)->conn.service_id,
ipoib_addr_get_qpn( &p_ib_arp->src_hw ) );
}
}
-#if DBG
- if( p_port->p_adapter->params.cm_enabled )
- {
- IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
- (" ARP %s from ENDPT[%p] state %d CM cap: %d QPN: %#x MAC: %02x:%02x:%02x:%02x:%02x:%02x\n",
- ((p_ib_arp->op == ARP_OP_REQ )? "REQUEST" : "REPLY"),
- *pp_src, endpt_cm_get_state( *pp_src ),
- ((cm_capable == IPOIB_CM_FLAG_RC)? 1: 0),
- cl_ntoh32( ipoib_addr_get_qpn( &p_ib_arp->src_hw ) ),
- (*pp_src)->mac.addr[0], (*pp_src)->mac.addr[1],
- (*pp_src)->mac.addr[2], (*pp_src)->mac.addr[3],
- (*pp_src)->mac.addr[4], (*pp_src)->mac.addr[5] ));
- }
-#endif
-
/* Now swizzle the data. */
p_arp->hw_type = ARP_HW_TYPE_ETH;
p_arp->hw_size = sizeof(mac_addr_t);
@@ -2817,7 +2798,21 @@
* remote port lets everyone know it was changed IP/MAC
* or just activated
*/
+ IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
+ (" BROADCAST ARP %#x from ENDPT state %d MAC: %02x:%02x:%02x:%02x:%02x:%02x\n",
+ p_ib_arp->op, endpt_cm_get_state( *pp_src ),
+ (*pp_src)->mac.addr[0], (*pp_src)->mac.addr[1],
+ (*pp_src)->mac.addr[2], (*pp_src)->mac.addr[3],
+ (*pp_src)->mac.addr[4], (*pp_src)->mac.addr[5] ));
+ if( endpt_cm_get_state( *pp_src ) == IPOIB_CM_CONNECTED )
+ {
+ p_port->endpt_mgr.conn_endpts--;
+ endpt_cm_set_state( *pp_src, IPOIB_CM_DISCONNECT );
+ NdisInterlockedInsertTailList( &p_port->endpt_mgr.remove_conns,
+ &(*pp_src)->list_entry, &p_port->endpt_mgr.remove_lock );
+ cl_event_signal( &p_port->endpt_mgr.event );
+ }
/* Guy: TODO: Check why this check fails in case of Voltaire IPR */
if ( !ipoib_is_voltaire_router_gid( &(*pp_src)->dgid ) &&
@@ -2841,12 +2836,24 @@
p_arp->dst_hw = p_dst->mac;
p_arp->dst_ip = p_ib_arp->dst_ip;
CL_ASSERT( p_dst->qpn == ipoib_addr_get_qpn( &p_ib_arp->dst_hw ) );
+ IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
+ ("ARP REPLY from ENDPT CM state %d MAC: %02x:%02x:%02x:%02x:%02x:%02x\n",
+ endpt_cm_get_state( *pp_src ),
+ (*pp_src)->mac.addr[0], (*pp_src)->mac.addr[1],
+ (*pp_src)->mac.addr[2], (*pp_src)->mac.addr[3],
+ (*pp_src)->mac.addr[4], (*pp_src)->mac.addr[5] ));
}
}
else /* we got ARP reqeust */
{
cl_memclr( &p_arp->dst_hw, sizeof(mac_addr_t) );
p_arp->dst_ip = p_ib_arp->dst_ip;
+ IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
+ ("ARP REQUEST from ENDPT CM state %d MAC: %02x:%02x:%02x:%02x:%02x:%02x\n",
+ endpt_cm_get_state( *pp_src ),
+ (*pp_src)->mac.addr[0], (*pp_src)->mac.addr[1],
+ (*pp_src)->mac.addr[2], (*pp_src)->mac.addr[3],
+ (*pp_src)->mac.addr[4], (*pp_src)->mac.addr[5] ));
}
/*
@@ -4113,11 +4120,9 @@
p_port->p_adapter->params.cm_enabled &&
p_desc->p_endpt->cm_flag == IPOIB_CM_FLAG_RC )
{
- cm_state_t cm_state;
- cm_state =
- ( cm_state_t )InterlockedCompareExchange( (volatile LONG *)&p_desc->p_endpt->conn.state,
- IPOIB_CM_CONNECT, IPOIB_CM_DISCONNECTED );
- switch( cm_state )
+ switch( ( cm_state_t )InterlockedCompareExchange(
+ (volatile LONG *)&p_desc->p_endpt->conn.state,
+ IPOIB_CM_CONNECT, IPOIB_CM_DISCONNECTED ) )
{
case IPOIB_CM_DISCONNECTED:
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
@@ -4135,7 +4140,7 @@
cl_qlist_insert_tail( &p_port->send_mgr.pending_list,
IPOIB_LIST_ITEM_FROM_PACKET( p_desc->p_pkt ) );
NdisInterlockedInsertTailList( &p_port->endpt_mgr.pending_conns,
- &p_desc->p_endpt->list_item,
+ &p_desc->p_endpt->list_entry,
&p_port->endpt_mgr.conn_lock );
cl_event_signal( &p_port->endpt_mgr.event );
return NDIS_STATUS_PENDING;
@@ -5032,8 +5037,6 @@
if( p_port->p_adapter->params.cm_enabled )
{
- cl_fmap_init( &p_port->endpt_mgr.conn_endpts, __gid_cmp );
-
NdisInitializeListHead( &p_port->endpt_mgr.pending_conns );
NdisAllocateSpinLock( &p_port->endpt_mgr.conn_lock );
cl_event_init( &p_port->endpt_mgr.event, FALSE );
@@ -5056,24 +5059,92 @@
IN void* p_context )
{
ib_api_status_t ib_status;
+ cl_status_t cl_status;
LIST_ENTRY *p_item;
+ cl_map_item_t *p_map_item;
ipoib_endpt_t *p_endpt;
ipoib_port_t *p_port =( ipoib_port_t *)p_context;
-
+
IPOIB_PRINT_EXIT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
("Starting Port [%d] Endpt CM thread \n", p_port->port_num ) );
while( !p_port->endpt_mgr.thread_is_done )
{
- cl_event_wait_on( &p_port->endpt_mgr.event, EVENT_NO_TIMEOUT, FALSE );
-
+ cl_status = cl_event_wait_on( &p_port->endpt_mgr.event, 20*SEC_TO_MICRO, FALSE );
+
+ if( cl_status == CL_TIMEOUT )
+ {
+ uint64_t time_stamp;
+
+ /* lookup for stalled connections */
+ while( p_port->endpt_rdr )
+ ;
+
+ cl_obj_lock( &p_port->obj );
+ p_map_item = cl_qmap_head( &p_port->endpt_mgr.lid_endpts );
+ time_stamp = KeQueryInterruptTime();
+ while( p_map_item != cl_qmap_end( &p_port->endpt_mgr.lid_endpts ) )
+ {
+ p_endpt = PARENT_STRUCT( p_map_item, ipoib_endpt_t, lid_item );
+ p_map_item = cl_qmap_next( p_map_item );
+
+ if( endpt_cm_get_state( p_endpt ) != IPOIB_CM_CONNECTED )
+ continue;
+
+ if( ( ( time_stamp - p_endpt->conn.time_stamp )/
+ HUNDREDNS_TO_SEC ) > ENDPT_CONN_TIMEOUT )
+ {
+ p_port->endpt_mgr.conn_endpts--;
+ endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECT );
+ NdisInterlockedInsertTailList( &p_port->endpt_mgr.remove_conns,
+ &p_endpt->list_entry, &p_port->endpt_mgr.remove_lock );
+ }
+ }
+ cl_obj_unlock( &p_port->obj );
+ }
+
+ /* walk through endpoints destroy list */
+ while( ( p_item = NdisInterlockedRemoveHeadList(
+ &p_port->endpt_mgr.remove_conns,
+ &p_port->endpt_mgr.remove_lock ) ) != NULL )
+ {
+ p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, list_entry );
+
+ endpt_cm_flush_recv( p_port, p_endpt );
+
+ if( endpt_cm_get_state( p_endpt ) == IPOIB_CM_DESTROY )
+ {
+ IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,
+ ("\nDESTROYING Endpt[%p] MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
+ p_endpt,
+ p_endpt->mac.addr[0], p_endpt->mac.addr[1],
+ p_endpt->mac.addr[2], p_endpt->mac.addr[3],
+ p_endpt->mac.addr[4], p_endpt->mac.addr[5] ) );
+ cl_obj_destroy( &p_endpt->obj );
+ }
+ else
+ {
+ endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECTED );
+ IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,
+ ("\n RESET Endpt[%p] MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
+ p_endpt,
+ p_endpt->mac.addr[0], p_endpt->mac.addr[1],
+ p_endpt->mac.addr[2], p_endpt->mac.addr[3],
+ p_endpt->mac.addr[4], p_endpt->mac.addr[5] ) );
+ }
+ }
+
+ /* walk through endpoints connect request list */
while( ( p_item = NdisInterlockedRemoveHeadList(
&p_port->endpt_mgr.pending_conns,
&p_port->endpt_mgr.conn_lock) ) != NULL )
{
- p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, list_item );
- if( p_port->endpt_mgr.thread_is_done )
+ p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, list_entry );
+ if( p_port->endpt_mgr.thread_is_done ||
+ !p_endpt->cm_flag ||
+ ( endpt_cm_get_state( p_endpt ) != IPOIB_CM_CONNECT ) ||
+ !p_endpt->conn.service_id || !p_endpt->dlid )
{
endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECTED );
continue;
@@ -5105,32 +5176,13 @@
}
if( ib_status != IB_SUCCESS && ib_status != IB_PENDING )
{
- endpt_cm_set_state( p_endpt, IPOIB_CM_DESTROY );
+ endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECT );
endpt_cm_flush_recv( p_port, p_endpt );
endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECTED );
}
}
}//while( p_item != NULL )
-
- while( ( p_item = NdisInterlockedRemoveHeadList(
- &p_port->endpt_mgr.remove_conns,
- &p_port->endpt_mgr.remove_lock ) ) != NULL )
- {
- p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, list_item );
-
- endpt_cm_set_state( p_endpt, IPOIB_CM_DESTROY );
-
- IPOIB_PRINT( TRACE_LEVEL_WARNING, IPOIB_DBG_INIT,
- ("\nDESTROYING Endpt[%p] MAC %02x:%02x:%02x:%02x:%02x:%02x\n",
- p_endpt,
- p_endpt->mac.addr[0], p_endpt->mac.addr[1],
- p_endpt->mac.addr[2], p_endpt->mac.addr[3],
- p_endpt->mac.addr[4], p_endpt->mac.addr[5] ) );
- endpt_cm_flush_recv( p_port, p_endpt );
- endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECTED );
- cl_obj_destroy( &p_endpt->obj );
- }
}
p_port->endpt_mgr.thread_is_done++;
@@ -5150,7 +5202,7 @@
CL_ASSERT( cl_is_fmap_empty( &p_port->endpt_mgr.gid_endpts ) );
if( p_port->p_adapter->params.cm_enabled )
{
- CL_ASSERT( cl_is_fmap_empty( &p_port->endpt_mgr.conn_endpts ) );
+ CL_ASSERT( !p_port->endpt_mgr.conn_endpts );
}
IPOIB_EXIT( IPOIB_DBG_INIT );
}
@@ -5185,7 +5237,6 @@
IN ipoib_port_t* const p_port )
{
cl_map_item_t *p_item;
- cl_fmap_item_t *p_fmap_item;
ipoib_endpt_t *p_endpt;
cl_qlist_t mc_list;
cl_qlist_t conn_list;
@@ -5253,31 +5304,12 @@
cl_qlist_insert_tail(
&mc_list, &p_endpt->mac_item.pool_item.list_item );
}
- /* destroy connected endpoints if any */
- else if( p_port->p_adapter->params.cm_enabled &&
- endpt_cm_get_state( p_endpt ) != IPOIB_CM_DISCONNECTED )
+ else if( p_endpt->h_av )
{
- p_fmap_item = cl_fmap_get( &p_port->endpt_mgr.conn_endpts, &p_endpt->dgid );
- if( p_fmap_item != cl_fmap_end( &p_port->endpt_mgr.conn_endpts ) )
- {
- cl_fmap_remove_item( &p_port->endpt_mgr.conn_endpts,
- &p_endpt->conn_item );
- }
- cl_qmap_remove_item( &p_port->endpt_mgr.mac_endpts,
- &p_endpt->mac_item );
- cl_fmap_remove_item( &p_port->endpt_mgr.gid_endpts,
- &p_endpt->gid_item );
-
- cl_qlist_insert_tail(
- &conn_list, &p_endpt->mac_item.pool_item.list_item );
- }
- if( p_endpt->h_av )
- {
/* Destroy the AV for all other endpoints. */
p_port->p_adapter->p_ifc->destroy_av( p_endpt->h_av );
p_endpt->h_av = NULL;
}
-
if( p_endpt->dlid )
{
cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,
@@ -5286,7 +5318,13 @@
("<__endptr_mgr_reset_all: setting p_endpt->dlid to 0\n"));
p_endpt->dlid = 0;
}
-
+ /* destroy connected endpoints if any */
+ if( p_port->p_adapter->params.cm_enabled &&
+ endpt_cm_get_state( p_endpt ) != IPOIB_CM_DISCONNECTED )
+ {
+ cl_qlist_insert_tail(
+ &conn_list, &p_endpt->conn_item );
+ }
}
#endif
cl_obj_unlock( &p_port->obj );
@@ -5295,7 +5333,7 @@
{
endpt_cm_destroy_conn( p_port,
PARENT_STRUCT( cl_qlist_remove_head( &conn_list ),
- ipoib_endpt_t, mac_item.pool_item.list_item ) );
+ ipoib_endpt_t, conn_item ), IPOIB_CM_DISCONNECT );
}
if(cl_qlist_count( &mc_list ) - local_exist)
@@ -5334,8 +5372,6 @@
IN ipoib_port_t* const p_port,
IN ipoib_endpt_t* const p_endpt )
{
- cl_fmap_item_t* p_fmap_item;
-
IPOIB_ENTER( IPOIB_DBG_ENDPT );
/* This function must be called from the receive path */
@@ -5353,16 +5389,6 @@
* in the LID map if the GID has the same subnet prefix as us.
*/
cl_fmap_remove_item( &p_port->endpt_mgr.gid_endpts, &p_endpt->gid_item );
- if( p_port->p_adapter->params.cm_enabled )
- {
- p_fmap_item = cl_fmap_get( &p_port->endpt_mgr.conn_endpts, &p_endpt->dgid );
-
- if( p_fmap_item != cl_fmap_end( &p_port->endpt_mgr.conn_endpts ) )
- {
- cl_fmap_remove_item( &p_port->endpt_mgr.conn_endpts,
- &p_endpt->conn_item );
- }
- }
if( p_endpt->dlid )
{
cl_qmap_remove_item( &p_port->endpt_mgr.lid_endpts,
@@ -5371,7 +5397,7 @@
cl_obj_unlock( &p_port->obj );
- endpt_cm_destroy_conn( p_port, p_endpt );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DESTROY );
IPOIB_EXIT( IPOIB_DBG_ENDPT );
}
@@ -5459,7 +5485,7 @@
p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, mac_item );
}
- p_path->resv0 = 0;
+ p_path->service_id = 0;
p_path->dgid = p_endpt->dgid;
p_path->sgid = p_port->p_local_endpt->dgid;
p_path->dlid = p_endpt->dlid;
@@ -5484,7 +5510,7 @@
p_port->p_local_endpt->dlid,
1,
p_port->ib_mgr.bcast_rec.pkey,
- sl,
+ sl, 0,
IB_PATH_SELECTOR_EXACTLY, p_port->ib_mgr.bcast_rec.mtu,
IB_PATH_SELECTOR_EXACTLY, p_port->ib_mgr.bcast_rec.rate,
IB_PATH_SELECTOR_EXACTLY, pkt_life,
@@ -5783,7 +5809,6 @@
IN const mac_addr_t mac )
{
cl_map_item_t *p_item;
- cl_fmap_item_t *p_fmap_item;
ipoib_endpt_t *p_endpt;
uint64_t key;
@@ -5812,17 +5837,6 @@
cl_fmap_remove_item(
&p_port->endpt_mgr.gid_endpts, &p_endpt->gid_item );
- if( p_port->p_adapter->params.cm_enabled )
- {
- p_fmap_item = cl_fmap_get( &p_port->endpt_mgr.conn_endpts, &p_endpt->dgid );
-
- if( p_fmap_item != cl_fmap_end( &p_port->endpt_mgr.conn_endpts ) )
- {
- cl_fmap_remove_item( &p_port->endpt_mgr.conn_endpts,
- &p_endpt->conn_item );
- }
- }
-
if( p_endpt->dlid )
{
cl_qmap_remove_item(
@@ -5831,7 +5845,7 @@
cl_obj_unlock( &p_port->obj );
- endpt_cm_destroy_conn( p_port, p_endpt );
+ endpt_cm_destroy_conn( p_port, p_endpt, IPOIB_CM_DESTROY );
#if DBG
cl_atomic_dec( &p_port->ref[ref_endpt_track] );
Index: kernel/ipoib_port.h
===================================================================
--- kernel/ipoib_port.h (revision 2004)
+++ kernel/ipoib_port.h (working copy)
@@ -487,7 +487,7 @@
cl_qmap_t mac_endpts;
cl_fmap_t gid_endpts;
cl_qmap_t lid_endpts;
- cl_fmap_t conn_endpts;
+ uint32_t conn_endpts;
LIST_ENTRY pending_conns;
LIST_ENTRY remove_conns;
NDIS_SPIN_LOCK conn_lock;
@@ -507,9 +507,6 @@
* lid_endpts
* Map of enpoints, keyed by LID. Only enpoints on the same subnet
* are inserted in the LID map.
-*
-* conn_endpts
-* Map of connected endpts, keyed by remote gid.
*********/
@@ -680,7 +677,7 @@
) {
if (Status != NDIS_STATUS_SUCCESS) {
IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
- ("Sending status other than Success to NDIS\n"));
+ ("Sending status %#x to NDIS\n", Status ));
}
NdisMSendComplete(MiniportAdapterHandle,Packet,Status);
}
@@ -751,7 +748,8 @@
void
endpt_cm_destroy_conn(
IN ipoib_port_t* const p_port,
- IN ipoib_endpt_t* const p_endpt );
+ IN ipoib_endpt_t* const p_endpt,
+ IN cm_state_t req_state );
void
endpt_cm_disconnect(
-------------- next part --------------
A non-text attachment was scrubbed...
Name: cm_timeout_03_19_09.patch
Type: application/octet-stream
Size: 25640 bytes
Desc: cm_timeout_03_19_09.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20090319/c71a865a/attachment.obj>
More information about the ofw
mailing list