[Openib-windows] Patch for not posting recvs while in RST state
Tzachi Dar
tzachid at mellanox.co.il
Mon May 8 02:37:22 PDT 2006
Hi Fab,
Attached is a patch that solves the posting of received packets when in
the QP is in RST state.
According to my checks it solves the problem, and it doesn't add any
locking or Interlocked operations.
(As always feel free to do cosmetic changes to the patch)
Thanks
Tzachi
Index: ipoib_port.c
===================================================================
--- ipoib_port.c (revision 333)
+++ ipoib_port.c (working copy)
@@ -58,7 +58,6 @@
#define MAX_SEND_WC 8
#define MAX_RECV_WC 16
-
ib_gid_t bcast_mgid_template = {
0xff, /* multicast field */
0x12, /* scope (to be filled in) */
@@ -676,7 +675,8 @@
p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );
/* Wait for all sends and receives to get flushed. */
- while( p_port->send_mgr.depth || p_port->recv_mgr.depth )
+ shutter_shut (&p_port->recv_mgr.shutter);
+ while( p_port->send_mgr.depth )
cl_thread_suspend( 0 );
/* Destroy the send and receive managers before closing the CA. */
@@ -1285,6 +1285,7 @@
("cl_malloc for PNDIS_PACKET array failed.\n") );
return IB_INSUFFICIENT_MEMORY;
}
+ shutter_init(&p_port->recv_mgr.shutter);
IPOIB_EXIT( IPOIB_DBG_INIT );
return IB_SUCCESS;
@@ -1298,7 +1299,6 @@
IPOIB_ENTER( IPOIB_DBG_INIT );
CL_ASSERT( cl_is_qlist_empty( &p_port->recv_mgr.done_list ) );
- CL_ASSERT( !p_port->recv_mgr.depth );
if( p_port->recv_mgr.recv_pkt_array )
cl_free( p_port->recv_mgr.recv_pkt_array );
@@ -1316,6 +1316,7 @@
ipoib_recv_desc_t *p_head = NULL, *p_tail = NULL, *p_next;
ib_api_status_t status;
ib_recv_wr_t *p_failed;
+ boolean_t AllowToPost = TRUE;
PERF_DECLARE( GetRecv );
PERF_DECLARE( PostRecv );
@@ -1336,10 +1337,19 @@
while( num_wrs-- )
{
- if( p_port->recv_mgr.depth == p_port->p_adapter->params.rq_depth )
+ long depth = shutter_use(&p_port->recv_mgr.shutter);
+ if (depth < 0) {
+ IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,
+ ("Already in shutdown\n") );
+ AllowToPost = FALSE;
+ // We will soon catch the state and fix it.
+ break;
+ }
+ if( depth == p_port->p_adapter->params.rq_depth )
{
IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,
("Receive queue already full\n") );
+ shutter_loose(&p_port->recv_mgr.shutter , 1 );
break;
}
@@ -1351,6 +1361,7 @@
{
IPOIB_TRACE( IPOIB_DBG_INFO | IPOIB_DBG_RECV,
("Out of receive descriptors!\n") );
+ shutter_loose(&p_port->recv_mgr.shutter , 1 );
break;
}
@@ -1366,7 +1377,6 @@
p_head = p_next;
- cl_atomic_inc( &p_port->recv_mgr.depth );
}
if( !p_head )
@@ -1376,11 +1386,19 @@
return IB_SUCCESS;
}
- cl_perf_start( PostRecv );
- status = p_port->p_adapter->p_ifc->post_recv(
- p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );
- cl_perf_stop( &p_port->p_adapter->perf, PostRecv );
-
+ if (AllowToPost == FALSE)
+ {
+ // Shutdown has happened, need to close all objects and leave
+ status = IB_CANCELED;
+ p_failed = &p_head->wr;
+ }
+ else
+ {
+ cl_perf_start( PostRecv );
+ status = p_port->p_adapter->p_ifc->post_recv(
+ p_port->ib_mgr.h_qp, &p_head->wr, &p_failed );
+ cl_perf_stop( &p_port->p_adapter->perf, PostRecv );
+ }
/*
* If we failed, fix up the work completion list and return those
* buffers to the pool
@@ -1397,7 +1415,7 @@
p_failed = p_failed->p_next;
__buf_mgr_put_recv( p_port, p_head, NULL );
- cl_atomic_dec( &p_port->recv_mgr.depth );
+ shutter_loose(&p_port->recv_mgr.shutter , 1 );
}
}
@@ -1558,7 +1576,7 @@
cl_atomic_dec( &p_port->endpt_rdr );
/* Update our posted depth. */
- cl_atomic_sub( &p_port->recv_mgr.depth, recv_cnt );
+ shutter_loose(&p_port->recv_mgr.shutter , recv_cnt );
cl_perf_start( BuildPktArray );
/* Notify NDIS of any and all possible receive buffers. */
@@ -4419,8 +4437,7 @@
IPOIB_ENTER( IPOIB_DBG_INIT );
/* Wait for all receives to get flushed. */
- while( p_port->recv_mgr.depth )
- cl_thread_suspend( 0 );
+ shutter_shut(&p_port->recv_mgr.shutter);
p_port->state = IB_QPS_RTS;
@@ -5082,6 +5099,7 @@
}
/* Prepost receives. */
+ shutter_init(&p_port->recv_mgr.shutter);
cl_spinlock_acquire( &p_port->recv_lock );
__recv_mgr_repost( p_port, p_port->p_adapter->params.rq_depth );
cl_spinlock_release( &p_port->recv_lock );
Index: ipoib_port.h
===================================================================
--- ipoib_port.h (revision 333)
+++ ipoib_port.h (working copy)
@@ -42,8 +42,10 @@
#include "ip_packet.h"
#include "ipoib_endpoint.h"
#include "ipoib_xfr_mgr.h"
+#include "ipoib_shutter.h"
+
/*
* Define to place receive buffer inline in receive descriptor.
*/
@@ -406,7 +408,7 @@
typedef struct _ipoib_recv_mgr
{
- atomic32_t depth;
+ shutter_t shutter;
NDIS_PACKET **recv_pkt_array;
Index: ipoib_shutter.h
===================================================================
--- ipoib_shutter.h (revision 0)
+++ ipoib_shutter.h (revision 0)
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2005 SilverStorm Technologies. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: ipoib_shutter.h 10 2005-05-24 00:33:03Z ftillier $
+ */
+
+
+
+#ifndef _IPOIB_SHUTTER_H_
+#define _IPOIB_SHUTTER_H_
+
+
+
+
+/*
+ * Define the max numbers of operations that can be simultaniously done
+ */
+#define MAX_OPERATIONS 0x10000000
+
+typedef struct _shutter_t {
+ long opertaions;
+
+} shutter_t;
+
+static inline void shutter_init(shutter_t* p_shutter)
+{
+ p_shutter->opertaions = 0;
+}
+
+static inline int shutter_use(shutter_t * p_shutter)
+{
+ long res;
+ res = InterlockedExchangeAdd(&p_shutter->opertaions, 1);
+ ASSERT(res < MAX_OPERATIONS);
+ if (res >= 0 ) {
+ // Everything went OK, we can do the operation
+ return res;
+ }
+ // The object was already shutdown
+ InterlockedExchangeAdd(&p_shutter->opertaions, -1);
+ return -1;
+}
+
+static inline void shutter_loose(shutter_t * p_shutter, long count)
+{
+ InterlockedExchangeAdd(&p_shutter->opertaions, -count);
+}
+
+static inline void shutter_shut(shutter_t * p_shutter)
+{
+ // Mark the counter as locked
+ long res;
+ LARGE_INTEGER interval;
+ res = InterlockedExchangeAdd(&p_shutter->opertaions, -MAX_OPERATIONS);
+ if (res == 0)
+ {
+ // The object had no user, quit
+ return;
+ }
+ if (res <= -MAX_OPERATIONS)
+ {
+ // This probably means that we have another bug, but I want to give
this
+ // a chance (someone has already started to shut) ?????????
+ InterlockedExchangeAdd(&p_shutter->opertaions, MAX_OPERATIONS);
+ }
+ // We now wait for the object to reach to -MAX_OPERATIONS
+ interval.QuadPart = 0;
+ for(;;) {
+ res = InterlockedExchangeAdd(&p_shutter->opertaions, 0);
+ if (res == -MAX_OPERATIONS) {
+ return;
+ }
+ KeDelayExecutionThread( KernelMode, FALSE, &interval );
+ }
+}
+
+#endif _IPOIB_SHUTTER_H_
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20060508/cb79f808/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: repost.patch
Type: application/octet-stream
Size: 7904 bytes
Desc: repost.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20060508/cb79f808/attachment.obj>
More information about the ofw
mailing list