[ofw] [PATCH] post 80% of SRQ buffers, code cleanup
Smith, Stan
stan.smith at intel.com
Thu Feb 24 10:04:01 PST 2011
Post 80% of SRQ buffers initially such that the SRQ buffer pool will not be required to grow as often, reduce RNR errors under stress.
CM code cleanup
migrate/rename cm_buf_mgr_t.recv_pool_depth to ib_mgr_t.srq_depth; clearer meaning in var naming.
add ib_mgr_t.srq_max_depth
output NDIS error code in hex so it's easier to find in NDIS.h
ipoib_port.cpp
Setup ib_mgr_t.srq vars max & 80% of max
Make the intent of cl_dbg_msg() output clear.
ipoib_cm.cpp:
Use new names for recv_pool_depth
RNR_retry @ 2 instead of 1 for correct operation.
Correct mislabeled CQ AsyncEvent debug print; was mislabeled as SRQ AsyncEvent.
Signed-off-by: stan smith <stan.smith at intel.com>
--- A/ulp/ipoib_NDIS6_CM/kernel/ipoib_endpoint.h Thu Feb 24 09:23:14 2011
+++ B/ulp/ipoib_NDIS6_CM/kernel/ipoib_endpoint.h Thu Feb 24 09:06:51 2011
@@ -53,7 +53,6 @@
cl_spinlock_t lock;
cl_qlist_t oop_list;
long posted;
- int32_t recv_pool_depth;
boolean_t pool_init;
} cm_buf_mgr_t;
/*
--- A/ulp/ipoib_NDIS6_CM/kernel/ipoib_port.h Thu Feb 24 09:22:30 2011
+++ B/ulp/ipoib_NDIS6_CM/kernel/ipoib_port.h Thu Feb 24 09:04:38 2011
@@ -107,6 +107,8 @@
ib_query_handle_t h_query;
ib_srq_handle_t h_srq;
atomic32_t srq_qp_cnt;
+ int32_t srq_max_depth;
+ int32_t srq_depth;
net32_t qpn;
ib_mr_handle_t h_mr;
@@ -1123,7 +1125,7 @@
if (NET_BUFFER_LIST_STATUS(NetBufferLists) != NDIS_STATUS_SUCCESS) {
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ALL,
- ("NBL completed with error %d to NDIS\n",
+ ("NBL completed with error %#x to NDIS\n",
NET_BUFFER_LIST_STATUS(NetBufferLists)));
}
IPOIB_PRINT( TRACE_LEVEL_VERBOSE, IPOIB_DBG_SEND,
--- A/ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp Thu Feb 24 09:34:29 2011
+++ B/ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp Thu Feb 24 09:05:46 2011
@@ -1442,6 +1442,12 @@
(p_port->p_ca_attrs->max_srq_wrs/2)) );
p_port->ib_mgr.srq_qp_cnt = 0;
+ p_port->ib_mgr.srq_max_depth = srq_attr.max_wr;
+
+ /* setup SRQ depth based on rq_depth property++: setup such that 80% of
+ * allocated SRQ (srq_max_depth) buffers are posted. See cm_buf_mgr_init().
+ */
+ p_port->ib_mgr.srq_depth = (srq_attr.max_wr / 5) * 4;
ib_status = p_port->p_adapter->p_ifc->create_srq( p_port->ib_mgr.h_pd,
&srq_attr,
@@ -6118,7 +6124,7 @@
{
IPOIB_PRINT_EXIT( TRACE_LEVEL_WARNING, IPOIB_DBG_SEND,
("No available WQEs.\n") );
- cl_dbg_out("HW is full\n");
+ cl_dbg_out("[IPoIB] HW send_q is full\n");
return NDIS_STATUS_PENDING;
}
--- A/ulp/ipoib_NDIS6_CM/kernel/ipoib_cm.cpp Thu Feb 24 09:45:24 2011
+++ B/ulp/ipoib_NDIS6_CM/kernel/ipoib_cm.cpp Thu Feb 24 09:20:29 2011
@@ -402,7 +402,7 @@
if( send_qp == FALSE && !p_endpt->conn.h_recv_cq )
{
memset( &create_cq, 0, sizeof( ib_cq_create_t ) );
- create_cq.size = p_port->p_adapter->params.rq_depth;
+ create_cq.size = p_port->ib_mgr.srq_depth;
create_cq.pfn_comp_cb = __cm_recv_cb;
ib_status = p_endpt->p_ifc->create_cq( p_port->ib_mgr.h_ca,
@@ -633,7 +633,7 @@
creq.flow_ctrl = FALSE; // srq attached qp does not support FC
creq.local_resp_timeout = ib_path_rec_pkt_life(&path_rec) + 1;
creq.rnr_nak_timeout = 7;
- creq.rnr_retry_cnt = 1; /* IPoIB CM RFC draft warns against retries */
+ creq.rnr_retry_cnt = 2;
creq.retry_cnt = 1; /* IPoIB CM RFC draft warns against retries */
//creq.pfn_cm_req_cb = (ib_pfn_cm_req_cb_t)NULL; no peer connections
@@ -1631,6 +1631,7 @@
PERF_DECLARE( CMSendCb );
PERF_DECLARE( CMPollSend );
PERF_DECLARE( CMFreeSendBuf );
+ PERF_DECLARE( PortResume );
IPOIB_ENTER( IPOIB_DBG_SEND );
@@ -1747,9 +1748,20 @@
p_endpt->conn.h_send_qp = NULL; // prevent Tx on invalid QP
__queue_tx_resource_free( p_port, p_endpt );
endpt_cm_set_state( p_endpt, IPOIB_CM_DISCONNECT_CLEANUP );
+ /* Resume any sends awaiting resources. */
+ cl_perf_start( PortResume );
+ ipoib_port_resume( p_port, TRUE, &complete_list );
+ cl_perf_stop( &p_port->p_adapter->perf, PortResume );
}
else
{
+ if (p_port->send_mgr.pending_list.count > 0)
+ {
+ /* Resume any sends awaiting resources. */
+ cl_perf_start( PortResume );
+ ipoib_port_resume( p_port, TRUE, &complete_list );
+ cl_perf_stop( &p_port->p_adapter->perf, PortResume );
+ }
/* Rearm the CQ. */
ib_status = p_ibal->rearm_cq( h_cq, FALSE );
CL_ASSERT( ib_status == IB_SUCCESS );
@@ -2073,18 +2085,18 @@
IPOIB_ENTER( IPOIB_DBG_RECV );
posted = p_port->cm_buf_mgr.posted;
- wanted = p_port->p_adapter->params.rq_depth - posted;
+ wanted = p_port->ib_mgr.srq_depth - posted;
#if DBG
- IPOIB_PRINT( TRACE_LEVEL_VERBOSE, IPOIB_DBG_RECV,
- ("Port[%d] posting %d RC bufs of limit(rq_depth %d) posted %d max %d\n",
- p_port->port_num, wanted, p_port->p_adapter->params.rq_depth,
- posted, p_port->cm_buf_mgr.recv_pool_depth) );
+ IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_CM/*XXX RECV*/,
+ ("Port[%d] posting RC bufs: wanted %d srq_depth %d posted %d srq max %d\n",
+ p_port->port_num, wanted, p_port->ib_mgr.srq_depth,
+ posted, p_port->ib_mgr.srq_max_depth) );
#endif
cl_spinlock_acquire( &p_port->cm_buf_mgr.lock);
- for( rx_cnt=posted; rx_cnt < p_port->p_adapter->params.rq_depth; rx_cnt++)
+ for( rx_cnt=posted; rx_cnt < p_port->ib_mgr.srq_depth; rx_cnt++)
{
/* Pull receives out of the pool to chain them up. */
p_next_desc = __cm_buf_mgr_get_recv_locked( p_port, p_endpt );
@@ -2174,9 +2186,9 @@
cl_obj_unlock( &p_port->obj );
cl_spinlock_acquire( &p_port->cm_buf_mgr.lock);
- rx_wanted = p_port->p_adapter->params.rq_depth - p_port->cm_buf_mgr.posted;
+ rx_wanted = p_port->ib_mgr.srq_depth - p_port->cm_buf_mgr.posted;
- while( p_port->cm_buf_mgr.posted < p_port->p_adapter->params.rq_depth )
+ while( p_port->cm_buf_mgr.posted < p_port->ib_mgr.srq_depth )
{
/* Pull receives out of the pool and chain them up. */
cl_perf_start( GetRecv );
@@ -2729,7 +2741,7 @@
p_port = ipoib_endpt_parent( p_endpt );
IPOIB_PRINT( TRACE_LEVEL_ERROR, IPOIB_DBG_ERROR,
- ("SRQ CQ AsyncEvent EP %s event '%s' vendor code %#I64d\n",
+ ("CQ AsyncEvent EP %s event '%s' vendor code %#I64d\n",
p_endpt->tag, ib_get_async_event_str(p_event_rec->code),
p_event_rec->vendor_specific) );
}
@@ -2904,15 +2916,11 @@
__cm_buf_mgr_construct( &p_port->cm_buf_mgr );
- p_port->cm_buf_mgr.recv_pool_depth =
- min( (uint32_t) p_port->p_adapter->params.rq_depth * 8,
- p_port->p_ca_attrs->max_srq_wrs/2 );
-
- DIPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_INIT,
- ("Port[%d] cm_recv_mgr.recv_pool_depth %d max_srq_wrs/2 %d\n",
+ DIPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_CM,
+ ("Port[%d] srq_max_depth %d srq_depth %d\n",
p_port->port_num,
- p_port->cm_buf_mgr.recv_pool_depth,
- p_port->p_ca_attrs->max_srq_wrs/2 ) );
+ p_port->ib_mgr.srq_max_depth,
+ p_port->ib_mgr.srq_depth) );
cl_qlist_init( &p_port->cm_buf_mgr.oop_list );
cl_status = cl_spinlock_init( &p_port->cm_buf_mgr.lock );
@@ -2954,7 +2962,7 @@
/* Allocate the receive descriptors pool */
cl_status = cl_qpool_init( &p_port->cm_buf_mgr.recv_pool,
- p_port->cm_buf_mgr.recv_pool_depth,
+ p_port->ib_mgr.srq_max_depth,
0,
0,
sizeof( ipoib_cm_recv_desc_t ),
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipoib_endpoint.h.patch
Type: application/octet-stream
Size: 318 bytes
Desc: ipoib_endpoint.h.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110224/aced15dd/attachment.obj>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipoib_port.h.patch
Type: application/octet-stream
Size: 712 bytes
Desc: ipoib_port.h.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110224/aced15dd/attachment-0001.obj>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipoib_port.cpp.patch
Type: application/octet-stream
Size: 873 bytes
Desc: ipoib_port.cpp.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110224/aced15dd/attachment-0002.obj>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipoib_cm.cpp.patch
Type: application/octet-stream
Size: 4856 bytes
Desc: ipoib_cm.cpp.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110224/aced15dd/attachment-0003.obj>
More information about the ofw
mailing list