[ofw] [Patch][IPoIB_NDIS6_CM] Datapath improvement
Smith, Stan
stan.smith at intel.com
Wed Dec 15 09:21:12 PST 2010
________________________________
From: Alex Naslednikov [mailto:xalex at mellanox.co.il]
Sent: Wednesday, December 15, 2010 8:42 AM
To: Smith, Stan; ofw at lists.openfabrics.org
Subject: RE: [ofw] [Patch][IPoIB_NDIS6_CM] Datapath improvement
Hello Stan,
This is definitely a bug. I missed that hdr field withing ipoib_port_t is defined as array of size 1 and not a pointer:
ipoib_prealloc_hdr_t hdr[1];
Thus, I was confused by the following allocation:
p_port = (ipoib_port_t *) cl_zalloc( sizeof(ipoib_port_t) +
(sizeof(ipoib_prealloc_hdr_t) * (p_adapter->params.sq_depth - 1)) );
The overall size is of course p_adapter->params.sq_depth and not p_adapter->params.sq_depth-1
An honest mistake, stuff happens.
Why the change to remove passing in 'alloc_size'? Either way is fine, just curious?
stan.
Please, see the patch for it:
Index: ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp
===================================================================
--- ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (revision 7041)
+++ ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (working copy)
@@ -109,8 +109,7 @@
******************************************************************************/
static void
__port_construct(
- IN ipoib_port_t* const p_port,
- IN int32_t alloc_size);
+ IN ipoib_port_t* const p_port );
static ib_api_status_t
__port_init(
@@ -689,8 +688,7 @@
static void
__port_construct(
- IN ipoib_port_t* const p_port,
- IN int32_t alloc_size)
+ IN ipoib_port_t* const p_port )
{
IPOIB_ENTER( IPOIB_DBG_INIT );
@@ -712,7 +710,7 @@
KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );
KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );
- for ( int i = 0; i < alloc_size; ++i )
+ for ( int i = 0; i < p_port->p_adapter->params.sq_depth ; ++i )
{
p_port->hdr[i].phys_addr = cl_get_physaddr( &p_port->hdr[i] );
p_port->hdr[i].resv = 0;
From: Smith, Stan [mailto:stan.smith at intel.com]
Sent: Thursday, December 09, 2010 7:50 PM
To: Alex Naslednikov; ofw at lists.openfabrics.org
Subject: RE: [ofw] [Patch][IPoIB_NDIS6_CM] Datapath improvement
Please see bug below....
________________________________
From: ofw-bounces at lists.openfabrics.org [mailto:ofw-bounces at lists.openfabrics.org] On Behalf Of Alex Naslednikov
Sent: Wednesday, December 01, 2010 2:28 AM
To: ofw at lists.openfabrics.org
Subject: [ofw] [Patch][IPoIB_NDIS6_CM] Datapath improvement
This patch removes calls to MmGetPhysAddr from datapath
Tested with SdpConnect.exe on Win2008x64 vs. R2, with and w\o LSO, with the following data sizes:
65536, 655360, 512000, 519317 (bytes)
In addition, it passed several regression runs
Signed-off by: Alexander Naslednikov (xalex at mellanox.co.il)
Index: ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp
===================================================================
--- ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (revision 3007)
+++ ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (working copy)
@@ -111,7 +111,8 @@
******************************************************************************/
static void
__port_construct(
- IN ipoib_port_t* const p_port );
+ IN ipoib_port_t* const p_port,
+ IN int32_t alloc_size);
static ib_api_status_t
__port_init(
@@ -638,6 +639,7 @@
CL_ASSERT( !p_adapter->p_port );
+ // Allocate PORT object along with "send queue depth" - 1 IPoIB headers
p_port = (ipoib_port_t *) cl_zalloc( sizeof(ipoib_port_t) +
(sizeof(ipoib_prealloc_hdr_t) * (p_adapter->params.sq_depth - 1)) );
if( !p_port )
@@ -652,7 +654,7 @@
gp_ipoib_port = p_port;
#endif
- __port_construct( p_port );
+ __port_construct( p_port, p_adapter->params.sq_depth -1 ); <===== remove '-1' as it shorts the hdr initialization by 1.
status = __port_init( p_port, p_adapter, p_pnp_rec );
if( status != IB_SUCCESS )
@@ -689,7 +691,8 @@
static void
__port_construct(
- IN ipoib_port_t* const p_port )
+ IN ipoib_port_t* const p_port,
+ IN int32_t alloc_size)
{
IPOIB_ENTER( IPOIB_DBG_INIT );
@@ -710,6 +713,12 @@
KeInitializeEvent( &p_port->sa_event, NotificationEvent, TRUE );
KeInitializeEvent( &p_port->leave_mcast_event, NotificationEvent, TRUE );
+
+ for ( int i = 0; i < alloc_size; ++i )
+ {
+ p_port->hdr[i].phys_addr = cl_get_physaddr( &p_port->hdr[i] );
+ p_port->hdr[i].resv = 0;
+ }
IPOIB_EXIT( IPOIB_DBG_INIT );
}
@@ -5278,9 +5287,7 @@
ASSERT( hdr_idx < s_buf->p_port->p_adapter->params.sq_depth );
/* Set up IPoIB Header */
- s_buf->p_port->hdr[hdr_idx].type = p_eth_hdr->type;
- s_buf->p_port->hdr[hdr_idx].resv = 0;
-
+ s_buf->p_port->hdr[hdr_idx].type = p_eth_hdr->type;
//Init send buffer to 0
s_buf->p_send_buf = NULL;
@@ -5309,8 +5316,7 @@
cl_perf_start( SendMgrFilter );
/* Put first DS to be IPoIB Header */
- p_desc->send_wr[0].local_ds[0].vaddr =
- cl_get_physaddr( &s_buf->p_port->hdr[hdr_idx] );
+ p_desc->send_wr[0].local_ds[0].vaddr = s_buf->p_port->hdr[hdr_idx].phys_addr;
p_desc->send_wr[0].local_ds[0].length = sizeof(ipoib_hdr_t);
p_desc->send_wr[0].local_ds[0].lkey = s_buf->p_port->ib_mgr.lkey;
Index: ulp/ipoib_NDIS6_CM/kernel/ipoib_port.h
===================================================================
--- ulp/ipoib_NDIS6_CM/kernel/ipoib_port.h (revision 2999)
+++ ulp/ipoib_NDIS6_CM/kernel/ipoib_port.h (working copy)
@@ -168,6 +168,12 @@
* resv
* Reserved portion of IPoIB header.
*********/
+
+typedef struct _ipoib_prealloc_hdr:public ipoib_hdr_t
+{
+ uint64_t phys_addr;
+} PACK_SUFFIX ipoib_prealloc_hdr_t;
+
static const uint32_t EthHeaderOffset = sizeof(eth_hdr_t);
//We reuse eth header to put there IPoIB header for LSO Net Buffers.
@@ -369,6 +375,7 @@
NDIS_TCP_IP_CHECKSUM_PACKET_INFO ndis_csum;
#if IPOIB_INLINE_RECV
recv_buf_t buf;
+ uint64_t phys_addr;
#else
recv_buf_t *p_buf;
#endif
@@ -626,7 +633,7 @@
ib_net16_t base_lid;
LONG n_no_progress;
PIO_WORKITEM pPoWorkItem;
- ipoib_hdr_t hdr[1]; /* Must be last! */
+ ipoib_prealloc_hdr_t hdr[1]; /* Must be last! */
} ipoib_port_t;
#pragma warning(default:4324)
Alexander (XaleX) Naslednikov
SW Networking Team
Mellanox Technologies
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20101215/7d073276/attachment.html>
More information about the ofw
mailing list