By default, use inline sends, and allow user to limit
inline send size through an environment variable.
To support existing deployments, winverbs make use of the same
environment setting as the ibal ND provider.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Note: The kernel HCA drivers report 0 for the max inline send
value for user space QPs, even if inline sends are enabled. I didn't
understand how to fix the kernel drivers, so they could report the
correct value. The winverbs ND code simply returns the value that it
uses when creating the QP.
This patch should go into WinOF 2.2 once committed to the trunk.
trunk/ulp/netdirect/user/nd_ep.cpp | 23 +++++++++++++++++++----
trunk/ulp/netdirect/user/nd_ep.h | 2 ++
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/trunk/ulp/netdirect/user/nd_ep.cpp b/trunk/ulp/netdirect/user/nd_ep.cpp
index d0fe6ea..7060b20 100644
--- a/trunk/ulp/netdirect/user/nd_ep.cpp
+++ b/trunk/ulp/netdirect/user/nd_ep.cpp
@@ -44,6 +44,16 @@ CNDEndpoint::CNDEndpoint(CNDConnector *pConnector)
m_pOutboundCq = NULL;
}
+STDMETHODIMP_(void) CNDEndpoint::
+InitMaxInline(void)
+{
+ TCHAR val[16];
+ DWORD ret;
+
+ ret = GetEnvironmentVariable("IBNDPROV_MAX_INLINE_SIZE", val, 16);
+ m_MaxInlineSend = (ret > 0 && ret <= 16) ? (SIZE_T) strtoul(val, NULL, 16) : 160;
+}
+
STDMETHODIMP CNDEndpoint::
Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
SIZE_T nInboundEntries, SIZE_T nOutboundEntries,
@@ -63,6 +73,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
m_pOutboundCq->AddRef();
m_InitiatorDepth = OutboundReadLimit;
m_ResponderResources = InboundReadLimit;
+ InitMaxInline();
RtlZeroMemory(&create, sizeof create);
create.pSendCq = pOutboundCq->m_pWvCq;
@@ -74,6 +85,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
create.ReceiveSge = nInboundSge;
create.InitiatorDepth = OutboundReadLimit;
create.ResponderResources = InboundReadLimit;
+ create.MaxInlineSend = m_MaxInlineSend;
create.QpType = WvQpTypeRc;
hr = m_pConnector->m_pAdapter->m_pWvPd->CreateConnectQueuePair(&create, &m_pWvQp);
@@ -97,7 +109,7 @@ Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
}
if (pMaxInlineData) {
- *pMaxInlineData = 0;
+ *pMaxInlineData = m_MaxInlineSend;
}
return ND_SUCCESS;
}
@@ -212,7 +224,8 @@ Send(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, DWORD Flags)
HRESULT hr;
pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl);
- opts = ConvertSendFlags(Flags) | (pSgl && pSgl[0].hMr ? 0 : WV_SEND_INLINE);
+ opts = ConvertSendFlags(Flags) |
+ (pResult->BytesTransferred <= m_MaxInlineSend ? WV_SEND_INLINE : 0);
hr = m_pWvQp->Send((UINT64) pResult, sgl, nSge, opts, 0);
return NDConvertWVStatus(hr);
}
@@ -278,7 +291,8 @@ Read(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
HRESULT hr;
pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl);
- opts = ConvertSendFlags(Flags) | (pSgl[0].hMr ? 0 : WV_SEND_INLINE);
+ opts = ConvertSendFlags(Flags) |
+ (pResult->BytesTransferred ? 0 : WV_SEND_INLINE);
addr = ntohll(pRemoteMwDescriptor->Base) + Offset;
hr = m_pWvQp->Read((UINT64) pResult, sgl, nSge, opts,
htonll(addr), pRemoteMwDescriptor->Token);
@@ -295,7 +309,8 @@ Write(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
HRESULT hr;
pResult->BytesTransferred = ConvertSgl(pSgl, nSge, sgl);
- opts = ConvertSendFlags(Flags) | (pSgl && pSgl[0].hMr ? 0 : WV_SEND_INLINE);
+ opts = ConvertSendFlags(Flags) |
+ (pResult->BytesTransferred <= m_MaxInlineSend ? WV_SEND_INLINE : 0);
addr = ntohll(pRemoteMwDescriptor->Base) + Offset;
hr = m_pWvQp->Write((UINT64) pResult, sgl, nSge, opts, 0,
htonll(addr), pRemoteMwDescriptor->Token);
diff --git a/trunk/ulp/netdirect/user/nd_ep.h b/trunk/ulp/netdirect/user/nd_ep.h
index a49c43e..78fac3f 100644
--- a/trunk/ulp/netdirect/user/nd_ep.h
+++ b/trunk/ulp/netdirect/user/nd_ep.h
@@ -111,6 +111,7 @@ public:
IWVConnectQueuePair *m_pWvQp;
SIZE_T m_InitiatorDepth;
SIZE_T m_ResponderResources;
+ SIZE_T m_MaxInlineSend;
protected:
CNDConnector *m_pConnector;
@@ -122,6 +123,7 @@ protected:
SIZE_T nInboundSge, SIZE_T nOutboundSge,
SIZE_T InboundReadLimit, SIZE_T OutboundReadLimit,
SIZE_T* pMaxInlineData);
+ STDMETHODIMP_(void) InitMaxInline();
STDMETHODIMP_(SIZE_T) ConvertSgl(const ND_SGE* pSgl, SIZE_T nSge, WV_SGE *pWvSgl);
STDMETHODIMP_(DWORD) ConvertSendFlags(DWORD Flags);
STDMETHODIMP_(DWORD) ConvertAccessFlags(DWORD Flags);