[ofw] [PATCH] winverbs/lib: cache pkey table
Sean Hefty
sean.hefty at intel.com
Mon Jan 18 11:43:35 PST 2010
Connection establishment rates take a huge performance hit
as a result of using FindPkey, which results in querying the
HCA to lookup current pkey information. Add a cache to the
userspace library for pkey information.
The cache is updated using the CWVDevice:Notify() routine.
Before any access is made to the cache, the caller checks
for changes to the partition tables. If a change is found,
then the cache is updated. Otherwise, the cache is accessed
to map pkey values to indices.
This results in the connection rate over winverbs almost
doubling.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
trunk/core/winverbs/user/wv_device.cpp | 149 +++++++++++++++++++++++++-------
trunk/core/winverbs/user/wv_device.h | 20 ++++
2 files changed, 137 insertions(+), 32 deletions(-)
diff --git a/trunk/core/winverbs/user/wv_device.cpp b/trunk/core/winverbs/user/wv_device.cpp
index 9aac788..b260d41 100644
--- a/trunk/core/winverbs/user/wv_device.cpp
+++ b/trunk/core/winverbs/user/wv_device.cpp
@@ -48,6 +48,8 @@ CWVDevice::CWVDevice(CWVProvider *pProvider)
m_pProvider = pProvider;
m_hFile = pProvider->m_hFile;
+ m_PortCount = 0;
+ m_pPorts = NULL;
m_hVerbsDevice = NULL;
m_hLib = NULL;
}
@@ -121,11 +123,56 @@ post:
hr = WvConvertIbStatus(stat);
}
+ if (SUCCEEDED(hr)) {
+ hr = InitPorts();
+ }
+
+ return hr;
+}
+
+STDMETHODIMP CWVDevice::
+InitPorts()
+{
+ WV_DEVICE_ATTRIBUTES attr;
+ WV_PORT *port;
+ HRESULT hr;
+
+ hr = Query(&attr);
+ if (FAILED(hr)) {
+ return hr;
+ }
+
+ m_pPorts = new WV_PORT[attr.PhysPortCount];
+ if (m_pPorts == NULL) {
+ return WV_NO_MEMORY;
+ }
+
+ RtlZeroMemory(m_pPorts, sizeof(WV_PORT) * attr.PhysPortCount);
+ for (m_PortCount = 0; m_PortCount < attr.PhysPortCount; m_PortCount++) {
+ port = &m_pPorts[m_PortCount];
+
+ port->m_Overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
+ if (port->m_Overlap.hEvent == NULL) {
+ hr = WV_INSUFFICIENT_RESOURCES;
+ break;
+ }
+
+ port->m_Overlap.hEvent = (HANDLE) ((ULONG_PTR) port->m_Overlap.hEvent | 1);
+
+ port->m_Flags = 0xFFFFFFFF;
+ hr = UpdatePort(m_PortCount + 1);
+ if (FAILED(hr)) {
+ CloseHandle(port->m_Overlap.hEvent);
+ break;
+ }
+ }
+
return hr;
}
CWVDevice::~CWVDevice()
{
+ WV_PORT *port;
DWORD bytes;
HRESULT hr;
@@ -137,6 +184,16 @@ CWVDevice::~CWVDevice()
m_Verbs.post_close_ca(m_hVerbsDevice, (ib_api_status_t) hr);
}
+ while (m_PortCount--) {
+ port = &m_pPorts[m_PortCount];
+ GetOverlappedResult(&port->m_Overlap, &bytes, TRUE);
+ CloseHandle(port->m_Overlap.hEvent);
+ }
+
+ if (m_pPorts != NULL) {
+ delete m_pPorts;
+ }
+
if (m_hLib != NULL) {
FreeLibrary(m_hLib);
}
@@ -252,6 +309,26 @@ QueryPort(UINT8 PortNumber, WV_PORT_ATTRIBUTES* pAttributes)
return HRESULT_FROM_WIN32(GetLastError());
}
+ pAttributes->PkeyTableLength = min(pAttributes->PkeyTableLength, WV_MAX_PKEYS);
+ return WV_SUCCESS;
+}
+
+STDMETHODIMP CWVDevice::
+UpdatePort(UINT8 PortNumber)
+{
+ WV_PORT *port = &m_pPorts[PortNumber - 1];
+ HRESULT hr;
+
+ if (port->m_Flags & WV_EVENT_PARTITION) {
+ UpdatePkeys(PortNumber);
+ }
+
+ port->m_Flags = 0;
+ hr = Notify(PortNumber, &port->m_Overlap, &port->m_Flags);
+ if (FAILED(hr) && hr != WV_IO_PENDING) {
+ return hr;
+ }
+
return WV_SUCCESS;
}
@@ -316,63 +393,73 @@ FindGid(UINT8 PortNumber, WV_GID *pGid, DWORD *pIndex)
}
STDMETHODIMP CWVDevice::
-QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey)
+UpdatePkeys(UINT8 PortNumber)
{
WV_IO_DEVICE_PORT_QUERY query;
- NET16 *pkeytable;
- DWORD npkey, bytes;
- HRESULT hr;
- CWVBuffer buf;
+ WV_PORT *port;
+ DWORD bytes;
- bytes = sizeof NET16 * (Index + 1);
- pkeytable = (NET16 *) buf.Get(bytes);
- if (pkeytable == NULL) {
- return WV_NO_MEMORY;
- }
+ port = &m_pPorts[PortNumber - 1];
+ bytes = sizeof(port->m_PkeyTable);
query.Id = m_Id;
query.PortNumber = PortNumber;
RtlZeroMemory(&query.Reserved, sizeof query.Reserved);
if (!WvDeviceIoControl(m_hFile, WV_IOCTL_DEVICE_PKEY_QUERY, &query,
- sizeof query, pkeytable, bytes, &bytes, NULL)) {
- hr = HRESULT_FROM_WIN32(GetLastError());
- goto out;
+ sizeof query, port->m_PkeyTable, bytes, &bytes, NULL)) {
+ port->m_PkeyCount = 0;
+ return HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ port->m_PkeyCount = (UINT16) (bytes / sizeof NET16);
+ return WV_SUCCESS;
+}
+
+STDMETHODIMP CWVDevice::
+QueryPkey(UINT8 PortNumber, UINT16 Index, NET16* pPkey)
+{
+ WV_PORT *port = &m_pPorts[PortNumber - 1];
+ HRESULT hr;
+
+ EnterCriticalSection(&m_CritSec);
+ if (HasOverlappedIoCompleted(&port->m_Overlap)) {
+ UpdatePort(PortNumber);
}
- npkey = bytes / sizeof NET16;
- if (Index >= npkey) {
+ if (Index < port->m_PkeyCount) {
+ *pPkey = port->m_PkeyTable[Index];
+ hr = WV_SUCCESS;
+ } else {
hr = WV_INVALID_PARAMETER_2;
- goto out;
}
- *pPkey = pkeytable[Index];
- hr = WV_SUCCESS;
-out:
- buf.Put();
+ LeaveCriticalSection(&m_CritSec);
return hr;
}
STDMETHODIMP CWVDevice::
FindPkey(UINT8 PortNumber, NET16 Pkey, UINT16 *pIndex)
{
- NET16 key;
+ WV_PORT *port = &m_pPorts[PortNumber - 1];
UINT16 index;
- HRESULT hr;
+ HRESULT hr = WV_INVALID_ADDRESS;
- for (index = 0; true; index++) {
- hr = QueryPkey(PortNumber, index, &key);
- if (FAILED(hr)) {
- return hr;
- }
+ EnterCriticalSection(&m_CritSec);
+ if (HasOverlappedIoCompleted(&port->m_Overlap)) {
+ UpdatePort(PortNumber);
+ }
- if (Pkey == key) {
- *pIndex = (UINT16) index;
- return WV_SUCCESS;
+ for (index = 0; index < port->m_PkeyCount; index++) {
+ if (Pkey == port->m_PkeyTable[index]) {
+ *pIndex = index;
+ hr = WV_SUCCESS;
+ break;
}
}
- return WV_INVALID_ADDRESS;
+ LeaveCriticalSection(&m_CritSec);
+ return hr;
}
STDMETHODIMP CWVDevice::
diff --git a/trunk/core/winverbs/user/wv_device.h b/trunk/core/winverbs/user/wv_device.h
index 7e98da1..9f4009c 100644
--- a/trunk/core/winverbs/user/wv_device.h
+++ b/trunk/core/winverbs/user/wv_device.h
@@ -38,6 +38,18 @@
#include "wv_provider.h"
#include "wv_base.h"
+#define WV_MAX_PKEYS 16
+
+typedef struct _WV_PORT
+{
+ OVERLAPPED m_Overlap;
+ DWORD m_Flags;
+ UINT16 m_PkeyCount;
+ NET16 m_PkeyTable[WV_MAX_PKEYS];
+
+} WV_PORT;
+
+
class CWVDevice : IWVDevice, public CWVBase
{
public:
@@ -105,7 +117,13 @@ public:
protected:
HMODULE m_hLib;
- STDMETHODIMP Open(NET64 Guid);
+ STDMETHODIMP Open(NET64 Guid);
+
+ WV_PORT *m_pPorts;
+ UINT8 m_PortCount;
+ STDMETHODIMP InitPorts();
+ STDMETHODIMP UpdatePort(UINT8 PortNumber);
+ STDMETHODIMP UpdatePkeys(UINT8 PortNumber);
};
#endif // __WV_DEVICE_H_
More information about the ofw
mailing list