[ofw] [RFC] [PATCH 5/12] winverbs: implement IWVDevice
Sean Hefty
sean.hefty at intel.com
Fri Mar 14 23:01:03 PDT 2008
Userspace WinVerb implementation for device objects. Most
calls are implemented, except for inherited IWVOverlapped routines.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
I started to implement GID and PKey caching, but removed the code. It
ends up being non-trivial, so I decided to defer it. My plan was to
update cached tables when QueryPort() was called, and let QueryGid() /
QueryPkey() return the cached values. But this requires synchronization
to the cache, QueryPort() could fail to obtain the tables, etc.
The IOCTL interfaces are defined to retrieve the entire tables,
which will make it easier to add caching later. But the WinVerb API
itself only returns 1 entry at a time, so for now we issue an IOCTL
for every query.
Index: core/winverbs/user/wv_device.cpp
===================================================================
--- core/winverbs/user/wv_device.cpp (revision 981)
+++ core/winverbs/user/wv_device.cpp (working copy)
@@ -27,20 +27,63 @@
* SOFTWARE.
*/
+#include <windows.h>
+#include <iba\ib_ci.h>
#include "wv_base.h"
#include "wv_device.h"
+#include "wv_cq.h"
+#include "..\wv_ioctl.h"
+#include "wv_pd.h"
+#define WV_MAX_LIB_NAME 32
+#ifdef _DEBUG_
+static char *WV_LIB_EXTENSION = "d.dll";
+#else
+static char *WV_LIB_EXTENSION = "dll";
+#endif
+
+CWVDevice::CWVDevice(CWVProvider *pProvider)
+{
+ pProvider->AddRef();
+ m_pProvider = pProvider;
+ m_hFile = pProvider->m_hFile;
+
+ m_hVerbsDevice = NULL;
+ m_Id = 0;
+
+ m_hLib = NULL;
+ m_nRef = 1;
+}
+
+CWVDevice::~CWVDevice()
+{
+ DWORD bytes;
+ HRESULT hr;
+
+ if (m_hVerbsDevice != NULL) {
+ m_Verbs.pre_close_ca(m_hVerbsDevice);
+ hr = DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_CLOSE, &m_Id, sizeof m_Id,
+ NULL, 0, &bytes, NULL) ?
+ WV_SUCCESS : HRESULT_FROM_WIN32(GetLastError());
+ m_Verbs.post_close_ca(m_hVerbsDevice, (ib_api_status_t) hr);
+ }
+
+ if (m_hLib != NULL) {
+ FreeLibrary(m_hLib);
+ }
+ m_pProvider->Release();
+}
+
STDMETHODIMP CWVDevice::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
- if (riid != IID_IUnknown && riid != IID_IWVDevice)
- {
+ if (riid != IID_IUnknown && riid != IID_IWVDevice) {
*ppvObj = NULL;
return E_NOINTERFACE;
}
*ppvObj = this;
- InterlockedIncrement(&m_nRef);
+ AddRef();
return WV_SUCCESS;
}
@@ -56,8 +99,9 @@
ULONG ref;
ref = (ULONG) InterlockedDecrement(&m_nRef);
- if (ref == 0)
+ if (ref == 0) {
delete this;
+ }
return ref;
}
@@ -78,149 +122,289 @@
return E_NOTIMPL;
}
+// Destructor will do necessary cleanup.
STDMETHODIMP CWVDevice::
Open(UINT64 Guid)
{
- UNREFERENCED_PARAMETER(Guid);
+ char libname[WV_MAX_LIB_NAME];
+ WV_IO_ID *pId;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
- return E_NOTIMPL;
-}
+ m_Guid = Guid;
+ if (!DeviceIoControl(m_hFile, WV_IOCTL_LIBRARY_QUERY, &m_Guid,
+ (DWORD) sizeof m_Guid, libname,
+ sizeof libname - sizeof WV_LIB_EXTENSION, &bytes, NULL)) {
+ return HRESULT_FROM_WIN32(GetLastError());
+ }
-//static void
-//SetDeviceCap(DWORD *pFlags, ib_ca_attr_t *pCaAttr)
-//{
-// *pFlags = 0;
-//
-// *pFlags |= pCaAttr->bad_pkey_ctr_support ? WV_DEVICE_BAD_PKEY_COUNTER : 0;
-// *pFlags |= pCaAttr->bad_qkey_ctr_support ? WV_DEVICE_BAD_QKEY_COUNTER : 0;
-// *pFlags |= pCaAttr->apm_support ? WV_DEVICE_PATH_MIGRATION : 0;
-// *pFlags |= pCaAttr->av_port_check ? WV_DEVICE_AH_PORT_CHECKING : 0;
-// *pFlags |= pCaAttr->change_primary_port ? WV_DEVICE_CHANGE_PHYSICAL_PORT : 0;
-// *pFlags |= pCaAttr->modify_wr_depth ? WV_DEVICE_RESIZE_MAX_WR : 0;
-// *pFlags |= pCaAttr->modify_srq_depth ? WV_DEVICE_SRQ_RESIZE : 0;
-// *pFlags |= pCaAttr->current_qp_state_support ? WV_DEVICE_QP_STATE_MODIFIER : 0;
-// *pFlags |= pCaAttr->shutdown_port_capability ? WV_DEVICE_SHUTDOWN_PORT : 0;
-// *pFlags |= pCaAttr->init_type_support ? WV_DEVICE_INIT_TYPE : 0;
-// *pFlags |= pCaAttr->port_active_event_support ? WV_DEVICE_PORT_ACTIVE_EVENT : 0;
-// *pFlags |= pCaAttr->system_image_guid_support ? WV_DEVICE_SYSTEM_IMAGE_GUID : 0;
-// //WV_DEVICE_RC_RNR_NAK_GENERATION - not set
-// //WV_DEVICE_BATCH_NOTIFY_CQ - not set
-//}
+ strcpy(libname + bytes, WV_LIB_EXTENSION);
+ m_hLib = LoadLibrary(libname);
+ if (m_hLib == NULL) {
+ return HRESULT_FROM_WIN32(GetLastError());
+ }
-//static void
-//SetDevicePages(DWORD *pFlags, ib_ca_attr_t *pCaAttr)
-//{
-// unsigned int i;
-// UINT32 size;
-//
-// *pFlags = 0;
-//
-// for (i = 0; i < pCaAttr->num_page_sizes; i++)
-// {
-// size = pCaAttr->p_page_size[i];
-// *pFlags |= (size & (size - 1)) ? 0 : size;
-// }
-//}
+ hr = WvGetUserVerbs(m_hLib, &m_Verbs);
+ if (FAILED(hr)) {
+ return hr;
+ }
-//static void
-//ConvertDeviceAttr(WV_DEVICE_ATTRIBUTES* pAttributes, ib_ca_attr_t *pCaAttr)
-//{
-// *((UINT64*)pAttributes->FwVersion) = pCaAttr->fw_ver; //TODO: check this: 64 byte array?
-// pAttributes->NodeGuid = pCaAttr->ca_guid;
-// pAttributes->SystemImageGuid = pCaAttr->system_image_guid;
-// pAttributes->VendorId = pCaAttr->vend_id;
-// pAttributes->VendorPartId = pCaAttr->dev_id;
-// pAttributes->HwVersion = pCaAttr->revision;
-//
-// SetDeviceCap(&pAttributes->CapabilityFlags, pCaAttr);
-// pAttributes->AtomicCapability = (WV_ATOMIC_CAPABILITIES) pCaAttr->atomicity;
-// SetDevicePages(&pAttributes->PageSizeCapabilityFlags, pCaAttr);
-//
-// pAttributes->MaxMrSize = pCaAttr->init_region_size;
-// pAttributes->MaxQp = pCaAttr->max_qps;
-// pAttributes->MaxQpWr = pCaAttr->max_wrs;
-// pAttributes->MaxSge = pCaAttr->max_sges;
-// pAttributes->MaxCq = pCaAttr->max_cqs;
-// pAttributes->MaxCqEntries = pCaAttr->max_cqes;
-// pAttributes->MaxMr = pCaAttr->init_regions;
-// pAttributes->MaxPd = pCaAttr->max_pds;
-// pAttributes->MaxQpResponderResources = pCaAttr->max_qp_resp_res;
-// pAttributes->MaxResponderResources = pCaAttr->max_resp_res;
-// pAttributes->MaxQpInitiatorDepth = pCaAttr->max_qp_init_depth;
-// pAttributes->MaxMw = pCaAttr->init_windows;
-// pAttributes->MaxMulticast = pCaAttr->max_mcast_grps;
-// pAttributes->MaxQpAttach = pCaAttr->max_qps_per_mcast_grp;
-// pAttributes->MaxMulticastQp = pCaAttr->max_mcast_qps;
-// pAttributes->MaxAh = pCaAttr->max_addr_handles;
-// pAttributes->MaxFmr = pCaAttr->max_fmr;
-// pAttributes->MaxMapPerFmr = pCaAttr->max_map_per_fmr;
-// pAttributes->MaxSrq = pCaAttr->max_srq;
-// pAttributes->MaxSrqWr = pCaAttr->max_srq_wrs;
-// pAttributes->MaxSrqSge = pCaAttr->max_srq_sges;
-// pAttributes->MaxPkeys = pCaAttr->max_partitions;
-// pAttributes->LocalAckDelay = pCaAttr->local_ack_delay;
-// pAttributes->PhysPortCount = pCaAttr->num_ports;
-//}
+ stat = m_Verbs.pre_open_ca(m_Guid, &verbsData, &m_hVerbsDevice);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+ bytes = sizeof WV_IO_ID + max(verbsData.input_size, verbsData.output_size);
+ pId = (WV_IO_ID *) buf.Get(bytes);
+ if (pId == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pId->Id = m_Guid;
+ pId->VerbInfo = verbsData.command;
+ RtlCopyMemory(pId + 1, verbsData.p_inout_buf, verbsData.input_size);
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_OPEN,
+ pId, sizeof WV_IO_ID + verbsData.input_size,
+ pId, sizeof WV_IO_ID + verbsData.output_size,
+ &bytes, NULL)) {
+ m_Id = pId->Id;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ stat = m_Verbs.post_open_ca(m_Guid, (ib_api_status_t) hr,
+ &m_hVerbsDevice, &verbsData);
+ if (SUCCEEDED(hr) && stat != IB_SUCCESS) {
+ hr = WvConvertIbStatus(stat);
+ }
+
+ return hr;
+}
+
STDMETHODIMP CWVDevice::
Query(WV_DEVICE_ATTRIBUTES* pAttributes)
{
- UNREFERENCED_PARAMETER(pAttributes);
+ WV_IO_DEVICE_ATTRIBUTES attr;
+ DWORD bytes;
- return E_NOTIMPL;
+ if (!DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_QUERY, &m_Id, sizeof m_Id,
+ &attr, sizeof attr, &bytes, NULL)) {
+ return HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ RtlCopyMemory(pAttributes->FwVersion, attr.FwVersion, 64);
+ pAttributes->NodeGuid = attr.NodeGuid;
+ pAttributes->SystemImageGuid = attr.SystemImageGuid;
+ pAttributes->VendorId = attr.VendorId;
+ pAttributes->VendorPartId = attr.VendorPartId;
+ pAttributes->HwVersion = attr.HwVersion;
+ pAttributes->CapabilityFlags = attr.CapabilityFlags;
+ pAttributes->AtomicCapability = (WV_ATOMIC_CAPABILITIES) attr.AtomicCapability;
+ pAttributes->PageSizeCapabilityFlags = attr.PageSizeCapabilityFlags;
+ pAttributes->MaxMrSize = attr.MaxMrSize;
+ pAttributes->MaxQp = attr.MaxQp;
+ pAttributes->MaxQpWr = attr.MaxQpWr;
+ pAttributes->MaxSge = attr.MaxSge;
+ pAttributes->MaxCq = attr.MaxCq;
+ pAttributes->MaxCqEntries = attr.MaxCqEntries;
+ pAttributes->MaxMr = attr.MaxMr;
+ pAttributes->MaxPd = attr.MaxPd;
+ pAttributes->MaxQpResponderResources = attr.MaxQpResponderResources;
+ pAttributes->MaxResponderResources = attr.MaxResponderResources;
+ pAttributes->MaxQpInitiatorDepth = attr.MaxQpInitiatorDepth;
+ pAttributes->MaxInlineSend = attr.MaxInlineSend;
+ pAttributes->MaxMw = attr.MaxMw;
+ pAttributes->MaxMulticast = attr.MaxMulticast;
+ pAttributes->MaxQpAttach = attr.MaxQpAttach;
+ pAttributes->MaxMulticastQp = attr.MaxMulticastQp;
+ pAttributes->MaxAh = attr.MaxAh;
+ pAttributes->MaxFmr = attr.MaxFmr;
+ pAttributes->MaxMapPerFmr = attr.MaxMapPerFmr;
+ pAttributes->MaxSrq = attr.MaxSrq;
+ pAttributes->MaxSrqWr = attr.MaxSrqWr;
+ pAttributes->MaxSrqSge = attr.MaxSrqSge;
+ pAttributes->MaxPkeys = attr.MaxPkeys;
+ pAttributes->LocalAckDelay = attr.LocalAckDelay;
+ pAttributes->PhysPortCount = attr.PhysPortCount;
+
+ return WV_SUCCESS;
}
STDMETHODIMP CWVDevice::
QueryPort(UINT8 PortNumber, WV_PORT_ATTRIBUTES* pAttributes)
{
- UNREFERENCED_PARAMETER(PortNumber);
- UNREFERENCED_PARAMETER(pAttributes);
+ WV_IO_DEVICE_PORT_QUERY query;
+ DWORD bytes;
- return E_NOTIMPL;
+ query.Id = m_Id;
+ query.PortNumber = PortNumber;
+ RtlZeroMemory(&query.Reserved, sizeof query.Reserved);
+
+ if (!DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_PORT_QUERY, &query,
+ sizeof query, pAttributes, sizeof *pAttributes,
+ &bytes, NULL)) {
+ return HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ return WV_SUCCESS;
}
+// TODO: cache GID table and updated with QueryPort()
STDMETHODIMP CWVDevice::
QueryGid(UINT8 PortNumber, DWORD Index, WV_GID* pGid)
{
- UNREFERENCED_PARAMETER(PortNumber);
- UNREFERENCED_PARAMETER(Index);
- UNREFERENCED_PARAMETER(pGid);
+ WV_IO_DEVICE_PORT_QUERY query;
+ WV_GID *gidtable;
+ DWORD ngid, bytes;
+ HRESULT hr;
+ CWVBuffer buf;
- return E_NOTIMPL;
+ bytes = sizeof WV_GID * (Index + 1);
+ gidtable = (WV_GID *) buf.Get(bytes);
+ if (gidtable == NULL) {
+ return WV_NO_MEMORY;
+ }
+
+ query.Id = m_Id;
+ query.PortNumber = PortNumber;
+ RtlZeroMemory(&query.Reserved, sizeof query.Reserved);
+
+ if (!DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_GID_QUERY, &query,
+ sizeof query, gidtable, bytes, &bytes, NULL)) {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ goto out;
+ }
+
+ ngid = bytes / sizeof WV_GID;
+ if (Index >= ngid) {
+ hr = WV_INVALID_PARAMETER_2;
+ goto out;
+ }
+ *pGid = gidtable[Index];
+ hr = WV_SUCCESS;
+
+out:
+ buf.Put();
+ return hr;
}
+// TODO: cache PKey table and update with QueryPort()
STDMETHODIMP CWVDevice::
QueryPkey(UINT8 PortNumber, DWORD Index, UINT16* pPkey)
{
- UNREFERENCED_PARAMETER(PortNumber);
- UNREFERENCED_PARAMETER(Index);
- UNREFERENCED_PARAMETER(pPkey);
+ WV_IO_DEVICE_PORT_QUERY query;
+ UINT16 *pkeytable;
+ DWORD npkey, bytes;
+ HRESULT hr;
+ CWVBuffer buf;
- return E_NOTIMPL;
+ bytes = sizeof UINT16 * (Index + 1);
+ pkeytable = (UINT16 *) buf.Get(bytes);
+ if (pkeytable == NULL) {
+ return WV_NO_MEMORY;
+ }
+
+ query.Id = m_Id;
+ query.PortNumber = PortNumber;
+ RtlZeroMemory(&query.Reserved, sizeof query.Reserved);
+
+ if (!DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_PKEY_QUERY, &query,
+ sizeof query, pkeytable, bytes, &bytes, NULL)) {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ goto out;
+ }
+
+ npkey = bytes / sizeof UINT16;
+ if (Index >= npkey) {
+ hr = WV_INVALID_PARAMETER_2;
+ goto out;
+ }
+ *pPkey = pkeytable[Index];
+ hr = WV_SUCCESS;
+
+out:
+ buf.Put();
+ return hr;
}
STDMETHODIMP CWVDevice::
CreateCompletionQueue(SIZE_T *pEntries, IWVCompletionQueue** ppCq)
{
- UNREFERENCED_PARAMETER(pEntries);
- UNREFERENCED_PARAMETER(ppCq);
+ HRESULT hr;
+ CWVCompletionQueue *cq;
- return E_NOTIMPL;
+ cq = new CWVCompletionQueue(this);
+ if (cq == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ cq->QueryInterface(IID_IWVCompletionQueue, (LPVOID*) ppCq);
+ cq->Release();
+
+ hr = cq->Create(pEntries);
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ cq->Release();
+err1:
+ *ppCq = NULL;
+ return hr;
}
STDMETHODIMP CWVDevice::
AllocateProtectionDomain(IWVProtectionDomain** ppPd)
{
- UNREFERENCED_PARAMETER(ppPd);
+ HRESULT hr;
+ CWVProtectionDomain *pd;
- return E_NOTIMPL;
+ pd = new CWVProtectionDomain(this);
+ if (pd == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ pd->QueryInterface(IID_IWVProtectionDomain, (LPVOID*) ppPd);
+ pd->Release();
+
+ hr = pd->Allocate();
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ pd->Release();
+err1:
+ *ppPd = NULL;
+ return hr;
}
STDMETHODIMP CWVDevice::
Notify(OVERLAPPED* pOverlapped, WV_EVENT* pEvent)
{
- UNREFERENCED_PARAMETER(pOverlapped);
- UNREFERENCED_PARAMETER(pEvent);
+ DWORD bytes;
+ HRESULT hr;
- return E_NOTIMPL;
+ if (DeviceIoControl(m_hFile, WV_IOCTL_DEVICE_NOTIFY, &m_Id, sizeof m_Id,
+ pEvent, sizeof WV_EVENT, &bytes, pOverlapped)) {
+ hr = WV_SUCCESS;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ return hr;
}
Index: core/winverbs/user/wv_device.h
===================================================================
--- core/winverbs/user/wv_device.h (revision 981)
+++ core/winverbs/user/wv_device.h (working copy)
@@ -32,7 +32,11 @@
#ifndef _WV_DEVICE_H_
#define _WV_DEVICE_H_
-#include "rdma\winverbs.h"
+#include <iba\ib_types.h>
+#include <iba\ib_uvp.h>
+#include <rdma\winverbs.h>
+#include "wv_provider.h"
+#include "..\wv_ioctl.h"
class CWVDevice : IWVDevice
{
@@ -56,22 +60,22 @@
STDMETHODIMP AllocateProtectionDomain(IWVProtectionDomain** ppPd);
STDMETHODIMP Notify(OVERLAPPED* pOverlapped, WV_EVENT* pEvent);
- CWVDevice(IWVProvider *pProvider)
- {
- pProvider->AddRef();
- m_pProvider = pProvider;
- m_nRef = 0;
- }
- ~CWVDevice()
- {
- m_pProvider->Release();
- }
+ CWVDevice(CWVProvider *pProvider);
+ ~CWVDevice();
STDMETHODIMP Open(UINT64 Guid);
-private:
+ CWVProvider *m_pProvider;
+ uvp_interface_t m_Verbs;
+ HANDLE m_hFile;
+
+ ib_ca_handle_t m_hVerbsDevice;
+ UINT64 m_Id;
+ UINT64 m_Guid;
+
+protected:
+ HMODULE m_hLib;
volatile LONG m_nRef;
- IWVProvider *m_pProvider;
};
#endif // __WV_DEVICE_H_
More information about the ofw
mailing list