[ofw] [RFC] [PATCH 7/12] winverbs: implement IWVProtectionDomain
Sean Hefty
sean.hefty at intel.com
Fri Mar 14 23:03:35 PDT 2008
Userspace WinVerb implementation for protection domains, address
handles, and memory windows. Most calls are implemented, except
for memory registration, which requires more thought.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: core/winverbs/user/wv_pd.cpp
===================================================================
--- core/winverbs/user/wv_pd.cpp (revision 971)
+++ core/winverbs/user/wv_pd.cpp (working copy)
@@ -29,47 +29,170 @@
#include "wv_base.h"
#include "wv_pd.h"
+#include "wv_srq.h"
+#include "wv_qp.h"
+CWVProtectionDomain::CWVProtectionDomain(CWVDevice *pDevice)
+{
+ pDevice->AddRef();
+ m_pDevice = pDevice;
+ m_pVerbs = &pDevice->m_Verbs;
+ m_hFile = pDevice->m_hFile;
+
+ m_hVerbsPd = NULL;
+ m_Id = 0;
+
+ m_nRef = 1;
+}
+
STDMETHODIMP CWVProtectionDomain::
+Allocate(void)
+{
+ WV_IO_ID *pId;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
+
+ stat = m_pVerbs->pre_allocate_pd(m_pDevice->m_hVerbsDevice, &verbsData,
+ &m_hVerbsPd);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = sizeof WV_IO_ID + max(verbsData.input_size, verbsData.output_size);
+ pId = (WV_IO_ID *) buf.Get(bytes);
+ if (pId == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pId->Id = m_pDevice->m_Id;
+ pId->VerbInfo = verbsData.command;
+ RtlCopyMemory(pId + 1, verbsData.p_inout_buf, verbsData.input_size);
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_PD_ALLOCATE,
+ pId, sizeof WV_IO_ID + verbsData.input_size,
+ pId, sizeof WV_IO_ID + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ m_Id = pId->Id;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_allocate_pd(m_pDevice->m_hVerbsDevice, (ib_api_status_t) hr,
+ &m_hVerbsPd, &verbsData);
+ return hr;
+}
+
+CWVProtectionDomain::~CWVProtectionDomain()
+{
+ DWORD bytes;
+ HRESULT hr;
+
+ if (m_Id != NULL) {
+ m_pVerbs->pre_deallocate_pd(m_hVerbsPd);
+ hr = DeviceIoControl(m_hFile, WV_IOCTL_PD_ALLOCATE, &m_Id, sizeof m_Id,
+ NULL, 0, &bytes, NULL) ?
+ WV_SUCCESS : HRESULT_FROM_WIN32(GetLastError());
+ m_pVerbs->post_deallocate_pd(m_hVerbsPd, (ib_api_status_t) hr);
+ }
+ m_pDevice->Release();
+}
+
+STDMETHODIMP CWVProtectionDomain::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
- UNREFERENCED_PARAMETER(riid);
- UNREFERENCED_PARAMETER(ppvObj);
+ if (riid != IID_IUnknown && riid != IID_IWVProtectionDomain) {
+ *ppvObj = NULL;
+ return E_NOINTERFACE;
+ }
- return E_NOTIMPL;
+ *ppvObj = this;
+ AddRef();
+ return WV_SUCCESS;
}
STDMETHODIMP_(ULONG) CWVProtectionDomain::
AddRef(void)
{
- return 0;
+ return InterlockedIncrement(&m_nRef);
}
STDMETHODIMP_(ULONG) CWVProtectionDomain::
Release(void)
{
- return 0;
+ ULONG ref;
+
+ ref = (ULONG) InterlockedDecrement(&m_nRef);
+ if (ref == 0) {
+ delete this;
+ }
+ return ref;
}
STDMETHODIMP CWVProtectionDomain::
CreateSharedReceiveQueue(SIZE_T MaxWr, SIZE_T MaxSge,
SIZE_T SrqLimit, IWVSharedReceiveQueue** ppSrq)
{
- UNREFERENCED_PARAMETER(MaxWr);
- UNREFERENCED_PARAMETER(MaxSge);
- UNREFERENCED_PARAMETER(SrqLimit);
- UNREFERENCED_PARAMETER(ppSrq);
+ HRESULT hr;
+ CWVSharedReceiveQueue *srq;
- return E_NOTIMPL;
+ srq = new CWVSharedReceiveQueue(this);
+ if (srq == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ srq->QueryInterface(IID_IWVSharedReceiveQueue, (LPVOID*) ppSrq);
+ srq->Release();
+
+ hr = srq->Create(MaxWr, MaxSge, SrqLimit);
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ srq->Release();
+err1:
+ *ppSrq = NULL;
+ return hr;
}
STDMETHODIMP CWVProtectionDomain::
CreateQueuePair(WV_QP_CREATE* pAttributes, IWVQueuePair** ppQp)
{
- UNREFERENCED_PARAMETER(pAttributes);
- UNREFERENCED_PARAMETER(ppQp);
+ HRESULT hr;
+ CWVQueuePair *qp;
- return E_NOTIMPL;
+ qp = new CWVQueuePair(this);
+ if (qp == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ qp->QueryInterface(IID_IWVQueuePair, (LPVOID*) ppQp);
+ qp->Release();
+
+ hr = qp->Create(pAttributes);
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ qp->Release();
+err1:
+ *ppQp = NULL;
+ return hr;
}
STDMETHODIMP CWVProtectionDomain::
@@ -98,60 +221,307 @@
STDMETHODIMP CWVProtectionDomain::
AllocateMemoryWindow(IWVMemoryWindow** ppMw)
{
- UNREFERENCED_PARAMETER(ppMw);
+ HRESULT hr;
+ CWVMemoryWindow *mw;
- return E_NOTIMPL;
+ mw = new CWVMemoryWindow(this);
+ if (mw == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ mw->QueryInterface(IID_IWVMemoryWindow, (LPVOID*) ppMw);
+ mw->Release();
+
+ hr = mw->Allocate();
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ mw->Release();
+err1:
+ *ppMw = NULL;
+ return hr;
}
STDMETHODIMP CWVProtectionDomain::
CreateAddressHandle(WV_ADDRESS_VECTOR* pAddress, IWVAddressHandle** ppAh)
{
- UNREFERENCED_PARAMETER(pAddress);
- UNREFERENCED_PARAMETER(ppAh);
+ HRESULT hr;
+ CWVAddressHandle *ah;
- return E_NOTIMPL;
+ ah = new CWVAddressHandle(this);
+ if (ah == NULL) {
+ hr = WV_NO_MEMORY;
+ goto err1;
+ }
+
+ ah->QueryInterface(IID_IWVAddressHandle, (LPVOID*) ppAh);
+ ah->Release();
+
+ hr = ah->Create(pAddress);
+ if (FAILED(hr)) {
+ goto err2;
+ }
+ return WV_SUCCESS;
+
+err2:
+ ah->Release();
+err1:
+ *ppAh = NULL;
+ return hr;
}
+CWVMemoryWindow::CWVMemoryWindow(CWVProtectionDomain *pPd)
+{
+ pPd->AddRef();
+ m_pPd = pPd;
+ m_pVerbs = pPd->m_pVerbs;
+ m_hFile = pPd->m_hFile;
+
+ m_hVerbsMw = NULL;
+ m_Id = 0;
+
+ m_nRef = 1;
+}
+
STDMETHODIMP CWVMemoryWindow::
+Allocate(void)
+{
+ WV_IO_ID *pId;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
+
+ stat = m_pVerbs->pre_create_mw(m_pPd->m_hVerbsPd, &verbsData, &m_hVerbsMw);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = sizeof WV_IO_ID + max(verbsData.input_size, verbsData.output_size);
+ pId = (WV_IO_ID *) buf.Get(bytes);
+ if (pId == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pId->Id = m_pPd->m_Id;
+ pId->VerbInfo = verbsData.command;
+ RtlCopyMemory(pId + 1, verbsData.p_inout_buf, verbsData.input_size);
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_MW_ALLOCATE,
+ pId, sizeof WV_IO_ID + verbsData.input_size,
+ pId, sizeof WV_IO_ID + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ m_Id = pId->Id;
+ m_Rkey = pId->Data;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_create_mw(m_pPd->m_hVerbsPd, (ib_api_status_t) hr,
+ m_Rkey, &m_hVerbsMw, &verbsData);
+ return hr;
+}
+
+CWVMemoryWindow::~CWVMemoryWindow()
+{
+ DWORD bytes;
+ HRESULT hr;
+
+ if (m_Id != NULL) {
+ m_pVerbs->pre_destroy_mw(m_hVerbsMw);
+ hr = DeviceIoControl(m_hFile, WV_IOCTL_MW_DEALLOCATE,
+ &m_Id, sizeof m_Id, NULL, 0, &bytes, NULL) ?
+ WV_SUCCESS : HRESULT_FROM_WIN32(GetLastError());
+ m_pVerbs->post_destroy_mw(m_hVerbsMw, (ib_api_status_t) hr);
+ }
+ m_pPd->Release();
+}
+
+STDMETHODIMP CWVMemoryWindow::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
- UNREFERENCED_PARAMETER(riid);
- UNREFERENCED_PARAMETER(ppvObj);
+ if (riid != IID_IUnknown && riid != IID_IWVMemoryWindow) {
+ *ppvObj = NULL;
+ return E_NOINTERFACE;
+ }
- return E_NOTIMPL;
+ *ppvObj = this;
+ AddRef();
+ return WV_SUCCESS;
}
STDMETHODIMP_(ULONG) CWVMemoryWindow::
AddRef(void)
{
- return 0;
+ return InterlockedIncrement(&m_nRef);
}
STDMETHODIMP_(ULONG) CWVMemoryWindow::
Release(void)
{
- return 0;
+ ULONG ref;
+
+ ref = (ULONG) InterlockedDecrement(&m_nRef);
+ if (ref == 0) {
+ delete this;
+ }
+ return ref;
}
+void WvVerbsConvertAv(ib_av_attr_t *pVerbsAv, WV_ADDRESS_VECTOR *pAv)
+{
+ if (pAv->GrhValid) {
+ RtlCopyMemory(&pVerbsAv->grh, &pAv->Grh, sizeof pAv->Grh);
+ pVerbsAv->grh_valid = 1;
+ } else {
+ pVerbsAv->grh_valid = 0;
+ }
+
+ pVerbsAv->port_num = pAv->PortNumber;
+ pVerbsAv->sl = pAv->ServiceLevel;
+ pVerbsAv->dlid = pAv->DLid;
+ pVerbsAv->static_rate = pAv->StaticRate;
+ pVerbsAv->path_bits = pAv->SourcePathBits;
+}
+
+void WvIoConvertAv(WV_IO_AV *pIoAv, WV_ADDRESS_VECTOR *pAv)
+{
+ if (pAv->GrhValid) {
+ RtlCopyMemory(pIoAv, &pAv->Grh, sizeof pAv->Grh);
+ pIoAv->GrhValid = 1;
+ } else {
+ pIoAv->GrhValid = 0;
+ }
+
+ pIoAv->DLid = pAv->DLid;
+ pIoAv->ServiceLevel = pAv->ServiceLevel;
+ pIoAv->SourcePathBits = pAv->SourcePathBits;
+ pIoAv->StaticRate = pAv->StaticRate;
+ pIoAv->PortNumber = pAv->PortNumber;
+}
+
+CWVAddressHandle::CWVAddressHandle(CWVProtectionDomain *pPd)
+{
+ pPd->AddRef();
+ m_pPd = pPd;
+ m_pVerbs = pPd->m_pVerbs;
+ m_hFile = pPd->m_hFile;
+
+ m_hVerbsAh = NULL;
+ m_Id = 0;
+
+ m_nRef = 1;
+}
+
STDMETHODIMP CWVAddressHandle::
+Create(WV_ADDRESS_VECTOR* pAddress)
+{
+ WV_IO_AH_CREATE *pav;
+ WV_IO_ID *pId;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ ib_av_attr_t av;
+ CWVBuffer buf;
+
+ WvVerbsConvertAv(&av, pAddress);
+ stat = m_pVerbs->pre_create_av(m_pPd->m_hVerbsPd, &av, &verbsData, &m_hVerbsAh);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = max(sizeof WV_IO_AH_CREATE + verbsData.input_size,
+ sizeof WV_IO_ID + verbsData.output_size);
+ pId = (WV_IO_ID *) buf.Get(bytes);
+ if (pId == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pav = (WV_IO_AH_CREATE *) pId;
+ pId->Id = m_pPd->m_Id;
+ pId->VerbInfo = verbsData.command;
+ WvIoConvertAv(&pav->AddressVector, pAddress);
+ RtlCopyMemory(pav + 1, verbsData.p_inout_buf, verbsData.input_size);
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_AH_CREATE,
+ pav, sizeof WV_IO_AH_CREATE + verbsData.input_size,
+ pId, sizeof WV_IO_ID + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ m_Id = pId->Id;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_create_av(m_pPd->m_hVerbsPd, (ib_api_status_t) hr,
+ &m_hVerbsAh, &verbsData);
+ return hr;
+}
+
+CWVAddressHandle::~CWVAddressHandle()
+{
+ DWORD bytes;
+ HRESULT hr;
+
+ if (m_Id != NULL) {
+ m_pVerbs->pre_destroy_av(m_hVerbsAh);
+ hr = DeviceIoControl(m_hFile, WV_IOCTL_AH_DESTROY,
+ &m_Id, sizeof m_Id, NULL, 0, &bytes, NULL) ?
+ WV_SUCCESS : HRESULT_FROM_WIN32(GetLastError());
+ m_pVerbs->post_destroy_av(m_hVerbsAh, (ib_api_status_t) hr);
+ }
+ m_pPd->Release();
+}
+
+STDMETHODIMP CWVAddressHandle::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
- UNREFERENCED_PARAMETER(riid);
- UNREFERENCED_PARAMETER(ppvObj);
+ if (riid != IID_IUnknown && riid != IID_IWVAddressHandle) {
+ *ppvObj = NULL;
+ return E_NOINTERFACE;
+ }
- return E_NOTIMPL;
+ *ppvObj = this;
+ AddRef();
+ return WV_SUCCESS;
}
STDMETHODIMP_(ULONG) CWVAddressHandle::
AddRef(void)
{
- return 0;
+ return InterlockedIncrement(&m_nRef);
}
STDMETHODIMP_(ULONG) CWVAddressHandle::
Release(void)
{
- return 0;
+ ULONG ref;
+
+ ref = (ULONG) InterlockedDecrement(&m_nRef);
+ if (ref == 0) {
+ delete this;
+ }
+ return ref;
}
+
Index: core/winverbs/user/wv_pd.h
===================================================================
--- core/winverbs/user/wv_pd.h (revision 971)
+++ core/winverbs/user/wv_pd.h (working copy)
@@ -33,6 +33,7 @@
#define _WV_PD_H_
#include "rdma\winverbs.h"
+#include "wv_device.h"
class CWVProtectionDomain : IWVProtectionDomain
{
@@ -53,6 +54,20 @@
STDMETHODIMP AllocateMemoryWindow(IWVMemoryWindow** ppMw);
STDMETHODIMP CreateAddressHandle(WV_ADDRESS_VECTOR* pAddress,
IWVAddressHandle** ppAh);
+
+ CWVProtectionDomain(CWVDevice *pDevice);
+ ~CWVProtectionDomain();
+ STDMETHODIMP Allocate();
+
+ CWVDevice *m_pDevice;
+ uvp_interface_t *m_pVerbs;
+ HANDLE m_hFile;
+
+ ib_pd_handle_t m_hVerbsPd;
+ UINT64 m_Id;
+
+protected:
+ volatile LONG m_nRef;
};
class CWVMemoryWindow : IWVMemoryWindow
@@ -62,9 +77,23 @@
STDMETHODIMP QueryInterface(REFIID riid, LPVOID FAR* ppvObj);
STDMETHODIMP_(ULONG) AddRef();
STDMETHODIMP_(ULONG) Release();
+
+ CWVMemoryWindow(CWVProtectionDomain *pPd);
+ ~CWVMemoryWindow();
+ STDMETHODIMP Allocate();
+
+ CWVProtectionDomain *m_pPd;
+ uvp_interface_t *m_pVerbs;
+ HANDLE m_hFile;
+
+ ib_mw_handle_t m_hVerbsMw;
+ UINT64 m_Id;
+
+ UINT32 m_Rkey;
+protected:
+ volatile LONG m_nRef;
};
-
class CWVAddressHandle : IWVAddressHandle
{
public:
@@ -72,6 +101,20 @@
STDMETHODIMP QueryInterface(REFIID riid, LPVOID FAR* ppvObj);
STDMETHODIMP_(ULONG) AddRef();
STDMETHODIMP_(ULONG) Release();
+
+ CWVAddressHandle(CWVProtectionDomain *pPd);
+ ~CWVAddressHandle();
+ STDMETHODIMP Create(WV_ADDRESS_VECTOR* pAddress);
+
+ CWVProtectionDomain *m_pPd;
+ uvp_interface_t *m_pVerbs;
+ HANDLE m_hFile;
+
+ ib_av_handle_t m_hVerbsAh;
+ UINT64 m_Id;
+
+protected:
+ volatile LONG m_nRef;
};
#endif // _WV_PD_H_
\ No newline at end of file
More information about the ofw
mailing list