[ofw] [RFC] [PATCH 8/12] winverbs: implement IWVSharedReceiveQueue
Sean Hefty
sean.hefty at intel.com
Fri Mar 14 23:04:17 PDT 2008
Userspace WinVerb implementation for shared receive queues. Most
calls are implemented, except for inherited IWVOverlapped routines.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: core/winverbs/user/wv_srq.cpp
===================================================================
--- core/winverbs/user/wv_srq.cpp (revision 971)
+++ core/winverbs/user/wv_srq.cpp (working copy)
@@ -30,25 +30,120 @@
#include "wv_base.h"
#include "wv_srq.h"
+CWVSharedReceiveQueue::CWVSharedReceiveQueue(CWVProtectionDomain *pPd)
+{
+ pPd->AddRef();
+ m_pPd = pPd;
+ m_pVerbs = pPd->m_pVerbs;
+ m_hFile = pPd->m_hFile;
+
+ m_hVerbsSrq = NULL;
+ m_Id = 0;
+
+ m_nRef = 1;
+}
+
STDMETHODIMP CWVSharedReceiveQueue::
+Create(SIZE_T MaxWr, SIZE_T MaxSge, SIZE_T SrqLimit)
+{
+ WV_IO_ID *pId;
+ WV_IO_SRQ_ATTRIBUTES *pattr;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
+ ib_srq_attr_t attr;
+
+ attr.max_sge = MaxSge;
+ attr.max_wr = MaxWr;
+ attr.srq_limit = SrqLimit;
+ stat = m_pVerbs->pre_create_srq(m_pPd->m_hVerbsPd, &attr,
+ &verbsData, &m_hVerbsSrq);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = max(sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.input_size,
+ sizeof WV_IO_ID + verbsData.output_size);
+ pattr = (WV_IO_SRQ_ATTRIBUTES *) buf.Get(bytes);
+ if (pattr == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pId = (WV_IO_ID *) pattr;
+ pattr->Id.Id = m_pPd->m_Id;
+ pattr->Id.VerbInfo = verbsData.command;
+ pattr->MaxSge = attr.max_sge;
+ pattr->MaxWr = attr.max_wr;
+ pattr->SrqLimit = attr.srq_limit;
+ RtlCopyMemory(pattr + 1, verbsData.p_inout_buf, verbsData.input_size);
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_SRQ_CREATE,
+ pattr, sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.input_size,
+ pId, sizeof WV_IO_ID + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ m_Id = pId->Id;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_create_srq(m_pPd->m_hVerbsPd, (ib_api_status_t) hr,
+ &m_hVerbsSrq, &verbsData);
+ return hr;
+}
+
+CWVSharedReceiveQueue::~CWVSharedReceiveQueue()
+{
+ DWORD bytes;
+ HRESULT hr;
+
+ if (m_Id != NULL) {
+ m_pVerbs->pre_destroy_srq(m_hVerbsSrq);
+ hr = DeviceIoControl(m_hFile, WV_IOCTL_SRQ_DESTROY, &m_Id, sizeof m_Id,
+ NULL, 0, &bytes, NULL) ?
+ WV_SUCCESS : HRESULT_FROM_WIN32(GetLastError());
+ m_pVerbs->post_destroy_srq(m_hVerbsSrq, (ib_api_status_t) hr);
+ }
+ m_pPd->Release();
+}
+
+STDMETHODIMP CWVSharedReceiveQueue::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
- UNREFERENCED_PARAMETER(riid);
- UNREFERENCED_PARAMETER(ppvObj);
+ if (riid != IID_IUnknown && riid != IID_IWVSharedReceiveQueue) {
+ *ppvObj = NULL;
+ return E_NOINTERFACE;
+ }
- return E_NOTIMPL;
+ *ppvObj = this;
+ AddRef();
+ return WV_SUCCESS;
}
STDMETHODIMP_(ULONG) CWVSharedReceiveQueue::
AddRef(void)
{
- return 0;
+ return InterlockedIncrement(&m_nRef);
}
STDMETHODIMP_(ULONG) CWVSharedReceiveQueue::
Release(void)
{
- return 0;
+ ULONG ref;
+
+ ref = (ULONG) InterlockedDecrement(&m_nRef);
+ if (ref == 0) {
+ delete this;
+ }
+ return ref;
}
STDMETHODIMP CWVSharedReceiveQueue::
@@ -71,36 +166,160 @@
STDMETHODIMP CWVSharedReceiveQueue::
Query(SIZE_T* pMaxWr, SIZE_T* pMaxSge, SIZE_T* pSrqLimit)
{
- UNREFERENCED_PARAMETER(pMaxWr);
- UNREFERENCED_PARAMETER(pMaxSge);
- UNREFERENCED_PARAMETER(pSrqLimit);
+ WV_IO_ID *pId;
+ WV_IO_SRQ_ATTRIBUTES *pattr;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
+ ib_srq_attr_t attr;
- return E_NOTIMPL;
+ stat = m_pVerbs->pre_query_srq(m_hVerbsSrq, &verbsData);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = max(sizeof WV_IO_ID + verbsData.input_size,
+ sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.output_size);
+ pId = (WV_IO_ID *) buf.Get(bytes);
+ if (pId == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pId->Id = m_Id;
+ pId->VerbInfo = verbsData.command;
+ RtlCopyMemory(pId + 1, verbsData.p_inout_buf, verbsData.input_size);
+ pattr = (WV_IO_SRQ_ATTRIBUTES *) pId;
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_SRQ_QUERY,
+ pId, sizeof WV_IO_ID + verbsData.input_size,
+ pattr, sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ attr.max_sge = pattr->MaxSge;
+ attr.max_wr = pattr->MaxWr;
+ attr.srq_limit = pattr->SrqLimit;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = pId->VerbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pId + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_query_srq(m_hVerbsSrq, (ib_api_status_t) hr,
+ &attr, &verbsData);
+ if (SUCCEEDED(hr)) {
+ *pMaxWr = attr.max_wr;
+ *pMaxSge = attr.max_sge;
+ *pSrqLimit = attr.srq_limit;
+ }
+ return hr;
}
STDMETHODIMP CWVSharedReceiveQueue::
Modify(SIZE_T MaxWr, SIZE_T SrqLimit)
{
- UNREFERENCED_PARAMETER(MaxWr);
- UNREFERENCED_PARAMETER(SrqLimit);
+ UINT32 *pverbInfo;
+ WV_IO_SRQ_ATTRIBUTES *pattr;
+ DWORD bytes;
+ ib_api_status_t stat;
+ HRESULT hr;
+ ci_umv_buf_t verbsData;
+ CWVBuffer buf;
+ ib_srq_attr_t attr;
- return E_NOTIMPL;
+ attr.max_wr = MaxWr;
+ attr.srq_limit = SrqLimit;
+ stat = m_pVerbs->pre_modify_srq(m_hVerbsSrq, &attr, (ib_srq_attr_mask_t)
+ (IB_SRQ_MAX_WR & IB_SRQ_LIMIT), &verbsData);
+ if (stat != IB_SUCCESS) {
+ return WvConvertIbStatus(stat);
+ }
+
+ bytes = max(sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.input_size,
+ sizeof UINT32 + verbsData.output_size);
+ pattr = (WV_IO_SRQ_ATTRIBUTES *) buf.Get(bytes);
+ if (pattr == NULL) {
+ hr = WV_NO_MEMORY;
+ goto post;
+ }
+
+ pattr->Id.Id = m_Id;
+ pattr->Id.VerbInfo = verbsData.command;
+ pattr->MaxWr = attr.max_wr;
+ pattr->SrqLimit = attr.srq_limit;
+ RtlCopyMemory(pattr + 1, verbsData.p_inout_buf, verbsData.input_size);
+ pverbInfo = (UINT32 *) pattr;
+
+ if (DeviceIoControl(m_hFile, WV_IOCTL_SRQ_MODIFY,
+ pattr, sizeof WV_IO_SRQ_ATTRIBUTES + verbsData.input_size,
+ pverbInfo, sizeof UINT32 + verbsData.output_size,
+ &bytes, NULL)) {
+ hr = WV_SUCCESS;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ verbsData.status = *pverbInfo;
+ RtlCopyMemory(verbsData.p_inout_buf, pverbInfo + 1, verbsData.output_size);
+ buf.Put();
+
+post:
+ m_pVerbs->post_modify_srq(m_hVerbsSrq, (ib_api_status_t) hr, &verbsData);
+ return hr;
}
STDMETHODIMP CWVSharedReceiveQueue::
PostReceive(UINT64 WrId, const WV_SGE* pSgl, SIZE_T nSge)
{
- UNREFERENCED_PARAMETER(WrId);
- UNREFERENCED_PARAMETER(pSgl);
- UNREFERENCED_PARAMETER(nSge);
+ ib_recv_wr_t wr, *pwr;
+ ib_api_status_t stat;
+ HRESULT hr;
+ CWVBuffer buf;
+ SIZE_T n;
- return E_NOTIMPL;
+ wr.p_next = NULL;
+ wr.wr_id = WrId;
+ wr.num_ds = nSge;
+
+ wr.ds_array = (ib_local_ds_t *) buf.Get(sizeof ib_local_ds_t * nSge);
+ if (wr.ds_array == NULL) {
+ return WV_NO_MEMORY;
+ }
+
+ for (n = 0; n < nSge; n++) {
+ wr.ds_array[n].vaddr = (UINT64) pSgl[n].pAddress;
+ wr.ds_array[n].length = (UINT32) pSgl[n].Length;
+ wr.ds_array[n].lkey = pSgl[n].Lkey;
+ }
+
+ stat = m_pVerbs->post_srq_recv(m_hVerbsSrq, &wr, &pwr);
+ if (stat == IB_SUCCESS) {
+ hr = WV_SUCCESS;
+ } else {
+ hr = WvConvertIbStatus(stat);
+ }
+
+ buf.Put();
+ return hr;
}
STDMETHODIMP CWVSharedReceiveQueue::
Notify(OVERLAPPED* pOverlapped)
{
- UNREFERENCED_PARAMETER(pOverlapped);
+ DWORD bytes;
+ HRESULT hr;
- return E_NOTIMPL;
+ if (DeviceIoControl(m_hFile, WV_IOCTL_SRQ_NOTIFY, &m_Id, sizeof m_Id,
+ NULL, 0, &bytes, pOverlapped)) {
+ hr = WV_SUCCESS;
+ } else {
+ hr = HRESULT_FROM_WIN32(GetLastError());
+ }
+
+ return hr;
}
Index: core/winverbs/user/wv_srq.h
===================================================================
--- core/winverbs/user/wv_srq.h (revision 971)
+++ core/winverbs/user/wv_srq.h (working copy)
@@ -33,6 +33,7 @@
#define _WV_SRQ_H_
#include "rdma\winverbs.h"
+#include "wv_pd.h"
class CWVSharedReceiveQueue : IWVSharedReceiveQueue
{
@@ -52,6 +53,20 @@
STDMETHODIMP Modify(SIZE_T MaxWr, SIZE_T SrqLimit);
STDMETHODIMP PostReceive(UINT64 WrId, const WV_SGE* pSgl, SIZE_T nSge);
STDMETHODIMP Notify(OVERLAPPED* pOverlapped);
+
+ CWVSharedReceiveQueue(CWVProtectionDomain *pPd);
+ ~CWVSharedReceiveQueue();
+ STDMETHODIMP Create(SIZE_T MaxWr, SIZE_T MaxSge, SIZE_T SrqLimit);
+
+ CWVProtectionDomain *m_pPd;
+ uvp_interface_t *m_pVerbs;
+ HANDLE m_hFile;
+
+ ib_srq_handle_t m_hVerbsSrq;
+ UINT64 m_Id;
+
+protected:
+ volatile LONG m_nRef;
};
#endif // _WV_SRQ_H_
\ No newline at end of file
More information about the ofw
mailing list