[ofw] [PATCH][RFC] winverbs/cq: CQ notification should be done in userspace
Fab Tillier
ftillier at windows.microsoft.com
Thu May 1 22:39:19 PDT 2008
>
>This changes CQ:Notify to call the userspace verbs provider for rearming
>the CQ. The overlapped request is still passed to the kernel and signaled
>directly when the CQ event is signaled.
>
>A user may also request notification of CQ error events only without
>rearming the CQ.
>
>Signed-off-by: Sean Hefty <sean.hefty at intel.com>
>---
>This patch follows the previous patch, along with a patch to signal all
>pending overlapped requests when an async event occurs.
>
>Index: core/winverbs/kernel/wv_cq.c
>===================================================================
>--- core/winverbs/kernel/wv_cq.c (revision 1112)
>+++ core/winverbs/kernel/wv_cq.c (working copy)
>@@ -281,30 +281,23 @@
>
> void WvCqNotify(WV_PROVIDER *pProvider, WDFREQUEST Request)
> {
>- WV_IO_ID *id;
>+ UINT64 *id;
> WV_COMPLETION_QUEUE *cq;
> NTSTATUS status;
>- ib_api_status_t ib_status;
>
>- status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &id, NULL);
>+ status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);
> if (!NT_SUCCESS(status)) {
> goto out;
> }
>
>- cq = WvCqAcquire(pProvider, id->Id);
>+ cq = WvCqAcquire(pProvider, *id);
> if (cq == NULL) {
> status = STATUS_NOT_FOUND;
> goto out;
> }
>
> WdfObjectAcquireLock(cq->Queue);
>- ib_status = cq->pVerbs->enable_cq_notify(cq->hVerbsCq,
>- id->Data ==
>WV_IO_CQ_NOTIFY_SOLICITED);
>- if (ib_status == IB_SUCCESS) {
>- status = WdfRequestForwardToIoQueue(Request, cq->Queue);
>- } else {
>- status = STATUS_UNSUCCESSFUL;
>- }
>+ status = WdfRequestForwardToIoQueue(Request, cq->Queue);
> WdfObjectReleaseLock(cq->Queue);
How do you distinguish between error-only and completion notify requests?
You could have a mix of error-only and completion requests, and upon completion only the completion requests should be completed.
On error, both error-only and completion notification requests should get completed.
> WvCqRelease(cq);
>
>@@ -314,40 +307,6 @@
> }
> }
>
>-void WvCqBatchNotify(WV_PROVIDER *pProvider, WDFREQUEST Request)
>-{
>- WV_IO_ID *id;
>- WV_COMPLETION_QUEUE *cq;
>- NTSTATUS status;
>- ib_api_status_t ib_status;
>-
>- status = WdfRequestRetrieveInputBuffer(Request, sizeof(WV_IO_ID), &id, NULL);
>- if (!NT_SUCCESS(status)) {
>- goto out;
>- }
>-
>- cq = WvCqAcquire(pProvider, id->Id);
>- if (cq == NULL) {
>- status = STATUS_NOT_FOUND;
>- goto out;
>- }
>-
>- WdfObjectAcquireLock(cq->Queue);
>- ib_status = cq->pVerbs->enable_ncomp_cq_notify(cq->hVerbsCq, id->Data);
>- if (ib_status == IB_SUCCESS) {
>- status = WdfRequestForwardToIoQueue(Request, cq->Queue);
>- } else {
>- status = STATUS_UNSUCCESSFUL;
>- }
>- WdfObjectReleaseLock(cq->Queue);
>- WvCqRelease(cq);
>-
>-out:
>- if (!NT_SUCCESS(status)) {
>- WdfRequestComplete(Request, status);
>- }
>-}
>-
> void WvCqCancel(WV_PROVIDER *pProvider, WDFREQUEST Request)
> {
> UINT64 *id;
>Index: core/winverbs/kernel/wv_driver.c
>===================================================================
>--- core/winverbs/kernel/wv_driver.c (revision 1111)
>+++ core/winverbs/kernel/wv_driver.c (working copy)
>@@ -235,9 +235,6 @@
> case WV_IOCTL_CQ_NOTIFY:
> WvCqNotify(prov, Request);
> break;
>- case WV_IOCTL_CQ_BATCH_NOTIFY:
>- WvCqBatchNotify(prov, Request);
>- break;
> case WV_IOCTL_CQ_CANCEL:
> WvCqCancel(prov, Request);
> break;
>Index: core/winverbs/user/wv_cq.cpp
>===================================================================
>--- core/winverbs/user/wv_cq.cpp (revision 1007)
>+++ core/winverbs/user/wv_cq.cpp (working copy)
>@@ -222,14 +222,21 @@
> STDMETHODIMP CWVCompletionQueue::
> Notify(WV_CQ_NOTIFY_TYPE Type, OVERLAPPED* pOverlapped)
> {
>- WV_IO_ID id;
> DWORD bytes;
> HRESULT hr;
>
>- id.Id = m_Id;
>- id.Data = (UINT32) Type;
>- if (DeviceIoControl(m_hFile, WV_IOCTL_CQ_NOTIFY, &id, sizeof id,
>+ if (DeviceIoControl(m_hFile, WV_IOCTL_CQ_NOTIFY, &m_Id, sizeof m_Id,
> NULL, 0, &bytes, pOverlapped)) {
Note that you expect DeviceIoControl to return FALSE with GetLastError returning ERROR_IO_PENDING.
>+ switch (Type) {
>+ case WvCqSolicited:
>+ m_pVerbs->rearm_cq(m_hVerbsCq, 1);
>+ break;
>+ case WvCqNextCompletion:
>+ m_pVerbs->rearm_cq(m_hVerbsCq, 0);
>+ break;
>+ default:
>+ break;
>+ }
> hr = WV_SUCCESS;
> } else {
> hr = HRESULT_FROM_WIN32(GetLastError());
>@@ -241,14 +248,12 @@
> STDMETHODIMP CWVCompletionQueue::
> BatchNotify(SIZE_T CompletedEntries, OVERLAPPED* pOverlapped)
> {
>- WV_IO_ID id;
> DWORD bytes;
> HRESULT hr;
>
>- id.Id = m_Id;
>- id.Data = (UINT32) CompletedEntries;
>- if (DeviceIoControl(m_hFile, WV_IOCTL_CQ_BATCH_NOTIFY, &id, sizeof id,
>+ if (DeviceIoControl(m_hFile, WV_IOCTL_CQ_NOTIFY, &m_Id, sizeof m_Id,
> NULL, 0, &bytes, pOverlapped)) {
>+ m_pVerbs->rearm_n_cq(m_hVerbsCq, (UINT32) CompletedEntries);
> hr = WV_SUCCESS;
> } else {
> hr = HRESULT_FROM_WIN32(GetLastError());
>Index: core/winverbs/wv_ioctl.h
>===================================================================
>--- core/winverbs/wv_ioctl.h (revision 1111)
>+++ core/winverbs/wv_ioctl.h (working copy)
>@@ -63,7 +63,6 @@
> WV_IO_FUNCTION_CQ_DESTROY,
> WV_IO_FUNCTION_CQ_RESIZE,
> WV_IO_FUNCTION_CQ_NOTIFY,
>- WV_IO_FUNCTION_CQ_BATCH_NOTIFY,
> WV_IO_FUNCTION_CQ_CANCEL,
> WV_IO_FUNCTION_SRQ_CREATE,
> WV_IO_FUNCTION_SRQ_DESTROY,
>@@ -183,14 +182,10 @@
> #define WV_IOCTL_CQ_RESIZE WV_IOCTL(WV_IO_FUNCTION_BASE + \
>
>WV_IO_FUNCTION_CQ_RESIZE)
>
>-// WV_IO_ID / none
>+// UINT64 Id / none
> #define WV_IOCTL_CQ_NOTIFY WV_IOCTL(WV_IO_FUNCTION_BASE + \
>
>WV_IO_FUNCTION_CQ_NOTIFY)
>
>-// WV_IO_ID / none
>-#define WV_IOCTL_CQ_BATCH_NOTIFY WV_IOCTL(WV_IO_FUNCTION_BASE + \
>-
>WV_IO_FUNCTION_CQ_BATCH_NOTIFY)
>-
> // UINT64 Id / none
> #define WV_IOCTL_CQ_CANCEL WV_IOCTL(WV_IO_FUNCTION_BASE + \
>
>WV_IO_FUNCTION_CQ_CANCEL)
>@@ -493,9 +488,6 @@
>
> } WV_IO_QP_CREATE;
>
>-#define WV_IO_CQ_NOTIFY_SOLICITED 1
>-#define WV_IO_CQ_NOTIFY_NEXT 2
>-
> #define WV_IO_QP_STATE_RESET 0
> #define WV_IO_QP_STATE_INIT 1
> #define WV_IO_QP_STATE_RTR 2
>Index: inc/user/rdma/winverbs.h
>===================================================================
>--- inc/user/rdma/winverbs.h (revision 1111)
>+++ inc/user/rdma/winverbs.h (working copy)
>@@ -177,7 +177,8 @@
>
> typedef enum _WV_CQ_NOTIFY_TYPE
> {
>- WvCqSolicited = 1,
>+ WvCqError,
>+ WvCqSolicited,
> WvCqNextCompletion
>
> } WV_CQ_NOTIFY_TYPE;
>
>
>_______________________________________________
>ofw mailing list
>ofw at lists.openfabrics.org
>http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
>
More information about the ofw
mailing list