[ofw] [PATCH] [RFC] winverbs/srq: add asynchronous notification support

[Sean Hefty]

Initial implementation for SRQ asynchronous notification handling.  This includes
SRQ errors, limit reached events, and last wqe reached events.  Canceling notification
is included. 

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: core/winverbs/kernel/wv_driver.c
===================================================================
--- core/winverbs/kernel/wv_driver.c	(revision 1099)
+++ core/winverbs/kernel/wv_driver.c	(working copy)
@@ -254,12 +254,10 @@
 		WvSrqModify(prov, Request);
 		break;
 	case WV_IOCTL_SRQ_NOTIFY:
-		//WvSrqNotify(prov, Request);
-		WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED);
+		WvSrqNotify(prov, Request);
 		break;
 	case WV_IOCTL_SRQ_CANCEL:
-		//WvSrqCancel(prov, Request);
-		WdfRequestComplete(Request, STATUS_NOT_IMPLEMENTED);
+		WvSrqCancel(prov, Request);
 		break;
 	case WV_IOCTL_QP_CREATE:
 		WvQpCreate(prov, Request);
Index: core/winverbs/kernel/wv_srq.c
===================================================================
--- core/winverbs/kernel/wv_srq.c	(revision 1090)
+++ core/winverbs/kernel/wv_srq.c	(working copy)
@@ -27,6 +27,7 @@
  * SOFTWARE.
  */
 
+#include "wv_driver.h"
 #include "wv_srq.h"
 #include "wv_ioctl.h"
 
@@ -74,9 +75,34 @@
 	pVerbsAttr->srq_limit = pAttr->SrqLimit;
 }
 
+static NTSTATUS WvSrqCompleteRequest(WV_SHARED_RECEIVE_QUEUE *pSrq, NTSTATUS Status)
+{
+	WDFREQUEST	request;
+	NTSTATUS	stat;
+
+	WdfObjectAcquireLock(pSrq->Queue);
+	stat = WdfIoQueueRetrieveNextRequest(pSrq->Queue, &request);
+	WdfObjectReleaseLock(pSrq->Queue);
+
+	if (NT_SUCCESS(stat)) {
+		WdfRequestComplete(request, Status);
+	}
+	return stat;
+}
+
 static void WvSrqEventHandler(ib_event_rec_t *pEvent)
 {
-	pEvent;
+	switch (pEvent->type) {
+	case IB_AE_SRQ_LIMIT_REACHED:
+		WvSrqCompleteRequest(pEvent->context, STATUS_ALERTED);
+		break;
+	case IB_AE_SRQ_QP_LAST_WQE_REACHED:
+		WvSrqCompleteRequest(pEvent->context, STATUS_NOTIFY_CLEANUP);
+		break;
+	default:
+		WvSrqCompleteRequest(pEvent->context, STATUS_UNEXPECTED_IO_ERROR);
+		break;
+	}
 }
 
 static NTSTATUS WvSrqAlloc(WV_PROTECTION_DOMAIN *pPd, WV_IO_SRQ_ATTRIBUTES *pAttr,
@@ -85,8 +111,10 @@
 	WV_SHARED_RECEIVE_QUEUE	*srq;
 	ib_srq_attr_t			attr;
 	ib_api_status_t			ib_status;
+	NTSTATUS				status;
+	WDF_IO_QUEUE_CONFIG		config;
 
-	srq = ExAllocatePoolWithTag(PagedPool, sizeof(WV_SHARED_RECEIVE_QUEUE), 'rsvw');
+	srq = ExAllocatePoolWithTag(NonPagedPool, sizeof(WV_SHARED_RECEIVE_QUEUE), 'rsvw');
 	if (srq == NULL) {
 		return STATUS_NO_MEMORY;
 	}
@@ -94,11 +122,18 @@
 	srq->Ref = 1;
 	KeInitializeEvent(&srq->Event, NotificationEvent, FALSE);
 
+	WDF_IO_QUEUE_CONFIG_INIT(&config, WdfIoQueueDispatchManual);
+	status = WdfIoQueueCreate(ControlDevice, &config,
+							  WDF_NO_OBJECT_ATTRIBUTES, &srq->Queue);
+	if (!NT_SUCCESS(status)) {
+		goto err1;
+	}
+
 	WvVerbsConvertSrq(&attr, pAttr);
 	ib_status = pPd->pVerbs->create_srq(pPd->hVerbsPd, srq, WvSrqEventHandler,
 										&attr, &srq->hVerbsSrq, pVerbsData);
 	if (ib_status != IB_SUCCESS) {
-		goto err;
+		goto err2;
 	}
 
 	srq->pPd = pPd;
@@ -107,7 +142,9 @@
 	*ppSrq = srq;
 	return STATUS_SUCCESS;
 
-err:
+err2:
+	WdfObjectDelete(srq->Queue);
+err1:
 	ExFreePool(srq);
 	return STATUS_UNSUCCESSFUL;
 }
@@ -211,6 +248,8 @@
 		pSrq->pVerbs->destroy_srq(pSrq->hVerbsSrq);
 	}
 
+	WdfIoQueuePurgeSynchronously(pSrq->Queue);
+	WdfObjectDelete(pSrq->Queue);
 	WvPdPut(pSrq->pPd);
 	ExFreePool(pSrq);
 }
@@ -297,4 +336,58 @@
 	len = outlen;
 complete:
 	WdfRequestCompleteWithInformation(Request, status, len);
-}
\ No newline at end of file
+}
+
+void WvSrqNotify(WV_PROVIDER *pProvider, WDFREQUEST Request)
+{
+	UINT64					*id;
+	WV_SHARED_RECEIVE_QUEUE	*srq;
+	NTSTATUS				status;
+
+	status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);
+	if (!NT_SUCCESS(status)) {
+		goto out;
+	}
+
+	srq = WvSrqAcquire(pProvider, *id);
+	if (srq == NULL) {
+		status = STATUS_NOT_FOUND;
+		goto out;
+	}
+
+	WdfObjectAcquireLock(srq->Queue);
+	status = WdfRequestForwardToIoQueue(Request, srq->Queue);
+	WdfObjectReleaseLock(srq->Queue);
+	WvSrqRelease(srq);
+
+out:
+	if (!NT_SUCCESS(status)) {
+		WdfRequestComplete(Request, status);
+	}
+}
+
+void WvSrqCancel(WV_PROVIDER *pProvider, WDFREQUEST Request)
+{
+	UINT64					*id;
+	WV_SHARED_RECEIVE_QUEUE	*srq;
+	NTSTATUS				status;
+
+	status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);
+	if (!NT_SUCCESS(status)) {
+		goto out;
+	}
+
+	srq = WvSrqAcquire(pProvider, *id);
+	if (srq == NULL) {
+		status = STATUS_NOT_FOUND;
+		goto out;
+	}
+
+	do {
+		status = WvSrqCompleteRequest(srq, STATUS_CANCELLED);
+	} while (NT_SUCCESS(status));
+	WvSrqRelease(srq);
+
+out:
+	WdfRequestComplete(Request, status);
+}
Index: core/winverbs/kernel/wv_srq.h
===================================================================
--- core/winverbs/kernel/wv_srq.h	(revision 1075)
+++ core/winverbs/kernel/wv_srq.h	(working copy)
@@ -50,6 +50,7 @@
 
 	KEVENT					Event;
 	LONG					Ref;
+	WDFQUEUE				Queue;
 
 }	WV_SHARED_RECEIVE_QUEUE;
 
@@ -64,5 +65,7 @@
 
 void WvSrqModify(WV_PROVIDER *pProvider, WDFREQUEST Request);
 void WvSrqQuery(WV_PROVIDER *pProvider, WDFREQUEST Request);
+void WvSrqNotify(WV_PROVIDER *pProvider, WDFREQUEST Request);
+void WvSrqCancel(WV_PROVIDER *pProvider, WDFREQUEST Request);
 
 #endif // _WV_SRQ_H_