[ofw] [PATCH v2 1/3] WinMAD: kernel filter driver

[Sean Hefty]

The WinMad driver is an upper filter driver for Infiniband HCAs.  It uses the
kernel IBAL MAD interfaces for access to qp 0 and 1.  It exposes a simple file
system interface to userspace for registering and deregistering MAD agents,
and supports file read/write operations to send/receive MADs.

WinMad does not implement SMI or GSI interfaces.  It only exposes a userspace
interface to MAD services that is optimized for Windows.

The driver supports asynchronous reads and writes using overlapped I/O, and
handles PnP device removal events with active userspace clients.

File writes results in sending MADs using a specified MAD agent.  File read
operations receive MADs across all registered agents for a single user.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
I'm re-posting the WinMad code for merging into the trunk, rather than posting
only the diffs against the changes in my tree.  I would like to merge all drivers
and libraries by the end of next week.  All files are available in the SVN winverbs
branch.

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef _WM_DRIVER_H_
#define _WM_DRIVER_H_

#include <ntddk.h>
#include <wdm.h>
#include <wdf.h>

#include <rdma\verbs.h>
#include <iba\ib_al_ifc.h>
#include "wm_ioctl.h"

#if WINVER <= _WIN32_WINNT_WINXP
#define KGUARDED_MUTEX FAST_MUTEX
#define KeInitializeGuardedMutex ExInitializeFastMutex
#define KeAcquireGuardedMutex ExAcquireFastMutex
#define KeReleaseGuardedMutex ExReleaseFastMutex
#endif

extern WDFDEVICE			ControlDevice;

typedef struct _WM_IB_PORT
{
	NET64					Guid;

}	WM_IB_PORT;

typedef struct _WM_IB_DEVICE
{
	LIST_ENTRY				Entry;
	LONG					Ref;
	KEVENT					Event;
	NET64					Guid;
	union
	{
		RDMA_INTERFACE_VERBS	VerbsInterface;
		ib_al_ifc_t				IbInterface;
	};
	ib_al_handle_t			hIbal;
	ib_pnp_handle_t			hPnp;
	//ib_ca_handle_t			hDevice;
	//ib_pd_handle_t			hPd;
	int						PortCount;
	WM_IB_PORT				*pPortArray;

}	WM_IB_DEVICE;

WM_IB_DEVICE *WmIbDeviceGet(NET64 Guid);
void WmIbDevicePut(WM_IB_DEVICE *pDevice);

#endif // _WM_DRIVER_H_

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef _WM_PROVIDER_H_
#define _WM_PROVIDER_H_

#include <ntddk.h>
#include <wdf.h>
#include <wdm.h>

#include <iba\ib_al.h>
#include "wm_driver.h"
#include "index_list.h"

typedef struct _WM_PROVIDER
{
	LIST_ENTRY			Entry;
	INDEX_LIST			RegIndex;
	WDFQUEUE			ReadQueue;

	ib_mad_element_t	*MadHead;
	ib_mad_element_t	*MadTail;

	KGUARDED_MUTEX		Lock;
	LONG				Ref;
	KEVENT				Event;
	LONG				Pending;
	LONG				Active;
	KEVENT				SharedEvent;
	LONG				Exclusive;
	KEVENT				ExclusiveEvent;

}	WM_PROVIDER;

void WmProviderGet(WM_PROVIDER *pProvider);
void WmProviderPut(WM_PROVIDER *pProvider);
NTSTATUS WmProviderInit(WM_PROVIDER *pProvider);
void WmProviderCleanup(WM_PROVIDER *pProvider);

void WmProviderRemoveHandler(WM_PROVIDER *pProvider, WM_IB_DEVICE *pDevice);
void WmProviderDisableRemove(WM_PROVIDER *pProvider);
void WmProviderEnableRemove(WM_PROVIDER *pProvider);

void WmProviderRead(WM_PROVIDER *pProvider, WDFREQUEST Request);
void WmProviderWrite(WM_PROVIDER *pProvider, WDFREQUEST Request);
void WmReceiveHandler(ib_mad_svc_handle_t hService, void *Context,
					  ib_mad_element_t *pMad);
void WmSendHandler(ib_mad_svc_handle_t hService, void *Context,
				   ib_mad_element_t *pMad);
void WmProviderFlushReceives(WM_PROVIDER *pProvider,
							 struct _WM_REGISTRATION *pRegistration);
void WmProviderCancel(WM_PROVIDER *pProvider, WDFREQUEST Request);

#endif // _WM_PROVIDER_H_

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef _WM_REG_H_
#define _WM_REG_H_

#include <ntddk.h>
#include <wdm.h>
#include <iba\ib_types.h>

#include "wm_driver.h"
#include "wm_provider.h"

typedef struct _WM_REGISTRATION
{
	WM_PROVIDER				*pProvider;
	WM_IB_DEVICE			*pDevice;

	ib_al_handle_t			hIbal;
	ib_ca_handle_t			hCa;
	ib_pd_handle_t			hPd;
	ib_qp_handle_t			hQp;
	ib_pool_key_t			hMadPool;
	ib_mad_svc_handle_t		hService;

	SIZE_T					Id;
	LONG					Ref;

}	WM_REGISTRATION;

void WmRegister(WM_PROVIDER *pProvider, WDFREQUEST Request);
void WmDeregister(WM_PROVIDER *pProvider, WDFREQUEST Request);
void WmRegFree(WM_REGISTRATION *pRegistration);
void WmRegRemoveHandler(WM_REGISTRATION *pRegistration);

WM_REGISTRATION *WmRegAcquire(WM_PROVIDER *pProvider, UINT64 Id);
void WmRegRelease(WM_REGISTRATION *pRegistration);

#endif // __WM_REG_H_

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <ntddk.h>
#include <wdf.h>
#include <wdmsec.h>
#include <ntstatus.h>
#include <initguid.h>

#include "wm_ioctl.h"
#include "wm_driver.h"
#include "wm_provider.h"
#include "wm_reg.h"

WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(WM_IB_DEVICE, WmIbDeviceGetContext)
WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(WM_PROVIDER, WmProviderGetContext)

WDFDEVICE				ControlDevice;
static LIST_ENTRY		DevList;
static LIST_ENTRY		ProvList;
static KGUARDED_MUTEX	Lock;

static EVT_WDF_DRIVER_DEVICE_ADD			WmIbDeviceAdd;
static EVT_WDF_OBJECT_CONTEXT_CLEANUP		WmIbDeviceCleanup;
static EVT_WDF_DEVICE_D0_ENTRY				WmPowerD0Entry;
static EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL	WmIoDeviceControl;
static EVT_WDF_IO_QUEUE_IO_READ				WmIoRead;
static EVT_WDF_IO_QUEUE_IO_WRITE			WmIoWrite;
static EVT_WDF_DEVICE_FILE_CREATE			WmFileCreate;
static EVT_WDF_FILE_CLEANUP					WmFileCleanup;
static EVT_WDF_FILE_CLOSE					WmFileClose;

static WM_IB_DEVICE *WmIbDeviceFind(NET64 Guid)
{
	WM_IB_DEVICE	*cur_dev, *dev = NULL;
	LIST_ENTRY		*entry;

	for (entry = DevList.Flink; entry != &DevList; entry = entry->Flink) {
		cur_dev = CONTAINING_RECORD(entry, WM_IB_DEVICE, Entry);
		if (cur_dev->Guid == Guid) {
			dev = cur_dev;
			break;
		}
	}
	return dev;
}

WM_IB_DEVICE *WmIbDeviceGet(NET64 Guid)
{
	WM_IB_DEVICE *dev;

	KeAcquireGuardedMutex(&Lock);
	dev = WmIbDeviceFind(Guid);
	if (dev != NULL) {
			InterlockedIncrement(&dev->Ref);
	}
	KeReleaseGuardedMutex(&Lock);
	return dev;
}

void WmIbDevicePut(WM_IB_DEVICE *pDevice)
{
	if (InterlockedDecrement(&pDevice->Ref) == 0) {
		KeSetEvent(&pDevice->Event, 0, FALSE);
	}
}

static VOID WmIoDeviceControl(WDFQUEUE Queue, WDFREQUEST Request,
							  size_t OutLen, size_t InLen, ULONG IoControlCode)
{
	WDFFILEOBJECT	file;
	WM_PROVIDER		*prov;
	UNREFERENCED_PARAMETER(OutLen);
	UNREFERENCED_PARAMETER(InLen);
	UNREFERENCED_PARAMETER(Queue);

	file = WdfRequestGetFileObject(Request);
	prov = WmProviderGetContext(file);

	switch (IoControlCode) {
	case WM_IOCTL_REGISTER:
		WmRegister(prov, Request);
		break;
	case WM_IOCTL_DEREGISTER:
		WmDeregister(prov, Request);
		break;
	case WM_IOCTL_CANCEL:
		WmProviderCancel(prov, Request);
		break;
	default:
		WdfRequestComplete(Request, STATUS_PROCEDURE_NOT_FOUND);
		break;
	}
}

static VOID WmIoRead(WDFQUEUE Queue, WDFREQUEST Request, size_t Length)
{
	WDFFILEOBJECT	file;
	WM_PROVIDER		*prov;
	UNREFERENCED_PARAMETER(Queue);

	file = WdfRequestGetFileObject(Request);
	prov = WmProviderGetContext(file);
	WmProviderRead(prov, Request);
}

static VOID WmIoWrite(WDFQUEUE Queue, WDFREQUEST Request, size_t Length)
{
	WDFFILEOBJECT	file;
	WM_PROVIDER		*prov;
	UNREFERENCED_PARAMETER(Queue);

	file = WdfRequestGetFileObject(Request);
	prov = WmProviderGetContext(file);
	WmProviderWrite(prov, Request);
}

static VOID WmFileCreate(WDFDEVICE Device, WDFREQUEST Request,
						 WDFFILEOBJECT FileObject)
{
	WM_PROVIDER	*prov = WmProviderGetContext(FileObject);
	UNREFERENCED_PARAMETER(Device);

	WmProviderInit(prov);
	KeAcquireGuardedMutex(&Lock);
	InsertHeadList(&ProvList, &prov->Entry);
	KeReleaseGuardedMutex(&Lock);
	WdfRequestComplete(Request, STATUS_SUCCESS);
}

static VOID WmFileCleanup(WDFFILEOBJECT FileObject)
{
	WM_PROVIDER *prov = WmProviderGetContext(FileObject);

	KeAcquireGuardedMutex(&Lock);
	RemoveEntryList(&prov->Entry);
	KeReleaseGuardedMutex(&Lock);
	WmProviderCleanup(prov);
}

static VOID WmFileClose(WDFFILEOBJECT FileObject)
{
	UNREFERENCED_PARAMETER(FileObject);
}

static VOID WmCreateControlDevice(WDFDRIVER Driver)
{
	PWDFDEVICE_INIT			pinit;
	WDF_FILEOBJECT_CONFIG	fileconfig;
	WDF_OBJECT_ATTRIBUTES	attr;
	WDF_IO_QUEUE_CONFIG		ioconfig;
	NTSTATUS				status;
	WDFQUEUE				queue;
	DECLARE_CONST_UNICODE_STRING(name, L"\\Device\\WinMad");
	DECLARE_CONST_UNICODE_STRING(symlink, L"\\DosDevices\\WinMad");

	pinit = WdfControlDeviceInitAllocate(Driver,
	
&SDDL_DEVOBJ_SYS_ALL_ADM_RWX_WORLD_RW_RES_R);
	if (pinit == NULL) {
			return;
	}

	WdfDeviceInitSetExclusive(pinit, FALSE);
	status = WdfDeviceInitAssignName(pinit, &name);
	if (!NT_SUCCESS(status)) {
		goto err1;
	}

	WDF_FILEOBJECT_CONFIG_INIT(&fileconfig, WmFileCreate, WmFileClose,
							   WmFileCleanup);
	WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attr, WM_PROVIDER);
	WdfDeviceInitSetFileObjectConfig(pinit, &fileconfig, &attr);

	WDF_OBJECT_ATTRIBUTES_INIT(&attr);
	status = WdfDeviceCreate(&pinit, &attr, &ControlDevice);
	if (!NT_SUCCESS(status)) {
		goto err1;
	}

	status = WdfDeviceCreateSymbolicLink(ControlDevice, &symlink);
	if (!NT_SUCCESS(status)) {
		goto err2;
	}

	WDF_IO_QUEUE_CONFIG_INIT_DEFAULT_QUEUE(&ioconfig, WdfIoQueueDispatchParallel);
	ioconfig.EvtIoDeviceControl = WmIoDeviceControl;
	ioconfig.EvtIoRead = WmIoRead;
	ioconfig.EvtIoWrite = WmIoWrite;
	status = WdfIoQueueCreate(ControlDevice, &ioconfig,
							  WDF_NO_OBJECT_ATTRIBUTES, &queue);
	if (!NT_SUCCESS(status)) {
		goto err2;
	}

	WdfControlFinishInitializing(ControlDevice);
	return;

err2:
	WdfObjectDelete(ControlDevice);
	return;
err1:
	WdfDeviceInitFree(pinit);
}

static ib_ca_attr_t *WmQueryCaAttributes(WM_IB_DEVICE *pDevice)
{
	ib_ca_attr_t	*attr;
	UINT32			size;
	ib_api_status_t	ib_status;

	size = 0;
	ib_status = pDevice->VerbsInterface.Verbs.
				query_ca(pDevice->VerbsInterface.Verbs.p_hca_dev, NULL, &size, NULL);
	if (ib_status != IB_INSUFFICIENT_MEMORY) {
		attr = NULL;
		goto out;
	}

	attr = ExAllocatePoolWithTag(PagedPool, size, 'acmw');
	if (attr == NULL) {
		goto out;
	}

	ib_status = pDevice->VerbsInterface.Verbs.
				query_ca(pDevice->VerbsInterface.Verbs.p_hca_dev, attr, &size, NULL);
	if (ib_status != IB_SUCCESS) {
		ExFreePool(attr);
		attr = NULL;
	}

out:
	return attr;
}

static NTSTATUS WmAddCa(WM_IB_DEVICE *pDevice)
{
	NTSTATUS		status;
	ib_api_status_t ib_status;
	ib_ca_attr_t	*attr;
	UINT32			size;
	UINT8			i;

	attr = WmQueryCaAttributes(pDevice);
	if (attr == NULL) {
		return STATUS_NO_MEMORY;
	}

	size = sizeof(WM_IB_PORT) * attr->num_ports;
	pDevice->pPortArray = ExAllocatePoolWithTag(PagedPool, size, 'pimw');
	if (pDevice->pPortArray == NULL) {
		status = STATUS_NO_MEMORY;
		goto out;
	}

	for (i = 0; i < attr->num_ports; i++) {
		pDevice->pPortArray[i].Guid = attr->p_port_attr[i].port_guid;
	}
	pDevice->PortCount = attr->num_ports;

	status = STATUS_SUCCESS;
out:
	ExFreePool(attr);
	return status;
}

static NTSTATUS WmPowerD0Entry(WDFDEVICE Device, WDF_POWER_DEVICE_STATE PreviousState)
{
	WM_IB_DEVICE	*dev;
	BOOLEAN			create;
	NTSTATUS		status;

	dev = WmIbDeviceGetContext(Device);
	status = WdfFdoQueryForInterface(Device, &GUID_RDMA_INTERFACE_VERBS,
									 (PINTERFACE) &dev->VerbsInterface,
									 sizeof(dev->VerbsInterface), VerbsVersion(2,
0),
									 NULL);
	if (!NT_SUCCESS(status)) {
		return status;
	}

	dev->Guid = dev->VerbsInterface.Verbs.guid;
	status = WmAddCa(dev);

	dev->VerbsInterface.InterfaceHeader.InterfaceDereference(dev->VerbsInterface.
	
InterfaceHeader.Context);
	if (!NT_SUCCESS(status)) {
		return status;
	}

	status = WdfFdoQueryForInterface(Device, &GUID_IB_AL_INTERFACE,
									 (PINTERFACE) &dev->IbInterface,
									 sizeof(dev->IbInterface),
									 AL_INTERFACE_VERSION, NULL);
	if (!NT_SUCCESS(status)) {
		return status;
	}

	KeAcquireGuardedMutex(&Lock);
	create = IsListEmpty(&DevList);
	InsertHeadList(&DevList, &dev->Entry);
	KeReleaseGuardedMutex(&Lock);

	if (create) {
		WmCreateControlDevice(WdfGetDriver());
	}
	return STATUS_SUCCESS;
}

static NTSTATUS WmIbDeviceAdd(WDFDRIVER Driver, PWDFDEVICE_INIT DeviceInit)
{
	WDF_OBJECT_ATTRIBUTES			attr;
	WDF_PNPPOWER_EVENT_CALLBACKS	power;
	WDFDEVICE						dev;
	WM_IB_DEVICE					*pdev;
	NTSTATUS						status;

	WdfFdoInitSetFilter(DeviceInit);

	WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attr, WM_IB_DEVICE);
	attr.EvtCleanupCallback = WmIbDeviceCleanup;

	WDF_PNPPOWER_EVENT_CALLBACKS_INIT(&power);
	power.EvtDeviceD0Entry = WmPowerD0Entry;
	WdfDeviceInitSetPnpPowerEventCallbacks(DeviceInit, &power);

	status = WdfDeviceCreate(&DeviceInit, &attr, &dev);
	if (!NT_SUCCESS(status)) {
		return status;
	}

	pdev = WmIbDeviceGetContext(dev);
	RtlZeroMemory(pdev, sizeof *pdev);
	pdev->Ref = 1;
	KeInitializeEvent(&pdev->Event, NotificationEvent, FALSE);

	return STATUS_SUCCESS;
}

static VOID WmIbDeviceCleanup(WDFDEVICE Device)
{
	WM_PROVIDER			*prov;
	WM_IB_DEVICE		*pdev;
	WM_REGISTRATION		*reg;
	LIST_ENTRY			*entry;
	BOOLEAN				destroy;
	WDFDEVICE			ctrldev;

	pdev = (WmIbDeviceGetContext(Device));

	KeAcquireGuardedMutex(&Lock);
	RemoveEntryList(&pdev->Entry);
	destroy = IsListEmpty(&DevList);
	ctrldev = ControlDevice;

	for (entry = ProvList.Flink; entry != &ProvList; entry = entry->Flink) {
		prov = CONTAINING_RECORD(entry, WM_PROVIDER, Entry);
		WmProviderRemoveHandler(prov, pdev);
	} 
	KeReleaseGuardedMutex(&Lock);

	if (InterlockedDecrement(&pdev->Ref) > 0) {
		KeWaitForSingleObject(&pdev->Event, Executive, KernelMode, FALSE, NULL);
	}

	pdev->IbInterface.wdm.InterfaceDereference(pdev->IbInterface.wdm.Context);
	if (pdev->pPortArray != NULL) {
		ExFreePool(pdev->pPortArray);
	}

	if (destroy) {
		WdfObjectDelete(ctrldev);
	}
}

NTSTATUS DriverEntry(PDRIVER_OBJECT DriverObject, PUNICODE_STRING RegistryPath)
{
	WDF_DRIVER_CONFIG		config;
	NTSTATUS				status;
	WDFDRIVER				driv;

	InitializeListHead(&DevList);
	InitializeListHead(&ProvList);
	KeInitializeGuardedMutex(&Lock);

	WDF_DRIVER_CONFIG_INIT(&config, WmIbDeviceAdd);
	status = WdfDriverCreate(DriverObject, RegistryPath, WDF_NO_OBJECT_ATTRIBUTES,
							 &config, &driv);
	if (!NT_SUCCESS(status)) {
		return status;
	}

	return STATUS_SUCCESS;
}

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "index_list.c"
#include "wm_driver.h"
#include "wm_ioctl.h"
#include "wm_provider.h"
#include "wm_reg.h"

void WmProviderGet(WM_PROVIDER *pProvider)
{
	InterlockedIncrement(&pProvider->Ref);
}

void WmProviderPut(WM_PROVIDER *pProvider)
{
	if (InterlockedDecrement(&pProvider->Ref) == 0) {
		KeSetEvent(&pProvider->Event, 0, FALSE);
	}
}

NTSTATUS WmProviderInit(WM_PROVIDER *pProvider)
{
	WDF_IO_QUEUE_CONFIG	config;
	NTSTATUS status;

	IndexListInit(&pProvider->RegIndex);
	pProvider->MadHead = NULL;
	pProvider->MadTail = NULL;

	KeInitializeGuardedMutex(&pProvider->Lock);
	pProvider->Ref = 1;
	KeInitializeEvent(&pProvider->Event, NotificationEvent, FALSE);

	pProvider->Pending = 0;
	pProvider->Active = 0;
	KeInitializeEvent(&pProvider->SharedEvent, NotificationEvent, FALSE);
	pProvider->Exclusive = 0;
	KeInitializeEvent(&pProvider->ExclusiveEvent, SynchronizationEvent, FALSE);

	WDF_IO_QUEUE_CONFIG_INIT(&config, WdfIoQueueDispatchManual);
	status = WdfIoQueueCreate(ControlDevice, &config,
							  WDF_NO_OBJECT_ATTRIBUTES, &pProvider->ReadQueue);
	return status;
}

static void WmInsertMad(WM_PROVIDER *pProvider, ib_mad_element_t *pMad)
{
	if (pProvider->MadHead == NULL) {
		pProvider->MadHead = pMad;
	} else {
		pProvider->MadTail->p_next = pMad;
	}
	pProvider->MadTail = pMad;
}

static ib_mad_element_t *WmRemoveMad(WM_PROVIDER *pProvider)
{
	ib_mad_element_t *mad;

	mad = pProvider->MadHead;
	if (mad != NULL) {
		pProvider->MadHead = (ib_mad_element_t *) mad->p_next;
		mad->p_next = NULL;
	}
	return mad;
}

void WmProviderFlushReceives(WM_PROVIDER *pProvider, WM_REGISTRATION *pRegistration)
{
	ib_mad_element_t	*mad, *next, *list;

	WdfObjectAcquireLock(pProvider->ReadQueue);
	list = pProvider->MadHead;
	pProvider->MadHead = NULL;

	for (mad = list; mad != NULL; mad = next) {
		next = mad->p_next;
		mad->p_next = NULL;

		if (mad->send_context1 == pRegistration) {
			pRegistration->pDevice->IbInterface.put_mad(mad);
		} else {
			WmInsertMad(pProvider, mad);
		}
	}
	WdfObjectReleaseLock(pProvider->ReadQueue);
}

void WmProviderCleanup(WM_PROVIDER *pProvider)
{
	WM_REGISTRATION		*reg;

	while ((reg = IndexListRemoveHead(&pProvider->RegIndex)) != NULL) {
		WmRegFree(reg);
	}

	if (InterlockedDecrement(&pProvider->Ref) > 0) {
		KeWaitForSingleObject(&pProvider->Event, Executive, KernelMode, FALSE, NULL);
	}

	WdfIoQueuePurgeSynchronously(pProvider->ReadQueue);
	WdfObjectDelete(pProvider->ReadQueue);

	IndexListDestroy(&pProvider->RegIndex);
}

// See comment above WmProviderRemoveHandler.
static void WmProviderLockRemove(WM_PROVIDER *pProvider)
{
	KeAcquireGuardedMutex(&pProvider->Lock);
	pProvider->Exclusive++;
	KeClearEvent(&pProvider->SharedEvent);
	while (pProvider->Active > 0) {
		KeReleaseGuardedMutex(&pProvider->Lock);
		KeWaitForSingleObject(&pProvider->ExclusiveEvent, Executive, KernelMode,
							  FALSE, NULL);
		KeAcquireGuardedMutex(&pProvider->Lock);
	}
	pProvider->Active++;
	KeReleaseGuardedMutex(&pProvider->Lock);
}

// See comment above WmProviderRemoveHandler.
static void WmProviderUnlockRemove(WM_PROVIDER *pProvider)
{
	KeAcquireGuardedMutex(&pProvider->Lock);
	pProvider->Exclusive--;
	pProvider->Active--;
	if (pProvider->Exclusive > 0) {
		KeSetEvent(&pProvider->ExclusiveEvent, 0, FALSE);
	} else if (pProvider->Pending > 0) {
		KeSetEvent(&pProvider->SharedEvent, 0, FALSE);
	}
	KeReleaseGuardedMutex(&pProvider->Lock);
}

/*
 * Must hold pProvider->Lock.  Function may release and re-acquire.
 * See comment above WmProviderRemoveHandler.
 */
void WmProviderDisableRemove(WM_PROVIDER *pProvider)
{
	while (pProvider->Exclusive > 0) {
		pProvider->Pending++;
		KeReleaseGuardedMutex(&pProvider->Lock);
		KeWaitForSingleObject(&pProvider->SharedEvent, Executive, KernelMode,
							  FALSE, NULL);
		KeAcquireGuardedMutex(&pProvider->Lock);
		pProvider->Pending--;
	}
	InterlockedIncrement(&pProvider->Active);
}

/*
 * No need to hold pProvider->Lock when releasing.
 * See comment above WmProviderRemoveHandler.
 */
void WmProviderEnableRemove(WM_PROVIDER *pProvider)
{
	InterlockedDecrement(&pProvider->Active);
	if (pProvider->Exclusive > 0) {
		KeSetEvent(&pProvider->ExclusiveEvent, 0, FALSE);
	}
}

/*
 * The remove handler blocks all other threads executing through this
 * provider until the remove has been processed.  Because device removal is
 * rare, we want a simple, optimized code path for all calls that access
 * the underlying hardware device, making use of any locks that we would
 * have to acquire anyway.  The locking for exclusive access can be
 * as ugly and slow as needed.
 */
void WmProviderRemoveHandler(WM_PROVIDER *pProvider, WM_IB_DEVICE *pDevice)
{
	WM_REGISTRATION *reg;
	SIZE_T i;

	WmProviderLockRemove(pProvider);
	IndexListForEach(&pProvider->RegIndex, i) {
		reg = IndexListAt(&pProvider->RegIndex, i);
		if (reg->pDevice == pDevice) {
			WmRegRemoveHandler(reg);
		}
	}
	WmProviderUnlockRemove(pProvider);
}

static NTSTATUS WmCopyRead(WM_PROVIDER *pProvider, WM_IO_MAD *pIoMad,
						   ib_mad_element_t *pMad, size_t *pLen)
{
	WM_REGISTRATION		*reg;

	reg = (WM_REGISTRATION *) pMad->send_context1;
	pIoMad->Id = reg->Id;

	pIoMad->Status = pMad->status;
	pIoMad->Timeout = pMad->timeout_ms;
	pIoMad->Retries = pMad->retry_cnt;
	pIoMad->Length = pMad->size;

	pIoMad->Address.Qpn = pMad->remote_qp;
	pIoMad->Address.Qkey = pMad->remote_qkey;
	pIoMad->Address.PkeyIndex = pMad->pkey_index;

	if ((pIoMad->Address.GrhValid = (UINT8) pMad->grh_valid)) {
		pIoMad->Address.VersionClassFlow = pMad->p_grh->ver_class_flow;
		pIoMad->Address.HopLimit = pMad->p_grh->hop_limit;
		pIoMad->Address.GidIndex = 0;	// TODO: update IBAL to use SGID index
		RtlCopyMemory(pIoMad->Address.Gid, pMad->p_grh->dest_gid.raw, 16);
	}

	pIoMad->Address.Lid = pMad->remote_lid;
	pIoMad->Address.ServiceLevel = pMad->remote_sl;
	pIoMad->Address.PathBits = pMad->path_bits;
	pIoMad->Address.StaticRate = 0;
	pIoMad->Address.Reserved = 0;

	if (*pLen >= sizeof(WM_IO_MAD) + pMad->size) {
		RtlCopyMemory(pIoMad + 1, pMad->p_mad_buf, pMad->size);
		*pLen = sizeof(WM_IO_MAD) + pMad->size;
		return STATUS_SUCCESS;
	} else {
		*pLen = sizeof(WM_IO_MAD);
		return STATUS_MORE_ENTRIES;
	}
}

void WmProviderRead(WM_PROVIDER *pProvider, WDFREQUEST Request)
{
	WM_REGISTRATION		*reg;
	NTSTATUS			status;
	WM_IO_MAD			*wmad;
	size_t				outlen, len = 0;

	status = WdfRequestRetrieveOutputBuffer(Request, sizeof(WM_IO_MAD), &wmad, &outlen);
	if (!NT_SUCCESS(status)) {
		goto out;
	}

	WdfObjectAcquireLock(pProvider->ReadQueue);
	if (pProvider->MadHead == NULL) {
		status = WdfRequestForwardToIoQueue(Request, pProvider->ReadQueue);
		WdfObjectReleaseLock(pProvider->ReadQueue);
		if (NT_SUCCESS(status)) {
			return;
		}
		goto out;
	}

	len = outlen;
	status = WmCopyRead(pProvider, wmad, pProvider->MadHead, &len);
	if (NT_SUCCESS(status)) {
		reg = (WM_REGISTRATION *) pProvider->MadHead->send_context1;
		reg->pDevice->IbInterface.put_mad(WmRemoveMad(pProvider));
	}
	WdfObjectReleaseLock(pProvider->ReadQueue);

out:
	WdfRequestCompleteWithInformation(Request, status, len);
}

static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UINT32 size)
{
	ib_al_ifc_t			*pifc;
	NTSTATUS			status;
	ib_mad_element_t	*mad;
	ib_api_status_t		ib_status;

	pifc = &pRegistration->pDevice->IbInterface;
	ib_status = pifc->get_mad(pRegistration->hMadPool, size, &mad);
	if (ib_status != IB_SUCCESS) {
		return STATUS_NO_MEMORY;
	}

	mad->context1 = pRegistration;
	RtlCopyMemory(mad->p_mad_buf, pIoMad + 1, size);
	mad->remote_qp = pIoMad->Address.Qpn;
	mad->remote_qkey = pIoMad->Address.Qkey;
	mad->resp_expected = (pIoMad->Timeout > 0);
	mad->timeout_ms = pIoMad->Timeout;
	mad->retry_cnt = pIoMad->Retries;

	if ((mad->grh_valid = pIoMad->Address.GrhValid)) {
		mad->p_grh->ver_class_flow = pIoMad->Address.VersionClassFlow;
		mad->p_grh->hop_limit = pIoMad->Address.HopLimit;
		// TODO: update IBAL to use SGID index
		// mad->p_grh->src_gid_index = pIoMad->Address.GidIndex;
		RtlCopyMemory(mad->p_grh->dest_gid.raw, pIoMad->Address.Gid, 16);
	}

	mad->remote_lid = pIoMad->Address.Lid;
	mad->remote_sl = pIoMad->Address.ServiceLevel;
	mad->pkey_index = pIoMad->Address.PkeyIndex;
	mad->path_bits = pIoMad->Address.PathBits;
	mad->p_mad_buf->trans_id &= 0xFFFFFFFF00000000;

	ib_status = pifc->send_mad(pRegistration->hService, mad, NULL);
	if (ib_status != IB_SUCCESS) {
		status = STATUS_UNSUCCESSFUL;
		goto err;
	}

	return STATUS_SUCCESS;

err:
	pRegistration->pDevice->IbInterface.put_mad(mad);
	return status;
}

void WmProviderWrite(WM_PROVIDER *pProvider, WDFREQUEST Request)
{
	WM_REGISTRATION		*reg;
	NTSTATUS			status;
	WM_IO_MAD			*wmad;
	size_t				inlen;

	status = WdfRequestRetrieveInputBuffer(Request, sizeof(WM_IO_MAD) + 24,
										   &wmad, &inlen);
	if (!NT_SUCCESS(status)) {
		goto out;
	}

	reg = WmRegAcquire(pProvider, wmad->Id);
	if (reg == NULL) {
		status = STATUS_NOT_FOUND;
		goto out;
	}

	status = WmSendMad(reg, wmad, (UINT32) (inlen - sizeof(WM_IO_MAD)));
	WmRegRelease(reg);

out:
	WdfRequestComplete(Request, status);
}

void WmReceiveHandler(ib_mad_svc_handle_t hService, void *Context,
					  ib_mad_element_t *pMad)
{
	WM_REGISTRATION	*reg;
	WM_PROVIDER		*prov = Context;
	WDFREQUEST		request;
	NTSTATUS		status;
	WM_IO_MAD		*wmad;
	size_t			len = 0;

	UNREFERENCED_PARAMETER(hService);

	WdfObjectAcquireLock(prov->ReadQueue);
	status = WdfIoQueueRetrieveNextRequest(prov->ReadQueue, &request);
	if (!NT_SUCCESS(status)) {
		WmInsertMad(prov, pMad);
		WdfObjectReleaseLock(prov->ReadQueue);
		return;
	}

	status = WdfRequestRetrieveOutputBuffer(request, sizeof(WM_IO_MAD), &wmad, &len);
	if (!NT_SUCCESS(status)) {
		reg = (WM_REGISTRATION *) pMad->send_context1;
		reg->pDevice->IbInterface.put_mad(pMad);
		goto out;
	}

	status = WmCopyRead(prov, wmad, pMad, &len);
	if (NT_SUCCESS(status)) {
		reg = (WM_REGISTRATION *) pMad->send_context1;
		reg->pDevice->IbInterface.put_mad(pMad);
	} else {
		WmInsertMad(prov, pMad);
	}
	WdfObjectReleaseLock(prov->ReadQueue);

out:
	WdfRequestCompleteWithInformation(request, status, len);	
}

void WmSendHandler(ib_mad_svc_handle_t hService, void *Context,
				   ib_mad_element_t *pMad)
{
	if (pMad->status == IB_SUCCESS) {
		((WM_REGISTRATION *) pMad->context1)->pDevice->IbInterface.put_mad(pMad);
	} else {
		pMad->send_context1 = (void*) pMad->context1;
		WmReceiveHandler(hService, Context, pMad);
	}
}

void WmProviderCancel(WM_PROVIDER *pProvider, WDFREQUEST Request)
{
	WDFREQUEST	request;
	NTSTATUS	status;

	WdfObjectAcquireLock(pProvider->ReadQueue);
	status = WdfIoQueueRetrieveNextRequest(pProvider->ReadQueue, &request);

	while (NT_SUCCESS(status)) {
		WdfRequestComplete(request, STATUS_CANCELLED);
		status = WdfIoQueueRetrieveNextRequest(pProvider->ReadQueue, &request);
	}
	WdfObjectReleaseLock(pProvider->ReadQueue);

	WdfRequestComplete(Request, status);
}

/*
 * Copyright (c) 2008 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <iba\ib_al.h>
#include "wm_driver.h"
#include "wm_reg.h"
#include "wm_ioctl.h"

WM_REGISTRATION *WmRegAcquire(WM_PROVIDER *pProvider, UINT64 Id)
{
	WM_REGISTRATION *reg;

	KeAcquireGuardedMutex(&pProvider->Lock);
	WmProviderDisableRemove(pProvider);
	reg = IndexListAt(&pProvider->RegIndex, (SIZE_T) Id);
	if (reg != NULL && reg->pDevice != NULL) {
		InterlockedIncrement(&reg->Ref);
	} else {
		reg = NULL;
		WmProviderEnableRemove(pProvider);
	}
	KeReleaseGuardedMutex(&pProvider->Lock);
	return reg;
}

void WmRegRelease(WM_REGISTRATION *pRegistration)
{
	WmProviderEnableRemove(pRegistration->pProvider);
	InterlockedDecrement(&pRegistration->Ref);
}

static WM_REGISTRATION *WmRegAlloc(WM_PROVIDER *pProvider)
{
	WM_REGISTRATION *reg;

	reg = ExAllocatePoolWithTag(PagedPool, sizeof(WM_REGISTRATION), 'grmw');
	if (reg == NULL) {
		return NULL;
	}

	RtlZeroMemory(reg, sizeof(WM_REGISTRATION));
	reg->Ref = 1;

	reg->pProvider = pProvider;
	WmProviderGet(pProvider);
	return reg;
}

static int WmConvertMethods(ib_mad_svc_t *svc, WM_IO_REGISTER *pAttributes)
{
	int i, j, unsolicited = 0;

	for (i = 0; i < 16; i++) {
		for (j = 0; j < 8; j++) {
			if (((pAttributes->Methods[i] >> j) & 0x01) != 0) {
				svc->method_array[i * 8 + j] = 1;
				unsolicited = 1;
			}
		}
	}
	return unsolicited;
}

static NTSTATUS WmRegInit(WM_REGISTRATION *pRegistration, WM_IO_REGISTER *pAttributes)
{
	WM_IB_DEVICE	*dev;
	ib_qp_create_t	attr;
	ib_mad_svc_t	svc;
	ib_api_status_t ib_status;
	NTSTATUS		status;

	RtlZeroMemory(&attr, sizeof attr);
	if (pAttributes->Qpn == 0) {
		attr.qp_type = IB_QPT_QP0_ALIAS;
	} else if (pAttributes->Qpn == IB_QP1) {
		attr.qp_type = IB_QPT_QP1_ALIAS;
	} else {
		return STATUS_BAD_NETWORK_PATH;
	}

	dev = WmIbDeviceGet(pAttributes->Guid);
	if (dev == NULL) {
		return STATUS_NO_SUCH_DEVICE;
	}

	if (--pAttributes->Port > dev->PortCount) {
		status = STATUS_INVALID_PORT_HANDLE;
		goto err1;
	}

	ib_status = dev->IbInterface.open_al(&pRegistration->hIbal);
	if (ib_status != IB_SUCCESS) {
		status = STATUS_UNSUCCESSFUL;
		goto err1;
	}

	ib_status = dev->IbInterface.open_ca(pRegistration->hIbal, pAttributes->Guid,
										 NULL, NULL, &pRegistration->hCa);
	if (ib_status != IB_SUCCESS) {
		goto err2;
	}

	ib_status = dev->IbInterface.alloc_pd(pRegistration->hCa, IB_PDT_ALIAS,
										  NULL, &pRegistration->hPd);
	if (ib_status != IB_SUCCESS) {
		goto err3;
	}

	attr.sq_depth = attr.rq_depth = 1;
	attr.sq_sge = attr.rq_sge = 1;
	attr.sq_signaled = 1;

	ib_status = dev->IbInterface.get_spl_qp(pRegistration->hPd,
	
dev->pPortArray[pAttributes->Port].Guid,
											&attr, pRegistration, NULL,
											&pRegistration->hMadPool,
											&pRegistration->hQp);
	if (ib_status != IB_SUCCESS) {
		status = STATUS_UNSUCCESSFUL;
		goto err4;
	}

	svc.mad_svc_context = pRegistration->pProvider;
	svc.pfn_mad_send_cb = WmSendHandler;
	svc.pfn_mad_recv_cb = WmReceiveHandler;
	svc.support_unsol = WmConvertMethods(&svc, pAttributes);
	svc.mgmt_class = pAttributes->Class;
	svc.mgmt_version = pAttributes->Version;
    svc.svc_type = IB_MAD_SVC_DEFAULT;

	ib_status = dev->IbInterface.reg_mad_svc(pRegistration->hQp, &svc,
											 &pRegistration->hService);
	if (ib_status != IB_SUCCESS) {
		status = STATUS_UNSUCCESSFUL;
		goto err5;
	}

	pRegistration->pDevice = dev;
	return STATUS_SUCCESS;

err5:
	dev->IbInterface.destroy_qp(pRegistration->hQp, NULL);
err4:
	dev->IbInterface.dealloc_pd(pRegistration->hPd, NULL);
err3:
	dev->IbInterface.close_ca(pRegistration->hCa, NULL);
err2:
	dev->IbInterface.close_al(pRegistration->hIbal);
err1:
	WmIbDevicePut(dev);
	pRegistration->pDevice = NULL;
	return status;
}

void WmRegister(WM_PROVIDER *pProvider, WDFREQUEST Request)
{
	WM_REGISTRATION	*reg;
	WM_IO_REGISTER	*attr;
	UINT64			*id;
	NTSTATUS		status;

	status = WdfRequestRetrieveInputBuffer(Request, sizeof(WM_IO_REGISTER), &attr, NULL);
	if (!NT_SUCCESS(status)) {
		goto err1;
	}
	status = WdfRequestRetrieveOutputBuffer(Request, sizeof(UINT64), &id, NULL);
	if (!NT_SUCCESS(status)) {
		goto err1;
	}

	reg = WmRegAlloc(pProvider);
	if (reg == NULL) {
		status = STATUS_NO_MEMORY;
		goto err1;
	}

	KeAcquireGuardedMutex(&pProvider->Lock);
	WmProviderDisableRemove(pProvider);
	KeReleaseGuardedMutex(&pProvider->Lock);

	status = WmRegInit(reg, attr);
	if (!NT_SUCCESS(status)) {
		goto err2;
	}

	KeAcquireGuardedMutex(&pProvider->Lock);
	reg->Id = IndexListInsertHead(&pProvider->RegIndex, reg);
	if (reg->Id == 0) {
		status = STATUS_NO_MEMORY;
		goto err2;
	}
	KeReleaseGuardedMutex(&pProvider->Lock);

	WmProviderEnableRemove(pProvider);
	*id = reg->Id;
	WdfRequestCompleteWithInformation(Request, status, sizeof(UINT64));
	return;

err2:
	WmRegFree(reg);
	WmProviderEnableRemove(pProvider);
err1:
	WdfRequestComplete(Request, status);
}

void WmDeregister(WM_PROVIDER *pProvider, WDFREQUEST Request)
{
	WM_REGISTRATION *reg;
	UINT64			*id;
	NTSTATUS		status;

	status = WdfRequestRetrieveInputBuffer(Request, sizeof(UINT64), &id, NULL);
	if (!NT_SUCCESS(status)) {
		goto out;
	}

	KeAcquireGuardedMutex(&pProvider->Lock);
	WmProviderDisableRemove(pProvider);
	reg = IndexListAt(&pProvider->RegIndex, (SIZE_T) *id);
	if (reg == NULL) {
		status = STATUS_NO_SUCH_DEVICE;
	} else if (reg->Ref > 1) {
		status = STATUS_ACCESS_DENIED;
	} else {
		IndexListRemove(&pProvider->RegIndex, (SIZE_T) *id);
		status = STATUS_SUCCESS;
	}
	KeReleaseGuardedMutex(&pProvider->Lock);

	if (NT_SUCCESS(status)) {
		WmRegFree(reg);
	}
	WmProviderEnableRemove(pProvider);
out:
	WdfRequestComplete(Request, status);
}

void WmRegFree(WM_REGISTRATION *pRegistatration)
{
	WmRegRemoveHandler(pRegistatration);
	WmProviderPut(pRegistatration->pProvider);
	ExFreePool(pRegistatration);
}

void WmRegRemoveHandler(WM_REGISTRATION *pRegistration)
{
	if (pRegistration->pDevice == NULL) {
		return;
	}

	pRegistration->pDevice->IbInterface.destroy_qp(pRegistration->hQp, NULL);
	pRegistration->pDevice->IbInterface.dealloc_pd(pRegistration->hPd, NULL);
	pRegistration->pDevice->IbInterface.close_ca(pRegistration->hCa, NULL);
	pRegistration->pDevice->IbInterface.close_al(pRegistration->hIbal);

	WmProviderFlushReceives(pRegistration->pProvider, pRegistration);
	WmIbDevicePut(pRegistration->pDevice);
	pRegistration->pDevice = NULL;
}