[ofw] [RFC] [PATCH 3/3] libibverbs: winof port of device routines

[Sean Hefty]

Implementation for libibverbs device related calls (get device list, open/
close device, get event, etc.).

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
/*
 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
 * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <stdio.h>
#include <infiniband/verbs.h>
#include <rdma/winverbs.h>

IWVProvider *prov;

struct verbs_device
{
	struct ibv_device	device;
	uint64_t			guid;
	uint8_t				phys_port_cnt;
};

struct verbs_port
{
	OVERLAPPED			overlap;
	DWORD				event_flag;
};

#define EVENT_PORT_NONE	-1

struct verbs_context
{
	struct ibv_context	context;
	struct verbs_device	device;
	HANDLE				*event;
	struct verbs_port	*port;
	uint8_t				event_port_num;
};

__declspec(dllexport)
struct ibv_device **ibv_get_device_list(int *num)
{
	WV_DEVICE_ATTRIBUTES attr;
	struct verbs_device *dev_array;
	struct ibv_device **pdev_array;
	UINT64 *guid;
	SIZE_T size, cnt;
	HRESULT hr;

	if (prov == NULL) {
		hr = WvGetObject(IID_IWVProvider, (LPVOID*) &prov);
		if (FAILED(hr)) {
			goto err1;
		}	
	}

	cnt = 0;
	size = sizeof(UINT64);

	while ((size / sizeof(UINT64)) > cnt) {
		if (cnt > 0) {
			delete guid;
		}

		cnt = size / sizeof(UINT64);
		guid = new UINT64[cnt];
		if (guid == NULL) {
			goto err1;
		}

		hr = prov->QueryDeviceList(guid, &size);
		if (FAILED(hr)) {
			goto err2;
		}
	}

	*num = (int) (size / sizeof(UINT64));
	dev_array = new struct verbs_device[*num];
	pdev_array = new struct ibv_device*[*num];
	if (dev_array == NULL || pdev_array == NULL) {
		goto err2;
	}

	for (cnt = 0; cnt < (SIZE_T) *num; cnt++) {
		pdev_array[cnt] = &dev_array[cnt].device;
		hr = prov->QueryDevice(guid[cnt], &attr);
		if (FAILED(hr)) {
			goto err3;
		}

		sscanf(dev_array[cnt].device.name, "ibv_device 0x%x", guid[cnt]);
		dev_array[cnt].device.node_type = IBV_NODE_UNKNOWN;
		dev_array[cnt].device.transport_type = (ibv_transport_type) attr.DeviceType;
		dev_array[cnt].guid = guid[cnt];
		dev_array[cnt].phys_port_cnt = attr.PhysPortCount;
	}

	return pdev_array;

err3:
	ibv_free_device_list(pdev_array);
err2:
	delete guid;
err1:
	*num = 0;
	return NULL;
}

__declspec(dllexport)
void ibv_free_device_list(struct ibv_device **list)
{
	delete CONTAINING_RECORD(list[0], struct verbs_device, device);
	delete list;
}

__declspec(dllexport)
const char *ibv_get_device_name(struct ibv_device *device)
{
	return device->name;
}

__declspec(dllexport)
uint64_t ibv_get_device_guid(struct ibv_device *device)
{
	return CONTAINING_RECORD(device, struct verbs_device, device)->guid;
}

__declspec(dllexport)
struct ibv_context *ibv_open_device(struct ibv_device *device)
{
	struct verbs_device *vdev;
	struct verbs_context *vcontext;
	HRESULT hr;
	int i;

	vdev = CONTAINING_RECORD(device, struct verbs_device, device);
	vcontext = new struct verbs_context;
	if (vcontext == NULL) {
		return NULL;
	}
	memcpy(&vcontext->device, vdev, sizeof(struct verbs_device));
	vcontext->event_port_num = EVENT_PORT_NONE;

	vcontext->port = new struct verbs_port[vdev->phys_port_cnt];
	if (vcontext->port == NULL) {
		goto err1;
	}

	vcontext->event = new HANDLE[vdev->phys_port_cnt];
	if (vcontext->event == NULL) {
		goto err2;
	}

	hr = prov->OpenDevice(vdev->guid, &vcontext->context.cmd_if);
	if (FAILED(hr)) {
		goto err3;
	}

	for (i = 0; i < vdev->phys_port_cnt; i++) {
		vcontext->event[i] = CreateEvent(NULL, FALSE, FALSE, NULL);
		if (vcontext->event[i] == NULL) {
			goto err4;
		}
		vcontext->port[i].overlap.hEvent = vcontext->event[i];
		vcontext->port[i].event_flag = 0;

		vcontext->context.cmd_if->Notify((UINT8) i,
										 &vcontext->port[i].overlap,
										 &vcontext->port[i].event_flag);
	}

	return &vcontext->context;

err4:
	while (--i >= 0) {
		CloseHandle(vcontext->event[i]);
	}
err3:
	delete vcontext->event;
err2:
	delete vcontext->port;
err1:
	delete vcontext;
	return NULL;
}

__declspec(dllexport)
int ibv_close_device(struct ibv_context *context)
{
	struct verbs_context *vcontext;
	int i;

	vcontext = CONTAINING_RECORD(context, struct verbs_context, context);
	context->cmd_if->CancelOverlappedRequests();

	for (i = 0; i < vcontext->device.phys_port_cnt; i++) {
		CloseHandle(vcontext->event[i]);
	}

	context->cmd_if->Release();
	delete vcontext->event;
	delete vcontext->port;
	delete vcontext;
	return 0;
}

static enum ibv_event_type ibv_get_port_event_state(struct verbs_context *vcontext)
{
	WV_PORT_ATTRIBUTES attr;
	HRESULT hr;

	hr = vcontext->context.cmd_if->QueryPort(vcontext->event_port_num, &attr);
	if (FAILED(hr)) {
		return IBV_EVENT_PORT_ERR;
	}

	return (attr.State == WvPortActive) ?
		   IBV_EVENT_PORT_ACTIVE : IBV_EVENT_PORT_ERR;
}

static int ibv_report_port_event(struct verbs_context *vcontext,
								 struct ibv_async_event *event)
{
	struct verbs_port *port;
	int ret = 0;

	port = &vcontext->port[vcontext->event_port_num];
	event->element.port_num = vcontext->event_port_num;

	if (port->event_flag & WV_EVENT_ERROR) {
		event->event_type = IBV_EVENT_DEVICE_FATAL;
		port->event_flag = 0;
	} else if (port->event_flag & WV_EVENT_STATE) {
		event->event_type = ibv_get_port_event_state(vcontext);
		port->event_flag = 0;
	} else if (port->event_flag & WV_EVENT_MANAGEMENT) {
		event->event_type = IBV_EVENT_SM_CHANGE;
		port->event_flag = 0;
	} else if (port->event_flag & WV_EVENT_LINK_ADDRESS) {
		event->event_type = IBV_EVENT_LID_CHANGE;
		port->event_flag &= ~WV_EVENT_LINK_ADDRESS;
	} else if (port->event_flag & WV_EVENT_PARTITION) {
		event->event_type = IBV_EVENT_PKEY_CHANGE;
		port->event_flag &= ~WV_EVENT_PARTITION;
	} else {
		port->event_flag = 0;
		ret = -1;
	}
	
	if (port->event_flag == 0) {
		vcontext->context.cmd_if->Notify(vcontext->event_port_num,
										 &port->overlap, &port->event_flag);
		vcontext->event_port_num = EVENT_PORT_NONE;
	}
	return ret;
}

__declspec(dllexport)
int ibv_get_async_event(struct ibv_context *context,
						struct ibv_async_event *event)
{
	struct verbs_context *vcontext;
	HRESULT hr;
	int i;

	vcontext = CONTAINING_RECORD(context, struct verbs_context, context);
	if (vcontext->event_port_num != EVENT_PORT_NONE) {
		if (ibv_report_port_event(vcontext, event) == 0) {
			return 0;
		}
	}

	hr = WaitForMultipleObjects(vcontext->device.phys_port_cnt,
								vcontext->event, FALSE, context->timeout);
	if (hr == WAIT_TIMEOUT) {
		return hr;
	} else if (hr == WAIT_FAILED) {
		return HRESULT_FROM_WIN32(GetLastError());
	}

	vcontext->event_port_num = (UINT8) hr;
	return ibv_report_port_event(vcontext, event);
}

__declspec(dllexport)
void ibv_ack_async_event(struct ibv_async_event *event)
{
	// Only device/port level events are currently supported
	// nothing to do here at the moment
}