[ofw] [RFC] [PATCH 3/3] port of libibumad to windows

Sean Hefty sean.hefty at intel.com
Tue Jul 1 19:02:15 PDT 2008


This ports the libibumad interfaces and basic functionality to Windows.  It relies
on winmad for its MAD transport, and libibverbs for additional functionality
(e.g. obtaining CA and port attributes).

The intent is to simplify porting and maintaining the OFED Infiniband management
utilities (diags, opensm, other MAD libraries), such that the WinOF and OFED
stacks can provide similar functionality and be kept more in sync.

The implementation focuses only on what I'm hoping is crucial to running the
existing OFED applications.  Debug related interfaces were not implemented.
Attention was given to ensure that performance using this interface would be
comparable to existing OFED and WinOF interfaces.  This will need to be verified
through testing.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
/*
 * Copyright (c) 2004-2007 Voltaire Inc.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <windows.h>
#include <stdio.h>

#include <infiniband/umad.h>
#include <infiniband/verbs.h>
#include "ibumad.h"

#define IB_OPENIB_OUI                 (0x001405)

#define UMAD_MAX_PKEYS	16

typedef struct um_port
{
	IWMProvider *prov;
	UINT64		dev_guid;
	OVERLAPPED	overlap;
	BOOL		pending;
	UINT8		port_num;

}	um_port_t;

CRITICAL_SECTION crit_sec;
um_port_t ports[UMAD_MAX_PORTS];


__declspec(dllexport)
int umad_init(void)
{
	InitializeCriticalSection(&crit_sec);
	return 0;
}

__declspec(dllexport)
int umad_done(void)
{
	return 0;
}

__declspec(dllexport)
int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
{
	struct ibv_device	**list;
	int					cnt, i;

	list = ibv_get_device_list(&cnt);
	if (list == NULL) {
		return 0;
	}

	for (i = 0; i < min(cnt, max); i++) {
		strcpy(cas[i], ibv_get_device_name(list[i]));
	}

	ibv_free_device_list(list);
	return i;
}

__declspec(dllexport)
int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
{
	umad_ca_t	ca;
	int			ports = 0, i;

	if (umad_get_ca(ca_name, &ca) < 0)
		return -1;

	if (ca.numports + 1 > max) {
		umad_release_ca(&ca);
		return -ENOMEM;
	}

	for (i = 0; i <= ca.numports; i++)
		portguids[ports++] = ca.ports[i]->port_guid;

	umad_release_ca(&ca);
	return ports;
}

static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
{
	ca->node_type = 1;	// HCA
	ca->numports = attr->phys_port_cnt;
	strncpy(ca->fw_ver, attr->fw_ver, 20);
	memset(ca->ca_type, 0, 40);		// TODO: determine what this should be
	sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
	ca->node_guid = attr->node_guid;
	ca->system_guid = attr->sys_image_guid;
}

static int umad_query_port(struct ibv_context *context, umad_port_t *port)
{
	ibv_port_attr	attr;
	ibv_gid			gid;
	int				i, ret;

	ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
	if (ret != 0) {
		return ret;
	}

	port->base_lid = attr.lid;
	port->lmc = attr.lmc;
	port->sm_lid = attr.sm_lid;
	port->sm_sl = attr.sm_sl;
	port->state = attr.state;
	port->phys_state = attr.phys_state;
	port->rate = attr.active_speed;
	port->capmask = attr.port_cap_flags;

	// Assume GID 0 contains port GUID and gid prefix
	ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
	if (ret != 0) {
		return ret;
	}

	port->gid_prefix = gid.global.subnet_prefix;
	port->port_guid = gid.global.interface_id;

	port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
	for (i = 0; i < (int) port->pkeys_size; i++) {
		ret = ibv_query_pkey(context,(uint8_t)  port->portnum, i, &port->pkeys[i]);
		if (ret != 0) {
			return ret;
		}
	}

	return 0;
}

__declspec(dllexport)
int	umad_get_ca(char *ca_name, umad_ca_t *ca)
{
	struct ibv_device	**list;
	struct ibv_context	*context;
	ibv_device_attr		dev_attr;
	int					cnt, i, ret = 0;
	uint8_t				*ports;
	size_t				port_size;

	list = ibv_get_device_list(&cnt);
	if (list == NULL) {
		return -ENOMEM;
	}

	for (i = 0; i < cnt; i++) {
		if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
			break;
		}
	}

	if (i == cnt) {
		ret = -EINVAL;
		goto free;
	}

	context = ibv_open_device(list[i]);
	if (context == NULL) {
		ret = -ENOMEM;
		goto free;
	}

	ret = ibv_query_device(context, &dev_attr);
	if (ret != 0) {
		goto close;
	}

	port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
	ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
	if (ports == NULL) {
		ret = -ENOMEM;
		goto close;
	}

	strcpy(ca->ca_name, ca_name);
	umad_convert_ca_attr(ca, &dev_attr);

	for (i = 0; i < dev_attr.phys_port_cnt; i++, ports += port_size) {

		ca->ports[i] = (umad_port_t *) ports;
		strcpy(ca->ports[i]->ca_name, ca_name);
		ca->ports[i]->portnum = i + 1;
		ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));

		ret = umad_query_port(context, ca->ports[i]);
		if (ret != 0) {
			goto close;
		}
	}

close:
	ibv_close_device(context);
free:
	ibv_free_device_list(list);
	if (ret != 0) {
		delete ca;
	}
	return ret;
}

__declspec(dllexport)
int	umad_release_ca(umad_ca_t *ca)
{
	delete ca->ports[0];
	return 0;
}

static uint64_t umad_get_ca_guid(char *ca_name)
{
	umad_ca_t	ca;
	uint64_t	guid;
	int			ret;

	ret = umad_get_ca(ca_name, &ca);
	if (ret != 0) {
		return 0;
	}

	guid = ca.node_guid;
	umad_release_ca(&ca);
	return guid;
}

__declspec(dllexport)
int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
{
	umad_ca_t	ca;
	int			ret;

	ret = umad_get_ca(ca_name, &ca);
	if (ret != 0) {
		return ret;
	}

	memcpy(port, ca.ports[portnum - 1], sizeof(umad_port_t));

	port->pkeys = new uint16_t[ca.ports[portnum - 1]->pkeys_size];
	if (port->pkeys == NULL) {
		ret = -ENOMEM;
		goto out;
	}

	memcpy(port->pkeys, ca.ports[portnum - 1]->pkeys,
		   sizeof(uint16_t) * ca.ports[portnum - 1]->pkeys_size);
out:
	umad_release_ca(&ca);
	return ret;
}

__declspec(dllexport)
int umad_release_port(umad_port_t *port)
{
	delete port->pkeys;
	return 0;
}

__declspec(dllexport)
int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
{
	return -EINVAL;
}

int umad_open_port(char *ca_name, int portnum)
{
	HRESULT	hr;
	int		portid;

	EnterCriticalSection(&crit_sec);
	for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
		if (ports[portid].prov == NULL) {
			break;
		}
	}

	if (portid == UMAD_MAX_PORTS) {
		portid = -ENOMEM;
		goto out;
	}

	ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
	if (ports[portid].overlap.hEvent == NULL) {
		portid = -ENOMEM;
		goto out;
	}

	hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
	if (FAILED(hr)) {
		CloseHandle(ports[portid].overlap.hEvent);
		portid = GetLastError() & 0x80000000;
		goto out;
	}

	ports[portid].dev_guid = umad_get_ca_guid(ca_name);
	ports[portid].port_num = (uint8_t) portnum;

out:
	LeaveCriticalSection(&crit_sec);
	return portid;
}

int umad_close_port(int portid)
{
	CloseHandle(ports[portid].overlap.hEvent);
	ports[portid].prov->Release();

	EnterCriticalSection(&crit_sec);
	ports[portid].prov = NULL;
	LeaveCriticalSection(&crit_sec);

	return 0;
}

__declspec(dllexport)
int	umad_set_grh_net(void *umad, void *mad_addr)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;
	struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;

	if (mad_addr) {
		mad->addr.grh_present = 1;
		memcpy(mad->addr.gid, addr->gid, 16);
		mad->addr.flow_label = addr->flow_label;
		mad->addr.hop_limit = addr->hop_limit;
		mad->addr.traffic_class = addr->traffic_class;
	} else
		mad->addr.grh_present = 0;
	return 0;
}

__declspec(dllexport)
int umad_set_grh(void *umad, void *mad_addr)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;
	struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;

	if (mad_addr) {
		mad->addr.grh_present = 1;
		memcpy(mad->addr.gid, addr->gid, 16);
		mad->addr.flow_label = htonl(addr->flow_label);
		mad->addr.hop_limit = addr->hop_limit;
		mad->addr.traffic_class = addr->traffic_class;
	} else
		mad->addr.grh_present = 0;
	return 0;
}

__declspec(dllexport)
int umad_set_pkey(void *umad, int pkey_index)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;

	mad->addr.pkey_index = (uint16_t) pkey_index;
	return 0;
}

__declspec(dllexport)
int umad_get_pkey(void *umad)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;

	return mad->addr.pkey_index;
}

__declspec(dllexport)
int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;

	mad->addr.qpn = htonl(dqp);
	mad->addr.lid = htons((uint16_t) dlid);
	mad->addr.qkey = htonl(qkey);
	mad->addr.sl = (uint8_t) sl;
	return 0;
}

__declspec(dllexport)
int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;

	mad->addr.qpn = dqp;
	mad->addr.lid = (uint16_t) dlid;
	mad->addr.qkey = qkey;
	mad->addr.sl = (uint8_t) sl;

	return 0;
}

__declspec(dllexport)
int umad_status(void *umad)
{
	return ((struct ib_user_mad *) umad)->status;
}

__declspec(dllexport)
ib_mad_addr_t *umad_get_mad_addr(void *umad)
{
	return &((struct ib_user_mad *) umad)->addr;
}

__declspec(dllexport)
void *umad_get_mad(void *umad)
{
	return ((struct ib_user_mad *)umad)->data;
}

__declspec(dllexport)
void *umad_alloc(int num, size_t size)
{
	return new uint8_t[num * size];
}

__declspec(dllexport)
void umad_free(void *umad)
{
	delete umad;
}

__declspec(dllexport)
size_t umad_size(void)
{
	return sizeof(struct ib_user_mad);
}

__declspec(dllexport)
int umad_send(int portid, int agentid, void *umad, int length,
			  int timeout_ms, int retries)
{
	struct ib_user_mad *mad = (struct ib_user_mad *) umad;
	HRESULT hr;

	mad->agent_id = agentid;
	mad->reserved_id = 0;

	mad->timeout_ms = (uint32_t) timeout_ms;
	mad->retries	= (uint32_t) retries;
	mad->length		= (uint32_t) length;

	hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
	if (FAILED(hr)) {
		return GetLastError();
	}

	return 0;
}

__declspec(dllexport)
int umad_recv(int portid, void *umad, int *length, int timeout_ms)
{
	WM_MAD		*mad = (WM_MAD *) umad;
	um_port_t	*port;
	HRESULT		hr;

	port = &ports[portid];
	hr = port->prov->Receive(mad, (size_t) length, &port->overlap);

	if (hr == ERROR_IO_PENDING) {
		if (port->pending && timeout_ms == 0) {
			do {
				hr = WaitForSingleObject(port->overlap.hEvent, 250);
			} while (hr == WAIT_TIMEOUT && port->pending);
		} else {
			hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
			if (hr == WAIT_TIMEOUT) {
				return -EWOULDBLOCK;
			}
		}
	}

	if (FAILED(hr)) {
		return -EIO;
	}

	if (mad->Length <= (UINT32) *length) {
		port->pending = FALSE;
	}

	*length = mad->Length;
	return 0;
}

__declspec(dllexport)
int umad_poll(int portid, int timeout_ms)
{
	WM_MAD		mad;
	um_port_t	*port;
	HRESULT		hr;

	port = &ports[portid];
	hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);

	if (hr == ERROR_IO_PENDING) {
		hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
		if (hr == WAIT_TIMEOUT) {
			return -ETIMEDOUT;
		}
	}

	if (FAILED(hr)) {
		return -EIO;
	}

	port->pending = TRUE;
	return 0;
}

__declspec(dllexport)
int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
					  uint8_t oui[3], long method_mask[16/sizeof(long)])
{
	WM_REGISTER reg;
	UINT64		id = 0;

	UNREFERENCED_PARAMETER(rmpp_version);

	reg.Guid = ports[portid].dev_guid;
	reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
	reg.Port = ports[portid].port_num;
	reg.Class = (uint8_t) mgmt_class;
	reg.Version = 1;
	memset(reg.Reserved, 0, sizeof(reg.Reserved));
	memcpy(reg.Oui, oui, sizeof(oui));
	memcpy(reg.Methods, method_mask, sizeof(reg.Methods));

	ports[portid].prov->Register(&reg, &id);

	return (int) id;
}

__declspec(dllexport)
int umad_register(int portid, int mgmt_class, int mgmt_version,
				  uint8_t rmpp_version, long method_mask[16/sizeof(long)])
{
	uint8_t oui[3];

	memset(oui, 0, 3);
	return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
}

__declspec(dllexport)
int umad_unregister(int portid, int agentid)
{
	ports[portid].pending = FALSE;
	return ports[portid].prov->Deregister((UINT64) agentid);
}

HANDLE umad_get_fd(int portid)
{
	return ports[portid].prov->GetFileHandle();
}

__declspec(dllexport)
int umad_debug(int level)
{
	UNREFERENCED_PARAMETER(level);
	return 0;
}

__declspec(dllexport)
void umad_addr_dump(ib_mad_addr_t *addr)
{
	UNREFERENCED_PARAMETER(addr);
}

__declspec(dllexport)
void umad_dump(void *umad)
{
	UNREFERENCED_PARAMETER(umad);
}
/*
 * Copyright (c) 2004-2007 Voltaire Inc.  All rights reserved.
 * Copyright (c) 2008 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenFabrics.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef UMAD_H
#define UMAD_H

#include <windows.h>
#include <iba\winmad.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Interfaces based on libibumad 1.2.0
 */

typedef unsigned __int8		uint8_t;
typedef unsigned __int16	uint16_t;
typedef unsigned __int32	uint32_t;
typedef unsigned __int64	uint64_t;

#define EIO			5
#define EWOULDBLOCK	11
#define ENOMEM		12
#define EINVAL		22
#define ETIMEDOUT	110

#define UMAD_ANY_PORT		1	// TODO: support this properly

// Allow casting to WM_MAD_AV
typedef struct ib_mad_addr
{
	uint32_t		qpn;
	uint32_t		qkey;
	uint32_t		flow_label;
	uint16_t		pkey_index;
	uint8_t			hop_limit;
	uint8_t			gid_index;
	uint8_t			gid[16];

	uint8_t			grh_present;
	uint8_t			reserved_grh;
	uint16_t		lid;
	uint8_t			sl;
	uint8_t			path_bits;
	uint8_t			reserved_rate;
	uint8_t			traffic_class;

}	ib_mad_addr_t;

// Allow casting to WM_MAD
#pragma warning(push)
#pragma warning(disable: 4200)
typedef struct ib_user_mad
{
	uint32_t		agent_id;
	uint32_t		reserved_id;
	ib_mad_addr_t	addr;

	uint32_t		status;
	uint32_t		timeout_ms;
	uint32_t		retries;
	uint32_t		length;
	uint8_t			data[0];

}	ib_user_mad_t;
#pragma warning(pop)

#define UMAD_CA_NAME_LEN	64
#define UMAD_CA_MAX_PORTS	10	/* 0 - 9 */
#define UMAD_MAX_PORTS		64

typedef struct umad_port
{
	char			ca_name[UMAD_CA_NAME_LEN];
	int				portnum;
	unsigned		base_lid;
	unsigned		lmc;
	unsigned		sm_lid;
	unsigned		sm_sl;
	unsigned		state;
	unsigned		phys_state;
	unsigned		rate;
	uint64_t		capmask;
	uint64_t		gid_prefix;
	uint64_t		port_guid;
	unsigned		pkeys_size;
	uint16_t		*pkeys;

}	umad_port_t;

typedef struct umad_ca
{
	char			ca_name[UMAD_CA_NAME_LEN];
	unsigned		node_type;
	int				numports;
	char			fw_ver[20];
	char			ca_type[40];
	char			hw_ver[20];
	uint64_t		node_guid;
	uint64_t		system_guid;
	umad_port_t		*ports[UMAD_CA_MAX_PORTS];

}	umad_ca_t;

__declspec(dllexport)
int umad_init(void);
__declspec(dllexport)
int umad_done(void);

__declspec(dllexport)
int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max);
__declspec(dllexport)
int	umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max);

__declspec(dllexport)
int	umad_get_ca(char *ca_name, umad_ca_t *ca);
__declspec(dllexport)
int	umad_release_ca(umad_ca_t *ca);

__declspec(dllexport)
int	umad_get_port(char *ca_name, int portnum, umad_port_t *port);
__declspec(dllexport)
int	umad_release_port(umad_port_t *port);

__declspec(dllexport)
int umad_get_issm_path(char *ca_name, int portnum, char path[], int max);

__declspec(dllexport)
int umad_open_port(char *ca_name, int portnum);
__declspec(dllexport)
int umad_close_port(int portid);

__declspec(dllexport)
void *umad_get_mad(void *umad);

__declspec(dllexport)
size_t umad_size(void);

__declspec(dllexport)
int umad_status(void *umad);

__declspec(dllexport)
ib_mad_addr_t	*umad_get_mad_addr(void *umad);
__declspec(dllexport)
int umad_set_grh_net(void *umad, void *mad_addr);
__declspec(dllexport)
int umad_set_grh(void *umad, void *mad_addr);
__declspec(dllexport)
int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey);
__declspec(dllexport)
int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey);
__declspec(dllexport)
int umad_set_pkey(void *umad, int pkey_index);
__declspec(dllexport)
int umad_get_pkey(void *umad);

__declspec(dllexport)
int umad_send(int portid, int agentid, void *umad, int length,
		  int timeout_ms, int retries);
__declspec(dllexport)
int umad_recv(int portid, void *umad, int *length, int timeout_ms);
__declspec(dllexport)
int umad_poll(int portid, int timeout_ms);
HANDLE umad_get_fd(int portid);

__declspec(dllexport)
int umad_register(int portid, int mgmt_class, int mgmt_version,
				  uint8_t rmpp_version, long method_mask[16/sizeof(long)]);
__declspec(dllexport)
int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
					  uint8_t oui[3], long method_mask[16/sizeof(long)]);
__declspec(dllexport)
int umad_unregister(int portid, int agentid);


__declspec(dllexport)
int umad_debug(int level);
__declspec(dllexport)
void umad_addr_dump(ib_mad_addr_t *addr);
__declspec(dllexport)
void umad_dump(void *umad);

__declspec(dllexport)
void *umad_alloc(int num, size_t size);

__declspec(dllexport)
void umad_free(void *umad);

#ifdef __cplusplus
}
#endif

#endif /* UMAD_H */





More information about the ofw mailing list