[ofw] [RFC] [PATCH 3/3] port of libibumad to windows
Sean Hefty
sean.hefty at intel.com
Tue Jul 1 19:02:15 PDT 2008
This ports the libibumad interfaces and basic functionality to Windows. It relies
on winmad for its MAD transport, and libibverbs for additional functionality
(e.g. obtaining CA and port attributes).
The intent is to simplify porting and maintaining the OFED Infiniband management
utilities (diags, opensm, other MAD libraries), such that the WinOF and OFED
stacks can provide similar functionality and be kept more in sync.
The implementation focuses only on what I'm hoping is crucial to running the
existing OFED applications. Debug related interfaces were not implemented.
Attention was given to ensure that performance using this interface would be
comparable to existing OFED and WinOF interfaces. This will need to be verified
through testing.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
/*
* Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#include <windows.h>
#include <stdio.h>
#include <infiniband/umad.h>
#include <infiniband/verbs.h>
#include "ibumad.h"
#define IB_OPENIB_OUI (0x001405)
#define UMAD_MAX_PKEYS 16
typedef struct um_port
{
IWMProvider *prov;
UINT64 dev_guid;
OVERLAPPED overlap;
BOOL pending;
UINT8 port_num;
} um_port_t;
CRITICAL_SECTION crit_sec;
um_port_t ports[UMAD_MAX_PORTS];
__declspec(dllexport)
int umad_init(void)
{
InitializeCriticalSection(&crit_sec);
return 0;
}
__declspec(dllexport)
int umad_done(void)
{
return 0;
}
__declspec(dllexport)
int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
{
struct ibv_device **list;
int cnt, i;
list = ibv_get_device_list(&cnt);
if (list == NULL) {
return 0;
}
for (i = 0; i < min(cnt, max); i++) {
strcpy(cas[i], ibv_get_device_name(list[i]));
}
ibv_free_device_list(list);
return i;
}
__declspec(dllexport)
int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
{
umad_ca_t ca;
int ports = 0, i;
if (umad_get_ca(ca_name, &ca) < 0)
return -1;
if (ca.numports + 1 > max) {
umad_release_ca(&ca);
return -ENOMEM;
}
for (i = 0; i <= ca.numports; i++)
portguids[ports++] = ca.ports[i]->port_guid;
umad_release_ca(&ca);
return ports;
}
static void umad_convert_ca_attr(umad_ca_t *ca, ibv_device_attr *attr)
{
ca->node_type = 1; // HCA
ca->numports = attr->phys_port_cnt;
strncpy(ca->fw_ver, attr->fw_ver, 20);
memset(ca->ca_type, 0, 40); // TODO: determine what this should be
sprintf(ca->hw_ver, "0x%x", attr->hw_ver);
ca->node_guid = attr->node_guid;
ca->system_guid = attr->sys_image_guid;
}
static int umad_query_port(struct ibv_context *context, umad_port_t *port)
{
ibv_port_attr attr;
ibv_gid gid;
int i, ret;
ret = ibv_query_port(context, (uint8_t) port->portnum, &attr);
if (ret != 0) {
return ret;
}
port->base_lid = attr.lid;
port->lmc = attr.lmc;
port->sm_lid = attr.sm_lid;
port->sm_sl = attr.sm_sl;
port->state = attr.state;
port->phys_state = attr.phys_state;
port->rate = attr.active_speed;
port->capmask = attr.port_cap_flags;
// Assume GID 0 contains port GUID and gid prefix
ret = ibv_query_gid(context, (uint8_t) port->portnum, 0, &gid);
if (ret != 0) {
return ret;
}
port->gid_prefix = gid.global.subnet_prefix;
port->port_guid = gid.global.interface_id;
port->pkeys_size = min(UMAD_MAX_PKEYS, attr.pkey_tbl_len);
for (i = 0; i < (int) port->pkeys_size; i++) {
ret = ibv_query_pkey(context,(uint8_t) port->portnum, i, &port->pkeys[i]);
if (ret != 0) {
return ret;
}
}
return 0;
}
__declspec(dllexport)
int umad_get_ca(char *ca_name, umad_ca_t *ca)
{
struct ibv_device **list;
struct ibv_context *context;
ibv_device_attr dev_attr;
int cnt, i, ret = 0;
uint8_t *ports;
size_t port_size;
list = ibv_get_device_list(&cnt);
if (list == NULL) {
return -ENOMEM;
}
for (i = 0; i < cnt; i++) {
if (!strcmp(ca_name, ibv_get_device_name(list[i]))) {
break;
}
}
if (i == cnt) {
ret = -EINVAL;
goto free;
}
context = ibv_open_device(list[i]);
if (context == NULL) {
ret = -ENOMEM;
goto free;
}
ret = ibv_query_device(context, &dev_attr);
if (ret != 0) {
goto close;
}
port_size = sizeof(umad_port_t) + sizeof(uint16_t) * UMAD_MAX_PKEYS;
ports = new uint8_t[port_size * dev_attr.phys_port_cnt];
if (ports == NULL) {
ret = -ENOMEM;
goto close;
}
strcpy(ca->ca_name, ca_name);
umad_convert_ca_attr(ca, &dev_attr);
for (i = 0; i < dev_attr.phys_port_cnt; i++, ports += port_size) {
ca->ports[i] = (umad_port_t *) ports;
strcpy(ca->ports[i]->ca_name, ca_name);
ca->ports[i]->portnum = i + 1;
ca->ports[i]->pkeys = (uint16_t *) (ports + sizeof(umad_port_t));
ret = umad_query_port(context, ca->ports[i]);
if (ret != 0) {
goto close;
}
}
close:
ibv_close_device(context);
free:
ibv_free_device_list(list);
if (ret != 0) {
delete ca;
}
return ret;
}
__declspec(dllexport)
int umad_release_ca(umad_ca_t *ca)
{
delete ca->ports[0];
return 0;
}
static uint64_t umad_get_ca_guid(char *ca_name)
{
umad_ca_t ca;
uint64_t guid;
int ret;
ret = umad_get_ca(ca_name, &ca);
if (ret != 0) {
return 0;
}
guid = ca.node_guid;
umad_release_ca(&ca);
return guid;
}
__declspec(dllexport)
int umad_get_port(char *ca_name, int portnum, umad_port_t *port)
{
umad_ca_t ca;
int ret;
ret = umad_get_ca(ca_name, &ca);
if (ret != 0) {
return ret;
}
memcpy(port, ca.ports[portnum - 1], sizeof(umad_port_t));
port->pkeys = new uint16_t[ca.ports[portnum - 1]->pkeys_size];
if (port->pkeys == NULL) {
ret = -ENOMEM;
goto out;
}
memcpy(port->pkeys, ca.ports[portnum - 1]->pkeys,
sizeof(uint16_t) * ca.ports[portnum - 1]->pkeys_size);
out:
umad_release_ca(&ca);
return ret;
}
__declspec(dllexport)
int umad_release_port(umad_port_t *port)
{
delete port->pkeys;
return 0;
}
__declspec(dllexport)
int umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
{
return -EINVAL;
}
int umad_open_port(char *ca_name, int portnum)
{
HRESULT hr;
int portid;
EnterCriticalSection(&crit_sec);
for (portid = 0; portid < UMAD_MAX_PORTS; portid++) {
if (ports[portid].prov == NULL) {
break;
}
}
if (portid == UMAD_MAX_PORTS) {
portid = -ENOMEM;
goto out;
}
ports[portid].overlap.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
if (ports[portid].overlap.hEvent == NULL) {
portid = -ENOMEM;
goto out;
}
hr = WmGetObject(IID_IWMProvider, (LPVOID*) &ports[portid].prov);
if (FAILED(hr)) {
CloseHandle(ports[portid].overlap.hEvent);
portid = GetLastError() & 0x80000000;
goto out;
}
ports[portid].dev_guid = umad_get_ca_guid(ca_name);
ports[portid].port_num = (uint8_t) portnum;
out:
LeaveCriticalSection(&crit_sec);
return portid;
}
int umad_close_port(int portid)
{
CloseHandle(ports[portid].overlap.hEvent);
ports[portid].prov->Release();
EnterCriticalSection(&crit_sec);
ports[portid].prov = NULL;
LeaveCriticalSection(&crit_sec);
return 0;
}
__declspec(dllexport)
int umad_set_grh_net(void *umad, void *mad_addr)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
if (mad_addr) {
mad->addr.grh_present = 1;
memcpy(mad->addr.gid, addr->gid, 16);
mad->addr.flow_label = addr->flow_label;
mad->addr.hop_limit = addr->hop_limit;
mad->addr.traffic_class = addr->traffic_class;
} else
mad->addr.grh_present = 0;
return 0;
}
__declspec(dllexport)
int umad_set_grh(void *umad, void *mad_addr)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
struct ib_mad_addr *addr = (struct ib_mad_addr *) mad_addr;
if (mad_addr) {
mad->addr.grh_present = 1;
memcpy(mad->addr.gid, addr->gid, 16);
mad->addr.flow_label = htonl(addr->flow_label);
mad->addr.hop_limit = addr->hop_limit;
mad->addr.traffic_class = addr->traffic_class;
} else
mad->addr.grh_present = 0;
return 0;
}
__declspec(dllexport)
int umad_set_pkey(void *umad, int pkey_index)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
mad->addr.pkey_index = (uint16_t) pkey_index;
return 0;
}
__declspec(dllexport)
int umad_get_pkey(void *umad)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
return mad->addr.pkey_index;
}
__declspec(dllexport)
int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
mad->addr.qpn = htonl(dqp);
mad->addr.lid = htons((uint16_t) dlid);
mad->addr.qkey = htonl(qkey);
mad->addr.sl = (uint8_t) sl;
return 0;
}
__declspec(dllexport)
int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
mad->addr.qpn = dqp;
mad->addr.lid = (uint16_t) dlid;
mad->addr.qkey = qkey;
mad->addr.sl = (uint8_t) sl;
return 0;
}
__declspec(dllexport)
int umad_status(void *umad)
{
return ((struct ib_user_mad *) umad)->status;
}
__declspec(dllexport)
ib_mad_addr_t *umad_get_mad_addr(void *umad)
{
return &((struct ib_user_mad *) umad)->addr;
}
__declspec(dllexport)
void *umad_get_mad(void *umad)
{
return ((struct ib_user_mad *)umad)->data;
}
__declspec(dllexport)
void *umad_alloc(int num, size_t size)
{
return new uint8_t[num * size];
}
__declspec(dllexport)
void umad_free(void *umad)
{
delete umad;
}
__declspec(dllexport)
size_t umad_size(void)
{
return sizeof(struct ib_user_mad);
}
__declspec(dllexport)
int umad_send(int portid, int agentid, void *umad, int length,
int timeout_ms, int retries)
{
struct ib_user_mad *mad = (struct ib_user_mad *) umad;
HRESULT hr;
mad->agent_id = agentid;
mad->reserved_id = 0;
mad->timeout_ms = (uint32_t) timeout_ms;
mad->retries = (uint32_t) retries;
mad->length = (uint32_t) length;
hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
if (FAILED(hr)) {
return GetLastError();
}
return 0;
}
__declspec(dllexport)
int umad_recv(int portid, void *umad, int *length, int timeout_ms)
{
WM_MAD *mad = (WM_MAD *) umad;
um_port_t *port;
HRESULT hr;
port = &ports[portid];
hr = port->prov->Receive(mad, (size_t) length, &port->overlap);
if (hr == ERROR_IO_PENDING) {
if (port->pending && timeout_ms == 0) {
do {
hr = WaitForSingleObject(port->overlap.hEvent, 250);
} while (hr == WAIT_TIMEOUT && port->pending);
} else {
hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
if (hr == WAIT_TIMEOUT) {
return -EWOULDBLOCK;
}
}
}
if (FAILED(hr)) {
return -EIO;
}
if (mad->Length <= (UINT32) *length) {
port->pending = FALSE;
}
*length = mad->Length;
return 0;
}
__declspec(dllexport)
int umad_poll(int portid, int timeout_ms)
{
WM_MAD mad;
um_port_t *port;
HRESULT hr;
port = &ports[portid];
hr = port->prov->Receive(&mad, sizeof mad, &port->overlap);
if (hr == ERROR_IO_PENDING) {
hr = WaitForSingleObject(port->overlap.hEvent, (DWORD) timeout_ms);
if (hr == WAIT_TIMEOUT) {
return -ETIMEDOUT;
}
}
if (FAILED(hr)) {
return -EIO;
}
port->pending = TRUE;
return 0;
}
__declspec(dllexport)
int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
uint8_t oui[3], long method_mask[16/sizeof(long)])
{
WM_REGISTER reg;
UINT64 id = 0;
UNREFERENCED_PARAMETER(rmpp_version);
reg.Guid = ports[portid].dev_guid;
reg.Qpn = (mgmt_class == 0x01 || mgmt_class == 0x81) ? 0 : htonl(1);
reg.Port = ports[portid].port_num;
reg.Class = (uint8_t) mgmt_class;
reg.Version = 1;
memset(reg.Reserved, 0, sizeof(reg.Reserved));
memcpy(reg.Oui, oui, sizeof(oui));
memcpy(reg.Methods, method_mask, sizeof(reg.Methods));
ports[portid].prov->Register(®, &id);
return (int) id;
}
__declspec(dllexport)
int umad_register(int portid, int mgmt_class, int mgmt_version,
uint8_t rmpp_version, long method_mask[16/sizeof(long)])
{
uint8_t oui[3];
memset(oui, 0, 3);
return umad_register_oui(portid, mgmt_class, rmpp_version, oui, method_mask);
}
__declspec(dllexport)
int umad_unregister(int portid, int agentid)
{
ports[portid].pending = FALSE;
return ports[portid].prov->Deregister((UINT64) agentid);
}
HANDLE umad_get_fd(int portid)
{
return ports[portid].prov->GetFileHandle();
}
__declspec(dllexport)
int umad_debug(int level)
{
UNREFERENCED_PARAMETER(level);
return 0;
}
__declspec(dllexport)
void umad_addr_dump(ib_mad_addr_t *addr)
{
UNREFERENCED_PARAMETER(addr);
}
__declspec(dllexport)
void umad_dump(void *umad)
{
UNREFERENCED_PARAMETER(umad);
}
/*
* Copyright (c) 2004-2007 Voltaire Inc. All rights reserved.
* Copyright (c) 2008 Intel Corporation. All rights reserved.
*
* This software is available to you under the OpenFabrics.org BSD license
* below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#pragma once
#ifndef UMAD_H
#define UMAD_H
#include <windows.h>
#include <iba\winmad.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* Interfaces based on libibumad 1.2.0
*/
typedef unsigned __int8 uint8_t;
typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t;
typedef unsigned __int64 uint64_t;
#define EIO 5
#define EWOULDBLOCK 11
#define ENOMEM 12
#define EINVAL 22
#define ETIMEDOUT 110
#define UMAD_ANY_PORT 1 // TODO: support this properly
// Allow casting to WM_MAD_AV
typedef struct ib_mad_addr
{
uint32_t qpn;
uint32_t qkey;
uint32_t flow_label;
uint16_t pkey_index;
uint8_t hop_limit;
uint8_t gid_index;
uint8_t gid[16];
uint8_t grh_present;
uint8_t reserved_grh;
uint16_t lid;
uint8_t sl;
uint8_t path_bits;
uint8_t reserved_rate;
uint8_t traffic_class;
} ib_mad_addr_t;
// Allow casting to WM_MAD
#pragma warning(push)
#pragma warning(disable: 4200)
typedef struct ib_user_mad
{
uint32_t agent_id;
uint32_t reserved_id;
ib_mad_addr_t addr;
uint32_t status;
uint32_t timeout_ms;
uint32_t retries;
uint32_t length;
uint8_t data[0];
} ib_user_mad_t;
#pragma warning(pop)
#define UMAD_CA_NAME_LEN 64
#define UMAD_CA_MAX_PORTS 10 /* 0 - 9 */
#define UMAD_MAX_PORTS 64
typedef struct umad_port
{
char ca_name[UMAD_CA_NAME_LEN];
int portnum;
unsigned base_lid;
unsigned lmc;
unsigned sm_lid;
unsigned sm_sl;
unsigned state;
unsigned phys_state;
unsigned rate;
uint64_t capmask;
uint64_t gid_prefix;
uint64_t port_guid;
unsigned pkeys_size;
uint16_t *pkeys;
} umad_port_t;
typedef struct umad_ca
{
char ca_name[UMAD_CA_NAME_LEN];
unsigned node_type;
int numports;
char fw_ver[20];
char ca_type[40];
char hw_ver[20];
uint64_t node_guid;
uint64_t system_guid;
umad_port_t *ports[UMAD_CA_MAX_PORTS];
} umad_ca_t;
__declspec(dllexport)
int umad_init(void);
__declspec(dllexport)
int umad_done(void);
__declspec(dllexport)
int umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max);
__declspec(dllexport)
int umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max);
__declspec(dllexport)
int umad_get_ca(char *ca_name, umad_ca_t *ca);
__declspec(dllexport)
int umad_release_ca(umad_ca_t *ca);
__declspec(dllexport)
int umad_get_port(char *ca_name, int portnum, umad_port_t *port);
__declspec(dllexport)
int umad_release_port(umad_port_t *port);
__declspec(dllexport)
int umad_get_issm_path(char *ca_name, int portnum, char path[], int max);
__declspec(dllexport)
int umad_open_port(char *ca_name, int portnum);
__declspec(dllexport)
int umad_close_port(int portid);
__declspec(dllexport)
void *umad_get_mad(void *umad);
__declspec(dllexport)
size_t umad_size(void);
__declspec(dllexport)
int umad_status(void *umad);
__declspec(dllexport)
ib_mad_addr_t *umad_get_mad_addr(void *umad);
__declspec(dllexport)
int umad_set_grh_net(void *umad, void *mad_addr);
__declspec(dllexport)
int umad_set_grh(void *umad, void *mad_addr);
__declspec(dllexport)
int umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey);
__declspec(dllexport)
int umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey);
__declspec(dllexport)
int umad_set_pkey(void *umad, int pkey_index);
__declspec(dllexport)
int umad_get_pkey(void *umad);
__declspec(dllexport)
int umad_send(int portid, int agentid, void *umad, int length,
int timeout_ms, int retries);
__declspec(dllexport)
int umad_recv(int portid, void *umad, int *length, int timeout_ms);
__declspec(dllexport)
int umad_poll(int portid, int timeout_ms);
HANDLE umad_get_fd(int portid);
__declspec(dllexport)
int umad_register(int portid, int mgmt_class, int mgmt_version,
uint8_t rmpp_version, long method_mask[16/sizeof(long)]);
__declspec(dllexport)
int umad_register_oui(int portid, int mgmt_class, uint8_t rmpp_version,
uint8_t oui[3], long method_mask[16/sizeof(long)]);
__declspec(dllexport)
int umad_unregister(int portid, int agentid);
__declspec(dllexport)
int umad_debug(int level);
__declspec(dllexport)
void umad_addr_dump(ib_mad_addr_t *addr);
__declspec(dllexport)
void umad_dump(void *umad);
__declspec(dllexport)
void *umad_alloc(int num, size_t size);
__declspec(dllexport)
void umad_free(void *umad);
#ifdef __cplusplus
}
#endif
#endif /* UMAD_H */
More information about the ofw
mailing list