[ofw] [RFC] 3/5: IB ACM: libibacm
Sean Hefty
sean.hefty at intel.com
Wed Sep 16 23:45:05 PDT 2009
Add an end-user library with simple interfaces for communicating
with the ib_acm service.
The linux and windows specific files for the library are simple and not
shown for this review
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
ib_acm.h: defines library interfaces.
These are the end-user application interfaces to the ib acm.
/*
* Copyright (c) 2009 Intel Corporation. All rights reserved.
*
* This software is available to you under the OpenFabrics.org BSD license
* below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#if !defined(IB_ACM_H)
#define IB_ACM_H
#include <infiniband/verbs.h>
#if defined(_WIN32)
#define LIB_EXPORT __declspec(dllexport)
#else
#define LIB_EXPORT
#endif
#ifdef __cplusplus
extern "C" {
#endif
struct ib_acm_dev_addr
{
uint64_t guid;
uint16_t pkey_index;
uint8_t port_num;
uint8_t reserved[5];
};
struct ib_acm_resolve_data
{
uint32_t reserved1;
uint8_t init_depth;
uint8_t resp_resources;
uint8_t packet_lifetime;
uint8_t mtu;
uint8_t reserved2[8];
};
/**
* ib_acm_resolve_name - Resolve path data between the specified names.
* Description:
* Discover path information, including identifying the local device,
* between the given the source and destination names.
* Notes:
* The source and destination names should match entries in acm_addr.cfg
* configuration files on their respective systems. Typically, the
* source and destination names will refer to system host names
* assigned to an Infiniband port.
*/
LIB_EXPORT
int ib_acm_resolve_name(char *src, char *dest,
struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
struct ib_acm_resolve_data *data);
/**
* ib_acm_resolve_ip - Resolve path data between the specified addresses.
* Description:
* Discover path information, including identifying the local device,
* between the given the source and destination addresses.
* Notes:
* The source and destination addresses should match entries in acm_addr.cfg
* configuration files on their respective systems. Typically, the
* source and destination addresses will refer to IP addresses assigned
* to an IPoIB instance.
*/
LIB_EXPORT
int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest,
struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
struct ib_acm_resolve_data *data);
#define IB_PATH_RECORD_REVERSIBLE 0x80
struct ib_path_record
{
uint64_t service_id;
union ibv_gid dgid;
union ibv_gid sgid;
uint16_t dlid;
uint16_t slid;
uint32_t flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/
uint8_t tclass;
uint8_t reversible_numpath; /* reversible-7:7 num path-6:0 */
uint16_t pkey;
uint16_t qosclass_sl; /* qos class-15:4 sl-3:0 */
uint8_t mtu; /* mtu selector-7:6 mtu-5:0 */
uint8_t rate; /* rate selector-7:6 rate-5:0 */
uint8_t packetlifetime; /* lifetime selector-7:6 lifetime-5:0 */
uint8_t preference;
uint8_t reserved[6];
};
/**
* ib_acm_resolve_path - Resolve path data meeting specified restrictions
* Description:
* Discover path information using the provided path record to
* restrict the discovery.
* Notes:
* Uses the provided path record as input into an query for path
* information. If successful, fills in any missing information. The
* caller must provide at least the source and destination LIDs as input.
*/
LIB_EXPORT
int ib_acm_resolve_path(struct ib_path_record *path);
/**
* ib_acm_query_path - Resolve path data meeting specified restrictions
* Description:
* Queries the IB SA for a path record using the provided path record to
* restrict the query.
* Notes:
* Uses the provided path record as input into an SA query for path
* information. If successful, fills in any missing information. The
* caller must provide at least the source and destination LIDs as input.
* Use of this call always results in sending a query to the IB SA.
*/
LIB_EXPORT
int ib_acm_query_path(struct ib_path_record *path);
/**
* ib_acm_convert_to_path - Convert resolved path data to a path record
* Description:
* Converts path information returned from resolving a host name or address
* to the format of an IB path record.
*/
LIB_EXPORT
int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,
struct ib_path_record *path);
#ifdef __cplusplus
}
#endif
#endif /* IB_ACM_H */
acm.h: defines the client/application side protocol of the ib acm service
/*
* Copyright (c) 2009 Intel Corporation. All rights reserved.
*
* This software is available to you under the OpenFabrics.org BSD license
* below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#if !defined(ACM_H)
#define ACM_H
#include <infiniband/ib_acm.h>
#define ACM_VERSION 1
#define ACM_OP_MASK 0x0F
#define ACM_OP_RESOLVE 0x01
#define ACM_OP_QUERY 0x02
//#define ACM_OP_CM 0x03
//#define ACM_OP_ACK_REQ 0x40 /* optional ack is required */
#define ACM_OP_ACK 0x80
#define ACM_STATUS_SUCCESS 0
#define ACM_STATUS_ENOMEM 1
#define ACM_STATUS_EINVAL 2
#define ACM_STATUS_ENODATA 3
#define ACM_STATUS_ENOTCONN 5
#define ACM_STATUS_ETIMEDOUT 6
#define ACM_STATUS_ESRCADDR 7
#define ACM_STATUS_ESRCTYPE 8
#define ACM_STATUS_EDESTADDR 9
#define ACM_STATUS_EDESTTYPE 10
struct acm_hdr
{
uint8_t version;
uint8_t opcode;
uint8_t status;
uint8_t param;
uint8_t dest_type;
uint8_t src_type;
uint8_t reserved[2];
uint64_t tid;
};
#define ACM_EP_TYPE_NAME 0x01
#define ACM_EP_TYPE_ADDRESS_IP 0x02
#define ACM_EP_TYPE_ADDRESS_IP6 0x03
#define ACM_EP_TYPE_DEVICE 0x10
#define ACM_EP_TYPE_AV 0x20
#define ACM_MAX_ADDRESS 32
union acm_ep_addr
{
uint8_t addr[ACM_MAX_ADDRESS];
uint8_t name[ACM_MAX_ADDRESS];
struct ib_acm_dev_addr dev;
struct ibv_ah_attr av;
};
struct acm_resolve_msg
{
struct acm_hdr hdr;
union acm_ep_addr src;
union acm_ep_addr dest;
struct ib_acm_resolve_data data;
};
//struct acm_cm_param
//{
// uint32_t qpn;
// uint8_t init_depth;
// uint8_t resp_resources;
// uint8_t retry_cnt;
// uint8_t rnr_retry_cnt;
// uint16_t src_port;
// uint16_t dest_port;
// uint8_t reserved[4];
//};
//struct acm_cm_msg
//{
// struct acm_hdr hdr;
// union acm_ep_addr src;
// union acm_ep_addr dest;
// struct acm_cm_param param;
//};
#define ACM_QUERY_PATH_RECORD 0x01
#define ACM_QUERY_SA 0x80
#define ACM_EP_TYPE_LID 0x01
#define ACM_EP_TYPE_GID 0x02
union acm_query_data
{
struct ib_path_record path;
};
struct acm_query_msg
{
struct acm_hdr hdr;
union acm_query_data data;
uint8_t reserved[16];
};
#define ACM_MSG_DATA_SIZE 80
struct acm_msg
{
struct acm_hdr hdr;
uint8_t data[ACM_MSG_DATA_SIZE];
};
#endif /* ACM_H */
libibacm.c: brain dead implementation that handles the client side protocol
of the ib acm for the user
/*
* Copyright (c) 2009 Intel Corporation. All rights reserved.
*
* This software is available to you under the OpenIB.org BSD license
* below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <osd.h>
#include <infiniband/ib_acm.h>
#include <infiniband/acm.h>
#include <stdio.h>
extern lock_t lock;
static SOCKET sock = INVALID_SOCKET;
static short server_port = 6125;
static int ready;
static int acm_init(void)
{
struct sockaddr_in addr;
int ret;
ret = osd_init();
if (ret)
return ret;
sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sock == INVALID_SOCKET) {
ret = socket_errno();
goto err1;
}
memset(&addr, 0, sizeof addr);
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = htons(server_port);
ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr));
if (ret)
goto err2;
ready = 1;
return 0;
err2:
closesocket(sock);
sock = INVALID_SOCKET;
err1:
osd_close();
return ret;
}
void LIB_DESTRUCTOR acm_cleanup(void)
{
if (sock != INVALID_SOCKET) {
shutdown(sock, SHUT_RDWR);
closesocket(sock);
}
}
static int acm_resolve(uint8_t *src, uint8_t *dest, uint8_t type,
struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
struct ib_acm_resolve_data *data)
{
struct acm_resolve_msg msg;
int ret;
lock_acquire(&lock);
if (!ready && (ret = acm_init()))
goto out;
memset(&msg, 0, sizeof msg);
msg.hdr.version = ACM_VERSION;
msg.hdr.opcode = ACM_OP_RESOLVE;
msg.hdr.dest_type = type;
msg.hdr.src_type = type;
switch (type) {
case ACM_EP_TYPE_NAME:
strncpy((char *) msg.src.name, (char *) src, ACM_MAX_ADDRESS);
strncpy((char *) msg.dest.name, (char *) dest, ACM_MAX_ADDRESS);
break;
case ACM_EP_TYPE_ADDRESS_IP:
memcpy(msg.src.addr, &((struct sockaddr_in *) src)->sin_addr, 4);
memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 4);
break;
case ACM_EP_TYPE_ADDRESS_IP6:
memcpy(msg.src.addr, &((struct sockaddr_in6 *) src)->sin6_addr, 16);
memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 16);
break;
case ACM_EP_TYPE_AV:
memcpy(&msg.src.av, src, sizeof(msg.src.av));
memcpy(&msg.dest.av, dest, sizeof(msg.dest.av));
break;
default:
ret = -1;
goto out;
}
ret = send(sock, (char *) &msg, sizeof msg, 0);
if (ret != sizeof msg)
goto out;
ret = recv(sock, (char *) &msg, sizeof msg, 0);
if (ret != sizeof msg)
goto out;
memcpy(dev_addr, &msg.src.dev, sizeof(*dev_addr));
*ah = msg.dest.av;
memcpy(data, &msg.data, sizeof(*data));
ret = 0;
out:
lock_release(&lock);
return ret;
}
LIB_EXPORT
int ib_acm_resolve_name(char *src, char *dest,
struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
struct ib_acm_resolve_data *data)
{
return acm_resolve((uint8_t *) src, (uint8_t *) dest,
ACM_EP_TYPE_NAME, dev_addr, ah, data);
}
LIB_EXPORT
int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest,
struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
struct ib_acm_resolve_data *data)
{
if (((struct sockaddr *) dest)->sa_family == AF_INET) {
return acm_resolve((uint8_t *) src, (uint8_t *) dest,
ACM_EP_TYPE_ADDRESS_IP, dev_addr, ah, data);
} else {
return acm_resolve((uint8_t *) src, (uint8_t *) dest,
ACM_EP_TYPE_ADDRESS_IP6, dev_addr, ah, data);
}
}
static int acm_query_path(struct ib_path_record *path, uint8_t query_sa)
{
struct acm_query_msg msg;
int ret;
lock_acquire(&lock);
if (!ready && (ret = acm_init()))
goto out;
memset(&msg, 0, sizeof msg);
msg.hdr.version = ACM_VERSION;
msg.hdr.opcode = ACM_OP_QUERY;
msg.hdr.param = ACM_QUERY_PATH_RECORD | query_sa;
if (path->dgid.global.interface_id || path->dgid.global.subnet_prefix) {
msg.hdr.dest_type = ACM_EP_TYPE_GID;
} else if (path->dlid) {
msg.hdr.dest_type = ACM_EP_TYPE_LID;
} else {
ret = -1;
goto out;
}
if (path->sgid.global.interface_id || path->sgid.global.subnet_prefix) {
msg.hdr.src_type = ACM_EP_TYPE_GID;
} else if (path->slid) {
msg.hdr.src_type = ACM_EP_TYPE_LID;
} else {
ret = -1;
goto out;
}
msg.data.path = *path;
ret = send(sock, (char *) &msg, sizeof msg, 0);
if (ret != sizeof msg)
goto out;
ret = recv(sock, (char *) &msg, sizeof msg, 0);
if (ret != sizeof msg)
goto out;
*path = msg.data.path;
ret = msg.hdr.status;
out:
lock_release(&lock);
return ret;
}
LIB_EXPORT
int ib_acm_query_path(struct ib_path_record *path)
{
return acm_query_path(path, ACM_QUERY_SA);
}
LIB_EXPORT
int ib_acm_resolve_path(struct ib_path_record *path)
{
return acm_query_path(path, 0);
}
static struct ibv_context *acm_open_device(uint64_t guid)
{
struct ibv_device **dev_array;
struct ibv_context *verbs = NULL;
int i, cnt;
dev_array = ibv_get_device_list(&cnt);
if (!dev_array)
return NULL;
for (i = 0; i < cnt; i++) {
if (guid == ibv_get_device_guid(dev_array[i])) {
verbs = ibv_open_device(dev_array[i]);
break;
}
}
ibv_free_device_list(dev_array);
return verbs;
}
LIB_EXPORT
int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,
struct ib_path_record *path)
{
struct ibv_context *verbs;
struct ibv_port_attr attr;
int ret;
verbs = acm_open_device(dev_addr->guid);
if (!verbs)
return -1;
if (ah->is_global) {
path->dgid = ah->grh.dgid;
ret = ibv_query_gid(verbs, dev_addr->port_num, ah->grh.sgid_index, &path->sgid);
if (ret)
goto out;
path->flowlabel_hoplimit =
htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit);
path->tclass = ah->grh.traffic_class;
}
path->dlid = htons(ah->dlid);
ret = ibv_query_port(verbs, dev_addr->port_num, &attr);
if (ret)
goto out;
path->slid = htons(attr.lid | ah->src_path_bits);
path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1;
ret = ibv_query_pkey(verbs, dev_addr->port_num, dev_addr->pkey_index, &path->pkey);
if (ret)
goto out;
path->pkey = htons(path->pkey);
path->qosclass_sl = htons((uint16_t) ah->sl);
path->mtu = (2 << 6) | data->mtu;
path->rate = (2 << 6) | ah->static_rate;
path->packetlifetime = (2 << 6) | data->packet_lifetime;
out:
ibv_close_device(verbs);
return ret;
}
More information about the ofw
mailing list