[ofa-general] [RFC] 3/5: IB ACM: libibacm

Sean Hefty sean.hefty at intel.com
Wed Sep 16 23:45:05 PDT 2009


Add an end-user library with simple interfaces for communicating
with the ib_acm service.

The linux and windows specific files for the library are simple and not
shown for this review

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---

ib_acm.h: defines library interfaces.
These are the end-user application interfaces to the ib acm.

/*
 * Copyright (c) 2009 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenFabrics.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if !defined(IB_ACM_H)
#define IB_ACM_H

#include <infiniband/verbs.h>

#if defined(_WIN32)
#define LIB_EXPORT __declspec(dllexport)
#else
#define LIB_EXPORT
#endif

#ifdef __cplusplus
extern "C" {
#endif

struct ib_acm_dev_addr
{
	uint64_t guid;
	uint16_t pkey_index;
	uint8_t  port_num;
	uint8_t  reserved[5];
};

struct ib_acm_resolve_data
{
	uint32_t reserved1;
	uint8_t  init_depth;
	uint8_t  resp_resources;
	uint8_t  packet_lifetime;
	uint8_t  mtu;
	uint8_t  reserved2[8];
};

/**
 * ib_acm_resolve_name - Resolve path data between the specified names.
 * Description:
 *   Discover path information, including identifying the local device,
 *   between the given the source and destination names.
 * Notes:
 *   The source and destination names should match entries in acm_addr.cfg
 *   configuration files on their respective systems.  Typically, the
 *   source and destination names will refer to system host names
 *   assigned to an Infiniband port.
 */
LIB_EXPORT
int ib_acm_resolve_name(char *src, char *dest,
	struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
	struct ib_acm_resolve_data *data);

/**
 * ib_acm_resolve_ip - Resolve path data between the specified addresses.
 * Description:
 *   Discover path information, including identifying the local device,
 *   between the given the source and destination addresses.
 * Notes:
 *   The source and destination addresses should match entries in acm_addr.cfg
 *   configuration files on their respective systems.  Typically, the
 *   source and destination addresses will refer to IP addresses assigned
 *   to an IPoIB instance.
 */
LIB_EXPORT
int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest,
	struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
	struct ib_acm_resolve_data *data);


#define IB_PATH_RECORD_REVERSIBLE 0x80

struct ib_path_record
{
	uint64_t        service_id;
	union ibv_gid   dgid;
	union ibv_gid   sgid;
	uint16_t        dlid;
	uint16_t        slid;
	uint32_t        flowlabel_hoplimit; /* resv-31:28 flow label-27:8 hop limit-7:0*/
	uint8_t         tclass;
	uint8_t         reversible_numpath; /* reversible-7:7 num path-6:0 */
	uint16_t        pkey;
	uint16_t        qosclass_sl;        /* qos class-15:4 sl-3:0 */
	uint8_t         mtu;                /* mtu selector-7:6 mtu-5:0 */
	uint8_t         rate;               /* rate selector-7:6 rate-5:0 */
	uint8_t         packetlifetime;     /* lifetime selector-7:6 lifetime-5:0 */
	uint8_t         preference;
	uint8_t         reserved[6];
};

/**
 * ib_acm_resolve_path - Resolve path data meeting specified restrictions
 * Description:
 *   Discover path information using the provided path record to
 *   restrict the discovery.
 * Notes:
 *   Uses the provided path record as input into an query for path
 *   information.  If successful, fills in any missing information.  The
 *   caller must provide at least the source and destination LIDs as input.
 */
LIB_EXPORT
int ib_acm_resolve_path(struct ib_path_record *path);

/**
 * ib_acm_query_path - Resolve path data meeting specified restrictions
 * Description:
 *   Queries the IB SA for a path record using the provided path record to
 *   restrict the query.
 * Notes:
 *   Uses the provided path record as input into an SA query for path
 *   information.  If successful, fills in any missing information.  The
 *   caller must provide at least the source and destination LIDs as input.
 *   Use of this call always results in sending a query to the IB SA.
 */
LIB_EXPORT
int ib_acm_query_path(struct ib_path_record *path);

/**
 * ib_acm_convert_to_path - Convert resolved path data to a path record
 * Description:
 *   Converts path information returned from resolving a host name or address
 *   to the format of an IB path record.
 */
LIB_EXPORT
int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
	struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,
	struct ib_path_record *path);

#ifdef __cplusplus
}
#endif

#endif /* IB_ACM_H */


acm.h: defines the client/application side protocol of the ib acm service
/*
 * Copyright (c) 2009 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenFabrics.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if !defined(ACM_H)
#define ACM_H

#include <infiniband/ib_acm.h>

#define ACM_VERSION 1

#define ACM_OP_MASK     0x0F
#define ACM_OP_RESOLVE  0x01
#define ACM_OP_QUERY    0x02
//#define ACM_OP_CM       0x03
//#define ACM_OP_ACK_REQ  0x40 /* optional ack is required */
#define ACM_OP_ACK      0x80

#define ACM_STATUS_SUCCESS         0
#define ACM_STATUS_ENOMEM          1
#define ACM_STATUS_EINVAL          2
#define ACM_STATUS_ENODATA         3
#define ACM_STATUS_ENOTCONN        5
#define ACM_STATUS_ETIMEDOUT       6
#define ACM_STATUS_ESRCADDR        7
#define ACM_STATUS_ESRCTYPE        8
#define ACM_STATUS_EDESTADDR       9
#define ACM_STATUS_EDESTTYPE      10

struct acm_hdr
{
	uint8_t  version;
	uint8_t  opcode;
	uint8_t  status;
	uint8_t  param;
	uint8_t  dest_type;
	uint8_t  src_type;
	uint8_t  reserved[2];
	uint64_t tid;
};

#define ACM_EP_TYPE_NAME        0x01
#define ACM_EP_TYPE_ADDRESS_IP  0x02
#define ACM_EP_TYPE_ADDRESS_IP6 0x03
#define ACM_EP_TYPE_DEVICE      0x10
#define ACM_EP_TYPE_AV          0x20

#define ACM_MAX_ADDRESS  32

union acm_ep_addr
{
	uint8_t                addr[ACM_MAX_ADDRESS];
	uint8_t                name[ACM_MAX_ADDRESS];
	struct ib_acm_dev_addr dev;
	struct ibv_ah_attr     av;
};

struct acm_resolve_msg
{
	struct acm_hdr             hdr;
	union  acm_ep_addr         src;
	union  acm_ep_addr         dest;
	struct ib_acm_resolve_data data;
};

//struct acm_cm_param
//{
//	uint32_t qpn;
//	uint8_t  init_depth;
//	uint8_t  resp_resources;
//	uint8_t  retry_cnt;
//	uint8_t  rnr_retry_cnt;
//	uint16_t src_port;
//	uint16_t dest_port;
//	uint8_t  reserved[4];
//};

//struct acm_cm_msg
//{
//	struct acm_hdr      hdr;
//	union  acm_ep_addr  src;
//	union  acm_ep_addr  dest;
//	struct acm_cm_param param;
//};

#define ACM_QUERY_PATH_RECORD  0x01
#define ACM_QUERY_SA           0x80

#define ACM_EP_TYPE_LID        0x01
#define ACM_EP_TYPE_GID        0x02

union acm_query_data
{
	struct ib_path_record  path;
};

struct acm_query_msg
{
	struct acm_hdr         hdr;
	union acm_query_data   data;
	uint8_t                reserved[16];
};

#define ACM_MSG_DATA_SIZE  80

struct acm_msg
{
	struct acm_hdr     hdr;
	uint8_t            data[ACM_MSG_DATA_SIZE];
};

#endif /* ACM_H */


libibacm.c: brain dead implementation that handles the client side protocol
of the ib acm for the user

/*
 * Copyright (c) 2009 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <osd.h>
#include <infiniband/ib_acm.h>
#include <infiniband/acm.h>
#include <stdio.h>

extern lock_t lock;
static SOCKET sock = INVALID_SOCKET;
static short server_port = 6125;
static int ready;

static int acm_init(void)
{
	struct sockaddr_in addr;
	int ret;

	ret = osd_init();
	if (ret)
		return ret;

	sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
	if (sock == INVALID_SOCKET) {
		ret = socket_errno();
		goto err1;
	}

	memset(&addr, 0, sizeof addr);
	addr.sin_family = AF_INET;
	addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
	addr.sin_port = htons(server_port);
	ret = connect(sock, (struct sockaddr *) &addr, sizeof(addr));
	if (ret)
		goto err2;

	ready = 1;
	return 0;

err2:
	closesocket(sock);
	sock = INVALID_SOCKET;
err1:
	osd_close();
	return ret;
}

void LIB_DESTRUCTOR acm_cleanup(void)
{
	if (sock != INVALID_SOCKET) {
		shutdown(sock, SHUT_RDWR);
		closesocket(sock);
	}
}

static int acm_resolve(uint8_t *src, uint8_t *dest, uint8_t type,
	struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
	struct ib_acm_resolve_data *data)
{
	struct acm_resolve_msg msg;
	int ret;

	lock_acquire(&lock);
	if (!ready && (ret = acm_init()))
		goto out;

	memset(&msg, 0, sizeof msg);
	msg.hdr.version = ACM_VERSION;
	msg.hdr.opcode = ACM_OP_RESOLVE;
	msg.hdr.dest_type = type;
	msg.hdr.src_type = type;

	switch (type) {
	case ACM_EP_TYPE_NAME:
		strncpy((char *) msg.src.name, (char *) src, ACM_MAX_ADDRESS);
		strncpy((char *) msg.dest.name, (char *) dest, ACM_MAX_ADDRESS);
		break;
	case ACM_EP_TYPE_ADDRESS_IP:
		memcpy(msg.src.addr, &((struct sockaddr_in *) src)->sin_addr, 4);
		memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 4);
		break;
	case ACM_EP_TYPE_ADDRESS_IP6:
		memcpy(msg.src.addr, &((struct sockaddr_in6 *) src)->sin6_addr, 16);
		memcpy(msg.dest.addr, &((struct sockaddr_in *) dest)->sin_addr, 16);
		break;
	case ACM_EP_TYPE_AV:
		memcpy(&msg.src.av, src, sizeof(msg.src.av));
		memcpy(&msg.dest.av, dest, sizeof(msg.dest.av));
		break;
	default:
		ret = -1;
		goto out;
	}
	
	ret = send(sock, (char *) &msg, sizeof msg, 0);
	if (ret != sizeof msg)
		goto out;

	ret = recv(sock, (char *) &msg, sizeof msg, 0);
	if (ret != sizeof msg)
		goto out;

	memcpy(dev_addr, &msg.src.dev, sizeof(*dev_addr));
	*ah = msg.dest.av;
	memcpy(data, &msg.data, sizeof(*data));
	ret = 0;

out:
	lock_release(&lock);
	return ret;
}

LIB_EXPORT
int ib_acm_resolve_name(char *src, char *dest,
	struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
	struct ib_acm_resolve_data *data)
{
	return acm_resolve((uint8_t *) src, (uint8_t *) dest,
		ACM_EP_TYPE_NAME, dev_addr, ah, data);
}

LIB_EXPORT
int ib_acm_resolve_ip(struct sockaddr *src, struct sockaddr *dest,
	struct ib_acm_dev_addr *dev_addr, struct ibv_ah_attr *ah,
	struct ib_acm_resolve_data *data)
{
	if (((struct sockaddr *) dest)->sa_family == AF_INET) {
		return acm_resolve((uint8_t *) src, (uint8_t *) dest,
			ACM_EP_TYPE_ADDRESS_IP, dev_addr, ah, data);
	} else {
		return acm_resolve((uint8_t *) src, (uint8_t *) dest,
			ACM_EP_TYPE_ADDRESS_IP6, dev_addr, ah, data);
	}
}

static int acm_query_path(struct ib_path_record *path, uint8_t query_sa)
{
	struct acm_query_msg msg;
	int ret;

	lock_acquire(&lock);
	if (!ready && (ret = acm_init()))
		goto out;

	memset(&msg, 0, sizeof msg);
	msg.hdr.version = ACM_VERSION;
	msg.hdr.opcode = ACM_OP_QUERY;
	msg.hdr.param = ACM_QUERY_PATH_RECORD | query_sa;

	if (path->dgid.global.interface_id || path->dgid.global.subnet_prefix) {
		msg.hdr.dest_type = ACM_EP_TYPE_GID;
	} else if (path->dlid) {
		msg.hdr.dest_type = ACM_EP_TYPE_LID;
	} else {
		ret = -1;
		goto out;
	}

	if (path->sgid.global.interface_id || path->sgid.global.subnet_prefix) {
		msg.hdr.src_type = ACM_EP_TYPE_GID;
	} else if (path->slid) {
		msg.hdr.src_type = ACM_EP_TYPE_LID;
	} else {
		ret = -1;
		goto out;
	}

	msg.data.path = *path;
	
	ret = send(sock, (char *) &msg, sizeof msg, 0);
	if (ret != sizeof msg)
		goto out;

	ret = recv(sock, (char *) &msg, sizeof msg, 0);
	if (ret != sizeof msg)
		goto out;

	*path = msg.data.path;
	ret = msg.hdr.status;

out:
	lock_release(&lock);
	return ret;
}

LIB_EXPORT
int ib_acm_query_path(struct ib_path_record *path)
{
	return acm_query_path(path, ACM_QUERY_SA);
}

LIB_EXPORT
int ib_acm_resolve_path(struct ib_path_record *path)
{
	return acm_query_path(path, 0);
}

static struct ibv_context *acm_open_device(uint64_t guid)
{
	struct ibv_device **dev_array;
	struct ibv_context *verbs = NULL;
	int i, cnt;

	dev_array = ibv_get_device_list(&cnt);
	if (!dev_array)
		return NULL;

	for (i = 0; i < cnt; i++) {
		if (guid == ibv_get_device_guid(dev_array[i])) {
			verbs = ibv_open_device(dev_array[i]);
			break;
		}
	}

	ibv_free_device_list(dev_array);
	return verbs;
}

LIB_EXPORT
int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
	struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,
	struct ib_path_record *path)
{
	struct ibv_context *verbs;
	struct ibv_port_attr attr;
	int ret;

	verbs = acm_open_device(dev_addr->guid);
	if (!verbs)
		return -1;

	if (ah->is_global) {
		path->dgid = ah->grh.dgid;
		ret = ibv_query_gid(verbs, dev_addr->port_num, ah->grh.sgid_index, &path->sgid);
		if (ret)
			goto out;

		path->flowlabel_hoplimit =
			htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit);
		path->tclass = ah->grh.traffic_class;
	}

	path->dlid = htons(ah->dlid);
	ret = ibv_query_port(verbs, dev_addr->port_num, &attr);
	if (ret)
		goto out;

	path->slid = htons(attr.lid | ah->src_path_bits);
	path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1;
	ret = ibv_query_pkey(verbs, dev_addr->port_num, dev_addr->pkey_index, &path->pkey);
	if (ret)
		goto out;

	path->pkey = htons(path->pkey);
	path->qosclass_sl = htons((uint16_t) ah->sl);
	path->mtu = (2 << 6) | data->mtu;
	path->rate = (2 << 6) | ah->static_rate;
	path->packetlifetime = (2 << 6) | data->packet_lifetime;

out:
	ibv_close_device(verbs);
	return ret;
}





More information about the general mailing list