[ofw] [RFC] ib cm: export CM only interface

Sean Hefty sean.hefty at intel.com
Mon Nov 17 11:46:18 PST 2008


This is for comments only - no testing has been done.

The following exports a CM only interface from IBAL.  This allows users to manage their QP states and avoid IBAL object
management overhead.  The interface sits over the existing 'CEP' interface to minimizes code changes, but defines
cleaner structures.
 
The interface is a direct call, callback interface.  It avoids thread context switching to better support an IOCTL based
consumer.  The interface will immediately be used by WinVerbs for RDMA CM support, and may later be used by a libibcm
compatibility library.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
/*
 * Copyright (c) 2008 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef _ib_cm_ifc_h_
#define _ib_cm_ifc_h_

#include <initguid.h>
#include <iba/ib_al_ifc.h>
#include <iba/ib_types.h>
#include <iba/ib_al.h>

typedef struct _iba_cm_id
{
	void	*context;
	void	(*callback)(struct _iba_cm_id *p_id, ib_mad_t *p_mad);
	net32_t	cid;

}	iba_cm_id;

typedef struct _iba_cm_req
{
	ib_net64_t					service_id;

	ib_path_rec_t				*p_primary_path;
	ib_path_rec_t				*p_alt_path;

	net32_t						qpn;
	ib_qp_type_t				qp_type;

	void						*p_pdata;
	uint8_t						pdata_len;

	uint8_t						max_cm_retries;
	uint8_t						resp_res;
	uint8_t						init_depth;
	uint8_t						remote_resp_timeout;
	uint8_t						flow_ctrl;
	uint8_t						local_resp_timeout;
	uint8_t						rnr_nak_timeout;
	uint8_t						rnr_retry_cnt;
	uint8_t						retry_cnt;

}	iba_cm_req;

typedef struct _iba_cm_rep
{
	net32_t						qpn;
	ib_qp_type_t				qp_type;

	void						*p_pdata;
	uint8_t						pdata_len;

	ib_access_t					access_ctrl;
	uint32_t					sq_depth;
	uint32_t					rq_depth;

	ib_cm_failover_t			failover_accepted;
	uint8_t						init_depth;
	uint8_t						target_ack_delay;
	uint8_t						flow_ctrl;
	uint8_t						rnr_nak_timeout;
	uint8_t						rnr_retry_cnt;

}	iba_cm_rep;

typedef struct _iba_cm_lap
{
	ib_path_rec_t				*p_alt_path;
	void						*p_pdata;
	uint8_t						pdata_len;
	uint8_t						remote_resp_timeout;

}	iba_cm_lap;

typedef struct _iba_cm_apr
{
	void						*p_pdata;
	ib_apr_info_t				*p_info;
	uint8_t						pdata_len;
	uint8_t						info_length;
	ib_apr_status_t				status;

}	iba_cm_apr;

typedef struct _iba_cm_sidr_req
{
	ib_net64_t					service_id;
	ib_path_rec_t				*p_path;
	void						*p_pdata;
	uint8_t						pdata_len;
	uint8_t						max_cm_retries;
	uint32_t					timeout;

}	iba_cm_sidr_req;

typedef struct _iba_cm_sidr_rep
{
	net32_t						qpn;
	net32_t						qkey;
	void						*p_pdata;
	void						*p_info;
	uint8_t						pdata_len;
	uint8_t						info_len;
	ib_sidr_status_t			status;

}	iba_cm_sidr_rep;

typedef struct _iba_cm_interface
{
	NTSTATUS		(*create_id)(void (*callback)(iba_cm_id *p_id, ib_mad_t *p_mad),
								 void *context, iba_cm_id **pp_id);
	void			(*destroy_id)(iba_cm_id *p_id);

	NTSTATUS		(*listen)(iba_cm_id *p_id, net64_t service_id, void *p_compare_buf,
							  uint8_t compare_len, uint8_t compare_offset);

	NTSTATUS		(*send_req)(iba_cm_id *p_id, iba_cm_req *p_req);
	NTSTATUS		(*send_rep)(iba_cm_id *p_id, iba_cm_rep *p_rep);
	NTSTATUS		(*send_rtu)(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len);

	NTSTATUS		(*send_dreq)(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len);
	NTSTATUS		(*send_drep)(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len);

	NTSTATUS		(*send_rej)(iba_cm_id *p_id, ib_rej_status_t status,
								void *p_ari, uint8_t ari_len,
								void *p_pdata, uint8_t pdata_len);
	NTSTATUS		(*send_mra)(iba_cm_id *p_id, uint8_t service_timeout,
								void *p_pdata, uint8_t pdata_len);

	NTSTATUS		(*send_lap)(iba_cm_id *p_id, iba_cm_lap *p_lap);
	NTSTATUS		(*send_apr)(iba_cm_id *p_id, iba_cm_apr *p_apr);

	NTSTATUS		(*send_sidr_req)(iba_cm_id *p_id, iba_cm_sidr_req *p_req);
	NTSTATUS		(*send_sidr_rep)(iba_cm_id *p_id, iba_cm_sidr_rep *p_rep);

	NTSTATUS		(*get_qp_attr)(iba_cm_id *p_id, ib_qp_state_t state,
								   ib_qp_mod_t *p_attr);

	NTSTATUS		(*migrate)(iba_cm_id *p_id);
	NTSTATUS		(*established)(iba_cm_id *p_id);

}	iba_cm_interface;

static inline USHORT IbaCmVersion(UINT8 Major, UINT8 Minor)
{
	return ((USHORT) Major << 8) | ((USHORT) Minor);
}

static inline UINT8 IbaCmVersionMajor(USHORT Version)
{
	return (UINT8) (Version >> 8);
}

static inline UINT8 IbaCmVersionMinor(USHORT Version)
{
	return (UINT8) Version;
}

// {EACC1466-BB2D-4478-B5BE-40EDF7EE08AB}
DEFINE_GUID(GUID_INFINIBAND_INTERFACE_CM, 0xeacc1466, 0xbb2d, 0x4478,
			0xb5, 0xbe, 0x40, 0xed, 0xf7, 0xee, 0x8, 0xab);

typedef struct _INFINIBAND_INTERFACE_CM
{
	INTERFACE			InterfaceHeader;
	iba_cm_interface	CM;

}	INFINIBAND_INTERFACE_CM;

#endif // _ib_cm_ifc_h_

/*
 * Copyright (c) 2008 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <iba/ib_cm_ifc.h>
#include "al_cm_cep.h"
#include "al_mgr.h"

static NTSTATUS
cm_create_id(void (*callback)(iba_cm_id *p_id, ib_mad_t *p_mad),
			 void *context, iba_cm_id **pp_id);

static NTSTATUS
convert_ib_status(ib_api_status_t status)
{
	switch (status) {
	case IB_SUCCESS:
		return STATUS_SUCCESS;
	case IB_INSUFFICIENT_RESOURCES:
		return STATUS_INSUFFICIENT_RESOURCES;
	case IB_INSUFFICIENT_MEMORY:
		return STATUS_NO_MEMORY;
	case IB_INVALID_PARAMETER:
	case IB_INVALID_SETTING:
		return STATUS_INVALID_PARAMETER;
	case IB_NOT_FOUND:
		return STATUS_NOT_FOUND;
	default:
		return STATUS_UNSUCCESSFUL;
	}
}

static void
cm_cep_handler(const ib_al_handle_t h_al, const net32_t cid)
{
	void				*context;
	net32_t				new_cid;
	ib_mad_element_t	*mad;
	iba_cm_id			*id, *listen_id;

	while (al_cep_poll(h_al, cid, &context, &new_cid, &mad) == IB_SUCCESS) {

		if (new_cid == AL_INVALID_CID) {
			id = (iba_cm_id *) context;
		} else {
			listen_id = (iba_cm_id *) context;

			id = ExAllocatePoolWithTag(NonPagedPool, sizeof(iba_cm_id), 'mcbi');
			if (id == NULL) {
				al_destroy_cep(gh_al, &new_cid, FALSE);
				ib_put_mad(mad);
				continue;
			}

			id->context = listen_id;
			id->callback = listen_id->callback;
			id->cid = new_cid;
		}

		id->callback(id, mad->p_mad_buf);
		ib_put_mad(mad);
	}
}

static NTSTATUS
cm_create_id(void (*callback)(iba_cm_id *p_id, ib_mad_t *p_mad),
			 void *context, iba_cm_id **pp_id)
{
	iba_cm_id		*id;
	ib_api_status_t	ib_status;

	id = ExAllocatePoolWithTag(NonPagedPool, sizeof(iba_cm_id), 'mcbi');
	if (id == NULL) {
		return STATUS_NO_MEMORY;
	}

	id->callback = callback;
	id->context = context;

	ib_status = al_create_cep(gh_al, cm_cep_handler, id, NULL, &id->cid);
	if (ib_status != IB_SUCCESS) {
		ExFreePool(id);
		return convert_ib_status(ib_status);
	}

	*pp_id = id;
	return STATUS_SUCCESS;
}

static void
cm_destroy_id(iba_cm_id *p_id)
{
	al_destroy_cep(gh_al, &p_id->cid, FALSE);
	ExFreePool(p_id);
}

static NTSTATUS
cm_listen(iba_cm_id *p_id, net64_t service_id, void *p_compare_buf,
		  uint8_t compare_len, uint8_t compare_offset)
{
	ib_cep_listen_t info;
	ib_api_status_t	ib_status;

	info.svc_id = service_id;
	info.port_guid = IB_ALL_PORTS;
	info.p_cmp_buf = p_compare_buf;
	info.cmp_len = compare_len;
	info.cmp_offset = compare_offset;
	
	ib_status = al_cep_listen(gh_al, p_id->cid, &info);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_req(iba_cm_id *p_id, iba_cm_req *p_req)
{
	ib_cm_req_t		req;
	ib_qp_mod_t		attr;
	ib_api_status_t ib_status;
	
	RtlZeroMemory(&req, sizeof req);
	req.svc_id = p_req->service_id;
	req.max_cm_retries = p_req->max_cm_retries;
	req.p_primary_path = p_req->p_primary_path;
	req.p_req_pdata = p_req->p_pdata;
	req.req_length = p_req->pdata_len;
	req.qp_type = p_req->qp_type;
	req.resp_res = p_req->resp_res;
	req.init_depth = p_req->init_depth;
	req.remote_resp_timeout = p_req->remote_resp_timeout;
	req.flow_ctrl = p_req->flow_ctrl;
	req.local_resp_timeout = p_req->local_resp_timeout;
	req.rnr_nak_timeout = p_req->rnr_nak_timeout;
	req.rnr_retry_cnt = p_req->rnr_retry_cnt;
	req.retry_cnt = p_req->retry_cnt;
	req.p_alt_path = p_req->p_alt_path;

	ib_status = kal_cep_pre_req(gh_al, p_id->cid, &req, p_req->qpn, &attr);
	if (ib_status != IB_SUCCESS) {
		return convert_ib_status(ib_status);
	}

	ib_status = al_cep_send_req(gh_al, p_id->cid);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_rep(iba_cm_id *p_id, iba_cm_rep *p_rep)
{
	ib_cm_rep_t		rep;
	ib_qp_mod_t		attr;
	net32_t			cid;
	ib_api_status_t ib_status;

	RtlZeroMemory(&rep, sizeof rep);
	rep.p_rep_pdata = p_rep->p_pdata;
	rep.rep_length = p_rep->pdata_len;
	rep.access_ctrl = p_rep->access_ctrl;
	rep.sq_depth = p_rep->sq_depth;
	rep.rq_depth = p_rep->rq_depth;
	rep.init_depth = p_rep->init_depth;
	rep.target_ack_delay = p_rep->target_ack_delay;
	rep.failover_accepted = p_rep->failover_accepted;
	rep.flow_ctrl = p_rep->flow_ctrl;
	rep.rnr_nak_timeout = p_rep->rnr_nak_timeout;
	rep.rnr_retry_cnt = p_rep->rnr_retry_cnt;

	ib_status = kal_cep_pre_rep(gh_al, p_id->cid, cm_cep_handler, p_id, NULL,
								&rep, p_rep->qpn, &cid, &attr);
	if (ib_status != IB_SUCCESS) {
		return convert_ib_status(ib_status);
	}

	ib_status = al_cep_send_rep(gh_al, p_id->cid);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_rtu(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_rtu(gh_al, p_id->cid, p_pdata, pdata_len);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_dreq(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_dreq(gh_al, p_id->cid, p_pdata, pdata_len);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_drep(iba_cm_id *p_id, void *p_pdata, uint8_t pdata_len)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_drep(gh_al, p_id->cid, p_pdata, pdata_len);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_rej(iba_cm_id *p_id, ib_rej_status_t status,
			void *p_ari, uint8_t ari_len,
			void *p_pdata, uint8_t pdata_len)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_rej(gh_al, p_id->cid, status, p_ari, ari_len,
						   p_pdata, pdata_len);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_mra(iba_cm_id *p_id, uint8_t service_timeout,
			void *p_pdata, uint8_t pdata_len)
{
	ib_cm_mra_t		mra;
	ib_api_status_t ib_status;

	mra.svc_timeout = service_timeout;
	mra.p_mra_pdata = p_pdata;
	mra.mra_length = pdata_len;

	ib_status = al_cep_mra(gh_al, p_id->cid, &mra);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_lap(iba_cm_id *p_id, iba_cm_lap *p_lap)
{
	ib_cm_lap_t		lap;
	ib_api_status_t	ib_status;

	RtlZeroMemory(&lap, sizeof lap);
	lap.p_lap_pdata = p_lap->p_pdata;
	lap.lap_length = p_lap->pdata_len;
	lap.remote_resp_timeout = p_lap->remote_resp_timeout;
	lap.p_alt_path = p_lap->p_alt_path;

	ib_status = al_cep_lap(gh_al, p_id->cid, &lap);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_apr(iba_cm_id *p_id, iba_cm_apr *p_apr)
{
	ib_cm_apr_t		apr;
	ib_qp_mod_t		attr;
	ib_api_status_t	ib_status;

	RtlZeroMemory(&apr, sizeof apr);
	apr.p_apr_pdata = p_apr->p_pdata;
	apr.apr_length = p_apr->pdata_len;
	apr.apr_status = p_apr->status;
	apr.info_length = p_apr->info_length;
	apr.p_info = p_apr->p_info;

	ib_status = al_cep_pre_apr(gh_al, p_id->cid, &apr, &attr);
	if (ib_status != IB_SUCCESS) {
		return convert_ib_status(ib_status);
	}

	ib_status = al_cep_send_apr(gh_al, p_id->cid);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_send_sidr_req(iba_cm_id *p_id, iba_cm_sidr_req *p_req)
{
	UNUSED_PARAM(p_id);
	UNUSED_PARAM(p_req);

	return STATUS_NOT_SUPPORTED;
}

static NTSTATUS
cm_send_sidr_rep(iba_cm_id *p_id, iba_cm_sidr_rep *p_rep)
{
	UNUSED_PARAM(p_id);
	UNUSED_PARAM(p_rep);

	return STATUS_NOT_SUPPORTED;
}

static NTSTATUS
cm_get_qp_attr(iba_cm_id *p_id, ib_qp_state_t state, ib_qp_mod_t *p_attr)
{
	ib_api_status_t	ib_status;

	switch (state) {
	case IB_QPS_INIT:
		ib_status = al_cep_get_init_attr(gh_al, p_id->cid, p_attr);
		break;
	case IB_QPS_RTR:
		ib_status = al_cep_get_rtr_attr(gh_al, p_id->cid, p_attr);
		break;
	case IB_QPS_RTS:
		ib_status = al_cep_get_rts_attr(gh_al, p_id->cid, p_attr);
		break;
	default:
		return STATUS_INVALID_PARAMETER;
	}

	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_migrate(iba_cm_id *p_id)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_migrate(gh_al, p_id->cid);
	return convert_ib_status(ib_status);
}

static NTSTATUS
cm_established(iba_cm_id *p_id)
{
	ib_api_status_t ib_status;

	ib_status = al_cep_established(gh_al, p_id->cid);
	return convert_ib_status(ib_status);
}

void cm_get_interface(iba_cm_interface *p_ifc)
{
	p_ifc->create_id = cm_create_id;
	p_ifc->destroy_id = cm_destroy_id;
	p_ifc->listen = cm_listen;
	p_ifc->send_req = cm_send_req;
	p_ifc->send_rep = cm_send_rep;
	p_ifc->send_rtu = cm_send_rtu;
	p_ifc->send_dreq = cm_send_dreq;
	p_ifc->send_drep = cm_send_drep;
	p_ifc->send_rej = cm_send_rej;
	p_ifc->send_mra = cm_send_mra;
	p_ifc->send_lap = cm_send_lap;
	p_ifc->send_apr = cm_send_apr;
	p_ifc->send_sidr_req = cm_send_sidr_req;
	p_ifc->send_sidr_rep = cm_send_sidr_rep;
	p_ifc->get_qp_attr = cm_get_qp_attr;
	p_ifc->migrate = cm_migrate;
	p_ifc->established = cm_establish;
}





More information about the ofw mailing list