[ofw] [ND 5/7] ND: endpoint APIs

Mon Apr 6 10:23:46 PDT 2009

/*
 * Copyright (c) 2009 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef _ND_ENDPOINT_H_
#define _ND_ENDPOINT_H_

#include <initguid.h>
#include <ndspi.h>
#include "nd_base.h"
#include "nd_connect.h"
#include "nd_cq.h"
#include "nd_adapter.h"

#define ND_MAX_SGE	8

class CNDEndpoint : INDEndpoint, public CNDBase
{
public:
	// IUnknown methods
	STDMETHODIMP QueryInterface(REFIID riid, LPVOID FAR* ppvObj);
	STDMETHODIMP_(ULONG) AddRef();
	STDMETHODIMP_(ULONG) Release();

	// INDEndpoint methods
	STDMETHODIMP Flush();
	STDMETHODIMP_(void) StartRequestBatch();
	STDMETHODIMP_(void) SubmitRequestBatch();
	STDMETHODIMP Send(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, DWORD Flags);
	STDMETHODIMP SendAndInvalidate(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
								   const ND_MW_DESCRIPTOR* pRemoteMwDescriptor,
								   DWORD Flags);
	STDMETHODIMP Receive(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge);
	STDMETHODIMP Bind(ND_RESULT* pResult, ND_MR_HANDLE hMr, INDMemoryWindow* pMw,
					  const void* pBuffer, SIZE_T BufferSize, DWORD Flags,
					  ND_MW_DESCRIPTOR* pMwDescriptor);
	STDMETHODIMP Invalidate(ND_RESULT* pResult, INDMemoryWindow* pMw, DWORD Flags);
	STDMETHODIMP Read(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
					  const ND_MW_DESCRIPTOR* pRemoteMwDescriptor,
					  ULONGLONG Offset, DWORD Flags);
	STDMETHODIMP Write(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
					   const ND_MW_DESCRIPTOR* pRemoteMwDescriptor,
					   ULONGLONG Offset, DWORD Flags);

	CNDEndpoint(CNDConnector *pConnector);
	~CNDEndpoint();
	void Delete() {delete this;}
	static STDMETHODIMP
	CreateInstance(CNDConnector *pConnector,
				   CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
				   SIZE_T nInboundEntries, SIZE_T nOutboundEntries,
				   SIZE_T nInboundSge, SIZE_T nOutboundSge,
				   SIZE_T InboundReadLimit, SIZE_T OutboundReadLimit,
				   SIZE_T* pMaxInlineData, INDEndpoint** ppEndpoint)
	{
		HRESULT hr;
		CNDEndpoint *ep;

		ep = new CNDEndpoint(pConnector);
		if (ep == NULL) {
			hr = ND_NO_MEMORY;
			goto err1;
		}

		hr = ep->Init(pInboundCq, pOutboundCq, nInboundEntries, nOutboundEntries,
					  nInboundSge, nOutboundSge, InboundReadLimit, OutboundReadLimit,
					  pMaxInlineData);
		if (FAILED(hr)) {
			goto err2;
		}

		*ppEndpoint = ep;
		return ND_SUCCESS;

	err2:
		ep->Release();
	err1:
		*ppEndpoint = NULL;
		return hr;
	}

protected:
	CNDConnector		*m_pConnector;
	CNDCompletionQueue	*m_pInboundCq;
	CNDCompletionQueue	*m_pOutboundCq;
	IWVConnectQueuePair	*m_pWvQp;

	STDMETHODIMP Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
					  SIZE_T nInboundEntries, SIZE_T nOutboundEntries,
					  SIZE_T nInboundSge, SIZE_T nOutboundSge,
					  SIZE_T InboundReadLimit, SIZE_T OutboundReadLimit,
					  SIZE_T* pMaxInlineData);
	STDMETHODIMP_(void)	ConvertSgl(const ND_SGE* pSgl, SIZE_T nSge, WV_SGE *pWvSgl);
	STDMETHODIMP_(DWORD) ConvertSendFlags(DWORD Flags);
};

#endif // _ND_ENDPOINT_H_

/*
 * Copyright (c) 2009 Intel Corporation. All rights reserved.
 *
 * This software is available to you under the OpenIB.org BSD license
 * below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "nd_ep.h"
#include "nd_adapter.h"
#include "nd_connect.h"
#include "nd_cq.h"
#include <netinet/in.h>

CNDEndpoint::CNDEndpoint(CNDConnector *pConnector)
{
	pConnector->AddRef();
	m_pConnector = pConnector;
	m_pWvQp = NULL;
	m_pInboundCq = NULL;
	m_pOutboundCq = NULL;
}

STDMETHODIMP CNDEndpoint::
Init(CNDCompletionQueue* pInboundCq, CNDCompletionQueue* pOutboundCq,
	 SIZE_T nInboundEntries, SIZE_T nOutboundEntries,
	 SIZE_T nInboundSge, SIZE_T nOutboundSge,
	 SIZE_T InboundReadLimit, SIZE_T OutboundReadLimit,
	 SIZE_T* pMaxInlineData)
{
	WV_QP_CREATE create;
	WV_QP_ATTRIBUTES attr;
	WV_DEVICE_ADDRESS *addr;
	DWORD opts;
	HRESULT hr;

	m_pInboundCq->AddRef();
	m_pOutboundCq->AddRef();

	RtlZeroMemory(&create, sizeof create);
	create.pSendCq = pOutboundCq->m_pWvCq;
	create.pReceiveCq = pInboundCq->m_pWvCq;
	create.Context = this;
	create.SendDepth = nOutboundEntries;
	create.SendSge = nOutboundSge;
	create.ReceiveDepth = nInboundEntries;
	create.ReceiveSge = nInboundSge;
	create.InitiatorDepth = OutboundReadLimit;
	create.ResponderResources = InboundReadLimit;
	create.QpType = WvQpTypeRc;

	hr = m_pConnector->m_pAdapter->m_pWvPd->CreateConnectQueuePair(&create, &m_pWvQp);
	if (FAILED(hr)) {
		return hr;
	}

	opts = WV_QP_ATTR_STATE | WV_QP_ATTR_PORT_NUMBER | WV_QP_ATTR_PKEY_INDEX;
	attr.QpState = WvQpStateInit;
	addr = &m_pConnector->m_pAdapter->m_DevAddress;
	attr.AddressVector.PortNumber = addr->PortNumber;
	hr = m_pConnector->m_pAdapter->m_pWvDevice->FindPkey(addr->PortNumber, addr->Pkey,
														 &attr.PkeyIndex);
	if (FAILED(hr)) {
		return hr;
	}

	hr = m_pWvQp->Modify(&attr, opts, NULL);
	if (FAILED(hr)) {
		return hr;
	}

	*pMaxInlineData = 0;
	return ND_SUCCESS;
}

CNDEndpoint::~CNDEndpoint()
{
	if (m_pWvQp != NULL) {
		m_pWvQp->Release();
	}
	if (m_pInboundCq != NULL) {
		m_pInboundCq->Release();
	}
	if (m_pOutboundCq != NULL) {
		m_pOutboundCq->Release();
	}
	m_pConnector->Release();
}

STDMETHODIMP CNDEndpoint::
QueryInterface(REFIID riid, LPVOID FAR* ppvObj)
{
	if (riid != IID_IUnknown && riid != IID_INDEndpoint) {
		*ppvObj = NULL;
		return E_NOINTERFACE;
	}

	*ppvObj = this;
	AddRef();
	return ND_SUCCESS;
}

STDMETHODIMP_(ULONG) CNDEndpoint::
AddRef(void)
{
	return CNDBase::AddRef();
}

STDMETHODIMP_(ULONG) CNDEndpoint::
Release(void)
{
	return CNDBase::Release();
}

STDMETHODIMP CNDEndpoint::
Flush(void)
{
	return ND_SUCCESS;
}

STDMETHODIMP_(void) CNDEndpoint::
StartRequestBatch(void)
{
	// no-op
}

STDMETHODIMP_(void) CNDEndpoint::
SubmitRequestBatch(void)
{
	// no-op
}

STDMETHODIMP_(void) CNDEndpoint::
ConvertSgl(const ND_SGE* pSgl, SIZE_T nSge, WV_SGE* pWvSgl)
{
	SIZE_T i;

	for (i = 0; i < nSge; i++) {
		pWvSgl[i].pAddress = pSgl[i].pAddr;
		pWvSgl[i].Length = (UINT32) pSgl[i].Length;
		pWvSgl[i].Lkey = pSgl[i].hMr ? ((ND_MR *) pSgl[i].hMr)->Keys.Lkey : 0;
	}
}

STDMETHODIMP_(DWORD) CNDEndpoint::
ConvertSendFlags(DWORD Flags)
{
	DWORD opts = 0;

	if (!(Flags & ND_OP_FLAG_SILENT_SUCCESS)) {
		opts |= WV_SEND_SIGNALED;
	}
	if (Flags & ND_OP_FLAG_READ_FENCE) {
		opts |= WV_SEND_FENCE;
	}
	if (Flags & ND_OP_FLAG_SEND_AND_SOLICIT_EVENT) {
		opts |= WV_SEND_SOLICITED;
	}
	return opts;
}

STDMETHODIMP CNDEndpoint::
Send(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge, DWORD Flags)
{
	WV_SGE sgl[ND_MAX_SGE];
	DWORD opts;

	ConvertSgl(pSgl, nSge, sgl);
	opts = ConvertSendFlags(Flags) | (pSgl[0].hMr ? 0 : WV_SEND_INLINE);
	return m_pWvQp->Send((UINT64) pResult, sgl, nSge, opts, 0);
}

STDMETHODIMP CNDEndpoint::
SendAndInvalidate(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
				  const ND_MW_DESCRIPTOR* pRemoteMwDescriptor, DWORD Flags)
{
	// TODO: MW not supported
	return ND_NOT_SUPPORTED;
}

STDMETHODIMP CNDEndpoint::
Receive(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge)
{
	WV_SGE sgl[ND_MAX_SGE];

	ConvertSgl(pSgl, nSge, sgl);
	return m_pWvQp->PostReceive((UINT64) pResult, sgl, nSge);
}

STDMETHODIMP CNDEndpoint::
Bind(ND_RESULT* pResult, ND_MR_HANDLE hMr, INDMemoryWindow* pMw,
	 const void* pBuffer, SIZE_T BufferSize, DWORD Flags,
	 ND_MW_DESCRIPTOR* pMwDescriptor)
{
	// TODO: MW not supported
	return ND_NOT_SUPPORTED;
}

STDMETHODIMP CNDEndpoint::
Invalidate(ND_RESULT* pResult, INDMemoryWindow* pMw, DWORD Flags)
{
	// TODO: MW not supported
	return ND_NOT_SUPPORTED;
}

STDMETHODIMP CNDEndpoint::
Read(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
	 const ND_MW_DESCRIPTOR* pRemoteMwDescriptor, ULONGLONG Offset, DWORD Flags)
{
	WV_SGE sgl[ND_MAX_SGE];
	DWORD opts;

	ConvertSgl(pSgl, nSge, sgl);
	opts = ConvertSendFlags(Flags) | (pSgl[0].hMr ? 0 : WV_SEND_INLINE);
	return m_pWvQp->Read((UINT64) pResult, sgl, nSge, opts,
						 pRemoteMwDescriptor->Base + htonll(Offset),
						 pRemoteMwDescriptor->Token);
}

STDMETHODIMP CNDEndpoint::
Write(ND_RESULT* pResult, const ND_SGE* pSgl, SIZE_T nSge,
	  const ND_MW_DESCRIPTOR* pRemoteMwDescriptor, ULONGLONG Offset, DWORD Flags)
{
	WV_SGE sgl[ND_MAX_SGE];
	DWORD opts;

	ConvertSgl(pSgl, nSge, sgl);
	opts = ConvertSendFlags(Flags) | (pSgl[0].hMr ? 0 : WV_SEND_INLINE);
	return m_pWvQp->Write((UINT64) pResult, sgl, nSge, opts, 0,
						  pRemoteMwDescriptor->Base + htonll(Offset),
						  pRemoteMwDescriptor->Token);
}