[ofw] Latest WinVerbs presentation & .h files.
Fab Tillier
ftillier at windows.microsoft.com
Wed Feb 13 22:34:29 PST 2008
This is an interesting take on verbs. Are these IB-specific as you've defined them, or are you designing them to also support iWARP devices?
>typedef union _WV_GID // Network byte order
>{
> UINT8 Raw[16];
> struct
> {
> UINT64 SubnetPrefix;
> UINT64 InterfaceId;
> } Global;
>
>} WV_GID;
What about multicast GID?
>typedef enum _WV_DEVICE_CAPABILITIES
While enums types in the DDK headers are generally UPPER_CASE, the enum values themselves are MixedCase.
>{
> WV_DEVICE_RESIZE_MAX_WR = 1,
> WV_DEVICE_BAD_PKEY_COUNTER = 1 << 1,
> WV_DEVICE_BAD_QKEY_COUNTER = 1 << 2,
> WV_DEVICE_RAW_MULTICAST = 1 << 3,
Is this needed?
> WV_DEVICE_PATH_MIGRATION = 1 << 4,
> WV_DEVICE_CHANGE_PHYSICAL_PORT = 1 << 5,
> WV_DEVICE_AH_PORT_CHECKING = 1 << 6,
> WV_DEVICE_QP_STATE_MODIFIER = 1 << 7,
> WV_DEVICE_SHUTDOWN_PORT = 1 << 8,
> WV_DEVICE_INIT_TYPE = 1 << 9,
> WV_DEVICE_PORT_ACTIVE_EVENT = 1 << 10,
> WV_DEVICE_SYSTEM_IMAGE_GUID = 1 << 11,
> WV_DEVICE_RC_RNR_NAK_GENERATION = 1 << 12,
> WV_DEVICE_SRQ_RESIZE = 1 << 13,
> WV_DEVICE_BATCH_NOTIFY_CQ = 1 << 14
What's this?
>
>} WV_DEVICE_CAPABILITIES;
>
>typedef enum _WV_ATOMIC_CAPABILITIES
>{
> WV_ATOMIC_NONE,
> WV_ATOMIC_HCA,
> WV_ATOMIC_GLOBAL
>
>} WV_ATOMIC_CAPABILITIES;
>
>typedef struct _WV_DEVICE_ATTRIBUTES
>{
> UINT8 FwVersion[64];
> UINT64 NodeGuid;
> UINT64 SystemImageGuid;
> UINT64 MaxMrSize;
> UINT64 PageSizeCapability;
> UINT32 VendorId;
> UINT32 VendorPartId;
Should this be DeviceId? Should it be moved to the top of the structure?
> UINT32 HwVersion;
> UINT32 MaxQp;
> UINT32 MaxQpWr;
> UINT32 DeviceCapabilityFlags;
> UINT32 MaxSge;
> UINT32 MaxCq;
> UINT32 MaxCqEntries;
> UINT32 MaxMr;
> UINT32 MaxPd;
> UINT32 MaxQpResponderResources;
> UINT32 MaxResponderResources;
> UINT32 MaxQpInitiatorDepth;
> WV_ATOMIC_CAPABILITIES AtomicCapbilities;
> UINT32 MaxMw;
> UINT32 MaxMulticast;
> UINT32 MaxQpAttach;
> UINT32 MaxMulticastQp;
> UINT32 MaxAh;
> UINT32 MaxFmr;
> UINT32 MaxMapPerFmr;
> UINT32 MaxSrq;
> UINT32 MaxSrqWr;
> UINT32 MaxSrqSge;
> UINT16 MaxPkeys;
> UINT8 LocalAckDelay;
> UINT8 PhysPortCount;
>
>} WV_DEVICE_ATTRIBUTES;
>
>typedef enum _WV_PORT_STATE
>{
> WV_PORT_NOP,
> WV_PORT_DOWN,
> WV_PORT_INIT,
> WV_PORT_ARMED,
> WV_PORT_ACTIVE,
> WV_PORT_ACTIVE_DEFER
>
>} WV_PORT_STATE;
>
>typedef struct _WV_PORT_ATTRIBUTES
>{
> WV_PORT_STATE State;
> UINT32 MaxMtu;
> UINT32 ActiveMtu;
> UINT32 GidTableLength;
> UINT32 PortCabilityFlags;
> UINT32 MaxMessageSize;
> UINT32 BadPkeyCounter;
> UINT32 QkeyViolationCounter;
> UINT16 PkeyTableLength;
> UINT16 Lid;
> UINT16 SmLid;
> UINT8 Lmc;
> UINT8 MaxVls;
> UINT8 SmSl;
> UINT8 SubneTimeout;
> UINT8 InitTypeReply;
> UINT8 ActiveWidth;
> UINT8 ActiveSpeed;
> UINT8 PhysicalState;
>
>} WV_PORT_ATTRIBUTES;
>
>typedef struct _WV_ADDRESS
>{
> SOCKET_ADDRESS Address;
> UINT64 DeviceGuid;
> UINT16 PKey;
> UINT8 PortNumber;
>
>} WV_ADDRESS;
>
You'll want to add #pragma warning(disable:4200) here.
>typedef struct _WV_ADDRESS_LIST
>{
> INT Count;
> WV_ADDRESS Address[0];
>
>} WV_ADDRESS_LIST;
You'll want to add #pragma warning(default:4200) here.
>
>typedef enum _WV_DEVICE_EVENT_TYPE
>{
> WV_DEVICE_ERROR,
> WV_DEVICE_PORT_ACTIVE,
> WV_DEVICE_PORT_DOWN
>
>} WV_DEVICE_EVENT_TYPE;
>
>typedef struct _WV_DEVICE_EVENT
>{
> HRESULT Status;
> WV_DEVICE_EVENT_TYPE Type;
> UINT8 PortNumber;
>
>} WV_DEVICE_EVENT;
>
>typedef enum _WV_CQ_NOTIFY_FLAGS
>{
> WV_CQ_SOLICITED = 1 << 0,
> WV_CQ_NEXT_COMP = 1 << 1,
>
>} WV_CQ_NOTIFY_FLAGS;
>
>enum _WV_COMPLETION_OPCODE
>{
> WV_WC_SEND,
> WV_WC_RDMA_WRITE,
> WV_WC_RECV,
> WV_WC_RDMA_READ,
> WV_WC_BIND_WINDOW,
> WV_WC_FETCH_ADD,
> WV_WC_COMPARE_SWAP,
> WV_WC_RECV_RDMA_WRITE
>
>} WV_COMPLETION_OPCODE;
>
>typedef enum _WV_COMPLETION_STATUS
>{
> WV_WC_SUCCESS,
> WV_WC_LOCAL_LENGTH_ERROR,
> WV_WC_LOCAL_OP_ERROR,
> WV_WC_LOCAL_PROTECTION_ERROR,
> WV_WC_WR_FLUSHED_ERROR,
> WV_WC_BIND_WINDOW_ERROR,
> WV_WC_REMOTE_ACCESS_ERROR,
> WV_WC_REMOTE_OP_ERROR,
> WV_WC_RNR_RETRY_ERROR,
> WV_WC_TIMEOUT_RETRY_ERROR,
> WV_WC_REMOTE_INVALID_REQUEST_ERROR,
> WV_WC_BAD_RESPONSE_ERROR,
> WV_WC_LOCAL_ACCESS_ERROR,
> WV_WC_UNKNOWN_ERROR
>
>} WV_COMPLETION_STATUS;
>
>enum _WV_COMPLETION_FLAGS
>{
> WV_WC_IMMEDIATE = 1 << 0,
> // Reserved = 1 << 1,
> WV_WC_GRH_VALID = 1 << 2,
> WV_WC_VENDOR_MASK = 0xFFFF0000
What's VENDOR_MASK used for? Does anyone actually use it historically in other stacks that have had it?
>
>} WV_COMPLETION_FLAGS;
>
>typedef struct _WV_COMPLETION
>{
> UINT64 Reserved;
No need for Reserved.
> UINT64 WrId;
> WV_COMPLETION_OPCODE Opcode;
> UINT32 Length;
> WV_COMPLETION_STATUS Status;
> UINT64 VendorCode;
>
> // Receive completion data
> WV_COMPLETION_FLAGS Flags;
> UINT32 ImmediateData; // Network byte order
> UINT32 SourceQp; // Network byte order
Qpn vs. Qp?
> UINT16 PkeyIndex;
> UINT16 SLid; // Network byte order
> UINT8 SL;
> UINT8 DLidPathBits;
What about the QP that a WR completed on? I'm thinking of the case where an app is using an SRQ and wants context of what QP the operation completed on. This would require apps to be able to set a context value for a QP that gets returned in the WC, haven't gotten to the QP creation to see if you allow a context.
>} WV_COMPLETION;
>
>DECLARE_HANDLE(WV_MR_HANDLE);
>
>typedef enum _WV_SEND_FLAGS
>{
> WV_SEND_IMMEDIATE = 0x00000001,
> WV_SEND_FENCE = 0x00000002,
> WV_SEND_SIGNALED = 0x00000004,
> WV_SEND_SOLICITED = 0x00000008,
> WV_SEND_INLINE = 0x00000010,
> WV_SEND_LOCAL = 0x00000020,
> WV_SEND_VENDOR_MASK = 0xFFFF0000
>
>} WV_SEND_FLAGS;
>
>typedef enum _WV_ATOMIC_OPERATION
>{
> WV_COMPARE_SWAP,
> WV_FETCH_ADD
>
>} WV_ATOMIC_OPERATION;
>
>typedef struct _WV_SGE
>{
> UINT64 Address;
> UINT32 Length;
> UINT32 LKey;
>
>} WV_SGE;
>
>typedef struct _WV_SEND_DATAGRAM
>{
> UINT64 WrId;
> IWVAddressHandle* pAddressHandle;
> WV_SGE* pSgl;
> UINT32 nSge;
> WV_SEND_FLAGS Flags;
> UINT32 ImmediateData; // Network byte order
> UINT32 DestinationQp; // Network byte order
Qpn vs. Qp?
> UINT32 DestinationQKey; // Network byte order
> UINT16 PKeyIndex;
>
>} WV_SEND_DATAGRAM;
>
>typedef struct _WV_ATOMIC
>{
> UINT64 WrId;
> UINT64 Address;
Is this network byte order or host order? It gets put on the wire, so I'd expect it to be network order.
> WV_SGE* pSgl;
> UINT32 nSge;
> WV_SEND_FLAGS Flags;
> UINT32 RKey;
> WV_ATOMIC_OPERATION Operation;
> UINT64 Value1; // Network byte order
> UINT64 Value2; // Network byte order
>
>} WV_ATOMIC;
>
>typedef struct _WV_GRH
>{
> UINT32 VersionClassFlow;
> UINT16 Reserved1;
> UINT8 Reserved2;
> UINT8 HopLimit;
> WV_GID SGid;
> WV_GID DGid;
>
>} WV_GRH;
>
>typedef struct _WV_ADDRESS_VECTOR
>{
> WV_GRH Grh;
> BOOL GrhValid;
> UINT16 DLid; // Network byte order
> UINT8 ServiceLevel;
> UINT8 SourcePathBits;
> UINT8 StaticRate;
> UINT8 PortNumber;
>
>} WV_ADDRESS_VECTOR;
>
>typedef enum _WV_SRQ_ATTRIBUTE_MASK
>{
> IB_SRQ_MAX_WR = 1 << 0,
> IB_SRQ_LIMIT = 1 << 1
>
>} WV_SRQ_ATTRIBUTE_MASK;
>
>typedef struct _WV_SRQ_ATTRIBUTES
>{
> UINT32 MaxWr;
Can this ever be zero?
> UINT32 MaxSge;
Can this ever be zero?
> UINT32 SrqLimit;
Can you reserve -1 to indicate no change?
If the first two of these can be zero and the SrqLimit can have a reserved value, is the structure needed or could these be function parameters? As individual function parameters, the attribute mask would go away.
>
>} WV_SRQ_ATTRIBUTES;
>
>typedef enum _WV_QP_TYPE
>{
> WV_QP_TYPE_RC,
> WV_QP_TYPE_UC,
> WV_QP_TYPE_RESERVED,
> WV_QP_TYPE_UD,
>
>} WV_QP_TYPE;
>
>typedef enum _WV_QP_STATE
>{
> WV_QP_STATE_RESET = (1 << 0)
> WV_QP_STATE_INIT = (1 << 1)
> WV_QP_STATE_RTR = (1 << 2)
> WV_QP_STATE_RTS = (1 << 3)
> WV_QP_STATE_SQD = (1 << 4)
> WV_QP_STATE_SQD_DRAINING = WV_QP_STATE_SQD | (1 << 5)
> WV_QP_STATE_SQD_DRAINED = WV_QP_STATE_SQD | (1 << 6)
> WV_QP_STATE_SQERR = (1 << 7)
> WV_QP_STATE_ERROR = (1 << 8)
> WV_QP_STATE_TIME_WAIT = 0xDEAD0000
>
>} WV_QP_STATE;
>
>typedef enum _WV_APM_STATE
>{
> WV_APM_MIGRATED = 1,
> WV_APM_REARM,
> WV_APM_ARMED
>
>} WV_APM_STATE;
>
>typedef struct _WV_QP_CREATE
>{
> IWVCompletionQueue* pSendCq;
> IWVCompletionQueue* pReceiveCq;
> IWVSharedReceiveQueue* pSharedReceiveQueue;
>
> SIZE_T SendDepth;
> SIZE_T SendSge;
> SIZE_T ReceiveDepth;
> SIZE_T ReceiveSge;
>
> WV_QP_TYPE QpType;
> BOOL SignalSends;
>
>} WV_QP_CREATE;
I have grown to dislike the send/receive terminology because there's so much more you can do than send on the send queue. I prefer active/passive, since requests posted to the active queue result in immediate HW processing, while requests posted to the passive queue are only processed when a corresponding request comes in from a remote peer.
>
>typedef struct _WV_QP_ATTRIBUTES
>{
> IWVProtectionDomain* pPd;
> IWVCompletionQueue* pSendCq;
> IWVCompletionQueue* pReceiveCq;
> IWVSharedReceiveQueue* pSharedReceiveQueue;
Is this useful to return?
> SIZE_T SendDepth;
> SIZE_T SendSge;
> SIZE_T ReceiveDepth;
> SIZE_T ReceiveSge;
> SIZE_T MaxInlineSend;
> SIZE_T InitiatorDepth;
> SIZE_T ResponderResources;
Is there any value in returning InitiatorDepth and ResponderResources? What can a user possibly do with this information?
>
> WV_QP_TYPE QpType;
> WV_QP_STATE CurrentQpState;
> WV_QP_STATE QpState;
What's the difference between these two states?
> WV_APM_STATE ApmState;
> UINT32 QpNumber; // Network byte order
Qpn vs. QpNumber? You use SendPsn and not SendPsNumber.
> UINT32 DestinationQpNumber; // Network byte order
> UINT32 QKey; // Network byte order
> UINT32 SendPsn; // Network byte order
> UINT32 ReceivePsn; // Network byte order
>
> WV_ADDRESS_VECTOR PrimaryAddressVector;
> WV_ADDRESS_VECTOR AlternateAddressVector;
> BOOL SignalSends;
Can this be changed once the QP is created? If not, is there value in returning it?
> DWORD AccessFlags;
> UINT16 PkeyIndex;
>
> UINT8 PathMtu;
> UINT8 LocalAckTimeout;
> UINT8 SequenceErrorRetryCount;
> UINT8 RnrRetryCount;
>
>} WV_QP_ATTRIBUTES;
>
>typedef enum _WV_QP_EVENT_TYPE
>{
> WV_QP_ERROR,
> WV_QP_MODIFY,
> WV_QP_COMMUNICATION_ESTABLISH,
> WV_QP_PATH_MIGRATION,
> WV_QP_SEND_QUEUE_DRAINED,
> WV_QP_LAST_SRQ_REQUEST
>
>} WV_QP_EVENT_TYPE;
>
>typedef struct _WV_QP_EVENT
>{
> HRESULT Status;
> WV_QP_EVENT_TYPE Type;
>
>} WV_QP_EVENT;
>
>typedef enum _WV_CM_EVENT_TYPE
>{
> WV_CM_EVENT_ADDR_RESOLVED,
> WV_CM_EVENT_ADDR_ERROR,
> WV_CM_EVENT_ROUTE_RESOLVED,
> WV_CM_EVENT_ROUTE_ERROR,
> WV_CM_EVENT_CONNECT_REQUEST,
> WV_CM_EVENT_LOOKUP_REQUEST = WV_CM_EVENT_CONNECT_REQUEST,
> WV_CM_EVENT_CONNECT_RESPONSE,
> WV_CM_EVENT_CONNECT_ERROR,
> WV_CM_EVENT_UNREACHABLE,
> WV_CM_EVENT_REJECTED,
> WV_CM_EVENT_ESTABLISHED,
> WV_CM_EVENT_DISCONNECTED,
> WV_CM_EVENT_DEVICE_REMOVAL,
> WV_CM_EVENT_MULTICAST_JOIN,
> WV_CM_EVENT_MULTICAST_ERROR
>
>} WV_CM_EVENT_TYPE;
>
>#define WV_CM_UDP_QKEY 0x01234567
>
>typedef struct _WV_CONNECT_PARAM
>{
> const VOID* pPrivateData;
> SIZE_T PrivateDataLength;
> SIZE_T ResponderResources;
> SIZE_T InitiatorDepth;
I find the iWARP terminology to be a bit easier to understand - InboundReadDepth and OutboundReadDepth.
> UINT8 RetryCount; // Ignored when accepting
> UINT8 RnrRetryCount;
>
>} WV_CONNECT_PARAM;
>
>typedef struct _WV_CONNECT_EVENT
>{
> WV_CM_EVENT_TYPE Event;
> HRESULT Status;
> WV_CONNECT_PARAM Connect;
>
>} WV_CONNECT_EVENT;
>
>typedef struct _WV_CONNECT_ATTRIBUTES
>{
> SOCKET_ADDRESS LocalAddress;
> SOCKET_ADDRESS PeerAddress;
> WV_CONNECT_EVENT LastEvent;
>
>} WV_CONNECT_ATTRIBUTES;
>
>typedef struct _WV_DATAGRAM_PARAM
>{
> const VOID* pPrivateData;
> SIZE_T PrivateDataLength;
> WV_ADDRESS_VECTOR AddressVector;
> UINT32 QpNumber;
> UINT32 QKey;
>
>} WV_DATAGRAM_PARAM;
>
>
>typedef struct _WV_DATAGRAM_EVENT
>{
> WV_CM_EVENT_TYPE Event;
> HRESULT Status;
> WV_DATAGRAM_PARAM Datagram;
>
>} WV_DATAGRAM_EVENT;
>
>
>typedef struct _WV_DATAGRAM_ATTRIBUTES
>{
> SOCKET_ADDRESS LocalAddress;
> SOCKET_ADDRESS PeerAddress;
> WV_DATAGRAM_EVENT LastEvent;
>
>} WV_DATAGRAM_ATTRIBUTES;
>
>
>#undef INTERFACE
>#define INTERFACE IWVOverlapped
>// {64687592-aa67-4b55-bc4b-e189bdd2fc4f}
>DEFINE_GUID(IID_IWVOverlapped, 0x64687592, 0xaa67, 0x4b55,
> 0xbc, 0x4b, 0xe1, 0x89, 0xbd, 0xd2, 0xfc, 0x4f);
>
>DECLARE_INTERFACE_(IWVOverlapped, IUnknown)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVCompletionQueue
>// {a5f6a8de-18a6-4086-aa93-7b66607a290a}
>DEFINE_GUID(IID_IWVCompletionQueue, 0xa5f6a8de, 0x18a6, 0x4086,
> 0xaa, 0x93, 0x7b, 0x66, 0x60, 0x7a, 0x29, 0x0a);
>
>DECLARE_INTERFACE_(IWVCompletionQueue, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVCompletionQueue methods
> STDMETHOD(Resize)(
> THIS_
> __inout SIZE_T* pEntries
> ) PURE;
>
> STDMETHOD(Peek)(
> THIS_
> __out SIZE_T* pCompletedEntries
> ) PURE;
>
> STDMETHOD(Notify)(
> THIS_
> __in WV_CQ_NOTIFY_FLAGS Flags,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(BatchNotify)(
> THIS_
> __in SIZE_T CompletedEntries,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD_(SIZE_T,Poll)(
> THIS_
> __inout_ecount(Entries) WV_COMPLETION* pCompletions[],
> __in SIZE_T Entries
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVMemoryWindow
>// {e8ae206f-b31f-4709-8dc9-6e87625f93fc}
>DEFINE_GUID(IID_IWVMemoryWindow, 0xe8ae206f, 0xb31f, 0x4709,
> 0x8d, 0xc9, 0x6e, 0x87, 0x62, 0x5f, 0x93, 0xfc);
>
>DECLARE_INTERFACE_(IWVMemoryWindow, IUnknown)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVAddressHandle
>// {95903fde-fdac-4007-92b7-e01286cd36e8}
>DEFINE_GUID(IID_IWVAddressHandle, 0x95903fde, 0xfdac, 0x4007,
> 0x92, 0xb7, 0xe0, 0x12, 0x86, 0xcd, 0x36, 0xe8);
>
>DECLARE_INTERFACE_(IWVAddressHandle, IUnknown)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVSharedReceiveQueue
>// {b2bf30e4-8c2e-4659-ac7a-4c5e0d2c7114}
>DEFINE_GUID(IID_IWVSharedReceiveQueue, 0xb2bf30e4, 0x8c2e, 0x4659,
> 0xac, 0x7a, 0x4c, 0x5e, 0x0d, 0x2c, 0x71, 0x14);
>
>DECLARE_INTERFACE_(IWVSharedReceiveQueue, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVSharedReceiveQueue methods
> STDMETHOD(Modify)(
> THIS_
> __in WV_SRQ_ATTRIBUTES* pAttributes,
> __in DWORD Mask);
> ) PURE;
See comment above about using the attribute values as parameters...
> STDMETHOD(PostReceive)(
> THIS_
> __in UINT64 WrId,
> __in_ecount(nSge) const WV_SGE* pSgl,
> __in UINT32 nSge
> ) PURE;
>
> /* Signaled on limit reached event */
> STDMETHOD(Notify)(
> THIS_
> __in OVERLAPPED* pOverlapped
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVQueuePair
>// {a847c13c-e617-489c-b0ab-2da73eb0adfd}
>DEFINE_GUID(IID_IWVQueuePair, 0xa847c13c, 0xe617, 0x489c,
> 0xb0, 0xab, 0x2d, 0xa7, 0x3e, 0xb0, 0xad, 0xfd);
>
>DECLARE_INTERFACE_(IWVQueuePair, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVQueuePair methods
> STDMETHOD(Query)(
> THIS_
> __out WV_QP_ATTRIBUTES* pAttributes
> ) PURE;
>
> STDMETHOD(Modify)(
> THIS_
> __in WV_QP_ATTRIBUTES* pAttributes,
> __in DWORD Options,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(SendDatagram)(
> THIS_
> __in WV_SEND_DATAGRAM* pDatagram
> ) PURE;
I think you should probably have two different QP classes, one for datagrams and one for connected QPs.
SendDatagram and the multicast operations would go into the datagram class, and the other operations would go in the connected QP. The endpoints would then have different CreateQp methods.
> STDMETHOD(Send)(
> THIS_
> __in UINT64 WrId,
> __in_ecount(nSge) const WV_SGE* pSgl
> __in UINT32 nSge,
> __in WV_SEND_FLAGS Flags,
> __in UINT32 ImmediateData
> ) PURE;
>
> STDMETHOD(Read)(
> THIS_
> __in UINT64 WrId,
> __in_ecount(nSge) const WV_SGE* pSgl
> __in UINT32 nSge,
> __in WV_SEND_FLAGS Flags,
> __in UINT64 Address,
> __in UINT32 RKey
> ) PURE;
>
> STDMETHOD(Write)(
> THIS_
> __in UINT64 WrId,
> __in_ecount(nSge) const WV_SGE* pSgl
> __in UINT32 nSge,
> __in WV_SEND_FLAGS Flags,
> __in UINT64 Address,
> __in UINT32 RKey,
> __in UINT32 ImmediateData
> ) PURE;
>
> STDMETHOD(Atomic)(
> THIS_
> __in WV_ATOMIC* pAtomic
Why not make these individual parameters like the other functions? If you split up AtomicAdd and AtomicCompareExchange, you end up with 7 and 8 parameters, respectively. Write has 7 and BindmemoryWindow has 8, so there's precedent. :)
> ) PURE;
>
> STDMETHOD(PostReceive)(
> THIS_
> __in UINT64 WrId,
> __in_ecount(nSge) const WV_SGE* pSgl
> __in UINT32 nSge,
> ) PURE;
>
> STDMETHOD(BindMemoryWindow)(
> THIS_
> __in IWVMemoryWindow* pMw,
> __in WV_MR_HANDLE hMr,
> __in UINT64 WrID,
> __in DWORD AccessFlags,
> __in DWORD SendFlags,
> __in_bcount(BufferLength) const VOID* pBuffer,
> __in SIZE_T BufferLength,
> __out UINT32 *RKey
> ) PURE;
Where's invalidate? Or do you just bind to zero length to invalidate? I guess this brings up a broader question of whether you intend to support Type 2 memory windows, or just Type 1, and whether you plan for this interface to support iWARP. I was going to ask how you were going to support send and invalidate, but I guess if all you support is type 1 then it doesn't matter.
> STDMETHOD(AttachMulticast)(
> THIS_
> __in WV_GID *pGid,
> __in UINT16 LID
> ) PURE;
>
> STDMETHOD(DetachMulticast)(
> THIS_
> __in WV_GID *pGid,
> __in UINT16 LID
> ) PURE;
These two above functions should be part of the datagram QP interface.
> STDMETHOD(Notify)(
> THIS_
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(GetEvent)(
> THIS_
> __out WV_QP_EVENT* pEvent
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVProtectionDomain
>// {a5633a12-dffc-4060-927d-9a600d7efb63}
>DEFINE_GUID(IID_IWVProtectionDomain, 0xa5633a12, 0xdffc, 0x4060,
> 0x92, 0x7d, 0x9a, 0x60, 0x0d, 0x7e, 0xfb, 0x63);
>
>DECLARE_INTERFACE_(IWVProtectionDomain, IUnknown)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVProtectionDomain methods
> STDMETHOD(CreateSharedReceiveQueue)(
> THIS_
> __in WV_SRQ_ATTRIBUTES* pAttributes,
> __deref_out IWVSharedReceiveQueue** ppSrq
> ) PURE;
>
> STDMETHOD(CreateQueuePair)(
> THIS_
> __in WV_QP_CREATE* pAttributes,
> __deref_out IWVQueuePair** ppQp
> ) PURE;
>
> STDMETHOD(RegisterMemory)(
> THIS_
> __in_bcount(BufferLength) const VOID* pBuffer,
> __in SIZE_T BufferLength
> __in DWORD AccessFlags,
Access flags on registrations are dumb. You're trying to protect the app from misusing a buffer within its own address space.
> __in OVERLAPPED* pOverlapped,
> __deref_out WV_MR_HANDLE* phMr,
> __out UINT32* pLKey,
You don't need the lkey - the MR handle is enough.
> __out UINT32* pRKey
This is going to be a bit hard to handle I think as an overlapped operation if you have multiple output parameters. The overlapped returns directly to the user, so the get multiple output buffers you need to do a bunch of extra work in the kernel to be able to access the buffer (like pinning it so that you can write to it from DISPATCH_LEVEL). You can't rely on the I/O manager doing all this for you.
One way to work around this is to have a single output buffer that is a structure containing the different values you want, but that's a bit ugly.
> ) PURE;
>
> STDMETHOD(DeregisterMemory)(
> THIS_
> __in WV_MR_HANDLE hMr,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(AllocateMemoryWindow)(
> THIS_
> __deref_out IWVMemoryWindow** ppMw
> ) PURE;
>
> STDMETHOD(CreateAddressHandle)(
> THIS_
> __in WV_ADDRESS_VECTOR* pAddress,
> __deref_out IWVAddressHandle** ppAh
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVEndpoint
>// {eb2fbd8e-b9b6-4b24-9a5e-94c26ae265f0}
>DEFINE_GUID(IID_IWVEndpoint, 0xeb2fbd8e, 0xb9b6, 0x4b24,
> 0x9a, 0x5e, 0x94, 0xc2, 0x6a, 0xe2, 0x65, 0xf0);
>
>DECLARE_INTERFACE_(IWVEndpoint, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVEndpoint methods
> STDMETHOD(BindAddress)(
> THIS_
> __in const struct sockaddr* pAddress
> ) PURE;
>
> STDMETHOD(ResolveAddress)(
> THIS_
> __in const struct sockaddr* pDestinationAddress,
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(ResolveRoute)(
> THIS_
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(CreateQueuePair)(
> THIS_
> __in IWVProtectionDomain* pPd,
> __in WV_QP_CREATE* pAttributes,
> __deref_out IWVQueuePair** ppQp
> ) PURE;
This should be part of the derived IWVConnectEndpoint (returning an IWVConnectQueuePair) and IWVDatagramEndpoint (returning an IWVDatagramQueuePair).
> STDMETHOD(Reject)(
> THIS_
> __in_bcount_opt(PrivateDataLength) const VOID* pPrivateData,
> __in SIZE_T PrivateDataLength
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVConnectEndpoint
>// {ac670274-1934-4759-a39c-eee01a8130b3}
>DEFINE_GUID(IID_IWVConnectEndpoint, 0xac670274, 0x1934, 0x4759,
> 0xa3, 0x9c, 0xee, 0xe0, 0x1a, 0x81, 0x30, 0xb3);
>
>DECLARE_INTERFACE_(IWVConnectEndpoint, IWVEndpoint)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVEndpoint methods
> __override STDMETHOD(BindAddress)(
> THIS_
> __in const struct sockaddr* pAddress
> ) PURE;
>
> __override STDMETHOD(ResolveAddress)(
> THIS_
> __in const struct sockaddr* pDestinationAddress,
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> __override STDMETHOD(ResolveRoute)(
> THIS_
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> __override STDMETHOD(CreateQueuePair)(
> THIS_
> __in IWVProtectionDomain* pPd,
> __in WV_QP_CREATE* pAttributes,
> __deref_out IWVQueuePair** ppQp
> ) PURE;
>
> __override STDMETHOD(Reject)(
> THIS_
> __in_bcount_opt(PrivateDataLength) const VOID* pPrivateData,
> __in SIZE_T PrivateDataLength
> ) PURE;
>
> // IWVConnectEndpoint methods
> STDMETHOD(Connect)(
> THIS_
> __in WV_CONNECT_PARAM* pParam,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(Accept)(
> THIS_
> __in WV_CONNECT_PARAM* pParam,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(Disconnect)(
> THIS_
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(Query)(
> THIS_
> __out WV_CONNECT_ATTRIBUTES* pAttributes
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVDatagramEndpoint
>// {1d879de6-f2af-4a8a-8893-52e0ab868130}
>DEFINE_GUID(IID_IWVDatagramEndpoint, 0x1d879de6, 0xf2af, 0x4a8a,
> 0x88, 0x93, 0x52, 0xe0, 0xab, 0x86, 0x81, 0x30);
>
>DECLARE_INTERFACE_(IWVDatagramEndpoint, IWVEndpoint)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVEndpoint methods
> __override STDMETHOD(BindAddress)(
> THIS_
> __in const struct sockaddr* pAddress
> ) PURE;
>
> __override STDMETHOD(ResolveAddress)(
> THIS_
> __in const struct sockaddr* pDestinationAddress,
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> __override STDMETHOD(ResolveRoute)(
> THIS_
> __in DWORD Milliseconds,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> __override STDMETHOD(CreateQueuePair)(
> THIS_
> __in IWVProtectionDomain* pPd,
> __in WV_QP_CREATE* pAttributes,
> __deref_out IWVQueuePair** ppQp
> ) PURE;
>
> __override STDMETHOD(Reject)(
> THIS_
> __in_bcount_opt(PrivateDataLength) const VOID* pPrivateData,
> __in SIZE_T PrivateDataLength
> ) PURE;
>
> // IWVDatagramEndpoint methods
> STDMETHOD(Lookup)(
> THIS_
> __in_bcount_opt(PrivateDataLength) const VOID* pPrivateData,
> __in SIZE_T PrivateDataLength,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(Accept)(
> THIS_
> __in WV_DATAGRAM_PARAM* pParam,
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(JoinMulticast)(
> THIS_
> __in const struct sockaddr* pAddress,
> __in OVERLAPPED* pOverlapped
> ) PURE;
Is a QP required for this to work? Will it create the QP object if it doesn't exist already? Does it perform the JoinMulticast on the QP after the SA has approved the join?
> STDMETHOD(LeaveMulticast)(
> THIS_
> __in const struct sockaddr* pAddress,
> __in OVERLAPPED* pOverlapped
> ) PURE;
Same as above.
> STDMETHOD(Query)(
> THIS_
> __out WV_DATAGRAM_ATTRIBUTES* pAttributes
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVListen
>// {5b839fd4-7410-441f-a2d9-5d75b1d8599b}
>DEFINE_GUID(IID_IWVListen, 0x5b839fd4, 0x7410, 0x441f,
> 0xa2, 0xd9, 0x5d, 0x75, 0xb1, 0xd8, 0x59, 0x9b);
>
>DECLARE_INTERFACE_(IWVListen, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVListen methods
> STDMETHOD(Accept)(
> THIS_
> __in IWVEndpoint* pEndpoint,
> __in OVERLAPPED* pOverlapped
> ) PURE;
So the connection sequence is IWVListen->Accept followed by IWVEndpoint->Accept?
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVDevice
>// {244af78c-b1ac-40e4-9896-271d58d591b8}
>DEFINE_GUID(IID_IWVDevice, 0x244af78c, 0xb1ac, 0x40e4,
> 0x98, 0x96, 0x27, 0x1d, 0x58, 0xd5, 0x91, 0xb8);
>
>DECLARE_INTERFACE_(IWVDevice, IWVOverlapped)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVOverlapped methods
> __override STDMETHOD(CancelOverlappedRequests)(
> THIS
> ) PURE;
>
> __override STDMETHOD(GetOverlappedResult)(
> THIS_
> __inout_opt OVERLAPPED *pOverlapped,
> __out SIZE_T *pNumberOfBytesTransferred,
> __in BOOL bWait
> ) PURE;
>
> // IWVDevice methods
> STDMETHOD(Query)(
> THIS_
> __out_bcount_part_opt(*pBufferSize, *pBufferSize) WV_DEVICE_ATTRIBUTES* pAttributes,
> __inout SIZE_T* pBufferSize
> ) PURE;
>
> STDMETHOD(QueryPort)(
> THIS_
> __in UINT8 PortNumber,
> __out_bcount_part_opt(*pBufferSize, *pBufferSize) WV_PORT_ATTRIBUTES* pAttributes,
> __inout SIZE_T* pBufferSize
> ) PURE;
>
> STDMETHOD(QueryGid)(
> THIS_
> __in UINT8 PortNumber,
> __in DWORD Index,
> __out WV_GID* pGid
> ) PURE;
Is there any value in being able to get the whole GID table and walking it?
> STDMETHOD(QueryPkey)(
> THIS_
> __in UINT8 PortNumber,
> __in DWORD Index,
> __out UINT16* pPkey
> ) PURE;
Same here - support to retrieve the whole PKey table?
> STDMETHOD(CreateCompletionQueue)(
> THIS_
> __inout SIZE_T *pEntries,
> __deref_out IWCompletionQueue** ppCq
> ) PURE;
>
> STDMETHOD(AllocateProtectionDomain)(
> THIS
> ) PURE;
Shouldn't this return a IVWProtectionDomain?
>
> STDMETHOD(Notify)(
> THIS_
> __in OVERLAPPED* pOverlapped
> ) PURE;
>
> STDMETHOD(GetEvent)(
> THIS_
> __in WV_DEVICE_EVENT* pEvent
> ) PURE;
>};
>
>
>#undef INTERFACE
>#define INTERFACE IWVProvider
>// {6901010c-17af-4894-a1dc-794d3611f262}
>DEFINE_GUID(IID_IIWVProvider, 0x6901010c, 0x17af, 0x4894,
> 0xa1, 0xdc, 0x79, 0x4d, 0x36, 0x11, 0xf2, 0x62);
>
>DECLARE_INTERFACE_(IWVProvider, IUnknown)
>{
> // IUnknown methods
> __override STDMETHOD(QueryInterface)(
> THIS_
> REFIID riid,
> LPVOID FAR* ppvObj
> ) PURE;
>
> __override STDMETHOD_(ULONG,AddRef)(
> THIS
> ) PURE;
>
> __override STDMETHOD_(ULONG,Release)(
> THIS
> ) PURE;
>
> // IWVProvider methods
> STDMETHOD(QueryDeviceList)(
> THIS_
> __inout_bcount_part_opt(*pBufferSize, *pBufferSize) UINT64* pGuidList,
> __inout SIZE_T* pBufferSize
> ) PURE;
>
> STDMETHOD(QueryDevice)(
> THIS_
> __in UINT64 Guid,
> __in_bcount_part_opt(*pBufferSize, *pBufferSize) WV_DEVICE_ATTRIBUTES* pAttributes,
> __inout SIZE_T* pBufferSize
> ) PURE;
>
> STDMETHOD(QueryAddressList)(
> THIS_
> __inout_bcount_part_opt(*pBufferSize, *pBufferSize) WV_ADDRESS_LIST* pAddressList,
> __inout SIZE_T* pBufferSize
> ) PURE;
>
> STDMETHOD(OpenDevice)(
> THIS_
> __in UINT64 Guid,
> __deref_out IWVDevice** ppDevice
> ) PURE;
>
> STDMETHOD(CreateConnectEndpoint)(
> THIS_
> __deref_out IWVConnectEndpoint** ppConnectEndpoint
> ) PURE;
>
> STDMETHOD(CreateDatagramEndpoint)(
> THIS_
> __deref_out IWVDatagramEndpoint** ppDatagramEndpoint
> ) PURE;
>
> STDMETHOD(CreateListen)(
> THIS
> __in const struct sockaddr* pAddress,
What parts of the address can be wildcarded? If you allow wildcards, do you also allow address sharing (i.e. a second listen for a specific address gets priority over a wildcard listen?)
> __in SIZE_T backlog,
> __deref_out IWVListen** ppListen
> ) PURE;
How do you distinguish between connected and datagram listens?
That's it for now...
-Fab
More information about the ofw
mailing list