[ofw] [PATCH] winverbs: convert RDMA address to network-byte order
Sean Hefty
sean.hefty at intel.com
Tue Jun 9 12:31:18 PDT 2009
Change the winverbs interface to accept remote RDMA addresses in network
byte order. This keeps it consistent with all other remote/wire data fields.
Unfortunately, this means that the remote address must be swizzled between
winverbs and the HCA libraries, which take the address in host order, but
the other fields in network order. Longer term, the HCA libraries can provide
optimized interfaces for winverbs, like posting a single work request.
For now, to avoid compatibility issues later, define the winverbs interface
using what's most reasonable. Update libibverbs accordingly.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\core\winverbs/user/wv_base.h
branches\winverbs\core\winverbs/user/wv_base.h
--- trunk\core\winverbs/user/wv_base.h 2008-11-05 16:43:19.529313500 -0800
+++ branches\winverbs\core\winverbs/user/wv_base.h 2009-06-09 11:22:24.193591900 -0700
@@ -70,7 +70,6 @@ protected:
#if defined(_WIN64)
#define WvConvertSgl(pSgl, nSge) ((ib_local_ds_t *) pSgl)
- #define WvConvertSends(pSend) ((ib_send_wr_t *) pSend)
#else
static inline ib_local_ds_t *WvConvertSgl(WV_SGE* pSgl, SIZE_T nSge)
{
@@ -81,16 +80,6 @@ protected:
}
return (ib_local_ds_t *) pSgl;
}
-
- static inline ib_send_wr_t *WvConvertSends(WV_SEND_REQUEST *pSend)
- {
- WV_SEND_REQUEST *preq;
-
- for (preq = pSend; preq != NULL; preq = preq->pNext) {
- WvConvertSgl(preq->pSgl, preq->nSge);
- }
- return (ib_send_wr_t *) pSend;
- }
#endif (_WIN64)
#endif // _WV_BASE_H_
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\core\winverbs/user/wv_qp.cpp
branches\winverbs\core\winverbs/user/wv_qp.cpp
--- trunk\core\winverbs/user/wv_qp.cpp 2009-03-25 12:11:46.356325900 -0700
+++ branches\winverbs\core\winverbs/user/wv_qp.cpp 2009-06-09 11:48:04.545241500 -0700
@@ -421,18 +421,31 @@ PostReceive(UINT64 WrId, WV_SGE* pSgl, S
STDMETHODIMP CWVQueuePair::
PostSend(WV_SEND_REQUEST *pSend, WV_SEND_REQUEST **ppFailed)
{
- ib_send_wr_t *wr;
+ WV_SEND_REQUEST *wr;
ib_api_status_t stat;
HRESULT hr;
- stat = m_pVerbs->post_send(m_hVerbsQp, WvConvertSends(pSend),
- (ppFailed == NULL) ? &wr : (ib_send_wr_t **) ppFailed);
+ for (wr = pSend; wr != NULL; wr = wr->pNext) {
+ if (wr->Opcode != WvSend) {
+ wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress);
+ }
+ WvConvertSgl(wr->pSgl, wr->nSge);
+ }
+
+ stat = m_pVerbs->post_send(m_hVerbsQp, (ib_send_wr_t *) pSend,
+ (ib_send_wr_t **) (ppFailed ? ppFailed : &wr));
if (stat == IB_SUCCESS) {
hr = WV_SUCCESS;
} else {
hr = WvConvertIbStatus(stat);
}
+ for (wr = pSend; wr != NULL; wr = wr->pNext) {
+ if (wr->Opcode != WvSend) {
+ wr->Wr.Rdma.RemoteAddress = _byteswap_uint64(wr->Wr.Rdma.RemoteAddress);
+ }
+ }
+
return hr;
}
Only in branches\winverbs\inc\kernel\rdma: _ntstatus_.h
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\inc\user\rdma/winverbs.h
branches\winverbs\inc\user\rdma/winverbs.h
--- trunk\inc\user\rdma/winverbs.h 2009-06-02 13:23:44.246541100 -0700
+++ branches\winverbs\inc\user\rdma/winverbs.h 2009-06-09 11:18:51.345933700 -0700
@@ -291,14 +291,14 @@ typedef struct _WV_SEND_REQUEST
{
struct
{
- UINT64 RemoteAddress;
+ NET64 RemoteAddress;
NET32 Rkey;
} Rdma;
struct
{
- UINT64 RemoteAddress;
+ NET64 RemoteAddress;
NET32 Rkey;
NET64 Compare;
NET64 Exchange;
@@ -307,7 +307,7 @@ typedef struct _WV_SEND_REQUEST
struct
{
- UINT64 RemoteAddress;
+ NET64 RemoteAddress;
NET32 Rkey;
NET64 Add;
UINT64 Reserved;
@@ -739,9 +739,9 @@ DECLARE_INTERFACE_(IWVSharedReceiveQueue
#undef INTERFACE
#define INTERFACE IWVQueuePair
-// {a847c13c-e617-489c-b0ab-2da73eb0adfd}
-DEFINE_GUID(IID_IWVQueuePair, 0xa847c13c, 0xe617, 0x489c,
- 0xb0, 0xab, 0x2d, 0xa7, 0x3e, 0xb0, 0xad, 0xfd);
+// {35C64226-6A1F-4c8d-9465-C6FEE8053CDD}
+DEFINE_GUID(IID_IWVQueuePair, 0x35c64226, 0x6a1f, 0x4c8d,
+ 0x94, 0x65, 0xc6, 0xfe, 0xe8, 0x5, 0x3c, 0xdd);
DECLARE_INTERFACE_(IWVQueuePair, IWVOverlapped)
{
@@ -802,9 +802,9 @@ DECLARE_INTERFACE_(IWVQueuePair, IWVOver
#undef INTERFACE
#define INTERFACE IWVConnectQueuePair
-// {A791309C-4244-4194-AD63-7D372CADE5E0}
-DEFINE_GUID(IID_IWVConnectQueuePair, 0xa791309c, 0x4244, 0x4194,
- 0xad, 0x63, 0x7d, 0x37, 0x2c, 0xad, 0xe5, 0xe0);
+// {DF905570-AEEB-4114-B30E-4DC3EB5A9AD6}
+DEFINE_GUID(IID_IWVConnectQueuePair, 0xdf905570, 0xaeeb, 0x4114,
+ 0xb3, 0xe, 0x4d, 0xc3, 0xeb, 0x5a, 0x9a, 0xd6);
DECLARE_INTERFACE_(IWVConnectQueuePair, IWVQueuePair)
{
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/verbs.cpp
branches\winverbs\ulp\libibverbs/src/verbs.cpp
--- trunk\ulp\libibverbs/src/verbs.cpp 2009-04-08 13:33:19.307495000 -0700
+++ branches\winverbs\ulp\libibverbs/src/verbs.cpp 2009-06-09 11:45:51.798639900 -0700
@@ -735,6 +735,7 @@ int ibv_post_send(struct ibv_qp *qp, str
}
if (cur_wr->opcode != 0) {
+ cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr);
cur_wr->wr.rdma.rkey = htonl(cur_wr->wr.rdma.rkey);
}
}
@@ -744,6 +745,7 @@ int ibv_post_send(struct ibv_qp *qp, str
for (cur_wr = wr; cur_wr != NULL; cur_wr = cur_wr->next) {
if (cur_wr->opcode != 0) {
cur_wr->wr.rdma.rkey = ntohl(cur_wr->wr.rdma.rkey);
+ cur_wr->wr.rdma.remote_addr = htonll(cur_wr->wr.rdma.remote_addr);
}
if ((cur_wr->send_flags & WV_SEND_IMMEDIATE) != 0) {
More information about the ofw
mailing list