[ofw] umad: do not modify input after sending
Hefty, Sean
sean.hefty at intel.com
Wed Apr 20 14:55:36 PDT 2011
Opensm relies on the libibumad layer not touching the input
after it has been written to the kernel. Fix this by adding direct
support for the umad address format. For the best performance, we
integrate this into the kernel winmad driver. If the 'version' field
is set, we use the winmad address format, otherwise, we assume umad
address format.
The address format only matters when a GRH is present, which typically
is not the case, and is not set by anything in tree. This patch
avoids translating the address formats unless the grh present flag is
set, which should provide a small performance improvement.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
trunk/core/winmad/kernel/wm_provider.c | 23 +++++++++++++++++-----
trunk/core/winmad/wm_ioctl.h | 25 +++++++++++++++++++++++-
trunk/inc/user/iba/winmad.h | 26 ++++++++++++++++++++++++-
trunk/ulp/libibumad/include/infiniband/umad.h | 2 +-
trunk/ulp/libibumad/src/umad.cpp | 13 +++----------
5 files changed, 71 insertions(+), 18 deletions(-)
diff --git a/trunk/core/winmad/kernel/wm_provider.c b/trunk/core/winmad/kernel/wm_provider.c
index f3926f9..6e73c9d 100644
--- a/trunk/core/winmad/kernel/wm_provider.c
+++ b/trunk/core/winmad/kernel/wm_provider.c
@@ -284,6 +284,23 @@ out:
WdfRequestCompleteWithInformation(Request, status, len);
}
+// If the Version is not set, use umad compatability address format
+static void WmConvertGrh(ib_grh_t *pGrh, WM_IO_MAD *pIoMad)
+{
+ if (RtlUlongByteSwap(pIoMad->Address.VersionClassFlow) >> 28) {
+ pGrh->ver_class_flow = pIoMad->Address.VersionClassFlow;
+ } else {
+ pGrh->ver_class_flow = RtlUlongByteSwap((6 << 28) |
+ (((uint32_t) pIoMad->UmadAddress.TrafficClass) << 20) |
+ (pIoMad->UmadAddress.FlowLabel & 0x000FFFFF));
+ }
+
+ pGrh->hop_limit = pIoMad->Address.HopLimit;
+ // TODO: update IBAL to use SGID index
+ // pGrh->src_gid_index = pIoMad->Address.GidIndex;
+ RtlCopyMemory(pGrh->dest_gid.raw, pIoMad->Address.Gid, 16);
+}
+
static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UINT32 size)
{
ib_al_ifc_t *pifc;
@@ -305,11 +322,7 @@ static NTSTATUS WmSendMad(WM_REGISTRATION *pRegistration, WM_IO_MAD *pIoMad, UIN
mad->retry_cnt = pIoMad->Retries;
if ((mad->grh_valid = pIoMad->Address.GrhValid)) {
- mad->p_grh->ver_class_flow = pIoMad->Address.VersionClassFlow;
- mad->p_grh->hop_limit = pIoMad->Address.HopLimit;
- // TODO: update IBAL to use SGID index
- // mad->p_grh->src_gid_index = pIoMad->Address.GidIndex;
- RtlCopyMemory(mad->p_grh->dest_gid.raw, pIoMad->Address.Gid, 16);
+ WmConvertGrh(mad->p_grh, pIoMad);
}
mad->remote_lid = pIoMad->Address.Lid;
diff --git a/trunk/core/winmad/wm_ioctl.h b/trunk/core/winmad/wm_ioctl.h
index 56063e0..6cdc220 100644
--- a/trunk/core/winmad/wm_ioctl.h
+++ b/trunk/core/winmad/wm_ioctl.h
@@ -98,6 +98,26 @@ typedef struct _WM_IO_MAD_AV
} WM_IO_MAD_AV;
+typedef struct _WM_IO_UMAD_AV
+{
+ NET32 Qpn;
+ NET32 Qkey;
+ UINT32 FlowLabel;
+ UINT16 PkeyIndex;
+ UINT8 HopLimit;
+ UINT8 GidIndex;
+ UINT8 Gid[16];
+
+ UINT8 TrafficClass;
+ UINT8 Reserved;
+ NET16 Lid;
+ UINT8 ServiceLevel;
+ UINT8 PathBits;
+ UINT8 StaticRate;
+ UINT8 GrhValid;
+
+} WM_IO_UMAD_AV;
+
#define WM_IO_SUCCESS 0
#define WM_IO_TIMEOUT 138
@@ -106,7 +126,10 @@ typedef struct _WM_IO_MAD_AV
typedef struct _WM_IO_MAD
{
UINT64 Id;
- WM_IO_MAD_AV Address;
+ union {
+ WM_IO_MAD_AV Address;
+ WM_IO_UMAD_AV UmadAddress;
+ };
UINT32 Status;
UINT32 Timeout;
diff --git a/trunk/inc/user/iba/winmad.h b/trunk/inc/user/iba/winmad.h
index 35261f5..4f06173 100644
--- a/trunk/inc/user/iba/winmad.h
+++ b/trunk/inc/user/iba/winmad.h
@@ -72,6 +72,27 @@ typedef struct _WM_MAD_AV
} WM_MAD_AV;
+// Simplify casting to WM_MAD_AV
+typedef struct _WM_UMAD_AV
+{
+ NET32 Qpn;
+ NET32 Qkey;
+ UINT32 FlowLabel;
+ UINT16 PkeyIndex;
+ UINT8 HopLimit;
+ UINT8 GidIndex;
+ UINT8 Gid[16];
+
+ UINT8 TrafficClass;
+ UINT8 ReservedGrh;
+ NET16 Lid;
+ UINT8 ServiceLevel;
+ UINT8 PathBits;
+ UINT8 StaticRate;
+ UINT8 GrhValid;
+
+} WM_UMAD_AV;
+
#define WM_SUCCESS 0
#define WM_TIMEOUT 138
@@ -80,7 +101,10 @@ typedef struct _WM_MAD_AV
typedef struct _WM_MAD
{
UINT64 Id;
- WM_MAD_AV Address;
+ union {
+ WM_MAD_AV Address;
+ WM_UMAD_AV UmadAddress;
+ };
UINT32 Status;
UINT32 Timeout;
diff --git a/trunk/ulp/libibumad/include/infiniband/umad.h b/trunk/ulp/libibumad/include/infiniband/umad.h
index f6b41ac..aceebf3 100644
--- a/trunk/ulp/libibumad/include/infiniband/umad.h
+++ b/trunk/ulp/libibumad/include/infiniband/umad.h
@@ -52,7 +52,7 @@ typedef unsigned __int64 uint64_t;
#define UMAD_MAX_DEVICES 20
#define UMAD_ANY_PORT 0
-// Allow casting to WM_MAD_AV
+// Allow casting to WM_UMAD_AV
typedef struct ib_mad_addr
{
uint32_t qpn;
diff --git a/trunk/ulp/libibumad/src/umad.cpp b/trunk/ulp/libibumad/src/umad.cpp
index d4759c7..19582dc 100644
--- a/trunk/ulp/libibumad/src/umad.cpp
+++ b/trunk/ulp/libibumad/src/umad.cpp
@@ -564,14 +564,6 @@ size_t umad_size(void)
return sizeof(struct ib_user_mad);
}
-static void umad_convert_addr(struct ib_mad_addr *addr, WM_MAD_AV *av)
-{
- av->VersionClassFlow = htonl((6 << 28) | (((uint32_t) addr->traffic_class) << 20) |
- (addr->flow_label & 0x000FFFFF));
- av->Reserved = 0;
- av->StaticRate = 0;
-}
-
static void umad_convert_av(WM_MAD_AV *av, struct ib_mad_addr *addr)
{
uint32_t ver_class_flow;
@@ -595,9 +587,10 @@ int umad_send(int portid, int agentid, void *umad, int length,
mad->retries = (uint32_t) retries;
mad->length = (uint32_t) length;
- umad_convert_addr(&mad->addr, &((WM_MAD *) mad)->Address);
+ mad->addr.reserved_grh = 0;
+ mad->addr.reserved_rate = 0;
+
hr = ports[portid].prov->Send((WM_MAD *) mad, NULL);
- umad_convert_av(&((WM_MAD *) mad)->Address, &mad->addr);
if (FAILED(hr)) {
_set_errno(EIO);
return GetLastError();
More information about the ofw
mailing list