<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<HTML><HEAD>
<META http-equiv=Content-Type content="text/html; charset=us-ascii">
<META content="MSHTML 6.00.2900.5512" name=GENERATOR></HEAD>
<BODY>
<DIV><FONT size=2><FONT size=1>
<P><FONT face=Arial size=2>MAD tracing mechanism works under the control of
flags, set in new Registry mlx4_bus parameter StatFlags.</FONT></P>
<P><FONT face=Arial size=2>The mechanism prints to debugger the IB headers of
packets sent over MLX transport, i.e. on QP0 and QP1.</FONT></P>
<P><FONT face=Arial size=2>The flags are:</FONT></P>
<P><FONT face=Arial size=2>0x0001 - print LRH</FONT></P>
<P><FONT face=Arial size=2>0x0002 - print BTH</FONT></P>
<P><FONT face=Arial size=2>0x0004 - print DETH</FONT></P>
<P><FONT face=Arial size=2>0x0008 - print GRH (it won't print i<SPAN
class=157434708-16112009>f</SPAN> the GRH is absent)</FONT></P>
<P><FONT face=Arial size=2>0x0010 - print some WQE info</FONT></P>
<P><FONT face=Arial size=2>0x0020 - print some more UD header info</FONT></P>
<P><FONT face=Arial size=2>0x0040 - print some send WR
info</FONT></P></FONT></FONT></DIV>
<DIV><FONT face=Arial size=2></FONT> </DIV>
<DIV><FONT face=Arial size=2></FONT> </DIV>
<DIV><FONT face=Arial size=2>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5099)<BR>@@
-93,6 +93,7 @@<BR> int
mod_interrupt_from_first;<BR> <BR> int
mod_affinity;<BR>+ int mod_stat_flags;<BR> PFDO_DEVICE_DATA
p_fdo[MAX_HCA_CARDS]; // for debug purposes<BR> }
GLOBALS;<BR> #pragma warning(default:4201) // nameless
struct/union<BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5099)<BR>@@ -34,6
+34,8 @@<BR> #define MLX4_QP_H<BR> <BR> #include
"device.h"<BR>+#include "ib\mlx4_ib.h"<BR>+#include
"ib_pack.h"<BR> <BR> #define
MLX4_INVALID_LKEY 0x100<BR> <BR>@@ -288,7 +290,22
@@<BR> __be32 byte_count;<BR> };<BR> <BR>+enum
{<BR>+ /*<BR>+ * Largest possible UD header: send with GRH and
immediate data.<BR>+ */<BR>+ MLX4_IB_UD_HEADER_SIZE =
72<BR>+};<BR> <BR>+struct mlx4_ib_sqp {<BR>+ struct
mlx4_ib_qp qp;<BR>+ int pkey_index;<BR>+ u32 qkey;<BR>+ u32 send_psn;<BR>+ struct
ib_ud_header ud_header;<BR>+ u8 header_buf[MLX4_IB_UD_HEADER_SIZE];<BR>+};<BR>+<BR> int
mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp
*qp,<BR> struct mlx4_qp_context
*context);<BR> <BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5099)<BR>@@ -36,6
+36,9 @@<BR> #include "qp.h"<BR> #include "user.h"<BR> <BR>+void
st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp *sqp, struct
mlx4_wqe_mlx_seg *mlx );<BR>+void st_print_mlx_send(struct mlx4_dev *mdev,
struct ib_qp *ibqp, ib_send_wr_t *wr);<BR>+<BR> enum
{<BR> MLX4_IB_ACK_REQ_FREQ = 8,<BR> };<BR>@@ -46,22 +49,6
@@<BR> };<BR> <BR> enum {<BR>- /*<BR>- * Largest
possible UD header: send with GRH and immediate data.<BR>-
*/<BR>- MLX4_IB_UD_HEADER_SIZE = 72<BR>-};<BR>-<BR>-struct
mlx4_ib_sqp {<BR>- struct
mlx4_ib_qp qp;<BR>- int pkey_index;<BR>- u32 qkey;<BR>- u32 send_psn;<BR>- struct
ib_ud_header ud_header;<BR>- u8 header_buf[MLX4_IB_UD_HEADER_SIZE];<BR>-};<BR>-<BR>-enum
{<BR> MLX4_IB_MIN_SQ_STRIDE = 6<BR> };<BR> <BR>@@ -1541,6
+1528,10 @@<BR> if
(bad_wr)<BR> *bad_wr =
wr;<BR> goto
out;<BR>+ <BR>+ /* statistics
*/<BR>+ st_print_mlx_send( dev, ibqp,
wr);<BR>+ st_print_mlx_header( dev, to_msqp(qp),
(void*)ctrl );<BR> }<BR> wqe
+= err;<BR> size += err / 16;<BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5099)<BR>@@
-21,7 +21,7
@@<BR> main.c \<BR> mr.c \<BR> qp.c \<BR>- srq.c \<BR>+ srq.c <BR> <BR> INCLUDES=..;..\inc;..\..\inc;..\core\$O;..\..\..\inc;..\..\..\..\..\inc;..\..\..\..\..\inc\kernel;<BR> <BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5099)<BR>@@
-30,6 +30,7 @@<BR> pci.c \<BR> pdo.c
\<BR> wmi.c
\<BR>+ stat.c<BR> <BR> PRECOMPILED_INCLUDE=precomp.h<BR> <BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5099)<BR>@@
-11,5 +11,6 @@<BR> #include "driver.h"<BR> #include
"cmd.h"<BR> #include <mlx4_debug.h><BR>+#include
"stat.h"<BR> <BR> <BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 0)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 5099)<BR>@@ -0,0
+1,159 @@<BR>+/*++<BR>+<BR>+Copyright (c) 2005-2009 Mellanox Technologies. All
rights reserved.<BR>+<BR>+Module
Name:<BR>+ bus_stat.h<BR>+<BR>+Abstract:<BR>+ Statistics Collector
header file<BR>+<BR>+Revision
History:<BR>+<BR>+Notes:<BR>+<BR>+--*/<BR>+<BR>+#include
"precomp.h"<BR>+<BR>+MLX4_ST_STAT g_stat;<BR>+<BR>+static void __print_grh(
struct mlx4_dev *mdev, struct ib_unpacked_grh *p)<BR>+{<BR>+ mlx4_dbg(mdev,
"\n\t ========== GRH ==========\n");<BR>+ mlx4_dbg(mdev, "\t
ip_version %02x",
p->ip_version);<BR>+ mlx4_dbg(mdev, "\t
traffic_class %02x",
p->traffic_class);<BR>+ mlx4_dbg(mdev, "\t
flow_label %08x",
<BR>+ be32_to_cpu(p->flow_label));<BR>+ mlx4_dbg(mdev, "\t
payload_length %04x",
<BR>+ be16_to_cpu(p->payload_length));<BR>+ mlx4_dbg(mdev,
"\t next_header %02x",
p->next_header);<BR>+ mlx4_dbg(mdev, "\t
hop_limit %02x",
p->hop_limit);<BR>+ mlx4_dbg(mdev, "\t
source_gid %08I64x:%08I64",
<BR>+ be64_to_cpu(p->source_gid.global.subnet_prefix),<BR>+ be64_to_cpu(p->source_gid.global.interface_id));<BR>+ mlx4_dbg(mdev,
"\t source_gid %08I64x:%08I64",
<BR>+ be64_to_cpu(p->destination_gid.global.subnet_prefix),<BR>+ be64_to_cpu(p->destination_gid.global.interface_id));<BR>+}<BR>+<BR>+static
void __print_deth( struct mlx4_dev *mdev, struct ib_unpacked_deth
*p)<BR>+{<BR>+ mlx4_dbg(mdev, "\n\t ========== DETH
==========\n");<BR>+ mlx4_dbg(mdev, "\t
qkey
%08x", <BR>+ be32_to_cpu(p->qkey));<BR>+ mlx4_dbg(mdev, "\t
source_qpn %08x",
<BR>+ be32_to_cpu(p->source_qpn));<BR>+}<BR>+<BR>+static void
__print_bth( struct mlx4_dev *mdev, struct ib_unpacked_bth
*p)<BR>+{<BR>+ mlx4_dbg(mdev, "\n\t ========== BTH
==========\n");<BR>+ mlx4_dbg(mdev, "\t
opcode %02x",
p->opcode);<BR>+ mlx4_dbg(mdev, "\t solicited_event %02x",
p->solicited_event);<BR>+ mlx4_dbg(mdev, "\t
mig_req %02x",
p->mig_req);<BR>+ mlx4_dbg(mdev, "\t header_version
%02x", p->transport_header_version);<BR>+ mlx4_dbg(mdev, "\t
pkey
%04x", <BR>+ be16_to_cpu(p->pkey));<BR>+ mlx4_dbg(mdev, "\t
destination_qpn %08x",
<BR>+ be32_to_cpu(p->destination_qpn));<BR>+ mlx4_dbg(mdev,
"\t ack_req %02x",
p->ack_req);<BR>+ mlx4_dbg(mdev, "\t
psn
%08x", <BR>+ be32_to_cpu(p->psn));<BR>+}<BR>+<BR>+static void
__print_lrh( struct mlx4_dev *mdev, struct ib_unpacked_lrh
*p)<BR>+{<BR>+ mlx4_dbg(mdev, "\n\t ========== LRH
==========\n");<BR>+ mlx4_dbg(mdev, "\t
virtual_lane %02x",
p->virtual_lane);<BR>+ mlx4_dbg(mdev, "\t
link_version %02x",
p->link_version);<BR>+ mlx4_dbg(mdev, "\t
service_level %02x",
p->service_level);<BR>+ mlx4_dbg(mdev, "\t link_next_header %02x",
p->link_next_header);<BR>+ mlx4_dbg(mdev, "\t
destination_lid %04x",
<BR>+ be16_to_cpu(p->destination_lid));<BR>+ mlx4_dbg(mdev,
"\t packet_length %04x",
<BR>+ be16_to_cpu(p->packet_length));<BR>+ mlx4_dbg(mdev, "\t
source_lid %04x",
<BR>+ be16_to_cpu(p->source_lid));<BR>+}<BR>+<BR>+static void
__print_ud_header( struct mlx4_dev *mdev, struct ib_ud_header
*p)<BR>+{<BR>+ mlx4_dbg(mdev, "\n\t ========== UD HEADER
==========\n");<BR>+<BR>+ mlx4_dbg(mdev, "\t grh_present %d, imm_present
%d, imm %08x",<BR>+ p->grh_present, p->immediate_present,
be32_to_cpu(p->immediate_data) );<BR>+<BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_LRH
)<BR>+ __print_lrh( mdev, &p->lrh );<BR>+<BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_BTH
)<BR>+ __print_bth( mdev, &p->bth );<BR>+ <BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_DETH
)<BR>+ __print_deth( mdev, &p->deth );<BR>+ <BR>+ if
( p->grh_present && (mdev->pdev->p_stat_dev->flags &
MLX4_MAD_TRACE_GRH) )<BR>+ __print_grh( mdev, &p->grh
);<BR>+ <BR>+}<BR>+<BR>+static void __print_mlx( struct mlx4_dev *mdev,
struct mlx4_wqe_mlx_seg *p)<BR>+{<BR>+ mlx4_dbg(mdev, "\n\t ========== MLX
WQE ==========\n");<BR>+ mlx4_dbg(mdev, "\t
owner
%02x", p->owner);<BR>+ mlx4_dbg(mdev, "\t
opcode %02x",
p->opcode);<BR>+ mlx4_dbg(mdev, "\t
size
%02x", p->size);<BR>+ mlx4_dbg(mdev, "\t
flags
%08x", <BR>+ be32_to_cpu(p->flags));<BR>+ mlx4_dbg(mdev, "\t
rlid
%04x", <BR>+ be16_to_cpu(p->rlid));<BR>+}<BR>+<BR>+void
st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp *sqp, struct
mlx4_wqe_mlx_seg *mlx )<BR>+{<BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_UDH
)<BR>+ __print_ud_header( mdev, &sqp->ud_header );<BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WQE
)<BR>+ __print_mlx( mdev, mlx );<BR>+}<BR>+<BR>+void
st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp, ib_send_wr_t
*wr)<BR>+{<BR>+ struct mlx4_ib_qp *qp = to_mqp(ibqp);<BR>+<BR>+ if (
mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WR )
{<BR>+ mlx4_dbg(mdev, "\n\t ========== SEND WR on QP %#x (%#x)
==========\n",<BR>+ ibqp->qp_num, qp->mqp.qpn
);<BR>+ mlx4_dbg(mdev, "\t
wr_type %d",
wr->wr_type);<BR>+ mlx4_dbg(mdev, "\t
wr_id
%08I64x", wr->wr_id);<BR>+ mlx4_dbg(mdev, "\t
send_opt %x",
wr->send_opt);<BR>+ mlx4_dbg(mdev, "\t
immediate_data %x",
be32_to_cpu(wr->immediate_data));<BR>+ mlx4_dbg(mdev, "\t
num_ds %d",
wr->num_ds);<BR>+ mlx4_dbg(mdev, "\t
ds[0].va %08x",
wr->ds_array[0].vaddr);<BR>+ mlx4_dbg(mdev, "\t
ds[0].length %x",
wr->ds_array[0].length);<BR>+ mlx4_dbg(mdev, "\t
ds[0].lkey %x",
wr->ds_array[0].lkey);<BR>+ }<BR>+}<BR>+<BR>+void st_dev_rmv(
PMLX4_ST_DEVICE p_stat )<BR>+{<BR>+ if ( p_stat
)<BR>+ p_stat->valid = FALSE;<BR>+}<BR>+<BR>+PMLX4_ST_DEVICE
st_dev_add()<BR>+{<BR>+ int i;<BR>+<BR>+ for ( i = 0; i <
MLX4_ST_MAX_DEVICES; ++i ) {<BR>+ if ( g_stat.dev[i].valid == FALSE )
{<BR>+ g_stat.dev[i].valid = TRUE;<BR>+ return
&g_stat.dev[i];<BR>+ }<BR>+ }<BR>+<BR>+ return
NULL;<BR>+}<BR>+<BR>+<BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 0)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 5099)<BR>@@ -0,0
+1,85 @@<BR>+/*++<BR>+<BR>+Copyright (c) 2005-2009 Mellanox Technologies. All
rights reserved.<BR>+<BR>+Module
Name:<BR>+ bus_stat.h<BR>+<BR>+Abstract:<BR>+ Statistics Collector
header file<BR>+<BR>+Revision
History:<BR>+<BR>+Notes:<BR>+<BR>+--*/<BR>+<BR>+#include
<wdf.h><BR>+#include "l2w.h"<BR>+#include "ib_pack.h"<BR>+#include
"qp.h"<BR>+#include "device.h"<BR>+<BR>+//<BR>+//
restrictions<BR>+//<BR>+<BR>+#define
MLX4_ST_MAX_DEVICES 8<BR>+<BR>+//<BR>+// enums<BR>+//
<BR>+<BR>+#define MLX4_MAD_TRACE_LRH (1 << 0)<BR>+#define
MLX4_MAD_TRACE_BTH (1 << 1)<BR>+#define
MLX4_MAD_TRACE_DETH (1 << 2)<BR>+#define
MLX4_MAD_TRACE_GRH (1 << 3)<BR>+#define
MLX4_MAD_TRACE_WQE (1 << 4)<BR>+#define
MLX4_MAD_TRACE_UDH (1 << 5)<BR>+#define
MLX4_MAD_TRACE_WR (1 << 6)<BR>+<BR>+<BR>+<BR>+//<BR>+//
structures<BR>+//<BR>+<BR>+// device<BR>+<BR>+typedef struct
_MLX4_ST_DEVICE<BR>+{<BR>+ boolean_t valid;<BR>+ PFDO_DEVICE_DATA p_fdo;<BR>+ WDFDEVICE h_wdf_device;<BR>+ ULONG flags;<BR>+ <BR>+}
MLX4_ST_DEVICE, *PMLX4_ST_DEVICE;<BR>+<BR>+// driver<BR>+typedef struct
_MLX4_ST_DRIVER<BR>+{<BR>+ GLOBALS *p_globals;<BR>+ WDFDRIVER h_wdf_driver; <BR>+ <BR>+}
MLX4_ST_DRIVER, *PMLX4_ST_DRIVER;<BR>+<BR>+// driver stack<BR>+<BR>+typedef
struct
_MLX4_ST_STAT<BR>+{<BR>+ MLX4_ST_DRIVER drv;<BR>+ MLX4_ST_DEVICE dev[MLX4_ST_MAX_DEVICES];<BR>+ <BR>+}
MLX4_ST_STAT, *PMLX4_ST_STAT;<BR>+<BR>+extern MLX4_ST_STAT g_stat;<BR>+<BR>+void
st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp *sqp, struct
mlx4_wqe_mlx_seg *mlx );<BR>+<BR>+void st_print_mlx_send(struct mlx4_dev *mdev,
struct ib_qp *ibqp, ib_send_wr_t *wr);<BR>+<BR>+void st_dev_rmv( PMLX4_ST_DEVICE
p_stat );<BR>+<BR>+PMLX4_ST_DEVICE st_dev_add();<BR>+<BR>+#pragma once<BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5099)<BR>@@
-582,6 +582,8 @@<BR> if (TargetState >
WdfPowerDeviceD0)<BR> __stop_card( p_fdo
);<BR> <BR>+ st_dev_rmv( p_fdo->pci_dev.p_stat_dev
);<BR>+<BR> MLX4_EXIT( MLX4_DBG_DRV );<BR> return
STATUS_SUCCESS;<BR> }<BR>@@ -1154,6 +1156,14
@@<BR> <BR> #endif<BR> <BR>+ //
statistics<BR>+ p_fdo->pci_dev.p_stat_dev = st_dev_add();<BR>+ if (
p_fdo->pci_dev.p_stat_dev )
{<BR>+ p_fdo->pci_dev.p_stat_dev->p_fdo =
p_fdo;<BR>+ p_fdo->pci_dev.p_stat_dev->h_wdf_device =
device;<BR>+ p_fdo->pci_dev.p_stat_dev->flags =
g.mod_stat_flags;<BR>+ }<BR>+<BR> status =
STATUS_SUCCESS;<BR> <BR> end: <BR>@@ -1234,6 +1244,9
@@<BR> //
"ProcessorAffinity"<BR> DECLARE_CONST_UNICODE_STRING(ProcessorAffinity,
L"ProcessorAffinity");<BR> <BR>+ // "Stat
Flags"<BR>+ DECLARE_CONST_UNICODE_STRING(StatFlags,
L"StatFlags");<BR>+<BR> ULONG value;<BR> WDFKEY hKey =
NULL;<BR> NTSTATUS status = STATUS_SUCCESS;<BR>@@ -1332,6 +1345,12
@@<BR> else<BR> g.mod_affinity =
0;<BR> <BR>+ status = WdfRegistryQueryULong(hKey,
&StatFlags, &value);<BR>+ if (NT_SUCCESS (status))
<BR>+ g.mod_stat_flags =
value;<BR>+ else<BR>+ g.mod_stat_flags =
0;<BR>+ <BR> <BR> WdfRegistryClose(hKey);<BR> status
= STATUS_SUCCESS;<BR>@@ -1424,6 +1443,11
@@<BR> //<BR> status =
__read_registry(&hDriver);<BR> <BR>+ //
statistics<BR>+ RtlZeroMemory( &g_stat, sizeof(g_stat)
);<BR>+ g_stat.drv.p_globals = &g;<BR>+ g_stat.drv.h_wdf_driver =
hDriver;<BR>+ <BR> // we don't matter the failure in the work
with Registry<BR> status = STATUS_SUCCESS;<BR> <BR>Index:
V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h<BR>===================================================================<BR>---
V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5098)<BR>+++
V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5099)<BR>@@ -162,6
+162,8 @@<BR> int ref_cnt; /* number of users
*/<BR> };<BR> <BR>+typedef struct _MLX4_ST_DEVICE
*PMLX4_ST_DEVICE;<BR>+<BR> // interface structure between Upper and Low
Layers of the driver<BR> struct pci_dev<BR> {<BR>@@ -200,6 +202,8
@@<BR> ULONG version;<BR> int
legacy_connect;<BR> #endif <BR>+ //
statistics<BR>+ PMLX4_ST_DEVICE
p_stat_dev;<BR> };<BR> <BR> /* DPC
*/<BR></DIV></FONT></BODY></HTML>