[ofw][patch][MLX4] added support for g_stat structure for MLX4_BUS driver + MAD tracing mechanism.
Leonid Keller
leonid at mellanox.co.il
Mon Nov 16 00:49:24 PST 2009
MAD tracing mechanism works under the control of flags, set in new
Registry mlx4_bus parameter StatFlags.
The mechanism prints to debugger the IB headers of packets sent over MLX
transport, i.e. on QP0 and QP1.
The flags are:
0x0001 - print LRH
0x0002 - print BTH
0x0004 - print DETH
0x0008 - print GRH (it won't print if the GRH is absent)
0x0010 - print some WQE info
0x0020 - print some more UD header info
0x0040 - print some send WR info
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5099)
@@ -93,6 +93,7 @@
int mod_interrupt_from_first;
int mod_affinity;
+ int mod_stat_flags;
PFDO_DEVICE_DATA p_fdo[MAX_HCA_CARDS]; // for debug purposes
} GLOBALS;
#pragma warning(default:4201) // nameless struct/union
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5099)
@@ -34,6 +34,8 @@
#define MLX4_QP_H
#include "device.h"
+#include "ib\mlx4_ib.h"
+#include "ib_pack.h"
#define MLX4_INVALID_LKEY 0x100
@@ -288,7 +290,22 @@
__be32 byte_count;
};
+enum {
+ /*
+ * Largest possible UD header: send with GRH and immediate data.
+ */
+ MLX4_IB_UD_HEADER_SIZE = 72
+};
+struct mlx4_ib_sqp {
+ struct mlx4_ib_qp qp;
+ int pkey_index;
+ u32 qkey;
+ u32 send_psn;
+ struct ib_ud_header ud_header;
+ u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
+};
+
int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
struct mlx4_qp_context *context);
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5099)
@@ -36,6 +36,9 @@
#include "qp.h"
#include "user.h"
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx );
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr);
+
enum {
MLX4_IB_ACK_REQ_FREQ = 8,
};
@@ -46,22 +49,6 @@
};
enum {
- /*
- * Largest possible UD header: send with GRH and immediate data.
- */
- MLX4_IB_UD_HEADER_SIZE = 72
-};
-
-struct mlx4_ib_sqp {
- struct mlx4_ib_qp qp;
- int pkey_index;
- u32 qkey;
- u32 send_psn;
- struct ib_ud_header ud_header;
- u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
-};
-
-enum {
MLX4_IB_MIN_SQ_STRIDE = 6
};
@@ -1541,6 +1528,10 @@
if (bad_wr)
*bad_wr = wr;
goto out;
+
+ /* statistics */
+ st_print_mlx_send( dev, ibqp, wr);
+ st_print_mlx_header( dev, to_msqp(qp), (void*)ctrl );
}
wqe += err;
size += err / 16;
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5099)
@@ -21,7 +21,7 @@
main.c \
mr.c \
qp.c \
- srq.c \
+ srq.c
INCLUDES=..;..\inc;..\..\inc;..\core\$O;..\..\..\inc;..\..\..\..\..\inc;
..\..\..\..\..\inc\kernel;
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5099)
@@ -30,6 +30,7 @@
pci.c \
pdo.c \
wmi.c \
+ stat.c
PRECOMPILED_INCLUDE=precomp.h
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5099)
@@ -11,5 +11,6 @@
#include "driver.h"
#include "cmd.h"
#include <mlx4_debug.h>
+#include "stat.h"
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 0)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 5099)
@@ -0,0 +1,159 @@
+/*++
+
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.
+
+Module Name:
+ bus_stat.h
+
+Abstract:
+ Statistics Collector header file
+
+Revision History:
+
+Notes:
+
+--*/
+
+#include "precomp.h"
+
+MLX4_ST_STAT g_stat;
+
+static void __print_grh( struct mlx4_dev *mdev, struct ib_unpacked_grh
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== GRH ==========\n");
+ mlx4_dbg(mdev, "\t ip_version %02x", p->ip_version);
+ mlx4_dbg(mdev, "\t traffic_class %02x", p->traffic_class);
+ mlx4_dbg(mdev, "\t flow_label %08x",
+ be32_to_cpu(p->flow_label));
+ mlx4_dbg(mdev, "\t payload_length %04x",
+ be16_to_cpu(p->payload_length));
+ mlx4_dbg(mdev, "\t next_header %02x", p->next_header);
+ mlx4_dbg(mdev, "\t hop_limit %02x", p->hop_limit);
+ mlx4_dbg(mdev, "\t source_gid %08I64x:%08I64",
+ be64_to_cpu(p->source_gid.global.subnet_prefix),
+ be64_to_cpu(p->source_gid.global.interface_id));
+ mlx4_dbg(mdev, "\t source_gid %08I64x:%08I64",
+ be64_to_cpu(p->destination_gid.global.subnet_prefix),
+ be64_to_cpu(p->destination_gid.global.interface_id));
+}
+
+static void __print_deth( struct mlx4_dev *mdev, struct
ib_unpacked_deth *p)
+{
+ mlx4_dbg(mdev, "\n\t ========== DETH ==========\n");
+ mlx4_dbg(mdev, "\t qkey %08x",
+ be32_to_cpu(p->qkey));
+ mlx4_dbg(mdev, "\t source_qpn %08x",
+ be32_to_cpu(p->source_qpn));
+}
+
+static void __print_bth( struct mlx4_dev *mdev, struct ib_unpacked_bth
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== BTH ==========\n");
+ mlx4_dbg(mdev, "\t opcode %02x", p->opcode);
+ mlx4_dbg(mdev, "\t solicited_event %02x", p->solicited_event);
+ mlx4_dbg(mdev, "\t mig_req %02x", p->mig_req);
+ mlx4_dbg(mdev, "\t header_version %02x",
p->transport_header_version);
+ mlx4_dbg(mdev, "\t pkey %04x",
+ be16_to_cpu(p->pkey));
+ mlx4_dbg(mdev, "\t destination_qpn %08x",
+ be32_to_cpu(p->destination_qpn));
+ mlx4_dbg(mdev, "\t ack_req %02x", p->ack_req);
+ mlx4_dbg(mdev, "\t psn %08x",
+ be32_to_cpu(p->psn));
+}
+
+static void __print_lrh( struct mlx4_dev *mdev, struct ib_unpacked_lrh
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== LRH ==========\n");
+ mlx4_dbg(mdev, "\t virtual_lane %02x", p->virtual_lane);
+ mlx4_dbg(mdev, "\t link_version %02x", p->link_version);
+ mlx4_dbg(mdev, "\t service_level %02x", p->service_level);
+ mlx4_dbg(mdev, "\t link_next_header %02x", p->link_next_header);
+ mlx4_dbg(mdev, "\t destination_lid %04x",
+ be16_to_cpu(p->destination_lid));
+ mlx4_dbg(mdev, "\t packet_length %04x",
+ be16_to_cpu(p->packet_length));
+ mlx4_dbg(mdev, "\t source_lid %04x",
+ be16_to_cpu(p->source_lid));
+}
+
+static void __print_ud_header( struct mlx4_dev *mdev, struct
ib_ud_header *p)
+{
+ mlx4_dbg(mdev, "\n\t ========== UD HEADER ==========\n");
+
+ mlx4_dbg(mdev, "\t grh_present %d, imm_present %d, imm %08x",
+ p->grh_present, p->immediate_present, be32_to_cpu(p->immediate_data)
);
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_LRH )
+ __print_lrh( mdev, &p->lrh );
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_BTH )
+ __print_bth( mdev, &p->bth );
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_DETH )
+ __print_deth( mdev, &p->deth );
+
+ if ( p->grh_present && (mdev->pdev->p_stat_dev->flags &
MLX4_MAD_TRACE_GRH) )
+ __print_grh( mdev, &p->grh );
+
+}
+
+static void __print_mlx( struct mlx4_dev *mdev, struct mlx4_wqe_mlx_seg
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== MLX WQE ==========\n");
+ mlx4_dbg(mdev, "\t owner %02x", p->owner);
+ mlx4_dbg(mdev, "\t opcode %02x", p->opcode);
+ mlx4_dbg(mdev, "\t size %02x", p->size);
+ mlx4_dbg(mdev, "\t flags %08x",
+ be32_to_cpu(p->flags));
+ mlx4_dbg(mdev, "\t rlid %04x",
+ be16_to_cpu(p->rlid));
+}
+
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx )
+{
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_UDH )
+ __print_ud_header( mdev, &sqp->ud_header );
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WQE )
+ __print_mlx( mdev, mlx );
+}
+
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr)
+{
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WR ) {
+ mlx4_dbg(mdev, "\n\t ========== SEND WR on QP %#x (%#x)
==========\n",
+ ibqp->qp_num, qp->mqp.qpn );
+ mlx4_dbg(mdev, "\t wr_type %d", wr->wr_type);
+ mlx4_dbg(mdev, "\t wr_id %08I64x", wr->wr_id);
+ mlx4_dbg(mdev, "\t send_opt %x", wr->send_opt);
+ mlx4_dbg(mdev, "\t immediate_data %x",
be32_to_cpu(wr->immediate_data));
+ mlx4_dbg(mdev, "\t num_ds %d", wr->num_ds);
+ mlx4_dbg(mdev, "\t ds[0].va %08x", wr->ds_array[0].vaddr);
+ mlx4_dbg(mdev, "\t ds[0].length %x", wr->ds_array[0].length);
+ mlx4_dbg(mdev, "\t ds[0].lkey %x", wr->ds_array[0].lkey);
+ }
+}
+
+void st_dev_rmv( PMLX4_ST_DEVICE p_stat )
+{
+ if ( p_stat )
+ p_stat->valid = FALSE;
+}
+
+PMLX4_ST_DEVICE st_dev_add()
+{
+ int i;
+
+ for ( i = 0; i < MLX4_ST_MAX_DEVICES; ++i ) {
+ if ( g_stat.dev[i].valid == FALSE ) {
+ g_stat.dev[i].valid = TRUE;
+ return &g_stat.dev[i];
+ }
+ }
+
+ return NULL;
+}
+
+
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 0)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 5099)
@@ -0,0 +1,85 @@
+/*++
+
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.
+
+Module Name:
+ bus_stat.h
+
+Abstract:
+ Statistics Collector header file
+
+Revision History:
+
+Notes:
+
+--*/
+
+#include <wdf.h>
+#include "l2w.h"
+#include "ib_pack.h"
+#include "qp.h"
+#include "device.h"
+
+//
+// restrictions
+//
+
+#define MLX4_ST_MAX_DEVICES 8
+
+//
+// enums
+//
+
+#define MLX4_MAD_TRACE_LRH (1 << 0)
+#define MLX4_MAD_TRACE_BTH (1 << 1)
+#define MLX4_MAD_TRACE_DETH (1 << 2)
+#define MLX4_MAD_TRACE_GRH (1 << 3)
+#define MLX4_MAD_TRACE_WQE (1 << 4)
+#define MLX4_MAD_TRACE_UDH (1 << 5)
+#define MLX4_MAD_TRACE_WR (1 << 6)
+
+
+
+//
+// structures
+//
+
+// device
+
+typedef struct _MLX4_ST_DEVICE
+{
+ boolean_t valid;
+ PFDO_DEVICE_DATA p_fdo;
+ WDFDEVICE h_wdf_device;
+ ULONG flags;
+
+} MLX4_ST_DEVICE, *PMLX4_ST_DEVICE;
+
+// driver
+typedef struct _MLX4_ST_DRIVER
+{
+ GLOBALS *p_globals;
+ WDFDRIVER h_wdf_driver;
+
+} MLX4_ST_DRIVER, *PMLX4_ST_DRIVER;
+
+// driver stack
+
+typedef struct _MLX4_ST_STAT
+{
+ MLX4_ST_DRIVER drv;
+ MLX4_ST_DEVICE dev[MLX4_ST_MAX_DEVICES];
+
+} MLX4_ST_STAT, *PMLX4_ST_STAT;
+
+extern MLX4_ST_STAT g_stat;
+
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx );
+
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr);
+
+void st_dev_rmv( PMLX4_ST_DEVICE p_stat );
+
+PMLX4_ST_DEVICE st_dev_add();
+
+#pragma once
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5099)
@@ -582,6 +582,8 @@
if (TargetState > WdfPowerDeviceD0)
__stop_card( p_fdo );
+ st_dev_rmv( p_fdo->pci_dev.p_stat_dev );
+
MLX4_EXIT( MLX4_DBG_DRV );
return STATUS_SUCCESS;
}
@@ -1154,6 +1156,14 @@
#endif
+ // statistics
+ p_fdo->pci_dev.p_stat_dev = st_dev_add();
+ if ( p_fdo->pci_dev.p_stat_dev ) {
+ p_fdo->pci_dev.p_stat_dev->p_fdo = p_fdo;
+ p_fdo->pci_dev.p_stat_dev->h_wdf_device = device;
+ p_fdo->pci_dev.p_stat_dev->flags = g.mod_stat_flags;
+ }
+
status = STATUS_SUCCESS;
end:
@@ -1234,6 +1244,9 @@
// "ProcessorAffinity"
DECLARE_CONST_UNICODE_STRING(ProcessorAffinity, L"ProcessorAffinity");
+ // "Stat Flags"
+ DECLARE_CONST_UNICODE_STRING(StatFlags, L"StatFlags");
+
ULONG value;
WDFKEY hKey = NULL;
NTSTATUS status = STATUS_SUCCESS;
@@ -1332,6 +1345,12 @@
else
g.mod_affinity = 0;
+ status = WdfRegistryQueryULong(hKey, &StatFlags, &value);
+ if (NT_SUCCESS (status))
+ g.mod_stat_flags = value;
+ else
+ g.mod_stat_flags = 0;
+
WdfRegistryClose(hKey);
status = STATUS_SUCCESS;
@@ -1424,6 +1443,11 @@
//
status = __read_registry(&hDriver);
+ // statistics
+ RtlZeroMemory( &g_stat, sizeof(g_stat) );
+ g_stat.drv.p_globals = &g;
+ g_stat.drv.h_wdf_driver = hDriver;
+
// we don't matter the failure in the work with Registry
status = STATUS_SUCCESS;
Index: V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5099)
@@ -162,6 +162,8 @@
int ref_cnt; /* number of users */
};
+typedef struct _MLX4_ST_DEVICE *PMLX4_ST_DEVICE;
+
// interface structure between Upper and Low Layers of the driver
struct pci_dev
{
@@ -200,6 +202,8 @@
ULONG version;
int legacy_connect;
#endif
+ // statistics
+ PMLX4_ST_DEVICE p_stat_dev;
};
/* DPC */
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20091116/15470188/attachment.html>
More information about the ofw
mailing list