[ofw][patch][MLX4] added support for g_stat structure for MLX4_BUS driver + MAD tracing mechanism.

Leonid Keller leonid at mellanox.co.il
Mon Nov 16 00:49:24 PST 2009


MAD tracing mechanism works under the control of flags, set in new
Registry mlx4_bus parameter StatFlags.

The mechanism prints to debugger the IB headers of packets sent over MLX
transport, i.e. on QP0 and QP1.

The flags are:

0x0001 - print LRH

0x0002 - print BTH

0x0004 - print DETH

0x0008 - print GRH (it won't print if the GRH is absent)

0x0010 - print some WQE info

0x0020 - print some more UD header info

0x0040 - print some send WR info

 
 
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/net/mlx4.h (revision 5099)
@@ -93,6 +93,7 @@
  int mod_interrupt_from_first;
 
  int mod_affinity;
+ int mod_stat_flags;
  PFDO_DEVICE_DATA p_fdo[MAX_HCA_CARDS]; // for debug purposes
 } GLOBALS;
 #pragma warning(default:4201) // nameless struct/union
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/inc/qp.h (revision 5099)
@@ -34,6 +34,8 @@
 #define MLX4_QP_H
 
 #include "device.h"
+#include "ib\mlx4_ib.h"
+#include "ib_pack.h"
 
 #define MLX4_INVALID_LKEY 0x100
 
@@ -288,7 +290,22 @@
  __be32   byte_count;
 };
 
+enum {
+ /*
+  * Largest possible UD header: send with GRH and immediate data.
+  */
+ MLX4_IB_UD_HEADER_SIZE  = 72
+};
 
+struct mlx4_ib_sqp {
+ struct mlx4_ib_qp qp;
+ int   pkey_index;
+ u32   qkey;
+ u32   send_psn;
+ struct ib_ud_header ud_header;
+ u8   header_buf[MLX4_IB_UD_HEADER_SIZE];
+};
+
 int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
     struct mlx4_qp_context *context);
 
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/qp.c (revision 5099)
@@ -36,6 +36,9 @@
 #include "qp.h"
 #include "user.h"
 
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx );
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr);
+
 enum {
  MLX4_IB_ACK_REQ_FREQ = 8,
 };
@@ -46,22 +49,6 @@
 };
 
 enum {
- /*
-  * Largest possible UD header: send with GRH and immediate data.
-  */
- MLX4_IB_UD_HEADER_SIZE  = 72
-};
-
-struct mlx4_ib_sqp {
- struct mlx4_ib_qp qp;
- int   pkey_index;
- u32   qkey;
- u32   send_psn;
- struct ib_ud_header ud_header;
- u8   header_buf[MLX4_IB_UD_HEADER_SIZE];
-};
-
-enum {
  MLX4_IB_MIN_SQ_STRIDE = 6
 };
 
@@ -1541,6 +1528,10 @@
     if (bad_wr)
      *bad_wr = wr;
     goto out;
+    
+    /* statistics */
+    st_print_mlx_send( dev, ibqp, wr);
+    st_print_mlx_header( dev, to_msqp(qp), (void*)ctrl );
    }
    wqe  += err;
    size += err / 16;
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/ib/SOURCES (revision 5099)
@@ -21,7 +21,7 @@
  main.c   \
  mr.c   \
  qp.c   \
- srq.c   \
+ srq.c   
 
 
INCLUDES=..;..\inc;..\..\inc;..\core\$O;..\..\..\inc;..\..\..\..\..\inc;
..\..\..\..\..\inc\kernel;
 
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/sources (revision 5099)
@@ -30,6 +30,7 @@
  pci.c \
  pdo.c \
  wmi.c \
+ stat.c
 
 PRECOMPILED_INCLUDE=precomp.h
 
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/precomp.h (revision 5099)
@@ -11,5 +11,6 @@
 #include "driver.h"
 #include "cmd.h"
 #include <mlx4_debug.h>
+#include "stat.h"
 
 
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 0)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.c (revision 5099)
@@ -0,0 +1,159 @@
+/*++
+
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.
+
+Module Name:
+ bus_stat.h
+
+Abstract:
+ Statistics Collector header file
+
+Revision History:
+
+Notes:
+
+--*/
+
+#include "precomp.h"
+
+MLX4_ST_STAT g_stat;
+
+static void __print_grh( struct mlx4_dev *mdev, struct ib_unpacked_grh
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== GRH ==========\n");
+ mlx4_dbg(mdev, "\t ip_version        %02x", p->ip_version);
+ mlx4_dbg(mdev, "\t traffic_class     %02x", p->traffic_class);
+ mlx4_dbg(mdev, "\t flow_label        %08x", 
+  be32_to_cpu(p->flow_label));
+ mlx4_dbg(mdev, "\t payload_length    %04x", 
+  be16_to_cpu(p->payload_length));
+ mlx4_dbg(mdev, "\t next_header       %02x", p->next_header);
+ mlx4_dbg(mdev, "\t hop_limit         %02x", p->hop_limit);
+ mlx4_dbg(mdev, "\t source_gid        %08I64x:%08I64", 
+  be64_to_cpu(p->source_gid.global.subnet_prefix),
+  be64_to_cpu(p->source_gid.global.interface_id));
+ mlx4_dbg(mdev, "\t source_gid        %08I64x:%08I64", 
+  be64_to_cpu(p->destination_gid.global.subnet_prefix),
+  be64_to_cpu(p->destination_gid.global.interface_id));
+}
+
+static void __print_deth( struct mlx4_dev *mdev, struct
ib_unpacked_deth *p)
+{
+ mlx4_dbg(mdev, "\n\t ========== DETH ==========\n");
+ mlx4_dbg(mdev, "\t qkey              %08x", 
+  be32_to_cpu(p->qkey));
+ mlx4_dbg(mdev, "\t source_qpn        %08x", 
+  be32_to_cpu(p->source_qpn));
+}
+
+static void __print_bth( struct mlx4_dev *mdev, struct ib_unpacked_bth
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== BTH ==========\n");
+ mlx4_dbg(mdev, "\t opcode            %02x", p->opcode);
+ mlx4_dbg(mdev, "\t solicited_event   %02x", p->solicited_event);
+ mlx4_dbg(mdev, "\t mig_req           %02x", p->mig_req);
+ mlx4_dbg(mdev, "\t header_version    %02x",
p->transport_header_version);
+ mlx4_dbg(mdev, "\t pkey              %04x", 
+  be16_to_cpu(p->pkey));
+ mlx4_dbg(mdev, "\t destination_qpn   %08x", 
+  be32_to_cpu(p->destination_qpn));
+ mlx4_dbg(mdev, "\t ack_req           %02x", p->ack_req);
+ mlx4_dbg(mdev, "\t psn               %08x", 
+  be32_to_cpu(p->psn));
+}
+
+static void __print_lrh( struct mlx4_dev *mdev, struct ib_unpacked_lrh
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== LRH ==========\n");
+ mlx4_dbg(mdev, "\t virtual_lane      %02x", p->virtual_lane);
+ mlx4_dbg(mdev, "\t link_version      %02x", p->link_version);
+ mlx4_dbg(mdev, "\t service_level     %02x", p->service_level);
+ mlx4_dbg(mdev, "\t link_next_header  %02x", p->link_next_header);
+ mlx4_dbg(mdev, "\t destination_lid   %04x", 
+  be16_to_cpu(p->destination_lid));
+ mlx4_dbg(mdev, "\t packet_length     %04x", 
+  be16_to_cpu(p->packet_length));
+ mlx4_dbg(mdev, "\t source_lid        %04x", 
+  be16_to_cpu(p->source_lid));
+}
+
+static void __print_ud_header( struct mlx4_dev *mdev, struct
ib_ud_header *p)
+{
+ mlx4_dbg(mdev, "\n\t ========== UD HEADER ==========\n");
+
+ mlx4_dbg(mdev, "\t grh_present %d, imm_present %d, imm %08x",
+  p->grh_present, p->immediate_present, be32_to_cpu(p->immediate_data)
);
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_LRH )
+  __print_lrh( mdev, &p->lrh );
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_BTH )
+  __print_bth( mdev, &p->bth );
+ 
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_DETH )
+  __print_deth( mdev, &p->deth );
+ 
+ if ( p->grh_present && (mdev->pdev->p_stat_dev->flags &
MLX4_MAD_TRACE_GRH) )
+  __print_grh( mdev, &p->grh );
+ 
+}
+
+static void __print_mlx( struct mlx4_dev *mdev, struct mlx4_wqe_mlx_seg
*p)
+{
+ mlx4_dbg(mdev, "\n\t ========== MLX WQE ==========\n");
+ mlx4_dbg(mdev, "\t owner             %02x", p->owner);
+ mlx4_dbg(mdev, "\t opcode            %02x", p->opcode);
+ mlx4_dbg(mdev, "\t size              %02x", p->size);
+ mlx4_dbg(mdev, "\t flags             %08x", 
+  be32_to_cpu(p->flags));
+ mlx4_dbg(mdev, "\t rlid              %04x", 
+  be16_to_cpu(p->rlid));
+}
+
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx )
+{
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_UDH )
+ __print_ud_header( mdev, &sqp->ud_header );
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WQE )
+  __print_mlx( mdev, mlx );
+}
+
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr)
+{
+ struct mlx4_ib_qp *qp = to_mqp(ibqp);
+
+ if ( mdev->pdev->p_stat_dev->flags & MLX4_MAD_TRACE_WR ) {
+  mlx4_dbg(mdev, "\n\t ========== SEND WR on QP %#x (%#x)
==========\n",
+   ibqp->qp_num, qp->mqp.qpn );
+  mlx4_dbg(mdev, "\t wr_type           %d", wr->wr_type);
+  mlx4_dbg(mdev, "\t wr_id             %08I64x", wr->wr_id);
+  mlx4_dbg(mdev, "\t send_opt          %x", wr->send_opt);
+  mlx4_dbg(mdev, "\t immediate_data    %x",
be32_to_cpu(wr->immediate_data));
+  mlx4_dbg(mdev, "\t num_ds            %d", wr->num_ds);
+  mlx4_dbg(mdev, "\t ds[0].va          %08x", wr->ds_array[0].vaddr);
+  mlx4_dbg(mdev, "\t ds[0].length      %x", wr->ds_array[0].length);
+  mlx4_dbg(mdev, "\t ds[0].lkey        %x", wr->ds_array[0].lkey);
+ }
+}
+
+void st_dev_rmv( PMLX4_ST_DEVICE p_stat )
+{
+ if ( p_stat )
+  p_stat->valid = FALSE;
+}
+
+PMLX4_ST_DEVICE st_dev_add()
+{
+ int i;
+
+ for ( i = 0; i < MLX4_ST_MAX_DEVICES; ++i ) {
+  if ( g_stat.dev[i].valid == FALSE ) {
+   g_stat.dev[i].valid = TRUE;
+   return &g_stat.dev[i];
+  }
+ }
+
+ return NULL;
+}
+
+
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 0)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/stat.h (revision 5099)
@@ -0,0 +1,85 @@
+/*++
+
+Copyright (c) 2005-2009 Mellanox Technologies. All rights reserved.
+
+Module Name:
+ bus_stat.h
+
+Abstract:
+ Statistics Collector header file
+
+Revision History:
+
+Notes:
+
+--*/
+
+#include <wdf.h>
+#include "l2w.h"
+#include "ib_pack.h"
+#include "qp.h"
+#include "device.h"
+
+//
+// restrictions
+//
+
+#define MLX4_ST_MAX_DEVICES   8
+
+//
+// enums
+// 
+
+#define MLX4_MAD_TRACE_LRH   (1 << 0)
+#define MLX4_MAD_TRACE_BTH   (1 << 1)
+#define MLX4_MAD_TRACE_DETH   (1 << 2)
+#define MLX4_MAD_TRACE_GRH   (1 << 3)
+#define MLX4_MAD_TRACE_WQE   (1 << 4)
+#define MLX4_MAD_TRACE_UDH   (1 << 5)
+#define MLX4_MAD_TRACE_WR   (1 << 6)
+
+
+
+//
+// structures
+//
+
+// device
+
+typedef struct _MLX4_ST_DEVICE
+{
+ boolean_t   valid;
+ PFDO_DEVICE_DATA p_fdo;
+ WDFDEVICE   h_wdf_device;
+ ULONG    flags;
+ 
+} MLX4_ST_DEVICE, *PMLX4_ST_DEVICE;
+
+// driver
+typedef struct _MLX4_ST_DRIVER
+{
+ GLOBALS    *p_globals;
+ WDFDRIVER   h_wdf_driver; 
+ 
+} MLX4_ST_DRIVER, *PMLX4_ST_DRIVER;
+
+// driver stack
+
+typedef struct _MLX4_ST_STAT
+{
+ MLX4_ST_DRIVER  drv;
+ MLX4_ST_DEVICE  dev[MLX4_ST_MAX_DEVICES];
+ 
+} MLX4_ST_STAT, *PMLX4_ST_STAT;
+
+extern MLX4_ST_STAT g_stat;
+
+void st_print_mlx_header( struct mlx4_dev *mdev, struct mlx4_ib_sqp
*sqp, struct mlx4_wqe_mlx_seg *mlx );
+
+void st_print_mlx_send(struct mlx4_dev *mdev, struct ib_qp *ibqp,
ib_send_wr_t *wr);
+
+void st_dev_rmv( PMLX4_ST_DEVICE p_stat );
+
+PMLX4_ST_DEVICE st_dev_add();
+
+#pragma once
Index: V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/bus/drv/drv.c (revision 5099)
@@ -582,6 +582,8 @@
  if (TargetState > WdfPowerDeviceD0)
   __stop_card( p_fdo );
 
+ st_dev_rmv( p_fdo->pci_dev.p_stat_dev );
+
  MLX4_EXIT( MLX4_DBG_DRV );
  return STATUS_SUCCESS;
 }
@@ -1154,6 +1156,14 @@
 
 #endif
 
+ // statistics
+ p_fdo->pci_dev.p_stat_dev = st_dev_add();
+ if ( p_fdo->pci_dev.p_stat_dev ) {
+  p_fdo->pci_dev.p_stat_dev->p_fdo = p_fdo;
+  p_fdo->pci_dev.p_stat_dev->h_wdf_device = device;
+  p_fdo->pci_dev.p_stat_dev->flags = g.mod_stat_flags;
+ }
+
  status = STATUS_SUCCESS;
 
 end: 
@@ -1234,6 +1244,9 @@
  // "ProcessorAffinity"
  DECLARE_CONST_UNICODE_STRING(ProcessorAffinity, L"ProcessorAffinity");
 
+ // "Stat Flags"
+ DECLARE_CONST_UNICODE_STRING(StatFlags, L"StatFlags");
+
  ULONG value;
  WDFKEY hKey = NULL;
  NTSTATUS status = STATUS_SUCCESS;
@@ -1332,6 +1345,12 @@
   else
    g.mod_affinity = 0;
   
+  status = WdfRegistryQueryULong(hKey, &StatFlags, &value);
+  if (NT_SUCCESS (status)) 
+   g.mod_stat_flags = value;
+  else
+   g.mod_stat_flags = 0;
+  
 
   WdfRegistryClose(hKey);
   status = STATUS_SUCCESS;
@@ -1424,6 +1443,11 @@
  //
  status = __read_registry(&hDriver);
 
+ // statistics
+ RtlZeroMemory( &g_stat, sizeof(g_stat) );
+ g_stat.drv.p_globals = &g;
+ g_stat.drv.h_wdf_driver = hDriver;
+ 
  // we don't matter the failure in the work with Registry
  status = STATUS_SUCCESS;
  
Index: V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h
===================================================================
--- V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5098)
+++ V:/svn/winib/trunk/hw/mlx4/kernel/inc/l2w.h (revision 5099)
@@ -162,6 +162,8 @@
  int   ref_cnt; /* number of users */
 };
 
+typedef struct _MLX4_ST_DEVICE *PMLX4_ST_DEVICE;
+
 // interface structure between Upper and Low Layers of the driver
 struct pci_dev
 {
@@ -200,6 +202,8 @@
  ULONG       version;
  int        legacy_connect;
 #endif 
+ // statistics
+ PMLX4_ST_DEVICE     p_stat_dev;
 };
 
 /* DPC */

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20091116/15470188/attachment.html>


More information about the ofw mailing list