[ofw] Patch: allow mlx4 users to add extra eqs for their internaluse

Sean Hefty sean.hefty at intel.com
Mon Jun 30 10:16:48 PDT 2008


Index: mlx4/kernel/bus/core/SOURCES
===================================================================
--- mlx4/kernel/bus/core/SOURCES (revision 1301)
+++ mlx4/kernel/bus/core/SOURCES (working copy)
@@ -43,7 +43,7 @@
 C_DEFINES = $(C_DEFINES) -DEVENT_TRACING
 
 RUN_WPP = $(SOURCES) -km -ext: .c .h .C .H \
- -scan:..\inc\mlx4_debug.h \
+ -scan:..\..\inc\mlx4_debug.h \
  -func:MLX4_PRINT(LEVEL,FLAGS,(MSG,...)) \
  -func:MLX4_PRINT_EXIT(LEVEL,FLAGS,(MSG,...)) 
 !ENDIF
Index: mlx4/kernel/bus/drv/drv.c
===================================================================
--- mlx4/kernel/bus/drv/drv.c (revision 1301)
+++ mlx4/kernel/bus/drv/drv.c (working copy)
@@ -917,11 +917,11 @@
   //
   status = WdfRegistryQueryULong(hKey, &debugLevel, &value);
   if (NT_SUCCESS (status)) 
-   g_mlx4_dbg_level = g.DebugPrintLevel = value;
+   g_mlx4_dbg_level = g.bwsd.DebugPrintLevel = value;
   
   status = WdfRegistryQueryULong(hKey, &debugFlags, &value);
   if (NT_SUCCESS (status)) 
-   g_mlx4_dbg_flags = g.DebugPrintFlags = value;
+   g_mlx4_dbg_flags = g.bwsd.DebugPrintFlags = value;
 
   status = WdfRegistryQueryULong(hKey, &numQp, &value);
   if (NT_SUCCESS (status)) 
@@ -1051,8 +1051,8 @@
 
 
  // global initializations
- g_mlx4_dbg_level = g.DebugPrintLevel = TRACE_LEVEL_VERBOSE;
- g_mlx4_dbg_flags = g.DebugPrintFlags = 0xffff;
+ g_mlx4_dbg_level = g.bwsd.DebugPrintLevel = TRACE_LEVEL_VERBOSE;
+ g_mlx4_dbg_flags = g.bwsd.DebugPrintFlags = 0xffff;
 
  MLX4_ENTER(MLX4_DBG_DRV);
  MLX4_PRINT(TRACE_LEVEL_INFORMATION, MLX4_DBG_DRV, 
Index: mlx4/kernel/bus/drv/drv.h
===================================================================
--- mlx4/kernel/bus/drv/drv.h (revision 1301)
+++ mlx4/kernel/bus/drv/drv.h (working copy)
@@ -56,6 +56,14 @@
 // The device extension of the bus itself.  From whence the PDO's are born.
 //
 
+
+typedef
+BOOLEAN
+(*PISR_FUNC)( 
+ IN PVOID  IsrContext
+ );
+
+
 typedef struct _FDO_DEVICE_DATA
 {
  BUS_WMI_STD_DATA   WmiData;
Index: mlx4/kernel/bus/drv/pdo.c
===================================================================
--- mlx4/kernel/bus/drv/pdo.c (revision 1301)
+++ mlx4/kernel/bus/drv/pdo.c (working copy)
@@ -226,6 +226,8 @@
 
  p_fdo->bus_ib_ifc.mlx4_interface.mlx4_INIT_PORT = mlx4_INIT_PORT;
  p_fdo->bus_ib_ifc.mlx4_interface.mlx4_CLOSE_PORT = mlx4_CLOSE_PORT;
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_add_eq = mlx4_add_eq;
+ p_fdo->bus_ib_ifc.mlx4_interface.mlx4_remove_eq = mlx4_remove_eq;
 
  
  //
Index: mlx4/kernel/bus/inc/bus_intf.h
===================================================================
--- mlx4/kernel/bus/inc/bus_intf.h (revision 1301)
+++ mlx4/kernel/bus/inc/bus_intf.h (working copy)
@@ -12,9 +12,9 @@
 enum mlx4_qp_state;
 struct mlx4_qp_context;
 enum mlx4_qp_optpar;
+struct mlx4_eq;
 
 
-
 typedef int (*MLX4_REGISTER_INTERFACE)(struct mlx4_interface *intf);
 typedef VOID (*MLX4_UNREGISTER_INTERFACE)(struct mlx4_interface *intf);
 
@@ -98,6 +98,20 @@
 typedef int (*MLX4_CLOSE_PORT)(struct mlx4_dev *dev, int port);
 
 
+typedef
+BOOLEAN
+(*PISR_FUNC)( 
+ IN PVOID  IsrContext
+ );



This is defined twice now.  Why not put it in a common header file?

 

+
+
+typedef int (*MLX4_ADD_EQ) (struct mlx4_dev *dev, int nent,
+     u8 intr, PISR_FUNC func, PVOID func_context ,
+     u8* p_eq_num, struct mlx4_eq ** p_eq);
+
+typedef VOID (*MLX4_REMOVE_EQ) (struct mlx4_dev *dev, u8 eq_num);
+
+
 struct mlx4_interface_ex {
  MLX4_PD_ALLOC       mlx4_pd_alloc;
  MLX4_PD_FREE        mlx4_pd_free;
@@ -141,6 +155,9 @@
 
  MLX4_INIT_PORT         mlx4_INIT_PORT;
  MLX4_CLOSE_PORT        mlx4_CLOSE_PORT;
+
+ MLX4_ADD_EQ            mlx4_add_eq;
+ MLX4_REMOVE_EQ         mlx4_remove_eq;
  
 };
 
@@ -154,7 +171,7 @@
  int       is_livefish;
  MLX4_REGISTER_INTERFACE     register_interface;
  MLX4_UNREGISTER_INTERFACE   unregister_interface;
- ULONG      port_id;
+ u8       port_id;
  struct VipBusIfc   *pVipBusIfc;
  
 } MLX4_BUS_IB_INTERFACE, *PMLX4_BUS_IB_INTERFACE;
Index: mlx4/kernel/bus/net/cq.c
===================================================================
--- mlx4/kernel/bus/net/cq.c (revision 1301)
+++ mlx4/kernel/bus/net/cq.c (working copy)
@@ -127,7 +127,6 @@
  u64 mtt_addr;
  int err;
 
- UNREFERENCED_PARAMETER(vector);
 #define COLLAPSED_SHIFT 18
 #define ENTRIES_SHIFT 24
 
@@ -161,7 +160,11 @@
  cq_context->flags = cpu_to_be32(!!collapsed << COLLAPSED_SHIFT);
  cq_context->logsize_usrpage = cpu_to_be32(
       (ilog2(nent) << ENTRIES_SHIFT) | uar->index);
- cq_context->comp_eqn        = (u8)priv->eq_table.eq[MLX4_EQ_COMP].eqn;
+ if (vector == 0) {
+  cq_context->comp_eqn        = (u8)priv->eq_table.eq[MLX4_EQ_COMP].eqn;
+ } else {
+  cq_context->comp_eqn        = (u8)priv->eq_table.eq[vector].eqn;
+ }
  cq_context->log_page_size   = (u8)(mtt->page_shift - MLX4_ICM_PAGE_SHIFT);
 
  mtt_addr = mlx4_mtt_addr(dev, mtt);
Index: mlx4/kernel/bus/net/eq.c
===================================================================
--- mlx4/kernel/bus/net/eq.c (revision 1303)
+++ mlx4/kernel/bus/net/eq.c (working copy)
@@ -308,6 +308,19 @@
   }
  }
 
+ for (i = MLX4_NUM_EQ; i <= priv->eq_table.max_extra_eqs; ++i) {
+  if (priv->eq_table.eq[i].isr) {
+   int ret = 0;
+   if ( next_eqe_sw(&priv->eq_table.eq[i]) ) {
+    ret = priv->eq_table.eq[i].isr(priv->eq_table.eq[i].ctx);
+    work |= ret;
+   }
+   else {

 

nit: the placement of curly braces around the else is inconsistent in this
patch.


+    eq_set_ci(&priv->eq_table.eq[i], 1);
+   }
+  }
+ }
+
  return (BOOLEAN)work;
 }
 
@@ -708,6 +721,65 @@
  return err;
 }
 
+
+int mlx4_add_eq(struct mlx4_dev *dev, int nent,
+     u8 intr, PISR_FUNC func, PVOID func_context ,
+     u8* p_eq_num, struct mlx4_eq ** p_eq)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int err;
+ u8 i, new_eq = 0;
+ for (i = MLX4_NUM_EQ; i < MLX4_NUM_EQ + MLX4_MAX_EXTRA_EQS ; i++) {
+  if(priv->eq_table.eq[MLX4_NUM_EQ].isr == NULL) {
+   new_eq = i;
+   break;
+  }
+ }
+ if (new_eq == 0)
+  return -ENOMEM;
+
+ err = mlx4_create_eq(dev, nent,
+        (dev->flags & MLX4_FLAG_MSI_X) ? MLX4_EQ_COMP : 0,
+        &priv->eq_table.eq[new_eq]);
+ if (err)
+  return err;
+
+ *p_eq = &priv->eq_table.eq[new_eq ];
+ *p_eq_num = new_eq;
+ priv->eq_table.eq[MLX4_NUM_EQ].isr = func;

 

Does func ever change?  Is it possible to just initialize it in the array once?


+ priv->eq_table.eq[MLX4_NUM_EQ].ctx = func_context;

 

Same question - does this ever change?


+ priv->eq_table.max_extra_eqs = max(priv->eq_table.max_extra_eqs, new_eq);
+ return 0;
+}
+
+void mlx4_remove_eq(struct mlx4_dev *dev, u8 eq_num)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ int err;
+ struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
+
+ priv->eq_table.eq[eq_num].isr = NULL;
+ priv->eq_table.eq[eq_num].ctx = NULL;
+
+ if (priv->eq_table.max_extra_eqs == eq_num)
+  priv->eq_table.max_extra_eqs--;
+
+ mlx4_free_eq(dev, &priv->eq_table.eq[eq_num]);
+
+
+ if (eq_table->have_irq) {
+  free_irq(dev->pdev->int_obj);
+
+
+  err = request_irq( dev, 
+   dev->pdev->int_info.u.Interrupt.Vector,
+   mlx4_interrupt, dev, 
+   mlx4_dpc, &priv->eq_table.eq[0],
+   &dev->pdev->int_obj );
+  // BUGBUG: how should the error be propogated ?
+ }
+}
+
 void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
 {
  struct mlx4_priv *priv = mlx4_priv(dev);
Index: mlx4/kernel/bus/net/mlx4.h
===================================================================
--- mlx4/kernel/bus/net/mlx4.h (revision 1301)
+++ mlx4/kernel/bus/net/mlx4.h (working copy)
@@ -68,7 +68,7 @@
 
 #pragma warning(disable:4201) // nameless struct/union
 typedef struct _GLOBALS {
- BUS_WMI_STD_DATA;
+ BUS_WMI_STD_DATA bwsd;
 
  int mod_num_qp;
  int mod_rdmarc_per_qp;
@@ -109,6 +109,8 @@
  MLX4_NUM_EQ
 };
 
+#define MLX4_MAX_EXTRA_EQS 5
+
 enum {
  MLX4_NUM_PDS  = 1 << 15
 };
@@ -172,8 +174,9 @@
  KDPC  dpc;  /* DPC routine */
  spinlock_t lock;  /* spinlock for simult DPCs */
  int   eq_ix;  /* EQ index - 0..MLX4_NUM_EQ */
- BOOLEAN (*isr)(int,void*); /* isr */
+ BOOLEAN (*isr)(void*); /* isr */

 

Is this the same as the two function typedef's above?  Are the typedefs even
needed then?


  void *  ctx;  /* isr ctx */
+ USHORT eq_no_progress;  /* used to look for stacked card */
 };
 
 struct mlx4_profile {
@@ -238,13 +241,14 @@
  void __iomem        *clr_int;
  u8 __iomem        *uar_map[(MLX4_NUM_EQ + 6) / 4];
  u32   clr_mask;
- struct mlx4_eq  eq[MLX4_NUM_EQ];
+ struct mlx4_eq  eq[MLX4_NUM_EQ + MLX4_MAX_EXTRA_EQS];
  u64   icm_virt;
  dma_addr_t icm_page;
  dma_addr_t  icm_dma;
  struct mlx4_icm_table cmpt_table;
  int   have_irq;
  u8   inta_pin;
+ u8      max_extra_eqs;

 

Can you explain what this is for?  You have MLX4_MAX_EXTRA_EQS.  I see where
it's initialized on create eq, then possibly decremented in remove.


 };
 
 struct mlx4_srq_table {
@@ -432,4 +436,16 @@
 #define ETH_FCS_LEN 4  /* Frame Check Sequence Length   */
 #define ETH_HLEN 14
 
+
+typedef
+BOOLEAN
+(*PISR_FUNC)( 
+ IN PVOID  IsrContext
+ );



A third definition.

 

+
+int mlx4_add_eq(struct mlx4_dev *dev, int nent,
+     u8 intr, PISR_FUNC func,PVOID func_context ,
+     u8* p_eq_num, struct mlx4_eq ** p_eq);
+
+void mlx4_remove_eq(struct mlx4_dev *dev, u8 eq_num);
 #endif /* MLX4_H */
Index: mlx4/kernel/inc/vip_dev.h
===================================================================
--- mlx4/kernel/inc/vip_dev.h (revision 1301)
+++ mlx4/kernel/inc/vip_dev.h (working copy)
@@ -25,7 +25,7 @@
 
 #define MTNIC_MAX_PORTS     2
 
-#define MAX_PORT_SIZE 120000
+#define MAX_PORT_SIZE 250000

 

This looks unrelated.


 #define MXE_INTERFACE_VERSION 2
 
 enum mtnic_state {
@@ -47,13 +47,15 @@
     LONG            ResetCount;
 
     // Objects that are needed in order to work with the hw
-//    struct mlx4_dev        *dev;
 
     u32                     priv_pdn;
     struct mlx4_uar         priv_uar;
     void __iomem            *uar_map;
     struct mlx4_mr          mr;
     spinlock_t              uar_lock;
+
+    struct mlx4_eq *        eq;
+    u8                      eq_number;
 } NicData_t;
 
 //typedef struct _VipBusIfc

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080630/3554005b/attachment.html>


More information about the ofw mailing list