[ofw] RE: [PATCH] mtcr

Fab Tillier ftillier at windows.microsoft.com
Wed Apr 9 09:54:00 PDT 2008


What is WA?

From: ofw-bounces at lists.openfabrics.org [mailto:ofw-bounces at lists.openfabrics.org] On Behalf Of Reuven Amitai
Sent: Wednesday, April 09, 2008 6:27 AM
To: ofw at lists.openfabrics.org
Subject: [ofw] [PATCH] mtcr

Hi,

The following patch adds WA for Hermon chip.

Thanks, Reuven.

Index: tools/mtcr/user/mtcr.c
===================================================================
--- tools/mtcr/user/mtcr.c    (revision 1047)
+++ tools/mtcr/user/mtcr.c    (working copy)
@@ -21,33 +21,82 @@
 #define USB_DEV_NAME    "mtusb-1"
 #define CLEAR(st)               memset(&(st), 0, sizeof(st))

+// HERMON ORDERING WA:
+#define HERMON_WA_BASE 0xf0384 // SEM BASE ADDR. SEM 0xf0380 is reserved for external tools usage.
+#define HERMON_WA_SIZE 3       // Size in entries

+#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL"

-#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL"
+/*
+ * DLL global variables
+ */
+#pragma data_seg(".sdata")
+
+/* Process / thread count */
+DWORD g_Slots[HERMON_WA_SIZE];     /* 3 slots */
+
 #ifdef DBG
 ULONG g_DebugLevel = DEBUG_LEVEL_MID;
 #else
 ULONG g_DebugLevel = DEBUG_LEVEL_LOW;
 #endif

+#pragma data_seg()
+
+
 //-----------------------------------------------------

 #define MAX_HCA_NUM 16

+// flags in below structure
+#define FIX_ORDERING_BUG     1

 typedef struct mfile_ibal_t {
     mfile            s;
     ib_al_handle_t   h_al;
     ib_ca_handle_t   h_ca;
     map_crspace      cr_map;
+
+    int            bugs;
+    int            slot_num;
+    u_int32_t      hermon_wa_slot; /* apply hermon cr write workaround */
+    int            hermon_wa_last_op_write;
+    u_int64_t      hermon_wa_max_retries;
+    u_int64_t      hermon_wa_num_of_writes;
+    u_int64_t      hermon_wa_num_of_retry_writes;
 } mfile_ibal;


+// Return slot_num or -1 when no free slots
+static int __get_slot()
+{
+     int i, prev_val;
+
+     for (i = 0; i < HERMON_WA_SIZE; ++i) {
+           prev_val = InterlockedCompareExchange(
+                 (LONG volatile* )&g_Slots[i], 0, 1 );
+           if ( prev_val )
+                 return i;
+     }
+     return -1;
+}
+
+static void __put_slot(int slot_num)
+{
+     int prev_val;
+     prev_val = InterlockedExchange(
+           (LONG volatile* )&g_Slots[slot_num], 1 );
+     if ( !prev_val )
+           DPRINT1(("Wrong semaphore %d value \n", slot_num));
+
+}
+
 BOOL APIENTRY DllMain( HANDLE hModule,
                        DWORD  ul_reason_for_call,
                        LPVOID lpReserved
                                          )
 {
+    int i;
     char* pszDbgLevel;
     switch (ul_reason_for_call)
     {
@@ -59,6 +108,9 @@
         if (pszDbgLevel) {
             g_DebugLevel = atol(pszDbgLevel);
         }
+           // set slot data
+           for (i = 0; i < HERMON_WA_SIZE; ++i)
+                 g_Slots[i] = 1;

         break;

@@ -85,13 +137,20 @@
 #define SINAI_4X_CONF_DEV_ID                            24205
 #define SINAI_8X_DEV_ID                                 25204
 #define SINAI_8X_CONF_DEV_ID                            25205
+#define HERMON_SDR_DEV_ID                               25408
+#define HERMON_DDR_DEV_ID                               25418
+#define HERMON_QDR_DEV_ID                               25428
+#define HERMON_DDR_PCI5_DEV_ID                          26418
+#define HERMON_QDR_PCI5_DEV_ID                          26428
+#define HERMON_CONF_DEV_ID                              401

 #define IS_CONF_DEV(dev_id)     \
-        ((dev_id == TAVOR_CONF_DEV_ID)    || \
-         (dev_id == ARBEL_TM_CONF_DEV_ID) || \
-         (dev_id == ARBEL_CONF_DEV_ID)    || \
-         (dev_id == SINAI_4X_CONF_DEV_ID) || \
-         (dev_id == SINAI_8X_CONF_DEV_ID))
+           ((dev_id == TAVOR_CONF_DEV_ID)    || \
+           (dev_id == ARBEL_TM_CONF_DEV_ID) || \
+           (dev_id == ARBEL_CONF_DEV_ID)    || \
+           (dev_id == SINAI_4X_CONF_DEV_ID) || \
+           (dev_id == SINAI_8X_CONF_DEV_ID) || \
+           (dev_id == HERMON_CONF_DEV_ID))

 #define MAX_DEV_NAME            32
 typedef struct {
@@ -115,7 +174,15 @@
         {       SINAI_4X_DEV_ID,        "InfiniHost_III_Lx",    MDEVS_TAVOR     },
         {       SINAI_4X_CONF_DEV_ID,   "InfiniHostBd",         MDEVS_TAVOR_CR  },
         {       SINAI_8X_DEV_ID,        "InfiniHost_III_Lx",    MDEVS_TAVOR     },
-        {       SINAI_8X_CONF_DEV_ID,   "InfiniHostBd",         MDEVS_TAVOR_CR  },
+        {       SINAI_8X_CONF_DEV_ID,    "InfiniHostBd",         MDEVS_TAVOR_CR    },
+
+                 {     HERMON_SDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR    },
+                 {     HERMON_DDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR    },
+                 {     HERMON_QDR_DEV_ID,      "ConnectX",             MDEVS_TAVOR_CR    },
+                 {     HERMON_DDR_PCI5_DEV_ID, "ConnectX",             MDEVS_TAVOR_CR    },
+                 {     HERMON_QDR_PCI5_DEV_ID, "ConnectX",             MDEVS_TAVOR_CR    },
+
+                 {     HERMON_CONF_DEV_ID, "ConnectXBd",         MDEVS_TAVOR_CR    },
 };
 #define DEVICE_DB_SIZE  (sizeof(db) / sizeof(DEVICE_DB_T))

@@ -123,6 +190,14 @@
 char *dsuffix[] = { "conf", "_cr", "_uar", "_ddr"};
 #define MASKS_SIZE      (sizeof(dmasks) / sizeof(Mdevs))

+static int is_hermon(USHORT dev_id)
+{
+     return (dev_id == HERMON_SDR_DEV_ID) ||
+           (dev_id == HERMON_DDR_DEV_ID) ||
+           (dev_id == HERMON_QDR_DEV_ID) ||
+           (dev_id == HERMON_DDR_PCI5_DEV_ID) ||
+           (dev_id == HERMON_QDR_PCI5_DEV_ID);
+}

 // Return: < 0 - Error.   > 0 - Numbern of characters written (including last '\0')
 int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, int name_len, int *cnt)
@@ -431,8 +506,70 @@
 }


+
 //
+// Hermon PCI ordering bug work around (in kernel):
 //
+// Toggle bit is done using a HW semaphore. 3 slots (sems) are allocated for MST tools and
+// are dynamically allocated by the mst_pci.o module. 1 slot is reserved for external tools usage.
+// This means to more than 3 tools that use the PCI dev can run simultaniously.
+//
+// Open:
+//   If the wa is enabled ("MTCR_HERMON_WA" env variable does NOT exist or exist and equals 1):
+//   and HERMON-A0 dev id is read:
+//   get a vacant slot - ioctl( , PCI_HERMON_WA ,);
+//   read the sem in the slot (to lock it).
+//
+//
+// Flush (hermon_wa_write_confirm_sem):
+//   If enabled on first read after write:
+//   write 0 to sem.
+//   read sem untlill sem = 0;
+//
+// Close/release
+//   Zero the slot.
+//
+
+int hermon_wa_write_confirm_sem(mfile* mf, u_int32_t addr) {
+    u_int64_t max_retries = 0x1000000;
+    u_int64_t retries = 0;
+    mfile_ibal *mfi = (mfile_ibal *)mf;
+
+    mfi->hermon_wa_num_of_writes++;
+
+    // clear sem
+    *((u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)) = 0;
+
+    while (retries < max_retries) {
+         u_int32_t slot_value ;
+
+         slot_value = __be32_to_cpu(*((volatile u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)));
+         if (slot_value == 0) {
+
+             // Good - The written value was read
+             if (retries) {
+                 // NOT REALLY NEEDED - just to collect statistics
+
+                 // printf("-D- Hermon WA addr %06x took %d retries\n", addr, retries);
+                 mfi->hermon_wa_num_of_retry_writes++;
+                 if (retries > mfi->hermon_wa_max_retries) {
+                     mfi->hermon_wa_max_retries = retries;
+                 }
+             }
+             return 4;
+         } else {
+             // sem is still locked from previous read - read passed the write
+             retries++;
+         }
+     }
+
+     DPRINT1(("-D- Hermon WA addr %06x failed after %ld retries\n", addr, retries));
+     return 0;
+}
+
+
+//
+//
 // List devices in their MST compatible names.
 // Each device type is indexed sepetrately.
 //
@@ -629,7 +766,7 @@
             }
         } else {

-            int bar_num;
+            int bar_num, slot_num;

             // Type of file
             mf->s.tp = MST_PCI;
@@ -649,8 +786,36 @@
             }

             mf->s.ptr = (void*)(ULONG_PTR)mf->cr_map.va;
-     }
-
+
+                 // check whether we need to perform the workaround
+                 if (is_hermon(dev_id)) {
+                       char* hermon_wa_env = getenv("MTCR_HERMON_WA");
+                       // if there is no env variable or it = "0" - we perform the workaround
+                       if (hermon_wa_env == NULL || strcmp(hermon_wa_env, "0")) {
+                             // perform the workaround only for A0 revision
+                             if (access_type == MDEVS_TAVOR_CR) {
+                                   u_int32_t rev = 0;
+                                   mread4((mfile*)mf, 0xf0014, &rev);
+                                   if (rev == 0xa00190)  { // TODO: Should I ignore REV here ?
+                                         slot_num = __get_slot();
+                                         if (slot_num < 0) // no free slot
+                                               goto ErrExit;
+                                         mf->bugs |= FIX_ORDERING_BUG;
+                                         mf->slot_num = slot_num;
+                                   }
+                             }
+                       }
+                 }
+
+                 // prepare for the workaround
+                 if (mf->bugs & FIX_ORDERING_BUG) {  // Hermon WA setup
+                       DPRINT1(("-D- Hermon WA setup\n"));
+                       mf->hermon_wa_slot = HERMON_WA_BASE + 4*slot_num;
+                       hermon_wa_write_confirm_sem((mfile*)mf, mf->hermon_wa_slot);
+                 }
+
+           }
+
     } else if (dev_id == DEVASYS_DEV_ID) {
         // Type of file
         h = usb_open();
@@ -702,6 +867,16 @@
         if (mf->tp == MST_PCICONF) {
             ibal_access(mfi->h_ca, 0x0, &stub, 4, FW_CLOSE_IF );
         } else if (mf->tp = MST_PCI) {
+
+            if (mfi->bugs & FIX_ORDERING_BUG) {
+                 __put_slot( mfi->slot_num );
+                    DPRINT1(("-D- Hermon WA stats:\n"));
+                    DPRINT1(("-D- : num of write flushes: %8ld\n", mfi->hermon_wa_num_of_writes));
+                    DPRINT1(("-D- : num of retry flushes: %8ld\n", mfi->hermon_wa_num_of_retry_writes));
+                    DPRINT1(("-D- : max_retries:          %8ld\n", mfi->hermon_wa_max_retries));
+            }
+
+
             if (mfi->cr_map.size) {

                 if (ibal_access(mfi->h_ca, 0x0, NULL, 0, FW_UNMAP_CRSPACE ) != IB_SUCCESS) {


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080409/9dbbd5eb/attachment.html>


More information about the ofw mailing list