[ofw] RE: [PATCH] mtcr
Fab Tillier
ftillier at windows.microsoft.com
Wed Apr 9 10:15:30 PDT 2008
Ok. Can you describe what the problem is that is being worked around?
Thanks,
-Fab
From: Reuven Amitai [mailto:reuven at mellanox.co.il]
Sent: Wednesday, April 09, 2008 10:14 AM
To: Fab Tillier; ofw at lists.openfabrics.org
Subject: RE: [PATCH] mtcr
Workaround.
________________________________
From: Fab Tillier [mailto:ftillier at windows.microsoft.com]
Sent: Wednesday, April 09, 2008 7:54 PM
To: Reuven Amitai; ofw at lists.openfabrics.org
Subject: RE: [PATCH] mtcr
What is WA?
From: ofw-bounces at lists.openfabrics.org [mailto:ofw-bounces at lists.openfabrics.org] On Behalf Of Reuven Amitai
Sent: Wednesday, April 09, 2008 6:27 AM
To: ofw at lists.openfabrics.org
Subject: [ofw] [PATCH] mtcr
Hi,
The following patch adds WA for Hermon chip.
Thanks, Reuven.
Index: tools/mtcr/user/mtcr.c
===================================================================
--- tools/mtcr/user/mtcr.c (revision 1047)
+++ tools/mtcr/user/mtcr.c (working copy)
@@ -21,33 +21,82 @@
#define USB_DEV_NAME "mtusb-1"
#define CLEAR(st) memset(&(st), 0, sizeof(st))
+// HERMON ORDERING WA:
+#define HERMON_WA_BASE 0xf0384 // SEM BASE ADDR. SEM 0xf0380 is reserved for external tools usage.
+#define HERMON_WA_SIZE 3 // Size in entries
+#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL"
-#define MTCR_DEBUG_ENV "MTCR_DEBUG_LEVEL"
+/*
+ * DLL global variables
+ */
+#pragma data_seg(".sdata")
+
+/* Process / thread count */
+DWORD g_Slots[HERMON_WA_SIZE]; /* 3 slots */
+
#ifdef DBG
ULONG g_DebugLevel = DEBUG_LEVEL_MID;
#else
ULONG g_DebugLevel = DEBUG_LEVEL_LOW;
#endif
+#pragma data_seg()
+
+
//-----------------------------------------------------
#define MAX_HCA_NUM 16
+// flags in below structure
+#define FIX_ORDERING_BUG 1
typedef struct mfile_ibal_t {
mfile s;
ib_al_handle_t h_al;
ib_ca_handle_t h_ca;
map_crspace cr_map;
+
+ int bugs;
+ int slot_num;
+ u_int32_t hermon_wa_slot; /* apply hermon cr write workaround */
+ int hermon_wa_last_op_write;
+ u_int64_t hermon_wa_max_retries;
+ u_int64_t hermon_wa_num_of_writes;
+ u_int64_t hermon_wa_num_of_retry_writes;
} mfile_ibal;
+// Return slot_num or -1 when no free slots
+static int __get_slot()
+{
+ int i, prev_val;
+
+ for (i = 0; i < HERMON_WA_SIZE; ++i) {
+ prev_val = InterlockedCompareExchange(
+ (LONG volatile* )&g_Slots[i], 0, 1 );
+ if ( prev_val )
+ return i;
+ }
+ return -1;
+}
+
+static void __put_slot(int slot_num)
+{
+ int prev_val;
+ prev_val = InterlockedExchange(
+ (LONG volatile* )&g_Slots[slot_num], 1 );
+ if ( !prev_val )
+ DPRINT1(("Wrong semaphore %d value \n", slot_num));
+
+}
+
BOOL APIENTRY DllMain( HANDLE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
+ int i;
char* pszDbgLevel;
switch (ul_reason_for_call)
{
@@ -59,6 +108,9 @@
if (pszDbgLevel) {
g_DebugLevel = atol(pszDbgLevel);
}
+ // set slot data
+ for (i = 0; i < HERMON_WA_SIZE; ++i)
+ g_Slots[i] = 1;
break;
@@ -85,13 +137,20 @@
#define SINAI_4X_CONF_DEV_ID 24205
#define SINAI_8X_DEV_ID 25204
#define SINAI_8X_CONF_DEV_ID 25205
+#define HERMON_SDR_DEV_ID 25408
+#define HERMON_DDR_DEV_ID 25418
+#define HERMON_QDR_DEV_ID 25428
+#define HERMON_DDR_PCI5_DEV_ID 26418
+#define HERMON_QDR_PCI5_DEV_ID 26428
+#define HERMON_CONF_DEV_ID 401
#define IS_CONF_DEV(dev_id) \
- ((dev_id == TAVOR_CONF_DEV_ID) || \
- (dev_id == ARBEL_TM_CONF_DEV_ID) || \
- (dev_id == ARBEL_CONF_DEV_ID) || \
- (dev_id == SINAI_4X_CONF_DEV_ID) || \
- (dev_id == SINAI_8X_CONF_DEV_ID))
+ ((dev_id == TAVOR_CONF_DEV_ID) || \
+ (dev_id == ARBEL_TM_CONF_DEV_ID) || \
+ (dev_id == ARBEL_CONF_DEV_ID) || \
+ (dev_id == SINAI_4X_CONF_DEV_ID) || \
+ (dev_id == SINAI_8X_CONF_DEV_ID) || \
+ (dev_id == HERMON_CONF_DEV_ID))
#define MAX_DEV_NAME 32
typedef struct {
@@ -115,7 +174,15 @@
{ SINAI_4X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR },
{ SINAI_4X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR },
{ SINAI_8X_DEV_ID, "InfiniHost_III_Lx", MDEVS_TAVOR },
- { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR },
+ { SINAI_8X_CONF_DEV_ID, "InfiniHostBd", MDEVS_TAVOR_CR },
+
+ { HERMON_SDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR },
+ { HERMON_DDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR },
+ { HERMON_QDR_DEV_ID, "ConnectX", MDEVS_TAVOR_CR },
+ { HERMON_DDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR },
+ { HERMON_QDR_PCI5_DEV_ID, "ConnectX", MDEVS_TAVOR_CR },
+
+ { HERMON_CONF_DEV_ID, "ConnectXBd", MDEVS_TAVOR_CR },
};
#define DEVICE_DB_SIZE (sizeof(db) / sizeof(DEVICE_DB_T))
@@ -123,6 +190,14 @@
char *dsuffix[] = { "conf", "_cr", "_uar", "_ddr"};
#define MASKS_SIZE (sizeof(dmasks) / sizeof(Mdevs))
+static int is_hermon(USHORT dev_id)
+{
+ return (dev_id == HERMON_SDR_DEV_ID) ||
+ (dev_id == HERMON_DDR_DEV_ID) ||
+ (dev_id == HERMON_QDR_DEV_ID) ||
+ (dev_id == HERMON_DDR_PCI5_DEV_ID) ||
+ (dev_id == HERMON_QDR_PCI5_DEV_ID);
+}
// Return: < 0 - Error. > 0 - Numbern of characters written (including last '\0')
int create_mst_names_by_dev_id(USHORT dev_id, int dev_ix, int mask, char *name, int name_len, int *cnt)
@@ -431,8 +506,70 @@
}
+
//
+// Hermon PCI ordering bug work around (in kernel):
//
+// Toggle bit is done using a HW semaphore. 3 slots (sems) are allocated for MST tools and
+// are dynamically allocated by the mst_pci.o module. 1 slot is reserved for external tools usage.
+// This means to more than 3 tools that use the PCI dev can run simultaniously.
+//
+// Open:
+// If the wa is enabled ("MTCR_HERMON_WA" env variable does NOT exist or exist and equals 1):
+// and HERMON-A0 dev id is read:
+// get a vacant slot - ioctl( , PCI_HERMON_WA ,);
+// read the sem in the slot (to lock it).
+//
+//
+// Flush (hermon_wa_write_confirm_sem):
+// If enabled on first read after write:
+// write 0 to sem.
+// read sem untlill sem = 0;
+//
+// Close/release
+// Zero the slot.
+//
+
+int hermon_wa_write_confirm_sem(mfile* mf, u_int32_t addr) {
+ u_int64_t max_retries = 0x1000000;
+ u_int64_t retries = 0;
+ mfile_ibal *mfi = (mfile_ibal *)mf;
+
+ mfi->hermon_wa_num_of_writes++;
+
+ // clear sem
+ *((u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)) = 0;
+
+ while (retries < max_retries) {
+ u_int32_t slot_value ;
+
+ slot_value = __be32_to_cpu(*((volatile u_int32_t *)((char *)mf->ptr + mfi->hermon_wa_slot)));
+ if (slot_value == 0) {
+
+ // Good - The written value was read
+ if (retries) {
+ // NOT REALLY NEEDED - just to collect statistics
+
+ // printf("-D- Hermon WA addr %06x took %d retries\n", addr, retries);
+ mfi->hermon_wa_num_of_retry_writes++;
+ if (retries > mfi->hermon_wa_max_retries) {
+ mfi->hermon_wa_max_retries = retries;
+ }
+ }
+ return 4;
+ } else {
+ // sem is still locked from previous read - read passed the write
+ retries++;
+ }
+ }
+
+ DPRINT1(("-D- Hermon WA addr %06x failed after %ld retries\n", addr, retries));
+ return 0;
+}
+
+
+//
+//
// List devices in their MST compatible names.
// Each device type is indexed sepetrately.
//
@@ -629,7 +766,7 @@
}
} else {
- int bar_num;
+ int bar_num, slot_num;
// Type of file
mf->s.tp = MST_PCI;
@@ -649,8 +786,36 @@
}
mf->s.ptr = (void*)(ULONG_PTR)mf->cr_map.va;
- }
-
+
+ // check whether we need to perform the workaround
+ if (is_hermon(dev_id)) {
+ char* hermon_wa_env = getenv("MTCR_HERMON_WA");
+ // if there is no env variable or it = "0" - we perform the workaround
+ if (hermon_wa_env == NULL || strcmp(hermon_wa_env, "0")) {
+ // perform the workaround only for A0 revision
+ if (access_type == MDEVS_TAVOR_CR) {
+ u_int32_t rev = 0;
+ mread4((mfile*)mf, 0xf0014, &rev);
+ if (rev == 0xa00190) { // TODO: Should I ignore REV here ?
+ slot_num = __get_slot();
+ if (slot_num < 0) // no free slot
+ goto ErrExit;
+ mf->bugs |= FIX_ORDERING_BUG;
+ mf->slot_num = slot_num;
+ }
+ }
+ }
+ }
+
+ // prepare for the workaround
+ if (mf->bugs & FIX_ORDERING_BUG) { // Hermon WA setup
+ DPRINT1(("-D- Hermon WA setup\n"));
+ mf->hermon_wa_slot = HERMON_WA_BASE + 4*slot_num;
+ hermon_wa_write_confirm_sem((mfile*)mf, mf->hermon_wa_slot);
+ }
+
+ }
+
} else if (dev_id == DEVASYS_DEV_ID) {
// Type of file
h = usb_open();
@@ -702,6 +867,16 @@
if (mf->tp == MST_PCICONF) {
ibal_access(mfi->h_ca, 0x0, &stub, 4, FW_CLOSE_IF );
} else if (mf->tp = MST_PCI) {
+
+ if (mfi->bugs & FIX_ORDERING_BUG) {
+ __put_slot( mfi->slot_num );
+ DPRINT1(("-D- Hermon WA stats:\n"));
+ DPRINT1(("-D- : num of write flushes: %8ld\n", mfi->hermon_wa_num_of_writes));
+ DPRINT1(("-D- : num of retry flushes: %8ld\n", mfi->hermon_wa_num_of_retry_writes));
+ DPRINT1(("-D- : max_retries: %8ld\n", mfi->hermon_wa_max_retries));
+ }
+
+
if (mfi->cr_map.size) {
if (ibal_access(mfi->h_ca, 0x0, NULL, 0, FW_UNMAP_CRSPACE ) != IB_SUCCESS) {
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080409/8c60b4b8/attachment.html>
More information about the ofw
mailing list