[openib-general] [PATCH] mthca - read byte count & read request size

[Eli Cohen]

Add control of PCIX max read byte count and PCI Express max read
request size parameters. This is due to the fact that some chipsets
may not work or may not work optimally with the default parameters.
Logic as follows: If the user sets a valid value to a parameter this
value wins. Otherwise the value read from the device is compared to
the default value defined by the spec. If the value read does not equal
the default it is assumed the BIOS configured it and the BIOS value is
retained. Otherwise the the former mthca configured defaults of 4K are
set.
To summarize, it assumes that if the BIOS configures a value it knows
best and we keep this value.


Signed-off-by: Eli Cohen <eli at mellanox.co.il>

Index: linux-2.6.14.2/drivers/infiniband/hw/mthca/mthca_main.c
===================================================================
--- linux-2.6.14.2.orig/drivers/infiniband/hw/mthca/mthca_main.c
+++ linux-2.6.14.2/drivers/infiniband/hw/mthca/mthca_main.c
@@ -69,6 +69,41 @@ MODULE_PARM_DESC(msi, "attempt to use MS
 
 #endif /* CONFIG_PCI_MSI */
 
+/* the PCIX max read byte count and PCI Express max read request size
+   parameters have default values as defined by the PRM which will
+   should give best performance in most cases. However there are some
+   chipsets that work better with other values or that do not work at
+   all with the default values */
+enum {
+	PCIX_MAX_RBC_512,
+	PCIX_MAX_RBC_1024,
+	PCIX_MAX_RBC_2048,
+	PCIX_MAX_RBC_4096,
+	PCIX_MAX_RBC_INVALID
+};
+
+#define CAP_PRMS_DESC " - some chipsets do not work optimally or do not work at " \
+		"all with the max value and this parameter can be used to tune "  \
+		"the optimal value"
+
+static int pcix_max_rbc = PCIX_MAX_RBC_INVALID;
+module_param(pcix_max_rbc, int, 0444);
+MODULE_PARM_DESC(pcix_max_rbc, "PCIX max read byte count" CAP_PRMS_DESC);
+
+enum {
+	PCIE_MAX_RRS_128,
+	PCIE_MAX_RRS_256,
+	PCIE_MAX_RRS_512,
+	PCIE_MAX_RRS_1024,
+	PCIE_MAX_RRS_2048,
+	PCIE_MAX_RRS_4096,
+	PCIE_MAX_RRS_INVALID
+};
+
+static int pcie_max_rrs = PCIE_MAX_RRS_INVALID;
+module_param(pcie_max_rrs, int, 0444);
+MODULE_PARM_DESC(pcie_max_rrs, "PCI Express max read read request size" CAP_PRMS_DESC);
+
 static const char mthca_version[] __devinitdata =
 	DRV_NAME ": Mellanox InfiniBand HCA driver v"
 	DRV_VERSION " (" DRV_RELDATE ")\n";
@@ -89,8 +124,9 @@ static int __devinit mthca_tune_pci(stru
 {
 	int cap;
 	u16 val;
+	int orig_val, set_val;
 
-	/* First try to max out Read Byte Count */
+	/* First determine Max Read Byte Count */
 	cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX);
 	if (cap) {
 		if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) {
@@ -98,12 +134,25 @@ static int __devinit mthca_tune_pci(stru
 				  "aborting.\n");
 			return -ENODEV;
 		}
-		val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2);
+
+		if (pcix_max_rbc < PCIX_MAX_RBC_INVALID)
+				     set_val = pcix_max_rbc;
+		else {
+			orig_val = (val & PCI_X_CMD_MAX_READ) >> 2;
+			if (orig_val != PCIX_MAX_RBC_512)
+				/* retain value set by the bios */
+				set_val = orig_val;
+			else
+				set_val = PCIX_MAX_RBC_4096;
+		}
+		val = (val & ~PCI_X_CMD_MAX_READ) | (set_val << 2);
 		if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) {
 			mthca_err(mdev, "Couldn't write PCI-X command register, "
 				  "aborting.\n");
 			return -ENODEV;
 		}
+		mthca_dbg(mdev, "setting read byte count to %d bytes\n",
+			  512 << set_val);
 	} else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
 		mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
 
@@ -114,12 +163,26 @@ static int __devinit mthca_tune_pci(stru
 				  "register, aborting.\n");
 			return -ENODEV;
 		}
-		val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12);
+
+		if (pcie_max_rrs < PCIE_MAX_RRS_INVALID)
+			set_val = pcie_max_rrs;
+		else {
+			orig_val = (val & PCI_EXP_DEVCTL_READRQ) >> 12;
+			if (orig_val != PCIE_MAX_RRS_512)
+				/* retain value set by the bios */
+				set_val = orig_val;
+			else
+				set_val = PCIE_MAX_RRS_4096;
+		}
+
+		val = (val & ~PCI_EXP_DEVCTL_READRQ) | (set_val << 12);
 		if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) {
 			mthca_err(mdev, "Couldn't write PCI Express device control "
 				  "register, aborting.\n");
 			return -ENODEV;
 		}
+		mthca_dbg(mdev, "setting max read request size to %d\n",
+			  128 << set_val);
 	} else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
 		mthca_info(mdev, "No PCI Express capability, "
 			   "not setting Max Read Request Size.\n");