[openib-general] Low cost MBs compatible with Mellanox PCIE Cards

[Jason Gunthorpe]

On Wed, Feb 15, 2006 at 04:52:17PM -0800, Roland Dreier wrote:

> I've not seen any problems with MSI/MSI-X with nforce4 and PCIe HCAs
> on both my Asus A8N-SLI and HP DL145G2 systems.

I've managed to make it work here too. I found an earlier email from
you about the HT MSI Mapping capability which proved to be the key.
The BIOS on this board doesn't enable it :|

People on this list may be interested in this patch which adds a PCI
quirk to force the HyperTransport MSI mapping function to be enabled
if the BIOS forgot about it. It makes my board work:

ib1{jgg}~#cat /proc/interrupts 
           CPU0
  0:     344750    IO-APIC-edge  timer
  1:          8    IO-APIC-edge  i8042
  2:          0          XT-PIC  cascade
  5:     141156   IO-APIC-level  ohci_hcd:usb1, ehci_hcd:usb2, eth0
  8:          4    IO-APIC-edge  rtc
 14:       1445    IO-APIC-edge  ide0
 15:         15    IO-APIC-edge  ide1
193:        179         PCI-MSI  ib_mthca

Thanks,
Jason

--- linux-2.6.15.4/drivers/pci/quirks.c	2006-02-16 12:08:59.000000000 -0700
+++ lin/drivers/pci/quirks.c	2006-02-16 12:12:30.000000000 -0700
@@ -1257,6 +1257,29 @@
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NCR, PCI_DEVICE_ID_NCR_53C810, fixup_rev1_53c810);
 
+#ifdef CONFIG_PCI_MSI
+static void __devinit fixup_ht_msi(struct pci_dev* dev)
+{
+	/* Some BIOS's do not enable the hypertransport MSI mapping capability
+	   on the chipset. This breaks MSI support.. */
+	int pos = pci_find_capability(dev,PCI_CAP_ID_HT);
+	while (pos != 0)
+	{
+		u32 cap;
+		pci_read_config_dword(dev,pos,&cap);
+		if (((cap >> 16) & PCI_HT_CMD_TYP) == PCI_HT_CMD_TYP_MSIM) {
+			if ((cap & PCI_HT_MSIM_ENABLE) == 0) {
+				printk("BIOS BUG: HyperTransport MSI mapping not enabled for %s, enabling.\n",pci_name(dev));
+				cap |= PCI_HT_MSIM_ENABLE;
+				pci_write_config_dword(dev,pos,cap);
+			}
+			break;
+		}
+		pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_HT);
+	}
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, fixup_ht_msi);
+#endif
 
 static void pci_do_fixups(struct pci_dev *dev, struct pci_fixup *f, struct pci_fixup *end)
 {
--- linux-2.6.15.4/include/linux/pci_regs.h	2006-02-16 12:09:05.000000000 -0700
+++ lin/include/linux/pci_regs.h	2006-02-16 12:12:30.000000000 -0700
@@ -196,12 +196,14 @@
 #define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
 #define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
 #define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
+#define  PCI_CAP_ID_HT          0x08    /* HyperTransport */
 #define  PCI_CAP_ID_SHPC 	0x0C	/* PCI Standard Hot-Plug Controller */
 #define  PCI_CAP_ID_EXP 	0x10	/* PCI Express */
 #define  PCI_CAP_ID_MSIX	0x11	/* MSI-X */
 #define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
 #define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
 #define PCI_CAP_SIZEOF		4
+#define PCI_HT_CMD_TYP          0xf800  /* Hypertransport capability type mask */
 
 /* Power Management Registers */
 
@@ -285,6 +287,10 @@
 #define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
 #define PCI_MSI_MASK_BIT	16	/* Mask bits register */
 
+/* HyperTransport MSI Mapping registers */
+#define PCI_HT_CMD_TYP_MSIM 0xa800      // MSI Mapping type
+#define PCI_HT_MSIM_ENABLE  (1<<16)
+
 /* CompactPCI Hotswap Register */
 
 #define PCI_CHSWP_CSR		2	/* Control and Status Register */