[openib-general] [PATCH] mthca - command interface
Eli Cohen
eli at mellanox.co.il
Mon Feb 13 04:19:31 PST 2006
This patch implements posting commands by issuing posted writes in a
manner similar to doorbells. The benefit of this method is that it does
not require polling the go bit before posting the command and thus can
lower CPU utilization.
Signed-off-by: Eli Cohen <eli at mellanox.co.il>
Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_dev.h
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_dev.h
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -128,6 +128,13 @@ struct mthca_cmd {
int free_head;
struct mthca_cmd_context *context;
u16 token_mask;
+ /* post commands like doorbells instead of hcr */
+ int can_post_doorbells;
+ int dbell_post;
+ void __iomem *dbell_map;
+ void __iomem *dbell_ptrs[8];
+ u64 dbell_base;
+ u16 dbell_offsets[8];
};
struct mthca_limits {
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -188,19 +188,40 @@ static inline int go_bit(struct mthca_de
swab32(1 << HCR_GO_BIT);
}
-static int mthca_cmd_post(struct mthca_dev *dev,
- u64 in_param,
- u64 out_param,
- u32 in_modifier,
- u8 op_modifier,
- u16 op,
- u16 token,
- int event)
-{
- int err = 0;
- mutex_lock(&dev->cmd.hcr_mutex);
+static void mthca_cmd_post_dbell(struct mthca_dev *dev,
+ u64 in_param,
+ u64 out_param,
+ u32 in_modifier,
+ u8 op_modifier,
+ u16 op,
+ u16 token)
+{
+ void __iomem **ptr = dev->cmd.dbell_ptrs;
+
+ writel((__force u32) cpu_to_be32(in_param >> 32), ptr[0]);
+ writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), ptr[1]);
+ writel((__force u32) cpu_to_be32(in_modifier), ptr[2]);
+ writel((__force u32) cpu_to_be32(out_param >> 32), ptr[3]);
+ writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), ptr[4]);
+ writel((__force u32) cpu_to_be32(token << 16), ptr[5]);
+ writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) |
+ (1 << HCA_E_BIT) |
+ (op_modifier << HCR_OPMOD_SHIFT) |
+ op), ptr[6]);
+ writel((__force u32) 0, ptr[7]);
+}
+
+static int mthca_cmd_post_hcr(struct mthca_dev *dev,
+ u64 in_param,
+ u64 out_param,
+ u32 in_modifier,
+ u8 op_modifier,
+ u16 op,
+ u16 token,
+ int event)
+{
if (event) {
unsigned long end = jiffies + GO_BIT_TIMEOUT;
@@ -210,10 +231,8 @@ static int mthca_cmd_post(struct mthca_d
}
}
- if (go_bit(dev)) {
- err = -EAGAIN;
- goto out;
- }
+ if (go_bit(dev))
+ return -EAGAIN;
/*
* We use writel (instead of something like memcpy_toio)
@@ -236,7 +255,29 @@ static int mthca_cmd_post(struct mthca_d
(op_modifier << HCR_OPMOD_SHIFT) |
op), dev->hcr + 6 * 4);
-out:
+ return 0;
+}
+
+static int mthca_cmd_post(struct mthca_dev *dev,
+ u64 in_param,
+ u64 out_param,
+ u32 in_modifier,
+ u8 op_modifier,
+ u16 op,
+ u16 token,
+ int event)
+{
+ int err = 0;
+
+ mutex_lock(&dev->cmd.hcr_mutex);
+
+ if (dev->cmd.dbell_post && event)
+ mthca_cmd_post_dbell(dev, in_param, out_param, in_modifier,
+ op_modifier, op, token);
+ else
+ err = mthca_cmd_post_hcr(dev, in_param, out_param, in_modifier,
+ op_modifier, op, token, event);
+
mutex_unlock(&dev->cmd.hcr_mutex);
return err;
}
@@ -461,6 +502,8 @@ void mthca_cmd_cleanup(struct mthca_dev
{
pci_pool_destroy(dev->cmd.pool);
iounmap(dev->hcr);
+ if (dev->cmd.dbell_post)
+ iounmap(dev->cmd.dbell_map);
}
/*
@@ -499,11 +542,54 @@ int mthca_cmd_use_events(struct mthca_de
--dev->cmd.token_mask;
dev->cmd.use_events = 1;
+
+
down(&dev->cmd.poll_sem);
return 0;
}
+
+/*
+ * attempt to post commands through doorbells
+ */
+int mthca_use_cmd_doorbells(struct mthca_dev *dev)
+{
+ int i;
+ u16 max_off = 0;
+ unsigned long pg1, pg2;
+ void __iomem *map_base;
+
+ if (!dev->cmd.can_post_doorbells)
+ return -ENODEV;
+
+ for (i=0; i<8; ++i)
+ if (dev->cmd.dbell_offsets[i] > max_off)
+ max_off = dev->cmd.dbell_offsets[i];
+
+ pg1 = dev->cmd.dbell_base & PAGE_MASK;
+ pg2 = (dev->cmd.dbell_base + max_off) & PAGE_MASK;
+
+ if (pg1 != pg2)
+ return -ENOMEM;
+
+ map_base = ioremap(dev->cmd.dbell_base, max_off +
+ sizeof(unsigned long));
+ if (!map_base) {
+ iounmap(map_base);
+ return -ENOMEM;
+ }
+
+ for (i=0; i<8; ++i)
+ dev->cmd.dbell_ptrs[i] = map_base +
+ dev->cmd.dbell_offsets[i];
+
+ dev->cmd.dbell_map = map_base;
+ dev->cmd.dbell_post = 1;
+
+ return 0;
+}
+
/*
* Switch back to polling (used when shutting down the device)
*/
@@ -667,8 +753,10 @@ int mthca_QUERY_FW(struct mthca_dev *dev
{
struct mthca_mailbox *mailbox;
u32 *outbox;
+ u32 tmp;
int err = 0;
u8 lg;
+ int i;
#define QUERY_FW_OUT_SIZE 0x100
#define QUERY_FW_VER_OFFSET 0x00
@@ -676,6 +764,11 @@ int mthca_QUERY_FW(struct mthca_dev *dev
#define QUERY_FW_ERR_START_OFFSET 0x30
#define QUERY_FW_ERR_SIZE_OFFSET 0x38
+
+#define QUERY_FW_CMD_DB_EN_OFFSET 0x10
+#define QUERY_FW_CMD_DB_OFFSET 0x50
+#define QUERY_FW_CMD_DB_BASE 0x60
+
#define QUERY_FW_START_OFFSET 0x20
#define QUERY_FW_END_OFFSET 0x28
@@ -708,6 +801,12 @@ int mthca_QUERY_FW(struct mthca_dev *dev
dev->cmd.max_cmds = 1 << lg;
MTHCA_GET(dev->catas_err.addr, outbox, QUERY_FW_ERR_START_OFFSET);
MTHCA_GET(dev->catas_err.size, outbox, QUERY_FW_ERR_SIZE_OFFSET);
+ MTHCA_GET(tmp, outbox, QUERY_FW_CMD_DB_EN_OFFSET);
+ dev->cmd.can_post_doorbells = tmp & 0x1;
+ MTHCA_GET(dev->cmd.dbell_base, outbox, QUERY_FW_CMD_DB_BASE);
+ for (i=0; i<8; ++i)
+ MTHCA_GET(dev->cmd.dbell_offsets[i], outbox,
+ QUERY_FW_CMD_DB_OFFSET + (i << 1));
mthca_dbg(dev, "FW version %012llx, max commands %d\n",
(unsigned long long) dev->fw_ver, dev->cmd.max_cmds);
@@ -1608,7 +1707,6 @@ int mthca_MODIFY_QP(struct mthca_dev *de
if (0) {
int i;
mthca_dbg(dev, "Dumping QP context:\n");
- printk(" opt param mask: %08x\n", be32_to_cpup(mailbox->buf));
for (i = 0; i < 0x100 / 4; ++i) {
if (i % 8 == 0)
printk(" [%02x] ", i * 4);
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.h
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_cmd.h
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.h
@@ -244,6 +244,7 @@ struct mthca_set_ib_param {
int mthca_cmd_init(struct mthca_dev *dev);
void mthca_cmd_cleanup(struct mthca_dev *dev);
int mthca_cmd_use_events(struct mthca_dev *dev);
+int mthca_use_cmd_doorbells(struct mthca_dev *dev);
void mthca_cmd_use_polling(struct mthca_dev *dev);
void mthca_cmd_event(struct mthca_dev *dev, u16 token,
u8 status, u64 out_param);
More information about the general
mailing list