[ofa-general] [PATCH] mlx4: implement MOD_STAT_CFG command to use for changing ConnectX page size

[Vladimir Sokolovsky]

 From 3bb8b713da6a0b2087201d6fb6c1a8d9274cf16e Mon Sep 17 00:00:00 2001
From: Vladimir Sokolovsky <vlad at mellanox.co.il>
Date: Sun, 18 May 2008 11:25:55 +0300
Subject: [PATCH] mlx4: implement MOD_STAT_CFG command to use for changing ConnectX page size

There is a bug in the OFED 1.3 mlx4 driver in mlx4_alloc_fmr which hardcoded
the minimum acceptable page_shift to be 12. However, new mlx4 firmware has a
minimum page_shift of 9 (log_pg_sz of 9 returned by QUERY_DEV_LIM) -- so that
ib_fmr_alloc fails for ULPs using the device minimum when creating FMRs.

To preserve firmware compatibility with released OFED drivers, the firmware
will continue to return 12 as before for log_page_sz in QUERY_DEV_CAP for these
drivers.
However, to enable new drivers to take advantage of the available smaller page
size, the mlx4 driver now first sets the log_pg_sz to the device minimum via
the MOD_STAT_CFG() command, and only then calls QUERY_DEV_CAP().
The QUERY_DEV_CAP() command then returns the new (lower) log_pg_sz value.

Signed-off-by: Jack Morgenstein <jackm at mellanox.co.il>
Signed-off-by: Vladimir Sokolovsky <vlad at mellanox.co.il>
---
  drivers/net/mlx4/fw.c    |   28 ++++++++++++++++++++++++++++
  drivers/net/mlx4/fw.h    |    6 ++++++
  drivers/net/mlx4/main.c  |   13 +++++++++++++
  include/linux/mlx4/cmd.h |    2 +-
  4 files changed, 48 insertions(+), 1 deletions(-)

diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index d82f275..2b5006b 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -101,6 +101,34 @@ static void dump_dev_cap_flags(struct mlx4_dev *dev, u32 flags)
  			mlx4_dbg(dev, "    %s\n", fname[i]);
  }

+int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg)
+{
+	struct mlx4_cmd_mailbox *mailbox;
+	u32 *inbox;
+	int err = 0;
+
+#define MOD_STAT_CFG_IN_SIZE		0x100
+
+#define MOD_STAT_CFG_PG_SZ_M_OFFSET	0x002
+#define MOD_STAT_CFG_PG_SZ_OFFSET	0x003
+
+	mailbox = mlx4_alloc_cmd_mailbox(dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+	inbox = mailbox->buf;
+
+	memset(inbox, 0, MOD_STAT_CFG_IN_SIZE);
+
+	MLX4_PUT(inbox, cfg->log_pg_sz, MOD_STAT_CFG_PG_SZ_OFFSET);
+	MLX4_PUT(inbox, cfg->log_pg_sz_m, MOD_STAT_CFG_PG_SZ_M_OFFSET);
+
+	err = mlx4_cmd(dev, mailbox->dma, 0, 0, MLX4_CMD_MOD_STAT_CFG,
+			MLX4_CMD_TIME_CLASS_A);
+
+	mlx4_free_cmd_mailbox(dev, mailbox);
+	return err;
+}
+
  int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
  {
  	struct mlx4_cmd_mailbox *mailbox;
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index 306cb9b..a0e046c 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -38,6 +38,11 @@
  #include "mlx4.h"
  #include "icm.h"

+struct mlx4_mod_stat_cfg {
+	u8 log_pg_sz;
+	u8 log_pg_sz_m;
+};
+
  struct mlx4_dev_cap {
  	int max_srq_sz;
  	int max_qp_sz;
@@ -162,5 +167,6 @@ int mlx4_SET_ICM_SIZE(struct mlx4_dev *dev, u64 icm_size, u64 *aux_pages);
  int mlx4_MAP_ICM_AUX(struct mlx4_dev *dev, struct mlx4_icm *icm);
  int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev);
  int mlx4_NOP(struct mlx4_dev *dev);
+int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg);

  #endif /* MLX4_FW_H */
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index a6aa49f..2a155ee 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -75,6 +75,11 @@ static char mlx4_version[] __devinitdata =
  	DRV_NAME ": Mellanox ConnectX core driver v"
  	DRV_VERSION " (" DRV_RELDATE ")\n";

+static int mlx4_log_pg_sz = 0;
+module_param(mlx4_log_pg_sz, int, 0444);
+MODULE_PARM_DESC(mlx4_log_pg_sz,
+		 "set FW log system min page size (0 gets native FW min. default=0)");
+
  static struct mlx4_profile default_profile = {
  	.num_qp		= 1 << 17,
  	.num_srq	= 1 << 16,
@@ -485,6 +490,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
  	struct mlx4_priv	  *priv = mlx4_priv(dev);
  	struct mlx4_adapter	   adapter;
  	struct mlx4_dev_cap	   dev_cap;
+	struct mlx4_mod_stat_cfg   mlx4_cfg;
  	struct mlx4_profile	   profile;
  	struct mlx4_init_hca_param init_hca;
  	u64 icm_size;
@@ -502,6 +508,13 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
  		return err;
  	}

+	mlx4_cfg.log_pg_sz_m = 1;
+	mlx4_cfg.log_pg_sz = (u8) mlx4_log_pg_sz;
+	err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
+	if (err)
+		mlx4_warn(dev, "Failed to override log_pg_sz parameter to %d\n",
+			  mlx4_log_pg_sz);
+
  	err = mlx4_dev_cap(dev, &dev_cap);
  	if (err) {
  		mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index 77323a7..3b563ed 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -169,7 +169,7 @@ static inline int mlx4_cmd_imm(struct mlx4_dev *dev, u64 in_param, u64 *out_para
  			       u32 in_modifier, u8 op_modifier, u16 op,
  			       unsigned long timeout)
  {
-	return __mlx4_cmd(dev, in_param, out_param, 1, in_modifier,
+	return __mlx4_cmd(dev, in_param, out_param, out_param ? 1 : 0, in_modifier,
  			  op_modifier, op, timeout);
  }

-- 
1.5.5.1