[ofa-general][PATCH 03/11 v3] mlx4: Different port type support

Yevgeny Petrilin yevgenyp at mellanox.co.il
Wed Aug 20 06:17:37 PDT 2008


From: Yevgeny Petrilin <yevgenyp at mellanox.co.il>
Date: Wed, 20 Aug 2008 10:03:23 +0300
Subject: [PATCH] mlx4: Different port type support

Multiprotocol supports different port types.
The port types are delivered through module parameters,
crossed with firmware capabilities.
Each consumer of mlx4_core should query for supported port types,
mlx4_ib can no longer assume that all phisical ports belong to it.

Signed-off-by: Yevgeny Petrilin <yevgenyp at mellanox.co.il>
---
 drivers/infiniband/hw/mlx4/mad.c     |    6 +-
 drivers/infiniband/hw/mlx4/main.c    |   12 ++++-
 drivers/infiniband/hw/mlx4/mlx4_ib.h |    2 +
 drivers/net/mlx4/fw.c                |    4 ++
 drivers/net/mlx4/fw.h                |    1 +
 drivers/net/mlx4/main.c              |   82 ++++++++++++++++++++++++++++++++++
 include/linux/mlx4/device.h          |   32 +++++++++++++
 7 files changed, 134 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index cdca3a5..606f1e2 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -298,7 +298,7 @@ int mlx4_ib_mad_init(struct mlx4_ib_dev *dev)
 	int p, q;
 	int ret;

-	for (p = 0; p < dev->dev->caps.num_ports; ++p)
+	for (p = 0; p < dev->num_ports; ++p)
 		for (q = 0; q <= 1; ++q) {
 			agent = ib_register_mad_agent(&dev->ib_dev, p + 1,
 						      q ? IB_QPT_GSI : IB_QPT_SMI,
@@ -314,7 +314,7 @@ int mlx4_ib_mad_init(struct mlx4_ib_dev *dev)
 	return 0;

 err:
-	for (p = 0; p < dev->dev->caps.num_ports; ++p)
+	for (p = 0; p < dev->num_ports; ++p)
 		for (q = 0; q <= 1; ++q)
 			if (dev->send_agent[p][q])
 				ib_unregister_mad_agent(dev->send_agent[p][q]);
@@ -327,7 +327,7 @@ void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev)
 	struct ib_mad_agent *agent;
 	int p, q;

-	for (p = 0; p < dev->dev->caps.num_ports; ++p) {
+	for (p = 0; p < dev->num_ports; ++p) {
 		for (q = 0; q <= 1; ++q) {
 			agent = dev->send_agent[p][q];
 			dev->send_agent[p][q] = NULL;
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index a3c2851..ff60f77 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -569,12 +569,16 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 	MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);

 	ibdev->dev = dev;
+	ibdev->ports_map = mlx4_get_ports_of_type(dev, MLX4_PORT_TYPE_IB);

 	strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
 	ibdev->ib_dev.owner		= THIS_MODULE;
 	ibdev->ib_dev.node_type		= RDMA_NODE_IB_CA;
 	ibdev->ib_dev.local_dma_lkey	= dev->caps.reserved_lkey;
-	ibdev->ib_dev.phys_port_cnt	= dev->caps.num_ports;
+	ibdev->num_ports = 0;
+	mlx4_foreach_port(i, ibdev->ports_map)
+		ibdev->num_ports++;
+	ibdev->ib_dev.phys_port_cnt     = ibdev->num_ports;
 	ibdev->ib_dev.num_comp_vectors	= 1;
 	ibdev->ib_dev.dma_device	= &dev->pdev->dev;

@@ -691,7 +695,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
 	struct mlx4_ib_dev *ibdev = ibdev_ptr;
 	int p;

-	for (p = 1; p <= dev->caps.num_ports; ++p)
+	for (p = 1; p <= ibdev->num_ports; ++p)
 		mlx4_CLOSE_PORT(dev, p);

 	mlx4_ib_mad_cleanup(ibdev);
@@ -706,6 +710,10 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
 			  enum mlx4_dev_event event, int port)
 {
 	struct ib_event ibev;
+	struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);
+
+	if (port > ibdev->num_ports)
+		return;

 	switch (event) {
 	case MLX4_DEV_EVENT_PORT_UP:
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 6e2b0dc..2b7a8d4 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -162,6 +162,8 @@ struct mlx4_ib_ah {
 struct mlx4_ib_dev {
 	struct ib_device	ib_dev;
 	struct mlx4_dev	       *dev;
+	u32			ports_map;
+	int			num_ports;
 	void __iomem	       *uar_map;

 	struct mlx4_uar		priv_uar;
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 40d8142..5136953 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -354,6 +354,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 			dev_cap->max_pkeys[i]	   = 1 << (field & 0xf);
 		}
 	} else {
+#define QUERY_PORT_SUPPORTED_TYPE_OFFSET	0x00
 #define QUERY_PORT_MTU_OFFSET			0x01
 #define QUERY_PORT_WIDTH_OFFSET			0x06
 #define QUERY_PORT_MAX_GID_PKEY_OFFSET		0x07
@@ -366,6 +367,9 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 			if (err)
 				goto out;

+			MLX4_GET(field, outbox,
+				 QUERY_PORT_SUPPORTED_TYPE_OFFSET);
+			dev_cap->supported_port_types[i] = field & 3;
 			MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET);
 			dev_cap->max_mtu[i]	   = field & 0xf;
 			MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET);
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index c34e726..acf6375 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -102,6 +102,7 @@ struct mlx4_dev_cap {
 	u32 reserved_lkey;
 	u64 max_icm_sz;
 	int max_gso_sz;
+	u8  supported_port_types[MLX4_MAX_PORTS + 1];
 	u8  log_max_macs[MLX4_MAX_PORTS + 1];
 	u8  log_max_vlans[MLX4_MAX_PORTS + 1];
 };
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index f172bb3..ba327ee 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -98,11 +98,48 @@ module_param_named(use_prio, use_prio, bool, 0444);
 MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
 		  "(0/1, default 0)");

+static char *port_type_arr[MLX4_MAX_PORTS] = {[0 ... (MLX4_MAX_PORTS-1)] = "ib"};
+module_param_array_named(port_type, port_type_arr, charp, NULL, 0444);
+MODULE_PARM_DESC(port_type, "Ports L2 type (ib/eth/auto, entry per port, "
+		  "comma seperated, default ib for all)");
+
+static int mlx4_check_port_params(struct mlx4_dev *dev,
+				  enum mlx4_port_type *port_type)
+{
+	if (port_type[0] != port_type[1] &&
+	    !(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
+		mlx4_err(dev, "Only same port types supported "
+			 "on this HCA, aborting.\n");
+		return -EINVAL;
+	}
+	if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
+	    (port_type[1] == MLX4_PORT_TYPE_IB)) {
+		mlx4_err(dev, "eth-ib configuration is not supported.\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static void mlx4_str2port_type(char **port_str,
+			       enum mlx4_port_type *port_type)
+{
+	int i;
+
+	for (i = 0; i < MLX4_MAX_PORTS; i++) {
+		if (!strcmp(port_str[i], "eth"))
+			port_type[i] = MLX4_PORT_TYPE_ETH;
+		else
+			port_type[i] = MLX4_PORT_TYPE_IB;
+	}
+}

 static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 {
 	int err;
 	int i;
+	enum mlx4_port_type port_type[MLX4_MAX_PORTS];
+
+	mlx4_str2port_type(port_type_arr, port_type);

 	err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
 	if (err) {
@@ -180,7 +217,24 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 	dev->caps.log_num_vlans = log_num_vlan;
 	dev->caps.log_num_prios = use_prio ? 3 : 0;

+	err = mlx4_check_port_params(dev, port_type);
+	if (err)
+		return err;
+
 	for (i = 1; i <= dev->caps.num_ports; ++i) {
+		if (!dev_cap->supported_port_types[i]) {
+			mlx4_warn(dev, "FW doesn't support Multi Protocol, "
+				  "loading IB only\n");
+			dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
+			continue;
+		}
+		if (port_type[i-1] & dev_cap->supported_port_types[i])
+			dev->caps.port_type[i] = port_type[i-1];
+		else {
+			mlx4_err(dev, "Requested port type for port %d "
+				 "not supported by HW\n", i);
+			return -ENODEV;
+		}
 		if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
 			dev->caps.log_num_macs = dev_cap->log_max_macs[i];
 			mlx4_warn(dev, "Requested number of MACs is too much "
@@ -1011,10 +1065,38 @@ static struct pci_driver mlx4_driver = {
 	.remove		= __devexit_p(mlx4_remove_one)
 };

+static int __init mlx4_verify_params(void)
+{
+	int i;
+
+	for (i = 0; i < MLX4_MAX_PORTS; ++i) {
+		if (strcmp(port_type_arr[i], "eth") &&
+		    strcmp(port_type_arr[i], "ib")) {
+			printk(KERN_WARNING "mlx4_core: bad port_type for "
+			       "port %d: %s\n", i, port_type_arr[i]);
+			return -1;
+		}
+	}
+	if ((log_num_mac < 0) || (log_num_mac > 7)) {
+		printk(KERN_WARNING "mlx4_core: bad num_mac: %d\n", log_num_mac);
+		return -1;
+	}
+
+	if ((log_num_vlan < 0) || (log_num_vlan > 7)) {
+		printk(KERN_WARNING "mlx4_core: bad num_vlan: %d\n", log_num_vlan);
+		return -1;
+	}
+
+	return 0;
+}
+
 static int __init mlx4_init(void)
 {
 	int ret;

+	if (mlx4_verify_params())
+		return -EINVAL;
+
 	ret = mlx4_catas_init();
 	if (ret)
 		return ret;
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 0a43891..758a50e 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -60,6 +60,7 @@ enum {
 	MLX4_DEV_CAP_FLAG_IPOIB_CSUM	= 1 <<  7,
 	MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR	= 1 <<  8,
 	MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR	= 1 <<  9,
+	MLX4_DEV_CAP_FLAG_DPDP		= 1 << 12,
 	MLX4_DEV_CAP_FLAG_MEM_WINDOW	= 1 << 16,
 	MLX4_DEV_CAP_FLAG_APM		= 1 << 17,
 	MLX4_DEV_CAP_FLAG_ATOMIC	= 1 << 18,
@@ -149,6 +150,11 @@ enum qp_region {
 	MLX4_QP_REGION_COUNT
 };

+enum mlx4_port_type {
+	MLX4_PORT_TYPE_IB	= 1 << 0,
+	MLX4_PORT_TYPE_ETH	= 1 << 1,
+};
+
 enum {
 	MLX4_NUM_FEXCH          = 64 * 1024,
 };
@@ -214,6 +220,7 @@ struct mlx4_caps {
 	int                     log_num_macs;
 	int                     log_num_vlans;
 	int                     log_num_prios;
+	enum mlx4_port_type	port_type[MLX4_MAX_PORTS + 1];
 };

 struct mlx4_buf_list {
@@ -368,6 +375,31 @@ struct mlx4_init_port_param {
 	u64			si_guid;
 };

+static inline void mlx4_query_steer_cap(struct mlx4_dev *dev, int *log_mac,
+					int *log_vlan, int *log_prio)
+{
+	*log_mac = dev->caps.log_num_macs;
+	*log_vlan = dev->caps.log_num_vlans;
+	*log_prio = dev->caps.log_num_prios;
+}
+
+static inline u32 mlx4_get_ports_of_type(struct mlx4_dev *dev,
+					 enum mlx4_port_type ptype)
+{
+	u32 ret = 0;
+	int i;
+
+	for (i = 1; i <= dev->caps.num_ports; ++i) {
+		if (dev->caps.port_type[i] == ptype)
+			ret |= 1 << (i-1);
+	}
+	return ret;
+}
+
+#define mlx4_foreach_port(port, bitmap) \
+	    for ((port) = 1; (port) <= MLX4_MAX_PORTS; (port)++) \
+		if (bitmap & 1 << ((port)-1))
+
 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
 		   struct mlx4_buf *buf);
 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf);
-- 
1.5.4






More information about the general mailing list