[ofa-general][PATCH 1/2] mlx4_core: Link sensing support
Yevgeny Petrilin
yevgenyp at mellanox.co.il
Tue Mar 3 04:42:54 PST 2009
At any time when port link is down (except to driver restart), and port
is configured to auto sensing mode, we try to sense port link type in
order to determine how to initialize the port.
If port type needs to be changed, all mlx4 interfaces are unregistered and then
registered again with the new port types. Sensing is done with intervals
of 3 seconds.
Signed-off-by: Yevgeny Petrilin <yevgenyp at mellanox.co.il>
---
drivers/net/mlx4/Makefile | 2 +-
drivers/net/mlx4/eq.c | 16 +++-
drivers/net/mlx4/intf.c | 4 +
drivers/net/mlx4/main.c | 94 +++++++++++++++++------
drivers/net/mlx4/mlx4.h | 26 +++++++
drivers/net/mlx4/sense.c | 174 +++++++++++++++++++++++++++++++++++++++++++
include/linux/mlx4/cmd.h | 1 +
include/linux/mlx4/device.h | 6 +-
8 files changed, 291 insertions(+), 32 deletions(-)
create mode 100644 drivers/net/mlx4/sense.c
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index a7a97bf..5a07b13 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -1,7 +1,7 @@
obj-$(CONFIG_MLX4_CORE) += mlx4_core.o
mlx4_core-y := alloc.o catas.o cmd.o cq.o eq.o fw.o icm.o intf.o main.o mcg.o \
- mr.o pd.o port.o profile.o qp.o reset.o srq.o
+ mr.o pd.o port.o profile.o qp.o reset.o srq.o sense.o
obj-$(CONFIG_MLX4_EN) += mlx4_en.o
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index 2c19bff..8830dcb 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -163,6 +163,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
int cqn;
int eqes_found = 0;
int set_ci = 0;
+ int port;
while ((eqe = next_eqe_sw(eq))) {
/*
@@ -203,11 +204,16 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
break;
case MLX4_EVENT_TYPE_PORT_CHANGE:
- mlx4_dispatch_event(dev,
- eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_ACTIVE ?
- MLX4_DEV_EVENT_PORT_UP :
- MLX4_DEV_EVENT_PORT_DOWN,
- be32_to_cpu(eqe->event.port_change.port) >> 28);
+ port = be32_to_cpu(eqe->event.port_change.port) >> 28;
+ if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
+ mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
+ port);
+ mlx4_priv(dev)->sense.do_sense_port[port] = 1;
+ } else {
+ mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP,
+ port);
+ mlx4_priv(dev)->sense.do_sense_port[port] = 0;
+ }
break;
case MLX4_EVENT_TYPE_CQ_ERROR:
diff --git a/drivers/net/mlx4/intf.c b/drivers/net/mlx4/intf.c
index 0e7eb10..30ef000 100644
--- a/drivers/net/mlx4/intf.c
+++ b/drivers/net/mlx4/intf.c
@@ -141,6 +141,8 @@ int mlx4_register_device(struct mlx4_dev *dev)
mutex_unlock(&intf_mutex);
mlx4_start_catas_poll(dev);
+ mlx4_start_sense(dev);
+
return 0;
}
@@ -149,6 +151,8 @@ void mlx4_unregister_device(struct mlx4_dev *dev)
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_interface *intf;
+ mlx4_stop_sense(dev);
+
mlx4_stop_catas_poll(dev);
mutex_lock(&intf_mutex);
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 8480f03..a50ae7a 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -98,24 +98,23 @@ module_param_named(use_prio, use_prio, bool, 0444);
MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
"(0/1, default 0)");
-static int mlx4_check_port_params(struct mlx4_dev *dev,
+int mlx4_check_port_params(struct mlx4_dev *dev,
enum mlx4_port_type *port_type)
{
int i;
for (i = 0; i < dev->caps.num_ports - 1; i++) {
- if (port_type[i] != port_type[i+1] &&
- !(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
- mlx4_err(dev, "Only same port types supported "
- "on this HCA, aborting.\n");
- return -EINVAL;
+ if (port_type[i] != port_type[i+1]) {
+ if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
+ mlx4_err(dev, "Only same port types supported "
+ "on this HCA, aborting.\n");
+ return -EINVAL;
+ }
+ if ((port_type[i] == MLX4_PORT_TYPE_ETH) ||
+ (port_type[i+1] == MLX4_PORT_TYPE_IB))
+ return -EINVAL;
}
}
- if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
- (port_type[1] == MLX4_PORT_TYPE_IB)) {
- mlx4_err(dev, "eth-ib configuration is not supported.\n");
- return -EINVAL;
- }
for (i = 0; i < dev->caps.num_ports; i++) {
if (!(port_type[i] & dev->caps.supported_type[i+1])) {
@@ -225,6 +224,9 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
else
dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
+ dev->caps.possible_type[i] = dev->caps.port_type[i];
+ mlx4_priv(dev)->sense.sense_allowed[i] =
+ dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO ? 1 : 0;
if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
dev->caps.log_num_macs = dev_cap->log_max_macs[i];
@@ -263,7 +265,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
* Change the port configuration of the device.
* Every user of this function must hold the port mutex.
*/
-static int mlx4_change_port_types(struct mlx4_dev *dev,
+int mlx4_change_port_types(struct mlx4_dev *dev,
enum mlx4_port_type *port_types)
{
int err = 0;
@@ -271,6 +273,8 @@ static int mlx4_change_port_types(struct mlx4_dev *dev,
int port;
for (port = 0; port < dev->caps.num_ports; port++) {
+ /* Change the port type only if the new type is different
+ * from the current, and not set to Auto */
if (port_types[port] != dev->caps.port_type[port + 1]) {
change = 1;
dev->caps.port_type[port + 1] = port_types[port];
@@ -302,10 +306,17 @@ static ssize_t show_port_type(struct device *dev,
struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
port_attr);
struct mlx4_dev *mdev = info->dev;
+ char type[8];
+
+ sprintf(type, "%s",
+ (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
+ "ib" : "eth");
+ if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
+ sprintf(buf, "auto (%s)\n", type);
+ else
+ sprintf(buf, "%s\n", type);
- return sprintf(buf, "%s\n",
- mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB ?
- "ib" : "eth");
+ return strlen(buf);
}
static ssize_t set_port_type(struct device *dev,
@@ -317,6 +328,7 @@ static ssize_t set_port_type(struct device *dev,
struct mlx4_dev *mdev = info->dev;
struct mlx4_priv *priv = mlx4_priv(mdev);
enum mlx4_port_type types[MLX4_MAX_PORTS];
+ enum mlx4_port_type new_types[MLX4_MAX_PORTS];
int i;
int err = 0;
@@ -324,24 +336,52 @@ static ssize_t set_port_type(struct device *dev,
info->tmp_type = MLX4_PORT_TYPE_IB;
else if (!strcmp(buf, "eth\n"))
info->tmp_type = MLX4_PORT_TYPE_ETH;
+ else if (!strcmp(buf, "auto\n"))
+ info->tmp_type = MLX4_PORT_TYPE_AUTO;
else {
mlx4_err(mdev, "%s is not supported port type\n", buf);
return -EINVAL;
}
mutex_lock(&priv->port_mutex);
- for (i = 0; i < mdev->caps.num_ports; i++)
+ /* Possible type is always the one that was delivered */
+ mdev->caps.possible_type[info->port] = info->tmp_type;
+
+ for (i = 0; i < mdev->caps.num_ports; i++) {
types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
- mdev->caps.port_type[i+1];
+ mdev->caps.possible_type[i+1];
+ if (types[i] == MLX4_PORT_TYPE_AUTO)
+ types[i] = mdev->caps.port_type[i+1];
+ }
+
+ if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
+ for (i = 1; i <= mdev->caps.num_ports; i++) {
+ if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
+ mdev->caps.possible_type[i] = mdev->caps.port_type[i];
+ err = -EINVAL;
+ }
+ }
+ }
+ if (err) {
+ mlx4_err(mdev, "Auto sensing is not supported on this HCA. "
+ "Set only 'eth' or 'ib' for both ports "
+ "(should be the same)\n");
+ goto out;
+ }
- err = mlx4_check_port_params(mdev, types);
+ mlx4_do_sense_ports(mdev, new_types, types);
+
+ err = mlx4_check_port_params(mdev, new_types);
if (err)
goto out;
- for (i = 1; i <= mdev->caps.num_ports; i++)
- priv->port[i].tmp_type = 0;
+ /* We are about to apply the changes after the configuration
+ * was verified, no need to remember the temporary types
+ * any more */
+ for (i = 0; i < mdev->caps.num_ports; i++)
+ priv->port[i + 1].tmp_type = 0;
- err = mlx4_change_port_types(mdev, types);
+ err = mlx4_change_port_types(mdev, new_types);
out:
mutex_unlock(&priv->port_mutex);
@@ -1113,14 +1153,21 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
goto err_port;
}
- err = mlx4_register_device(dev);
+ err = mlx4_sense_init(dev);
if (err)
goto err_port;
+ err = mlx4_register_device(dev);
+ if (err)
+ goto err_sense;
+
pci_set_drvdata(pdev, dev);
return 0;
+err_sense:
+ mlx4_sense_cleanup(dev);
+
err_port:
for (port = 1; port <= dev->caps.num_ports; port++)
mlx4_cleanup_port_info(&priv->port[port]);
@@ -1182,13 +1229,12 @@ static void mlx4_remove_one(struct pci_dev *pdev)
int p;
if (dev) {
+ mlx4_sense_cleanup(dev);
mlx4_unregister_device(dev);
-
for (p = 1; p <= dev->caps.num_ports; p++) {
mlx4_cleanup_port_info(&priv->port[p]);
mlx4_CLOSE_PORT(dev, p);
}
-
mlx4_cleanup_mcg_table(dev);
mlx4_cleanup_qp_table(dev);
mlx4_cleanup_srq_table(dev);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index e0213ba..347d994 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -40,6 +40,8 @@
#include <linux/mutex.h>
#include <linux/radix-tree.h>
#include <linux/timer.h>
+#include <linux/random.h>
+#include <linux/workqueue.h>
#include <linux/mlx4/device.h>
#include <linux/mlx4/driver.h>
@@ -276,6 +278,15 @@ struct mlx4_port_info {
struct mlx4_vlan_table vlan_table;
};
+struct mlx4_sense {
+ struct mlx4_dev *dev;
+ u8 do_sense_port[MLX4_MAX_PORTS + 1];
+ u8 sense_allowed[MLX4_MAX_PORTS + 1];
+ struct delayed_work sense_poll;
+ struct workqueue_struct *sense_wq;
+ u32 resched;
+};
+
struct mlx4_priv {
struct mlx4_dev dev;
@@ -305,6 +316,7 @@ struct mlx4_priv {
struct mlx4_uar driver_uar;
void __iomem *kar;
struct mlx4_port_info port[MLX4_MAX_PORTS + 1];
+ struct mlx4_sense sense;
struct mutex port_mutex;
};
@@ -313,6 +325,8 @@ static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
return container_of(dev, struct mlx4_priv, dev);
}
+#define MLX4_SENSE_RANGE (HZ * 3)
+
u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap);
void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj);
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align);
@@ -379,6 +393,18 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type);
void mlx4_handle_catas_err(struct mlx4_dev *dev);
+void mlx4_do_sense_ports(struct mlx4_dev *dev,
+ enum mlx4_port_type *stype,
+ enum mlx4_port_type *defaults);
+void mlx4_start_sense(struct mlx4_dev *dev);
+void mlx4_stop_sense(struct mlx4_dev *dev);
+int mlx4_sense_init(struct mlx4_dev *dev);
+void mlx4_sense_cleanup(struct mlx4_dev *dev);
+int mlx4_check_port_params(struct mlx4_dev *dev,
+ enum mlx4_port_type *port_type);
+int mlx4_change_port_types(struct mlx4_dev *dev,
+ enum mlx4_port_type *port_types);
+
void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
diff --git a/drivers/net/mlx4/sense.c b/drivers/net/mlx4/sense.c
new file mode 100644
index 0000000..f3c5411
--- /dev/null
+++ b/drivers/net/mlx4/sense.c
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/errno.h>
+#include <linux/if_ether.h>
+
+#include <linux/mlx4/cmd.h>
+
+#include "mlx4.h"
+
+
+static int mlx4_SENSE_PORT(struct mlx4_dev *dev, int port,
+ enum mlx4_port_type *type)
+{
+ u64 out_param;
+ int err = 0;
+
+ err = mlx4_cmd_imm(dev, 0, &out_param, port, 0,
+ MLX4_CMD_SENSE_PORT, MLX4_CMD_TIME_CLASS_B);
+ if (err) {
+ mlx4_err(dev, "Sense command failed for port: %d\n", port);
+ return err;
+ }
+
+ if (out_param > 2) {
+ mlx4_err(dev, "Sense returned illegal value: 0x%llx\n", out_param);
+ return EINVAL;
+ }
+
+ *type = out_param;
+ return 0;
+}
+
+void mlx4_do_sense_ports(struct mlx4_dev *dev,
+ enum mlx4_port_type *stype,
+ enum mlx4_port_type *defaults)
+{
+ struct mlx4_sense *sense = &mlx4_priv(dev)->sense;
+ int err;
+ int i;
+
+ for (i = 1; i <= dev->caps.num_ports; i++) {
+ stype[i-1] = 0;
+ if (sense->do_sense_port[i] && sense->sense_allowed[i] &&
+ dev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
+ err = mlx4_SENSE_PORT(dev, i, &stype[i-1]);
+ if (err)
+ stype[i-1] = defaults[i-1];
+ } else
+ stype[i-1] = defaults[i-1];
+ }
+
+ /*
+ * Adjust port configuration:
+ * If port 1 sensed nothing and port 2 is IB, set both as IB
+ * If port 2 sensed nothing and port 1 is Eth, set both as Eth
+ */
+ if (stype[0] == MLX4_PORT_TYPE_ETH) {
+ for (i = 1; i < dev->caps.num_ports; i++)
+ stype[i] = stype[i] ? stype[i] : MLX4_PORT_TYPE_ETH;
+ }
+ if (stype[dev->caps.num_ports - 1] == MLX4_PORT_TYPE_IB) {
+ for (i = 0; i < dev->caps.num_ports - 1; i++)
+ stype[i] = stype[i] ? stype[i] : MLX4_PORT_TYPE_IB;
+ }
+
+ /*
+ * If sensed nothing, remain in current configuration.
+ */
+ for (i = 0; i < dev->caps.num_ports; i++)
+ stype[i] = stype[i] ? stype[i] : defaults[i];
+
+}
+
+static void mlx4_sense_port(struct work_struct *work)
+{
+ struct delayed_work *delay = container_of(work, struct delayed_work, work);
+ struct mlx4_sense *sense = container_of(delay, struct mlx4_sense,
+ sense_poll);
+ struct mlx4_dev *dev = sense->dev;
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ enum mlx4_port_type stype[MLX4_MAX_PORTS];
+
+ mutex_lock(&priv->port_mutex);
+ mlx4_do_sense_ports(dev, stype, &dev->caps.port_type[1]);
+
+ if (mlx4_check_port_params(dev, stype))
+ goto sense_again;
+
+ if (mlx4_change_port_types(dev, stype))
+ mlx4_err(dev, "Failed to change port_types\n");
+
+sense_again:
+ mutex_unlock(&priv->port_mutex);
+ if (sense->resched)
+ queue_delayed_work(sense->sense_wq , &sense->sense_poll,
+ round_jiffies(MLX4_SENSE_RANGE));
+}
+
+void mlx4_start_sense(struct mlx4_dev *dev)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_sense *sense = &priv->sense;
+
+ if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP))
+ return;
+
+ sense->resched = 1;
+ queue_delayed_work(sense->sense_wq , &sense->sense_poll,
+ round_jiffies(MLX4_SENSE_RANGE));
+}
+
+
+void mlx4_stop_sense(struct mlx4_dev *dev)
+{
+ mlx4_priv(dev)->sense.resched = 0;
+}
+
+int mlx4_sense_init(struct mlx4_dev *dev)
+{
+ struct mlx4_priv *priv = mlx4_priv(dev);
+ struct mlx4_sense *sense = &priv->sense;
+ int port;
+
+ sense->dev = dev;
+ sense->sense_wq = create_singlethread_workqueue("mlx4_sense");
+ if (!sense->sense_wq)
+ return -ENOMEM;
+
+ for (port = 1; port <= dev->caps.num_ports; port++)
+ sense->do_sense_port[port] = 1;
+
+ INIT_DELAYED_WORK_DEFERRABLE(&sense->sense_poll, mlx4_sense_port);
+
+ return 0;
+}
+
+void mlx4_sense_cleanup(struct mlx4_dev *dev)
+{
+ mlx4_stop_sense(dev);
+ cancel_delayed_work_sync(&mlx4_priv(dev)->sense.sense_poll);
+ destroy_workqueue(mlx4_priv(dev)->sense.sense_wq);
+}
+
diff --git a/include/linux/mlx4/cmd.h b/include/linux/mlx4/cmd.h
index cf9c679..0f82293 100644
--- a/include/linux/mlx4/cmd.h
+++ b/include/linux/mlx4/cmd.h
@@ -55,6 +55,7 @@ enum {
MLX4_CMD_CLOSE_PORT = 0xa,
MLX4_CMD_QUERY_HCA = 0xb,
MLX4_CMD_QUERY_PORT = 0x43,
+ MLX4_CMD_SENSE_PORT = 0x4d,
MLX4_CMD_SET_PORT = 0xc,
MLX4_CMD_ACCESS_DDR = 0x2e,
MLX4_CMD_MAP_ICM = 0xffa,
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 8f659cc..3aff8a6 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -155,8 +155,9 @@ enum mlx4_qp_region {
};
enum mlx4_port_type {
- MLX4_PORT_TYPE_IB = 1 << 0,
- MLX4_PORT_TYPE_ETH = 1 << 1,
+ MLX4_PORT_TYPE_IB = 1,
+ MLX4_PORT_TYPE_ETH = 2,
+ MLX4_PORT_TYPE_AUTO = 3
};
enum mlx4_special_vlan_idx {
@@ -237,6 +238,7 @@ struct mlx4_caps {
enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
u8 supported_type[MLX4_MAX_PORTS + 1];
u32 port_mask;
+ enum mlx4_port_type possible_type[MLX4_MAX_PORTS + 1];
};
struct mlx4_buf_list {
--
1.6.1.3
More information about the general
mailing list