[ewg] [PATCHv4] IB/ipoib: Fix ipoib handling for pkey reordering
Moni Levy
myopenib at gmail.com
Thu Mar 29 07:46:12 PDT 2007
This issue was found during partitioning & SM fail over testing. The fix was tested over the weekend with pkey reshuffling, removal and addition every few seconds concurrent with OFED restart. Please look at the "IB/cache: Add ib_cache report for cache in process" patch also.
Changes from v1:
* added flush flag to ipoib_ib_dev_stop(), ipoib_ib_dev_down() alike
* fixed a bug in device extraction from the work struct
* removed some warnings in case they are caused due to missing PKEY as
this seems like a valid flow now.
Changes from v2:
* less/fixed debug prints - (MST remark)
* flush_restart_qp stuff renamed to just restart_qp (MST remark)
* the patch now depends on Roland's "IPoIB: Only handle async
events for one port"
Changed from v3:
* We now reschedule that qp_restart_task in case the PKEY cache was not
coherent. Applies over the "IB/cache: Add ib_cache report for cache
in process" patch
SM reconfiguration or failover possibly causes a shuffling of the values in the
port pkey table. The current implementation only queries for the index of the
pkey once, when it creates the device QP and after that moves it into working
state, and hence does not address this scenario. Fix this by using the
PKEY_CHANGE event as a trigger to reconfigure the device QP.
Signed-off-by: Moni Levy <monil at voltaire.com>
---
drivers/infiniband/ulp/ipoib/ipoib.h | 4 +
drivers/infiniband/ulp/ipoib/ipoib_ib.c | 58 +++++++++++++++++++------
drivers/infiniband/ulp/ipoib/ipoib_main.c | 5 +-
drivers/infiniband/ulp/ipoib/ipoib_multicast.c | 11 ++--
drivers/infiniband/ulp/ipoib/ipoib_verbs.c | 7 ++-
5 files changed, 64 insertions(+), 21 deletions(-)
Index: b/drivers/infiniband/ulp/ipoib/ipoib.h
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib.h 2007-03-29 08:12:09.129621280 +0200
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h 2007-03-29 11:32:58.861338222 +0200
@@ -205,6 +205,7 @@ struct ipoib_dev_priv {
struct delayed_work pkey_task;
struct delayed_work mcast_task;
struct work_struct flush_task;
+ struct work_struct restart_qp_task;
struct work_struct restart_task;
struct delayed_work ah_reap_task;
@@ -334,12 +335,13 @@ struct ipoib_dev_priv *ipoib_intf_alloc(
int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port);
void ipoib_ib_dev_flush(struct work_struct *work);
+void ipoib_ib_dev_restart_qp(struct work_struct *work);
void ipoib_ib_dev_cleanup(struct net_device *dev);
int ipoib_ib_dev_open(struct net_device *dev);
int ipoib_ib_dev_up(struct net_device *dev);
int ipoib_ib_dev_down(struct net_device *dev, int flush);
-int ipoib_ib_dev_stop(struct net_device *dev);
+int ipoib_ib_dev_stop(struct net_device *dev, int flush);
int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port);
void ipoib_dev_cleanup(struct net_device *dev);
Index: b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2007-03-29 08:12:09.147618072 +0200
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c 2007-03-29 11:52:58.867503247 +0200
@@ -415,21 +415,23 @@ int ipoib_ib_dev_open(struct net_device
ret = ipoib_init_qp(dev);
if (ret) {
- ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
- return -1;
+ if (ret != -ENOENT && ret != -ESTALE) {
+ ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
+ }
+ return ret;
}
ret = ipoib_ib_post_receives(dev);
if (ret) {
ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
- ipoib_ib_dev_stop(dev);
+ ipoib_ib_dev_stop(dev, 1);
return -1;
}
ret = ipoib_cm_dev_open(dev);
if (ret) {
ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
- ipoib_ib_dev_stop(dev);
+ ipoib_ib_dev_stop(dev, 1);
return -1;
}
@@ -459,7 +461,7 @@ int ipoib_ib_dev_up(struct net_device *d
ipoib_pkey_dev_check_presence(dev);
if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
- ipoib_dbg(priv, "PKEY is not assigned.\n");
+ ipoib_dbg(priv, "pkey is not assigned.\n");
return 0;
}
@@ -508,7 +510,7 @@ static int recvs_pending(struct net_devi
return pending;
}
-int ipoib_ib_dev_stop(struct net_device *dev)
+int ipoib_ib_dev_stop(struct net_device *dev, int flush)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct ib_qp_attr qp_attr;
@@ -581,7 +583,8 @@ timeout:
/* Wait for all AHs to be reaped */
set_bit(IPOIB_STOP_REAPER, &priv->flags);
cancel_delayed_work(&priv->ah_reap_task);
- flush_workqueue(ipoib_workqueue);
+ if (flush)
+ flush_workqueue(ipoib_workqueue);
begin = jiffies;
@@ -622,13 +625,17 @@ int ipoib_ib_dev_init(struct net_device
return 0;
}
-void ipoib_ib_dev_flush(struct work_struct *work)
+static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int restart_qp)
{
- struct ipoib_dev_priv *cpriv, *priv =
- container_of(work, struct ipoib_dev_priv, flush_task);
+ struct ipoib_dev_priv *cpriv;
struct net_device *dev = priv->dev;
- if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags) ) {
+ /*
+ * ipoib_ib_dev_stop() below may not find the PKey and leave the
+ * IPOIB_FLAG_INITIALIZED flag off so flush in that case with restart_qp
+ * flag on is Ok.
+ */
+ if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags) && !restart_qp) {
ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
return;
}
@@ -642,6 +649,15 @@ void ipoib_ib_dev_flush(struct work_stru
ipoib_ib_dev_down(dev, 0);
+ if (restart_qp) {
+ ipoib_dbg(priv, "restarting the device QP\n");
+ if (test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags) )
+ ipoib_ib_dev_stop(dev, 0);
+ /* The pkey cache was not coherent we should retry */
+ if (ipoib_ib_dev_open(dev) == -ESTALE);
+ queue_work(ipoib_workqueue, &priv->restart_qp_task);
+ }
+
/*
* The device could have been brought down between the start and when
* we get here, don't bring it back up if it's not configured up
@@ -655,11 +671,29 @@ void ipoib_ib_dev_flush(struct work_stru
/* Flush any child interfaces too */
list_for_each_entry(cpriv, &priv->child_intfs, list)
- ipoib_ib_dev_flush(&cpriv->flush_task);
+ __ipoib_ib_dev_flush(cpriv, restart_qp);
mutex_unlock(&priv->vlan_mutex);
}
+void ipoib_ib_dev_flush(struct work_struct *work)
+{
+ struct ipoib_dev_priv *priv =
+ container_of(work, struct ipoib_dev_priv, flush_task);
+ /* We only restart the QP in case of pkey change event */
+ ipoib_dbg(priv, "Flushing %s\n", priv->dev->name);
+ __ipoib_ib_dev_flush(priv, 0);
+}
+
+void ipoib_ib_dev_restart_qp(struct work_struct *work)
+{
+ struct ipoib_dev_priv *priv =
+ container_of(work, struct ipoib_dev_priv, restart_qp_task);
+ /* We only restart the QP in case of pkey change event */
+ ipoib_dbg(priv, "Flushing %s and restarting it's QP\n", priv->dev->name);
+ __ipoib_ib_dev_flush(priv, 1);
+}
+
void ipoib_ib_dev_cleanup(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
Index: b/drivers/infiniband/ulp/ipoib/ipoib_main.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c 2007-03-29 08:12:09.161615577 +0200
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c 2007-03-29 11:32:58.904330521 +0200
@@ -107,7 +107,7 @@ int ipoib_open(struct net_device *dev)
return -EINVAL;
if (ipoib_ib_dev_up(dev)) {
- ipoib_ib_dev_stop(dev);
+ ipoib_ib_dev_stop(dev, 1);
return -EINVAL;
}
@@ -152,7 +152,7 @@ static int ipoib_stop(struct net_device
flush_workqueue(ipoib_workqueue);
ipoib_ib_dev_down(dev, 1);
- ipoib_ib_dev_stop(dev);
+ ipoib_ib_dev_stop(dev, 1);
if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
struct ipoib_dev_priv *cpriv;
@@ -993,6 +993,7 @@ static void ipoib_setup(struct net_devic
INIT_DELAYED_WORK(&priv->pkey_task, ipoib_pkey_poll);
INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task);
INIT_WORK(&priv->flush_task, ipoib_ib_dev_flush);
+ INIT_WORK(&priv->restart_qp_task, ipoib_ib_dev_restart_qp);
INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
}
Index: b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2007-03-29 08:12:09.176612904 +0200
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2007-03-29 11:32:58.932325506 +0200
@@ -232,9 +232,10 @@ static int ipoib_mcast_join_finish(struc
ret = ipoib_mcast_attach(dev, be16_to_cpu(mcast->mcmember.mlid),
&mcast->mcmember.mgid);
if (ret < 0) {
- ipoib_warn(priv, "couldn't attach QP to multicast group "
- IPOIB_GID_FMT "\n",
- IPOIB_GID_ARG(mcast->mcmember.mgid));
+ if (ret != -ENXIO) /* No pkey found */
+ ipoib_warn(priv, "couldn't attach QP to multicast group "
+ IPOIB_GID_FMT "\n",
+ IPOIB_GID_ARG(mcast->mcmember.mgid));
clear_bit(IPOIB_MCAST_FLAG_ATTACHED, &mcast->flags);
return ret;
@@ -312,7 +313,7 @@ ipoib_mcast_sendonly_join_complete(int s
status = ipoib_mcast_join_finish(mcast, &multicast->rec);
if (status) {
- if (mcast->logcount++ < 20)
+ if (mcast->logcount++ < 20 && status != -ENXIO)
ipoib_dbg_mcast(netdev_priv(dev), "multicast join failed for "
IPOIB_GID_FMT ", status %d\n",
IPOIB_GID_ARG(mcast->mcmember.mgid), status);
@@ -416,7 +417,7 @@ static int ipoib_mcast_join_complete(int
", status %d\n",
IPOIB_GID_ARG(mcast->mcmember.mgid),
status);
- } else {
+ } else if (status != -ENXIO) {
ipoib_warn(priv, "multicast join failed for "
IPOIB_GID_FMT ", status %d\n",
IPOIB_GID_ARG(mcast->mcmember.mgid),
Index: b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
===================================================================
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2007-03-29 08:12:09.210606845 +0200
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c 2007-03-29 11:32:58.956321208 +0200
@@ -52,8 +52,10 @@ int ipoib_mcast_attach(struct net_device
if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index)) {
clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
ret = -ENXIO;
+ ipoib_dbg(priv, "pkey %X not found\n", priv->pkey);
goto out;
}
+ ipoib_dbg(priv, "pkey %X found at index %d\n", priv->pkey, pkey_index);
set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
/* set correct QKey for QP */
@@ -260,7 +262,6 @@ void ipoib_event(struct ib_event_handler
container_of(handler, struct ipoib_dev_priv, event_handler);
if ((record->event == IB_EVENT_PORT_ERR ||
- record->event == IB_EVENT_PKEY_CHANGE ||
record->event == IB_EVENT_PORT_ACTIVE ||
record->event == IB_EVENT_LID_CHANGE ||
record->event == IB_EVENT_SM_CHANGE ||
@@ -268,5 +269,9 @@ void ipoib_event(struct ib_event_handler
record->element.port_num == priv->port) {
ipoib_dbg(priv, "Port state change event\n");
queue_work(ipoib_workqueue, &priv->flush_task);
+ } else if (record->event == IB_EVENT_PKEY_CHANGE &&
+ record->element.port_num == priv->port) {
+ ipoib_dbg(priv, "pkey change event on port:%d\n", priv->port);
+ queue_work(ipoib_workqueue, &priv->restart_qp_task);
}
}
More information about the ewg
mailing list