[openib-general] [PATCH] timeout wq code
Sean Hefty
mshefty at ichips.intel.com
Thu Oct 21 16:08:49 PDT 2004
Code to use a work queue to time out MADs. There is one work queue per port. (Completion handling code was not changed.)
I'm working on creating a few simple test cases to verify MAD functionality (registration, timeouts, sends, receives, and RMPP in the future), but these are not yet done.
-Sean
Index: access/ib_mad_priv.h
===================================================================
--- access/ib_mad_priv.h (revision 1036)
+++ access/ib_mad_priv.h (working copy)
@@ -58,6 +58,7 @@
#include <linux/pci.h>
#include <linux/kthread.h>
+#include <linux/workqueue.h>
#include <ib_mad.h>
#include <ib_smi.h>
@@ -112,6 +113,8 @@
spinlock_t lock;
struct list_head send_list;
struct list_head wait_list;
+ struct work_struct work;
+ unsigned long timeout;
atomic_t refcount;
wait_queue_head_t wait;
@@ -149,6 +152,7 @@
spinlock_t reg_lock;
struct ib_mad_mgmt_class_table *version[MAX_MGMT_VERSION];
struct list_head agent_list;
+ struct workqueue_struct *wq;
spinlock_t send_list_lock;
struct list_head send_posted_mad_list;
Index: access/ib_mad.c
===================================================================
--- access/ib_mad.c (revision 1036)
+++ access/ib_mad.c (working copy)
@@ -87,6 +87,7 @@
static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_send_wc *mad_send_wc);
+static void timeout_sends(void *data);
/*
* Returns a ib_mad_port_private structure or NULL for a device/port.
@@ -264,6 +265,7 @@
spin_lock_init(&mad_agent_priv->lock);
INIT_LIST_HEAD(&mad_agent_priv->send_list);
INIT_LIST_HEAD(&mad_agent_priv->wait_list);
+ INIT_WORK(&mad_agent_priv->work, timeout_sends, mad_agent_priv);
atomic_set(&mad_agent_priv->refcount, 1);
init_waitqueue_head(&mad_agent_priv->wait);
mad_agent_priv->port_priv = port_priv;
@@ -299,6 +301,9 @@
*/
cancel_mads(mad_agent_priv);
+ cancel_delayed_work(&mad_agent_priv->work);
+ flush_workqueue(mad_agent_priv->port_priv->wq);
+
spin_lock_irqsave(&mad_agent_priv->port_priv->reg_lock, flags);
remove_mad_reg_req(mad_agent_priv);
list_del(&mad_agent_priv->agent_list);
@@ -398,7 +403,8 @@
mad_send_wr->tid = send_wr->wr.ud.mad_hdr->tid;
mad_send_wr->agent = mad_agent;
/* Timeout will be updated after send completes. */
- mad_send_wr->timeout = cur_send_wr->wr.ud.timeout_ms;
+ mad_send_wr->timeout = msecs_to_jiffies(cur_send_wr->wr.
+ ud.timeout_ms);
/* One reference for each work request to QP + response. */
mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);
mad_send_wr->status = IB_WC_SUCCESS;
@@ -422,9 +428,7 @@
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
*bad_send_wr = cur_send_wr;
- if (atomic_dec_and_test(&mad_agent_priv->refcount))
- wake_up(&mad_agent_priv->wait);
-
+ atomic_dec(&mad_agent_priv->refcount);
return ret;
}
cur_send_wr= next_send_wr;
@@ -1007,6 +1011,30 @@
return;
}
+static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
+{
+ struct ib_mad_send_wr_private *mad_send_wr;
+ unsigned long delay;
+
+ if (list_empty(&mad_agent_priv->wait_list)) {
+ cancel_delayed_work(&mad_agent_priv->work);
+ } else {
+ mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
+ struct ib_mad_send_wr_private,
+ agent_list);
+
+ if (time_after(mad_agent_priv->timeout, mad_send_wr->timeout)) {
+ mad_agent_priv->timeout = mad_send_wr->timeout;
+ cancel_delayed_work(&mad_agent_priv->work);
+ delay = mad_send_wr->timeout - jiffies;
+ if ((long)delay <= 0)
+ delay = 1;
+ queue_delayed_work(mad_agent_priv->port_priv->wq,
+ &mad_agent_priv->work, delay);
+ }
+ }
+}
+
static void wait_for_response(struct ib_mad_agent_private *mad_agent_priv,
struct ib_mad_send_wr_private *mad_send_wr )
{
@@ -1027,6 +1055,13 @@
break;
}
list_add(&mad_send_wr->agent_list, list_item);
+
+ /* Re-schedule a work item if we have a shorter timeout. */
+ if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list) {
+ cancel_delayed_work(&mad_agent_priv->work);
+ queue_delayed_work(mad_agent_priv->port_priv->wq,
+ &mad_agent_priv->work, delay);
+ }
}
/*
@@ -1059,6 +1094,7 @@
/* Remove send from MAD agent and notify client of completion */
list_del(&mad_send_wr->agent_list);
+ adjust_timeout(mad_agent_priv);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
if (mad_send_wr->status != IB_WC_SUCCESS )
@@ -1233,6 +1269,7 @@
}
list_del(&mad_send_wr->agent_list);
+ adjust_timeout(mad_agent_priv);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
@@ -1250,6 +1287,47 @@
}
EXPORT_SYMBOL(ib_cancel_mad);
+static void timeout_sends(void *data)
+{
+ struct ib_mad_agent_private *mad_agent_priv;
+ struct ib_mad_send_wr_private *mad_send_wr;
+ struct ib_mad_send_wc mad_send_wc;
+ unsigned long flags, delay;
+
+ mad_agent_priv = (struct ib_mad_agent_private *)data;
+
+ mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
+ mad_send_wc.vendor_err = 0;
+
+ spin_lock_irqsave(&mad_agent_priv->lock, flags);
+ while (!list_empty(&mad_agent_priv->wait_list)) {
+ mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
+ struct ib_mad_send_wr_private,
+ agent_list);
+
+ if (time_after(mad_send_wr->timeout, jiffies)) {
+ delay = mad_send_wr->timeout - jiffies;
+ if ((long)delay <= 0)
+ delay = 1;
+ queue_delayed_work(mad_agent_priv->port_priv->wq,
+ &mad_agent_priv->work, delay);
+ break;
+ }
+
+ list_del(&mad_send_wr->agent_list);
+ spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+
+ mad_send_wc.wr_id = mad_send_wr->wr_id;
+ mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
+ &mad_send_wc);
+
+ kfree(mad_send_wr);
+ atomic_dec(&mad_agent_priv->refcount);
+ spin_lock_irqsave(&mad_agent_priv->lock, flags);
+ }
+ spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+}
+
/*
* IB MAD thread
*/
@@ -1756,14 +1834,20 @@
for (i = 0; i < IB_MAD_QPS_CORE; i++)
INIT_LIST_HEAD(&port_priv->recv_posted_mad_list[i]);
+ port_priv->wq = create_workqueue("ib_mad");
+ if (!port_priv->wq) {
+ ret = -ENOMEM;
+ goto error8;
+ }
+
ret = ib_mad_thread_init(port_priv);
if (ret)
- goto error8;
+ goto error9;
ret = ib_mad_port_start(port_priv);
if (ret) {
printk(KERN_ERR PFX "Couldn't start port\n");
- goto error9;
+ goto error10;
}
spin_lock_irqsave(&ib_mad_port_list_lock, flags);
@@ -1772,8 +1856,10 @@
return 0;
-error9:
+error10:
kthread_stop(port_priv->mad_thread);
+error9:
+ destroy_workqueue(port_priv->wq);
error8:
ib_destroy_qp(port_priv->qp[1]);
error7:
@@ -1814,6 +1900,7 @@
ib_mad_port_stop(port_priv);
kthread_stop(port_priv->mad_thread);
+ destroy_workqueue(port_priv->wq);
ib_destroy_qp(port_priv->qp[1]);
ib_destroy_qp(port_priv->qp[0]);
ib_dereg_mr(port_priv->mr);
--
More information about the general
mailing list