[openib-general] [PATCH] timeout wq code

Sean Hefty mshefty at ichips.intel.com
Thu Oct 21 16:08:49 PDT 2004


Code to use a work queue to time out MADs.  There is one work queue per port.  (Completion handling code was not changed.)

I'm working on creating a few simple test cases to verify MAD functionality (registration, timeouts, sends, receives, and RMPP in the future), but these are not yet done.

-Sean


Index: access/ib_mad_priv.h
===================================================================
--- access/ib_mad_priv.h	(revision 1036)
+++ access/ib_mad_priv.h	(working copy)
@@ -58,6 +58,7 @@
 
 #include <linux/pci.h>
 #include <linux/kthread.h>
+#include <linux/workqueue.h>
 #include <ib_mad.h>
 #include <ib_smi.h>
 
@@ -112,6 +113,8 @@
 	spinlock_t lock;
 	struct list_head send_list;
 	struct list_head wait_list;
+	struct work_struct work;
+	unsigned long timeout;
 
 	atomic_t refcount;
 	wait_queue_head_t wait;
@@ -149,6 +152,7 @@
 	spinlock_t reg_lock;
 	struct ib_mad_mgmt_class_table *version[MAX_MGMT_VERSION];
 	struct list_head agent_list;
+	struct workqueue_struct *wq;
 
 	spinlock_t send_list_lock;
 	struct list_head send_posted_mad_list;
Index: access/ib_mad.c
===================================================================
--- access/ib_mad.c	(revision 1036)
+++ access/ib_mad.c	(working copy)
@@ -87,6 +87,7 @@
 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
 static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
 				    struct ib_mad_send_wc *mad_send_wc);
+static void timeout_sends(void *data);
 
 /*
  * Returns a ib_mad_port_private structure or NULL for a device/port.
@@ -264,6 +265,7 @@
 	spin_lock_init(&mad_agent_priv->lock);
 	INIT_LIST_HEAD(&mad_agent_priv->send_list);
 	INIT_LIST_HEAD(&mad_agent_priv->wait_list);
+	INIT_WORK(&mad_agent_priv->work, timeout_sends, mad_agent_priv);
 	atomic_set(&mad_agent_priv->refcount, 1);
 	init_waitqueue_head(&mad_agent_priv->wait);
 	mad_agent_priv->port_priv = port_priv;
@@ -299,6 +301,9 @@
 	 */
 	cancel_mads(mad_agent_priv);
 
+	cancel_delayed_work(&mad_agent_priv->work);
+	flush_workqueue(mad_agent_priv->port_priv->wq);
+
 	spin_lock_irqsave(&mad_agent_priv->port_priv->reg_lock, flags);
 	remove_mad_reg_req(mad_agent_priv);
 	list_del(&mad_agent_priv->agent_list);
@@ -398,7 +403,8 @@
 		mad_send_wr->tid = send_wr->wr.ud.mad_hdr->tid;
 		mad_send_wr->agent = mad_agent;
 		/* Timeout will be updated after send completes. */
-		mad_send_wr->timeout = cur_send_wr->wr.ud.timeout_ms;
+		mad_send_wr->timeout = msecs_to_jiffies(cur_send_wr->wr.
+							ud.timeout_ms);
 		/* One reference for each work request to QP + response. */
 		mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);
 		mad_send_wr->status = IB_WC_SUCCESS;
@@ -422,9 +428,7 @@
 			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
 			*bad_send_wr = cur_send_wr;
-			if (atomic_dec_and_test(&mad_agent_priv->refcount))
-				wake_up(&mad_agent_priv->wait);
-
+			atomic_dec(&mad_agent_priv->refcount);
 			return ret;		
 		}
 		cur_send_wr= next_send_wr;
@@ -1007,6 +1011,30 @@
 	return;
 }
 
+static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
+{
+	struct ib_mad_send_wr_private *mad_send_wr;
+	unsigned long delay;
+
+	if (list_empty(&mad_agent_priv->wait_list)) {
+		cancel_delayed_work(&mad_agent_priv->work);
+	} else {
+		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
+					 struct ib_mad_send_wr_private,
+					 agent_list);
+
+		if (time_after(mad_agent_priv->timeout, mad_send_wr->timeout)) {
+			mad_agent_priv->timeout = mad_send_wr->timeout;
+			cancel_delayed_work(&mad_agent_priv->work);
+			delay = mad_send_wr->timeout - jiffies;
+			if ((long)delay <= 0)
+				delay = 1;
+			queue_delayed_work(mad_agent_priv->port_priv->wq,
+					   &mad_agent_priv->work, delay);
+		}
+	}
+}
+
 static void wait_for_response(struct ib_mad_agent_private *mad_agent_priv,
 			      struct ib_mad_send_wr_private *mad_send_wr )
 {
@@ -1027,6 +1055,13 @@
 			break;
 	}
 	list_add(&mad_send_wr->agent_list, list_item);
+
+	/* Re-schedule a work item if we have a shorter timeout. */
+	if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list) {
+		cancel_delayed_work(&mad_agent_priv->work);
+		queue_delayed_work(mad_agent_priv->port_priv->wq,
+				   &mad_agent_priv->work, delay);
+	}
 }
 
 /*
@@ -1059,6 +1094,7 @@
 
 	/* Remove send from MAD agent and notify client of completion */
 	list_del(&mad_send_wr->agent_list);
+	adjust_timeout(mad_agent_priv);
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 	
 	if (mad_send_wr->status != IB_WC_SUCCESS )
@@ -1233,6 +1269,7 @@
 	}
 
 	list_del(&mad_send_wr->agent_list);
+	adjust_timeout(mad_agent_priv);
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
 	mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
@@ -1250,6 +1287,47 @@
 }
 EXPORT_SYMBOL(ib_cancel_mad);
 
+static void timeout_sends(void *data)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+	struct ib_mad_send_wr_private *mad_send_wr;
+	struct ib_mad_send_wc mad_send_wc;
+	unsigned long flags, delay;
+
+	mad_agent_priv = (struct ib_mad_agent_private *)data;
+
+	mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
+	mad_send_wc.vendor_err = 0;
+
+	spin_lock_irqsave(&mad_agent_priv->lock, flags);
+	while (!list_empty(&mad_agent_priv->wait_list)) {
+		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
+					 struct ib_mad_send_wr_private,
+					 agent_list);
+
+		if (time_after(mad_send_wr->timeout, jiffies)) {
+			delay = mad_send_wr->timeout - jiffies;
+			if ((long)delay <= 0)
+				delay = 1;
+			queue_delayed_work(mad_agent_priv->port_priv->wq,
+					   &mad_agent_priv->work, delay);
+			break;
+		}
+
+		list_del(&mad_send_wr->agent_list);
+		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+
+		mad_send_wc.wr_id = mad_send_wr->wr_id;
+		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
+						   &mad_send_wc);
+
+		kfree(mad_send_wr);
+		atomic_dec(&mad_agent_priv->refcount);
+		spin_lock_irqsave(&mad_agent_priv->lock, flags);
+	}
+	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+}
+
 /*
  * IB MAD thread
  */
@@ -1756,14 +1834,20 @@
 	for (i = 0; i < IB_MAD_QPS_CORE; i++)
 		INIT_LIST_HEAD(&port_priv->recv_posted_mad_list[i]);
 
+	port_priv->wq = create_workqueue("ib_mad");
+	if (!port_priv->wq) {
+		ret = -ENOMEM;
+		goto error8;
+	}
+
 	ret = ib_mad_thread_init(port_priv);
 	if (ret)
-		goto error8;
+		goto error9;
 
 	ret = ib_mad_port_start(port_priv);
 	if (ret) {
 		printk(KERN_ERR PFX "Couldn't start port\n");
-		goto error9;
+		goto error10;
 	}
 
 	spin_lock_irqsave(&ib_mad_port_list_lock, flags);
@@ -1772,8 +1856,10 @@
 
 	return 0;
 
-error9:
+error10:
 	kthread_stop(port_priv->mad_thread);
+error9:
+	destroy_workqueue(port_priv->wq);
 error8:
 	ib_destroy_qp(port_priv->qp[1]);
 error7:
@@ -1814,6 +1900,7 @@
 
 	ib_mad_port_stop(port_priv);
 	kthread_stop(port_priv->mad_thread);
+	destroy_workqueue(port_priv->wq);
 	ib_destroy_qp(port_priv->qp[1]);
 	ib_destroy_qp(port_priv->qp[0]);
 	ib_dereg_mr(port_priv->mr);


-- 



More information about the general mailing list