[openib-general] [PATCH] ib_mad: Track count of posted sends and receives

[Hal Rosenstock]

ib_mad: Track count of posted sends and receives
Also, Encode QP number in WRID of receive WRs
Updated TODO (with this and added sysfs support to futures)

Index: ib_mad.c
===================================================================
--- ib_mad.c	(revision 787)
+++ ib_mad.c	(working copy)
@@ -330,6 +330,7 @@
 		spin_lock_irqsave(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
 		list_add_tail((struct list_head *)mad_send_wr,
 			      &((struct ib_mad_port_private
*)mad_agent->device->mad)->send_posted_mad_list);
+		((struct ib_mad_port_private
*)mad_agent->device->mad)->send_posted_mad_count++;
 		spin_unlock_irqrestore(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
 
 		ret = ib_post_send(mad_agent->qp, &wr, &bad_wr);
@@ -337,6 +338,7 @@
 			/* Unlink from posted send MAD list */
 			spin_unlock_irqrestore(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
 			list_del((struct list_head *)send_wr);
+			((struct ib_mad_port_private
*)mad_agent->device->mad)->send_posted_mad_count--;
 			spin_unlock_irqrestore(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
 			*bad_send_wr = cur_send_wr;
 			printk(KERN_NOTICE "ib_mad_post_send failed\n");
@@ -539,6 +541,20 @@
 	}
 }
 
+static int convert_qpnum(u32 qp_num)
+{
+	/* 
+	 * No redirection currently!!!
+	 * QP0 and QP1 only
+	 * Ultimately, will need table of QP numbers and table index
+	 * as QP numbers will not be packed once redirection supported
+	 */
+	if (qp_num > 1) {
+		printk(KERN_ERR "QP number %d invalid\n", qp_num);
+	}
+	return qp_num;
+}
+
 static void ib_mad_recv_done_handler(struct ib_mad_port_private *priv,
 				     struct ib_wc *wc)
 {
@@ -547,7 +563,11 @@
 				     *head = (struct ib_mad_private_header
*)&priv->recv_posted_mad_list;
 	struct ib_mad_private *recv = NULL;
 	unsigned long flags;
+	u32 qp_num;
 
+	/* WC WRID is the QP number */
+	qp_num = wc->wr_id;
+
 	/* Find entry on posted MAD receive list which corresponds to this
completion */
 	spin_lock_irqsave(&priv->recv_list_lock, flags);
 	list_for_each(entry, head) {
@@ -555,6 +575,7 @@
 			recv = (struct ib_mad_private *)entry;	
 			/* Remove from posted receive MAD list */
 			list_del((struct list_head *)entry);
+			priv->recv_posted_mad_count[convert_qpnum(qp_num)]--;
 			break;
 		}
 	}
@@ -603,6 +624,7 @@
 			send_wr = entry;
 			/* Remove from posted send MAD list */
 			list_del((struct list_head *)entry);
+			priv->send_posted_mad_count--;	
 			break;
 		}
 	}
@@ -768,11 +790,12 @@
 	recv_wr.sg_list = &sg_list;
 	recv_wr.num_sge = 1;
 	recv_wr.recv_flags = IB_RECV_SIGNALED;
-	recv_wr.wr_id = (unsigned long)mad_priv;
+	recv_wr.wr_id = qp->qp_num; /* 32 bits left */
 
 	/* Link receive WR into posted receive MAD list */
 	spin_lock_irqsave(&priv->recv_list_lock, flags);
-	list_add_tail((struct list_head *)mad_priv,
&priv->recv_posted_mad_list[qp->qp_num]); /* This works now as only QP0
and 1 (no redirection)!!! */
+	list_add_tail((struct list_head *)mad_priv,
&priv->recv_posted_mad_list[convert_qpnum(qp->qp_num)]);
+	priv->recv_posted_mad_count[convert_qpnum(qp->qp_num)]++;
 	spin_unlock_irqrestore(&priv->recv_list_lock, flags);
 
 	pci_unmap_addr_set(&mad_priv->header.buf, mapping, sg_list.addr);
@@ -782,6 +805,7 @@
 		/* Unlink from posted receive MAD list */
 		spin_lock_irqsave(&priv->recv_list_lock, flags);
 		list_del((struct list_head *)mad_priv);
+		priv->recv_posted_mad_count[convert_qpnum(qp->qp_num)]--;
 		spin_unlock_irqrestore(&priv->recv_list_lock, flags);
 
 		pci_unmap_single(priv->device->dma_device,
@@ -830,6 +854,7 @@
 
 		}
 		INIT_LIST_HEAD(&priv->recv_posted_mad_list[i]);
+		priv->recv_posted_mad_count[i] = 0;
 		spin_unlock_irqrestore(&priv->recv_list_lock, flags);
 	}
 }
@@ -846,9 +871,10 @@
 	spin_lock_irqsave(&priv->send_list_lock, flags);
 	while (!list_empty(&priv->send_posted_mad_list)) {
 		list_del(priv->send_posted_mad_list.next);
-		/* Call completion handler ? */
+		/* Call completion handler with some status ? */
 	}
 	INIT_LIST_HEAD(&priv->send_posted_mad_list);
+	priv->send_posted_mad_count = 0;
 	spin_unlock_irqrestore(&priv->send_list_lock, flags);
 }
 
@@ -1153,8 +1179,10 @@
 	spin_lock_init(&priv->recv_list_lock);
 	spin_lock_init(&priv->send_list_lock);
 	INIT_LIST_HEAD(&priv->send_posted_mad_list);
+	priv->send_posted_mad_count = 0;
 	for (i = 0; i < IB_MAD_QPS_SUPPORTED; i++) {
 		INIT_LIST_HEAD(&priv->recv_posted_mad_list[i]);
+		priv->recv_posted_mad_count[i] = 0;
 	}
 
 	ib_mad_thread_init(priv);
Index: ib_mad_priv.h
===================================================================
--- ib_mad_priv.h	(revision 787)
+++ ib_mad_priv.h	(working copy)
@@ -132,9 +132,12 @@
 	struct ib_mr *mr;
 
 	spinlock_t send_list_lock;
+	struct list_head send_posted_mad_list;
+	int send_posted_mad_count;
+
 	spinlock_t recv_list_lock;
-	struct list_head send_posted_mad_list;
 	struct list_head recv_posted_mad_list[IB_MAD_QPS_SUPPORTED];
+	int recv_posted_mad_count[IB_MAD_QPS_SUPPORTED];
 
 	struct ib_mad_thread_data thread_data;
 };
Index: TODO
===================================================================
--- TODO	(revision 787)
+++ TODO	(working copy)
@@ -3,9 +3,7 @@
 OpenIB MAD Layer
 
 Short Term
-Track count of posted sends and receives
 Support call of ib_mad_post_send from any context
-Encode QP number in WRID of receive WRs
 Fix list handling
 Use wait queue and wait_event rather than signals and semaphores
 Finish coding receive path
@@ -16,6 +14,7 @@
 Use tasklets/softirq rather than process context
 
 Futures
+sysfs support for MAD layer (statistics, debug support, etc.)
 RMPP support
 Redirection support (including receive list per QP)
 Replace locking with RCU