[openib-general] ip over ib tx/rx split

[Michael S. Tsirkin]

Hi!
The following simple patch splits the tx and rx completions
for ip over ib to separate cqs.
This shall get better performance on SMP due to reduced cq lock contention.

I dont realy see any negative effect this may have.

Later I plan to look into switching the send cq to polling mode
to reduce interrupts, along the lines of what Ido proposed here.

With this approach, it may be also possible split the completion routine
to send a receive side, thus saving some conditional branches on interrupt path.
I left that for later, too.

Unfortunately I only have one machine alive at the moment,
so it had only limited testing :)
Could someone please tell me what effect it has?

Comments are welcome.

MST

Index: ulp/ipoib/ipoib_verbs.c
===================================================================
--- ulp/ipoib/ipoib_verbs.c	(revision 1716)
+++ ulp/ipoib/ipoib_verbs.c	(working copy)
@@ -175,24 +175,31 @@ int ipoib_transport_dev_init(struct net_
 		return -ENODEV;
 	}
 
-	priv->cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev,
-				IPOIB_TX_RING_SIZE + IPOIB_RX_RING_SIZE + 1);
-	if (IS_ERR(priv->cq)) {
-		printk(KERN_WARNING "%s: failed to create CQ\n", ca->name);
+	priv->tx_cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev,
+				IPOIB_TX_RING_SIZE);
+	if (IS_ERR(priv->tx_cq)) {
+		printk(KERN_WARNING "%s: failed to create TX CQ\n", ca->name);
 		goto out_free_pd;
 	}
 
-	if (ib_req_notify_cq(priv->cq, IB_CQ_NEXT_COMP))
-		goto out_free_cq;
+	priv->rx_cq = ib_create_cq(priv->ca, ipoib_ib_completion, NULL, dev,
+				IPOIB_RX_RING_SIZE);
+	if (IS_ERR(priv->rx_cq)) {
+		printk(KERN_WARNING "%s: failed to create RX CQ\n", ca->name);
+		goto out_free_tx_cq;
+	}
+
+	if (ib_req_notify_cq(priv->rx_cq, IB_CQ_NEXT_COMP))
+		goto out_free_rx_cq;
 
 	priv->mr = ib_get_dma_mr(priv->pd, IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(priv->mr)) {
 		printk(KERN_WARNING "%s: ib_get_dma_mr failed\n", ca->name);
-		goto out_free_cq;
+		goto out_free_rx_cq;
 	}
 
-	init_attr.send_cq = priv->cq;
-	init_attr.recv_cq = priv->cq,
+	init_attr.send_cq = priv->tx_cq;
+	init_attr.recv_cq = priv->rx_cq,
 
 	priv->qp = ib_create_qp(priv->pd, &init_attr);
 	if (IS_ERR(priv->qp)) {
@@ -216,8 +223,11 @@ int ipoib_transport_dev_init(struct net_
 out_free_mr:
 	ib_dereg_mr(priv->mr);
 
-out_free_cq:
-	ib_destroy_cq(priv->cq);
+out_free_rx_cq:
+	ib_destroy_cq(priv->rx_cq);
+
+out_free_tx_cq:
+	ib_destroy_cq(priv->tx_cq);
 
 out_free_pd:
 	ib_dealloc_pd(priv->pd);
@@ -239,8 +249,11 @@ void ipoib_transport_dev_cleanup(struct 
 	if (ib_dereg_mr(priv->mr))
 		ipoib_warn(priv, "ib_dereg_mr failed\n");
 
-	if (ib_destroy_cq(priv->cq))
-		ipoib_warn(priv, "ib_cq_destroy failed\n");
+	if (ib_destroy_cq(priv->rx_cq))
+		ipoib_warn(priv, "ib_cq_destroy for rx cq failed\n");
+
+	if (ib_destroy_cq(priv->tx_cq))
+		ipoib_warn(priv, "ib_cq_destroy for tx cq failed\n");
 
 	if (ib_dealloc_pd(priv->pd))
 		ipoib_warn(priv, "ib_dealloc_pd failed\n");
Index: ulp/ipoib/ipoib.h
===================================================================
--- ulp/ipoib/ipoib.h	(revision 1716)
+++ ulp/ipoib/ipoib.h	(working copy)
@@ -137,7 +137,8 @@ struct ipoib_dev_priv {
 	u16           	  pkey;
 	struct ib_pd  	 *pd;
 	struct ib_mr  	 *mr;
-	struct ib_cq  	 *cq;
+	struct ib_cq  	 *tx_cq;
+	struct ib_cq  	 *rx_cq;
 	struct ib_qp  	 *qp;
 	u32           	  qkey;
 
-- 
MST - Michael S. Tsirkin