[ofa-general] [PATCH] IB/ipoib: use vmap with allocation of tx ring

[Eli Cohen]

IB/ipoib: use vmap with allocation of tx ring

With the introduction of s/g support in IPOIB, the size of struct ipoib_tx_buf
has increased since it reserves room for the fragments. This caused allocations
to fail when large send queues are required. This patch uses an array of pages
and maps them with vmap to increase the certainty of the allocation to succeed.

Signed-off-by: Eli Cohen <eli at mellanox.co.il>
---

I used the alloc and free functions as global functions since I think
we may use the for other allocations in IPOIB.


  drivers/infiniband/ulp/ipoib/ipoib.h      |    9 +++++
  drivers/infiniband/ulp/ipoib/ipoib_main.c |   53 ++++++++++++++++++++++++++---
  2 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index f9b7caa..78a99d6 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -186,6 +186,12 @@ enum ipoib_cm_state {
  	IPOIB_CM_RX_FLUSH  /* Last WQE Reached event observed */
  };

+struct ipoib_vmap {
+       void           *ptr;
+       struct page   **page_arr;
+       int             npages;
+};
+
  struct ipoib_cm_rx {
  	struct ib_cm_id	       *id;
  	struct ib_qp	       *qp;
@@ -293,6 +299,7 @@ struct ipoib_dev_priv {
  	struct ipoib_rx_buf *rx_ring;

  	spinlock_t	     tx_lock;
+	struct ipoib_vmap    tx_vmap_ring;
  	struct ipoib_tx_buf *tx_ring;
  	unsigned	     tx_head;
  	unsigned	     tx_tail;
@@ -458,6 +465,8 @@ int ipoib_vlan_delete(struct net_device *pdev, unsigned short pkey);
  void ipoib_pkey_poll(struct work_struct *work);
  int ipoib_pkey_dev_delay_open(struct net_device *dev);
  void ipoib_drain_cq(struct net_device *dev);
+int ipoib_vmalloc(struct ipoib_vmap *buf, int size);
+void ipoib_vfree(struct ipoib_vmap *buf);

  #ifdef CONFIG_INFINIBAND_IPOIB_CM

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index f96477a..f21fd14 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -92,6 +92,49 @@ static struct ib_client ipoib_client = {
  	.remove = ipoib_remove_one
  };

+int ipoib_vmalloc(struct ipoib_vmap *buf, int size)
+{
+	int     i;
+	int     npages = ALIGN(size, PAGE_SIZE) / PAGE_SIZE;
+	int     ret = -ENOMEM;
+
+	buf->page_arr = kmalloc(npages * sizeof buf->page_arr[0], GFP_KERNEL);
+	if (!buf->page_arr)
+		goto out;
+
+	for (i = 0; i < npages; ++i) {
+		buf->page_arr[i] = alloc_page(GFP_KERNEL);
+		if (!buf->page_arr[i])
+			goto page_fail;
+	}
+
+	buf->npages = npages;
+	buf->ptr = vmap(buf->page_arr, buf->npages, VM_MAP, PAGE_KERNEL);
+	if (!buf->ptr)
+		goto page_fail;
+
+	memset(buf->ptr, 0, size);
+	return 0;
+	
+page_fail:
+	for (; i > 0; --i)
+		__free_page(buf->page_arr[i - 1]);
+
+	kfree(buf->page_arr);
+out:
+	return ret;
+}
+
+void ipoib_vfree(struct ipoib_vmap *buf)
+{
+	int     i;
+
+	for (i = 0; i < buf->npages; ++i)
+		__free_page(buf->page_arr[i]);
+
+	kfree(buf->page_arr);
+}
+
  int ipoib_open(struct net_device *dev)
  {
  	struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -887,13 +930,13 @@ int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
  		goto out;
  	}

-	priv->tx_ring = kzalloc(ipoib_sendq_size * sizeof *priv->tx_ring,
-				GFP_KERNEL);
-	if (!priv->tx_ring) {
+	if (ipoib_vmalloc(&priv->tx_vmap_ring, ipoib_sendq_size *
+			  sizeof *priv->tx_ring)) {
  		printk(KERN_WARNING "%s: failed to allocate TX ring (%d entries)\n",
  		       ca->name, ipoib_sendq_size);
  		goto out_rx_ring_cleanup;
  	}
+	priv->tx_ring = priv->tx_vmap_ring.ptr;

  	/* priv->tx_head, tx_tail & tx_outstanding are already 0 */

@@ -903,7 +946,7 @@ int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
  	return 0;

  out_tx_ring_cleanup:
-	kfree(priv->tx_ring);
+	ipoib_vfree(&priv->tx_vmap_ring);

  out_rx_ring_cleanup:
  	kfree(priv->rx_ring);
@@ -928,7 +971,7 @@ void ipoib_dev_cleanup(struct net_device *dev)
  	ipoib_ib_dev_cleanup(dev);

  	kfree(priv->rx_ring);
-	kfree(priv->tx_ring);
+	ipoib_vfree(&priv->tx_vmap_ring);

  	priv->rx_ring = NULL;
  	priv->tx_ring = NULL;
-- 
1.5.3.8