[ofa-general] [PATCH] mlx4: fix fast registration implementation

[Jack Morgenstein]

The low-level driver modified the page-list addresses for FRWR post send
to big-endian, and set a "present" bit. This caused problems later when the
ULP attempted to unmap the pages in the page-list (using the list addresses which
were assumed to be still in CPU-endian order).

The cause of the crash was found by Vu Pham of Mellanox.
The fix is along the lines suggested by Steve Wise in comment #21 in Bugzilla 1571.

This patch fixes Bugzilla 1571.

Signed-off-by: Jack Morgenstein <jackm at dev.mellanox.co.il>

---
Roland, please take this for kernel 2.6.30.

diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 9974e88..a8c0bc4 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -86,6 +86,7 @@ struct mlx4_ib_mr {
 
 struct mlx4_ib_fast_reg_page_list {
 	struct ib_fast_reg_page_list	ibfrpl;
+	u64				*mapped_page_list;
 	dma_addr_t			map;
 };
 
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 8e4d26d..fddf583 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -231,16 +231,22 @@ struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device
 	if (!mfrpl)
 		return ERR_PTR(-ENOMEM);
 
-	mfrpl->ibfrpl.page_list = dma_alloc_coherent(&dev->dev->pdev->dev,
+	mfrpl->ibfrpl.page_list = kmalloc(size, GFP_KERNEL);
+	if (!mfrpl->ibfrpl.page_list)
+		goto err_free;
+
+	mfrpl->mapped_page_list = dma_alloc_coherent(&dev->dev->pdev->dev,
 						     size, &mfrpl->map,
 						     GFP_KERNEL);
 	if (!mfrpl->ibfrpl.page_list)
-		goto err_free;
+		goto err_free_mfrpl;
 
 	WARN_ON(mfrpl->map & 0x3f);
 
 	return &mfrpl->ibfrpl;
 
+err_free_mfrpl:
+	kfree(mfrpl->ibfrpl.page_list);
 err_free:
 	kfree(mfrpl);
 	return ERR_PTR(-ENOMEM);
@@ -252,8 +258,9 @@ void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
 	struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(page_list);
 	int size = page_list->max_page_list_len * sizeof (u64);
 
-	dma_free_coherent(&dev->dev->pdev->dev, size, page_list->page_list,
+	dma_free_coherent(&dev->dev->pdev->dev, size, mfrpl->mapped_page_list,
 			  mfrpl->map);
+	kfree(mfrpl->ibfrpl.page_list);
 	kfree(mfrpl);
 }
 
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index f385a24..20724ae 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1365,7 +1365,7 @@ static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr)
 	int i;
 
 	for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i)
-		wr->wr.fast_reg.page_list->page_list[i] =
+		mfrpl->mapped_page_list[i] =
 			cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] |
 				    MLX4_MTT_FLAG_PRESENT);