[openib-general] [PATCH] ib_sync_cq ( was Re: RFC: ib_set_comp_handler)

[Michael S. Tsirkin]

Roland, Sean,
With the following patch, it becomes legal for clients to modify comp_handler
or cq_context fields in ib_cq structure of an existing cq.

To avoid races, and to make it possible for hw layer to cache
these values, I added a new API ib_sync_cq which must be called
after one of comp_handler and cq_context is changed.

I plan to use this capability in sdp, to disable cq events while a connection
is destroyed. I expect this to be useful in other scenarious (IPoIB NAPI?).

Comments?

---

Make it possible to flush completion events for a specific cq.

Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>

Index: linux-2.6.13/drivers/infiniband/hw/mthca/mthca_cq.c
===================================================================
--- linux-2.6.13.orig/drivers/infiniband/hw/mthca/mthca_cq.c	2005-09-11 17:52:37.000000000 +0300
+++ linux-2.6.13/drivers/infiniband/hw/mthca/mthca_cq.c	2005-09-12 10:33:13.000000000 +0300
@@ -789,6 +789,15 @@ err_out:
 	return err;
 }
 
+void mthca_sync_cq(struct ib_cq *ibcq)
+{
+	struct mthca_dev *dev = to_mdev(ibcq->device);
+	if (dev->mthca_flags & MTHCA_FLAG_MSI_X)
+		synchronize_irq(dev->eq_table.eq[MTHCA_EQ_COMP].msi_x_vector);
+	else
+		synchronize_irq(dev->pdev->irq);
+}
+
 void mthca_free_cq(struct mthca_dev *dev,
 		   struct mthca_cq *cq)
 {
Index: linux-2.6.13/drivers/infiniband/include/rdma/ib_verbs.h
===================================================================
--- linux-2.6.13.orig/drivers/infiniband/include/rdma/ib_verbs.h	2005-09-11 10:24:36.000000000 +0300
+++ linux-2.6.13/drivers/infiniband/include/rdma/ib_verbs.h	2005-09-12 10:40:33.000000000 +0300
@@ -884,6 +884,7 @@ struct ib_device {
 	struct ib_cq *             (*create_cq)(struct ib_device *device, int cqe,
 						struct ib_ucontext *context,
 						struct ib_udata *udata);
+	void                       (*sync_cq)(struct ib_cq *cq);
 	int                        (*destroy_cq)(struct ib_cq *cq);
 	int                        (*resize_cq)(struct ib_cq *cq, int *cqe);
 	int                        (*poll_cq)(struct ib_cq *cq, int num_entries,
@@ -1227,6 +1228,16 @@ struct ib_cq *ib_create_cq(struct ib_dev
 			   void *cq_context, int cqe);
 
 /**
+ * ib_sync_cq - flush CQ completion event handler.
+ *              This must be used after modifying comp_handler or cq_context.
+ * @cq: The CQ to flush events for.
+ */
+static inline void ib_sync_cq(struct ib_cq *cq)
+{
+	return cq->device->sync_cq(cq);
+}
+
+/**
  * ib_resize_cq - Modifies the capacity of the CQ.
  * @cq: The CQ to resize.
  * @cqe: The minimum size of the CQ.
Index: linux-2.6.13/drivers/infiniband/hw/mthca/mthca_provider.c
===================================================================
--- linux-2.6.13.orig/drivers/infiniband/hw/mthca/mthca_provider.c	2005-09-11 10:24:37.000000000 +0300
+++ linux-2.6.13/drivers/infiniband/hw/mthca/mthca_provider.c	2005-09-12 10:34:00.000000000 +0300
@@ -1090,6 +1090,7 @@ int mthca_register_device(struct mthca_d
 	dev->ib_dev.destroy_qp           = mthca_destroy_qp;
 	dev->ib_dev.create_cq            = mthca_create_cq;
 	dev->ib_dev.destroy_cq           = mthca_destroy_cq;
+	dev->ib_dev.sync_cq              = mthca_sync_cq;
 	dev->ib_dev.poll_cq              = mthca_poll_cq;
 	dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
 	dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;

-- 
MST