[ofa-general] [RFC PATCH 3/5] rdma/cma: simplify locking needed for serialization of callbacks

[Or Gerlitz]

The rdma-cm has some logic in place to make sure that callbacks on an ID are
delivered to the consumer in serialized manner, specifically it has code to protect
against the device removal racing with a callback now being delivered to the user.

This patch simplifies this logic by using a mutex per ID instead of the wait queue
and atomic variable. I have left the disable/enable_remove notation such that the patch
would be easier to read, but if this approach is accepted, I think we want to change
it to disable/enable_callback

Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>

Index: linux-2.6.26-rc3/drivers/infiniband/core/cma.c
===================================================================
--- linux-2.6.26-rc3.orig/drivers/infiniband/core/cma.c	2008-05-26 15:11:17.000000000 +0300
+++ linux-2.6.26-rc3/drivers/infiniband/core/cma.c	2008-05-26 15:22:11.000000000 +0300
@@ -126,8 +126,7 @@ struct rdma_id_private {

 	struct completion	comp;
 	atomic_t		refcount;
-	wait_queue_head_t	wait_remove;
-	atomic_t		dev_remove;
+	struct mutex		handler_mutex;

 	int			backlog;
 	int			timeout_ms;
@@ -359,7 +358,7 @@ static int cma_disable_remove(struct rdm

 	spin_lock_irqsave(&id_priv->lock, flags);
 	if (id_priv->state == state) {
-		atomic_inc(&id_priv->dev_remove);
+		mutex_lock(&id_priv->handler_mutex);
 		ret = 0;
 	} else
 		ret = -EINVAL;
@@ -369,8 +368,7 @@ static int cma_disable_remove(struct rdm

 static void cma_enable_remove(struct rdma_id_private *id_priv)
 {
-	if (atomic_dec_and_test(&id_priv->dev_remove))
-		wake_up(&id_priv->wait_remove);
+	mutex_unlock(&id_priv->handler_mutex);
 }

 static int cma_has_cm_dev(struct rdma_id_private *id_priv)
@@ -395,8 +393,7 @@ struct rdma_cm_id *rdma_create_id(rdma_c
 	mutex_init(&id_priv->qp_mutex);
 	init_completion(&id_priv->comp);
 	atomic_set(&id_priv->refcount, 1);
-	init_waitqueue_head(&id_priv->wait_remove);
-	atomic_set(&id_priv->dev_remove, 0);
+	mutex_init(&id_priv->handler_mutex);
 	INIT_LIST_HEAD(&id_priv->listen_list);
 	INIT_LIST_HEAD(&id_priv->mc_list);
 	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
@@ -1118,7 +1115,7 @@ static int cma_req_handler(struct ib_cm_
 		goto out;
 	}

-	atomic_inc(&conn_id->dev_remove);
+	mutex_lock(&conn_id->handler_mutex);
 	mutex_lock(&lock);
 	ret = cma_acquire_dev(conn_id);
 	mutex_unlock(&lock);
@@ -1296,7 +1293,7 @@ static int iw_conn_req_handler(struct iw
 		goto out;
 	}
 	conn_id = container_of(new_cm_id, struct rdma_id_private, id);
-	atomic_inc(&conn_id->dev_remove);
+	mutex_lock(&conn_id->handler_mutex);
 	conn_id->state = CMA_CONNECT;

 	dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
@@ -1588,7 +1585,7 @@ static void cma_work_handler(struct work
 	struct rdma_id_private *id_priv = work->id;
 	int destroy = 0;

-	atomic_inc(&id_priv->dev_remove);
+	mutex_lock(&id_priv->handler_mutex);
 	if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
 		goto out;

@@ -1760,7 +1757,7 @@ static void addr_handler(int status, str
 	struct rdma_cm_event event;

 	memset(&event, 0, sizeof event);
-	atomic_inc(&id_priv->dev_remove);
+	mutex_lock(&id_priv->handler_mutex);

 	/*
 	 * Grab mutex to block rdma_destroy_id() from removing the device while
@@ -2756,22 +2753,26 @@ static int cma_remove_id_dev(struct rdma
 {
 	struct rdma_cm_event event;
 	enum cma_state state;
-
+	int ret = 0;
+
 	/* Record that we want to remove the device */
 	state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
 	if (state == CMA_DESTROYING)
 		return 0;

 	cma_cancel_operation(id_priv, state);
-	wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
+	mutex_lock(&id_priv->handler_mutex);

 	/* Check for destruction from another callback. */
 	if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
-		return 0;
+		goto out;

 	memset(&event, 0, sizeof event);
 	event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
-	return id_priv->id.event_handler(&id_priv->id, &event);
+	ret = id_priv->id.event_handler(&id_priv->id, &event);
+out:
+	mutex_unlock(&id_priv->handler_mutex);
+	return ret;
 }

 static void cma_process_remove(struct cma_device *cma_dev)