[ofa-general] Re: [PATCH V2 - libibverbs] Added reference count to completion event channels

[Roland Dreier]

OK, I got bored and tried to implement this using a mutex in the
ibv_context structure.  How does this (compile tested only) patch
look?

 - R.


diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 2ae50ab..acc1b82 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -573,11 +573,14 @@ struct ibv_qp {
 };
 
 struct ibv_comp_channel {
+	struct ibv_context     *context;
 	int			fd;
+	int			refcnt;
 };
 
 struct ibv_cq {
 	struct ibv_context     *context;
+	struct ibv_comp_channel *channel;
 	void		       *cq_context;
 	uint32_t		handle;
 	int			cqe;
@@ -680,12 +683,13 @@ struct ibv_context_ops {
 };
 
 struct ibv_context {
-	struct ibv_device         *device;
-	struct ibv_context_ops	   ops;
-	int                        cmd_fd;
-	int                        async_fd;
-	int                        num_comp_vectors;
-	void			  *abi_compat;
+	struct ibv_device      *device;
+	struct ibv_context_ops	ops;
+	int			cmd_fd;
+	int			async_fd;
+	int			num_comp_vectors;
+	pthread_mutex_t		mutex;
+	void		       *abi_compat;
 };
 
 /**
diff --git a/src/cmd.c b/src/cmd.c
index f7d3fde..a0bfaad 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -75,7 +75,9 @@ static int ibv_cmd_get_context_v2(struct ibv_context *context,
 
 	context->async_fd         = resp->async_fd;
 	context->num_comp_vectors = 1;
+	t->channel.context        = context;
 	t->channel.fd		  = cq_fd;
+	t->channel.refcnt	  = 0;
 	context->abi_compat       = t;
 
 	return 0;
diff --git a/src/device.c b/src/device.c
index bca1ce9..3abc1eb 100644
--- a/src/device.c
+++ b/src/device.c
@@ -138,6 +138,7 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 
 	context->device = device;
 	context->cmd_fd = cmd_fd;
+	pthread_mutex_init(&context->mutex, NULL);
 
 	return context;
 
diff --git a/src/verbs.c b/src/verbs.c
index 56513e4..5334af1 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -226,7 +226,9 @@ struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context)
 		return NULL;
 	}
 
-	channel->fd = resp.fd;
+	channel->context = context;
+	channel->fd      = resp.fd;
+	channel->refcnt  = 0;
 
 	return channel;
 }
@@ -240,23 +242,44 @@ static int ibv_destroy_comp_channel_v2(struct ibv_comp_channel *channel)
 
 int ibv_destroy_comp_channel(struct ibv_comp_channel *channel)
 {
-	if (abi_ver <= 2)
-		return ibv_destroy_comp_channel_v2(channel);
+	int ret;
+
+	pthread_mutex_lock(&channel->context->mutex);
+
+	if (channel->refcnt) {
+		ret = EBUSY;
+		goto out;
+	}
+
+	if (abi_ver <= 2) {
+		ret = ibv_destroy_comp_channel_v2(channel);
+		goto out;
+	}
 
 	close(channel->fd);
 	free(channel);
+	ret = 0;
 
-	return 0;
+out:
+	pthread_mutex_unlock(&channel->context->mutex);
+
+	return ret;
 }
 
 struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_context,
 			       struct ibv_comp_channel *channel, int comp_vector)
 {
-	struct ibv_cq *cq = context->ops.create_cq(context, cqe, channel,
-						   comp_vector);
+	struct ibv_cq *cq;
+
+	pthread_mutex_lock(&context->mutex);
+
+	cq = context->ops.create_cq(context, cqe, channel, comp_vector);
 
 	if (cq) {
 		cq->context    	     	   = context;
+		cq->channel		   = channel;
+		if (channel)
+			++channel->refcnt;
 		cq->cq_context 	     	   = cq_context;
 		cq->comp_events_completed  = 0;
 		cq->async_events_completed = 0;
@@ -264,6 +287,8 @@ struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_co
 		pthread_cond_init(&cq->cond, NULL);
 	}
 
+	pthread_mutex_unlock(&context->mutex);
+
 	return cq;
 }
 default_symver(__ibv_create_cq, ibv_create_cq);
@@ -279,7 +304,17 @@ default_symver(__ibv_resize_cq, ibv_resize_cq);
 
 int __ibv_destroy_cq(struct ibv_cq *cq)
 {
-	return cq->context->ops.destroy_cq(cq);
+	int ret;
+
+	pthread_mutex_lock(&cq->context->mutex);
+
+	ret = cq->context->ops.destroy_cq(cq);
+	if (cq->channel && !ret)
+		--cq->channel->refcnt;
+
+	pthread_mutex_unlock(&cq->context->mutex);
+
+	return ret;
 }
 default_symver(__ibv_destroy_cq, ibv_destroy_cq);