[openib-general] RFC: CMA backlog (was Re: CMA backlog)

[Michael S. Tsirkin]

Quoting r. Sean Hefty <sean.hefty at intel.com>:
> >Yes, that was my thinking. To avoid touching all users, maybe the simplest way
> >is to make ib_cm discard the new cm_id without reject if the client callback
> >returned -ENOMEM?
> >
> >If you consider that in out of memory situation sending reject will also likely
> >fail, this might be a good idea, regardless.
> >
> >Sounds good?
> 
> I'd like to get some other feedback, but this approach sounds reasonable.

Here's an untested patch that does this. Comments?

Signed-off-by: Jack Morgenstein <jackm at mellanox.co.il>

Index: src/drivers/infiniband/core/cma.c
===================================================================
--- src.orig/drivers/infiniband/core/cma.c	2006-06-07 11:33:04.359936000 +0300
+++ src/drivers/infiniband/core/cma.c	2006-06-15 13:44:07.030643000 +0300
@@ -118,7 +118,8 @@ struct rdma_id_private {
 	wait_queue_head_t	wait_remove;
 	atomic_t		dev_remove;
 
 	int			backlog;
+	atomic_t		curr_backlog;
 	int			timeout_ms;
 	struct ib_sa_query	*query;
 	int			query_id;
@@ -328,6 +329,7 @@ struct rdma_cm_id* rdma_create_id(rdma_c
 	atomic_set(&id_priv->dev_remove, 0);
 	INIT_LIST_HEAD(&id_priv->listen_list);
 	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
+	atomic_set(&id_priv->curr_backlog, 0);
 
 	return &id_priv->id;
 }
@@ -1022,6 +1024,9 @@ static int cma_listen_handler(struct rdm
 {
 	struct rdma_id_private *id_priv = id->context;
 
+	if (atomic_read(&id_priv->curr_backlog) > id_priv->backlog)
+		return -ENOMEM;
+
 	id->context = id_priv->id.context;
 	id->event_handler = id_priv->id.event_handler;
 	return id_priv->id.event_handler(id, event);
@@ -1870,6 +1875,25 @@ out:
 }
 EXPORT_SYMBOL(rdma_disconnect);
 
+
+void rdma_backlog_added_one(struct rdma_cm_id *id)
+{
+	struct rdma_id_private *id_priv;
+
+	id_priv = container_of(id, struct rdma_id_private, id);
+	atomic_inc(&id_priv->curr_backlog);
+}
+EXPORT_SYMBOL(rdma_backlog_added_one);
+
+void rdma_backlog_removed_one(struct rdma_cm_id *id)
+{
+	struct rdma_id_private *id_priv;
+
+	id_priv = container_of(id, struct rdma_id_private, id);
+	atomic_dec(&id_priv->curr_backlog);
+}
+EXPORT_SYMBOL(rdma_backlog_removed_one);
+
 static void cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
Index: src/drivers/infiniband/include/rdma/rdma_cm.h
===================================================================
--- src.orig/drivers/infiniband/include/rdma/rdma_cm.h	2006-05-10 11:18:37.538572000 +0300
+++ src/drivers/infiniband/include/rdma/rdma_cm.h	2006-06-15 15:49:37.708725000 +0300
@@ -252,5 +252,21 @@ int rdma_reject(struct rdma_cm_id *id, c
  */
 int rdma_disconnect(struct rdma_cm_id *id);
 
+/**
+ * rdma_backlog_added_one - This function is called by the passive side to
+ *   notify cma that one connection request has been added to backlog queue.
+ *
+ * No error checking is done here (e.g., if backlog is already at max, etc)
+ */
+void rdma_backlog_added_one(struct rdma_cm_id *id);
+
+/**
+ * rdma_backlog_added_one - This function is called by the passive side to
+ *   notify cma that one connection request has been added to backlog queue.
+ *
+ * No error checking is done here (e.g., if queue was already empty)
+ */
+void rdma_backlog_removed_one(struct rdma_cm_id *id);
+
 #endif /* RDMA_CM_H */
 
Index: src/drivers/infiniband/core/cm.c
===================================================================
--- src.orig/drivers/infiniband/core/cm.c	2006-06-07 11:33:04.109937000 +0300
+++ src/drivers/infiniband/core/cm.c	2006-06-15 15:50:34.222140000 +0300
@@ -701,6 +701,19 @@ static void cm_reset_to_idle(struct cm_i
 	}
 }
 
+static void ib_destroy_cm_id_no_resp(struct ib_cm_id *cm_id)
+{
+	struct cm_id_private *cm_id_priv;
+
+	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
+
+	cm_free_id(cm_id->local_id);
+	cm_deref_id(cm_id_priv);
+	kfree(cm_id_priv->compare_data);
+	kfree(cm_id_priv->private_data);
+	kfree(cm_id_priv);
+}
+
 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
 {
 	struct cm_id_private *cm_id_priv;
@@ -1162,7 +1177,10 @@ static void cm_process_work(struct cm_id
 		cm_free_work(work);
 	}
 	cm_deref_id(cm_id_priv);
-	if (ret)
+
+	if (ret == -ENOMEM)
+		ib_destroy_cm_id_no_resp(&cm_id_priv->id);
+	else if (ret)
 		ib_destroy_cm_id(&cm_id_priv->id);
 }
 

-- 
MST