[ofa-general] [PATCH] IB/iSER: Release connection resources when receiving a RDMA_CM_EVENT_DEVICE_REMOVAL event

[Erez Zilber]

When a RDMA_CM_EVENT_DEVICE_REMOVAL event is raised, iSER should release the
connection resources except for the rdma cm id (which will be released by the
cma itself).

This behavior is necessary if IB modules are unloaded while open-iscsi is still
running. Currently, iSER just initiates a BUG() call.

Signed-off-by: Erez Zilber <erezz at voltaire.com>
---
 drivers/infiniband/ulp/iser/iscsi_iser.h |    2 ++
 drivers/infiniband/ulp/iser/iser_verbs.c |   18 ++++++++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 1ee867b..9fe0b3f 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -249,6 +249,8 @@ struct iser_conn {
 	struct iser_page_vec         *page_vec;     /* represents SG to fmr maps*
 						     * maps serialized as tx is*/
 	struct list_head	     conn_list;       /* entry in ig conn list */
+	wait_queue_head_t            rem_wait;
+	int                          dev_removed;
 };
 
 struct iscsi_iser_conn {
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 993f0a8..9beddb9 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -219,7 +219,8 @@ static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
 	if (ib_conn->qp != NULL)
 		rdma_destroy_qp(ib_conn->cma_id);
 
-	if (ib_conn->cma_id != NULL)
+	/* if the device was removed, the cma will call rdma_destroy_id itself */
+	if (ib_conn->cma_id != NULL && !ib_conn->dev_removed)
 		rdma_destroy_id(ib_conn->cma_id);
 
 	ib_conn->fmr_pool = NULL;
@@ -325,7 +326,10 @@ static void iser_conn_release(struct iser_conn *ib_conn)
 		iser_device_try_release(device);
 	if (ib_conn->iser_conn)
 		ib_conn->iser_conn->ib_conn = NULL;
-	kfree(ib_conn);
+	if (ib_conn->dev_removed)
+		wake_up_interruptible(&ib_conn->rem_wait);
+	else
+		kfree(ib_conn);
 }
 
 /**
@@ -451,6 +455,7 @@ static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
 static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 {
 	int ret = 0;
+	struct iser_conn *ib_conn;
 
 	iser_err("event %d conn %p id %p\n",event->event,cma_id->context,cma_id);
 
@@ -476,8 +481,12 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
 		iser_disconnected_handler(cma_id);
 		break;
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
-		iser_err("Device removal is currently unsupported\n");
-		BUG();
+		ib_conn = (struct iser_conn *)cma_id->context;
+		ib_conn->dev_removed = 1;
+		iser_disconnected_handler(cma_id);
+		wait_event_interruptible(ib_conn->rem_wait, ib_conn->state == ISER_CONN_DOWN);
+		kfree(ib_conn);
+		ret = 1;
 		break;
 	default:
 		iser_err("Unexpected RDMA CM event (%d)\n", event->event);
@@ -497,6 +506,7 @@ int iser_conn_init(struct iser_conn **ibconn)
 	}
 	ib_conn->state = ISER_CONN_INIT;
 	init_waitqueue_head(&ib_conn->wait);
+	init_waitqueue_head(&ib_conn->rem_wait);
 	atomic_set(&ib_conn->post_recv_buf_count, 0);
 	atomic_set(&ib_conn->post_send_buf_count, 0);
 	INIT_LIST_HEAD(&ib_conn->conn_list);
-- 
1.5.3.6

Roland,

This patch was built against your 2.6.26 branch. Can you add it to your list?

Thanks,
Erez