[openib-general] [PATCH] iser: simplified the rdma connection termination scheme
Or Gerlitz
ogerlitz at voltaire.com
Mon Feb 13 07:12:46 PST 2006
simplified the rdma connection termination scheme, consolidated the flows
for SYNC and ASYNC termination into one.
Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>
Index: iscsi_iser.h
===================================================================
--- iscsi_iser.h (revision 5383)
+++ iscsi_iser.h (revision 5384)
@@ -146,12 +146,11 @@ struct iser_hdr {
#define ISER_OBJECT_NAME_SIZE 64
enum iser_ib_conn_state {
- ISER_CONN_INIT, /* descriptor allocd, no conn */
- ISER_CONN_PENDING, /* about to be established */
- ISER_CONN_UP, /* up and running */
- ISER_CONN_SYNC_TERM, /* synchronous termination */
- ISER_CONN_ASYNC_TERM, /* asynchronous termination */
- ISER_CONN_DOWN, /* shut down */
+ ISER_CONN_INIT, /* descriptor allocd, no conn */
+ ISER_CONN_PENDING, /* in the process of being established */
+ ISER_CONN_UP, /* up and running */
+ ISER_CONN_TERMINATING, /* in the process of being terminated */
+ ISER_CONN_DOWN, /* shut down */
ISER_CONN_STATES_NUM
};
@@ -457,11 +456,7 @@ int iser_conn_establish(struct iser_con
struct sockaddr_in *dst_addr,
struct sockaddr_in *src_addr);
-void iser_conn_release(struct iser_conn *p_iser_conn);
-
-void iser_conn_async_terminate(struct iser_conn *p_iser_conn);
-
-int iser_conn_sync_terminate(struct iscsi_iser_conn *p_iser_conn);
+void iser_conn_terminate(struct iser_conn *ib_conn);
void iser_rcv_completion(struct iser_desc *p_desc,
unsigned long dto_xfer_len);
Index: iser_verbs.c
===================================================================
--- iser_verbs.c (revision 5383)
+++ iser_verbs.c (revision 5384)
@@ -50,6 +50,7 @@
static void iser_cq_tasklet_fn(unsigned long data);
static void iser_cq_callback(struct ib_cq *cq, void *cq_context);
static void iser_comp_error_worker(void *data);
+static void iser_conn_release(struct iser_conn *p_iser_conn);
static void iser_cq_event_callback(struct ib_event *cause, void *context)
{
@@ -282,134 +283,21 @@ static void iser_adaptor_try_release(str
up(&ig.adaptor_list_sem);
}
-static char *conn_state_name[ISER_CONN_STATES_NUM + 1] = {
- "INITIAL ",
- "PENDING ",
- "UP ",
- "SYNC_TERM ",
- "ASYNC_TERM ",
- "DOWN ",
-
- "ILLEGAL "
-};
-
/**
- * iser_conn_get_state_name - Retrieves symbolic name of a conn state
- *
- * returns conn name string
+ * iser_conn_terminate - Triggers start of the disconnect procedures and wait
+ * for them to be done
*/
-static char *iser_conn_get_state_name(struct iser_conn *p_iser_conn)
+void iser_conn_terminate(struct iser_conn *ib_conn)
{
- enum iser_ib_conn_state state;
-
- state = atomic_read(&p_iser_conn->state);
- return conn_state_name[state < ISER_CONN_STATES_NUM ?
- state : ISER_CONN_STATES_NUM];
-}
-
-/**
- * iser_conn_sync_terminate - Triggers start of the disconnect procedures
- */
-int iser_conn_sync_terminate(struct iscsi_iser_conn *p_iser_conn)
-{
- struct iser_conn *ib_conn = p_iser_conn->ib_conn;
int err = 0;
- switch (atomic_read(&ib_conn->state)) {
- case ISER_CONN_UP:
- atomic_set(&ib_conn->state, ISER_CONN_SYNC_TERM);
- err = rdma_disconnect(ib_conn->cma_id);
- if (err)
- iser_bug("Failed to disc.gracefully, conn: 0x%p\n",
- p_iser_conn);
- wait_event_interruptible(ib_conn->wait,
- (atomic_read(&ib_conn->state) ==
- ISER_CONN_DOWN));
- break;
-
- /* this state is possible here if async termination races with iscsi *
- * conn stop, in that case the state is aligned to SYNC */
- case ISER_CONN_ASYNC_TERM:
- atomic_set(&ib_conn->state, ISER_CONN_SYNC_TERM);
- wait_event_interruptible(ib_conn->wait,
- (atomic_read(&ib_conn->state) ==
- ISER_CONN_DOWN));
- break;
-
- case ISER_CONN_DOWN:
- /* this may happen only when iSCSI is being notified */
- break;
-
- default:
- iser_err("called when in state %s\n",
- iser_conn_get_state_name(ib_conn));
- err = -EPERM;
- break;
- }
- return err;
-}
-
-/**
-* iser_conn_async_terminate - Triggers start of the disconn procedures
-*/
-void iser_conn_async_terminate(struct iser_conn *p_iser_conn)
-{
- /* if the state is UP it means that the conn is being async terminated *
- * as of the iSCSI layer. We need to initiate a disconnection after *
- * which we will notify the iSCSI layer */
- if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP) {
- iser_err("Conn. 0x%p is being terminated asynchronously\n", p_iser_conn);
- atomic_set(&p_iser_conn->state, ISER_CONN_ASYNC_TERM);
- rdma_disconnect(p_iser_conn->cma_id);
- } else
- iser_err("called when in state %s - doing nothing\n",
- iser_conn_get_state_name(p_iser_conn));
-}
-
-/* iser_complete_conn_termination - Checks if the conn may be terminated *
- * and terminates if possible */
-static void iser_complete_conn_termination(struct iser_conn *p_iser_conn)
-{
- int rcv_buf_count;
- int send_buf_count;
-
- rcv_buf_count = atomic_read(&p_iser_conn->post_recv_buf_count);
- send_buf_count = atomic_read(&p_iser_conn->post_send_buf_count);
-
- /* Check if this conn may be terminated now */
- if (rcv_buf_count == 0 &&
- send_buf_count == 0 && p_iser_conn->disc_evt_flag) {
- unsigned int cur_conn_state;
-
- cur_conn_state = atomic_read(&p_iser_conn->state);
- if (cur_conn_state != ISER_CONN_ASYNC_TERM &&
- cur_conn_state != ISER_CONN_SYNC_TERM) {
- iser_bug("Illegal state (%s)",
- iser_conn_get_state_name(p_iser_conn));
- }
- atomic_set(&p_iser_conn->state, ISER_CONN_DOWN);
-
- iser_dbg("Conn:0x%p, may be terminated:%s\n", p_iser_conn,
- (cur_conn_state == ISER_CONN_ASYNC_TERM ?
- "ASYNC" : "SYNC"));
-
- /* Notify the upper layer about asynch terminations */
- if (cur_conn_state == ISER_CONN_ASYNC_TERM)
- iscsi_iser_conn_failure(p_iser_conn->p_iscsi_conn,
- ISCSI_ERR_CONN_FAILED);
-
- if (cur_conn_state == ISER_CONN_SYNC_TERM)
- wake_up_interruptible(
- &p_iser_conn->wait);
- else
- iser_conn_release(p_iser_conn);
- } else {
- iser_dbg("Conn:0x%p not terminated now, disc_event:%d,"
- " post_recv_cnt:%d, post_send_cnt:%d, state:%s)\n",
- p_iser_conn, p_iser_conn->disc_evt_flag,
- rcv_buf_count, send_buf_count,
- iser_conn_get_state_name(p_iser_conn));
- }
+ atomic_set(&ib_conn->state, ISER_CONN_TERMINATING);
+ err = rdma_disconnect(ib_conn->cma_id);
+ if (err)
+ iser_bug("Failed to disconnect, conn: 0x%p err %d\n",ib_conn,err);
+ wait_event_interruptible(ib_conn->wait,
+ (atomic_read(&ib_conn->state) == ISER_CONN_DOWN));
+ iser_conn_release(ib_conn);
}
static void iser_connect_error(struct rdma_cm_id *cma_id)
@@ -421,8 +309,8 @@ static void iser_connect_error(struct rd
atomic_set(&p_iser_conn->state, ISER_CONN_DOWN);
wake_up_interruptible(&p_iser_conn->wait);
} else
- iser_err("Unexpected evt for conn.state: %s\n",
- iser_conn_get_state_name(p_iser_conn));
+ iser_err("Unexpected evt for conn.state: %d\n",
+ atomic_read(&p_iser_conn->state));
}
static void iser_addr_handler(struct rdma_cm_id *cma_id)
@@ -489,8 +377,6 @@ static void iser_disconnected_handler(st
{
struct iser_conn *p_iser_conn;
- rdma_disconnect(cma_id);
-
p_iser_conn = (struct iser_conn *)cma_id->context;
p_iser_conn->disc_evt_flag = 1;
@@ -501,13 +387,16 @@ static void iser_disconnected_handler(st
wake_up_interruptible(&p_iser_conn->wait);
} else {
if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP) {
- atomic_set(&p_iser_conn->state, ISER_CONN_ASYNC_TERM);
- iser_dbg("Conn 0x%p is being terminated"
- " asynchronously\n", p_iser_conn);
+ atomic_set(&p_iser_conn->state, ISER_CONN_TERMINATING);
+ iscsi_iser_conn_failure(p_iser_conn->p_iscsi_conn,
+ ISCSI_ERR_CONN_FAILED);
+ }
+ /* Complete the termination process if no posts are pending */
+ if ((atomic_read(&p_iser_conn->post_recv_buf_count) == 0) &&
+ (atomic_read(&p_iser_conn->post_send_buf_count) == 0)) {
+ atomic_set(&p_iser_conn->state, ISER_CONN_DOWN);
+ wake_up_interruptible(&p_iser_conn->wait);
}
- /* Complete the termination process if possible */
- /* (no events pending) */
- iser_complete_conn_termination(p_iser_conn);
}
}
@@ -621,7 +510,7 @@ connect_failure:
/**
* Frees all conn objects and deallocs conn descriptor
*/
-void iser_conn_release(struct iser_conn *p_iser_conn)
+static void iser_conn_release(struct iser_conn *p_iser_conn)
{
struct iscsi_iser_conn *p_iscsi_conn;
struct iser_adaptor *p_iser_adaptor = p_iser_conn->p_adaptor;
@@ -829,10 +718,18 @@ static void iser_comp_error_worker(void
if (p_iser_conn == NULL)
iser_bug("NULL p_desc->p_conn \n");
- if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP)
- iser_conn_async_terminate(p_iser_conn);
+ if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP) {
+ atomic_set(&p_iser_conn->state, ISER_CONN_TERMINATING);
+ iscsi_iser_conn_failure(p_iser_conn->p_iscsi_conn,
+ ISCSI_ERR_CONN_FAILED);
+ }
- iser_complete_conn_termination(p_iser_conn);
+ /* complete the termination process if disconnect event was delivered *
+ * note there are no more non completed posts to the QP */
+ if (p_iser_conn->disc_evt_flag) {
+ atomic_set(&p_iser_conn->state, ISER_CONN_DOWN);
+ wake_up_interruptible(&p_iser_conn->wait);
+ }
}
static void iser_handle_comp_error(struct iser_desc *p_desc)
Index: iser_initiator.c
===================================================================
--- iser_initiator.c (revision 5383)
+++ iser_initiator.c (revision 5384)
@@ -463,11 +463,7 @@ int iser_send_command(struct iscsi_iser_
send_command_error:
if (p_send_dto != NULL)
iser_dto_buffs_release(p_send_dto);
- if (p_iser_conn != NULL) {
- /* drop the conn, open tasks are deleted during shutdown */
- iser_err("send cmd failed, drop conn:0x%p\n", p_iser_conn);
- iser_conn_async_terminate(p_iser_conn->ib_conn);
- }
+ iser_err("conn %p failed err %d\n",p_iser_conn, err);
return err;
}
@@ -545,12 +541,7 @@ send_data_out_error:
iser_dto_buffs_release(p_send_dto);
if (tx_desc != NULL)
kmem_cache_free(ig.desc_cache, tx_desc);
-
- if (p_iser_conn != NULL) {
- /* drop the conn, open tasks are deleted during shutdown */
- iser_err("send dout failed, drop conn:0x%p\n", p_iser_conn);
- iser_conn_async_terminate(p_iser_conn->ib_conn);
- }
+ iser_err("conn %p failed err %d\n",p_iser_conn, err);
return err;
}
@@ -613,11 +604,7 @@ int iser_send_control(struct iscsi_iser_
send_control_error:
if (p_send_dto != NULL)
iser_dto_buffs_release(p_send_dto);
- if (p_iser_conn != NULL) {
- /* drop the conn, open tasks are deleted during shutdown */
- iser_err("send ctrl failed, drop conn:0x%p\n", p_iser_conn);
- iser_conn_async_terminate(p_iser_conn->ib_conn);
- }
+ iser_err("conn %p failed err %d\n",p_iser_conn, err);
return err;
}
Index: iscsi_iser.c
===================================================================
--- iscsi_iser.c (revision 5383)
+++ iscsi_iser.c (revision 5384)
@@ -524,6 +524,9 @@ iscsi_iser_mtask_xmit(struct iscsi_iser_
error = iser_send_control(conn, mtask);
+ if (error && error != -EAGAIN)
+ iscsi_iser_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
+
return error;
}
@@ -1181,13 +1184,7 @@ iscsi_iser_conn_stop(iscsi_connh_t connh
/* starts conn teardown process, waits until all previously *
* posted buffers get flushed, deallocates all conn resources */
- if (atomic_read(&conn->ib_conn->state) == ISER_CONN_UP) {
- iser_conn_sync_terminate(conn);
- iser_conn_release(conn->ib_conn);
- }
- else
- iser_err("conn state is %d doing nothing\n",
- atomic_read(&conn->ib_conn->state));
+ iser_conn_terminate(conn->ib_conn);
sock_release(conn->sock);
conn->sock = NULL;
More information about the general
mailing list