[openib-general] [PATCH] iser: always call rdma_disconnect/destroy_qp from sleepable context
Or Gerlitz
ogerlitz at voltaire.com
Wed Feb 1 06:52:28 PST 2006
make sure that rdma_disconnect & rdma_destroy_qp etc are always called
from sleepable context
Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>
Index: ulp/iser/iser_conn.c
===================================================================
--- ulp/iser/iser_conn.c (revision 5235)
+++ ulp/iser/iser_conn.c (revision 5251)
@@ -279,7 +279,7 @@ int iser_conn_sync_terminate(struct iscs
case ISER_CONN_UP:
case ISER_CONN_PENDING:
atomic_set(&p_iser_conn->ib_conn->state, ISER_CONN_SYNC_TERM);
- err = iser_disconnect(p_iser_conn->ib_conn);
+ err = rdma_disconnect(p_iser_conn->ib_conn->cma_id);
if (err)
iser_bug("Failed to disc.gracefully, conn: 0x%p\n",
p_iser_conn);
@@ -309,22 +309,24 @@ int iser_conn_sync_terminate(struct iscs
}
/**
- * iser_conn_async_terminate - Triggers start of the disconn procedures
- */
+* iser_conn_async_terminate - Triggers start of the disconn procedures
+*/
int iser_conn_async_terminate(struct iser_conn *p_iser_conn)
{
- int err = 0;
+ int err;
+ /* if the state is UP it means that the conn is being async terminated *
+ * as of the iSCSI layer. We need to initiate a disconnection after *
+ * which we will notify the iSCSI layer */
if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP) {
+ iser_err("Conn. 0x%p is being terminated asynchronously\n", p_iser_conn);
atomic_set(&p_iser_conn->state, ISER_CONN_ASYNC_TERM);
- iser_dbg("Conn.0x%p is being terminated asynchronously\n",
- p_iser_conn);
- err = iser_disconnect(p_iser_conn);
+ err = rdma_disconnect(p_iser_conn->cma_id);
} else {
- iser_err("called when in state %s\n",
+ iser_err("called when in state %s\n",
iser_conn_get_state_name(p_iser_conn));
- err = -EPERM;
- }
+ err = -EPERM;
+ }
return err;
}
Index: ulp/iser/iscsi_iser.h
===================================================================
--- ulp/iser/iscsi_iser.h (revision 5235)
+++ ulp/iser/iscsi_iser.h (revision 5251)
@@ -265,6 +265,7 @@ struct iscsi_iser_conn
atomic_t post_send_buf_count;
wait_queue_head_t disconnect_wait_q; /* sync conn term uses this */
+ struct work_struct comperror_work; /* sleepable contex for conn term */
int c_stage; /* connection state */
int stop_stage; /* conn_stop() flag: */
@@ -486,6 +487,7 @@ int iser_conn_async_terminate(struct ise
int iser_conn_sync_terminate(struct iscsi_iser_conn *p_iser_conn);
+
int iser_complete_conn_termination(struct iscsi_iser_conn *p_iser_conn);
void iser_adaptor_add_conn(struct iser_adaptor *p_iser_adaptor,
@@ -506,8 +508,6 @@ int iser_dto_add_regd_buff(struct iser_d
void iser_dto_free(struct iser_dto *p_dto);
-int iser_dto_completion_error(struct iser_desc *p_desc);
-
void iser_dto_send_create(struct iscsi_iser_conn *p_iser_conn,
struct iser_desc *tx_desc);
@@ -636,4 +636,5 @@ void iser_unreg_mem(struct iser_mem_reg
int iser_post_recv(struct iser_desc *p_rx_desc);
int iser_start_send(struct iser_desc *p_tx_desc);
+void iser_comp_error_worker(void *data);
#endif
Index: ulp/iser/iser_verbs.c
===================================================================
--- ulp/iser/iser_verbs.c (revision 5235)
+++ ulp/iser/iser_verbs.c (revision 5251)
@@ -318,8 +318,6 @@ static void iser_route_handler(struct rd
conn_param.retry_count = 7;
conn_param.rnr_retry_count = 6;
- /* #warning "the target must prefix iSER SID with OPENIB OUI prefix" */
-
ret = rdma_connect(cma_id, &conn_param);
if (ret) {
iser_err("failure connecting: %d\n", ret);
@@ -443,23 +441,6 @@ iser_connect(struct iser_conn *p_iser_co
}
/**
- * iser_disconnect - disconnects from the target
- *
- * returns 0 on success, -1 on failure
- */
-int iser_disconnect(struct iser_conn *p_iser_conn)
-{
- int ret;
-
- ret = rdma_disconnect(p_iser_conn->cma_id);
- if (ret) {
- iser_err("rdma_disconnet failed: %d\n", ret);
- return -1;
- }
- return 0;
-}
-
-/**
* iser_reg_phys_mem - Register physical memory
*
* returns: 0 on success, -1 on failure
@@ -641,34 +622,43 @@ int iser_start_send(struct iser_desc *p_
return ret_val;
}
-static void iser_handle_comp_error(enum ib_wc_status status,
- struct iser_desc *p_desc)
+void iser_comp_error_worker(void *data)
{
- int ret_val;
- struct iscsi_iser_conn *p_iser_conn = p_desc->dto.p_conn;
-
+ struct iscsi_iser_conn *p_iser_conn = data;
+ int err;
if(p_iser_conn == NULL)
iser_bug("NULL p_desc->p_conn \n");
- /* Since the cma doesn't notify us on CONNECTION_EVENT_BROKEN *
- * we need to initiate a disconn */
- if (status != IB_WC_WR_FLUSH_ERR) {
- iser_dbg("Calling iser_conn_async_terminate\n");
- ret_val = iser_conn_async_terminate(p_iser_conn->ib_conn);
- if (ret_val != 0)
- iser_err("Failed to async term conn:0x%p\n",
- p_iser_conn);
- }
- /* If this event is unsolicited this means that the conn is *
- * being async terminated from the iSCSI layer's perspective. */
- if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP) {
- atomic_set(&p_iser_conn->state, ISER_CONN_ASYNC_TERM);
- iser_dbg("Conn. 0x%p is being terminated asynchronously\n", p_iser_conn);
- }
- /* Handle completion Error */
- ret_val = iser_dto_completion_error(p_desc);
- if (ret_val && ret_val != -EAGAIN)
- iser_err("Failed to handle ERROR DTO completion\n");
+ if (atomic_read(&p_iser_conn->state) == ISER_CONN_UP)
+ err = iser_conn_async_terminate(p_iser_conn->ib_conn);
+
+ err = iser_complete_conn_termination(p_iser_conn);
+ if (err && err != -EAGAIN)
+ iser_err("Failed to handle ERROR completion\n");
+
+ iser_err("iser_complete_conn_termination return %d\n", err);
+}
+
+static void iser_handle_comp_error(struct iser_desc *p_desc)
+{
+ struct iser_dto *p_dto = &p_desc->dto;
+ struct iscsi_iser_conn *p_iser_conn = p_dto->p_conn;
+
+ iser_dto_free(p_dto);
+
+ if(p_desc->type == ISCSI_RX) {
+ kfree(p_desc->data);
+ kmem_cache_free(ig.desc_cache, p_desc);
+ atomic_dec(&p_iser_conn->post_recv_buf_count);
+ } else { /* type is TX control/command/dataout */
+ if(p_desc->type == ISCSI_TX_DATAOUT)
+ kmem_cache_free(ig.desc_cache, p_desc);
+ atomic_dec(&p_iser_conn->post_send_buf_count);
+ }
+
+ if( atomic_read(&p_iser_conn->post_recv_buf_count) == 0 &&
+ atomic_read(&p_iser_conn->post_send_buf_count) == 0)
+ schedule_work(&p_iser_conn->comperror_work);
}
void iser_cq_tasklet_fn(unsigned long data)
@@ -691,9 +681,10 @@ void iser_cq_tasklet_fn(unsigned long da
iser_rcv_completion(p_desc, xfer_len);
} else /* type == ISCSI_TX_CONTROL/SCSI_CMD/DOUT */
iser_snd_completion(p_desc);
- } else
- /* #warning "we better do a context jump here" */
- iser_handle_comp_error(wc.status, p_desc);
+ } else {
+ iser_err("comp w. error op %d status %d\n",p_desc->type,wc.status);
+ iser_handle_comp_error(p_desc);
+ }
}
/* #warning "it is assumed here that arming CQ only once its empty would not"
* "cause interrupts to be missed" */
Index: ulp/iser/iser_dto.c
===================================================================
--- ulp/iser/iser_dto.c (revision 5235)
+++ ulp/iser/iser_dto.c (revision 5251)
@@ -87,40 +87,6 @@ void iser_dto_free(struct iser_dto *p_dt
iser_dto_buffs_release(p_dto);
}
-/**
- * iser_dto_completion_error - Handles error in DTO completion
- *
- * returns 0 on success, -1 on failure
- */
-int iser_dto_completion_error(struct iser_desc *p_desc)
-{
- struct iscsi_iser_conn *p_iser_conn;
- int err;
- struct iser_dto *p_dto = &p_desc->dto;
-
- p_iser_conn = p_dto->p_conn;
- if (p_iser_conn == NULL)
- iser_bug("NULL conn in p_dto:0x%p\n", p_dto);
-
- iser_dto_free(p_dto);
-
- if(p_desc->type == ISCSI_RX) {
- kfree(p_desc->data);
- kmem_cache_free(ig.desc_cache, p_desc);
- atomic_dec(&p_iser_conn->post_recv_buf_count);
- }
- else { /* type is TX control/command/dataout */
- if(p_desc->type == ISCSI_TX_DATAOUT)
- kmem_cache_free(ig.desc_cache, p_desc);
- atomic_dec(&p_iser_conn->post_send_buf_count);
- }
-
- err = iser_complete_conn_termination(p_iser_conn);
-
- return err;
-}
-
-/* iser_dto_get_rx_pdu_data - gets received PDU descriptor & data from rx DTO */
/**
* Creates a new send DTO descriptor,
Index: ulp/iser/iscsi_iser.c
===================================================================
--- ulp/iser/iscsi_iser.c (revision 5235)
+++ ulp/iser/iscsi_iser.c (revision 5251)
@@ -1086,8 +1086,8 @@ static iscsi_connh_t iscsi_iser_conn_cre
atomic_set(&conn->post_recv_buf_count, 0);
atomic_set(&conn->post_send_buf_count, 0);
init_waitqueue_head(&conn->disconnect_wait_q);
-
spin_lock_init(&conn->lock);
+ INIT_WORK(&conn->comperror_work, iser_comp_error_worker, conn);
return iscsi_handle(conn);
More information about the general
mailing list