[openib-general] [PATCH] iser: added -EAGAIN scheme to the tx posting and completion code
Or Gerlitz
ogerlitz at voltaire.com
Mon Jan 30 23:44:35 PST 2006
commited in r5228
added -EAGAIN scheme to the tx posting and completion code. When the QP is
full and we can't post to it, TX is suspended till there's room.
Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>
Index: ulp/iser/iser_conn.c
===================================================================
--- ulp/iser/iser_conn.c (revision 5182)
+++ ulp/iser/iser_conn.c (revision 5228)
@@ -204,7 +204,7 @@ int iser_conn_set_full_featured_mode(str
int i, err = 0;
/* no need to keep it in a var, we are after login so if this should
* be negotiated, by now the result should be available here */
- int initial_post_recv_bufs_num = ISER_INITIAL_POST_RECV + 2;
+ int initial_post_recv_bufs_num = ISER_MAX_RX_MISC_PDUS;
iser_dbg("Initially post: %d\n", initial_post_recv_bufs_num);
Index: ulp/iser/iscsi_iser.h
===================================================================
--- ulp/iser/iscsi_iser.h (revision 5182)
+++ ulp/iser/iscsi_iser.h (revision 5228)
@@ -303,6 +303,7 @@ struct iscsi_iser_conn
struct list_head item; /* maintains list of conns */
unsigned long suspend_tx;
+ spinlock_t lock;
char name[ISER_OBJECT_NAME_SIZE];
};
@@ -328,13 +329,11 @@ struct iscsi_iser_cmd_task {
struct iscsi_iser_conn *conn;
spinlock_t task_lock;
enum iser_task_status status;
+ int command_sent;
int datasn; /* DataSN */
uint32_t unsol_datasn;
int sent;
- struct scatterlist *sg; /* per-cmd SG list */
- struct scatterlist *bad_sg; /* assert statement */
- int sg_count; /* SG's to process */
int imm_count; /* imm-data (bytes) */
int unsol_count; /* unsolicited (bytes)*/
@@ -344,7 +343,6 @@ struct iscsi_iser_cmd_task {
struct scsi_cmnd *sc; /* associated SCSI cmd*/
int total_length;
- int data_offset;
struct iscsi_iser_mgmt_task *mtask; /* tmf mtask in progr */
unsigned int post_send_count; /* posted send buffers pending completion */
@@ -592,58 +590,30 @@ void iser_task_set_status(struct iscsi_i
/* ---------------------------------------------------------------------
* CONSTANTS & MACROS
* ------------------------------------------------------------------ */
-#define ISER_MAX_NOP_IN 2
-#define ISER_MAX_ASYNC_EVT 2
+/* iSER Initiator QP settings */
+/* Maximal bounds on asynchronous PDUs received by iSER Initiator */
-#define ISER_MAX_LOGIN_REQ 1
-#define ISER_MAX_TEXT_REQ 1
-#define ISER_MAX_NOP_OUT 2
-#define ISER_MAX_TASK_MGT_REQ 2
-#define ISER_MAX_LOGOUT_REQ 1
-
-#define ISER_MAX_IMMEDIATE_CMDS 2
-
-#define ISER_MIN_RECV_DSL (8*1024) /* 8K */
-#define ISER_MAX_FIRST_BURST (128*1024) /* 128K */
-
-#define ISER_MAX_CTRLS_PER_CMD(first_burst,recv_dsl,imm) \
- ((first_burst / recv_dsl) + \
- (first_burst % recv_dsl > 0 ? 1 : 0) + \
- (imm ? 0 : 1))
-
- /* Maximal bounds on asynchronous PDUs received by iSER Initiator */
-#define ISER_MAX_RX_MISC_PDUS (ISER_MAX_NOP_IN + \
- ISER_MAX_ASYNC_EVT)
-
-#define ISER_MAX_TX_MISC_PDUS (ISER_MAX_TEXT_REQ + \
- ISER_MAX_NOP_OUT + \
- ISER_MAX_TASK_MGT_REQ + \
- ISER_MAX_LOGOUT_REQ)
+#define ISER_MAX_RX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */
-#define ISER_MAX_RX_CMD_RESP ISCSI_ISER_XMIT_CMDS_MAX
+#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), *
+ * SCSI_TMFUNC(2), LOGOUT(1) */
+#define ISER_QP_MAX_RECV_DTOS (ISCSI_ISER_XMIT_CMDS_MAX + \
+ ISER_MAX_RX_MISC_PDUS + \
+ ISER_MAX_TX_MISC_PDUS)
-/* iSER Initiator QP settings */
-#define ISER_AVG_TASK_RELATED_SEND(first_burst, recv_dsl,imm,max_cmds) \
- (max_cmds * (ISER_MAX_CTRLS_PER_CMD(first_burst,recv_dsl,imm) + \
- ISER_MAX_IMMEDIATE_CMDS))
-
-#define ISER_INITIAL_POST_RECV ISER_MAX_RX_MISC_PDUS
-
-#define ISER_QP_AVG_POST_RECV (ISER_MAX_RX_CMD_RESP + \
- ISER_MAX_RX_MISC_PDUS + \
- ISER_MAX_TX_MISC_PDUS)
-
-#define ISER_QP_MAX_RECV_DTOS (ISER_QP_AVG_POST_RECV + 8)
-
-#define ISER_QP_MAX_REQ_DTOS \
- (ISER_AVG_TASK_RELATED_SEND( \
- ISER_MAX_FIRST_BURST, \
- ISER_MIN_RECV_DSL, \
- 1, \
- ISCSI_ISER_XMIT_CMDS_MAX) + \
- ISER_MAX_TX_MISC_PDUS + \
- ISER_MAX_RX_MISC_PDUS)
+/* the max TX (send) WR supported by the iSER QP is defined by *
+ * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
+ * to have at max for SCSI command. The tx posting & completion handling code *
+ * supports -EAGAIN scheme where tx is suspended till the QP has room for more *
+ * send WR. D=8 comes from 64K/8K */
+
+#define ISER_INFLIGHT_DATAOUTS 8
+
+#define ISER_QP_MAX_REQ_DTOS (ISCSI_ISER_XMIT_CMDS_MAX * \
+ (1 + ISER_INFLIGHT_DATAOUTS) + \
+ ISER_MAX_TX_MISC_PDUS + \
+ ISER_MAX_RX_MISC_PDUS)
/* iSER Initiator CQ settings */
#define ISCSI_ISER_MAX_CONN 8
Index: ulp/iser/iser_initiator.c
===================================================================
--- ulp/iser/iser_initiator.c (revision 5182)
+++ ulp/iser/iser_initiator.c (revision 5228)
@@ -266,6 +266,22 @@ iser_prepare_write_cmd(struct iscsi_iser
return 0;
}
+static int
+iser_check_xmit(struct iscsi_iser_conn *conn, void *task)
+{
+ int rc = 0;
+
+ spin_lock_bh(&conn->lock);
+ if(atomic_read(&conn->post_send_buf_count) == ISER_QP_MAX_REQ_DTOS) {
+ iser_dbg("%ld can't xmit task %p, suspending tx\n",jiffies,task);
+ set_bit(SUSPEND_BIT, &conn->suspend_tx);
+ rc = -EAGAIN;
+ }
+ spin_unlock_bh(&conn->lock);
+ return rc;
+}
+
+
/**
* iser_send_command - send command PDU
*/
@@ -284,6 +300,8 @@ int iser_send_command(struct iscsi_iser_
iser_err("Failed to send, conn: 0x%p is not up\n", p_iser_conn->ib_conn);
return -EPERM;
}
+ if(iser_check_xmit(p_iser_conn, p_ctask))
+ return -EAGAIN;
edtl = ntohl(hdr->data_length);
@@ -372,6 +390,9 @@ int iser_send_data_out(struct iscsi_iser
return -EPERM;
}
+ if(iser_check_xmit(p_iser_conn, p_ctask))
+ return -EAGAIN;
+
itt = ntohl(hdr->itt);
data_seg_len = ntoh24(hdr->dlength);
buf_offset = ntohl(hdr->offset);
@@ -455,6 +476,9 @@ int iser_send_control(struct iscsi_iser_
return -EPERM;
}
+ if(iser_check_xmit(p_iser_conn,p_mtask))
+ return -EAGAIN;
+
/* build the tx desc regd header and add it to the tx desc dto */
p_mtask->desc.type = ISCSI_TX_CONTROL;
p_send_dto = &p_mtask->desc.dto;
@@ -627,6 +651,14 @@ void iser_snd_completion(struct iser_des
atomic_dec(&p_iser_conn->post_send_buf_count);
+ spin_lock(&p_iser_conn->lock);
+ if(p_iser_conn->suspend_tx) {
+ iser_dbg("%ld resuming tx\n",jiffies);
+ clear_bit(SUSPEND_BIT, &p_iser_conn->suspend_tx);
+ schedule_work(&p_iser_conn->xmitwork);
+ }
+ spin_unlock(&p_iser_conn->lock);
+
/* if the last sent PDU of the task, task can be freed */
if (p_dto->p_task != NULL &&
iser_task_post_send_count_dec_and_test(p_dto->p_task))
Index: ulp/iser/iscsi_iser.c
===================================================================
--- ulp/iser/iscsi_iser.c (revision 5182)
+++ ulp/iser/iscsi_iser.c (revision 5228)
@@ -104,8 +104,7 @@ static void iscsi_iser_cmd_init(struct i
MAX_COMMAND_SIZE - sc->cmd_len);
ctask->mtask = NULL;
- ctask->sent = 0;
- ctask->sg_count = 0;
+ ctask->command_sent = 0;
ctask->total_length = sc->request_bufflen;
@@ -168,9 +167,6 @@ static void iscsi_iser_cmd_init(struct i
* call it again later, or recover. '0' return code means successful
* xmit.
*
- * Management xmit state machine consists of two states:
- * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
- * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
**/
static int iscsi_iser_mtask_xmit(struct iscsi_iser_conn *conn,
struct iscsi_iser_mgmt_task *mtask)
@@ -179,12 +175,8 @@ static int iscsi_iser_mtask_xmit(struct
debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id, mtask->itt);
- /* Send the control */
error = iser_send_control(conn, mtask);
- if (error)
- printk(KERN_ERR "send_control failed\n");
-
return error;
}
@@ -253,10 +245,9 @@ static int iscsi_iser_ctask_xmit_unsol_d
/* Send the command */
error = iser_send_data_out(conn, ctask, &hdr);
if (error) {
- printk(KERN_ERR "send_data_out failed\n");
+ ctask->unsol_datasn--;
goto iscsi_iser_ctask_xmit_unsol_data_exit;
}
-
ctask->unsol_count -= ctask->data_count;
debug_scsi("Need to send %d more as data-out PDUs\n",
ctask->unsol_count);
@@ -281,22 +272,19 @@ static int iscsi_iser_ctask_xmit(struct
return error;
/* Send the cmd PDU */
- error = iser_send_command(conn, ctask);
- if (error) {
- printk(KERN_ERR "Couldn't send a cmd PDU\n");
- goto iscsi_iser_ctask_xmit_exit;
+ if(!ctask->command_sent) {
+ error = iser_send_command(conn, ctask);
+ if (error)
+ goto iscsi_iser_ctask_xmit_exit;
+ ctask->command_sent = 1;
}
/* Send unsolicited data-out PDU(s) if necessary */
- if (ctask->unsol_count) {
+ if (ctask->unsol_count)
error = iscsi_iser_ctask_xmit_unsol_data(conn, ctask);
- if (error)
- printk(KERN_ERR "Couldn't send unsolicited "
- "data-out PDU(s)\n");
- }
iscsi_iser_ctask_xmit_exit:
- if(error)
+ if(error && error != -EAGAIN)
iscsi_iser_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
return error;
}
@@ -1099,6 +1087,8 @@ static iscsi_connh_t iscsi_iser_conn_cre
atomic_set(&conn->post_send_buf_count, 0);
init_waitqueue_head(&conn->disconnect_wait_q);
+ spin_lock_init(&conn->lock);
+
return iscsi_handle(conn);
login_mtask_alloc_fail:
More information about the general
mailing list