[openib-general] [PATCH] iser: align fmr pool params to scsi midlayer host template
Or Gerlitz
ogerlitz at voltaire.com
Sun Jan 15 04:14:07 PST 2006
was commited to r5002
Correlate the fmr pool params with the template posted to the scsi
midlayer and have fmr pool per ib connection.
Signed-off-by: Or Gerlitz <ogerlitz.voltaire.com>
Index: ulp/iser/iscsi_iser.h
===================================================================
--- ulp/iser/iscsi_iser.h (revision 4984)
+++ ulp/iser/iscsi_iser.h (working copy)
@@ -30,8 +30,9 @@
#define ISCSI_ISER_XMIT_CMDS_MAX 128 /* must be power of 2 */
#define ISCSI_ISER_MGMT_CMDS_MAX 32 /* must be power of 2 */
-#define ISER_SG_TABLESIZE SG_ALL
-#define ISER_CMD_PER_LUN 128
+ /* support upto 512KB in one RDMA */
+#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> PAGE_SHIFT)
+#define ISCSI_ISER_CMD_PER_LUN ISCSI_ISER_XMIT_CMDS_MAX
#define ISCSI_ISER_MAX_LUN 256
#define ISCSI_ISER_MAX_CMD_LEN 16
@@ -148,6 +149,7 @@ struct iser_conn
atomic_t state; /* rdma connection state */
struct rdma_cm_id *cma_id;
struct ib_qp *qp;
+ struct ib_fmr_pool *fmr_pool;
struct iser_adaptor *p_adaptor; /* adaptor context */
struct list_head adaptor_list; /* entry in the adaptor's conns list */
@@ -198,10 +200,8 @@ struct iscsi_iser_conn
int id; /* iSCSI CID */
spinlock_t lock; /* MERGE_FIXME: can it be removed */
- int max_xmit_dlength; /* FIXME change it to be target_max_recv_dsl */
- int initiator_max_recv_dsl;
- int target_max_recv_dsl;
-
+ int max_recv_dlength; /* == initiator_max_recv_dsl */
+ int max_xmit_dlength; /* == target_max_recv_dsl */
unsigned int max_outstand_cmds; /* MERGE_FIXME need2 review */
/* abort */
@@ -273,7 +273,7 @@ struct iscsi_iser_data_task {
struct iscsi_data hdr; /* PDU */
struct list_head item; /* data queue item */
};
-#define ISCSI_DTASK_DEFAULT_MAX ISER_SG_TABLESIZE * PAGE_SIZE / 512
+#define ISCSI_DTASK_DEFAULT_MAX ISCSI_ISER_SG_TABLESIZE * PAGE_SIZE / 512
struct iscsi_iser_session
{
Index: ulp/iser/iser_verbs.c
===================================================================
--- ulp/iser/iser_verbs.c (revision 4985)
+++ ulp/iser/iser_verbs.c (working copy)
@@ -71,7 +71,6 @@ static void iser_qp_event_callback(struc
int iser_create_adaptor_ib_res(struct iser_adaptor *p_iser_adaptor)
{
struct ib_device *device = p_iser_adaptor->device;
- struct ib_fmr_pool_param params;
strcpy(p_iser_adaptor->name, device->name);
iser_dbg("setting device name %s as adatptor name\n", device->name);
@@ -100,25 +99,8 @@ int iser_create_adaptor_ib_res(struct is
if (IS_ERR(p_iser_adaptor->mr))
goto dma_mr_err;
- params.max_pages_per_fmr = ISER_MAX_CMD_SIZE >> PAGE_SHIFT;
- params.pool_size = ISER_MAX_OUTSTAND_CMDS * ISER_MAX_CONN;
- params.dirty_watermark = 32;
- params.cache = 0;
- params.flush_function = NULL;
- params.access = (IB_ACCESS_LOCAL_WRITE |
- IB_ACCESS_REMOTE_WRITE |
- IB_ACCESS_REMOTE_READ);
-
- p_iser_adaptor->fmr_pool = ib_create_fmr_pool(p_iser_adaptor->pd, ¶ms);
- if (IS_ERR(p_iser_adaptor->fmr_pool)) {
- iser_err("failed to create FMR pool\n");
- goto fmr_pool_err;
- }
-
return 0;
-fmr_pool_err:
- ib_dereg_mr(p_iser_adaptor->mr);
dma_mr_err:
tasklet_kill(&p_iser_adaptor->cq_tasklet);
cq_arm_err:
@@ -138,41 +120,51 @@ pd_err:
*/
int iser_free_adaptor_ib_res(struct iser_adaptor *p_iser_adaptor)
{
- /* do we need to deallocate any resource ? */
- if (p_iser_adaptor->fmr_pool == NULL)
- return 0;
+ BUG_ON(p_iser_adaptor->mr == NULL);
tasklet_kill(&p_iser_adaptor->cq_tasklet);
- (void)ib_destroy_fmr_pool(p_iser_adaptor->fmr_pool);
(void)ib_dereg_mr(p_iser_adaptor->mr);
(void)ib_destroy_cq(p_iser_adaptor->cq);
(void)ib_dealloc_pd(p_iser_adaptor->pd);
- p_iser_adaptor->fmr_pool = NULL;
- p_iser_adaptor->mr = NULL;
- p_iser_adaptor->cq = NULL;
- p_iser_adaptor->pd = NULL;
+ p_iser_adaptor->mr = NULL;
+ p_iser_adaptor->cq = NULL;
+ p_iser_adaptor->pd = NULL;
return 0;
}
/**
- * iser_create_qp - Creates a Queue-Pair (QP)
+ * iser_create_ib_conn_res - Creates FMR pool and Queue-Pair (QP)
*
* returns 0 on success, -1 on failure
*/
-int iser_create_qp(struct iser_conn *p_iser_conn)
+int iser_create_ib_conn_res(struct iser_conn *p_iser_conn)
{
struct iser_adaptor *p_iser_adaptor;
struct ib_qp_init_attr init_attr;
int ret;
+ struct ib_fmr_pool_param params;
+
+ BUG_ON(p_iser_conn->p_adaptor == NULL);
- if (p_iser_conn->p_adaptor == NULL) {
- iser_err("NULL adaptor in conn, p_conn: 0x%p\n",
- p_iser_conn);
- return -1;
- }
p_iser_adaptor = p_iser_conn->p_adaptor;
+
+ params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE;
+ params.pool_size = ISCSI_ISER_XMIT_CMDS_MAX;
+ params.dirty_watermark = 32;
+ params.cache = 0;
+ params.flush_function = NULL;
+ params.access = (IB_ACCESS_LOCAL_WRITE |
+ IB_ACCESS_REMOTE_WRITE |
+ IB_ACCESS_REMOTE_READ);
+
+ p_iser_conn->fmr_pool = ib_create_fmr_pool(p_iser_adaptor->pd, ¶ms);
+ if (IS_ERR(p_iser_conn->fmr_pool)) {
+ ret = PTR_ERR(p_iser_conn->fmr_pool);
+ goto fmr_pool_err;
+ }
+
memset(&init_attr, 0, sizeof init_attr);
init_attr.event_handler = iser_qp_event_callback;
@@ -187,35 +179,42 @@ int iser_create_qp(struct iser_conn *p_i
init_attr.qp_type = IB_QPT_RC;
ret = rdma_create_qp(p_iser_conn->cma_id, p_iser_adaptor->pd, &init_attr);
- if (ret) {
- iser_err("unable to create qp: %d\n", ret);
- return -1;
- }
+ if (ret)
+ goto qp_err;
+
p_iser_conn->qp = p_iser_conn->cma_id->qp;
- iser_err("setting conn %p qp cma_id %p qp %p\n",
- p_iser_conn,p_iser_conn->cma_id, p_iser_conn->cma_id->qp);
+ iser_err("setting conn %p cma_id %p: fmr_pool %p qp %p\n",
+ p_iser_conn, p_iser_conn->cma_id,
+ p_iser_conn->fmr_pool, p_iser_conn->cma_id->qp);
+ return ret;
+
+qp_err:
+ (void)ib_destroy_fmr_pool(p_iser_conn->fmr_pool);
+fmr_pool_err:
+ iser_err("unable to create fmr pool or qp for ib_conn: %d\n", ret);
return ret;
}
/**
- * iser_free_qp_and_id - Releases the QP and CMA ID objects
+ * iser_free_ib_conn_res - Releases the FMR pool, QP and CMA ID objects
*
* Also starts the conn termination. May be called more than
* once, actually initiates releasing QP/ID only for the first call.
*
* returns 0 on success, -1 on failure
*/
-int iser_free_qp_and_id(struct iser_conn *p_iser_conn)
+int iser_free_ib_conn_res(struct iser_conn *p_iser_conn)
{
- if (p_iser_conn == NULL) {
- iser_err("NULL conn to free\n");
- return -1;
- }
+ BUG_ON(p_iser_conn == NULL);
- iser_err("free-ing conn %p conn->qp %p conn->cma_id %p\n",
- p_iser_conn,p_iser_conn->qp, p_iser_conn->cma_id);
+ iser_err("free-ing conn %p cma_id %p fmr pool %p qp %p\n",
+ p_iser_conn, p_iser_conn->cma_id,
+ p_iser_conn->fmr_pool, p_iser_conn->qp);
/* qp is created only once both addr & route are resolved */
+ if (p_iser_conn->fmr_pool != NULL)
+ ib_destroy_fmr_pool(p_iser_conn->fmr_pool);
+
if (p_iser_conn->qp != NULL)
rdma_destroy_qp(p_iser_conn->cma_id);
@@ -224,6 +223,7 @@ int iser_free_qp_and_id(struct iser_conn
else
iser_bug("we are not supposed to be called twice\n");
+ p_iser_conn->fmr_pool = NULL;
p_iser_conn->qp = NULL;
p_iser_conn->cma_id = NULL;
@@ -307,7 +307,7 @@ static void iser_route_handler(struct rd
struct rdma_conn_param conn_param;
int ret;
- ret = iser_create_qp((struct iser_conn *)cma_id->context);
+ ret = iser_create_ib_conn_res((struct iser_conn *)cma_id->context);
if (ret)
goto failure;
@@ -471,7 +471,7 @@ int iser_disconnect(struct iser_conn *p_
* returns: 0 on success, -1 on failure
*/
int
-iser_reg_phys_mem(struct iser_adaptor *p_iser_adaptor,
+iser_reg_phys_mem(struct iser_conn *p_iser_conn,
struct iser_page_vec *page_vec,
enum ib_access_flags access_flags,
struct iser_mem_reg *mem_reg)
@@ -484,7 +484,7 @@ iser_reg_phys_mem(struct iser_adaptor *p
page_list = page_vec->pages;
io_addr = page_list[0];
- mem = ib_fmr_pool_map_phys(p_iser_adaptor->fmr_pool,
+ mem = ib_fmr_pool_map_phys(p_iser_conn->fmr_pool,
page_list,
page_vec->length,
&io_addr);
Index: ulp/iser/iser.h
===================================================================
--- ulp/iser/iser.h (revision 4984)
+++ ulp/iser/iser.h (working copy)
@@ -99,8 +99,6 @@ struct iser_regd_buf {
struct iser_adaptor;
-#define ISER_MAX_CONN 4
-
struct iser_adaptor {
struct list_head ig_list; /* entry in ig adaptors list */
@@ -109,7 +107,6 @@ struct iser_adaptor {
struct ib_pd *pd;
struct ib_cq *cq;
struct ib_mr *mr;
- struct ib_fmr_pool *fmr_pool;
struct tasklet_struct cq_tasklet;
Index: ulp/iser/iser_verbs.h
===================================================================
--- ulp/iser/iser_verbs.h (revision 4984)
+++ ulp/iser/iser_verbs.h (working copy)
@@ -48,11 +48,8 @@
#define ISER_MAX_TASK_MGT_REQ 2
#define ISER_MAX_LOGOUT_REQ 1
-#define ISER_MAX_OUTSTAND_CMDS 64
#define ISER_MAX_IMMEDIATE_CMDS 2
-#define ISER_MAX_CMD_SIZE 0x80000 /* 512KB */
-
#define ISER_MIN_RECV_DSL (8*1024) /* 8K */
#define ISER_MAX_FIRST_BURST (128*1024) /* 128K */
@@ -64,13 +61,14 @@
/* Maximal bounds on asynchronous PDUs received by iSER Initiator */
#define ISER_MAX_RX_MISC_PDUS (ISER_MAX_NOP_IN + \
ISER_MAX_ASYNC_EVT)
+
#define ISER_MAX_TX_MISC_PDUS (ISER_MAX_TEXT_REQ + \
ISER_MAX_NOP_OUT + \
ISER_MAX_TASK_MGT_REQ + \
ISER_MAX_LOGOUT_REQ)
-#define ISER_MAX_RX_CMD_RESP ISER_MAX_OUTSTAND_CMDS
-#define ISER_MAX_TX_CMDS (ISER_MAX_OUTSTAND_CMDS + \
- ISER_MAX_IMMEDIATE_CMDS)
+
+#define ISER_MAX_RX_CMD_RESP ISCSI_ISER_XMIT_CMDS_MAX
+
/* iSER Initiator QP settings */
#define ISER_AVG_TASK_RELATED_SEND(first_burst, recv_dsl,imm,max_cmds) \
@@ -90,19 +88,15 @@
ISER_MAX_FIRST_BURST, \
ISER_MIN_RECV_DSL, \
1, \
- ISER_MAX_OUTSTAND_CMDS) + \
+ ISCSI_ISER_XMIT_CMDS_MAX) + \
ISER_MAX_TX_MISC_PDUS + \
ISER_MAX_RX_MISC_PDUS)
/* iSER Initiator CQ settings */
-#define ISER_CQ_MAX_RECV_DTOS (ISER_QP_MAX_RECV_DTOS * \
- ISER_MAX_CONN)
-#define ISER_CQ_MAX_REQ_DTOS (ISER_QP_MAX_REQ_DTOS * \
- ISER_MAX_CONN)
-#define ISER_MAX_CQ_LEN (ISER_CQ_MAX_RECV_DTOS +\
- ISER_CQ_MAX_REQ_DTOS)
+#define ISCSI_ISER_MAX_CONN 8
-#define ISER_MAX_TOTAL_QLEN ISER_MAX_QLEN
+#define ISER_MAX_CQ_LEN ((ISER_QP_MAX_RECV_DTOS + ISER_QP_MAX_REQ_DTOS) *\
+ ISCSI_ISER_MAX_CONN)
int iser_create_adaptor_ib_res(struct iser_adaptor *p_iser_adaptor);
@@ -117,7 +111,9 @@ int iser_disconnect(struct iser_conn *p_
int iser_free_qp_and_id(struct iser_conn *p_iser_conn);
-int iser_reg_phys_mem(struct iser_adaptor *p_iser_adaptor,
+int iser_free_ib_conn_res(struct iser_conn *p_iser_conn);
+
+int iser_reg_phys_mem(struct iser_conn *p_iser_conn,
struct iser_page_vec *page_vec,
enum ib_access_flags access_flags,
struct iser_mem_reg *p_mem_reg);
Index: ulp/iser/iser_conn.c
===================================================================
--- ulp/iser/iser_conn.c (revision 4985)
+++ ulp/iser/iser_conn.c (working copy)
@@ -373,7 +373,7 @@ void iser_conn_release(struct iser_conn
struct iscsi_iser_conn *p_iscsi_conn;
if (atomic_read(&p_iser_conn->state) == ISER_CONN_DOWN) {
- iser_free_qp_and_id(p_iser_conn); /* qp/id freed only once */
+ iser_free_ib_conn_res(p_iser_conn); /* qp/id freed only once */
iser_adaptor_remove_conn(p_iser_conn);
p_iscsi_conn = p_iser_conn->p_iscsi_conn;
Index: ulp/iser/iser_initiator.c
===================================================================
--- ulp/iser/iser_initiator.c (revision 4984)
+++ ulp/iser/iser_initiator.c (working copy)
@@ -57,7 +57,7 @@ static void iser_dma_unmap_task_data(str
static int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *p_iser_task,
enum iser_data_dir cmd_dir)
{
- struct iser_adaptor *p_iser_adaptor = p_iser_task->conn->ib_conn->p_adaptor;
+ struct iser_conn *p_iser_conn = p_iser_task->conn->ib_conn;
struct iser_page_vec *page_vec = NULL;
struct iser_regd_buf *p_regd_buf = NULL;
enum ib_access_flags priv_flags = IB_ACCESS_LOCAL_WRITE;
@@ -106,7 +106,7 @@ static int iser_reg_rdma_mem(struct iscs
return -ENOMEM;
}
page_vec_len = iser_page_vec_build(p_mem, page_vec, 0, cnt_to_reg);
- err = iser_reg_phys_mem(p_iser_adaptor, page_vec, priv_flags,
+ err = iser_reg_phys_mem(p_iser_conn, page_vec, priv_flags,
&p_regd_buf->reg);
iser_page_vec_free(page_vec);
if (err) {
Index: ulp/iser/iscsi_iser.c
===================================================================
--- ulp/iser/iscsi_iser.c (revision 4984)
+++ ulp/iser/iscsi_iser.c (working copy)
@@ -892,10 +892,10 @@ static struct scsi_host_template iscsi_i
ISCSI_VERSION_STR,
.queuecommand = iscsi_iser_queuecommand,
.can_queue = ISCSI_ISER_XMIT_CMDS_MAX - 1,
- .sg_tablesize = ISER_SG_TABLESIZE,
- .cmd_per_lun = ISER_CMD_PER_LUN,
+ .sg_tablesize = ISCSI_ISER_SG_TABLESIZE,
+ .cmd_per_lun = ISCSI_ISER_CMD_PER_LUN,
.eh_abort_handler = iscsi_iser_eh_abort,
- .eh_host_reset_handler = iscsi_iser_eh_host_reset,
+ .eh_host_reset_handler = iscsi_iser_eh_host_reset,
.use_clustering = DISABLE_CLUSTERING,
.proc_name = "iscsi_iser",
.this_id = -1,
@@ -1318,6 +1318,9 @@ static int iscsi_iser_conn_set_param(isc
spin_unlock_bh(&session->lock);
switch (param) {
+ case ISCSI_PARAM_MAX_RECV_DLENGTH:
+ /* TBD */
+ break;
case ISCSI_PARAM_MAX_XMIT_DLENGTH:
conn->max_xmit_dlength = value;
break;
More information about the general
mailing list