[openib-general] [PATCH 1/6] [RFC] iscsi_iser header file
Or Gerlitz
ogerlitz at voltaire.com
Wed Feb 22 06:25:49 PST 2006
+ some of the defines here replicate thos in drivers/scsi/iscsi_tcp.h so
merging them is possible
+ a cleanup we plan is to reduce the usage of the iser dbg/err/bug macros,
convert the remaining iser_bug calls into standard BUG() calls.
+ the wire structures are iser_hdr (below) and variuos iscsi_hdr
(from scsi/iscsi_proto.h)
--- /ulp/iser-x/iscsi_iser.h 2006-02-22 15:06:45.000000000 +0200
+++ /ulp/iser/iscsi_iser.h 2006-02-22 13:48:55.000000000 +0200
@@ -1 +1,501 @@
+/*
+ * iSER transport for the Open iSCSI Initiator & iSER transport internals
+ *
+ * Copyright (C) 2004 Dmitry Yusupov
+ * Copyright (C) 2004 Alex Aizman
+ * Copyright (C) 2005 Mike Christie
+ * based on code maintained by open-iscsi at googlegroups.com
+ *
+ * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: iscsi_iser.h 5459 2006-02-22 11:00:48Z ogerlitz $
+ */
+#ifndef __ISCSI_ISER_H__
+#define __ISCSI_ISER_H__
+#include <linux/types.h>
+#include <linux/net.h>
+#include <scsi/iscsi_proto.h>
+#include <scsi/iscsi_if.h>
+
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+
+/* XXX remove this compatibility hack when 2.6.16 is released */
+#include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,16)
+#include <linux/mutex-backport.h>
+#else
+#include <linux/mutex.h>
+#endif /* XXX end of hack */
+
+#include <linux/mempool.h>
+#include <linux/uio.h>
+
+#include <linux/socket.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_fmr_pool.h>
+#include <rdma/rdma_cm.h>
+
+#define PFX "iser:"
+
+#define iser_dbg(fmt, arg...) \
+ do { \
+ if (iser_debug_level > 0) \
+ printk(KERN_DEBUG PFX "%s:" fmt,\
+ __func__ , ## arg); \
+ } while (0)
+
+#define iser_err(fmt, arg...) \
+ do { \
+ printk(KERN_ERR PFX "%s:" fmt, \
+ __func__ , ## arg); \
+ } while (0)
+
+#define iser_bug(fmt,arg...) \
+ do { \
+ printk(KERN_ERR PFX "%s: PANIC! " fmt, \
+ __func__ , ## arg); \
+ BUG(); \
+ } while(0)
+
+#define ISCSI_ISER_XMIT_CMDS_MAX 128 /* must be power of 2 */
+#define ISCSI_ISER_MGMT_CMDS_MAX 32 /* must be power of 2 */
+ /* support upto 512KB in one RDMA */
+#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> PAGE_SHIFT)
+#define ISCSI_ISER_CMD_PER_LUN ISCSI_ISER_XMIT_CMDS_MAX
+#define ISCSI_ISER_MAX_LUN 256
+#define ISCSI_ISER_MAX_CMD_LEN 16
+
+#define ISCSI_MGMT_ITT_OFFSET 0xa00
+
+/* Session's states */
+#define ISCSI_STATE_FREE 1
+#define ISCSI_STATE_LOGGED_IN 2
+#define ISCSI_STATE_FAILED 3
+#define ISCSI_STATE_TERMINATE 4
+
+/* Connection's states */
+#define ISCSI_CONN_INITIAL_STAGE 0
+#define ISCSI_CONN_STARTED 1
+#define ISCSI_CONN_STOPPED 2
+#define ISCSI_CONN_CLEANUP_WAIT 3
+
+/* Connection suspend "bit" */
+#define SUSPEND_BIT 1
+
+/* Task Mgmt states */
+#define TMABORT_INITIAL 0x0
+#define TMABORT_SUCCESS 0x1
+#define TMABORT_FAILED 0x2
+#define TMABORT_TIMEDOUT 0x3
+
+#define ITT_MASK (0xfff)
+#define CID_SHIFT 12
+#define CID_MASK (0xffff<<CID_SHIFT)
+#define AGE_SHIFT 28
+#define AGE_MASK (0xf<<AGE_SHIFT)
+
+/* QP settings */
+/* Maximal bounds on received asynchronous PDUs */
+#define ISER_MAX_RX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */
+
+#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), *
+ * SCSI_TMFUNC(2), LOGOUT(1) */
+
+#define ISER_QP_MAX_RECV_DTOS (ISCSI_ISER_XMIT_CMDS_MAX + \
+ ISER_MAX_RX_MISC_PDUS + \
+ ISER_MAX_TX_MISC_PDUS)
+
+/* the max TX (send) WR supported by the iSER QP is defined by *
+ * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
+ * to have at max for SCSI command. The tx posting & completion handling code *
+ * supports -EAGAIN scheme where tx is suspended till the QP has room for more *
+ * send WR. D=8 comes from 64K/8K */
+
+#define ISER_INFLIGHT_DATAOUTS 8
+
+#define ISER_QP_MAX_REQ_DTOS (ISCSI_ISER_XMIT_CMDS_MAX * \
+ (1 + ISER_INFLIGHT_DATAOUTS) + \
+ ISER_MAX_TX_MISC_PDUS + \
+ ISER_MAX_RX_MISC_PDUS)
+
+#define ISER_VER 0x10
+#define ISER_WSV 0x08
+#define ISER_RSV 0x04
+
+struct iser_hdr {
+ u8 flags;
+ u8 rsvd[3];
+ __be32 write_stag; /* write rkey */
+ __be64 write_va;
+ __be32 read_stag; /* read rkey */
+ __be64 read_va;
+} __attribute__((packed));
+
+
+/* Length of an object name string */
+#define ISER_OBJECT_NAME_SIZE 64
+
+enum iser_ib_conn_state {
+ ISER_CONN_INIT, /* descriptor allocd, no conn */
+ ISER_CONN_PENDING, /* in the process of being established */
+ ISER_CONN_UP, /* up and running */
+ ISER_CONN_TERMINATING, /* in the process of being terminated */
+ ISER_CONN_DOWN, /* shut down */
+ ISER_CONN_STATES_NUM
+};
+
+enum iser_task_status {
+ ISER_TASK_STATUS_INIT = 0,
+ ISER_TASK_STATUS_STARTED,
+ ISER_TASK_STATUS_COMPLETED
+};
+
+enum iser_data_dir {
+ ISER_DIR_IN = 0, /* to initiator */
+ ISER_DIR_OUT, /* from initiator */
+ ISER_DIRS_NUM
+};
+
+enum iser_buf_type {
+ ISER_BUF_TYPE_SINGLE = 0, /* single contiguous buffer */
+ ISER_BUF_TYPE_SCATTERLIST, /* struct scatterlist array */
+ ISER_BUF_TYPES_NUM
+};
+
+struct iser_data_buf {
+ enum iser_buf_type type; /* single or scatterlist */
+ void *p_buf; /* single -> data scatterlist -> sg */
+ unsigned int size; /* data len for single, nentries for sg */
+ dma_addr_t dma_addr; /* returned by dma_map_single */
+ unsigned int dma_nents; /* returned by dma_map_sg for */
+};
+
+/* fwd declarations */
+struct iser_adaptor;
+struct iscsi_iser_conn;
+struct iscsi_iser_session;
+struct iscsi_iser_mgmt_task;
+struct iscsi_iser_cmd_task;
+
+struct iser_mem_reg {
+ u32 lkey;
+ u32 rkey;
+ u64 va;
+ u64 len;
+ void *mem_h;
+};
+
+struct iser_regd_buf {
+ struct iser_mem_reg reg; /* memory registration info */
+ void *virt_addr;
+ struct iser_adaptor *p_adaptor; /* p_adaptor->device for dma_unmap */
+ dma_addr_t dma_addr; /* if non zero, addr for dma_unmap */
+ enum dma_data_direction direction; /* direction for dma_unmap */
+ unsigned int data_size;
+ atomic_t ref_count; /* refcount, freed when dec to 0 */
+};
+
+#define MAX_REGD_BUF_VECTOR_LEN 2
+
+struct iser_dto {
+ struct iscsi_iser_cmd_task *p_task;
+ struct iscsi_iser_conn *p_conn;
+ int notify_enable;
+
+ /* vector of registered buffers */
+ unsigned int regd_vector_len;
+ struct iser_regd_buf *regd[MAX_REGD_BUF_VECTOR_LEN];
+
+ /* offset into the registered buffer may be specified */
+ unsigned int offset[MAX_REGD_BUF_VECTOR_LEN];
+
+ /* a smaller size may be specified, if 0, then full size is used */
+ unsigned int used_sz[MAX_REGD_BUF_VECTOR_LEN];
+};
+
+enum iser_desc_type {
+ ISCSI_RX,
+ ISCSI_TX_CONTROL ,
+ ISCSI_TX_SCSI_COMMAND,
+ ISCSI_TX_DATAOUT
+};
+
+struct iser_desc {
+ struct iser_hdr iser_header;
+ struct iscsi_hdr iscsi_header;
+ struct iser_regd_buf hdr_regd_buf;
+ void *data; /* used by RX & TX_CONTROL */
+ struct iser_regd_buf data_regd_buf; /* used by RX & TX_CONTROL */
+ enum iser_desc_type type;
+ struct iser_dto dto;
+};
+
+struct iser_adaptor {
+ struct ib_device *device;
+ struct ib_pd *pd;
+ struct ib_cq *cq;
+ struct ib_mr *mr;
+ struct tasklet_struct cq_tasklet;
+ struct list_head ig_list; /* entry in ig adaptors list */
+ int refcount;
+ char name[ISER_OBJECT_NAME_SIZE];
+};
+
+struct iser_conn
+{
+ atomic_t state; /* rdma connection state */
+ struct iser_adaptor *p_adaptor; /* adaptor context */
+ struct rdma_cm_id *cma_id; /* CMA ID */
+ struct ib_qp *qp; /* QP */
+ struct ib_fmr_pool *fmr_pool; /* pool of IB FMRs */
+ int disc_evt_flag; /* disconn event delivered */
+ wait_queue_head_t wait; /* waitq for conn/disconn */
+ struct iscsi_iser_conn *p_iscsi_conn; /* iscsi conn for upcalls */
+ atomic_t post_recv_buf_count; /* posted rx count */
+ atomic_t post_send_buf_count; /* posted tx count */
+ struct work_struct comperror_work; /* conn term sleepable ctx*/
+ char name[ISER_OBJECT_NAME_SIZE];
+};
+
+struct iscsi_iser_queue {
+ struct kfifo *queue; /* FIFO Queue */
+ void **pool; /* Pool of elements */
+ int max; /* Max number of elements */
+};
+
+struct iscsi_iser_conn {
+ struct socket *sock; /* iSER socket */
+ struct iser_conn *ib_conn; /* iSER IB conn */
+ int stop_stage; /* conn_stop() flag: *
+ * stop to recover, *
+ * stop to terminate */
+ /* iSCSI connection-wide sequencing */
+ uint32_t exp_statsn;
+
+ /* control data */
+ int id; /* CID */
+ struct iscsi_iser_session *session; /* parent session */
+ struct list_head item; /* maintains list of conns */
+ int c_stage; /* connection state */
+ struct iscsi_iser_mgmt_task *login_mtask;/* mtask for login/text */
+ struct iscsi_iser_mgmt_task *mtask; /* xmit mtask in progress */
+ struct iscsi_iser_cmd_task *ctask; /* xmit ctask in progress */
+
+ /* xmit */
+ struct kfifo *immqueue; /* immediate xmit queue */
+ struct kfifo *mgmtqueue; /* mgmt (control) xmit queue */
+ struct kfifo *xmitqueue; /* data-path cmd queue */
+ struct work_struct xmitwork; /* per-conn. xmit workqueue */
+ struct mutex xmitmutex; /* serializes connection xmit,
+ * access to kfifos: *
+ * xmitqueue, *
+ * immqueue, mgmtqueue */
+ unsigned long suspend_tx; /* suspend Tx */
+ spinlock_t lock;
+
+ /* abort */
+ wait_queue_head_t ehwait; /* used in eh_abort() */
+ struct iscsi_tm tmhdr;
+ struct timer_list tmabort_timer; /* abort timer */
+ int tmabort_state; /* see TMABORT_INITIAL,etc.*/
+
+ /* negotiated params */
+ int max_recv_dlength; /* initiator_max_recv_dsl*/
+ int max_xmit_dlength; /* target_max_recv_dsl */
+};
+
+struct iscsi_iser_session {
+ /* iSCSI session-wide sequencing */
+ uint32_t cmdsn;
+ uint32_t exp_cmdsn;
+ uint32_t max_cmdsn;
+
+ /* configuration */
+ int initial_r2t_en;
+ int max_r2t;
+ int imm_data_en;
+ int first_burst;
+ int max_burst;
+ int time2wait;
+ int time2retain;
+ int pdu_inorder_en;
+ int dataseq_inorder_en;
+ int erl;
+
+ /* control data */
+ struct Scsi_Host *host;
+ int id;
+ struct iscsi_iser_conn *leadconn; /* leading connection */
+ spinlock_t lock; /* protects session state, *
+ * sequence numbers, *
+ * session resources: *
+ * - cmdpool, *
+ * - mgmtpool, */
+ volatile int state; /* session state */
+ int conn_cnt;
+ int age; /* counts session re-opens */
+
+ struct list_head connections; /* list of connections */
+ int cmds_max; /* size of cmds array */
+ struct iscsi_iser_cmd_task **cmds; /* Original Cmds arr */
+ struct iscsi_iser_queue cmdpool; /* PDU's pool */
+ int mgmtpool_max;/* size of mgmt array */
+ struct iscsi_iser_mgmt_task **mgmt_cmds; /* Original mgmt arr */
+ struct iscsi_iser_queue mgmtpool; /* Mgmt PDU's pool */
+};
+
+struct iscsi_iser_mgmt_task {
+ struct iser_desc desc;
+ struct iscsi_hdr *hdr; /* mgmt. PDU header points *
+ * to desc.iscsi_hdr */
+ char *data; /* mgmt payload points to *
+ * desc.data */
+ int data_count; /* counts data to be sent */
+ uint32_t itt; /* this ITT */
+};
+
+struct iscsi_iser_cmd_task {
+ struct iser_desc desc;
+
+ struct iscsi_cmd *hdr; /* iSCSI PDU header points *
+ * to desc.iscsi_hdr */
+ int itt; /* this ITT */
+ int datasn; /* DataSN */
+ uint32_t unsol_datasn;
+ int imm_count; /* imm-data (bytes) */
+ int unsol_count; /* unsolicited (bytes) */
+ int data_count; /* remaining Data-Out */
+ int rdma_data_count;/* RDMA bytes */
+ struct scsi_cmnd *sc; /* associated SCSI cmd */
+ int total_length;
+ struct iscsi_iser_conn *conn; /* used connection */
+ struct iscsi_iser_mgmt_task *mtask; /* tmf mtask in progr */
+
+ enum iser_task_status status;
+ int command_sent; /* set if command sent */
+ int dir[ISER_DIRS_NUM]; /* set if dir use*/
+ struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */
+ unsigned long data_len[ISER_DIRS_NUM]; /* total data len*/
+ struct iser_data_buf data[ISER_DIRS_NUM]; /* orig. data des*/
+ struct iser_data_buf data_copy[ISER_DIRS_NUM];/* contig. copy */
+};
+
+struct iser_page_vec {
+ u64 *pages;
+ int length;
+ int offset;
+ int data_size;
+};
+
+struct iser_global {
+ struct mutex adaptor_list_mutex;/* */
+ struct list_head adaptor_list; /* all iSER adaptors */
+
+ kmem_cache_t *desc_cache;
+};
+
+extern struct iser_global ig;
+extern int iser_debug_level;
+
+/* allocate connection resources needed for rdma functionality */
+int iser_conn_set_full_featured_mode(struct iscsi_iser_conn *p_iser_conn);
+
+int iser_send_control(struct iscsi_iser_conn *p_iser_conn,
+ struct iscsi_iser_mgmt_task *p_mtask);
+
+int iser_send_command(struct iscsi_iser_conn *p_iser_conn,
+ struct iscsi_iser_cmd_task *p_ctask);
+
+int iser_send_data_out(struct iscsi_iser_conn *p_iser_conn,
+ struct iscsi_iser_cmd_task *p_ctask,
+ struct iscsi_data *hdr);
+
+int iscsi_iser_hdr_recv(struct iscsi_iser_conn *conn, struct iscsi_hdr *hdr,
+ char *rx_data);
+
+void iscsi_iser_conn_failure(struct iscsi_iser_conn *conn, enum iscsi_err err);
+
+
+void iser_conn_init(struct iser_conn *p_iser_conn);
+
+int iser_conn_establish(struct iser_conn *p_iser_conn,
+ struct sockaddr_in *dst_addr,
+ struct sockaddr_in *src_addr);
+
+void iser_conn_terminate(struct iser_conn *ib_conn);
+
+void iser_rcv_completion(struct iser_desc *p_desc,
+ unsigned long dto_xfer_len);
+
+void iser_snd_completion(struct iser_desc *p_desc);
+
+void iser_ctask_rdma_init(struct iscsi_iser_cmd_task *p_iser_task);
+void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *p_iser_task);
+
+void iser_dto_buffs_release(struct iser_dto *p_dto);
+
+int iser_regd_buff_release(struct iser_regd_buf *p_regd_buf);
+
+void iser_reg_single(struct iser_adaptor *p_iser_adaptor,
+ struct iser_regd_buf *p_regd_buf,
+ enum dma_data_direction direction);
+
+int iser_sg_size(struct iser_data_buf *p_mem);
+
+void iser_start_rdma_unaligned_sg(struct iscsi_iser_cmd_task *p_iser_task,
+ enum iser_data_dir cmd_dir);
+
+void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_cmd_task *p_iser_task);
+
+int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *p_iser_task,
+ enum iser_data_dir cmd_dir);
+
+int iser_connect(struct iser_conn *p_iser_conn,
+ struct sockaddr_in *src_addr, struct sockaddr_in *dst_addr);
+
+int iser_reg_page_vec(struct iser_conn *p_iser_conn,
+ struct iser_page_vec *page_vec,
+ struct iser_mem_reg *p_mem_reg);
+
+void iser_unreg_mem(struct iser_mem_reg *mem_reg);
+
+int iser_post_recv(struct iser_desc *p_rx_desc);
+int iser_post_send(struct iser_desc *p_tx_desc);
+#endif
More information about the general
mailing list