[openib-general] [iSER]use iSER on x86_64
Dan Bar Dov
danb at voltaire.com
Mon Aug 8 10:09:24 PDT 2005
Ian, hi
Please send patches for the modifications you made for x86_64. In any case, we'll be upgrading the ISER implementation to be openIB/kDAPL based (DAT1.2, but with the openIB DAT headers). It will support x86_64.
To test you'll have to have InfiniBand hardware (HCA+switch) plus an ISER target.
Dan
> -----Original Message-----
> From: openib-general-bounces at openib.org
> [mailto:openib-general-bounces at openib.org] On Behalf Of Ian Jiang
> Sent: Sunday, August 07, 2005 3:36 AM
> To: openib-general at openib.org
> Subject: [openib-general] [iSER]use iSER on x86_64
>
> [I sent this mail the day before yesterday, but it did not
> appear on the list. So I just try again.]
>
> I modified several files under iSER/datamover/ to use it on
> my x86_64 machine.
> I also edited the iSER/make.conf file where necessary.
> I could compile the iSER without any errors by now, but I am
> not sure if it can works with the iSCSI initiator or the
> kDAPL, because I have no idea to test it yet.
>
> The iSER is obtained from
> https://openib.org/svn/gen2/ulps/iser/
> The dat headers related could be either
> http://sourceforge.net/projects/dapl/dapl_beta2.06/dat/
> or
> http://www.datcollaborative.org/dat_headers_1.2.tgz/
>
> Any suggestion is appriciated!
>
>
> Ian Jiang
> ianjiang91 at hotmail.com
> ----
> Computer Architecture Laboratory
> Institute of Computing Technology
> Chinese Academy of Sciences
> Beijing,P.R.China
> Zip code: 100080
> Tel: +86-10-62564394(office)
>
>
> ======================
> iser-datamover.patches
> ======================
>
> --- iser_conn.c.old 2005-08-04 01:40:23.000000000 +0800
> +++ iser_conn.c 2005-08-05 06:36:56.000000000 +0800
> @@ -602,8 +602,11 @@
> /* Find the connection */
> p_iser_conn = hash_find_iser_conn(iscsi_conn_h);
> if (p_iser_conn == NULL) {
> - IERROR("Connection not found, conn_h: %d\n",
> - (unsigned) iscsi_conn_h);
> + /* by IanJiang, ianjiang at ict.ac.cn */
> + // IERROR("Connection not found, conn_h: %d\n",
> + IERROR("Connection not found, conn_h: %ld\n",
> + // (unsigned) iscsi_conn_h);
> + (unsigned long) iscsi_conn_h);
> return ISER_INVALID_CONN;
> }
>
> --- iser_dto.c.old 2005-08-04 05:59:17.000000000 +0800
> +++ iser_dto.c 2005-08-05 06:37:28.000000000 +0800
> @@ -356,7 +356,9 @@
>
> /* Get the VA of the headers registered memory */
> p_recv_buf =
> - (unsigned char *) (unsigned int)
> p_dto->regd[0]->virt_buf.p_buf;
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// (unsigned char *) (unsigned int)
> p_dto->regd[0]->virt_buf.p_buf;
> + (unsigned char *) (unsigned long)
> p_dto->regd[0]->virt_buf.p_buf;
> /* Account for the offset within it */
> p_recv_buf += p_dto->offset[0];
> /* Skip the iSER header to get the iSCSI PDU BHS */
> --- iser_global.c.old 2005-08-04 01:40:04.000000000 +0800
> +++ iser_global.c 2005-08-05 06:38:06.000000000 +0800
> @@ -571,7 +571,9 @@
> unsigned port) /*!< IN - listening port */
> {
> struct iser_entity_t *p_entity;
> - unsigned entity_index = (unsigned) api_h;
> +/* by IanJiang, ianjiang at ict.ac.cn */
> + unsigned long entity_index = (unsigned long) api_h;
> +// unsigned entity_index = (unsigned) api_h;
>
> if (entity_index >= iser_global.num_entities) {
> return ISER_ILLEGAL_PARAM;
> --- iser_initiator.c.old 2005-08-04 01:39:47.000000000 +0800
> +++ iser_initiator.c 2005-08-05 06:45:48.000000000 +0800
> @@ -276,15 +276,21 @@
> p_iser_conn = hash_find_iser_conn(iscsi_conn_h);
>
> if (p_iser_conn == NULL) {
> - IERROR("Failed to find connection,
> iscsi_conn_h: %X\n",
> - (unsigned) iscsi_conn_h);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> + //IERROR("Failed to find connection,
> iscsi_conn_h: %X\n",
> + IERROR("Failed to find connection,
> iscsi_conn_h: %lX\n",
> + //(unsigned) iscsi_conn_h);
> + (unsigned long) iscsi_conn_h);
> iser_pdu_print((char *) __func__, NULL,
> p_bhs->buf, NULL);
> iser_ret = ISER_INVALID_CONN;
> goto send_control_error;
> }
> if (atomic_read(&p_iser_conn->state) != ISER_CONN_UP) {
> - IERROR("Connection is not up, iscsi_conn_h:
> %X, p_conn:
> 0x%p\n",
> - (unsigned) iscsi_conn_h, p_iser_conn);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> + //IERROR("Connection is not up, iscsi_conn_h:
> %X, p_conn:
> 0x%p\n",
> + IERROR("Connection is not up, iscsi_conn_h:
> %lX, p_conn:
> 0x%p\n",
> + //(unsigned ) iscsi_conn_h, p_iser_conn);
> + (unsigned long) iscsi_conn_h, p_iser_conn);
> iser_pdu_print((char *) __func__, NULL,
> p_bhs->buf, NULL);
> iser_ret = ISER_FAILURE;
> goto send_control_error;
> --- iser_kdapl.c.old 2005-08-04 02:12:25.000000000 +0800
> +++ iser_kdapl.c 2005-08-05 06:46:13.000000000 +0800
> @@ -49,6 +49,9 @@
> #include "iser_procfs.h"
> #include "iser_trace.h"
>
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +#define DAT_MEM_OPT_DONT_CARE DAT_MEM_OPTIMIZE_DONT_CARE
> +
> /*
> ---------------------------------------------------------------------
> * CONSTANTS & MACROS
> *
> ------------------------------------------------------------------ */
> --- iser_memory.c.old 2005-08-04 02:14:55.000000000 +0800
> +++ iser_memory.c 2005-08-05 06:47:51.000000000 +0800
> @@ -683,10 +683,14 @@
> list_entry(p_list, struct iser_buf_pool_region_t,
> pool_list);
> n += sprintf(p_str + n,
> - "\t%3d: mem[0x%p sz:%7d t:%d]
> lmr[h:0x%08x
> va:0x%08x sz:%7d]\n",
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// "\t%3d: mem[0x%p sz:%7d t:%d]
> lmr[h:0x%08x
> va:0x%08x sz:%7d]\n",
> + "\t%3d: mem[0x%p sz:%7d t:%d]
> lmr[h:0x%08lx
> va:0x%08x sz:%7d]\n",
> p_pool_region->id,
> p_pool_region->buf.p_buf,
> p_pool_region->buf.size,
> p_pool_region->buf.type,
> - (unsigned)
> p_pool_region->mem_reg.lmr_handle,
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// (unsigned)
> p_pool_region->mem_reg.lmr_handle,
> + (unsigned long)
> p_pool_region->mem_reg.lmr_handle,
> (unsigned)
> p_pool_region->mem_reg.lmr_triplet.
> virtual_address,
> (unsigned)
> p_pool_region->mem_reg.lmr_triplet.
> @@ -958,13 +962,18 @@
> ITRACE_ENTRY();
>
> if (p_iser_adaptor->regd_mem.virt_buf.size > 0) {
> /* if any
> pre-regd
> buffer */
> - unsigned start_regd_buf =
> - (unsigned)
> p_iser_adaptor->regd_mem.virt_buf.p_buf;
> - unsigned end_regd_buf =
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// unsigned start_regd_buf =
> + unsigned long start_regd_buf =
> +// (unsigned)
> p_iser_adaptor->regd_mem.virt_buf.p_buf;
> + (unsigned long)
> p_iser_adaptor->regd_mem.virt_buf.p_buf;
> +// unsigned end_regd_buf =
> + unsigned long end_regd_buf =
> start_regd_buf +
> p_iser_adaptor->regd_mem.virt_buf.size;
>
> ITRACE(ISER_TRACE_BUFFERS,
> - "Looking up buf: 0x%08x - 0x%08x in
> cache: 0x%08x -
> 0x%08x\n",
> +// "Looking up buf: 0x%08x - 0x%08x in
> cache: 0x%08x -
> 0x%08x\n",
> + "Looking up buf: 0x%08x - 0x%08x in cache:
> + 0x%08lx -
> 0x%08lx\n",
> buf_addr, buf_addr + buf_size, start_regd_buf,
> end_regd_buf);
> if (start_regd_buf <= buf_addr
> --- iser_procfs.c.old 2005-08-04 06:01:50.000000000 +0800
> +++ iser_procfs.c 2005-08-05 06:48:33.000000000 +0800
> @@ -207,9 +207,13 @@
> p_iser_conn = (struct iser_conn_t *) data;
>
> page[0] = '\0';
> - n += sprintf(buf, "iSCSI handle: 0x%08X\nkDAPL EP handle:
> 0x%08X\n",
> - (unsigned) p_iser_conn->iscsi_conn_h,
> - (unsigned) p_iser_conn->ep_handle);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// n += sprintf(buf, "iSCSI handle: 0x%08X\nkDAPL EP handle:
> 0x%08X\n",
> + n += sprintf(buf, "iSCSI handle: 0x%08lX\nkDAPL EP handle:
> 0x%08lX\n",
> +// (unsigned) p_iser_conn->iscsi_conn_h,
> + (unsigned long) p_iser_conn->iscsi_conn_h,
> +// (unsigned along) p_iser_conn->ep_handle);
> + (unsigned long) p_iser_conn->ep_handle);
> strcat(page, buf);
>
> n += sprintf(buf, "State: %s\n",
> iser_conn_get_state_name(p_iser_conn));
> --- iser_utils.c.old 2005-08-04 02:42:16.000000000 +0800
> +++ iser_utils.c 2005-08-05 06:48:55.000000000 +0800
> @@ -32,6 +32,10 @@
> * $Id: iser_utils.c,v 1.38 2005/01/31 08:02:00 danb Exp $ */
>
> +/* by IanJiang, ianjiang at ict.ac.cn
> + * replace u32 by u64 in 5 places
> + */
> +
> #include <linux/module.h>
> #include <linux/kernel.h>
> #include <linux/slab.h>
> @@ -102,7 +106,7 @@
>
> ITRACE_ENTRY();
>
> - hash_val = hash_func(iser_task->itt ^ (u32)
> iser_task->p_conn);
> + hash_val = hash_func(iser_task->itt ^ (u64)
> iser_task->p_conn);
>
> ITRACE(ISER_TRACE_HASHTABLES,
> "p_task: 0x%p, p_conn: 0x%p, itt: %d, hash_val
> = %d\n", @@ -126,7 +130,7 @@ */ struct iser_task_t *
> hash_find_iser_task(struct iser_conn_t *iser_conn, /*!<
> IN - part of
> hash
> key */
> - u32 itt) /*!< IN - part of hash key */
> + u64 itt) /*!< IN - part of hash key */
> {
> int hash_val;
> struct list_head *p_bucket;
> @@ -135,7 +139,7 @@
>
> ITRACE_ENTRY();
>
> - hash_val = hash_func(itt ^ (u32) iser_conn);
> + hash_val = hash_func(itt ^ (u64) iser_conn);
> p_bucket = &(iser_global.task_hash.bucket_head[hash_val]);
>
> spin_lock(&iser_global.task_hash.lock);
> @@ -151,8 +155,11 @@
> spin_unlock(&iser_global.task_hash.lock);
>
> ITRACE(ISER_TRACE_HASHTABLES,
> - "p_conn: 0x%p, itt: %d, hash_val = %d, found p_task:
> 0x%p\n",
> - iser_conn, itt, hash_val, iser_task);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// "p_conn: 0x%p, itt: %d, hash_val = %d, found p_task:
> 0x%p\n",
> +// iser_conn, itt, hash_val, iser_task);
> + "p_conn: 0x%p, itt: %ld, hash_val = %d, found p_task:
> 0x%p\n",
> + iser_conn, (unsigned long)itt, hash_val, iser_task);
>
> ITRACE_EXIT();
> return iser_task;
> @@ -188,7 +195,7 @@
>
> ITRACE_ENTRY();
>
> - hash_val = hash_func((u32) iser_conn->iscsi_conn_h);
> + hash_val = hash_func((u64) iser_conn->iscsi_conn_h);
>
> spin_lock(&iser_global.conn_hash.lock);
> INIT_LIST_HEAD(&iser_conn->hash_list);
> @@ -216,7 +223,7 @@
>
> ITRACE_ENTRY();
>
> - hash_val = hash_func((u32) iscsi_conn_h);
> + hash_val = hash_func((u64) iscsi_conn_h);
> p_bucket = &(iser_global.conn_hash.bucket_head[hash_val]);
>
> spin_lock(&iser_global.conn_hash.lock);
> @@ -349,8 +356,11 @@
> virt_addr = p_iovec_virt[i].iov_base;
> p_phys[i].addr = virt_to_phys(virt_addr);
> ITRACE(ISER_TRACE_BUFFERS,
> - "IOVEC[%d] virt: 0x%08X -> phys:
> 0x%08X, sz: %d\n",
> i,
> - (unsigned) virt_addr, p_phys[i].addr,
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// "IOVEC[%d] virt: 0x%08X -> phys:
> 0x%08X, sz: %d\n",
> i,
> +// (unsigned) virt_addr, p_phys[i].addr,
> + "IOVEC[%d] virt: 0x%08lX -> phys: 0x%08X, sz:
> %ld\n", i,
> + (unsigned long) virt_addr, p_phys[i].addr,
> p_iovec_virt[i].iov_len);
> p_phys[i].size = p_iovec_virt[i].iov_len;
> total_sz += p_iovec_virt[i].iov_len; @@ -393,7
> +403,9 @@
> }
>
> for (i = 0; i < p_data->size; i++) {
> - p_phys[i].addr = (uint32_t) p_iovec_phys[i].iov_base;
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// p_phys[i].addr = (uint32_t) p_iovec_phys[i].iov_base;
> + p_phys[i].addr = (uint64_t) p_iovec_phys[i].iov_base;
> p_phys[i].size = p_iovec_phys[i].iov_len;
> total_sz += p_iovec_phys[i].iov_len;
>
> @@ -440,10 +452,14 @@
> "starting scatterlist conversion - %d
> elements\n", p_data->size);
> for (i = 0; i < p_data->size; i++) {
> p_phys[i].addr = page_to_phys(p_sg[i].page) +
> p_sg[i].offset;
> +/* by IanJiang, ianjiang at ict.ac.cn */
> ITRACE(ISER_TRACE_BUFFERS,
> - "SCATTER[%d] page: 0x%08X + 0x%8X ->
> phys: 0x%08X,
> sz: %d\n",
> - i, (unsigned) p_sg[i].page, p_sg[i].offset,
> - p_phys[i].addr, p_sg[i].length);
> +// "SCATTER[%d] page: 0x%08X + 0x%8X ->
> phys: 0x%08X,
> sz: %d\n",
> + "SCATTER[%d] page: 0x%08lX + 0x%8X -> phys:
> + 0x%08lX,
> sz: %d\n",
> +// i, (unsigned) p_sg[i].page, p_sg[i].offset,
> + i, (unsigned long) p_sg[i].page,
> p_sg[i].offset,
> +// p_phys[i].addr, p_sg[i].length);
> + (unsigned long) p_phys[i].addr,
> p_sg[i].length);
> p_phys[i].size = p_sg[i].length;
> total_sz += p_sg[i].length;
> }
> @@ -501,7 +517,9 @@
> void *
> iser_phys_to_virt(void *phys_addr)
> {
> - return phys_to_virt((unsigned) phys_addr);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// return phys_to_virt((unsigned) phys_addr);
> + return phys_to_virt((unsigned long) phys_addr);
> }
>
> /**
> @@ -535,7 +553,9 @@
> {
> struct iovec *p_iovec;
> p_iovec = (struct iovec *) p_iovec_phys;
> - return phys_to_virt((unsigned) p_iovec[i].iov_base);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +// return phys_to_virt((unsigned) p_iovec[i].iov_base);
> + return phys_to_virt((unsigned long) p_iovec[i].iov_base);
> }
>
> /**
> --- iser_utils.h.old 2005-07-01 17:41:00.000000000 +0800
> +++ iser_utils.h 2005-08-05 06:49:32.000000000 +0800
> @@ -48,7 +48,9 @@
> * ISER TASK-SPECIFIC HASH MANAGEMENT
> *
> ------------------------------------------------------------------ */
>
> -struct iser_task_t *hash_find_iser_task(struct iser_conn_t
> *iser_conn, u32 itt);
> +/* by IanJiang, ianjiang at ict.ac.cn */
> +//struct iser_task_t *hash_find_iser_task(struct iser_conn_t
> +*iser_conn,
> u32 itt);
> +struct iser_task_t *hash_find_iser_task(struct iser_conn_t
> *iser_conn,
> +u64
> itt);
>
> void
> hash_add_iser_task(struct iser_task_t *iser_task);
>
> _________________________________________________________________
> 与联机的朋友进行交流，请使用 MSN Messenger: http://messenger.msn.com/cn
>
> _______________________________________________
> openib-general mailing list
> openib-general at openib.org
> http://openib.org/mailman/listinfo/openib-general
>
> To unsubscribe, please visit
> http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list