[ofa-general] [PATCH RFC] rds: add iwarp support

Olaf Kirch olaf.kirch at ORACLE.COM
Tue Jul 8 13:55:00 PDT 2008


Hi Jon,

thanks for doing this! I'll merge everything tomorrow, and try
to break it down into a series of nice patches, and it all goes
well I will at least push the bcopy part to my 1.4 tree.

> This patch reduces the default send_wr and recv_wr to sizes that Chelsio
> can handle, as the sizes before were too large.  Optimally, this should
> be determined with a query of the device.  I have added that to my todo
> list as well

Yes, that'll be nice to do properly. I'll think about this a little
tomorrow.

> The lguid and fguid cannot be used for iWARP as those structs do not
> exists and give off a nasty illegal pointer deference.  I removed the
> references to those, as the appear to only be for debug messages.

Fine with me.

> Lastly, this has the port space differentiation (which is the only way
> TCP and iWARP can happily co-exist).

Okay.

Olaf

On Tuesday 08 July 2008 22:27:55 Jon Mason wrote:
> Hey Olaf,
> With the patch below on top of your previous patch, I am able to get it
> working over iWARP (though RDMA is still breaking, but in a new and
> different way).
> 
> The patch modifies rds_ib_laddr_check to use rdma_bind_addr to determine
> if it is a IB/iWARP device by creating a temporary rdma_cm_id and
> binding to the address passed in.  I'm sure this can be optimized, but
> I'll leave that as a todo for later.
> 
> This patch reduces the default send_wr and recv_wr to sizes that Chelsio
> can handle, as the sizes before were too large.  Optimally, this should
> be determined with a query of the device.  I have added that to my todo
> list as well
> 
> The lguid and fguid cannot be used for iWARP as those structs do not
> exists and give off a nasty illegal pointer deference.  I removed the
> references to those, as the appear to only be for debug messages.
> 
> Lastly, this has the port space differentiation (which is the only way
> TCP and iWARP can happily co-exist).
> 
> Let me know what you think.
> 
> Thanks,
> Jon
> 
> Signed-off-by: Jon Mason <jon at opengridcomputing.com>
> 
> diff --git a/net/rds/ib.c b/net/rds/ib.c
> index cd2dc7c..067fab0 100644
> --- a/net/rds/ib.c
> +++ b/net/rds/ib.c
> @@ -158,17 +158,29 @@ void rds_ib_remove_one(struct ib_device *device)
>   */
>  static int rds_ib_laddr_check(__be32 addr)
>  {
> -	struct net_device *dev;
> -	int ret;
> +	int ret = -EADDRNOTAVAIL;
> +	struct rdma_cm_id *cm_id;
> +	struct sockaddr_in sin;
> +
> +	cm_id = rdma_create_id(NULL, NULL, RDMA_PS_TCP);
> +	if (cm_id == NULL) {
> +		printk("rdma_create_id failed\n");
> +		return -EADDRNOTAVAIL;
> +	}
>  
> -	dev = ip_dev_find(addr);
> -	if (dev && dev->type == ARPHRD_INFINIBAND) {
> -		dev_put(dev);
> -		ret = 0;
> -	} else
> +	memset(&sin, 0, sizeof(sin));
> +	sin.sin_family = AF_INET;
> +	sin.sin_addr.s_addr = addr;
> +
> +	/* rdma_bind_addr will fail for non-IB/iWARP devices */
> +	ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
> +        if (ret)
>  		ret = -EADDRNOTAVAIL;
>  
> -	rdsdebug("addr %u.%u.%u.%u ret %d\n", NIPQUAD(addr), ret);
> +	rdsdebug("addr %u.%u.%u.%u ret %d node type %d\n",
> +		 NIPQUAD(addr), ret, cm_id->device?cm_id->device->node_type:-1);
> +
> +	rdma_destroy_id(cm_id);
>  
>  	return ret;
>  }
> diff --git a/net/rds/ib.h b/net/rds/ib.h
> index 947977c..2924fda 100644
> --- a/net/rds/ib.h
> +++ b/net/rds/ib.h
> @@ -7,14 +7,16 @@
>  
>  #define RDS_IB_RESOLVE_TIMEOUT_MS	5000
>  
> -#define RDS_FMR_SIZE			256
> +/* FIXME - use ib_query_device to determine proper value */
> +#define RDS_FMR_SIZE			20 
>  #define RDS_FMR_POOL_SIZE		2048
>  
> -#define RDS_IB_MAX_SGE			8
> +#define RDS_IB_MAX_SGE		8
>  #define RDS_IB_RECV_SGE 		2
>  
> -#define RDS_IB_DEFAULT_RECV_WR		1024
> -#define RDS_IB_DEFAULT_SEND_WR		256
> +/* FIXME - call ib_query_device to determine sane values for this based on the HW */
> +#define RDS_IB_DEFAULT_RECV_WR	512
> +#define RDS_IB_DEFAULT_SEND_WR	128
>  
>  #define RDS_IB_SUPPORTED_PROTOCOLS	0x00000003	/* minor versions supported */
>  
> diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
> index a49e394..fd56481 100644
> --- a/net/rds/ib_cm.c
> +++ b/net/rds/ib_cm.c
> @@ -296,7 +296,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
>  	 */
>  	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
>  	if (ret) {
> -		rdsdebug("ib_req_notify_cq failed: %d\n", ret);
> +		rdsdebug("rdma_create_qp failed: %d\n", ret);
>  		goto out;
>  	}
>  
> @@ -388,8 +388,6 @@ static u32 rds_ib_protocol_compatible(const struct rds_ib_connect_private *dp)
>  static int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
>  				    struct rdma_cm_event *event)
>  {
> -	__be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
> -	__be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
>  	const struct rds_ib_connect_private *dp = event->param.conn.private_data;
>  	struct rds_ib_connect_private dp_rep;
>  	struct rds_connection *conn = NULL;
> @@ -404,11 +402,9 @@ static int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
>  	if (!version)
>  		goto out;
>  
> -	rdsdebug("saddr %u.%u.%u.%u daddr %u.%u.%u.%u RDSv%u.%u lguid 0x%llx fguid "
> -		 "0x%llx\n", NIPQUAD(dp->dp_saddr), NIPQUAD(dp->dp_daddr),
> -		 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version),
> -		 (unsigned long long)be64_to_cpu(lguid),
> -		 (unsigned long long)be64_to_cpu(fguid));
> +	rdsdebug("saddr %u.%u.%u.%u daddr %u.%u.%u.%u RDSv%u.%u\n",
> +		 NIPQUAD(dp->dp_saddr), NIPQUAD(dp->dp_daddr),
> +		 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version));
>  
>  	conn = rds_conn_create(dp->dp_daddr, dp->dp_saddr, &rds_ib_transport,
>  			       GFP_KERNEL);
> @@ -628,7 +624,7 @@ int rds_ib_conn_connect(struct rds_connection *conn)
>  
>  	dest.sin_family = AF_INET;
>  	dest.sin_addr.s_addr = (__force u32)conn->c_faddr;
> -	dest.sin_port = (__force u16)htons(RDS_PORT);
> +	dest.sin_port = (__force u16)htons(RDS_IB_PORT);
>  
>  	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
>  				(struct sockaddr *)&dest,
> @@ -813,7 +809,7 @@ int __init rds_ib_listen_init(void)
>  
>  	sin.sin_family = PF_INET,
>  	sin.sin_addr.s_addr = (__force u32)htonl(INADDR_ANY);
> -	sin.sin_port = (__force u16)htons(RDS_PORT);
> +	sin.sin_port = (__force u16)htons(RDS_IB_PORT);
>  
>  	/*
>  	 * XXX I bet this binds the cm_id to a device.  If we want to support
> @@ -833,7 +829,7 @@ int __init rds_ib_listen_init(void)
>  		goto out;
>  	}
>  
> -	rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT);
> +	rdsdebug("cm %p listening on port %u\n", cm_id, RDS_IB_PORT);
>  
>  	rds_ib_listen_id = cm_id;
>  	cm_id = NULL;
> diff --git a/net/rds/rds.h b/net/rds/rds.h
> index 03031e2..6c17a4d 100644
> --- a/net/rds/rds.h
> +++ b/net/rds/rds.h
> @@ -26,8 +26,8 @@
>   *
>   * port 18633 was the version that had ack frames on the wire.
>   */
> -#define RDS_PORT	18634
> -
> +#define RDS_TCP_PORT	18634
> +#define RDS_IB_PORT	18635
>  
>  #ifndef AF_RDS
>  #define AF_RDS          28      /* Reliable Datagram Socket     */
> diff --git a/net/rds/tcp_connect.c b/net/rds/tcp_connect.c
> index 0389a99..298e372 100644
> --- a/net/rds/tcp_connect.c
> +++ b/net/rds/tcp_connect.c
> @@ -96,7 +96,7 @@ int rds_tcp_conn_connect(struct rds_connection *conn)
>  
>  	dest.sin_family = AF_INET;
>  	dest.sin_addr.s_addr = (__force u32)conn->c_faddr;
> -	dest.sin_port = (__force u16)htons(RDS_PORT);
> +	dest.sin_port = (__force u16)htons(RDS_TCP_PORT);
>  
>  	/* 
>  	 * once we call connect() we can start getting callbacks and they
> diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c
> index caeacbe..50709b7 100644
> --- a/net/rds/tcp_listen.c
> +++ b/net/rds/tcp_listen.c
> @@ -159,7 +159,7 @@ int __init rds_tcp_listen_init(void)
>  
>  	sin.sin_family = PF_INET,
>  	sin.sin_addr.s_addr = (__force u32)htonl(INADDR_ANY);
> -	sin.sin_port = (__force u16)htons(RDS_PORT);
> +	sin.sin_port = (__force u16)htons(RDS_TCP_PORT);
>  
>  	ret = sock->ops->bind(sock, (struct sockaddr *)&sin, sizeof(sin));
>  	if (ret < 0)
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
> 
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
> 



-- 
Olaf Kirch  |  --- o --- Nous sommes du soleil we love when we play
okir at lst.de |    / | \   sol.dhoop.naytheet.ah kin.ir.samse.qurax



More information about the general mailing list