[ofa-general] RE: tcp/rdma port unification patch in librdmacm

[Or Gerlitz]

Sean Hefty wrote:
> I'm undecided on the best way to expose this.  Always enabled?  Enabled during
> the build?  Enabled through a new API?  Other?

Hi Sean,
 
Experimenting with the kernel patch (below) applied on Linus tree, such that the rdma_cm module is loaded with the param being ON and running rping, I didn't see any extra socket listed in the netstat -natp output. So to start with (I tend to think it was explained in the past... sorry for asking again) - is it correct that the approach taken by this patch is to consume only a --port-- from the tcp port space but not to have actually a visible socket? is this approach applicable to user space?

As for enabling this always on librdmacm, this can't work - since it will break IB user space app that attempts to open two listeners: TCP one on port X and IB one on SID(PS_TCP, X) - for example for X=3260 and the app being the Linux stgt target. Also Steve has mentioned earlier on this thread that kernel apps are safe from this problem - I am not sure this is the case for Lustre.

Or.


RDMA/CMA: Allocate PS_TCP ports from the host TCP port space.

This is needed for iwarp providers that support native and rdma
connections over the same interface.

Optionally turned on by the new rdma_cm unify_tcp_port_space module option.

Signed-off-by: Steve Wise <swise at opengridcomputing.com>
Signed-off-by: Faisal Latif <faisal.latif at intel.com>
Signed-off-by: Chien Tung <chien.tin.tung at intel.com>
---

 drivers/infiniband/core/cma.c |   43 ++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 42 insertions(+), 1 deletions(-)

Index: linus-linux-2.6/drivers/infiniband/core/cma.c
===================================================================
--- linus-linux-2.6.orig/drivers/infiniband/core/cma.c
+++ linus-linux-2.6/drivers/infiniband/core/cma.c
@@ -55,6 +55,11 @@ MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
 MODULE_LICENSE("Dual BSD/GPL");
 
+int unify_tcp_port_space = 0;
+module_param(unify_tcp_port_space, int, 0644);
+MODULE_PARM_DESC(unify_tcp_port_space, "Unify the host TCP and RDMA port "
+		 "space allocation (default=0)");
+
 #define CMA_CM_RESPONSE_TIMEOUT 20
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
@@ -118,6 +123,7 @@ struct rdma_id_private {
 	struct rdma_cm_id	id;
 
 	struct rdma_bind_list	*bind_list;
+	struct socket		*sock;
 	struct hlist_node	node;
 	struct list_head	list; /* listen_any_list or cma_device.list */
 	struct list_head	listen_list; /* per device listens */
@@ -813,6 +819,8 @@ static void cma_release_port(struct rdma
 		kfree(bind_list);
 	}
 	mutex_unlock(&lock);
+	if (id_priv->sock)
+		sock_release(id_priv->sock);
 }
 
 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
@@ -2042,6 +2050,34 @@ static int cma_use_port(struct idr *ps, 
 	return 0;
 }
 
+static int cma_get_tcp_port(struct rdma_id_private *id_priv)
+{
+	int ret;
+	int size;
+	struct socket *sock;
+
+	ret = sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &sock);
+	if (ret)
+		return ret;
+	ret = sock->ops->bind(sock,
+			(struct sockaddr *) &id_priv->id.route.addr.src_addr,
+			ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
+	if (ret) {
+		sock_release(sock);
+		return ret;
+	}
+	size = ip_addr_size(&id_priv->id.route.addr.src_addr);
+	ret = sock->ops->getname(sock,
+			(struct sockaddr *) &id_priv->id.route.addr.src_addr,
+			&size, 0);
+	if (ret) {
+		sock_release(sock);
+		return ret;
+	}
+	id_priv->sock = sock;
+	return 0;
+}
+
 static int cma_get_port(struct rdma_id_private *id_priv)
 {
 	struct idr *ps;
@@ -2053,6 +2089,11 @@ static int cma_get_port(struct rdma_id_p
 		break;
 	case RDMA_PS_TCP:
 		ps = &tcp_ps;
+		if (unify_tcp_port_space) {
+			ret = cma_get_tcp_port(id_priv);
+			if (ret)
+				goto out;
+		}
 		break;
 	case RDMA_PS_UDP:
 		ps = &udp_ps;
@@ -2070,7 +2111,7 @@ static int cma_get_port(struct rdma_id_p
 	else
 		ret = cma_use_port(ps, id_priv);
 	mutex_unlock(&lock);
-
+out:
 	return ret;
 }