[openib-general] [PATCH 2/3] CMA Changes and iWARP CM

Tom Tucker tom at opengridcomputing.com
Thu Mar 9 16:28:49 PST 2006


This patch includes the modifications to the CMA, and uCMA needed to
support the new iWARP CM and includes updates to the following files:
        core/cma.c 
        core/ucma.c

Signed-off-by: Tom Tucker <tom at opengridcomputing.com>

Index: infiniband/core/cma.c
===================================================================
--- infiniband/core/cma.c	(revision 5632)
+++ infiniband/core/cma.c	(working copy)
@@ -42,6 +42,7 @@
 #include <rdma/ib_cache.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_local_sa.h>
+#include <rdma/iw_cm.h>
 
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
@@ -111,6 +112,7 @@
 	int			query_id;
 	union {
 		struct ib_cm_id	*ib;
+		struct iw_cm_id	*iw;
 	} cm_id;
 
 	u32			seq_num;
@@ -230,13 +232,23 @@
 	id_priv->cma_dev = NULL;
 }
 
-static int cma_acquire_ib_dev(struct rdma_id_private *id_priv)
+static int cma_acquire_dev(struct rdma_id_private *id_priv)
 {
+	enum rdma_node_type dev_type = id_priv->id.route.addr.dev_addr.dev_type;
 	struct cma_device *cma_dev;
 	union ib_gid *gid;
 	int ret = -ENODEV;
 
-	gid = ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+	switch (rdma_node_get_transport(dev_type)) {
+	case RDMA_TRANSPORT_IB:
+		gid = ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+		break;
+	case RDMA_TRANSPORT_IWARP:
+		gid = iw_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+		break;
+	default:
+		return -ENODEV;
+	}
 
 	mutex_lock(&lock);
 	list_for_each_entry(cma_dev, &dev_list, list) {
@@ -251,18 +263,6 @@
 	return ret;
 }
 
-static int cma_acquire_dev(struct rdma_id_private *id_priv)
-{
-	enum rdma_node_type dev_type = id_priv->id.route.addr.dev_addr.dev_type;
-
-	switch (rdma_node_get_transport(dev_type)) {
-	case RDMA_TRANSPORT_IB:
-		return cma_acquire_ib_dev(id_priv);
-	default:
-		return -ENODEV;
-	}
-}
-
 static void cma_deref_id(struct rdma_id_private *id_priv)
 {
 	if (atomic_dec_and_test(&id_priv->refcount))
@@ -320,6 +320,16 @@
 					  IB_QP_PKEY_INDEX | IB_QP_PORT);
 }
 
+static int cma_init_iw_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
+{
+	struct ib_qp_attr qp_attr;
+
+	qp_attr.qp_state = IB_QPS_INIT;
+	qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE;
+
+	return ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_ACCESS_FLAGS);
+}
+
 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 		   struct ib_qp_init_attr *qp_init_attr)
 {
@@ -339,6 +349,9 @@
 	case RDMA_TRANSPORT_IB:
 		ret = cma_init_ib_qp(id_priv, qp);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_init_iw_qp(id_priv, qp);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -431,6 +444,10 @@
 		if (qp_attr->qp_state == IB_QPS_RTR)
 			qp_attr->rq_psn = id_priv->seq_num;
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
+					qp_attr_mask);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -588,6 +605,10 @@
 	 		if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 				ib_destroy_cm_id(id_priv->cm_id.ib);
 			break;
+		case RDMA_TRANSPORT_IWARP:
+	 		if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+				iw_destroy_cm_id(id_priv->cm_id.iw);
+			break;
 		default:
 			break;
 		}
@@ -651,6 +672,10 @@
 	 		if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 				ib_destroy_cm_id(id_priv->cm_id.ib);
 			break;
+		case RDMA_TRANSPORT_IWARP:
+	 		if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+				iw_destroy_cm_id(id_priv->cm_id.iw);
+			break;
 		default:
 			break;
 		}
@@ -837,7 +862,7 @@
 	}
 
 	atomic_inc(&conn_id->dev_remove);
-	ret = cma_acquire_ib_dev(conn_id);
+	ret = cma_acquire_dev(conn_id);
 	if (ret) {
 		ret = -ENODEV;
 		cma_release_remove(conn_id);
@@ -900,6 +925,113 @@
 	}
 }
 
+static int cma_iw_handler(struct iw_cm_id* iw_id, struct iw_cm_event* iw_event)
+{
+	struct rdma_id_private *id_priv = iw_id->context;
+	enum rdma_cm_event_type event = 0;
+	int ret = 0;
+
+	atomic_inc(&id_priv->dev_remove);
+
+	switch (iw_event->event) {
+	case IW_CM_EVENT_LLP_DISCONNECT:
+	case IW_CM_EVENT_LLP_RESET:
+	case IW_CM_EVENT_LLP_TIMEOUT:
+	case IW_CM_EVENT_CLOSE:
+		event = RDMA_CM_EVENT_DISCONNECTED;
+		break;
+	case IW_CM_EVENT_CONNECT_REPLY:
+		if (iw_event->status)
+			event = RDMA_CM_EVENT_REJECTED;
+		else
+			event = RDMA_CM_EVENT_ESTABLISHED;
+		break;
+	case IW_CM_EVENT_ESTABLISHED:
+		event = RDMA_CM_EVENT_ESTABLISHED;
+		break;
+	default:
+		BUG_ON(1);
+		break;
+	}	
+
+	ret = cma_notify_user(id_priv, event, iw_event->status, 
+			      iw_event->private_data, 
+			      iw_event->private_data_len);
+	if (ret) {
+		/* Destroy the CM ID by returning a non-zero value. */
+		id_priv->cm_id.iw = NULL;
+		cma_exch(id_priv, CMA_DESTROYING);
+		cma_release_remove(id_priv);
+		rdma_destroy_id(&id_priv->id);
+		return ret;
+	}
+
+	cma_release_remove(id_priv);
+	return ret;
+}
+
+static int iw_conn_req_handler(struct iw_cm_id *cm_id, 
+			       struct iw_cm_event *iw_event)
+{
+	struct rdma_cm_id* new_cm_id;
+	struct rdma_id_private *listen_id, *conn_id;
+	struct sockaddr_in* sin;
+	int ret;
+
+	listen_id = cm_id->context;
+	atomic_inc(&listen_id->dev_remove);
+	if (!cma_comp(listen_id, CMA_LISTEN)) {
+		ret = -ECONNABORTED;
+		goto out;
+	}
+
+	/* Create a new RDMA id the new IW CM ID */
+	new_cm_id = rdma_create_id(listen_id->id.event_handler, 
+				   listen_id->id.context,
+				   RDMA_PS_TCP);
+	if (!new_cm_id) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	conn_id = container_of(new_cm_id, struct rdma_id_private, id);
+	atomic_inc(&conn_id->dev_remove);
+	conn_id->state = CMA_CONNECT;
+
+	/* New connection inherits device and address from parent */
+	memcpy(&new_cm_id->route.addr.dev_addr,
+	       &listen_id->id.route.addr.dev_addr,
+	       sizeof(new_cm_id->route.addr.dev_addr));
+	/* New connection inherits device from parent */
+	ret = cma_acquire_dev(conn_id);
+	if (ret)
+		goto out;
+
+	conn_id->cm_id.iw = cm_id;
+	cm_id->context = conn_id;
+	cm_id->cm_handler = cma_iw_handler;
+
+	sin = (struct sockaddr_in*)&new_cm_id->route.addr.src_addr;
+	*sin = iw_event->local_addr;
+
+	sin = (struct sockaddr_in*)&new_cm_id->route.addr.dst_addr;
+	*sin = iw_event->remote_addr;
+
+	ret = cma_notify_user(conn_id, RDMA_CM_EVENT_CONNECT_REQUEST, 0,
+			      iw_event->private_data,
+			      iw_event->private_data_len);
+	if (ret) {
+		/* Destroy the CM ID by returning a non-zero value. */
+		conn_id->cm_id.iw = NULL;
+		cma_exch(conn_id, CMA_DESTROYING);
+		cma_release_remove(conn_id);
+		rdma_destroy_id(&conn_id->id);
+	}
+
+out:
+	cma_release_remove(listen_id);
+	return ret;
+}
+
 static int cma_ib_listen(struct rdma_id_private *id_priv)
 {
 	struct ib_cm_compare_data compare_data;
@@ -929,6 +1061,30 @@
 	return ret;
 }
 
+static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
+{
+	int ret;
+	struct sockaddr_in* sin;
+
+	id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device, 
+					    iw_conn_req_handler,
+					    id_priv);
+	if (IS_ERR(id_priv->cm_id.iw))
+		return PTR_ERR(id_priv->cm_id.iw);
+
+	sin = (struct sockaddr_in*)&id_priv->id.route.addr.src_addr;
+	id_priv->cm_id.iw->local_addr = *sin;
+
+	ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
+
+	if (ret) {
+		iw_destroy_cm_id(id_priv->cm_id.iw);
+		id_priv->cm_id.iw = NULL;
+	}
+
+	return ret;
+}
+
 static int cma_duplicate_listen(struct rdma_id_private *id_priv)
 {
 	struct rdma_id_private *cur_id_priv;
@@ -1014,6 +1170,9 @@
 		case RDMA_TRANSPORT_IB:
 			ret = cma_ib_listen(id_priv);
 			break;
+		case RDMA_TRANSPORT_IWARP:
+			ret = cma_iw_listen(id_priv, backlog);
+			break;
 		default:
 			ret = -ENOSYS;
 			break;
@@ -1156,6 +1315,23 @@
 	return ret;
 }
 
+static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
+{
+	struct cma_work *work;
+
+	work = kzalloc(sizeof *work, GFP_KERNEL);
+	if (!work)
+		return -ENOMEM;
+
+	work->id = id_priv;
+	INIT_WORK(&work->work, cma_work_handler, work);
+	work->old_state = CMA_ROUTE_QUERY;
+	work->new_state = CMA_ROUTE_RESOLVED;
+	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+	queue_work(rdma_wq, &work->work);
+	return 0;
+}
+
 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
 {
 	struct rdma_id_private *id_priv;
@@ -1170,6 +1346,9 @@
 	case RDMA_TRANSPORT_IB:
 		ret = cma_resolve_ib_route(id_priv, timeout_ms);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_resolve_iw_route(id_priv, timeout_ms);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1315,6 +1494,7 @@
 		ret = rdma_resolve_ip(src_addr, dst_addr,
 				      &id->route.addr.dev_addr,
 				      timeout_ms, addr_handler, id_priv);
+
 	if (ret)
 		goto err;
 
@@ -1446,6 +1626,37 @@
 	return ret;
 }
 
+static int cma_connect_iw(struct rdma_id_private *id_priv,
+			  struct rdma_conn_param *conn_param)
+{
+	struct iw_cm_id* cm_id;
+	struct sockaddr_in* sin;
+	int ret;
+
+	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
+	if (IS_ERR(cm_id)) {
+		ret = PTR_ERR(cm_id);
+		goto out;
+	}
+
+	id_priv->cm_id.iw = cm_id;
+
+	sin = (struct sockaddr_in*)&id_priv->id.route.addr.src_addr;
+	cm_id->local_addr = *sin;
+
+	sin = (struct sockaddr_in*)&id_priv->id.route.addr.dst_addr;
+	cm_id->remote_addr = *sin;
+
+	cm_id->qp = id_priv->id.qp;
+	cm_id->qp_num = id_priv->qp_num;
+
+	ret = iw_cm_connect(cm_id, conn_param->private_data, 
+			    conn_param->private_data_len);
+						
+out:
+	return ret;
+}
+
 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 {
 	struct rdma_id_private *id_priv;
@@ -1465,6 +1676,9 @@
 	case RDMA_TRANSPORT_IB:
 		ret = cma_connect_ib(id_priv, conn_param);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_connect_iw(id_priv, conn_param);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1527,6 +1741,13 @@
 		else
 			ret = cma_rep_recv(id_priv);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		id_priv->cm_id.iw->qp = id_priv->id.qp;
+		id_priv->cm_id.iw->qp_num = id_priv->qp_num;
+
+		ret = iw_cm_accept(id_priv->cm_id.iw, conn_param->private_data,
+				   conn_param->private_data_len);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1559,6 +1780,10 @@
 				     IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
 				     private_data, private_data_len);
 		break;
+	case RDMA_TRANSPORT_IWARP: 
+		ret = iw_cm_reject(id_priv->cm_id.iw, 
+				   private_data, private_data_len);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1586,6 +1811,9 @@
 		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
 			ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = iw_cm_disconnect(id_priv->cm_id.iw);
+		break;
 	default:
 		break;
 	}
Index: infiniband/core/ucma.c
===================================================================
--- infiniband/core/ucma.c	(revision 5632)
+++ infiniband/core/ucma.c	(working copy)
@@ -450,6 +450,20 @@
 	}
 }
 
+static void ucma_copy_iw_route(struct rdma_ucm_query_route_resp *resp,
+			       struct rdma_route *route)
+{
+	struct rdma_dev_addr *dev_addr;
+
+	resp->num_paths = 0;
+	dev_addr = &route->addr.dev_addr;
+	memset(&resp->ib_route[0], 0, sizeof(resp->ib_route[0]));
+	memcpy(&resp->ib_route[0].dgid, iw_addr_get_dgid(dev_addr),
+	       sizeof(union ib_gid));
+	memcpy(&resp->ib_route[0].sgid, iw_addr_get_sgid(dev_addr),
+	       sizeof(union ib_gid));
+}
+
 static ssize_t ucma_query_route(struct ucma_file *file,
 				const char __user *inbuf,
 				int in_len, int out_len)
@@ -488,6 +502,10 @@
 	switch (rdma_node_get_transport(ctx->cm_id->device->node_type)) {
 	case RDMA_TRANSPORT_IB:
 		ucma_copy_ib_route(&resp, &ctx->cm_id->route);
+		break;
+	case RDMA_TRANSPORT_IWARP:
+		ucma_copy_iw_route(&resp, &ctx->cm_id->route);
+		break;
 	default:
 		break;
 	}




More information about the general mailing list