[openib-general] [PATCH 2/3] iWARP and RDMA CMA
Tom Tucker
tom at opengridcomputing.com
Wed Mar 8 07:55:53 PST 2006
This patch includes the modifications to the CMA, and uCMA needed to
support the new iWARP CM and includes updates to the following files:
core/cma.c
core/ucma.c
Signed-off-by: Tom Tucker <tom at opengridcomputing.com>
Index: cma.c
===================================================================
--- cma.c (revision 5632)
+++ cma.c (working copy)
@@ -42,6 +42,8 @@
#include <rdma/ib_cache.h>
#include <rdma/ib_cm.h>
#include <rdma/ib_local_sa.h>
+#include <rdma/iw_cm.h>
+#include <rdma/ib_sa.h>
MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("Generic RDMA CM Agent");
@@ -111,6 +113,7 @@
int query_id;
union {
struct ib_cm_id *ib;
+ struct iw_cm_id *iw;
} cm_id;
u32 seq_num;
@@ -230,13 +233,23 @@
id_priv->cma_dev = NULL;
}
-static int cma_acquire_ib_dev(struct rdma_id_private *id_priv)
+static int cma_acquire_dev(struct rdma_id_private *id_priv)
{
+ enum rdma_node_type dev_type = id_priv->id.route.addr.dev_addr.dev_type;
struct cma_device *cma_dev;
union ib_gid *gid;
int ret = -ENODEV;
- gid = ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+ switch (rdma_node_get_transport(dev_type)) {
+ case RDMA_TRANSPORT_IB:
+ gid = ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+ break;
+ case RDMA_TRANSPORT_IWARP:
+ gid = iw_addr_get_sgid(&id_priv->id.route.addr.dev_addr);
+ break;
+ default:
+ return -ENODEV;
+ }
mutex_lock(&lock);
list_for_each_entry(cma_dev, &dev_list, list) {
@@ -251,18 +264,6 @@
return ret;
}
-static int cma_acquire_dev(struct rdma_id_private *id_priv)
-{
- enum rdma_node_type dev_type = id_priv->id.route.addr.dev_addr.dev_type;
-
- switch (rdma_node_get_transport(dev_type)) {
- case RDMA_TRANSPORT_IB:
- return cma_acquire_ib_dev(id_priv);
- default:
- return -ENODEV;
- }
-}
-
static void cma_deref_id(struct rdma_id_private *id_priv)
{
if (atomic_dec_and_test(&id_priv->refcount))
@@ -320,6 +321,16 @@
IB_QP_PKEY_INDEX | IB_QP_PORT);
}
+static int cma_init_iw_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
+{
+ struct ib_qp_attr qp_attr;
+
+ qp_attr.qp_state = IB_QPS_INIT;
+ qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE;
+
+ return ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_ACCESS_FLAGS);
+}
+
int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
struct ib_qp_init_attr *qp_init_attr)
{
@@ -339,6 +350,9 @@
case RDMA_TRANSPORT_IB:
ret = cma_init_ib_qp(id_priv, qp);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = cma_init_iw_qp(id_priv, qp);
+ break;
default:
ret = -ENOSYS;
break;
@@ -431,6 +445,10 @@
if (qp_attr->qp_state == IB_QPS_RTR)
qp_attr->rq_psn = id_priv->seq_num;
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
+ qp_attr_mask);
+ break;
default:
ret = -ENOSYS;
break;
@@ -588,6 +606,10 @@
if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
ib_destroy_cm_id(id_priv->cm_id.ib);
break;
+ case RDMA_TRANSPORT_IWARP:
+ if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+ iw_destroy_cm_id(id_priv->cm_id.iw);
+ break;
default:
break;
}
@@ -651,6 +673,10 @@
if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
ib_destroy_cm_id(id_priv->cm_id.ib);
break;
+ case RDMA_TRANSPORT_IWARP:
+ if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+ iw_destroy_cm_id(id_priv->cm_id.iw);
+ break;
default:
break;
}
@@ -837,7 +863,7 @@
}
atomic_inc(&conn_id->dev_remove);
- ret = cma_acquire_ib_dev(conn_id);
+ ret = cma_acquire_dev(conn_id);
if (ret) {
ret = -ENODEV;
cma_release_remove(conn_id);
@@ -900,6 +926,118 @@
}
}
+static int cma_iw_handler(struct iw_cm_id* iw_id, struct iw_cm_event* event)
+{
+ struct rdma_id_private *id_priv = iw_id->context;
+ enum rdma_cm_event_type event_type = 0;
+ int ret = 0;
+
+ atomic_inc(&id_priv->dev_remove);
+
+ switch (event->event) {
+ case IW_CM_EVENT_LLP_DISCONNECT:
+ case IW_CM_EVENT_LLP_RESET:
+ case IW_CM_EVENT_LLP_TIMEOUT:
+ case IW_CM_EVENT_CLOSE:
+ event_type = RDMA_CM_EVENT_DISCONNECTED;
+ break;
+
+ case IW_CM_EVENT_CONNECT_REPLY: {
+ if (event->status)
+ event_type = RDMA_CM_EVENT_REJECTED;
+ else
+ event_type = RDMA_CM_EVENT_ESTABLISHED;
+ break;
+ }
+
+ case IW_CM_EVENT_ESTABLISHED:
+ event_type = RDMA_CM_EVENT_ESTABLISHED;
+ break;
+
+ default:
+ BUG_ON(1);
+ break;
+
+ }
+
+ ret = cma_notify_user(id_priv,
+ event_type,
+ event->status,
+ event->private_data,
+ event->private_data_len);
+ if (ret) {
+ /* Destroy the CM ID by returning a non-zero value. */
+ id_priv->cm_id.iw = NULL;
+ cma_exch(id_priv, CMA_DESTROYING);
+ cma_release_remove(id_priv);
+ rdma_destroy_id(&id_priv->id);
+ return ret;
+ }
+
+ cma_release_remove(id_priv);
+ return ret;
+}
+
+static int iw_conn_req_handler(struct iw_cm_id *cm_id,
+ struct iw_cm_event *iw_event)
+{
+ struct rdma_cm_id* new_cm_id;
+ struct rdma_id_private *listen_id, *conn_id;
+ struct sockaddr_in* sin;
+ int ret;
+
+ listen_id = cm_id->context;
+ atomic_inc(&listen_id->dev_remove);
+ if (!cma_comp(listen_id, CMA_LISTEN)) {
+ ret = -ECONNABORTED;
+ goto out;
+ }
+
+ /* Create a new RDMA id the new IW CM ID */
+ new_cm_id = rdma_create_id(listen_id->id.event_handler,
+ listen_id->id.context,
+ RDMA_PS_TCP);
+ if (!new_cm_id) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ conn_id = container_of(new_cm_id, struct rdma_id_private, id);
+ atomic_inc(&conn_id->dev_remove);
+ conn_id->state = CMA_CONNECT;
+
+ /* New connection inherits device and address from parent */
+ memcpy(&new_cm_id->route.addr.dev_addr,
+ &listen_id->id.route.addr.dev_addr,
+ sizeof(new_cm_id->route.addr.dev_addr));
+ /* New connection inherits device from parent */
+ cma_acquire_dev(conn_id);
+
+ conn_id->cm_id.iw = cm_id;
+ cm_id->context = conn_id;
+ cm_id->cm_handler = cma_iw_handler;
+
+ sin = (struct sockaddr_in*)&new_cm_id->route.addr.src_addr;
+ *sin = iw_event->local_addr;
+
+ sin = (struct sockaddr_in*)&new_cm_id->route.addr.dst_addr;
+ *sin = iw_event->remote_addr;
+
+ ret = cma_notify_user(conn_id, RDMA_CM_EVENT_CONNECT_REQUEST, 0,
+ iw_event->private_data,
+ iw_event->private_data_len);
+ if (ret) {
+ /* Destroy the CM ID by returning a non-zero value. */
+ conn_id->cm_id.iw = NULL;
+ cma_exch(conn_id, CMA_DESTROYING);
+ cma_release_remove(conn_id);
+ rdma_destroy_id(&conn_id->id);
+ }
+
+out:
+ cma_release_remove(listen_id);
+ return ret;
+}
+
static int cma_ib_listen(struct rdma_id_private *id_priv)
{
struct ib_cm_compare_data compare_data;
@@ -929,6 +1067,30 @@
return ret;
}
+static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
+{
+ int ret;
+ struct sockaddr_in* sin;
+
+ id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
+ iw_conn_req_handler,
+ id_priv);
+ if (IS_ERR(id_priv->cm_id.iw))
+ return PTR_ERR(id_priv->cm_id.iw);
+
+ sin = (struct sockaddr_in*)&id_priv->id.route.addr.src_addr;
+ id_priv->cm_id.iw->local_addr = *sin;
+
+ ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
+
+ if (ret) {
+ iw_destroy_cm_id(id_priv->cm_id.iw);
+ id_priv->cm_id.iw = NULL;
+ }
+
+ return ret;
+}
+
static int cma_duplicate_listen(struct rdma_id_private *id_priv)
{
struct rdma_id_private *cur_id_priv;
@@ -1014,6 +1176,9 @@
case RDMA_TRANSPORT_IB:
ret = cma_ib_listen(id_priv);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = cma_iw_listen(id_priv, backlog);
+ break;
default:
ret = -ENOSYS;
break;
@@ -1156,6 +1321,23 @@
return ret;
}
+static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
+{
+ struct cma_work *work;
+
+ work = kzalloc(sizeof *work, GFP_KERNEL);
+ if (!work)
+ return -ENOMEM;
+
+ work->id = id_priv;
+ INIT_WORK(&work->work, cma_work_handler, work);
+ work->old_state = CMA_ROUTE_QUERY;
+ work->new_state = CMA_ROUTE_RESOLVED;
+ work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+ queue_work(rdma_wq, &work->work);
+ return 0;
+}
+
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
struct rdma_id_private *id_priv;
@@ -1170,6 +1352,9 @@
case RDMA_TRANSPORT_IB:
ret = cma_resolve_ib_route(id_priv, timeout_ms);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = cma_resolve_iw_route(id_priv, timeout_ms);
+ break;
default:
ret = -ENOSYS;
break;
@@ -1315,6 +1500,7 @@
ret = rdma_resolve_ip(src_addr, dst_addr,
&id->route.addr.dev_addr,
timeout_ms, addr_handler, id_priv);
+
if (ret)
goto err;
@@ -1446,6 +1632,37 @@
return ret;
}
+static int cma_connect_iw(struct rdma_id_private *id_priv,
+ struct rdma_conn_param *conn_param)
+{
+ struct iw_cm_id* cm_id;
+ struct sockaddr_in* sin;
+ int ret;
+
+ cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
+ if (IS_ERR(cm_id)) {
+ ret = PTR_ERR(cm_id);
+ goto out;
+ }
+
+ id_priv->cm_id.iw = cm_id;
+
+ sin = (struct sockaddr_in*)&id_priv->id.route.addr.src_addr;
+ cm_id->local_addr = *sin;
+
+ sin = (struct sockaddr_in*)&id_priv->id.route.addr.dst_addr;
+ cm_id->remote_addr = *sin;
+
+ cm_id->qp = id_priv->id.qp;
+ cm_id->qp_num = id_priv->qp_num;
+
+ ret = iw_cm_connect(cm_id, conn_param->private_data,
+ conn_param->private_data_len);
+
+out:
+ return ret;
+}
+
int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
struct rdma_id_private *id_priv;
@@ -1465,6 +1682,9 @@
case RDMA_TRANSPORT_IB:
ret = cma_connect_ib(id_priv, conn_param);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = cma_connect_iw(id_priv, conn_param);
+ break;
default:
ret = -ENOSYS;
break;
@@ -1527,6 +1747,13 @@
else
ret = cma_rep_recv(id_priv);
break;
+ case RDMA_TRANSPORT_IWARP:
+ id_priv->cm_id.iw->qp = id_priv->id.qp;
+ id_priv->cm_id.iw->qp_num = id_priv->qp_num;
+
+ ret = iw_cm_accept(id_priv->cm_id.iw, conn_param->private_data,
+ conn_param->private_data_len);
+ break;
default:
ret = -ENOSYS;
break;
@@ -1559,6 +1786,10 @@
IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
private_data, private_data_len);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = iw_cm_reject(id_priv->cm_id.iw,
+ private_data, private_data_len);
+ break;
default:
ret = -ENOSYS;
break;
@@ -1586,6 +1817,9 @@
if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
break;
+ case RDMA_TRANSPORT_IWARP:
+ ret = iw_cm_disconnect(id_priv->cm_id.iw);
+ break;
default:
break;
}
Index: ucma.c
===================================================================
--- ucma.c (revision 5632)
+++ ucma.c (working copy)
@@ -450,6 +450,21 @@
}
}
+static void ucma_copy_iw_route(struct rdma_ucm_query_route_resp *resp,
+ struct rdma_route *route)
+{
+ struct rdma_dev_addr *dev_addr;
+
+ resp->num_paths = 0;
+ dev_addr = &route->addr.dev_addr;
+ memset(&resp->ib_route[0], 0, sizeof(resp->ib_route[0]));
+ memcpy(&resp->ib_route[0].dgid, iw_addr_get_dgid(dev_addr),
+ sizeof(union ib_gid));
+ memcpy(&resp->ib_route[0].sgid, iw_addr_get_sgid(dev_addr),
+ sizeof(union ib_gid));
+ resp->ib_route[0].pkey = 0;
+}
+
static ssize_t ucma_query_route(struct ucma_file *file,
const char __user *inbuf,
int in_len, int out_len)
@@ -488,6 +503,10 @@
switch (rdma_node_get_transport(ctx->cm_id->device->node_type)) {
case RDMA_TRANSPORT_IB:
ucma_copy_ib_route(&resp, &ctx->cm_id->route);
+ break;
+ case RDMA_TRANSPORT_IWARP:
+ ucma_copy_iw_route(&resp, &ctx->cm_id->route);
+ break;
default:
break;
}
More information about the general
mailing list