[openib-general] [RFC/PATCH v3] rdma/cma: add RDMA_PS_IPOIB port space

Or Gerlitz ogerlitz at voltaire.com
Tue Jan 23 07:38:07 PST 2007


Add to the RDMA CM an IPoIB port space (RDMA_PS_IPOIB) whose semantics are similar
to those of RDMA_PS_UDP where RDMA_PS_IPOIB IDs allow for inter operability with
IPoIB on some traffic patterns.

For RDMA_PS_UDP and RDMA_PS_IPOIB IDs, the qkey is stored in struct rdma_id_private
and delivered also in ADDR_RESOLVED and CONNECT_REQUEST events. The user space library
learns the qkey from these events and use them when it is called to create UD QP.

The IB UD qkey used by RDMA_PS_IPOIB IDs is that of the related ipoib broadcast group
where the qkey used by RDMA_PS_UDP IDs is hard defined "rdma cm qkey".

Creation of RDMA_PS_IPOIB IDs by proceeses is controlled by the linux kernel
capabilities subsystem.

Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>

Index: rdma-dev/drivers/infiniband/core/cma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/cma.c	2007-01-21 12:11:16.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/cma.c	2007-01-23 15:45:52.000000000 +0200
@@ -71,6 +71,7 @@ static struct workqueue_struct *cma_wq;
 static DEFINE_IDR(sdp_ps);
 static DEFINE_IDR(tcp_ps);
 static DEFINE_IDR(udp_ps);
+static DEFINE_IDR(ipoib_ps);

 struct cma_device {
 	struct list_head	list;
@@ -136,6 +137,7 @@ struct rdma_id_private {
 	u32			seq_num;
 	u32			qp_num;
 	u8			srq;
+	u32			qkey;
 };

 struct cma_multicast {
@@ -323,6 +325,10 @@ struct rdma_cm_id *rdma_create_id(rdma_c
 {
 	struct rdma_id_private *id_priv;

+	/* XXX - work around this till capabilities work fine for non root users */
+	if (ps == RDMA_PS_IPOIB && !capable(CAP_NET_BROADCAST))
+		return ERR_PTR(-EACCES);
+
 	id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 	if (!id_priv)
 		return ERR_PTR(-ENOMEM);
@@ -884,6 +890,31 @@ out:
 	return ret;
 }

+static int cma_set_qkey(struct rdma_id_private *id_priv, struct rdma_cm_event *event)
+{
+	struct ib_sa_mcmember_rec rec;
+	struct rdma_dev_addr *dev_addr;
+	int ret;
+
+	if (id_priv->id.ps == RDMA_PS_IPOIB) {
+		dev_addr = &id_priv->id.route.addr.dev_addr;
+		ib_addr_get_mgid(dev_addr, &rec.mgid);
+		ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
+					     &rec.mgid, &rec);
+		if (ret)
+			return -EINVAL;
+		id_priv->qkey = rec.qkey;
+		event->param.ud.qkey = rec.qkey;
+	}
+
+	if (id_priv->id.ps == RDMA_PS_UDP) {
+		id_priv->qkey = RDMA_UD_QKEY;
+		event->param.ud.qkey = RDMA_UD_QKEY;
+	}
+
+	return 0;
+}
+
 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 					       struct ib_cm_event *ib_event)
 {
@@ -999,7 +1030,7 @@ static int cma_req_handler(struct ib_cm_
 	memset(&event, 0, sizeof event);
 	offset = cma_user_data_offset(listen_id->id.ps);
 	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
-	if (listen_id->id.ps == RDMA_PS_UDP) {
+	if (listen_id->id.ps == RDMA_PS_UDP || listen_id->id.ps == RDMA_PS_IPOIB) {
 		conn_id = cma_new_udp_id(&listen_id->id, ib_event);
 		event.param.ud.private_data = ib_event->private_data + offset;
 		event.param.ud.private_data_len =
@@ -1020,7 +1051,11 @@ static int cma_req_handler(struct ib_cm_
 	mutex_unlock(&lock);
 	if (ret)
 		goto release_conn_id;
-
+
+	ret = cma_set_qkey(conn_id, &event);
+	if (ret)
+		goto release_conn_id;
+
 	conn_id->cm_id.ib = cm_id;
 	cm_id->context = conn_id;
 	cm_id->cm_handler = cma_ib_handler;
@@ -1600,6 +1635,7 @@ static void addr_handler(int status, str
 {
 	struct rdma_id_private *id_priv = context;
 	struct rdma_cm_event event;
+	int ret;

 	memset(&event, 0, sizeof event);
 	atomic_inc(&id_priv->dev_remove);
@@ -1627,6 +1663,11 @@ static void addr_handler(int status, str
 		memcpy(&id_priv->id.route.addr.src_addr, src_addr,
 		       ip_addr_size(src_addr));
 		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+		ret = cma_set_qkey(id_priv, &event);
+		if (ret) {
+			event.event = RDMA_CM_EVENT_ADDR_ERROR;
+			event.status = ret;
+		}
 	}

 	if (id_priv->id.event_handler(&id_priv->id, &event)) {
@@ -1822,6 +1863,9 @@ static int cma_get_port(struct rdma_id_p
 	case RDMA_PS_UDP:
 		ps = &udp_ps;
 		break;
+	case RDMA_PS_IPOIB:
+		ps = &ipoib_ps;
+		break;
 	default:
 		return -EPROTONOSUPPORT;
 	}
@@ -1936,7 +1980,9 @@ static int cma_sidr_rep_handler(struct i
 			event.status = ib_event->param.sidr_rep_rcvd.status;
 			break;
 		}
-		if (rep->qkey != RDMA_UD_QKEY) {
+		if (rep->qkey != id_priv->qkey) {
+			printk(KERN_WARNING "qkey mismatch %.8x client qkey %.8x\n",
+				rep->qkey, id_priv->qkey);
 			event.event = RDMA_CM_EVENT_UNREACHABLE;
 			event.status = -EINVAL;
 			break;
@@ -2135,7 +2181,7 @@ int rdma_connect(struct rdma_cm_id *id,

 	switch (rdma_node_get_transport(id->device->node_type)) {
 	case RDMA_TRANSPORT_IB:
-		if (id->ps == RDMA_PS_UDP)
+		if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
 			ret = cma_resolve_ib_udp(id_priv, conn_param);
 		else
 			ret = cma_connect_ib(id_priv, conn_param);
@@ -2231,7 +2277,7 @@ static int cma_send_sidr_rep(struct rdma
 	rep.status = status;
 	if (status == IB_SIDR_SUCCESS) {
 		rep.qp_num = id_priv->qp_num;
-		rep.qkey = RDMA_UD_QKEY;
+		rep.qkey = id_priv->qkey;
 	}
 	rep.private_data = private_data;
 	rep.private_data_len = private_data_len;
@@ -2255,7 +2301,7 @@ int rdma_accept(struct rdma_cm_id *id, s

 	switch (rdma_node_get_transport(id->device->node_type)) {
 	case RDMA_TRANSPORT_IB:
-		if (id->ps == RDMA_PS_UDP)
+		if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
 			ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
 						conn_param->private_data,
 						conn_param->private_data_len);
@@ -2316,7 +2362,7 @@ int rdma_reject(struct rdma_cm_id *id, c

 	switch (rdma_node_get_transport(id->device->node_type)) {
 	case RDMA_TRANSPORT_IB:
-		if (id->ps == RDMA_PS_UDP)
+		if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
 			ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
 						private_data, private_data_len);
 		else
Index: rdma-dev/include/rdma/rdma_cm.h
===================================================================
--- rdma-dev.orig/include/rdma/rdma_cm.h	2007-01-18 13:46:06.000000000 +0200
+++ rdma-dev/include/rdma/rdma_cm.h	2007-01-23 12:35:35.000000000 +0200
@@ -58,6 +58,7 @@ enum rdma_cm_event_type {

 enum rdma_port_space {
 	RDMA_PS_SDP  = 0x0001,
+	RDMA_PS_IPOIB = 0x0002,
 	RDMA_PS_TCP  = 0x0106,
 	RDMA_PS_UDP  = 0x0111,
 	RDMA_PS_SCTP = 0x0183
Index: rdma-dev/drivers/infiniband/core/ucma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/ucma.c	2007-01-18 13:46:06.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/ucma.c	2007-01-23 15:45:05.000000000 +0200
@@ -257,7 +257,7 @@ static int ucma_event_handler(struct rdm
 	ucma_set_event_context(ctx, event, uevent);
 	uevent->resp.event = event->event;
 	uevent->resp.status = event->status;
-	if (cm_id->ps == RDMA_PS_UDP)
+	if (cm_id->ps == RDMA_PS_UDP || cm_id->ps == RDMA_PS_IPOIB)
 		ucma_copy_ud_event(&uevent->resp.param.ud, &event->param.ud);
 	else
 		ucma_copy_conn_event(&uevent->resp.param.conn,





More information about the general mailing list