[openib-general] [RFC/PATCH v2] rdma/cma: use the ipoib broadcast group qkey

[Or Gerlitz]

Modify the kernel rdma cm use the ipoib broadcast group qkey instead a qkey of its
own for its UD IDs/QPs. For RDMA_PS_UDP ID, the qkey is stored in struct rdma_id_private
and delivered also in ADDR_RESOLVED and CONNECT_REQUEST events. The user space library
learns the qkey from these events and use them when it is called to create UD QP.

Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>

Index: rdma-dev/drivers/infiniband/core/cma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/cma.c	2007-01-21 12:11:16.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/cma.c	2007-01-22 21:52:30.000000000 +0200
@@ -136,6 +136,7 @@ struct rdma_id_private {
 	u32			seq_num;
 	u32			qp_num;
 	u8			srq;
+	u32			qkey;
 };

 struct cma_multicast {
@@ -884,6 +885,21 @@ out:
 	return ret;
 }

+static int get_broadcast_group_qkey(struct rdma_id_private *id_priv)
+{
+	struct ib_sa_mcmember_rec rec;
+	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
+	int ret;
+
+	ib_addr_get_mgid(dev_addr, &rec.mgid);
+	ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
+				     &rec.mgid, &rec);
+	if (ret)
+		return -EINVAL;
+	id_priv->qkey = rec.qkey;
+	return 0;
+}
+
 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 					       struct ib_cm_event *ib_event)
 {
@@ -1020,7 +1036,14 @@ static int cma_req_handler(struct ib_cm_
 	mutex_unlock(&lock);
 	if (ret)
 		goto release_conn_id;
-
+
+	if (conn_id->id.ps == RDMA_PS_UDP) {
+		ret = get_broadcast_group_qkey(conn_id);
+		if (ret)
+			goto release_conn_id;
+		event.param.ud.qkey = conn_id->qkey;
+	}
+
 	conn_id->cm_id.ib = cm_id;
 	cm_id->context = conn_id;
 	cm_id->cm_handler = cma_ib_handler;
@@ -1600,6 +1623,7 @@ static void addr_handler(int status, str
 {
 	struct rdma_id_private *id_priv = context;
 	struct rdma_cm_event event;
+	int ret;

 	memset(&event, 0, sizeof event);
 	atomic_inc(&id_priv->dev_remove);
@@ -1627,6 +1651,14 @@ static void addr_handler(int status, str
 		memcpy(&id_priv->id.route.addr.src_addr, src_addr,
 		       ip_addr_size(src_addr));
 		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+		if (id_priv->id.ps == RDMA_PS_UDP) {
+			ret = get_broadcast_group_qkey(id_priv);
+			if (ret) {
+				event.event = RDMA_CM_EVENT_ADDR_ERROR;
+				event.status = ret;
+			} else
+				event.param.ud.qkey = id_priv->qkey;
+		}
 	}

 	if (id_priv->id.event_handler(&id_priv->id, &event)) {
@@ -1936,7 +1968,9 @@ static int cma_sidr_rep_handler(struct i
 			event.status = ib_event->param.sidr_rep_rcvd.status;
 			break;
 		}
-		if (rep->qkey != RDMA_UD_QKEY) {
+		if (rep->qkey != id_priv->qkey) {
+			printk(KERN_WARNING "qkey mismatch %.8x client qkey %.8x\n",
+				rep->qkey, id_priv->qkey);
 			event.event = RDMA_CM_EVENT_UNREACHABLE;
 			event.status = -EINVAL;
 			break;
@@ -2231,7 +2265,7 @@ static int cma_send_sidr_rep(struct rdma
 	rep.status = status;
 	if (status == IB_SIDR_SUCCESS) {
 		rep.qp_num = id_priv->qp_num;
-		rep.qkey = RDMA_UD_QKEY;
+		rep.qkey = id_priv->qkey;
 	}
 	rep.private_data = private_data;
 	rep.private_data_len = private_data_len;
Index: rdma-dev/include/rdma/rdma_cm_ib.h
===================================================================
--- rdma-dev.orig/include/rdma/rdma_cm_ib.h	2007-01-18 13:43:37.000000000 +0200
+++ rdma-dev/include/rdma/rdma_cm_ib.h	2007-01-22 21:59:34.000000000 +0200
@@ -44,7 +44,4 @@
 int rdma_set_ib_paths(struct rdma_cm_id *id,
 		      struct ib_sa_path_rec *path_rec, int num_paths);

-/* Global qkey for UD QPs and multicast groups. */
-#define RDMA_UD_QKEY 0x01234567
-
 #endif /* RDMA_CM_IB_H */