[openib-general] [RFC/PATCH] rdma/cma: use the ipoib broadcast group qkey

[Or Gerlitz]

Sean,

Please let me know what you think - my intention is to have the group
type effect only whether or not to set the rdmacm signature byte on the
mgid and as for the qkey, just make the ipoib broadcast group qkey being
used instread a qkey defined by the rdma cm. The patch is not completed
yet in the sense that the qkey associated with the rdma cm kernel id
should be exported to user space (on the client side it would be on the
addr resolve event flow and on the server side on the conn req event flow)
to be set by librdmacm into the user UD QP on the time rdma_create_qp is called.

change the kernel rdma cm use the ipoib broadcast group qkey instead a qkey of its own.

Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>

Index: rdma-dev/drivers/infiniband/core/cma.c
===================================================================
--- rdma-dev.orig/drivers/infiniband/core/cma.c	2007-01-21 12:11:16.000000000 +0200
+++ rdma-dev/drivers/infiniband/core/cma.c	2007-01-22 14:05:22.000000000 +0200
@@ -136,6 +136,7 @@ struct rdma_id_private {
 	u32			seq_num;
 	u32			qp_num;
 	u8			srq;
+	u32			qkey;
 };

 struct cma_multicast {
@@ -884,6 +885,21 @@ out:
 	return ret;
 }

+static int get_broadcast_group_qkey(struct rdma_id_private *id_priv)
+{
+	struct ib_sa_mcmember_rec rec;
+	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
+	int ret;
+
+	ib_addr_get_mgid(dev_addr, &rec.mgid);
+	ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
+				     &rec.mgid, &rec);
+	if (ret)
+		return -EINVAL;
+	id_priv->qkey = rec.qkey;
+	return 0;
+}
+
 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 					       struct ib_cm_event *ib_event)
 {
@@ -1021,6 +1037,10 @@ static int cma_req_handler(struct ib_cm_
 	if (ret)
 		goto release_conn_id;

+	ret = get_broadcast_group_qkey(conn_id);
+	if (ret)
+		goto release_conn_id;
+
 	conn_id->cm_id.ib = cm_id;
 	cm_id->context = conn_id;
 	cm_id->cm_handler = cma_ib_handler;
@@ -1600,6 +1620,7 @@ static void addr_handler(int status, str
 {
 	struct rdma_id_private *id_priv = context;
 	struct rdma_cm_event event;
+	int ret;

 	memset(&event, 0, sizeof event);
 	atomic_inc(&id_priv->dev_remove);
@@ -1626,6 +1647,11 @@ static void addr_handler(int status, str
 	} else {
 		memcpy(&id_priv->id.route.addr.src_addr, src_addr,
 		       ip_addr_size(src_addr));
+		ret = get_broadcast_group_qkey(id_priv);
+		if (ret) {
+			event.event = RDMA_CM_EVENT_ADDR_ERROR;
+			event.status = ret;
+		}
 		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
 	}

@@ -1936,7 +1962,9 @@ static int cma_sidr_rep_handler(struct i
 			event.status = ib_event->param.sidr_rep_rcvd.status;
 			break;
 		}
-		if (rep->qkey != RDMA_UD_QKEY) {
+		if (rep->qkey != id_priv->qkey) {
+			printk(KERN_WARNING "qkey mismatch %.8x client qkey %.8x\n",
+				rep->qkey, id_priv->qkey);
 			event.event = RDMA_CM_EVENT_UNREACHABLE;
 			event.status = -EINVAL;
 			break;
@@ -2231,7 +2259,7 @@ static int cma_send_sidr_rep(struct rdma
 	rep.status = status;
 	if (status == IB_SIDR_SUCCESS) {
 		rep.qp_num = id_priv->qp_num;
-		rep.qkey = RDMA_UD_QKEY;
+		rep.qkey = id_priv->qkey;
 	}
 	rep.private_data = private_data;
 	rep.private_data_len = private_data_len;