[openib-general] [PATCH v2][RDMA CM] IB mcast fix

[Steve Wise]

Version 2:
- Updated SIDR CMA code to use global qkey.
- Updated udaddy.c to use global qkey.
- Tested with cmatose, udaddy, mckey over mthca/IB. 

-----


Set the QKEY to a common global value for all UD QPs and multicast
groups created by the RDMA CM.

Signed-off-by: swise at opengridcomputing.com



Index: src/userspace/librdmacm/include/rdma/rdma_cma_ib.h
===================================================================
--- src/userspace/librdmacm/include/rdma/rdma_cma_ib.h	(revision 9004)
+++ src/userspace/librdmacm/include/rdma/rdma_cma_ib.h	(working copy)
@@ -59,4 +59,10 @@
 		      struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
 		      uint32_t *remote_qkey);
 
+/*
+ * Global qkey value for all UD QPs and multicast groups created via the 
+ * RDMA CM.
+ */
+#define RDMA_UD_QKEY 0x01234567
+
 #endif /* RDMA_CMA_IB_H */
Index: src/userspace/librdmacm/src/cma.c
===================================================================
--- src/userspace/librdmacm/src/cma.c	(revision 9004)
+++ src/userspace/librdmacm/src/cma.c	(working copy)
@@ -701,7 +701,7 @@
 
 	qp_attr.port_num = id_priv->id.port_num;
 	qp_attr.qp_state = IBV_QPS_INIT;
-	qp_attr.qkey = ntohs(rdma_get_src_port(&id_priv->id));
+	qp_attr.qkey = RDMA_UD_QKEY;
 	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
 					  IBV_QP_PORT | IBV_QP_QKEY);
 	if (ret)
Index: src/userspace/librdmacm/examples/udaddy.c
===================================================================
--- src/userspace/librdmacm/examples/udaddy.c	(revision 9004)
+++ src/userspace/librdmacm/examples/udaddy.c	(working copy)
@@ -434,7 +434,7 @@
 	node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,
 					 node->cma_id->port_num);
 	node->remote_qpn = ntohl(wc->imm_data);
-	node->remote_qkey = ntohs(rdma_get_dst_port(node->cma_id));
+	node->remote_qkey = RDMA_UD_QKEY;
 }
 
 static int poll_cqs(void)
Index: src/linux-kernel/infiniband/include/rdma/rdma_cm_ib.h
===================================================================
--- src/linux-kernel/infiniband/include/rdma/rdma_cm_ib.h	(revision 9004)
+++ src/linux-kernel/infiniband/include/rdma/rdma_cm_ib.h	(working copy)
@@ -82,4 +82,10 @@
  */
 int rdma_set_ib_req_info(struct rdma_cm_id *id, struct ib_cm_req_opt *info);
 
+/*
+ * Global qkey value for all UD QPs and multicast groups created via the 
+ * RDMA CM.
+ */
+#define RDMA_UD_QKEY 0x01234567
+
 #endif /* RDMA_CM_IB_H */
Index: src/linux-kernel/infiniband/core/cma.c
===================================================================
--- src/linux-kernel/infiniband/core/cma.c	(revision 9004)
+++ src/linux-kernel/infiniband/core/cma.c	(working copy)
@@ -1822,7 +1822,7 @@
 			break;
 		}
 		route = &id_priv->id.route;
-		if (rep->qkey != ntohs(cma_port(&route->addr.dst_addr))) {
+		if (rep->qkey != RDMA_UD_QKEY) {
 			event = RDMA_CM_EVENT_UNREACHABLE;
 			status = -EINVAL;
 			break;
@@ -2012,7 +2012,7 @@
 	rep.status = status;
 	if (status == IB_SIDR_SUCCESS) {
 		rep.qp_num = id_priv->qp_num;
-		rep.qkey = ntohs(cma_port(&id_priv->id.route.addr.src_addr));
+		rep.qkey = RDMA_UD_QKEY;
 	}
 
 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
@@ -2172,7 +2172,7 @@
 	ib_addr_get_sgid(dev_addr, &rec.port_gid);
 	rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
 	rec.join_state = 1;
-	rec.qkey = sin->sin_addr.s_addr;
+	rec.qkey = cpu_to_be32(RDMA_UD_QKEY);
 
 	comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
 		    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |