[openib-general] [PATCH 2/2] librdmacm: add support to join IPOIB multicast groups

[Sean Hefty]

Add to the librdmacm an IPOIB port space that allows interoperability with
IPoIB multicast traffic.  Use of the RDMA_PS_IPOIB is limited to multicast
join/leave.

Rename the RDMA_UD_QKEY to RDMA_UDP_QKEY to signify that the qkey is only
used with the RDMA_PS_UDP port space.  Update mckey to allow testing the
RDMA_PS_IPOIB.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
I will commit the mckey changes separately, since that patch is from Or.

diff --git a/examples/mckey.c b/examples/mckey.c
index 39f77d7..d4f83b8 100644
--- a/examples/mckey.c
+++ b/examples/mckey.c
@@ -78,6 +78,7 @@ static int message_count = 10;
 static int is_sender;
 static char *dst_addr;
 static char *src_addr;
+static enum rdma_port_space port_space = RDMA_PS_UDP;
 
 static int create_message(struct cmatest_node *node)
 {
@@ -328,7 +329,7 @@ static int alloc_nodes(void)
 	for (i = 0; i < connections; i++) {
 		test.nodes[i].id = i;
 		ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,
-				     &test.nodes[i], RDMA_PS_UDP);
+				     &test.nodes[i], port_space);
 		if (ret)
 			goto err;
 	}
@@ -478,7 +479,7 @@ int main(int argc, char **argv)
 {
 	int op, ret;
 
-	while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
+	while ((op = getopt(argc, argv, "m:sb:c:C:S:p:")) != -1) {
 		switch (op) {
 		case 'm':
 			dst_addr = optarg;
@@ -498,6 +499,9 @@ int main(int argc, char **argv)
 		case 'S':
 			message_size = atoi(optarg);
 			break;
+		case 'p':
+			port_space = strtol(optarg, NULL, 0);
+			break;
 		default:
 			printf("usage: %s\n", argv[0]);
 			printf("\t-m multicast_address\n");
@@ -506,6 +510,8 @@ int main(int argc, char **argv)
 			printf("\t[-c connections]\n");
 			printf("\t[-C message_count]\n");
 			printf("\t[-S message_size]\n");
+			printf("\t[-p port_space - %#x for UDP (default), "
+			       "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
 			exit(1);
 		}
 	}
diff --git a/examples/udaddy.c b/examples/udaddy.c
index ab9ace6..153e39c 100644
--- a/examples/udaddy.c
+++ b/examples/udaddy.c
@@ -420,7 +420,7 @@ static void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc)
 	node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,
 					 node->cma_id->port_num);
 	node->remote_qpn = ntohl(wc->imm_data);
-	node->remote_qkey = RDMA_UD_QKEY;
+	node->remote_qkey = RDMA_UDP_QKEY;
 }
 
 static int poll_cqs(void)
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
index 88a25b2..e30f8cd 100644
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -61,15 +61,16 @@ enum rdma_cm_event_type {
 };
 
 enum rdma_port_space {
+	RDMA_PS_IPOIB= 0x0002,
 	RDMA_PS_TCP  = 0x0106,
 	RDMA_PS_UDP  = 0x0111,
 };
 
 /*
- * Global qkey value for all UD QPs and multicast groups created via the 
+ * Global qkey value for UDP QPs and multicast groups created via the 
  * RDMA CM.
  */
-#define RDMA_UD_QKEY 0x01234567
+#define RDMA_UDP_QKEY 0x01234567
 
 struct ib_addr {
 	union ibv_gid	sgid;
diff --git a/src/cma.c b/src/cma.c
index 7ab685b..6a0d076 100644
--- a/src/cma.c
+++ b/src/cma.c
@@ -698,7 +698,7 @@ static int ucma_init_ud_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
 
 	qp_attr.port_num = id_priv->id.port_num;
 	qp_attr.qp_state = IBV_QPS_INIT;
-	qp_attr.qkey = RDMA_UD_QKEY;
+	qp_attr.qkey = RDMA_UDP_QKEY;	/* Will override PS_IPOIB on join */
 	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
 					  IBV_QP_PORT | IBV_QP_QKEY);
 	if (ret)
@@ -729,7 +729,7 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
 	if (!qp)
 		return -ENOMEM;
 
-	if (id->ps == RDMA_PS_UDP)
+	if (id->ps == RDMA_PS_UDP || id->ps == RDMA_PS_IPOIB)
 		ret = ucma_init_ud_qp(id_priv, qp);
 	else
 		ret = ucma_init_ib_qp(id_priv, qp);
@@ -1136,14 +1136,25 @@ static int ucma_process_establish(struct rdma_cm_id *id)
 
 static int ucma_process_join(struct cma_event *evt)
 {
+	int ret;
+
 	evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
 	evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
 
-	if (evt->id_priv->id.qp)
-		return ibv_attach_mcast(evt->id_priv->id.qp,
-					&evt->mc->mgid, evt->mc->mlid);
-	else
+	if (!evt->id_priv->id.qp)
 		return 0;
+
+	if (evt->id_priv->id.ps == RDMA_PS_IPOIB) {
+		struct ibv_qp_attr qp_attr;
+
+		qp_attr.qkey = evt->event.param.ud.qkey;
+		ret = ibv_modify_qp(evt->id_priv->id.qp, &qp_attr, IBV_QP_QKEY);
+		if (ret)
+			return ret;
+	}
+
+	return ibv_attach_mcast(evt->id_priv->id.qp, &evt->mc->mgid,
+				evt->mc->mlid);
 }
 
 static void ucma_copy_conn_event(struct cma_event *event,