[openib-general] [RFC/PATCH] librdmacm: use the ipoib broadcast group qkey

[Or Gerlitz]

Modify librdmacm use a qkey for its UD IDs/QPs delivered to it by the
rdma cm kernel code instead the a hard coded RDMA_UD_QKEY. For RDMA_PS_UDP
ID, the qkey is provided by the kernel in ADDR_RESOLVED and CONNECT_REQUEST
events and is stored by the library in struct cma_id_private. Later
the library use the qkey when it is called to create a UD QP.

Signed-off-by: Or Gerlitz <ogerlitz at voltaire.com>

Index: librdmacm/src/cma.c
===================================================================
--- librdmacm.orig/src/cma.c	2007-01-22 21:21:37.000000000 +0200
+++ librdmacm/src/cma.c	2007-01-22 21:57:13.000000000 +0200
@@ -116,6 +116,7 @@ struct cma_id_private {
 	pthread_mutex_t	  mut;
 	uint32_t	  handle;
 	struct cma_multicast *mc_list;
+	uint32_t	  qkey;
 };

 struct cma_multicast {
@@ -687,7 +688,7 @@ static int ucma_init_ud_qp(struct cma_id

 	qp_attr.port_num = id_priv->id.port_num;
 	qp_attr.qp_state = IBV_QPS_INIT;
-	qp_attr.qkey = RDMA_UD_QKEY;
+	qp_attr.qkey = id_priv->qkey;
 	ret = ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX |
 					  IBV_QP_PORT | IBV_QP_QKEY);
 	if (ret)
@@ -1169,6 +1170,7 @@ int rdma_get_cm_event(struct rdma_event_
 	struct ucma_abi_get_event *cmd;
 	struct cma_event *evt;
 	void *msg;
+	struct cma_id_private *id_priv;
 	int ret, size;

 	ret = cma_dev_cnt ? 0 : ucma_init();
@@ -1199,6 +1201,9 @@ retry:
 		evt->event.status = ucma_query_route(&evt->id_priv->id);
 		if (evt->event.status)
 			evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
+		else if (evt->id_priv->id.ps == RDMA_PS_UDP) {
+                        evt->id_priv->qkey = resp->param.ud.qkey;
+		}
 		break;
 	case RDMA_CM_EVENT_ROUTE_RESOLVED:
 		evt->id_priv = (void *) (uintptr_t) resp->uid;
@@ -1211,12 +1216,16 @@ retry:
 		evt->id_priv = (void *) (uintptr_t) resp->uid;
 		if (evt->id_priv->id.ps == RDMA_PS_TCP)
 			ucma_copy_conn_event(evt, &resp->param.conn);
-		else
+		else
 			ucma_copy_ud_event(evt, &resp->param.ud);

 		ret = ucma_process_conn_req(evt, resp->id);
 		if (ret)
 			goto retry;
+		if (evt->id_priv->id.ps == RDMA_PS_UDP) {
+			id_priv = container_of(evt->event.id, struct cma_id_private, id);
+			id_priv->qkey = resp->param.ud.qkey;
+		}
 		break;
 	case RDMA_CM_EVENT_CONNECT_RESPONSE:
 		evt->id_priv = (void *) (uintptr_t) resp->uid;
Index: librdmacm/examples/udaddy.c
===================================================================
--- librdmacm.orig/examples/udaddy.c	2007-01-22 21:19:52.000000000 +0200
+++ librdmacm/examples/udaddy.c	2007-01-22 22:02:07.000000000 +0200
@@ -415,6 +415,13 @@ static void destroy_nodes(void)
 	free(test.nodes);
 }

+/*
+ * Global qkey value for all UD QPs and multicast groups created via the
+ * RDMA CM.
+ * XXX FIXME - enhance test to not assume a pre defined qkey
+ */
+#define RDMA_UD_QKEY 0x01234567
+
 static void create_reply_ah(struct cmatest_node *node, struct ibv_wc *wc)
 {
 	node->ah = ibv_create_ah_from_wc(node->pd, wc, node->mem,
Index: librdmacm/include/rdma/rdma_cma.h
===================================================================
--- librdmacm.orig/include/rdma/rdma_cma.h	2007-01-22 21:56:13.000000000 +0200
+++ librdmacm/include/rdma/rdma_cma.h	2007-01-22 21:56:32.000000000 +0200
@@ -65,12 +65,6 @@ enum rdma_port_space {
 	RDMA_PS_UDP  = 0x0111,
 };

-/*
- * Global qkey value for all UD QPs and multicast groups created via the
- * RDMA CM.
- */
-#define RDMA_UD_QKEY 0x01234567
-
 struct ib_addr {
 	union ibv_gid	sgid;
 	union ibv_gid	dgid;