[openib-general] [PATCH] librdmacm: update userspace to match kernel latest kernel ABI
Sean Hefty
sean.hefty at intel.com
Thu Nov 30 17:14:08 PST 2006
This updates the librdmacm to ABI 3, which reworks event reporting to
include connect data, and adds UD QP support.
This also updates the sample programs.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
diff --git a/Makefile.am b/Makefile.am
index 8fc24a2..6a1f579 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,25 +18,24 @@ endif
src_librdmacm_la_SOURCES = src/cma.c
src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script)
-bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey
+bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy
+#examples/mckey
examples_ucmatose_SOURCES = examples/cmatose.c
examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
examples_rping_SOURCES = examples/rping.c
examples_rping_LDADD = $(top_builddir)/src/librdmacm.la
examples_udaddy_SOURCES = examples/udaddy.c
examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
-examples_mckey_SOURCES = examples/mckey.c
-examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
+#examples_mckey_SOURCES = examples/mckey.c
+#examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
librdmacmincludedir = $(includedir)/rdma
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
- include/rdma/rdma_cma.h \
- include/rdma/rdma_cma_ib.h
+ include/rdma/rdma_cma.h
EXTRA_DIST = include/rdma/rdma_cma_abi.h \
include/rdma/rdma_cma.h \
- include/rdma/rdma_cma_ib.h \
src/librdmacm.map \
librdmacm.spec.in
diff --git a/examples/cmatose.c b/examples/cmatose.c
index ca46402..4479fd4 100644
--- a/examples/cmatose.c
+++ b/examples/cmatose.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,6 +41,7 @@
#include <sys/socket.h>
#include <netdb.h>
#include <byteswap.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
@@ -52,12 +53,6 @@ static inline uint64_t cpu_to_be64(uint6
static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
#endif
-/*
- * To execute:
- * Server: rdma_cmatose
- * Client: rdma_cmatose <dst_ip>
- */
-
struct cmatest_node {
int id;
struct rdma_cm_id *cma_id;
@@ -85,7 +80,8 @@ static struct cmatest test;
static int connections = 1;
static int message_size = 100;
static int message_count = 10;
-static int is_server;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -377,7 +373,7 @@ static int alloc_nodes(void)
for (i = 0; i < connections; i++) {
test.nodes[i].id = i;
- if (!is_server) {
+ if (dst_addr) {
ret = rdma_create_id(test.channel,
&test.nodes[i].cma_id,
&test.nodes[i], RDMA_PS_TCP);
@@ -460,6 +456,28 @@ static int disconnect_events(void)
return ret;
}
+static int get_addr(char *dst, struct sockaddr_in *addr)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ *addr = *(struct sockaddr_in *) res->ai_addr;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
static int run_server(void)
{
struct rdma_cm_id *listen_id;
@@ -472,12 +490,18 @@ static int run_server(void)
return ret;
}
- test.src_in.sin_family = PF_INET;
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
+ if (ret)
+ goto out;
+ } else
+ test.src_in.sin_family = PF_INET;
+
test.src_in.sin_port = 7471;
ret = rdma_bind_addr(listen_id, test.src_addr);
if (ret) {
printf("cmatose: bind address failed: %d\n", ret);
- return ret;
+ goto out;
}
ret = rdma_listen(listen_id, 0);
@@ -528,40 +552,18 @@ out:
return ret;
}
-static int get_addr(char *dst, struct sockaddr_in *addr)
-{
- struct addrinfo *res;
- int ret;
-
- ret = getaddrinfo(dst, NULL, NULL, &res);
- if (ret) {
- printf("getaddrinfo failed - invalid hostname or IP address\n");
- return ret;
- }
-
- if (res->ai_family != PF_INET) {
- ret = -1;
- goto out;
- }
-
- *addr = *(struct sockaddr_in *) res->ai_addr;
-out:
- freeaddrinfo(res);
- return ret;
-}
-
-static int run_client(char *dst, char *src)
+static int run_client(void)
{
int i, ret, ret2;
printf("cmatose: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -570,7 +572,7 @@ static int run_client(char *dst, char *s
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("cmatose: failure getting addr: %d\n", ret);
@@ -597,7 +599,6 @@ static int run_client(char *dst, char *s
}
printf("data transfers complete\n");
-
}
ret = 0;
@@ -611,13 +612,35 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc > 3) {
- printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
+ switch (op) {
+ case 's':
+ dst_addr = optarg;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_server = (argc == 1);
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -633,10 +656,10 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- if (is_server)
- ret = run_server();
+ if (dst_addr)
+ ret = run_client();
else
- ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
+ ret = run_server();
printf("test complete\n");
destroy_nodes();
diff --git a/examples/mckey.c b/examples/mckey.c
index 96250de..31dbac6 100644
--- a/examples/mckey.c
+++ b/examples/mckey.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -42,9 +42,9 @@
#include <netdb.h>
#include <byteswap.h>
#include <unistd.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
-#include <rdma/rdma_cma_ib.h>
struct cmatest_node {
int id;
@@ -76,6 +76,8 @@ static int connections = 1;
static int message_size = 100;
static int message_count = 10;
static int is_sender;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -239,19 +241,12 @@ err:
return ret;
}
-static int join_handler(struct cmatest_node *node)
+static int join_handler(struct cmatest_node *node,
+ struct rdma_ud_param *param)
{
- struct ibv_ah_attr ah_attr;
- int ret;
-
- ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
- &node->remote_qpn, &node->remote_qkey);
- if (ret) {
- printf("mckey: failure getting destination attributes\n");
- goto err;
- }
-
- node->ah = ibv_create_ah(node->pd, &ah_attr);
+ node->remote_qpn = param->qp_num;
+ node->remote_qkey = param->qkey;
+ node->ah = ibv_create_ah(node->pd, ¶m->ah_attr);
if (!node->ah) {
printf("mckey: failure creating address handle\n");
goto err;
@@ -262,7 +257,7 @@ static int join_handler(struct cmatest_n
return 0;
err:
connect_error();
- return ret;
+ return -1;
}
static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
@@ -274,7 +269,7 @@ static int cma_handler(struct rdma_cm_id
ret = addr_handler(cma_id->context);
break;
case RDMA_CM_EVENT_MULTICAST_JOIN:
- ret = join_handler(cma_id->context);
+ ret = join_handler(cma_id->context, &event->param.ud);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -411,18 +406,21 @@ out:
return ret;
}
-static int run(char *dst, char *src)
+static int run(void)
{
int i, ret;
- printf("mckey: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (is_sender)
+ printf("mckey: starting client\n");
+ else
+ printf("mckey: starting server\n");
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -431,7 +429,7 @@ static int run(char *dst, char *src)
printf("mckey: joining\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("mckey: failure getting addr: %d\n", ret);
@@ -472,14 +470,39 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc < 3 || argc > 4) {
- printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n",
- argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
+ switch (op) {
+ case 'm':
+ dst_addr = optarg;
+ break;
+ case 's':
+ is_sender = 1;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t-m multicast_address\n");
+ printf("\t[-s(ender)]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_sender = (argv[1][0] == 's');
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -494,7 +517,7 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- ret = run(argv[2], (argc == 4) ? argv[3] : NULL);
+ ret = run();
printf("test complete\n");
destroy_nodes();
diff --git a/examples/udaddy.c b/examples/udaddy.c
index f6896c2..ab9ace6 100644
--- a/examples/udaddy.c
+++ b/examples/udaddy.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,15 +41,9 @@
#include <sys/socket.h>
#include <netdb.h>
#include <byteswap.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
-#include <rdma/rdma_cma_ib.h>
-
-/*
- * To execute:
- * Server: udaddy
- * Client: udaddy [server_addr [src_addr]]
- */
struct cmatest_node {
int id;
@@ -80,7 +74,8 @@ static struct cmatest test;
static int connections = 1;
static int message_size = 100;
static int message_count = 10;
-static int is_server;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -246,7 +241,6 @@ static int route_handler(struct cmatest_
memset(&conn_param, 0, sizeof conn_param);
conn_param.qp_num = node->cma_id->qp->qp_num;
- conn_param.qp_type = node->cma_id->qp->qp_type;
conn_param.retry_count = 5;
ret = rdma_connect(node->cma_id, &conn_param);
if (ret) {
@@ -284,7 +278,6 @@ static int connect_handler(struct rdma_c
memset(&conn_param, 0, sizeof conn_param);
conn_param.qp_num = node->cma_id->qp->qp_num;
- conn_param.qp_type = node->cma_id->qp->qp_type;
ret = rdma_accept(node->cma_id, &conn_param);
if (ret) {
printf("udaddy: failure accepting: %d\n", ret);
@@ -303,19 +296,12 @@ err1:
return ret;
}
-static int resolved_handler(struct cmatest_node *node)
+static int resolved_handler(struct cmatest_node *node,
+ struct rdma_cm_event *event)
{
- struct ibv_ah_attr ah_attr;
- int ret;
-
- ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
- &node->remote_qpn, &node->remote_qkey);
- if (ret) {
- printf("udaddy: failure getting destination attributes\n");
- goto err;
- }
-
- node->ah = ibv_create_ah(node->pd, &ah_attr);
+ node->remote_qpn = event->param.ud.qp_num;
+ node->remote_qkey = event->param.ud.qkey;
+ node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);
if (!node->ah) {
printf("udaddy: failure creating address handle\n");
goto err;
@@ -326,7 +312,7 @@ static int resolved_handler(struct cmate
return 0;
err:
connect_error();
- return ret;
+ return -1;
}
static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
@@ -344,7 +330,7 @@ static int cma_handler(struct rdma_cm_id
ret = connect_handler(cma_id);
break;
case RDMA_CM_EVENT_ESTABLISHED:
- ret = resolved_handler(cma_id->context);
+ ret = resolved_handler(cma_id->context, event);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -404,7 +390,7 @@ static int alloc_nodes(void)
for (i = 0; i < connections; i++) {
test.nodes[i].id = i;
- if (!is_server) {
+ if (dst_addr) {
ret = rdma_create_id(test.channel,
&test.nodes[i].cma_id,
&test.nodes[i], RDMA_PS_UDP);
@@ -475,6 +461,28 @@ static int connect_events(void)
return ret;
}
+static int get_addr(char *dst, struct sockaddr_in *addr)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ *addr = *(struct sockaddr_in *) res->ai_addr;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
static int run_server(void)
{
struct rdma_cm_id *listen_id;
@@ -487,7 +495,13 @@ static int run_server(void)
return ret;
}
- test.src_in.sin_family = PF_INET;
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
+ if (ret)
+ goto out;
+ } else
+ test.src_in.sin_family = PF_INET;
+
test.src_in.sin_port = 7174;
ret = rdma_bind_addr(listen_id, test.src_addr);
if (ret) {
@@ -526,40 +540,18 @@ out:
return ret;
}
-static int get_addr(char *dst, struct sockaddr_in *addr)
-{
- struct addrinfo *res;
- int ret;
-
- ret = getaddrinfo(dst, NULL, NULL, &res);
- if (ret) {
- printf("getaddrinfo failed - invalid hostname or IP address\n");
- return ret;
- }
-
- if (res->ai_family != PF_INET) {
- ret = -1;
- goto out;
- }
-
- *addr = *(struct sockaddr_in *) res->ai_addr;
-out:
- freeaddrinfo(res);
- return ret;
-}
-
-static int run_client(char *dst, char *src)
+static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -568,7 +560,7 @@ static int run_client(char *dst, char *s
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("udaddy: failure getting addr: %d\n", ret);
@@ -601,13 +593,35 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc > 3) {
- printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
+ switch (op) {
+ case 's':
+ dst_addr = optarg;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_server = (argc == 1);
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -622,10 +636,10 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- if (is_server)
- ret = run_server();
+ if (dst_addr)
+ ret = run_client();
else
- ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
+ ret = run_server();
printf("test complete\n");
destroy_nodes();
diff --git a/include/rdma/rdma_cma.h b/include/rdma/rdma_cma.h
old mode 100644
new mode 100755
index 6b20b12..711348a
--- a/include/rdma/rdma_cma.h
+++ b/include/rdma/rdma_cma.h
@@ -55,9 +55,7 @@ enum rdma_cm_event_type {
RDMA_CM_EVENT_REJECTED,
RDMA_CM_EVENT_ESTABLISHED,
RDMA_CM_EVENT_DISCONNECTED,
- RDMA_CM_EVENT_DEVICE_REMOVAL,
- RDMA_CM_EVENT_MULTICAST_JOIN,
- RDMA_CM_EVENT_MULTICAST_ERROR
+ RDMA_CM_EVENT_DEVICE_REMOVAL
};
enum rdma_port_space {
@@ -65,11 +63,11 @@ enum rdma_port_space {
RDMA_PS_UDP = 0x0111,
};
-/* Protocol levels for get/set options. */
-enum {
- RDMA_PROTO_IP = 0,
- RDMA_PROTO_IB = 1,
-};
+/*
+ * Global qkey value for all UD QPs and multicast groups created via the
+ * RDMA CM.
+ */
+#define RDMA_UD_QKEY 0x01234567
struct ib_addr {
union ibv_gid sgid;
@@ -78,8 +76,12 @@ struct ib_addr {
};
struct rdma_addr {
- struct sockaddr_in6 src_addr;
- struct sockaddr_in6 dst_addr;
+ struct sockaddr src_addr;
+ uint8_t src_pad[sizeof(struct sockaddr_storage) -
+ sizeof(struct sockaddr)];
+ struct sockaddr dst_addr;
+ uint8_t dst_pad[sizeof(struct sockaddr_storage) -
+ sizeof(struct sockaddr)];
union {
struct ib_addr ibaddr;
} addr;
@@ -105,11 +107,25 @@ struct rdma_cm_id {
uint8_t port_num;
};
-struct rdma_multicast_data {
- void *context;
- struct sockaddr addr;
- uint8_t pad[sizeof(struct sockaddr_in6) -
- sizeof(struct sockaddr)];
+struct rdma_conn_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ uint8_t responder_resources;
+ uint8_t initiator_depth;
+ uint8_t flow_control;
+ uint8_t retry_count; /* ignored when accepting */
+ uint8_t rnr_retry_count;
+ /* Fields below ignored if a QP is created on the rdma_cm_id. */
+ uint8_t srq;
+ uint32_t qp_num;
+};
+
+struct rdma_ud_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ struct ibv_ah_attr ah_attr;
+ uint32_t qp_num;
+ uint32_t qkey;
};
struct rdma_cm_event {
@@ -117,8 +133,10 @@ struct rdma_cm_event {
struct rdma_cm_id *listen_id;
enum rdma_cm_event_type event;
int status;
- void *private_data;
- uint8_t private_data_len;
+ union {
+ struct rdma_conn_param conn;
+ struct rdma_ud_param ud;
+ } param;
};
/**
@@ -210,20 +228,6 @@ int rdma_create_qp(struct rdma_cm_id *id
*/
void rdma_destroy_qp(struct rdma_cm_id *id);
-struct rdma_conn_param {
- const void *private_data;
- uint8_t private_data_len;
- uint8_t responder_resources;
- uint8_t initiator_depth;
- uint8_t flow_control;
- uint8_t retry_count; /* ignored when accepting */
- uint8_t rnr_retry_count;
- /* Fields below ignored if a QP is created on the rdma_cm_id. */
- uint8_t srq;
- uint32_t qp_num;
- enum ibv_qp_type qp_type;
-};
-
/**
* rdma_connect - Initiate an active connection request.
*
@@ -255,28 +259,25 @@ int rdma_reject(struct rdma_cm_id *id, c
uint8_t private_data_len);
/**
- * rdma_disconnect - This function disconnects the associated QP and
- * transitions it into the error state.
- */
-int rdma_disconnect(struct rdma_cm_id *id);
-
-/**
- * rdma_join_multicast - Join the multicast group specified by the given
- * address.
- * @id: Communication identifier associated with the request.
- * @addr: Multicast address identifying the group to join.
- * @context: User-defined context associated with the join request. The
- * context is returned to the user through the private_data field in
- * the rdma_cm_event.
+ * rdma_notify - Notifies the RDMA CM of an asynchronous event that has
+ * occurred on the connection.
+ * @id: Connection identifier to transition to established.
+ * @event: Asynchronous event.
+ *
+ * This routine should be invoked by users to notify the CM of relevant
+ * communication events. Events that should be reported to the CM and
+ * when to report them are:
+ *
+ * IB_EVENT_COMM_EST - Used when a message is received on a connected
+ * QP before an RTU has been received.
*/
-int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
- void *context);
+int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event);
/**
- * rdma_leave_multicast - Leave the multicast group specified by the given
- * address.
+ * rdma_disconnect - This function disconnects the associated QP and
+ * transitions it into the error state.
*/
-int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr);
+int rdma_disconnect(struct rdma_cm_id *id);
/**
* rdma_get_cm_event - Retrieves the next pending communications event,
@@ -302,40 +303,17 @@ int rdma_get_cm_event(struct rdma_event_
*/
int rdma_ack_cm_event(struct rdma_cm_event *event);
-/**
- * rdma_get_option - Retrieve options for an rdma_cm_id.
- * @id: Communication identifier to retrieve option for.
- * @level: Protocol level of the option to retrieve.
- * @optname: Name of the option to retrieve.
- * @optval: Buffer to receive the returned options.
- * @optlen: On input, the size of the %optval buffer. On output, the
- * size of the returned data.
- */
-int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t *optlen);
-
-/**
- * rdma_set_option - Set options for an rdma_cm_id.
- * @id: Communication identifier to set option for.
- * @level: Protocol level of the option to set.
- * @optname: Name of the option to set.
- * @optval: Reference to the option data.
- * @optlen: The size of the %optval buffer.
- */
-int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t optlen);
-
static inline uint16_t rdma_get_src_port(struct rdma_cm_id *id)
{
- return id->route.addr.src_addr.sin6_family == PF_INET6 ?
- id->route.addr.src_addr.sin6_port :
+ return id->route.addr.src_addr.sa_family == PF_INET6 ?
+ ((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port :
((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port;
}
static inline uint16_t rdma_get_dst_port(struct rdma_cm_id *id)
{
- return id->route.addr.dst_addr.sin6_family == PF_INET6 ?
- id->route.addr.dst_addr.sin6_port :
+ return id->route.addr.dst_addr.sa_family == PF_INET6 ?
+ ((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port :
((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port;
}
diff --git a/include/rdma/rdma_cma_abi.h b/include/rdma/rdma_cma_abi.h
index 45d5c37..cd2d0aa 100644
--- a/include/rdma/rdma_cma_abi.h
+++ b/include/rdma/rdma_cma_abi.h
@@ -33,14 +33,15 @@
#ifndef RDMA_CMA_ABI_H
#define RDMA_CMA_ABI_H
+#include <infiniband/kern-abi.h>
#include <infiniband/sa-kern-abi.h>
/*
* This file must be kept in sync with the kernel's version of rdma_user_cm.h
*/
-#define RDMA_USER_CM_MIN_ABI_VERSION 1
-#define RDMA_USER_CM_MAX_ABI_VERSION 2
+#define RDMA_USER_CM_MIN_ABI_VERSION 3
+#define RDMA_USER_CM_MAX_ABI_VERSION 3
#define RDMA_MAX_PRIVATE_DATA 256
@@ -60,9 +61,7 @@ enum {
UCMA_CMD_GET_EVENT,
UCMA_CMD_GET_OPTION,
UCMA_CMD_SET_OPTION,
- UCMA_CMD_GET_DST_ATTR,
- UCMA_CMD_JOIN_MCAST,
- UCMA_CMD_LEAVE_MCAST
+ UCMA_CMD_NOTIFY
};
struct ucma_abi_cmd_hdr {
@@ -71,11 +70,6 @@ struct ucma_abi_cmd_hdr {
__u16 out;
};
-struct ucma_abi_create_id_v1 {
- __u64 uid;
- __u64 response;
-};
-
struct ucma_abi_create_id {
__u64 uid;
__u64 response;
@@ -133,7 +127,7 @@ struct ucma_abi_query_route_resp {
struct ucma_abi_conn_param {
__u32 qp_num;
- __u32 qp_type;
+ __u32 reserved;
__u8 private_data[RDMA_MAX_PRIVATE_DATA];
__u8 private_data_len;
__u8 srq;
@@ -145,6 +139,15 @@ struct ucma_abi_conn_param {
__u8 valid;
};
+struct ucma_abi_ud_param {
+ __u32 qp_num;
+ __u32 qkey;
+ struct ibv_kern_ah_attr ah_attr;
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+ __u8 private_data_len;
+ __u8 reserved[7];
+};
+
struct ucma_abi_connect {
struct ucma_abi_conn_param conn_param;
__u32 id;
@@ -180,27 +183,9 @@ struct ucma_abi_init_qp_attr {
__u32 qp_state;
};
-struct ucma_abi_join_mcast {
- __u32 id;
- struct sockaddr_in6 addr;
- __u64 uid;
-};
-
-struct ucma_abi_leave_mcast {
- __u32 id;
- struct sockaddr_in6 addr;
-};
-
-struct ucma_abi_dst_attr_resp {
- __u32 remote_qpn;
- __u32 remote_qkey;
- struct ibv_kern_ah_attr ah_attr;
-};
-
-struct ucma_abi_get_dst_attr {
- __u64 response;
- struct sockaddr_in6 addr;
+struct ucma_abi_notify {
__u32 id;
+ __u32 event;
};
struct ucma_abi_get_event {
@@ -212,30 +197,10 @@ struct ucma_abi_event_resp {
__u32 id;
__u32 event;
__u32 status;
- __u8 private_data_len;
- __u8 reserved[3];
- __u8 private_data[RDMA_MAX_PRIVATE_DATA];
-};
-
-struct ucma_abi_get_option {
- __u64 response;
- __u64 optval;
- __u32 id;
- __u32 level;
- __u32 optname;
- __u32 optlen;
-};
-
-struct ucma_abi_get_option_resp {
- __u32 optlen;
-};
-
-struct ucma_abi_set_option {
- __u64 optval;
- __u32 id;
- __u32 level;
- __u32 optname;
- __u32 optlen;
+ union {
+ struct ucma_abi_conn_param conn;
+ struct ucma_abi_ud_param ud;
+ } param;
};
#endif /* RDMA_CMA_ABI_H */
diff --git a/include/rdma/rdma_cma_ib.h b/include/rdma/rdma_cma_ib.h
deleted file mode 100644
index 6c33c61..0000000
--- a/include/rdma/rdma_cma_ib.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2006 Intel Corporation. All rights reserved.
- *
- * This Software is licensed under one of the following licenses:
- *
- * 1) under the terms of the "Common Public License 1.0" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/cpl.php.
- *
- * 2) under the terms of the "The BSD License" a copy of which is
- * available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/bsd-license.php.
- *
- * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
- * copy of which is available from the Open Source Initiative, see
- * http://www.opensource.org/licenses/gpl-license.php.
- *
- * Licensee has the right to choose one of the above licenses.
- *
- * Redistributions of source code must retain the above copyright
- * notice and one of the license notices.
- *
- * Redistributions in binary form must reproduce both the above copyright
- * notice, one of the license notices in the documentation
- * and/or other materials provided with the distribution.
- *
- */
-
-#if !defined(RDMA_CMA_IB_H)
-#define RDMA_CMA_IB_H
-
-#include <rdma/rdma_cma.h>
-
-
-/* IB specific option names for get/set. */
-enum {
- IB_PATH_OPTIONS = 1, /* struct ibv_kern_path_rec */
- IB_CM_REQ_OPTIONS = 2 /* struct ib_cm_req_opt */
-};
-
-struct ib_cm_req_opt {
- uint8_t remote_cm_response_timeout;
- uint8_t local_cm_response_timeout;
- uint8_t max_cm_retries;
-};
-
-/**
- * rdma_get_dst_attr - Retrieve information about a UDP destination.
- * @id: Connection identifier associated with the request.
- * @addr: Address of remote destination to retrieve information about.
- * @ah_attr: Address handle attributes. A caller uses these attributes to
- * create an address handle when communicating with the destination.
- * @qpn: The remote QP number associated with the UDP address.
- * @qkey: The QKey of the remote QP.
- *
- * Users must have called rdma_connect() to resolve the destination information.
- */
-int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr,
- struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
- uint32_t *remote_qkey);
-
-/*
- * Global qkey value for all UD QPs and multicast groups created via the
- * RDMA CM.
- */
-#define RDMA_UD_QKEY 0x01234567
-
-#endif /* RDMA_CMA_IB_H */
diff --git a/src/cma.c b/src/cma.c
old mode 100644
new mode 100755
index 3ccfcb8..043dab8
--- a/src/cma.c
+++ b/src/cma.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -54,7 +54,6 @@
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
-#include <rdma/rdma_cma_ib.h>
#define PFX "librdmacm: "
@@ -118,6 +117,12 @@ struct cma_id_private {
uint32_t handle;
};
+struct cma_event {
+ struct rdma_cm_event event;
+ uint8_t private_data[RDMA_MAX_PRIVATE_DATA];
+ struct cma_id_private *id_priv;
+};
+
static struct cma_device *cma_dev_array;
static int cma_dev_cnt;
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER;
@@ -335,41 +340,6 @@ err: ucma_free_id(id_priv);
return NULL;
}
-static int ucma_create_id_v1(struct rdma_event_channel *channel,
- struct rdma_cm_id **id, void *context,
- enum rdma_port_space ps)
-{
- struct ucma_abi_create_id_resp *resp;
- struct ucma_abi_create_id_v1 *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- if (ps != RDMA_PS_TCP) {
- fprintf(stderr, "librdmacm: Kernel ABI does not support "
- "requested port space.\n");
- return -EPROTONOSUPPORT;
- }
-
- id_priv = ucma_alloc_id(channel, context, ps);
- if (!id_priv)
- return -ENOMEM;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
- cmd->uid = (uintptr_t) id_priv;
-
- ret = write(channel->fd, msg, size);
- if (ret != size)
- goto err;
-
- id_priv->handle = resp->id;
- *id = &id_priv->id;
- return 0;
-
-err: ucma_free_id(id_priv);
- return ret;
-}
-
int rdma_create_id(struct rdma_event_channel *channel,
struct rdma_cm_id **id, void *context,
enum rdma_port_space ps)
@@ -384,9 +354,6 @@ int rdma_create_id(struct rdma_event_cha
if (ret)
return ret;
- if (abi_ver == 1)
- return ucma_create_id_v1(channel, id, context, ps);
-
id_priv = ucma_alloc_id(channel, context, ps);
if (!id_priv)
return -ENOMEM;
@@ -492,9 +459,9 @@ static int ucma_query_route(struct rdma_
sizeof id->route.addr.addr.ibaddr.dgid);
id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
memcpy(&id->route.addr.src_addr, &resp->src_addr,
- sizeof id->route.addr.src_addr);
+ sizeof resp->src_addr);
memcpy(&id->route.addr.dst_addr, &resp->dst_addr,
- sizeof id->route.addr.dst_addr);
+ sizeof resp->dst_addr);
if (!id_priv->cma_dev && resp->node_guid) {
ret = ucma_get_device(id_priv, resp->node_guid);
@@ -696,7 +663,7 @@ static int ucma_init_ib_qp(struct cma_id
qp_attr.port_num = id_priv->id.port_num;
qp_attr.qp_state = IBV_QPS_INIT;
- qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
+ qp_attr.qp_access_flags = 0;
return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
IBV_QP_PKEY_INDEX | IBV_QP_PORT);
}
@@ -767,11 +734,9 @@ void rdma_destroy_qp(struct rdma_cm_id *
static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
struct rdma_conn_param *src,
- uint32_t qp_num,
- enum ibv_qp_type qp_type, uint8_t srq)
+ uint32_t qp_num, uint8_t srq)
{
dst->qp_num = qp_num;
- dst->qp_type = qp_type;
dst->srq = srq;
dst->responder_resources = src->responder_resources;
dst->initiator_depth = src->initiator_depth;
@@ -799,12 +764,11 @@ int rdma_connect(struct rdma_cm_id *id,
cmd->id = id_priv->handle;
if (id->qp)
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num, id->qp->qp_type,
+ id->qp->qp_num,
(id->qp->srq != NULL));
else
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
conn_param->qp_num,
- conn_param->qp_type,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -852,12 +816,11 @@ int rdma_accept(struct rdma_cm_id *id, s
cmd->uid = (uintptr_t) id_priv;
if (id->qp)
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num, id->qp->qp_type,
+ id->qp->qp_num,
(id->qp->srq != NULL));
else
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
conn_param->qp_num,
- conn_param->qp_type,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -894,6 +857,25 @@ int rdma_reject(struct rdma_cm_id *id, c
return 0;
}
+int rdma_notify(struct rdma_cm_id *id, enum ibv_event_type event)
+{
+ struct ucma_abi_notify *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_NOTIFY, size);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->event = event;
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
int rdma_disconnect(struct rdma_cm_id *id)
{
struct ucma_abi_disconnect *cmd;
@@ -925,80 +907,6 @@ int rdma_disconnect(struct rdma_cm_id *i
return 0;
}
-int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
- void *context)
-{
- struct ucma_abi_join_mcast *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size, addrlen;
-
- addrlen = ucma_addrlen(addr);
- if (!addrlen)
- return -EINVAL;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_JOIN_MCAST, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
- cmd->uid = (uintptr_t) context;
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- return 0;
-}
-
-int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
-{
- struct ucma_abi_leave_mcast *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size, addrlen;
- struct ibv_ah_attr ah_attr;
- uint32_t qp_info;
-
- addrlen = ucma_addrlen(addr);
- if (!addrlen)
- return -EINVAL;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LEAVE_MCAST, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
-
- if (id->qp) {
- ret = rdma_get_dst_attr(id, addr, &ah_attr, &qp_info, &qp_info);
- if (ret)
- goto out;
-
- ret = ibv_detach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
- if (ret)
- goto out;
- }
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- ret = (ret > 0) ? -ENODATA : ret;
-out:
- return ret;
-}
-
-static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
- struct ucma_abi_event_resp *src)
-{
- dst->event = src->event;
- dst->status = src->status;
- dst->private_data_len = src->private_data_len;
- if (src->private_data_len) {
- dst->private_data = dst + 1;
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- } else
- dst->private_data = NULL;
-}
-
static void ucma_complete_event(struct cma_id_private *id_priv)
{
pthread_mutex_lock(&id_priv->mut);
@@ -1009,36 +917,34 @@ static void ucma_complete_event(struct c
int rdma_ack_cm_event(struct rdma_cm_event *event)
{
- struct rdma_cm_id *id;
+ struct cma_event *evt;
if (!event)
return -EINVAL;
- id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
- event->listen_id : event->id;
+ evt = container_of(event, struct cma_event, event);
- ucma_complete_event(container_of(id, struct cma_id_private, id));
- free(event);
+ ucma_complete_event(evt->id_priv);
+ free(evt);
return 0;
}
-static int ucma_process_conn_req(struct rdma_cm_event *event,
+static int ucma_process_conn_req(struct cma_event *evt,
uint32_t handle)
{
- struct cma_id_private *listen_id_priv, *id_priv;
+ struct cma_id_private *id_priv;
int ret;
- listen_id_priv = container_of(event->id, struct cma_id_private, id);
- id_priv = ucma_alloc_id(event->id->channel, event->id->context,
- event->id->ps);
+ id_priv = ucma_alloc_id(evt->id_priv->id.channel,
+ evt->id_priv->id.context, evt->id_priv->id.ps);
if (!id_priv) {
- ucma_destroy_kern_id(event->id->channel->fd, handle);
+ ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
ret = -ENOMEM;
goto err;
}
- event->listen_id = event->id;
- event->id = &id_priv->id;
+ evt->event.listen_id = &evt->id_priv->id;
+ evt->event.id = &id_priv->id;
id_priv->handle = handle;
ret = ucma_query_route(&id_priv->id);
@@ -1049,7 +955,7 @@ static int ucma_process_conn_req(struct
return 0;
err:
- ucma_complete_event(listen_id_priv);
+ ucma_complete_event(evt->id_priv);
return ret;
}
@@ -1093,34 +999,42 @@ static int ucma_process_establish(struct
return ret;
}
-static void ucma_process_mcast(struct rdma_cm_id *id, struct rdma_cm_event *evt)
+static void ucma_copy_conn_event(struct cma_event *event,
+ struct ucma_abi_conn_param *src)
{
- struct ucma_abi_join_mcast kmc_data;
- struct rdma_multicast_data *mc_data;
- struct ibv_ah_attr ah_attr;
- uint32_t qp_info;
+ struct rdma_conn_param *dst = &event->event.param.conn;
- kmc_data = *(struct ucma_abi_join_mcast *) evt->private_data;
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = &event->private_data;
+ memcpy(&event->private_data, src->private_data,
+ src->private_data_len);
+ }
- mc_data = evt->private_data;
- mc_data->context = (void *) (uintptr_t) kmc_data.uid;
- memcpy(&mc_data->addr, &kmc_data.addr,
- ucma_addrlen((struct sockaddr *) &kmc_data.addr));
+ dst->responder_resources = src->responder_resources;
+ dst->initiator_depth = src->initiator_depth;
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+ dst->srq = src->srq;
+ dst->qp_num = src->qp_num;
+}
- if (evt->status || !id->qp)
- return;
+static void ucma_copy_ud_event(struct cma_event *event,
+ struct ucma_abi_ud_param *src)
+{
+ struct rdma_ud_param *dst = &event->event.param.ud;
- evt->status = rdma_get_dst_attr(id, &mc_data->addr, &ah_attr,
- &qp_info, &qp_info);
- if (evt->status)
- goto err;
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = &event->private_data;
+ memcpy(&event->private_data, src->private_data,
+ src->private_data_len);
+ }
- evt->status = ibv_attach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
- if (evt->status)
- goto err;
- return;
-err:
- evt->event = RDMA_CM_EVENT_MULTICAST_ERROR;
+ ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
+ dst->qp_num = src->qp_num;
+ dst->qkey = src->qkey;
}
int rdma_get_cm_event(struct rdma_event_channel *channel,
@@ -1128,8 +1042,7 @@ int rdma_get_cm_event(struct rdma_event_
{
struct ucma_abi_event_resp *resp;
struct ucma_abi_get_event *cmd;
- struct cma_id_private *id_priv;
- struct rdma_cm_event *evt;
+ struct cma_event *evt;
void *msg;
int ret, size;
@@ -1140,155 +1053,102 @@ int rdma_get_cm_event(struct rdma_event_
if (!event)
return -EINVAL;
- evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
+ evt = malloc(sizeof *evt);
if (!evt)
return -ENOMEM;
retry:
+ memset(evt, 0, sizeof *evt);
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
ret = write(channel->fd, msg, size);
if (ret != size) {
free(evt);
return (ret > 0) ? -ENODATA : ret;
}
-
- id_priv = (void *) (uintptr_t) resp->uid;
- evt->id = &id_priv->id;
- ucma_copy_event_from_kern(evt, resp);
- switch (evt->event) {
+ evt->event.event = resp->event;
+ switch (resp->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
- evt->status = ucma_query_route(&id_priv->id);
- if (evt->status)
- evt->event = RDMA_CM_EVENT_ADDR_ERROR;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
- evt->status = ucma_query_route(&id_priv->id);
- if (evt->status)
- evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->id.ps == RDMA_PS_TCP)
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ else
+ ucma_copy_ud_event(evt, &resp->param.ud);
+
ret = ucma_process_conn_req(evt, resp->id);
if (ret)
goto retry;
break;
case RDMA_CM_EVENT_CONNECT_RESPONSE:
- evt->status = ucma_process_conn_resp(id_priv);
- if (!evt->status)
- evt->event = RDMA_CM_EVENT_ESTABLISHED;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ evt->event.status = ucma_process_conn_resp(evt->id_priv);
+ if (!evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
else {
- evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
- id_priv->connect_error = 1;
+ evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
+ evt->id_priv->connect_error = 1;
}
break;
case RDMA_CM_EVENT_ESTABLISHED:
- if (id_priv->id.ps == RDMA_PS_UDP)
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ if (evt->id_priv->id.ps == RDMA_PS_UDP) {
+ ucma_copy_ud_event(evt, &resp->param.ud);
break;
+ }
- evt->status = ucma_process_establish(&id_priv->id);
- if (evt->status) {
- evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
- id_priv->connect_error = 1;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ evt->event.status = ucma_process_establish(&evt->id_priv->id);
+ if (evt->event.status) {
+ evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
+ evt->id_priv->connect_error = 1;
}
break;
case RDMA_CM_EVENT_REJECTED:
- if (id_priv->connect_error) {
- ucma_complete_event(id_priv);
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->connect_error) {
+ ucma_complete_event(evt->id_priv);
goto retry;
}
- ucma_modify_qp_err(evt->id);
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ ucma_modify_qp_err(evt->event.id);
break;
case RDMA_CM_EVENT_DISCONNECTED:
- if (id_priv->connect_error) {
- ucma_complete_event(id_priv);
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->connect_error) {
+ ucma_complete_event(evt->id_priv);
goto retry;
}
- break;
- case RDMA_CM_EVENT_MULTICAST_JOIN:
- case RDMA_CM_EVENT_MULTICAST_ERROR:
- ucma_process_mcast(&id_priv->id, evt);
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
break;
default:
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ if (evt->id_priv->id.ps == RDMA_PS_TCP)
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ else
+ ucma_copy_ud_event(evt, &resp->param.ud);
break;
}
- *event = evt;
- return 0;
-}
-
-int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t *optlen)
-{
- struct ucma_abi_get_option_resp *resp;
- struct ucma_abi_get_option *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_OPTION, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- cmd->optval = (uintptr_t) optval;
- cmd->level = level;
- cmd->optname = optname;
- cmd->optlen = *optlen;
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- *optlen = resp->optlen;
- return 0;
-}
-
-int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t optlen)
-{
- struct ucma_abi_set_option *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_SET_OPTION, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- cmd->optval = (uintptr_t) optval;
- cmd->level = level;
- cmd->optname = optname;
- cmd->optlen = optlen;
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- return 0;
-}
-
-int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr,
- struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
- uint32_t *remote_qkey)
-{
- struct ucma_abi_dst_attr_resp *resp;
- struct ucma_abi_get_dst_attr *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size, addrlen;
-
- addrlen = ucma_addrlen(addr);
- if (!addrlen)
- return -EINVAL;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_DST_ATTR, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- ibv_copy_ah_attr_from_kern(ah_attr, &resp->ah_attr);
- *remote_qpn = resp->remote_qpn;
- *remote_qkey = resp->remote_qkey;
+ *event = &evt->event;
return 0;
}
diff --git a/src/librdmacm.map b/src/librdmacm.map
index 52986ca..ed3f35a 100644
--- a/src/librdmacm.map
+++ b/src/librdmacm.map
@@ -1,4 +1,4 @@
-RDMACM_1.0 {
+RDMACM_2.0 {
global:
rdma_create_event_channel;
rdma_destroy_event_channel;
@@ -19,8 +19,6 @@ RDMACM_1.0 {
rdma_get_option;
rdma_set_option;
rdma_get_dst_attr;
- rdma_join_multicast;
- rdma_leave_multicast;
rdma_get_devices;
rdma_free_devices;
local: *;
More information about the general
mailing list