[ofw] [PATCH 10/11] librdmacm: add rdma_server / rdma_client
Hefty, Sean
sean.hefty at intel.com
Mon Aug 16 16:24:02 PDT 2010
New test apps for synchronous operation and simplified connection
establishment.
From: Sean Hefty <sean.hefty at intel.com>
---
trunk/ulp/librdmacm/examples/dirs | 4
trunk/ulp/librdmacm/examples/rdma_client/SOURCES | 28 ++
trunk/ulp/librdmacm/examples/rdma_client/makefile | 7
.../librdmacm/examples/rdma_client/rdma_client.c | 134 +++++++++
trunk/ulp/librdmacm/examples/rdma_server/SOURCES | 28 ++
trunk/ulp/librdmacm/examples/rdma_server/makefile | 7
.../librdmacm/examples/rdma_server/rdma_server.c | 148 ++++++++++
trunk/ulp/librdmacm/include/rdma/rdma_verbs.h | 297 ++++++++++++++++++++
8 files changed, 652 insertions(+), 1 deletions(-)
create mode 100644 trunk/ulp/librdmacm/examples/rdma_client/SOURCES
create mode 100644 trunk/ulp/librdmacm/examples/rdma_client/makefile
create mode 100644 trunk/ulp/librdmacm/examples/rdma_client/rdma_client.c
create mode 100644 trunk/ulp/librdmacm/examples/rdma_server/SOURCES
create mode 100644 trunk/ulp/librdmacm/examples/rdma_server/makefile
create mode 100644 trunk/ulp/librdmacm/examples/rdma_server/rdma_server.c
create mode 100644 trunk/ulp/librdmacm/include/rdma/rdma_verbs.h
diff --git a/trunk/ulp/librdmacm/examples/dirs b/trunk/ulp/librdmacm/examples/dirs
index 2bdf906..498dd65 100644
--- a/trunk/ulp/librdmacm/examples/dirs
+++ b/trunk/ulp/librdmacm/examples/dirs
@@ -1,2 +1,4 @@
DIRS = \
- cmatose
+ cmatose \
+ rdma_server \
+ rdma_client
diff --git a/trunk/ulp/librdmacm/examples/rdma_client/SOURCES b/trunk/ulp/librdmacm/examples/rdma_client/SOURCES
new file mode 100644
index 0000000..3073041
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_client/SOURCES
@@ -0,0 +1,28 @@
+TARGETNAME = rdma_client
+TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)
+TARGETTYPE = PROGRAM
+
+UMTYPE = console
+UMENTRY = main
+
+USE_MSVCRT = 1
+
+SOURCES = rdma_client.c
+
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;\
+ ..\..\..\libibverbs\include;..\..\..\..\inc\user\linux;
+
+TARGETLIBS = \
+ $(SDK_LIB_PATH)\kernel32.lib \
+ $(SDK_LIB_PATH)\advapi32.lib \
+ $(SDK_LIB_PATH)\user32.lib \
+ $(SDK_LIB_PATH)\ole32.lib \
+ $(SDK_LIB_PATH)\ws2_32.lib \
+!if $(FREEBUILD)
+ $(TARGETPATH)\*\libibverbs.lib \
+ $(TARGETPATH)\*\librdmacm.lib
+!else
+ $(TARGETPATH)\*\libibverbsd.lib \
+ $(TARGETPATH)\*\librdmacmd.lib
+!endif
+
diff --git a/trunk/ulp/librdmacm/examples/rdma_client/makefile b/trunk/ulp/librdmacm/examples/rdma_client/makefile
new file mode 100644
index 0000000..a0c0627
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_client/makefile
@@ -0,0 +1,7 @@
+#
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source
+# file to this component. This file merely indirects to the real make file
+# that is shared by all the driver components of the OpenIB Windows project.
+#
+
+!INCLUDE ..\..\..\..\inc\openib.def
diff --git a/trunk/ulp/librdmacm/examples/rdma_client/rdma_client.c b/trunk/ulp/librdmacm/examples/rdma_client/rdma_client.c
new file mode 100644
index 0000000..e0a394e
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_client/rdma_client.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "..\..\..\..\etc\user\getopt.c"
+#include <rdma/rdma_verbs.h>
+#include <rdma/rdma_cma.h>
+
+static char *server = "127.0.0.1";
+static char *port = "7471";
+
+struct rdma_cm_id *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(server, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.qp_context = id;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_connect(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ return 0;
+}
+
+int __cdecl main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "s:p:")) != -1) {
+ switch (op) {
+ case 's':
+ server = optarg;
+ break;
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_client: start\n");
+ ret = run();
+ printf("rdma_client: end %d\n", ret);
+ return ret;
+}
diff --git a/trunk/ulp/librdmacm/examples/rdma_server/SOURCES b/trunk/ulp/librdmacm/examples/rdma_server/SOURCES
new file mode 100644
index 0000000..70396d0
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_server/SOURCES
@@ -0,0 +1,28 @@
+TARGETNAME = rdma_server
+TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)
+TARGETTYPE = PROGRAM
+
+UMTYPE = console
+UMENTRY = main
+
+USE_MSVCRT = 1
+
+SOURCES = rdma_server.c
+
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;\
+ ..\..\..\libibverbs\include;..\..\..\..\inc\user\linux;
+
+TARGETLIBS = \
+ $(SDK_LIB_PATH)\kernel32.lib \
+ $(SDK_LIB_PATH)\advapi32.lib \
+ $(SDK_LIB_PATH)\user32.lib \
+ $(SDK_LIB_PATH)\ole32.lib \
+ $(SDK_LIB_PATH)\ws2_32.lib \
+!if $(FREEBUILD)
+ $(TARGETPATH)\*\libibverbs.lib \
+ $(TARGETPATH)\*\librdmacm.lib
+!else
+ $(TARGETPATH)\*\libibverbsd.lib \
+ $(TARGETPATH)\*\librdmacmd.lib
+!endif
+
diff --git a/trunk/ulp/librdmacm/examples/rdma_server/makefile b/trunk/ulp/librdmacm/examples/rdma_server/makefile
new file mode 100644
index 0000000..a0c0627
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_server/makefile
@@ -0,0 +1,7 @@
+#
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source
+# file to this component. This file merely indirects to the real make file
+# that is shared by all the driver components of the OpenIB Windows project.
+#
+
+!INCLUDE ..\..\..\..\inc\openib.def
diff --git a/trunk/ulp/librdmacm/examples/rdma_server/rdma_server.c b/trunk/ulp/librdmacm/examples/rdma_server/rdma_server.c
new file mode 100644
index 0000000..8be1577
--- /dev/null
+++ b/trunk/ulp/librdmacm/examples/rdma_server/rdma_server.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "..\..\..\..\etc\user\getopt.c"
+#include <rdma/rdma_verbs.h>
+#include <rdma/rdma_cma.h>
+
+static char *port = "7471";
+
+struct rdma_cm_id *listen_id, *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_flags = RAI_PASSIVE;
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(NULL, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&listen_id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("rdma_listen %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_request(listen_id, &id);
+ if (ret) {
+ printf("rdma_get_request %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_accept(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_send_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_send_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ rdma_destroy_ep(listen_id);
+ return 0;
+}
+
+int __cdecl main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "p:")) != -1) {
+ switch (op) {
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_server: start\n");
+ ret = run();
+ printf("rdma_server: end %d\n", ret);
+ return ret;
+}
diff --git a/trunk/ulp/librdmacm/include/rdma/rdma_verbs.h b/trunk/ulp/librdmacm/include/rdma/rdma_verbs.h
new file mode 100644
index 0000000..feb01be
--- /dev/null
+++ b/trunk/ulp/librdmacm/include/rdma/rdma_verbs.h
@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(RDMA_VERBS_H)
+#define RDMA_VERBS_H
+
+#include <assert.h>
+#include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
+#include <errno.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static __inline int rdma_seterrno(int ret)
+{
+ if (ret) {
+ errno = ret;
+ ret = -1;
+ }
+ return ret;
+}
+
+/*
+ * Memory registration helpers.
+ */
+static __inline struct ibv_mr *
+rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
+}
+
+static __inline struct ibv_mr *
+rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_READ);
+}
+
+static __inline struct ibv_mr *
+rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE);
+}
+
+static __inline int
+rdma_dereg_mr(struct ibv_mr *mr)
+{
+ return rdma_seterrno(ibv_dereg_mr(mr));
+}
+
+
+/*
+ * Vectored send, receive, and RDMA operations.
+ * Support multiple scatter-gather entries.
+ */
+static __inline int
+rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge)
+{
+ struct ibv_recv_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+
+ return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_READ;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_WRITE;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+/*
+ * Simple send, receive, and RDMA calls.
+ */
+static __inline int
+rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr)
+{
+ struct ibv_sge sge;
+
+ assert((addr >= mr->addr) &&
+ (((uint8_t) addr + length) <= ((uint8_t) mr->addr + mr->length)));
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_recvv(id, context, &sge, 1);
+}
+
+static __inline int
+rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_sendv(id, context, &sge, 1, flags);
+}
+
+static __inline int
+rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static __inline int
+rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static __inline int
+rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ struct ibv_ah *ah, uint32_t remote_qpn)
+{
+ struct ibv_send_wr wr, *bad;
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+ wr.wr.ud.ah = ah;
+ wr.wr.ud.remote_qpn = remote_qpn;
+ wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+// Comment out until patch to automatically create CQs
+static __inline int
+rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ goto out;
+
+ ret = ibv_req_notify_cq(id->send_cq, 0);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ while (!(ret = ibv_poll_cq(id->send_cq, 1, wc))) {
+ ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ assert(cq == id->send_cq && context == id);
+ ibv_ack_cq_events(id->send_cq, 1);
+ }
+out:
+ return (ret < 0) ? rdma_seterrno(ret) : ret;
+}
+
+static __inline int
+rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ goto out;
+
+ ret = ibv_req_notify_cq(id->recv_cq, 0);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ while (!(ret = ibv_poll_cq(id->recv_cq, 1, wc))) {
+ ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ assert(cq == id->recv_cq && context == id);
+ ibv_ack_cq_events(id->recv_cq, 1);
+ }
+out:
+ return (ret < 0) ? rdma_seterrno(ret) : ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDMA_CMA_H */
More information about the ofw
mailing list