[openib-general] [PATCHv1] userspace CMA
Sean Hefty
sean.hefty at intel.com
Thu Nov 10 12:37:48 PST 2005
The following patch adds support for the userspace RDMA CMA.
More specifically:
- Adds common user/kernel data structures and copy routines in ib_uverbs
And libibverbs.
- Updates ib_ucm and libibcm to use common structures and copy routines.
- Adds a new kernel module, rdma_ucm, to support userspace CMA.
- Adds a userspace CMA library, librdmacm.
- Adds a userspace CMA test program, ucmatose.
- Updates the kernel CMA to support a userspace CMA library.
- Adds support for user's to transition QP states.
- Adds backlog parameter to rdma_listen API. (not used yet)
Please respond with any feedback. Thanks.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
Index: userspace/libibverbs/include/infiniband/sa.h
===================================================================
--- userspace/libibverbs/include/infiniband/sa.h (revision 0)
+++ userspace/libibverbs/include/infiniband/sa.h (revision 0)
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2004 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: sa.h 2616 2005-06-15 15:22:39Z halr $
+ */
+
+#ifndef IB_SA_H
+#define IB_SA_H
+
+#include <infiniband/verbs.h>
+
+enum ib_sa_rate {
+ IB_SA_RATE_2_5_GBPS = 2,
+ IB_SA_RATE_5_GBPS = 5,
+ IB_SA_RATE_10_GBPS = 3,
+ IB_SA_RATE_20_GBPS = 6,
+ IB_SA_RATE_30_GBPS = 4,
+ IB_SA_RATE_40_GBPS = 7,
+ IB_SA_RATE_60_GBPS = 8,
+ IB_SA_RATE_80_GBPS = 9,
+ IB_SA_RATE_120_GBPS = 10
+};
+
+static inline int ib_sa_rate_enum_to_int(enum ib_sa_rate rate)
+{
+ switch (rate) {
+ case IB_SA_RATE_2_5_GBPS: return 1;
+ case IB_SA_RATE_5_GBPS: return 2;
+ case IB_SA_RATE_10_GBPS: return 4;
+ case IB_SA_RATE_20_GBPS: return 8;
+ case IB_SA_RATE_30_GBPS: return 12;
+ case IB_SA_RATE_40_GBPS: return 16;
+ case IB_SA_RATE_60_GBPS: return 24;
+ case IB_SA_RATE_80_GBPS: return 32;
+ case IB_SA_RATE_120_GBPS: return 48;
+ default: return -1;
+ }
+}
+
+struct ib_sa_path_rec {
+ /* reserved */
+ /* reserved */
+ union ibv_gid dgid;
+ union ibv_gid sgid;
+ uint16_t dlid;
+ uint16_t slid;
+ int raw_traffic;
+ /* reserved */
+ uint32_t flow_label;
+ uint8_t hop_limit;
+ uint8_t traffic_class;
+ int reversible;
+ uint8_t numb_path;
+ uint16_t pkey;
+ /* reserved */
+ uint8_t sl;
+ uint8_t mtu_selector;
+ uint8_t mtu;
+ uint8_t rate_selector;
+ uint8_t rate;
+ uint8_t packet_life_time_selector;
+ uint8_t packet_life_time;
+ uint8_t preference;
+};
+
+struct ib_sa_mcmember_rec {
+ union ibv_gid mgid;
+ union ibv_gid port_gid;
+ uint32_t qkey;
+ uint16_t mlid;
+ uint8_t mtu_selector;
+ uint8_t mtu;
+ uint8_t traffic_class;
+ uint16_t pkey;
+ uint8_t rate_selector;
+ uint8_t rate;
+ uint8_t packet_life_time_selector;
+ uint8_t packet_life_time;
+ uint8_t sl;
+ uint32_t flow_label;
+ uint8_t hop_limit;
+ uint8_t scope;
+ uint8_t join_state;
+ int proxy_join;
+};
+
+struct ib_sa_service_rec {
+ uint64_t id;
+ union ibv_gid gid;
+ uint16_t pkey;
+ /* uint16_t resv; */
+ uint32_t lease;
+ uint8_t key[16];
+ uint8_t name[64];
+ uint8_t data8[16];
+ uint16_t data16[8];
+ uint32_t data32[4];
+ uint64_t data64[2];
+};
+
+#endif /* IB_SA_H */
Index: userspace/libibverbs/include/infiniband/marshall.h
===================================================================
--- userspace/libibverbs/include/infiniband/marshall.h (revision 0)
+++ userspace/libibverbs/include/infiniband/marshall.h (revision 0)
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef INFINIBAND_MARSHALL_H
+#define INFINIBAND_MARSHALL_H
+
+#include <infiniband/verbs.h>
+#include <infiniband/sa.h>
+#include <infiniband/kern-abi.h>
+#include <infiniband/sa_kern-abi.h>
+
+#ifdef __cplusplus
+# define BEGIN_C_DECLS extern "C" {
+# define END_C_DECLS }
+#else /* !__cplusplus */
+# define BEGIN_C_DECLS
+# define END_C_DECLS
+#endif /* __cplusplus */
+
+BEGIN_C_DECLS
+
+void ib_copy_qp_attr_from_kern(struct ibv_qp_attr *dst,
+ struct ibv_kern_qp_attr *src);
+
+void ib_copy_path_rec_from_kern(struct ib_sa_path_rec *dst,
+ struct ib_kern_path_rec *src);
+
+void ib_copy_path_rec_to_kern(struct ib_kern_path_rec *dst,
+ struct ib_sa_path_rec *src);
+
+END_C_DECLS
+
+#endif /* INFINIBAND_MARSHALL_H */
Index: userspace/libibverbs/include/infiniband/kern-abi.h
===================================================================
--- userspace/libibverbs/include/infiniband/kern-abi.h (revision 4017)
+++ userspace/libibverbs/include/infiniband/kern-abi.h (working copy)
@@ -357,6 +357,64 @@
__u32 async_events_reported;
};
+struct ibv_kern_global_route {
+ __u8 dgid[16];
+ __u32 flow_label;
+ __u8 sgid_index;
+ __u8 hop_limit;
+ __u8 traffic_class;
+ __u8 reserved;
+};
+
+struct ibv_kern_ah_attr {
+ struct ibv_kern_global_route grh;
+ __u16 dlid;
+ __u8 sl;
+ __u8 src_path_bits;
+ __u8 static_rate;
+ __u8 is_global;
+ __u8 port_num;
+ __u8 reserved;
+};
+
+struct ibv_kern_qp_attr {
+ __u32 qp_attr_mask;
+ __u32 qp_state;
+ __u32 cur_qp_state;
+ __u32 path_mtu;
+ __u32 path_mig_state;
+ __u32 qkey;
+ __u32 rq_psn;
+ __u32 sq_psn;
+ __u32 dest_qp_num;
+ __u32 qp_access_flags;
+
+ struct ibv_kern_ah_attr ah_attr;
+ struct ibv_kern_ah_attr alt_ah_attr;
+
+ /* ib_qp_cap */
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+
+ __u16 pkey_index;
+ __u16 alt_pkey_index;
+ __u8 en_sqd_async_notify;
+ __u8 sq_draining;
+ __u8 max_rd_atomic;
+ __u8 max_dest_rd_atomic;
+ __u8 min_rnr_timer;
+ __u8 port_num;
+ __u8 timeout;
+ __u8 retry_cnt;
+ __u8 rnr_retry;
+ __u8 alt_port_num;
+ __u8 alt_timeout;
+ __u8 reserved[5];
+};
+
struct ibv_create_qp {
__u32 command;
__u16 in_words;
@@ -532,26 +590,6 @@
__u32 bad_wr;
};
-struct ibv_kern_global_route {
- __u8 dgid[16];
- __u32 flow_label;
- __u8 sgid_index;
- __u8 hop_limit;
- __u8 traffic_class;
- __u8 reserved;
-};
-
-struct ibv_kern_ah_attr {
- struct ibv_kern_global_route grh;
- __u16 dlid;
- __u8 sl;
- __u8 src_path_bits;
- __u8 static_rate;
- __u8 is_global;
- __u8 port_num;
- __u8 reserved;
-};
-
struct ibv_create_ah {
__u32 command;
__u16 in_words;
Index: userspace/libibverbs/src/libibverbs.map
===================================================================
--- userspace/libibverbs/src/libibverbs.map (revision 4017)
+++ userspace/libibverbs/src/libibverbs.map (working copy)
@@ -57,5 +57,8 @@
ibv_cmd_destroy_ah;
ibv_cmd_attach_mcast;
ibv_cmd_detach_mcast;
+ ib_copy_qp_attr_from_kern;
+ ib_copy_path_rec_from_kern;
+ ib_copy_path_rec_to_kern;
local: *;
};
Index: userspace/libibverbs/src/marshall.c
===================================================================
--- userspace/libibverbs/src/marshall.c (revision 0)
+++ userspace/libibverbs/src/marshall.c (revision 0)
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <infiniband/marshall.h>
+
+static void ib_copy_ah_attr_from_kern(struct ibv_ah_attr *dst,
+ struct ibv_kern_ah_attr *src)
+{
+ memcpy(dst->grh.dgid.raw, src->grh.dgid, sizeof dst->grh.dgid);
+ dst->grh.flow_label = src->grh.flow_label;
+ dst->grh.sgid_index = src->grh.sgid_index;
+ dst->grh.hop_limit = src->grh.hop_limit;
+ dst->grh.traffic_class = src->grh.traffic_class;
+
+ dst->dlid = src->dlid;
+ dst->sl = src->sl;
+ dst->src_path_bits = src->src_path_bits;
+ dst->static_rate = src->static_rate;
+ dst->is_global = src->is_global;
+ dst->port_num = src->port_num;
+}
+
+void ib_copy_qp_attr_from_kern(struct ibv_qp_attr *dst,
+ struct ibv_kern_qp_attr *src)
+{
+ dst->cur_qp_state = src->cur_qp_state;
+ dst->path_mtu = src->path_mtu;
+ dst->path_mig_state = src->path_mig_state;
+ dst->qkey = src->qkey;
+ dst->rq_psn = src->rq_psn;
+ dst->sq_psn = src->sq_psn;
+ dst->dest_qp_num = src->dest_qp_num;
+ dst->qp_access_flags = src->qp_access_flags;
+
+ dst->cap.max_send_wr = src->max_send_wr;
+ dst->cap.max_recv_wr = src->max_recv_wr;
+ dst->cap.max_send_sge = src->max_send_sge;
+ dst->cap.max_recv_sge = src->max_recv_sge;
+ dst->cap.max_inline_data = src->max_inline_data;
+
+ ib_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
+ ib_copy_ah_attr_from_kern(&dst->alt_ah_attr, &src->alt_ah_attr);
+
+ dst->pkey_index = src->pkey_index;
+ dst->alt_pkey_index = src->alt_pkey_index;
+ dst->en_sqd_async_notify = src->en_sqd_async_notify;
+ dst->sq_draining = src->sq_draining;
+ dst->max_rd_atomic = src->max_rd_atomic;
+ dst->max_dest_rd_atomic = src->max_dest_rd_atomic;
+ dst->min_rnr_timer = src->min_rnr_timer;
+ dst->port_num = src->port_num;
+ dst->timeout = src->timeout;
+ dst->retry_cnt = src->retry_cnt;
+ dst->rnr_retry = src->rnr_retry;
+ dst->alt_port_num = src->alt_port_num;
+ dst->alt_timeout = src->alt_timeout;
+}
+
+void ib_copy_path_rec_from_kern(struct ib_sa_path_rec *dst,
+ struct ib_kern_path_rec *src)
+{
+ memcpy(dst->dgid.raw, src->dgid, sizeof dst->dgid);
+ memcpy(dst->sgid.raw, src->sgid, sizeof dst->sgid);
+
+ dst->dlid = src->dlid;
+ dst->slid = src->slid;
+ dst->raw_traffic = src->raw_traffic;
+ dst->flow_label = src->flow_label;
+ dst->hop_limit = src->hop_limit;
+ dst->traffic_class = src->traffic_class;
+ dst->reversible = src->reversible;
+ dst->numb_path = src->numb_path;
+ dst->pkey = src->pkey;
+ dst->sl = src->sl;
+ dst->mtu_selector = src->mtu_selector;
+ dst->mtu = src->mtu;
+ dst->rate_selector = src->rate_selector;
+ dst->rate = src->rate;
+ dst->packet_life_time = src->packet_life_time;
+ dst->preference = src->preference;
+ dst->packet_life_time_selector = src->packet_life_time_selector;
+}
+
+void ib_copy_path_rec_to_kern(struct ib_kern_path_rec *dst,
+ struct ib_sa_path_rec *src)
+{
+ memcpy(dst->dgid, src->dgid.raw, sizeof src->dgid);
+ memcpy(dst->sgid, src->sgid.raw, sizeof src->sgid);
+
+ dst->dlid = src->dlid;
+ dst->slid = src->slid;
+ dst->raw_traffic = src->raw_traffic;
+ dst->flow_label = src->flow_label;
+ dst->hop_limit = src->hop_limit;
+ dst->traffic_class = src->traffic_class;
+ dst->reversible = src->reversible;
+ dst->numb_path = src->numb_path;
+ dst->pkey = src->pkey;
+ dst->sl = src->sl;
+ dst->mtu_selector = src->mtu_selector;
+ dst->mtu = src->mtu;
+ dst->rate_selector = src->rate_selector;
+ dst->rate = src->rate;
+ dst->packet_life_time = src->packet_life_time;
+ dst->preference = src->preference;
+ dst->packet_life_time_selector = src->packet_life_time_selector;
+}
Index: userspace/libibverbs/Makefile.am
===================================================================
--- userspace/libibverbs/Makefile.am (revision 4017)
+++ userspace/libibverbs/Makefile.am (working copy)
@@ -14,7 +14,8 @@
libibverbs_version_script =
endif
-src_libibverbs_la_SOURCES = src/cmd.c src/device.c src/init.c src/memory.c src/verbs.c
+src_libibverbs_la_SOURCES = src/cmd.c src/device.c src/init.c src/marshall.c \
+ src/memory.c src/verbs.c
src_libibverbs_la_LDFLAGS = -version-info 1 -export-dynamic \
$(libibverbs_version_script)
src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
@@ -40,7 +41,8 @@
libibverbsincludedir = $(includedir)/infiniband
libibverbsinclude_HEADERS = include/infiniband/arch.h include/infiniband/driver.h \
- include/infiniband/kern-abi.h include/infiniband/opcode.h include/infiniband/verbs.h
+ include/infiniband/kern-abi.h include/infiniband/opcode.h include/infiniband/verbs.h \
+ include/infiniband/sa_kern-abi.h include/infiniband/sa.h include/infiniband/marshall.h
man_MANS = man/ibv_asyncwatch.1 man/ibv_devices.1 man/ibv_devinfo.1 \
man/ibv_rc_pingpong.1 man/ibv_uc_pingpong.1 man/ibv_ud_pingpong.1 \
@@ -53,6 +55,8 @@
EXTRA_DIST = include/infiniband/driver.h include/infiniband/kern-abi.h \
include/infiniband/opcode.h include/infiniband/verbs.h src/ibverbs.h \
+ include/infiniband/marshall.h include/sa_kern-abi.h \
+ include/infiniband/sa.h \
src/libibverbs.map libibverbs.spec.in $(man_MANS) $(DEBIAN)
dist-hook: libibverbs.spec
Index: userspace/librdmacm/include/rdma/rdma_cma_abi.h
===================================================================
--- userspace/librdmacm/include/rdma/rdma_cma_abi.h (revision 0)
+++ userspace/librdmacm/include/rdma/rdma_cma_abi.h (revision 0)
@@ -0,0 +1,186 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RDMA_CMA_ABI_H
+#define RDMA_CMA_ABI_H
+
+#include <infiniband/sa_kern-abi.h>
+
+/*
+ * This file must be kept in sync with the kernel's version of rdma_user_cm.h
+ */
+
+#define RDMA_USER_CM_MIN_ABI_VERSION 1
+#define RDMA_USER_CM_MAX_ABI_VERSION 1
+
+#define RDMA_MAX_PRIVATE_DATA 256
+
+enum {
+ UCMA_CMD_CREATE_ID,
+ UCMA_CMD_DESTROY_ID,
+ UCMA_CMD_BIND_ADDR,
+ UCMA_CMD_RESOLVE_ADDR,
+ UCMA_CMD_RESOLVE_ROUTE,
+ UCMA_CMD_QUERY_ROUTE,
+ UCMA_CMD_CONNECT,
+ UCMA_CMD_LISTEN,
+ UCMA_CMD_ACCEPT,
+ UCMA_CMD_REJECT,
+ UCMA_CMD_DISCONNECT,
+ UCMA_CMD_INIT_QP_ATTR,
+ UCMA_CMD_GET_EVENT
+};
+
+struct ucma_abi_cmd_hdr {
+ __u32 cmd;
+ __u16 in;
+ __u16 out;
+};
+
+struct ucma_abi_create_id {
+ __u64 uid;
+ __u64 response;
+};
+
+struct ucma_abi_create_id_resp {
+ __u32 id;
+};
+
+struct ucma_abi_destroy_id {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_destroy_id_resp {
+ __u32 events_reported;
+};
+
+struct ucma_abi_bind_addr {
+ __u64 response;
+ struct sockaddr_in6 addr;
+ __u32 id;
+};
+
+struct ucma_abi_bind_addr_resp {
+ __u64 node_guid;
+};
+
+struct ucma_abi_resolve_addr {
+ struct sockaddr_in6 src_addr;
+ struct sockaddr_in6 dst_addr;
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct ucma_abi_resolve_route {
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct ucma_abi_query_route {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_query_route_resp {
+ __u64 node_guid;
+ struct ib_kern_path_rec ib_route[2];
+ struct sockaddr_in6 src_addr;
+ __u32 num_paths;
+};
+
+struct ucma_abi_conn_param {
+ __u32 qp_num;
+ __u32 qp_type;
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+ __u8 private_data_len;
+ __u8 srq;
+ __u8 responder_resources;
+ __u8 initiator_depth;
+ __u8 flow_control;
+ __u8 retry_count;
+ __u8 rnr_retry_count;
+ __u8 valid;
+};
+
+struct ucma_abi_connect {
+ struct ucma_abi_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_listen {
+ __u32 id;
+ __u32 backlog;
+};
+
+struct ucma_abi_accept {
+ __u64 uid;
+ struct ucma_abi_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_reject {
+ __u32 id;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+struct ucma_abi_disconnect {
+ __u32 id;
+};
+
+struct ucma_abi_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct ucma_abi_get_event {
+ __u64 response;
+};
+
+struct ucma_abi_event_resp {
+ __u64 uid;
+ __u32 id;
+ __u32 event;
+ __u32 status;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+#endif /* RDMA_CMA_ABI_H */
Index: userspace/librdmacm/include/rdma/rdma_cma.h
===================================================================
--- userspace/librdmacm/include/rdma/rdma_cma.h (revision 0)
+++ userspace/librdmacm/include/rdma/rdma_cma.h (revision 0)
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) 2005 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ */
+
+#if !defined(RDMA_CMA_H)
+#define RDMA_CMA_H
+
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <infiniband/verbs.h>
+#include <infiniband/sa.h>
+
+/*
+ * Upon receiving a device removal event, users must destroy the associated
+ * RDMA identifier and release all resources allocated with the device.
+ */
+enum rdma_cm_event_type {
+ RDMA_CM_EVENT_ADDR_RESOLVED,
+ RDMA_CM_EVENT_ADDR_ERROR,
+ RDMA_CM_EVENT_ROUTE_RESOLVED,
+ RDMA_CM_EVENT_ROUTE_ERROR,
+ RDMA_CM_EVENT_CONNECT_REQUEST,
+ RDMA_CM_EVENT_CONNECT_RESPONSE,
+ RDMA_CM_EVENT_CONNECT_ERROR,
+ RDMA_CM_EVENT_UNREACHABLE,
+ RDMA_CM_EVENT_REJECTED,
+ RDMA_CM_EVENT_ESTABLISHED,
+ RDMA_CM_EVENT_DISCONNECTED,
+ RDMA_CM_EVENT_DEVICE_REMOVAL,
+};
+
+struct ib_addr {
+ union ibv_gid sgid;
+ union ibv_gid dgid;
+ uint16_t pkey;
+};
+
+struct rdma_addr {
+ struct sockaddr_in6 src_addr;
+ struct sockaddr_in6 dst_addr;
+ union {
+ struct ib_addr ibaddr;
+ } addr;
+};
+
+struct rdma_route {
+ struct rdma_addr addr;
+ struct ib_sa_path_rec *path_rec;
+ int num_paths;
+};
+
+struct rdma_cm_id {
+ struct ibv_context *verbs;
+ void *context;
+ struct ibv_qp *qp;
+ struct rdma_route route;
+};
+
+struct rdma_cm_event {
+ struct rdma_cm_id *id;
+ struct rdma_cm_id *listen_id;
+ enum rdma_cm_event_type event;
+ int status;
+ void *private_data;
+ uint8_t private_data_len;
+};
+
+int rdma_create_id(struct rdma_cm_id **id, void *context);
+
+int rdma_destroy_id(struct rdma_cm_id *id);
+
+/**
+ * rdma_bind_addr - Bind an RDMA identifier to a source address and
+ * associated RDMA device, if needed.
+ *
+ * @id: RDMA identifier.
+ * @addr: Local address information. Wildcard values are permitted.
+ *
+ * This associates a source address with the RDMA identifier before calling
+ * rdma_listen. If a specific local address is given, the RDMA identifier will
+ * be bound to a local RDMA device.
+ */
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr);
+
+/**
+ * rdma_resolve_addr - Resolve destination and optional source addresses
+ * from IP addresses to an RDMA address. If successful, the specified
+ * rdma_cm_id will be bound to a local device.
+ *
+ * @id: RDMA identifier.
+ * @src_addr: Source address information. This parameter may be NULL.
+ * @dst_addr: Destination address information.
+ * @timeout_ms: Time to wait for resolution to complete.
+ */
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ struct sockaddr *dst_addr, int timeout_ms);
+
+/**
+ * rdma_resolve_route - Resolve the RDMA address bound to the RDMA identifier
+ * into route information needed to establish a connection.
+ *
+ * This is called on the client side of a connection.
+ * Users must have first called rdma_resolve_addr to resolve a dst_addr
+ * into an RDMA address before calling this routine.
+ */
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
+
+/**
+ * rdma_create_qp - Allocate a QP and associate it with the specified RDMA
+ * identifier.
+ *
+ * QPs allocated to an rdma_cm_id will automatically be transitioned by the CMA
+ * through their states.
+ */
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr);
+
+/**
+ * rdma_destroy_qp - Deallocate the QP associated with the specified RDMA
+ * identifier.
+ *
+ * Users must destroy any QP associated with an RDMA identifier before
+ * destroying the RDMA ID.
+ */
+void rdma_destroy_qp(struct rdma_cm_id *id);
+
+struct rdma_conn_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ uint8_t responder_resources;
+ uint8_t initiator_depth;
+ uint8_t flow_control;
+ uint8_t retry_count; /* ignored when accepting */
+ uint8_t rnr_retry_count;
+};
+
+/**
+ * rdma_connect - Initiate an active connection request.
+ *
+ * Users must have resolved a route for the rdma_cm_id to connect with
+ * by having called rdma_resolve_route before calling this routine.
+ */
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_listen - This function is called by the passive side to
+ * listen for incoming connection requests.
+ *
+ * Users must have bound the rdma_cm_id to a local address by calling
+ * rdma_bind_addr before calling this routine.
+ */
+int rdma_listen(struct rdma_cm_id *id, int backlog);
+
+/**
+ * rdma_accept - Called to accept a connection request.
+ * @id: Connection identifier associated with the request.
+ * @conn_param: Information needed to establish the connection.
+ */
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_reject - Called on the passive side to reject a connection request.
+ */
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len);
+
+/**
+ * rdma_disconnect - This function disconnects the associated QP.
+ */
+int rdma_disconnect(struct rdma_cm_id *id);
+
+/**
+ * rdma_get_cm_event - Retrieves the next pending communications event,
+ * if no event is pending waits for an event.
+ * @event: Allocated information about the next communication event.
+ * Event should be freed using rdma_ack_cm_event()
+ *
+ * A RDMA_CM_EVENT_CONNECT_REQUEST communication events result
+ * in the allocation of a new @rdma_cm_id.
+ * Clients are responsible for destroying the new @rdma_cm_id.
+ */
+int rdma_get_cm_event(struct rdma_cm_event **event);
+
+/**
+ * rdma_ack_cm_event - Free a communications event.
+ * @event: Event to be released.
+ *
+ * All events which are allocated by rdma_get_cm_event() must be released,
+ * there should be a one-to-one correspondence between successful gets
+ * and acks.
+ */
+int rdma_ack_cm_event(struct rdma_cm_event *event);
+
+#endif /* RDMA_CMA_H */
Index: userspace/librdmacm/AUTHORS
===================================================================
--- userspace/librdmacm/AUTHORS (revision 0)
+++ userspace/librdmacm/AUTHORS (revision 0)
@@ -0,0 +1 @@
+Sean Hefty <sean.hefty at intel.com>
Index: userspace/librdmacm/configure.in
===================================================================
--- userspace/librdmacm/configure.in (revision 0)
+++ userspace/librdmacm/configure.in (revision 0)
@@ -0,0 +1,50 @@
+dnl Process this file with autoconf to produce a configure script.
+
+AC_PREREQ(2.57)
+AC_INIT(librdmacm, 0.9.0, openib-general at openib.org)
+AC_CONFIG_SRCDIR([src/cma.c])
+AC_CONFIG_AUX_DIR(config)
+AM_CONFIG_HEADER(config.h)
+AM_INIT_AUTOMAKE(librdmacm, 0.9.0)
+AC_DISABLE_STATIC
+AM_PROG_LIBTOOL
+
+AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of ib libraries],
+[ if test x$enableval = xno ; then
+ disable_libcheck=yes
+ fi
+])
+
+dnl Checks for programs
+AC_PROG_CC
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_C_CONST
+AC_CHECK_SIZEOF(long)
+
+dnl Checks for libraries
+if test "$disable_libcheck" != "yes"
+then
+AC_CHECK_LIB(ibverbs, ibv_get_devices, [],
+ AC_MSG_ERROR([ibv_get_devices() not found. librdmacm requires libibverbs.]))
+fi
+
+dnl Checks for header files.
+if test "$disable_libcheck" != "yes"
+then
+AC_CHECK_HEADER(infiniband/verbs.h, [],
+ AC_MSG_ERROR([<infiniband/verbs.h> not found. Is libibverbs installed?]))
+fi
+AC_HEADER_STDC
+
+AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
+ if test -n "`$LD --help < /dev/null 2>/dev/null | grep version-script`"; then
+ ac_cv_version_script=yes
+ else
+ ac_cv_version_script=no
+ fi)
+
+AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")
+
+AC_CONFIG_FILES([Makefile librdmacm.spec])
+AC_OUTPUT
Index: userspace/librdmacm/INSTALL
===================================================================
Index: userspace/librdmacm/src/cma.c
===================================================================
--- userspace/librdmacm/src/cma.c (revision 0)
+++ userspace/librdmacm/src/cma.c (revision 0)
@@ -0,0 +1,887 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: cm.c 3453 2005-09-15 21:43:21Z sean.hefty $
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <string.h>
+#include <glob.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdint.h>
+#include <poll.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include <infiniband/marshall.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_cma_abi.h>
+
+#define PFX "librdmacm: "
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
+static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
+#else
+static inline uint64_t htonll(uint64_t x) { return x; }
+static inline uint64_t ntohll(uint64_t x) { return x; }
+#endif
+
+#define CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, type, size) \
+do { \
+ struct ucma_abi_cmd_hdr *hdr; \
+ \
+ size = sizeof(*hdr) + sizeof(*cmd); \
+ msg = alloca(size); \
+ if (!msg) \
+ return -ENOMEM; \
+ hdr = msg; \
+ cmd = msg + sizeof(*hdr); \
+ hdr->cmd = type; \
+ hdr->in = sizeof(*cmd); \
+ hdr->out = sizeof(*resp); \
+ memset(cmd, 0, sizeof(*cmd)); \
+ resp = alloca(sizeof(*resp)); \
+ if (!resp) \
+ return -ENOMEM; \
+ cmd->response = (uintptr_t)resp;\
+} while (0)
+
+#define CMA_CREATE_MSG_CMD(msg, cmd, type, size) \
+do { \
+ struct ucma_abi_cmd_hdr *hdr; \
+ \
+ size = sizeof(*hdr) + sizeof(*cmd); \
+ msg = alloca(size); \
+ if (!msg) \
+ return -ENOMEM; \
+ hdr = msg; \
+ cmd = msg + sizeof(*hdr); \
+ hdr->cmd = type; \
+ hdr->in = sizeof(*cmd); \
+ hdr->out = 0; \
+ memset(cmd, 0, sizeof(*cmd)); \
+} while (0)
+
+struct cma_device {
+ struct ibv_context *verbs;
+ uint64_t guid;
+ int port_cnt;
+};
+
+struct cma_id_private {
+ struct rdma_cm_id id;
+ struct cma_device *cma_dev;
+ int events_completed;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+ uint32_t handle;
+};
+
+static struct dlist *dev_list;
+static struct dlist *cma_dev_list;
+int cma_fd;
+
+#define container_of(ptr, type, field) \
+ ((type *) ((void *)ptr - offsetof(type, field)))
+
+static void __attribute__((constructor)) rdma_cma_init(void)
+{
+ struct ibv_device *dev;
+ struct cma_device *cma_dev;
+ struct ibv_device_attr attr;
+ int ret;
+
+ cma_fd = open("/dev/infiniband/rdma_cm", O_RDWR);
+ if (cma_fd < 0)
+ abort();
+
+ cma_dev_list = dlist_new(sizeof *cma_dev);
+ dev_list = ibv_get_devices();
+ if (!cma_dev_list || !dev_list)
+ abort();
+
+ dlist_for_each_data(dev_list, dev, struct ibv_device) {
+ cma_dev = malloc(sizeof *cma_dev);
+ if (!cma_dev)
+ abort();
+
+ cma_dev->guid = ibv_get_device_guid(dev);
+ cma_dev->verbs = ibv_open_device(dev);
+ if (!cma_dev->verbs)
+ abort();
+
+ ret = ibv_query_device(cma_dev->verbs, &attr);
+ if (ret)
+ abort();
+
+ cma_dev->port_cnt = attr.phys_port_cnt;
+ dlist_push(cma_dev_list, cma_dev);
+ }
+}
+
+static void __attribute__((destructor)) rdma_cma_fini(void)
+{
+ struct cma_device *cma_dev;
+
+ if (!cma_dev_list)
+ return;
+
+ dlist_for_each_data(cma_dev_list, cma_dev, struct cma_device)
+ ibv_close_device(cma_dev->verbs);
+
+ dlist_destroy(cma_dev_list);
+ close(cma_fd);
+}
+
+static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
+{
+ struct cma_device *cma_dev;
+
+ dlist_for_each_data(cma_dev_list, cma_dev, struct cma_device)
+ if (cma_dev->guid == guid) {
+ id_priv->cma_dev = cma_dev;
+ id_priv->id.verbs = cma_dev->verbs;
+ return 0;
+ }
+
+ return -ENODEV;
+}
+
+static void ucma_free_id(struct cma_id_private *id_priv)
+{
+ pthread_cond_destroy(&id_priv->cond);
+ pthread_mutex_destroy(&id_priv->mut);
+ if (id_priv->id.route.path_rec)
+ free(id_priv->id.route.path_rec);
+ free(id_priv);
+}
+
+static struct cma_id_private *ucma_alloc_id(void *context)
+{
+ struct cma_id_private *id_priv;
+
+ id_priv = malloc(sizeof *id_priv);
+ if (!id_priv)
+ return NULL;
+
+ memset(id_priv, 0, sizeof *id_priv);
+ id_priv->id.context = context;
+ pthread_mutex_init(&id_priv->mut, NULL);
+ if (pthread_cond_init(&id_priv->cond, NULL))
+ goto err;
+
+ return id_priv;
+
+err: ucma_free_id(id_priv);
+ return NULL;
+}
+
+int rdma_create_id(struct rdma_cm_id **id, void *context)
+{
+ struct ucma_abi_create_id_resp *resp;
+ struct ucma_abi_create_id *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ id_priv = ucma_alloc_id(context);
+ if (!id_priv)
+ return -ENOMEM;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
+ cmd->uid = (uintptr_t) id_priv;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ goto err;
+
+ id_priv->handle = resp->id;
+ *id = &id_priv->id;
+ return 0;
+
+err: ucma_free_id(id_priv);
+ return ret;
+}
+
+static int ucma_destroy_kern_id(uint32_t handle)
+{
+ struct ucma_abi_destroy_id_resp *resp;
+ struct ucma_abi_destroy_id *cmd;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_DESTROY_ID, size);
+ cmd->id = handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return resp->events_reported;
+}
+
+int rdma_destroy_id(struct rdma_cm_id *id)
+{
+ struct cma_id_private *id_priv;
+ int ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ ret = ucma_destroy_kern_id(id_priv->handle);
+ if (ret < 0)
+ return ret;
+
+ pthread_mutex_lock(&id_priv->mut);
+ while (id_priv->events_completed < ret)
+ pthread_cond_wait(&id_priv->cond, &id_priv->mut);
+ pthread_mutex_unlock(&id_priv->mut);
+
+ ucma_free_id(id_priv);
+ return 0;
+}
+
+static int ucma_addrlen(struct sockaddr *addr)
+{
+ if (!addr)
+ return 0;
+
+ switch (addr->sa_family) {
+ case PF_INET:
+ return sizeof(struct sockaddr_in);
+ case PF_INET6:
+ return sizeof(struct sockaddr_in6);
+ default:
+ return 0;
+ }
+}
+
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
+{
+ struct ucma_abi_bind_addr_resp *resp;
+ struct ucma_abi_bind_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, addrlen;
+
+ addrlen = ucma_addrlen(addr);
+ if (!addrlen)
+ return -EINVAL;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_BIND_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ memcpy(&cmd->addr, addr, addrlen);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ if (resp->node_guid) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ }
+
+ memcpy(&id->route.addr.src_addr, addr, addrlen);
+ return 0;
+}
+
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ struct sockaddr *dst_addr, int timeout_ms)
+{
+ struct ucma_abi_resolve_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, daddrlen;
+
+ daddrlen = ucma_addrlen(dst_addr);
+ if (!daddrlen)
+ return -EINVAL;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if (src_addr)
+ memcpy(&cmd->src_addr, src_addr, ucma_addrlen(src_addr));
+ memcpy(&cmd->dst_addr, dst_addr, daddrlen);
+ cmd->timeout_ms = timeout_ms;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ memcpy(&id->route.addr.dst_addr, dst_addr, daddrlen);
+ return 0;
+}
+
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
+{
+ struct ucma_abi_resolve_route *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->timeout_ms = timeout_ms;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask)
+{
+ struct ucma_abi_init_qp_attr *cmd;
+ struct ibv_kern_qp_attr *resp;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_INIT_QP_ATTR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->qp_state = qp_attr->qp_state;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ ib_copy_qp_attr_from_kern(qp_attr, resp);
+ *qp_attr_mask = resp->qp_attr_mask;
+ return 0;
+}
+
+static int ucma_modify_qp_rtr(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ if (!id->qp)
+ return -EINVAL;
+
+ /* Need to update QP attributes from default values. */
+ qp_attr.qp_state = IBV_QPS_INIT;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ ret = ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+ if (ret)
+ return ret;
+
+ qp_attr.qp_state = IBV_QPS_RTR;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+}
+
+static int ucma_modify_qp_rts(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_RTS;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+}
+
+static int ucma_modify_qp_err(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+
+ if (!id->qp)
+ return 0;
+
+ qp_attr.qp_state = IBV_QPS_ERR;
+ return ibv_modify_qp(id->qp, &qp_attr, IBV_QP_STATE);
+}
+
+static int ucma_find_gid(struct cma_device *cma_dev, union ibv_gid *gid,
+ uint8_t *port_num)
+{
+ int port, ret, i;
+ union ibv_gid chk_gid;
+
+ for (port = 1; port <= cma_dev->port_cnt; port++)
+ for (i = 0, ret = 0; !ret; i++) {
+ ret = ibv_query_gid(cma_dev->verbs, port, i, &chk_gid);
+ if (!ret && !memcmp(gid, &chk_gid, sizeof *gid)) {
+ *port_num = port;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int ucma_find_pkey(struct cma_device *cma_dev, uint8_t port_num,
+ uint16_t pkey, uint16_t *pkey_index)
+{
+ int ret, i;
+ uint16_t chk_pkey;
+
+ for (i = 0, ret = 0; !ret; i++) {
+ ret = ibv_query_pkey(cma_dev->verbs, port_num, i, &chk_pkey);
+ if (!ret && pkey == chk_pkey) {
+ *pkey_index = (uint16_t) i;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int ucma_init_ib_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
+{
+ struct ibv_qp_attr qp_attr;
+ struct ib_addr *ibaddr;
+ int ret;
+
+ qp_attr.qp_state = IBV_QPS_INIT;
+ qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
+
+ ibaddr = &id_priv->id.route.addr.addr.ibaddr;
+ ret = ucma_find_gid(id_priv->cma_dev, &ibaddr->sgid, &qp_attr.port_num);
+ if (ret)
+ return ret;
+
+ ret = ucma_find_pkey(id_priv->cma_dev, qp_attr.port_num, ibaddr->pkey,
+ &qp_attr.pkey_index);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
+ IBV_QP_PKEY_INDEX | IBV_QP_PORT);
+}
+
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct cma_id_private *id_priv;
+ struct ibv_qp *qp;
+ int ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ if (id->verbs != pd->context)
+ return -EINVAL;
+
+ qp = ibv_create_qp(pd, qp_init_attr);
+ if (!qp)
+ return -ENOMEM;
+
+ ret = ucma_init_ib_qp(id_priv, qp);
+ if (ret)
+ goto err;
+
+ id->qp = qp;
+ return 0;
+err:
+ ibv_destroy_qp(qp);
+ return ret;
+}
+
+void rdma_destroy_qp(struct rdma_cm_id *id)
+{
+ ibv_destroy_qp(id->qp);
+}
+
+static int ucma_query_route(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_route_resp *resp;
+ struct ucma_abi_query_route *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, i;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY_ROUTE, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ if (resp->num_paths) {
+ id->route.path_rec = malloc(sizeof *id->route.path_rec *
+ resp->num_paths);
+ if (!id->route.path_rec)
+ return -ENOMEM;
+
+ id->route.num_paths = resp->num_paths;
+ for (i = 0; i < resp->num_paths; i++)
+ ib_copy_path_rec_from_kern(&id->route.path_rec[i],
+ &resp->ib_route[i]);
+ }
+
+ memcpy(id->route.addr.addr.ibaddr.sgid.raw, resp->ib_route[0].sgid,
+ sizeof id->route.addr.addr.ibaddr.sgid);
+ memcpy(id->route.addr.addr.ibaddr.dgid.raw, resp->ib_route[0].dgid,
+ sizeof id->route.addr.addr.ibaddr.dgid);
+ id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
+ memcpy(&id->route.addr.src_addr, &resp->src_addr,
+ sizeof id->route.addr.src_addr);
+
+ if (!id_priv->cma_dev) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
+ struct rdma_conn_param *src,
+ struct ibv_qp *qp)
+{
+ dst->qp_num = qp->qp_num;
+ dst->qp_type = qp->qp_type;
+ dst->srq = (qp->srq != NULL);
+ dst->responder_resources = src->responder_resources;
+ dst->initiator_depth = src->initiator_depth;
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+ dst->valid = 1;
+
+ if (src->private_data && src->private_data_len) {
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ dst->private_data_len = src->private_data_len;
+ } else
+ src->private_data_len = 0;
+}
+
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
+{
+ struct ucma_abi_connect *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_CONNECT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_listen(struct rdma_cm_id *id, int backlog)
+{
+ struct ucma_abi_listen *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LISTEN, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->backlog = backlog;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
+{
+ struct ucma_abi_accept *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_rtr(id);
+ if (ret)
+ return ret;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->uid = (uintptr_t) id_priv;
+ ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ucma_modify_qp_err(id);
+ return (ret > 0) ? -ENODATA : ret;
+ }
+
+ return 0;
+}
+
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len)
+{
+ struct ucma_abi_reject *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_REJECT, size);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if (private_data && private_data_len) {
+ memcpy(cmd->private_data, private_data, private_data_len);
+ cmd->private_data_len = private_data_len;
+ } else
+ cmd->private_data_len = 0;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_disconnect(struct rdma_cm_id *id)
+{
+ struct ucma_abi_disconnect *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_err(id);
+ if (ret)
+ return ret;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_DISCONNECT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
+ struct ucma_abi_event_resp *src)
+{
+ dst->event = src->event;
+ dst->status = src->status;
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = dst + 1;
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ } else
+ dst->private_data = NULL;
+}
+
+static void ucma_complete_event(struct cma_id_private *id_priv)
+{
+ pthread_mutex_lock(&id_priv->mut);
+ id_priv->events_completed++;
+ pthread_cond_signal(&id_priv->cond);
+ pthread_mutex_unlock(&id_priv->mut);
+}
+
+int rdma_ack_cm_event(struct rdma_cm_event *event)
+{
+ struct rdma_cm_id *id;
+
+ if (!event)
+ return -EINVAL;
+
+ id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
+ event->listen_id : event->id;
+
+ ucma_complete_event(container_of(id, struct cma_id_private, id));
+ free(event);
+ return 0;
+}
+
+static int ucma_process_conn_req(struct rdma_cm_event *event,
+ uint32_t handle)
+{
+ struct cma_id_private *listen_id_priv, *id_priv;
+ int ret;
+
+ listen_id_priv = container_of(event->id, struct cma_id_private, id);
+ id_priv = ucma_alloc_id(event->id->context);
+ if (!id_priv) {
+ ucma_destroy_kern_id(handle);
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ event->listen_id = event->id;
+ event->id = &id_priv->id;
+ id_priv->handle = handle;
+
+ ret = ucma_query_route(&id_priv->id);
+ if (ret) {
+ rdma_destroy_id(&id_priv->id);
+ goto err;
+ }
+
+ return 0;
+err:
+ ucma_complete_event(listen_id_priv);
+ return ret;
+}
+
+static int ucma_process_conn_resp(struct cma_id_private *id_priv)
+{
+ struct ucma_abi_accept *cmd;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_rtr(&id_priv->id);
+ if (ret)
+ goto err;
+
+ ret = ucma_modify_qp_rts(&id_priv->id);
+ if (ret)
+ goto err;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ret = (ret > 0) ? -ENODATA : ret;
+ goto err;
+ }
+
+ return 0;
+err:
+ ucma_modify_qp_err(&id_priv->id);
+ return ret;
+}
+
+static int ucma_process_establish(struct rdma_cm_id *id)
+{
+ int ret;
+
+ ret = ucma_modify_qp_rts(id);
+ if (ret)
+ ucma_modify_qp_err(id);
+
+ return ret;
+}
+
+int rdma_get_cm_event(struct rdma_cm_event **event)
+{
+ struct ucma_abi_event_resp *resp;
+ struct ucma_abi_get_event *cmd;
+ struct cma_id_private *id_priv;
+ struct rdma_cm_event *evt;
+ void *msg;
+ int ret, size;
+
+ if (!event)
+ return -EINVAL;
+
+ evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
+ if (!evt)
+ return -ENOMEM;
+
+retry:
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ret = (ret > 0) ? -ENODATA : ret;
+ goto err;
+ }
+
+ id_priv = (void *) (uintptr_t) resp->uid;
+ evt->id = &id_priv->id;
+ ucma_copy_event_from_kern(evt, resp);
+
+ switch (evt->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ evt->status = ucma_query_route(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_ADDR_ERROR;
+ break;
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ evt->status = ucma_query_route(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
+ break;
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ ret = ucma_process_conn_req(evt, resp->id);
+ if (ret)
+ goto retry;
+ break;
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ evt->status = ucma_process_conn_resp(id_priv);
+ if (!evt->status)
+ evt->event = RDMA_CM_EVENT_ESTABLISHED;
+ else
+ evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
+ break;
+ case RDMA_CM_EVENT_ESTABLISHED:
+ evt->status = ucma_process_establish(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
+ break;
+ case RDMA_CM_EVENT_REJECTED:
+ ucma_modify_qp_err(evt->id);
+ break;
+ default:
+ break;
+ }
+
+ *event = evt;
+ return 0;
+err:
+ free(evt);
+ return ret;
+}
Index: userspace/librdmacm/src/librdmacm.map
===================================================================
--- userspace/librdmacm/src/librdmacm.map (revision 0)
+++ userspace/librdmacm/src/librdmacm.map (revision 0)
@@ -0,0 +1,18 @@
+RDMACM_1.0 {
+ global:
+ rdma_create_id;
+ rdma_destroy_id;
+ rdma_bind_addr;
+ rdma_resolve_addr;
+ rdma_resolve_route;
+ rdma_create_qp;
+ rdma_destroy_qp;
+ rdma_connect;
+ rdma_listen;
+ rdma_accept;
+ rdma_reject;
+ rdma_disconnect;
+ rdma_get_cm_event;
+ rdma_ack_cm_event;
+ local: *;
+};
Index: userspace/librdmacm/ChangeLog
===================================================================
Index: userspace/librdmacm/COPYING
===================================================================
--- userspace/librdmacm/COPYING (revision 0)
+++ userspace/librdmacm/COPYING (revision 0)
@@ -0,0 +1,378 @@
+This software is available to you under a choice of one of two
+licenses. You may choose to be licensed under the terms of the the
+OpenIB.org BSD license or the GNU General Public License (GPL) Version
+2, both included below.
+
+Copyright (c) 2005 Intel Corporation. All rights reserved.
+
+==================================================================
+
+ OpenIB.org BSD license
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+==================================================================
+
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+ 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: userspace/librdmacm/librdmacm.spec.in
===================================================================
--- userspace/librdmacm/librdmacm.spec.in (revision 0)
+++ userspace/librdmacm/librdmacm.spec.in (revision 0)
@@ -0,0 +1,40 @@
+# $Id: $
+
+%define prefix /usr
+%define ver @VERSION@
+%define RELEASE 1
+%define rel %{?CUSTOM_RELEASE} %{!?CUSTOM_RELEASE:%RELEASE}
+
+Summary: Userspace RDMA Connection Manager.
+Name: librdmacm
+Version: %ver
+Release: %rel
+Copyright: Dual GPL/BSD
+Group: System Environment/Libraries
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+Source: http://openib.org/downloads/%{name}-%{version}.tar.gz
+Url: http://openib.org/
+
+%description
+Along with the OpenIB kernel drivers, librdmacm provides a userspace
+RDMA Connection Managment API.
+
+%prep
+%setup -q
+
+%build
+%configure
+make
+
+%install
+make DESTDIR=${RPM_BUILD_ROOT} install
+# remove unpackaged files from the buildroot
+rm -f $RPM_BUILD_ROOT%{_libdir}/*.la
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root)
+%{_libdir}/librdmacm*.so.*
+%doc AUTHORS COPYING ChangeLog NEWS README
Index: userspace/librdmacm/Makefile.am
===================================================================
--- userspace/librdmacm/Makefile.am (revision 0)
+++ userspace/librdmacm/Makefile.am (revision 0)
@@ -0,0 +1,36 @@
+# $Id: Makefile.am 3373 2005-09-12 16:34:20Z roland $
+INCLUDES = -I$(srcdir)/include
+
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+
+rdmacmlibdir = $(libdir)
+
+rdmacmlib_LTLIBRARIES = src/librdmacm.la
+
+src_rdmacm_la_CFLAGS = -g -Wall -D_GNU_SOURCE
+
+if HAVE_LD_VERSION_SCRIPT
+ rdmacm_version_script = -Wl,--version-script=$(srcdir)/src/librdmacm.map
+else
+ rdmacm_version_script =
+endif
+
+src_librdmacm_la_SOURCES = src/cma.c
+src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script)
+
+bin_PROGRAMS = examples/ucmatose
+examples_ucmatose_SOURCES = examples/cmatose.c
+examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
+
+librdmacmincludedir = $(includedir)/rdma
+
+librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
+ include/rdma/rdma_cma.h
+
+EXTRA_DIST = include/rdma/rdma_cma_abi.h \
+ include/rdma/rdma_cma.h \
+ src/librdmacm.map \
+ librdmacm.spec.in
+
+dist-hook: librdmacm.spec
+ cp librdmacm.spec $(distdir)
Index: userspace/librdmacm/autogen.sh
===================================================================
--- userspace/librdmacm/autogen.sh (revision 0)
+++ userspace/librdmacm/autogen.sh (revision 0)
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+set -x
+aclocal -I config
+libtoolize --force --copy
+autoheader
+automake --foreign --add-missing --copy
+autoconf
Property changes on: userspace/librdmacm/autogen.sh
___________________________________________________________________
Name: svn:executable
+ *
Index: userspace/librdmacm/NEWS
===================================================================
Index: userspace/librdmacm/README
===================================================================
--- userspace/librdmacm/README (revision 0)
+++ userspace/librdmacm/README (revision 0)
@@ -0,0 +1,29 @@
+This README is for userspace RDMA cm library.
+
+Building
+
+To make this directory, run:
+./autogen.sh && ./configure && make && make install
+
+Typically the autogen and configure steps only need be done the first
+time unless configure.in or Makefile.am changes.
+
+Libraries are installed by default at /usr/local/lib.
+
+Device files
+
+The userspace CMA uses a single device file regardless of the number
+of adapters or ports present.
+
+To create the appropriate character device file automatically with
+udev, a rule like
+
+ KERNEL="ucma", NAME="infiniband/%k", MODE="0666"
+
+can be used. This will create the device node named
+
+ /dev/infiniband/ucma
+
+or you can create it manually
+
+ mknod /dev/infiniband/ucma c 231 255
Index: userspace/librdmacm/examples/cmatose.c
===================================================================
--- userspace/librdmacm/examples/cmatose.c (revision 0)
+++ userspace/librdmacm/examples/cmatose.c (revision 0)
@@ -0,0 +1,583 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+
+#include <rdma/rdma_cma.h>
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+static inline uint32_t cpu_to_be32(uint32_t x) { return x; }
+#else
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
+#endif
+
+/*
+ * To execute:
+ * Server: rdma_cmatose
+ * Client: rdma_cmatose "dst_ip=ip"
+ */
+
+struct cmatest_node {
+ int id;
+ struct rdma_cm_id *cma_id;
+ int connected;
+ struct ibv_pd *pd;
+ struct ibv_cq *cq;
+ struct ibv_mr *mr;
+ void *mem;
+};
+
+struct cmatest {
+ struct cmatest_node *nodes;
+ int conn_index;
+ int connects_left;
+ int disconnects_left;
+
+ struct sockaddr_in addr_in;
+ struct sockaddr *addr;
+};
+
+static struct cmatest test;
+static int connections = 1;
+static int message_size = 100;
+static int message_count = 10;
+static int is_server;
+
+static int create_message(struct cmatest_node *node)
+{
+ if (!message_size)
+ message_count = 0;
+
+ if (!message_count)
+ return 0;
+
+ node->mem = malloc(message_size);
+ if (!node->mem) {
+ printf("failed message allocation\n");
+ return -1;
+ }
+ node->mr = ibv_reg_mr(node->pd, node->mem, message_size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!node->mr) {
+ printf("failed to reg MR\n");
+ goto err;
+ }
+ return 0;
+err:
+ free(node->mem);
+ return -1;
+}
+
+static int init_node(struct cmatest_node *node)
+{
+ struct ibv_qp_init_attr init_qp_attr;
+ int cqe, ret;
+
+ node->pd = ibv_alloc_pd(node->cma_id->verbs);
+ if (!node->pd) {
+ ret = -ENOMEM;
+ printf("cmatose: unable to allocate PD\n");
+ goto out;
+ }
+
+ cqe = message_count ? message_count * 2 : 2;
+ node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);
+ if (!node->cq) {
+ ret = -ENOMEM;
+ printf("cmatose: unable to create CQ\n");
+ goto out;
+ }
+
+ memset(&init_qp_attr, 0, sizeof init_qp_attr);
+ init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+ init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+ init_qp_attr.cap.max_send_sge = 1;
+ init_qp_attr.cap.max_recv_sge = 1;
+ init_qp_attr.qp_context = node;
+ init_qp_attr.sq_sig_all = 1;
+ init_qp_attr.qp_type = IBV_QPT_RC;
+ init_qp_attr.send_cq = node->cq;
+ init_qp_attr.recv_cq = node->cq;
+ ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
+ if (ret) {
+ printf("cmatose: unable to create QP: %d\n", ret);
+ goto out;
+ }
+
+ ret = create_message(node);
+ if (ret) {
+ printf("cmatose: failed to create messages: %d\n", ret);
+ goto out;
+ }
+out:
+ return ret;
+}
+
+static int post_recvs(struct cmatest_node *node)
+{
+ struct ibv_recv_wr recv_wr, *recv_failure;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!message_count)
+ return 0;
+
+ recv_wr.next = NULL;
+ recv_wr.sg_list = &sge;
+ recv_wr.num_sge = 1;
+ recv_wr.wr_id = (uintptr_t) node;
+
+ sge.length = message_size;
+ sge.lkey = node->mr->lkey;
+ sge.addr = (uintptr_t) node->mem;
+
+ for (i = 0; i < message_count && !ret; i++ ) {
+ ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
+ if (ret) {
+ printf("failed to post receives: %d\n", ret);
+ break;
+ }
+ }
+ return ret;
+}
+
+static int post_sends(struct cmatest_node *node)
+{
+ struct ibv_send_wr send_wr, *bad_send_wr;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!node->connected || !message_count)
+ return 0;
+
+ send_wr.next = NULL;
+ send_wr.sg_list = &sge;
+ send_wr.num_sge = 1;
+ send_wr.opcode = IBV_WR_SEND;
+ send_wr.send_flags = 0;
+ send_wr.wr_id = (unsigned long)node;
+
+ sge.length = message_size;
+ sge.lkey = node->mr->lkey;
+ sge.addr = (uintptr_t) node->mem;
+
+ for (i = 0; i < message_count && !ret; i++)
+ ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
+
+ return ret;
+}
+
+static void connect_error(void)
+{
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void addr_handler(struct cmatest_node *node)
+{
+ int ret;
+
+ ret = rdma_resolve_route(node->cma_id, 2000);
+ if (ret) {
+ printf("cmatose: resolve route failed: %d\n", ret);
+ connect_error();
+ }
+}
+
+static void route_handler(struct cmatest_node *node)
+{
+ struct rdma_conn_param conn_param;
+ int ret;
+
+ ret = init_node(node);
+ if (ret)
+ goto err;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto err;
+
+ memset(&conn_param, 0, sizeof conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ conn_param.retry_count = 5;
+ ret = rdma_connect(node->cma_id, &conn_param);
+ if (ret) {
+ printf("cmatose: failure connecting: %d\n", ret);
+ goto err;
+ }
+ return;
+err:
+ connect_error();
+}
+
+static int connect_handler(struct rdma_cm_id *cma_id)
+{
+ struct cmatest_node *node;
+ struct rdma_conn_param conn_param;
+ int ret;
+
+ if (test.conn_index == connections) {
+ ret = -ENOMEM;
+ goto err1;
+ }
+ node = &test.nodes[test.conn_index++];
+
+ node->cma_id = cma_id;
+ cma_id->context = node;
+
+ ret = init_node(node);
+ if (ret)
+ goto err2;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto err2;
+
+ memset(&conn_param, 0, sizeof conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ ret = rdma_accept(node->cma_id, &conn_param);
+ if (ret) {
+ printf("cmatose: failure accepting: %d\n", ret);
+ goto err2;
+ }
+ return 0;
+
+err2:
+ node->cma_id = NULL;
+ connect_error();
+err1:
+ printf("cmatose: failing connection request\n");
+ rdma_reject(cma_id, NULL, 0);
+ return ret;
+}
+
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
+{
+ int ret = 0;
+
+ switch (event->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ addr_handler(cma_id->context);
+ break;
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ route_handler(cma_id->context);
+ break;
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ ret = connect_handler(cma_id);
+ break;
+ case RDMA_CM_EVENT_ESTABLISHED:
+ ((struct cmatest_node *) cma_id->context)->connected = 1;
+ test.connects_left--;
+ break;
+ case RDMA_CM_EVENT_ADDR_ERROR:
+ case RDMA_CM_EVENT_ROUTE_ERROR:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_REJECTED:
+ printf("cmatose: event: %d, error: %d\n", event->event,
+ event->status);
+ connect_error();
+ break;
+ case RDMA_CM_EVENT_DISCONNECTED:
+ rdma_disconnect(cma_id);
+ test.disconnects_left--;
+ break;
+ case RDMA_CM_EVENT_DEVICE_REMOVAL:
+ /* Cleanup will occur after test completes. */
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static void destroy_node(struct cmatest_node *node)
+{
+ if (!node->cma_id)
+ return;
+
+ if (node->cma_id->qp)
+ rdma_destroy_qp(node->cma_id);
+
+ if (node->cq)
+ ibv_destroy_cq(node->cq);
+
+ if (node->mem) {
+ ibv_dereg_mr(node->mr);
+ free(node->mem);
+ }
+
+ if (node->pd)
+ ibv_dealloc_pd(node->pd);
+
+ /* Destroy the RDMA ID after all device resources */
+ rdma_destroy_id(node->cma_id);
+}
+
+static int alloc_nodes(void)
+{
+ int ret, i;
+
+ test.nodes = malloc(sizeof *test.nodes * connections);
+ if (!test.nodes) {
+ printf("cmatose: unable to allocate memory for test nodes\n");
+ return -ENOMEM;
+ }
+ memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+ for (i = 0; i < connections; i++) {
+ test.nodes[i].id = i;
+ if (!is_server) {
+ ret = rdma_create_id(&test.nodes[i].cma_id,
+ &test.nodes[i]);
+ if (ret)
+ goto err;
+ }
+ }
+ return 0;
+err:
+ while (--i >= 0)
+ rdma_destroy_id(test.nodes[i].cma_id);
+ free(test.nodes);
+ return ret;
+}
+
+static void destroy_nodes(void)
+{
+ int i;
+
+ for (i = 0; i < connections; i++)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+}
+
+static int poll_cqs(void)
+{
+ struct ibv_wc wc[8];
+ int done, i, ret;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (done = 0; done < message_count; done += ret) {
+ ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+ if (ret < 0) {
+ printf("cmatose: failed polling CQ: %d\n", ret);
+ return ret;
+ }
+ }
+ }
+ return 0;
+}
+
+static void connect_events(void)
+{
+ struct rdma_cm_event *event;
+ int err = 0;
+
+ while (test.connects_left && !err) {
+ err = rdma_get_cm_event(&event);
+ if (!err) {
+ cma_handler(event->id, event);
+ rdma_ack_cm_event(event);
+ }
+ }
+}
+
+static void disconnect_events(void)
+{
+ struct rdma_cm_event *event;
+ int err = 0;
+
+ while (test.disconnects_left && !err) {
+ err = rdma_get_cm_event(&event);
+ if (!err) {
+ cma_handler(event->id, event);
+ rdma_ack_cm_event(event);
+ }
+ }
+}
+
+static void run_server(void)
+{
+ struct rdma_cm_id *listen_id;
+ int i, ret;
+
+ printf("cmatose: starting server\n");
+ ret = rdma_create_id(&listen_id, &test);
+ if (ret) {
+ printf("cmatose: listen request failed\n");
+ return;
+ }
+
+ test.addr_in.sin_family = PF_INET;
+ test.addr_in.sin_port = 7471;
+ ret = rdma_bind_addr(listen_id, test.addr);
+ if (ret) {
+ printf("cmatose: bind address failed: %d\n", ret);
+ return;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("cmatose: failure trying to listen: %d\n", ret);
+ goto out;
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("initiating data transfers\n");
+ for (i = 0; i < connections; i++)
+ if (post_sends(&test.nodes[i]))
+ goto out;
+
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("data transfers complete\n");
+
+ }
+ printf("cmatose: disconnecting\n");
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ test.nodes[i].connected = 0;
+ rdma_disconnect(test.nodes[i].cma_id);
+ }
+
+ disconnect_events();
+ printf("disconnected\n");
+out:
+ rdma_destroy_id(listen_id);
+}
+
+static int get_dst_addr(char *dst)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ test.addr_in = *(struct sockaddr_in *) res->ai_addr;
+ test.addr_in.sin_port = 7471;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
+static void run_client(char *dst)
+{
+ int i, ret;
+
+ printf("cmatose: starting client\n");
+ ret = get_dst_addr(dst);
+ if (ret)
+ return;
+
+ printf("cmatose: connecting\n");
+ for (i = 0; i < connections; i++) {
+ ret = rdma_resolve_addr(test.nodes[i].cma_id, NULL,
+ test.addr, 2000);
+ if (ret) {
+ printf("cmatose: failure getting addr: %d\n", ret);
+ connect_error();
+ }
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+
+ printf("sending replies\n");
+ for (i = 0; i < connections; i++)
+ if (post_sends(&test.nodes[i]))
+ goto out;
+
+ printf("data transfers complete\n");
+
+ }
+out:
+ disconnect_events();
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 1 && argc != 2) {
+ printf("usage: %s [server_addr]\n", argv[0]);
+ exit(1);
+ }
+ is_server = (argc == 1);
+
+ test.addr = (struct sockaddr *) &test.addr_in;
+ test.connects_left = connections;
+ test.disconnects_left = connections;
+ if (alloc_nodes())
+ exit(1);
+
+ if (is_server)
+ run_server();
+ else
+ run_client(argv[1]);
+
+ printf("test complete\n");
+ destroy_nodes();
+ return 0;
+}
Index: userspace/libibcm/include/infiniband/cm_abi.h
===================================================================
--- userspace/libibcm/include/infiniband/cm_abi.h (revision 4017)
+++ userspace/libibcm/include/infiniband/cm_abi.h (working copy)
@@ -37,6 +37,9 @@
#define CM_ABI_H
#include <linux/types.h>
+#include <infiniband/sa.h>
+#include <infiniband/marshall.h>
+
/*
* This file must be kept in sync with the kernel's version of ib_user_cm.h
*/
@@ -114,58 +117,6 @@
__u32 qp_state;
};
-struct cm_abi_ah_attr {
- __u8 grh_dgid[16];
- __u32 grh_flow_label;
- __u16 dlid;
- __u16 reserved;
- __u8 grh_sgid_index;
- __u8 grh_hop_limit;
- __u8 grh_traffic_class;
- __u8 sl;
- __u8 src_path_bits;
- __u8 static_rate;
- __u8 is_global;
- __u8 port_num;
-};
-
-struct cm_abi_init_qp_attr_resp {
- __u32 qp_attr_mask;
- __u32 qp_state;
- __u32 cur_qp_state;
- __u32 path_mtu;
- __u32 path_mig_state;
- __u32 qkey;
- __u32 rq_psn;
- __u32 sq_psn;
- __u32 dest_qp_num;
- __u32 qp_access_flags;
-
- struct cm_abi_ah_attr ah_attr;
- struct cm_abi_ah_attr alt_ah_attr;
-
- /* ibv_qp_cap */
- __u32 max_send_wr;
- __u32 max_recv_wr;
- __u32 max_send_sge;
- __u32 max_recv_sge;
- __u32 max_inline_data;
-
- __u16 pkey_index;
- __u16 alt_pkey_index;
- __u8 en_sqd_async_notify;
- __u8 sq_draining;
- __u8 max_rd_atomic;
- __u8 max_dest_rd_atomic;
- __u8 min_rnr_timer;
- __u8 port_num;
- __u8 timeout;
- __u8 retry_cnt;
- __u8 rnr_retry;
- __u8 alt_port_num;
- __u8 alt_timeout;
-};
-
struct cm_abi_listen {
__u64 service_id;
__u64 service_mask;
@@ -184,28 +135,6 @@
__u8 reserved[3];
};
-struct cm_abi_path_rec {
- __u8 dgid[16];
- __u8 sgid[16];
- __u16 dlid;
- __u16 slid;
- __u32 raw_traffic;
- __u32 flow_label;
- __u32 reversible;
- __u32 mtu;
- __u16 pkey;
- __u8 hop_limit;
- __u8 traffic_class;
- __u8 numb_path;
- __u8 sl;
- __u8 mtu_selector;
- __u8 rate_selector;
- __u8 rate;
- __u8 packet_life_time_selector;
- __u8 packet_life_time;
- __u8 preference;
-};
-
struct cm_abi_req {
__u32 id;
__u32 qpn;
@@ -308,8 +237,8 @@
};
struct cm_abi_req_event_resp {
- struct cm_abi_path_rec primary_path;
- struct cm_abi_path_rec alternate_path;
+ struct ib_kern_path_rec primary_path;
+ struct ib_kern_path_rec alternate_path;
__u64 remote_ca_guid;
__u32 remote_qkey;
__u32 remote_qpn;
@@ -353,7 +282,7 @@
};
struct cm_abi_lap_event_resp {
- struct cm_abi_path_rec path;
+ struct ib_kern_path_rec path;
};
struct cm_abi_apr_event_resp {
Index: userspace/libibcm/include/infiniband/cm.h
===================================================================
--- userspace/libibcm/include/infiniband/cm.h (revision 4017)
+++ userspace/libibcm/include/infiniband/cm.h (working copy)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004 Intel Corporation. All rights reserved.
+ * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
*
Index: userspace/libibcm/configure.in
===================================================================
--- userspace/libibcm/configure.in (revision 4017)
+++ userspace/libibcm/configure.in (working copy)
@@ -27,8 +27,6 @@
then
AC_CHECK_LIB(ibverbs, ibv_get_devices, [],
AC_MSG_ERROR([ibv_get_devices() not found. libibcm requires libibverbs.]))
-AC_CHECK_LIB(ibat, ib_at_route_by_ip, [],
- AC_MSG_ERROR([ib_at_route_by_ip() not found. libibcm requires libibat.]))
fi
dnl Checks for header files.
@@ -36,8 +34,8 @@
then
AC_CHECK_HEADER(infiniband/verbs.h, [],
AC_MSG_ERROR([<infiniband/verbs.h> not found. Is libibverbs installed?]))
-AC_CHECK_HEADER(infiniband/at.h, [],
- AC_MSG_ERROR([<infiniband/at.h> not found. Is libibat installed?]))
+AC_CHECK_HEADER(infiniband/marshall.h, [],
+ AC_MSG_ERROR([<infiniband/marshall.h> not found. Is libibverbs installed?]))
fi
AC_HEADER_STDC
Index: userspace/libibcm/src/cm.c
===================================================================
--- userspace/libibcm/src/cm.c (revision 4017)
+++ userspace/libibcm/src/cm.c (working copy)
@@ -52,6 +52,7 @@
#include <infiniband/cm.h>
#include <infiniband/cm_abi.h>
+#include <infiniband/marshall.h>
#define PFX "libibcm: "
@@ -266,33 +267,6 @@
return NULL;
}
-static void cm_param_path_get(struct cm_abi_path_rec *abi,
- struct ib_sa_path_rec *sa)
-{
- memcpy(abi->dgid, sa->dgid.raw, sizeof(union ibv_gid));
- memcpy(abi->sgid, sa->sgid.raw, sizeof(union ibv_gid));
-
- abi->dlid = sa->dlid;
- abi->slid = sa->slid;
-
- abi->raw_traffic = sa->raw_traffic;
- abi->flow_label = sa->flow_label;
- abi->reversible = sa->reversible;
- abi->mtu = sa->mtu;
- abi->pkey = sa->pkey;
-
- abi->hop_limit = sa->hop_limit;
- abi->traffic_class = sa->traffic_class;
- abi->numb_path = sa->numb_path;
- abi->sl = sa->sl;
- abi->mtu_selector = sa->mtu_selector;
- abi->rate_selector = sa->rate_selector;
- abi->rate = sa->rate;
- abi->packet_life_time_selector = sa->packet_life_time_selector;
- abi->packet_life_time = sa->packet_life_time;
- abi->preference = sa->preference;
-}
-
static void ib_cm_free_id(struct cm_id_private *cm_id_priv)
{
pthread_cond_destroy(&cm_id_priv->cond);
@@ -407,65 +381,11 @@
return 0;
}
-static void ib_cm_copy_ah_attr(struct ibv_ah_attr *dest_attr,
- struct cm_abi_ah_attr *src_attr)
-{
- memcpy(dest_attr->grh.dgid.raw, src_attr->grh_dgid,
- sizeof dest_attr->grh.dgid);
- dest_attr->grh.flow_label = src_attr->grh_flow_label;
- dest_attr->grh.sgid_index = src_attr->grh_sgid_index;
- dest_attr->grh.hop_limit = src_attr->grh_hop_limit;
- dest_attr->grh.traffic_class = src_attr->grh_traffic_class;
-
- dest_attr->dlid = src_attr->dlid;
- dest_attr->sl = src_attr->sl;
- dest_attr->src_path_bits = src_attr->src_path_bits;
- dest_attr->static_rate = src_attr->static_rate;
- dest_attr->is_global = src_attr->is_global;
- dest_attr->port_num = src_attr->port_num;
-}
-
-static void ib_cm_copy_qp_attr(struct ibv_qp_attr *dest_attr,
- struct cm_abi_init_qp_attr_resp *src_attr)
-{
- dest_attr->cur_qp_state = src_attr->cur_qp_state;
- dest_attr->path_mtu = src_attr->path_mtu;
- dest_attr->path_mig_state = src_attr->path_mig_state;
- dest_attr->qkey = src_attr->qkey;
- dest_attr->rq_psn = src_attr->rq_psn;
- dest_attr->sq_psn = src_attr->sq_psn;
- dest_attr->dest_qp_num = src_attr->dest_qp_num;
- dest_attr->qp_access_flags = src_attr->qp_access_flags;
-
- dest_attr->cap.max_send_wr = src_attr->max_send_wr;
- dest_attr->cap.max_recv_wr = src_attr->max_recv_wr;
- dest_attr->cap.max_send_sge = src_attr->max_send_sge;
- dest_attr->cap.max_recv_sge = src_attr->max_recv_sge;
- dest_attr->cap.max_inline_data = src_attr->max_inline_data;
-
- ib_cm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
- ib_cm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
-
- dest_attr->pkey_index = src_attr->pkey_index;
- dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
- dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
- dest_attr->sq_draining = src_attr->sq_draining;
- dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
- dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
- dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
- dest_attr->port_num = src_attr->port_num;
- dest_attr->timeout = src_attr->timeout;
- dest_attr->retry_cnt = src_attr->retry_cnt;
- dest_attr->rnr_retry = src_attr->rnr_retry;
- dest_attr->alt_port_num = src_attr->alt_port_num;
- dest_attr->alt_timeout = src_attr->alt_timeout;
-}
-
int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
struct ibv_qp_attr *qp_attr,
int *qp_attr_mask)
{
- struct cm_abi_init_qp_attr_resp *resp;
+ struct ibv_kern_qp_attr *resp;
struct cm_abi_init_qp_attr *cmd;
void *msg;
int result;
@@ -483,7 +403,7 @@
return (result > 0) ? -ENODATA : result;
*qp_attr_mask = resp->qp_attr_mask;
- ib_cm_copy_qp_attr(qp_attr, resp);
+ ib_copy_qp_attr_from_kern(qp_attr, resp);
return 0;
}
@@ -511,8 +431,8 @@
int ib_cm_send_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param)
{
- struct cm_abi_path_rec *p_path;
- struct cm_abi_path_rec *a_path;
+ struct ib_kern_path_rec *p_path;
+ struct ib_kern_path_rec *a_path;
struct cm_abi_req *cmd;
void *msg;
int result;
@@ -543,7 +463,7 @@
if (!p_path)
return -ENOMEM;
- cm_param_path_get(p_path, param->primary_path);
+ ib_copy_path_rec_to_kern(p_path, param->primary_path);
cmd->primary_path = (uintptr_t) p_path;
}
@@ -552,7 +472,7 @@
if (!a_path)
return -ENOMEM;
- cm_param_path_get(a_path, param->alternate_path);
+ ib_copy_path_rec_to_kern(a_path, param->alternate_path);
cmd->alternate_path = (uintptr_t) a_path;
}
@@ -758,7 +678,7 @@
void *private_data,
uint8_t private_data_len)
{
- struct cm_abi_path_rec *abi_path;
+ struct ib_kern_path_rec *abi_path;
struct cm_abi_lap *cmd;
void *msg;
int result;
@@ -772,7 +692,7 @@
if (!abi_path)
return -ENOMEM;
- cm_param_path_get(abi_path, alternate_path);
+ ib_copy_path_rec_to_kern(abi_path, alternate_path);
cmd->path = (uintptr_t) abi_path;
}
@@ -791,7 +711,7 @@
int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
struct ib_cm_sidr_req_param *param)
{
- struct cm_abi_path_rec *abi_path;
+ struct ib_kern_path_rec *abi_path;
struct cm_abi_sidr_req *cmd;
void *msg;
int result;
@@ -812,7 +732,7 @@
if (!abi_path)
return -ENOMEM;
- cm_param_path_get(abi_path, param->path);
+ ib_copy_path_rec_to_kern(abi_path, param->path);
cmd->path = (uintptr_t) abi_path;
}
@@ -862,39 +782,6 @@
return 0;
}
-/*
- * event processing
- */
-static void cm_event_path_get(struct ib_sa_path_rec *upath,
- struct cm_abi_path_rec *kpath)
-{
- if (!kpath || !upath)
- return;
-
- memcpy(upath->dgid.raw, kpath->dgid, sizeof upath->dgid);
- memcpy(upath->sgid.raw, kpath->sgid, sizeof upath->sgid);
-
- upath->dlid = kpath->dlid;
- upath->slid = kpath->slid;
- upath->raw_traffic = kpath->raw_traffic;
- upath->flow_label = kpath->flow_label;
- upath->hop_limit = kpath->hop_limit;
- upath->traffic_class = kpath->traffic_class;
- upath->reversible = kpath->reversible;
- upath->numb_path = kpath->numb_path;
- upath->pkey = kpath->pkey;
- upath->sl = kpath->sl;
- upath->mtu_selector = kpath->mtu_selector;
- upath->mtu = kpath->mtu;
- upath->rate_selector = kpath->rate_selector;
- upath->rate = kpath->rate;
- upath->packet_life_time = kpath->packet_life_time;
- upath->preference = kpath->preference;
-
- upath->packet_life_time_selector =
- kpath->packet_life_time_selector;
-}
-
static void cm_event_req_get(struct ib_cm_req_event_param *ureq,
struct cm_abi_req_event_resp *kreq)
{
@@ -913,8 +800,10 @@
ureq->srq = kreq->srq;
ureq->port = kreq->port;
- cm_event_path_get(ureq->primary_path, &kreq->primary_path);
- cm_event_path_get(ureq->alternate_path, &kreq->alternate_path);
+ ib_copy_path_rec_from_kern(ureq->primary_path, &kreq->primary_path);
+ if (ureq->alternate_path)
+ ib_copy_path_rec_from_kern(ureq->alternate_path,
+ &kreq->alternate_path);
}
static void cm_event_rep_get(struct ib_cm_rep_event_param *urep,
@@ -1058,8 +947,8 @@
case IB_CM_LAP_RECEIVED:
evt->param.lap_rcvd.alternate_path = path_b;
path_b = NULL;
- cm_event_path_get(evt->param.lap_rcvd.alternate_path,
- &resp->u.lap_resp.path);
+ ib_copy_path_rec_from_kern(evt->param.lap_rcvd.alternate_path,
+ &resp->u.lap_resp.path);
break;
case IB_CM_APR_RECEIVED:
evt->param.apr_rcvd.ap_status = resp->u.apr_resp.status;
Index: userspace/libibcm/src/libibcm.map
===================================================================
--- userspace/libibcm/src/libibcm.map (revision 4017)
+++ userspace/libibcm/src/libibcm.map (working copy)
@@ -1,9 +1,9 @@
-IBCM_1.0 {
+IBCM_4.0 {
global:
- ib_cm_event_get;
- ib_cm_event_put;
- ib_cm_get_fd;
+ ib_cm_get_event;
+ ib_cm_ack_event;
+ ib_cm_get_device;
ib_cm_create_id;
ib_cm_destroy_id;
ib_cm_attr_id;
@@ -20,5 +20,6 @@
ib_cm_send_apr;
ib_cm_send_sidr_req;
ib_cm_send_sidr_rep;
+ ib_cm_init_qp_attr;
local: *;
};
Index: userspace/libibcm/Makefile.am
===================================================================
--- userspace/libibcm/Makefile.am (revision 4017)
+++ userspace/libibcm/Makefile.am (working copy)
@@ -18,10 +18,6 @@
src_libibcm_la_SOURCES = src/cm.c
src_libibcm_la_LDFLAGS = -avoid-version $(ucm_version_script)
-bin_PROGRAMS = examples/ucmpost
-examples_ucmpost_SOURCES = examples/cmpost.c
-examples_ucmpost_LDADD = $(top_builddir)/src/libibcm.la
-
libibcmincludedir = $(includedir)/infiniband
libibcminclude_HEADERS = include/infiniband/cm_abi.h \
Index: userspace/libibcm/README
===================================================================
--- userspace/libibcm/README (revision 4017)
+++ userspace/libibcm/README (working copy)
@@ -12,18 +12,17 @@
Device files
-The userspace CM uses a single device file regardless of the number
-of adapters or ports present.
+The userspace CM uses a device file per adapter present.
To create the appropriate character device file automatically with
udev, a rule like
- KERNEL="ucm", NAME="infiniband/%k", MODE="0666"
+ KERNEL="ucm*", NAME="infiniband/%k", MODE="0666"
can be used. This will create the device node named
- /dev/infiniband/ucm
+ /dev/infiniband/ucm0
-or you can create it manually
+for the first HCA in the system, or you can create it manually
- mknod /dev/infiniband/ucm c 231 255
+ mknod /dev/infiniband/ucm0 c 231 255
Index: linux-kernel/infiniband/include/rdma/ib_user_verbs.h
===================================================================
--- linux-kernel/infiniband/include/rdma/ib_user_verbs.h (revision 4017)
+++ linux-kernel/infiniband/include/rdma/ib_user_verbs.h (working copy)
@@ -311,6 +311,64 @@
__u32 async_events_reported;
};
+struct ib_uverbs_global_route {
+ __u8 dgid[16];
+ __u32 flow_label;
+ __u8 sgid_index;
+ __u8 hop_limit;
+ __u8 traffic_class;
+ __u8 reserved;
+};
+
+struct ib_uverbs_ah_attr {
+ struct ib_uverbs_global_route grh;
+ __u16 dlid;
+ __u8 sl;
+ __u8 src_path_bits;
+ __u8 static_rate;
+ __u8 is_global;
+ __u8 port_num;
+ __u8 reserved;
+};
+
+struct ib_uverbs_qp_attr {
+ __u32 qp_attr_mask;
+ __u32 qp_state;
+ __u32 cur_qp_state;
+ __u32 path_mtu;
+ __u32 path_mig_state;
+ __u32 qkey;
+ __u32 rq_psn;
+ __u32 sq_psn;
+ __u32 dest_qp_num;
+ __u32 qp_access_flags;
+
+ struct ib_uverbs_ah_attr ah_attr;
+ struct ib_uverbs_ah_attr alt_ah_attr;
+
+ /* ib_qp_cap */
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+
+ __u16 pkey_index;
+ __u16 alt_pkey_index;
+ __u8 en_sqd_async_notify;
+ __u8 sq_draining;
+ __u8 max_rd_atomic;
+ __u8 max_dest_rd_atomic;
+ __u8 min_rnr_timer;
+ __u8 port_num;
+ __u8 timeout;
+ __u8 retry_cnt;
+ __u8 rnr_retry;
+ __u8 alt_port_num;
+ __u8 alt_timeout;
+ __u8 reserved[5];
+};
+
struct ib_uverbs_create_qp {
__u64 response;
__u64 user_handle;
@@ -487,26 +545,6 @@
__u32 bad_wr;
};
-struct ib_uverbs_global_route {
- __u8 dgid[16];
- __u32 flow_label;
- __u8 sgid_index;
- __u8 hop_limit;
- __u8 traffic_class;
- __u8 reserved;
-};
-
-struct ib_uverbs_ah_attr {
- struct ib_uverbs_global_route grh;
- __u16 dlid;
- __u8 sl;
- __u8 src_path_bits;
- __u8 static_rate;
- __u8 is_global;
- __u8 port_num;
- __u8 reserved;
-};
-
struct ib_uverbs_create_ah {
__u64 response;
__u64 user_handle;
Index: linux-kernel/infiniband/include/rdma/rdma_user_cm.h
===================================================================
--- linux-kernel/infiniband/include/rdma/rdma_user_cm.h (revision 0)
+++ linux-kernel/infiniband/include/rdma/rdma_user_cm.h (revision 0)
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RDMA_USER_CM_H
+#define RDMA_USER_CM_H
+
+#include <linux/types.h>
+#include <linux/in6.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_user_sa.h>
+
+#define RDMA_USER_CM_ABI_VERSION 1
+
+#define RDMA_MAX_PRIVATE_DATA 256
+
+enum {
+ RDMA_USER_CM_CMD_CREATE_ID,
+ RDMA_USER_CM_CMD_DESTROY_ID,
+ RDMA_USER_CM_CMD_BIND_ADDR,
+ RDMA_USER_CM_CMD_RESOLVE_ADDR,
+ RDMA_USER_CM_CMD_RESOLVE_ROUTE,
+ RDMA_USER_CM_CMD_QUERY_ROUTE,
+ RDMA_USER_CM_CMD_CONNECT,
+ RDMA_USER_CM_CMD_LISTEN,
+ RDMA_USER_CM_CMD_ACCEPT,
+ RDMA_USER_CM_CMD_REJECT,
+ RDMA_USER_CM_CMD_DISCONNECT,
+ RDMA_USER_CM_CMD_INIT_QP_ATTR,
+ RDMA_USER_CM_CMD_GET_EVENT
+};
+
+/*
+ * command ABI structures.
+ */
+struct rdma_ucm_cmd_hdr {
+ __u32 cmd;
+ __u16 in;
+ __u16 out;
+};
+
+struct rdma_ucm_create_id {
+ __u64 uid;
+ __u64 response;
+};
+
+struct rdma_ucm_create_id_resp {
+ __u32 id;
+};
+
+struct rdma_ucm_destroy_id {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct rdma_ucm_destroy_id_resp {
+ __u32 events_reported;
+};
+
+struct rdma_ucm_bind_addr {
+ __u64 response;
+ struct sockaddr_in6 addr;
+ __u32 id;
+};
+
+struct rdma_ucm_bind_addr_resp {
+ __u64 node_guid;
+};
+
+struct rdma_ucm_resolve_addr {
+ struct sockaddr_in6 src_addr;
+ struct sockaddr_in6 dst_addr;
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct rdma_ucm_resolve_route {
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct rdma_ucm_query_route {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct rdma_ucm_query_route_resp {
+ __u64 node_guid;
+ struct ib_user_path_rec ib_route[2];
+ struct sockaddr_in6 src_addr;
+ __u32 num_paths;
+};
+
+struct rdma_ucm_conn_param {
+ __u32 qp_num;
+ __u32 qp_type;
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+ __u8 private_data_len;
+ __u8 srq;
+ __u8 responder_resources;
+ __u8 initiator_depth;
+ __u8 flow_control;
+ __u8 retry_count;
+ __u8 rnr_retry_count;
+ __u8 valid;
+};
+
+struct rdma_ucm_connect {
+ struct rdma_ucm_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct rdma_ucm_listen {
+ __u32 id;
+ __u32 backlog;
+};
+
+struct rdma_ucm_accept {
+ __u64 uid;
+ struct rdma_ucm_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct rdma_ucm_reject {
+ __u32 id;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+struct rdma_ucm_disconnect {
+ __u32 id;
+};
+
+struct rdma_ucm_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct rdma_ucm_get_event {
+ __u64 response;
+};
+
+struct rdma_ucm_event_resp {
+ __u64 uid;
+ __u32 id;
+ __u32 event;
+ __u32 status;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+#endif /* RDMA_USER_CM_H */
Index: linux-kernel/infiniband/include/rdma/rdma_cm.h
===================================================================
--- linux-kernel/infiniband/include/rdma/rdma_cm.h (revision 4017)
+++ linux-kernel/infiniband/include/rdma/rdma_cm.h (working copy)
@@ -44,6 +44,7 @@
RDMA_CM_EVENT_ROUTE_RESOLVED,
RDMA_CM_EVENT_ROUTE_ERROR,
RDMA_CM_EVENT_CONNECT_REQUEST,
+ RDMA_CM_EVENT_CONNECT_RESPONSE,
RDMA_CM_EVENT_CONNECT_ERROR,
RDMA_CM_EVENT_UNREACHABLE,
RDMA_CM_EVENT_REJECTED,
@@ -137,6 +138,9 @@
/**
* rdma_create_qp - Allocate a QP and associate it with the specified RDMA
* identifier.
+ *
+ * QPs allocated to an rdma_cm_id will automatically be transitioned by the CMA
+ * through their states.
*/
int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
struct ib_qp_init_attr *qp_init_attr);
@@ -150,6 +154,28 @@
*/
void rdma_destroy_qp(struct rdma_cm_id *id);
+/**
+ * rdma_init_qp_attr - Initializes the QP attributes for use in transitioning
+ * to a specified QP state.
+ * @id: Communication identifier associated with the QP attributes to
+ * initialize.
+ * @qp_attr: On input, specifies the desired QP state. On output, the
+ * mandatory and desired optional attributes will be set in order to
+ * modify the QP to the specified state.
+ * @qp_attr_mask: The QP attribute mask that may be used to transition the
+ * QP to the specified state.
+ *
+ * Users must set the @qp_attr->qp_state to the desired QP state. This call
+ * will set all required attributes for the given transition, along with
+ * known optional attributes. Users may override the attributes returned from
+ * this call before calling ib_modify_qp.
+ *
+ * Users that wish to have their QP automatically transitioned through its
+ * states can associate a QP with the rdma_cm_id by calling rdma_create_qp().
+ */
+int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
+ int *qp_attr_mask);
+
struct rdma_conn_param {
const void *private_data;
u8 private_data_len;
@@ -158,6 +184,10 @@
u8 flow_control;
u8 retry_count; /* ignored when accepting */
u8 rnr_retry_count;
+ /* Fields below ignored if a QP is created on the rdma_cm_id. */
+ u8 srq;
+ u32 qp_num;
+ enum ib_qp_type qp_type;
};
/**
@@ -175,10 +205,18 @@
* Users must have bound the rdma_cm_id to a local address by calling
* rdma_bind_addr before calling this routine.
*/
-int rdma_listen(struct rdma_cm_id *id);
+int rdma_listen(struct rdma_cm_id *id, int backlog);
/**
- * rdma_accept - Called on the passive side to accept a connection request
+ * rdma_accept - Called to accept a connection request or response.
+ * @id: Connection identifier associated with the request.
+ * @conn_param: Information needed to establish the connection. This must be
+ * provided if accepting a connection request. If accepting a connection
+ * response, this parameter must be NULL.
+ *
+ * Typically, this routine is only called by the listener to accept a connection
+ * request. It must also be called on the active side of a connection if the
+ * user is performing their own QP transitions.
*/
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
Index: linux-kernel/infiniband/include/rdma/ib_user_cm.h
===================================================================
--- linux-kernel/infiniband/include/rdma/ib_user_cm.h (revision 4017)
+++ linux-kernel/infiniband/include/rdma/ib_user_cm.h (working copy)
@@ -36,7 +36,7 @@
#ifndef IB_USER_CM_H
#define IB_USER_CM_H
-#include <linux/types.h>
+#include <rdma/ib_user_sa.h>
#define IB_USER_CM_ABI_VERSION 4
@@ -110,58 +110,6 @@
__u32 qp_state;
};
-struct ib_ucm_ah_attr {
- __u8 grh_dgid[16];
- __u32 grh_flow_label;
- __u16 dlid;
- __u16 reserved;
- __u8 grh_sgid_index;
- __u8 grh_hop_limit;
- __u8 grh_traffic_class;
- __u8 sl;
- __u8 src_path_bits;
- __u8 static_rate;
- __u8 is_global;
- __u8 port_num;
-};
-
-struct ib_ucm_init_qp_attr_resp {
- __u32 qp_attr_mask;
- __u32 qp_state;
- __u32 cur_qp_state;
- __u32 path_mtu;
- __u32 path_mig_state;
- __u32 qkey;
- __u32 rq_psn;
- __u32 sq_psn;
- __u32 dest_qp_num;
- __u32 qp_access_flags;
-
- struct ib_ucm_ah_attr ah_attr;
- struct ib_ucm_ah_attr alt_ah_attr;
-
- /* ib_qp_cap */
- __u32 max_send_wr;
- __u32 max_recv_wr;
- __u32 max_send_sge;
- __u32 max_recv_sge;
- __u32 max_inline_data;
-
- __u16 pkey_index;
- __u16 alt_pkey_index;
- __u8 en_sqd_async_notify;
- __u8 sq_draining;
- __u8 max_rd_atomic;
- __u8 max_dest_rd_atomic;
- __u8 min_rnr_timer;
- __u8 port_num;
- __u8 timeout;
- __u8 retry_cnt;
- __u8 rnr_retry;
- __u8 alt_port_num;
- __u8 alt_timeout;
-};
-
struct ib_ucm_listen {
__be64 service_id;
__be64 service_mask;
@@ -180,28 +128,6 @@
__u8 reserved[3];
};
-struct ib_ucm_path_rec {
- __u8 dgid[16];
- __u8 sgid[16];
- __be16 dlid;
- __be16 slid;
- __u32 raw_traffic;
- __be32 flow_label;
- __u32 reversible;
- __u32 mtu;
- __be16 pkey;
- __u8 hop_limit;
- __u8 traffic_class;
- __u8 numb_path;
- __u8 sl;
- __u8 mtu_selector;
- __u8 rate_selector;
- __u8 rate;
- __u8 packet_life_time_selector;
- __u8 packet_life_time;
- __u8 preference;
-};
-
struct ib_ucm_req {
__u32 id;
__u32 qpn;
@@ -304,8 +230,8 @@
};
struct ib_ucm_req_event_resp {
- struct ib_ucm_path_rec primary_path;
- struct ib_ucm_path_rec alternate_path;
+ struct ib_user_path_rec primary_path;
+ struct ib_user_path_rec alternate_path;
__be64 remote_ca_guid;
__u32 remote_qkey;
__u32 remote_qpn;
@@ -349,7 +275,7 @@
};
struct ib_ucm_lap_event_resp {
- struct ib_ucm_path_rec path;
+ struct ib_user_path_rec path;
};
struct ib_ucm_apr_event_resp {
Index: linux-kernel/infiniband/include/rdma/ib_user_sa.h
===================================================================
--- linux-kernel/infiniband/include/rdma/ib_user_sa.h (revision 0)
+++ linux-kernel/infiniband/include/rdma/ib_user_sa.h (revision 0)
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef IB_USER_SA_H
+#define IB_USER_SA_H
+
+#include <linux/types.h>
+
+struct ib_user_path_rec {
+ __u8 dgid[16];
+ __u8 sgid[16];
+ __be16 dlid;
+ __be16 slid;
+ __u32 raw_traffic;
+ __be32 flow_label;
+ __u32 reversible;
+ __u32 mtu;
+ __be16 pkey;
+ __u8 hop_limit;
+ __u8 traffic_class;
+ __u8 numb_path;
+ __u8 sl;
+ __u8 mtu_selector;
+ __u8 rate_selector;
+ __u8 rate;
+ __u8 packet_life_time_selector;
+ __u8 packet_life_time;
+ __u8 preference;
+};
+
+#endif /* IB_USER_SA_H */
Index: linux-kernel/infiniband/include/rdma/ib_marshall.h
===================================================================
--- linux-kernel/infiniband/include/rdma/ib_marshall.h (revision 0)
+++ linux-kernel/infiniband/include/rdma/ib_marshall.h (revision 0)
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(IB_USER_MARSHALL_H)
+#define IB_USER_MARSHALL_H
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_sa.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_user_sa.h>
+
+void ib_copy_qp_attr_to_user(struct ib_uverbs_qp_attr *dst,
+ struct ib_qp_attr *src);
+
+void ib_copy_path_rec_to_user(struct ib_user_path_rec *dst,
+ struct ib_sa_path_rec *src);
+
+void ib_copy_path_rec_from_user(struct ib_sa_path_rec *dst,
+ struct ib_user_path_rec *src);
+
+#endif /* IB_USER_MARSHALL_H */
Index: linux-kernel/infiniband/core/uverbs_marshall.c
===================================================================
--- linux-kernel/infiniband/core/uverbs_marshall.c (revision 0)
+++ linux-kernel/infiniband/core/uverbs_marshall.c (revision 0)
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <rdma/ib_marshall.h>
+
+static void ib_copy_ah_attr_to_user(struct ib_uverbs_ah_attr *dst,
+ struct ib_ah_attr *src)
+{
+ memcpy(dst->grh.dgid, src->grh.dgid.raw, sizeof src->grh.dgid);
+ dst->grh.flow_label = src->grh.flow_label;
+ dst->grh.sgid_index = src->grh.sgid_index;
+ dst->grh.hop_limit = src->grh.hop_limit;
+ dst->grh.traffic_class = src->grh.traffic_class;
+ dst->dlid = src->dlid;
+ dst->sl = src->sl;
+ dst->src_path_bits = src->src_path_bits;
+ dst->static_rate = src->static_rate;
+ dst->is_global = src->ah_flags & IB_AH_GRH ? 1 : 0;
+ dst->port_num = src->port_num;
+}
+
+void ib_copy_qp_attr_to_user(struct ib_uverbs_qp_attr *dst,
+ struct ib_qp_attr *src)
+{
+ dst->cur_qp_state = src->cur_qp_state;
+ dst->path_mtu = src->path_mtu;
+ dst->path_mig_state = src->path_mig_state;
+ dst->qkey = src->qkey;
+ dst->rq_psn = src->rq_psn;
+ dst->sq_psn = src->sq_psn;
+ dst->dest_qp_num = src->dest_qp_num;
+ dst->qp_access_flags = src->qp_access_flags;
+
+ dst->max_send_wr = src->cap.max_send_wr;
+ dst->max_recv_wr = src->cap.max_recv_wr;
+ dst->max_send_sge = src->cap.max_send_sge;
+ dst->max_recv_sge = src->cap.max_recv_sge;
+ dst->max_inline_data = src->cap.max_inline_data;
+
+ ib_copy_ah_attr_to_user(&dst->ah_attr, &src->ah_attr);
+ ib_copy_ah_attr_to_user(&dst->alt_ah_attr, &src->alt_ah_attr);
+
+ dst->pkey_index = src->pkey_index;
+ dst->alt_pkey_index = src->alt_pkey_index;
+ dst->en_sqd_async_notify = src->en_sqd_async_notify;
+ dst->sq_draining = src->sq_draining;
+ dst->max_rd_atomic = src->max_rd_atomic;
+ dst->max_dest_rd_atomic = src->max_dest_rd_atomic;
+ dst->min_rnr_timer = src->min_rnr_timer;
+ dst->port_num = src->port_num;
+ dst->timeout = src->timeout;
+ dst->retry_cnt = src->retry_cnt;
+ dst->rnr_retry = src->rnr_retry;
+ dst->alt_port_num = src->alt_port_num;
+ dst->alt_timeout = src->alt_timeout;
+}
+EXPORT_SYMBOL(ib_copy_qp_attr_to_user);
+
+void ib_copy_path_rec_to_user(struct ib_user_path_rec *dst,
+ struct ib_sa_path_rec *src)
+{
+ memcpy(dst->dgid, src->dgid.raw, sizeof src->dgid);
+ memcpy(dst->sgid, src->sgid.raw, sizeof src->sgid);
+
+ dst->dlid = src->dlid;
+ dst->slid = src->slid;
+ dst->raw_traffic = src->raw_traffic;
+ dst->flow_label = src->flow_label;
+ dst->hop_limit = src->hop_limit;
+ dst->traffic_class = src->traffic_class;
+ dst->reversible = src->reversible;
+ dst->numb_path = src->numb_path;
+ dst->pkey = src->pkey;
+ dst->sl = src->sl;
+ dst->mtu_selector = src->mtu_selector;
+ dst->mtu = src->mtu;
+ dst->rate_selector = src->rate_selector;
+ dst->rate = src->rate;
+ dst->packet_life_time = src->packet_life_time;
+ dst->preference = src->preference;
+ dst->packet_life_time_selector = src->packet_life_time_selector;
+}
+EXPORT_SYMBOL(ib_copy_path_rec_to_user);
+
+void ib_copy_path_rec_from_user(struct ib_sa_path_rec *dst,
+ struct ib_user_path_rec *src)
+{
+ memcpy(dst->dgid.raw, src->dgid, sizeof dst->dgid);
+ memcpy(dst->sgid.raw, src->sgid, sizeof dst->sgid);
+
+ dst->dlid = src->dlid;
+ dst->slid = src->slid;
+ dst->raw_traffic = src->raw_traffic;
+ dst->flow_label = src->flow_label;
+ dst->hop_limit = src->hop_limit;
+ dst->traffic_class = src->traffic_class;
+ dst->reversible = src->reversible;
+ dst->numb_path = src->numb_path;
+ dst->pkey = src->pkey;
+ dst->sl = src->sl;
+ dst->mtu_selector = src->mtu_selector;
+ dst->mtu = src->mtu;
+ dst->rate_selector = src->rate_selector;
+ dst->rate = src->rate;
+ dst->packet_life_time = src->packet_life_time;
+ dst->preference = src->preference;
+ dst->packet_life_time_selector = src->packet_life_time_selector;
+}
+EXPORT_SYMBOL(ib_copy_path_rec_from_user);
Index: linux-kernel/infiniband/core/Makefile
===================================================================
--- linux-kernel/infiniband/core/Makefile (revision 4017)
+++ linux-kernel/infiniband/core/Makefile (working copy)
@@ -3,7 +3,7 @@
obj-$(CONFIG_INFINIBAND) += ib_core.o ib_mad.o ib_ping.o ib_cm.o \
ib_sa.o ib_at.o ib_addr.o rdma_cm.o
obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o
-obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o ib_ucm.o ib_uat.o
+obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o ib_ucm.o ib_uat.o rdma_ucm.o
ib_core-y := packer.o ud_header.o verbs.o sysfs.o \
device.o fmr_pool.o cache.o
@@ -16,13 +16,16 @@
rdma_cm-y := cma.o
+rdma_ucm-y := ucma.o
+
ib_addr-y := addr.o
ib_sa-y := sa_query.o
ib_umad-y := user_mad.o
-ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_mem.o
+ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_mem.o \
+ uverbs_marshall.o
ib_ucm-y := ucm.o
Index: linux-kernel/infiniband/core/cma.c
===================================================================
--- linux-kernel/infiniband/core/cma.c (revision 4017)
+++ linux-kernel/infiniband/core/cma.c (working copy)
@@ -101,6 +101,10 @@
struct ib_sa_query *query;
int query_id;
struct ib_cm_id *cm_id;
+
+ u32 qp_num;
+ enum ib_qp_type qp_type;
+ u8 srq;
};
struct cma_addr {
@@ -294,6 +298,9 @@
goto err;
id->qp = qp;
+ id_priv->qp_num = qp->qp_num;
+ id_priv->qp_type = qp->qp_type;
+ id_priv->srq = (qp->srq != NULL);
return 0;
err:
ib_destroy_qp(qp);
@@ -307,51 +314,82 @@
}
EXPORT_SYMBOL(rdma_destroy_qp);
-static int cma_modify_ib_qp_rtr(struct rdma_id_private *id_priv)
+static int cma_modify_qp_rtr(struct rdma_cm_id *id)
{
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
+ if (!id->qp)
+ return 0;
+
/* Need to update QP attributes from default values. */
qp_attr.qp_state = IB_QPS_INIT;
- ret = ib_cm_init_qp_attr(id_priv->cm_id, &qp_attr, &qp_attr_mask);
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
if (ret)
return ret;
- ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
+ ret = ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
if (ret)
return ret;
qp_attr.qp_state = IB_QPS_RTR;
- ret = ib_cm_init_qp_attr(id_priv->cm_id, &qp_attr, &qp_attr_mask);
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
if (ret)
return ret;
- qp_attr.rq_psn = id_priv->id.qp->qp_num;
- return ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
+ return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
-static int cma_modify_ib_qp_rts(struct rdma_id_private *id_priv)
+static int cma_modify_qp_rts(struct rdma_cm_id *id)
{
struct ib_qp_attr qp_attr;
int qp_attr_mask, ret;
+ if (!id->qp)
+ return 0;
+
qp_attr.qp_state = IB_QPS_RTS;
- ret = ib_cm_init_qp_attr(id_priv->cm_id, &qp_attr, &qp_attr_mask);
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
if (ret)
return ret;
- return ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
+ return ib_modify_qp(id->qp, &qp_attr, qp_attr_mask);
}
static int cma_modify_qp_err(struct rdma_cm_id *id)
{
struct ib_qp_attr qp_attr;
+ if (!id->qp)
+ return 0;
+
qp_attr.qp_state = IB_QPS_ERR;
return ib_modify_qp(id->qp, &qp_attr, IB_QP_STATE);
}
+int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
+ int *qp_attr_mask)
+{
+ struct rdma_id_private *id_priv;
+ int ret;
+
+ id_priv = container_of(id, struct rdma_id_private, id);
+ switch (id_priv->id.device->node_type) {
+ case IB_NODE_CA:
+ ret = ib_cm_init_qp_attr(id_priv->cm_id, qp_attr,
+ qp_attr_mask);
+ if (qp_attr->qp_state == IB_QPS_RTR)
+ qp_attr->rq_psn = id_priv->qp_num;
+ break;
+ default:
+ ret = -ENOSYS;
+ break;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(rdma_init_qp_attr);
+
static int cma_verify_addr(struct cma_addr *addr,
struct sockaddr_in *ip_addr)
{
@@ -497,14 +535,14 @@
{
int ret;
- ret = cma_modify_ib_qp_rtr(id_priv);
+ ret = cma_modify_qp_rtr(&id_priv->id);
if (ret)
goto reject;
- ret = cma_modify_ib_qp_rts(id_priv);
+ ret = cma_modify_qp_rts(&id_priv->id);
if (ret)
goto reject;
-
+
ret = ib_send_cm_rtu(id_priv->cm_id, NULL, 0);
if (ret)
goto reject;
@@ -521,7 +559,7 @@
{
int ret;
- ret = cma_modify_ib_qp_rts(id_priv);
+ ret = cma_modify_qp_rts(&id_priv->id);
if (ret)
goto reject;
@@ -551,9 +589,12 @@
status = -ETIMEDOUT;
break;
case IB_CM_REP_RECEIVED:
- status = cma_rep_recv(id_priv);
- event = status ? RDMA_CM_EVENT_CONNECT_ERROR :
- RDMA_CM_EVENT_ESTABLISHED;
+ if (id_priv->id.qp) {
+ status = cma_rep_recv(id_priv);
+ event = status ? RDMA_CM_EVENT_CONNECT_ERROR :
+ RDMA_CM_EVENT_ESTABLISHED;
+ } else
+ event = RDMA_CM_EVENT_CONNECT_RESPONSE;
private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
break;
case IB_CM_RTU_RECEIVED:
@@ -765,7 +806,7 @@
cma_attach_to_dev(dev_id_priv, cma_dev);
list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
- ret = rdma_listen(id);
+ ret = rdma_listen(id, 0);
if (ret)
goto err;
@@ -792,7 +833,7 @@
return ret;
}
-int rdma_listen(struct rdma_cm_id *id)
+int rdma_listen(struct rdma_cm_id *id, int backlog)
{
struct rdma_id_private *id_priv;
int ret;
@@ -877,7 +918,7 @@
memset(&path_rec, 0, sizeof path_rec);
path_rec.sgid = addr->sgid;
path_rec.dgid = addr->dgid;
- path_rec.pkey = addr->pkey;
+ path_rec.pkey = cpu_to_be16(addr->pkey);
path_rec.numb_path = 1;
id_priv->query_id = ib_sa_path_rec_get(id_priv->id.device,
@@ -1062,8 +1103,8 @@
req.alternate_path = &route->path_rec[1];
req.service_id = cma_get_service_id(&route->addr.dst_addr);
- req.qp_num = id_priv->id.qp->qp_num;
- req.qp_type = IB_QPT_RC;
+ req.qp_num = id_priv->qp_num;
+ req.qp_type = id_priv->qp_type;
req.starting_psn = req.qp_num;
req.responder_resources = conn_param->responder_resources;
req.initiator_depth = conn_param->initiator_depth;
@@ -1073,7 +1114,7 @@
req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
req.max_cm_retries = CMA_MAX_CM_RETRIES;
- req.srq = id_priv->id.qp->srq ? 1 : 0;
+ req.srq = id_priv->srq ? 1 : 0;
ret = ib_send_cm_req(id_priv->cm_id, &req);
out:
@@ -1090,6 +1131,12 @@
if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
return -EINVAL;
+ if (!id->qp) {
+ id_priv->qp_num = conn_param->qp_num;
+ id_priv->qp_type = conn_param->qp_type;
+ id_priv->srq = conn_param->srq;
+ }
+
switch (id->device->node_type) {
case IB_NODE_CA:
ret = cma_connect_ib(id_priv, conn_param);
@@ -1114,12 +1161,12 @@
struct ib_cm_rep_param rep;
int ret;
- ret = cma_modify_ib_qp_rtr(id_priv);
+ ret = cma_modify_qp_rtr(&id_priv->id);
if (ret)
return ret;
memset(&rep, 0, sizeof rep);
- rep.qp_num = id_priv->id.qp->qp_num;
+ rep.qp_num = id_priv->qp_num;
rep.starting_psn = rep.qp_num;
rep.private_data = conn_param->private_data;
rep.private_data_len = conn_param->private_data_len;
@@ -1129,7 +1176,7 @@
rep.failover_accepted = 0;
rep.flow_control = conn_param->flow_control;
rep.rnr_retry_count = conn_param->rnr_retry_count;
- rep.srq = id_priv->id.qp->srq ? 1 : 0;
+ rep.srq = id_priv->srq ? 1 : 0;
return ib_send_cm_rep(id_priv->cm_id, &rep);
}
@@ -1143,9 +1190,18 @@
if (!cma_comp(id_priv, CMA_CONNECT))
return -EINVAL;
+ if (!id->qp && conn_param) {
+ id_priv->qp_num = conn_param->qp_num;
+ id_priv->qp_type = conn_param->qp_type;
+ id_priv->srq = conn_param->srq;
+ }
+
switch (id->device->node_type) {
case IB_NODE_CA:
- ret = cma_accept_ib(id_priv, conn_param);
+ if (conn_param)
+ ret = cma_accept_ib(id_priv, conn_param);
+ else
+ ret = cma_rep_recv(id_priv);
break;
default:
ret = -ENOSYS;
Index: linux-kernel/infiniband/core/ucm.c
===================================================================
--- linux-kernel/infiniband/core/ucm.c (revision 4017)
+++ linux-kernel/infiniband/core/ucm.c (working copy)
@@ -47,6 +47,7 @@
#include <rdma/ib_cm.h>
#include <rdma/ib_user_cm.h>
+#include <rdma/ib_marshall.h>
MODULE_AUTHOR("Libor Michalek");
MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");
@@ -202,36 +203,6 @@
return NULL;
}
-static void ib_ucm_event_path_get(struct ib_ucm_path_rec *upath,
- struct ib_sa_path_rec *kpath)
-{
- if (!kpath || !upath)
- return;
-
- memcpy(upath->dgid, kpath->dgid.raw, sizeof *upath->dgid);
- memcpy(upath->sgid, kpath->sgid.raw, sizeof *upath->sgid);
-
- upath->dlid = kpath->dlid;
- upath->slid = kpath->slid;
- upath->raw_traffic = kpath->raw_traffic;
- upath->flow_label = kpath->flow_label;
- upath->hop_limit = kpath->hop_limit;
- upath->traffic_class = kpath->traffic_class;
- upath->reversible = kpath->reversible;
- upath->numb_path = kpath->numb_path;
- upath->pkey = kpath->pkey;
- upath->sl = kpath->sl;
- upath->mtu_selector = kpath->mtu_selector;
- upath->mtu = kpath->mtu;
- upath->rate_selector = kpath->rate_selector;
- upath->rate = kpath->rate;
- upath->packet_life_time = kpath->packet_life_time;
- upath->preference = kpath->preference;
-
- upath->packet_life_time_selector =
- kpath->packet_life_time_selector;
-}
-
static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq,
struct ib_cm_req_event_param *kreq)
{
@@ -250,8 +221,10 @@
ureq->srq = kreq->srq;
ureq->port = kreq->port;
- ib_ucm_event_path_get(&ureq->primary_path, kreq->primary_path);
- ib_ucm_event_path_get(&ureq->alternate_path, kreq->alternate_path);
+ ib_copy_path_rec_to_user(&ureq->primary_path, kreq->primary_path);
+ if (kreq->alternate_path)
+ ib_copy_path_rec_to_user(&ureq->alternate_path,
+ kreq->alternate_path);
}
static void ib_ucm_event_rep_get(struct ib_ucm_rep_event_resp *urep,
@@ -321,8 +294,8 @@
info = evt->param.rej_rcvd.ari;
break;
case IB_CM_LAP_RECEIVED:
- ib_ucm_event_path_get(&uvt->resp.u.lap_resp.path,
- evt->param.lap_rcvd.alternate_path);
+ ib_copy_path_rec_to_user(&uvt->resp.u.lap_resp.path,
+ evt->param.lap_rcvd.alternate_path);
uvt->data_len = IB_CM_LAP_PRIVATE_DATA_SIZE;
uvt->resp.present = IB_UCM_PRES_ALTERNATE;
break;
@@ -634,65 +607,11 @@
return result;
}
-static void ib_ucm_copy_ah_attr(struct ib_ucm_ah_attr *dest_attr,
- struct ib_ah_attr *src_attr)
-{
- memcpy(dest_attr->grh_dgid, src_attr->grh.dgid.raw,
- sizeof src_attr->grh.dgid);
- dest_attr->grh_flow_label = src_attr->grh.flow_label;
- dest_attr->grh_sgid_index = src_attr->grh.sgid_index;
- dest_attr->grh_hop_limit = src_attr->grh.hop_limit;
- dest_attr->grh_traffic_class = src_attr->grh.traffic_class;
-
- dest_attr->dlid = src_attr->dlid;
- dest_attr->sl = src_attr->sl;
- dest_attr->src_path_bits = src_attr->src_path_bits;
- dest_attr->static_rate = src_attr->static_rate;
- dest_attr->is_global = (src_attr->ah_flags & IB_AH_GRH);
- dest_attr->port_num = src_attr->port_num;
-}
-
-static void ib_ucm_copy_qp_attr(struct ib_ucm_init_qp_attr_resp *dest_attr,
- struct ib_qp_attr *src_attr)
-{
- dest_attr->cur_qp_state = src_attr->cur_qp_state;
- dest_attr->path_mtu = src_attr->path_mtu;
- dest_attr->path_mig_state = src_attr->path_mig_state;
- dest_attr->qkey = src_attr->qkey;
- dest_attr->rq_psn = src_attr->rq_psn;
- dest_attr->sq_psn = src_attr->sq_psn;
- dest_attr->dest_qp_num = src_attr->dest_qp_num;
- dest_attr->qp_access_flags = src_attr->qp_access_flags;
-
- dest_attr->max_send_wr = src_attr->cap.max_send_wr;
- dest_attr->max_recv_wr = src_attr->cap.max_recv_wr;
- dest_attr->max_send_sge = src_attr->cap.max_send_sge;
- dest_attr->max_recv_sge = src_attr->cap.max_recv_sge;
- dest_attr->max_inline_data = src_attr->cap.max_inline_data;
-
- ib_ucm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
- ib_ucm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
-
- dest_attr->pkey_index = src_attr->pkey_index;
- dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
- dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
- dest_attr->sq_draining = src_attr->sq_draining;
- dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
- dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
- dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
- dest_attr->port_num = src_attr->port_num;
- dest_attr->timeout = src_attr->timeout;
- dest_attr->retry_cnt = src_attr->retry_cnt;
- dest_attr->rnr_retry = src_attr->rnr_retry;
- dest_attr->alt_port_num = src_attr->alt_port_num;
- dest_attr->alt_timeout = src_attr->alt_timeout;
-}
-
static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file,
const char __user *inbuf,
int in_len, int out_len)
{
- struct ib_ucm_init_qp_attr_resp resp;
+ struct ib_uverbs_qp_attr resp;
struct ib_ucm_init_qp_attr cmd;
struct ib_ucm_context *ctx;
struct ib_qp_attr qp_attr;
@@ -715,7 +634,7 @@
if (result)
goto out;
- ib_ucm_copy_qp_attr(&resp, &qp_attr);
+ ib_copy_qp_attr_to_user(&resp, &qp_attr);
if (copy_to_user((void __user *)(unsigned long)cmd.response,
&resp, sizeof(resp)))
@@ -790,7 +709,7 @@
static int ib_ucm_path_get(struct ib_sa_path_rec **path, u64 src)
{
- struct ib_ucm_path_rec ucm_path;
+ struct ib_user_path_rec upath;
struct ib_sa_path_rec *sa_path;
*path = NULL;
@@ -802,36 +721,14 @@
if (!sa_path)
return -ENOMEM;
- if (copy_from_user(&ucm_path, (void __user *)(unsigned long)src,
- sizeof(ucm_path))) {
+ if (copy_from_user(&upath, (void __user *)(unsigned long)src,
+ sizeof(upath))) {
kfree(sa_path);
return -EFAULT;
}
- memcpy(sa_path->dgid.raw, ucm_path.dgid, sizeof sa_path->dgid);
- memcpy(sa_path->sgid.raw, ucm_path.sgid, sizeof sa_path->sgid);
-
- sa_path->dlid = ucm_path.dlid;
- sa_path->slid = ucm_path.slid;
- sa_path->raw_traffic = ucm_path.raw_traffic;
- sa_path->flow_label = ucm_path.flow_label;
- sa_path->hop_limit = ucm_path.hop_limit;
- sa_path->traffic_class = ucm_path.traffic_class;
- sa_path->reversible = ucm_path.reversible;
- sa_path->numb_path = ucm_path.numb_path;
- sa_path->pkey = ucm_path.pkey;
- sa_path->sl = ucm_path.sl;
- sa_path->mtu_selector = ucm_path.mtu_selector;
- sa_path->mtu = ucm_path.mtu;
- sa_path->rate_selector = ucm_path.rate_selector;
- sa_path->rate = ucm_path.rate;
- sa_path->packet_life_time = ucm_path.packet_life_time;
- sa_path->preference = ucm_path.preference;
-
- sa_path->packet_life_time_selector =
- ucm_path.packet_life_time_selector;
-
+ ib_copy_path_rec_from_user(sa_path, &upath);
*path = sa_path;
return 0;
}
Index: linux-kernel/infiniband/core/ucma.c
===================================================================
--- linux-kernel/infiniband/core/ucma.c (revision 0)
+++ linux-kernel/infiniband/core/ucma.c (revision 0)
@@ -0,0 +1,788 @@
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/poll.h>
+#include <linux/idr.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <linux/miscdevice.h>
+
+#include <rdma/rdma_user_cm.h>
+#include <rdma/ib_marshall.h>
+#include <rdma/rdma_cm.h>
+
+MODULE_AUTHOR("Sean Hefty");
+MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access");
+MODULE_LICENSE("Dual BSD/GPL");
+
+struct ucma_file {
+ struct semaphore mutex;
+ struct file *filp;
+ struct list_head ctxs;
+ struct list_head events;
+ wait_queue_head_t poll_wait;
+};
+
+struct ucma_context {
+ int id;
+ wait_queue_head_t wait;
+ atomic_t ref;
+ int events_reported;
+
+ struct ucma_file *file;
+ struct rdma_cm_id *cm_id;
+ __u64 uid;
+
+ struct list_head events; /* list of pending events. */
+ struct list_head file_list; /* member in file ctx list */
+};
+
+struct ucma_event {
+ struct ucma_context *ctx;
+ struct list_head file_list; /* member in file event list */
+ struct list_head ctx_list; /* member in ctx event list */
+ struct rdma_cm_id *cm_id;
+ struct rdma_ucm_event_resp resp;
+};
+
+static DECLARE_MUTEX(ctx_mutex);
+static DEFINE_IDR(ctx_idr);
+
+static struct ucma_context* ucma_get_ctx(struct ucma_file *file, int id)
+{
+ struct ucma_context *ctx;
+
+ down(&ctx_mutex);
+ ctx = idr_find(&ctx_idr, id);
+ if (!ctx)
+ ctx = ERR_PTR(-ENOENT);
+ else if (ctx->file != file)
+ ctx = ERR_PTR(-EINVAL);
+ else
+ atomic_inc(&ctx->ref);
+ up(&ctx_mutex);
+
+ return ctx;
+}
+
+static void ucma_put_ctx(struct ucma_context *ctx)
+{
+ if (atomic_dec_and_test(&ctx->ref))
+ wake_up(&ctx->wait);
+}
+
+static inline int ucma_new_cm_id(int event)
+{
+ return event == RDMA_CM_EVENT_CONNECT_REQUEST;
+}
+
+static void ucma_cleanup_events(struct ucma_context *ctx)
+{
+ struct ucma_event *uevent;
+
+ down(&ctx->file->mutex);
+ list_del(&ctx->file_list);
+ while (!list_empty(&ctx->events)) {
+
+ uevent = list_entry(ctx->events.next, struct ucma_event,
+ ctx_list);
+ list_del(&uevent->file_list);
+ list_del(&uevent->ctx_list);
+
+ /* clear incoming connections. */
+ if (ucma_new_cm_id(uevent->resp.event))
+ rdma_destroy_id(uevent->cm_id);
+
+ kfree(uevent);
+ }
+ up(&ctx->file->mutex);
+}
+
+static struct ucma_context* ucma_alloc_ctx(struct ucma_file *file)
+{
+ struct ucma_context *ctx;
+ int ret;
+
+ ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return NULL;
+
+ memset(ctx, 0, sizeof *ctx);
+ atomic_set(&ctx->ref, 1);
+ init_waitqueue_head(&ctx->wait);
+ ctx->file = file;
+ INIT_LIST_HEAD(&ctx->events);
+
+ do {
+ ret = idr_pre_get(&ctx_idr, GFP_KERNEL);
+ if (!ret)
+ goto error;
+
+ down(&ctx_mutex);
+ ret = idr_get_new(&ctx_idr, ctx, &ctx->id);
+ up(&ctx_mutex);
+ } while (ret == -EAGAIN);
+
+ if (ret)
+ goto error;
+
+ list_add_tail(&ctx->file_list, &file->ctxs);
+ return ctx;
+
+error:
+ kfree(ctx);
+ return NULL;
+}
+
+static int ucma_event_handler(struct rdma_cm_id *cm_id,
+ struct rdma_cm_event *event)
+{
+ struct ucma_event *uevent;
+ struct ucma_context *ctx = cm_id->context;
+
+ uevent = kmalloc(sizeof(*uevent), GFP_KERNEL);
+ if (!uevent)
+ return ucma_new_cm_id(event->event); /* Destroy new IDs. */
+
+ memset(uevent, 0, sizeof(*uevent));
+ uevent->ctx = ctx;
+ uevent->cm_id = cm_id;
+ uevent->resp.uid = ctx->uid;
+ uevent->resp.id = ctx->id;
+ uevent->resp.event = event->event;
+ uevent->resp.status = event->status;
+ if ((uevent->resp.private_data_len = event->private_data_len))
+ memcpy(uevent->resp.private_data, event->private_data,
+ event->private_data_len);
+
+ down(&ctx->file->mutex);
+ list_add_tail(&uevent->file_list, &ctx->file->events);
+ list_add_tail(&uevent->ctx_list, &ctx->events);
+ wake_up_interruptible(&ctx->file->poll_wait);
+ up(&ctx->file->mutex);
+ return 0;
+}
+
+static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct ucma_context *ctx;
+ struct rdma_ucm_get_event cmd;
+ struct ucma_event *uevent;
+ int ret = 0;
+ DEFINE_WAIT(wait);
+
+ if (out_len < sizeof(struct rdma_ucm_event_resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ down(&file->mutex);
+ while (list_empty(&file->events)) {
+ if (file->filp->f_flags & O_NONBLOCK) {
+ ret = -EAGAIN;
+ break;
+ }
+
+ if (signal_pending(current)) {
+ ret = -ERESTARTSYS;
+ break;
+ }
+
+ prepare_to_wait(&file->poll_wait, &wait, TASK_INTERRUPTIBLE);
+ up(&file->mutex);
+ schedule();
+ down(&file->mutex);
+ finish_wait(&file->poll_wait, &wait);
+ }
+
+ if (ret)
+ goto done;
+
+ uevent = list_entry(file->events.next, struct ucma_event, file_list);
+
+ if (ucma_new_cm_id(uevent->resp.event)) {
+ ctx = ucma_alloc_ctx(file);
+ if (!ctx) {
+ ret = -ENOMEM;
+ goto done;
+ }
+
+ ctx->cm_id = uevent->cm_id;
+ ctx->cm_id->context = ctx;
+ uevent->resp.id = ctx->id;
+ }
+
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &uevent->resp, sizeof(uevent->resp))) {
+ ret = -EFAULT;
+ goto done;
+ }
+
+ list_del(&uevent->file_list);
+ list_del(&uevent->ctx_list);
+ uevent->ctx->events_reported++;
+ kfree(uevent);
+done:
+ up(&file->mutex);
+ return ret;
+}
+
+static ssize_t ucma_create_id(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_create_id cmd;
+ struct rdma_ucm_create_id_resp resp;
+ struct ucma_context *ctx;
+ int ret;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ down(&file->mutex);
+ ctx = ucma_alloc_ctx(file);
+ up(&file->mutex);
+ if (!ctx)
+ return -ENOMEM;
+
+ ctx->uid = cmd.uid;
+ ctx->cm_id = rdma_create_id(ucma_event_handler, ctx);
+ if (IS_ERR(ctx->cm_id)) {
+ ret = PTR_ERR(ctx->cm_id);
+ goto err1;
+ }
+
+ resp.id = ctx->id;
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp))) {
+ ret = -EFAULT;
+ goto err2;
+ }
+ return 0;
+
+err2:
+ rdma_destroy_id(ctx->cm_id);
+err1:
+ down(&ctx_mutex);
+ idr_remove(&ctx_idr, ctx->id);
+ up(&ctx_mutex);
+ kfree(ctx);
+ return ret;
+}
+
+static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_destroy_id cmd;
+ struct rdma_ucm_destroy_id_resp resp;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ down(&ctx_mutex);
+ ctx = idr_find(&ctx_idr, cmd.id);
+ if (!ctx)
+ ctx = ERR_PTR(-ENOENT);
+ else if (ctx->file != file)
+ ctx = ERR_PTR(-EINVAL);
+ else
+ idr_remove(&ctx_idr, ctx->id);
+ up(&ctx_mutex);
+
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ atomic_dec(&ctx->ref);
+ wait_event(ctx->wait, !atomic_read(&ctx->ref));
+
+ /* No new events will be generated after destroying the id. */
+ rdma_destroy_id(ctx->cm_id);
+ /* Cleanup events not yet reported to the user. */
+ ucma_cleanup_events(ctx);
+
+ resp.events_reported = ctx->events_reported;
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ ret = -EFAULT;
+
+ kfree(ctx);
+ return ret;
+}
+
+static ssize_t ucma_bind_addr(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_bind_addr cmd;
+ struct rdma_ucm_bind_addr_resp resp;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr);
+ if (ret)
+ goto out;
+
+ if (ctx->cm_id->device)
+ resp.node_guid = ctx->cm_id->device->node_guid;
+ else
+ resp.node_guid = 0;
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ ret = -EFAULT;
+
+out:
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_resolve_addr(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_resolve_addr cmd;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr,
+ (struct sockaddr *) &cmd.dst_addr,
+ cmd.timeout_ms);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_resolve_route(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_resolve_route cmd;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_resolve_route(ctx->cm_id, cmd.timeout_ms);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,
+ struct rdma_route *route)
+{
+ struct ib_addr *ibaddr;
+
+ resp->num_paths = route->num_paths;
+ switch (route->num_paths) {
+ case 0:
+ ibaddr = &route->addr.addr.ibaddr;
+ memcpy(&resp->ib_route[0].dgid, ibaddr->dgid.raw,
+ sizeof ibaddr->dgid);
+ memcpy(&resp->ib_route[0].sgid, ibaddr->sgid.raw,
+ sizeof ibaddr->sgid);
+ resp->ib_route[0].pkey = cpu_to_be16(ibaddr->pkey);
+ break;
+ case 2:
+ ib_copy_path_rec_to_user(&resp->ib_route[1],
+ &route->path_rec[1]);
+ /* fall through */
+ case 1:
+ ib_copy_path_rec_to_user(&resp->ib_route[0],
+ &route->path_rec[0]);
+ break;
+ default:
+ break;
+ }
+}
+
+static ssize_t ucma_query_route(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_query_route cmd;
+ struct rdma_ucm_query_route_resp resp;
+ struct ucma_context *ctx;
+ struct sockaddr *addr;
+ int ret = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ if (!ctx->cm_id->device) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ addr = &ctx->cm_id->route.addr.src_addr;
+ memcpy(&resp.src_addr, addr, addr->sa_family == AF_INET ?
+ sizeof(struct sockaddr_in) :
+ sizeof(struct sockaddr_in6));
+ resp.node_guid = ctx->cm_id->device->node_guid;
+ switch (ctx->cm_id->device->node_type) {
+ case IB_NODE_CA:
+ ucma_copy_ib_route(&resp, &ctx->cm_id->route);
+ default:
+ break;
+ }
+
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ ret = -EFAULT;
+
+out:
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static void ucma_copy_conn_param(struct rdma_conn_param *dst_conn,
+ struct rdma_ucm_conn_param *src_conn)
+{
+ dst_conn->private_data = src_conn->private_data;
+ dst_conn->private_data_len = src_conn->private_data_len;
+ dst_conn->responder_resources =src_conn->responder_resources;
+ dst_conn->initiator_depth = src_conn->initiator_depth;
+ dst_conn->flow_control = src_conn->flow_control;
+ dst_conn->retry_count = src_conn->retry_count;
+ dst_conn->rnr_retry_count = src_conn->rnr_retry_count;
+ dst_conn->srq = src_conn->srq;
+ dst_conn->qp_num = src_conn->qp_num;
+ dst_conn->qp_type = src_conn->qp_type;
+}
+
+static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_connect cmd;
+ struct rdma_conn_param conn_param;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ if (!cmd.conn_param.valid)
+ return -EINVAL;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ucma_copy_conn_param(&conn_param, &cmd.conn_param);
+ ret = rdma_connect(ctx->cm_id, &conn_param);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_listen cmd;
+ struct ucma_context *ctx;
+ int ret;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_listen(ctx->cm_id, cmd.backlog);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_accept cmd;
+ struct rdma_conn_param conn_param;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ if (cmd.conn_param.valid) {
+ ctx->uid = cmd.uid;
+ ucma_copy_conn_param(&conn_param, &cmd.conn_param);
+ ret = rdma_accept(ctx->cm_id, &conn_param);
+ } else
+ ret = rdma_accept(ctx->cm_id, NULL);
+
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_reject cmd;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_reject(ctx->cm_id, cmd.private_data, cmd.private_data_len);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_disconnect(struct ucma_file *file, const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_disconnect cmd;
+ struct ucma_context *ctx;
+ int ret = 0;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ ret = rdma_disconnect(ctx->cm_id);
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t ucma_init_qp_attr(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct rdma_ucm_init_qp_attr cmd;
+ struct ib_uverbs_qp_attr resp;
+ struct ucma_context *ctx;
+ struct ib_qp_attr qp_attr;
+ int ret = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ucma_get_ctx(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ resp.qp_attr_mask = 0;
+ memset(&qp_attr, 0, sizeof qp_attr);
+ qp_attr.qp_state = cmd.qp_state;
+ ret = rdma_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask);
+ if (ret)
+ goto out;
+
+ ib_copy_qp_attr_to_user(&resp, &qp_attr);
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ ret = -EFAULT;
+
+out:
+ ucma_put_ctx(ctx);
+ return ret;
+}
+
+static ssize_t (*ucma_cmd_table[])(struct ucma_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len) = {
+ [RDMA_USER_CM_CMD_CREATE_ID] = ucma_create_id,
+ [RDMA_USER_CM_CMD_DESTROY_ID] = ucma_destroy_id,
+ [RDMA_USER_CM_CMD_BIND_ADDR] = ucma_bind_addr,
+ [RDMA_USER_CM_CMD_RESOLVE_ADDR] = ucma_resolve_addr,
+ [RDMA_USER_CM_CMD_RESOLVE_ROUTE]= ucma_resolve_route,
+ [RDMA_USER_CM_CMD_QUERY_ROUTE] = ucma_query_route,
+ [RDMA_USER_CM_CMD_CONNECT] = ucma_connect,
+ [RDMA_USER_CM_CMD_LISTEN] = ucma_listen,
+ [RDMA_USER_CM_CMD_ACCEPT] = ucma_accept,
+ [RDMA_USER_CM_CMD_REJECT] = ucma_reject,
+ [RDMA_USER_CM_CMD_DISCONNECT] = ucma_disconnect,
+ [RDMA_USER_CM_CMD_INIT_QP_ATTR] = ucma_init_qp_attr,
+ [RDMA_USER_CM_CMD_GET_EVENT] = ucma_get_event
+};
+
+static ssize_t ucma_write(struct file *filp, const char __user *buf,
+ size_t len, loff_t *pos)
+{
+ struct ucma_file *file = filp->private_data;
+ struct rdma_ucm_cmd_hdr hdr;
+ ssize_t ret;
+
+ if (len < sizeof(hdr))
+ return -EINVAL;
+
+ if (copy_from_user(&hdr, buf, sizeof(hdr)))
+ return -EFAULT;
+
+ if (hdr.cmd < 0 || hdr.cmd >= ARRAY_SIZE(ucma_cmd_table))
+ return -EINVAL;
+
+ if (hdr.in + sizeof(hdr) > len)
+ return -EINVAL;
+
+ ret = ucma_cmd_table[hdr.cmd](file, buf + sizeof(hdr), hdr.in, hdr.out);
+ if (!ret)
+ ret = len;
+
+ return ret;
+}
+
+static unsigned int ucma_poll(struct file *filp, struct poll_table_struct *wait)
+{
+ struct ucma_file *file = filp->private_data;
+ unsigned int mask = 0;
+
+ poll_wait(filp, &file->poll_wait, wait);
+
+ if (!list_empty(&file->events))
+ mask = POLLIN | POLLRDNORM;
+
+ return mask;
+}
+
+static int ucma_open(struct inode *inode, struct file *filp)
+{
+ struct ucma_file *file;
+
+ file = kmalloc(sizeof *file, GFP_KERNEL);
+ if (!file)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&file->events);
+ INIT_LIST_HEAD(&file->ctxs);
+ init_waitqueue_head(&file->poll_wait);
+ init_MUTEX(&file->mutex);
+
+ filp->private_data = file;
+ file->filp = filp;
+ return 0;
+}
+
+static int ucma_close(struct inode *inode, struct file *filp)
+{
+ struct ucma_file *file = filp->private_data;
+ struct ucma_context *ctx;
+
+ down(&file->mutex);
+ while (!list_empty(&file->ctxs)) {
+ ctx = list_entry(file->ctxs.next, struct ucma_context,
+ file_list);
+ up(&file->mutex);
+
+ down(&ctx_mutex);
+ idr_remove(&ctx_idr, ctx->id);
+ up(&ctx_mutex);
+
+ rdma_destroy_id(ctx->cm_id);
+ ucma_cleanup_events(ctx);
+ kfree(ctx);
+
+ down(&file->mutex);
+ }
+ up(&file->mutex);
+ kfree(file);
+ return 0;
+}
+
+static struct file_operations ucma_fops = {
+ .owner = THIS_MODULE,
+ .open = ucma_open,
+ .release = ucma_close,
+ .write = ucma_write,
+ .poll = ucma_poll,
+};
+
+static struct miscdevice ucma_misc = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "rdma_cm",
+ .fops = &ucma_fops,
+};
+
+static int __init ucma_init(void)
+{
+ return misc_register(&ucma_misc);
+}
+
+static void __exit ucma_cleanup(void)
+{
+ misc_deregister(&ucma_misc);
+ idr_destroy(&ctx_idr);
+}
+
+module_init(ucma_init);
+module_exit(ucma_cleanup);
More information about the general
mailing list