[openib-general] [PATCH 2/2] librdmacm: add multicast test program to examples

Sean Hefty sean.hefty at intel.com
Mon Jun 12 10:43:38 PDT 2006


Simple multicast test program.  When run, the client creates a QP
and joins it to a multicast group.  It then either sends or receives
messages on the group.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
diff -up svn3/gen2/trunk/src/userspace/librdmacm/librdmacm.spec.in svn/gen2/trunk/src/userspace/librdmacm/librdmacm.spec.in
--- svn3/gen2/trunk/src/userspace/librdmacm/librdmacm.spec.in	2006-06-06 17:35:31.000000000 -0700
+++ svn/gen2/trunk/src/userspace/librdmacm/librdmacm.spec.in	2006-06-01 14:53:47.000000000 -0700
@@ -67,3 +67,4 @@ rm -rf $RPM_BUILD_ROOT
 %{_bindir}/rping
 %{_bindir}/ucmatose
 %{_bindir}/udaddy
+%{_bindir}/mckey
diff -up svn3/gen2/trunk/src/userspace/librdmacm/Makefile.am svn/gen2/trunk/src/userspace/librdmacm/Makefile.am
--- svn3/gen2/trunk/src/userspace/librdmacm/Makefile.am	2006-06-06 17:35:31.000000000 -0700
+++ svn/gen2/trunk/src/userspace/librdmacm/Makefile.am	2006-06-06 14:48:23.000000000 -0700
@@ -18,13 +18,15 @@ endif
 src_librdmacm_la_SOURCES = src/cma.c
 src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script)
 
-bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy
+bin_PROGRAMS = examples/ucmatose examples/rping examples/udaddy examples/mckey
 examples_ucmatose_SOURCES = examples/cmatose.c
 examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
 examples_rping_SOURCES = examples/rping.c
 examples_rping_LDADD = $(top_builddir)/src/librdmacm.la
 examples_udaddy_SOURCES = examples/udaddy.c
 examples_udaddy_LDADD = $(top_builddir)/src/librdmacm.la
+examples_mckey_SOURCES = examples/mckey.c
+examples_mckey_LDADD = $(top_builddir)/src/librdmacm.la
 
 librdmacmincludedir = $(includedir)/rdma
 
diff -upN svn3/gen2/trunk/src/userspace/librdmacm/examples/mckey.c svn/gen2/trunk/src/userspace/librdmacm/examples/mckey.c
--- svn3/gen2/trunk/src/userspace/librdmacm/examples/mckey.c	1969-12-31 16:00:00.000000000 -0800
+++ svn/gen2/trunk/src/userspace/librdmacm/examples/mckey.c	2006-06-06 12:56:35.000000000 -0700
@@ -0,0 +1,505 @@
+/*
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+#include <unistd.h>
+
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_cma_ib.h>
+
+struct cmatest_node {
+	int			id;
+	struct rdma_cm_id	*cma_id;
+	int			connected;
+	struct ibv_pd		*pd;
+	struct ibv_cq		*cq;
+	struct ibv_mr		*mr;
+	struct ibv_ah		*ah;
+	uint32_t		remote_qpn;
+	uint32_t		remote_qkey;
+	void			*mem;
+};
+
+struct cmatest {
+	struct rdma_event_channel *channel;
+	struct cmatest_node	*nodes;
+	int			conn_index;
+	int			connects_left;
+
+	struct sockaddr_in	dst_in;
+	struct sockaddr		*dst_addr;
+	struct sockaddr_in	src_in;
+	struct sockaddr		*src_addr;
+};
+
+static struct cmatest test;
+static int connections = 1;
+static int message_size = 100;
+static int message_count = 10;
+static int is_sender;
+
+static int create_message(struct cmatest_node *node)
+{
+	if (!message_size)
+		message_count = 0;
+
+	if (!message_count)
+		return 0;
+
+	node->mem = malloc(message_size + sizeof(struct ibv_grh));
+	if (!node->mem) {
+		printf("failed message allocation\n");
+		return -1;
+	}
+	node->mr = ibv_reg_mr(node->pd, node->mem,
+			      message_size + sizeof(struct ibv_grh),
+			      IBV_ACCESS_LOCAL_WRITE);
+	if (!node->mr) {
+		printf("failed to reg MR\n");
+		goto err;
+	}
+	return 0;
+err:
+	free(node->mem);
+	return -1;
+}
+
+static int init_node(struct cmatest_node *node)
+{
+	struct ibv_qp_init_attr init_qp_attr;
+	int cqe, ret;
+
+	node->pd = ibv_alloc_pd(node->cma_id->verbs);
+	if (!node->pd) {
+		ret = -ENOMEM;
+		printf("cmatose: unable to allocate PD\n");
+		goto out;
+	}
+
+	cqe = message_count ? message_count * 2 : 2;
+	node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);
+	if (!node->cq) {
+		ret = -ENOMEM;
+		printf("cmatose: unable to create CQ\n");
+		goto out;
+	}
+
+	memset(&init_qp_attr, 0, sizeof init_qp_attr);
+	init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+	init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+	init_qp_attr.cap.max_send_sge = 1;
+	init_qp_attr.cap.max_recv_sge = 1;
+	init_qp_attr.qp_context = node;
+	init_qp_attr.sq_sig_all = 0;
+	init_qp_attr.qp_type = IBV_QPT_UD;
+	init_qp_attr.send_cq = node->cq;
+	init_qp_attr.recv_cq = node->cq;
+	ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
+	if (ret) {
+		printf("cmatose: unable to create QP: %d\n", ret);
+		goto out;
+	}
+
+	ret = create_message(node);
+	if (ret) {
+		printf("cmatose: failed to create messages: %d\n", ret);
+		goto out;
+	}
+out:
+	return ret;
+}
+
+static int post_recvs(struct cmatest_node *node)
+{
+	struct ibv_recv_wr recv_wr, *recv_failure;
+	struct ibv_sge sge;
+	int i, ret = 0;
+
+	if (!message_count)
+		return 0;
+
+	recv_wr.next = NULL;
+	recv_wr.sg_list = &sge;
+	recv_wr.num_sge = 1;
+	recv_wr.wr_id = (uintptr_t) node;
+
+	sge.length = message_size + sizeof(struct ibv_grh);
+	sge.lkey = node->mr->lkey;
+	sge.addr = (uintptr_t) node->mem;
+
+	for (i = 0; i < message_count && !ret; i++ ) {
+		ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
+		if (ret) {
+			printf("failed to post receives: %d\n", ret);
+			break;
+		}
+	}
+	return ret;
+}
+
+static int post_sends(struct cmatest_node *node, int signal_flag)
+{
+	struct ibv_send_wr send_wr, *bad_send_wr;
+	struct ibv_sge sge;
+	int i, ret = 0;
+
+	if (!node->connected || !message_count)
+		return 0;
+
+	send_wr.next = NULL;
+	send_wr.sg_list = &sge;
+	send_wr.num_sge = 1;
+	send_wr.opcode = IBV_WR_SEND_WITH_IMM;
+	send_wr.send_flags = IBV_SEND_INLINE | signal_flag;
+	send_wr.wr_id = (unsigned long)node;
+	send_wr.imm_data = htonl(node->cma_id->qp->qp_num);
+
+	send_wr.wr.ud.ah = node->ah;
+	send_wr.wr.ud.remote_qpn = node->remote_qpn;
+	send_wr.wr.ud.remote_qkey = node->remote_qkey;
+
+	sge.length = message_size - sizeof(struct ibv_grh);
+	sge.lkey = node->mr->lkey;
+	sge.addr = (uintptr_t) node->mem;
+
+	for (i = 0; i < message_count && !ret; i++) {
+		ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
+		if (ret) 
+			printf("failed to post sends: %d\n", ret);
+	}
+	return ret;
+}
+
+static void connect_error(void)
+{
+	test.connects_left--;
+}
+
+static int addr_handler(struct cmatest_node *node)
+{
+	int ret;
+
+	ret = init_node(node);
+	if (ret)
+		goto err;
+
+	if (!is_sender) {
+		ret = post_recvs(node);
+		if (ret)
+			goto err;
+	}
+
+	ret = rdma_join_multicast(node->cma_id, test.dst_addr, node);
+	if (ret) {
+		printf("cmatose: failure joining: %d\n", ret);
+		goto err;
+	}
+	return 0;
+err:
+	connect_error();
+	return ret;
+}
+
+static int join_handler(struct cmatest_node *node)
+{
+	struct ibv_ah_attr ah_attr;
+	int ret;
+
+	ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
+				&node->remote_qpn, &node->remote_qkey);
+	if (ret) {
+		printf("mckey: failure getting destination attributes\n");
+		goto err;
+	}
+
+	node->ah = ibv_create_ah(node->pd, &ah_attr);
+	if (!node->ah) {
+		printf("mckey: failure creating address handle\n");
+		goto err;
+	}
+
+	node->connected = 1;
+	test.connects_left--;
+	return 0;
+err:
+	connect_error();
+	return ret;
+}
+
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
+{
+	int ret = 0;
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		ret = addr_handler(cma_id->context);
+		break;
+	case RDMA_CM_EVENT_MULTICAST_JOIN:
+		ret = join_handler(cma_id->context);
+		break;
+	case RDMA_CM_EVENT_ADDR_ERROR:
+	case RDMA_CM_EVENT_ROUTE_ERROR:
+	case RDMA_CM_EVENT_MULTICAST_ERROR:
+		printf("cmatose: event: %d, error: %d\n", event->event,
+			event->status);
+		connect_error();
+		ret = event->status;
+		break;
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		/* Cleanup will occur after test completes. */
+		break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+static void destroy_node(struct cmatest_node *node)
+{
+	if (!node->cma_id)
+		return;
+
+	if (node->ah)
+		ibv_destroy_ah(node->ah);
+
+	if (node->cma_id->qp)
+		rdma_destroy_qp(node->cma_id);
+
+	if (node->cq)
+		ibv_destroy_cq(node->cq);
+
+	if (node->mem) {
+		ibv_dereg_mr(node->mr);
+		free(node->mem);
+	}
+
+	if (node->pd)
+		ibv_dealloc_pd(node->pd);
+
+	/* Destroy the RDMA ID after all device resources */
+	rdma_destroy_id(node->cma_id);
+}
+
+static int alloc_nodes(void)
+{
+	int ret, i;
+
+	test.nodes = malloc(sizeof *test.nodes * connections);
+	if (!test.nodes) {
+		printf("cmatose: unable to allocate memory for test nodes\n");
+		return -ENOMEM;
+	}
+	memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+	for (i = 0; i < connections; i++) {
+		test.nodes[i].id = i;
+		ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,
+				     &test.nodes[i], RDMA_PS_UDP);
+		if (ret)
+			goto err;
+	}
+	return 0;
+err:
+	while (--i >= 0)
+		rdma_destroy_id(test.nodes[i].cma_id);
+	free(test.nodes);
+	return ret;
+}
+
+static void destroy_nodes(void)
+{
+	int i;
+
+	for (i = 0; i < connections; i++)
+		destroy_node(&test.nodes[i]);
+	free(test.nodes);
+}
+
+static int poll_cqs(void)
+{
+	struct ibv_wc wc[8];
+	int done, i, ret;
+
+	for (i = 0; i < connections; i++) {
+		if (!test.nodes[i].connected)
+			continue;
+
+		for (done = 0; done < message_count; done += ret) {
+			ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+			if (ret < 0) {
+				printf("cmatose: failed polling CQ: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+	return 0;
+}
+
+static int connect_events(void)
+{
+	struct rdma_cm_event *event;
+	int ret = 0;
+
+	while (test.connects_left && !ret) {
+		ret = rdma_get_cm_event(test.channel, &event);
+		if (!ret) {
+			ret = cma_handler(event->id, event);
+			rdma_ack_cm_event(event);
+		}
+	}
+	return ret;
+}
+
+static int get_addr(char *dst, struct sockaddr_in *addr)
+{
+	struct addrinfo *res;
+	int ret;
+
+	ret = getaddrinfo(dst, NULL, NULL, &res);
+	if (ret) {
+		printf("getaddrinfo failed - invalid hostname or IP address\n");
+		return ret;
+	}
+
+	if (res->ai_family != PF_INET) {
+		ret = -1;
+		goto out;
+	}
+
+	*addr = *(struct sockaddr_in *) res->ai_addr;
+out:
+	freeaddrinfo(res);
+	return ret;
+}
+
+static int run(char *dst, char *src)
+{
+	int i, ret;
+
+	printf("cmatose: starting client\n");
+	if (src) {
+		ret = get_addr(src, &test.src_in);
+		if (ret)
+			return ret;
+	}
+
+	ret = get_addr(dst, &test.dst_in);
+	if (ret)
+		return ret;
+
+	test.dst_in.sin_port = 7174;
+
+	printf("cmatose: joining\n");
+	for (i = 0; i < connections; i++) {
+		ret = rdma_resolve_addr(test.nodes[i].cma_id,
+					src ? test.src_addr : NULL,
+					test.dst_addr, 2000);
+		if (ret) {
+			printf("cmatose: failure getting addr: %d\n", ret);
+			connect_error();
+			return ret;
+		}
+	}
+
+	ret = connect_events();
+	if (ret)
+		goto out;
+
+	/*
+	 * Pause to give SM chance to configure switches.  We don't want to
+	 * handle reliability issue in this simple test program.
+	 */
+	sleep(3);
+
+	if (message_count) {
+		if (is_sender) {
+			printf("initiating data transfers\n");
+			for (i = 0; i < connections; i++) {
+				ret = post_sends(&test.nodes[i], 0);
+				if (ret)
+					goto out;
+			}
+		} else {
+			printf("receiving data transfers\n");
+			ret = poll_cqs();
+			if (ret)
+				goto out;
+		}
+		printf("data transfers complete\n");
+	}
+out:
+	return ret;
+}
+
+int main(int argc, char **argv)
+{
+	int ret;
+
+	if (argc < 3 || argc > 4) {
+		printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n",
+		       argv[0]);
+		exit(1);
+	}
+	is_sender = (argv[1][0] == 's');
+
+	test.dst_addr = (struct sockaddr *) &test.dst_in;
+	test.src_addr = (struct sockaddr *) &test.src_in;
+	test.connects_left = connections;
+
+	test.channel = rdma_create_event_channel();
+	if (!test.channel) {
+		printf("failed to create event channel\n");
+		exit(1);
+	}
+
+	if (alloc_nodes())
+		exit(1);
+
+	ret = run(argv[2], (argc == 4) ? argv[3] : NULL);
+
+	printf("test complete\n");
+	destroy_nodes();
+	rdma_destroy_event_channel(test.channel);
+
+	printf("return status %d\n", ret);
+	return ret;
+}





More information about the general mailing list