[openib-general] [PATCH 5/6] [RFC] stupid test program for resize CQ

Roland Dreier rolandd at cisco.com
Mon Jan 30 09:17:27 PST 2006


Here's a simple test program I wrote that just creates a loopback QP
on the first port of the first HCA it finds, and tries to keep 100
RDMA writes queued all the time.  In another thread, it resizes the CQ
at random intervals.  The first thread makes sure that all the
completions expected do arrive in the expected order.

To build it, just compile and link with "-libverbs".  It doesn't take
any command line parameters to run.  If it's working correctly, the
output should just look like an endless stream of

    Resizing to  500... resized to  511
          500000 writes done
    Resizing to  600... resized to 1023
    Resizing to  500... resized to  511
    Resizing to  800... resized to 1023
         1000000 writes done
         1500000 writes done
    Resizing to 1100... resized to 2047
    Resizing to  100... resized to  127
    Resizing to 1100... resized to 2047
         2000000 writes done

If either the "Resizing" or "xxx writes done" lines stop appearing,
then something went wrong.

---

/*
 * Copyright (c) 2006 Cisco Systems.  All rights reserved.
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License version
 *	2 as published by the Free Software Foundation.
 *
 * $Id$
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <pthread.h>

#include <infiniband/verbs.h>

static int depth = 100;
static int page_size;

static uint16_t get_local_lid(struct ibv_context *context, int port)
{
	struct ibv_port_attr attr;

	if (ibv_query_port(context, port, &attr))
		return 0;

	return attr.lid;
}

static void *resize_task(void *cq_ptr)
{
	struct ibv_cq *cq = cq_ptr;
	int new_size;

	while (1) {
		usleep(drand48() * 1000000);
		new_size = (lrand48() % 11 + 1) * depth;
		printf("Resizing to %4d... ", new_size);
		fflush(stdout);
		if (ibv_resize_cq(cq, new_size)) {
			fprintf(stderr, "\nResize failed\n");
			exit(1);
		}
		printf("resized to %4d\n", cq->cqe);
		fflush(stdout);
	}
}

static int post_write(uint64_t wrid, void *buf, struct ibv_qp *qp, struct ibv_mr *mr)
{
	struct ibv_sge list = {
		.addr 	= (uintptr_t) buf,
		.length = 1,
		.lkey 	= mr->lkey
	};
	struct ibv_send_wr wr = {
		.wr_id 	    = wrid,
		.sg_list    = &list,
		.num_sge    = 1,
		.opcode     = IBV_WR_RDMA_WRITE,
		.send_flags = IBV_SEND_SIGNALED,
		.wr.rdma    = {
			.remote_addr = (uintptr_t) buf + 8,
			.rkey        = mr->rkey
		}
	};
	struct ibv_send_wr *bad_wr;

	return ibv_post_send(qp, &wr, &bad_wr);
}

int main(int argc, char *argv[])
{
	struct ibv_device      **dev_list;
	struct ibv_device 	*ib_dev;
	struct ibv_context	*context;
	struct ibv_pd		*pd;
	struct ibv_mr		*mr;
	struct ibv_cq		*cq;
	struct ibv_qp		*qp;
	struct ibv_wc		 wc;
	void			*buf;
	uint16_t		 lid;
	uint64_t		 wrid, exp_wrid;
	int			 i;
	pthread_t		 resize_thread;

	srand48(getpid() * time(NULL));
	page_size = sysconf(_SC_PAGESIZE);

	dev_list = ibv_get_device_list(NULL);
	if (!dev_list) {
		fprintf(stderr, "No IB devices found\n");
		return 1;
	}

	ib_dev = *dev_list;
	if (!ib_dev) {
		fprintf(stderr, "No IB devices found\n");
		return 1;
	}

	buf = memalign(page_size, page_size);
	if (!buf) {
		fprintf(stderr, "Couldn't allocate work buf.\n");
		return 1;
	}

	context = ibv_open_device(ib_dev);
	if (!context) {
		fprintf(stderr, "Couldn't get context for %s\n",
			ibv_get_device_name(ib_dev));
		return 1;
	}

	lid = get_local_lid(context, 1);

	pd = ibv_alloc_pd(context);
	if (!pd) {
		fprintf(stderr, "Couldn't allocate PD\n");
		return 1;
	}

	mr = ibv_reg_mr(pd, buf, page_size,
			IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
	if (!mr) {
		fprintf(stderr, "Couldn't allocate MR\n");
		return 1;
	}

	cq = ibv_create_cq(context, depth, NULL, NULL, 0);
	if (!cq) {
		fprintf(stderr, "Couldn't create CQ\n");
		return 1;
	}

	{
		struct ibv_qp_init_attr attr = {
			.send_cq = cq,
			.recv_cq = cq,
			.cap     = {
				.max_send_wr  = depth,
				.max_recv_wr  = 0,
				.max_send_sge = 1,
				.max_recv_sge = 1
			},
			.qp_type = IBV_QPT_RC
		};

		qp = ibv_create_qp(pd, &attr);
		if (!qp)  {
			fprintf(stderr, "Couldn't create QP\n");
			return 1;
		}
	}

	{
		struct ibv_qp_attr attr;

		attr.qp_state        = IBV_QPS_INIT;
		attr.pkey_index      = 0;
		attr.port_num        = 1;
		attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE;

		if (ibv_modify_qp(qp, &attr,
				  IBV_QP_STATE              |
				  IBV_QP_PKEY_INDEX         |
				  IBV_QP_PORT               |
				  IBV_QP_ACCESS_FLAGS)) {
			fprintf(stderr, "Failed to modify QP to INIT\n");
			return 1;
		}

		attr.qp_state		= IBV_QPS_RTR;
		attr.path_mtu		= IBV_MTU_1024;
		attr.dest_qp_num	= qp->qp_num;
		attr.rq_psn 		= 1;
		attr.max_dest_rd_atomic	= 4;
		attr.min_rnr_timer	= 12;
		attr.ah_attr.is_global	= 0;
		attr.ah_attr.dlid	= lid;
		attr.ah_attr.sl		= 0;
		attr.ah_attr.src_path_bits = 0;
		attr.ah_attr.port_num	= 1;

		if (ibv_modify_qp(qp, &attr,
				  IBV_QP_STATE              |
				  IBV_QP_AV                 |
				  IBV_QP_PATH_MTU           |
				  IBV_QP_DEST_QPN           |
				  IBV_QP_RQ_PSN             |
				  IBV_QP_MAX_DEST_RD_ATOMIC |
				  IBV_QP_MIN_RNR_TIMER)) {
			fprintf(stderr, "Failed to modify QP to RTR\n");
			return 1;
		}

		attr.qp_state 	    = IBV_QPS_RTS;
		attr.timeout 	    = 14;
		attr.retry_cnt 	    = 7;
		attr.rnr_retry 	    = 7;
		attr.sq_psn 	    = 1;
		attr.max_rd_atomic  = 4;
		if (ibv_modify_qp(qp, &attr,
			  IBV_QP_STATE              |
			  IBV_QP_TIMEOUT            |
			  IBV_QP_RETRY_CNT          |
			  IBV_QP_RNR_RETRY          |
			  IBV_QP_SQ_PSN             |
			  IBV_QP_MAX_QP_RD_ATOMIC)) {
			fprintf(stderr, "Failed to modify QP to RTS\n");
			return 1;
		}
	}

	wrid = exp_wrid = 0;

	if (pthread_create(&resize_thread, NULL, resize_task, cq)) {
		fprintf(stderr, "Couldn't start resize_task\n");
		return 1;
	}

	for (i = 0; i < depth; ++i) {
		if (post_write(wrid, buf, qp, mr)) {
			fprintf(stderr, "Couldn't post work request %lld\n",
				(long long) wrid);
			return 1;
		}
		++wrid;
	}

	while (1) {
		i = ibv_poll_cq(cq, 1, &wc);
		if (i < 0) {
			fprintf(stderr, "poll CQ failed %d\n", i);
			return 1;
		}

		if (i) {
			if (wc.status != IBV_WC_SUCCESS) {
				fprintf(stderr, "Failed status %d for wr_id %lld\n",
					wc.status, (long long) wc.wr_id);
				return 1;
			}

			if (wc.wr_id != exp_wrid) {
				fprintf(stderr, "wr_id mismatch %lld != %lld\n",
					(long long) wc.wr_id, (long long) exp_wrid);
				return 1;
			}

			++exp_wrid;

			if (!(exp_wrid % 500000))
				printf("%12lld writes done\n", (long long) exp_wrid);

			if (post_write(wrid, buf, qp, mr)) {
				fprintf(stderr, "Couldn't post work request %lld\n",
					(long long) wrid);
				return 1;
			}
			++wrid;
		}
	}

	return 0;
}



More information about the general mailing list