[openib-general] [PATCH 5/5] [RFC] stupid test program for resize CQ
Roland Dreier
rolandd at cisco.com
Thu Jan 26 11:23:33 PST 2006
Here's a simple test program I wrote that just creates a loopback QP
on the first port of the first HCA it finds, and tries to keep 100
RDMA writes queued all the time. In another thread, it resizes the CQ
at random intervals. The first thread makes sure that all the
completions expected do arrive in the expected order.
To build it, just compile and link with "-libverbs". It doesn't take
any command line parameters to run. If it's working correctly, the
output should just look like an endless stream of
Resizing to 500... resized to 511
500000 writes done
Resizing to 600... resized to 1023
Resizing to 500... resized to 511
Resizing to 800... resized to 1023
1000000 writes done
1500000 writes done
Resizing to 1100... resized to 2047
Resizing to 100... resized to 127
Resizing to 1100... resized to 2047
2000000 writes done
If either the "Resizing" or "xxx writes done" lines stop appearing,
then something went wrong.
---
/*
* Copyright (c) 2006 Cisco Systems. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* $Id: rc_pingpong.c 5046 2006-01-17 17:20:37Z roland $
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <pthread.h>
#include <infiniband/verbs.h>
static int depth = 100;
static int page_size;
static uint16_t get_local_lid(struct ibv_context *context, int port)
{
struct ibv_port_attr attr;
if (ibv_query_port(context, port, &attr))
return 0;
return attr.lid;
}
static void *resize_task(void *cq_ptr)
{
struct ibv_cq *cq = cq_ptr;
int new_size;
while (1) {
usleep(drand48() * 1000000);
new_size = (lrand48() % 11 + 1) * depth;
printf("Resizing to %4d... ", new_size);
fflush(stdout);
if (ibv_resize_cq(cq, new_size)) {
fprintf(stderr, "\nResize failed\n");
exit(1);
}
printf("resized to %4d\n", cq->cqe);
fflush(stdout);
}
}
static int post_write(uint64_t wrid, void *buf, struct ibv_qp *qp, struct ibv_mr *mr)
{
struct ibv_sge list = {
.addr = (uintptr_t) buf,
.length = 1,
.lkey = mr->lkey
};
struct ibv_send_wr wr = {
.wr_id = wrid,
.sg_list = &list,
.num_sge = 1,
.opcode = IBV_WR_RDMA_WRITE,
.send_flags = IBV_SEND_SIGNALED,
.wr.rdma = {
.remote_addr = (uintptr_t) buf + 8,
.rkey = mr->rkey
}
};
struct ibv_send_wr *bad_wr;
return ibv_post_send(qp, &wr, &bad_wr);
}
int main(int argc, char *argv[])
{
struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct ibv_context *context;
struct ibv_pd *pd;
struct ibv_mr *mr;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_wc wc;
void *buf;
uint16_t lid;
uint64_t wrid, exp_wrid;
int i;
pthread_t resize_thread;
srand48(getpid() * time(NULL));
page_size = sysconf(_SC_PAGESIZE);
dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
buf = memalign(page_size, page_size);
if (!buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return 1;
}
context = ibv_open_device(ib_dev);
if (!context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
return 1;
}
lid = get_local_lid(context, 1);
pd = ibv_alloc_pd(context);
if (!pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return 1;
}
mr = ibv_reg_mr(pd, buf, page_size,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if (!mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return 1;
}
cq = ibv_create_cq(context, depth, NULL, NULL, 0);
if (!cq) {
fprintf(stderr, "Couldn't create CQ\n");
return 1;
}
{
struct ibv_qp_init_attr attr = {
.send_cq = cq,
.recv_cq = cq,
.cap = {
.max_send_wr = depth,
.max_recv_wr = 0,
.max_send_sge = 1,
.max_recv_sge = 1
},
.qp_type = IBV_QPT_RC
};
qp = ibv_create_qp(pd, &attr);
if (!qp) {
fprintf(stderr, "Couldn't create QP\n");
return 1;
}
}
{
struct ibv_qp_attr attr;
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = 1;
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
return 1;
}
attr.qp_state = IBV_QPS_RTR;
attr.path_mtu = IBV_MTU_1024;
attr.dest_qp_num = qp->qp_num;
attr.rq_psn = 1;
attr.max_dest_rd_atomic = 4;
attr.min_rnr_timer = 12;
attr.ah_attr.is_global = 0;
attr.ah_attr.dlid = lid;
attr.ah_attr.sl = 0;
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = 1;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
IBV_QP_PATH_MTU |
IBV_QP_DEST_QPN |
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
fprintf(stderr, "Failed to modify QP to RTR\n");
return 1;
}
attr.qp_state = IBV_QPS_RTS;
attr.timeout = 14;
attr.retry_cnt = 7;
attr.rnr_retry = 7;
attr.sq_psn = 1;
attr.max_rd_atomic = 4;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
IBV_QP_RETRY_CNT |
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
fprintf(stderr, "Failed to modify QP to RTS\n");
return 1;
}
}
wrid = exp_wrid = 0;
if (pthread_create(&resize_thread, NULL, resize_task, cq)) {
fprintf(stderr, "Couldn't start resize_task\n");
return 1;
}
for (i = 0; i < depth; ++i) {
if (post_write(wrid, buf, qp, mr)) {
fprintf(stderr, "Couldn't post work request %lld\n",
(long long) wrid);
return 1;
}
++wrid;
}
while (1) {
i = ibv_poll_cq(cq, 1, &wc);
if (i < 0) {
fprintf(stderr, "poll CQ failed %d\n", i);
return 1;
}
if (i) {
if (wc.status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed status %d for wr_id %lld\n",
wc.status, (long long) wc.wr_id);
return 1;
}
if (wc.wr_id != exp_wrid) {
fprintf(stderr, "wr_id mismatch %lld != %lld\n",
(long long) wc.wr_id, (long long) exp_wrid);
return 1;
}
++exp_wrid;
if (!(exp_wrid % 500000))
printf("%12lld writes done\n", (long long) exp_wrid);
if (post_write(wrid, buf, qp, mr)) {
fprintf(stderr, "Couldn't post work request %lld\n",
(long long) wrid);
return 1;
}
++wrid;
}
}
return 0;
}
More information about the general
mailing list