[openib-general] [PATCH 5/6] [RFC] stupid test program for resize CQ
Roland Dreier
rolandd at cisco.com
Mon Jan 30 09:17:27 PST 2006
Here's a simple test program I wrote that just creates a loopback QP
on the first port of the first HCA it finds, and tries to keep 100
RDMA writes queued all the time. In another thread, it resizes the CQ
at random intervals. The first thread makes sure that all the
completions expected do arrive in the expected order.
To build it, just compile and link with "-libverbs". It doesn't take
any command line parameters to run. If it's working correctly, the
output should just look like an endless stream of
Resizing to 500... resized to 511
500000 writes done
Resizing to 600... resized to 1023
Resizing to 500... resized to 511
Resizing to 800... resized to 1023
1000000 writes done
1500000 writes done
Resizing to 1100... resized to 2047
Resizing to 100... resized to 127
Resizing to 1100... resized to 2047
2000000 writes done
If either the "Resizing" or "xxx writes done" lines stop appearing,
then something went wrong.
---
/*
* Copyright (c) 2006 Cisco Systems. All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License version
* 2 as published by the Free Software Foundation.
*
* $Id$
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <pthread.h>
#include <infiniband/verbs.h>
static int depth = 100;
static int page_size;
static uint16_t get_local_lid(struct ibv_context *context, int port)
{
struct ibv_port_attr attr;
if (ibv_query_port(context, port, &attr))
return 0;
return attr.lid;
}
static void *resize_task(void *cq_ptr)
{
struct ibv_cq *cq = cq_ptr;
int new_size;
while (1) {
usleep(drand48() * 1000000);
new_size = (lrand48() % 11 + 1) * depth;
printf("Resizing to %4d... ", new_size);
fflush(stdout);
if (ibv_resize_cq(cq, new_size)) {
fprintf(stderr, "\nResize failed\n");
exit(1);
}
printf("resized to %4d\n", cq->cqe);
fflush(stdout);
}
}
static int post_write(uint64_t wrid, void *buf, struct ibv_qp *qp, struct ibv_mr *mr)
{
struct ibv_sge list = {
.addr = (uintptr_t) buf,
.length = 1,
.lkey = mr->lkey
};
struct ibv_send_wr wr = {
.wr_id = wrid,
.sg_list = &list,
.num_sge = 1,
.opcode = IBV_WR_RDMA_WRITE,
.send_flags = IBV_SEND_SIGNALED,
.wr.rdma = {
.remote_addr = (uintptr_t) buf + 8,
.rkey = mr->rkey
}
};
struct ibv_send_wr *bad_wr;
return ibv_post_send(qp, &wr, &bad_wr);
}
int main(int argc, char *argv[])
{
struct ibv_device **dev_list;
struct ibv_device *ib_dev;
struct ibv_context *context;
struct ibv_pd *pd;
struct ibv_mr *mr;
struct ibv_cq *cq;
struct ibv_qp *qp;
struct ibv_wc wc;
void *buf;
uint16_t lid;
uint64_t wrid, exp_wrid;
int i;
pthread_t resize_thread;
srand48(getpid() * time(NULL));
page_size = sysconf(_SC_PAGESIZE);
dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
ib_dev = *dev_list;
if (!ib_dev) {
fprintf(stderr, "No IB devices found\n");
return 1;
}
buf = memalign(page_size, page_size);
if (!buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return 1;
}
context = ibv_open_device(ib_dev);
if (!context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
return 1;
}
lid = get_local_lid(context, 1);
pd = ibv_alloc_pd(context);
if (!pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return 1;
}
mr = ibv_reg_mr(pd, buf, page_size,
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
if (!mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return 1;
}
cq = ibv_create_cq(context, depth, NULL, NULL, 0);
if (!cq) {
fprintf(stderr, "Couldn't create CQ\n");
return 1;
}
{
struct ibv_qp_init_attr attr = {
.send_cq = cq,
.recv_cq = cq,
.cap = {
.max_send_wr = depth,
.max_recv_wr = 0,
.max_send_sge = 1,
.max_recv_sge = 1
},
.qp_type = IBV_QPT_RC
};
qp = ibv_create_qp(pd, &attr);
if (!qp) {
fprintf(stderr, "Couldn't create QP\n");
return 1;
}
}
{
struct ibv_qp_attr attr;
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = 1;
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
return 1;
}
attr.qp_state = IBV_QPS_RTR;
attr.path_mtu = IBV_MTU_1024;
attr.dest_qp_num = qp->qp_num;
attr.rq_psn = 1;
attr.max_dest_rd_atomic = 4;
attr.min_rnr_timer = 12;
attr.ah_attr.is_global = 0;
attr.ah_attr.dlid = lid;
attr.ah_attr.sl = 0;
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = 1;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_AV |
IBV_QP_PATH_MTU |
IBV_QP_DEST_QPN |
IBV_QP_RQ_PSN |
IBV_QP_MAX_DEST_RD_ATOMIC |
IBV_QP_MIN_RNR_TIMER)) {
fprintf(stderr, "Failed to modify QP to RTR\n");
return 1;
}
attr.qp_state = IBV_QPS_RTS;
attr.timeout = 14;
attr.retry_cnt = 7;
attr.rnr_retry = 7;
attr.sq_psn = 1;
attr.max_rd_atomic = 4;
if (ibv_modify_qp(qp, &attr,
IBV_QP_STATE |
IBV_QP_TIMEOUT |
IBV_QP_RETRY_CNT |
IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN |
IBV_QP_MAX_QP_RD_ATOMIC)) {
fprintf(stderr, "Failed to modify QP to RTS\n");
return 1;
}
}
wrid = exp_wrid = 0;
if (pthread_create(&resize_thread, NULL, resize_task, cq)) {
fprintf(stderr, "Couldn't start resize_task\n");
return 1;
}
for (i = 0; i < depth; ++i) {
if (post_write(wrid, buf, qp, mr)) {
fprintf(stderr, "Couldn't post work request %lld\n",
(long long) wrid);
return 1;
}
++wrid;
}
while (1) {
i = ibv_poll_cq(cq, 1, &wc);
if (i < 0) {
fprintf(stderr, "poll CQ failed %d\n", i);
return 1;
}
if (i) {
if (wc.status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed status %d for wr_id %lld\n",
wc.status, (long long) wc.wr_id);
return 1;
}
if (wc.wr_id != exp_wrid) {
fprintf(stderr, "wr_id mismatch %lld != %lld\n",
(long long) wc.wr_id, (long long) exp_wrid);
return 1;
}
++exp_wrid;
if (!(exp_wrid % 500000))
printf("%12lld writes done\n", (long long) exp_wrid);
if (post_write(wrid, buf, qp, mr)) {
fprintf(stderr, "Couldn't post work request %lld\n",
(long long) wrid);
return 1;
}
++wrid;
}
}
return 0;
}
More information about the general
mailing list