[openib-general] [PATCH 6/6] [RFC] stupid test kernel module for resize CQ
Roland Dreier
rolandd at cisco.com
Mon Jan 30 09:17:27 PST 2006
And here's a simple test kernel module I wrote to test resizing kernel
CQs. It is similar to the userspace test program, and should produce
endless output like
Resizing to 800... resized to 1023
28328000000 writes done
when loaded.
If either the "Resizing" or "xxx writes done" lines stop appearing,
then something went wrong.
---
/*
* Copyright (c) 2006 Cisco Systems. All rights reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License version
* 2 as published by the Free Software Foundation.
*
* $Id$
*/
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/kthread.h>
#include <linux/random.h>
#include <linux/dma-mapping.h>
#include <linux/mutex.h>
#include <rdma/ib_verbs.h>
MODULE_LICENSE("GPL");
static int depth = 100;
static int wcsize = 2;
static u32 seed;
DEFINE_MUTEX(seed_mutex);
struct kcq_ctx {
struct ib_device *ib_dev;
struct ib_pd *pd;
struct ib_mr *mr;
struct ib_cq *cq;
struct ib_qp *qp;
void *buf;
dma_addr_t dma;
u64 wrid;
u64 exp_wrid;
struct task_struct *poll_task;
struct work_struct resize_work;
};
static u16 get_local_lid(struct ib_device *device, int port)
{
struct ib_port_attr attr;
if (ib_query_port(device, port, &attr))
return 0;
return attr.lid;
}
static u32 rand(void)
{
u32 ret;
mutex_lock(&seed_mutex);
/* 3-shift-register generator with period 2^32-1 */
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
ret = seed;
mutex_unlock(&seed_mutex);
return ret;
}
static void kcq_resize_work(void *ctx_ptr)
{
struct kcq_ctx *ctx = ctx_ptr;
int new_size = (rand() % 11 + 1) * depth;
printk(KERN_ERR "Resizing to %4d... ", new_size);
if (ib_resize_cq(ctx->cq, new_size))
printk("failed\n");
else
printk("resized to %4d\n", ctx->cq->cqe);
schedule_delayed_work(&ctx->resize_work, msecs_to_jiffies(rand() % 1000));
}
static int post_write(u64 wrid, dma_addr_t buf, struct ib_qp *qp, struct ib_mr *mr)
{
struct ib_sge list = {
.addr = buf,
.length = 1,
.lkey = mr->lkey
};
struct ib_send_wr wr = {
.wr_id = wrid,
.sg_list = &list,
.num_sge = 1,
.opcode = IB_WR_RDMA_WRITE,
.send_flags = IB_SEND_SIGNALED,
.wr.rdma = {
.remote_addr = buf + 8,
.rkey = mr->rkey
}
};
struct ib_send_wr *bad_wr;
return ib_post_send(qp, &wr, &bad_wr);
}
static int kcq_poll_thread(void *ctx_ptr)
{
struct kcq_ctx *ctx = ctx_ptr;
struct ib_wc *wc;
int i, n;
wc = kmalloc(wcsize * sizeof *wc, GFP_KERNEL);
if (!wc)
goto stall;
for (i = 0; i < depth; ++i) {
if (post_write(ctx->wrid, ctx->dma, ctx->qp, ctx->mr)) {
printk(KERN_ERR "Couldn't post work request %lld\n",
(long long) ctx->wrid);
goto stall;
}
++ctx->wrid;
}
schedule_delayed_work(&ctx->resize_work, msecs_to_jiffies(rand() % 1000));
while (!kthread_should_stop()) {
cond_resched();
n = ib_poll_cq(ctx->cq, wcsize, wc);
if (n < 0) {
printk(KERN_ERR "poll CQ failed %d\n", n);
goto stall;
}
for (i = 0; i < n; ++i) {
if (wc[i].status != IB_WC_SUCCESS) {
printk(KERN_ERR "Failed status %d for wr_id %lld\n",
wc[i].status, (long long) wc[i].wr_id);
goto stall;
}
if (wc[i].wr_id != ctx->exp_wrid) {
printk(KERN_ERR "wr_id mismatch %lld != %lld\n",
(long long) wc[i].wr_id, (long long) ctx->exp_wrid);
goto stall;
}
++ctx->exp_wrid;
if (!(ctx->exp_wrid % 500000))
printk(KERN_ERR "%12lld writes done\n", (long long) ctx->exp_wrid);
if (post_write(ctx->wrid, ctx->dma, ctx->qp, ctx->mr)) {
printk(KERN_ERR "Couldn't post work request %lld\n",
(long long) ctx->wrid);
goto stall;
}
++ctx->wrid;
}
}
return 0;
stall:
while (!kthread_should_stop())
schedule_timeout_interruptible(1);
return -1;
}
static void kcq_add_one(struct ib_device *device);
static void kcq_remove_one(struct ib_device *device);
static struct ib_client kcq_client = {
.name = "kcq",
.add = kcq_add_one,
.remove = kcq_remove_one
};
static void kcq_add_one(struct ib_device *device)
{
struct kcq_ctx *ctx;
ctx = kzalloc(sizeof *ctx, GFP_KERNEL);
if (!ctx)
return;
INIT_WORK(&ctx->resize_work, kcq_resize_work, ctx);
ctx->pd = ib_alloc_pd(device);
if (IS_ERR(ctx->pd)) {
printk(KERN_ERR "%s: Couldn't allocate PD\n", device->name);
goto err;
}
ctx->buf = dma_alloc_coherent(device->dma_device, PAGE_SIZE,
&ctx->dma, GFP_KERNEL);
if (!ctx->buf) {
printk(KERN_ERR "%s: Couldn't allocate buf\n", device->name);
goto err_pd;
}
ctx->mr = ib_get_dma_mr(ctx->pd,
IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE);
if (IS_ERR(ctx->mr)) {
printk(KERN_ERR "%s: Couldn't allocate MR\n", device->name);
goto err_buf;
}
ctx->cq = ib_create_cq(device, NULL, NULL, NULL, depth);
if (IS_ERR(ctx->cq)) {
printk(KERN_ERR "%s: Couldn't allocate CQ\n", device->name);
goto err_mr;
}
{
struct ib_qp_init_attr attr = {
.send_cq = ctx->cq,
.recv_cq = ctx->cq,
.cap = {
.max_send_wr = depth,
.max_recv_wr = 0,
.max_send_sge = 1,
.max_recv_sge = 1
},
.sq_sig_type = IB_SIGNAL_ALL_WR,
.qp_type = IB_QPT_RC
};
ctx->qp = ib_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
printk(KERN_ERR "%s: Couldn't create QP\n", device->name);
goto err_cq;
}
}
{
struct ib_qp_attr attr;
attr.qp_state = IB_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = 1;
attr.qp_access_flags = IB_ACCESS_REMOTE_WRITE;
if (ib_modify_qp(ctx->qp, &attr,
IB_QP_STATE |
IB_QP_PKEY_INDEX |
IB_QP_PORT |
IB_QP_ACCESS_FLAGS)) {
printk(KERN_ERR "%s: Failed to modify QP to INIT\n", device->name);
goto err_qp;
}
attr.qp_state = IB_QPS_RTR;
attr.path_mtu = IB_MTU_1024;
attr.dest_qp_num = ctx->qp->qp_num;
attr.rq_psn = 1;
attr.max_dest_rd_atomic = 4;
attr.min_rnr_timer = 12;
attr.ah_attr.ah_flags = 0;
attr.ah_attr.dlid = get_local_lid(device, 1);
attr.ah_attr.sl = 0;
attr.ah_attr.src_path_bits = 0;
attr.ah_attr.port_num = 1;
if (ib_modify_qp(ctx->qp, &attr,
IB_QP_STATE |
IB_QP_AV |
IB_QP_PATH_MTU |
IB_QP_DEST_QPN |
IB_QP_RQ_PSN |
IB_QP_MAX_DEST_RD_ATOMIC |
IB_QP_MIN_RNR_TIMER)) {
printk(KERN_ERR "%s: Failed to modify QP to RTR\n", device->name);
goto err_qp;
}
attr.qp_state = IB_QPS_RTS;
attr.timeout = 14;
attr.retry_cnt = 7;
attr.rnr_retry = 7;
attr.sq_psn = 1;
attr.max_rd_atomic = 4;
if (ib_modify_qp(ctx->qp, &attr,
IB_QP_STATE |
IB_QP_TIMEOUT |
IB_QP_RETRY_CNT |
IB_QP_RNR_RETRY |
IB_QP_SQ_PSN |
IB_QP_MAX_QP_RD_ATOMIC)) {
printk(KERN_ERR "%s: Failed to modify QP to RTS\n", device->name);
goto err_qp;
}
}
ctx->poll_task = kthread_create(kcq_poll_thread, ctx,
"kcq-%s", device->name);
if (IS_ERR(ctx->poll_task)) {
printk(KERN_ERR "%s: Failed to start poll thread\n", device->name);
goto err_qp;
}
wake_up_process(ctx->poll_task);
ib_set_client_data(device, &kcq_client, ctx);
return;
err_qp:
ib_destroy_qp(ctx->qp);
err_cq:
ib_destroy_cq(ctx->cq);
err_mr:
ib_dereg_mr(ctx->mr);
err_buf:
dma_free_coherent(device->dma_device, PAGE_SIZE, ctx->buf, ctx->dma);
err_pd:
ib_dealloc_pd(ctx->pd);
err:
kfree(ctx);
}
static void kcq_remove_one(struct ib_device *device)
{
struct kcq_ctx *ctx;
ctx = ib_get_client_data(device, &kcq_client);
if (!ctx)
return;
cancel_rearming_delayed_work(&ctx->resize_work);
kthread_stop(ctx->poll_task);
ib_destroy_qp(ctx->qp);
ib_destroy_cq(ctx->cq);
ib_dereg_mr(ctx->mr);
dma_free_coherent(device->dma_device, PAGE_SIZE, ctx->buf, ctx->dma);
ib_dealloc_pd(ctx->pd);
kfree(ctx);
}
static int __init kcq_init(void)
{
get_random_bytes(&seed, sizeof seed);
return ib_register_client(&kcq_client);
}
static void __exit kcq_cleanup(void)
{
ib_unregister_client(&kcq_client);
}
module_init(kcq_init);
module_exit(kcq_cleanup);
More information about the general
mailing list