[openib-general] create QP failure

[Roland Dreier]

Hmm, I couldn't duplicate your problem here.  The test program below
just prints

max_send_wr 65535, max_recv_wr 65535, max_send_sge 28, max_recv_sge 28 max_inline_data 0
max_send_wr 65535, max_recv_wr 65535, max_send_sge 28, max_recv_sge 28 max_inline_data 476

but creates both QPs successfully.  Kyle, can you run the program
below and tell me what it prints on your system?

Michael -- I did see one problem on Arbel with MemFree FW 5.1.0.  The
FW reports a max descriptor size of 496 and max SG entries of 30.  But
496 byte descriptors are not big enough for a send with 30 SG entries,
so an attempt to create a QP with the max parameters fails with

max_send_wr 16384, max_recv_wr 16384, max_send_sge 30, max_recv_sge 30 max_inline_data 0
Couldn't create QP #1

What do you think the best way to handle this is?

Thanks,
  Roland

here's the test program... it can be built with "gcc -o foo foo.c -libverbs"
and doesn't take any options to run.

/*
 * Copyright (c) 2006 Cisco Systems.  All rights reserved.
 *
 *	This program is free software; you can redistribute it and/or
 *	modify it under the terms of the GNU General Public License version
 *	2 as published by the Free Software Foundation.
 *
 * $Id$
 */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>

#include <infiniband/verbs.h>

int main(int argc, char *argv[])
{
	struct ibv_device      **dev_list;
	struct ibv_device 	*ib_dev;
	struct ibv_device_attr   devattr;
	struct ibv_context	*context;
	struct ibv_pd		*pd;
	struct ibv_cq		*cq;
	struct ibv_qp		*qp;
	struct ibv_wc		 wc;

	dev_list = ibv_get_device_list(NULL);
	if (!dev_list) {
		fprintf(stderr, "No IB devices found\n");
		return 1;
	}

	ib_dev = *dev_list;
	if (!ib_dev) {
		fprintf(stderr, "No IB devices found\n");
		return 1;
	}

	context = ibv_open_device(ib_dev);
	if (!context) {
		fprintf(stderr, "Couldn't get context for %s\n",
			ibv_get_device_name(ib_dev));
		return 1;
	}

	if (ibv_query_device(context, &devattr)) {
		fprintf(stderr, "Couldn't query device attrs\n");
		return 1;
	}

	pd = ibv_alloc_pd(context);
	if (!pd) {
		fprintf(stderr, "Couldn't allocate PD\n");
		return 1;
	}

	cq = ibv_create_cq(context, 1, NULL, NULL, 0);
	if (!cq) {
		fprintf(stderr, "Couldn't create CQ\n");
		return 1;
	}

	{
		struct ibv_qp_init_attr attr = {
			.send_cq = cq,
			.recv_cq = cq,
			.cap     = {
				.max_send_wr  = devattr.max_qp_wr,
				.max_recv_wr  = devattr.max_qp_wr,
				.max_send_sge = devattr.max_sge,
				.max_recv_sge = devattr.max_sge
			},
			.qp_type = IBV_QPT_RC
		};

		printf("max_send_wr %d, max_recv_wr %d, max_send_sge %d, max_recv_sge %d max_inline_data %d\n",
		       attr.cap.max_send_wr,  attr.cap.max_recv_wr,
		       attr.cap.max_send_sge, attr.cap.max_recv_sge,
		       attr.cap.max_inline_data);

		qp = ibv_create_qp(pd, &attr);
		if (!qp)  {
			fprintf(stderr, "Couldn't create QP #1\n");
			return 1;
		}

		printf("max_send_wr %d, max_recv_wr %d, max_send_sge %d, max_recv_sge %d max_inline_data %d\n",
		       attr.cap.max_send_wr,  attr.cap.max_recv_wr,
		       attr.cap.max_send_sge, attr.cap.max_recv_sge,
		       attr.cap.max_inline_data);

		qp = ibv_create_qp(pd, &attr);
		if (!qp)  {
			fprintf(stderr, "Couldn't create QP #2\n");
			return 1;
		}
	}

	return 0;
}