[ofa-general] Memory registration redux

[Woodruff, Robert J]

Hay Roland,

One question from my MPI guys.  Looks like you have added the ability
to have more than one version of the device to allow future versions,
i.e., the .intf_version in the register call. 

struct ummunot_register_ioctl r = {
		.intf_version	= UMMUNOT_INTF_VERSION,
		.start		= (unsigned long) buf,
		.end		= (unsigned long) buf + size,
		.user_cookie	= cookie,
	};


I assume there will be some ioctl to allow a program to discover at runtime
the version(s) of the device that are supported on a particular system ?

woody
 

-----Original Message-----
From: general-bounces at lists.openfabrics.org [mailto:general-bounces at lists.openfabrics.org] On Behalf Of Roland Dreier
Sent: Tuesday, May 26, 2009 4:14 PM
To: Jason Gunthorpe
Cc: Pavel Shamis; Hans Westgaard Ry; Dontje; Lenny Verkhovsky; H??kon Bugge; Donald Kerr; OpenFabrics General; Supalov, Alexander
Subject: Re: [ofa-general] Memory registration redux

Here's the test program:

#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <linux/types.h>
#include <linux/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>

#define UMMUNOT_INTF_VERSION		1

enum {
	UMMUNOT_EVENT_TYPE_INVAL	= 0,
	UMMUNOT_EVENT_TYPE_LAST		= 1,
};

enum {
	UMMUNOT_EVENT_FLAG_HINT		= 1 << 0,
};

/*
 * If type field is INVAL, then user_cookie_counter holds the
 * user_cookie for the region being reported; if the HINT flag is set
 * then hint_start/hint_end hold the start and end of the mapping that
 * was invalidated.  (If HINT is not set, then multiple events
 * invalidated parts of the registered range and hint_start/hint_end
 * should be ignored)
 *
 * If type is LAST, then the read operation has emptied the list of
 * invalidated regions, and user_cookie_counter holds the value of the
 * kernel's generation counter when the empty list occurred.  The
 * other fields are not filled in for this event.
 */
struct ummunot_event {
	__u32	type;
	__u32	flags;
	__u64	hint_start;
	__u64	hint_end;
	__u64	user_cookie_counter;
};

struct ummunot_register_ioctl {
	__u32	intf_version;	/* in */
	__u32	reserved1;
	__u64	start;		/* in */
	__u64	end;		/* in */
	__u64	user_cookie;	/* in */
};

#define UMMUNOT_MAGIC			'U'

#define UMMUNOT_REGISTER_REGION		_IOWR(UMMUNOT_MAGIC, 1, \
					      struct ummunot_register_ioctl)
#define UMMUNOT_UNREGISTER_REGION	_IOW(UMMUNOT_MAGIC, 2, __u64)

static int umn_fd;
static volatile unsigned long long *umn_counter;

static int umn_init(void)
{
	umn_fd = open("/dev/ummunot", O_RDONLY);
	if (umn_fd < 0) {
		perror("open");
		return 1;
	}

	umn_counter = mmap(NULL, sizeof *umn_counter, PROT_READ,
			   MAP_SHARED, umn_fd, 0);
	if (umn_counter == MAP_FAILED) {
		perror("mmap");
		return 1;
	}

	return 0;
}

static int umn_register(void *buf, size_t size, __u64 cookie)
{
	struct ummunot_register_ioctl r = {
		.intf_version	= UMMUNOT_INTF_VERSION,
		.start		= (unsigned long) buf,
		.end		= (unsigned long) buf + size,
		.user_cookie	= cookie,
	};

	if (ioctl(umn_fd, UMMUNOT_REGISTER_REGION, &r)) {
		perror("ioctl");
		return 1;
	}

	return 0;
}

static int umn_unregister(__u64 cookie)
{
	if (ioctl(umn_fd, UMMUNOT_UNREGISTER_REGION, &cookie)) {
		perror("ioctl");
		return 1;
	}

	return 0;
}

int main(int argc, char *argv[])
{
	int page_size = sysconf(_SC_PAGESIZE);
	void *t;

	if (umn_init())
		return 1;

	if (*umn_counter != 0) {
		fprintf(stderr, "counter = %lld (expected 0)\n", *umn_counter);
		return 1;
	}

	t = mmap(NULL, 3 * page_size, PROT_READ,
		 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);

	if (umn_register(t, 3 * page_size, 123))
		return 1;

	munmap(t + page_size, page_size);

	printf("ummunot events: %lld\n", *umn_counter);

	if (*umn_counter > 0) {
		struct ummunot_event ev[2];
		int len;
		int i;

		len = read(umn_fd, &ev, sizeof ev);
		printf("read %d events (%d tot)\n", len / sizeof ev[0], len);

		for (i = 0; i < len / sizeof ev[0]; ++i) {
			switch (ev[i].type) {
			case UMMUNOT_EVENT_TYPE_INVAL:
				printf("[%3d]: inval cookie %lld\n",
				       i, ev[i].user_cookie_counter);
				if (ev[i].flags & UMMUNOT_EVENT_FLAG_HINT)
					printf("  hint %llx...%lx\n",
					       ev[i].hint_start, ev[i].hint_end);
				break;
			case UMMUNOT_EVENT_TYPE_LAST:
				printf("[%3d]: empty up to %lld\n",
				       i, ev[i].user_cookie_counter);
				break;
			default:
				printf("[%3d]: unknown event type %d\n",
				       i, ev[i].type);
				break;
			}
		}
	}

	umn_unregister(123);
	munmap(t, page_size);

	printf("ummunot events: %lld\n", *umn_counter);

	return 0;
}
_______________________________________________
general mailing list
general at lists.openfabrics.org
http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general

To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general