[ofa-general] Memory registration redux

[Roland Dreier]

Here's the test program:

#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <linux/types.h>
#include <linux/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>

#define UMMUNOT_INTF_VERSION		1

enum {
	UMMUNOT_EVENT_TYPE_INVAL	= 0,
	UMMUNOT_EVENT_TYPE_LAST		= 1,
};

enum {
	UMMUNOT_EVENT_FLAG_HINT		= 1 << 0,
};

/*
 * If type field is INVAL, then user_cookie_counter holds the
 * user_cookie for the region being reported; if the HINT flag is set
 * then hint_start/hint_end hold the start and end of the mapping that
 * was invalidated.  (If HINT is not set, then multiple events
 * invalidated parts of the registered range and hint_start/hint_end
 * should be ignored)
 *
 * If type is LAST, then the read operation has emptied the list of
 * invalidated regions, and user_cookie_counter holds the value of the
 * kernel's generation counter when the empty list occurred.  The
 * other fields are not filled in for this event.
 */
struct ummunot_event {
	__u32	type;
	__u32	flags;
	__u64	hint_start;
	__u64	hint_end;
	__u64	user_cookie_counter;
};

struct ummunot_register_ioctl {
	__u32	intf_version;	/* in */
	__u32	reserved1;
	__u64	start;		/* in */
	__u64	end;		/* in */
	__u64	user_cookie;	/* in */
};

#define UMMUNOT_MAGIC			'U'

#define UMMUNOT_REGISTER_REGION		_IOWR(UMMUNOT_MAGIC, 1, \
					      struct ummunot_register_ioctl)
#define UMMUNOT_UNREGISTER_REGION	_IOW(UMMUNOT_MAGIC, 2, __u64)

static int umn_fd;
static volatile unsigned long long *umn_counter;

static int umn_init(void)
{
	umn_fd = open("/dev/ummunot", O_RDONLY);
	if (umn_fd < 0) {
		perror("open");
		return 1;
	}

	umn_counter = mmap(NULL, sizeof *umn_counter, PROT_READ,
			   MAP_SHARED, umn_fd, 0);
	if (umn_counter == MAP_FAILED) {
		perror("mmap");
		return 1;
	}

	return 0;
}

static int umn_register(void *buf, size_t size, __u64 cookie)
{
	struct ummunot_register_ioctl r = {
		.intf_version	= UMMUNOT_INTF_VERSION,
		.start		= (unsigned long) buf,
		.end		= (unsigned long) buf + size,
		.user_cookie	= cookie,
	};

	if (ioctl(umn_fd, UMMUNOT_REGISTER_REGION, &r)) {
		perror("ioctl");
		return 1;
	}

	return 0;
}

static int umn_unregister(__u64 cookie)
{
	if (ioctl(umn_fd, UMMUNOT_UNREGISTER_REGION, &cookie)) {
		perror("ioctl");
		return 1;
	}

	return 0;
}

int main(int argc, char *argv[])
{
	int page_size = sysconf(_SC_PAGESIZE);
	void *t;

	if (umn_init())
		return 1;

	if (*umn_counter != 0) {
		fprintf(stderr, "counter = %lld (expected 0)\n", *umn_counter);
		return 1;
	}

	t = mmap(NULL, 3 * page_size, PROT_READ,
		 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, -1, 0);

	if (umn_register(t, 3 * page_size, 123))
		return 1;

	munmap(t + page_size, page_size);

	printf("ummunot events: %lld\n", *umn_counter);

	if (*umn_counter > 0) {
		struct ummunot_event ev[2];
		int len;
		int i;

		len = read(umn_fd, &ev, sizeof ev);
		printf("read %d events (%d tot)\n", len / sizeof ev[0], len);

		for (i = 0; i < len / sizeof ev[0]; ++i) {
			switch (ev[i].type) {
			case UMMUNOT_EVENT_TYPE_INVAL:
				printf("[%3d]: inval cookie %lld\n",
				       i, ev[i].user_cookie_counter);
				if (ev[i].flags & UMMUNOT_EVENT_FLAG_HINT)
					printf("  hint %llx...%lx\n",
					       ev[i].hint_start, ev[i].hint_end);
				break;
			case UMMUNOT_EVENT_TYPE_LAST:
				printf("[%3d]: empty up to %lld\n",
				       i, ev[i].user_cookie_counter);
				break;
			default:
				printf("[%3d]: unknown event type %d\n",
				       i, ev[i].type);
				break;
			}
		}
	}

	umn_unregister(123);
	munmap(t, page_size);

	printf("ummunot events: %lld\n", *umn_counter);

	return 0;
}