[openib-general] Re: [PATCH] signal driven io support for ib_uverbs

[Gleb Natapov]

On Wed, Jul 06, 2005 at 02:35:03PM -0700, Roland Dreier wrote:
> How does this patch look?
Actually I take may words about f_setown() back. Kernel dose not
set it by default. So unless you set it in the kernel or from
userspace using fcntl(F_SETOWN) you don't receive SIGIO signal.

The question is what is the right way? Do it from the kernel or
left it to userspace? Parts of the kernel do f_setown() from the
kernel others don't.

Other then that the patch is working OK.

> 
> Do you have a test program you're using to check SIGIO-based event
> handling?  If so can you post it?
Actually I am using it as part of MPI implementation, but I've crafted
patch for asyncwatch.c to use signals. Try to comment F_SETOWN bit,
you wont get events.


Index: libibverbs/examples/asyncwatch.c
===================================================================
--- libibverbs/examples/asyncwatch.c	(revision 2810)
+++ libibverbs/examples/asyncwatch.c	(working copy)
@@ -39,6 +39,14 @@
 #include <stdio.h>
 #include <endian.h>
 #include <byteswap.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <unistd.h>
+
 
 #include <infiniband/verbs.h>
 
@@ -48,12 +56,120 @@ static inline uint64_t be64_to_cpu(uint6
 static inline uint64_t be64_to_cpu(uint64_t x) { return x; }
 #endif
 
+struct ibv_context *context;
+
+void async_sigaction (int sig, siginfo_t *siginfo, void *ign)
+{
+        int ret;
+        struct ibv_async_event event;
+
+        if (siginfo->si_code != POLL_IN)
+        {
+                fprintf (stderr, "siginfo->si_code == %d\n", siginfo->si_code);
+                return;
+        }
+
+        if (siginfo->si_fd != context->async_fd)
+        {
+                fprintf (stderr, "siginfo->si_fd != context->async_fd\n");
+                return;
+        }
+
+        while(!(ret = ibv_get_async_event (context, &event)))
+        {
+		printf("  event_type %d, port %d\n", event.event_type,
+		       event.element.port_num);
+        }
+
+        if (errno == EAGAIN) /* no more events to process */
+                return;
+
+        /* other error */
+	fprintf (stderr, "Error in ibv_get_async_event\n");
+	exit (1);
+        return;
+}
+
+void async_event_setup (void)
+{
+	struct sigaction act;
+	int fdflags;
+
+	act.sa_sigaction = async_sigaction;
+	sigemptyset (&act.sa_mask);
+	act.sa_flags = SA_RESTART | SA_SIGINFO;
+	act.sa_restorer = NULL;
+
+	siginterrupt (SIGIO, 0); /* do not interrupt systemcalls */
+	if (sigaction (SIGIO, &act, NULL) != 0) {
+		perror ("sigaction");
+		exit (1);
+	}
+
+	fdflags = fcntl (context->async_fd, F_GETFL);
+
+	if (fdflags == -1) {
+		perror ("fcntl (F_GETFL)");
+		exit (1);
+	}
+
+	if (fcntl (context->async_fd, F_SETFL,
+				fdflags | O_NONBLOCK | O_ASYNC) == -1) {
+		perror ("fcntl (F_ETFL)");
+		exit (1);
+	}
+
+	if (fcntl (context->async_fd, F_SETSIG, SIGIO) == -1) {
+		perror ("fcntl (F_SETSIG)");
+		exit (1);
+	}
+
+	if (fcntl (context->async_fd, F_SETOWN, getpid()) == -1) {
+		perror ("fcntl (F_SETOWN)");
+		exit (1);
+	}
+}
+
+static void usage(const char *argv0)
+{
+	printf("Usage:\n");
+	printf("  %s            wait for asynchronous events\n", argv0);
+        printf("\n");
+        printf("Options:\n");
+        printf("  -s, --sig      wait for events using signals\n");
+}
+
 int main(int argc, char *argv[])
 {
 	struct dlist *dev_list;
 	struct ibv_device *ib_dev;
-	struct ibv_context *context;
 	struct ibv_async_event event;
+	int sig = 0;
+
+	while (1)
+	{
+		int c;
+
+		static struct option long_options[] = {
+			{ .name = "sig",    .has_arg = 0, .val = 's' },
+			{ 0 }
+		};
+
+		c = getopt_long(argc, argv, "s", long_options, NULL);
+
+		if (c == -1)
+			break;
+
+		switch (c)
+		{
+		case 's':
+			sig = 1;
+			break;
+		default:
+			usage (argv[0]);
+			return 1;
+		}
+	}
 
 	dev_list = ibv_get_devices();
 
@@ -75,12 +191,20 @@ int main(int argc, char *argv[])
 	printf("%s: async event FD %d\n",
 	       ibv_get_device_name(ib_dev), context->async_fd);
 
+	if (sig)
+		async_event_setup();
+
 	while (1) {
-		if (ibv_get_async_event(context, &event))
-			return 1;
 
-		printf("  event_type %d, port %d\n", event.event_type,
-		       event.element.port_num);
+		if (sig) {
+			pause ();
+		} else {
+			if (ibv_get_async_event(context, &event))
+				return 1;
+
+			printf("  event_type %d, port %d\n", event.event_type,
+		       	event.element.port_num);
+		}
 	}
 
 	return 0;
 
--
			Gleb.