[openib-general] Re: unable to run ibv_uc_pingpong and other example binaries
Joe Damato
damato at psc.edu
Tue Jul 19 09:28:06 PDT 2005
Michael S. Tsirkin wrote:
>Quoting r. Joe Damato <damato at psc.edu>:
>
>
>>Subject: unable to run ibv_uc_pingpong and other example binaries
>>
>>Hello,
>>
>> First I would like to apologize for the long email.
>>
>> I am currently using the gen2 revision 2734 openib source and kernel
>>2.6.11. I have compiled and installed libibverbs, libmthca and libibcm.
>>I have also created the /dev/infiniband devices, and I have my card
>>flashed to the latest firmware version.
>>
>>[root at frodo examples]# pwd
>>/cluster/src/OPENIB_SRC/openib-2734/gen2/trunk/src/userspace/libibverbs/examples
>>[root at frodo examples]# ./ibv_uc_pingpong
>>
>>
>
>You want to run .libs/ibv_uc_pingpong I think.
>The one you are running is a shell script.
>
>
>
OK, just tried it, same problem:
[root at frodo examples]# pwd
/cluster/src/OPENIB_SRC/openib-2734/gen2/trunk/src/userspace/libibverbs/examples
[root at frodo examples]# .libs/ibv_uc_pingpong
libucm: Error <-1:6> couldn't open IB cm device </dev/infiniband/ucm>
libibverbs: Fatal: couldn't open sysfs class 'infiniband_verbs'.
No IB devices found
>>libucm: Error <-1:6> couldn't open IB cm device </dev/infiniband/ucm>
>>libibverbs: Fatal: couldn't open sysfs class 'infiniband_verbs'.
>>No IB devices found
>>[root at frodo examples]# ./ibv_rc_pingpong
>>libucm: Error <-1:6> couldn't open IB cm device </dev/infiniband/ucm>
>>libibverbs: Fatal: couldn't open sysfs class 'infiniband_verbs'.
>>No IB devices found
>>
>>
>
>Hmm. I didnt know libibverbs depends on libucm.
>Can you strace it?
>
>
>
sure. strace:
[root at frodo examples]# strace .libs/ibv_uc_pingpong
execve(".libs/ibv_uc_pingpong", [".libs/ibv_uc_pingpong"], [/* 22 vars
*/]) = 0
brk(0) = 0x504000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2aaaaaaab000
uname({sys="Linux", node="frodo.psc.edu", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/tls/x86_64/libibverbs.so.1", O_RDONLY) = -1 ENOENT
(No such file or directory)
open("/usr/local/lib/tls/libibverbs.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/x86_64/libibverbs.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/libibverbs.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\33\0\0"...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=91160, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2aaaaaaac000
mmap(NULL, 1066272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2aaaaaaad000
mprotect(0x2aaaaaab2000, 1045792, PROT_NONE) = 0
mmap(0x2aaaaabb1000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x2aaaaabb1000
close(3) = 0
open("/usr/local/lib/tls/x86_64/libsysfs.so.1", O_RDONLY) = -1 ENOENT
(No such file or directory)
open("/usr/local/lib/tls/libsysfs.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/lib/x86_64/libsysfs.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/libsysfs.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=40696, ...}) = 0
mmap(NULL, 40696, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2aaaaabb2000
close(3) = 0
open("/usr/lib64/libsysfs.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300/`\257"...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=52728, ...}) = 0
mmap(0x33af600000, 1097280, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x33af600000
mprotect(0x33af60c000, 1048128, PROT_NONE) = 0
mmap(0x33af70b000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x33af70b000
close(3) = 0
open("/usr/local/lib/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT
(No such file or directory)
open("/usr/local/lib/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0V\240\341"...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=115080, ...}) = 0
mmap(0x34e1a00000, 1131384, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x34e1a00000
mprotect(0x34e1a0f000, 1069944, PROT_NONE) = 0
mmap(0x34e1b0f000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x34e1b0f000
mmap(0x34e1b11000, 13176, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x34e1b11000
close(3) = 0
open("/usr/local/lib/tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/lib/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/lib64/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\17 "...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=23184, ...}) = 0
mmap(0x34e1200000, 1061096, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x34e1200000
mprotect(0x34e1202000, 1052904, PROT_NONE) = 0
mmap(0x34e1302000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x34e1302000
close(3) = 0
open("/usr/local/lib/tls/x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/lib/tls/libc.so.6", O_RDONLY) = -1 ENOENT (No such file
or directory)
open("/usr/local/lib/x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/lib64/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\305"...,
640) = 640
lseek(3, 624, SEEK_SET) = 624
read(3, "\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\2\0\0\0\4\0\0\0"..., 32)
= 32
fstat(3, {st_mode=S_IFREG|0755, st_size=1553232, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2aaaaabbc000
mmap(0x34e0f00000, 2321544, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x34e0f00000
mprotect(0x34e102e000, 1084552, PROT_NONE) = 0
mmap(0x34e112d000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12d000) = 0x34e112d000
mmap(0x34e1133000, 15496, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x34e1133000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2aaaaabbd000
arch_prctl(ARCH_SET_FS, 0x2aaaaabbd800) = 0
mprotect(0x34e112d000, 16384, PROT_READ) = 0
mprotect(0x34e1302000, 4096, PROT_READ) = 0
mprotect(0x34e1b0f000, 4096, PROT_READ) = 0
mprotect(0x34e0e19000, 4096, PROT_READ) = 0
munmap(0x2aaaaabb2000, 40696) = 0
set_tid_address(0x2aaaaabbd890) = 13505
rt_sigaction(SIGRTMIN, {0x34e1a05190, [], SA_RESTORER|SA_SIGINFO,
0x34e1a0c380}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x34e1a05210, [],
SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x34e1a0c380}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
_sysctl({{CTL_KERN, KERN_VERSION}, 2, 0x7ffffffff620, 35, (nil), 0}) = 0
brk(0) = 0x504000
brk(0x525000) = 0x525000
futex(0x34e13030a8, FUTEX_WAKE, 2147483647) = 0
getuid() = 0
geteuid() = 0
open("/usr/local/lib/infiniband", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
getdents64(3, /* 7 entries */, 4096) = 208
getdents64(3, /* 0 entries */, 4096) = 0
close(3) = 0
open("/usr/local/lib/infiniband/libibcm.so", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\f\0"...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=42601, ...}) = 0
mmap(NULL, 1058376, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2aaaaabbe000
mprotect(0x2aaaaabc1000, 1046088, PROT_NONE) = 0
mmap(0x2aaaaacc0000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x2aaaaacc0000
close(3) = 0
open("/dev/infiniband/ucm", O_RDWR) = -1 ENXIO (No such device or address)
write(2, "libucm: Error <-1:6> couldn\'t op"..., 70libucm: Error <-1:6>
couldn't open IB cm device </dev/infiniband/ucm>
) = 70
munmap(0x2aaaaabbe000, 1058376) = 0
open("/usr/local/lib/infiniband/mthca.so", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\24\0"...,
640) = 640
fstat(3, {st_mode=S_IFREG|0755, st_size=125471, ...}) = 0
mmap(NULL, 1070328, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x2aaaaabbe000
mprotect(0x2aaaaabc3000, 1049848, PROT_NONE) = 0
mmap(0x2aaaaacc3000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x2aaaaacc3000
close(3) = 0
open("/proc/mounts", O_RDONLY) = 3
futex(0x34e11346e0, FUTEX_WAKE, 2147483647) = 0
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x2aaaaacc4000
read(3, "rootfs / rootfs rw 0 0\n/dev /dev"..., 1024) = 601
close(3) = 0
munmap(0x2aaaaacc4000, 4096) = 0
lstat("/sys/class/infiniband_verbs", 0x7ffffffff580) = -1 ENOENT (No
such file or directory)
write(2, "libibverbs: Fatal: couldn\'t open"..., 65libibverbs: Fatal:
couldn't open sysfs class 'infiniband_verbs'.
) = 65
write(2, "No IB devices found\n", 20No IB devices found
) = 20
exit_group(1) = ?
>>but....
>>
>>[root at frodo examples]# cd /dev/
>>[root at frodo dev]# ls infiniband/
>>issm0 issm1 ucm umad0 umad1 uverbs0 uverbs1
>>[root at frodo dev]# ls -al infiniband/
>>total 0
>>drw-rw-rw- 2 root root 180 Jul 19 11:36 .
>>drwxr-xr-x 8 root root 13940 Jul 19 11:35 ..
>>crw-rw-rw- 1 root root 231, 64 Jul 19 11:35 issm0
>>crw-rw-rw- 1 root root 231, 65 Jul 19 11:35 issm1
>>crw-rw-rw- 1 root root 231, 255 Jul 19 11:36 ucm
>>crw-rw-rw- 1 root root 231, 0 Jul 19 11:35 umad0
>>crw-rw-rw- 1 root root 231, 1 Jul 19 11:35 umad1
>>crw-rw-rw- 1 root root 231, 192 Jul 19 11:36 uverbs0
>>crw-rw-rw- 1 root root 231, 193 Jul 19 11:36 uverbs1
>>
>>
>
>BTW, you probably dont want umad and issm world-accessible, it can be
>a security hole: these are management interfaces.
>
>
>
ok -- i'll fix that -- THANKS for the warning.
>>[...]
>>I currently have 8 other machines with IB, however when trying to
>>run ibv_rc_pingpong they all report the same error (each of the 8
>>systems are running the same kernel and all cards are flashed to the
>>latest firmware):
>>
>>ost0# ./ibv_rc_pingpong
>>Couldn't get local LID
>>
>>but when checking ibstat it shows that Port 2 is active
>>
>>
>
>I think these tests assume that since local LID is 0 sm isnt running.
>Try running opensm on one of the nodes.
>
>
>
opensm is running on frodo which is connected to the same IB switch as
ost0 (all nodes are running on the same switch) so opensm is ALREADY
running for this IB set up, but just for the heck of it:
[root at ost0 examples]$ /usr/local/ib/bin/opensm
-------------------------------------------------
OpenSM Rev:openib-1.0.0
Command Line Arguments:
Log File: /var/log/osm.log
-------------------------------------------------
using default guid 0x2c90200003f42
Suspended
[root at ost0 examples]$ bg
[1] /usr/local/ib/bin/opensm &
[root at ost0 examples]$ ./ibv_uc_pingpong
Couldn't get local LID
Any other suggestions?
Thanks -
Joe Damato
More information about the general
mailing list