[openib-general] ifconfig down stuck
Michael S. Tsirkin
mst at mellanox.co.il
Wed Nov 15 01:20:44 PST 2006
running nightly tests on latest IB code from Linus' git tree (backported
to 2.6.9), ifconfig down got stuck at some point. I see a ton of
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
unregister_netdevice: waiting for ib0 to become free. Usage count = -61
in dmesg
sysrq trace below.
I also noticed that ib_ucm:ib_ucm_cleanup seems to be stuck in
mutex_lock - apparently ib_ucm module was being removed.
Not sure whether these are related problems.
SysRq : Show State
sibling
task PC pid father child younger older
init S 000000000000000b 0 1 0 2 (NOTLB)
00000100c7fb1d78 0000000000000002 00000100c7fb1e58 ffffffff8018f019
0000000000000000 00000100c7fb1e58 000000d033715120 0000000300000246
0000010133ba77f0 00000000000007ce
Call Trace:<ffffffff8018f019>{dput+56} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff8011026a>{system_call+126}
migration/0 S 000001000105b7e0 0 2 1 3 (L-TLB)
0000010133b59ec8 0000000000000046 000001000105b7e0 0000001900000076
0000010132979030 0000000000000076 0000010001044a40 0000000000000001
0000010133bab7f0 0000000000000185
Call Trace:<ffffffff80134f1e>{migration_thread+323} <ffffffff80134ddb>{migration_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ksoftirqd/0 S 0000000000000000 0 3 1 4 2 (L-TLB)
0000010133b5df08 0000000000000046 000001012e2dd030 000000190000007a
0000010133695030 000000000000007a 0000010001044a40 0000000000000000
0000010133bab030 00000000000000e9
Call Trace:<ffffffff8013cc90>{ksoftirqd+0} <ffffffff8013cccc>{ksoftirqd+60}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
migration/1 S 000001000105b7e0 0 4 1 5 3 (L-TLB)
0000010133b5fec8 0000000000000046 000001000105b7e0 0000001900000074
00000101326077f0 0000000000000074 000001000104ca40 0000000100000001
0000010133bac7f0 0000000000000128
Call Trace:<ffffffff80134f1e>{migration_thread+323} <ffffffff80134ddb>{migration_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ksoftirqd/1 S 0000000000000001 0 5 1 6 4 (L-TLB)
00000100c7f21f08 0000000000000046 000001013362d7f0 0000001900000079
0000010133695030 0000000000000079 000001000104ca40 0000000100000000
0000010133bac030 00000000000000fa
Call Trace:<ffffffff8013ca84>{tasklet_action+103} <ffffffff8013cc90>{ksoftirqd+0}
<ffffffff8013cccc>{ksoftirqd+60} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
migration/2 S 000001000105b7e0 0 6 1 7 5 (L-TLB)
00000100c7f25ec8 0000000000000046 000001000105b7e0 0000001900000074
0000010132979030 0000000000000074 0000010001054a40 0000000200000001
0000010133b727f0 00000000000001a2
Call Trace:<ffffffff80134f1e>{migration_thread+323} <ffffffff80134ddb>{migration_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ksoftirqd/2 S 0000000000000002 0 7 1 8 6 (L-TLB)
00000100c7f57f08 0000000000000046 0000010133b1b030 0000001900000079
0000010050f5e7f0 0000000000000079 0000010001054a40 0000000200000000
0000010133b72030 0000000000000143
Call Trace:<ffffffff8013ca84>{tasklet_action+103} <ffffffff8013cc90>{ksoftirqd+0}
<ffffffff8013cccc>{ksoftirqd+60} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
migration/3 S 00000100010437e0 0 8 1 9 7 (L-TLB)
00000100c7f59ec8 0000000000000046 00000100010437e0 0000001900000074
00000101326077f0 0000000000000074 000001000105ca40 0000000300000001
00000100c7f2f7f0 0000000000000191
Call Trace:<ffffffff80134f1e>{migration_thread+323} <ffffffff80134ddb>{migration_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ksoftirqd/3 S 0000000000000000 0 9 1 10 8 (L-TLB)
0000010005d8df08 0000000000000046 000001013296d030 0000001900000077
0000010132b1e7f0 0000000000000077 000001000105ca40 0000000300000000
00000100c7f2f030 0000000000000153
Call Trace:<ffffffff8013cc90>{ksoftirqd+0} <ffffffff8013cccc>{ksoftirqd+60}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
events/0 S ffffffff80162f6a 0 10 1 14 11 9 (L-TLB)
0000010133833e68 0000000000000046 ffffffff803d4300 0000000000000246
0000000000000246 0000000000000246 000000010321acaf 0000000000000246
000001013380e7f0 000000000000166a
Call Trace:<ffffffff80162f6a>{cache_reap+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
events/1 S ffffffff80162f6a 0 11 1 391 12 10 (L-TLB)
0000010133835e68 0000000000000046 0000010133ba7030 000001000104eec0
0000000000000005 0000000000000246 000000010321b13f 0000000100000246
000001013380e030 0000000000000d63
Call Trace:<ffffffff80162f6a>{cache_reap+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
events/2 S ffffffff80162f6a 0 12 1 2069 13 11 (L-TLB)
0000010133839e68 0000000000000046 0000010133baa7f0 0000010001056ec0
000000000000000a 0000000000000246 000000010321ab60 0000000200000246
000001013380f7f0 00000000000012e3
Call Trace:<ffffffff80162f6a>{cache_reap+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
events/3 S ffffffff80162f6a 0 13 1 392 48 12 (L-TLB)
000001013383be68 0000000000000046 0000010133baa030 0000000000000000
0000000000000000 0000000000000246 000000010321ae1f 0000000300000246
000001013380f030 0000000000000a70
Call Trace:<ffffffff80162f6a>{cache_reap+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
khelper S 0000010021105bd8 0 14 10 15 (L-TLB)
000001013383de68 0000000000000046 00000100010437e0 000000190000006a
0000010026ae3030 000000000000006a 0000010001044a40 0000000000000001
00000101338117f0 000000000000016b
Call Trace:<ffffffff80147171>{__call_usermodehelper+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
kacpid S 0000010005dac040 0 15 10 44 14 (L-TLB)
00000100c7d71e68 0000000000000046 0000000000000000 0000000000000072
0000000000000000 0000000000000001 ffffffff80467880 0000000000000000
0000010133811030 00000000000007fa
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133dfa>{__wake_up_common+67} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
kblockd/0 S ffffffff8024e2d9 0 44 10 45 15 (L-TLB)
00000100c7db1e68 0000000000000046 00000060ffffffff 0000000000000000
000000ac00000246 00000101336babc8 0000010005f41780 0000000005ec40c0
00000100c7d4c7f0 0000000000000266
Call Trace:<ffffffff8024e2d9>{blk_unplug_work+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
kblockd/1 S 0000010005ec42a0 0 45 10 46 44 (L-TLB)
00000100c7db3e68 0000000000000046 00000060ffffffff 0000001900000074
00000101336d4030 0000000000000074 000001000104ca40 0000000105ec40c0
00000100c7d4c030 0000000000000819
Call Trace:<ffffffff8024e2d9>{blk_unplug_work+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
kblockd/2 S ffffffff8024e2d9 0 46 10 47 45 (L-TLB)
00000100c7db7e68 0000000000000046 00000060ffffffff 0000000000000000
000000c900000246 00000101336babc8 0000010005f41780 0000000205ec40c0
00000100c7d5d7f0 00000000000006a9
Call Trace:<ffffffff8024e2d9>{blk_unplug_work+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}<0>unregister_netdevice: waiting for ib0 to become free. Usage count = -61
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
kblockd/3 S ffffffff8024e2d9 0 47 10 75 46 (L-TLB)
00000100c7db9e68 0000000000000046 00000060ffffffff 0000000000000000
000000a500000246 00000101336babc8 0000010005f41780 0000000305ec40c0
00000100c7d5d030 000000000000025a
Call Trace:<ffffffff8024e2d9>{blk_unplug_work+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
khubd S 0000000000000000 0 48 1 77 13 (L-TLB)
00000100c7ddbe78 0000000000000046 000001013230c030 000000000000007d
0000000000000000 0000000000000000 ffffffff80467880 0000000000000246
00000100c7da07f0 0000000000000959
Call Trace:<ffffffff8015d666>{free_pages_bulk+692} <ffffffff80287641>{hub_thread+2921}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80110f47>{child_rip+8} <ffffffff80286ad8>{hub_thread+0}
<ffffffff80110f3f>{child_rip+0}
pdflush S ffffffff8014b4f0 0 75 10 76 47 (L-TLB)
00000100c7df1ec8 0000000000000046 0000000000000000 000000000000006a
00000100c7dc57f0 00000000c7d75240 ffffffff80467880 00000000010437e0
00000100c7dc57f0 00000000000004e0
Call Trace:<ffffffff8013407a>{set_user_nice+259} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8015fec8>{pdflush+191} <ffffffff8015fe09>{pdflush+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
pdflush S ffffffff8014b4f0 0 76 10 78 75 (L-TLB)
0000010005e03ec8 0000000000000046 0000010005d2bc88 000000000003ce19
000000010321b576 00000100c7fb1de8 00000000fffffffc 00000001c7fb1dd8
00000100c7dc5030 00000000000002ac
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8015fec8>{pdflush+191}
<ffffffff8015fe09>{pdflush+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
aio/0 S 00000100c7d9eec0 0 78 10 79 76 (L-TLB)
0000010005e27e68 0000000000000046 0000000000000000 000000000000006b
0000000000000000 0000000000000008 ffffffff80467880 0000000000000000
00000100c7dc7030 00000000000005d2
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133dfa>{__wake_up_common+67} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
kswapd0 S 0000010005e05f08 0 77 1 225 48 (L-TLB)
0000010005e05eb8 0000000000000046 0000010005e05e08 000000000000007d
0000000000000000 0000000000000000 ffffffff80467880 00000000801346b7
00000100c7dc77f0 0000000000000d5c
Call Trace:<ffffffff80165ea0>{kswapd+231} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80110f47>{child_rip+8}
<ffffffff80165db9>{kswapd+0} <ffffffff80110f3f>{child_rip+0}
aio/1 S 00000100c7d9ef40 0 79 10 80 78 (L-TLB)
0000010005e2be68 0000000000000046 0000000000000000 0000000000000000
0000000000000000 0000000000000008 000000000000231e 0000000100000000
00000100c7df97f0 0000000000000cf8
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133dfa>{__wake_up_common+67} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
aio/2 S 00000100c7d9efc0 0 80 10 81 79 (L-TLB)
0000010005e2de68 0000000000000046 0000000000000000 0000000000000000
0000000000000000 0000000000000008 000000000000231e 0000000200000000
00000100c7df9030 0000000000000d5a
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133dfa>{__wake_up_common+67} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
aio/3 S 00000100c7d9f040 0 81 10 1987 80 (L-TLB)
0000010005e2fe68 0000000000000046 0000000000000000 0000000000000000
0000000000000000 0000000000000007 0000000000001f36 0000000300000000
00000100c7dfa7f0 0000000000000c32
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133dfa>{__wake_up_common+67} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
kseriod S 0000000000000000 0 225 1 359 77 (L-TLB)
00000100c7dddec8 0000000000000046 00000100c7ddde18 000001000000007a
ffffffff80132215 0000000000000000 ffffffff80467880 00000000c7daf218
00000100c7da0030 0000000000000e74
Call Trace:<ffffffff80132215>{deactivate_task+37} <ffffffff80240ffc>{serio_thread+469}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff8013a7e5>{do_exit+3137}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80110f47>{child_rip+8}
<ffffffff801d633b>{selinux_d_instantiate+0} <ffffffff801d633b>{selinux_d_instantiate+0}
<ffffffff801d633b>{selinux_d_instantiate+0} <ffffffff80240e27>{serio_thread+0}
<ffffffff80110f3f>{child_rip+0}
scsi_eh_0 S 0000000000573710 0 359 1 400 225 (L-TLB)
0000010005d4ddf8 0000000000000046 00000101336ba800 0000010133b1bbd2
0000000000000018 0000000000000008 000000000000231e 0000000100000246
0000010133b1b7f0 000000000000294f
Call Trace:<ffffffff80309517>{__down_interruptible+203} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8030aeb1>{__down_failed_interruptible+53}
<ffffffffa0006825>{:scsi_mod:.text.lock.scsi_error+45}
<ffffffff8013a7e5>{do_exit+3137} <ffffffff80110f47>{child_rip+8}
<ffffffffa0005ab9>{:scsi_mod:scsi_error_handler+0}
<ffffffff80110f3f>{child_rip+0}
ata/0 S ffffffff80309800 0 391 11 (L-TLB)
0000010132919e68 0000000000000046 0000000000000000 0000000000000000
0000000000000000 0000000000000009 0000000000002706 0000000000000000
00000101336957f0 0000000000001664
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
ata/1 S 000001013372a900 0 392 13 393 (L-TLB)
0000010132841e68 0000000000000046 0000000100000001 0000001900000075
0000010133ba77f0 0000000000000075 000001000104ca40 0000000100000100
0000010133b457f0 0000000000000e3d
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
ata/2 S ffffffff80309800 0 393 13 394 392 (L-TLB)
0000010132935e68 0000000000000046 0000000100000001 0000000000000002
0000010133b459d0 0000000000000008 000000000000231e 0000000200000001
0000010005f617f0 00000000000010d9
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
ata/3 S 000001013372aa00 0 394 13 393 (L-TLB)
0000010132937e68 0000000000000046 000001000104b7e0 0000001900000086
00000100c7f2f030 0000000000000086 000001000105ca40 0000000333679240
0000010005f61030 0000000000000d9d
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
scsi_eh_1 S 0000010133782900 0 400 1 413 359 (L-TLB)
000001013291bdf8 0000000000000046 0000010133b13000 000001013378abd2
000001013378a7f0 0000000000000006 0000000000001acc 0000000200000246
000001013378a7f0 0000000000000e6a
Call Trace:<ffffffff80309517>{__down_interruptible+203} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8030aeb1>{__down_failed_interruptible+53}
<ffffffffa0006825>{:scsi_mod:.text.lock.scsi_error+45}
<ffffffff8013a7e5>{do_exit+3137} <ffffffff80110f47>{child_rip+8}
<ffffffffa0005ab9>{:scsi_mod:scsi_error_handler+0}
<ffffffff80110f3f>{child_rip+0}
kjournald D ffffffff80309800 0 413 1 1573 400 (L-TLB)
0000010005fa7e78 ffffffff8030a14d 0000010133baa7f0 0000001900000073
00000101336d4030 0000000000000073 0000010001044a40 000000022d0ec240
0000010005f9b030 000000000000021d
Call Trace:<ffffffff8030a14d>{thread_return+88} <ffffffff8013fdf3>{del_timer+107}
<ffffffffa0078914>{:jbd:kjournald+250} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffffa0078814>{:jbd:commit_timeout+0}
<ffffffff80110f47>{child_rip+8} <ffffffffa007881a>{:jbd:kjournald+0}
<ffffffff80110f3f>{child_rip+0}
udevd S 0000000000000006 0 1573 1 2087 413 (NOTLB)
0000010131e5bd78 0000000000000002 0000010133baa7f0 00000000000000d2
000000d00000f810 0000000000000246 0000000000000246 000000020000e780
0000010132235030 0000000000001a10
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff802ad3d3>{datagram_poll+39}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff8011026a>{system_call+126}
kauditd S ffffffff8014b4f0 0 1987 10 15262 81 (L-TLB)
00000101315f3ea8 0000000000000046 00000004bc4af1e4 000001013380e7f0
00000100010437e0 0000000000000000 0000000000000000 00000000bc465a6c
00000101329797f0 00000000000009f8
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff80154360>{kauditd_thread+0}
<ffffffff801544df>{kauditd_thread+383} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
kmirrord S ffffffff8014b4f0 0 2069 12 15263 (L-TLB)
0000010130999e68 0000000000000046 ffffffff801476af 00000101317b37a8
000000000000000a 0000000000000000 0000000000000008 0000000230998000
0000010005d5e7f0 0000000000001a3b
Call Trace:<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
kjournald S 0000000000000000 0 2087 1 2690 1573 (L-TLB)
00000101304d3e78 0000000000000046 0000010133baa7f0 0000010130663af8
0000010133b66a98 0000000000000005 0000000000000000 00000002000003e8
000001013362d030 0000000000002484
Call Trace:<ffffffffa0078a14>{:jbd:kjournald+506} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffffa0078814>{:jbd:commit_timeout+0}
<ffffffff80110f47>{child_rip+8} <ffffffffa007881a>{:jbd:kjournald+0}
<ffffffff80110f3f>{child_rip+0}
dhclient S ffffffff80309800 0 2690 1 2905 2087 (NOTLB)
0000010131151d78 0000000000000002 0000010132839080 ffffffff802a72c8
000001002afdf800 0000000000000246 000000d000000000 0000000000000246
000001013230c7f0 000000000000123c
Call Trace:<ffffffff802a72c8>{sock_recvmsg+284} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff801932d4>{dnotify_parent+34}
<ffffffff8011026a>{system_call+126}
syslogd D 000001012fd1fef8 0 2905 1 2909 2690 (NOTLB)
000001012fd1fd98 0000000000000006 00000100010537e0 0000000000000002
000001012fd1fd18 ffffffff801321e3 00000100010537e0 0000000000000003
00000101336d4030 0000000000000190
Call Trace:<ffffffff801321e3>{activate_task+124} <ffffffffa0078c3c>{:jbd:log_wait_commit+202}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffffa0073ab2>{:jbd:journal_stop+536} <ffffffffa0073b08>{:jbd:journal_force_commit+32}
<ffffffffa0090774>{:ext3:ext3_force_commit+36} <ffffffff80197fd0>{__writeback_single_inode+311}
<ffffffff80198315>{sync_inode+52} <ffffffffa0087027>{:ext3:ext3_sync_file+191}
<ffffffff8017ac12>{sys_fsync+155} <ffffffff8011026a>{system_call+126}
klogd S 000000000000006e 0 2909 1 2920 2905 (NOTLB)
0000010130821be8 0000000000000006 0000010133ba7030 ffffffff80132155
0000000000000080 0000000000000080 00000000000000d0 0000000100000008
000001013229b030 000000000000067f
Call Trace:<ffffffff80132155>{recalc_task_prio+337} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffff801356aa>{prepare_to_wait_exclusive+21} <ffffffff80304899>{unix_wait_for_peer+163}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff803053ab>{unix_dgram_sendmsg+957} <ffffffff802a75b4>{sock_aio_write+306}
<ffffffff801795c1>{do_sync_write+173} <ffffffff80137d3a>{do_syslog+482}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff801932d4>{dnotify_parent+34}
<ffffffff801796cf>{vfs_write+226} <ffffffff801797a4>{sys_write+69}
<ffffffff8011026a>{system_call+126}
irqbalance S 0000000000000000 0 2920 1 2932 2909 (NOTLB)
000001012fe43ee8 0000000000000006 0000000000001000 0000002a9556d000
000001012fd63558 0000002a95600000 0000010130191558 000000019556d000
0000010132af2030 000000000000fe35
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff80140811>{sys_nanosleep+192}
<ffffffff8011026a>{system_call+126}
portmap S 7fffffffffffffff 0 2932 1 2952 2920 (NOTLB)
000001012f89fe88 0000000000000006 000001012f89fe48 0000001c2abb4070
000001012f89fec8 ffffffff00000010 000000d02f89fe48 0000000100000246
000001013230c030 0000000000005875
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff802cfe27>{tcp_poll+44} <ffffffff8018bf33>{sys_poll+604}
<ffffffff8018b399>{__pollwait+0} <ffffffff8011026a>{system_call+126}
rpc.statd S 0000000000000007 0 2952 1 3553 2932 (NOTLB)
000001012f839d78 0000000000000006 0000000000000071 ffffffff802a7143
0000000000000000 000001012f839e58 000000d033788950 0000000300000246
000001013237b030 000000000000974f
Call Trace:<ffffffff802a7143>{sock_sendmsg+271} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffff802cfe27>{tcp_poll+44} <ffffffff8018b7ff>{do_select+939}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff8011026a>{system_call+126}
rpc.idmapd S 0000000000000000 0 3553 1 3592 2952 (NOTLB)
000001012f4f3e78 0000000000000006 0000010133ba7030 0000000102e0fd5c
0000000000000000 000001012f4f3e88 0000000000000005 0000000100000005
0000010132607030 00000000000007a0
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff8019da63>{sys_epoll_wait+403}
<ffffffff8014441b>{sys_rt_sigaction+97} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8011026a>{system_call+126}
rpciod S 000001012f6ce000 0 3592 1 3593 3553 (L-TLB)
000001012f519ec8 0000000000000046 000001002f962c88 0000000000000001
ffffffffa0247060 ffffffffa01cef37 0000010001044a40 000000003395b800
00000101336d67f0 00000000000004cf
Call Trace:<ffffffffa01cef37>{:sunrpc:__rpc_execute+867} <ffffffffa01cf505>{:sunrpc:rpciod+483}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff80110f47>{child_rip+8} <ffffffffa01cf322>{:sunrpc:rpciod+0}
<ffffffff80110f3f>{child_rip+0}
lockd S 7fffffffffffffff 0 3593 1 3623 3592 (L-TLB)
000001012f599e18 0000000000000046 0000000000000000 0000000000000246
0000000000000246 ffffffff802ac378 0000000000003700 000000002fcb4780
0000010132b1e030 0000000000004e89
Call Trace:<ffffffff802ac378>{skb_dequeue+80} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffffa01d19c7>{:sunrpc:svc_reserve+75} <ffffffff8013558a>{add_wait_queue+18}
<ffffffffa01d296e>{:sunrpc:svc_recv+786} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffffa01fb088>{:lockd:lockd+0}
<ffffffffa01fb1fa>{:lockd:lockd+370} <ffffffff80110f47>{child_rip+8}
<ffffffffa01fb088>{:lockd:lockd+0} <ffffffffa01fb088>{:lockd:lockd+0}
<ffffffff80110f3f>{child_rip+0}
ypbind S 7fffffffffffffff 0 3623 1 3624 3593 (NOTLB)
000001012f037e88 0000000000000002 0000000000000008 000001012f1dec80
000001012fd7bc80 ffffffff802db1b3 000000d000000000 0000000000000246
00000101321b07f0 000000000000c2bc
Call Trace:<ffffffff802db1b3>{tcp_rcv_state_process+3789} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffff8013558a>{add_wait_queue+18} <ffffffff802cfe27>{tcp_poll+44}
<ffffffff8018bf33>{sys_poll+604} <ffffffff8018b399>{__pollwait+0}
<ffffffff8011026a>{system_call+126}
ypbind S 0000003dd8b0d1c0 0 3624 1 3626 3623 (NOTLB)
000001012f059e48 0000000000000002 0000000000000000 0000000000000000
0000000000000246 0000000000000003 000001012f059de8 0000000080133e50
0000010132af27f0 000000000000127a
Call Trace:<ffffffff801ea749>{__up_read+16} <ffffffff8014c9d7>{do_futex+706}
<ffffffff801795c1>{do_sync_write+173} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffff801415e2>{dequeue_signal+58} <ffffffff80143a78>{sys_rt_sigtimedwait+487}
<ffffffff8014cc9b>{sys_futex+203} <ffffffff801797bf>{sys_write+96}
<ffffffff8011026a>{system_call+126}
ypbind D 0000000000000002 0 3626 1 3810 3624 (NOTLB)
000001012f05baa8 0000000000000002 00000101328337f0 0000000000000206
0000000000000001 00000000324da000 0000001000000001 0000000000000246
00000101328337f0 0000000000001828
Call Trace:<ffffffff803093df>{__down+147} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8030ae77>{__down_failed+53} <ffffffff802b214a>{.text.lock.dev+85}
<ffffffff802c4171>{netlink_dump_start+307} <ffffffff802b75c8>{rtnetlink_rcv+861}
<ffffffff802c3bbb>{netlink_data_ready+22} <ffffffff802c33b1>{netlink_sendskb+113}
<ffffffff802c3b90>{netlink_sendmsg+694} <ffffffff802a7143>{sock_sendmsg+271}
<ffffffff8015c093>{__generic_file_write_nolock+158}
<ffffffff801eb4b3>{vsnprintf+1406} <ffffffff80135752>{autoremove_wake_function+0}
<ffffffff802a6ecb>{sockfd_lookup+16} <ffffffff802a8612>{sys_sendto+195}
<ffffffff802a84a9>{sys_getsockname+125} <ffffffff802c2a00>{netlink_insert+321}
<ffffffff80178d11>{fd_install+42} <ffffffff802a6eb3>{sock_map_fd+59}
<ffffffff8011026a>{system_call+126}
smartd S 0000000000000000 0 3810 1 3820 3626 (NOTLB)
000001012f55bee8 0000000000000006 0000010133bbea00 0000010133b96400
000001012b8d01c0 ffffffff80250ebb 0000000000000000 0000000100000246
00000101327b0030 0000000000003027
Call Trace:<ffffffff80250ebb>{blkdev_ioctl+1674} <ffffffffa002607c>{:sd_mod:scsi_disk_put+81}
<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff80140811>{sys_nanosleep+192}
<ffffffff8011026a>{system_call+126}
acpid S 0000000000000005 0 3820 1 3898 3810 (NOTLB)
00000101303c9d78 0000000000000006 000001012f221508 0000000000000246
0000010132868580 00000101303c9d30 0000000000000000 000000000000000e
0000010005f9b7f0 000000000001f502
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8018b7ff>{do_select+939}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff801932d4>{dnotify_parent+34} <ffffffff8011026a>{system_call+126}
sshd S 0000000000000004 0 3898 1 10192 3921 3820 (NOTLB)
000001012e927d78 0000000000000002 0000010133baa7f0 000001000000e780
0000000000000000 00000000000000d2 000000d00000f810 0000000200000246
000001013237b7f0 00000000000066f8
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff802cfe27>{tcp_poll+44}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff8018f019>{dput+56}
<ffffffff8011026a>{system_call+126}
xinetd S 0000000000000009 0 3921 1 3941 3898 (NOTLB)
000001012ea39d78 0000000000000002 0000010133ba7030 0000001900000076
000001003a1e17f0 0000000000000076 000000d001054a40 0000000100000246
000001013229b7f0 00000000000017df
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff802cfe27>{tcp_poll+44}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff8011026a>{system_call+126}
rpc.rquotad S 000001012eb5fcc0 0 3941 1 3958 3921 (NOTLB)
000001012eac7e88 0000000000000002 0000010131d63568 0000001900000074
0000010132979030 0000000000000074 0000010001044a40 0000000000000246
0000010131c56030 000000000001cd61
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff802cfe27>{tcp_poll+44} <ffffffff8018bf33>{sys_poll+604}
<ffffffff8018b399>{__pollwait+0} <ffffffff8011026a>{system_call+126}
nfsd S 000001000104b7e0 0 3958 1 3959 3941 (L-TLB)
000001012eb0bde8 0000000000000046 0000010133baa030 0000001900000073
000001013208f7f0 0000000000000073 000001000105ca40 000000032fcb4b00
000001012ef59030 0000000000001b50
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b55>{schedule_tail+93} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3959 1 3960 3958 (L-TLB)
000001012eb0fde8 0000000000000046 000001012ef59030 0000001900000073
0000010132774030 0000000000000073 000001000105ca40 00000003e2c10000
000001013208f7f0 000000000000106b
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3960 1 3961 3959 (L-TLB)
000001012e441de8 0000000000000046 000001013208f7f0 0000001900000073
000001012ef597f0 0000000000000073 000001000105ca40 000000030001e831
0000010132774030 0000000000000e1f
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3961 1 3962 3960 (L-TLB)
000001012e437de8 0000000000000046 0000010132774030 0000001900000073
0000010133b17030 0000000000000073 000001000105ca40 0000000300013ebd
000001012ef597f0 0000000000000d68
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3962 1 3963 3961 (L-TLB)
000001012eba3de8 0000000000000046 000001012ef597f0 0000001900000073
00000101322ef030 0000000000000073 000001000105ca40 0000000333b17030
0000010133b17030 0000000000000d3e
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3963 1 3964 3962 (L-TLB)
000001012e495de8 0000000000000046 0000010133b17030 0000001900000073
00000101325b67f0 0000000000000073 000001000105ca40 00000003322ef030
00000101322ef030 0000000000000d5a
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000001000104b7e0 0 3964 1 3965 3963 (L-TLB)
000001012e467de8 0000000000000046 00000101322ef030 0000001900000073
00000101321b0030 0000000000000073 000001000105ca40 00000003325b67f0
00000101325b67f0 0000000000000d36
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
nfsd S 000000000036ee80 0 3965 1 3969 3964 (L-TLB)
000001012eb91de8 0000000000000046 00000101325b67f0 000000190000007d
00000101322357f0 000000000000007d 000001000105ca40 00000003328c29a8
00000101321b0030 0000000000000d19
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffffa01d296e>{:sunrpc:svc_recv+786}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c9479>{:nfsd:nfsd+381}
<ffffffff80132b2f>{schedule_tail+55} <ffffffff80110f47>{child_rip+8}
<ffffffffa02c92fc>{:nfsd:nfsd+0} <ffffffffa02c92fc>{:nfsd:nfsd+0}
<ffffffff80110f3f>{child_rip+0}
rpc.mountd S 000001012eb5f7c0 0 3969 1 3999 3965 (NOTLB)
000001012ebfbd78 0000000000000006 00000000000000bf 0000001900000074
000001013296d030 0000000000000074 0000010001044a40 0000000000000246
0000010133b177f0 0000000000021f84
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff802cfe27>{tcp_poll+44}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff801ece86>{strncpy_from_user+91}
<ffffffff8011026a>{system_call+126}
sendmail S 0000000000000005 0 3999 1 4007 3969 (NOTLB)
000001012e0b7d78 0000000000000006 0000000000000022 00000100b0621015
00000000ffffffff 0000000000000002 000000d030620ffe 0000000100000246
000001013378a030 00000000000020ae
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff8018b7ff>{do_select+939}
<ffffffff8018b399>{__pollwait+0} <ffffffff8017b195>{invalidate_inode_buffers+12}
<ffffffff8018bb7e>{sys_select+820} <ffffffff8018f019>{dput+56}
<ffffffff8011026a>{system_call+126}
sendmail S 0000000000000000 0 4007 1 4047 3999 (NOTLB)
000001012e1dbf68 0000000000000006 0000000103346adf 0000000000000246
0000000000000246 ffffffff8013fcb4 0000000000000004 000000030036f0a4
0000010132833030 0000000000009ae2
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8013bdc6>{do_setitimer+332}
<ffffffff80144529>{sys_pause+23} <ffffffff8011026a>{system_call+126}
gpm S 00000100010437e0 0 4047 1 4117 4007 (NOTLB)
000001012dda3d78 0000000000000002 00000000000000bf 0000001900000086
0000010133b72030 0000000000000086 0000010001054a40 000000028015ade7
00000100c7dfa030 000000000000db72
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff8018b7ff>{do_select+939}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff8011026a>{system_call+126}
crond S 0000000000000000 0 4117 1 4140 4047 (NOTLB)
000001012e235ee8 0000000000000002 000001012e235e58 000001012e235ef8
000001012e235ef8 ffffffff80181d23 000000000000080a 0000000300043b57
00000101336d47f0 0000000000000e4f
Call Trace:<ffffffff80181d23>{cp_new_stat+233} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff80140811>{sys_nanosleep+192} <ffffffff8011026a>{system_call+126}
xfs S 0000000000000004 0 4140 1 4159 4117 (NOTLB)
000001012de31d78 0000000000000002 ffffffff803d4300 0000000000000246
00000101326077f0 0000000000000074 0000010001054a40 0000000000000046
000001012dc5b030 000000000000526c
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffff8030ac17>{schedule_timeout+367}
<ffffffff801406de>{process_timeout+0} <ffffffff8018b7ff>{do_select+939}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff8011026a>{system_call+126}
atd S 0000000000000000 0 4159 1 4175 4140 (NOTLB)
000001012decbee8 0000000000000006 000000002efdb354 ffffffff8018b1b8
000001012decbf38 00000000a0073ad8 000001012decbf38 000000022decbf38
000001013296d7f0 0000000000002579
Call Trace:<ffffffff8018b1b8>{filldir64+0} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff80140811>{sys_nanosleep+192} <ffffffff8011026a>{system_call+126}
dbus-daemon-1 S 7fffffffffffffff 0 4175 1 4188 4159 (NOTLB)
000001012d857e88 0000000000000002 000001012d857e18 0000000000000246
000001012dc5b7f0 0000000000000000 000001012d857f50 000000022d857f50
000001012ef5a7f0 00000000000a3e3c
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8018bf33>{sys_poll+604} <ffffffff8018b399>{__pollwait+0}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
cups-config-d S 7fffffffffffffff 0 4188 1 4199 4175 (NOTLB)
000001012d8c9e88 0000000000000006 000001012e3cf1f8 0000000000000246
000001012d8c9e18 0000000000000000 000001012d8c9f50 000000022d8c9f50
00000101336d6030 000000000001cd2c
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8018bf33>{sys_poll+604} <ffffffff8018b399>{__pollwait+0}
<ffffffff8011026a>{system_call+126}
hald S 00000000000007d1 0 4199 1 4217 4188 (NOTLB)
000001012d9dbe88 0000000000000006 0000010133ba7030 ffffffff8018bfe0
0040ea5b00008001 0000010132adb688 000000d000000000 0000000100000246
00000101328387f0 0000000000001db1
Call Trace:<ffffffff8018bfe0>{sys_poll+777} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff802ad3ac>{datagram_poll+0} <ffffffff8018bf33>{sys_poll+604}
<ffffffff8018b399>{__pollwait+0} <ffffffff8011026a>{system_call+126}
python S 0000000000000000 0 4217 1 5216 4365 4199 (NOTLB)
000001012da69e08 0000000000000006 0000000000000202 ffffffff80169144
0000010133678040 000001012e333018 000000000060a2c8 000000022dad3108
000001013208f030 00000000000049ed
Call Trace:<ffffffff80169144>{do_wp_page+1181} <ffffffff8018473c>{pipe_wait+128}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff80181d23>{cp_new_stat+233}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff8016b255>{__vma_link+66}
<ffffffff80184987>{pipe_readv+526} <ffffffff80184a28>{pipe_read+26}
<ffffffff801794df>{vfs_read+207} <ffffffff80179736>{sys_read+69}
<ffffffff8011026a>{system_call+126}
python S 0000000000000000 0 4365 1 4962 4368 4217 (NOTLB)
000001012c62dd78 0000000000000002 000001012c62de58 000001012c62de58
0000000000000000 ffffffff8018599c 00000000fffffffe 0000000080186cff
0000010131c567f0 000000000000150a
Call Trace:<ffffffff8018599c>{path_release+12} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018728a>{__user_walk+94}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff8011026a>{system_call+126}
agetty S 0000000000000001 0 4368 1 4369 4365 (NOTLB)
0000010132b85d88 0000000000000002 0000000000000075 0000000000000000
0000010137ecc5e8 ffffffff8015ade7 0000000000000001 0000000205d69b18
00000101329267f0 000000000000fb18
Call Trace:<ffffffff8015ade7>{filemap_nopage+384} <ffffffff8024251d>{uart_start+38}
<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80226a1d>{tty_read+230}
<ffffffff801794df>{vfs_read+207} <ffffffff80179736>{sys_read+69}
<ffffffff8011026a>{system_call+126}
mingetty S 000001013377c0c0 0 4369 1 4370 4368 (NOTLB)
000001012c6b3d88 0000000000000006 0000000d00000075 0000001900000076
00000101322ef7f0 0000000000000076 0000010001044a40 0000000005d69b18
00000101323cd7f0 0000000000090cc0
Call Trace:<ffffffff80232de1>{complement_pos+12} <ffffffff80232de1>{complement_pos+12}
<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133e50>{__wake_up+54} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80226a1d>{tty_read+230} <ffffffff801794df>{vfs_read+207}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
mingetty S 000001012d4f9280 0 4370 1 4371 4369 (NOTLB)
000001012c1ebd88 0000000000000002 0000000d00000075 000000190000006f
000001012e2dd030 000000000000006f 0000010001044a40 0000000005d69b18
00000101326977f0 000000000003d013
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133e50>{__wake_up+54} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80226a1d>{tty_read+230} <ffffffff801794df>{vfs_read+207}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
mingetty S 000001012bf227c0 0 4371 1 4372 4370 (NOTLB)
000001012ee7dd88 0000000000000002 0000000d00000075 0000001900000076
000001012cacc7f0 0000000000000076 0000010001044a40 0000000005d69b18
00000101322ef7f0 0000000000019b8b
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133e50>{__wake_up+54} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80226a1d>{tty_read+230} <ffffffff801794df>{vfs_read+207}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
mingetty S 000001013222f0c0 0 4372 1 4373 4371 (NOTLB)
000001012d941d88 0000000000000002 0000000d00000075 0000001900000076
000001012cacc7f0 0000000000000076 0000010001054a40 0000000205d69b18
000001012e5eb030 000000000001c064
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133e50>{__wake_up+54} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80226a1d>{tty_read+230} <ffffffff801794df>{vfs_read+207}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
mingetty S 000001013222fcc0 0 4373 1 4374 4372 (NOTLB)
000001012d905d88 0000000000000002 0000000d00000075 0000001900000072
000001012e31a030 0000000000000072 0000010001054a40 0000000205d69b18
000001012cacc7f0 0000000000030f7c
Call Trace:<ffffffff8030ab88>{schedule_timeout+224} <ffffffff8013558a>{add_wait_queue+18}
<ffffffff8022bda8>{read_chan+1059} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133e50>{__wake_up+54} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80226a1d>{tty_read+230} <ffffffff801794df>{vfs_read+207}
<ffffffff80179736>{sys_read+69} <ffffffff8011026a>{system_call+126}
mingetty S 0000000000000001 0 4374 1 30324 4373 (NOTLB)
000001012dac9d88 0000000000000002 0000000d00000075 000001012dac9dd8
0000010137ec00ff ffffffff8015ade7 0000000000000001 0000000105d69b18
000001012cacc030 00000000000331d5
Call Trace:<ffffffff8015ade7>{filemap_nopage+384} <ffffffff8030ab88>{schedule_timeout+224}
<ffffffff8013558a>{add_wait_queue+18} <ffffffff8022bda8>{read_chan+1059}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133e50>{__wake_up+54}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80226a1d>{tty_read+230}
<ffffffff801794df>{vfs_read+207} <ffffffff80179736>{sys_read+69}
<ffffffff8011026a>{system_call+126}
python S ffffffff80309800 0 4962 4365 10190 (NOTLB)
000001012bbe3d78 0000000000000002 000001012bbe3e58 000001012bbe3e58
0000000000000000 ffffffff8018599c 00000000fffffffe 0000000080186cff
000001012e2dd030 0000000000004440
Call Trace:<ffffffff8018599c>{path_release+12} <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018728a>{__user_walk+94}
<ffffffff8018b399>{__pollwait+0} <ffffffff8018bb7e>{sys_select+820}
<ffffffff8011026a>{system_call+126}
cupsd D 0000000000000002 0 30324 1 4314 4374 (NOTLB)
00000100ada0daa8 0000000000000002 0000010132926030 0000000000000202
0000000000000001 0000000080161936 0000001000000001 0000000000000246
0000010132926030 0000000000001c1f
Call Trace:<ffffffff803093df>{__down+147} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8030ae77>{__down_failed+53} <ffffffff802b214a>{.text.lock.dev+85}
<ffffffff802c4171>{netlink_dump_start+307} <ffffffff802b75c8>{rtnetlink_rcv+861}
<ffffffff802c3bbb>{netlink_data_ready+22} <ffffffff802c33b1>{netlink_sendskb+113}
<ffffffff802c3b90>{netlink_sendmsg+694} <ffffffff802a7143>{sock_sendmsg+271}
<ffffffff8013fdf3>{del_timer+107} <ffffffff801eb4b3>{vsnprintf+1406}
<ffffffff80135752>{autoremove_wake_function+0} <ffffffff802a6ecb>{sockfd_lookup+16}
<ffffffff802a8612>{sys_sendto+195} <ffffffff802a84a9>{sys_getsockname+125}
<ffffffff802c2a00>{netlink_insert+321} <ffffffff80178d11>{fd_install+42}
<ffffffff802a6eb3>{sock_map_fd+59} <ffffffff8011026a>{system_call+126}
ib_cm/0 S ffffffffa00f5b5b 0 15262 10 15264 1987 (L-TLB)
000001005fc95e68 0000000000000046 ffffffff803d4300 0000001900000069
000001013380e7f0 0000000000000246 0000000018664492 0000000000000206
00000100243cf7f0 0000000000000132
Call Trace:<ffffffffa00f5b5b>{:ib_cm:cm_work_handler+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_cm/1 S ffffffffa00f5b5b 0 15263 12 15265 2069 (L-TLB)
0000010028807e68 0000000000000046 0000010133ba7030 0000000000000212
0000010028807dd8 0000000000000246 00000000186644be 0000000100000206
000001009e8aa7f0 00000000000001a6
Call Trace:<ffffffffa00f5b5b>{:ib_cm:cm_work_handler+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_cm/2 S ffffffffa00f5b5b 0 15264 10 15289 15262 (L-TLB)
00000100a5cc9e68 0000000000000046 0000010133baa7f0 0000000000000206
00000000ffffff98 0000000000000246 0000000018664493 0000000200000206
00000100af56f7f0 00000000000001ee
Call Trace:<ffffffffa00f5b5b>{:ib_cm:cm_work_handler+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_cm/3 S ffffffffa00f5b5b 0 15265 12 15286 15263 (L-TLB)
0000010041b45e68 0000000000000046 0000010133baa030 0000000000000297
0000010041b45dd8 0000000000000246 000000001866909f 0000000300000206
0000010048d027f0 000000000000033e
Call Trace:<ffffffffa00f5b5b>{:ib_cm:cm_work_handler+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
mthcacatas S ffffffff80309800 0 15286 12 15329 15265 (L-TLB)
0000010021105ce8 0000000000000046 0000010133baa7f0 ffffffffa02b8910
0000010069648000 0000000000000006 000000010321bb00 0000000269648000
0000010026ae3030 00000000000000db
Call Trace:<ffffffff8013fcb4>{__mod_timer+293} <ffffffffa012d22c>{:ib_mthca:catas_reset+0}
<ffffffff8030ac17>{schedule_timeout+367} <ffffffff801406de>{process_timeout+0}
<ffffffff802b1c55>{netdev_run_todo+515} <ffffffffa01ac519>{:ib_ipoib:ipoib_remove_one+100}
<ffffffffa00d2229>{:ib_core:ib_unregister_device+90}
<ffffffffa011917d>{:ib_mthca:__mthca_remove_one+48}
<ffffffffa0119290>{:ib_mthca:__mthca_restart_one+23}
<ffffffffa012d2f2>{:ib_mthca:catas_reset+198} <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_mad1 S ffffffffa00e31db 0 15289 10 15290 15264 (L-TLB)
00000100be61de68 0000000000000046 00000100be61ddc8 ffffffffa00f03c0
00000100be61de48 0000000000000206 0000000000000202 0000000200000206
000001012e2dd7f0 00000000000002d3
Call Trace:<ffffffffa00e31db>{:ib_mad:timeout_sends+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_mad2 S ffffffffa00e2911 0 15290 10 15471 15289 (L-TLB)
000001003c963e68 0000000000000046 00000100c7d914e0 000001009ed377c0
0000000000000212 000001003c963e08 00000100c7d91400 00000000a00e2e7a
00000100a0f607f0 00000000000000b5
Call Trace:<ffffffffa00e2911>{:ib_mad:ib_mad_completion_handler+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
ipoib S ffffffff80309800 0 15329 12 15466 15286 (L-TLB)
0000010034f87e68 0000000000000046 ffffffff803d4300 0000000000000000
0000ffff1b4012ff ffffffff00000000 00000000000080fe 00000000abc90200
00000100a9988030 000000000000010a
Call Trace:<ffffffffa01ad340>{:ib_ipoib:ipoib_reap_ah+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
ib_addr_wq S ffffffff80309800 0 15466 12 15329 (L-TLB)
000001002edcfe68 0000000000000046 ffffffff803d4300 0000001900000073
00000101336d4030 0000000000000073 0000010001044a40 0000000000000000
0000010129cad7f0 00000000000000d3
Call Trace:<ffffffffa01ba2e5>{:ib_addr:process_req+0} <ffffffff80147791>{worker_thread+226}
<ffffffff80133da9>{default_wake_function+0} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b4c7>{kthread+200}
<ffffffff80110f47>{child_rip+8} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
iw_cm_wq S 0000010032d0ec40 0 15471 10 15481 15290 (L-TLB)
00000100c6473e68 0000000000000046 00000100010537e0 0000001900000074
0000010133ba77f0 0000000000000074 0000010001044a40 0000000000000000
00000100326b5030 00000000000017e3
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
rdma_cm_wq S 0000000000000002 0 15481 10 15486 15471 (L-TLB)
0000010060f0fe68 0000000000000046 0000000000000001 0000001900000075
0000010133ba77f0 0000000000000075 0000010001044a40 0000000080467658
000001002e97c7f0 0000000000000d04
Call Trace:<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff801476af>{worker_thread+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
sdp S ffffffffa0355248 0 15486 10 15481 (L-TLB)
0000010060e4be68 0000000000000046 0000000000000000 00000100326b57f0
ffffffff80135752 0000010060e4bdc0 0000010060e4bdc0 00000002bd30e000
00000100326b57f0 00000000000004ad
Call Trace:<ffffffff80135752>{autoremove_wake_function+0} <ffffffffa0355248>{:ib_sdp:sdp_destroy_work+0}
<ffffffff80147791>{worker_thread+226} <ffffffff80133da9>{default_wake_function+0}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0} <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200} <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0} <ffffffff8014b3ff>{kthread+0}
<ffffffff80110f3f>{child_rip+0}
modprobe D 0000000000000000 0 4314 1 9803 30324 (NOTLB)
00000100268f3de8 0000000000000002 000001009e8aa030 0000010050f4e040
000001009e8aa030 000001000104b7e0 000001000105b7e0 000000018030a0f5
000001009e8aa030 00000000000012a9
Call Trace:<ffffffff803093df>{__down+147} <ffffffff80133da9>{default_wake_function+0}
<ffffffff8030ae77>{__down_failed+53} <ffffffffa00d270d>{:ib_core:.text.lock.device+65}
<ffffffffa010bb39>{:ib_ucm:ib_ucm_cleanup+13} <ffffffff8014e230>{sys_delete_module+479}
<ffffffff801ea645>{__up_write+20} <ffffffff8016cf85>{sys_munmap+94}
<ffffffff8011026a>{system_call+126}
ifconfig D 0000000000000000 0 5216 4217 (NOTLB)
00000100196c5d48 0000000000000002 fffffffffffffff4 fffffffffffffff4
000001001f4eea98 000001001ae4c618 000000012f7cf8b8 00000000196c5d48
000001012e5eb7f0 00000000002913b4
Call Trace:<ffffffff80123ed3>{do_page_fault+575} <ffffffff803093df>{__down+147}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8030ae77>{__down_failed+53}
<ffffffff802b214a>{.text.lock.dev+85} <ffffffff802eeb9a>{devinet_ioctl+1671}
<ffffffff802efb1c>{inet_ioctl+124} <ffffffff802a7aa5>{sock_ioctl+699}
<ffffffff8018ae05>{sys_ioctl+853} <ffffffff8011026a>{system_call+126}
ifconfig D 0000000000000000 0 9803 1 10123 4314 (NOTLB)
0000010037121d48 0000000000000002 fffffffffffffff4 fffffffffffffff4
00000100b0206ad8 00000100606fd618 000000012f7cf8b8 0000000037121d48
0000010112e2d030 0000000000169063
Call Trace:<ffffffff80123ed3>{do_page_fault+575} <ffffffff803093df>{__down+147}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8030ae77>{__down_failed+53}
<ffffffff802b214a>{.text.lock.dev+85} <ffffffff802eeb9a>{devinet_ioctl+1671}
<ffffffff802efb1c>{inet_ioctl+124} <ffffffff802a7aa5>{sock_ioctl+699}
<ffffffff8018ae05>{sys_ioctl+853} <ffffffff8011026a>{system_call+126}
ifconfig D 0000000000000000 0 10123 1 9803 (NOTLB)
0000010014d2fd48 0000000000000002 fffffffffffffff4 fffffffffffffff4
00000100798c1048 0000010037c4c618 000000012f7cf8b8 0000000014d2fd48
000001008ffc17f0 000000000011d9f1
Call Trace:<ffffffff80123ed3>{do_page_fault+575} <ffffffff803093df>{__down+147}
<ffffffff80133da9>{default_wake_function+0} <ffffffff8030ae77>{__down_failed+53}
<ffffffff802b214a>{.text.lock.dev+85} <ffffffff802eeb9a>{devinet_ioctl+1671}
<ffffffff802efb1c>{inet_ioctl+124} <ffffffff802a7aa5>{sock_ioctl+699}
<ffffffff8018ae05>{sys_ioctl+853} <ffffffff8011026a>{system_call+126}
ps X 000001008b197f18 0 10190 4962 (L-TLB)
000001008b197ef8 0000000000000046 000001013291ec80 0000010112e2d7f0
0000000000000046 ffffffff80142adf 0000000000000011 0000000000040001
0000010112e2d7f0 00000000016c9ead
Call Trace:<ffffffff80142adf>{do_notify_parent+382} <ffffffff80178e44>{filp_close+103}
<ffffffff8030a260>{__cond_resched+23} <ffffffff8013a7e5>{do_exit+3137}
<ffffffff801796e5>{vfs_write+248} <ffffffff8013a8f6>{sys_exit_group+0}
<ffffffff8011026a>{system_call+126}
sshd S ffffffff80309800 0 10192 3898 10194 (NOTLB)
000001003ec8bd78 0000000000000006 000001012e8573b0 0000000000000246
0000000000000246 ffffffff802d1bd9 000000d032979030 0000000000000246
00000100af56f030 0000000000001ddc
Call Trace:<ffffffff802d1bd9>{tcp_sendmsg+3755} <ffffffff802264f1>{tty_ldisc_try+60}
<ffffffff8030ab88>{schedule_timeout+224} <ffffffff80226624>{tty_ldisc_deref+103}
<ffffffff8018b7ff>{do_select+939} <ffffffff8018b399>{__pollwait+0}
<ffffffff8018bb7e>{sys_select+820} <ffffffff801932d4>{dnotify_parent+34}
<ffffffff8011026a>{system_call+126}
bash R running task 0 10194 10192 (NOTLB)
--
MST
More information about the general
mailing list