[ofa-general] crash in ipoib
Sean Hefty
mshefty at ichips.intel.com
Thu Jun 14 09:20:54 PDT 2007
Here's the capture from the network console
<5> [...network console startup...]
<5> Unable to handle kernel NULL pointer dereference at 0000000000000008
RIP:
<5> <4>Warning: kfree_skb on hard IRQ ffffffff802bb055
<5> Warning: kfree_skb on hard IRQ ffffffff802bb055
<5> Warning: kfree_skb on hard IRQ ffffffff802bb055
<5> Warning: kfree_skb on hard IRQ ffffffff802bb055
<5> <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> PML4 dcc2f067 PGD 102087067 PMD 0
<5> Oops: 0002 [1] SMP
<5> CPU 1
<5> Modules linked in: netconsole det(U) nfs lockd nfs_acl autofs4
i2c_dev i2c_core sunrpc rdma_ucm(U) ib_vnic(U) ib_sdp(U) rdma_cm(U)
iw_cm(U) ib_addr(U) ib_local_sa(U) ib_ipath(U) ipt_REJECT ipt_state
ip_conntrack iptable_filter ip_tables dm_mirror dm_mod button battery ac
joydev uhci_hcd ehci_hcd hw_random ib_mthca(U) ib_ipoib(U) ib_umad(U)
ib_ucm(U) ib_uverbs(U) ib_cm(U) ib_sa(U) ib_mad(U) ib_core(U) md5 ipv6
e1000(U) ahci ext3 jbd ata_piix libata sd_mod scsi_mod
<5> Pid: 1584, comm: ib_cm/1 Tainted: PF 2.6.9-42.ELsmp
<5> RIP: 0010:[<ffffffffa0146b60>]
<ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> RSP: 0018:0000010005d7b940 EFLAGS: 00010046
<5> RAX: 0000000000000000 RBX: 000001010d3a8e00 RCX: 0000000000000000
<5> RDX: 000001010d3a8e10 RSI: 00000101191b3990 RDI: 00000101191b3380
<5> RBP: 000001011302b680 R08: 0000000000000010 R09: 0000010119301e00
<5> R10: 000000000000001f R11: 00000000000000e4 R12: 0000000000000206
<5> R13: 00000101191b3380 R14: 00000101191b3000 R15: 0000000000000030
<5> FS: 0000000000000000(0000) GS:ffffffff804e5100(0000)
knlGS:0000000000000000
<5> CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b
<5> CR2: 0000000000000008 CR3: 0000000005d68000 CR4: 00000000000006e0
<5> Process ib_cm/1 (pid: 1584, threadinfo 0000010119c14000, task
000001011a9c5030)
<5> Stack: 0000000000000206 0000000000000030 0000000000000206
0000010110e8fb00
<5> 00000101191b37b8 0000000000000206 00000000dc62401c
0000000400000206
<5> 0100000082000001 000000041a9121c0
<5> Call Trace:<IRQ> <ffffffffa0141fa0>{:ib_ipoib:ipoib_ib_completion+144}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffffa015c225>{:ib_mthca:mthca_eq_int+221}
<ffffffff80113209>{do_IRQ+266}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffff80112f4a>{handle_IRQ_event+41}
<ffffffff801131c4>{do_IRQ+197}
<5> <ffffffff80110833>{ret_from_intr+0}
<ffffffff801ec72d>{csum_partial+1209}
<5> <ffffffff802abeb1>{skb_checksum+308}
<ffffffffa01ca846>{:ip_conntrack:tcp_error+312}
<5> <ffffffffa01c9197>{:ip_conntrack:ip_conntrack_in+163}
<5> <ffffffff802c8e30>{ip_local_deliver_finish+0}
<ffffffff802b92fe>{nf_hook_slow+184}
<5> <ffffffff802b8f1e>{nf_iterate+82}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802b92ba>{nf_hook_slow+116}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802c98e8>{ip_rcv+1119}
<ffffffff802b066b>{netif_receive_skb+791}
<5> <ffffffff802b0730>{process_backlog+136}
<ffffffff802b0884>{net_rx_action+203}
<5> <ffffffff8013c738>{__do_softirq+88}
<ffffffff8013c7e1>{do_softirq+49}
<5> <ffffffff80113247>{do_IRQ+328}
<ffffffff80110833>{ret_from_intr+0}
<5> <EOI> <ffffffff8030b0cb>{_spin_unlock_irqrestore+47}
<5> <ffffffffa0119ee7>{:ib_cm:ib_send_cm_rep+812}
<ffffffffa01468ef>{:ib_ipoib:ipoib_cm_rx_handler+821}
<5> <ffffffffa0146530>{:ib_ipoib:ipoib_cm_rx_event_handler+0}
<5> <ffffffffa00f4711>{:ib_core:ib_find_cached_pkey+192}
<5> <ffffffffa011b09d>{:ib_cm:cm_process_work+101}
<ffffffffa011ba49>{:ib_cm:cm_req_handler+2398}
<5> <ffffffffa011be2e>{:ib_cm:cm_work_handler+0}
<ffffffffa011be5c>{:ib_cm:cm_work_handler+46}
<5> <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200}
<5> <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0}
<5> <ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
<5>
<5>
<5> Code: 48 89 48 08 48 89 01 49 8b 86 90 09 00 00 48 89 50 08 48 89
<5> RIP <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378} RSP
<0000010005d7b940>
<5> CR2: 0000000000000008
<5> <0>Kernel panic - not syncing: Oops
<5> Badness in panic at kernel/panic.c:118
<5>
<5> Call Trace:<IRQ> <ffffffff80137a86>{panic+527}
<ffffffff8013fcb4>{__mod_timer+293}
<5> <ffffffff80232de1>{complement_pos+12}
<ffffffff801f89e8>{vgacon_cursor+213}
<5> <ffffffff801f8913>{vgacon_cursor+0}
<ffffffff801239b6>{bust_spinlocks+62}
<5> <ffffffff80111b07>{oops_end+65}
<ffffffff80124148>{do_page_fault+1204}
<5> <ffffffffa016ba02>{:ib_mthca:mthca_tavor_post_srq_recv+839}
<5> <ffffffffa0145fdf>{:ib_ipoib:ipoib_cm_post_receive+119}
<5> <ffffffff80161936>{cache_alloc_refill+390}
<ffffffff80110d91>{error_exit+0}
<5> <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> <ffffffffa0141fa0>{:ib_ipoib:ipoib_ib_completion+144}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffffa015c225>{:ib_mthca:mthca_eq_int+221}
<ffffffff80113209>{do_IRQ+266}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffff80112f4a>{handle_IRQ_event+41}
<ffffffff801131c4>{do_IRQ+197}
<5> <ffffffff80110833>{ret_from_intr+0}
<ffffffff801ec72d>{csum_partial+1209}
<5> <ffffffff802abeb1>{skb_checksum+308}
<ffffffffa01ca846>{:ip_conntrack:tcp_error+312}
<5> <ffffffffa01c9197>{:ip_conntrack:ip_conntrack_in+163}
<5> <ffffffff802c8e30>{ip_local_deliver_finish+0}
<ffffffff802b92fe>{nf_hook_slow+184}
<5> <ffffffff802b8f1e>{nf_iterate+82}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802b92ba>{nf_hook_slow+116}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802c98e8>{ip_rcv+1119}
<ffffffff802b066b>{netif_receive_skb+791}
<5> <ffffffff802b0730>{process_backlog+136}
<ffffffff802b0884>{net_rx_action+203}
<5> <ffffffff8013c738>{__do_softirq+88}
<ffffffff8013c7e1>{do_softirq+49}
<5> <ffffffff80113247>{do_IRQ+328}
<ffffffff80110833>{ret_from_intr+0}
<5> <EOI> <ffffffff8030b0cb>{_spin_unlock_irqrestore+47}
<5> <ffffffffa0119ee7>{:ib_cm:ib_send_cm_rep+812}
<ffffffffa01468ef>{:ib_ipoib:ipoib_cm_rx_handler+821}
<5> <ffffffffa0146530>{:ib_ipoib:ipoib_cm_rx_event_handler+0}
<5> <ffffffffa00f4711>{:ib_core:ib_find_cached_pkey+192}
<5> <ffffffffa011b09d>{:ib_cm:cm_process_work+101}
<ffffffffa011ba49>{:ib_cm:cm_req_handler+2398}
<5> <ffffffffa011be2e>{:ib_cm:cm_work_handler+0}
<ffffffffa011be5c>{:ib_cm:cm_work_handler+46}
<5> <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200}
<5> <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0}
<5> <ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
<5>
<5> Badness in i8042_panic_blink at drivers/input/serio/i8042.c:987
<5>
<5> Call Trace:<IRQ> <ffffffff80241feb>{i8042_panic_blink+238}
<ffffffff80137a34>{panic+445}
<5> <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff80232de1>{complement_pos+12}
<5> <ffffffff801f89e8>{vgacon_cursor+213}
<ffffffff801f8913>{vgacon_cursor+0}
<5> <ffffffff801239b6>{bust_spinlocks+62}
<ffffffff80111b07>{oops_end+65}
<5> <ffffffff80124148>{do_page_fault+1204}
<ffffffffa016ba02>{:ib_mthca:mthca_tavor_post_srq_recv+839}
<5> <ffffffffa0145fdf>{:ib_ipoib:ipoib_cm_post_receive+119}
<5> <ffffffff80161936>{cache_alloc_refill+390}
<ffffffff80110d91>{error_exit+0}
<5> <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> <ffffffffa0141fa0>{:ib_ipoib:ipoib_ib_completion+144}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffffa015c225>{:ib_mthca:mthca_eq_int+221}
<ffffffff80113209>{do_IRQ+266}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffff80112f4a>{handle_IRQ_event+41}
<ffffffff801131c4>{do_IRQ+197}
<5> <ffffffff80110833>{ret_from_intr+0}
<ffffffff801ec72d>{csum_partial+1209}
<5> <ffffffff802abeb1>{skb_checksum+308}
<ffffffffa01ca846>{:ip_conntrack:tcp_error+312}
<5> <ffffffffa01c9197>{:ip_conntrack:ip_conntrack_in+163}
<5> <ffffffff802c8e30>{ip_local_deliver_finish+0}
<ffffffff802b92fe>{nf_hook_slow+184}
<5> <ffffffff802b8f1e>{nf_iterate+82}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802b92ba>{nf_hook_slow+116}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802c98e8>{ip_rcv+1119}
<ffffffff802b066b>{netif_receive_skb+791}
<5> <ffffffff802b0730>{process_backlog+136}
<ffffffff802b0884>{net_rx_action+203}
<5> <ffffffff8013c738>{__do_softirq+88}
<ffffffff8013c7e1>{do_softirq+49}
<5> <ffffffff80113247>{do_IRQ+328}
<ffffffff80110833>{ret_from_intr+0}
<5> <EOI> <ffffffff8030b0cb>{_spin_unlock_irqrestore+47}
<5> <ffffffffa0119ee7>{:ib_cm:ib_send_cm_rep+812}
<ffffffffa01468ef>{:ib_ipoib:ipoib_cm_rx_handler+821}
<5> <ffffffffa0146530>{:ib_ipoib:ipoib_cm_rx_event_handler+0}
<5> <ffffffffa00f4711>{:ib_core:ib_find_cached_pkey+192}
<5> <ffffffffa011b09d>{:ib_cm:cm_process_work+101}
<ffffffffa011ba49>{:ib_cm:cm_req_handler+2398}
<5> <ffffffffa011be2e>{:ib_cm:cm_work_handler+0}
<ffffffffa011be5c>{:ib_cm:cm_work_handler+46}
<5> <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200}
<5> <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0}
<5> <ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
<5>
<5> Badness in i8042_panic_blink at drivers/input/serio/i8042.c:990
<5>
<5> Call Trace:<IRQ> <ffffffff8024207d>{i8042_panic_blink+384}
<ffffffff80137a34>{panic+445}
<5> <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff80232de1>{complement_pos+12}
<5> <ffffffff801f89e8>{vgacon_cursor+213}
<ffffffff801f8913>{vgacon_cursor+0}
<5> <ffffffff801239b6>{bust_spinlocks+62}
<ffffffff80111b07>{oops_end+65}
<5> <ffffffff80124148>{do_page_fault+1204}
<ffffffffa016ba02>{:ib_mthca:mthca_tavor_post_srq_recv+839}
<5> <ffffffffa0145fdf>{:ib_ipoib:ipoib_cm_post_receive+119}
<5> <ffffffff80161936>{cache_alloc_refill+390}
<ffffffff80110d91>{error_exit+0}
<5> <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> <ffffffffa0141fa0>{:ib_ipoib:ipoib_ib_completion+144}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffffa015c225>{:ib_mthca:mthca_eq_int+221}
<ffffffff80113209>{do_IRQ+266}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffff80112f4a>{handle_IRQ_event+41}
<ffffffff801131c4>{do_IRQ+197}
<5> <ffffffff80110833>{ret_from_intr+0}
<ffffffff801ec72d>{csum_partial+1209}
<5> <ffffffff802abeb1>{skb_checksum+308}
<ffffffffa01ca846>{:ip_conntrack:tcp_error+312}
<5> <ffffffffa01c9197>{:ip_conntrack:ip_conntrack_in+163}
<5> <ffffffff802c8e30>{ip_local_deliver_finish+0}
<ffffffff802b92fe>{nf_hook_slow+184}
<5> <ffffffff802b8f1e>{nf_iterate+82}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802b92ba>{nf_hook_slow+116}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802c98e8>{ip_rcv+1119}
<ffffffff802b066b>{netif_receive_skb+791}
<5> <ffffffff802b0730>{process_backlog+136}
<ffffffff802b0884>{net_rx_action+203}
<5> <ffffffff8013c738>{__do_softirq+88}
<ffffffff8013c7e1>{do_softirq+49}
<5> <ffffffff80113247>{do_IRQ+328}
<ffffffff80110833>{ret_from_intr+0}
<5> <EOI> <ffffffff8030b0cb>{_spin_unlock_irqrestore+47}
<5> <ffffffffa0119ee7>{:ib_cm:ib_send_cm_rep+812}
<ffffffffa01468ef>{:ib_ipoib:ipoib_cm_rx_handler+821}
<5> <ffffffffa0146530>{:ib_ipoib:ipoib_cm_rx_event_handler+0}
<5> <ffffffffa00f4711>{:ib_core:ib_find_cached_pkey+192}
<5> <ffffffffa011b09d>{:ib_cm:cm_process_work+101}
<ffffffffa011ba49>{:ib_cm:cm_req_handler+2398}
<5> <ffffffffa011be2e>{:ib_cm:cm_work_handler+0}
<ffffffffa011be5c>{:ib_cm:cm_work_handler+46}
<5> <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200}
<5> <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0}
<5> <ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
<5>
<5> Badness in i8042_panic_blink at drivers/input/serio/i8042.c:992
<5>
<5> Call Trace:<IRQ> <ffffffff802420e2>{i8042_panic_blink+485}
<ffffffff80137a34>{panic+445}
<5> <ffffffff8013fcb4>{__mod_timer+293}
<ffffffff80232de1>{complement_pos+12}
<5> <ffffffff801f89e8>{vgacon_cursor+213}
<ffffffff801f8913>{vgacon_cursor+0}
<5> <ffffffff801239b6>{bust_spinlocks+62}
<ffffffff80111b07>{oops_end+65}
<5> <ffffffff80124148>{do_page_fault+1204}
<ffffffffa016ba02>{:ib_mthca:mthca_tavor_post_srq_recv+839}
<5> <ffffffffa0145fdf>{:ib_ipoib:ipoib_cm_post_receive+119}
<5> <ffffffff80161936>{cache_alloc_refill+390}
<ffffffff80110d91>{error_exit+0}
<5> <ffffffffa0146b60>{:ib_ipoib:ipoib_cm_handle_rx_wc+378}
<5> <ffffffffa0141fa0>{:ib_ipoib:ipoib_ib_completion+144}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffffa015c225>{:ib_mthca:mthca_eq_int+221}
<ffffffff80113209>{do_IRQ+266}
<5> <ffffffffa015c7d3>{:ib_mthca:mthca_tavor_interrupt+95}
<5> <ffffffff80112f4a>{handle_IRQ_event+41}
<ffffffff801131c4>{do_IRQ+197}
<5> <ffffffff80110833>{ret_from_intr+0}
<ffffffff801ec72d>{csum_partial+1209}
<5> <ffffffff802abeb1>{skb_checksum+308}
<ffffffffa01ca846>{:ip_conntrack:tcp_error+312}
<5> <ffffffffa01c9197>{:ip_conntrack:ip_conntrack_in+163}
<5> <ffffffff802c8e30>{ip_local_deliver_finish+0}
<ffffffff802b92fe>{nf_hook_slow+184}
<5> <ffffffff802b8f1e>{nf_iterate+82}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802b92ba>{nf_hook_slow+116}
<ffffffff802c9232>{ip_rcv_finish+0}
<5> <ffffffff802c98e8>{ip_rcv+1119}
<ffffffff802b066b>{netif_receive_skb+791}
<5> <ffffffff802b0730>{process_backlog+136}
<ffffffff802b0884>{net_rx_action+203}
<5> <ffffffff8013c738>{__do_softirq+88}
<ffffffff8013c7e1>{do_softirq+49}
<5> <ffffffff80113247>{do_IRQ+328}
<ffffffff80110833>{ret_from_intr+0}
<5> <EOI> <ffffffff8030b0cb>{_spin_unlock_irqrestore+47}
<5> <ffffffffa0119ee7>{:ib_cm:ib_send_cm_rep+812}
<ffffffffa01468ef>{:ib_ipoib:ipoib_cm_rx_handler+821}
<5> <ffffffffa0146530>{:ib_ipoib:ipoib_cm_rx_event_handler+0}
<5> <ffffffffa00f4711>{:ib_core:ib_find_cached_pkey+192}
<5> <ffffffffa011b09d>{:ib_cm:cm_process_work+101}
<ffffffffa011ba49>{:ib_cm:cm_req_handler+2398}
<5> <ffffffffa011be2e>{:ib_cm:cm_work_handler+0}
<ffffffffa011be5c>{:ib_cm:cm_work_handler+46}
<5> <ffffffff80147852>{worker_thread+419}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff80133dfa>{__wake_up_common+67}
<ffffffff80133da9>{default_wake_function+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff801476af>{worker_thread+0}
<5> <ffffffff8014b4f0>{keventd_create_kthread+0}
<ffffffff8014b4c7>{kthread+200}
<5> <ffffffff80110f47>{child_rip+8}
<ffffffff8014b4f0>{keventd_create_kthread+0}
<5> <ffffffff8014b3ff>{kthread+0} <ffffffff80110f3f>{child_rip+0}
<5>
More information about the general
mailing list