Complete report #1420761
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa351e1ca
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa3481fa8
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa348201b
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa35f9f93
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x33
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa35fbb8f
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa35fbcc3
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x23
|
|
7
|
exit_mmap
|
vmlinux |
0xffffffffa36094fb
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xdb
|
|
8
|
|
vmlinux |
0xffffffffa3566bd5
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x175
|
|
9
|
mmput
|
vmlinux |
0xffffffffa349c07f
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x6f
|
|
10
|
do_exit
|
vmlinux |
0xffffffffa34a6128
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x288
|
|
11
|
|
vmlinux |
0xffffffffa34ea151
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x31
|
|
12
|
|
vmlinux |
0xffffffffa34e8619
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xa9
|
|
13
|
do_group_exit
|
vmlinux |
0xffffffffa34a694f
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x3f
|
|
14
|
get_signal_to_deliver
|
vmlinux |
0xffffffffa34b825e
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x1ce
|
|
15
|
do_signal
|
vmlinux |
0xffffffffa342d5b7
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x57
|
|
16
|
|
vmlinux |
0xffffffffa3bb6e5a
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x9a
|
|
17
|
|
vmlinux |
0xffffffffa34cf340
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x50
|
|
18
|
|
vmlinux |
0xffffffffa3bb6e1b
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x5b
|
|
19
|
do_notify_resume
|
vmlinux |
0xffffffffa342dcca
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x7a
|
|
20
|
int_signal
|
vmlinux |
0xffffffffa3bc56f7
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [runc:30658]
Modules linked in: uinput veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq snd_seq_device snd_pcm snd_timer
ppdev snd soundcore pcspkr virtio_rng sg parport_pc joydev i2c_piix4 parport virtio_balloon ip_tables xfs libcrc32c sr_mod cdrom ata_generic qxl pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm virtio_net virtio_console virtio_blk net_failover ata_piix failover libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw virtio_pci floppy virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 30658 Comm: runc Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.92.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff93cfcc143180 ti: ffff93cf416a8000 task.ti: ffff93cf416a8000
RIP: 0010:[<ffffffffa351e1ca>] [<ffffffffa351e1ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff93cf416abaf8 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00000000000000fc RCX: ffff93cfffd60348
RDX: 0000000000000005 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff93cf416abb30 R08: ffff93ceb9311400 R09: ffffffffa379c669
R10: ffff93cfffc9f160 R11: ffffe24f44c1ea00 R12: 0000000000000292
R13: ffff93cf416abaa8 R14: 0000000000000000 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff93cfffc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000fc7cc002 CR3: 0000000148e10000 CR4: 00000000001606e0
Call Trace:
[<ffffffffa3481fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa348201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa35f9f93>] tlb_flush_mmu_tlbonly+0x33/0xd0
[<ffffffffa35fbb8f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa35fbcc3>] tlb_finish_mmu+0x23/0x40
[<ffffffffa36094fb>] exit_mmap+0xdb/0x1a0
[<ffffffffa3566bd5>] ? __delayacct_add_tsk+0x175/0x1a0
[<ffffffffa349c07f>] mmput+0x6f/0x100
[<ffffffffa34a6128>] do_exit+0x288/0xa30
[<ffffffffa34ea151>] ? put_prev_entity+0x31/0x400
[<ffffffffa34e8619>] ? pick_next_entity+0xa9/0x190
[<ffffffffa34a694f>] do_group_exit+0x3f/0xa0
[<ffffffffa34b825e>] get_signal_to_deliver+0x1ce/0x5e0
[<ffffffffa342d5b7>] do_signal+0x57/0x6f0
[<ffffffffa3bb6e5a>] ? do_nanosleep+0x9a/0x100
[<ffffffffa34cf340>] ? hrtimer_get_res+0x50/0x50
[<ffffffffa3bb6e1b>] ? do_nanosleep+0x5b/0x100
[<ffffffffa342dcca>] do_notify_resume+0x7a/0xd0
[<ffffffffa3bc56f7>] int_signal+0x12/0x17
Code: 48 63 35 7e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 15 a4 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #810269
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff867150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8667d398
Build id: 3.10.0-1062.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8667d40b
Build id: 3.10.0-1062.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff867ea0ef
Build id: 3.10.0-1062.el7.x86_64
|
0x2f
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff867ebc2f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff867ebd53
Build id: 3.10.0-1062.el7.x86_64
|
0x23
|
|
7
|
exit_mmap
|
vmlinux |
0xffffffff867f910b
Build id: 3.10.0-1062.el7.x86_64
|
0xdb
|
|
8
|
|
vmlinux |
0xffffffff8675c225
Build id: 3.10.0-1062.el7.x86_64
|
0x175
|
|
9
|
mmput
|
vmlinux |
0xffffffff866971f7
Build id: 3.10.0-1062.el7.x86_64
|
0x67
|
|
10
|
do_exit
|
vmlinux |
0xffffffff866a0ed8
Build id: 3.10.0-1062.el7.x86_64
|
0x288
|
|
11
|
|
vmlinux |
0xffffffff867114d5
Build id: 3.10.0-1062.el7.x86_64
|
0x195
|
|
12
|
do_group_exit
|
vmlinux |
0xffffffff866a171f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
13
|
get_signal_to_deliver
|
vmlinux |
0xffffffff866b252e
Build id: 3.10.0-1062.el7.x86_64
|
0x1ce
|
|
14
|
do_signal
|
vmlinux |
0xffffffff8662c527
Build id: 3.10.0-1062.el7.x86_64
|
0x57
|
|
15
|
|
vmlinux |
0xffffffff86d7e09a
Build id: 3.10.0-1062.el7.x86_64
|
0x9a
|
|
16
|
|
vmlinux |
0xffffffff866c8d10
Build id: 3.10.0-1062.el7.x86_64
|
0x50
|
|
17
|
|
vmlinux |
0xffffffff86d7e05b
Build id: 3.10.0-1062.el7.x86_64
|
0x5b
|
|
18
|
do_notify_resume
|
vmlinux |
0xffffffff8662cc32
Build id: 3.10.0-1062.el7.x86_64
|
0x72
|
|
19
|
int_signal
|
vmlinux |
0xffffffff86d8c23b
Build id: 3.10.0-1062.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [csi-snapshotter:61127]
Modules linked in: qax_tq_base(OE) xt_set ipt_rpfilter xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel veth xt_recent xt_statistic xt_nat ipt_REJECT nf_reject_ipv4 iptable_mangle xt_comment xt_mark ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi overlay(T) sunrpc ppdev pcspkr virtio_balloon joydev i2c_piix4 sg parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console virtio_blk virtio_scsi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm ata_piix drm libata virtio_pci virtio_ring serio_raw floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: qax_tq_base]
CPU: 0 PID: 61127 Comm: csi-snapshotter Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9e68dfb4a0e0 ti: ffff9e5a954d0000 task.ti: ffff9e5a954d0000
RIP: 0010:[<ffffffff867150ca>] [<ffffffff867150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9e5a954d3af8 EFLAGS: 00000202
RAX: 000000000000000f RBX: 00000000000000fc RCX: ffff9e6a7d1dfc10
RDX: 000000000000000f RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff9e5a954d3b30 R08: ffff9e5bffdf7000 R09: ffffffff8697f629
R10: ffff9e6a7ce1f160 R11: ffffe49679b3cc00 R12: 0000000000000292
R13: ffff9e5a954d3aa8 R14: 0000000000000000 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff9e6a7ce00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001cfd4e4 CR3: 0000000149210000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8667d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8667d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff867ea0ef>] tlb_flush_mmu_tlbonly+0x2f/0xc0
[<ffffffff867ebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff867ebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffff867f910b>] exit_mmap+0xdb/0x1a0
[<ffffffff8675c225>] ? __delayacct_add_tsk+0x175/0x1a0
[<ffffffff866971f7>] mmput+0x67/0xf0
[<ffffffff866a0ed8>] do_exit+0x288/0xa50
[<ffffffff867114d5>] ? futex_wait+0x195/0x280
[<ffffffff866a171f>] do_group_exit+0x3f/0xa0
[<ffffffff866b252e>] get_signal_to_deliver+0x1ce/0x5e0
[<ffffffff8662c527>] do_signal+0x57/0x6f0
[<ffffffff86d7e09a>] ? do_nanosleep+0x9a/0x100
[<ffffffff866c8d10>] ? hrtimer_get_res+0x50/0x50
[<ffffffff86d7e05b>] ? do_nanosleep+0x5b/0x100
[<ffffffff8662cc32>] do_notify_resume+0x72/0xc0
[<ffffffff86d8c23b>] int_signal+0x12/0x17
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 34 87 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89