Complete report #243208
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff85916e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8587e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8587e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff859ede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff859ef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff859efa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff859f2f3a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffff859ed42d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffff85f95176
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x17
|
|
10
|
|
vmlinux |
0xffffffff85961ec4
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xf4
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff85f94f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [zabbix_agent2:32371]
Modules linked in: fidelis(OE) tcp_diag inet_diag nfnetlink_queue nfnetlink_log veth vxlan ip6_udp_tunnel udp_tunnel xt_nat xt_statistic ipt_REJECT nf_reject_ipv4 nf_tables ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs iptable_mangle xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc nfnetlink overlay(T) sunrpc sb_edac ppdev iosf_mbi kvm_intel kvm irqbypass crc32_pclmul parport_pc parport joydev ghash_clmulni_intel virtio_balloon i2c_piix4 aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr cryptd ip_tables xfs libcrc32c ata_generic pata_acpi cirrus drm_kms_helper virtio_net net_failover virtio_blk failover syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel virtio_pci virtio_ring floppy serio_raw virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: fidelis]
CPU: 18 PID: 32371 Comm: zabbix_agent2 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: OpenStack Foundation OpenStack Nova, BIOS 1.10.2-1.1~u16.04+mcp2 04/01/2014
task: ffff9011623a8000 ti: ffff902995ca4000 task.ti: ffff902995ca4000
RIP: 0010:[<ffffffff85916e4a>] [<ffffffff85916e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff902995ca7c70 EFLAGS: 00000202
RAX: 000000000000001e RBX: 000800008587e6a0 RCX: ffff902f7f5a08a8
RDX: 000000000000001e RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff902995ca7ca8 R08: ffff901105377000 R09: ffffffff85b86479
R10: ffff902f7f29f160 R11: ffffeeb9c711ee00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff902995ca7c20 R15: 0000000000000001
FS: 00007f699dffb700(0000) GS:ffff902f7f280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00080b000 CR3: 00000001fdbce000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8587e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8587e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff859ede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff859ef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff859efa73>] tlb_finish_mmu+0x23/0x30
[<ffffffff859f2f3a>] zap_page_range+0xea/0x130
[<ffffffff859ed42d>] SyS_madvise+0x49d/0xac0
[<ffffffff85f95176>] ? sysret_audit+0x17/0x21
[<ffffffff85961ec4>] ? ftrace_ops_list_func+0xf4/0x120
[<ffffffff85f94f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 86 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1414563
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa7b1e25a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa7a81fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa7a8201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa7bfa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa7bfbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa7bfbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa7bff2ca
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffa7bf96bd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffa7acf340
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x50
|
|
10
|
|
vmlinux |
0xffffffffa81b6e1b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x5b
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa81c539a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [transport:13908]
Modules linked in: uinput nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq snd_seq_device snd_pcm ppdev snd_timer sg parport_pc snd parport pcspkr joydev soundcore i2c_piix4 virtio_balloon ip_tables xfs libcrc32c sd_mod
crc_t10dif crct10dif_generic crct10dif_common ata_generic pata_acpi virtio_console ata_piix libata e1000 crc32c_intel virtio_pci serio_raw virtio_ring floppy virtio dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 13908 Comm: transport Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: SANGFOR Standard PC (i440FX + PIIX, 1996)/Bochs, BIOS Bochs 01/01/2011
task: ffff9278f0ec4200 ti: ffff92793620c000 task.ti: ffff92793620c000
RIP: 0010:[<ffffffffa7b1e25a>] [<ffffffffa7b1e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff92793620fc70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000a7a81b10 RCX: ffff92793fc604c8
RDX: 0000000000000001 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff92793620fca8 R08: ffff92787aee1c00 R09: ffffffffa7d9c7f9
R10: ffff92793fd9f160 R11: ffffcf0687229000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff92793620fc20 R15: 0000000000000001
FS: 00007f7b9b524700(0000) GS:ffff92793fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff7ef147b90 CR3: 00000000866fa000 CR4: 00000000000206e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa7a81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa7a8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa7bfa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa7bfbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa7bfbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffa7bff2ca>] zap_page_range+0xea/0x140
[<ffffffffa7bf96bd>] SyS_madvise+0x49d/0xac0
[<ffffffffa7acf340>] ? hrtimer_get_res+0x50/0x50
[<ffffffffa81b6e1b>] ? do_nanosleep+0x5b/0x100
[<ffffffffa81c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 75 a8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #657280
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffab51761a
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffab47e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffab47e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffab5ede36
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffab5ef8ff
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffab5efa23
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x23
|
|
7
|
madvise_free_single_vma
|
vmlinux |
0xffffffffab5ec4e6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x116
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffab5ed421
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4d1
|
|
9
|
|
vmlinux |
0xffffffffab4ca2d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x50
|
|
10
|
|
vmlinux |
0xffffffffabb84dbb
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x5b
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffabb92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [dockerd:1749]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 sb_edac snd_rawmidi snd_ac97_codec iosf_mbi ppdev crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq aesni_intel lrw gf128mul glue_helper vmw_balloon snd_seq_device ablk_helper snd_pcm cryptd snd_timer joydev snd pcspkr soundcore sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common
crc32c_intel nfit ata_piix libnvdimm mptspi serio_raw libata e1000 scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1749 Comm: dockerd Kdump: loaded Tainted: G L ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9b21c71f3150 ti: ffff9b23d1eb4000 task.ti: ffff9b23d1eb4000
RIP: 0010:[<ffffffffab51761a>] [<ffffffffab51761a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9b23d1eb7c70 EFLAGS: 00000202
RAX: 0000000000000007 RBX: 00080000ab47e730 RCX: ffff9b23ed7e06c8
RDX: 0000000000000007 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9b23d1eb7ca8 R08: ffff9b2139ccc400 R09: ffffffffab7851b9
R10: ffff9b23ed79f160 R11: ffffe04f8e809c00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9b23d1eb7c20 R15: 000000c001fa7000
FS: 00007f6be6fed700(0000) GS:ffff9b23ed780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055c43ead2000 CR3: 000000041eaa8000 CR4: 00000000003607e0
Call Trace:
[<ffffffffab47e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffab47e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffab5ede36>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffab5ef8ff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffab5efa23>] tlb_finish_mmu+0x23/0x30
[<ffffffffab5ec4e6>] madvise_free_single_vma+0x116/0x180
[<ffffffffab5ed421>] SyS_madvise+0x4d1/0xab0
[<ffffffffab4ca2d0>] ? hrtimer_get_res+0x50/0x50
[<ffffffffabb84dbb>] ? do_nanosleep+0x5b/0x100
[<ffffffffabb92ed2>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 15 ac f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89