Complete report #1198652
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8c71e25a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8c681fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8c68201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8c7fa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8c7fbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8c7fbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8c7ff2ca
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff8c9226b9
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xc9
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff8c7f96bd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff8c75f95b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x5b
|
|
11
|
tracesys
|
vmlinux |
0xffffffff8cdc562e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [conn23:5105]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi ppdev snd_ac97_codec aesni_intel vmw_balloon
ac97_bus lrw snd_seq gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm joydev pcspkr snd_timer snd sg vmw_vmci soundcore i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel mptspi e1000 scsi_transport_spi serio_raw mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 2 PID: 5105 Comm: conn23 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a6fd805e300 ti: ffff8a6fe1874000 task.ti: ffff8a6fe1874000
RIP: 0010:[<ffffffff8c71e25a>] [<ffffffff8c71e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a6fe1877c70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 000800008c681b10 RCX: ffff8a707a6604b0
RDX: 0000000000000001 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8a6fe1877ca8 R08: ffff8a70459e4000 R09: ffffffff8c99c7f9
R10: ffff8a707a69f160 R11: fffff15b8037ea00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8a6fe1877c20 R15: 0000000000000001
FS: 00007f0d54f20700(0000) GS:ffff8a707a680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f157622d2a0 CR3: 000000008a852000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8c681fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8c68201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8c7fa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff8c7fbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8c7fbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffff8c7ff2ca>] zap_page_range+0xea/0x140
[<ffffffff8c9226b9>] ? file_has_perm+0xc9/0x160
[<ffffffff8c7f96bd>] SyS_madvise+0x49d/0xac0
[<ffffffff8c75f95b>] ? __seccomp_filter+0x5b/0x310
[<ffffffff8cdc562e>] tracesys+0xa6/0xcc
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 35 8d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #882630
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb611722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb607e258
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb607e2cb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb61ee666
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb61f012f
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb61f0253
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb61f371a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffb61edc0d
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffb61573eb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x5b
|
|
10
|
|
vmlinux |
0xffffffffb60daf40
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20
|
|
11
|
tracesys
|
vmlinux |
0xffffffffb679a226
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [kube-apiserver:13049]
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [kube-apiserver:12602]
Modules linked in: xt_statistic vxlan ip6_udp_tunnel udp_tunnel xt_nat nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter tun overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg pcspkr joydev vmw_vmci i2c_piix4 parport_pc parport
ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm nfit ata_piix libahci libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 10 PID: 12602 Comm: kube-apiserver Tainted: G L ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff90ae5cf20000 ti: ffff90b42dad8000 task.ti: ffff90b42dad8000
RIP: 0010:[<ffffffffb611722a>] [<ffffffffb611722a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff90b42dadbc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000b607ddd0 RCX: ffff90ae67c207c8
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff90b42dadbca8 R08: ffff90aef0df6000 R09: ffffffffb63869d9
R10: ffff90b47fc9f160 R11: fffff61ceeffb200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff90b42dadbc20 R15: 0000000000000001
FS: 000000c00008c490(0000) GS:ffff90b47fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c007cc2038 CR3: 0000000beda50000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb607e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb607e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb61ee666>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb61f012f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb61f0253>] tlb_finish_mmu+0x23/0x30
[<ffffffffb61f371a>] zap_page_range+0xea/0x130
[<ffffffffb61edc0d>] SyS_madvise+0x49d/0xac0
[<ffffffffb61573eb>] ? __seccomp_filter+0x5b/0x300
[<ffffffffb60daf40>] ? wake_up_state+0x20/0x20
[<ffffffffb679a226>] tracesys+0xa6/0xcc
Code: 48 63 35 7e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 d5 b6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #867366
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa971e23a
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa9681fa8
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa968201b
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa97fa07e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa97fbbff
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa97fbd33
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa97ff27a
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa9802503
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xdb3
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa97f966d
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa975f93b
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x5b
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa9dc562e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [popen.rb:41:4193]
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [memory_killer:4192]
Modules linked in: veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel snd_timer lrw gf128mul ppdev snd glue_helper ablk_helper
cryptd vmw_balloon soundcore sg pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 crct10dif_pclmul drm crct10dif_common crc32c_intel ata_piix mptspi nfit scsi_transport_spi libata serio_raw mptscsih libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 4192 Comm: memory_killer Kdump: loaded Tainted: G ------------ T 3.10.0-1160.99.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a8d6a5a1080 ti: ffff8a8d1748c000 task.ti: ffff8a8d1748c000
RIP: 0010:[<ffffffffa971e23a>] [<ffffffffa971e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a8d1748fc70 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 00080000a9681b10 RCX: ffff8a8df57206f0
RDX: 0000000000000004 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8a8d1748fca8 R08: ffff8a8d3a88c800 R09: ffffffffa999c7b9
R10: ffff8a8df57df160 R11: ffffcb0645a91e00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff8a8d1748fbe0 R15: 0000000000000282
FS: 00007f78c7260700(0000) GS:ffff8a8df57c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f79090f3338 CR3: 000000004d0d4000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa9681fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa968201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa97fa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa97fbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa97fbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffffa97ff27a>] zap_page_range+0xea/0x140
[<ffffffffa9802503>] ? handle_mm_fault+0xdb3/0x1190
[<ffffffffa97f966d>] SyS_madvise+0x49d/0xac0
[<ffffffffa975f93b>] ? __seccomp_filter+0x5b/0x310
[<ffffffffa9dc562e>] tracesys+0xa6/0xcc
Code: 48 63 35 4e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 35 aa f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #733799
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa0b170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa0a7e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa0a7e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa0bee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa0bf013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa0bf0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa0bf372a
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffa0bedc1d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffa0b5729b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x5b
|
|
10
|
tracesys
|
vmlinux |
0xffffffffa119a226
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [java:9490]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 nfit libnvdimm coretemp ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif cdrom crct10dif_generic ata_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm pata_acpi mptspi drm scsi_transport_spi ata_piix mptscsih mptbase crct10dif_pclmul libata crct10dif_common crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks
floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 8 PID: 9490 Comm: java Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9057c66b6300 ti: ffff905251180000 task.ti: ffff905251180000
RIP: 0010:[<ffffffffa0b170da>] [<ffffffffa0b170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff905251183c70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000a0a7ddd0 RCX: ffff9052f7c20850
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff905251183ca8 R08: ffff90557b0cfc00 R09: ffffffffa0d86c09
R10: ffff9056f7c1f160 R11: ffffe80d54635a00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff905251183c20 R15: 0000000000000001
FS: 00007fbe9b2d9700(0000) GS:ffff9056f7c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc0c000ad40 CR3: 0000000236dd2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa0a7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa0a7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa0bee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa0bf013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa0bf0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffa0bf372a>] zap_page_range+0xea/0x130
[<ffffffffa0bedc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffa0b5729b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffa119a226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 75 a1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #639707
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffaab170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffaaa7e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffaaa7e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffaabee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffaabf013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffaabf0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffaabf372a
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffaabf66a2
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb22
|
|
9
|
|
vmlinux |
0xffffffffaabfd36c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x39c
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffaabedc1d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffffaab5729b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x5b
|
|
12
|
tracesys
|
vmlinux |
0xffffffffab19a226
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [elasticsearch[s:26317]
Modules linked in: 8021q garp mrp stp llc ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr sg joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod crc_t10dif crct10dif_generic drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel nfit serio_raw libnvdimm vmw_pvscsi vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 26317 Comm: elasticsearch[s Kdump: loaded Tainted: G L ------------ 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff98a1f8750000 ti: ffff98a283548000 task.ti: ffff98a283548000
RIP: 0010:[<ffffffffaab170da>] [<ffffffffaab170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff98a28354bc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000aaa7ddd0 RCX: ffff98a57fc20628
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff98a28354bca8 R08: ffff98a2f6bc0c00 R09: ffffffffaad86c09
R10: ffff98a57fc9f160 R11: ffffe0decb05ca00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff98a28354bbe0 R15: 0000000000000282
FS: 00007f806d2c7700(0000) GS:ffff98a57fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f81a28472b0 CR3: 00000003d7942000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffaaa7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffaaa7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffaabee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffaabf013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffaabf0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffaabf372a>] zap_page_range+0xea/0x130
[<ffffffffaabf66a2>] ? handle_mm_fault+0xb22/0xfb0
[<ffffffffaabfd36c>] ? do_mmap+0x39c/0x590
[<ffffffffaabedc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffaab5729b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffab19a226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 75 ab f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #628378
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa49170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa487e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa487e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa49ee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa49f013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa49f0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa49f372a
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa48dada8
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffffa4a9d8fe
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x31e
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffa49edc1d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffffa48dadf0
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffa4f99f92
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [prometheus:2159]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq crc32_pclmul ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm lrw gf128mul snd_timer glue_helper snd ablk_helper cryptd vmw_vmci vmw_balloon soundcore ppdev
i2c_piix4 joydev pcspkr sg parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea crct10dif_pclmul crct10dif_common sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm serio_raw ata_piix drm nfit e1000 libnvdimm mptspi scsi_transport_spi libata mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 2159 Comm: prometheus Kdump: loaded Not tainted 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff8f5d31dd9080 ti: ffff8f5e8fb6c000 task.ti: ffff8f5e8fb6c000
RIP: 0010:[<ffffffffa49170da>] [<ffffffffa49170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8f5e8fb6fc70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000a487ddd0 RCX: ffff8f5eb56e0600
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8f5e8fb6fca8 R08: ffff8f5dfa88b000 R09: ffffffffa4b86c09
R10: ffff8f5eb565f160 R11: ffffe32746f1da00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff8f5e8fb6fbe0 R15: 0000000000000282
FS: 00007fd5fce84700(0000) GS:ffff8f5eb5640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3ddd688000 CR3: 0000000220700000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffa487e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa487e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa49ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa49f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa49f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffa49f372a>] zap_page_range+0xea/0x130
[<ffffffffa48dada8>] ? wake_up_q+0x58/0x80
[<ffffffffa4a9d8fe>] ? ep_poll+0x31e/0x360
[<ffffffffa49edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffa48dadf0>] ? wake_up_state+0x20/0x20
[<ffffffffa4f99f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 55 a5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #619140
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff93b1e0ba
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff93a81f88
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff93a81ffb
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff93bf9f4e
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff93bfba8f
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff93bfbbc3
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff93bff10a
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff93ae1008
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff93bf94dd
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff93b5f7bb
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x5b
|
|
11
|
|
vmlinux |
0xffffffff93b1c620
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x80
|
|
12
|
tracesys
|
vmlinux |
0xffffffff941c562e
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [auditbeat:1151]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14604(E) snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core iosf_mbi snd_hwdep crc32_pclmul snd_seq ghash_clmulni_intel aesni_intel snd_seq_device snd_pcm lrw gf128mul snd_timer glue_helper snd ablk_helper cryptd soundcore joydev virtio_rng virtio_balloon i2c_piix4 pcspkr ip_tables xfs libcrc32c ata_generic virtio_blk virtio_console pata_acpi qxl drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm ata_piix e1000 drm libata serio_raw crct10dif_pclmul crct10dif_common virtio_pci crc32c_intel virtio_ring floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 1151 Comm: auditbeat Tainted: P E ------------ 3.10.0-1160.83.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.15.0-2.module+el8.6.0+20659+3dcf7c70 04/01/2014
task: ffff935c3102d280 ti: ffff935c328f8000 task.ti: ffff935c328f8000
RIP: 0010:[<ffffffff93b1e0ba>] [<ffffffff93b1e0ba>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff935c328fbc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0008000093a81af0 RCX: ffff935c3fc203c0
RDX: 0000000000000000 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff935c328fbca8 R08: ffff935af9312000 R09: ffffffff93d9c179
R10: ffff935c3fd5f160 R11: fffff46185c1d100 R12: 0000000000000282
R13: 0000000000000282 R14: ffff935c328fbc20 R15: 0000000000000001
FS: 00007fcca7fff700(0000) GS:ffff935c3fd40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c001b5f000 CR3: 00000000ba09a000 CR4: 00000000001606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff93a81f88>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff93a81ffb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff93bf9f4e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff93bfba8f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff93bfbbc3>] tlb_finish_mmu+0x23/0x40
[<ffffffff93bff10a>] zap_page_range+0xea/0x140
[<ffffffff93ae1008>] ? wake_up_q+0x58/0x90
[<ffffffff93bf94dd>] SyS_madvise+0x49d/0xac0
[<ffffffff93b5f7bb>] ? __seccomp_filter+0x5b/0x310
[<ffffffff93b1c620>] ? SyS_futex+0x80/0x190
[<ffffffff941c562e>] tracesys+0xa6/0xcc
Code: 48 63 35 8e 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 75 94 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #586493
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa8b16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa8a7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa8a7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa8bedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa8bef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa8bef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa8bf2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffa8bed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffa8b56dfb
Build id: 3.10.0-1160.el7.x86_64
|
0x5b
|
|
10
|
|
vmlinux |
0xffffffffa8c4a6fa
Build id: 3.10.0-1160.el7.x86_64
|
0x1ca
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa9194226
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [util.rb:23:101209]
Modules linked in: tcp_lp veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi ppdev snd_ac97_codec ac97_bus aesni_intel
snd_seq lrw gf128mul snd_seq_device glue_helper ablk_helper vmw_balloon snd_pcm cryptd snd_timer joydev pcspkr snd soundcore i2c_piix4 sg vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt ata_generic pata_acpi crct10dif_pclmul mptspi fb_sys_fops crct10dif_common ttm scsi_transport_spi drm ata_piix e1000 mptscsih crc32c_intel serio_raw mptbase nfit libnvdimm libata drm_panel_orientation_quirks fuse
CPU: 1 PID: 101209 Comm: util.rb:23 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff88958b05b180 ti: ffff88969f8fc000 task.ti: ffff88969f8fc000
RIP: 0010:[<ffffffffa8b16c0a>] [<ffffffffa8b16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88969f8ffc70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000a8a7e6a0 RCX: ffff8897af2a0600
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88969f8ffca8 R08: ffff8896fe34b000 R09: ffffffffa8d85f99
R10: ffff8897af25f160 R11: ffffe442c436ac00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff88969f8ffc20 R15: 0000000000000001
FS: 00007fe3a33ef700(0000) GS:ffff8897af240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa515170000 CR3: 00000001391a4000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa8a7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa8a7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa8bedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa8bef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa8bef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffa8bf2dba>] zap_page_range+0xea/0x130
[<ffffffffa8bed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffa8b56dfb>] ? __seccomp_filter+0x5b/0x300
[<ffffffffa8c4a6fa>] ? __check_object_size+0x1ca/0x250
[<ffffffffa9194226>] tracesys+0xa6/0xcc
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 a9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #586400
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8c916c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8c87e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8c87e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8c9edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8c9ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8c9ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8c9f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff8c9f5d32
Build id: 3.10.0-1160.el7.x86_64
|
0xb22
|
|
9
|
|
vmlinux |
0xffffffff8c9fc9d8
Build id: 3.10.0-1160.el7.x86_64
|
0x378
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffff8c9ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffff8c956dfb
Build id: 3.10.0-1160.el7.x86_64
|
0x5b
|
|
12
|
tracesys
|
vmlinux |
0xffffffff8cf94226
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 62s! [FilePath.localP:6846]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec lrw ac97_bus
gf128mul ppdev glue_helper ablk_helper snd_seq snd_seq_device cryptd vmw_balloon snd_pcm snd_timer snd joydev pcspkr soundcore sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom ata_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops pata_acpi mptspi ttm scsi_transport_spi crct10dif_pclmul mptscsih crct10dif_common ata_piix drm mptbase nfit e1000 crc32c_intel serio_raw drm_panel_orientation_quirks libata libnvdimm fuse
CPU: 1 PID: 6846 Comm: FilePath.localP Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff99a752df6300 ti: ffff99a75297c000 task.ti: ffff99a75297c000
RIP: 0010:[<ffffffff8c916c0a>] [<ffffffff8c916c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff99a75297fc70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000800008c87e6a0 RCX: ffff99a7796e0600
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff99a75297fca8 R08: ffff99a770fe3000 R09: ffffffff8cb85f99
R10: ffff99a77965f160 R11: fffff2ce82589600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff99a75297fc20 R15: 0000000000000001
FS: 00007fce3ed70700(0000) GS:ffff99a779640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fce5e86b530 CR3: 00000000996c6000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8c87e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8c87e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8c9edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8c9ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8c9ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff8c9f2dba>] zap_page_range+0xea/0x130
[<ffffffff8c9f5d32>] ? handle_mm_fault+0xb22/0xfb0
[<ffffffff8c9fc9d8>] ? do_mmap+0x378/0x530
[<ffffffff8c9ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffff8c956dfb>] ? __seccomp_filter+0x5b/0x300
[<ffffffff8cf94226>] tracesys+0xa6/0xcc
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 8d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #579523
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8b916bda
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8b87e258
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8b87e2cb
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8b9ee176
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8b9efc3f
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8b9efd63
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8b9f322a
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff8b8dad78
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffff8b912170
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffff8b9ed71d
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffff8b956d9b
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x5b
|
|
12
|
tracesys
|
vmlinux |
0xffffffff8bf96226
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [JS Helper:7567]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache macvlan dummy xt_nat veth tcp_lp nf_conntrack_netlink xt_addrtype br_netfilter xt_multiport xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vboxnetadp(OE) vboxnetflt(OE) vboxdrv(OE) vmw_vsock_vmci_transport vsock
sunrpc sb_edac iosf_mbi kvm_intel ppdev kvm vmw_balloon irqbypass pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc32_pclmul crc32c_intel ahci crc_t10dif crct10dif_generic crct10dif_pclmul ghash_clmulni_intel crct10dif_common ata_piix libahci drm aesni_intel nfit libata glue_helper lrw gf128mul ablk_helper libnvdimm cryptd serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 7567 Comm: JS Helper Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.31.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8c8ba2278000 ti: ffff8c8ba236c000 task.ti: ffff8c8ba236c000
RIP: 0010:[<ffffffff8b916bda>] [<ffffffff8b916bda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8c8ba236fc70 EFLAGS: 00000202
RAX: 0000000000000007 RBX: 000800008b87ddd0 RCX: ffff8c8bfd7e0678
RDX: 0000000000000007 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8c8ba236fca8 R08: ffff8c7d59683c00 R09: ffffffff8bb866a9
R10: ffff8c8bfd71f160 R11: fffffe569e03d000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8c8ba236fc20 R15: 0000000000000001
FS: 00007fda72402700(0000) GS:ffff8c8bfd700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f71a7ffc318 CR3: 0000000f98ae8000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8b87e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8b87e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8b9ee176>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8b9efc3f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8b9efd63>] tlb_finish_mmu+0x23/0x30
[<ffffffff8b9f322a>] zap_page_range+0xea/0x130
[<ffffffff8b8dad78>] ? wake_up_q+0x58/0x80
[<ffffffff8b912170>] ? futex_wake+0x90/0x180
[<ffffffff8b9ed71d>] SyS_madvise+0x49d/0xac0
[<ffffffff8b956d9b>] ? __seccomp_filter+0x5b/0x300
[<ffffffff8bf96226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 8a c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 55 8c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 8a c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #554487
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa8916c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa887e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa887e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa89edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa89ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa89ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa89f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa89fc9d8
Build id: 3.10.0-1160.el7.x86_64
|
0x378
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa89ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa8956dfb
Build id: 3.10.0-1160.el7.x86_64
|
0x5b
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa8f94226
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 25s! [java:25014]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock sg vmw_balloon ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel joydev lrw gf128mul glue_helper ablk_helper cryptd vmw_vmci parport_pc parport pcspkr i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea crc_t10dif sysfillrect crct10dif_generic sysimgblt fb_sys_fops ttm drm ahci crct10dif_pclmul nfit ata_piix libahci crct10dif_common crc32c_intel serio_raw libata libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash
dm_log dm_mod fuse
CPU: 1 PID: 25014 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a6a9aea6300 ti: ffff8a6cbc8fc000 task.ti: ffff8a6cbc8fc000
RIP: 0010:[<ffffffffa8916c0a>] [<ffffffffa8916c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a6cbc8ffc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000a887e6a0 RCX: ffff8a6a75a20558
RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff8a6cbc8ffca8 R08: ffff8a692e7ac000 R09: ffffffffa8b85f99
R10: ffff8a6a75a5f160 R11: ffffd8a9ccd5e200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8a6cbc8ffc20 R15: 0000000000000001
FS: 00007f417cece700(0000) GS:ffff8a6a75a40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f71167b3200 CR3: 00000004fc898000 CR4: 0000000000760fe0
PKRU: 55555554
Call Trace:
[<ffffffffa887e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa887e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa89edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa89ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa89ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffa89f2dba>] zap_page_range+0xea/0x130
[<ffffffffa89fc9d8>] ? do_mmap+0x378/0x530
[<ffffffffa89ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffa8956dfb>] ? __seccomp_filter+0x5b/0x300
[<ffffffffa8f94226>] tracesys+0xa6/0xcc
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 a9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #464940
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffab716e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffab67e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffab67e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffab7ede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffab7ef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffab7efa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffab7f2f3a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffab89d23e
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x31e
|
|
9
|
|
vmlinux |
0xffffffffab84d163
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x93
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffab7ed42d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x49d
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffabd94f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 25s! [k3s-server:55926]
Modules linked in: xt_statistic xt_owner xt_REDIRECT nf_nat_redirect veth ip6t_MASQUERADE nf_nat_masquerade_ipv6 nf_conntrack_netlink xt_nat ip6table_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_physdev xt_NFLOG nfnetlink_log xt_limit vxlan ip6_udp_tunnel xt_mark udp_tunnel xt_multiport xt_addrtype xt_comment ip_set nfnetlink xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter cmac arc4 md4 nls_utf8 br_netfilter bridge cifs stp ccm llc dns_resolver overlay(T) vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi kvm_intel
kvm irqbypass crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel libata nfit serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 55926 Comm: k3s-server Tainted: G L ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9614795a9080 ti: ffff96177f448000 task.ti: ffff96177f448000
RIP: 0010:[<ffffffffab716e4a>] [<ffffffffab716e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff96177f44bc70 EFLAGS: 00000202
RAX: 000000000000000b RBX: 00080000ab67e6a0 RCX: ffff96189d8dfc10
RDX: 000000000000000b RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff96177f44bca8 R08: ffff9611ffdd7000 R09: ffffffffab986479
R10: ffff96189d61f160 R11: fffff68014ac7000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff96177f44bc20 R15: 0000000000000001
FS: 00007f4e146bbf38(0000) GS:ffff96189d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f58004f7000 CR3: 000000017338e000 CR4: 00000000001607f0
Call Trace:
[<ffffffffab67e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffab67e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffab7ede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffab7ef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffab7efa73>] tlb_finish_mmu+0x23/0x30
[<ffffffffab7f2f3a>] zap_page_range+0xea/0x130
[<ffffffffab89d23e>] ? ep_poll+0x31e/0x360
[<ffffffffab84d163>] ? do_sync_read+0x93/0xe0
[<ffffffffab7ed42d>] SyS_madvise+0x49d/0xac0
[<ffffffffabd94f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 35 ac f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #443679
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb0f16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb0e7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb0e7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb0fedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb0fef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb0fef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb0ff2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffb0fed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffb0f56dfb
Build id: 3.10.0-1160.el7.x86_64
|
0x5b
|
|
10
|
tracesys
|
vmlinux |
0xffffffffb1594226
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [popen.rb:42:14674]
Modules linked in: veth br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul ppdev glue_helper ablk_helper cryptd snd_timer vmw_balloon snd soundcore joydev sg pcspkr i2c_piix4 vmw_vmci parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm nfit libnvdimm drm serio_raw mptspi e1000 scsi_transport_spi ata_piix mptscsih libata mptbase drm_panel_orientation_quirks
fuse [last unloaded: ip_tables]
CPU: 0 PID: 14674 Comm: popen.rb:42 Kdump: loaded Tainted: G B L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9c85837ea100 ti: ffff9c852e8f4000 task.ti: ffff9c852e8f4000
RIP: 0010:[<ffffffffb0f16c0a>] [<ffffffffb0f16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c852e8f7c70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000c0000b0e7e6a0 RCX: ffff9c85b96dfc10
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9c852e8f7ca8 R08: ffff9c85b91d7000 R09: ffffffffb1185f99
R10: ffff9c85b961f160 R11: ffffcfd502589e00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff9c852e8f7be0 R15: 0000000000000282
FS: 00007f4cc8453700(0000) GS:ffff9c85b9600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4cbfdf7000 CR3: 000000003592e000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb0e7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb0e7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb0fedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb0fef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb0fef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffb0ff2dba>] zap_page_range+0xea/0x130
[<ffffffffb0fed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffb0f56dfb>] ? __seccomp_filter+0x5b/0x300
[<ffffffffb1594226>] tracesys+0xa6/0xcc
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 b5 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #379506
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffad9170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffad87e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffad87e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffad9ee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffad9f013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffad9f0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffad9f372a
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffad8dada8
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffad9edc1d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffad95729b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x5b
|
|
11
|
|
vmlinux |
0xffffffffad8dadf0
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20
|
|
12
|
|
vmlinux |
0xffffffffad9156f0
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x80
|
|
13
|
tracesys
|
vmlinux |
0xffffffffadf9a226
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [metricbeat:28762]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13109(E) snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq iosf_mbi snd_seq_device snd_pcm crc32_pclmul ghash_clmulni_intel aesni_intel snd_timer lrw gf128mul glue_helper ablk_helper snd cryptd soundcore ppdev pcspkr joydev virtio_rng parport_pc virtio_balloon parport i2c_piix4 ip_tables xfs libcrc32c ata_generic pata_acpi virtio_console virtio_blk qxl drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul ata_piix crct10dif_common crc32c_intel serio_raw libata e1000 virtio_pci drm_panel_orientation_quirks virtio_ring virtio floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 28762 Comm: metricbeat Tainted: P EL ------------ 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7860+a7792d29 04/01/2014
task: ffff88015e003180 ti: ffff8801bb770000 task.ti: ffff8801bb770000
RIP: 0010:[<ffffffffad9170da>] [<ffffffffad9170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8801bb773c70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000ad87ddd0 RCX: ffff8802bfc9fc10
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff8801bb773ca8 R08: ffff88027fd52000 R09: ffffffffadb86c09
R10: ffff8802bfc1f160 R11: ffffd952c6d7cb00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8801bb773c20 R15: 0000000000000001
FS: 00007fe4fdffb700(0000) GS:ffff8802bfc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd7880eed28 CR3: 00000000b79a8000 CR4: 00000000001606f0
Call Trace:
[<ffffffffad87e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffad87e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffad9ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffad9f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffad9f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffad9f372a>] zap_page_range+0xea/0x130
[<ffffffffad8dada8>] ? wake_up_q+0x58/0x80
[<ffffffffad9edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffad95729b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffad8dadf0>] ? wake_up_state+0x20/0x20
[<ffffffffad9156f0>] ? SyS_futex+0x80/0x190
[<ffffffffadf9a226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 55 ae f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #350868
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9b5170da
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9b47e258
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9b47e2cb
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9b5ee676
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9b5f013f
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9b5f0263
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff9b5f372a
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff9b4dada8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff9b5edc1d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff9b55729b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x5b
|
|
11
|
|
vmlinux |
0xffffffff9b4dadf0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
|
12
|
tracesys
|
vmlinux |
0xffffffff9bb96226
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [filebeat:17852]
Modules linked in: tcp_lp rfcomm veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter binfmt_misc ip6_tables iptable_filter des_generic md4 nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock bnep sunrpc snd_seq_midi snd_seq_midi_event squashfs loop ppdev snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus vmw_balloon snd_seq btusb ghash_clmulni_intel snd_seq_device btrtl btbcm btintel bluetooth snd_pcm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev snd_timer rfkill snd sg soundcore
vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi scsi_transport_spi libata mptscsih mptbase crct10dif_pclmul crct10dif_common e1000 nfit crc32c_intel libnvdimm serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 5 PID: 17852 Comm: filebeat Kdump: loaded Tainted: G ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffffa04e43014200 ti: ffffa04e44c98000 task.ti: ffffa04e44c98000
RIP: 0010:[<ffffffff9b5170da>] [<ffffffff9b5170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa04e44c9bc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009b47ddd0 RCX: ffffa053f3a206a0
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa04e44c9bca8 R08: ffffa0475d4a4000 R09: ffffffff9b786bf9
R10: ffffa053f3b5f160 R11: ffffd38f414ea800 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa04e44c9bc20 R15: 0000000000000001
FS: 00007f94e37fe700(0000) GS:ffffa053f3b40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00003e8a0067f000 CR3: 0000000db8b16000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9b47e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9b47e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9b5ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9b5f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9b5f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffff9b5f372a>] zap_page_range+0xea/0x130
[<ffffffff9b4dada8>] ? wake_up_q+0x58/0x80
[<ffffffff9b5edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffff9b55729b>] ? __seccomp_filter+0x5b/0x300
[<ffffffff9b4dadf0>] ? wake_up_state+0x20/0x20
[<ffffffff9bb96226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 15 9c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #344951
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffaa3170da
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffaa27e258
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffaa27e2cb
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffaa3ee676
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffaa3f013f
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffaa3f0263
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffaa3f372a
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffaa2dada8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffaa3edc1d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffaa35729b
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x5b
|
|
11
|
tracesys
|
vmlinux |
0xffffffffaa996226
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [metricbeat:26567]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_12905(E) xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay snd_hda_codec_generic snd_hda_intel iosf_mbi snd_hda_codec crc32_pclmul ppdev snd_hda_core ghash_clmulni_intel joydev aesni_intel snd_hwdep snd_seq lrw snd_seq_device gf128mul glue_helper snd_pcm ablk_helper cryptd virtio_balloon virtio_rng snd_timer snd parport_pc i2c_piix4 parport soundcore pcspkr ip_tables xfs libcrc32c virtio_blk virtio_console ata_generic pata_acpi qxl drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm e1000 ata_piix crct10dif_pclmul crct10dif_common virtio_pci libata crc32c_intel virtio_ring serio_raw virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 26567 Comm: metricbeat Tainted: P E ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7860+a7792d29 04/01/2014
task: ffff9f686ef74200 ti: ffff9f68254c4000 task.ti: ffff9f68254c4000
RIP: 0010:[<ffffffffaa3170da>] [<ffffffffaa3170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9f68254c7c70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000aa27ddd0 RCX: ffff9f693fc20370
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9f68254c7ca8 R08: ffff9f687aec2000 R09: ffffffffaa586bf9
R10: ffff9f693fd9f160 R11: ffffe5a50733ad00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9f68254c7c20 R15: 0000000000000001
FS: 00007f542f445700(0000) GS:ffff9f693fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c001bff000 CR3: 00000000b8ce6000 CR4: 00000000001606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaa27e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffaa27e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffaa3ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffaa3f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffaa3f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffaa3f372a>] zap_page_range+0xea/0x130
[<ffffffffaa2dada8>] ? wake_up_q+0x58/0x80
[<ffffffffaa3edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffaa35729b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffaa996226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 f5 aa f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #279315
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb8d170da
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb8c7e258
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb8c7e2cb
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb8dee676
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb8df013f
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb8df0263
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb8df372a
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffb8dedc1d
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x49d
|
|
9
|
?
poll_select_copy_remaining
|
vmlinux |
0xffffffffb8e64924
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x144
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb9395f92
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [docker-containe:1928]
Modules linked in: tcp_lp xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event sb_edac coretemp snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus iosf_mbi crc32_pclmul snd_seq ghash_clmulni_intel aesni_intel lrw pcspkr gf128mul
glue_helper snd_seq_device snd_pcm ablk_helper cryptd snd_timer snd ppdev soundcore vmw_balloon sg joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel nfit ata_piix libnvdimm mptspi scsi_transport_spi libata mptscsih serio_raw e1000 mptbase drm_panel_orientation_quirks fuse
CPU: 10 PID: 1928 Comm: docker-containe Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8d9e21f29080 ti: ffff8d9c35804000 task.ti: ffff8d9c35804000
RIP: 0010:[<ffffffffb8d170da>] [<ffffffffb8d170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8d9c35807c70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000b8c7ddd0 RCX: ffff8d9e33620768
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8d9c35807ca8 R08: ffff8d9cf89f0000 R09: ffffffffb8f86bf9
R10: ffff8d9e3389f160 R11: ffffe73249b92800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8d9c35807c20 R15: 0000000000000001
FS: 00007f1029c6c700(0000) GS:ffff8d9e33880000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6635b50990 CR3: 00000000b66ba000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb8c7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb8c7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb8dee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb8df013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb8df0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffb8df372a>] zap_page_range+0xea/0x130
[<ffffffffb8dedc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffb8e64924>] ? poll_select_copy_remaining+0x144/0x180
[<ffffffffb9395f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 95 b9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #279314
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb09170da
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb087e258
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb087e2cb
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb09ee676
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb09f013f
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb09f0263
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb09f372a
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb08dada8
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb09edc1d
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x49d
|
|
10
|
?
poll_select_copy_remaining
|
vmlinux |
0xffffffffb0a64924
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x144
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb0f95f92
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [docker-containe:1811]
Modules linked in: tcp_lp xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event sb_edac snd_ens1371 coretemp snd_rawmidi snd_ac97_codec ac97_bus iosf_mbi snd_seq crc32_pclmul snd_seq_device ghash_clmulni_intel snd_pcm aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd snd_timer joydev snd pcspkr soundcore ppdev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel nfit libata libnvdimm mptspi scsi_transport_spi e1000 serio_raw mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 14 PID: 1811 Comm: docker-containe Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9eea63078000 ti: ffff9eea5ec4c000 task.ti: ffff9eea5ec4c000
RIP: 0010:[<ffffffffb09170da>] [<ffffffffb09170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9eea5ec4fc70 EFLAGS: 00000202
RAX: 0000000000000008 RBX: 00080000b087ddd0 RCX: ffff9eea73820808
RDX: 0000000000000008 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9eea5ec4fca8 R08: ffff9ee9389f2000 R09: ffffffffb0b86bf9
R10: ffff9eea7399f160 R11: fffffe7d82d09a00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9eea5ec4fc20 R15: ffffffffb19dbb80
FS: 00007fb3bc346700(0000) GS:ffff9eea73980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe7886c1c99 CR3: 00000002a31cc000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb087e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb087e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb09ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb09f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb09f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffb09f372a>] zap_page_range+0xea/0x130
[<ffffffffb08dada8>] ? wake_up_q+0x58/0x80
[<ffffffffb09edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffb0a64924>] ? poll_select_copy_remaining+0x144/0x180
[<ffffffffb0f95f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 55 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #266040
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa4b16bda
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa4a7e258
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa4a7e2cb
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa4bee176
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa4befc3f
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa4befd63
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa4bf322a
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffa4bed71d
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffa4b56d9b
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x5b
|
|
10
|
|
vmlinux |
0xffffffffa4ae22e9
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xa9
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa5196226
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [java:22608]
Modules linked in: tcp_lp veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi crc32_pclmul snd_seq_device ghash_clmulni_intel snd_pcm aesni_intel snd_timer snd lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon soundcore joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom
crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common nfit crc32c_intel drm libnvdimm e1000 ata_piix serio_raw mptspi scsi_transport_spi libata mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 1 PID: 22608 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.31.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9d9cb4d1c200 ti: ffff9d9be6fbc000 task.ti: ffff9d9be6fbc000
RIP: 0010:[<ffffffffa4b16bda>] [<ffffffffa4b16bda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9d9be6fbfc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000a4a7ddd0 RCX: ffff9d9e2d620600
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9d9be6fbfca8 R08: ffff9d9b75ccb000 R09: ffffffffa4d866a9
R10: ffff9d9e2d65f160 R11: ffffdbbe0cb6de00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9d9be6fbfc20 R15: 0000000000000001
FS: 00007feada30a700(0000) GS:ffff9d9e2d640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6d15428810 CR3: 00000000091d2000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffa4a7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa4a7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa4bee176>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa4befc3f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa4befd63>] tlb_finish_mmu+0x23/0x30
[<ffffffffa4bf322a>] zap_page_range+0xea/0x130
[<ffffffffa4bed71d>] SyS_madvise+0x49d/0xac0
[<ffffffffa4b56d9b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffa4ae22e9>] ? pick_next_entity+0xa9/0x190
[<ffffffffa5196226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 8a c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 75 a5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 8a c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #171883
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff90116eea
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9007e868
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9007e8db
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff901edf56
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff901efa1f
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff901efb43
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff901f300a
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff900dada8
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff901ed4fd
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff901570ab
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x5b
|
|
11
|
|
vmlinux |
0xffffffff900dadf0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x20
|
|
12
|
|
vmlinux |
0xffffffff90115500
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x80
|
|
13
|
tracesys
|
vmlinux |
0xffffffff90796226
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [metricbeat:1202]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) ext4 mbcache jbd2 snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq iosf_mbi crc32_pclmul snd_seq_device ghash_clmulni_intel ppdev snd_pcm joydev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd virtio_rng snd_timer virtio_balloon snd parport_pc soundcore i2c_piix4 parport pcspkr ip_tables xfs libcrc32c ata_generic pata_acpi virtio_blk virtio_console qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel e1000 floppy
virtio_pci virtio_ring virtio serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 1202 Comm: metricbeat Tainted: G ------------ T 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module_el8.3.0+555+a55c8938 04/01/2014
task: ffff92f1f8bf4200 ti: ffff92f1fa390000 task.ti: ffff92f1fa390000
RIP: 0010:[<ffffffff90116eea>] [<ffffffff90116eea>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff92f1fa393c70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009007e6a0 RCX: ffff92f23fc20348
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff92f1fa393ca8 R08: ffff92f1f43c1c00 R09: ffffffff90386549
R10: ffff92f23fd1f160 R11: fffff5e301253800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff92f1fa393c20 R15: 0000000000000001
FS: 00007f115bf7f700(0000) GS:ffff92f23fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1193067000 CR3: 00000000b8c16000 CR4: 00000000001606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9007e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9007e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff901edf56>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff901efa1f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff901efb43>] tlb_finish_mmu+0x23/0x30
[<ffffffff901f300a>] zap_page_range+0xea/0x130
[<ffffffff900dada8>] ? wake_up_q+0x58/0x80
[<ffffffff901ed4fd>] SyS_madvise+0x49d/0xac0
[<ffffffff901570ab>] ? __seccomp_filter+0x5b/0x300
[<ffffffff900dadf0>] ? wake_up_state+0x20/0x20
[<ffffffff90115500>] ? SyS_futex+0x80/0x190
[<ffffffff90796226>] tracesys+0xa6/0xcc
Code: 48 63 35 fe 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 d5 90 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #171865
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9bb16dda
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9ba7e868
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9ba7e8db
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9bbede26
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9bbef8ef
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9bbefa13
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff9bbf2eda
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff9badb078
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff9bbed3cd
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff9bb56f9b
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x5b
|
|
11
|
tracesys
|
vmlinux |
0xffffffff9c196226
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [auditbeat:4275]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) ext4 mbcache jbd2 snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep iosf_mbi crc32_pclmul snd_seq snd_seq_device ghash_clmulni_intel snd_pcm aesni_intel ppdev lrw joydev parport_pc gf128mul glue_helper parport virtio_balloon snd_timer virtio_rng ablk_helper cryptd pcspkr snd i2c_piix4 soundcore ip_tables xfs libcrc32c ata_generic pata_acpi virtio_console virtio_blk qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul crct10dif_common drm libata e1000 crc32c_intel serio_raw
virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 4275 Comm: auditbeat Tainted: G ------------ T 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.11.0-2.el7 04/01/2014
task: ffff89e27cbe4200 ti: ffff89e32ead0000 task.ti: ffff89e32ead0000
RIP: 0010:[<ffffffff9bb16dda>] [<ffffffff9bb16dda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff89e32ead3c70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 000800009ba7e6a0 RCX: ffff89e33fca0348
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff89e32ead3ca8 R08: ffff89e27cf41c00 R09: ffffffff9bd86419
R10: ffff89e33fd1f160 R11: ffffda29401f3200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff89e32ead3c20 R15: 0000000000000001
FS: 00007f3ced7fb700(0000) GS:ffff89e33fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000ea3000 CR3: 00000000029a6000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9ba7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9ba7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9bbede26>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9bbef8ef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9bbefa13>] tlb_finish_mmu+0x23/0x30
[<ffffffff9bbf2eda>] zap_page_range+0xea/0x130
[<ffffffff9badb078>] ? wake_up_q+0x58/0x80
[<ffffffff9bbed3cd>] SyS_madvise+0x49d/0xac0
[<ffffffff9bb56f9b>] ? __seccomp_filter+0x5b/0x300
[<ffffffff9c196226>] tracesys+0xa6/0xcc
Code: 48 63 35 0e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 75 9c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 dd 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #24259
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa9b16c5a
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa9a7e868
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa9a7e8db
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa9bedd46
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa9bef80f
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa9bef933
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa9bf2dfa
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffa9bed2ed
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffa9b56e4b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x5b
|
|
10
|
tracesys
|
vmlinux |
0xffffffffaa195226
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [java:32848]
Modules linked in: tcp_lp xt_nat veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event xfs libcrc32c crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec aesni_intel lrw gf128mul ac97_bus pcspkr joydev ppdev snd_seq snd_seq_device snd_pcm glue_helper ablk_helper cryptd vmw_balloon snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2
sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi drm nfit scsi_transport_spi mptscsih libata crct10dif_pclmul libnvdimm crct10dif_common crc32c_intel serio_raw mptbase e1000 drm_panel_orientation_quirks fuse
CPU: 13 PID: 32848 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff88d2e2cfe300 ti: ffff88d2d9c38000 task.ti: ffff88d2d9c38000
RIP: 0010:[<ffffffffa9b16c5a>] [<ffffffffa9b16c5a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88d2d9c3bc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000a9a7e6a0 RCX: ffff88d5086207e0
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88d2d9c3bca8 R08: ffff88c97c366000 R09: ffffffffa9d85fd9
R10: ffff88d50895f160 R11: ffffdd9768c19400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff88d2d9c3bc20 R15: 0000000000000001
FS: 00007f8f90ee1700(0000) GS:ffff88d508940000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb6e62cd000 CR3: 0000000712302000 CR4: 00000000003407e0
Call Trace:
[<ffffffffa9a7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa9a7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa9bedd46>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa9bef80f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa9bef933>] tlb_finish_mmu+0x23/0x30
[<ffffffffa9bf2dfa>] zap_page_range+0xea/0x130
[<ffffffffa9bed2ed>] SyS_madvise+0x49d/0xac0
[<ffffffffa9b56e4b>] ? __seccomp_filter+0x5b/0x300
[<ffffffffaa195226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 aa f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1412922
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb931e25a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb9281fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb928201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb93fa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb93fbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb93fbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb93ff2ca
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb92e1118
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb93f96bd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffb92e8699
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xa9
|
|
11
|
|
vmlinux |
0xffffffffb92e918c
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3c
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffb99c539a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 24s! [containerd-shim:2067]
Modules linked in: tcp_lp veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log ext4 mbcache jbd2 snd_seq_midi snd_seq_midi_event dm_mod kvm_intel kvm ppdev irqbypass snd_ens1371 crc32_pclmul snd_rawmidi ghash_clmulni_intel snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr snd_timer snd vmw_vmci soundcore i2c_piix4 parport_pc parport ip_tables
xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi mptscsih e1000 mptbase serio_raw drm_panel_orientation_quirks fuse
CPU: 2 PID: 2067 Comm: containerd-shim Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 03/24/2025
task: ffff8aad5d6f6300 ti: ffff8aad5d70c000 task.ti: ffff8aad5d70c000
RIP: 0010:[<ffffffffb931e25a>] [<ffffffffb931e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8aad5d70fc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000b9281b10 RCX: ffff8aadf6420628
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8aad5d70fca8 R08: ffff8aadf5264800 R09: ffffffffb959c7f9
R10: ffff8aadf649f160 R11: ffffe080411cca00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8aad5d70fc20 R15: ffffffffba40db80
FS: 000000c000210090(0000) GS:ffff8aadf6480000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3a215cefd8 CR3: 000000009e8a8000 CR4: 0000000000760fe0
PKRU: 55555554
Call Trace:
[<ffffffffb9281fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb928201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb93fa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffb93fbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb93fbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffb93ff2ca>] zap_page_range+0xea/0x140
[<ffffffffb92e1118>] ? wake_up_q+0x58/0x90
[<ffffffffb93f96bd>] SyS_madvise+0x49d/0xac0
[<ffffffffb92e8699>] ? pick_next_entity+0xa9/0x190
[<ffffffffb92e918c>] ? set_next_entity+0x3c/0x100
[<ffffffffb99c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 f5 b9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1168369
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa091e07a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa0881f48
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa0881fbb
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa09f9f0e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa09fba4f
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa09fbb83
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa09ff0ca
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa08e0fc8
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa09f949d
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa091c5e0
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x80
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa0fc539a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [containerd-shim:18386]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc cmac arc4 md4 nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock vfat fat snd_hda_codec_generic snd_hda_intel snd_hda_codec sb_edac snd_hda_core iosf_mbi crc32_pclmul snd_hwdep ghash_clmulni_intel snd_seq aesni_intel snd_seq_device lrw gf128mul glue_helper snd_pcm ablk_helper ppdev cryptd snd_timer snd soundcore joydev vmw_balloon pcspkr sg i2c_piix4 vmw_vmci tpm_crb parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix sd_mod drm
crc_t10dif crct10dif_generic libata crct10dif_pclmul crct10dif_common nfit crc32c_intel serio_raw libnvdimm pcnet32 vmw_pvscsi mii drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 18386 Comm: containerd-shim Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.81.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.21100432.B64.2301110304 01/11/2023
task: ffff8b56d3b86300 ti: ffff8b56d3b60000 task.ti: ffff8b56d3b60000
RIP: 0010:[<ffffffffa091e07a>] [<ffffffffa091e07a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b56d3b63c70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000a0881ab0 RCX: ffff8b57756e0600
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8b56d3b63ca8 R08: ffff8b56ba863800 R09: ffffffffa0b9c139
R10: ffff8b577565f160 R11: ffffbc7445a90e00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8b56d3b63c20 R15: ffffffffa1a0db80
FS: 000000c00029c490(0000) GS:ffff8b5775640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0000bb010 CR3: 00000001c9e32000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa0881f48>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa0881fbb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa09f9f0e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa09fba4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa09fbb83>] tlb_finish_mmu+0x23/0x40
[<ffffffffa09ff0ca>] zap_page_range+0xea/0x140
[<ffffffffa08e0fc8>] ? wake_up_q+0x58/0x90
[<ffffffffa09f949d>] SyS_madvise+0x49d/0xac0
[<ffffffffa091c5e0>] ? SyS_futex+0x80/0x190
[<ffffffffa0fc539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 55 a1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1020379
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb7d1e23a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb7c81fa8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb7c8201b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb7dfa07e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb7dfbbff
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb7dfbd33
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb7dff27a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb7ce1118
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb7df966d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb83c539a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [containerd:18006]
Modules linked in: xt_owner xt_REDIRECT nf_nat_redirect xt_multiport ipt_rpfilter iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel rpcsec_gss_krb5 veth auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport dummy nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6_tables xt_mark xt_comment iptable_mangle xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter scini(POE) ip_vs nf_conntrack binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon
sg parport_pc joydev parport pcspkr vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih drm libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc overlay(T)
CPU: 5 PID: 18006 Comm: containerd Tainted: P W OEL ------------ T 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa05ce4a90000 ti: ffffa05cc6328000 task.ti: ffffa05cc6328000
RIP: 0010:[<ffffffffb7d1e23a>] [<ffffffffb7d1e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa05cc632bc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000b7c81b10 RCX: ffffa05dfbc20750
RDX: 0000000000000000 RSI: 000000000000000c RDI: 0000000000000000
RBP: ffffa05cc632bca8 R08: ffffa05e7d4ef000 R09: ffffffffb7f9c7b9
R10: ffffa05efbc9f160 R11: fffff0f300105400 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa05cc632bc20 R15: 0000000000000001
FS: 00007fb8c1e43700(0000) GS:ffffa05efbc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0037f8000 CR3: 00000000b6876000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb7c81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb7c8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb7dfa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffb7dfbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb7dfbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffffb7dff27a>] zap_page_range+0xea/0x140
[<ffffffffb7ce1118>] ? wake_up_q+0x58/0x90
[<ffffffffb7df966d>] SyS_madvise+0x49d/0xac0
[<ffffffffb83c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 8e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 95 b8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #929947
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9051e1ce
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff90481fa8
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9048201b
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff905fa00e
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff905fbb8f
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff905fbcc3
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff905ff20a
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff904e1118
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff905f95fd
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x49d
|
|
10
|
tracesys
|
vmlinux |
0xffffffff90bc562e
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 23s! [s1-perf:21699]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_addrtype br_netfilter bridge stp llc overlay(T) xt_owner ebtable_filter ebtables devlink ip6_tables iptable_filter binfmt_misc vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect sysimgblt fb_sys_fops crc_t10dif ttm crct10dif_generic ata_piix drm nfit libata crct10dif_pclmul crct10dif_common crc32c_intel libnvdimm serio_raw
vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 21699 Comm: s1-perf Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.92.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff99e0d7b49080 ti: ffff99e0f8a18000 task.ti: ffff99e0f8a18000
RIP: 0010:[<ffffffff9051e1ce>] [<ffffffff9051e1ce>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff99e0f8a1bc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0008000090481b10 RCX: ffff99f0ffc206f0
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff99e0f8a1bca8 R08: ffff99e21955c000 R09: ffffffff9079c669
R10: ffff99f0ffddf160 R11: fffffc74801c8400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff99e0f8a1bc20 R15: 0000000000000001
FS: 00007ff55f7fe700(0000) GS:ffff99f0ffdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbb10448d70 CR3: 000000023f0a4000 CR4: 00000000001607e0
Call Trace:
[<ffffffff90481fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9048201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff905fa00e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff905fbb8f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff905fbcc3>] tlb_finish_mmu+0x23/0x40
[<ffffffff905ff20a>] zap_page_range+0xea/0x140
[<ffffffff904e1118>] ? wake_up_q+0x58/0x90
[<ffffffff905f95fd>] SyS_madvise+0x49d/0xac0
[<ffffffff90bc562e>] tracesys+0xa6/0xcc
Code: 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 15 91 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 4d 18 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #924808
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8e11722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8e07e258
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8e07e2cb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8e1ee666
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8e1f012f
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8e1f0253
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8e1f371a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff8e0daef8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x58
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 71s! [frpc:83226]
Modules linked in: xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm snd_ens1371 irqbypass snd_rawmidi crc32_pclmul snd_ac97_codec ac97_bus ghash_clmulni_intel snd_seq aesni_intel lrw snd_seq_device gf128mul snd_pcm snd_timer ppdev glue_helper ablk_helper cryptd snd vmw_balloon soundcore joydev pcspkr sg parport_pc
parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops mptspi ttm crct10dif_pclmul crct10dif_common scsi_transport_spi ata_piix crc32c_intel mptscsih drm nfit libata mptbase serio_raw libnvdimm e1000 drm_panel_orientation_quirks fuse
CPU: 6 PID: 83226 Comm: frpc Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a820ad14200 ti: ffff8a81c15c0000 task.ti: ffff8a81c15c0000
RIP: 0010:[<ffffffff8e11722a>] [<ffffffff8e11722a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff8a81c15c3c70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800008e07ddd0 RCX: ffff8a822d6a06c8
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8a81c15c3ca8 R08: ffff8a7f75ccc400 R09: ffffffff8e3869d9
R10: ffff8a822d79f160 R11: ffffd8a589457c00 R12: 0000000000000282
R13: 0000000000000000 R14: ffff8a81c15c3be0 R15: 0000000000000282
FS: 0000000000000000(0000) GS:ffff8a822d780000(0063) knlGS:0000000009442294
CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033
CR2: 000016bd7ca74000 CR3: 0000000073f1a000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8e07e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8e07e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8e1ee666>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8e1f012f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8e1f0253>] tlb_finish_mmu+0x23/0x30
[<ffffffff8e1f371a>] zap_page_range+0xea/0x130
[<ffffffff8e0daef8>] ? wake_up_q+0x58/0x80
Complete report #499706
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff827170de
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8267e258
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8267e2cb
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff827ee676
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff827f013f
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff827f0263
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff827f372a
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff826dada8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff827edc1d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff826dadf0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff82d95f92
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [zabbix_agent2:54908]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag nf_conntrack_netlink xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ipmi_devintf ipmi_msghandler nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod squashfs loop iosf_mbi crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon joydev sg pcspkr vmw_vmci i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi ata_piix mptscsih crct10dif_pclmul crct10dif_common nfit drm mptbase libata crc32c_intel libnvdimm serio_raw e1000 drm_panel_orientation_quirks fuse
CPU: 2 PID: 54908 Comm: zabbix_agent2 Not tainted 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff92aaecf50000 ti: ffff92a7759bc000 task.ti: ffff92a7759bc000
RIP: 0010:[<ffffffff827170de>] [<ffffffff827170de>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff92a7759bfc70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000800008267ddd0 RCX: ffff92aaed6e0628
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff92a7759bfca8 R08: ffff92a835ccb400 R09: ffffffff82986bf9
R10: ffff92aaed69f160 R11: ffffe6df1023a400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff92a7759bfc20 R15: 0000000000000001
FS: 00007f67467fc700(0000) GS:ffff92aaed680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001089987 CR3: 00000003d23f0000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8267e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8267e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff827ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff827f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff827f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffff827f372a>] zap_page_range+0xea/0x130
[<ffffffff826dada8>] ? wake_up_q+0x58/0x80
[<ffffffff827edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffff826dadf0>] ? wake_up_state+0x20/0x20
[<ffffffff82d95f92>] system_call_fastpath+0x25/0x2a
Code: 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 35 83 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #281226
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbcb16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbca7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbca7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffbcbedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffbcbef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbcbef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffbcbf2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffbcadaaa8
Build id: 3.10.0-1160.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffbcbed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffbd193f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [n9e-agent:129795]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi sb_edac snd_ac97_codec coretemp ac97_bus snd_seq snd_seq_device iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw snd_pcm gf128mul glue_helper ablk_helper ppdev snd_timer cryptd
vmw_balloon snd soundcore pcspkr joydev vmw_vmci sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi scsi_transport_spi e1000 mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 129795 Comm: n9e-agent Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9af4f4fcb180 ti: ffff9b0184f74000 task.ti: ffff9b0184f74000
RIP: 0010:[<ffffffffbcb16c0a>] [<ffffffffbcb16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9b0184f77c70 EFLAGS: 00000202
RAX: 000000000000000f RBX: 00080000bca7e6a0 RCX: ffff9b037d9e0678
RDX: 000000000000000f RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9b0184f77ca8 R08: ffff9af4e4345000 R09: ffffffffbcd85f99
R10: ffff9b037d71f160 R11: ffffda7dc5f7d400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9b0184f77c20 R15: 0000000000000001
FS: 00007f09cb7fe700(0000) GS:ffff9b037d700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6081c90158 CR3: 00000001d5eee000 CR4: 00000000001607e0
Call Trace:
[<ffffffffbca7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbca7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffbcbedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffbcbef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffbcbef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffbcbf2dba>] zap_page_range+0xea/0x130
[<ffffffffbcadaaa8>] ? wake_up_q+0x58/0x80
[<ffffffffbcbed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffbd193f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 bd f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #162419
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb6b150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb6a7d398
Build id: 3.10.0-1062.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb6a7d40b
Build id: 3.10.0-1062.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb6bea166
Build id: 3.10.0-1062.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb6bebc2f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb6bebd53
Build id: 3.10.0-1062.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb6beea5a
Build id: 3.10.0-1062.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb6ada068
Build id: 3.10.0-1062.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb6be976d
Build id: 3.10.0-1062.el7.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb718bede
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 43s! [prometheus:78719]
Modules linked in: tcp_lp fuse xt_multiport ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs dummy iptable_raw xt_CT ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc nfit libnvdimm nls_utf8 isofs ppdev coretemp vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel
aesni_intel lrw gf128mul sg pcspkr glue_helper ablk_helper cryptd joydev parport_pc parport vmw_vmci pcc_cpufreq i2c_piix4 br_netfilter bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi drm scsi_transport_spi ata_piix mptscsih crct10dif_pclmul crct10dif_common crc32c_intel libata mptbase serio_raw vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 78719 Comm: prometheus Tainted: G L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9495d1f48000 ti: ffff948bfcf24000 task.ti: ffff948bfcf24000
RIP: 0010:[<ffffffffb6b150ca>] [<ffffffffb6b150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff948bfcf27c70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000b6a7d1d0 RCX: ffff94983d6dfc10
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff948bfcf27ca8 R08: ffff9489bfdd7000 R09: ffffffffb6d7f629
R10: ffffd55535e23fa0 R11: ffffe954b875ec00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff948bfcf27c20 R15: 0000000000000001
FS: 000000c00f694410(0000) GS:ffff94983d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0223a7000 CR3: 0000000e76a3a000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb6a7d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb6a7d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb6bea166>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb6bebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb6bebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffffb6beea5a>] zap_page_range+0xea/0x130
[<ffffffffb6ada068>] ? wake_up_q+0x58/0x80
[<ffffffffb6be976d>] SyS_madvise+0x49d/0xac0
[<ffffffffb718bede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 74 b7 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1249396
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb4116c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb407e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb407e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb41edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb41ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb41ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb41f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb40daaa8
Build id: 3.10.0-1160.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffffb4112040
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffb41ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb4793f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [JS Helper:97005]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec aesni_intel ac97_bus snd_seq vmw_balloon lrw gf128mul glue_helper ablk_helper snd_seq_device cryptd snd_pcm snd_timer joydev snd
sg pcspkr soundcore vmw_vmci i2c_piix4 parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel ata_piix nfit mptspi scsi_transport_spi libata mptscsih libnvdimm e1000 mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 97005 Comm: JS Helper Kdump: loaded Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa01f25b54200 ti: ffffa01e879ec000 task.ti: ffffa01e879ec000
RIP: 0010:[<ffffffffb4116c0a>] [<ffffffffb4116c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa01e879efc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000b407e6a0 RCX: ffffa01f7d6206c8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa01e879efca8 R08: ffffa01ebe34c400 R09: ffffffffb4385f99
R10: ffffa01f7d79f160 R11: ffffea6142d2e400 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa01e879efc20 R15: 0000000000000001
FS: 00007fda39406700(0000) GS:ffffa01f7d780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fda294d2ff0 CR3: 000000003d08e000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb407e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb407e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb41edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb41ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb41ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffb41f2dba>] zap_page_range+0xea/0x130
[<ffffffffb40daaa8>] ? wake_up_q+0x58/0x80
[<ffffffffb4112040>] ? futex_wake+0x90/0x180
[<ffffffffb41ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffb4793f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 d5 b4 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1203125
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbc7170da
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbc67e258
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbc67e2cb
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffbc7ee676
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffbc7f013f
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbc7f0263
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffbc7f372a
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffbc712670
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x90
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffbc7edc1d
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffbcd95f92
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [MallocTrimThrea:122844]
Modules linked in: veeamsnap(OE) tcp_lp tun bridge stp llc ebtable_filter ebtables xt_comment devlink ip6table_filter ip6_tables ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter iptable_raw iptable_security xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi crc32_pclmul ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm lrw snd_timer gf128mul snd glue_helper ablk_helper cryptd soundcore ppdev vmw_balloon sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix libnvdimm drm crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi mptscsih e1000 libata serio_raw mptbase drm_panel_orientation_quirks fuse
CPU: 1 PID: 122844 Comm: MallocTrimThrea Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.41.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9e707dd46300 ti: ffff9e71a565c000 task.ti: ffff9e71a565c000
RIP: 0010:[<ffffffffbc7170da>] [<ffffffffbc7170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9e71a565fc70 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000bc67ddd0 RCX: ffff9e71b76e0600
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9e71a565fca8 R08: ffff9e717b88b000 R09: ffffffffbc986bd9
R10: ffff9e71b765f160 R11: ffffd93806d8f400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9e71a565fc20 R15: 0000000000000001
FS: 00007f0f88be4700(0000) GS:ffff9e71b7640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000071adef CR3: 00000001b55f4000 CR4: 00000000003607e0
Call Trace:
[<ffffffffbc67e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbc67e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffbc7ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffbc7f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffbc7f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffbc7f372a>] zap_page_range+0xea/0x130
[<ffffffffbc712670>] ? futex_wake+0x90/0x180
[<ffffffffbc7edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffbcd95f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 35 bd f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1178591
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa1d1e25e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa1c81fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa1c8201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa1dfa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa1dfbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa1dfbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa1dff2ca
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa1ce1118
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffffa1d196b0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffa1df96bd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x49d
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa23c562e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 25s! [s1-agent:1343]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter cts nfsv3 nfs_acl cmac arc4 md4 nls_utf8 rpcsec_gss_krb5 cifs nfsv4 nfs ccm dns_resolver lockd grace fscache vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd joydev pcspkr sg parport_pc parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt ahci
fb_sys_fops ttm libahci drm sd_mod ata_piix nfit crc_t10dif crct10dif_generic libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 1343 Comm: s1-agent Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9e2928ad5280 ti: ffff9e264ceb8000 task.ti: ffff9e264ceb8000
RIP: 0010:[<ffffffffa1d1e25e>] [<ffffffffa1d1e25e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9e264cebbc70 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00080000a1c81b10 RCX: ffff9e297fd60650
RDX: 0000000000000005 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff9e264cebbca8 R08: ffff9e26b63c1000 R09: ffffffffa1f9c7f9
R10: ffff9e297fcdf160 R11: ffffe254d0ba2400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9e264cebbc20 R15: 0000000000000001
FS: 00007fa7a61b6700(0000) GS:ffff9e297fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055e3c5341620 CR3: 00000002a7278000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffa1c81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa1c8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa1dfa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa1dfbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa1dfbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffa1dff2ca>] zap_page_range+0xea/0x140
[<ffffffffa1ce1118>] ? wake_up_q+0x58/0x90
[<ffffffffa1d196b0>] ? futex_wake+0x90/0x190
[<ffffffffa1df96bd>] SyS_madvise+0x49d/0xac0
[<ffffffffa23c562e>] tracesys+0xa6/0xcc
Code: 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 95 a2 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #1005564
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffacd1e25a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffacc81fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffacc8201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffacdfa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffacdfbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffacdfbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffacdff2ca
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffacd196b0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x90
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffacdf96bd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffacce5d58
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x78
|
|
11
|
|
vmlinux |
0xffffffffaccec6ab
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x8b
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffad3c539a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [bwa:15744]
Modules linked in: tcp_lp nls_utf8 isofs xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 devlink tun ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc ext4 mbcache jbd2 nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr virtio_balloon i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi
virtio_net net_failover virtio_scsi failover bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci ata_piix libahci drm crct10dif_pclmul virtio_pci crct10dif_common crc32c_intel libata virtio_ring serio_raw drm_panel_orientation_quirks virtio floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 23 PID: 15744 Comm: bwa Tainted: G L ------------ 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
task: ffffa0f5b400b180 ti: ffffa0f58ffa0000 task.ti: ffffa0f58ffa0000
RIP: 0010:[<ffffffffacd1e25a>] [<ffffffffacd1e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa0f58ffa3c70 EFLAGS: 00000202
RAX: 000000000000007f RBX: 00080000acc81b10 RCX: ffffa0f5bb5e0970
RDX: 000000000000007f RSI: 0000000000000080 RDI: 0000000000000040
RBP: ffffa0f58ffa3ca8 R08: ffffa01b22900808 R09: ffffffffacf9c7f9
R10: ffffa0f5b9bdf160 R11: ffffda6f3652ee00 R12: 0000000000000282
R13: 0000000000000018 R14: ffffa0f58ffa3be0 R15: 0000000000000282
FS: 00007f9874fe1700(0000) GS:ffffa0f5b9bc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f83a42d6000 CR3: 000000c7e1406000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffacc81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffacc8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffacdfa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffacdfbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffacdfbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffacdff2ca>] zap_page_range+0xea/0x140
[<ffffffffacd196b0>] ? futex_wake+0x90/0x190
[<ffffffffacdf96bd>] SyS_madvise+0x49d/0xac0
[<ffffffffacce5d58>] ? __enqueue_entity+0x78/0x80
[<ffffffffaccec6ab>] ? put_prev_entity+0x8b/0x400
[<ffffffffad3c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 95 ad f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1003448
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb191e25e
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb1881fa8
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb188201b
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb19fa0ce
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb19fbc4f
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb19fbd83
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb19ff2ca
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb18e1118
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffffb19196b0
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffb19f96bd
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffffb18e8699
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xa9
|
|
12
|
|
vmlinux |
0xffffffffb18e918c
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x3c
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffffb1fc539a
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [containerd-shim:4762]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_addrtype br_netfilter bridge stp llc iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper cryptd vmw_balloon sg pcspkr joydev vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm nfit sd_mod libahci ata_piix crc_t10dif crct10dif_generic libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash
dm_log dm_mod fuse
CPU: 3 PID: 4762 Comm: containerd-shim Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9d0fe4c09080 ti: ffff9d13eef80000 task.ti: ffff9d13eef80000
RIP: 0010:[<ffffffffb191e25e>] [<ffffffffb191e25e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9d13eef83c70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000b1881b10 RCX: ffff9d147fca0650
RDX: 0000000000000002 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff9d13eef83ca8 R08: ffff9d10b2bf6000 R09: ffffffffb1b9c7f9
R10: ffff9d147fcdf160 R11: ffffe2b6d21de200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9d13eef83c20 R15: 0000000000000001
FS: 000000c000200090(0000) GS:ffff9d147fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000156000 CR3: 00000004aa8c0000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb1881fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb188201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb19fa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffb19fbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb19fbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffb19ff2ca>] zap_page_range+0xea/0x140
[<ffffffffb18e1118>] ? wake_up_q+0x58/0x90
[<ffffffffb19196b0>] ? futex_wake+0x90/0x190
[<ffffffffb19f96bd>] SyS_madvise+0x49d/0xac0
[<ffffffffb18e8699>] ? pick_next_entity+0xa9/0x190
[<ffffffffb18e918c>] ? set_next_entity+0x3c/0x100
[<ffffffffb1fc539a>] system_call_fastpath+0x25/0x2a
Code: 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 55 b2 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #976538
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9311e25a
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff93081fa8
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9308201b
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff931fa0ce
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff931fbc4f
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff931fbd83
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff931ff2ca
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff930e1118
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffff931196b0
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffff931f96bd
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x49d
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff937c539a
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [mariadbd:25492]
Modules linked in: veeamsnap(OE) binfmt_misc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache vfat fat sg pcspkr joydev virtio_balloon i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi virtio_net net_failover virtio_console virtio_scsi failover bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci virtio_ring serio_raw virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 25492 Comm: mariadbd Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 3.20230228-4~bpo11+3 02/15/2024
task: ffff977ca7bec200 ti: ffff977ad6dd4000 task.ti: ffff977ad6dd4000
RIP: 0010:[<ffffffff9311e25a>] [<ffffffff9311e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff977ad6dd7c70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0008000093081b10 RCX: ffff977ccfc60428
RDX: 0000000000000001 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff977ad6dd7ca8 R08: ffff9777ef221400 R09: ffffffff9339c7f9
R10: ffff977ccfc9f160 R11: ffffde0c80f40600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff977ad6dd7c20 R15: 0000000000000001
FS: 00007f97d93f4700(0000) GS:ffff977ccfc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056367dffcbb8 CR3: 00000005c1a74000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff93081fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9308201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff931fa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff931fbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff931fbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffff931ff2ca>] zap_page_range+0xea/0x140
[<ffffffff930e1118>] ? wake_up_q+0x58/0x90
[<ffffffff931196b0>] ? futex_wake+0x90/0x190
[<ffffffff931f96bd>] SyS_madvise+0x49d/0xac0
[<ffffffff937c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 d5 93 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #924807
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8e11722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8e07e258
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8e07e2cb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8e1ee666
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8e1f012f
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8e1f0253
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8e1f371a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff8e0daef8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x58
|
|
9
|
|
vmlinux |
0xffffffff8e1127c0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x90
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffff8e1edc0d
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x49d
|
|
11
|
|
vmlinux |
0xffffffff8e0e2429
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa9
|
|
12
|
ia32_do_call
|
vmlinux |
0xffffffff8e79e68e
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4c
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 68s! [frpc:83226]
Modules linked in: xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm snd_ens1371 irqbypass snd_rawmidi crc32_pclmul snd_ac97_codec ac97_bus ghash_clmulni_intel snd_seq aesni_intel lrw snd_seq_device gf128mul snd_pcm snd_timer ppdev glue_helper ablk_helper cryptd snd vmw_balloon soundcore joydev pcspkr sg parport_pc
parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops mptspi ttm crct10dif_pclmul crct10dif_common scsi_transport_spi ata_piix crc32c_intel mptscsih drm nfit libata mptbase serio_raw libnvdimm e1000 drm_panel_orientation_quirks fuse
CPU: 6 PID: 83226 Comm: frpc Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a820ad14200 ti: ffff8a81c15c0000 task.ti: ffff8a81c15c0000
RIP: 0010:[<ffffffff8e11722a>] [<ffffffff8e11722a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff8a81c15c3c70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800008e07ddd0 RCX: ffff8a822d6a06c8
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8a81c15c3ca8 R08: ffff8a7f75ccc400 R09: ffffffff8e3869d9
R10: ffff8a822d79f160 R11: ffffd8a589457c00 R12: 0000000000000282
R13: 0000000000000000 R14: ffff8a81c15c3be0 R15: 0000000000000282
FS: 0000000000000000(0000) GS:ffff8a822d780000(0063) knlGS:0000000009442294
CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033
CR2: 000016bd7ca74000 CR3: 0000000073f1a000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8e07e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8e07e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8e1ee666>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8e1f012f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8e1f0253>] tlb_finish_mmu+0x23/0x30
[<ffffffff8e1f371a>] zap_page_range+0xea/0x130
[<ffffffff8e0daef8>] ? wake_up_q+0x58/0x80
[<ffffffff8e1127c0>] ? futex_wake+0x90/0x180
[<ffffffff8e1edc0d>] SyS_madvise+0x49d/0xac0
[<ffffffff8e0e2429>] ? pick_next_entity+0xa9/0x190
[<ffffffff8e79e68e>] ia32_do_call+0x4c/0x4c
Code: 48 63 35 7e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 d5 8e f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 87 c4 00 eb b7 0f b6 4d cc 4c 89
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [PLANE:33:plane-0] flip_done timed out
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [PLANE:33:plane-0] flip_done timed out
Complete report #586584
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8c87e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
2
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8c87e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
3
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8c9edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
4
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8c9ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8c9ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
6
|
zap_page_range
|
vmlinux |
0xffffffff8c9f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
7
|
|
vmlinux |
0xffffffff8c912040
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffff8c9ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff8cf93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 100s! [vmtoolsd:683]
FS: 00007faa599a0700(0000) GS:ffff99a779640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fce5e86f150 CR3: 000000004671e000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8c87e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8c87e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8c9edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8c9ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8c9ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff8c9f2dba>] zap_page_range+0xea/0x130
[<ffffffff8c912040>] ? futex_wake+0x90/0x180
[<ffffffff8c9ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffff8cf93f92>] system_call_fastpath+0x25/0x2a
Complete report #581873
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa8b16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa8a7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa8a7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa8bedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa8bef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa8bef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa8bf2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa8b12040
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa8bed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffa9193f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [server.rb:286:98299]
Modules linked in: tcp_lp veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi ppdev snd_ac97_codec ac97_bus aesni_intel
snd_seq lrw gf128mul snd_seq_device glue_helper ablk_helper vmw_balloon snd_pcm cryptd snd_timer joydev pcspkr snd soundcore i2c_piix4 sg vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt ata_generic pata_acpi crct10dif_pclmul mptspi fb_sys_fops crct10dif_common ttm scsi_transport_spi drm ata_piix e1000 mptscsih crc32c_intel serio_raw mptbase nfit libnvdimm libata drm_panel_orientation_quirks fuse
CPU: 0 PID: 98299 Comm: server.rb:286 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8895ba961080 ti: ffff8896c697c000 task.ti: ffff8896c697c000
RIP: 0010:[<ffffffffa8b16c0a>] [<ffffffffa8b16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8896c697fc70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000c0000a8a7e6a0 RCX: ffff8897af29fc10
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8896c697fca8 R08: ffff8896ffdd7000 R09: ffffffffa8d85f99
R10: ffff8897af21f160 R11: ffffe442c1312200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8896c697fc20 R15: ffffffffa8a2b621
FS: 00007f1298177700(0000) GS:ffff8897af200000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0e39bac000 CR3: 000000010f7a2000 CR4: 00000000003607f0
Call Trace:
[<ffffffffa8a7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa8a7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa8bedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa8bef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa8bef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffa8bf2dba>] zap_page_range+0xea/0x130
[<ffffffffa8b12040>] ? futex_wake+0x90/0x180
[<ffffffffa8bed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffa9193f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 a9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1375738
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa651e2b8
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x208
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa6481fa8
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa648201b
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa65fa12e
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa65fbcaf
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa65fbde3
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa65ff32a
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa660931c
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x39c
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa65f971d
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x49d
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffa6bc539a
Build id: 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [process reaper:46982]
Modules linked in: sunrpc vfat fat ppdev nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev sg pcspkr parport_pc parport i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix xen_netfront xen_blkfront libata crct10dif_pclmul crct10dif_common crc32c_intel floppy serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod tun fuse
CPU: 7 PID: 46982 Comm: process reaper Kdump: loaded Tainted: G L ------------ 3.10.0-1160.119.1.el7.tuxcare.els19.x86_64 #1
Hardware name: Xen HVM domU, BIOS 4.13 09/09/2025
task: ffff9d22045a1080 ti: ffff9d2323914000 task.ti: ffff9d2323914000
RIP: 0010:[<ffffffffa651e2b8>] [<ffffffffa651e2b8>] smp_call_function_many+0x208/0x270
RSP: 0018:ffff9d2323917c70 EFLAGS: 00000202
RAX: 000000000000000d RBX: ffff9d2323917c10 RCX: ffff9d29ee5605b8
RDX: 000000000000000d RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff9d2323917ca8 R08: ffff9d236c79a800 R09: 0000000000000001
R10: ffff9d236c79a800 R11: 00007f07dad19000 R12: ffff9d2323917c00
R13: ffffffffa6877ff6 R14: ffff9d2323917be0 R15: ffffffffa655b78e
FS: 00007f07dad4e700(0000) GS:ffff9d29ee3c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc0b5c204b1 CR3: 000000018c920000 CR4: 00000000007606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa6481fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa648201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa65fa12e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa65fbcaf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa65fbde3>] tlb_finish_mmu+0x23/0x40
[<ffffffffa65ff32a>] zap_page_range+0xea/0x140
[<ffffffffa660931c>] ? do_mmap+0x39c/0x590
[<ffffffffa65f971d>] SyS_madvise+0x49d/0xac0
[<ffffffffa6bc539a>] system_call_fastpath+0x25/0x2a
Code: 29 00 48 63 35 0e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 15 a7 f6 41 20 01 74 cd 0f 1f 44 00 00 <f3> 90 f6 41 20 01 75 f8 48 63 35 dd 17 c4 00 eb b7 0f b6 4d cc
Complete report #1363664
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb311e07a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb3081f48
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb3081fbb
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb31f9f0e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb31fba4f
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb31fbb83
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb31ff0ca
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb31194d0
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x90
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb31f949d
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffb31ea203
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xf3
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb37c539a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [java:55853]
Modules linked in: nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq sb_edac snd_seq_device snd_pcm iosf_mbi snd_timer crc32_pclmul ghash_clmulni_intel aesni_intel lrw snd gf128mul glue_helper ppdev ablk_helper cryptd soundcore vmw_balloon sg pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix nfit drm libnvdimm libata crc32c_intel e1000 mptspi serio_raw scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 55853 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.81.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff90c691722100 ti: ffff90c8e7028000 task.ti: ffff90c8e7028000
RIP: 0010:[<ffffffffb311e07a>] [<ffffffffb311e07a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff90c8e702bc70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000b3081ab0 RCX: ffff90c8e9660650
RDX: 0000000000000001 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff90c8e702bca8 R08: ffff90c53255c000 R09: ffffffffb339c139
R10: ffff90c8e96df160 R11: fffffa350b455400 R12: 0000000000000282
R13: 0000000000000018 R14: ffff90c8e702bbe0 R15: 0000000000000282
FS: 00007f146bd17700(0000) GS:ffff90c8e96c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6b77484c38 CR3: 00000004f55de000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb3081f48>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb3081fbb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb31f9f0e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffb31fba4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb31fbb83>] tlb_finish_mmu+0x23/0x40
[<ffffffffb31ff0ca>] zap_page_range+0xea/0x140
[<ffffffffb31194d0>] ? futex_wake+0x90/0x190
[<ffffffffb31f949d>] SyS_madvise+0x49d/0xac0
[<ffffffffb31ea203>] ? vm_mmap_pgoff+0xf3/0x120
[<ffffffffb37c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 d5 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1302727
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa5516c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa547e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa547e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa55edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa55ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa55ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa55f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa54daaa8
Build id: 3.10.0-1160.el7.x86_64
|
0x58
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa55ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa55de1bb
Build id: 3.10.0-1160.el7.x86_64
|
0xeb
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa5b93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [Metrics-HealthC:23553]
Modules linked in: xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) nfit libnvdimm snd_hda_codec_generic snd_hda_intel snd_hda_codec iosf_mbi crc32_pclmul ghash_clmulni_intel snd_hda_core snd_hwdep aesni_intel snd_seq lrw gf128mul snd_seq_device snd_pcm ppdev glue_helper ablk_helper cryptd snd_timer i2c_piix4 snd soundcore joydev parport_pc virtio_balloon parport pcspkr ip_tables xfs libcrc32c qxl drm_kms_helper ata_generic pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm virtio_console virtio_blk drm ata_piix 8139too libata crct10dif_pclmul crct10dif_common floppy crc32c_intel virtio_pci
serio_raw virtio_ring 8139cp virtio drm_panel_orientation_quirks mii dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 23553 Comm: Metrics-HealthC Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff8dd8f6cdd280 ti: ffff8dd661b74000 task.ti: ffff8dd661b74000
RIP: 0010:[<ffffffffa5516c0a>] [<ffffffffa5516c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8dd661b77c70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000a547e6a0 RCX: ffff8dda7fc60398
RDX: 0000000000000001 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff8dd661b77ca8 R08: ffff8dd69cbb9c00 R09: ffffffffa5785f99
R10: ffff8dda7fd1f160 R11: fffff87190a1d600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8dd661b77c20 R15: 0000000000000001
FS: 00007f8cfbdfd640(0000) GS:ffff8dda7fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055dd0e4e0330 CR3: 0000000449bc2000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa547e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa547e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa55edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa55ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa55ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffa55f2dba>] zap_page_range+0xea/0x130
[<ffffffffa54daaa8>] ? wake_up_q+0x58/0x80
[<ffffffffa55ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffffa55de1bb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffffa5b93f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 15 a6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #913573
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff90516c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9047e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9047e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff905edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff905ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff905ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff905f2dba
Build id: 3.10.0-1160.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffff905ed2ad
Build id: 3.10.0-1160.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffff905de1bb
Build id: 3.10.0-1160.el7.x86_64
|
0xeb
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff90b93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [java:39553]
Modules linked in: vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper cryptd vmw_balloon pcspkr sg joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix mptspi libata scsi_transport_spi mptscsih serio_raw nfit e1000 mptbase libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 39553 Comm: java Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff97d75a6eb180 ti: ffff97d42e1c0000 task.ti: ffff97d42e1c0000
RIP: 0010:[<ffffffff90516c0a>] [<ffffffff90516c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff97d42e1c3c70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009047e6a0 RCX: ffff97d8a3620600
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff97d42e1c3ca8 R08: ffff97d37ef5b800 R09: ffffffff90785f99
R10: ffff97d8a365f160 R11: ffffee74157b5600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff97d42e1c3c20 R15: 0000000000000001
FS: 00007f07744d0700(0000) GS:ffff97d8a3640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000558b231a36f8 CR3: 0000000035f8c000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff9047e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9047e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff905edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff905ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff905ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff905f2dba>] zap_page_range+0xea/0x130
[<ffffffff905ed2ad>] SyS_madvise+0x49d/0xac0
[<ffffffff905de1bb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffff90b93f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 15 91 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #807602
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa31170d8
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x208
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa307e258
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa307e2cb
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa31ee676
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa31f013f
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa31f0263
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa31f372a
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa31fd36c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x39c
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa31edc1d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa31dedab
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xeb
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa3795f92
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ssh-agent: stdo:23537]
Modules linked in: xt_comment xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink tun vmw_vsock_vmci_transport vsock sunrpc vfat fat sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci i2c_piix4 parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common mptspi libata crc32c_intel scsi_transport_spi mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ipmi_msghandler]
CPU: 4 PID: 23537 Comm: ssh-agent: stdo Tainted: G W L ------------ 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.0.B64.1704120155 04/12/2017
task: ffff9d141f2fb180 ti: ffff9d0ea5fbc000 task.ti: ffff9d0ea5fbc000
RIP: 0010:[<ffffffffa31170d8>] [<ffffffffa31170d8>] smp_call_function_many+0x208/0x270
RSP: 0018:ffff9d0ea5fbfc70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000a307ddd0 RCX: ffff9d157fca0678
RDX: 0000000000000002 RSI: 0000000000000007 RDI: 0000000000000000
RBP: ffff9d0ea5fbfca8 R08: ffff9d0ca5624c00 R09: ffffffffa3386bf9
R10: ffff9d157fd1f160 R11: ffffed871bb1a400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9d0ea5fbfc20 R15: 0000000000000001
FS: 00007ff4d81c3700(0000) GS:ffff9d157fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff4d81c0908 CR3: 000000093e804000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa307e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa307e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa31ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa31f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa31f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffa31f372a>] zap_page_range+0xea/0x130
[<ffffffffa31fd36c>] ? do_mmap+0x39c/0x590
[<ffffffffa31edc1d>] SyS_madvise+0x49d/0xac0
[<ffffffffa31dedab>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffffa3795f92>] system_call_fastpath+0x25/0x2a
Code: 28 00 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 d5 a3 f6 41 20 01 74 cd 0f 1f 44 00 00 <f3> 90 f6 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc
Complete report #589266
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa111718a
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa107e258
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa107e2cb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa11ee6a6
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa11f016f
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa11f0293
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa11f375a
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa11fd34c
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x39c
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffa11edc4d
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffa11deddb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xeb
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa179a226
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [RemoteInvocatio:30691]
Modules linked in: binfmt_misc unix_diag af_packet_diag netlink_diag udp_diag tcp_diag inet_diag uinput nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink cmac arc4 md4 nls_utf8 cifs ccm dns_resolver overlay(T) dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul sg glue_helper joydev pcspkr ablk_helper cryptd i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_net net_failover virtio_console
virtio_scsi failover qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy fuse
CPU: 0 PID: 30691 Comm: RemoteInvocatio Tainted: G L ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
task: ffff978a267f4200 ti: ffff978ad644c000 task.ti: ffff978ad644c000
RIP: 0010:[<ffffffffa111718a>] [<ffffffffa111718a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff978ad644fc70 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 00080000a107ddd0 RCX: ffff978dbfd1fc10
RDX: 0000000000000004 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff978ad644fca8 R08: ffff978affd4f800 R09: ffffffffa13869f9
R10: ffff978dbfc1f160 R11: fffff42684b13500 R12: 0000000000000282
R13: 0000000000000282 R14: ffff978ad644fc20 R15: 0000000000000001
FS: 00007fbf078f8700(0000) GS:ffff978dbfc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbf364a0768 CR3: 000000009fede000 CR4: 0000000000160ff0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa107e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa107e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa11ee6a6>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa11f016f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa11f0293>] tlb_finish_mmu+0x23/0x30
[<ffffffffa11f375a>] zap_page_range+0xea/0x130
[<ffffffffa11fd34c>] ? do_mmap+0x39c/0x590
[<ffffffffa11edc4d>] SyS_madvise+0x49d/0xac0
[<ffffffffa11deddb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffffa179a226>] tracesys+0xa6/0xcc
Code: 48 63 35 be 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 d5 a1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 8d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #243730
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb4b16a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb4a7e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb4a7e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb4bec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb4bedb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb4bedc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb4bf098a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffb4bfad78
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x378
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffffb4beb69d
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffffb4bddffb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xeb
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb518dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [java:5257]
Modules linked in: cts rpcsec_gss_krb5 veth nfsd nfsv3 nfs_acl nfs lockd grace fscache openafs(POE) xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) nls_utf8 isofs vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr i2c_piix4 parport_pc vmw_vmci parport binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit crc32c_intel serio_raw libnvdimm vmxnet3 ata_generic drm_panel_orientation_quirks pata_acpi sd_mod
crc_t10dif crct10dif_generic sr_mod crct10dif_pclmul crct10dif_common cdrom mptspi scsi_transport_spi mptscsih mptbase ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 5257 Comm: java Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8c7ec2a2b150 ti: ffff8c76edfc8000 task.ti: ffff8c76edfc8000
RIP: 0010:[<ffffffffb4b16a7a>] [<ffffffffb4b16a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8c76edfcbc70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000b4a7e2b0 RCX: ffff8c7effc60678
RDX: 0000000000000001 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff8c76edfcbca8 R08: ffff8c742d600800 R09: ffffffffb4d81839
R10: ffff8c7effd1f160 R11: fffffb7a8251ac00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8c76edfcbc20 R15: 0000000000000001
FS: 00007f8f1361f700(0000) GS:ffff8c7effd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8f13609eb8 CR3: 000000003653e000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb4a7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb4a7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb4bec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb4bedb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb4bedc83>] tlb_finish_mmu+0x23/0x30
[<ffffffffb4bf098a>] zap_page_range+0xea/0x130
[<ffffffffb4bfad78>] ? do_mmap+0x378/0x530
[<ffffffffb4beb69d>] SyS_madvise+0x49d/0xac0
[<ffffffffb4bddffb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffffb518dede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 74 b5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #75681
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8bf16e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8be7e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8be7e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8bfede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8bfef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8bfefa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff8bff2f3a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffff8bfed42d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffff8bfde32b
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xeb
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff8c594f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [Worker-97:27210]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc nfsv3 nfs_acl nfs lockd ebtable_filter grace ebtables fscache ip6table_filter devlink ip6_tables iptable_filter mvfs(OE) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul vmw_balloon glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common crc32c_intel ata_piix
nfit libata serio_raw mptspi libnvdimm scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks floppy fuse
CPU: 4 PID: 27210 Comm: Worker-97 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff9c4ccf1fd280 ti: ffff9c4b0c83c000 task.ti: ffff9c4b0c83c000
RIP: 0010:[<ffffffff8bf16e4a>] [<ffffffff8bf16e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c4b0c83fc70 EFLAGS: 00000202
RAX: 0000000000000005 RBX: ffffffff8be7e6a0 RCX: ffff9c4dee760678
RDX: 0000000000000005 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff9c4b0c83fca8 R08: ffff9c4b3a343800 R09: ffffffff8c186479
R10: ffff9c4dee71f160 R11: ffffe2b146631a00 R12: 0000000000000286
R13: ffff9c4b0c83fc20 R14: ffff9c4b38740000 R15: 0000000000000001
FS: 00007fa65fafe700(0000) GS:ffff9c4dee700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000015cdd840e000 CR3: 0000000405532000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8be7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8be7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8bfede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8bfef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8bfefa73>] tlb_finish_mmu+0x23/0x30
[<ffffffff8bff2f3a>] zap_page_range+0xea/0x130
[<ffffffff8bfed42d>] SyS_madvise+0x49d/0xac0
[<ffffffff8bfde32b>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffff8c594f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 b5 8c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #73650
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff91516e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9147e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9147e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff915ede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff915ef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff915efa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff915f2f3a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffff915fcb7c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x39c
|
|
9
|
SyS_madvise
|
vmlinux |
0xffffffff915ed42d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x49d
|
|
10
|
|
vmlinux |
0xffffffff915de32b
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xeb
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff91b94f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [systemd-journal:15084]
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [java:15334]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter sunrpc ext4 mbcache jbd2 dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev i2c_piix4 sg joydev pcspkr virtio_balloon parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi bochs_drm drm_kms_helper virtio_net net_failover failover syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm ahci ata_piix libahci crc32c_intel
libata serio_raw virtio_pci virtio_ring drm_panel_orientation_quirks virtio floppy fuse
CPU: 2 PID: 15334 Comm: java Tainted: G L ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.0-0-g63451fca13-prebuilt.qemu-project.org 04/01/2014
task: ffff8e565ff9b180 ti: ffff8e5538f9c000 task.ti: ffff8e5538f9c000
RIP: 0010:[<ffffffff91516e4a>] [<ffffffff91516e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8e5538f9fc70 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 000800009147e6a0 RCX: ffff8e5937ca0628
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff8e5538f9fca8 R08: ffff8e567937a800 R09: ffffffff91786479
R10: ffff8e5937d1f160 R11: ffffe44449aa9c00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8e5538f9fc20 R15: 0000000000000001
FS: 00007f9d6f206700(0000) GS:ffff8e5937d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9d6f1f1a18 CR3: 00000003f2a4a000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9147e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9147e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff915ede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff915ef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff915efa73>] tlb_finish_mmu+0x23/0x30
[<ffffffff915f2f3a>] zap_page_range+0xea/0x130
[<ffffffff915fcb7c>] ? do_mmap+0x39c/0x590
[<ffffffff915ed42d>] SyS_madvise+0x49d/0xac0
[<ffffffff915de32b>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffff91b94f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 15 92 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #56223
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9bd16a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9bc7e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9bc7e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9bdec026
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9bdedaef
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9bdedc13
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffff9bdf091a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffff9bdeb62d
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffff9bdddfdb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xeb
|
|
10
|
tracesys
|
vmlinux |
0xffffffff9c38e172
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [elasticsearch[c:59626]
Modules linked in: tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter fuse vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper snd_ens1371 vmw_balloon snd_rawmidi snd_ac97_codec nfit ablk_helper cryptd ac97_bus snd_seq libnvdimm snd_seq_device snd_pcm pcspkr snd_timer snd joydev soundcore vmw_vmci sg i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi libata scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common mptbase crc32c_intel e1000 drm_panel_orientation_quirks serio_raw dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 59626 Comm: elasticsearch[c Tainted: G L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88a2cad30000 ti: ffff88a2ed32c000 task.ti: ffff88a2ed32c000
RIP: 0010:[<ffffffff9bd16a6a>] [<ffffffff9bd16a6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88a2ed32fc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009bc7e2b0 RCX: ffff88a4f5620600
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88a2ed32fca8 R08: ffff88a43c84bc00 R09: ffffffff9bf817e9
R10: ffff88a4f565f160 R11: ffffbbce02e23200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff88a2ed32fc20 R15: 0000000000000001
FS: 00007f0694c6e700(0000) GS:ffff88a4f5640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055af9bb36acc CR3: 00000002217e8000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9bc7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9bc7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9bdec026>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9bdedaef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9bdedc13>] tlb_finish_mmu+0x23/0x30
[<ffffffff9bdf091a>] zap_page_range+0xea/0x130
[<ffffffff9bdeb62d>] SyS_madvise+0x49d/0xac0
[<ffffffff9bdddfdb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffff9c38e172>] tracesys+0xa6/0xcc
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 94 9c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #9570
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb2716a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb267e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb267e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb27ec026
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb27edaef
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb27edc13
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffb27f091a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xea
|
|
8
|
SyS_madvise
|
vmlinux |
0xffffffffb27eb62d
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x49d
|
|
9
|
|
vmlinux |
0xffffffffb27ddfdb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xeb
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb2d8dede
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [RMI TCP Connect:17128]
Modules linked in: tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter binfmt_misc sunrpc nfit libnvdimm ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper sg ablk_helper cryptd joydev pcspkr virtio_balloon parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom cirrus ata_generic drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm virtio_console virtio_blk ata_piix crct10dif_pclmul drm 8139too libata crct10dif_common crc32c_intel virtio_pci virtio_ring serio_raw 8139cp floppy drm_panel_orientation_quirks
mii virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 17128 Comm: RMI TCP Connect Kdump: loaded Tainted: G L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff8b31ef1541c0 ti: ffff8b30cfdc0000 task.ti: ffff8b30cfdc0000
RIP: 0010:[<ffffffffb2716a6a>] [<ffffffffb2716a6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b30cfdc3c70 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000b267e2b0 RCX: ffff8b323a120320
RDX: 0000000000000002 RSI: 000000000000000a RDI: 0000000000000000
RBP: ffff8b30cfdc3ca8 R08: ffff8b3237ab9c00 R09: ffffffffb29817e9
R10: ffff8b323a09f160 R11: ffffd6dcc5d46a00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8b30cfdc3c20 R15: 0000000000000001
FS: 00007f2f0f4cc700(0000) GS:ffff8b323a080000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000013460001 CR3: 00000000a7120000 CR4: 00000000007606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb267e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb267e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb27ec026>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb27edaef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb27edc13>] tlb_finish_mmu+0x23/0x30
[<ffffffffb27f091a>] zap_page_range+0xea/0x130
[<ffffffffb27eb62d>] SyS_madvise+0x49d/0xac0
[<ffffffffb27ddfdb>] ? vm_mmap_pgoff+0xeb/0x120
[<ffffffffb2d8dede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 34 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #997488
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa931e23a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa9281fa8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa928201b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa93fa07e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa93fbbff
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa93fbd33
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x23
|
|
7
|
zap_page_range
|
vmlinux |
0xffffffffa93ff27a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xea
|
|
8
|
|
vmlinux |
0xffffffffa940228a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb3a
|
|
9
|
|
vmlinux |
0xffffffffa940926c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x39c
|
|
10
|
SyS_madvise
|
vmlinux |
0xffffffffa93f966d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x49d
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa99c539a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 25s! [InactivityMonit:7913]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 auth_rpcgss binfmt_misc sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif crct10dif_generic ahci drm ata_piix libahci
nfit libata crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 7 PID: 7913 Comm: InactivityMonit Kdump: loaded Tainted: G L ------------ 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff97f95a6d0000 ti: ffff97f9c6f7c000 task.ti: ffff97f9c6f7c000
RIP: 0010:[<ffffffffa931e23a>] [<ffffffffa931e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff97f9c6f7fc70 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000a9281b10 RCX: ffff97f90da20750
RDX: 0000000000000000 RSI: 000000000000001e RDI: 0000000000000000
RBP: ffff97f9c6f7fca8 R08: ffff97ede0592000 R09: ffffffffa959c7b9
R10: ffff97f90dbdf160 R11: fffff947e3fbf800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff97f9c6f7fc20 R15: 0000000000000001
FS: 00007fc04fafa700(0000) GS:ffff97f90dbc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc1313062b0 CR3: 000000190c528000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffa9281fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa928201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa93fa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffa93fbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa93fbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffffa93ff27a>] zap_page_range+0xea/0x140
[<ffffffffa940228a>] ? handle_mm_fault+0xb3a/0x1190
[<ffffffffa940926c>] ? do_mmap+0x39c/0x590
[<ffffffffa93f966d>] SyS_madvise+0x49d/0xac0
[<ffffffffa99c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 8e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 f5 a9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 18 c4 00 eb b7 0f b6 4d cc 4c 89