Complete report #1251564
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8fb1e23a
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8fa81fa8
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8fa8201b
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8fbfa07e
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8fbfbbff
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8fbfbd33
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff8fc058b4
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff8fc07f15
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff8fc08155
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff8fc09482
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff901c539a
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 54s! [java:26795]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netbios_ns nf_conntrack_broadcast xt_nat xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables binfmt_misc tcp_lp tun devlink bridge stp llc ip_set vmw_vsock_vmci_transport vsock sunrpc ppdev sb_edac vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul nfit glue_helper libnvdimm ablk_helper cryptd joydev sg pcspkr vmw_vmci
i2c_piix4 parport_pc parport xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi crct10dif_pclmul crct10dif_common libata scsi_transport_spi crc32c_intel serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 6 PID: 26795 Comm: java Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.102.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8f2fee948000 ti: ffff8f2aa1010000 task.ti: ffff8f2aa1010000
RIP: 0010:[<ffffffff8fb1e23a>] [<ffffffff8fb1e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8f2aa1013cb0 EFLAGS: 00000202
RAX: 0000000000000008 RBX: 000800008fa81b10 RCX: ffff8f329f4206c8
RDX: 0000000000000008 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff8f2aa1013ce8 R08: ffff8f2bec52a000 R09: ffffffff8fd9c7b9
R10: ffff8f329f39f160 R11: ffffeeee1a995a00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8f2aa1013c60 R15: ffff8f2fee948000
FS: 00007fa6451b7700(0000) GS:ffff8f329f380000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa6c6464280 CR3: 000000058db36000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8fa81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8fa8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8fbfa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff8fbfbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8fbfbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffff8fc058b4>] unmap_region+0xf4/0x150
[<ffffffff8fc07f15>] do_munmap+0x2a5/0x480
[<ffffffff8fc08155>] vm_munmap+0x65/0xb0
[<ffffffff8fc09482>] SyS_munmap+0x22/0x30
[<ffffffff901c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 4e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 75 90 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #887159
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb271e23a
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb2681fa8
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb268201b
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb27fa07e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb27fbbff
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb27fbd33
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb28058b4
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb2807f15
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb2808155
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb2809482
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffffb2dc562e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [elasticsearch[s:9533]
Modules linked in: traps(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 crc32_pclmul ghash_clmulni_intel ppdev aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod crc_t10dif crct10dif_generic drm ata_piix nfit crct10dif_pclmul crct10dif_common crc32c_intel libata libnvdimm serio_raw vmxnet3
vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: traps]
CPU: 4 PID: 9533 Comm: elasticsearch[s Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.99.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/28/2020
task: ffff92c4ccdd1080 ti: ffff92c4cce34000 task.ti: ffff92c4cce34000
RIP: 0010:[<ffffffffb271e23a>] [<ffffffffb271e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff92c4cce37cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffffffffb2681b10 RCX: ffff92c9ffc20650
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff92c4cce37ce8 R08: ffff92bb19551400 R09: ffffffffb299c7b9
R10: ffff92c9ffd1f160 R11: fffff282b148ae00 R12: ffff92c4cce37c80
R13: ffffffffb299c6ef R14: ffff92c4cce37c30 R15: ffff92c4ccdd1080
FS: 00007fb29807b700(0000) GS:ffff92c9ffd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb8e100a000 CR3: 0000000fe3b20000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb2681fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb268201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb27fa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffb27fbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb27fbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffffb28058b4>] unmap_region+0xf4/0x150
[<ffffffffb2807f15>] do_munmap+0x2a5/0x480
[<ffffffffb2808155>] vm_munmap+0x65/0xb0
[<ffffffffb2809482>] SyS_munmap+0x22/0x30
[<ffffffffb2dc562e>] tracesys+0xa6/0xcc
Code: 48 63 35 4e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 35 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #826110
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9dd1173e
Build id: 3.10.0-957.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9dc7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9dc7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff9dde4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9dde5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff9ddef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff9ddf1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff9ddf1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff9ddf2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
10
|
tracesys
|
vmlinux |
0xffffffff9e37505b
Build id: 3.10.0-957.el7.x86_64
|
0xa3
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [java:106495]
Modules linked in: veth nfnetlink_queue nfnetlink_log bluetooth rfkill xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq joydev snd_seq_device pcspkr snd_pcm snd_timer snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic
crct10dif_pclmul crct10dif_common crc32c_intel pata_acpi vmwgfx serio_raw drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix e1000 libata mptspi scsi_transport_spi nfit mptscsih libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 106495 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8f15782e2080 ti: ffff8f168b278000 task.ti: ffff8f168b278000
RIP: 0010:[<ffffffff9dd1173e>] [<ffffffff9dd1173e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff8f168b27bcb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff8f1845020600
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8f168b27bcf0 R08: ffff8f15786cc400 R09: ffffffff9df755c9
R10: ffff8f18450df160 R11: fffff4f40db9b200 R12: 0000000000000296
R13: ffff8f168b27bc68 R14: 0000000000000001 R15: fffff4f40b231b80
FS: 00007fba091cab10(0000) GS:ffff8f18450c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2da8281508 CR3: 0000000094ad0000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9dc7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9dc7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff9dde4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff9dde5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff9ddef014>] unmap_region+0xf4/0x140
[<ffffffff9ddf1605>] do_munmap+0x2a5/0x480
[<ffffffff9ddf1845>] vm_munmap+0x65/0xb0
[<ffffffff9ddf2ad2>] SyS_munmap+0x22/0x30
[<ffffffff9e37505b>] tracesys+0xa3/0xc9
Code: 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 94 9e f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #814616
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff82f1e23e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff82e81fa8
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff82e8201b
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff82ffa07e
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff82ffbbff
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff82ffbd33
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff830058b4
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff83007f15
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff83008155
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff83009482
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff835c539a
Build id: 3.10.0-1160.99.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [grafana:2525]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack vboxsf(OE) ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iosf_mbi kvm_intel kvm snd_intel8x0 irqbypass crc32_pclmul snd_ac97_codec ppdev ac97_bus snd_seq ghash_clmulni_intel snd_seq_device snd_pcm aesni_intel joydev lrw gf128mul glue_helper ablk_helper cryptd snd_timer vboxguest(OE) pcspkr i2c_piix4 parport_pc sg snd soundcore parport
video ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 2525 Comm: grafana Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.99.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff8b8456d7c200 ti: ffff8b848d1f0000 task.ti: ffff8b848d1f0000
RIP: 0010:[<ffffffff82f1e23e>] [<ffffffff82f1e23e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff8b848d1f3cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0008000082e81b10 RCX: ffff8b849fc20348
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff8b848d1f3ce8 R08: ffff8b81f62fdc00 R09: ffffffff8319c7b9
R10: ffff8b849fd1f160 R11: fffff5ed50336000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8b848d1f3c60 R15: 00007f4da0881000
FS: 00007f4d9f810f38(0000) GS:ffff8b849fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f02968f2000 CR3: 00000004075d2000 CR4: 00000000000606e0
Call Trace:
[<ffffffff82e81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff82e8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff82ffa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff82ffbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff82ffbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffff830058b4>] unmap_region+0xf4/0x150
[<ffffffff83007f15>] do_munmap+0x2a5/0x480
[<ffffffff83008155>] vm_munmap+0x65/0xb0
[<ffffffff83009482>] SyS_munmap+0x22/0x30
[<ffffffff835c539a>] system_call_fastpath+0x25/0x2a
Code: 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 b5 83 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #752569
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fb892
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106f508
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106f5d8
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811af0f3
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b0415
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b74e4
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff811b9aa2
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x292
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff811b9ce5
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff811badf2
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff816b89fd
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 47s! [mono:116843]
Modules linked in: file_protector(OE) snapapi26(OE) tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun nf_conntrack_ftp ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel ext4 aesni_intel lrw gf128mul glue_helper ablk_helper mbcache cryptd jbd2 ppdev vmw_balloon joydev
pcspkr nfit sg libnvdimm parport_pc parport i2c_piix4 shpchp vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt crc32c_intel fb_sys_fops ttm mptspi scsi_transport_spi drm mptscsih ata_piix mptbase serio_raw libata vmxnet3 i2c_core [last unloaded: edd]
CPU: 0 PID: 116843 Comm: mono Tainted: G OEL ------------ 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8801002bbf40 ti: ffff8803043f0000 task.ti: ffff8803043f0000
RIP: 0010:[<ffffffff810fb892>] [<ffffffff810fb892>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff8803043f3cc0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000800008106f340 RCX: ffff8803316dbe20
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8803043f3cf8 R08: ffff88017fd8d400 R09: ffffffff81324cb9
R10: ffff88033161b9c0 R11: ffffea00000d0400 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8803043f3c70 R15: ffff8801002bbf40
FS: 00007f2be46e5780(0000) GS:ffff880331600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2bc3b4b3c6 CR3: 000000032c65e000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8106f508>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106f5d8>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811af0f3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b0415>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b74e4>] unmap_region+0xf4/0x140
[<ffffffff811b9aa2>] do_munmap+0x292/0x470
[<ffffffff811b9ce5>] vm_munmap+0x65/0xb0
[<ffffffff811badf2>] SyS_munmap+0x22/0x30
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 76 b6 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 8e b1 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 45 b6 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #749898
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb691718a
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb687e258
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb687e2cb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb69ee6a6
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb69f016f
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb69f0293
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb69f9a84
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb6a4da20
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x90
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb69fc025
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb69fc265
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb69fd562
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffb6f99f92
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb6f99ed5
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [java:72811]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) sunrpc ppdev sg joydev virtio_balloon parport_pc parport i2c_piix4 pcspkr ip_tables xfs libcrc32c sr_mod cdrom virtio_net ata_generic pata_acpi net_failover virtio_console virtio_scsi failover virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ata_piix ttm libata drm serio_raw virtio_pci virtio_ring floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 72811 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff956abb255280 ti: ffff956bb64f0000 task.ti: ffff956bb64f0000
RIP: 0010:[<ffffffffb691718a>] [<ffffffffb691718a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff956bb64f3cb0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000b687ddd0 RCX: ffff956d2dee0428
RDX: 0000000000000003 RSI: 0000000000000060 RDI: 0000000000000000
RBP: ffff956bb64f3ce8 R08: ffff956a763bb400 R09: ffffffffb6b869f9
R10: ffff956d2de9f160 R11: ffffdc6ccf49c000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff956bb64f3c60 R15: 0000000000000000
FS: 00007fa9be97d700(0000) GS:ffff956d2de80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa960065fb0 CR3: 00000003d7e8e000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb687e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb687e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb69ee6a6>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb69f016f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb69f0293>] tlb_finish_mmu+0x23/0x30
[<ffffffffb69f9a84>] unmap_region+0xf4/0x140
[<ffffffffb6a4da20>] ? do_sync_write+0x90/0xe0
[<ffffffffb69fc025>] do_munmap+0x2a5/0x480
[<ffffffffb69fc265>] vm_munmap+0x65/0xb0
[<ffffffffb69fd562>] SyS_munmap+0x22/0x30
[<ffffffffb6f99f92>] system_call_fastpath+0x25/0x2a
[<ffffffffb6f99ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 be 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 55 b7 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 8d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #506895
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
10
|
tracesys
|
vmlinux |
0xffffffff816b51d2
Build id: 3.10.0-693.el7.x86_64
|
0xdd
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [java:114787]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun overlay(T) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371
snd_rawmidi snd_ac97_codec ac97_bus snd_seq ppdev snd_seq_device vmw_balloon snd_pcm pcspkr joydev snd_timer snd i2c_piix4 soundcore vmw_vmci sg shpchp nfit libnvdimm parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata mptspi scsi_transport_spi mptscsih serio_raw mptbase e1000 i2c_core
CPU: 2 PID: 114787 Comm: java Tainted: G L ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff88034efa9fa0 ti: ffff88025ef68000 task.ti: ffff88025ef68000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff88025ef6bcf0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000800008106e4a0 RCX: ffff8803702daf48
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88025ef6bd28 R08: ffff88017afb3800 R09: ffffffff813227d9
R10: ffff880370299c80 R11: ffffea000c0dd000 R12: 0000000000000286
R13: 0000000000000286 R14: ffff88025ef6bca0 R15: 00007f5559340000
FS: 00007f5598269700(0000) GS:ffff880370280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f55443bd1b0 CR3: 000000019a66d000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000107553cc0 ffff88034d0be108 ffff88034d0bddc0 0000000000000000
ffffffffffffffff ffff88034d0be108 ffff88014ef49d88 ffff88025ef6bd78
ffffffff8106e668 ffff88034d0bddc0 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b51d2>] tracesys+0xdd/0xe2
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #413015
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9f711a0a
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9f67a628
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9f67a698
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff9f7e4607
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9f7e5f05
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff9f7ef5a4
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff9f7f1b95
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x2a5
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff9f7f1dd5
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff9f7f3062
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff9fd75ddb
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [mysqld:368]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock iptable_filter dm_mirror dm_region_hash dm_log dm_mod ppdev vmw_balloon sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul
crct10dif_common crc32c_intel vmwgfx serio_raw drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm vmxnet3 drm ata_piix mptspi scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks floppy
CPU: 8 PID: 368 Comm: mysqld Tainted: G W L ------------ 3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9af268430000 ti: ffff9b048f460000 task.ti: ffff9b048f460000
RIP: 0010:[<ffffffff9f711a0a>] [<ffffffff9f711a0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9b048f463cb8 EFLAGS: 00000202
RAX: 000000000000000a RBX: 000000000001b700 RCX: ffff9b059e8a06c8
RDX: 000000000000000a RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff9b048f463cf0 R08: ffff9aef1b057000 R09: ffffffff9f975d69
R10: ffff9b059e81f160 R11: ffffd2529e33c000 R12: ffff9b048f463c68
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000200
FS: 00007fceb437a700(0000) GS:ffff9b059e800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3e88c15000 CR3: 000000064c71a000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9f67a628>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9f67a698>] flush_tlb_mm_range+0x68/0x140
[<ffffffff9f7e4607>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff9f7e5f05>] tlb_finish_mmu+0x55/0x60
[<ffffffff9f7ef5a4>] unmap_region+0xf4/0x140
[<ffffffff9f7f1b95>] do_munmap+0x2a5/0x480
[<ffffffff9f7f1dd5>] vm_munmap+0x65/0xb0
[<ffffffff9f7f3062>] SyS_munmap+0x22/0x30
[<ffffffff9fd75ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 7e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 bc 34 a0 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #398066
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb8d1722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb8c7e258
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb8c7e2cb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb8dee666
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb8df012f
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb8df0253
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb8df9a44
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb8dfc035
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb8dfc275
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb8dfd572
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb9399f92
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x25
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb9399ed5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [Wt-780:4312]
Modules linked in: vboxsf(OE) veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ext4 mbcache jbd2 binfmt_misc overlay(T) nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter snd_intel8x0 snd_ac97_codec ac97_bus snd_seq ppdev snd_seq_device joydev snd_pcm snd_timer pcspkr snd vboxguest(OE) sg parport_pc i2c_piix4 soundcore video parport
sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common sr_mod vmwgfx cdrom ata_generic drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ahci ttm libahci drm serio_raw ata_piix e1000 crc32c_intel libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 4312 Comm: Wt-780 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff9ca28ab43180 ti: ffff9ca51d84c000 task.ti: ffff9ca51d84c000
RIP: 0010:[<ffffffffb8d1722a>] [<ffffffffb8d1722a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9ca51d84fcb0 EFLAGS: 00010202
RAX: 0000000000000001 RBX: ffffffffb8c7ddd0 RCX: ffff9ca5dfc9fc10
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9ca51d84fce8 R08: ffff9ca33fd52000 R09: ffffffffb8f869d9
R10: ffff9ca5dfc1f160 R11: ffffe8570d76ff00 R12: ffff9ca51d84fc80
R13: ffffffffb8f8692f R14: ffff9ca33fd52d08 R15: ffff9ca28ab43180
FS: 00007f0e061e2700(0000) GS:ffff9ca5dfc00000(0000) knlGS:0000000b226f0000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000b22b57006 CR3: 0000000407f9c000 CR4: 00000000000006f0
Call Trace:
[<ffffffffb8c7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb8c7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb8dee666>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb8df012f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb8df0253>] tlb_finish_mmu+0x23/0x30
[<ffffffffb8df9a44>] unmap_region+0xf4/0x140
[<ffffffffb8dfc035>] do_munmap+0x2a5/0x480
[<ffffffffb8dfc275>] vm_munmap+0x65/0xb0
[<ffffffffb8dfd572>] SyS_munmap+0x22/0x30
[<ffffffffb9399f92>] system_call_fastpath+0x25/0x2a
[<ffffffffb9399ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 7e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 95 b9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #313774
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb8b1181a
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb8a7a598
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb8a7a608
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffb8be42e7
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb8be5be5
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb8bef284
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffffb8bf1875
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x2a5
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffffb8bf1ab5
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffffb8bf2d42
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb9174ddb
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x22
|
|
11
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb9174d21
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#38 stuck for 23s! [sanlock:52234]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver vhost_net vhost macvtap macvlan ebt_arp tun nfsv3 nfs_acl nfs lockd grace fscache bnx2i libiscsi devlink bnx2fc cnic uio fcoe libfcoe libfc scsi_transport_fc scsi_tgt ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack geneve ip6_udp_tunnel udp_tunnel openvswitch ip_set nfnetlink ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter softdog dcdbas dm_service_time dm_multipath amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass joydev
pcspkr sg sp5100_tco k10temp fam15h_power i2c_piix4 ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_cpufreq bridge bonding ip_tables ext4 mbcache jbd2 dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c uas usb_storage sd_mod sr_mod crc_t10dif crct10dif_generic cdrom mgag200 i2c_algo_bit drm_kms_helper syscopyarea crct10dif_pclmul crct10dif_common sysfillrect crc32_pclmul sysimgblt crc32c_intel fb_sys_fops ttm ghash_clmulni_intel aesni_intel lrw gf128mul drm glue_helper ablk_helper cryptd serio_raw scsi_transport_iscsi megaraid_sas drm_panel_orientation_quirks ahci libahci 8021q garp mrp libata stp llc bnx2 sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 38 PID: 52234 Comm: sanlock Kdump: loaded Tainted: G L ------------ 3.10.0-957.1.3.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R815/04Y8PT, BIOS 3.4.1 05/04/2018
task: ffff8a38bbda9040 ti: ffff8a38b3678000 task.ti: ffff8a38b3678000
RIP: 0010:[<ffffffffb8b1181a>] [<ffffffffb8b1181a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a38b367bcb8 EFLAGS: 00000202
RAX: 0000000000000023 RBX: 000000000001b700 RCX: ffff8a38bfce0e50
RDX: 0000000000000023 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff8a38b367bcf0 R08: ffff8a315fc0c000 R09: ffffffffb8d75839
R10: ffff8a38bfd9f160 R11: ffffeaa31fd1c000 R12: ffff8a38b367bc68
R13: ffffeaa31feb4d80 R14: 0000000000000001 R15: ffffeaa31fbf7b00
FS: 00007f658f441700(0000) GS:ffff8a38bfd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe9decee140 CR3: 0000002ff8b3e000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb8a7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb8a7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb8be42e7>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffb8be5be5>] tlb_finish_mmu+0x55/0x60
[<ffffffffb8bef284>] unmap_region+0xf4/0x140
[<ffffffffb8bf1875>] do_munmap+0x2a5/0x480
[<ffffffffb8bf1ab5>] vm_munmap+0x65/0xb0
[<ffffffffb8bf2d42>] SyS_munmap+0x22/0x30
[<ffffffffb9174ddb>] system_call_fastpath+0x22/0x27
[<ffffffffb9174d21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 6e 82 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 74 b9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 3d 82 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #298467
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8691173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8687a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8687a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff869e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff869e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff869ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff869f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff869f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff869f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff86f74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [360entclient:21862]
Modules linked in: kbase(OE) binfmt_misc udp_diag tcp_diag inet_diag fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel virtio_balloon aesni_intel parport_pc lrw parport gf128mul glue_helper i2c_piix4 ablk_helper cryptd joydev pcspkr ip_tables ext4 mbcache jbd2 virtio_console virtio_blk
virtio_net ata_generic pata_acpi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm floppy ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw drm_panel_orientation_quirks virtio_pci virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod [last unloaded: kbase]
CPU: 7 PID: 21862 Comm: 360entclient Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: Fedora Project OpenStack Nova, BIOS 0.5.1 01/01/2011
task: ffff9b5fb861b0c0 ti: ffff9b5f897c0000 task.ti: ffff9b5f897c0000
RIP: 0010:[<ffffffff8691173a>] [<ffffffff8691173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9b5f897c3cb8 EFLAGS: 00000202
RAX: 0000000000000006 RBX: 00000000000000fc RCX: ffff9b5fbf3a03c0
RDX: 0000000000000006 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9b5f897c3cf0 R08: ffff9b51294f4800 R09: ffffffff86b755c9
R10: ffff9b5fbf3df160 R11: fffffa0bfea4ac00 R12: 0000000000000296
R13: ffff9b5f897c3c68 R14: 0000000000000001 R15: fffffa0bd7a57180
FS: 00007f8205730700(0000) GS:ffff9b5fbf3c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8207fd7438 CR3: 0000000ff82f4000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8687a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8687a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff869e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff869e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff869ef014>] unmap_region+0xf4/0x140
[<ffffffff869f1605>] do_munmap+0x2a5/0x480
[<ffffffff869f1845>] vm_munmap+0x65/0xb0
[<ffffffff869f2ad2>] SyS_munmap+0x22/0x30
[<ffffffff86f74ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 54 87 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #128414
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [java:13244]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi coretemp snd_ac97_codec ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm lrw gf128mul glue_helper snd_timer ablk_helper snd cryptd soundcore pcspkr ppdev sg vmw_balloon joydev parport_pc parport shpchp i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata mptspi serio_raw scsi_transport_spi e1000 mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 13244 Comm: java Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2012
task: ffff8801bb2a8000 ti: ffff880044750000 task.ti: ffff880044750000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff880044753cf0 EFLAGS: 00000202
RAX: 0000000000000007 RBX: ffffffff8106e4a0 RCX: ffff8802367da7c8
RDX: 0000000000000007 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff880044753d28 R08: ffff88017ceab400 R09: ffffffff813227d9
R10: ffff880236659c80 R11: ffffea0001111a00 R12: 0000000000000282
R13: ffff880044753ca0 R14: ffff880044753c70 R15: ffff8801bb2a8000
FS: 00007f1a2cb1c700(0000) GS:ffff880236640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f592b1ba000 CR3: 0000000208f5f000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001bb2a8000 ffff88003541a288 ffff880035419f40 0000000000000000
ffffffffffffffff ffff88003541a288 ffff88020f6ce0d8 ffff880044753d78
ffffffff8106e668 ffff880035419f40 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #110971
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbab1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbaa7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbaa7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffbabe4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbabe5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffbabef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
do_munmap
|
vmlinux |
0xffffffffbabf1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
8
|
vm_munmap
|
vmlinux |
0xffffffffbabf1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
9
|
SyS_munmap
|
vmlinux |
0xffffffffbabf2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffbb174ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [mysqld:21785]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp iosf_mbi kvm_intel ppdev kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul vmw_balloon glue_helper ablk_helper cryptd joydev pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec
ac97_bus snd_seq sg snd_seq_device snd_pcm snd_timer snd soundcore parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ata_generic pata_acpi serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci ata_piix drm e1000 libata vmw_pvscsi drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 21785 Comm: mysqld Kdump: loaded Not tainted 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8bc2d6516180 ti: ffff8bc41da0c000 task.ti: ffff8bc41da0c000
RIP: 0010:[<ffffffffbab1173a>] [<ffffffffbab1173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8bc41da0fcb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff8bc4758204b8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8bc41da0fcf0 R08: ffff8bc3bc8db800 R09: ffffffffbad755c9
R10: ffff8bc47589f160 R11: ffffef620824cc00 R12: 0000000000000296
R13: ffff8bc41da0fc68 R14: 0000000000000000 R15: 0000000000000200
FS: 00007f64b5e1b700(0000) GS:ffff8bc475880000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f632699a000 CR3: 00000001baf74000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffbaa7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbaa7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffbabe4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffbabe5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffbabef014>] unmap_region+0xf4/0x140
[<ffffffffbabf1605>] do_munmap+0x2a5/0x480
[<ffffffffbabf1845>] vm_munmap+0x65/0xb0
[<ffffffffbabf2ad2>] SyS_munmap+0x22/0x30
[<ffffffffbb174ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 74 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #24260
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff99716a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9967e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9967e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff997ec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff997edb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff997edc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff997f7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff997f9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff997f9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff997faf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffff99d8e172
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [aux:33659]
Modules linked in: tcp_lp fuse veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event xfs ppdev vmw_balloon snd_ens1371 snd_rawmidi crc32_pclmul snd_ac97_codec libcrc32c ghash_clmulni_intel ac97_bus snd_seq aesni_intel snd_seq_device lrw gf128mul snd_pcm glue_helper snd_timer snd joydev ablk_helper pcspkr soundcore cryptd sg parport_pc parport vmw_vmci i2c_piix4 ip_tables ext4 mbcache
jbd2 sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix nfit drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi libnvdimm libata serio_raw scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks
CPU: 11 PID: 33659 Comm: aux Kdump: loaded Tainted: G ------------ T 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff92cafbac62a0 ti: ffff92c11797c000 task.ti: ffff92c11797c000
RIP: 0010:[<ffffffff99716a7a>] [<ffffffff99716a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff92c11797fcb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800009967e2b0 RCX: ffff92cd886a0790
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff92c11797fce8 R08: ffff92c1ec665c00 R09: ffffffff99981839
R10: ffff92cd888df160 R11: ffffc3dae7d18400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff92c11797fc60 R15: 00007feba6634000
FS: 00007feba76cf700(0000) GS:ffff92cd888c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdbca606b80 CR3: 000000082b678000 CR4: 00000000003407e0
Call Trace:
[<ffffffff9967e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9967e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff997ec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff997edb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff997edc83>] tlb_finish_mmu+0x23/0x30
[<ffffffff997f7474>] unmap_region+0xf4/0x140
[<ffffffff997f9a75>] do_munmap+0x2a5/0x480
[<ffffffff997f9cb5>] vm_munmap+0x65/0xb0
[<ffffffff997faf52>] SyS_munmap+0x22/0x30
[<ffffffff99d8e172>] tracesys+0xa6/0xcc
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 34 9a f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #14524
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9e51761a
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9e47e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9e47e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9e5ede36
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9e5ef8ff
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9e5efa23
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9e5f9234
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff9e5fb825
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff9e5fba65
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff9e5fcd02
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff9eb92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [5_scheduler:2809]
Modules linked in: xt_set ipt_rpfilter xt_multiport ip_set_hash_ip ip_set_hash_net vxlan ip6_udp_tunnel udp_tunnel ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_comment xt_mark veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log
dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device snd_pcm ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_timer snd joydev pcspkr sg soundcore i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel nfit ata_piix libnvdimm libata serio_raw mptspi scsi_transport_spi mptscsih e1000 mptbase drm_panel_orientation_quirks
CPU: 1 PID: 2809 Comm: 5_scheduler Tainted: G ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff964cbe0062a0 ti: ffff964cbfa20000 task.ti: ffff964cbfa20000
RIP: 0010:[<ffffffff9e51761a>] [<ffffffff9e51761a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff964cbfa23cb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800009e47e730 RCX: ffff964cc02a0600
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff964cbfa23ce8 R08: ffff964bbb9cb000 R09: ffffffff9e7851b9
R10: ffff964cc025f160 R11: ffffccec02757800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff964cbfa23c60 R15: 00007f8ef99bf000
FS: 00007f8ffc433700(0000) GS:ffff964cc0240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8eff4b3000 CR3: 000000027f91a000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9e47e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9e47e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9e5ede36>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9e5ef8ff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9e5efa23>] tlb_finish_mmu+0x23/0x30
[<ffffffff9e5f9234>] unmap_region+0xf4/0x140
[<ffffffff9e5fb825>] do_munmap+0x2a5/0x480
[<ffffffff9e5fba65>] vm_munmap+0x65/0xb0
[<ffffffff9e5fcd02>] SyS_munmap+0x22/0x30
[<ffffffff9eb92ed2>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 15 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #10220
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff84d16a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff84c7e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff84c7e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff84dec026
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff84dedaef
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff84dedc13
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff84df7404
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff84df9a05
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff84df9c45
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff84dfaee2
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffff8538e172
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [C2 CompilerThre:22824]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon sb_edac iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm nfit crct10dif_pclmul libata mptspi crct10dif_common crc32c_intel libnvdimm scsi_transport_spi serio_raw mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 22824 Comm: C2 CompilerThre Kdump: loaded Tainted: G W L ------------ T 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff97c6c23fa0e0 ti: ffff97c6c4e80000 task.ti: ffff97c6c4e80000
RIP: 0010:[<ffffffff84d16a6a>] [<ffffffff84d16a6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff97c6c4e83cb0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffffffff84c7e2b0 RCX: ffff97c6ffc606c8
RDX: 0000000000000001 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff97c6c4e83ce8 R08: ffff97c0344b2400 R09: ffffffff84f817e9
R10: ffff97c6ffd9f160 R11: ffffcd858db3f800 R12: ffff97c6c4e83c80
R13: ffffffff84f8173f R14: ffff97c6c23fa0e0 R15: 0000000000000000
FS: 00007f4e29c1e700(0000) GS:ffff97c6ffd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4dda3a6010 CR3: 00000007fbe3e000 CR4: 00000000001607e0
Call Trace:
[<ffffffff84c7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff84c7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff84dec026>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff84dedaef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff84dedc13>] tlb_finish_mmu+0x23/0x30
[<ffffffff84df7404>] unmap_region+0xf4/0x140
[<ffffffff84df9a05>] do_munmap+0x2a5/0x480
[<ffffffff84df9c45>] vm_munmap+0x65/0xb0
[<ffffffff84dfaee2>] SyS_munmap+0x22/0x30
[<ffffffff8538e172>] tracesys+0xa6/0xcc
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 94 85 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1242
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9c116a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9c07e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9c07e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9c1ec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9c1edb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9c1edc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9c1f7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff9c1f9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff9c1f9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff9c1faf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff9c78dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9c78de21
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [java:11777]
Modules linked in: xt_nat veth dcdbas nf_conntrack_netlink nfnetlink br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay vmw_vsock_vmci_transport vsock xt_comment xt_conntrack iptable_filter ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ext4 mbcache jbd2 sb_edac ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul parport_pc glue_helper ablk_helper sg vmw_vmci parport cryptd pcspkr i2c_piix4 joydev binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi crct10dif_pclmul libata
scsi_transport_spi crct10dif_common nfit crc32c_intel mptscsih e1000 serio_raw libnvdimm mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 11777 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8c32c177b150 ti: ffff8c2a24208000 task.ti: ffff8c2a24208000
RIP: 0010:[<ffffffff9c116a7a>] [<ffffffff9c116a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8c2a2420bcb0 EFLAGS: 00000202
RAX: 0000000000000012 RBX: ffffffff9c07e2b0 RCX: ffff8c32cd4a06a0
RDX: 0000000000000012 RSI: 0000000000000014 RDI: 0000000000000000
RBP: ffff8c2a2420bce8 R08: ffff8c27acd52000 R09: ffffffff9c381839
R10: ffff8c32cd15f160 R11: ffffe39af1a9c800 R12: 0000000000000286
R13: ffff8c2a2420bc60 R14: 0000000000000000 R15: ffff8c32c177b150
FS: 00007f008cfa5700(0000) GS:ffff8c32cd140000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f000b584004 CR3: 00000008f37de000 CR4: 00000000000407e0
Call Trace:
[<ffffffff9c07e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9c07e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9c1ec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9c1edb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9c1edc83>] tlb_finish_mmu+0x23/0x30
[<ffffffff9c1f7474>] unmap_region+0xf4/0x140
[<ffffffff9c1f9a75>] do_munmap+0x2a5/0x480
[<ffffffff9c1f9cb5>] vm_munmap+0x65/0xb0
[<ffffffff9c1faf52>] SyS_munmap+0x22/0x30
[<ffffffff9c78dede>] system_call_fastpath+0x25/0x2a
[<ffffffff9c78de21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 d4 9c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1321404
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffacf1e25a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xfffffffface81fa8
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xfffffffface8201b
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffacffa0ce
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffacffbc4f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffacffbd83
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffad005904
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffad16c764
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x14
|
|
9
|
|
vmlinux |
0xffffffffad005f11
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffffad007f65
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffffad0081a5
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffffad0094d2
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x22
|
|
13
|
tracesys
|
vmlinux |
0xffffffffad5c562e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#49 stuck for 22s! [s1-agent:2275]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport binfmt_misc nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci ata_piix libahci
drm nfit crct10dif_pclmul mptspi crct10dif_common libata scsi_transport_spi libnvdimm crc32c_intel mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 49 PID: 2275 Comm: s1-agent Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff982cdeee3180 ti: ffff982ff1c0c000 task.ti: ffff982ff1c0c000
RIP: 0010:[<ffffffffacf1e25a>] [<ffffffffacf1e25a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff982ff1c0fcb0 EFLAGS: 00000202
RAX: 000000000000001a RBX: 00080000ace81b10 RCX: ffff982f9a520e58
RDX: 000000000000001a RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff982ff1c0fce8 R08: ffff983022c23400 R09: ffffffffad19c7f9
R10: ffff9834e9d5f160 R11: fffffd50efd14200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff982ff1c0fc60 R15: 00007f56e2ffd000
FS: 00007f571cff9700(0000) GS:ffff9834e9d40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f591bb01000 CR3: 0000000ad9828000 CR4: 00000000000607e0
Call Trace:
[<fffffffface81fa8>] native_flush_tlb_others+0xb8/0xc0
[<fffffffface8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffacffa0ce>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffacffbc4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffacffbd83>] tlb_finish_mmu+0x23/0x40
[<ffffffffad005904>] unmap_region+0xf4/0x150
[<ffffffffad16c764>] ? blk_finish_plug+0x14/0x50
[<ffffffffad005f11>] ? __vma_rb_erase+0x121/0x230
[<ffffffffad007f65>] do_munmap+0x2a5/0x480
[<ffffffffad0081a5>] vm_munmap+0x65/0xb0
[<ffffffffad0094d2>] SyS_munmap+0x22/0x30
[<ffffffffad5c562e>] tracesys+0xa6/0xcc
Code: 48 63 35 ae 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 c0 19 b5 ad f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 7d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #781326
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffadd150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffadc7d398
Build id: 3.10.0-1062.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffadc7d40b
Build id: 3.10.0-1062.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffaddea166
Build id: 3.10.0-1062.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffaddebc2f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffaddebd53
Build id: 3.10.0-1062.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffaddf5544
Build id: 3.10.0-1062.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffaddf5b01
Build id: 3.10.0-1062.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffaddf7b45
Build id: 3.10.0-1062.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffaddf7d85
Build id: 3.10.0-1062.el7.x86_64
|
0x65
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [python:127079]
Modules linked in: xt_nat veth binfmt_misc tcp_lp nls_utf8 isofs rfcomm fuse nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel btusb snd_ens1371 btrtl snd_rawmidi btbcm snd_ac97_codec btintel bluetooth ac97_bus aesni_intel snd_seq snd_seq_device lrw gf128mul snd_pcm glue_helper ablk_helper cryptd snd_timer joydev rfkill snd pcspkr sg vmw_vmci soundcore i2c_piix4 parport_pc parport
ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix crct10dif_pclmul crct10dif_common nfit scsi_transport_spi crc32c_intel mptscsih libata mptbase libnvdimm serio_raw e1000 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 127079 Comm: python Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffffa10fa13b0000 ti: ffffa10f81768000 task.ti: ffffa10f81768000
RIP: 0010:[<ffffffffadd150ca>] [<ffffffffadd150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa10f8176bcb0 EFLAGS: 00000202
RAX: 000000000000000a RBX: 00080000adc7d1d0 RCX: ffffa1113189fc10
RDX: 000000000000000a RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa10f8176bce8 R08: ffffa10f7fdd7000 R09: ffffffffadf7f629
R10: ffffa1113161f160 R11: ffffe82a8c79e600 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa10f8176bc60 R15: 00007f80a2fc0000
FS: 00007f8072ffd700(0000) GS:ffffa11131600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f80a2fc0000 CR3: 000000032a6ba000 CR4: 00000000007607f0
PKRU: 55555554
Call Trace:
[<ffffffffadc7d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffadc7d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffaddea166>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffaddebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffaddebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffffaddf5544>] unmap_region+0xf4/0x140
[<ffffffffaddf5b01>] ? __vma_rb_erase+0x121/0x220
[<ffffffffaddf7b45>] do_munmap+0x2a5/0x480
[<ffffffffaddf7d85>] vm_munmap+0x65/0xb0
Complete report #1197541
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fd5fa
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff81070b98
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff81070c68
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811b18c3
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b2be5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b9cb4
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811ba271
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811bc285
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811bc4c5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811bd5d2
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816c0715
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1c
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816c0661
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [qaxsafed:23170]
Modules linked in: qax_tq_base(OE) cfg80211 rfkill gsch(OE) redirfs(OE) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dsa_filter(POE) snd_hda_codec_generic snd_hda_intel cirrus snd_hda_codec ttm snd_hda_core drm_kms_helper snd_hwdep snd_seq snd_seq_device syscopyarea sysfillrect sysimgblt fb_sys_fops drm virtio_net virtio_console pcspkr virtio_scsi snd_pcm joydev snd_timer snd virtio_balloon sg soundcore shpchp
i2c_piix4 i2c_core virtio_pci virtio_ring virtio ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi ata_piix libata serio_raw floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: qax_tq_base]
CPU: 1 PID: 23170 Comm: qaxsafed Tainted: P OE ------------ 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: inspur Standard PC (i440FX + PIIX, 1996), BIOS 1.15.0-2.module_kos5+70+5e037788 04/01/2014
task: ffff880137bd8fd0 ti: ffff8800a7c80000 task.ti: ffff8800a7c80000
RIP: 0010:[<ffffffff810fd5fa>] [<ffffffff810fd5fa>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8800a7c83cc0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000810709d0 RCX: ffff88013fd1c3b8
RDX: 0000000000000002 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff8800a7c83cf8 R08: ffff880139495000 R09: ffffffff81328989
R10: ffff88013fc9ba00 R11: ffffea00017f1800 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8800a7c83c70 R15: 00007fa04bc53000
FS: 00007f9f82c18700(0000) GS:ffff88013fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa04bc53000 CR3: 00000000b0842000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81070b98>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff81070c68>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811b18c3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b2be5>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b9cb4>] unmap_region+0xf4/0x140
[<ffffffff811ba271>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811bc285>] do_munmap+0x295/0x470
[<ffffffff811bc4c5>] vm_munmap+0x65/0xb0
[<ffffffff811bd5d2>] SyS_munmap+0x22/0x30
[<ffffffff816c0715>] system_call_fastpath+0x1c/0x21
[<ffffffff816c0661>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 8e 1b a3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b2 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 1b a3 00 eb b7 0f b6 4d cc 4c 89
Complete report #1140051
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9cb16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9ca7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9ca7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9cbedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9cbef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9cbef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9cbf90e4
Build id: 3.10.0-1160.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9cbf96a1
Build id: 3.10.0-1160.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff9cbfb6d5
Build id: 3.10.0-1160.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff9cbfb915
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff9cbfcbb2
Build id: 3.10.0-1160.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff9d193f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [systemd:1]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw vmw_vsock_vmci_transport nf_conntrack vsock ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 ppdev vmw_balloon btusb snd_ens1371 btrtl snd_rawmidi btbcm snd_ac97_codec crc32_pclmul btintel ac97_bus ghash_clmulni_intel bluetooth snd_seq snd_seq_device aesni_intel lrw gf128mul
glue_helper ablk_helper snd_pcm cryptd joydev pcspkr snd_timer rfkill snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common nfit crc32c_intel mptspi libata scsi_transport_spi libnvdimm mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 14 PID: 1 Comm: systemd Kdump: loaded Tainted: G B L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9fcbdcfb0000 ti: ffff9fcbdcbbc000 task.ti: ffff9fcbdcbbc000
RIP: 0010:[<ffffffff9cb16c0a>] [<ffffffff9cb16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9fcbdcbbfcb0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 000800009ca7e6a0 RCX: ffff9fcfa9660808
RDX: 0000000000000001 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9fcbdcbbfce8 R08: ffff9fcbdd3dec00 R09: ffffffff9cd85f99
R10: ffff9fcfa999f160 R11: fffff90114995200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9fcbdcbbfc60 R15: 00007fcf3c745000
FS: 00007fcf3c72e940(0000) GS:ffff9fcfa9980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcf3c745000 CR3: 000000052aeac000 CR4: 00000000007407e0
PKRU: 55555554
Call Trace:
[<ffffffff9ca7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9ca7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9cbedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9cbef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9cbef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff9cbf90e4>] unmap_region+0xf4/0x140
[<ffffffff9cbf96a1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9cbfb6d5>] do_munmap+0x2a5/0x480
[<ffffffff9cbfb915>] vm_munmap+0x65/0xb0
[<ffffffff9cbfcbb2>] SyS_munmap+0x22/0x30
[<ffffffff9d193f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 9d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1041961
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa4b170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa4a7e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa4a7e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa4bee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa4bf013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa4bf0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffa4bf9a54
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc05753c4
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffffa4bfa011
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffffa4bfc045
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffffa4bfc285
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffffa4bfd582
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffffa5199f92
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa5199ed5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa2
|
|
15
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa4ace852
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xe2
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [java:681]
c4
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink sunrpc dm_mirror dm_region_hash dm_log dm_mod ext4 mbcache jbd2 ppdev sg parport_pc virtio_balloon virtio_rng parport i2c_piix4 pcspkr joydev ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops virtio_net ttm net_failover virtio_console virtio_blk failover drm ata_piix libata virtio_pci virtio_ring serio_raw floppy virtio drm_panel_orientation_quirks fuse
CPU: 8 PID: 681 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff8bef3c144200 ti: ffff8bef3c070000 task.ti: ffff8bef3c070000
RIP: 0010:[<ffffffffa4b170da>] [<ffffffffa4b170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8bef3c073cb0 EFLAGS: 00000202
RAX: 0000000000000014 RBX: ffffffffa4a7ddd0 RCX: ffff8befbf320518
RDX: 0000000000000014 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff8bef3c073ce8 R08: ffff8be119504800 R09: ffffffffa4d86c09
R10: ffff8befbf01f160 R11: ffffe7f13e718800 R12: 0000000000000286
R13: ffff8bef3c073c60 R14: 00007fb3340ba000 R15: 00007fb3340b9000
FS: 00007fb1fc8e6700(0000) GS:ffff8befbf000000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb3340b9000 CR3: 0000000fcec62000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa4a7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa4a7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa4bee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa4bf013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa4bf0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffa4bf9a54>] unmap_region+0xf4/0x140
[<ffffffffc05753c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffffa4bfa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa4bfc045>] do_munmap+0x2a5/0x480
[<ffffffffa4bfc285>] vm_munmap+0x65/0xb0
[<ffffffffa4bfd582>] SyS_munmap+0x22/0x30
[<ffffffffa5199f92>] system_call_fastpath+0x25/0x2a
[<ffffffffa5199ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffa4ace852>] smpboot_thread_fn+0xe2/0x1a0
Complete report #950124
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff98d1e07a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff98c81f48
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff98c81fbb
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff98df9f0e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff98dfba4f
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff98dfbb83
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff98e05524
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff98e05b31
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff98e07b75
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff98e07db5
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff98e090e2
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff993c539a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [360entclient:125542]
Modules linked in: nfring(OE) xt_NFQUEUE nfnetlink_log ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev sg lrw gf128mul joydev pcspkr glue_helper ablk_helper cryptd vmw_balloon vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sd_mod sysfillrect sysimgblt
fb_sys_fops crc_t10dif ttm crct10dif_generic ahci ata_piix libahci drm crct10dif_pclmul crct10dif_common crc32c_intel libata nfit libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: nfring]
CPU: 3 PID: 125542 Comm: 360entclient Kdump: loaded Tainted: GF OE ------------ 3.10.0-1160.81.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff93ce8a3b6300 ti: ffff93c723a54000 task.ti: ffff93c723a54000
RIP: 0010:[<ffffffff98d1e07a>] [<ffffffff98d1e07a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff93c723a57cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0008000098c81ab0 RCX: ffff93ce9d620650
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff93c723a57ce8 R08: ffff93c7ec55c000 R09: ffffffff98f9c139
R10: ffff93ce9d6df160 R11: ffffc3ead434ec00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff93c723a57c60 R15: 00007fbcc02fc000
FS: 00007fbcd7fff700(0000) GS:ffff93ce9d6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbcc02fc000 CR3: 00000006f17a8000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff98c81f48>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff98c81fbb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff98df9f0e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff98dfba4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff98dfbb83>] tlb_finish_mmu+0x23/0x40
[<ffffffff98e05524>] unmap_region+0xf4/0x150
[<ffffffff98e05b31>] ? __vma_rb_erase+0x121/0x230
[<ffffffff98e07b75>] do_munmap+0x2a5/0x480
[<ffffffff98e07db5>] vm_munmap+0x65/0xb0
[<ffffffff98e090e2>] SyS_munmap+0x22/0x30
[<ffffffff993c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 95 99 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #913116
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff97b16c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff97a7e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff97a7e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff97bedd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff97bef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff97bef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff97bf90e4
Build id: 3.10.0-1160.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff98186e74
Build id: 3.10.0-1160.el7.x86_64
|
0x594
|
|
9
|
|
vmlinux |
0xffffffff97bf96a1
Build id: 3.10.0-1160.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff97bfb6d5
Build id: 3.10.0-1160.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff97bfb915
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff97bfcbb2
Build id: 3.10.0-1160.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff98193f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [connection:30437]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg i2c_piix4 parport_pc vmw_vmci parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper
sd_mod syscopyarea crc_t10dif sysfillrect sysimgblt crct10dif_generic ahci fb_sys_fops ttm nfit libahci libnvdimm crct10dif_pclmul crct10dif_common drm ata_piix crc32c_intel serio_raw vmxnet3 libata drm_panel_orientation_quirks vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 30437 Comm: connection Kdump: loaded Not tainted 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff934e467f8000 ti: ffff934e523e4000 task.ti: ffff934e523e4000
RIP: 0010:[<ffffffff97b16c0a>] [<ffffffff97b16c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff934e523e7cb0 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 0008000097a7e6a0 RCX: ffff936c3f360678
RDX: 0000000000000005 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff934e523e7ce8 R08: ffff934dc5b57400 R09: ffffffff97d85f99
R10: ffff936c3f31f160 R11: ffffde1f8782ee00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff934e523e7c60 R15: 00007fb7fb827000
FS: 00007fb7f8afc700(0000) GS:ffff936c3f300000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb7c2362000 CR3: 0000000034842000 CR4: 00000000003607e0
Call Trace:
[<ffffffff97a7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff97a7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff97bedd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff97bef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff97bef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff97bf90e4>] unmap_region+0xf4/0x140
[<ffffffff98186e74>] ? __schedule+0x594/0x860
[<ffffffff97bf96a1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff97bfb6d5>] do_munmap+0x2a5/0x480
[<ffffffff97bfb915>] vm_munmap+0x65/0xb0
[<ffffffff97bfcbb2>] SyS_munmap+0x22/0x30
[<ffffffff98193f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 98 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #904439
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb791718a
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb787e258
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb787e2cb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb79ee6a6
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb79f016f
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb79f0293
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb79f9a84
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb79fa041
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb79fc025
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb79fc265
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb79fd562
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffffb7f9a226
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [sisamddaemon:2429]
Modules linked in: ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sisap(POE) sisevt(PE) sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon ppdev vmw_vmci sg joydev i2c_piix4 pcspkr parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod crc_t10dif crct10dif_generic drm nfit ata_piix libnvdimm crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 14 PID: 2429 Comm: sisamddaemon Kdump: loaded Tainted: P OE ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9eb723daa100 ti: ffff9eb6f9514000 task.ti: ffff9eb6f9514000
RIP: 0010:[<ffffffffb791718a>] [<ffffffffb791718a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9eb6f9517cb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: ffffffffb787ddd0 RCX: ffff9eb743aa0808
RDX: 0000000000000002 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9eb6f9517ce8 R08: ffff9eb2b0992400 R09: ffffffffb7b869f9
R10: ffff9eb743d9f160 R11: fffff0be82399600 R12: 0000000000000286
R13: ffff9eb6f9517c60 R14: 00007efe172ba000 R15: 00007efe172b9000
FS: 00007efd87fff700(0000) GS:ffff9eb743d80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efe172b9000 CR3: 0000000600ec8000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb787e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb787e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb79ee6a6>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb79f016f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb79f0293>] tlb_finish_mmu+0x23/0x30
[<ffffffffb79f9a84>] unmap_region+0xf4/0x140
[<ffffffffb79fa041>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb79fc025>] do_munmap+0x2a5/0x480
[<ffffffffb79fc265>] vm_munmap+0x65/0xb0
[<ffffffffb79fd562>] SyS_munmap+0x22/0x30
[<ffffffffb7f9a226>] tracesys+0xa6/0xcc
Code: 48 63 35 be 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 55 b8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 8d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #895426
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9f1150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9f07d398
Build id: 3.10.0-1062.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9f07d40b
Build id: 3.10.0-1062.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9f1ea166
Build id: 3.10.0-1062.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9f1ebc2f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9f1ebd53
Build id: 3.10.0-1062.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9f1f5544
Build id: 3.10.0-1062.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9f0e375c
Build id: 3.10.0-1062.el7.x86_64
|
0x14c
|
|
9
|
|
vmlinux |
0xffffffff9f1f5b01
Build id: 3.10.0-1062.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff9f1f7b45
Build id: 3.10.0-1062.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff9f1f7d85
Build id: 3.10.0-1062.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff9f1f9022
Build id: 3.10.0-1062.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff9f78bede
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9f78be21
Build id: 3.10.0-1062.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [Index Provenanc:156577]
Modules linked in: binfmt_misc iptable_filter ext4 ppdev mbcache jbd2 joydev sg virtio_balloon parport_pc parport pcspkr i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_console virtio_net virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix virtio_pci virtio_ring libata floppy serio_raw virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 9 PID: 156577 Comm: Index Provenanc Kdump: loaded Tainted: G L ------------ 3.10.0-1062.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffffa022696241c0 ti: ffffa01f51694000 task.ti: ffffa01f51694000
RIP: 0010:[<ffffffff9f1150ca>] [<ffffffff9f1150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa01f51697cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c00009f07d1d0 RCX: ffffa024dce204f0
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffffa01f51697ce8 R08: ffffa01e344ad400 R09: ffffffff9f37f629
R10: ffffa024dd05f160 R11: ffffd78f4b873c00 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa01f51697c60 R15: ffffa01f51697c60
FS: 00007f848696d700(0000) GS:ffffa024dd040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f84ea7a5000 CR3: 00000004e4944000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9f07d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9f07d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9f1ea166>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9f1ebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9f1ebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffff9f1f5544>] unmap_region+0xf4/0x140
[<ffffffff9f0e375c>] ? update_curr+0x14c/0x1e0
[<ffffffff9f1f5b01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9f1f7b45>] do_munmap+0x2a5/0x480
[<ffffffff9f1f7d85>] vm_munmap+0x65/0xb0
[<ffffffff9f1f9022>] SyS_munmap+0x22/0x30
[<ffffffff9f78bede>] system_call_fastpath+0x25/0x2a
[<ffffffff9f78be21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 d4 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #880661
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbc11e07a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbc081f48
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbc081fbb
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffbc1f9f0e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffbc1fba4f
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbc1fbb83
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffbc205524
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffbc205b31
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffbc207b75
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffbc207db5
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffbc2090e2
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffbc7c539a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 21s! [C2 CompilerThre:6994]
Modules linked in: af_packet_diag netlink_diag vsock_diag tcp_diag udp_diag inet_diag unix_diag xt_comment xt_conntrack iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter vmw_vsock_vmci_transport vsock sb_edac ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci parport_pc parport i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata mptspi scsi_transport_spi nfit mptscsih crct10dif_pclmul crct10dif_common crc32c_intel serio_raw libnvdimm mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash
dm_log dm_mod fuse
CPU: 4 PID: 6994 Comm: C2 CompilerThre Tainted: G W OEL ------------ 3.10.0-1160.81.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff990261a8d280 ti: ffff9902351e8000 task.ti: ffff9902351e8000
RIP: 0010:[<ffffffffbc11e07a>] [<ffffffffbc11e07a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9902351ebcb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000bc081ab0 RCX: ffff9909ffc20678
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff9902351ebce8 R08: ffff99032c413000 R09: ffffffffbc39c139
R10: ffff9909ffd1f160 R11: fffff9b841ecc800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9902351ebc60 R15: 00007f041c1c3000
FS: 00007f03e8128700(0000) GS:ffff9909ffd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f041c1c3000 CR3: 000000003059e000 CR4: 00000000000607e0
Call Trace:
[<ffffffffbc081f48>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbc081fbb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffbc1f9f0e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffffbc1fba4f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffbc1fbb83>] tlb_finish_mmu+0x23/0x40
[<ffffffffbc205524>] unmap_region+0xf4/0x150
[<ffffffffbc205b31>] ? __vma_rb_erase+0x121/0x230
[<ffffffffbc207b75>] do_munmap+0x2a5/0x480
[<ffffffffbc207db5>] vm_munmap+0x65/0xb0
[<ffffffffbc2090e2>] SyS_munmap+0x22/0x30
[<ffffffffbc7c539a>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 d5 bc f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #873384
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff82f1e23e
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff82e81fa8
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff82e8201b
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff82ffa07e
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xae
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff82ffbbff
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff82ffbd33
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff830058b4
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff82ed8f30
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x90
|
|
9
|
|
vmlinux |
0xffffffff83005ec1
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff83007f15
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff83008155
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff83009482
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff835c539a
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [grep:24344]
Modules linked in: vmw_vsock_vmci_transport vsock vfat fat ext4 mbcache jbd2 sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel pcspkr aesni_intel lrw joydev gf128mul glue_helper ppdev vmw_balloon ablk_helper sg cryptd parport_pc parport i2c_piix4 vmw_vmci tpm_crb ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci ata_piix sd_mod crc_t10dif libahci crct10dif_generic nfit drm libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 24344 Comm: grep Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.102.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.18227214.B64.2106252220 06/25/2021
task: ffff92a02471a100 ti: ffff929f02554000 task.ti: ffff929f02554000
RIP: 0010:[<ffffffff82f1e23e>] [<ffffffff82f1e23e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff929f02557cb0 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 0008000082e81b10 RCX: ffff92a13fd606c8
RDX: 0000000000000005 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff929f02557ce8 R08: ffff929e76353800 R09: ffffffff8319c7b9
R10: ffff92a13fd9f160 R11: fffff5b1103a0000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff929f02557c60 R15: 00007eff571a3000
FS: 00007eff5719c740(0000) GS:ffff92a13fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007eff569d3b30 CR3: 00000003d4a72000 CR4: 00000000001607e0
Call Trace:
[<ffffffff82e81fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff82e8201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff82ffa07e>] tlb_flush_mmu_tlbonly+0xae/0xd0
[<ffffffff82ffbbff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff82ffbd33>] tlb_finish_mmu+0x23/0x40
[<ffffffff830058b4>] unmap_region+0xf4/0x150
[<ffffffff82ed8f30>] ? __wake_up_common_lock+0x90/0xc0
[<ffffffff83005ec1>] ? __vma_rb_erase+0x121/0x230
[<ffffffff83007f15>] do_munmap+0x2a5/0x480
[<ffffffff83008155>] vm_munmap+0x65/0xb0
[<ffffffff83009482>] SyS_munmap+0x22/0x30
[<ffffffff835c539a>] system_call_fastpath+0x25/0x2a
Code: 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 b5 83 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #862075
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8e11760a
Build id: 3.10.0-1127.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8e07e8f8
Build id: 3.10.0-1127.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8e07e96b
Build id: 3.10.0-1127.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8e1eddb6
Build id: 3.10.0-1127.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8e1ef87f
Build id: 3.10.0-1127.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8e1ef9a3
Build id: 3.10.0-1127.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff8e1f91b4
Build id: 3.10.0-1127.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff8e1f9771
Build id: 3.10.0-1127.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff8e1fb7a5
Build id: 3.10.0-1127.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff8e1fb9e5
Build id: 3.10.0-1127.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff8e1fcc82
Build id: 3.10.0-1127.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffff8e793166
Build id: 3.10.0-1127.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [C1 CompilerThre:17699]
Modules linked in: ip6table_filter ip6_tables xt_owner tcp_diag inet_diag binfmt_misc iptable_filter vmw_vsock_vmci_transport vsock sb_edac ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr cryptd joydev sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod ahci crc_t10dif ata_piix libahci crct10dif_generic drm nfit libata crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: talpa_syscallhook]
CPU: 1 PID: 17699 Comm: C1 CompilerThre Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff990bb6c49070 ti: ffff98fcc6068000 task.ti: ffff98fcc6068000
RIP: 0010:[<ffffffff8e11760a>] [<ffffffff8e11760a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff98fcc606bcb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008e07e730 RCX: ffff990bbf220600
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff98fcc606bce8 R08: ffff98fd29526800 R09: ffffffff8e385099
R10: ffff990bbf25f160 R11: ffffe965c228f800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff98fcc606bc60 R15: 00007fa6374fb000
FS: 00007fa5febf0700(0000) GS:ffff990bbf240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa6374fb000 CR3: 000000095d474000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8e07e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8e07e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8e1eddb6>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8e1ef87f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8e1ef9a3>] tlb_finish_mmu+0x23/0x30
[<ffffffff8e1f91b4>] unmap_region+0xf4/0x140
[<ffffffff8e1f9771>] ? __vma_rb_erase+0x121/0x220
[<ffffffff8e1fb7a5>] do_munmap+0x2a5/0x480
[<ffffffff8e1fb9e5>] vm_munmap+0x65/0xb0
[<ffffffff8e1fcc82>] SyS_munmap+0x22/0x30
[<ffffffff8e793166>] tracesys+0xa6/0xcc
Code: 48 63 35 1e 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 0f d5 8e f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 7a c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #752469
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbdd150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbdc7d398
Build id: 3.10.0-1062.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbdc7d40b
Build id: 3.10.0-1062.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffbddea166
Build id: 3.10.0-1062.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffbddebc2f
Build id: 3.10.0-1062.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbddebd53
Build id: 3.10.0-1062.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffbddf5544
Build id: 3.10.0-1062.el7.x86_64
|
0xf4
|
|
8
|
|
qax_tq_base |
0xffffffffc06c375c
Build id: 3.10.0-1062.el7.x86_64
|
0xdc
|
|
9
|
|
vmlinux |
0xffffffffbddf5b01
Build id: 3.10.0-1062.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffffbddf7b45
Build id: 3.10.0-1062.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffffbddf7d85
Build id: 3.10.0-1062.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffffbddf9022
Build id: 3.10.0-1062.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffffbe38bede
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbe38be21
Build id: 3.10.0-1062.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [java:73637]
Modules linked in: qax_tq_base(OE) ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_common rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set ipt_rpfilter xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel veth xt_recent xt_nat xt_statistic ipt_REJECT nf_reject_ipv4 xt_comment xt_mark iptable_mangle ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi overlay(T) sunrpc ppdev pcspkr virtio_balloon joydev i2c_piix4 sg parport_pc parport ip_tables xfs libcrc32c
sr_mod cdrom ata_generic pata_acpi virtio_console virtio_net virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci virtio_ring floppy serio_raw virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: qax_tq_base]
CPU: 12 PID: 73637 Comm: java Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff957666a10000 ti: ffff9572d0eb0000 task.ti: ffff9572d0eb0000
RIP: 0010:[<ffffffffbdd150ca>] [<ffffffffbdd150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9572d0eb3cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000bdc7d1d0 RCX: ffff95777ce20568
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff9572d0eb3ce8 R08: ffff9568e95d5800 R09: ffffffffbdf7f629
R10: ffffcfe1fb12d2a0 R11: ffffd1aab0e6c200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9572d0eb3c60 R15: 00007fea254d2000
FS: 00007fe9809fe700(0000) GS:ffff95777d100000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fea254d2000 CR3: 0000000be73dc000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffbdc7d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbdc7d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffbddea166>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffbddebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffbddebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffffbddf5544>] unmap_region+0xf4/0x140
[<ffffffffc06c375c>] ? do_hook_ops+0xdc/0x110 [qax_tq_base]
[<ffffffffbddf5b01>] ? __vma_rb_erase+0x121/0x220
[<ffffffffbddf7b45>] do_munmap+0x2a5/0x480
[<ffffffffbddf7d85>] vm_munmap+0x65/0xb0
[<ffffffffbddf9022>] SyS_munmap+0x22/0x30
[<ffffffffbe38bede>] system_call_fastpath+0x25/0x2a
[<ffffffffbe38be21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 94 be f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #619236
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff943170da
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9427e258
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9427e2cb
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff943ee676
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff943f013f
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff943f0263
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff943f9a54
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff943fa011
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff943fc045
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff943fc285
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff943fd582
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff94995f92
Build id: 3.10.0-1160.42.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [dockerd:1872]
Modules linked in: binfmt_misc ip_vs_rr xt_ipvs ip_vs xt_REDIRECT nf_nat_redirect vxlan ip6_udp_tunnel udp_tunnel xt_mark xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper
cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 CyProtectDrv(POE) CyProtectDrvOpen(OE) ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect sysimgblt fb_sys_fops crc_t10dif ahci ttm crct10dif_generic ata_piix libahci drm crct10dif_pclmul nfit crct10dif_common crc32c_intel libata libnvdimm vmxnet3 serio_raw vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 11 PID: 1872 Comm: dockerd Kdump: loaded Tainted: P W OEL ------------ T 3.10.0-1160.42.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9990aa5b4200 ti: ffff999070edc000 task.ti: ffff999070edc000
RIP: 0010:[<ffffffff943170da>] [<ffffffff943170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff999070edfcb0 EFLAGS: 00000202
RAX: 0000000000000006 RBX: 000800009427ddd0 RCX: ffff9990ad7a0790
RDX: 0000000000000006 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff999070edfce8 R08: ffff998df5c6d800 R09: ffffffff94586bd9
R10: ffff9990ad8df160 R11: ffffe55dcb8c4400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff999070edfc60 R15: ffffffff943bd91e
FS: 00007fb7377fe700(0000) GS:ffff9990ad8c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb74804d008 CR3: 0000000409904000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9427e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9427e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff943ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff943f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff943f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffff943f9a54>] unmap_region+0xf4/0x140
[<ffffffff943fa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffff943fc045>] do_munmap+0x2a5/0x480
[<ffffffff943fc285>] vm_munmap+0x65/0xb0
[<ffffffff943fd582>] SyS_munmap+0x22/0x30
[<ffffffff94995f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 f5 94 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #534602
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff82116e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8207e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8207e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff821ede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff821ef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff821efa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff821f9264
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff821f9821
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff821fb855
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff821fba95
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff821fcd92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff82794f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#103 stuck for 23s! [evdefer/10:14358]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14203(E) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache uinput devlink binfmt_misc ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter falcon_lsm_pinned_14108(E) snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq snd_seq_device snd_pcm qxl snd_timer ttm drm_kms_helper iTCO_wdt syscopyarea
crc32_pclmul ghash_clmulni_intel aesni_intel lrw sysfillrect gf128mul sysimgblt glue_helper ppdev fb_sys_fops ablk_helper snd iTCO_vendor_support joydev cryptd parport_pc drm virtio_rng pcspkr i2c_i801 lpc_ich soundcore virtio_balloon parport drm_panel_orientation_quirks nfsd nfs_acl lockd auth_rpcgss grace sunrpc ip_tables xfs libcrc32c virtio_net net_failover virtio_console virtio_blk failover crct10dif_pclmul ahci virtio_pci libahci crct10dif_common serio_raw crc32c_intel libata virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: falcon_kal]
CPU: 103 PID: 14358 Comm: evdefer/10 Kdump: loaded Tainted: P W EL ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: Red Hat KVM/RHEL-AV, BIOS 1.13.0-2.module+el8.4.0+534+4680a14e 04/01/2014
task: ffff8b1f00b84200 ti: ffff8b13a6b74000 task.ti: ffff8b13a6b74000
RIP: 0010:[<ffffffff82116e4a>] [<ffffffff82116e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b13a6b77cb0 EFLAGS: 00000202
RAX: 0000000000000035 RBX: 000800008207e6a0 RCX: ffff8cfbac361428
RDX: 0000000000000035 RSI: 00000000000000ff RDI: 0000000000000000
RBP: ffff8b13a6b77ce8 R08: ffff8b10ac352400 R09: ffffffff82386479
R10: ffff8cfbacfdf160 R11: fffff7083281ea00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8b13a6b77c60 R15: 00007fcdfe03b000
FS: 00007fcdef2a6700(0000) GS:ffff8cfbacfc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcdfe03b000 CR3: 0000000470f76000 CR4: 0000000000760ee0
PKRU: 55555554
Call Trace:
[<ffffffff8207e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8207e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff821ede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff821ef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff821efa73>] tlb_finish_mmu+0x23/0x30
[<ffffffff821f9264>] unmap_region+0xf4/0x140
[<ffffffff821f9821>] ? __vma_rb_erase+0x121/0x220
[<ffffffff821fb855>] do_munmap+0x2a5/0x480
[<ffffffff821fba95>] vm_munmap+0x65/0xb0
[<ffffffff821fcd92>] SyS_munmap+0x22/0x30
[<ffffffff82794f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 d5 82 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #520961
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffadd170da
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffadc7e258
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffadc7e2cb
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffaddee676
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffaddf013f
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffaddf0263
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffaddf9a54
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffaddfa011
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffaddfc045
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffaddfc285
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffaddfd582
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffae395f92
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffae395ed5
Build id: 3.10.0-1160.41.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [f2b/f.sshd:1828]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink nfsv3 nfs fscache joydev i2c_piix4 sg virtio_rng virtio_balloon pcspkr edac_mce_amd openafs(POE) nfsd auth_rpcgss nfs_acl lockd grace binfmt_misc sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_net net_failover virtio_console virtio_scsi failover qxl drm_kms_helper crct10dif_pclmul syscopyarea crct10dif_common sysfillrect crc32_pclmul sysimgblt fb_sys_fops crc32c_intel ttm ghash_clmulni_intel ata_piix
drm libata aesni_intel serio_raw glue_helper lrw gf128mul virtio_pci floppy ablk_helper cryptd virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 1828 Comm: f2b/f.sshd Kdump: loaded Tainted: P OE ------------ 3.10.0-1160.41.1.el7.x86_64 #1
Hardware name: oVirt RHEL, BIOS 1.16.0-1.module_el8.7.0+1140+ff0772f9 04/01/2014
task: ffff9052e2f1c200 ti: ffff9052e1cc4000 task.ti: ffff9052e1cc4000
RIP: 0010:[<ffffffffadd170da>] [<ffffffffadd170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9052e1cc7cb0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000adc7ddd0 RCX: ffff9052f32e0600
RDX: 0000000000000003 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9052e1cc7ce8 R08: ffff9051382fe800 R09: ffffffffadf86bd9
R10: ffff9052f325f160 R11: fffff5baccc7b200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9052e1cc7c60 R15: 00007f79f88f7000
FS: 00007f79e9be3700(0000) GS:ffff9052f3240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f79f88f7000 CR3: 0000000035ac0000 CR4: 00000000000407e0
Call Trace:
[<ffffffffadc7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffadc7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffaddee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffaddf013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffaddf0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffaddf9a54>] unmap_region+0xf4/0x140
[<ffffffffaddfa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffffaddfc045>] do_munmap+0x2a5/0x480
[<ffffffffaddfc285>] vm_munmap+0x65/0xb0
[<ffffffffaddfd582>] SyS_munmap+0x22/0x30
[<ffffffffae395f92>] system_call_fastpath+0x25/0x2a
[<ffffffffae395ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 95 ae f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #513188
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa1916c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa187e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa187e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa19edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa19ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa19ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffa19f90e4
Build id: 3.10.0-1160.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffa1a4d060
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
9
|
|
vmlinux |
0xffffffffa19f96a1
Build id: 3.10.0-1160.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffffa19fb6d5
Build id: 3.10.0-1160.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffffa19fb915
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffffa19fcbb2
Build id: 3.10.0-1160.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffffa1f93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa1f93ed5
Build id: 3.10.0-1160.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 50s! [DrCOMX:10033]
Modules linked in: tun devlink bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 snd_rawmidi ghash_clmulni_intel snd_ac97_codec ppdev ac97_bus snd_seq snd_seq_device vmw_balloon snd_pcm aesni_intel lrw gf128mul snd_timer glue_helper ablk_helper snd cryptd vmw_vmci sg pcspkr joydev soundcore i2c_piix4 parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 mptspi scsi_transport_spi mptscsih drm mptbase nfit libnvdimm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 0 PID: 10033 Comm: DrCOMX Kdump: loaded Not tainted 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff97f393bb2100 ti: ffff97f2fb258000 task.ti: ffff97f2fb258000
RIP: 0010:[<ffffffffa1916c0a>] [<ffffffffa1916c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff97f2fb25bcb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: ffffffffa187e6a0 RCX: ffff97f45ee9fc10
RDX: 0000000000000002 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff97f2fb25bce8 R08: ffff97edbfd6f400 R09: ffffffffa1b85f99
R10: ffff97f45ee1f160 R11: fffff0d040d76c00 R12: 0000000000000286
R13: ffff97f2fb25bc60 R14: 00007f624a9a8000 R15: 00007f624a9a7000
FS: 00007f623f7fe700(0000) GS:ffff97f45ee00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f624a9a7000 CR3: 00000006c8462000 CR4: 00000000000407f0
Call Trace:
[<ffffffffa187e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa187e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa19edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa19ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa19ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffffa19f90e4>] unmap_region+0xf4/0x140
[<ffffffffa1a4d060>] ? do_sync_write+0x90/0xe0
[<ffffffffa19f96a1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa19fb6d5>] do_munmap+0x2a5/0x480
[<ffffffffa19fb915>] vm_munmap+0x65/0xb0
[<ffffffffa19fcbb2>] SyS_munmap+0x22/0x30
[<ffffffffa1f93f92>] system_call_fastpath+0x25/0x2a
[<ffffffffa1f93ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 a2 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #475178
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffff9fd163d2
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x102
|
|
2
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9fc7e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
3
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9fdec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
4
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9fdedb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9fdedc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff9fdf7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff9fdf7a31
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff9fdf9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff9fdf9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff9fdfaf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa038dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa038de21
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [python:119291]
Modules linked in: binfmt_misc xt_multiport nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc ppdev virtio_balloon pcspkr sg joydev parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console virtio_blk
virtio_scsi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 119291 Comm: python Kdump: loaded Tainted: G L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9db349eb9070 ti: ffff9db03a400000 task.ti: ffff9db03a400000
RIP: 0010:[<ffffffff9fd163d2>] [<ffffffff9fd163d2>] generic_exec_single+0x102/0x1c0
RSP: 0018:ffff9db03a403c20 EFLAGS: 00000202
RAX: 00000000000000a0 RBX: ffff9db03a403bf0 RCX: 0000000000000020
RDX: 00000000ffffffff RSI: 00000000000000a0 RDI: 0000000000000282
RBP: ffff9db03a403c70 R08: ffffffffa0413cc8 R09: ffffffff9fe00718
R10: ffff9db3ace1f0a0 R11: fffff6975071ce80 R12: ffffffff9fdc8144
R13: ffff9db03a403cb8 R14: 0000000000000002 R15: 0000000000000000
FS: 00007fef49541740(0000) GS:ffff9db3ace00000(0000) knlGS:0000000000000000
[<ffffffff9fc7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9fdec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9fdedb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9fdedc83>] tlb_finish_mmu+0x23/0x30
[<ffffffff9fdf7474>] unmap_region+0xf4/0x140
[<ffffffff9fdf7a31>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9fdf9a75>] do_munmap+0x2a5/0x480
[<ffffffff9fdf9cb5>] vm_munmap+0x65/0xb0
[<ffffffff9fdfaf52>] SyS_munmap+0x22/0x30
[<ffffffffa038dede>] system_call_fastpath+0x25/0x2a
[<ffffffffa038de21>] ? system_call_after_swapgs+0xae/0x146
Code: 89 de 48 03 14 c5 20 e8 94 a0 48 89 df e8 37 23 28 00 84 c0 75 53 45 85 ed 74 16 f6 43 20 01 74 10 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28 00 00
Complete report #439203
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9f91720a
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9f87e258
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9f87e2cb
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9f9ee676
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9f9f013f
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9f9f0263
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9f9f9a54
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9f9fa011
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff9f9fc045
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff9f9fc285
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff9f9fd582
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffff9ff9a226
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [metricbeat:5491]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13501(E) snd_hda_codec_generic snd_hda_intel snd_hda_codec iosf_mbi snd_hda_core snd_hwdep snd_seq crc32_pclmul ghash_clmulni_intel snd_seq_device aesni_intel lrw snd_pcm gf128mul glue_helper pcspkr snd_timer ppdev ablk_helper snd joydev cryptd virtio_rng soundcore parport_pc i2c_piix4 parport virtio_balloon ip_tables xfs libcrc32c ata_generic pata_acpi virtio_console virtio_blk qxl drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm e1000 libata crct10dif_pclmul crct10dif_common crc32c_intel virtio_pci serio_raw virtio_ring floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 5491 Comm: metricbeat Tainted: P E ------------ 3.10.0-1160.66.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7860+a7792d29 04/01/2014
task: ffff8b2310694200 ti: ffff8b21a1e20000 task.ti: ffff8b21a1e20000
RIP: 0010:[<ffffffff9f91720a>] [<ffffffff9f91720a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b21a1e23cb0 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 000800009f87ddd0 RCX: ffff8b233fd5fc10
RDX: 0000000000000005 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff8b21a1e23ce8 R08: ffff8b22ffd4f800 R09: ffffffff9fb869e9
R10: ffff8b233fc1f160 R11: ffffd1dec092a000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8b21a1e23c60 R15: 00007fc009e85000
FS: 00007fbfbbfff700(0000) GS:ffff8b233fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc009e85000 CR3: 00000001b8ef6000 CR4: 00000000001606f0
Call Trace:
[<ffffffff9f87e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9f87e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9f9ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9f9f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9f9f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffff9f9f9a54>] unmap_region+0xf4/0x140
[<ffffffff9f9fa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9f9fc045>] do_munmap+0x2a5/0x480
[<ffffffff9f9fc285>] vm_munmap+0x65/0xb0
[<ffffffff9f9fd582>] SyS_munmap+0x22/0x30
[<ffffffff9ff9a226>] tracesys+0xa6/0xcc
Code: 48 63 35 9e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 55 a0 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #406442
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9e7170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9e67e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9e67e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9e7ee676
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9e7f013f
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9e7f0263
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9e7f9a54
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc05133c4
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffff9e7fa011
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff9e7fc045
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff9e7fc285
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff9e7fd582
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff9ed99f92
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [hz._hzInstance_:57026]
Modules linked in: synosnap(OE) veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock bnep sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 btusb snd_rawmidi btrtl snd_ac97_codec btbcm sb_edac ac97_bus btintel snd_seq iosf_mbi
snd_seq_device snd_pcm sg bluetooth crc32_pclmul ghash_clmulni_intel aesni_intel rfkill lrw gf128mul glue_helper ppdev ablk_helper snd_timer snd vmw_balloon joydev pcspkr cryptd soundcore parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi ata_piix nfit scsi_transport_spi drm mptscsih libata libnvdimm e1000 serio_raw mptbase drm_panel_orientation_quirks fuse
CPU: 10 PID: 57026 Comm: hz._hzInstance_ Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffffa12719ba9080 ti: ffffa127433e4000 task.ti: ffffa127433e4000
RIP: 0010:[<ffffffff9e7170da>] [<ffffffff9e7170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa127433e7cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009e67ddd0 RCX: ffffa1277d020768
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa127433e7ce8 R08: ffffa122afd88800 R09: ffffffff9e986c09
R10: ffffa1277d29f160 R11: ffffe71c42c9c400 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa127433e7c60 R15: 00007fa35c065000
FS: 00007fa33f7f7700(0000) GS:ffffa1277d280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa35c065000 CR3: 000000063c186000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9e67e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9e67e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9e7ee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9e7f013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9e7f0263>] tlb_finish_mmu+0x23/0x30
[<ffffffff9e7f9a54>] unmap_region+0xf4/0x140
[<ffffffffc05133c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffff9e7fa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9e7fc045>] do_munmap+0x2a5/0x480
[<ffffffff9e7fc285>] vm_munmap+0x65/0xb0
[<ffffffff9e7fd582>] SyS_munmap+0x22/0x30
[<ffffffff9ed99f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 35 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #370515
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb7f1519e
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb7e7d478
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb7e7d4eb
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb7fea756
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb7fec21f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb7fec343
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb7ff5b34
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb7ff60f1
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb7ff8135
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb7ff8375
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb7ff9612
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffb858bede
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [evdefer/3:14855]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13109(E) falcon_lsm_pinned_13005(E) nls_utf8 isofs fuse rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache devlink binfmt_misc falcon_lsm_pinned_12905(E) ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_nat nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_hda_codec_generic
lrw gf128mul glue_helper ablk_helper cryptd snd_hda_intel joydev snd_hda_codec virtio_balloon snd_hda_core sg snd_hwdep snd_seq parport_pc snd_seq_device parport snd_pcm snd_timer snd pcspkr i2c_piix4 soundcore auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata serio_raw crct10dif_pclmul crct10dif_common virtio_pci crc32c_intel floppy virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: falcon_kal]
CPU: 3 PID: 14855 Comm: evdefer/3 Kdump: loaded Tainted: P EL ------------ 3.10.0-1062.4.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffffa0d716e28000 ti: ffffa0d711364000 task.ti: ffffa0d711364000
RIP: 0010:[<ffffffffb7f1519e>] [<ffffffffb7f1519e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffffa0d711367cb0 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00080000b7e7d2b0 RCX: ffffa0d71f160370
RDX: 0000000000000005 RSI: 0000000000000018 RDI: 0000000000000000
RBP: ffffa0d711367ce8 R08: ffffa0a8f6c07c00 R09: ffffffffb817fe99
R10: ffffa0d71f0df160 R11: fffffbf03cb5b400 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa0d711367c60 R15: ffffffffb7e7d2b0
FS: 00007fb0f7ab4700(0000) GS:ffffa0d71f0c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000095fe9003 CR3: 0000002e66b96000 CR4: 00000000001606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb7e7d478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb7e7d4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb7fea756>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb7fec21f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb7fec343>] tlb_finish_mmu+0x23/0x30
[<ffffffffb7ff5b34>] unmap_region+0xf4/0x140
[<ffffffffb7ff60f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb7ff8135>] do_munmap+0x2a5/0x480
[<ffffffffb7ff8375>] vm_munmap+0x65/0xb0
[<ffffffffb7ff9612>] SyS_munmap+0x22/0x30
[<ffffffffb858bede>] system_call_fastpath+0x25/0x2a
Code: 66 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d6 b4 b8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 3d 66 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #364195
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff87316a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8727e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8727e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff873ec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff873edb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff873edc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff873f7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff873f7a31
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff873f9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff873f9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff873faf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff8798dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff8798de21
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [360entclient:3552]
Modules linked in: tcp_lp fuse kbase(OE) xt_NFQUEUE nfnetlink_log nfnetlink nfring(OE) vmsecmod(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nf_conntrack_ipv6 nf_defrag_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache sunrpc ppdev sg parport_pc pcspkr parport virtio_balloon joydev i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_console virtio_net virtio_blk virtio_scsi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci virtio_ring serio_raw floppy virtio
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmsec_nfq]
CPU: 9 PID: 3552 Comm: 360entclient Kdump: loaded Tainted: GF OEL ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9eaebee5a0e0 ti: ffff9eaeadb50000 task.ti: ffff9eaeadb50000
RIP: 0010:[<ffffffff87316a7a>] [<ffffffff87316a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9eaeadb53cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008727e2b0 RCX: ffff9eaf7ce20540
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff9eaeadb53ce8 R08: ffff9ea0e95dcc00 R09: ffffffff87581839
R10: ffff9eaf7d05f160 R11: ffffdd7dffdada00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9eaeadb53c60 R15: 00007f20dadc8000
FS: 00007f21377e4700(0000) GS:ffff9eaf7d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f20dadc8000 CR3: 0000000ff1af8000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8727e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8727e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff873ec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff873edb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff873edc83>] tlb_finish_mmu+0x23/0x30
[<ffffffff873f7474>] unmap_region+0xf4/0x140
[<ffffffff873f7a31>] ? __vma_rb_erase+0x121/0x220
[<ffffffff873f9a75>] do_munmap+0x2a5/0x480
[<ffffffff873f9cb5>] vm_munmap+0x65/0xb0
[<ffffffff873faf52>] SyS_munmap+0x22/0x30
[<ffffffff8798dede>] system_call_fastpath+0x25/0x2a
[<ffffffff8798de21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 f4 87 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #343782
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffadd16a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffadc7e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffadc7e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffaddec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffaddedb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffaddedc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffaddf7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc060e0c4
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffffaddf7a31
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffffaddf9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffffaddf9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffffaddfaf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffffae38dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffae38de21
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [icsfilesec:116627]
Modules linked in: tcp_lp fuse kbase(OE) xt_NFQUEUE nfnetlink_log nfnetlink nfring(OE) vmsecmod(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nf_conntrack_ipv6 nf_defrag_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache sunrpc ppdev joydev virtio_balloon sg i2c_piix4 pcspkr parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_console virtio_net virtio_blk virtio_scsi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_pci virtio_ring serio_raw floppy virtio
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmsec_nfq]
CPU: 8 PID: 116627 Comm: icsfilesec Kdump: loaded Tainted: GF OEL ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9c1d36613150 ti: ffff9c2ce82c4000 task.ti: ffff9c2ce82c4000
RIP: 0010:[<ffffffffadd16a7a>] [<ffffffffadd16a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c2ce82c7cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000adc7e2b0 RCX: ffff9c2cfce20518
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff9c2ce82c7ce8 R08: ffff9c1e695dc800 R09: ffffffffadf81839
R10: ffff9c2cfd01f160 R11: fffff0548b9aec00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9c2ce82c7c60 R15: 00007ff6cee16000
FS: 00007ff69bfff700(0000) GS:ffff9c2cfd000000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff6cee16000 CR3: 00000001db766000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffadc7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffadc7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffaddec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffaddedb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffaddedc83>] tlb_finish_mmu+0x23/0x30
[<ffffffffaddf7474>] unmap_region+0xf4/0x140
[<ffffffffc060e0c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffffaddf7a31>] ? __vma_rb_erase+0x121/0x220
[<ffffffffaddf9a75>] do_munmap+0x2a5/0x480
[<ffffffffaddf9cb5>] vm_munmap+0x65/0xb0
[<ffffffffaddfaf52>] SyS_munmap+0x22/0x30
[<ffffffffae38dede>] system_call_fastpath+0x25/0x2a
[<ffffffffae38de21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 94 ae f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #332232
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff86516dda
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8647e868
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8647e8db
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff865ede26
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff865ef8ef
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff865efa13
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff865f9204
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc05813c4
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffff865f97c1
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff865fb7f5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff865fba35
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff865fcd32
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff86b95f92
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff86b95ed5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 24s! [pcsd:1213]
Modules linked in: ext4 mbcache jbd2 drbd(OE) squashfs loop iosf_mbi kvm_intel kvm snd_hda_codec_generic irqbypass snd_hda_intel crc32_pclmul snd_hda_codec ghash_clmulni_intel aesni_intel snd_hda_core snd_hwdep snd_seq lrw gf128mul sg joydev pcspkr glue_helper snd_seq_device snd_pcm ablk_helper snd_timer snd soundcore cryptd i2c_piix4 virtio_balloon nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_console virtio_blk qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci 8139too e1000 ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw virtio_pci virtio_ring virtio 8139cp drm_panel_orientation_quirks mii floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 1 PID: 1213 Comm: pcsd Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ArchLinux 1.15.0-1 04/01/2014
task: ffff9c18a9dbb180 ti: ffff9c180328c000 task.ti: ffff9c180328c000
RIP: 0010:[<ffffffff86516dda>] [<ffffffff86516dda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c180328fcb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008647e6a0 RCX: ffff9c18bfc20400
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9c180328fce8 R08: ffff9c18b9f55800 R09: ffffffff86786419
R10: ffff9c18bfc9f160 R11: fffff975024f4900 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9c180328fc60 R15: 00007fe30b8bc000
FS: 00007fe2edc0d700(0000) GS:ffff9c18bfc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe30b8bc000 CR3: 000000013196c000 CR4: 0000000000060ee0
Call Trace:
[<ffffffff8647e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8647e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff865ede26>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff865ef8ef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff865efa13>] tlb_finish_mmu+0x23/0x30
[<ffffffff865f9204>] unmap_region+0xf4/0x140
[<ffffffffc05813c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffff865f97c1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff865fb7f5>] do_munmap+0x2a5/0x480
[<ffffffff865fba35>] vm_munmap+0x65/0xb0
[<ffffffff865fcd32>] SyS_munmap+0x22/0x30
[<ffffffff86b95f92>] system_call_fastpath+0x25/0x2a
[<ffffffff86b95ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 0e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 15 87 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 dd 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #325695
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff84316c0e
Build id: 3.10.0-1160.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8427e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8427e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff843edd06
Build id: 3.10.0-1160.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff843ef7cf
Build id: 3.10.0-1160.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff843ef8f3
Build id: 3.10.0-1160.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff843f90e4
Build id: 3.10.0-1160.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc06bd3c4
Build id: 3.10.0-1160.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffff843f96a1
Build id: 3.10.0-1160.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff843fb6d5
Build id: 3.10.0-1160.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff843fb915
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff843fcbb2
Build id: 3.10.0-1160.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff84993f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 28s! [pool:128888]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 sb_edac snd_rawmidi snd_ac97_codec coretemp ac97_bus iosf_mbi crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device aesni_intel lrw snd_pcm gf128mul glue_helper snd_timer ablk_helper cryptd snd
ppdev vmw_balloon soundcore pcspkr joydev sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi libata mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 128888 Comm: pool Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff88fa919fa100 ti: ffff88fa0cfe4000 task.ti: ffff88fa0cfe4000
RIP: 0010:[<ffffffff84316c0e>] [<ffffffff84316c0e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff88fa0cfe7cb0 EFLAGS: 00000202
RAX: 000000000000000a RBX: 000800008427e6a0 RCX: ffff8905bd8a0650
RDX: 000000000000000a RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88fa0cfe7ce8 R08: ffff88f724344800 R09: ffffffff84585f99
R10: ffff8905bd6df160 R11: ffffe08a53836a00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff88fa0cfe7c20 R15: 0000000000000282
FS: 00007fe24a776700(0000) GS:ffff8905bd6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe25fad6000 CR3: 0000000110ba4000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8427e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8427e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff843edd06>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff843ef7cf>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff843ef8f3>] tlb_finish_mmu+0x23/0x30
[<ffffffff843f90e4>] unmap_region+0xf4/0x140
[<ffffffffc06bd3c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffff843f96a1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff843fb6d5>] do_munmap+0x2a5/0x480
[<ffffffff843fb915>] vm_munmap+0x65/0xb0
[<ffffffff843fcbb2>] SyS_munmap+0x22/0x30
[<ffffffff84993f92>] system_call_fastpath+0x25/0x2a
Code: 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 f5 84 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #280635
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb4f170da
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb4e7e258
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb4e7e2cb
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb4fee676
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb4ff013f
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb4ff0263
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb4ff9a54
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb4ffa011
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb4ffc045
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb4ffc285
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb4ffd582
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffffb5596226
Build id: 3.10.0-1160.45.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [Ruby-0-Thread-1:6355]
Modules linked in: vboxnetadp(OE) vboxnetflt(OE) vboxdrv(OE) veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) sb_edac sg i2c_piix4 iosf_mbi crc32_pclmul ghash_clmulni_intel virtio_balloon pcspkr aesni_intel lrw joydev gf128mul glue_helper ablk_helper cryptd ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom virtio_net net_failover virtio_scsi failover ata_generic pata_acpi bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ata_piix ttm drm libata crct10dif_pclmul crct10dif_common crc32c_intel virtio_pci virtio_ring serio_raw virtio drm_panel_orientation_quirks floppy dm_mirror
dm_region_hash dm_log dm_mod
CPU: 1 PID: 6355 Comm: Ruby-0-Thread-1 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
task: ffff8e5073f05280 ti: ffff8e4f40264000 task.ti: ffff8e4f40264000
RIP: 0010:[<ffffffffb4f170da>] [<ffffffffb4f170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8e4f40267cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c0000b4e7ddd0 RCX: ffff8e507fc20600
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff8e4f40267ce8 R08: ffff8e4fbaee9000 R09: ffffffffb5186bf9
R10: ffff8e507fc5f160 R11: ffffefc7077d9600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8e4f40267c60 R15: 00007f63d7501000
FS: 00007f629631d700(0000) GS:ffff8e507fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f63d7501000 CR3: 0000000234c0a000 CR4: 0000000000160fe0
Call Trace:
[<ffffffffb4e7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb4e7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb4fee676>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb4ff013f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb4ff0263>] tlb_finish_mmu+0x23/0x30
[<ffffffffb4ff9a54>] unmap_region+0xf4/0x140
[<ffffffffb4ffa011>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb4ffc045>] do_munmap+0x2a5/0x480
[<ffffffffb4ffc285>] vm_munmap+0x65/0xb0
[<ffffffffb4ffd582>] SyS_munmap+0x22/0x30
[<ffffffffb5596226>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 b5 b5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #268919
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa0b16e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa0a7e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa0a7e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa0bede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa0bef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa0befa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffa0bf9264
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffa0bf9821
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffa0bfb855
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffa0bfba95
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffa0bfcd92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffa1194f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [UndertowEventHa:5658]
Modules linked in: gsch(OE) redirfs(OE) bmhook(OE) tmhook(OE) dsa_filter(POE) dsa_filter_hook(OE) tcp_lp rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 ppdev vmw_balloon sg joydev parport_pc parport sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul i2c_piix4
vmw_vmci glue_helper ablk_helper pcspkr cryptd ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi crct10dif_pclmul crct10dif_common mptscsih libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: tmhook]
CPU: 6 PID: 5658 Comm: UndertowEventHa Kdump: loaded Tainted: P OEL ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8ce6fc4bb180 ti: ffff8ce56ce8c000 task.ti: ffff8ce56ce8c000
RIP: 0010:[<ffffffffa0b16e4a>] [<ffffffffa0b16e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8ce56ce8fcb0 EFLAGS: 00000202
RAX: 000000000000000b RBX: 00080000a0a7e6a0 RCX: ffff8ce70d8e06c8
RDX: 000000000000000b RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8ce56ce8fce8 R08: ffff8cdc21bac400 R09: ffffffffa0d86479
R10: ffff8ce70d79f160 R11: ffffd5b9c4ccfe00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8ce56ce8fc60 R15: 00007f58749a1000
FS: 00007f5869e28700(0000) GS:ffff8ce70d780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f58749a1000 CR3: 0000000a8aad2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa0a7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa0a7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa0bede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa0bef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa0befa73>] tlb_finish_mmu+0x23/0x30
[<ffffffffa0bf9264>] unmap_region+0xf4/0x140
[<ffffffffa0bf9821>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa0bfb855>] do_munmap+0x2a5/0x480
[<ffffffffa0bfba95>] vm_munmap+0x65/0xb0
[<ffffffffa0bfcd92>] SyS_munmap+0x22/0x30
[<ffffffffa1194f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 75 a1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #245149
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff94516dda
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9447e868
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9447e8db
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff945ede26
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff945ef8ef
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff945efa13
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff945f9204
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff945f97c1
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff945fb7f5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff945fba35
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff945fcd32
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff94b95f92
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff94b95ed5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [python3:23282]
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [mongod:23099]
Modules linked in: binfmt_misc vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev pcspkr sg vmw_balloon vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect sysimgblt crc_t10dif fb_sys_fops crct10dif_generic ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw nfit vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 23099 Comm: mongod Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffffa0af5b486300 ti: ffffa0b0e60f4000 task.ti: ffffa0b0e60f4000
RIP: 0010:[<ffffffff94516dda>] [<ffffffff94516dda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa0b0e60f7cb0 EFLAGS: 00000202
RAX: 0000000000000009 RBX: ffffffff9447e6a0 RCX: ffffa0b0ffce0728
RDX: 0000000000000009 RSI: 000000000000000c RDI: 0000000000000000
RBP: ffffa0b0e60f7ce8 R08: ffffa0afffc17c00 R09: ffffffff94786419
R10: ffffa0b0ffc1f160 R11: fffff5be889ad400 R12: 0000000000000286
R13: ffffa0b0e60f7c60 R14: 00007fe9f1036000 R15: 00007fe9f1035000
FS: 00007fe998421700(0000) GS:ffffa0b0ffc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe9f1035000 CR3: 000000014117e000 CR4: 00000000000407e0
Call Trace:
[<ffffffff9447e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9447e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff945ede26>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff945ef8ef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff945efa13>] tlb_finish_mmu+0x23/0x30
[<ffffffff945f9204>] unmap_region+0xf4/0x140
[<ffffffff945f97c1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff945fb7f5>] do_munmap+0x2a5/0x480
[<ffffffff945fba35>] vm_munmap+0x65/0xb0
[<ffffffff945fcd32>] SyS_munmap+0x22/0x30
[<ffffffff94b95f92>] system_call_fastpath+0x25/0x2a
[<ffffffff94b95ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 0e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 15 95 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 dd 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #160732
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff88b16a7a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff88a7e478
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff88a7e4eb
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff88bec096
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff88bedb5f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff88bedc83
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff88bf7474
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff88bf7a31
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff88bf9a75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff88bf9cb5
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff88bfaf52
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffff8918e172
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [pool-2996-threa:2893]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace sunrpc fscache xt_set ip_set_hash_net vxlan ip6_udp_tunnel udp_tunnel xt_nat xt_mark xfrm6_mode_tunnel xfrm4_mode_tunnel veth esp4 drbg ansi_cprng ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon sg parport_pc parport joydev pcspkr i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci ata_piix crct10dif_pclmul crct10dif_common crc32c_intel nfit libata serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 2893 Comm: pool-2996-threa Kdump: loaded Tainted: G W ------------ T 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8d8579f18000 ti: ffff8d84d0d64000 task.ti: ffff8d84d0d64000
RIP: 0010:[<ffffffff88b16a7a>] [<ffffffff88b16a7a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8d84d0d67cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0008000088a7e2b0 RCX: ffff8d8cffc206a0
RDX: 0000000000000000 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff8d84d0d67ce8 R08: ffff8d8634520800 R09: ffffffff88d81839
R10: ffff8d8cffd9f160 R11: fffff7ee00258800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8d84d0d67c60 R15: 00007effa9669000
FS: 00007eff4a9b4700(0000) GS:ffff8d8cffd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007effa9669000 CR3: 000000080ab46000 CR4: 00000000000007e0
Call Trace:
[<ffffffff88a7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff88a7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff88bec096>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff88bedb5f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff88bedc83>] tlb_finish_mmu+0x23/0x30
[<ffffffff88bf7474>] unmap_region+0xf4/0x140
[<ffffffff88bf7a31>] ? __vma_rb_erase+0x121/0x220
[<ffffffff88bf9a75>] do_munmap+0x2a5/0x480
[<ffffffff88bf9cb5>] vm_munmap+0x65/0xb0
[<ffffffff88bfaf52>] SyS_munmap+0x22/0x30
[<ffffffff8918e172>] tracesys+0xa6/0xcc
Code: 48 63 35 ee 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 74 89 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #138395
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff93116eea
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9307e868
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9307e8db
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff931edf56
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff931efa1f
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff931efb43
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff931f9334
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff933577a4
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x14
|
|
9
|
|
vmlinux |
0xffffffff931f98f1
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff931fb925
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff931fbb65
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff931fce62
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff93795f92
Build id: 3.10.0-1160.24.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [Plex Media Serv:30321]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink arc4 md4 nls_utf8 cifs ccm dns_resolver nf_nat_ftp nf_conntrack_ftp nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sd_mod sysimgblt fb_sys_fops ttm crc_t10dif crct10dif_generic ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 30321 Comm: Plex Media Serv Kdump: loaded Tainted: G L ------------ 3.10.0-1160.24.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff99aebbcd6300 ti: ffff99ac99748000 task.ti: ffff99ac99748000
RIP: 0010:[<ffffffff93116eea>] [<ffffffff93116eea>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff99ac9974bcb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800009307e6a0 RCX: ffff99af7fca0650
RDX: 0000000000000002 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff99ac9974bce8 R08: ffff99acb9f42000 R09: ffffffff93386549
R10: ffff99af7fcdf160 R11: fffffecb4b4d3400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff99ac9974bc60 R15: 00007f8caa7fc000
FS: 00007f8c3e7fc700(0000) GS:ffff99af7fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f32741c1000 CR3: 000000041c7f8000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9307e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9307e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff931edf56>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff931efa1f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff931efb43>] tlb_finish_mmu+0x23/0x30
[<ffffffff931f9334>] unmap_region+0xf4/0x140
[<ffffffff933577a4>] ? blk_finish_plug+0x14/0x40
[<ffffffff931f98f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff931fb925>] do_munmap+0x2a5/0x480
[<ffffffff931fbb65>] vm_munmap+0x65/0xb0
[<ffffffff931fce62>] SyS_munmap+0x22/0x30
[<ffffffff93795f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 fe 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 d5 93 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #76889
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9af16e4a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9ae7e868
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9ae7e8db
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9afede86
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9afef94f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9afefa73
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9aff9264
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9aff9821
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff9affb855
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff9affba95
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff9affcd92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff9b594f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9b594ed5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [sisidsdaemon:2946]
Modules linked in: binfmt_misc falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_10912(E) iptable_filter xt_DSCP iptable_mangle vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi ata_piix scsi_transport_spi nfit mptscsih libata serio_raw mptbase libnvdimm vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 2946 Comm: sisidsdaemon Kdump: loaded Tainted: P EL ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8be6133cb180 ti: ffff8be535d70000 task.ti: ffff8be535d70000
RIP: 0010:[<ffffffff9af16e4a>] [<ffffffff9af16e4a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8be535d73cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffffffff9ae7e6a0 RCX: ffff8bf4fee20628
RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff8be535d73ce8 R08: ffff8be663b43c00 R09: ffffffff9b186479
R10: ffff8bf4fee9f160 R11: fffffdedc599e000 R12: 0000000000000286
R13: ffff8be535d73c60 R14: 00007f6724e96000 R15: 00007f6724e95000
FS: 00007f670f7fe700(0000) GS:ffff8bf4fee80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6724e95000 CR3: 000000013278a000 CR4: 00000000000407e0
Call Trace:
[<ffffffff9ae7e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9ae7e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9afede86>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9afef94f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9afefa73>] tlb_finish_mmu+0x23/0x30
[<ffffffff9aff9264>] unmap_region+0xf4/0x140
[<ffffffff9aff9821>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9affb855>] do_munmap+0x2a5/0x480
[<ffffffff9affba95>] vm_munmap+0x65/0xb0
[<ffffffff9affcd92>] SyS_munmap+0x22/0x30
[<ffffffff9b594f92>] system_call_fastpath+0x25/0x2a
[<ffffffff9b594ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 de 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 b5 9b f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #36375
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8391761a
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8387e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8387e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff839ede36
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff839ef8ff
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff839efa23
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff839f9234
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff839f97f1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff839fb825
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff839fba65
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff839fcd02
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffff83f93166
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [elasticsearch[G:2338]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus ppdev snd_seq snd_seq_device snd_pcm vmw_balloon snd_timer coretemp snd sg joydev pcspkr soundcore i2c_piix4 vmw_vmci parport_pc parport sch_fq ip_tables xfs libcrc32c
sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm ata_piix drm mptspi nfit libata scsi_transport_spi libnvdimm serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 2338 Comm: elasticsearch[G Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9d261fb48000 ti: ffff9d2840430000 task.ti: ffff9d2840430000
RIP: 0010:[<ffffffff8391761a>] [<ffffffff8391761a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9d2840433cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008387e730 RCX: ffff9d295d620790
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9d2840433ce8 R08: ffff9d22b4556400 R09: ffffffff83b851b9
R10: ffff9d295d91f160 R11: ffffedd4a071ba00 R12: 0000000000000282
R13: 0000000000000018 R14: ffff9d2840433c20 R15: 0000000000000282
FS: 00007f4fb2131700(0000) GS:ffff9d295d900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5161372000 CR3: 00000007aa6a6000 CR4: 00000000000007e0
Call Trace:
[<ffffffff8387e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8387e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff839ede36>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff839ef8ff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff839efa23>] tlb_finish_mmu+0x23/0x30
[<ffffffff839f9234>] unmap_region+0xf4/0x140
[<ffffffff839f97f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff839fb825>] do_munmap+0x2a5/0x480
[<ffffffff839fba65>] vm_munmap+0x65/0xb0
[<ffffffff839fcd02>] SyS_munmap+0x22/0x30
[<ffffffff83f93166>] tracesys+0xa6/0xcc
Code: 48 63 35 ce 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 55 84 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #22724
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb9b16a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb9a7e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb9a7e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb9bec026
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb9bedaef
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb9bedc13
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb9bf7404
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb9bf79c1
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb9bf9a05
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb9bf9c45
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb9bfaee2
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffba18dede
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [flussonic:26908]
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper vmw_balloon ablk_helper e1000e cryptd pcspkr ptp sg joydev pps_core parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod crc_t10dif ahci drm crct10dif_generic libahci ata_piix nfit libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 26908 Comm: flussonic Kdump: loaded Tainted: G W L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/03/2018
task: ffff8cea83d4e2a0 ti: ffff8ceb919b0000 task.ti: ffff8ceb919b0000
RIP: 0010:[<ffffffffb9b16a6a>] [<ffffffffb9b16a6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8ceb919b3cb0 EFLAGS: 00000202
RAX: 0000000000000007 RBX: 00080000b9a7e2b0 RCX: ffff8cec7fde0678
RDX: 0000000000000007 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff8ceb919b3ce8 R08: ffff8cebfcc10c00 R09: ffffffffb9d817e9
R10: ffff8cec7fd1f160 R11: fffff19c00231000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8ceb919b3c60 R15: 00007f980a1fd000
FS: 00007f992013f700(0000) GS:ffff8cec7fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9812644898 CR3: 00000001eff6a000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb9a7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb9a7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb9bec026>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb9bedaef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb9bedc13>] tlb_finish_mmu+0x23/0x30
[<ffffffffb9bf7404>] unmap_region+0xf4/0x140
[<ffffffffb9bf79c1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb9bf9a05>] do_munmap+0x2a5/0x480
[<ffffffffb9bf9c45>] vm_munmap+0x65/0xb0
[<ffffffffb9bfaee2>] SyS_munmap+0x22/0x30
[<ffffffffba18dede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 74 ba f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #20137
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9c91761e
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9c87e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9c87e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9c9ede36
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9c9ef8ff
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9c9efa23
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9c9f9234
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9c9f97f1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff9c9fb825
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff9c9fba65
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff9c9fcd02
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff9cf92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 33s! [reader#1:1187]
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Modules linked in:
vsock_diag unix_diag af_packet_diag netlink_diag udp_diag tcp_diag inet_diag binfmt_misc ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr cryptd joydev sg parport_pc parport i2c_piix4 vmw_vmci CyProtectDrv(OE) ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod
crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi nfit libata scsi_transport_spi libnvdimm crct10dif_pclmul mptscsih crct10dif_common crc32c_intel serio_raw mptbase drm_panel_orientation_quirks vmxnet3 dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 1187 Comm: reader#1 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8be418d662a0 ti: ffff8be408f4c000 task.ti: ffff8be408f4c000
RIP: 0010:[<ffffffff9c91761e>] [<ffffffff9c91761e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff8be408f4fcb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800009c87e730 RCX: ffff8be43fc9fc10
RDX: 0000000000000002 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff8be408f4fce8 R08: ffff8bdd7fd55000 R09: ffffffff9cb851b9
R10: ffff8be43fc1f160 R11: fffff878a07c4a00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8be408f4fc60 R15: 00007f3e7ecb6000
FS: 00007f3e743cc700(0000) GS:ffff8be43fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee20 CR3: 0000000035db8000 CR4: 00000000001607f0
Call Trace:
[<ffffffff9c87e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9c87e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9c9ede36>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9c9ef8ff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9c9efa23>] tlb_finish_mmu+0x23/0x30
[<ffffffff9c9f9234>] unmap_region+0xf4/0x140
[<ffffffff9c9f97f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9c9fb825>] do_munmap+0x2a5/0x480
[<ffffffff9c9fba65>] vm_munmap+0x65/0xb0
[<ffffffff9c9fcd02>] SyS_munmap+0x22/0x30
[<ffffffff9cf92ed2>] system_call_fastpath+0x25/0x2a
Code: 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 55 9d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #17342
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb3316c5e
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb327e868
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb327e8db
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffb33edd46
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffb33ef80f
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb33ef933
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffb33f9124
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffb33f96e1
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb33fb715
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb33fb955
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb33fcbf2
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffb3994f92
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb3994ed5
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [psvc:26836]
Modules linked in: vmfsrtx_export(POE) vmfsrtx_bridge(POE) vmfsrtx_bridge_helper(OE) vmfsrtx_dev(POE) vmfsrtx_dev_helper(POE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd joydev pcspkr vmw_balloon sg parport_pc vmw_vmci
parport i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit crct10dif_pclmul crct10dif_common crc32c_intel libnvdimm ata_piix mptspi scsi_transport_spi mptscsih libata mptbase serio_raw vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: vmfsrtx_bridge_helper]
CPU: 2 PID: 26836 Comm: psvc Kdump: loaded Tainted: P W OEL ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9e2fc7bc2100 ti: ffff9e2f60bdc000 task.ti: ffff9e2f60bdc000
RIP: 0010:[<ffffffffb3316c5e>] [<ffffffffb3316c5e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9e2f60bdfcb0 EFLAGS: 00000202
RAX: 0000000000000004 RBX: ffffffffb327e6a0 RCX: ffff9e4d7ef20628
RDX: 0000000000000004 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff9e2f60bdfce8 R08: ffff9e2f05f53c00 R09: ffffffffb3585fd9
R10: ffff9e4d7ee9f160 R11: ffffd88246fef800 R12: 0000000000000286
R13: ffff9e2f60bdfc60 R14: 00007f649bd13000 R15: 00007f649bd12000
FS: 00007f6472ffd700(0000) GS:ffff9e4d7ee80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f649bd12000 CR3: 00000001d13b0000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb327e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb327e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb33edd46>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffb33ef80f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffb33ef933>] tlb_finish_mmu+0x23/0x30
[<ffffffffb33f9124>] unmap_region+0xf4/0x140
[<ffffffffb33f96e1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb33fb715>] do_munmap+0x2a5/0x480
[<ffffffffb33fb955>] vm_munmap+0x65/0xb0
[<ffffffffb33fcbf2>] SyS_munmap+0x22/0x30
[<ffffffffb3994f92>] system_call_fastpath+0x25/0x2a
[<ffffffffb3994ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 f5 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #15522
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9e9150ca
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9e87d398
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9e87d40b
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff9e9ea166
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff9e9ebc2f
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9e9ebd53
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff9e9f5544
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff9eb50cc4
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x14
|
|
9
|
|
vmlinux |
0xffffffff9e9f5b01
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff9e9f7b45
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff9e9f7d85
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff9e9f9022
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x22
|
|
13
|
tracesys
|
vmlinux |
0xffffffff9ef8d172
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [s1-agent:121307]
Modules linked in: btrfs raid6_pq xor msdos fat ext4 mbcache jbd2 udp_diag tcp_diag inet_diag binfmt_misc tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun bridge stp llc devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg i2c_piix4 vmw_vmci parport_pc
parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi libata mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 10 PID: 121307 Comm: s1-agent Kdump: loaded Tainted: G W L ------------ 3.10.0-1062.1.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff98750347e2a0 ti: ffff98750c054000 task.ti: ffff98750c054000
RIP: 0010:[<ffffffff9e9150ca>] [<ffffffff9e9150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff98750c057cb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800009e87d1d0 RCX: ffff987d1d620718
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff98750c057ce8 R08: ffff98767451e000 R09: ffffffff9eb7f629
R10: ffff987d1d89f160 R11: fffff8e619f1a400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff98750c057c60 R15: 00007f02848f2000
FS: 00007f0232ffd700(0000) GS:ffff987d1d880000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000016c5238 CR3: 0000000108cde000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9e87d398>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9e87d40b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9e9ea166>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff9e9ebc2f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff9e9ebd53>] tlb_finish_mmu+0x23/0x30
[<ffffffff9e9f5544>] unmap_region+0xf4/0x140
[<ffffffff9eb50cc4>] ? blk_finish_plug+0x14/0x40
[<ffffffff9e9f5b01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9e9f7b45>] do_munmap+0x2a5/0x480
[<ffffffff9e9f7d85>] vm_munmap+0x65/0xb0
[<ffffffff9e9f9022>] SyS_munmap+0x22/0x30
[<ffffffff9ef8d172>] tracesys+0xa6/0xcc
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 54 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #3683
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa6d16a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa6c7e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa6c7e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa6dec026
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa6dedaef
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa6dedc13
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffa6df7404
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffa6df79c1
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffa6df9a05
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffa6df9c45
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffa6dfaee2
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffa738dede
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [setroubleshootd:86700]
Modules linked in: xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables veth tcp_lp fuse br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vmw_vmci vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 vmw_balloon snd_rawmidi snd_ac97_codec ac97_bus aesni_intel snd_seq lrw gf128mul glue_helper snd_seq_device snd_pcm ablk_helper cryptd snd_timer pcspkr joydev snd sg soundcore parport_pc parport pcc_cpufreq i2c_piix4 xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi serio_raw drm mptscsih ata_piix e1000 nfit libnvdimm libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 3 PID: 86700 Comm: setroubleshootd Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff96b282d4b150 ti: ffff96b29cc30000 task.ti: ffff96b29cc30000
RIP: 0010:[<ffffffffa6d16a6a>] [<ffffffffa6d16a6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff96b29cc33cb0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00080000a6c7e2b0 RCX: ffff96b33a6604d8
RDX: 0000000000000001 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff96b29cc33ce8 R08: ffff96b3065c4c00 R09: ffffffffa6f817e9
R10: ffff96b33a6df160 R11: ffffd2cc40fd4000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff96b29cc33c60 R15: 00007f003cc4d000
FS: 00007f003cc30740(0000) GS:ffff96b33a6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f003cc4d000 CR3: 000000001cdae000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa6c7e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa6c7e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa6dec026>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa6dedaef>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa6dedc13>] tlb_finish_mmu+0x23/0x30
[<ffffffffa6df7404>] unmap_region+0xf4/0x140
[<ffffffffa6df79c1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa6df9a05>] do_munmap+0x2a5/0x480
[<ffffffffa6df9c45>] vm_munmap+0x65/0xb0
[<ffffffffa6dfaee2>] SyS_munmap+0x22/0x30
[<ffffffffa738dede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 94 a7 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1617
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8ff1761a
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8fe7e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8fe7e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8ffede36
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8ffef8ff
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8ffefa23
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff8fff9234
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc0384344
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffff8fff97f1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff8fffb825
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff8fffba65
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff8fffcd02
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff90592ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
|
14
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff90592e15
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [ns-slapd:5566]
Modules linked in: kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev parport_pc joydev parport i2c_piix4 virtio_balloon pcspkr sch_fq_codel auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi qxl drm_kms_helper virtio_net net_failover failover syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common virtio_pci crc32c_intel floppy serio_raw virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 5566 Comm: ns-slapd Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20200516_175120-felixonmars2 04/01/2014
task: ffffa11678a08000 ti: ffffa11672244000 task.ti: ffffa11672244000
RIP: 0010:[<ffffffff8ff1761a>] [<ffffffff8ff1761a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa11672247cb0 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 000800008fe7e730 RCX: ffffa11800120428
RDX: 0000000000000004 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffffa11672247ce8 R08: ffffa1173cf55400 R09: ffffffff901851b9
R10: ffffa1180009f160 R11: ffffebf988d6e500 R12: 0000000000000282
R13: 0000000000000282 R14: ffffa11672247c60 R15: 00007f08376b9000
FS: 00007f080ef67700(0000) GS:ffffa11800080000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f08376b9000 CR3: 00000000b8a12000 CR4: 0000000000020ee0
Call Trace:
[<ffffffff8fe7e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8fe7e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8ffede36>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8ffef8ff>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8ffefa23>] tlb_finish_mmu+0x23/0x30
[<ffffffff8fff9234>] unmap_region+0xf4/0x140
[<ffffffffc0384344>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffff8fff97f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff8fffb825>] do_munmap+0x2a5/0x480
[<ffffffff8fffba65>] vm_munmap+0x65/0xb0
[<ffffffff8fffcd02>] SyS_munmap+0x22/0x30
[<ffffffff90592ed2>] system_call_fastpath+0x25/0x2a
[<ffffffff90592e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 ce 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b5 90 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1419
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8cd1519a
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8cc7d478
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8cc7d4eb
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff8cdea756
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff8cdec21f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8cdec343
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff8cdf5b34
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xf4
|
|
8
|
|
xfs |
0xffffffffc04370c4
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xe4
|
|
9
|
|
vmlinux |
0xffffffff8cdf60f1
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x121
|
|
10
|
do_munmap
|
vmlinux |
0xffffffff8cdf8135
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x2a5
|
|
11
|
vm_munmap
|
vmlinux |
0xffffffff8cdf8375
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x65
|
|
12
|
SyS_munmap
|
vmlinux |
0xffffffff8cdf9612
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x22
|
|
13
|
system_call_fastpath
|
vmlinux |
0xffffffff8d38bede
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [java:18159]
Modules linked in: xt_nat veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm irqbypass snd_ens1371 snd_rawmidi snd_ac97_codec ppdev ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device aesni_intel vmw_balloon snd_pcm lrw gf128mul glue_helper snd_timer ablk_helper cryptd snd soundcore pcspkr joydev nfit i2c_piix4 sg vmw_vmci libnvdimm parport_pc
parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix serio_raw libata mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 18159 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-1062.4.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8b43187f20e0 ti: ffff8b42e53e0000 task.ti: ffff8b42e53e0000
RIP: 0010:[<ffffffff8cd1519a>] [<ffffffff8cd1519a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b42e53e3cb0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 000800008cc7d2b0 RCX: ffff8b456d660678
RDX: 0000000000000001 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8b42e53e3ce8 R08: ffff8b42b9cad400 R09: ffffffff8cf7fe99
R10: ffff8b456d71f160 R11: ffffe76d46c3a000 R12: 0000000000000282
R13: 0000000000000018 R14: ffff8b42e53e3c20 R15: 0000000000000282
FS: 00007f9f84c95700(0000) GS:ffff8b456d700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9f9c793000 CR3: 0000000200b98000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8cc7d478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8cc7d4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8cdea756>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff8cdec21f>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff8cdec343>] tlb_finish_mmu+0x23/0x30
[<ffffffff8cdf5b34>] unmap_region+0xf4/0x140
[<ffffffffc04370c4>] ? xfs_iunlock+0xe4/0x120 [xfs]
[<ffffffff8cdf60f1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff8cdf8135>] do_munmap+0x2a5/0x480
[<ffffffff8cdf8375>] vm_munmap+0x65/0xb0
[<ffffffff8cdf9612>] SyS_munmap+0x22/0x30
[<ffffffff8d38bede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 6e 66 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d6 94 8d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 3d 66 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #826
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff949175ce
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9487e8f8
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9487e96b
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffff949ede16
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffff949ef8df
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffff949efa03
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffff949f9214
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffff949f97d1
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff949fb805
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff949fba45
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff949fcce2
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff94f92ed2
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff94f92e15
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [Timer-11:14741]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop overlay iptable_filter ppdev pcspkr joydev parport_pc i2c_piix4 parport virtio_balloon ip_tables xfs libcrc32c ata_generic pata_acpi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata virtio_blk e1000 virtio_pci virtio_ring serio_raw floppy drm_panel_orientation_quirks virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 14741 Comm: Timer-11 Kdump: loaded Not tainted 3.10.0-1127.10.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff9ea846bfb150 ti: ffff9ea8a0878000 task.ti: ffff9ea8a0878000
RIP: 0010:[<ffffffff949175ce>] [<ffffffff949175ce>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9ea8a087bcb0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffffffff9487e730 RCX: ffff9ea91fca0428
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9ea8a087bce8 R08: ffff9ea8fd6bdc00 R09: ffffffff94b85179
R10: ffff9ea91fd1f160 R11: ffffe943c0d6af00 R12: ffff9ea8a087bc80
R13: ffffffff94b850cf R14: ffff9ea8ffc01500 R15: 00007fb825b76000
FS: 00007fb81836e700(0000) GS:ffff9ea91fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb825b76000 CR3: 0000000180318000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9487e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9487e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff949ede16>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffff949ef8df>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffff949efa03>] tlb_finish_mmu+0x23/0x30
[<ffffffff949f9214>] unmap_region+0xf4/0x140
[<ffffffff949f97d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff949fb805>] do_munmap+0x2a5/0x480
[<ffffffff949fba45>] vm_munmap+0x65/0xb0
[<ffffffff949fcce2>] SyS_munmap+0x22/0x30
[<ffffffff94f92ed2>] system_call_fastpath+0x25/0x2a
[<ffffffff94f92e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 10 55 95 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 6d 7b c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #377
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa3f175ca
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa3e7e8f8
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa3e7e96b
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x6b
|
|
4
|
tlb_flush_mmu_tlbonly
|
vmlinux |
0xffffffffa3fede16
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xa6
|
|
5
|
arch_tlb_finish_mmu
|
vmlinux |
0xffffffffa3fef8df
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x3f
|
|
6
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa3fefa03
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x23
|
|
7
|
unmap_region
|
vmlinux |
0xffffffffa3ff9214
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xf4
|
|
8
|
|
vmlinux |
0xffffffffa3ff97d1
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffa3ffb805
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffa3ffba45
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffa3ffcce2
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffa4592ed2
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0x25
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa4592e15
Build id: 3.10.0-1127.13.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [C2 CompilerThre:27836]
Modules linked in: nf_conntrack_netlink nfnetlink binfmt_misc vboxnetadp(OE) vboxnetflt(OE) xt_conntrack iptable_filter xt_REDIRECT nf_nat_redirect vboxdrv(OE) iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack sg ppdev joydev pcspkr parport_pc virtio_balloon parport i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi virtio_net virtio_console net_failover virtio_scsi failover bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 27836 Comm: C2 CompilerThre Kdump: loaded Tainted: G OEL ------------ 3.10.0-1127.13.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.2-0-gf9626ccb91-prebuilt.qemu-project.org 04/01/2014
task: ffff8f9c51f69070 ti: ffff8f9aca03c000 task.ti: ffff8f9aca03c000
RIP: 0010:[<ffffffffa3f175ca>] [<ffffffffa3f175ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8f9aca03fcb0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000a3e7e730 RCX: ffff8f9d3fd9fc10
RDX: 0000000000000003 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff8f9aca03fce8 R08: ffff8f9bffd52000 R09: ffffffffa4185179
R10: ffff8f9d3fc1f160 R11: ffffcf5d0811c400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8f9aca03fc60 R15: 00007fef13d89000
FS: 00007feee521c700(0000) GS:ffff8f9d3fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fef13d89000 CR3: 000000022b986000 CR4: 00000000000006f0
Call Trace:
[<ffffffffa3e7e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa3e7e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa3fede16>] tlb_flush_mmu_tlbonly+0xa6/0xc0
[<ffffffffa3fef8df>] arch_tlb_finish_mmu+0x3f/0x80
[<ffffffffa3fefa03>] tlb_finish_mmu+0x23/0x30
[<ffffffffa3ff9214>] unmap_region+0xf4/0x140
[<ffffffffa3ff97d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa3ffb805>] do_munmap+0x2a5/0x480
[<ffffffffa3ffba45>] vm_munmap+0x65/0xb0
[<ffffffffa3ffcce2>] SyS_munmap+0x22/0x30
[<ffffffffa4592ed2>] system_call_fastpath+0x25/0x2a
[<ffffffffa4592e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 de 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 10 b5 a4 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ad 7b c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1197547
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [qaxsafed:7269]
Modules linked in: qax_tq_base(OE) udp_diag binfmt_misc tcp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables ilocker(OE) nfsv3 nfs fscache gsch(OE) redirfs(OE) tun dsa_filter(POE) ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev
pcspkr joydev vmw_balloon sg i2c_piix4 parport_pc parport shpchp vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul mptspi crct10dif_common crc32c_intel scsi_transport_spi libata mptscsih serio_raw e1000 mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: qax_tq_base]
CPU: 4 PID: 7269 Comm: qaxsafed Tainted: P W OEL ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8804142a1fa0 ti: ffff88000f234000 task.ti: ffff88000f234000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff88000f237cf0 EFLAGS: 00000202
RAX: 0000000000000006 RBX: ffffffff8106e4a0 RCX: ffff88043fd9af98
RDX: 0000000000000006 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff88000f237d28 R08: ffff88017a311000 R09: ffffffff813227d9
R10: ffff88043fd19c80 R11: ffffea001008f000 R12: ffff88000f237cc0
R13: ffffffff8132272f R14: ffff88000f237e08 R15: 00007f13b850d000
FS: 00007f13d8731700(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe4ef35c000 CR3: 0000000222e75000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001142a1fa0 ffff88024bb3dac8 ffff88024bb3d780 0000000000000000
ffffffffffffffff ffff88024bb3dac8 ffff88041eb7e360 ffff88000f237d78
ffffffff8106e668 ffff88024bb3d780 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #1179445
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb2507e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb2477018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb2477088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffb25c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb25c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb25ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffb25cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb25ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb25cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb25ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffffb2b1fa51
Build id: 3.10.0-862.el7.x86_64
|
0x9d
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [1_aux:101835]
Modules linked in: tcp_lp veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) nls_utf8 isofs rfcomm fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmhgfs(OE) vmw_vsock_vmci_transport vsock bnep sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec ac97_bus lrw gf128mul glue_helper ppdev btusb ablk_helper cryptd snd_seq btrtl btbcm btintel snd_seq_device bluetooth snd_pcm joydev snd_timer pcspkr sg snd rfkill nfit soundcore i2c_piix4 vmw_vmci shpchp libnvdimm parport_pc
parport ip_tables xfs libcrc32c sr_mod sd_mod crc_t10dif cdrom crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel mptspi libata e1000 scsi_transport_spi mptscsih serio_raw mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 101835 Comm: 1_aux Kdump: loaded Tainted: G OE ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff924c1da2af70 ti: ffff924c777b4000 task.ti: ffff924c777b4000
RIP: 0010:[<ffffffffb2507e3a>] [<ffffffffb2507e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff924c777b7cb8 EFLAGS: 00000202
RAX: 0000000000000006 RBX: 00000000000000fc RCX: ffff924d7959cf28
RDX: 0000000000000006 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff924c777b7cf0 R08: ffff924bbad2c000 R09: ffffffffb274a3d9
R10: ffff924d7951bb00 R11: ffffee46cc66a400 R12: 0000000000000296
R13: ffff924c777b7c68 R14: 0000000000000001 R15: ffffee46c828fd00
FS: 00007f23f5dc66c0(0000) GS:ffff924d79500000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005587f716d618 CR3: 00000002e6e8a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb2477018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb2477088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb25c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffb25c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffb25ca624>] unmap_region+0xf4/0x140
[<ffffffffb25cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb25ccbf5>] do_munmap+0x295/0x470
[<ffffffffb25cce35>] vm_munmap+0x65/0xb0
[<ffffffffb25ce052>] SyS_munmap+0x22/0x30
[<ffffffffb2b1fa51>] tracesys+0x9d/0xc3
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 13 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #1154591
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#28 stuck for 21s! [cap_process:4748]
Modules linked in: tcp_lp fuse udp_diag tcp_diag inet_diag xt_comment xt_multiport xt_set ip_set_hash_ip ip_set nf_conntrack_netlink nfnetlink xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack sfesp(OE) ipt_REJECT nf_reject_ipv4 tun bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg nfit i2c_piix4 libnvdimm shpchp parport_pc vmw_vmci parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod vmwgfx crc_t10dif crct10dif_generic
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000e ahci libahci crct10dif_pclmul ata_piix crct10dif_common crc32c_intel serio_raw ptp libata pps_core i2c_core vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 28 PID: 4748 Comm: cap_process Tainted: G W OEL ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/09/2019
task: ffff8807fc22cf10 ti: ffff8800b2470000 task.ti: ffff8800b2470000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff8800b2473cf0 EFLAGS: 00000202
RAX: 0000000000000019 RBX: 000800008106e4a0 RCX: ffff88083f85b3c8
RDX: 0000000000000019 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff8800b2473d28 R08: ffff8804ba7a0800 R09: ffffffff813227d9
R10: ffff88083f919c80 R11: ffffea00002b3e00 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8800b2473ca0 R15: 00007f236dd63000
FS: 00007f2314b5e700(0000) GS:ffff88083f900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f236dd63000 CR3: 00000000349db000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001fc22cf10 ffff88042b3fbb88 ffff88042b3fb840 00007f236dd63000
00007f236dd64000 ffff88042b3fbb88 ffff8803d768d950 ffff8800b2473d78
ffffffff8106e668 ffff88042b3fb840 00007f236dd63000 00007f236dd64000
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #811398
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb6707e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb6677018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb6677088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffb67c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb67c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb67ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc05c6bec
Build id: 3.10.0-862.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffffb67cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb67ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb67cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb67ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [java:974735]
Modules linked in: ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi ac97_bus crc32_pclmul ghash_clmulni_intel snd_seq aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm snd_timer snd ppdev soundcore vmw_balloon vmw_vmci i2c_piix4 joydev pcspkr sg shpchp nfit libnvdimm parport_pc parport binfmt_misc xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi ata_piix crct10dif_pclmul scsi_transport_spi crct10dif_common drm libata crc32c_intel e1000 mptscsih mptbase serio_raw i2c_core [last unloaded: ip_tables]
CPU: 14 PID: 974735 Comm: java Tainted: G W L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff8be87ef83f40 ti: ffff8be61d710000 task.ti: ffff8be61d710000
RIP: 0010:[<ffffffffb6707e3a>] [<ffffffffb6707e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8be61d713cb8 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff8beebd6dd0b8
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8be61d713cf0 R08: ffff8be02966e800 R09: ffffffffb694a3d9
R10: ffff8beebd99bb00 R11: fffff89f8e52b200 R12: 0000000000000296
R13: 0000000000000018 R14: ffff8be61d713c28 R15: 0000000000000296
FS: 00002b99c381b700(0000) GS:ffff8beebd980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b98bffb0000 CR3: 0000000c512c0000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb6677018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb6677088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb67c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffb67c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffb67ca624>] unmap_region+0xf4/0x140
[<ffffffffc05c6bec>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffffb67cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb67ccbf5>] do_munmap+0x295/0x470
[<ffffffffb67cce35>] vm_munmap+0x65/0xb0
[<ffffffffb67ce052>] SyS_munmap+0x22/0x30
CPU: 13 PID: 974732 Comm: java Tainted: G W L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Complete report #733921
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
?
copy_user_generic_unrolled
|
vmlinux |
0xffffffff81330240
Build id: 3.10.0-693.el7.x86_64
|
0xa0
|
|
8
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 68s! [java:51709]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm sb_edac snd_timer edac_core snd nfit soundcore ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr vmw_balloon libnvdimm joydev sg shpchp i2c_piix4 parport_pc parport vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi ata_piix scsi_transport_spi libata mptscsih mptbase e1000 serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 51709 Comm: java Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8804112ccf10 ti: ffff88023f744000 task.ti: ffff88023f744000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff88023f747cf0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800008106e4a0 RCX: ffff88042d69af70
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88023f747d28 R08: ffff88017a395000 R09: ffffffff813227d9
R10: ffff88042d6d9c80 R11: ffffea000b28b000 R12: 0000000000000286
R13: 0000000000000286 R14: ffff88023f747ca0 R15: 00007fd8fbaa1000
FS: 00007fd8de07a700(0000) GS:ffff88042d6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd8fbaa1000 CR3: 000000038fa50000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001112ccf10 ffff88041e3d6748 ffff88041e3d6400 00007fd8fbaa1000
00007fd8fbaa2000 ffff88041e3d6748 ffff88018865be60 ffff88023f747d78
ffffffff8106e668 ffff88041e3d6400 00007fd8fbaa1000 00007fd8fbaa2000
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff81330240>] ? copy_user_generic_unrolled+0xa0/0xc0
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #604465
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fd5fa
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff81070b98
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff81070c68
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811b18c3
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b2be5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b9cb4
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b6781
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x691
|
|
8
|
|
vmlinux |
0xffffffff811ba271
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff811bc285
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff811bc4c5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff811bd5d2
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff816c0715
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [gnome-shell:1863]
Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event sb_edac edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul snd_ens1371 glue_helper snd_rawmidi ablk_helper snd_ac97_codec cryptd ac97_bus snd_seq snd_seq_device snd_pcm ppdev vmw_balloon snd_timer snd joydev pcspkr
sg soundcore vmw_vmci nfit i2c_piix4 shpchp libnvdimm parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c vmwgfx drm_kms_helper sr_mod sd_mod cdrom crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ata_generic ttm pata_acpi drm mptspi scsi_transport_spi mptscsih ata_piix mptbase libata e1000 crct10dif_pclmul crct10dif_common crc32c_intel serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1863 Comm: gnome-shell Tainted: G L ------------ 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8804288e3f40 ti: ffff88040073c000 task.ti: ffff88040073c000
RIP: 0010:[<ffffffff810fd5fa>] [<ffffffff810fd5fa>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88040073fcc0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00080000810709d0 RCX: ffff88042d61cd58
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88040073fcf8 R08: ffff880179ca4800 R09: ffffffff81328989
R10: ffff88042d79ba00 R11: ffffea000c8cf800 R12: 0000000000000286
R13: 0000000000000286 R14: ffff88040073fc70 R15: ffffffff812da063
FS: 00007fa180adeac0(0000) GS:ffff88042d780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa11ef35000 CR3: 000000041daa4000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81070b98>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff81070c68>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811b18c3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b2be5>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b9cb4>] unmap_region+0xf4/0x140
[<ffffffff811b6781>] ? handle_mm_fault+0x691/0xfa0
[<ffffffff811ba271>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811bc285>] do_munmap+0x295/0x470
[<ffffffff811bc4c5>] vm_munmap+0x65/0xb0
[<ffffffff811bd5d2>] SyS_munmap+0x22/0x30
[<ffffffff816c0715>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 8e 1b a3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b2 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 1b a3 00 eb b7 0f b6 4d cc 4c 89
Complete report #534482
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbaf1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbae7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbae7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffbafe4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbafe5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffbafef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffbafef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffbaff1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffbaff1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffbaff2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffbb574ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbb574d21
Build id: 3.10.0-957.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [mysqld:11996]
Modules linked in: udp_diag binfmt_misc resguard_linux(OE) secmodel_linux(OE) tcp_diag inet_diag ip6t_MASQUERADE nf_nat_masquerade_ipv6 ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_filter ip6_tables iptable_nat nf_nat_ipv4 nf_nat tun ipt_REJECT nf_reject_ipv4 xt_limit xt_conntrack iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev sg parport_pc virtio_balloon parport joydev i2c_piix4 pcspkr nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_console virtio_net virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy
fuse
CPU: 8 PID: 11996 Comm: mysqld Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9de9f5656180 ti: ffff9decf77f0000 task.ti: ffff9decf77f0000
RIP: 0010:[<ffffffffbaf1173a>] [<ffffffffbaf1173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9decf77f3cb8 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 00000000000000fc RCX: ffffd5acbd700c20
RDX: 0000000000000004 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff9decf77f3cf0 R08: ffff9decd0f20400 R09: ffffffffbb1755c9
R10: ffff9deded01f160 R11: fffff7880a5c9c00 R12: 0000000000000296
R13: ffff9decf77f3c68 R14: 0000000000000001 R15: fffff7880508a3c0
FS: 00007fab4cb91700(0000) GS:ffff9deded000000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa763ffe008 CR3: 000000040a6c4000 CR4: 00000000000006e0
Call Trace:
[<ffffffffbae7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbae7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffbafe4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffbafe5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffbafef014>] unmap_region+0xf4/0x140
[<ffffffffbafef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffbaff1605>] do_munmap+0x2a5/0x480
[<ffffffffbaff1845>] vm_munmap+0x65/0xb0
[<ffffffffbaff2ad2>] SyS_munmap+0x22/0x30
[<ffffffffbb574ddb>] system_call_fastpath+0x22/0x27
[<ffffffffbb574d21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 b4 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #509787
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffba907e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffba877018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffba877088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffba9c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffba9c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffba9ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffbaa1a1f0
Build id: 3.10.0-862.el7.x86_64
|
0x90
|
|
8
|
|
vmlinux |
0xffffffffba9cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffba9ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffba9cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffba9ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffffbaf1fa51
Build id: 3.10.0-862.el7.x86_64
|
0x9d
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [elasticsearch[n:18425]
Modules linked in: xt_statistic vxlan ip6_udp_tunnel udp_tunnel ipt_REJECT nf_reject_ipv4 nf_tables(T) ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter iptable_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment xt_mark xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter bridge stp llc overlay(T) iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter ext4 mbcache jbd2 sunrpc sg 8139too virtio_balloon pcspkr joydev ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common ata_generic pata_acpi ata_piix libata 8139cp virtio_pci virtio_ring mii serio_raw virtio dm_mirror
dm_region_hash dm_log dm_mod
CPU: 4 PID: 18425 Comm: elasticsearch[n Kdump: loaded Tainted: G ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff88684b268000 ti: ffff886739ff8000 task.ti: ffff886739ff8000
RIP: 0010:[<ffffffffba907e3a>] [<ffffffffba907e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff886739ffbcb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0000000000019680 RCX: ffff886d6f21cd28
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff886739ffbcf0 R08: ffff885adf734c00 R09: ffffffffbab4a3d9
R10: ffff886d6f31bb00 R11: fffff3d98b88d200 R12: ffff886739ffbc30
R13: 0000000000000286 R14: 0000000000000010 R15: ffffffffba859653
FS: 00007ff0ce1fd700(0000) GS:ffff886d6f300000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff0ccdd8000 CR3: 0000000db62a8000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffba877018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffba877088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffba9c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffba9c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffba9ca624>] unmap_region+0xf4/0x140
[<ffffffffbaa1a1f0>] ? do_sync_write+0x90/0xe0
[<ffffffffba9cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffba9ccbf5>] do_munmap+0x295/0x470
[<ffffffffba9cce35>] vm_munmap+0x65/0xb0
[<ffffffffba9ce052>] SyS_munmap+0x22/0x30
[<ffffffffbaf1fa51>] tracesys+0x9d/0xc3
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 53 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #496090
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9b11173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9b07a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9b07a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff9b1e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff9b1e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff9b1ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc03db8bc
Build id: 3.10.0-957.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffff9b1ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff9b1f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff9b1f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff9b1f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff9b774ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [java:8478]
Modules linked in: vmw_vsock_vmci_transport vsock vfat fat crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg parport_pc vmw_vmci parport tpm_crb i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic e1000e crct10dif_pclmul crct10dif_common serio_raw crc32c_intel vmwgfx ptp pps_core drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci mptsas libahci drm ata_piix scsi_transport_sas nfit mptscsih libata libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 8478 Comm: java Kdump: loaded Tainted: G W L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.17369862.B64.2012240522 12/24/2020
task: ffff9f623a59b0c0 ti: ffff9f6991300000 task.ti: ffff9f6991300000
RIP: 0010:[<ffffffff9b11173a>] [<ffffffff9b11173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9f6991303cb8 EFLAGS: 00000202
RAX: 000000000000000e RBX: 00000000000000fc RCX: ffff9f699f5a04b8
RDX: 000000000000000e RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9f6991303cf0 R08: ffff9f62f4bfb800 R09: ffffffff9b3755c9
R10: ffff9f699f29f160 R11: fffff1071e2c8400 R12: 0000000000000296
R13: ffff9f6991303c68 R14: 0000000000000001 R15: fffff1071a246c80
FS: 00007fdf151e4700(0000) GS:ffff9f699f280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdf117d5000 CR3: 0000000814b30000 CR4: 0000000000760fe0
PKRU: 55555554
Call Trace:
[<ffffffff9b07a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9b07a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff9b1e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff9b1e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff9b1ef014>] unmap_region+0xf4/0x140
[<ffffffffc03db8bc>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffff9b1ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff9b1f1605>] do_munmap+0x2a5/0x480
[<ffffffff9b1f1845>] vm_munmap+0x65/0xb0
[<ffffffff9b1f2ad2>] SyS_munmap+0x22/0x30
[<ffffffff9b774ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 d4 9b f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #493105
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fd5fa
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff81070b98
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff81070c68
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811b18c3
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b2be5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b9cb4
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811ba271
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811bc285
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811bc4c5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811bd5d2
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816c0715
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1c
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816c0661
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [libvirtd:1438]
Modules linked in: ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter scsi_transport_iscsi softdog iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel dm_service_time aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg i2c_piix4 virtio_rng pcspkr joydev virtio_balloon nfsd auth_rpcgss nfs_acl dm_multipath lockd grace sunrpc 8021q garp mrp bridge stp llc bonding ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_net virtio_scsi virtio_console qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel i2c_core virtio_pci serio_raw virtio_ring floppy virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 1438 Comm: libvirtd Not tainted 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: Red Hat RHEL/RHEL-AV, BIOS 1.14.0-1.module+el8.4.0+8855+a9e237a9 04/01/2014
task: ffff8800364c0fd0 ti: ffff880074900000 task.ti: ffff880074900000
RIP: 0010:[<ffffffff810fd5fa>] [<ffffffff810fd5fa>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff880074903cc0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00080000810709d0 RCX: ffff88007d2dcab8
RDX: 0000000000000003 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff880074903cf8 R08: ffff880077d9b400 R09: ffffffff81328989
R10: ffff88007d29ba00 R11: ffffea0001dba400 R12: 0000000000000286
R13: 0000000000000286 R14: ffff880074903c70 R15: 00007fe716fc5000
FS: 00007fe703f8f700(0000) GS:ffff88007d280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe716fc5000 CR3: 0000000077bd4000 CR4: 00000000000606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81070b98>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff81070c68>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811b18c3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b2be5>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b9cb4>] unmap_region+0xf4/0x140
[<ffffffff811ba271>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811bc285>] do_munmap+0x295/0x470
[<ffffffff811bc4c5>] vm_munmap+0x65/0xb0
[<ffffffff811bd5d2>] SyS_munmap+0x22/0x30
[<ffffffff816c0715>] system_call_fastpath+0x1c/0x21
[<ffffffff816c0661>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 8e 1b a3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b2 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 1b a3 00 eb b7 0f b6 4d cc 4c 89
Complete report #468702
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b5009
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [nbrmms:2277]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13707(E) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev vmw_balloon pcspkr joydev nfit sg libnvdimm vmw_vmci i2c_piix4 parport_pc shpchp parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix ahci libahci mptspi scsi_transport_spi i2c_core libata mptscsih serio_raw mptbase vmxnet3 dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 2277 Comm: nbrmms Tainted: P EL ------------ 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/28/2020
task: ffff880fe6e40000 ti: ffff880fb7fa0000 task.ti: ffff880fb7fa0000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff880fb7fa3cf0 EFLAGS: 00000202
RAX: 0000000000000004 RBX: ffffffff8106e4a0 RCX: ffff88103fd1af20
RDX: 0000000000000004 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff880fb7fa3d28 R08: ffff880169407400 R09: ffffffff813229e9
R10: ffff88103fc99c80 R11: ffffea003e71c200 R12: ffff880fb7fa3cc0
R13: ffffffff8132293f R14: ffff880fb7fa3e08 R15: 00007f7ada64b000
FS: 00007f7abdcda700(0000) GS:ffff88103fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7ada64b000 CR3: 0000000fcfb4b000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001e6e40000 ffff880ffaa7bb88 ffff880ffaa7b840 0000000000000000
ffffffffffffffff ffff880ffaa7bb88 ffff880fe97df518 ffff880fb7fa3d78
ffffffff8106e668 ffff880ffaa7b840 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #448717
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb991173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb987a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb987a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffb99e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb99e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb99ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffb99ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb99f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb99f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb99f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb9f74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [auth_server:19349]
Modules linked in: nvidia_uvm(OE) xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc nvidia_drm(POE) nvidia_modeset(POE) vfat fat ext4 skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel mbcache jbd2 kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul
glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support nvidia(POE) pcspkr joydev sg mei_me mei lpc_ich i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler acpi_cpufreq wmi acpi_power_meter acpi_pad sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper crct10dif_pclmul syscopyarea crct10dif_common sysfillrect crc32c_intel i40e sysimgblt fb_sys_fops ttm drm ptp pps_core ahci libahci libata drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 19 PID: 19349 Comm: auth_server Kdump: loaded Tainted: P OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Supermicro X11DPG-OT/X11DPG-OT-CPU, BIOS 3.1 04/30/2019
task: ffff9e98b1232080 ti: ffff9e98a66b0000 task.ti: ffff9e98a66b0000
RIP: 0010:[<ffffffffb991173a>] [<ffffffffb991173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9e98a66b3cb8 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 000000e000000013 RCX: ffff9e98de760908
RDX: 0000000000000005 RSI: 00000000000000e0 RDI: 0000000000000000
RBP: ffff9e98a66b3cf0 R08: ffff9e91f31b3c00 R09: ffffffffb9b755c9
R10: ffff9e98deadf160 R11: fffff3ff2119b000 R12: 000000fcb9b7551f
R13: 0000000000000296 R14: 0000000000000296 R15: ffff9e98a66b3c68
FS: 00007fa72c625700(0000) GS:ffff9e98deac0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa72e342000 CR3: 0000001025b02000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb987a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb987a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb99e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffb99e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffb99ef014>] unmap_region+0xf4/0x140
[<ffffffffb99ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb99f1605>] do_munmap+0x2a5/0x480
[<ffffffffb99f1845>] vm_munmap+0x65/0xb0
[<ffffffffb99f2ad2>] SyS_munmap+0x22/0x30
[<ffffffffb9f74ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 54 ba f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #436624
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff99307e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff99277018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff99277088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffff993c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff993c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff993ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff993cabff
Build id: 3.10.0-862.el7.x86_64
|
0x13f
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff993ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff993cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff993ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffff9991fa51
Build id: 3.10.0-862.el7.x86_64
|
0x9d
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [java:13972]
Modules linked in: tcp_lp nls_utf8 isofs fuse veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec aesni_intel lrw ppdev ac97_bus gf128mul glue_helper snd_seq ablk_helper cryptd snd_seq_device vmw_balloon snd_pcm snd_timer snd pcspkr joydev nfit soundcore sg libnvdimm parport_pc vmw_vmci parport i2c_piix4 shpchp ip_tables xfs libcrc32c sr_mod cdrom ata_generic
pata_acpi sd_mod crc_t10dif crct10dif_generic ahci crct10dif_pclmul libahci crct10dif_common ata_piix vmwgfx drm_kms_helper mptspi scsi_transport_spi crc32c_intel syscopyarea mptscsih sysfillrect sysimgblt fb_sys_fops ttm drm e1000 libata serio_raw mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 13972 Comm: java Kdump: loaded Tainted: G ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8dcd7ca63f40 ti: ffff8dcd74fb4000 task.ti: ffff8dcd74fb4000
RIP: 0010:[<ffffffff99307e3a>] [<ffffffff99307e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8dcd74fb7cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff8dcdf361cdb8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8dcd74fb7cf0 R08: ffff8dccbb4ab800 R09: ffffffff9954a3d9
R10: ffff8dcdf369bb00 R11: ffffc2d6c05ec400 R12: 0000000000000296
R13: ffff8dcd74fb7c68 R14: 0000000000000000 R15: 0000000000000200
FS: 00007fb56ae11b10(0000) GS:ffff8dcdf3680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb56adfd618 CR3: 0000000291732000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff99277018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff99277088>] flush_tlb_mm_range+0x68/0x140
[<ffffffff993c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffff993c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffff993ca624>] unmap_region+0xf4/0x140
[<ffffffff993cabff>] ? __vma_rb_erase+0x13f/0x220
[<ffffffff993ccbf5>] do_munmap+0x295/0x470
[<ffffffff993cce35>] vm_munmap+0x65/0xb0
[<ffffffff993ce052>] SyS_munmap+0x22/0x30
[<ffffffff9991fa51>] tracesys+0x9d/0xc3
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 f3 99 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #406830
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb591173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb587a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb587a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffb59e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb59e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb59ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffb5f678e8
Build id: 3.10.0-957.el7.x86_64
|
0x5d8
|
|
8
|
|
vmlinux |
0xffffffffb59ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffb59f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffb59f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffb59f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
12
|
tracesys
|
vmlinux |
0xffffffffb5f7505b
Build id: 3.10.0-957.el7.x86_64
|
0xa3
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [elasticsearch[9:8854]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr btusb snd_ens1371 btrtl btbcm snd_rawmidi btintel bluetooth snd_ac97_codec ac97_bus snd_seq snd_seq_device rfkill snd_pcm snd_timer snd soundcore sg nfit parport_pc parport libnvdimm vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw e1000 ata_generic
pata_acpi floppy vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 8854 Comm: elasticsearch[9 Kdump: loaded Tainted: G ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9cb95ea62080 ti: ffff9cb95f15c000 task.ti: ffff9cb95f15c000
RIP: 0010:[<ffffffffb591173a>] [<ffffffffb591173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9cb95f15fcb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000000000001b700 RCX: ffff9cb97c620490
RDX: 0000000000000000 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff9cb95f15fcf0 R08: ffff9cb97b653800 R09: ffffffffb5b755c9
R10: ffff9cb97c6df160 R11: fffff6980083b600 R12: ffff9cb95f15fc68
R13: ffffffffc0312227 R14: ffffffffb5a1be42 R15: ffff9cb93678fd00
FS: 00007f82033ff700(0000) GS:ffff9cb97c6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f82a410b000 CR3: 0000000052ddc000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb587a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb587a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb59e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffb59e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffb59ef014>] unmap_region+0xf4/0x140
[<ffffffffb5f678e8>] ? __schedule+0x5d8/0x890
[<ffffffffb59ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb59f1605>] do_munmap+0x2a5/0x480
[<ffffffffb59f1845>] vm_munmap+0x65/0xb0
[<ffffffffb59f2ad2>] SyS_munmap+0x22/0x30
[<ffffffffb5f7505b>] tracesys+0xa3/0xc9
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 54 b6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #399824
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa3f0816a
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa3e772d8
Build id: 3.10.0-862.3.3.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa3e77348
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffa3fc07d7
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa3fc1fb5
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffa3fca964
Build id: 3.10.0-862.3.3.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffa3fcaf21
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffa3fccf35
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffa3fcd175
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffa3fce392
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffa452082f
Build id: 3.10.0-862.3.3.el7.x86_64
|
0x1c
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa452077b
Build id: 3.10.0-862.3.3.el7.x86_64
|
0xc8
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [spades-hammer:13600]
Modules linked in: fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vboxsf(OE) sunrpc vboxguest(OE) snd_intel8x0 vmwgfx snd_ac97_codec ttm ac97_bus ppdev drm_kms_helper snd_seq joydev syscopyarea snd_seq_device sysfillrect snd_pcm sysimgblt iosf_mbi fb_sys_fops snd_timer snd drm sg soundcore i2c_piix4 pcspkr parport_pc video parport i2c_core ip_tables
xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod crct10dif_common cdrom ata_generic pata_acpi ata_piix libata crc32c_intel serio_raw e1000 dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 13600 Comm: spades-hammer Kdump: loaded Tainted: G OEL ------------ 3.10.0-862.3.3.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff9576cc5ddee0 ti: ffff957757f90000 task.ti: ffff957757f90000
RIP: 0010:[<ffffffffa3f0816a>] [<ffffffffa3f0816a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff957757f93cb8 EFLAGS: 00010202
RAX: 0000000000000001 RBX: 0000000000019680 RCX: ffff95775fc9cc20
RDX: 0000000000000001 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff957757f93cf0 R08: ffff95775949d400 R09: ffffffffa414a819
R10: ffff95775fd9bb00 R11: ffffd74602954200 R12: ffffffffa414a76f
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000200
FS: 00007f6312bfe700(0000) GS:ffff95775fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f631140f000 CR3: 00000000ac05a000 CR4: 00000000000206e0
Call Trace:
[<ffffffffa3e772d8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa3e77348>] flush_tlb_mm_range+0x68/0x140
[<ffffffffa3fc07d7>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffa3fc1fb5>] tlb_finish_mmu+0x55/0x60
[<ffffffffa3fca964>] unmap_region+0xf4/0x140
[<ffffffffa3fcaf21>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa3fccf35>] do_munmap+0x295/0x470
[<ffffffffa3fcd175>] vm_munmap+0x65/0xb0
[<ffffffffa3fce392>] SyS_munmap+0x22/0x30
[<ffffffffa452082f>] system_call_fastpath+0x1c/0x21
[<ffffffffa452077b>] ? system_call_after_swapgs+0xc8/0x160
Code: 48 63 35 9e b2 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 52 b3 a4 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d b2 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #393264
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fb892
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106f508
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106f5d8
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811af0f3
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b0415
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b74e4
Build id: 3.10.0-693.17.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b7aa1
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b9aa2
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b9ce5
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811badf2
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b89fd
Build id: 3.10.0-693.17.1.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [ds_am:2449]
Modules linked in: vfat fat isofs xfs libcrc32c binfmt_misc bmhook(OE) tmhook(OE) dsa_filter(POE) dsa_filter_hook(OE) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod btrfs sb_edac edac_core iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd raid6_pq xor joydev pcspkr sg nfit parport_pc parport libnvdimm vmw_vmci shpchp i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ahci ata_piix libahci libata serio_raw vmxnet3 i2c_core vmw_pvscsi floppy
CPU: 8 PID: 2449 Comm: ds_am Tainted: P OEL ------------ 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8805a4e1dee0 ti: ffff88058b65c000 task.ti: ffff88058b65c000
RIP: 0010:[<ffffffff810fb892>] [<ffffffff810fb892>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff88058b65fcc0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000800008106f340 RCX: ffff8805a769cd80
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88058b65fcf8 R08: ffff880176d3d800 R09: ffffffff81324cb9
R10: ffff8805a781b9c0 R11: ffffea0000430800 R12: 0000000000000286
R13: 0000000000000286 R14: ffff88058b65fc70 R15: 00007fbca92e5000
FS: 00007fbc8a051700(0000) GS:ffff8805a7800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbca92e5000 CR3: 0000000581a98000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8106f508>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106f5d8>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811af0f3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b0415>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b74e4>] unmap_region+0xf4/0x140
[<ffffffff811b7aa1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b9aa2>] do_munmap+0x292/0x470
[<ffffffff811b9ce5>] vm_munmap+0x65/0xb0
[<ffffffff811badf2>] SyS_munmap+0x22/0x30
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 76 b6 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 8e b1 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 45 b6 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #385194
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa571173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa567a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa567a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffa57e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa57e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffa57ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffa57ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffa57f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffa57f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffa57f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffffa5d7505b
Build id: 3.10.0-957.el7.x86_64
|
0xa3
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [com.alibaba.nac:20806]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype xt_CHECKSUM br_netfilter iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_midi snd_seq_midi_event joydev pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd sg soundcore parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul
crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi vmwgfx e1000 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi libata mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 20806 Comm: com.alibaba.nac Kdump: loaded Tainted: G ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff958d5a840000 ti: ffff958ce12e0000 task.ti: ffff958ce12e0000
RIP: 0010:[<ffffffffa571173a>] [<ffffffffa571173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff958ce12e3cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff958d796204b8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff958ce12e3cf0 R08: ffff958d7762b800 R09: ffffffffa59755c9
R10: ffff958d7969f160 R11: ffffec68405ffc00 R12: 0000000000000296
R13: ffff958ce12e3c68 R14: 0000000000000001 R15: ffffec6844785700
FS: 00007f1cc54c4700(0000) GS:ffff958d79680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1d5da40000 CR3: 000000011db2c000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa567a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa567a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffa57e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffa57e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffa57ef014>] unmap_region+0xf4/0x140
[<ffffffffa57ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa57f1605>] do_munmap+0x2a5/0x480
[<ffffffffa57f1845>] vm_munmap+0x65/0xb0
[<ffffffffa57f2ad2>] SyS_munmap+0x22/0x30
[<ffffffffa5d7505b>] tracesys+0xa3/0xc9
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 34 a6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #344835
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810fd5fa
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff81070b98
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff81070c68
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811b18c3
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811b2be5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b9cb4
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811ba271
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811bc285
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811bc4c5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811bd5d2
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffff816c0991
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x9d
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 106s! [java:5356]
Modules linked in: xt_nat veth fuse ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc vboxsf(OE) overlay(T) ext4 mbcache jbd2 joydev ppdev parport_pc vboxguest(OE) sg parport video pcspkr i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm serio_raw libata e1000 crc32c_intel i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 5356 Comm: java Tainted: G OE ------------ T 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff88049f220000 ti: ffff8804622ac000 task.ti: ffff8804622ac000
RIP: 0010:[<ffffffff810fd5fa>] [<ffffffff810fd5fa>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8804622afcc0 EFLAGS: 00010202
RAX: 0000000000000001 RBX: 0000000000000010 RCX: ffff88051fc5be60
RDX: 0000000000000001 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff8804622afcf8 R08: ffff88017fd42c00 R09: ffffffff81328989
R10: ffff88051fc1ba00 R11: ffffea001379b100 R12: ffffffffffffff02
R13: 0000000000000286 R14: 00000000000000fc R15: 00000000000008fc
FS: 00007fd75388d700(0000) GS:ffff88051fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd73c000000 CR3: 00000004ce3c8000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81070b98>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff81070c68>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811b18c3>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811b2be5>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b9cb4>] unmap_region+0xf4/0x140
[<ffffffff811ba271>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811bc285>] do_munmap+0x295/0x470
[<ffffffff811bc4c5>] vm_munmap+0x65/0xb0
[<ffffffff811bd5d2>] SyS_munmap+0x22/0x30
[<ffffffff816c0991>] tracesys+0x9d/0xc3
Code: 48 63 35 8e 1b a3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b2 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 1b a3 00 eb b7 0f b6 4d cc 4c 89
Complete report #294077
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a6
Build id: 3.10.0-693.el7.x86_64
|
0x206
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [icsfilesec:55562]
Modules linked in: vmsec_nfq(OE) nf_conntrack_ipv6 nf_defrag_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nfnetlink_queue vmsecmod(OE) tun ip_set nfnetlink bridge stp llc nf_conntrack sg i2c_piix4 ppdev pcspkr virtio_balloon joydev parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata virtio_pci floppy virtio_ring i2c_core serio_raw virtio dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmsec_nfq]
CPU: 5 PID: 55562 Comm: icsfilesec Tainted: G OEL ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff88044ddeeeb0 ti: ffff8804cdaf8000 task.ti: ffff8804cdaf8000
RIP: 0010:[<ffffffff810f99a6>] [<ffffffff810f99a6>] smp_call_function_many+0x206/0x260
RSP: 0018:ffff8804cdafbcf0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008106e4a0 RCX: ffff88081da1ae08
RDX: 0000000000000000 RSI: 0000000000000070 RDI: 0000000000000000
RBP: ffff8804cdafbd28 R08: ffff880174b4d000 R09: ffffffff813227d9
R10: ffff88081db59c80 R11: ffffea0015108200 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8804cdafbca0 R15: 00007fead309e000
FS: 00007feab9ffb700(0000) GS:ffff88081db40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fead309e000 CR3: 00000000030c4000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
000000014ddeeeb0 ffff880819f5e108 ffff880819f5ddc0 0000000000000000
ffffffffffffffff ffff880819f5e108 ffff8807e1380e58 ffff8804cdafbd78
ffffffff8106e668 ffff880819f5ddc0 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #269238
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb250b09a
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb2479e58
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb2479ec8
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffb25c33c7
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb25c4cc5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb25cdab4
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffb25ce071
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb25d0085
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb25d02c5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb25d14e2
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffb2b2579b
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [systemd:1]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter gsch(OE) redirfs(OE) event_collector_1_9_8557(POE) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon
ppdev sg pcspkr joydev parport_pc parport vmw_vmci shpchp i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect crc_t10dif sysimgblt crct10dif_generic fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw i2c_core vmxnet3 vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1 Comm: systemd Kdump: loaded Tainted: P W OEL ------------ 3.10.0-862.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9537ecc38000 ti: ffff9537ecc80000 task.ti: ffff9537ecc80000
RIP: 0010:[<ffffffffb250b09a>] [<ffffffffb250b09a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9537ecc83cb8 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00000000000000fc RCX: ffff954f8d01d028
RDX: 0000000000000005 RSI: 000000000000000a RDI: 0000000000000000
RBP: ffff9537ecc83cf0 R08: ffff9543ad778c00 R09: ffffffffb274edf9
R10: ffff954f8d11bb40 R11: ffffe2c2b1f2b000 R12: 0000000000000296
R13: ffff9537ecc83c68 R14: 0000000000000001 R15: ffffe2c2b93a1e80
FS: 00007fd91b808940(0000) GS:ffff954f8d100000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd91b819000 CR3: 0000000c8d6ce000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb2479e58>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb2479ec8>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb25c33c7>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffb25c4cc5>] tlb_finish_mmu+0x55/0x60
[<ffffffffb25cdab4>] unmap_region+0xf4/0x140
[<ffffffffb25ce071>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb25d0085>] do_munmap+0x295/0x470
[<ffffffffb25d02c5>] vm_munmap+0x65/0xb0
[<ffffffffb25d14e2>] SyS_munmap+0x22/0x30
[<ffffffffb2b2579b>] system_call_fastpath+0x22/0x27
Code: 48 63 35 ee 84 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 54 13 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 84 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #254554
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a6
Build id: 3.10.0-693.el7.x86_64
|
0x206
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [java:21225]
Modules linked in: nfsv3 nfs fscache vmw_vsock_vmci_transport vsock coretemp ppdev vmw_balloon pcspkr joydev sg shpchp parport_pc vmw_vmci parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper crc32c_intel serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi drm scsi_transport_spi vmxnet3 mptscsih ata_piix mptbase libata i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 21225 Comm: java Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff8804223c0000 ti: ffff88031cce0000 task.ti: ffff88031cce0000
RIP: 0010:[<ffffffff810f99a6>] [<ffffffff810f99a6>] smp_call_function_many+0x206/0x260
RSP: 0018:ffff88031cce3cf0 EFLAGS: 00000202
RAX: 0000000000000003 RBX: ffffffff8106e4a0 RCX: ffff88043fd9a7c8
RDX: 0000000000000003 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff88031cce3d28 R08: ffff88017a25d000 R09: ffffffff813227d9
R10: ffff88043fc99c80 R11: ffffea0010adfe00 R12: ffff88031cce3cc0
R13: ffffffff8132272f R14: ffff88031cce3c70 R15: ffff88043fc96cc0
FS: 00007fdb60ad9700(0000) GS:ffff88043fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fdb60ba1010 CR3: 000000008d716000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001223c0000 ffff88034ad673c8 ffff88034ad67080 00007fdb60bfc000
00007fdb60c02000 ffff88034ad673c8 ffff8803c40bbd80 ffff88031cce3d78
ffffffff8106e668 ffff88034ad67080 00007fdb60bfc000 00007fdb60c02000
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
Complete report #222195
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbe71185a
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbe67a5c8
Build id: 3.10.0-957.5.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbe67a638
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffbe7e4427
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbe7e5d25
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffbe7ef3c4
Build id: 3.10.0-957.5.1.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc03148bc
Build id: 3.10.0-957.5.1.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffffbe7ef981
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffbe7f19b5
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffbe7f1bf5
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffbe7f2e82
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffbed74ddb
Build id: 3.10.0-957.5.1.el7.x86_64
|
0x22
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbed74d21
Build id: 3.10.0-957.5.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [java:24353]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev vmw_balloon coretemp pcspkr joydev nfit sg libnvdimm parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common sr_mod cdrom crc32c_intel serio_raw ata_generic pata_acpi floppy vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000e ata_piix drm mptspi scsi_transport_spi ptp pps_core mptscsih libata mptbase drm_panel_orientation_quirks
CPU: 1 PID: 24353 Comm: java Kdump: loaded Tainted: G W ------------ 3.10.0-957.5.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9631b0c8c100 ti: ffff963159ae0000 task.ti: ffff963159ae0000
RIP: 0010:[<ffffffffbe71185a>] [<ffffffffbe71185a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff963159ae3cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000000000001b700 RCX: ffff9631bfc20490
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff963159ae3cf0 R08: ffff9630fcf2cc00 R09: ffffffffbe975ba9
R10: ffff9631bfc9f160 R11: ffffee4fc8a19700 R12: ffffffffbe975aff
R13: ffff9630ffc01500 R14: 0000000000000001 R15: ffffffffbe81d47a
FS: 00007f3de39f9700(0000) GS:ffff9631bfc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3e547f0000 CR3: 000000009ec4a000 CR4: 00000000000007e0
Call Trace:
[<ffffffffbe67a5c8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbe67a638>] flush_tlb_mm_range+0x68/0x140
[<ffffffffbe7e4427>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffbe7e5d25>] tlb_finish_mmu+0x55/0x60
[<ffffffffbe7ef3c4>] unmap_region+0xf4/0x140
[<ffffffffc03148bc>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffffbe7ef981>] ? __vma_rb_erase+0x121/0x220
[<ffffffffbe7f19b5>] do_munmap+0x2a5/0x480
[<ffffffffbe7f1bf5>] vm_munmap+0x65/0xb0
[<ffffffffbe7f2e82>] SyS_munmap+0x22/0x30
[<ffffffffbed74ddb>] system_call_fastpath+0x22/0x27
[<ffffffffbed74d21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 2e 82 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 34 bf f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 fd 81 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #213219
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a6
Build id: 3.10.0-693.el7.x86_64
|
0x206
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [sqlservr:2583]
Modules linked in: tcp_lp rfcomm tun lp ip_set nfnetlink bridge stp llc fuse bnep snd_seq_midi snd_seq_midi_event coretemp iosf_mbi kvm_intel kvm ppdev irqbypass crc32_pclmul snd_ens1371 snd_rawmidi btusb snd_ac97_codec btrtl btbcm btintel bluetooth ghash_clmulni_intel ac97_bus snd_seq aesni_intel lrw snd_seq_device gf128mul glue_helper snd_pcm ablk_helper vmw_balloon cryptd snd_timer rfkill joydev pcspkr snd sg soundcore parport_pc parport shpchp i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc xfs libcrc32c ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata mptspi scsi_transport_spi mptscsih mptbase crct10dif_pclmul crct10dif_common crc32c_intel e1000 serio_raw i2c_core floppy
dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 2 PID: 2583 Comm: sqlservr Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/31/2013
task: ffff88003599bf40 ti: ffff88008d29c000 task.ti: ffff88008d29c000
RIP: 0010:[<ffffffff810f99a6>] [<ffffffff810f99a6>] smp_call_function_many+0x206/0x260
RSP: 0018:ffff88008d29fcf0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffffffff8106e4a0 RCX: ffff8800bb61a7a8
RDX: 0000000000000000 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff88008d29fd28 R08: ffff880086453400 R09: ffffffff813227d9
R10: ffff8800bb699c80 R11: ffffea0002b4a800 R12: 0000000000000282
R13: 0000000000000018 R14: ffff88008d29fc68 R15: 0000000000000282
FS: 00007f211f536700(0000) GS:ffff8800bb680000(0000) knlGS:0000000181ca0000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f211ee21778 CR3: 000000008ebed000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
0000000101301d80 ffff8800b3b5b548 ffff8800b3b5b200 0000000000000000
ffffffffffffffff ffff8800b3b5b548 ffff880090a896c8 ffff88008d29fd78
ffffffff8106e668 ffff8800b3b5b200 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #200853
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa4b07e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa4a77018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa4a77088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffa4bc0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffa4bc1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffa4bca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc0356bec
Build id: 3.10.0-862.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffffa4bcabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffa4bccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffa4bcce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffa4bce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffa511f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [BgSchPool:2351]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev nfit pcspkr sg vmw_balloon libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper ahci syscopyarea sysfillrect sd_mod sysimgblt fb_sys_fops ttm ata_piix crc_t10dif libahci crct10dif_generic drm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmw_pvscsi
vmxnet3 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 2351 Comm: BgSchPool Kdump: loaded Not tainted 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff91588178af70 ti: ffff915885fe8000 task.ti: ffff915885fe8000
RIP: 0010:[<ffffffffa4b07e3a>] [<ffffffffa4b07e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff915885febcb8 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00000000000000fc RCX: ffff9158bfc9cf00
RDX: 0000000000000002 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff915885febcf0 R08: ffff9155fa357400 R09: ffffffffa4d4a3d9
R10: ffff9158bfcdbb00 R11: ffffca4c8e62ec00 R12: 0000000000000296
R13: ffff915885febc68 R14: 0000000000000001 R15: ffffca4c810f68c0
FS: 00007ff1153f6700(0000) GS:ffff9158bfcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff1431d2000 CR3: 000000041c336000 CR4: 00000000003407e0
Call Trace:
[<ffffffffa4a77018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa4a77088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffa4bc0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffa4bc1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffa4bca624>] unmap_region+0xf4/0x140
[<ffffffffc0356bec>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffffa4bcabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffa4bccbf5>] do_munmap+0x295/0x470
[<ffffffffa4bcce35>] vm_munmap+0x65/0xb0
[<ffffffffa4bce052>] SyS_munmap+0x22/0x30
[<ffffffffa511f7d5>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 73 a5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #191272
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb9707e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb9677018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb9677088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffb97c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffb97c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffb97ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffb97cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffb97ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffb97cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffb97ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffffb9d1fa51
Build id: 3.10.0-862.el7.x86_64
|
0x9d
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [java:8107]
Modules linked in: tcp_lp fuse tcp_diag inet_diag xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set veth nf_conntrack_netlink nfnetlink xt_nat xt_statistic ip_vs_sh ip_vs_wrr ip_vs_rr cfg80211 rfkill ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack ipt_REJECT nf_reject_ipv4 ip_vs nf_conntrack tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel snd_ens1371 ppdev snd_rawmidi snd_ac97_codec ac97_bus
snd_seq vmw_balloon aesni_intel snd_seq_device lrw gf128mul glue_helper ablk_helper cryptd snd_pcm snd_timer pcspkr joydev snd nfit vmw_vmci i2c_piix4 sg soundcore libnvdimm shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel mptspi libata scsi_transport_spi e1000 serio_raw mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 8107 Comm: java Kdump: loaded Tainted: G W L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9c9589570fd0 ti: ffff9c9794eac000 task.ti: ffff9c9794eac000
RIP: 0010:[<ffffffffb9707e3a>] [<ffffffffb9707e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c9794eafcb8 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 00000000000000fc RCX: ffff9c992571cf50
RDX: 0000000000000004 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9c9794eafcf0 R08: ffff9c9476544c00 R09: ffffffffb994a3d9
R10: ffff9c992575bb00 R11: ffffe2d04ff8d400 R12: 0000000000000296
R13: ffff9c9794eafc68 R14: ffffffff000000a8 R15: ffffffffb97f8139
FS: 00007f775d6a1700(0000) GS:ffff9c9925740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7745d7f000 CR3: 00000000b1242000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb9677018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb9677088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb97c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffb97c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffb97ca624>] unmap_region+0xf4/0x140
[<ffffffffb97cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffb97ccbf5>] do_munmap+0x295/0x470
[<ffffffffb97cce35>] vm_munmap+0x65/0xb0
[<ffffffffb97ce052>] SyS_munmap+0x22/0x30
[<ffffffffb9d1fa51>] tracesys+0x9d/0xc3
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 33 ba f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #183920
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9791173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9787a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9787a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff979e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff979e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff979ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff979ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff979f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff979f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff979f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
tracesys
|
vmlinux |
0xffffffff97f7505b
Build id: 3.10.0-957.el7.x86_64
|
0xa3
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [7_scheduler:20966]
Modules linked in: tcp_lp fuse xt_nat veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec joydev ac97_bus pcspkr snd_seq snd_seq_device snd_pcm sg snd_timer snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sd_mod sr_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks ata_piix libata nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 20966 Comm: 7_scheduler Kdump: loaded Tainted: G W L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9225a9a3b0c0 ti: ffff92255032c000 task.ti: ffff92255032c000
RIP: 0010:[<ffffffff9791173a>] [<ffffffff9791173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff92255032fcb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9225b3620600
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff92255032fcf0 R08: ffff92247b4b3c00 R09: ffffffff97b755c9
R10: ffff9225b36df160 R11: ffffe0a8c7fdbc00 R12: 0000000000000296
R13: ffff92255032fc68 R14: 0000000000000001 R15: ffffe0a8ca2de640
FS: 00007f6180d27b20(0000) GS:ffff9225b36c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7c965e45f0 CR3: 0000000249a04000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9787a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9787a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff979e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff979e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff979ef014>] unmap_region+0xf4/0x140
[<ffffffff979ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff979f1605>] do_munmap+0x2a5/0x480
[<ffffffff979f1845>] vm_munmap+0x65/0xb0
[<ffffffff979f2ad2>] SyS_munmap+0x22/0x30
[<ffffffff97f7505b>] tracesys+0xa3/0xc9
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 54 98 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #183416
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbad0b0ca
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbac79e58
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbac79ec8
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffbadc3487
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffbadc4d85
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffbadcdb74
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc043dbdc
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffffbadce131
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffbadd0145
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffbadd0385
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffbadd15a2
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffbb32579b
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x22
|
|
13
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbb3256e1
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [java:16266]
Modules linked in: tcp_lp rfcomm fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel btusb btrtl btbcm btintel aesni_intel bluetooth lrw snd_ens1371 gf128mul glue_helper snd_rawmidi ablk_helper cryptd snd_ac97_codec ppdev ac97_bus snd_seq vmw_balloon snd_seq_device snd_pcm rfkill snd_timer pcspkr snd sg vmw_vmci joydev soundcore i2c_piix4 nfit shpchp libnvdimm parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul drm crct10dif_common libata crc32c_intel mptspi e1000 scsi_transport_spi serio_raw mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 16266 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffffa1469e276eb0 ti: ffffa145d0be4000 task.ti: ffffa145d0be4000
RIP: 0010:[<ffffffffbad0b0ca>] [<ffffffffbad0b0ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa145d0be7cb8 EFLAGS: 00000202
RAX: 0000000000000009 RBX: 00000000000000fc RCX: ffffa146ab85ce98
RDX: 0000000000000009 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa145d0be7cf0 R08: ffffa14377d3ec00 R09: ffffffffbaf4ef19
R10: ffffa146ab79bb40 R11: ffffe69391437600 R12: 0000000000000296
R13: ffffa145d0be7c68 R14: 0000000000000001 R15: ffffe6938fd54b80
FS: 00007f95da216700(0000) GS:ffffa146ab780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f95f40d0000 CR3: 00000003b4396000 CR4: 00000000000607e0
Call Trace:
[<ffffffffbac79e58>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbac79ec8>] flush_tlb_mm_range+0x68/0x140
[<ffffffffbadc3487>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffbadc4d85>] tlb_finish_mmu+0x55/0x60
[<ffffffffbadcdb74>] unmap_region+0xf4/0x140
[<ffffffffc043dbdc>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffffbadce131>] ? __vma_rb_erase+0x121/0x220
[<ffffffffbadd0145>] do_munmap+0x295/0x470
[<ffffffffbadd0385>] vm_munmap+0x65/0xb0
[<ffffffffbadd15a2>] SyS_munmap+0x22/0x30
[<ffffffffbb32579b>] system_call_fastpath+0x22/0x27
[<ffffffffbb3256e1>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 fe 84 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 54 93 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 84 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #182538
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8cb12b6a
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8ca7b788
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8ca7b7f8
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff8cbe57d7
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff8cbe70d5
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff8cbf0754
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc04fb67c
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffff8cbf0d11
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffff8cbf2d45
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x2a5
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffff8cbf2f85
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffff8cbf4212
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffff8d176ddb
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [httpd:3298]
Modules linked in: binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter unix_diag vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 ppdev snd_rawmidi iosf_mbi snd_ac97_codec ac97_bus crc32_pclmul vmw_balloon snd_seq ghash_clmulni_intel aesni_intel snd_seq_device lrw gf128mul glue_helper snd_pcm ablk_helper cryptd snd_timer snd pcspkr joydev soundcore sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common
crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops serio_raw ttm ata_piix mptspi scsi_transport_spi drm mptscsih libata mptbase e1000 nfit libnvdimm drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 3298 Comm: httpd Kdump: loaded Tainted: G L ------------ 3.10.0-957.27.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff88d0b16b4100 ti: ffff88d0203fc000 task.ti: ffff88d0203fc000
RIP: 0010:[<ffffffff8cb12b6a>] [<ffffffff8cb12b6a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88d0203ffcb8 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000000000001b700 RCX: ffff88d0be6e04b8
RDX: 0000000000000003 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff88d0203ffcf0 R08: ffff88d0b71bb400 R09: ffffffff8cd774a9
R10: ffff88d0be69f160 R11: ffffdf2944c52000 R12: ffff88d0203ffc30
R13: 0000000000000286 R14: 0000000000000010 R15: ffffffff8ca5c493
FS: 00007f257f0a8700(0000) GS:ffff88d0be680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f25958b5000 CR3: 00000001327ce000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8ca7b788>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8ca7b7f8>] flush_tlb_mm_range+0x68/0x140
[<ffffffff8cbe57d7>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff8cbe70d5>] tlb_finish_mmu+0x55/0x60
[<ffffffff8cbf0754>] unmap_region+0xf4/0x140
[<ffffffffc04fb67c>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffff8cbf0d11>] ? __vma_rb_erase+0x121/0x220
[<ffffffff8cbf2d45>] do_munmap+0x2a5/0x480
[<ffffffff8cbf2f85>] vm_munmap+0x65/0xb0
[<ffffffff8cbf4212>] SyS_munmap+0x22/0x30
[<ffffffff8d176ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 5e 75 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 bf 74 8d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 2d 75 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #168330
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff82d1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff82c7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff82c7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff82de4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff82de5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff82def014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff82def5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff82df1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff82df1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff82df2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff83374ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [360entclient:17174]
Modules linked in: kbase(OE) udp_diag tcp_diag inet_diag binfmt_misc fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd parport_pc parport virtio_balloon joydev i2c_piix4 pcspkr ip_tables ext4 mbcache jbd2 ata_generic pata_acpi
virtio_blk virtio_console virtio_net cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix floppy libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 17174 Comm: 360entclient Kdump: loaded Tainted: G OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: Fedora Project OpenStack Nova, BIOS 0.5.1 01/01/2011
task: ffff9c0e0aa75140 ti: ffff9c0dee370000 task.ti: ffff9c0dee370000
RIP: 0010:[<ffffffff82d1173a>] [<ffffffff82d1173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9c0dee373cb8 EFLAGS: 00000202
RAX: 0000000000000009 RBX: 00000000000000fc RCX: ffff9c0e3f460398
RDX: 0000000000000009 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9c0dee373cf0 R08: ffff9bffa94f4400 R09: ffffffff82f755c9
R10: ffff9c0e3f39f160 R11: fffff81ed4acd600 R12: 0000000000000296
R13: ffff9c0dee373c68 R14: 0000000000000001 R15: fffff81ec4366c80
FS: 00007fad2379b700(0000) GS:ffff9c0e3f380000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faceabcb000 CR3: 0000000fea2f0000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff82c7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff82c7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff82de4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff82de5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff82def014>] unmap_region+0xf4/0x140
[<ffffffff82def5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff82df1605>] do_munmap+0x2a5/0x480
[<ffffffff82df1845>] vm_munmap+0x65/0xb0
[<ffffffff82df2ad2>] SyS_munmap+0x22/0x30
[<ffffffff83374ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 94 83 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #161556
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffadf07e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffade77018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffade77088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffadfc0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffadfc1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffadfca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
xfs |
0xffffffffc03a5bec
Build id: 3.10.0-862.el7.x86_64
|
0xec
|
|
8
|
|
vmlinux |
0xffffffffadfcabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
9
|
do_munmap
|
vmlinux |
0xffffffffadfccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
10
|
vm_munmap
|
vmlinux |
0xffffffffadfcce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
11
|
SyS_munmap
|
vmlinux |
0xffffffffadfce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
12
|
system_call_fastpath
|
vmlinux |
0xffffffffae51f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [TaskSchedulerFo:32809]
Modules linked in: tcp_lp fuse xt_statistic nf_conntrack_netlink xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set nfnetlink nfsv3 veth xt_nat ip_vs_sh ip_vs_wrr ip_vs_rr rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark ip_vs xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul snd_ens1371 snd_rawmidi snd_ac97_codec ghash_clmulni_intel ac97_bus snd_seq ppdev
snd_seq_device aesni_intel snd_pcm lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_timer joydev pcspkr snd sg vmw_vmci soundcore nfit i2c_piix4 shpchp libnvdimm parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel mptspi scsi_transport_spi libata mptscsih serio_raw mptbase e1000 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 32809 Comm: TaskSchedulerFo Kdump: loaded Tainted: G W L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff93525b25cf10 ti: ffff93508cd64000 task.ti: ffff93508cd64000
RIP: 0010:[<ffffffffadf07e3a>] [<ffffffffadf07e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff93508cd67cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff93529cc1cd90
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff93508cd67cf0 R08: ffff9352813d8c00 R09: ffffffffae14a3d9
R10: ffff93529cc5bb00 R11: ffffcdeec2e00200 R12: 0000000000000296
R13: 0000000000000018 R14: ffff93508cd67c28 R15: 0000000000000296
FS: 00007f73f8e59700(0000) GS:ffff93529cc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f73fc656000 CR3: 000000004dfb8000 CR4: 00000000003607e0
Call Trace:
[<ffffffffade77018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffade77088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffadfc0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffadfc1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffadfca624>] unmap_region+0xf4/0x140
[<ffffffffc03a5bec>] ? xfs_iunlock+0xec/0x130 [xfs]
[<ffffffffadfcabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffadfccbf5>] do_munmap+0x295/0x470
[<ffffffffadfcce35>] vm_munmap+0x65/0xb0
[<ffffffffadfce052>] SyS_munmap+0x22/0x30
[<ffffffffae51f7d5>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 b3 ae f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #15517
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106e668
Build id: 3.10.0-693.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106e738
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.63
|
vmlinux |
0xffffffff811ad053
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ae375
Build id: 3.10.0-693.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b5444
Build id: 3.10.0-693.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b5a01
Build id: 3.10.0-693.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b7a02
Build id: 3.10.0-693.el7.x86_64
|
0x292
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7c45
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8d52
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [in:imjournal:1215]
Modules linked in: binfmt_misc vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg vmw_balloon joydev pcspkr parport_pc i2c_piix4 vmw_vmci parport shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi ata_piix scsi_transport_spi crct10dif_pclmul mptscsih crct10dif_common libata drm serio_raw crc32c_intel mptbase vmxnet3 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1215 Comm: in:imjournal Tainted: G W L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/17/2015
task: ffff88022c7beeb0 ti: ffff8802357a8000 task.ti: ffff8802357a8000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff8802357abcf0 EFLAGS: 00000202
RAX: 0000000000000009 RBX: ffffffff8106e4a0 RCX: ffff88023745b058
RDX: 0000000000000009 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff8802357abd28 R08: ffff88017cdc7800 R09: ffffffff813227d9
R10: ffff880237399c80 R11: ffffea000b81f400 R12: 0000000000000282
R13: ffff8802357abca0 R14: ffff8802357abe08 R15: 00007fefc22f6000
FS: 00007fefbdfcc700(0000) GS:ffff880237380000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fefc22f6000 CR3: 000000022ff9d000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
000000012c7beeb0 ffff880235670fc8 ffff880235670c80 00007fefc22f6000
00007fefc22f7000 ffff880235670fc8 ffff880230ae8948 ffff8802357abd78
ffffffff8106e668 ffff880235670c80 00007fefc22f6000 00007fefc22f7000
Call Trace:
[<ffffffff8106e668>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106e738>] flush_tlb_mm_range+0x68/0x140
[<ffffffff811ad053>] tlb_flush_mmu.part.63+0x33/0xc0
[<ffffffff811ae375>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b5444>] unmap_region+0xf4/0x140
[<ffffffff811b5a01>] ? __vma_rb_erase+0x121/0x220
[<ffffffff811b7a02>] do_munmap+0x292/0x470
[<ffffffff811b7c45>] vm_munmap+0x65/0xb0
[<ffffffff811b8d52>] SyS_munmap+0x22/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #14410
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffac707e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffac677018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffac677088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffffac7c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffac7c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffac7ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffac7cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffac7ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffac7cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffac7ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffacd1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [gnome-shell:2077]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac snd_ens1371 coretemp snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ghash_clmulni_intel ac97_bus aesni_intel snd_seq lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm ppdev vmw_balloon i2c_piix4
snd_timer pcspkr snd sg soundcore joydev vmw_vmci shpchp nfit libnvdimm parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata e1000 serio_raw mptspi scsi_transport_spi mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 8 PID: 2077 Comm: gnome-shell Tainted: G L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9a4ef5748000 ti: ffff9a59fe260000 task.ti: ffff9a59fe260000
RIP: 0010:[<ffffffffac707e3a>] [<ffffffffac707e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9a59fe263cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9a5a2041cfc8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9a59fe263cf0 R08: ffff9a502e3ed000 R09: ffffffffac94a3d9
R10: ffff9a5a2061bb00 R11: ffffec74ed7b1400 R12: 0000000000000296
R13: 0000000000000018 R14: ffff9a59fe263c28 R15: 0000000000000296
FS: 00007f0a1d0b0ac0(0000) GS:ffff9a5a20600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f095f86a000 CR3: 0000000b45e92000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffac677018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffac677088>] flush_tlb_mm_range+0x68/0x140
[<ffffffffac7c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffffac7c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffffac7ca624>] unmap_region+0xf4/0x140
[<ffffffffac7cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffac7ccbf5>] do_munmap+0x295/0x470
[<ffffffffac7cce35>] vm_munmap+0x65/0xb0
[<ffffffffac7ce052>] SyS_munmap+0x22/0x30
[<ffffffffacd1f7d5>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 33 ad f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #13279
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff98f0862a
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff98e77788
Build id: 3.10.0-862.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff98e777f8
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffff98fc0c57
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff98fc2435
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff98fcade4
Build id: 3.10.0-862.9.1.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff98fcb3bf
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x13f
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff98fcd3b5
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff98fcd5f5
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff98fce812
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff99520795
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [java:20256]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon sg pcspkr joydev nfit libnvdimm parport_pc i2c_piix4 vmw_vmci parport shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix
serio_raw mptspi scsi_transport_spi e1000 libata mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 20256 Comm: java Kdump: loaded Tainted: G W L ------------ 3.10.0-862.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9a9eb5160000 ti: ffff9aa04df4c000 task.ti: ffff9aa04df4c000
RIP: 0010:[<ffffffff98f0862a>] [<ffffffff98f0862a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9aa04df4fcb8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00000000000000fc RCX: ffff9aa0bfc5ce70
RDX: 0000000000000001 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff9aa04df4fcf0 R08: ffff9a9ffcf36400 R09: ffffffff9914ae19
R10: ffff9aa0bfd5bb40 R11: ffffcbf486851600 R12: 0000000000000296
R13: ffff9aa04df4fc68 R14: ffffcbf482544b80 R15: ffffffff99028af1
FS: 00007f74f4149700(0000) GS:ffff9aa0bfd40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1aa89949b1 CR3: 00000000b5b5c000 CR4: 00000000001607e0
Call Trace:
[<ffffffff98e77788>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff98e777f8>] flush_tlb_mm_range+0x68/0x140
[<ffffffff98fc0c57>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffff98fc2435>] tlb_finish_mmu+0x55/0x60
[<ffffffff98fcade4>] unmap_region+0xf4/0x140
[<ffffffff98fcb3bf>] ? __vma_rb_erase+0x13f/0x220
[<ffffffff98fcd3b5>] do_munmap+0x295/0x470
[<ffffffff98fcd5f5>] vm_munmap+0x65/0xb0
[<ffffffff98fce812>] SyS_munmap+0x22/0x30
[<ffffffff99520795>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 9e ae c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 b3 99 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d ae c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #12316
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff88907e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff88877018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff88877088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.75
|
vmlinux |
0xffffffff889c0497
Build id: 3.10.0-862.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff889c1c75
Build id: 3.10.0-862.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff889ca624
Build id: 3.10.0-862.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff889cabe1
Build id: 3.10.0-862.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff889ccbf5
Build id: 3.10.0-862.el7.x86_64
|
0x295
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff889cce35
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff889ce052
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff88f1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff88f1f721
Build id: 3.10.0-862.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 37s! [ftdc:4048]
Modules linked in: binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype tun xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev sg pcspkr floppy parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 ata_piix mptspi scsi_transport_spi
crct10dif_pclmul crct10dif_common crc32c_intel mptscsih serio_raw libata mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 4048 Comm: ftdc Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9704eea4eeb0 ti: ffff970282ae0000 task.ti: ffff970282ae0000
RIP: 0010:[<ffffffff88907e3a>] [<ffffffff88907e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff970282ae3cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 0000000000019680 RCX: ffff97059ee1cd90
RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff970282ae3cf0 R08: ffff96fef4413c00 R09: ffffffff88b4a3d9
R10: ffff97059ee9bb00 R11: ffffce0554cbde00 R12: ffff970282ae3c68
R13: 00007f0014580000 R14: 0000000000000001 R15: ffffce0541ac2ac0
FS: 00007f000659c700(0000) GS:ffff97059ee80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0014580000 CR3: 00000004d27c6000 CR4: 00000000000007e0
Call Trace:
[<ffffffff88877018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff88877088>] flush_tlb_mm_range+0x68/0x140
[<ffffffff889c0497>] tlb_flush_mmu.part.75+0x37/0xe0
[<ffffffff889c1c75>] tlb_finish_mmu+0x55/0x60
[<ffffffff889ca624>] unmap_region+0xf4/0x140
[<ffffffff889cabe1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff889ccbf5>] do_munmap+0x295/0x470
[<ffffffff889cce35>] vm_munmap+0x65/0xb0
[<ffffffff889ce052>] SyS_munmap+0x22/0x30
[<ffffffff88f1f7d5>] system_call_fastpath+0x1c/0x21
[<ffffffff88f1f721>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 53 89 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #10858
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffab71173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffab67a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffab67a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffffab7e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffffab7e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffffab7ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffffab7ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffffab7f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffffab7f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffffab7f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffffabd74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [polkitd:2863]
Modules linked in: tcp_lp isofs ext4 mbcache jbd2 rfcomm fuse cmac xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ipmi_devintf ipmi_msghandler drbg ansi_cprng ccm ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep sunrpc vfat fat arc4 snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core rtsx_pci_ms nouveau snd_hwdep snd_seq ath10k_pci ath10k_core ath
mac80211 snd_seq_device edac_mce_amd kvm_amd snd_pcm kvm mxm_wmi video i2c_algo_bit ttm snd_timer drm_kms_helper snd cfg80211 irqbypass soundcore syscopyarea sysfillrect btusb btrtl btbcm dell_wmi btintel dell_smbios memstick bluetooth sysimgblt dcdbas sparse_keymap rfkill fb_sys_fops rtsx_pci_sdmmc dell_wmi_descriptor drm crc32_pclmul ghash_clmulni_intel drm_panel_orientation_quirks aesni_intel lrw gf128mul glue_helper ablk_helper wmi cryptd sg wdat_wdt rtsx_pci pcspkr sdhci_acpi r8169 sdhci mii ccp k10temp i2c_piix4 pinctrl_amd gpio_amdpt acpi_cpufreq mmc_core iosf_mbi i2c_designware_platform i2c_designware_core binfmt_misc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ahci libahci libata dm_mirror dm_region_hash
dm_log dm_mod
CPU: 6 PID: 2863 Comm: polkitd Tainted: G W ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: Dell Inc. Inspiron 5675/07PR60, BIOS 1.1.2 05/04/2017
task: ffff8b22beef0000 ti: ffff8b22a1f90000 task.ti: ffff8b22a1f90000
RIP: 0010:[<ffffffffab71173a>] [<ffffffffab71173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8b22a1f93cb8 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 000000000001b700 RCX: ffff8b22b6320678
RDX: 0000000000000004 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff8b22a1f93cf0 R08: ffff8b21bceec800 R09: ffffffffab9755c9
R10: ffff8b22b639f160 R11: ffffdb3388f8ae00 R12: ffff8b22a1f93c68
R13: ffff8b21bfc03500 R14: 0000000000000001 R15: ffffffffab81d04a
FS: 00007f0a399b4900(0000) GS:ffff8b22b6380000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0a399da000 CR3: 000000026af9c000 CR4: 00000000003407e0
Call Trace:
[<ffffffffab67a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffab67a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffab7e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffffab7e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffffab7ef014>] unmap_region+0xf4/0x140
[<ffffffffab7ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffffab7f1605>] do_munmap+0x2a5/0x480
[<ffffffffab7f1845>] vm_munmap+0x65/0xb0
[<ffffffffab7f2ad2>] SyS_munmap+0x22/0x30
[<ffffffffabd74ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 34 ac f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #9654
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f9f72
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x202
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8106c5f8
Build id: 3.10.0-514.26.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8106c6c9
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x69
|
|
4
|
tlb_flush_mmu.part.61
|
vmlinux |
0xffffffff811ac733
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x33
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff811ada25
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff811b4944
Build id: 3.10.0-514.26.2.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff811b4edf
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x13f
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff811b6e75
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x245
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff811b7091
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x41
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff811b8042
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff81697809
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [beam.smp:2831]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec intel_powerclamp ac97_bus snd_seq snd_seq_device coretemp snd_pcm ppdev snd_timer snd soundcore vmw_balloon pcspkr sg parport_pc parport i2c_piix4 shpchp vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crc32c_intel drm
mptspi scsi_transport_spi ata_piix mptscsih mptbase libata e1000 serio_raw i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 2831 Comm: beam.smp Tainted: G L ------------ 3.10.0-514.26.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/31/2013
task: ffff88009535edd0 ti: ffff880098ef0000 task.ti: ffff880098ef0000
RIP: 0010:[<ffffffff810f9f72>] [<ffffffff810f9f72>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff880098ef3d10 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffffffff8106c430 RCX: ffff8800bb41ad48
RDX: 0000000000000000 RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff880098ef3d48 R08: ffff880086433400 R09: ffffffff81318499
R10: ffff8800bb499b40 R11: ffffea0002548000 R12: 0000000000000286
R13: ffff880098ef3cc0 R14: ffff880098ef3e28 R15: 00007f803d90a000
FS: 00007f803e23b700(0000) GS:ffff8800bb480000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f6eb65ce47e CR3: 000000009aeaf000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001fffff000 ffff8800358ac808 ffff8800358ac4c0 0000000000000000
ffffffffffffffff ffff8800358ac808 ffff8800358cc798 ffff880098ef3d98
ffffffff8106c5f8 ffff8800358ac4c0 0000000000000000 ffffffffffffffff
Call Trace:
[<ffffffff8106c5f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8106c6c9>] flush_tlb_mm_range+0x69/0x140
[<ffffffff811ac733>] tlb_flush_mmu.part.61+0x33/0xc0
[<ffffffff811ada25>] tlb_finish_mmu+0x55/0x60
[<ffffffff811b4944>] unmap_region+0xf4/0x140
[<ffffffff811b4edf>] ? __vma_rb_erase+0x13f/0x220
[<ffffffff811b6e75>] do_munmap+0x245/0x420
[<ffffffff811b7091>] vm_munmap+0x41/0x60
[<ffffffff811b8042>] SyS_munmap+0x22/0x30
[<ffffffff81697809>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 da 9e 00 89 c2 39 f0 0f 8d 86 fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 99 ad 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 d9 9e 00 eb b7 0f b6 4d cc 4c 89
Complete report #8954
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff84d1173e
Build id: 3.10.0-957.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff84c7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff84c7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff84de4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff84de5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff84def014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff84def5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff84df1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff84df1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff84df2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff85374ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
12
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff85374d21
Build id: 3.10.0-957.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [Glowroot-Gauge-:2732]
Modules linked in: nfsv3 nfs_acl nfs lockd grace fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter qxl ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ppdev drm sg pcspkr drm_panel_orientation_quirks virtio_balloon joydev i2c_piix4 parport_pc parport auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi virtio_console
serio_raw floppy ata_piix libata e1000 virtio_pci virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 2732 Comm: Glowroot-Gauge- Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff9d0ee0d5e180 ti: ffff9d0f2f138000 task.ti: ffff9d0f2f138000
RIP: 0010:[<ffffffff84d1173e>] [<ffffffff84d1173e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9d0f2f13bcb8 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00000000000000fc RCX: ffff9d103fd1f610
RDX: 0000000000000002 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9d0f2f13bcf0 R08: ffff9d0fffd42000 R09: ffffffff84f755c9
R10: ffff9d103fc1f160 R11: ffffe9a705037c00 R12: 0000000000000296
R13: ffff9d0f2f13bc68 R14: 0000000000000001 R15: ffffe9a70569b280
FS: 00007f7a4f5f5700(0000) GS:ffff9d103fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7ad4fd1000 CR3: 00000001a640a000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff84c7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff84c7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff84de4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff84de5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff84def014>] unmap_region+0xf4/0x140
[<ffffffff84def5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff84df1605>] do_munmap+0x2a5/0x480
[<ffffffff84df1845>] vm_munmap+0x65/0xb0
[<ffffffff84df2ad2>] SyS_munmap+0x22/0x30
[<ffffffff85374ddb>] system_call_fastpath+0x22/0x27
[<ffffffff85374d21>] ? system_call_after_swapgs+0xae/0x146
Code: 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 94 85 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #7450
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8631173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8627a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8627a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
tlb_flush_mmu.part.76
|
vmlinux |
0xffffffff863e4077
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
5
|
tlb_finish_mmu
|
vmlinux |
0xffffffff863e5975
Build id: 3.10.0-957.el7.x86_64
|
0x55
|
|
6
|
unmap_region
|
vmlinux |
0xffffffff863ef014
Build id: 3.10.0-957.el7.x86_64
|
0xf4
|
|
7
|
|
vmlinux |
0xffffffff863ef5d1
Build id: 3.10.0-957.el7.x86_64
|
0x121
|
|
8
|
do_munmap
|
vmlinux |
0xffffffff863f1605
Build id: 3.10.0-957.el7.x86_64
|
0x2a5
|
|
9
|
vm_munmap
|
vmlinux |
0xffffffff863f1845
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
10
|
SyS_munmap
|
vmlinux |
0xffffffff863f2ad2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff86974ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [C1 CompilerThre:17930]
Modules linked in: fuse xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop binfmt_misc ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc coretemp ext4 iosf_mbi mbcache jbd2 crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon
joydev parport_pc parport sg vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx serio_raw drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 ata_piix drm mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 17930 Comm: C1 CompilerThre Kdump: loaded Not tainted 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9a1d5f68c100 ti: ffff9a1d7eb10000 task.ti: ffff9a1d7eb10000
RIP: 0010:[<ffffffff8631173a>] [<ffffffff8631173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9a1d7eb13cb8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9a1ded6204b8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9a1d7eb13cf0 R08: ffff9a1b39cb3800 R09: ffffffff865755c9
R10: ffff9a1ded69f160 R11: ffffd90c90b26a00 R12: 0000000000000296
R13: ffff9a1d7eb13c68 R14: 0000000000000001 R15: ffffd90c888e15c0
FS: 00007f63dd7ec700(0000) GS:ffff9a1ded680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f63dc9bd000 CR3: 00000001e77c0000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8627a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8627a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff863e4077>] tlb_flush_mmu.part.76+0x37/0xe0
[<ffffffff863e5975>] tlb_finish_mmu+0x55/0x60
[<ffffffff863ef014>] unmap_region+0xf4/0x140
[<ffffffff863ef5d1>] ? __vma_rb_erase+0x121/0x220
[<ffffffff863f1605>] do_munmap+0x2a5/0x480
[<ffffffff863f1845>] vm_munmap+0x65/0xb0
[<ffffffff863f2ad2>] SyS_munmap+0x22/0x30
[<ffffffff86974ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 f4 86 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89