Complete report #1352931
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff84f1720a
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff84fc7940
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff84f17359
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff84fc4825
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff84fc96ab
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff8501cc69
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff84ff6887
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xcb7
|
|
8
|
|
vmlinux |
0xffffffff84e7dff5
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x115
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffff85594653
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x213
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffff85594975
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffff85590778
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [vasp_std:69790]
Modules linked in: tcp_lp tun devlink bridge stp llc ip_set vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev vmw_balloon iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec lrw gf128mul ac97_bus glue_helper ablk_helper snd_seq cryptd snd_seq_device snd_pcm joydev pcspkr snd_timer sg snd vmw_vmci soundcore i2c_piix4 parport_pc parport xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul drm crct10dif_common nfit mptspi crc32c_intel scsi_transport_spi libata mptscsih libnvdimm e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
[last unloaded: ip_tables]
CPU: 4 PID: 69790 Comm: vasp_std Kdump: loaded Tainted: G L ------------ 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9cbad6a49080 ti: ffff9cbafc120000 task.ti: ffff9cbafc120000
RIP: 0010:[<ffffffff84f1720a>] [<ffffffff84f1720a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9cbafc123bb0 EFLAGS: 00000202
RAX: 0000000000000010 RBX: 0008000084fc7940 RCX: ffff9cbafda20678
RDX: 0000000000000010 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9cbafc123be8 R08: ffff9c9c730fbc00 R09: ffffffff85186a19
R10: ffff9cbafd71f160 R11: fffff14f717ec000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9cbafc123b60 R15: 0000000001320122
FS: 00002b9cc8fbb900(0000) GS:ffff9cbafd700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000001ad35000 CR3: 0000001ced0fa000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff84fc7940>] ? drain_pages+0xb0/0xb0
[<ffffffff84f17359>] on_each_cpu_mask+0x29/0x70
[<ffffffff84fc4825>] drain_all_pages+0xb5/0xc0
[<ffffffff84fc96ab>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff8501cc69>] alloc_pages_vma+0xa9/0x200
[<ffffffff84ff6887>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff84e7dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffff85594653>] __do_page_fault+0x213/0x500
[<ffffffff85594975>] do_page_fault+0x35/0x90
[<ffffffff85590778>] page_fault+0x28/0x30
Code: 48 63 35 9e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 b5 85 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1274099
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffad31e23a
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffad3d28c0
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffad31e389
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffad3cf6c5
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffad3d46ab
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x8db
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffffad4293c9
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffad4022ef
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0xb9f
|
|
8
|
|
vmlinux |
0xffffffffad46f4dd
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x4d
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffffad9bf7d3
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x213
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffffad9bfb05
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffffad9bb7b8
Build id: 3.10.0-1160.105.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 33s! [crond:92926]
Modules linked in: tun ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel e1000 mptspi scsi_transport_spi serio_raw mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 7 PID: 92926 Comm: crond Kdump: loaded Tainted: G L ------------ 3.10.0-1160.105.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff88dcd9f82100 ti: ffff88dc4940c000 task.ti: ffff88dc4940c000
RIP: 0010:[<ffffffffad31e23a>] [<ffffffffad31e23a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff88dc4940fbb0 EFLAGS: 00000202
RAX: 0000000000000008 RBX: 00080000ad3d28c0 RCX: ffff88ddf78206f0
RDX: 0000000000000008 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88dc4940fbe8 R08: ffff88ddbb88e800 R09: ffffffffad59c7b9
R10: ffff88ddf77df160 R11: fffff7d40266f200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff88dc4940fb60 R15: 0000000001320122
FS: 00007f00e9a0f840(0000) GS:ffff88ddf77c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055567c15a8b8 CR3: 00000000b904c000 CR4: 00000000001607e0
Call Trace:
[<ffffffffad3d28c0>] ? drain_pages+0xb0/0xb0
[<ffffffffad31e389>] on_each_cpu_mask+0x29/0x70
[<ffffffffad3cf6c5>] drain_all_pages+0xb5/0xd0
[<ffffffffad3d46ab>] __alloc_pages_nodemask+0x8db/0xbf0
[<ffffffffad4293c9>] alloc_pages_vma+0xa9/0x210
[<ffffffffad4022ef>] handle_mm_fault+0xb9f/0x1190
[<ffffffffad46f4dd>] ? do_filp_open+0x4d/0xc0
[<ffffffffad9bf7d3>] __do_page_fault+0x213/0x510
[<ffffffffad9bfb05>] do_page_fault+0x35/0x90
[<ffffffffad9bb7b8>] page_fault+0x28/0x30
Code: 48 63 35 8e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 f5 ad f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1201149
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb5d170da
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb5dc78e0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffb5d17229
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffb5dc47c5
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffb5dc964b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffffb5e1cc49
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffb5df6837
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xcb7
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffb6390653
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffb6390975
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x35
|
|
10
|
|
vmlinux |
0xffffffffb638cac9
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xaa
|
|
11
|
page_fault
|
vmlinux |
0xffffffffb638c778
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [node:24137]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc squashfs loop sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci parport i2c_piix4 binfmt_misc ip_tables ext4 mbcache jbd2 ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper e1000e crct10dif_pclmul crct10dif_common ahci nfit ata_piix syscopyarea crc32c_intel sysfillrect sysimgblt fb_sys_fops ttm libahci serio_raw libata ptp drm pps_core libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 24137 Comm: node Kdump: loaded Not tainted 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff930d79771080 ti: ffff9310ef218000 task.ti: ffff9310ef218000
RIP: 0010:[<ffffffffb5d170da>] [<ffffffffb5d170da>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9310ef21bbb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c0000b5dc78e0 RCX: ffff93117fc20600
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9310ef21bbe8 R08: ffff930eb6356000 R09: ffffffffb5f86bf9
R10: ffff93117fc9f160 R11: ffffe872c8806c00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9310ef21bb60 R15: 0000000001320122
FS: 00007f10a0931700(0000) GS:ffff93117fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00001b6b74ccb000 CR3: 00000001844aa000 CR4: 00000000000607e0
Call Trace:
[<ffffffffb5dc78e0>] ? drain_pages+0xb0/0xb0
[<ffffffffb5d17229>] on_each_cpu_mask+0x29/0x70
[<ffffffffb5dc47c5>] drain_all_pages+0xb5/0xc0
[<ffffffffb5dc964b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffffb5e1cc49>] alloc_pages_vma+0xa9/0x200
[<ffffffffb5df6837>] handle_mm_fault+0xcb7/0xfb0
[<ffffffffb6390653>] __do_page_fault+0x213/0x500
[<ffffffffb6390975>] do_page_fault+0x35/0x90
[<ffffffffb638cac9>] ? error_swapgs+0xaa/0xc0
[<ffffffffb638c778>] page_fault+0x28/0x30
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 95 b6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1040739
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
|
vmlinux |
0xffffffff98fd28c0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb0
|
|
2
|
on_each_cpu_mask
|
vmlinux |
0xffffffff98f1e389
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x29
|
|
3
|
drain_all_pages
|
vmlinux |
0xffffffff98fcf6c5
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb5
|
|
4
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff98fd46ab
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x8db
|
|
5
|
alloc_pages_vma
|
vmlinux |
0xffffffff990293c9
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xa9
|
|
6
|
handle_mm_fault
|
vmlinux |
0xffffffff990022ef
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb9f
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff995bf7d3
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff995bfb05
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff995bb7b8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 25s! [java:30604]
PKRU: 55555554
Call Trace:
[<ffffffff98fd28c0>] ? drain_pages+0xb0/0xb0
[<ffffffff98f1e389>] on_each_cpu_mask+0x29/0x70
[<ffffffff98fcf6c5>] drain_all_pages+0xb5/0xd0
[<ffffffff98fd46ab>] __alloc_pages_nodemask+0x8db/0xbf0
[<ffffffff990293c9>] alloc_pages_vma+0xa9/0x210
[<ffffffff990022ef>] handle_mm_fault+0xb9f/0x1190
[<ffffffff995bf7d3>] __do_page_fault+0x213/0x510
[<ffffffff995bfb05>] do_page_fault+0x35/0x90
[<ffffffff995bb7b8>] page_fault+0x28/0x30
Complete report #1007955
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff90b1720a
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff90bc7940
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff90b17359
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff90bc4825
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff90bc96ab
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff90c1cc69
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff90bf6887
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xcb7
|
|
8
|
|
vmlinux |
0xffffffff90ae4896
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x86
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffff91194653
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x213
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffff91194975
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffff91190778
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [node:3181]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul
glue_helper ablk_helper ppdev joydev cryptd pcspkr vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel nfit ata_piix drm mptspi libnvdimm scsi_transport_spi libata serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 3181 Comm: node Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff96f62b684200 ti: ffff96f53c9c4000 task.ti: ffff96f53c9c4000
RIP: 0010:[<ffffffff90b1720a>] [<ffffffff90b1720a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff96f53c9c7bb0 EFLAGS: 00000202
RAX: 0000000000000007 RBX: 000c000090bc7940 RCX: ffff96f6bb5e0678
RDX: 0000000000000007 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff96f53c9c7be8 R08: ffff96f67b98bc00 R09: ffffffff90d86a19
R10: ffff96f6bb51f160 R11: ffffd20bc5cd4200 R12: 0000000000000282
R13: 0000000000000282 R14: ffff96f53c9c7b60 R15: 0000000001320122
FS: 00007fe0d14b3c18(0000) GS:ffff96f6bb500000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00001176beb4e000 CR3: 0000000106a3c000 CR4: 00000000003607e0
Call Trace:
[<ffffffff90bc7940>] ? drain_pages+0xb0/0xb0
[<ffffffff90b17359>] on_each_cpu_mask+0x29/0x70
[<ffffffff90bc4825>] drain_all_pages+0xb5/0xc0
[<ffffffff90bc96ab>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff90c1cc69>] alloc_pages_vma+0xa9/0x200
[<ffffffff90bf6887>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff90ae4896>] ? update_curr+0x86/0x1e0
[<ffffffff91194653>] __do_page_fault+0x213/0x500
[<ffffffff91194975>] do_page_fault+0x35/0x90
[<ffffffff91190778>] page_fault+0x28/0x30
Code: 48 63 35 9e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 75 91 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #887046
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffba51e23a
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffba5d28c0
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffba51e389
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffba5cf6c5
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffba5d46ab
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x8db
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffffba6293c9
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffba6022ef
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb9f
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffbabbf7d3
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffbabbfb05
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffffbabbb7b8
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [synology-backup:128819]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter br_netfilter bridge stp llc overlay(T) ip_vs nf_conntrack vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi btusb btrtl crc32_pclmul btbcm ac97_bus btintel ghash_clmulni_intel snd_seq bluetooth aesni_intel snd_seq_device lrw gf128mul snd_pcm rfkill glue_helper ablk_helper snd_timer cryptd ppdev snd soundcore pcspkr vmw_balloon sg joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul
crct10dif_common nfit crc32c_intel nvme libata serio_raw e1000 libnvdimm nvme_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod synosnap(OE)
CPU: 8 PID: 128819 Comm: synology-backup Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.102.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff98e5365a0000 ti: ffff98e22bfd8000 task.ti: ffff98e22bfd8000
RIP: 0010:[<ffffffffba51e23a>] [<ffffffffba51e23a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff98e22bfdbbb0 EFLAGS: 00000202
RAX: 0000000000000009 RBX: 00080000ba5d28c0 RCX: ffff98e8dd860718
RDX: 0000000000000009 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff98e22bfdbbe8 R08: ffff98e22c55d400 R09: ffffffffba79c7b9
R10: ffff98e8dd81f160 R11: fffff0cbd1d65600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff98e22bfdbb60 R15: 0000000001320122
FS: 00007fdbcb7fe700(0000) GS:ffff98e8dd800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdb80101918 CR3: 000000081cbee000 CR4: 00000000003607e0
Call Trace:
[<ffffffffba5d28c0>] ? drain_pages+0xb0/0xb0
[<ffffffffba51e389>] on_each_cpu_mask+0x29/0x70
[<ffffffffba5cf6c5>] drain_all_pages+0xb5/0xd0
[<ffffffffba5d46ab>] __alloc_pages_nodemask+0x8db/0xbf0
[<ffffffffba6293c9>] alloc_pages_vma+0xa9/0x210
[<ffffffffba6022ef>] handle_mm_fault+0xb9f/0x1190
[<ffffffffbabbf7d3>] __do_page_fault+0x213/0x510
[<ffffffffbabbfb05>] do_page_fault+0x35/0x90
[<ffffffffbabbb7b8>] page_fault+0x28/0x30
Code: 48 63 35 4e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 40 19 15 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #834741
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa091e0ba
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffa09d2730
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffa091e209
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffa09cf535
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffa09d451b
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x8db
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffffa0a29049
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffa0a02297
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xcb7
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffa0fbf7d3
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffa0fbfb05
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffffa0fbb7b8
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 1732s! [systemd-journal:38249]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg joydev vmw_balloon pcspkr parport_pc parport snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device
snd_pcm snd_timer snd soundcore vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm serio_raw e1000 ata_piix mptspi nfit scsi_transport_spi libata mptscsih libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 38249 Comm: systemd-journal Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.83.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9cb208410000 ti: ffff9cb20840c000 task.ti: ffff9cb20840c000
RIP: 0010:[<ffffffffa091e0ba>] [<ffffffffa091e0ba>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9cb20840fbb0 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00080000a09d2730 RCX: ffff9cb43569fc10
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9cb20840fbe8 R08: ffff9cb37fdd7000 R09: ffffffffa0b9c179
R10: ffff9cb43561f160 R11: fffff72b448ef600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9cb20840fb60 R15: 0000000001320122
FS: 00007f438d774880(0000) GS:ffff9cb435600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000555c3a3f77b8 CR3: 0000000008576000 CR4: 00000000003607f0
Call Trace:
[<ffffffffa09d2730>] ? drain_pages+0xb0/0xb0
[<ffffffffa091e209>] on_each_cpu_mask+0x29/0x70
[<ffffffffa09cf535>] drain_all_pages+0xb5/0xd0
[<ffffffffa09d451b>] __alloc_pages_nodemask+0x8db/0xbf0
[<ffffffffa0a29049>] alloc_pages_vma+0xa9/0x210
[<ffffffffa0a02297>] handle_mm_fault+0xcb7/0xfb0
[<ffffffffa0fbf7d3>] __do_page_fault+0x213/0x510
[<ffffffffa0fbfb05>] do_page_fault+0x35/0x90
[<ffffffffa0fbb7b8>] page_fault+0x28/0x30
Code: 48 63 35 8e 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 55 a1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 19 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #734530
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9ed1720a
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff9edc7940
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff9ed17359
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff9edc4825
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff9edc96ab
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff9ee1cc69
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff9edf6887
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xcb7
|
|
8
|
|
vmlinux |
0xffffffff9ee4da43
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x93
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffff9f394653
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x213
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffff9f394975
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffff9f390778
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [fping:80572]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables devlink tun ipmi_devintf ipmi_msghandler ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 xt_conntrack ip6table_filter ip6_tables iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle vmw_vsock_vmci_transport vsock bnep sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus aesni_intel btusb snd_seq btrtl btbcm btintel lrw gf128mul glue_helper ablk_helper snd_seq_device bluetooth snd_pcm ppdev rfkill cryptd snd_timer snd soundcore joydev pcspkr vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci sch_fq_codel ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel ata_piix nfit drm libnvdimm libata serio_raw mptspi scsi_transport_spi mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 80572 Comm: fping Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9a17808ab180 ti: ffff9a18343e4000 task.ti: ffff9a18343e4000
RIP: 0010:[<ffffffff9ed1720a>] [<ffffffff9ed1720a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9a18343e7bb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c00009edc7940 RCX: ffff9a1837620650
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9a18343e7be8 R08: ffff9a17fb88b800 R09: ffffffff9ef86a19
R10: ffff9a18376df160 R11: ffffe63505301c00 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9a18343e7b60 R15: 0000000001320122
FS: 00007f59b72ca740(0000) GS:ffff9a18376c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffeafb3af90 CR3: 000000015ce50000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9edc7940>] ? drain_pages+0xb0/0xb0
[<ffffffff9ed17359>] on_each_cpu_mask+0x29/0x70
[<ffffffff9edc4825>] drain_all_pages+0xb5/0xc0
[<ffffffff9edc96ab>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff9ee1cc69>] alloc_pages_vma+0xa9/0x200
[<ffffffff9edf6887>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff9ee4da43>] ? do_sync_write+0x93/0xe0
[<ffffffff9f394653>] __do_page_fault+0x213/0x500
[<ffffffff9f394975>] do_page_fault+0x35/0x90
[<ffffffff9f390778>] page_fault+0x28/0x30
Code: 48 63 35 9e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 95 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 6d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #441038
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff81d170da
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff81dc78e0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff81d17229
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff81dc47c5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff81dc964b
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff81e1cc49
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff81df6837
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xcb7
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff82390653
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff82390975
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffff8238c778
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [freshclam:8482]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper pcspkr cryptd sg joydev vmw_vmci parport_pc i2c_piix4 parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper
sd_mod crc_t10dif syscopyarea sysfillrect crct10dif_generic sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci drm crc32c_intel libahci ata_piix libata serio_raw nfit libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 8482 Comm: freshclam Kdump: loaded Tainted: G L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff993206d34200 ti: ffff9931fbe8c000 task.ti: ffff9931fbe8c000
RIP: 0010:[<ffffffff81d170da>] [<ffffffff81d170da>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9931fbe8fbb0 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c000081dc78e0 RCX: ffff9932efc20628
RDX: 0000000000000000 RSI: 0000000000000003 RDI: 0000000000000000
RBP: ffff9931fbe8fbe8 R08: ffff9932e8fb4000 R09: ffffffff81f86bf9
R10: ffff9932efd1f160 R11: ffffeb90c4a1f000 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9931fbe8fb60 R15: 0000000001320122
FS: 00007f3fa2c38880(0000) GS:ffff9932efd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3f85a7a4e8 CR3: 000000003f4a8000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81dc78e0>] ? drain_pages+0xb0/0xb0
[<ffffffff81d17229>] on_each_cpu_mask+0x29/0x70
[<ffffffff81dc47c5>] drain_all_pages+0xb5/0xc0
[<ffffffff81dc964b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff81e1cc49>] alloc_pages_vma+0xa9/0x200
[<ffffffff81df6837>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff82390653>] __do_page_fault+0x213/0x500
[<ffffffff82390975>] do_page_fault+0x35/0x90
[<ffffffff8238c778>] page_fault+0x28/0x30
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 95 82 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #428632
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8731722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff873c78f0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff87317379
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff873c47d5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff873c965b
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff8741cc39
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa9
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff873f6827
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xcb7
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff87994653
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff87994975
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x35
|
|
10
|
|
vmlinux |
0xffffffff87990ac9
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xaa
|
|
11
|
page_fault
|
vmlinux |
0xffffffff87990778
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [gnome-shell:6677]
Modules linked in: tcp_lp nls_utf8 isofs veth vboxsf(OE) xt_addrtype br_netfilter overlay(T) xt_CHECKSUM tun devlink xt_REDIRECT nf_nat_redirect xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_intel8x0 snd_ac97_codec ac97_bus snd_seq snd_seq_device joydev snd_pcm ppdev snd_timer vboxguest(OE) pcspkr snd sg soundcore parport_pc
video parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci ata_piix serio_raw e1000 libata crc32c_intel drm_panel_orientation_quirks fuse
CPU: 3 PID: 6677 Comm: gnome-shell Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff890597de6300 ti: ffff89059992c000 task.ti: ffff89059992c000
RIP: 0010:[<ffffffff8731722a>] [<ffffffff8731722a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff89059992fbb0 EFLAGS: 00010202
RAX: 0000000000000000 RBX: ffff89059992fb70 RCX: ffff89069fc20450
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff89059992fbe8 R08: ffff890698492000 R09: ffffffff875869d9
R10: ffff89069fd9f160 R11: ffffd41fc06ef500 R12: ffff89059992fb80
R13: ffffffff8758692f R14: ffff89069b001500 R15: 0000000001320122
FS: 00007efe3f57ca80(0000) GS:ffff89069fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000003c990ec CR3: 000000001971a000 CR4: 00000000000206e0
Call Trace:
[<ffffffff873c78f0>] ? drain_pages+0xb0/0xb0
[<ffffffff87317379>] on_each_cpu_mask+0x29/0x70
[<ffffffff873c47d5>] drain_all_pages+0xb5/0xc0
[<ffffffff873c965b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff8741cc39>] alloc_pages_vma+0xa9/0x200
[<ffffffff873f6827>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff87994653>] __do_page_fault+0x213/0x500
[<ffffffff87994975>] do_page_fault+0x35/0x90
[<ffffffff87990ac9>] ? error_swapgs+0xaa/0xc0
[<ffffffff87990778>] page_fault+0x28/0x30
Code: 48 63 35 7e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 f5 87 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #402836
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa8b1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffa8bbf550
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffa8b11889
Build id: 3.10.0-957.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffa8bbc435
Build id: 3.10.0-957.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffffa915d228
Build id: 3.10.0-957.el7.x86_64
|
0x510
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffa8bc0e95
Build id: 3.10.0-957.el7.x86_64
|
0x405
|
|
7
|
alloc_pages_vma
|
vmlinux |
0xffffffffa8c11105
Build id: 3.10.0-957.el7.x86_64
|
0xb5
|
|
8
|
handle_pte_fault
|
vmlinux |
0xffffffffa8be96c7
Build id: 3.10.0-957.el7.x86_64
|
0x887
|
|
9
|
handle_mm_fault
|
vmlinux |
0xffffffffa8bebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
10
|
__do_page_fault
|
vmlinux |
0xffffffffa916f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
11
|
do_page_fault
|
vmlinux |
0xffffffffa916f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
12
|
page_fault
|
vmlinux |
0xffffffffa916b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [java:18985]
Modules linked in: vmw_vsock_vmci_transport vsock vfat fat crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr cryptd ppdev vmw_balloon joydev sg vmw_vmci parport_pc parport i2c_piix4 tpm_crb ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx e1000e serio_raw ptp drm_kms_helper pps_core syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptsas scsi_transport_sas ahci mptscsih drm mptbase nfit libahci ata_piix libnvdimm libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 9 PID: 18985 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.17369862.B64.2012240522 12/24/2020
task: ffff91f9d6298000 ti: ffff91f4f632c000 task.ti: ffff91f4f632c000
RIP: 0010:[<ffffffffa8b1173a>] [<ffffffffa8b1173a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff91f4f632fac0 EFLAGS: 00000202
RAX: 000000000000000e RBX: 000c0000a8bbf550 RCX: ffff91fadf5a06f0
RDX: 000000000000000e RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff91f4f632faf8 R08: ffff91f434bfd400 R09: ffffffffa8d755c9
R10: ffff91fadf45f160 R11: ffffdbee8a7bce00 R12: 0000000000000286
R13: 0000000000000286 R14: ffff91f4f632fa70 R15: 0000000001320122
FS: 00007f704fbff700(0000) GS:ffff91fadf440000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000007a7dbf000 CR3: 00000006733c8000 CR4: 0000000000760fe0
PKRU: 55555554
Call Trace:
[<ffffffffa8bbf550>] ? drain_pages+0xb0/0xb0
[<ffffffffa8b11889>] on_each_cpu_mask+0x29/0x70
[<ffffffffa8bbc435>] drain_all_pages+0xb5/0xc0
[<ffffffffa915d228>] __alloc_pages_slowpath+0x510/0x724
[<ffffffffa8bc0e95>] __alloc_pages_nodemask+0x405/0x420
[<ffffffffa8c11105>] alloc_pages_vma+0xb5/0x200
[<ffffffffa8be96c7>] handle_pte_fault+0x887/0xd10
[<ffffffffa8bebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffa916f5e3>] __do_page_fault+0x203/0x500
[<ffffffffa916f915>] do_page_fault+0x35/0x90
[<ffffffffa916b758>] page_fault+0x28/0x30
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 74 a9 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #392938
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb2916c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb29c74a0
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffb2916d59
Build id: 3.10.0-1160.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffb29c4385
Build id: 3.10.0-1160.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffffb2f7c6b2
Build id: 3.10.0-1160.el7.x86_64
|
0x510
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffb29c8d76
Build id: 3.10.0-1160.el7.x86_64
|
0x436
|
|
7
|
alloc_pages_vma
|
vmlinux |
0xffffffffb2a1c269
Build id: 3.10.0-1160.el7.x86_64
|
0xa9
|
|
8
|
handle_mm_fault
|
vmlinux |
0xffffffffb29f5ec7
Build id: 3.10.0-1160.el7.x86_64
|
0xcb7
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffffb2f8e653
Build id: 3.10.0-1160.el7.x86_64
|
0x213
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffffb2f8e975
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffffb2f8a778
Build id: 3.10.0-1160.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [util.rb:23:60859]
Modules linked in: tcp_lp veth br_netfilter tun devlink overlay(T) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc parport snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore vmw_vmci i2c_piix4 xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper ata_piix serio_raw mptspi syscopyarea scsi_transport_spi sysfillrect sysimgblt fb_sys_fops libata ttm mptscsih nfit drm e1000 libnvdimm mptbase drm_panel_orientation_quirks
fuse [last unloaded: ip_tables]
CPU: 1 PID: 60859 Comm: util.rb:23 Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff95e008318000 ti: ffff95e028bc8000 task.ti: ffff95e028bc8000
RIP: 0010:[<ffffffffb2916c0a>] [<ffffffffb2916c0a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff95e028bcbb48 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff95e0ba6e0488
RDX: 0000000000000003 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff95e028bcbb80 R08: ffff95e082b83000 R09: ffffffffb2b85f99
R10: ffff95e0ba65f160 R11: ffffe48d4037c800 R12: 0000000000000296
R13: ffff95e028bcbaf8 R14: 0000000001320122 R15: 0000000076415727
FS: 00007f4ff7dfd700(0000) GS:ffff95e0ba640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4febb6f000 CR3: 00000000b9550000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb29c74a0>] ? drain_pages+0xb0/0xb0
[<ffffffffb2916d59>] on_each_cpu_mask+0x29/0x70
[<ffffffffb29c4385>] drain_all_pages+0xb5/0xc0
[<ffffffffb2f7c6b2>] __alloc_pages_slowpath+0x510/0x729
[<ffffffffb29c8d76>] __alloc_pages_nodemask+0x436/0x450
[<ffffffffb2a1c269>] alloc_pages_vma+0xa9/0x200
[<ffffffffb29f5ec7>] handle_mm_fault+0xcb7/0xfb0
[<ffffffffb2f8e653>] __do_page_fault+0x213/0x500
[<ffffffffb2f8e975>] do_page_fault+0x35/0x90
[<ffffffffb2f8a778>] page_fault+0x28/0x30
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 b3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #323466
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8b907e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff8b99cb10
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff8b907f89
Build id: 3.10.0-862.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff8b9999f5
Build id: 3.10.0-862.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8b99e860
Build id: 3.10.0-862.el7.x86_64
|
0x8a0
|
|
6
|
alloc_pages_vma
|
vmlinux |
0xffffffff8b9ebd65
Build id: 3.10.0-862.el7.x86_64
|
0xb5
|
|
7
|
handle_pte_fault
|
vmlinux |
0xffffffff8b9c57ec
Build id: 3.10.0-862.el7.x86_64
|
0x7ec
|
|
8
|
handle_mm_fault
|
vmlinux |
0xffffffff8b9c747d
Build id: 3.10.0-862.el7.x86_64
|
0x39d
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffff8bf1a597
Build id: 3.10.0-862.el7.x86_64
|
0x197
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffff8bf1a925
Build id: 3.10.0-862.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffff8bf16768
Build id: 3.10.0-862.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [java:2166]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi crc32_pclmul ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm lrw gf128mul btusb btrtl glue_helper ablk_helper cryptd snd_timer snd soundcore btbcm btintel bluetooth ppdev rfkill joydev pcspkr nfit sg parport_pc parport libnvdimm shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi libata mptscsih serio_raw mptbase e1000 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 2166 Comm: java Tainted: G L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff973f90d20000 ti: ffff973f90d3c000 task.ti: ffff973f90d3c000
RIP: 0010:[<ffffffff8b907e3a>] [<ffffffff8b907e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff973f90d3fb30 EFLAGS: 00000202
RAX: 0000000000000006 RBX: 000c00008b99cb10 RCX: ffff973fbb79cd48
RDX: 0000000000000006 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff973f90d3fb68 R08: ffff973fba6c3800 R09: ffffffff8bb4a3d9
R10: ffff973fbb69bb00 R11: ffffd35c81e01600 R12: 0000000000000286
R13: 0000000000000286 R14: ffff973f90d3fae0 R15: 0000000001320122
FS: 00007fecb8c8a700(0000) GS:ffff973fbb680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f521b06cff8 CR3: 0000000050c42000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8b99cb10>] ? drain_pages+0xb0/0xb0
[<ffffffff8b907f89>] on_each_cpu_mask+0x29/0x70
[<ffffffff8b9999f5>] drain_all_pages+0xb5/0xc0
[<ffffffff8b99e860>] __alloc_pages_nodemask+0x8a0/0xbb0
[<ffffffff8b9ebd65>] alloc_pages_vma+0xb5/0x200
[<ffffffff8b9c57ec>] handle_pte_fault+0x7ec/0xc30
[<ffffffff8b9c747d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff8bf1a597>] __do_page_fault+0x197/0x4f0
[<ffffffff8bf1a925>] do_page_fault+0x35/0x90
[<ffffffff8bf16768>] page_fault+0x28/0x30
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 53 8c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #116462
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f9f22
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x202
|
|
2
|
|
vmlinux |
0xffffffff81189d70
Build id: 3.10.0-514.16.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff810fa06a
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x2a
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff81186c55
Build id: 3.10.0-514.16.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff816824b1
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x511
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8118b5c5
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x405
|
|
7
|
alloc_pages_vma
|
vmlinux |
0xffffffff811d272a
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x9a
|
|
8
|
handle_mm_fault
|
vmlinux |
0xffffffff811b19ef
Build id: 3.10.0-514.16.1.el7.x86_64
|
0xc6f
|
|
9
|
__do_page_fault
|
vmlinux |
0xffffffff81692644
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x154
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffff81692975
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x35
|
|
11
|
page_fault
|
vmlinux |
0xffffffff8168eb88
Build id: 3.10.0-514.16.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [git:111669]
Modules linked in: isofs fuse tun ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event intel_powerclamp snd_ens1371 coretemp snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device iosf_mbi crc32_pclmul ghash_clmulni_intel snd_pcm aesni_intel lrw snd_timer gf128mul glue_helper snd ablk_helper soundcore cryptd ppdev vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm crct10dif_common ata_piix crc32c_intel serio_raw libata drm mptspi scsi_transport_spi e1000 mptscsih
mptbase i2c_core fjes [last unloaded: ip_tables]
CPU: 2 PID: 111669 Comm: git Tainted: G L ------------ 3.10.0-514.16.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88008a9fce70 ti: ffff8801d873c000 task.ti: ffff8801d873c000
RIP: 0010:[<ffffffff810f9f22>] [<ffffffff810f9f22>] smp_call_function_many+0x202/0x260
RSP: 0000:ffff8801d873fb80 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000c000081189d70 RCX: ffff88023561a6a0
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8801d873fbb8 R08: ffff88017cf14400 R09: ffffffff813181a9
R10: ffff880235699b40 R11: ffffea00024bca00 R12: 0000000000000286
R13: 0000000000000000 R14: ffff8801d873faf0 R15: 0000000000000286
FS: 00007f6467fff700(0000) GS:ffff880235680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f63d3918000 CR3: 00000000b8bc2000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001000000c0 ffffffff81eef000 ffffffff81eef000 ffffffff81189d70
0000000000000000 0000000000000002 ffff88023ffd8000 ffff8801d873fbe8
ffffffff810fa06a 0000000000000003 0000000000000080 0000000000000080
Call Trace:
[<ffffffff81189d70>] ? drain_pages+0xb0/0xb0
[<ffffffff810fa06a>] on_each_cpu_mask+0x2a/0x60
[<ffffffff81186c55>] drain_all_pages+0xb5/0xc0
[<ffffffff816824b1>] __alloc_pages_slowpath+0x511/0x725
[<ffffffff8118b5c5>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff811d272a>] alloc_pages_vma+0x9a/0x150
[<ffffffff811b19ef>] handle_mm_fault+0xc6f/0xfe0
[<ffffffff81692644>] __do_page_fault+0x154/0x450
[<ffffffff81692975>] do_page_fault+0x35/0x90
[<ffffffff8168eb88>] page_fault+0x28/0x30
Code: 48 63 35 76 99 9e 00 89 c2 39 f0 0f 8d 86 fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 58 ad 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 45 99 9e 00 eb b7 0f b6 4d cc 4c 89
Complete report #114589
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8f11760a
Build id: 3.10.0-1127.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff8f1c7850
Build id: 3.10.0-1127.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff8f117759
Build id: 3.10.0-1127.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff8f1c46c5
Build id: 3.10.0-1127.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff8f77b2f5
Build id: 3.10.0-1127.el7.x86_64
|
0x510
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8f1c9146
Build id: 3.10.0-1127.el7.x86_64
|
0x436
|
|
7
|
alloc_pages_vma
|
vmlinux |
0xffffffff8f21c2a9
Build id: 3.10.0-1127.el7.x86_64
|
0xa9
|
|
8
|
handle_mm_fault
|
vmlinux |
0xffffffff8f1f5f97
Build id: 3.10.0-1127.el7.x86_64
|
0xcb7
|
|
9
|
|
vmlinux |
0xffffffff8f24c663
Build id: 3.10.0-1127.el7.x86_64
|
0x93
|
|
10
|
__do_page_fault
|
vmlinux |
0xffffffff8f78d653
Build id: 3.10.0-1127.el7.x86_64
|
0x213
|
|
11
|
do_page_fault
|
vmlinux |
0xffffffff8f78d975
Build id: 3.10.0-1127.el7.x86_64
|
0x35
|
|
12
|
page_fault
|
vmlinux |
0xffffffff8f789778
Build id: 3.10.0-1127.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [cc1:107860]
Modules linked in: qingteng(OE) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi snd_seq_midi kvm_intel snd_seq_midi_event kvm irqbypass crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq aesni_intel lrw snd_seq_device gf128mul ppdev glue_helper ablk_helper snd_pcm vmw_balloon cryptd snd_timer pcspkr joydev snd soundcore sg parport_pc
parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif ata_generic pata_acpi vmwgfx crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi drm nfit e1000 serio_raw mptscsih mptbase libata libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 107860 Comm: cc1 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9b74b93c3150 ti: ffff9b749d534000 task.ti: ffff9b749d534000
RIP: 0010:[<ffffffff8f11760a>] [<ffffffff8f11760a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9b749d537b48 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00000000000000fc RCX: ffff9b74bb69fc10
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9b749d537b80 R08: ffff9b74bf9d7000 R09: ffffffff8f385099
R10: ffff9b74bb61f160 R11: fffff72ac03c6000 R12: 0000000000000296
R13: ffff9b749d537af8 R14: 0000000001320122 R15: 000000008b10e7cd
FS: 00007f7e52ef1840(0000) GS:ffff9b74bb600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7e47c9b000 CR3: 0000000015338000 CR4: 00000000003607f0
Call Trace:
[<ffffffff8f1c7850>] ? drain_pages+0xb0/0xb0
[<ffffffff8f117759>] on_each_cpu_mask+0x29/0x70
[<ffffffff8f1c46c5>] drain_all_pages+0xb5/0xc0
[<ffffffff8f77b2f5>] __alloc_pages_slowpath+0x510/0x729
[<ffffffff8f1c9146>] __alloc_pages_nodemask+0x436/0x450
[<ffffffff8f21c2a9>] alloc_pages_vma+0xa9/0x200
[<ffffffff8f1f5f97>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff8f24c663>] ? do_sync_write+0x93/0xe0
[<ffffffff8f78d653>] __do_page_fault+0x213/0x500
[<ffffffff8f78d975>] do_page_fault+0x35/0x90
[<ffffffff8f789778>] page_fault+0x28/0x30
Code: 48 63 35 1e 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 0f d5 8f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 7a c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #18854
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff96116c5a
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff961c74e0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff96116da9
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff961c43c5
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff9677c772
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x510
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff961c8db6
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x436
|
|
7
|
alloc_pages_vma
|
vmlinux |
0xffffffff9621c2a9
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa9
|
|
8
|
handle_mm_fault
|
vmlinux |
0xffffffff961f5f07
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xcb7
|
|
9
|
|
vmlinux |
0xffffffff9624cfc3
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x93
|
|
10
|
__do_page_fault
|
vmlinux |
0xffffffff9678f653
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x213
|
|
11
|
do_page_fault
|
vmlinux |
0xffffffff9678f975
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x35
|
|
12
|
|
vmlinux |
0xffffffff9678bac9
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xaa
|
|
13
|
page_fault
|
vmlinux |
0xffffffff9678b778
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [yum:27878]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink binfmt_misc ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul pcspkr vmw_balloon glue_helper ablk_helper joydev cryptd sg i2c_piix4 vmw_vmci parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c
sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common mptspi crc32c_intel scsi_transport_spi serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 27878 Comm: yum Tainted: G W L ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9607dfac5280 ti: ffff960658ee0000 task.ti: ffff960658ee0000
RIP: 0010:[<ffffffff96116c5a>] [<ffffffff96116c5a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff960658ee3b48 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00000000000000fc RCX: ffff96083fd60600
RDX: 0000000000000005 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff960658ee3b80 R08: ffff960579f41000 R09: ffffffff96385fd9
R10: ffff96083fc5f160 R11: fffff4c300307800 R12: 0000000000000296
R13: ffff960658ee3af8 R14: 0000000001320122 R15: 000000002967b14e
FS: 00007f45abb6e740(0000) GS:ffff96083fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f45556ff000 CR3: 0000000144540000 CR4: 00000000000607e0
Call Trace:
[<ffffffff961c74e0>] ? drain_pages+0xb0/0xb0
[<ffffffff96116da9>] on_each_cpu_mask+0x29/0x70
[<ffffffff961c43c5>] drain_all_pages+0xb5/0xc0
[<ffffffff9677c772>] __alloc_pages_slowpath+0x510/0x729
[<ffffffff961c8db6>] __alloc_pages_nodemask+0x436/0x450
[<ffffffff9621c2a9>] alloc_pages_vma+0xa9/0x200
[<ffffffff961f5f07>] handle_mm_fault+0xcb7/0xfb0
[<ffffffff9624cfc3>] ? do_sync_read+0x93/0xe0
[<ffffffff9678f653>] __do_page_fault+0x213/0x500
[<ffffffff9678f975>] do_page_fault+0x35/0x90
[<ffffffff9678bac9>] ? error_swapgs+0xaa/0xc0
[<ffffffff9678b778>] page_fault+0x28/0x30