Complete report #924795
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8e11722a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff8e0ca660
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x30
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff8e1172ed
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffff8e0ca75c
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffff8e107760
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffff8e0a38f5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffff8e24b14a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1ca
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffff8e0a39fc
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xac
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff8e799f92
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 70s! [vmtoolsd:745]
Modules linked in: xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm snd_ens1371 irqbypass snd_rawmidi crc32_pclmul snd_ac97_codec ac97_bus ghash_clmulni_intel snd_seq aesni_intel lrw snd_seq_device gf128mul snd_pcm snd_timer ppdev glue_helper ablk_helper cryptd snd vmw_balloon soundcore joydev pcspkr sg parport_pc
parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops mptspi ttm crct10dif_pclmul crct10dif_common scsi_transport_spi ata_piix crc32c_intel mptscsih drm nfit libata mptbase serio_raw libnvdimm e1000 drm_panel_orientation_quirks fuse
CPU: 5 PID: 745 Comm: vmtoolsd Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a82288c6300 ti: ffff8a7e35fd8000 task.ti: ffff8a7e35fd8000
RIP: 0010:[<ffffffff8e11722a>] [<ffffffff8e11722a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a7e35fdbe18 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00000000000000fc RCX: ffff8a822d6a06a0
RDX: 0000000000000002 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8a7e35fdbe50 R08: ffff8a7f75ccc000 R09: ffffffff8e3869d9
R10: ffff8a822d75f160 R11: ffffd8a590558e00 R12: 0000000000000296
R13: 0000000000000018 R14: ffff8a7e35fdbd88 R15: 0000000000000296
FS: 00007fb7cc587780(0000) GS:ffff8a822d740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056248a7bf090 CR3: 000000042a780000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8e0ca660>] ? hrtimer_cancel+0x30/0x30
[<ffffffff8e1172ed>] on_each_cpu+0x2d/0x60
[<ffffffff8e0ca75c>] clock_was_set+0x1c/0x30
[<ffffffff8e107760>] do_settimeofday+0x120/0x130
[<ffffffff8e0a38f5>] do_sys_settimeofday+0xa5/0x100
[<ffffffff8e24b14a>] ? __check_object_size+0x1ca/0x250
[<ffffffff8e0a39fc>] SyS_settimeofday+0xac/0xf0
[<ffffffff8e799f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 7e 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 d5 8e f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #861791
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
|
vmlinux |
0xffffffff810b4dd0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff810f9a5d
Build id: 3.10.0-693.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffff810b4ecc
Build id: 3.10.0-693.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffff810ea000
Build id: 3.10.0-693.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffff8108f785
Build id: 3.10.0-693.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffff810a326b
Build id: 3.10.0-693.el7.x86_64
|
0x3b
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffff8108f868
Build id: 3.10.0-693.el7.x86_64
|
0x88
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 55s! [vmtoolsd:718]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_ens1371 snd_rawmidi snd_ac97_codec pcspkr ac97_bus snd_seq snd_seq_device joydev
snd_pcm sg nfit snd_timer snd libnvdimm soundcore vmw_vmci i2c_piix4 shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix drm libata e1000 crc32c_intel mptspi scsi_transport_spi serio_raw mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 718 Comm: vmtoolsd Not tainted 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff88003583eeb0 ti: ffff8800761e0000 task.ti: ffff8800761e0000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff8800761e3e48 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff88007b61a780
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8800761e3e80 R08: ffff88007a5ab800 R09: ffffffff813227d9
R10: ffff88007b659c80 R11: ffffea0001e57200 R12: 0000000000000296
R13: 0000000000000018 R14: ffff8800761e3db8 R15: 0000000000000296
FS: 00007f6f63cc5880(0000) GS:ffff88007b640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6f63cdf000 CR3: 00000000743b9000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000001761e7080 ffffffff81b1c820 ffff8800761e3f50 ffffffff810b4dd0
0000000000000000 00007fff971c3d50 00007fff971c3d38 ffff8800761e3ea8
ffffffff810f9a5d ffff8800761e3f50 0000000000000283 00007fff971c3d48
Call Trace:
[<ffffffff810b4dd0>] ? hrtimer_cancel+0x40/0x40
[<ffffffff810f9a5d>] on_each_cpu+0x2d/0x60
[<ffffffff810b4ecc>] clock_was_set+0x1c/0x30
[<ffffffff810ea000>] do_settimeofday+0x120/0x130
[<ffffffff8108f785>] do_sys_settimeofday+0xa5/0x100
[<ffffffff810a326b>] ? do_sys_times+0x3b/0xc0
[<ffffffff8108f868>] SyS_settimeofday+0x88/0xc0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #531117
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa3707e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffa36bf2e0
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffa3707efd
Build id: 3.10.0-862.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffffa36bf3dc
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffffa36f8360
Build id: 3.10.0-862.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffffa3699575
Build id: 3.10.0-862.el7.x86_64
|
0xa5
|
|
7
|
SyS_settimeofday
|
vmlinux |
0xffffffffa3699658
Build id: 3.10.0-862.el7.x86_64
|
0x88
|
|
8
|
system_call_fastpath
|
vmlinux |
0xffffffffa3d1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [vmtoolsd:740]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq ppdev aesni_intel snd_seq_device lrw gf128mul glue_helper ablk_helper cryptd snd_pcm vmw_balloon joydev pcspkr snd_timer sg
snd vmw_vmci soundcore i2c_piix4 nfit libnvdimm parport_pc shpchp parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea ata_piix sysfillrect sysimgblt fb_sys_fops mptspi ttm libata scsi_transport_spi crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw mptscsih mptbase e1000 i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 740 Comm: vmtoolsd Tainted: G L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9417f362eeb0 ti: ffff9417f63d8000 task.ti: ffff9417f63d8000
RIP: 0010:[<ffffffffa3707e3a>] [<ffffffffa3707e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9417f63dbe18 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9417fa61cd20
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9417f63dbe50 R08: ffff9417c65cb400 R09: ffffffffa394a3d9
R10: ffff9417fa65bb00 R11: ffffd1d5c29efa00 R12: 0000000000000296
R13: ffff9417f63dbdc8 R14: ffffffffa382c51d R15: ffff9417f63dbd80
FS: 00007f819cc2c880(0000) GS:ffff9417fa640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f06db104000 CR3: 0000000035b60000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffa36bf2e0>] ? hrtimer_cancel+0x40/0x40
[<ffffffffa3707efd>] on_each_cpu+0x2d/0x60
[<ffffffffa36bf3dc>] clock_was_set+0x1c/0x30
[<ffffffffa36f8360>] do_settimeofday+0x120/0x130
[<ffffffffa3699575>] do_sys_settimeofday+0xa5/0x100
[<ffffffffa3699658>] SyS_settimeofday+0x88/0xc0
[<ffffffffa3d1f7d5>] system_call_fastpath+0x1c/0x21
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 33 a4 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #430728
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff88d16eea
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff88cca450
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x30
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff88d16fad
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffff88cca54c
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffff88d072c0
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffff88ca36a5
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffff88e4a9aa
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x1ca
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffff88ca37ac
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xac
|
|
9
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff89395ec9
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x96
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff89395f92
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x25
|
|
11
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff89395ed5
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [timesync:806]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay vboxsf(OE) sunrpc vboxvideo(OE) snd_intel8x0 ttm ppdev hid_multitouch drm_kms_helper snd_ac97_codec joydev syscopyarea sysfillrect ac97_bus sysimgblt snd_seq fb_sys_fops drm snd_seq_device pcspkr snd_pcm sg drm_panel_orientation_quirks iosf_mbi snd_timer vboxguest(OE) snd i2c_piix4 soundcore parport_pc parport video ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi serio_raw ahci ata_piix libahci e1000 libata crc32c_intel dm_mirror
dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 806 Comm: timesync Kdump: loaded Tainted: G W OE ------------ 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff95e9fafb3180 ti: ffff95e9f43e4000 task.ti: ffff95e9f43e4000
RIP: 0010:[<ffffffff88d16eea>] [<ffffffff88d16eea>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff95e9f43e7e18 EFLAGS: 00010202
RAX: 0000000000000002 RBX: 000000000001b840 RCX: ffff95eaa011fc10
RDX: 0000000000000002 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff95e9f43e7e50 R08: ffff95e9ffd52000 R09: ffffffff88f86549
R10: ffff95eaa001f160 R11: fffff7a683332500 R12: ffffffff88f8649f
R13: ffff95e95b4550d0 R14: ffffffffc02ca0de R15: ffff95e9f43e7d88
FS: 00007fb7e1501700(0000) GS:ffff95eaa0000000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055ec01f4a010 CR3: 0000000176fc2000 CR4: 00000000000206f0
Call Trace:
[<ffffffff88cca450>] ? hrtimer_cancel+0x30/0x30
[<ffffffff88d16fad>] on_each_cpu+0x2d/0x60
[<ffffffff88cca54c>] clock_was_set+0x1c/0x30
[<ffffffff88d072c0>] do_settimeofday+0x120/0x130
[<ffffffff88ca36a5>] do_sys_settimeofday+0xa5/0x100
[<ffffffff88e4a9aa>] ? __check_object_size+0x1ca/0x250
[<ffffffff88ca37ac>] SyS_settimeofday+0xac/0xf0
[<ffffffff89395ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff89395f92>] system_call_fastpath+0x25/0x2a
[<ffffffff89395ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 fe 87 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 15 95 89 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 87 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #364896
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbd9170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffbd8ca510
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x30
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffbd91719d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffffbd8ca60c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffffbd907610
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffffbd8a3765
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffffbda4b13a
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x1ca
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffffbd8a386c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xac
|
|
9
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbdf99ec9
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x96
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffbdf99f92
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x25
|
|
11
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffbdf99ed5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 62s! [timesync:10367]
Modules linked in: vboxsf(OE) veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun binfmt_misc overlay(T) nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter snd_intel8x0 snd_ac97_codec ac97_bus snd_seq ppdev snd_seq_device joydev sg snd_pcm parport_pc snd_timer video parport vboxguest(OE) i2c_piix4 snd soundcore pcspkr sch_fq_codel ip_tables
xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci serio_raw libahci drm ata_piix crc32c_intel e1000 drm_panel_orientation_quirks libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 10367 Comm: timesync Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff97b2a1c24200 ti: ffff97b2ba438000 task.ti: ffff97b2ba438000
RIP: 0010:[<ffffffffbd9170da>] [<ffffffffbd9170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff97b2ba43be18 EFLAGS: 00010202
RAX: 0000000000000002 RBX: 000000000001b840 RCX: ffff97b45fd20400
RDX: 0000000000000002 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff97b2ba43be50 R08: ffff97b3baebd800 R09: ffffffffbdb86c09
R10: ffff97b45fc9f160 R11: ffffdba204e54600 R12: ffffffffbdb86b5f
R13: ffff97b3d1fc5a90 R14: ffffffffc04b30de R15: ffff97b2ba43bd88
FS: 00007f29d596b700(0000) GS:ffff97b45fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f3940703000 CR3: 0000000075342000 CR4: 00000000000006e0
Call Trace:
[<ffffffffbd8ca510>] ? hrtimer_cancel+0x30/0x30
[<ffffffffbd91719d>] on_each_cpu+0x2d/0x60
[<ffffffffbd8ca60c>] clock_was_set+0x1c/0x30
[<ffffffffbd907610>] do_settimeofday+0x120/0x130
[<ffffffffbd8a3765>] do_sys_settimeofday+0xa5/0x100
[<ffffffffbda4b13a>] ? __check_object_size+0x1ca/0x250
[<ffffffffbd8a386c>] SyS_settimeofday+0xac/0xf0
[<ffffffffbdf99ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffbdf99f92>] system_call_fastpath+0x25/0x2a
[<ffffffffbdf99ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 55 be f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #345350
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff89516eaa
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff894ca520
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x30
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff89516f6d
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffff894ca61c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffff89507280
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffff894a3705
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffff8964a8da
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x1ca
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffff894a380c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xac
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff89b93f92
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 21s! [vmtoolsd:1242]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack vmw_vsock_vmci_transport vsock ip_set nfnetlink ebtable_filter ebtables fuse ip6table_filter ip6_tables iptable_filter vmhgfs(OE) sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 crc32_pclmul ppdev snd_ens1371 snd_rawmidi snd_ac97_codec ghash_clmulni_intel ac97_bus snd_seq vmw_balloon snd_seq_device snd_pcm aesni_intel lrw gf128mul glue_helper ablk_helper
cryptd snd_timer joydev sg pcspkr snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm libnvdimm ata_piix crct10dif_pclmul crct10dif_common mptspi crc32c_intel libata scsi_transport_spi mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1242 Comm: vmtoolsd Tainted: G OE ------------ 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff9516fd3c3180 ti: ffff9517b45f8000 task.ti: ffff9517b45f8000
RIP: 0010:[<ffffffff89516eaa>] [<ffffffff89516eaa>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9517b45fbe18 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9517b56206c8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9517b45fbe50 R08: ffff9516fe34c400 R09: ffffffff89786199
R10: ffff9517b579f160 R11: ffffd10105d8e800 R12: 0000000000000296
R13: 0000000000000018 R14: ffff9517b45fbd88 R15: 0000000000000296
FS: 00007fd6078f2740(0000) GS:ffff9517b5780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffeb6460ff0 CR3: 000000022f20e000 CR4: 00000000003607e0
Call Trace:
[<ffffffff894ca520>] ? hrtimer_cancel+0x30/0x30
[<ffffffff89516f6d>] on_each_cpu+0x2d/0x60
[<ffffffff894ca61c>] clock_was_set+0x1c/0x30
[<ffffffff89507280>] do_settimeofday+0x120/0x130
[<ffffffff894a3705>] do_sys_settimeofday+0xa5/0x100
[<ffffffff8964a8da>] ? __check_object_size+0x1ca/0x250
[<ffffffff894a380c>] SyS_settimeofday+0xac/0xf0
[<ffffffff89b93f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 7e 86 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 15 8a f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 4d 86 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #298387
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff810f99a2
Build id: 3.10.0-693.el7.x86_64
|
0x202
|
|
2
|
|
vmlinux |
0xffffffff810b4dd0
Build id: 3.10.0-693.el7.x86_64
|
0x40
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff810f9a5d
Build id: 3.10.0-693.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffff810b4ecc
Build id: 3.10.0-693.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffff810ea000
Build id: 3.10.0-693.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffff8108f785
Build id: 3.10.0-693.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
|
0x39d
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffff8108f868
Build id: 3.10.0-693.el7.x86_64
|
0x88
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [vmtoolsd:832]
Modules linked in: tcp_lp binfmt_misc nls_utf8 isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi sg snd_ac97_codec ac97_bus snd_seq snd_seq_device iosf_mbi crc32_pclmul snd_pcm ghash_clmulni_intel snd_timer ppdev pcspkr aesni_intel snd joydev soundcore lrw gf128mul glue_helper ablk_helper cryptd nfit vmw_balloon libnvdimm vmw_vmci i2c_piix4 shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel drm libata mptspi scsi_transport_spi serio_raw mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 832 Comm: vmtoolsd Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8802321d8000 ti: ffff88017c6dc000 task.ti: ffff88017c6dc000
RIP: 0010:[<ffffffff810f99a2>] [<ffffffff810f99a2>] smp_call_function_many+0x202/0x260
RSP: 0018:ffff88017c6dfe48 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff88023561af20
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff88017c6dfe80 R08: ffff88017cf3b800 R09: ffffffff813227d9
R10: ffff880235659c80 R11: ffffea0001b2e600 R12: 0000000000000296
R13: ffff88017c6dfdf8 R14: 0000000000000000 R15: ffff8802321d8000
FS: 00007fc3ce24b880(0000) GS:ffff880235640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000606240 CR3: 0000000226ea0000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000135d0fb00 ffffffff81b1c820 ffff88017c6dff50 ffffffff810b4dd0
0000000000000000 00007ffdced418f0 00007ffdced418d8 ffff88017c6dfea8
ffffffff810f9a5d ffff88017c6dff50 0000000000000287 00007ffdced418e8
Call Trace:
[<ffffffff810b4dd0>] ? hrtimer_cancel+0x40/0x40
[<ffffffff810f9a5d>] on_each_cpu+0x2d/0x60
[<ffffffff810b4ecc>] clock_was_set+0x1c/0x30
[<ffffffff810ea000>] do_settimeofday+0x120/0x130
[<ffffffff8108f785>] do_sys_settimeofday+0xa5/0x100
[<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
[<ffffffff8108f868>] SyS_settimeofday+0x88/0xc0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 48 63 35 26 45 a2 00 89 c2 39 f0 0f 8d 85 fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 fd b0 81 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 f5 44 a2 00 eb b7 0f b6 4d cc 4c 89
Complete report #14588
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb1f16cba
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb1eca9d0
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x30
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffb1f16d7d
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x2d
|
|
4
|
clock_was_set
|
vmlinux |
0xffffffffb1ecaacc
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1c
|
|
5
|
do_settimeofday
|
vmlinux |
0xffffffffb1f07010
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x120
|
|
6
|
do_sys_settimeofday
|
vmlinux |
0xffffffffb1ea3fa5
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xa5
|
|
7
|
|
vmlinux |
0xffffffffb204764a
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1ca
|
|
8
|
SyS_settimeofday
|
vmlinux |
0xffffffffb1ea40ac
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xac
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffffb258dede
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [vmtoolsd:762]
Modules linked in: tcp_lp nls_utf8 isofs rfcomm fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock bnep sunrpc dm_mirror dm_region_hash dm_log dm_mod btusb iosf_mbi btrtl btbcm btintel crc32_pclmul bluetooth ghash_clmulni_intel ppdev rfkill aesni_intel vmw_balloon lrw gf128mul joydev glue_helper ablk_helper pcspkr cryptd sg snd_seq_midi snd_seq_midi_event parport_pc parport snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd i2c_piix4 soundcore vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod
cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx nfit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops libnvdimm ttm crct10dif_pclmul crct10dif_common drm ata_piix mptspi scsi_transport_spi crc32c_intel libata mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks
CPU: 3 PID: 762 Comm: vmtoolsd Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff886131f062a0 ti: ffff8860b68fc000 task.ti: ffff8860b68fc000
RIP: 0010:[<ffffffffb1f16cba>] [<ffffffffb1f16cba>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8860b68ffe18 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff886139620650
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff8860b68ffe50 R08: ffff886137623c00 R09: ffffffffb21819f9
R10: ffff8861396df160 R11: fffffd3940d82000 R12: 0000000000000296
R13: ffff8860b68ffdc8 R14: ffffffffb205c68d R15: ffffffffb2025e1a
FS: 00007fb9cd00c880(0000) GS:ffff8861396c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fab06b01000 CR3: 00000000b73b2000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb1eca9d0>] ? hrtimer_cancel+0x30/0x30
[<ffffffffb1f16d7d>] on_each_cpu+0x2d/0x60
[<ffffffffb1ecaacc>] clock_was_set+0x1c/0x30
[<ffffffffb1f07010>] do_settimeofday+0x120/0x130
[<ffffffffb1ea3fa5>] do_sys_settimeofday+0xa5/0x100
[<ffffffffb204764a>] ? __check_object_size+0x1ca/0x250
[<ffffffffb1ea40ac>] SyS_settimeofday+0xac/0xf0
[<ffffffffb258dede>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ee 5c c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 e8 b4 b2 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 bd 5c c4 00 eb b7 0f b6 4d cc 4c 89