Complete report #588169
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
get_online_cpus
|
vmlinux |
0xffffffffae49c52a
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x3a
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffffae5d8cb2
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x22
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffae4baf9f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffae4bc036
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffae4bbf10
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffae4c2e81
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffae4c2db0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffaeb76c1d
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x7
|
|
9
|
|
vmlinux |
0xffffffffae4c2db0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 41s! [kworker/6:0:23493]
Modules linked in: binfmt_misc netfilter_edr(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat tun bridge stp llc ebtable_filter ebtables ip6_tables devlink tcp_diag inet_diag vmw_vsock_vmci_transport vsock ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter ext4 mbcache jbd2 nfit libnvdimm iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev sg pcspkr parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul
crct10dif_common libata mptspi e1000 scsi_transport_spi crc32c_intel mptscsih serio_raw mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 23493 Comm: kworker/6:0 Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
Workqueue: events vmstat_shepherd
task: ffff8f56213c6180 ti: ffff8f54e8f4c000 task.ti: ffff8f54e8f4c000
RIP: 0010:[<ffffffffae49c52a>] [<ffffffffae49c52a>] get_online_cpus+0x3a/0x50
RSP: 0000:ffff8f54e8f4fde8 EFLAGS: 00000202
RAX: ffff8f56213c6180 RBX: ffff8f54e8f4fdb8 RCX: ffff8f54e8f4ffd8
RDX: 0000000000000000 RSI: 4100000000000000 RDI: ffffffffaf0479e8
RBP: ffff8f54e8f4fde8 R08: ffffffffaf098828 R09: 000143d98e898820
R10: 000143d98e898820 R11: 0000000000000001 R12: ffff8f54e8f4fda0
R13: 0000000000000001 R14: 0000000000000001 R15: ffff8f567fd9ab80
FS: 0000000000000000(0000) GS:ffff8f567fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f13f9812bb0 CR3: 0000000035306000 CR4: 00000000003607e0
Call Trace:
[<ffffffffae5d8cb2>] vmstat_shepherd+0x22/0xe0
[<ffffffffae4baf9f>] process_one_work+0x17f/0x440
[<ffffffffae4bc036>] worker_thread+0x126/0x3c0
[<ffffffffae4bbf10>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffae4c2e81>] kthread+0xd1/0xe0
[<ffffffffae4c2db0>] ? insert_kthread_work+0x40/0x40
[<ffffffffaeb76c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffae4c2db0>] ? insert_kthread_work+0x40/0x40
Code: 48 8b 04 25 80 0e 01 00 48 39 05 d2 b4 ba 00 74 1f 48 c7 c7 e8 79 04 af e8 64 bc 6c 00 48 c7 c7 e8 79 04 af 83 05 e6 b4 ba 00 01 <e8> f1 bb 6c 00 5d c3 0f 1f 44 00 00 66 2e 0f 1f 84 00 00 00 00
Complete report #574694
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffffac2bc3ce
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffffac3df99d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffac2bde3f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffac2bef56
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffac2bee30
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffac2c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xd1
|
|
7
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffac994ec9
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x96
|
|
8
|
|
vmlinux |
0xffffffffac2c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x40
|
|
9
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffac994df7
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x21
|
|
10
|
|
vmlinux |
0xffffffffac2c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 26s! [kworker/12:1:63651]
Modules linked in: binfmt_misc xt_set nf_conntrack_netlink ip_set_hash_net ip_set nfnetlink iptable_raw vxlan ip6_udp_tunnel udp_tunnel xt_mark xfrm6_mode_tunnel xfrm4_mode_tunnel esp4 drbg ansi_cprng veth xt_nat xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio tun loop bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 aesni_intel snd_rawmidi lrw snd_ac97_codec gf128mul glue_helper ac97_bus snd_seq ablk_helper
cryptd ppdev snd_seq_device snd_pcm sg vmw_balloon snd_timer snd pcspkr soundcore joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel nfit ata_piix libnvdimm mptspi libata scsi_transport_spi serio_raw e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 12 PID: 63651 Comm: kworker/12:1 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: events vmstat_shepherd
task: ffff89100a07a100 ti: ffff890cda230000 task.ti: ffff890cda230000
RIP: 0010:[<ffffffffac2bc3ce>] [<ffffffffac2bc3ce>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffff890cda233de0 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff890cda233db8 RCX: 0000000000000002
RDX: ffffffffaca13a50 RSI: 0000000000000096 RDI: 0000000000000246
RBP: ffff890cda233de8 R08: 0000000000000001 R09: ffff89129d65fd50
R10: 0000000000000006 R11: 0000000000000005 R12: ffff890cda233d98
R13: 0000000000000000 R14: 0000000000000001 R15: ffff89129d657de0
FS: 0000000000000000(0000) GS:ffff89129d900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f11a841e010 CR3: 0000000756728000 CR4: 00000000000607e0
Call Trace:
[<ffffffffac3df99d>] vmstat_shepherd+0x8d/0xe0
[<ffffffffac2bde3f>] process_one_work+0x17f/0x440
[<ffffffffac2bef56>] worker_thread+0x126/0x3c0
[<ffffffffac2bee30>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffac2c5e11>] kthread+0xd1/0xe0
[<ffffffffac994ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffac2c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffac994df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffac2c5d40>] ? insert_kthread_work+0x40/0x40
Code: 66 90 55 48 89 e5 53 9c 58 66 66 90 66 90 48 89 c3 fa 66 66 90 66 66 90 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <66> 66 90 66 90 5b 44 89 c0 5d c3 0f 1f 80 00 00 00 00 e8 ab fc
Complete report #564414
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffffa96c16be
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffffa97eb8ed
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffa96c319f
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffa96c42e6
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffa96c41c0
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x2b0
|
|
6
|
kthread
|
vmlinux |
0xffffffffa96cb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffa96cb400
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa9dc51dd
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x7
|
|
9
|
|
vmlinux |
0xffffffffa96cb400
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 28s! [kworker/4:1:49]
Modules linked in: vboxsf(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set sep5(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter socperf3(OE) pax(OE) sunrpc snd_intel8x0 joydev snd_ac97_codec ac97_bus vboxguest(OE) snd_seq ppdev snd_seq_device snd_pcm snd_timer snd soundcore sg pcspkr i2c_piix4 parport_pc parport video ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif crct10dif_generic cdrom crct10dif_common
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm ata_piix drm_panel_orientation_quirks e1000 libata crc32c_intel serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 49 Comm: kworker/4:1 Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: events vmstat_shepherd
task: ffff8b468e75a100 ti: ffff8b468d690000 task.ti: ffff8b468d690000
RIP: 0010:[<ffffffffa96c16be>] [<ffffffffa96c16be>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffff8b468d693de0 EFLAGS: 00010246
RAX: 0000000000000001 RBX: 0000000000000086 RCX: ffff8b43f6318000
RDX: 00000000000008fd RSI: 0000000000000096 RDI: 0000000000000246
RBP: ffff8b468d693de8 R08: 0000000000000001 R09: ffff8b468f11fd50
R10: 0000000000000006 R11: 0000000000000005 R12: ffff8b468d693d98
R13: 00000000000008fd R14: 0000000000000001 R15: ffff8b468f117de0
FS: 0000000000000000(0000) GS:ffff8b468f400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f08d7f93f60 CR3: 000000040d36c000 CR4: 00000000000206e0
Call Trace:
[<ffffffffa97eb8ed>] vmstat_shepherd+0x8d/0xe0
[<ffffffffa96c319f>] process_one_work+0x17f/0x440
[<ffffffffa96c42e6>] worker_thread+0x126/0x3c0
[<ffffffffa96c41c0>] ? manage_workers.isra.26+0x2b0/0x2b0
[<ffffffffa96cb4d1>] kthread+0xd1/0xe0
[<ffffffffa96cb400>] ? insert_kthread_work+0x40/0x40
[<ffffffffa9dc51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa96cb400>] ? insert_kthread_work+0x40/0x40
Code: 00 00 55 48 89 e5 53 9c 58 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <0f> 1f 44 00 00 5b 44 89 c0 5d c3 cc cc cc cc 0f 1f 00 e8 9b fc
Complete report #463955
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffffa26bc7ae
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffffa27df52d
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffa26be21f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffa26bf336
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffa26bf210
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffa26c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffa26c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa2d8dd37
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffffa26c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 50s! [ksoftirqd/4:29]
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 50s! [kworker/7:2:32590]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13601(E) vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper vmw_balloon ablk_helper cryptd pcspkr sg joydev parport_pc vmw_vmci parport i2c_piix4 nfsd nfs_acl lockd binfmt_misc grace auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit ttm libnvdimm ata_piix drm libata mptspi scsi_transport_spi crct10dif_pclmul crct10dif_common mptscsih crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy fuse
CPU: 7 PID: 32590 Comm: kworker/7:2 Kdump: loaded Tainted: P EL ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
Workqueue: events vmstat_shepherd
task: ffff88dbeb379070 ti: ffff88dbb4b44000 task.ti: ffff88dbb4b44000
RIP: 0010:[<ffffffffa26bc7ae>] [<ffffffffa26bc7ae>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffff88dbb4b47de0 EFLAGS: 00000246
RAX: 0000000000000001 RBX: 0000000000000082 RCX: ffff88d93a3c4000
RDX: 00000000000008fd RSI: 0000000000000096 RDI: 0000000000000246
RBP: ffff88dbb4b47de8 R08: 0000000000000001 R09: 0101010101010101
R10: 0000000000000006 R11: 0000000000000005 R12: ffff88dbb4b47d98
R13: 0000000000000000 R14: 0000000000000001 R15: ffff88dbffc57da0
FS: 0000000000000000(0000) GS:ffff88dbffdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001548d60 CR3: 0000000095e10000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa27df52d>] vmstat_shepherd+0x8d/0xe0
[<ffffffffa26be21f>] process_one_work+0x17f/0x440
[<ffffffffa26bf336>] worker_thread+0x126/0x3c0
[<ffffffffa26bf210>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa26c61f1>] kthread+0xd1/0xe0
[<ffffffffa26c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2d8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa26c6120>] ? insert_kthread_work+0x40/0x40
Code: 00 00 55 48 89 e5 53 9c 58 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <0f> 1f 44 00 00 5b 44 89 c0 5d c3 0f 1f 80 00 00 00 00 e8 ab fc
Complete report #229888
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffffad2b820e
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffffad3d7c4d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffad2b9ebf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffad2baf56
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffad2bae30
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffad2c1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffad2c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffad975c1d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7
|
|
9
|
|
vmlinux |
0xffffffffad2c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 29s! [kworker/2:1:73144]
Modules linked in: fuse vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device joydev pcspkr snd_pcm snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport binfmt_misc sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx crct10dif_pclmul drm_kms_helper crct10dif_common crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit serio_raw ata_piix mptspi libata libnvdimm scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 73144 Comm: kworker/2:1 Not tainted 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
Workqueue: events vmstat_shepherd
task: ffffa1317ca11040 ti: ffffa130988a0000 task.ti: ffffa130988a0000
RIP: 0010:[<ffffffffad2b820e>] [<ffffffffad2b820e>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffffa130988a3de0 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffa130988a3db8 RCX: 0000000000000001
RDX: ffffffffada12ed0 RSI: 0000000000000096 RDI: 0000000000000246
RBP: ffffa130988a3de8 R08: 0000000000000001 R09: ffffa13213a1f750
R10: 0000000000000006 R11: 0000000000000005 R12: ffffa130988a3d98
R13: 0000000000000000 R14: 0000000000000000 R15: ffffa13213a17d00
FS: 0000000000000000(0000) GS:ffffa13213a80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd628ea4000 CR3: 00000000b832a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffad3d7c4d>] vmstat_shepherd+0x8d/0xe0
[<ffffffffad2b9ebf>] process_one_work+0x17f/0x440
[<ffffffffad2baf56>] worker_thread+0x126/0x3c0
[<ffffffffad2bae30>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffad2c1da1>] kthread+0xd1/0xe0
[<ffffffffad2c1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffad975c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffad2c1cd0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 55 48 89 e5 53 9c 58 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <0f> 1f 44 00 00 5b 44 89 c0 5d c3 0f 1f 80 00 00 00 00 e8 1b fe
Complete report #215377
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffff8acbc3fe
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffff8ade03fd
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffff8acbde6f
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffff8acbef86
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffff8acbee60
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffff8acc5e41
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffff8acc5d70
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8b395df7
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffff8acc5d70
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [kworker/2:0:3375]
Modules linked in: binfmt_misc xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg vmw_vmci parport_pc
parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul crct10dif_common drm crc32c_intel mptspi scsi_transport_spi serio_raw libata mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 3375 Comm: kworker/2:0 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.36.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: events vmstat_shepherd
task: ffff931704e06300 ti: ffff9317037fc000 task.ti: ffff9317037fc000
RIP: 0010:[<ffffffff8acbc3fe>] [<ffffffff8acbc3fe>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffff9317037ffde0 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff9317037ffd80 RCX: ffff9317763c0000
RDX: 0000000000000000 RSI: 0000000000000096 RDI: 0000000000000246
RBP: ffff9317037ffde8 R08: 0000000000000001 R09: ffff931776366300
R10: 0000000000000006 R11: 0000000000000005 R12: ffff9317037ffd98
R13: 0000000000000000 R14: 0000000000000003 R15: ffff931a3fd97de0
FS: 0000000000000000(0000) GS:ffff931a3fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4839da9000 CR3: 0000000119a52000 CR4: 00000000000407e0
Call Trace:
[<ffffffff8ade03fd>] vmstat_shepherd+0x8d/0xe0
[<ffffffff8acbde6f>] process_one_work+0x17f/0x440
[<ffffffff8acbef86>] worker_thread+0x126/0x3c0
[<ffffffff8acbee60>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff8acc5e41>] kthread+0xd1/0xe0
[<ffffffff8acc5d70>] ? insert_kthread_work+0x40/0x40
[<ffffffff8b395df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8acc5d70>] ? insert_kthread_work+0x40/0x40
Code: 66 90 55 48 89 e5 53 9c 58 66 66 90 66 90 48 89 c3 fa 66 66 90 66 66 90 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <66> 66 90 66 90 5b 44 89 c0 5d c3 0f 1f 80 00 00 00 00 e8 ab fc
Complete report #199411
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
queue_delayed_work_on
|
vmlinux |
0xffffffff85cb12be
Build id: 3.10.0-862.el7.x86_64
|
0x2e
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffff85db4abd
Build id: 3.10.0-862.el7.x86_64
|
0x8d
|
|
3
|
process_one_work
|
vmlinux |
0xffffffff85cb2dff
Build id: 3.10.0-862.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffff85cb3ac6
Build id: 3.10.0-862.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffff85cb39a0
Build id: 3.10.0-862.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffff85cbae31
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffff85cbad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff8631f637
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffff85cbad60
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 30s! [xfsaild/dm-0:545]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 30s! [kworker/0:2:29174]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel nfsv4 dns_resolver nfs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 fscache rpcsec_gss_krb5 nf_conntrack_netlink br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_REJECT nf_reject_ipv4 xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype xt_set tun bridge stp llc xt_mark ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_bitmap_port ip_set_hash_ipport ip_set nfnetlink dummy xt_comment iptable_nat nf_nat_ipv4 ebtable_filter ebtables nf_nat ip6table_filter ip6_tables iptable_filter devlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack vmw_vsock_vmci_transport vsock crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd sg parport_pc
parport vmw_balloon joydev pcspkr vmw_vmci i2c_piix4 shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc sch_fq_codel ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci ata_piix mptspi i2c_core libata scsi_transport_spi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw mptscsih vmxnet3 mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 29174 Comm: kworker/0:2 Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
Workqueue: events vmstat_shepherd
task: ffff94981b79bf40 ti: ffff949795e10000 task.ti: ffff949795e10000
RIP: 0010:[<ffffffff85cb12be>] [<ffffffff85cb12be>] queue_delayed_work_on+0x2e/0x50
RSP: 0018:ffff949795e13de0 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffff85ca445d RCX: 0000000000000038
RDX: 00000000000000ff RSI: 0000000000000046 RDI: 0000000000000246
RBP: ffff949795e13de8 R08: 0000000000000001 R09: ffff94997fc5c0a0
R10: 0000000000000006 R11: 0000000000000005 R12: ffff949795e13d98
R13: 0000000000000000 R14: 0000000000000001 R15: ffff94997fc55c00
FS: 0000000000000000(0000) GS:ffff94997fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fb3cc684ad0 CR3: 0000000424a60000 CR4: 00000000000007f0
Call Trace:
[<ffffffff85db4abd>] vmstat_shepherd+0x8d/0xe0
[<ffffffff85cb2dff>] process_one_work+0x17f/0x440
[<ffffffff85cb3ac6>] worker_thread+0x126/0x3c0
[<ffffffff85cb39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff85cbae31>] kthread+0xd1/0xe0
[<ffffffff85cbad60>] ? insert_kthread_work+0x40/0x40
[<ffffffff8631f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff85cbad60>] ? insert_kthread_work+0x40/0x40
Code: 66 90 55 48 89 e5 53 9c 58 66 66 90 66 90 48 89 c3 fa 66 66 90 66 66 90 f0 0f ba 2a 00 19 c0 45 31 c0 85 c0 74 17 48 89 df 57 9d <66> 66 90 66 90 5b 44 89 c0 5d c3 0f 1f 80 00 00 00 00 e8 ab fc
Complete report #13050
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
get_online_cpus
|
vmlinux |
0xffffffff9ea9fa17
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x17
|
|
2
|
vmstat_shepherd
|
vmlinux |
0xffffffff9ebdfb02
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x22
|
|
3
|
process_one_work
|
vmlinux |
0xffffffff9eabe6bf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffff9eabf7d6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffff9eabf6b0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffff9eac6691
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffff9eac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9f192d37
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffff9eac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [kworker/0:0:4004]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm serio_raw drm mptspi scsi_transport_spi mptscsih mptbase ata_piix vmxnet3 libata
drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 4004 Comm: kworker/0:0 Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
Workqueue: events vmstat_shepherd
task: ffff9af10846d230 ti: ffff9af01030c000 task.ti: ffff9af01030c000
RIP: 0010:[<ffffffff9ea9fa17>] [<ffffffff9ea9fa17>] get_online_cpus+0x17/0x50
RSP: 0018:ffff9af01030fde8 EFLAGS: 00010246
RAX: ffff9af10846d230 RBX: ffff9af01030fdb8 RCX: ffff9af10846d230
RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff9af13fc1acc0
RBP: ffff9af01030fde8 R08: ffffffff9f69a308 R09: 0000000000000001
R10: 0000000000000001 R11: ffff9aef09aac900 R12: 000000000001acc0
R13: ffff9af01030ffd8 R14: ffff9af01030ffd8 R15: ffff9af01030ffd8
FS: 0000000000000000(0000) GS:ffff9af13fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc5868a8000 CR3: 00000002365a4000 CR4: 00000000000007f0
Call Trace:
[<ffffffff9ebdfb02>] vmstat_shepherd+0x22/0xe0
[<ffffffff9eabe6bf>] process_one_work+0x17f/0x440
[<ffffffff9eabf7d6>] worker_thread+0x126/0x3c0
[<ffffffff9eabf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff9eac6691>] kthread+0xd1/0xe0
[<ffffffff9eac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9eac65c0>] ? insert_kthread_work+0x40/0x40
Code: 9f 48 89 e5 e8 bb 45 6e 00 5d c3 66 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 e8 02 66 6e 00 65 48 8b 04 25 c0 0e 01 00 <48> 39 05 c2 87 ba 00 74 1f 48 c7 c7 e8 81 64 9f e8 e4 45 6e 00