Complete report #1421716
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff997ba385
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x15
|
|
2
|
blk_timeout_work
|
vmlinux |
0xffffffff99372fe3
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x133
|
|
3
|
process_one_work
|
vmlinux |
0xffffffff990c32ef
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffff990c4436
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffff990c4310
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x2b0
|
|
6
|
kthread
|
vmlinux |
0xffffffff990cb621
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffff990cb550
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff997c51dd
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x7
|
|
9
|
|
vmlinux |
0xffffffff990cb550
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 3243s! [kworker/1:1H:904]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vboxsf(OE) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc snd_intel8x0 snd_ac97_codec ac97_bus ppdev snd_seq snd_seq_device snd_pcm sg joydev snd_timer pcspkr vboxguest(OE) parport_pc snd parport soundcore i2c_piix4 video ip_tables xfs libcrc32c sd_mod crc_t10dif
crct10dif_generic crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci ata_piix libahci crc32c_intel serio_raw drm libata e1000 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 904 Comm: kworker/1:1H Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
Workqueue: kblockd blk_timeout_work
task: ffff8cc735104200 ti: ffff8cc7d1890000 task.ti: ffff8cc7d1890000
RIP: 0010:[<ffffffff997ba385>] [<ffffffff997ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8cc7d1893dc0 EFLAGS: 00010282
RAX: 0000000000000001 RBX: ffff8cc81fc93940 RCX: 0000000000000000
RDX: 0000000000000001 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8cc7d1893dc0 R08: ffff8cc81fc94d88 R09: cce2b4cb155f0828
R10: cce2b4cb155f0828 R11: ffff8cc7962a7f00 R12: ffff8cc735ff0548
R13: ffff8cc81fc93940 R14: ffff8cc7d1893d30 R15: ffff8cc7267e1ad0
FS: 0000000000000000(0000) GS:ffff8cc81fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00001f14ed3bb388 CR3: 000000008fc02000 CR4: 00000000000206e0
Call Trace:
[<ffffffff99372fe3>] blk_timeout_work+0x133/0x150
[<ffffffff990c32ef>] process_one_work+0x17f/0x440
[<ffffffff990c4436>] worker_thread+0x126/0x3c0
[<ffffffff990c4310>] ? manage_workers.isra.26+0x2b0/0x2b0
[<ffffffff990cb621>] kthread+0xd1/0xe0
[<ffffffff990cb550>] ? insert_kthread_work+0x40/0x40
[<ffffffff997c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff990cb550>] ? insert_kthread_work+0x40/0x40
Code: aa 42 96 ff 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 8a 42 96 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #585277
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb576a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
blk_timeout_work
|
vmlinux |
0xffffffffb534d253
Build id: 3.10.0-957.el7.x86_64
|
0x133
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffb50b9d4f
Build id: 3.10.0-957.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffb50bade6
Build id: 3.10.0-957.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffb50bacc0
Build id: 3.10.0-957.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffb50c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffb50c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb5774c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffffb50c1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 65s! [kworker/1:1H:4872]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon joydev sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm vmw_pvscsi nfit ata_piix libnvdimm libata drm_panel_orientation_quirks dm_mirror dm_region_hash
dm_log dm_mod
CPU: 1 PID: 4872 Comm: kworker/1:1H Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
Workqueue: kblockd blk_timeout_work
task: ffff9ec8fa25d140 ti: ffff9ec8fb694000 task.ti: ffff9ec8fb694000
RIP: 0010:[<ffffffffb576a0f5>] [<ffffffffb576a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9ec8fb697dc0 EFLAGS: 00000282
RAX: ffff9ec875ee8828 RBX: ffff9ec97fd13900 RCX: 0000000000000000
RDX: 0000000000000001 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9ec8fb697dc0 R08: ffff9ec84150d950 R09: 84ddc70e554e8828
R10: 84ddc70e554e8828 R11: 0000000000000000 R12: ffff9ec875ee8548
R13: ffff9ec97fd13900 R14: ffff9ec8fb697d30 R15: ffff9ec845808ed0
FS: 0000000000000000(0000) GS:ffff9ec97fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f995a58de10 CR3: 00000000044d2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb534d253>] blk_timeout_work+0x133/0x150
[<ffffffffb50b9d4f>] process_one_work+0x17f/0x440
[<ffffffffb50bade6>] worker_thread+0x126/0x3c0
[<ffffffffb50bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffb50c1c31>] kthread+0xd1/0xe0
[<ffffffffb50c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffb5774c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb50c1b60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #182993
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa0f883a5
Build id: 3.10.0-1127.el7.x86_64
|
0x15
|
|
2
|
blk_timeout_work
|
vmlinux |
0xffffffffa0b5c9e3
Build id: 3.10.0-1127.el7.x86_64
|
0x133
|
|
3
|
process_one_work
|
vmlinux |
0xffffffffa08be6bf
Build id: 3.10.0-1127.el7.x86_64
|
0x17f
|
|
4
|
worker_thread
|
vmlinux |
0xffffffffa08bf7d6
Build id: 3.10.0-1127.el7.x86_64
|
0x126
|
|
5
|
|
vmlinux |
0xffffffffa08bf6b0
Build id: 3.10.0-1127.el7.x86_64
|
0x2a0
|
|
6
|
kthread
|
vmlinux |
0xffffffffa08c6691
Build id: 3.10.0-1127.el7.x86_64
|
0xd1
|
|
7
|
|
vmlinux |
0xffffffffa08c65c0
Build id: 3.10.0-1127.el7.x86_64
|
0x40
|
|
8
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffa0f92d37
Build id: 3.10.0-1127.el7.x86_64
|
0x21
|
|
9
|
|
vmlinux |
0xffffffffa08c65c0
Build id: 3.10.0-1127.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [kworker/0:1H:404]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev snd_seq_midi snd_seq_midi_event aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel nfit libnvdimm drm ata_piix libata serio_raw mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 404 Comm: kworker/0:1H Kdump: loaded Tainted: G L ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
Workqueue: kblockd blk_timeout_work
task: ffff98cab4433150 ti: ffff98cab45a8000 task.ti: ffff98cab45a8000
RIP: 0010:[<ffffffffa0f883a5>] [<ffffffffa0f883a5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff98cab45abdc0 EFLAGS: 00000282
RAX: ffff98cab5f01b58 RBX: ffff98cbb961acc0 RCX: 0000000000000000
RDX: 0000000000000000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff98cab45abdc0 R08: 0000000000000001 R09: 9cd4c10a48901b58
R10: 9cd4c10a48901b58 R11: ffff98cb13f53f00 R12: ffff98cab45abd68
R13: ffffffffa08af728 R14: ffff98cab45abd30 R15: ffff98cbb0eb3750
FS: 0000000000000000(0000) GS:ffff98cbb9600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000e52c40 CR3: 00000000aabe2000 CR4: 00000000003607f0
Call Trace:
[<ffffffffa0b5c9e3>] blk_timeout_work+0x133/0x150
[<ffffffffa08be6bf>] process_one_work+0x17f/0x440
[<ffffffffa08bf7d6>] worker_thread+0x126/0x3c0
[<ffffffffa08bf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa08c6691>] kthread+0xd1/0xe0
[<ffffffffa08c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa0f92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa08c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48