Complete report #529714
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8163d94b
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x1b
|
|
2
|
ata_sff_error_handler
|
libata |
0xffffffffa0069bb3
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x83
|
|
3
|
ata_bmdma_error_handler
|
libata |
0xffffffffa006a1c8
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x58
|
|
4
|
ata_scsi_port_error_handler
|
libata |
0xffffffffa0067489
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x529
|
|
5
|
ata_scsi_error
|
libata |
0xffffffffa00678e0
Build id: 3.10.0-327.22.2.el7.x86_64
|
0xa0
|
|
6
|
scsi_error_handler
|
vmlinux |
0xffffffff8141d713
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x143
|
|
7
|
|
vmlinux |
0xffffffff8141d5d0
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x260
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.22.2.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff816467d8
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.22.2.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#1 stuck for 38s! [scsi_eh_1:274]
Modules linked in: nfsv3 nfs_acl nfs lockd grace fscache binfmt_misc falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14108(E) vmw_vsock_vmci_transport vsock sunrpc ppdev crc32_pclmul vmw_balloon ghash_clmulni_intel ext4 mbcache jbd2 aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg parport_pc parport vmw_vmci i2c_piix4 shpchp ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi vmwgfx drm_kms_helper ttm drm i2c_core ata_piix libata floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 274 Comm: scsi_eh_1 Tainted: P E ------------ 3.10.0-327.22.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff88042508dc00 ti: ffff880424a14000 task.ti: ffff880424a14000
RIP: 0010:[<ffffffff8163d94b>] [<ffffffff8163d94b>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff880424a17d38 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffff880424a17d20 RCX: 0000000000000006
RDX: 0000000000000000 RSI: 0000000000000202 RDI: 0000000000000202
RBP: ffff880424a17d40 R08: 0000000000000092 R09: 0000000000000597
R10: 6574796220363335 R11: 656c63206f742073 R12: 0000000000000202
R13: ffff880424a17cc0 R14: 0000000000000002 R15: ffffffffa0075ae5
FS: 0000000000000000(0000) GS:ffff88043fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f030683a710 CR3: 000000040ea75000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8804248cc000 ffff880424a17d78 ffffffffa0069bb3 ffff8804248cc000
ffff8804248cc158 0000000000000202 0000000000000246 0000000000000000
ffff880424a17da0 ffffffffa006a1c8 ffff8804248cc000 0000000000000001
Call Trace:
[<ffffffffa0069bb3>] ata_sff_error_handler+0x83/0xf0 [libata]
[<ffffffffa006a1c8>] ata_bmdma_error_handler+0x58/0x110 [libata]
[<ffffffffa0067489>] ata_scsi_port_error_handler+0x529/0x8e0 [libata]
[<ffffffffa00678e0>] ata_scsi_error+0xa0/0xe0 [libata]
[<ffffffff8141d713>] scsi_error_handler+0x143/0x8a0
[<ffffffff8141d5d0>] ? scsi_eh_get_sense+0x260/0x260
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff816467d8>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 08 e8 aa 63 a4 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6
Complete report #221244
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8168d7ab
Build id: 3.10.0-514.el7.x86_64
|
0x1b
|
|
2
|
ata_sff_error_handler
|
libata |
0xffffffffa0097703
Build id: 3.10.0-514.el7.x86_64
|
0x83
|
|
3
|
ata_bmdma_error_handler
|
libata |
0xffffffffa0097d18
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
|
4
|
ata_scsi_port_error_handler
|
libata |
0xffffffffa0094869
Build id: 3.10.0-514.el7.x86_64
|
0x529
|
|
5
|
ata_scsi_error
|
libata |
0xffffffffa0094cc0
Build id: 3.10.0-514.el7.x86_64
|
0xa0
|
|
6
|
scsi_error_handler
|
vmlinux |
0xffffffff81456d03
Build id: 3.10.0-514.el7.x86_64
|
0xf3
|
|
7
|
|
vmlinux |
0xffffffff81456c10
Build id: 3.10.0-514.el7.x86_64
|
0x250
|
|
8
|
kthread
|
vmlinux |
0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810bf8d6
Build id: 3.10.0-514.el7.x86_64
|
0x56
|
|
10
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
|
11
|
ret_from_fork
|
vmlinux |
0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
|
12
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 42s! [scsi_eh_1:333]
Modules linked in: fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock intel_powerclamp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nfit ppdev vmw_balloon sg libnvdimm pcspkr parport_pc parport i2c_piix4 shpchp vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi mptscsih crct10dif_pclmul
crct10dif_common mptbase ata_piix drm crc32c_intel serio_raw libata vmxnet3 i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 333 Comm: scsi_eh_1 Not tainted 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff88079d881f60 ti: ffff88079d894000 task.ti: ffff88079d894000
RIP: 0010:[<ffffffff8168d7ab>] [<ffffffff8168d7ab>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff88079d897d38 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffff88079d897d20 RCX: 0000000000000000
RDX: 0000000000000000 RSI: 0000000000000202 RDI: 0000000000000202
RBP: ffff88079d897d40 R08: 0000000000000096 R09: 000000000000058c
R10: 6574796220363335 R11: 656c63206f742073 R12: 0000000000000202
R13: ffff88079d897cc0 R14: 0000000000000002 R15: ffffffffa00a637e
FS: 0000000000000000(0000) GS:ffff8807c2d80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f844ee7d320 CR3: 00000000019ba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88079d878000 ffff88079d897d78 ffffffffa0097703 ffff88079d878000
ffff88079d878158 0000000000000202 0000000000000246 0000000000000000
ffff88079d897da0 ffffffffa0097d18 ffff88079d878000 0000000000000001
Call Trace:
[<ffffffffa0097703>] ata_sff_error_handler+0x83/0xf0 [libata]
[<ffffffffa0097d18>] ata_bmdma_error_handler+0x58/0x110 [libata]
[<ffffffffa0094869>] ata_scsi_port_error_handler+0x529/0x8e0 [libata]
[<ffffffffa0094cc0>] ata_scsi_error+0xa0/0xe0 [libata]
[<ffffffff81456d03>] scsi_error_handler+0xf3/0x8b0
[<ffffffff81456c10>] ? scsi_eh_get_sense+0x250/0x250
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810bf8d6>] ? finish_task_switch+0x56/0x180
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 08 e8 da 11 a0 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6
Complete report #13819
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8168d7ab
Build id: 3.10.0-514.el7.x86_64
|
0x1b
|
|
2
|
ata_sff_error_handler
|
libata |
0xffffffffa0083703
Build id: 3.10.0-514.el7.x86_64
|
0x83
|
|
3
|
ata_bmdma_error_handler
|
libata |
0xffffffffa0083d18
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
|
4
|
ata_scsi_port_error_handler
|
libata |
0xffffffffa0080869
Build id: 3.10.0-514.el7.x86_64
|
0x529
|
|
5
|
ata_scsi_error
|
libata |
0xffffffffa0080cc0
Build id: 3.10.0-514.el7.x86_64
|
0xa0
|
|
6
|
scsi_error_handler
|
vmlinux |
0xffffffff81456d03
Build id: 3.10.0-514.el7.x86_64
|
0xf3
|
|
7
|
|
vmlinux |
0xffffffff81456c10
Build id: 3.10.0-514.el7.x86_64
|
0x250
|
|
8
|
kthread
|
vmlinux |
0xffffffff810b052f
Build id: 3.10.0-514.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81696518
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810b0460
Build id: 3.10.0-514.el7.x86_64
|
0x140
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [scsi_eh_2:303]
Modules linked in: tcp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun xt_AUDIT nf_log_ipv4 nf_log_common xt_LOG xt_limit xt_set ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set_hash_ip ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock bnep dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event intel_powerclamp iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 aesni_intel lrw snd_rawmidi
snd_ac97_codec ac97_bus snd_seq btusb gf128mul glue_helper btrtl snd_seq_device btbcm btintel ablk_helper snd_pcm bluetooth cryptd snd_timer snd vmw_balloon soundcore ppdev sg rfkill pcspkr nfit parport_pc parport libnvdimm shpchp vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul vmwgfx crct10dif_common crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm serio_raw ata_piix mptspi libata scsi_transport_spi e1000 mptscsih mptbase i2c_core fjes
CPU: 5 PID: 303 Comm: scsi_eh_2 Tainted: G L ------------ 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff880231f23ec0 ti: ffff8800359a4000 task.ti: ffff8800359a4000
RIP: 0010:[<ffffffff8168d7ab>] [<ffffffff8168d7ab>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0000:ffff8800359a7d38 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffff8800359a7d20 RCX: 0000000000000000
RDX: 0000000000000000 RSI: 0000000000000202 RDI: 0000000000000202
RBP: ffff8800359a7d40 R08: 0000000000000096 R09: 0000000000000b2a
R10: 6574796220363335 R11: 656c63206f742073 R12: 0000000000000202
R13: ffff8800359a7cc0 R14: 0000000000000002 R15: ffffffffa009237e
FS: 0000000000000000(0000) GS:ffff880235740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4a67647500 CR3: 00000000ae1ae000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880035978000 ffff8800359a7d78 ffffffffa0083703 ffff880035978000
ffff880035978158 0000000000000202 0000000000000246 0000000000000000
ffff8800359a7da0 ffffffffa0083d18 ffff880035978000 0000000000000001
Call Trace:
[<ffffffffa0083703>] ata_sff_error_handler+0x83/0xf0 [libata]
[<ffffffffa0083d18>] ata_bmdma_error_handler+0x58/0x110 [libata]
[<ffffffffa0080869>] ata_scsi_port_error_handler+0x529/0x8e0 [libata]
[<ffffffffa0080cc0>] ata_scsi_error+0xa0/0xe0 [libata]
[<ffffffff81456d03>] scsi_error_handler+0xf3/0x8b0
[<ffffffff81456c10>] ? scsi_eh_get_sense+0x250/0x250
[<ffffffff810b052f>] kthread+0xcf/0xe0
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81696518>] ret_from_fork+0x58/0x90
[<ffffffff810b0460>] ? kthread_create_on_node+0x140/0x140
Code: 08 e8 da 11 a0 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6
Complete report #270050
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8163c9cb
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x1b
|
|
2
|
ata_bmdma_error_handler
|
libata |
0xffffffffa005b1c0
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x50
|
|
3
|
ata_scsi_port_error_handler
|
libata |
0xffffffffa0058489
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x529
|
|
4
|
ata_scsi_error
|
libata |
0xffffffffa00588e0
Build id: 3.10.0-327.3.1.el7.x86_64
|
0xa0
|
|
5
|
scsi_error_handler
|
vmlinux |
0xffffffff8141ce53
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x143
|
|
6
|
|
vmlinux |
0xffffffff8141cd10
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x260
|
|
7
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.3.1.el7.x86_64
|
0xcf
|
|
8
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x140
|
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff81645818
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x58
|
|
10
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#1 stuck for 22s! [scsi_eh_0:303]
Modules linked in: ppdev ext4 mbcache jbd2 i2c_piix4 sg parport_pc parport pcspkr ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi virtio_blk cirrus syscopyarea sysfillrect sysimgblt drm_kms_helper ttm ata_piix drm serio_raw virtio_pci e1000 floppy i2c_core libata virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 303 Comm: scsi_eh_0 Tainted: G L ------------ 3.10.0-327.3.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2007
task: ffff880405aed080 ti: ffff880036a7c000 task.ti: ffff880036a7c000
RIP: 0010:[<ffffffff8163c9cb>] [<ffffffff8163c9cb>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff880036a7fd70 EFLAGS: 00000202
RAX: 0000000000000050 RBX: ffff880036ab2800 RCX: 000000000000582a
RDX: 00000000000003f6 RSI: 0000000000000202 RDI: 0000000000000202
RBP: ffff880036a7fd78 R08: 0000000000000202 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000004 R12: ffff880036a7fd30
R13: 0000000000000001 R14: ffffffff8109e6ea R15: ffff880036a7fd40
FS: 0000000000000000(0000) GS:ffff88041fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f419e35e384 CR3: 0000000035c59000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880036908000 ffff880036a7fda0 ffffffffa005b1c0 ffff880036908000
0000000000000001 0000000000000000 ffff880036a7fde8 ffffffffa0058489
ffff88003690b548 ffff880036909e70 ffff880036ab0000 ffff880036a7fdf8
Call Trace:
[<ffffffffa005b1c0>] ata_bmdma_error_handler+0x50/0x110 [libata]
[<ffffffffa0058489>] ata_scsi_port_error_handler+0x529/0x8e0 [libata]
[<ffffffffa00588e0>] ata_scsi_error+0xa0/0xe0 [libata]
[<ffffffff8141ce53>] scsi_error_handler+0x143/0x8a0
[<ffffffff8141cd10>] ? scsi_eh_get_sense+0x260/0x260
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645818>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 08 e8 2a 73 a4 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6