Info

Permalink
Function
_raw_spin_unlock_irqrestore
First occurrence
2020-10-21
Last occurrence
2025-12-20
State
NEW
Components
kernel
Quality
-60

Statistics

Operating system Count
CentOS 7 362
Architecture Count
x86_64 362
Related packages Count
kernel 362
 0:3.10.0-1160.31.1.el7 132
 0:3.10.0-862.6.3.el7 29
 0:3.10.0-1160.el7 27
 0:3.10.0-1127.el7 22
 0:3.10.0-862.11.6.el7 21
 0:3.10.0-1127.19.1.el7 17
 0:3.10.0-1160.49.1.el7 15
 0:3.10.0-693.el7 12
 0:3.10.0-1160.108.1.el7 8
 0:3.10.0-1160.83.1.el7 8
 0:3.10.0-1160.88.1.el7 8
 0:3.10.0-1160.118.1.el7 7
 0:3.10.0-1160.11.1.el7 6
 0:3.10.0-1160.6.1.el7 5
 0:3.10.0-693.21.1.el7 5
 0:3.10.0-1160.15.2.el7 5
 0:3.10.0-1160.76.1.el7 5
 0:3.10.0-1062.el7 4
 0:3.10.0-1127.13.1.el7 3
 0:3.10.0-1160.114.2.el7 3
 0:3.10.0-1062.9.1.el7 2
 0:3.10.0-1160.105.1.el7 2
 0:3.10.0-1160.119.1.el7 2
 0:3.10.0-1062.18.1.el7 2
 0:3.10.0-1160.71.1.el7 2
 0:3.10.0-1160.92.1.el7 2
 0:3.10.0-957.1.3.el7 2
 0:3.10.0-1160.59.1.el7 1
 0:3.10.0-1160.62.1.el7 1
 0:3.10.0-957.12.1.el7 1
 0:3.10.0-1160.66.1.el7 1
 0:3.10.0-1160.80.1.el7 1
 0:3.10.0-957.12.2.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1182914
Frame # Function Binary Source or offset Line
1
retint_careful
vmlinux 0xffffffff9b9bb550
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xe
2
mptspi_qcmd
mptspi 0xffffffffc04c72f0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x50
3
scsi_dispatch_cmd
vmlinux 0xffffffff9b703030
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xb0
4
scsi_request_fn
vmlinux 0xffffffff9b70cbdc
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x4ac
5
? deadline_add_request
vmlinux 0xffffffff9b58e21e
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x6e
6
__blk_run_queue
vmlinux 0xffffffff9b5687d9
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x39
7
blk_queue_bio
vmlinux 0xffffffff9b56c703
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x3b3
8
generic_make_request
vmlinux 0xffffffff9b56a537
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x157
9
? bvec_alloc
vmlinux 0xffffffff9b49b9a2
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x92
10
submit_bio
vmlinux 0xffffffff9b56a7e0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x70
11
? bio_alloc_bioset
vmlinux 0xffffffff9b49bc43
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x213
12
xfs_add_to_ioend
xfs 0xffffffffc06fd255
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x145
13
xfs_do_writepage
xfs 0xffffffffc06fd8a5
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x1e5
14
write_cache_pages
vmlinux 0xffffffff9b3d5a5c
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x21c
15
? xfs_vm_writepages
xfs 0xffffffffc06fd6c0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xb0
16
xfs_vm_writepages
xfs 0xffffffffc06fd67b
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x6b
17
do_writepages
vmlinux 0xffffffff9b3d6b01
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x21
18
__writeback_single_inode
vmlinux 0xffffffff9b48cd40
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x40
19
writeback_sb_inodes
vmlinux 0xffffffff9b48d90c
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x1cc
20
__writeback_inodes_wb
vmlinux 0xffffffff9b48dc17
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xa7
21
wb_writeback
vmlinux 0xffffffff9b48e0f3
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x263
22
bdi_writeback_workfn
vmlinux 0xffffffff9b48ecff
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x2cf
23
process_one_work
vmlinux 0xffffffff9b2c32ef
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x17f
24
worker_thread
vmlinux 0xffffffff9b2c4436
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x126
25
? manage_workers.isra.26
vmlinux 0xffffffff9b2c4310
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x2b0
26
kthread
vmlinux 0xffffffff9b2cb621
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xd1
27
? insert_kthread_work
vmlinux 0xffffffff9b2cb550
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x40
28
ret_from_fork_nospec_begin
vmlinux 0xffffffff9b9c51dd
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x7
29
? insert_kthread_work
vmlinux 0xffffffff9b2cb550
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 34s! [JRaft-Global-El:24617]
Modules linked in: binfmt_misc veth ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_statistic xt_nat ip6table_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 vxlan ip6_udp_tunnel udp_tunnel xt_physdev xt_NFLOG nfnetlink_log xt_limit xt_mark xt_multiport xt_comment ip_set tcp_lp nls_utf8 isofs nf_conntrack_netlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache overlay(T) br_netfilter bridge stp llc vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus ppdev snd_seq vmw_balloon snd_seq_device snd_pcm snd_timer snd soundcore pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common nfit libata crc32c_intel libnvdimm mptspi e1000 scsi_transport_spi mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 24617 Comm: JRaft-Global-El Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff8c67dd81d280 ti: ffff8c69b8244000 task.ti: ffff8c69b8244000
RIP: 0010:[<ffffffff9b9bb550>]  [<ffffffff9b9bb550>] retint_careful+0xe/0x32
RSP: 0000:ffff8c69b8247f88  EFLAGS: 00000203
RAX: 0140000000000000 RBX: ffffffff9b9b7920 RCX: ffff8c69b8244000
RDX: 0000000000000008 RSI: ffff8c69b8247f78 RDI: 000000000000fe0e
RBP: 00000000c46d32d0 R08: 0000000000000001 R09: ffff8c67dd81d280
R10: 000000000060a2aa R11: 0000000000000001 R12: 0000000000000000
R13: ffff8c68eebf12c0 R14: ffff8c6a5d61acc0 R15: ffff8c68eebf12c0
FS:  00007f0c2d08e700(0000) GS:ffff8c6a5d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6bd0ae5c00 CR3: 0000000687da6000 CR4: 00000000007607f0
PKRU: 55555554
Call Trace:
[<ffffffffc04c72f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9b703030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff9b70cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9b58e21e>] ? deadline_add_request+0x6e/0x90
[<ffffffff9b5687d9>] __blk_run_queue+0x39/0x50
[<ffffffff9b56c703>] blk_queue_bio+0x3b3/0x400
[<ffffffff9b56a537>] generic_make_request+0x157/0x390
[<ffffffff9b49b9a2>] ? bvec_alloc+0x92/0x120
[<ffffffff9b56a7e0>] submit_bio+0x70/0x150
[<ffffffff9b49bc43>] ? bio_alloc_bioset+0x213/0x310
Code: 8b 54 24 30 48 8b 74 24 38 48 8b 7c 24 40 48 83 c4 50 48 cf 0f 1f 40 00 0f 1f 00 48 cf 0f ba e2 03 73 2c fb 0f 1f 80 00 00 00 00 <57> e8 ba d5 ff ff 5f 65 48 8b 0c 25 b8 0e 01 00 48 81 e9 d8 3f 
[<ffffffffc06fd255>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc06fd8a5>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff9b3d5a5c>] write_cache_pages+0x21c/0x480
[<ffffffffc06fd6c0>] ? xfs_vm_writepages+0xb0/0xb0 [xfs]
[<ffffffffc06fd67b>] xfs_vm_writepages+0x6b/0xb0 [xfs]
[<ffffffff9b3d6b01>] do_writepages+0x21/0x50
[<ffffffff9b48cd40>] __writeback_single_inode+0x40/0x260
[<ffffffff9b48d90c>] writeback_sb_inodes+0x1cc/0x430
[<ffffffff9b48dc17>] __writeback_inodes_wb+0xa7/0xe0
[<ffffffff9b48e0f3>] wb_writeback+0x263/0x2f0
[<ffffffff9b48ecff>] bdi_writeback_workfn+0x2cf/0x470
[<ffffffff9b2c32ef>] process_one_work+0x17f/0x440
[<ffffffff9b2c4436>] worker_thread+0x126/0x3c0
[<ffffffff9b2c4310>] ? manage_workers.isra.26+0x2b0/0x2b0
[<ffffffff9b2cb621>] kthread+0xd1/0xe0
[<ffffffff9b2cb550>] ? insert_kthread_work+0x40/0x40
[<ffffffff9b9c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff9b2cb550>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #1104356
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8c36b395
Build id: 3.10.0-957.12.2.el7.x86_64 		0x15
2
ata_scsi_queuecmd
libata 0xffffffffc03ded65
Build id: 3.10.0-957.12.2.el7.x86_64 		0x155
3
? ata_scsi_invalid_field
libata 0xffffffffc03dba40
Build id: 3.10.0-957.12.2.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8c0d1630
Build id: 3.10.0-957.12.2.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8c0dabcc
Build id: 3.10.0-957.12.2.el7.x86_64 		0x4cc
6
? deadline_add_request
vmlinux 0xffffffff8bf67d9e
Build id: 3.10.0-957.12.2.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff8bf43789
Build id: 3.10.0-957.12.2.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff8bf47483
Build id: 3.10.0-957.12.2.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff8bf45417
Build id: 3.10.0-957.12.2.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff8be7e882
Build id: 3.10.0-957.12.2.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff8bf456c0
Build id: 3.10.0-957.12.2.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff8be7eb23
Build id: 3.10.0-957.12.2.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc05a7165
Build id: 3.10.0-957.12.2.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc05a7787
Build id: 3.10.0-957.12.2.el7.x86_64 		0x1d7
15
write_cache_pages
vmlinux 0xffffffff8bdc253c
Build id: 3.10.0-957.12.2.el7.x86_64 		0x21c
16
? xfs_aops_discard_page
xfs 0xffffffffc05a75b0
Build id: 3.10.0-957.12.2.el7.x86_64 		0x150
17
xfs_vm_writepages
xfs 0xffffffffc05a73ab
Build id: 3.10.0-957.12.2.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff8bdc3651
Build id: 3.10.0-957.12.2.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff8be70070
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
20
writeback_sb_inodes
vmlinux 0xffffffff8be70b04
Build id: 3.10.0-957.12.2.el7.x86_64 		0x1c4
21
__writeback_inodes_wb
vmlinux 0xffffffff8be70e6f
Build id: 3.10.0-957.12.2.el7.x86_64 		0x9f
22
wb_writeback
vmlinux 0xffffffff8be716a3
Build id: 3.10.0-957.12.2.el7.x86_64 		0x263
23
bdi_writeback_workfn
vmlinux 0xffffffff8be7202b
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2cb
24
process_one_work
vmlinux 0xffffffff8bcb9e3f
Build id: 3.10.0-957.12.2.el7.x86_64 		0x17f
25
worker_thread
vmlinux 0xffffffff8bcbaed6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x126
26
? manage_workers.isra.25
vmlinux 0xffffffff8bcbadb0
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2a0
27
kthread
vmlinux 0xffffffff8bcc1d21
Build id: 3.10.0-957.12.2.el7.x86_64 		0xd1
28
? insert_kthread_work
vmlinux 0xffffffff8bcc1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffff8c375c37
Build id: 3.10.0-957.12.2.el7.x86_64 		0x21
30
? insert_kthread_work
vmlinux 0xffffffff8bcc1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [kworker/u4:1:10826]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr sg joydev vmw_balloon nfit libnvdimm parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c sd_mod crc_t10dif
crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ahci e1000 libahci ata_generic pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 10826 Comm: kworker/u4:1 Tainted: G           OE  ------------   3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
Workqueue: writeback bdi_writeback_workfn (flush-253:2)
task: ffff92d40a74d140 ti: ffff92d3e3e94000 task.ti: ffff92d3e3e94000
RIP: 0010:[<ffffffff8c36b395>]  [<ffffffff8c36b395>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff92d3e3e97780  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff92d3a2c90000 RCX: 0000000000000000
RDX: ffff92d3a2b8c818 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff92d3e3e97780 R08: 0000000000000000 R09: 000000018f31afff
R10: ffff92d4327c20d8 R11: ffff92d4360ee380 R12: ffff92d3e3e97728
R13: ffffffff8bfa18a3 R14: ffff92d3e3e97728 R15: ffff92d4327c2000
FS:  0000000000000000(0000) GS:ffff92d43fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f927f6fa000 CR3: 00000001a2a10000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc03ded65>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc03dba40>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffff8c0d1630>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c0dabcc>] scsi_request_fn+0x4cc/0x680
[<ffffffff8bf67d9e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8bf43789>] __blk_run_queue+0x39/0x50
[<ffffffff8bf47483>] blk_queue_bio+0x3b3/0x400
[<ffffffff8bf45417>] generic_make_request+0x147/0x380
[<ffffffff8be7e882>] ? bvec_alloc+0x92/0x120
[<ffffffff8bf456c0>] submit_bio+0x70/0x150
[<ffffffff8be7eb23>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc05a7165>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc05a7787>] xfs_do_writepage+0x1d7/0x510 [xfs]
[<ffffffff8bdc253c>] write_cache_pages+0x21c/0x470
[<ffffffffc05a75b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc05a73ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8bdc3651>] do_writepages+0x21/0x50
[<ffffffff8be70070>] __writeback_single_inode+0x40/0x260
[<ffffffff8be70b04>] writeback_sb_inodes+0x1c4/0x490
[<ffffffff8be70e6f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8be716a3>] wb_writeback+0x263/0x2f0
[<ffffffff8be7202b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff8bcb9e3f>] process_one_work+0x17f/0x440
[<ffffffff8bcbaed6>] worker_thread+0x126/0x3c0
[<ffffffff8bcbadb0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffff8bcc1d21>] kthread+0xd1/0xe0
[<ffffffff8bcc1c50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8c375c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8bcc1c50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #1032865
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffa71ba385
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x15
2
e1000_update_stats
e1000 0xffffffffc01486d4
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x944
3
e1000_watchdog
e1000 0xffffffffc014877d
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x6d
4
process_one_work
vmlinux 0xffffffffa6ac319f
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x17f
5
worker_thread
vmlinux 0xffffffffa6ac42e6
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x126
6
? manage_workers.isra.26
vmlinux 0xffffffffa6ac41c0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2b0
7
kthread
vmlinux 0xffffffffa6acb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd1
8
? insert_kthread_work
vmlinux 0xffffffffa6acb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
9
ret_from_fork_nospec_begin
vmlinux 0xffffffffa71c51dd
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7
10
? insert_kthread_work
vmlinux 0xffffffffa6acb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
11
__blk_run_queue
vmlinux 0xffffffffa6d68119
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x39
12
blk_queue_bio
vmlinux 0xffffffffa6d6c043
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x3b3
13
generic_make_request
vmlinux 0xffffffffa6d69e77
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x157
14
? bvec_alloc
vmlinux 0xffffffffa6c9b3b2
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x92
15
submit_bio
vmlinux 0xffffffffa6d6a120
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x70
16
? bio_alloc_bioset
vmlinux 0xffffffffa6c9b653
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x213
17
xfs_add_to_ioend
xfs 0xffffffffc0381255
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x145
18
xfs_do_writepage
xfs 0xffffffffc03818a5
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x1e5
19
write_cache_pages
vmlinux 0xffffffffa6bd583c
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x21c
20
? xfs_vm_writepages
xfs 0xffffffffc03816c0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xb0
21
xfs_vm_writepages
xfs 0xffffffffc038167b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x6b
22
do_writepages
vmlinux 0xffffffffa6bd68e1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x21
23
__writeback_single_inode
vmlinux 0xffffffffa6c8c750
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
24
? wake_up_bit
vmlinux 0xffffffffa6acc5f5
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x25
25
writeback_sb_inodes
vmlinux 0xffffffffa6c8d31c
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x1cc
26
__writeback_inodes_wb
vmlinux 0xffffffffa6c8d627
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xa7
27
wb_writeback
vmlinux 0xffffffffa6c8db03
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x263
28
bdi_writeback_workfn
vmlinux 0xffffffffa6c8e60c
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x1cc
29
process_one_work
vmlinux 0xffffffffa6ac319f
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x17f
30
worker_thread
vmlinux 0xffffffffa6ac42e6
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x126
31
? manage_workers.isra.26
vmlinux 0xffffffffa6ac41c0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2b0
32
kthread
vmlinux 0xffffffffa6acb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd1
33
? insert_kthread_work
vmlinux 0xffffffffa6acb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
34
ret_from_fork_nospec_begin
vmlinux 0xffffffffa71c51dd
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7
35
? insert_kthread_work
vmlinux 0xffffffffa6acb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 33s! [kworker/0:0:21299]
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel snd_ens1371 lrw gf128mul snd_rawmidi glue_helper snd_ac97_codec ablk_helper cryptd ac97_bus snd_seq snd_seq_device snd_pcm snd_timer ppdev snd soundcore i2c_piix4 joydev pcspkr vmw_balloon sg vmw_vmci parport_pc parport binfmt_misc fuse ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi nfit e1000 scsi_transport_spi mptscsih serio_raw libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 21299 Comm: kworker/0:0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
Workqueue: events e1000_watchdog [e1000]
task: ffffa07f27901080 ti: ffffa07c1b08c000 task.ti: ffffa07c1b08c000
RIP: 0010:[<ffffffffa71ba385>]  [<ffffffffa71ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa07c1b08fd90  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000086 RCX: 0004fa2131c5d5c1
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffffa07c1b08fd90 R08: ffffa07f27c470b8 R09: 7e03016d650470b0
R10: 7e03016d650470b0 R11: fffff7059492f680 R12: ffffffffa6ab3a7c
R13: ffffa07c1b08fd48 R14: 0000000000000086 R15: ffffa07e7aa24980
FS:  0000000000000000(0000) GS:ffffa07f29600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f97b5bef000 CR3: 000000009b300000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc01486d4>] e1000_update_stats+0x944/0x980 [e1000]
[<ffffffffc014877d>] e1000_watchdog+0x6d/0x550 [e1000]
[<ffffffffa6ac319f>] process_one_work+0x17f/0x440
[<ffffffffa6ac42e6>] worker_thread+0x126/0x3c0
[<ffffffffa6ac41c0>] ? manage_workers.isra.26+0x2b0/0x2b0
[<ffffffffa6acb4d1>] kthread+0xd1/0xe0
[<ffffffffa6acb400>] ? insert_kthread_work+0x40/0x40
[<ffffffffa71c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa6acb400>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48 
[<ffffffffa6d68119>] __blk_run_queue+0x39/0x50
[<ffffffffa6d6c043>] blk_queue_bio+0x3b3/0x400
[<ffffffffa6d69e77>] generic_make_request+0x157/0x390
[<ffffffffa6c9b3b2>] ? bvec_alloc+0x92/0x120
[<ffffffffa6d6a120>] submit_bio+0x70/0x150
[<ffffffffa6c9b653>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc0381255>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc03818a5>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffffa6bd583c>] write_cache_pages+0x21c/0x480
[<ffffffffc03816c0>] ? xfs_vm_writepages+0xb0/0xb0 [xfs]
[<ffffffffc038167b>] xfs_vm_writepages+0x6b/0xb0 [xfs]
[<ffffffffa6bd68e1>] do_writepages+0x21/0x50
[<ffffffffa6c8c750>] __writeback_single_inode+0x40/0x260
[<ffffffffa6acc5f5>] ? wake_up_bit+0x25/0x30
[<ffffffffa6c8d31c>] writeback_sb_inodes+0x1cc/0x430
[<ffffffffa6c8d627>] __writeback_inodes_wb+0xa7/0xe0
[<ffffffffa6c8db03>] wb_writeback+0x263/0x2f0
[<ffffffffa6c8e60c>] bdi_writeback_workfn+0x1cc/0x470
[<ffffffffa6ac319f>] process_one_work+0x17f/0x440
[<ffffffffa6ac42e6>] worker_thread+0x126/0x3c0
[<ffffffffa6ac41c0>] ? manage_workers.isra.26+0x2b0/0x2b0
[<ffffffffa6acb4d1>] kthread+0xd1/0xe0
[<ffffffffa6acb400>] ? insert_kthread_work+0x40/0x40
[<ffffffffa71c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa6acb400>] ? insert_kthread_work+0x40/0x40
Code: 61 4b 56 59 c7 45 a4 0a 00 00 00 89 4d d0 48 89 45 c0 48 89 45 c8 0f 1f 00 65 c7 05 1d 0e 57 59 00 00 00 00 fb 66 0f 1f 44 00 00 <49> c7 c4 c0 70 60 a7 eb 0e 0f 1f 44 00 00 49 83 c4 08 41 d1 ef
Complete report #444857
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8c98a3f5
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0347632
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8c6ec580
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8c6e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8c6ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8c57846e
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff8c553869
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff8c557673
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff8c5554f7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff8c48c1e2
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff8c5557a0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff8c48c483
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc0574e45
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc0575475
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffff8c3c9efc
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0575290
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc057525b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff8c3caf51
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff8c47d8d0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffff8c2c6ec5
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffff8c47e454
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffff8c47e75f
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffff8c47ec33
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x263
24
bdi_writeback_workfn
vmlinux 0xffffffff8c47f82b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffff8c2bde3f
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffff8c2bef56
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x126
27
? manage_workers.isra.26
vmlinux 0xffffffff8c2bee30
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffff8c2c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffff8c2c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffff8c994ddd
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x7
31
? insert_kthread_work
vmlinux 0xffffffff8c2c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [kworker/u16:2:25937]
Modules linked in: vmsecmod(OE) tcp_lp tcp_diag udp_diag inet_diag binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4
ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix sd_mod crc_t10dif crct10dif_generic libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: vmsecmod]
CPU: 2 PID: 25937 Comm: kworker/u16:2 Tainted: G        W  OEL ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
Workqueue: writeback bdi_writeback_workfn (flush-253:2)
task: ffff96691a960000 ti: ffff9666bd018000 task.ti: ffff9666bd018000
RIP: 0010:[<ffffffff8c98a3f5>]  [<ffffffff8c98a3f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9666bd01b750  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffff8c5b2803 RCX: ffffffff8ce34f40
RDX: ffffaa4bc1ba0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9666bd01b750 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9669ad976000 R11: ffff9669ae3fe000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9668dab31080 R15: ffffffff8c6eb239
FS:  0000000000000000(0000) GS:ffff9669bfc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000022d4228 CR3: 000000013ee10000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc0347632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8c6ec580>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8c6e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c6ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8c57846e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8c553869>] __blk_run_queue+0x39/0x50
[<ffffffff8c557673>] blk_queue_bio+0x3b3/0x400
[<ffffffff8c5554f7>] generic_make_request+0x147/0x380
[<ffffffff8c48c1e2>] ? bvec_alloc+0x92/0x120
[<ffffffff8c5557a0>] submit_bio+0x70/0x150
[<ffffffff8c48c483>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc0574e45>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc0575475>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff8c3c9efc>] write_cache_pages+0x21c/0x470
[<ffffffffc0575290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc057525b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8c3caf51>] do_writepages+0x21/0x50
[<ffffffff8c47d8d0>] __writeback_single_inode+0x40/0x260
[<ffffffff8c2c6ec5>] ? wake_up_bit+0x25/0x30
[<ffffffff8c47e454>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff8c47e75f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8c47ec33>] wb_writeback+0x263/0x2f0
[<ffffffff8c47f82b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff8c2bde3f>] process_one_work+0x17f/0x440
[<ffffffff8c2bef56>] worker_thread+0x126/0x3c0
[<ffffffff8c2bee30>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff8c2c5e11>] kthread+0xd1/0xe0
[<ffffffff8c2c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffff8c994ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff8c2c5d40>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #311928
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff885893e5
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x15
2
ata_scsi_queuecmd
libata 0xffffffffc0341d45
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x155
3
? ata_scsi_invalid_field
libata 0xffffffffc033ea20
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff882e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff882ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8817818e
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff881536d9
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff881573e3
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff88155367
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff8808c362
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff88155610
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff8808c603
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc0519e45
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc051a475
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffff87fc9f5c
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc051a290
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc051a25b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff87fcafb1
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff8807d8d0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffff87ec6f25
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffff8807e454
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffff8807e75f
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffff8807ec33
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x263
24
? bdi_dirty_limit
vmlinux 0xffffffff87fca720
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
25
bdi_writeback_workfn
vmlinux 0xffffffff8807f72c
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1cc
26
process_one_work
vmlinux 0xffffffff87ebde9f
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x17f
27
worker_thread
vmlinux 0xffffffff87ebefb6
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x126
28
? manage_workers.isra.26
vmlinux 0xffffffff87ebee90
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2a0
29
kthread
vmlinux 0xffffffff87ec5e71
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xd1
30
? insert_kthread_work
vmlinux 0xffffffff87ec5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
31
ret_from_fork_nospec_begin
vmlinux 0xffffffff88593df7
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
32
? insert_kthread_work
vmlinux 0xffffffff87ec5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 54s! [kworker/u256:1:25897]
sd 1:0:0:0: [sda] tag#0 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_TIMEOUT cmd_age=90s
sd 1:0:0:0: [sda] tag#0 CDB: Write(10) 2a 00 0b 08 3c a0 00 00 08 00
blk_update_request: I/O error, dev sda, sector 185089184
Modules linked in: nfnetlink_queue nfnetlink_log nfnetlink bluetooth rfkill iptable_filter vmw_vsock_vmci_transport vsock ppdev vmw_balloon pcspkr joydev sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm nfit serio_raw ata_piix libnvdimm libata vmw_pvscsi vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 25897 Comm: kworker/u256:1 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/28/2020
Workqueue: writeback bdi_writeback_workfn (flush-253:2)
task: ffff9ef1b7fb8000 ti: ffff9ef411248000 task.ti: ffff9ef411248000
RIP: 0010:[<ffffffff885893e5>]  [<ffffffff885893e5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9ef41124b780  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffffc034da3a RCX: 000d85dd8f9da17c
RDX: 0000000000001068 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9ef41124b780 R08: 0000000000009000 R09: a7496a86da9d25a0
R10: 969075a9d25a81e6 R11: 1f6a7496871c6a74 R12: ffff9ef075ca0158
R13: ffffffffc034ab5b R14: ffff9ef075ca0158 R15: ffffffffc034bf16
FS:  0000000000000000(0000) GS:ffff9ef46d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe7d2a6b000 CR3: 00000000ba0c6000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc0341d45>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc033ea20>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffff882e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff882ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8817818e>] ? deadline_add_request+0x6e/0x90
[<ffffffff881536d9>] __blk_run_queue+0x39/0x50
[<ffffffff881573e3>] blk_queue_bio+0x3b3/0x400
[<ffffffff88155367>] generic_make_request+0x147/0x380
[<ffffffff8808c362>] ? bvec_alloc+0x92/0x120
[<ffffffff88155610>] submit_bio+0x70/0x150
[<ffffffff8808c603>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc0519e45>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc051a475>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff87fc9f5c>] write_cache_pages+0x21c/0x470
[<ffffffffc051a290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc051a25b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff87fcafb1>] do_writepages+0x21/0x50
[<ffffffff8807d8d0>] __writeback_single_inode+0x40/0x260
[<ffffffff87ec6f25>] ? wake_up_bit+0x25/0x30
[<ffffffff8807e454>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff8807e75f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8807ec33>] wb_writeback+0x263/0x2f0
[<ffffffff87fca720>] ? bdi_dirty_limit+0x40/0xe0
[<ffffffff8807f72c>] bdi_writeback_workfn+0x1cc/0x460
[<ffffffff87ebde9f>] process_one_work+0x17f/0x440
[<ffffffff87ebefb6>] worker_thread+0x126/0x3c0
[<ffffffff87ebee90>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff87ec5e71>] kthread+0xd1/0xe0
[<ffffffff87ec5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff88593df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff87ec5da0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #244422
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00bf3be
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00da71e
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01522e0
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa26e4120
Build id: 3.10.0-1160.31.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffa26ed84c
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffffa257869e
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffffa25539f9
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffffa25577a3
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffffa2555687
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffffa248c2e2
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffffa2555930
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffffa248c583
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc0337e45
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc0338475
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffffa23ca45c
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0338290
Build id: 3.10.0-1160.31.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc033825b
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffffa23cb4b1
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffffa247d9d0
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffffa22c6ee5
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffffa247e554
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffffa247e85f
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffffa247ed33
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x263
24
bdi_writeback_workfn
vmlinux 0xffffffffa247f92b
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffffa22bde5f
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffffa22bef76
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x126
27
? manage_workers.isra.26
vmlinux 0xffffffffa22bee50
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffffa22c5e31
Build id: 3.10.0-1160.31.1.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffffa22c5d60
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2995df7
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x21
31
? insert_kthread_work
vmlinux 0xffffffffa22c5d60
Build id: 3.10.0-1160.31.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [kworker/u128:0:30388]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_12502(E) falcon_lsm_pinned_12304(E) falcon_lsm_pinned_12303(E) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter falcon_lsm_pinned_12207(E) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ext4 mbcache jbd2 sb_edac iosf_mbi
crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel nfit drm serio_raw libnvdimm ata_piix mptspi scsi_transport_spi libata vmxnet3 mptscsih mptbase drm_panel_orientation_quirks fuse [last unloaded: falcon_kal]
CPU: 2 PID: 30388 Comm: kworker/u128:0 Tainted: P            E  ------------   3.10.0-1160.31.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
Workqueue: writeback bdi_writeback_workfn (flush-8:0)
task: ffff9db1f3156300 ti: ffff9db0c0d08000 task.ti: ffff9db0c0d08000
RIP: 0010:[<ffffffffc00bf3be>]  [<ffffffffc00bf3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9db0c0d0b6e8  EFLAGS: 00000246
RAX: ffffadb940e00000 RBX: ffff9db0723c27f0 RCX: 000000000000001d
RDX: ffff9db1f177e600 RSI: ffff9db1f752b000 RDI: 000000000000000e
RBP: ffff9db0c0d0b6f8 R08: 0000000000000003 R09: 0000000196f24fff
R10: ffff9db1f0d99800 R11: ffff9daff7419f80 R12: ffff9dafe8391f80
R13: ffff9db0c0d0b65c R14: ffffffffa255c3bb R15: ffff9db0c0d0b6a0
FS:  0000000000000000(0000) GS:ffff9db1f6680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f635804b000 CR3: 00000000b1be6000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc00da71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01522e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa26e4120>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa26ed84c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa257869e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa25539f9>] __blk_run_queue+0x39/0x50
[<ffffffffa25577a3>] blk_queue_bio+0x3b3/0x400
[<ffffffffa2555687>] generic_make_request+0x147/0x380
[<ffffffffa248c2e2>] ? bvec_alloc+0x92/0x120
[<ffffffffa2555930>] submit_bio+0x70/0x150
[<ffffffffa248c583>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc0337e45>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc0338475>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffffa23ca45c>] write_cache_pages+0x21c/0x470
[<ffffffffc0338290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc033825b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa23cb4b1>] do_writepages+0x21/0x50
[<ffffffffa247d9d0>] __writeback_single_inode+0x40/0x260
[<ffffffffa22c6ee5>] ? wake_up_bit+0x25/0x30
[<ffffffffa247e554>] writeback_sb_inodes+0x1c4/0x430
[<ffffffffa247e85f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffffa247ed33>] wb_writeback+0x263/0x2f0
[<ffffffffa247f92b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffffa22bde5f>] process_one_work+0x17f/0x440
[<ffffffffa22bef76>] worker_thread+0x126/0x3c0
[<ffffffffa22bee50>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa22c5e31>] kthread+0xd1/0xe0
[<ffffffffa22c5d60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2995df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa22c5d60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 d3 0c
Complete report #192183
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03b73be
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc04d571e
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04ae2e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8c8e2990
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8c8ec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8c7771ae
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff8c7527a9
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff8c7564b3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff8c754437
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff8c68b6a2
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff8c7546e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff8c68b943
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc05f3ec5
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc05f44f5
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffff8c5ca20c
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc05f4310
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc05f42db
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff8c5cb261
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff8c67cc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
20
writeback_sb_inodes
vmlinux 0xffffffff8c67d7c4
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1c4
21
__writeback_inodes_wb
vmlinux 0xffffffff8c67dacf
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x9f
22
wb_writeback
vmlinux 0xffffffff8c67dfa3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x263
23
bdi_writeback_workfn
vmlinux 0xffffffff8c67eb9b
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2cb
24
process_one_work
vmlinux 0xffffffff8c4be6bf
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x17f
25
worker_thread
vmlinux 0xffffffff8c4bf7d6
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x126
26
? manage_workers.isra.26
vmlinux 0xffffffff8c4bf6b0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2a0
27
kthread
vmlinux 0xffffffff8c4c6691
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xd1 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 25s! [kworker/u8:0:7343]
Modules linked in: binfmt_misc tcp_diag inet_diag vfat fat isofs ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter xt_time nf_log_ipv4 nf_log_common nf_nat_ftp xt_conntrack nf_conntrack_ftp xt_LOG xt_limit kcare(OE) ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 xt_REDIRECT nf_nat_redirect iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_owner xt_multiport ext4 mbcache jbd2 loop sunrpc ppdev vmw_balloon pcspkr coretemp joydev sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper crc32c_intel syscopyarea sysfillrect
sysimgblt serio_raw fb_sys_fops ttm mptspi scsi_transport_spi ata_piix mptscsih drm libata vmxnet3 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 7343 Comm: kworker/u8:0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff9c05bc230000 ti: ffff9c03dda90000 task.ti: ffff9c03dda90000
RIP: 0010:[<ffffffffc03b73be>]  [<ffffffffc03b73be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9c03dda936e8  EFLAGS: 00000246
RAX: ffffa8fac1bc0000 RBX: ffff9c03939a07f0 RCX: 0000000000000008
RDX: ffff9c07ab4a1e00 RSI: ffff9c07abeb4000 RDI: 000000000000000e
RBP: ffff9c03dda936f8 R08: 0000000000000003 R09: 00000002dd596fff
R10: ffff9c07aac4c800 R11: ffff9c038c8b6e00 R12: ffff9c0482dd1b00
R13: ffff9c03dda9365c R14: ffffffff8c75b0cb R15: ffff9c03dda936a0
FS:  0000000000000000(0000) GS:ffff9c07bfc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000004d74c0 CR3: 000000000aef8000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc04d571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04ae2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8c8e2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c8ec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8c7771ae>] ? deadline_add_request+0x6e/0x90
[<ffffffff8c7527a9>] __blk_run_queue+0x39/0x50
[<ffffffff8c7564b3>] blk_queue_bio+0x3b3/0x400
[<ffffffff8c754437>] generic_make_request+0x147/0x380
[<ffffffff8c68b6a2>] ? bvec_alloc+0x92/0x120
[<ffffffff8c7546e0>] submit_bio+0x70/0x150
[<ffffffff8c68b943>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc05f3ec5>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc05f44f5>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff8c5ca20c>] write_cache_pages+0x21c/0x470
[<ffffffffc05f4310>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc05f42db>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8c5cb261>] do_writepages+0x21/0x50
[<ffffffff8c67cc40>] __writeback_single_inode+0x40/0x260
[<ffffffff8c67d7c4>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff8c67dacf>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8c67dfa3>] wb_writeback+0x263/0x2f0
[<ffffffff8c67eb9b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff8c4be6bf>] process_one_work+0x17f/0x440
[<ffffffff8c4bf7d6>] worker_thread+0x126/0x3c0
[<ffffffff8c4bf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff8c4c6691>] kthread+0xd1/0xe0
Complete report #160201
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffa498a3f5
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x15
2
ata_scsi_queuecmd
libata 0xffffffffc0422d45
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x155
3
? ata_scsi_invalid_field
libata 0xffffffffc041fa20
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa46e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffa46ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffffa457846e
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffffa4553869
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffffa4557673
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffffa45554f7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffffa448c1e2
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffffa45557a0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffffa448c483
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc060fe45
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc0610475
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffffa43c9efc
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0610290
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc061025b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffffa43caf51
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffffa447d8d0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffffa42c6ec5
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffffa447e454
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffffa447e75f
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffffa447ec33
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x263
24
? get_nr_inodes
vmlinux 0xffffffffa446a85c
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4c
25
bdi_writeback_workfn
vmlinux 0xffffffffa447f82b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2cb
26
process_one_work
vmlinux 0xffffffffa42bde3f
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x17f
27
worker_thread
vmlinux 0xffffffffa42bef56
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x126
28
? manage_workers.isra.26
vmlinux 0xffffffffa42bee30
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2a0
29
kthread
vmlinux 0xffffffffa42c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xd1
30
? insert_kthread_work
vmlinux 0xffffffffa42c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
31
ret_from_fork_nospec_begin
vmlinux 0xffffffffa4994df7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21
32
? insert_kthread_work
vmlinux 0xffffffffa42c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 27s! [kworker/u8:1:5241]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter sunrpc ext4 mbcache jbd2 dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper sg ablk_helper cryptd virtio_balloon i2c_piix4 pcspkr joydev ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_net net_failover failover bochs_drm ahci drm_kms_helper syscopyarea sysfillrect libahci sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix drm crc32c_intel virtio_pci serio_raw virtio_ring
virtio libata drm_panel_orientation_quirks floppy fuse
CPU: 3 PID: 5241 Comm: kworker/u8:1 Tainted: G             L ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.13.0-48-gd9c812dda519-prebuilt.qemu.org 04/01/2014
Workqueue: writeback bdi_writeback_workfn (flush-8:0)
task: ffff883ade98b180 ti: ffff883926fb8000 task.ti: ffff883926fb8000
RIP: 0010:[<ffffffffa498a3f5>]  [<ffffffffa498a3f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff883926fbb780  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff883926fbb750 RCX: 0000000000000000
RDX: ffff8838793ecc18 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff883926fbb780 R08: 0000000000000000 R09: 00000002a8d91fff
R10: 0000000000000000 R11: 0000000000000001 R12: ffffffffa431791b
R13: ffff883926fbb700 R14: ffff883a5d101498 R15: ffff883a5d48f800
FS:  0000000000000000(0000) GS:ffff883b37d80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f48295c9000 CR3: 000000035ce10000 CR4: 0000000000160fe0
Call Trace:
[<ffffffffc0422d45>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc041fa20>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffffa46e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa46ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa457846e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa4553869>] __blk_run_queue+0x39/0x50
[<ffffffffa4557673>] blk_queue_bio+0x3b3/0x400
[<ffffffffa45554f7>] generic_make_request+0x147/0x380
[<ffffffffa448c1e2>] ? bvec_alloc+0x92/0x120
[<ffffffffa45557a0>] submit_bio+0x70/0x150
[<ffffffffa448c483>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc060fe45>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc0610475>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffffa43c9efc>] write_cache_pages+0x21c/0x470
[<ffffffffc0610290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc061025b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa43caf51>] do_writepages+0x21/0x50
[<ffffffffa447d8d0>] __writeback_single_inode+0x40/0x260
[<ffffffffa42c6ec5>] ? wake_up_bit+0x25/0x30
[<ffffffffa447e454>] writeback_sb_inodes+0x1c4/0x430
[<ffffffffa447e75f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffffa447ec33>] wb_writeback+0x263/0x2f0
[<ffffffffa446a85c>] ? get_nr_inodes+0x4c/0x70
[<ffffffffa447f82b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffffa42bde3f>] process_one_work+0x17f/0x440
[<ffffffffa42bef56>] worker_thread+0x126/0x3c0
[<ffffffffa42bee30>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa42c5e11>] kthread+0xd1/0xe0
[<ffffffffa42c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffa4994df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa42c5d40>] ? insert_kthread_work+0x40/0x40
Code: 16 ce 98 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 f6 cd 98 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #147880
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff816ab845
Build id: 3.10.0-693.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc004b632
Build id: 3.10.0-693.el7.x86_64 		0x442
3
? scsi_test_unit_ready
vmlinux 0xffffffff8146d190
Build id: 3.10.0-693.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8146540a
Build id: 3.10.0-693.el7.x86_64 		0xaa
5
scsi_request_fn
vmlinux 0xffffffff8146e1cf
Build id: 3.10.0-693.el7.x86_64 		0x4df
6
__blk_run_queue
vmlinux 0xffffffff812f71d3
Build id: 3.10.0-693.el7.x86_64 		0x33
7
blk_queue_bio
vmlinux 0xffffffff812faad3
Build id: 3.10.0-693.el7.x86_64 		0x363
8
generic_make_request
vmlinux 0xffffffff812f8b45
Build id: 3.10.0-693.el7.x86_64 		0x105
9
? bvec_alloc
vmlinux 0xffffffff8123b472
Build id: 3.10.0-693.el7.x86_64 		0x92
10
submit_bio
vmlinux 0xffffffff812f8dc0
Build id: 3.10.0-693.el7.x86_64 		0x70
11
? bio_alloc_bioset
vmlinux 0xffffffff8123b713
Build id: 3.10.0-693.el7.x86_64 		0x213
12
xfs_add_to_ioend
xfs 0xffffffffc02474f5
Build id: 3.10.0-693.el7.x86_64 		0xc5
13
xfs_do_writepage
xfs 0xffffffffc0247b65
Build id: 3.10.0-693.el7.x86_64 		0x1e5
14
write_cache_pages
vmlinux 0xffffffff8118dc91
Build id: 3.10.0-693.el7.x86_64 		0x251
15
? xfs_aops_discard_page
xfs 0xffffffffc0247980
Build id: 3.10.0-693.el7.x86_64 		0x150
16
xfs_vm_writepages
xfs 0xffffffffc024778d
Build id: 3.10.0-693.el7.x86_64 		0xbd
17
do_writepages
vmlinux 0xffffffff8118effe
Build id: 3.10.0-693.el7.x86_64 		0x1e
18
__writeback_single_inode
vmlinux 0xffffffff8122d7d0
Build id: 3.10.0-693.el7.x86_64 		0x40
19
writeback_sb_inodes
vmlinux 0xffffffff8122e414
Build id: 3.10.0-693.el7.x86_64 		0x1c4
20
__writeback_inodes_wb
vmlinux 0xffffffff8122e77f
Build id: 3.10.0-693.el7.x86_64 		0x9f
21
wb_writeback
vmlinux 0xffffffff8122efb3
Build id: 3.10.0-693.el7.x86_64 		0x263
22
? get_nr_inodes
vmlinux 0xffffffff8121bc3c
Build id: 3.10.0-693.el7.x86_64 		0x4c
23
bdi_writeback_workfn
vmlinux 0xffffffff8122f45b
Build id: 3.10.0-693.el7.x86_64 		0x2cb
24
process_one_work
vmlinux 0xffffffff810a881a
Build id: 3.10.0-693.el7.x86_64 		0x17a
25
worker_thread
vmlinux 0xffffffff810a94e6
Build id: 3.10.0-693.el7.x86_64 		0x126
26
? manage_workers.isra.24
vmlinux 0xffffffff810a93c0
Build id: 3.10.0-693.el7.x86_64 		0x2a0
27
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64 		0xcf
28
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64 		0x40
29
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64 		0x58
30
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [kworker/u4:0:29658]
Modules linked in: vmw_vsock_vmci_transport vsock coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev nfit pcspkr joydev sg libnvdimm vmw_balloon i2c_piix4 shpchp vmw_vmci parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 ahci libahci vmw_pvscsi ata_piix libata i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 29658 Comm: kworker/u4:0 Not tainted 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff8801370e5ee0 ti: ffff8800a81d4000 task.ti: ffff8800a81d4000
RIP: 0010:[<ffffffff816ab845>]  [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8800a81d7748  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff8800a81d7708 RCX: 0000000035daefff
RDX: ffffc90000760000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8800a81d7748 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8800b59da700 R11: ffff8800367e2000 R12: 0000000000000020
R13: ffff8800ab29e780 R14: ffffffff8146bdb9 R15: ffff8800a81d76c0
FS:  0000000000000000(0000) GS:ffff88013fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f6905414000 CR3: 00000000b71f0000 CR4: 00000000001407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8800a81d77c0 ffffffffc004b632 ffff8800a1ff9000 ffff880035c28000
ffff880035d4b080 ffff880135db67e8 ffffffff8146d190 0000000000000006
ffff880135db6754 0000000000000282 ffff8800b59da700 0000000000000000
Call Trace:
[<ffffffffc004b632>] pvscsi_queue+0x442/0x7e1 [vmw_pvscsi]
[<ffffffff8146d190>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8146540a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e1cf>] scsi_request_fn+0x4df/0x680
[<ffffffff812f71d3>] __blk_run_queue+0x33/0x40
[<ffffffff812faad3>] blk_queue_bio+0x363/0x3a0
[<ffffffff812f8b45>] generic_make_request+0x105/0x310
[<ffffffff8123b472>] ? bvec_alloc+0x92/0x120
[<ffffffff812f8dc0>] submit_bio+0x70/0x150
[<ffffffff8123b713>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc02474f5>] xfs_add_to_ioend+0xc5/0x150 [xfs]
[<ffffffffc0247b65>] xfs_do_writepage+0x1e5/0x530 [xfs]
[<ffffffff8118dc91>] write_cache_pages+0x251/0x4d0
[<ffffffffc0247980>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc024778d>] xfs_vm_writepages+0xbd/0xe0 [xfs]
[<ffffffff8118effe>] do_writepages+0x1e/0x40
[<ffffffff8122d7d0>] __writeback_single_inode+0x40/0x220
[<ffffffff8122e414>] writeback_sb_inodes+0x1c4/0x490
[<ffffffff8122e77f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8122efb3>] wb_writeback+0x263/0x2f0
[<ffffffff8121bc3c>] ? get_nr_inodes+0x4c/0x70
[<ffffffff8122f45b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #64692
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc038c3be
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03db71e
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03a72e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff984e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff984ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffff9837818e
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff983536d9
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff983573e3
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff98355367
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff9828c362
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff98355610
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff9828c603
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc061ce45
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc061d475
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffff981c9f5c
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc061d290
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc061d25b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff981cafb1
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff9827d8d0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffff980c6f25
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffff9827e454
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffff9827e75f
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffff9827ec33
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x263
24
bdi_writeback_workfn
vmlinux 0xffffffff9827f82b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffff980bde9f
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffff980befb6
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x126
27
? manage_workers.isra.26
vmlinux 0xffffffff980bee90
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffff980c5e71
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffff980c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffff98793df7
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
31
? insert_kthread_work
vmlinux 0xffffffff980c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 26s! [kworker/u16:2:3992]
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix serio_raw mptspi scsi_transport_spi libata mptscsih e1000 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 3992 Comm: kworker/u16:2 Kdump: loaded Tainted: P        W  OEL ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff9cf2cfa4d280 ti: ffff9cf26d2a4000 task.ti: ffff9cf26d2a4000
RIP: 0010:[<ffffffffc038c3be>]  [<ffffffffc038c3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9cf26d2a76e8  EFLAGS: 00000246
RAX: ffffb56f40f40000 RBX: ffff9cf0f5e1d7f0 RCX: 000000000000000a
RDX: ffff9cf2f5a2aa00 RSI: ffff9cf2313c4000 RDI: 000000000000000e
RBP: ffff9cf26d2a76f8 R08: 0000000000000003 R09: 0000000050498fff
R10: ffff9cf2f4872000 R11: ffff9cf1c8305340 R12: ffff9cf2eb426780
R13: ffff9cf26d2a765c R14: ffffffff9835bffb R15: ffff9cf26d2a76a0
FS:  0000000000000000(0000) GS:ffff9cf2ffd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f172c89f000 CR3: 00000000b072e000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc03db71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a72e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff984e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff984ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9837818e>] ? deadline_add_request+0x6e/0x90
[<ffffffff983536d9>] __blk_run_queue+0x39/0x50
[<ffffffff983573e3>] blk_queue_bio+0x3b3/0x400
[<ffffffff98355367>] generic_make_request+0x147/0x380
[<ffffffff9828c362>] ? bvec_alloc+0x92/0x120
[<ffffffff98355610>] submit_bio+0x70/0x150
[<ffffffff9828c603>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc061ce45>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc061d475>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff981c9f5c>] write_cache_pages+0x21c/0x470
[<ffffffffc061d290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc061d25b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff981cafb1>] do_writepages+0x21/0x50
[<ffffffff9827d8d0>] __writeback_single_inode+0x40/0x260
[<ffffffff980c6f25>] ? wake_up_bit+0x25/0x30
[<ffffffff9827e454>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff9827e75f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff9827ec33>] wb_writeback+0x263/0x2f0
[<ffffffff9827f82b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff980bde9f>] process_one_work+0x17f/0x440
[<ffffffff980befb6>] worker_thread+0x126/0x3c0
[<ffffffff980bee90>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff980c5e71>] kthread+0xd1/0xe0
[<ffffffff980c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff98793df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff980c5da0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 a3 39
Complete report #23097
Frame # Function Binary Source or offset Line
1
scsi_request_fn
vmlinux 0xffffffff8bcebd05
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x135
2
? deadline_add_request
vmlinux 0xffffffff8bb7716e
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x6e
3
__blk_run_queue
vmlinux 0xffffffff8bb52769
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x39
4
blk_queue_bio
vmlinux 0xffffffff8bb56473
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x3b3
5
generic_make_request
vmlinux 0xffffffff8bb543f7
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x147
6
? bvec_alloc
vmlinux 0xffffffff8ba8b6b2
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x92
7
submit_bio
vmlinux 0xffffffff8bb546a0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x70
8
? bio_alloc_bioset
vmlinux 0xffffffff8ba8b953
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x213
9
xfs_add_to_ioend
xfs 0xffffffffc03efec5
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x145
10
xfs_do_writepage
xfs 0xffffffffc03f04f5
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x1e5
11
write_cache_pages
vmlinux 0xffffffff8b9ca1bc
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x21c
12
? xfs_vm_writepages
xfs 0xffffffffc03f0310
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xa0
13
xfs_vm_writepages
xfs 0xffffffffc03f02db
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x6b
14
do_writepages
vmlinux 0xffffffff8b9cb211
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x21
15
__writeback_single_inode
vmlinux 0xffffffff8ba7cc30
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
16
? wake_up_bit
vmlinux 0xffffffff8b8c7745
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x25
17
writeback_sb_inodes
vmlinux 0xffffffff8ba7d7c4
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x1c4
18
__writeback_inodes_wb
vmlinux 0xffffffff8ba7dacf
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x9f
19
wb_writeback
vmlinux 0xffffffff8ba7dfb3
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x263
20
? bdi_dirty_limit
vmlinux 0xffffffff8b9ca980
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
21
bdi_writeback_workfn
vmlinux 0xffffffff8ba7eaac
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x1cc
22
process_one_work
vmlinux 0xffffffff8b8be6bf
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x17f
23
worker_thread
vmlinux 0xffffffff8b8bf7d6
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x126
24
? manage_workers.isra.26
vmlinux 0xffffffff8b8bf6b0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x2a0
25
kthread
vmlinux 0xffffffff8b8c6691
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xd1
26
? insert_kthread_work
vmlinux 0xffffffff8b8c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
27
ret_from_fork_nospec_begin
vmlinux 0xffffffff8bf92d37
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x21
28
? insert_kthread_work
vmlinux 0xffffffff8b8c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 21s! [kworker/u129:1:47245]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd sg pcspkr joydev vmw_balloon parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix libata drm crct10dif_pclmul crct10dif_common mptspi crc32c_intel
scsi_transport_spi e1000 mptscsih serio_raw mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 47245 Comm: kworker/u129:1 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1127.13.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/17/2015
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff8b0fdcd8a0e0 ti: ffff8b13f8ad8000 task.ti: ffff8b13f8ad8000
RIP: 0010:[<ffffffff8bcebd05>]  [<ffffffff8bcebd05>] scsi_request_fn+0x135/0x680
RSP: 0018:ffff8b13f8adb800  EFLAGS: 00000202
RAX: 000000017410cb60 RBX: ffff8b0fd31d4900 RCX: 00000001741132ce
RDX: 0000000174111f31 RSI: ffff8b0fd31d4bd0 RDI: ffff8b0ef4545c10
RBP: ffff8b13f8adb858 R08: ffff8b0ef4545d58 R09: 0000000000000004
R10: ffff8b10fbc75000 R11: ffff8b0f7aa41880 R12: ffff8b13f8adb790
R13: ffffffffc029bf0f R14: ffff8b13f8adb770 R15: ffff8b13b08d4f00
FS:  0000000000000000(0000) GS:ffff8b0ffb580000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0fea7fa030 CR3: 0000000082fd8000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8bb7716e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8bb52769>] __blk_run_queue+0x39/0x50
[<ffffffff8bb56473>] blk_queue_bio+0x3b3/0x400
[<ffffffff8bb543f7>] generic_make_request+0x147/0x380
[<ffffffff8ba8b6b2>] ? bvec_alloc+0x92/0x120
[<ffffffff8bb546a0>] submit_bio+0x70/0x150
[<ffffffff8ba8b953>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc03efec5>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc03f04f5>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff8b9ca1bc>] write_cache_pages+0x21c/0x470
[<ffffffffc03f0310>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc03f02db>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8b9cb211>] do_writepages+0x21/0x50
[<ffffffff8ba7cc30>] __writeback_single_inode+0x40/0x260
[<ffffffff8b8c7745>] ? wake_up_bit+0x25/0x30
[<ffffffff8ba7d7c4>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff8ba7dacf>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8ba7dfb3>] wb_writeback+0x263/0x2f0
[<ffffffff8b9ca980>] ? bdi_dirty_limit+0x40/0xe0
[<ffffffff8ba7eaac>] bdi_writeback_workfn+0x1cc/0x460
[<ffffffff8b8be6bf>] process_one_work+0x17f/0x440
[<ffffffff8b8bf7d6>] worker_thread+0x126/0x3c0
[<ffffffff8b8bf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff8b8c6691>] kthread+0xd1/0xe0
[<ffffffff8b8c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff8bf92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8b8c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 85 40 04 00 00 a8 02 0f 85 18 01 00 00 4c 89 f7 e8 40 9c e6 ff 49 8b bd 58 04 00 00 c6 07 00 0f 1f 40 00 fb 66 0f 1f 44 00 00 <4d> 8b be 00 01 00 00 4d 85 ff 0f 84 3d 04 00 00 49 8b 85 40 04
Complete report #20766
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffa896a175
Build id: 3.10.0-957.1.3.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc04e5622
Build id: 3.10.0-957.1.3.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffa86d9100
Build id: 3.10.0-957.1.3.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffa86d0bd0
Build id: 3.10.0-957.1.3.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffa86da16c
Build id: 3.10.0-957.1.3.el7.x86_64 		0x4cc
6
? deadline_add_request
vmlinux 0xffffffffa856786e
Build id: 3.10.0-957.1.3.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffffa8543269
Build id: 3.10.0-957.1.3.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffffa8546f63
Build id: 3.10.0-957.1.3.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffffa8544ef7
Build id: 3.10.0-957.1.3.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffffa847e3b2
Build id: 3.10.0-957.1.3.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffffa85451a0
Build id: 3.10.0-957.1.3.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffffa847e653
Build id: 3.10.0-957.1.3.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc05fb165
Build id: 3.10.0-957.1.3.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc05fb787
Build id: 3.10.0-957.1.3.el7.x86_64 		0x1d7
15
write_cache_pages
vmlinux 0xffffffffa83c22a4
Build id: 3.10.0-957.1.3.el7.x86_64 		0x254
16
? xfs_aops_discard_page
xfs 0xffffffffc05fb5b0
Build id: 3.10.0-957.1.3.el7.x86_64 		0x150
17
xfs_vm_writepages
xfs 0xffffffffc05fb3ab
Build id: 3.10.0-957.1.3.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffffa83c33f1
Build id: 3.10.0-957.1.3.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffffa846fba0
Build id: 3.10.0-957.1.3.el7.x86_64 		0x40
20
writeback_sb_inodes
vmlinux 0xffffffffa8470634
Build id: 3.10.0-957.1.3.el7.x86_64 		0x1c4
21
__writeback_inodes_wb
vmlinux 0xffffffffa847099f
Build id: 3.10.0-957.1.3.el7.x86_64 		0x9f
22
wb_writeback
vmlinux 0xffffffffa84711d3
Build id: 3.10.0-957.1.3.el7.x86_64 		0x263
23
? get_nr_inodes
vmlinux 0xffffffffa845cfec
Build id: 3.10.0-957.1.3.el7.x86_64 		0x4c
24
bdi_writeback_workfn
vmlinux 0xffffffffa8471b5b
Build id: 3.10.0-957.1.3.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffffa82b9d4f
Build id: 3.10.0-957.1.3.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffffa82bade6
Build id: 3.10.0-957.1.3.el7.x86_64 		0x126
27
? manage_workers.isra.25
vmlinux 0xffffffffa82bacc0
Build id: 3.10.0-957.1.3.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffffa82c1c31
Build id: 3.10.0-957.1.3.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffffa82c1b60
Build id: 3.10.0-957.1.3.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffa8974c37
Build id: 3.10.0-957.1.3.el7.x86_64 		0x21
31
? insert_kthread_work
vmlinux 0xffffffffa82c1b60
Build id: 3.10.0-957.1.3.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 41s! [kworker/u2:1:16118]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter devlink ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw vmw_balloon gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi ahci libahci vmwgfx drm_kms_helper ata_generic pata_acpi syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm floppy drm ata_piix libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 16118 Comm: kworker/u2:1 Kdump: loaded Tainted: G             L ------------   3.10.0-957.1.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/17/2015
Workqueue: writeback bdi_writeback_workfn (flush-253:1)
task: ffff9f76b656e180 ti: ffff9f763f474000 task.ti: ffff9f763f474000
RIP: 0010:[<ffffffffa896a175>]  [<ffffffffa896a175>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff9f763f477740  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffffa85a1363 RCX: ffffffffa8e343c0
RDX: ffffbac9c0780000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9f763f477740 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9f763595b8d8 R11: ffff9f7634ffc000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9f7622445000 R15: ffffffffa86d7e59
FS:  0000000000000000(0000) GS:ffff9f773fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f15fbb7a000 CR3: 00000000a89d2000 CR4: 00000000000607f0
Call Trace:
[<ffffffffc04e5622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffa86d9100>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffa86d0bd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa86da16c>] scsi_request_fn+0x4cc/0x680
[<ffffffffa856786e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa8543269>] __blk_run_queue+0x39/0x50
[<ffffffffa8546f63>] blk_queue_bio+0x3b3/0x400
[<ffffffffa8544ef7>] generic_make_request+0x147/0x380
[<ffffffffa847e3b2>] ? bvec_alloc+0x92/0x120
[<ffffffffa85451a0>] submit_bio+0x70/0x150
[<ffffffffa847e653>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc05fb165>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc05fb787>] xfs_do_writepage+0x1d7/0x510 [xfs]
[<ffffffffa83c22a4>] write_cache_pages+0x254/0x4e0
[<ffffffffc05fb5b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc05fb3ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa83c33f1>] do_writepages+0x21/0x50
[<ffffffffa846fba0>] __writeback_single_inode+0x40/0x260
[<ffffffffa8470634>] writeback_sb_inodes+0x1c4/0x490
[<ffffffffa847099f>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffffa84711d3>] wb_writeback+0x263/0x2f0
[<ffffffffa845cfec>] ? get_nr_inodes+0x4c/0x70
[<ffffffffa8471b5b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffffa82b9d4f>] process_one_work+0x17f/0x440
[<ffffffffa82bade6>] worker_thread+0x126/0x3c0
[<ffffffffa82bacc0>] ? manage_workers.isra.25+0x2a0/0x2a0
[<ffffffffa82c1c31>] kthread+0xd1/0xe0
[<ffffffffa82c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa8974c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa82c1b60>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #16531
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01e83be
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc023c71e
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc029d2e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff84ce2990
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff84cec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x4ac
6
? deadline_add_request
vmlinux 0xffffffff84b771ae
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff84b527a9
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff84b564b3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff84b54437
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffff84a8b6a2
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff84b546e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff84a8b943
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc043cec5
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc043d4f5
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffff849ca20c
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc043d310
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc043d2db
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffff849cb261
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff84a7cc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
20
writeback_sb_inodes
vmlinux 0xffffffff84a7d7c4
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1c4
21
__writeback_inodes_wb
vmlinux 0xffffffff84a7dacf
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x9f
22
wb_writeback
vmlinux 0xffffffff84a7dfa3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x263
23
bdi_writeback_workfn
vmlinux 0xffffffff84a7eb9b
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2cb
24
process_one_work
vmlinux 0xffffffff848be6bf
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x17f
25
worker_thread
vmlinux 0xffffffff848bf7d6
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x126
26
? manage_workers.isra.26
vmlinux 0xffffffff848bf6b0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2a0
27
kthread
vmlinux 0xffffffff848c6691
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xd1
28
? insert_kthread_work
vmlinux 0xffffffff848c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffff84f92d37
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
30
? insert_kthread_work
vmlinux 0xffffffff848c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [kworker/u4:1:23302]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd sg vmw_balloon pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom ata_generic vmwgfx pata_acpi drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix crct10dif_pclmul crct10dif_common scsi_transport_spi libata crc32c_intel mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 23302 Comm: kworker/u4:1 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff90a5c5ddd230 ti: ffff90a5637c8000 task.ti: ffff90a5637c8000
RIP: 0010:[<ffffffffc01e83be>]  [<ffffffffc01e83be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff90a5637cb6e8  EFLAGS: 00000246
RAX: ffffb0ce80fc0000 RBX: ffff90a4f642c7f0 RCX: 0000000000000004
RDX: ffff90a5f63f2c00 RSI: ffff90a5f6396000 RDI: 000000000000000e
RBP: ffff90a5637cb6f8 R08: 0000000000000003 R09: 000000016c1c8fff
R10: ffff90a5c6548800 R11: ffff90a47f591a40 R12: ffff90a5637f6a80
R13: ffff90a5637cb65c R14: ffffffff84b5b0cb R15: ffff90a5637cb6a0
FS:  0000000000000000(0000) GS:ffff90a5ffd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2906506000 CR3: 00000000ba28c000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc023c71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc029d2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff84ce2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff84cec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff84b771ae>] ? deadline_add_request+0x6e/0x90
[<ffffffff84b527a9>] __blk_run_queue+0x39/0x50
[<ffffffff84b564b3>] blk_queue_bio+0x3b3/0x400
[<ffffffff84b54437>] generic_make_request+0x147/0x380
[<ffffffff84a8b6a2>] ? bvec_alloc+0x92/0x120
[<ffffffff84b546e0>] submit_bio+0x70/0x150
[<ffffffff84a8b943>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc043cec5>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc043d4f5>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffff849ca20c>] write_cache_pages+0x21c/0x470
[<ffffffffc043d310>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc043d2db>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff849cb261>] do_writepages+0x21/0x50
[<ffffffff84a7cc40>] __writeback_single_inode+0x40/0x260
[<ffffffff84a7d7c4>] writeback_sb_inodes+0x1c4/0x430
[<ffffffff84a7dacf>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff84a7dfa3>] wb_writeback+0x263/0x2f0
[<ffffffff84a7eb9b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff848be6bf>] process_one_work+0x17f/0x440
[<ffffffff848bf7d6>] worker_thread+0x126/0x3c0
[<ffffffff848bf6b0>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffff848c6691>] kthread+0xd1/0xe0
[<ffffffff848c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff84f92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff848c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 63 1f
Complete report #11106
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc005b3be
Build id: 3.10.0-693.21.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00d172e
Build id: 3.10.0-693.21.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00452b0
Build id: 3.10.0-693.21.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8146d8f0
Build id: 3.10.0-693.21.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8147678f
Build id: 3.10.0-693.21.1.el7.x86_64 		0x4df
6
__blk_run_queue
vmlinux 0xffffffff812fd1d9
Build id: 3.10.0-693.21.1.el7.x86_64 		0x39
7
blk_queue_bio
vmlinux 0xffffffff81300b33
Build id: 3.10.0-693.21.1.el7.x86_64 		0x363
8
generic_make_request
vmlinux 0xffffffff812feb7b
Build id: 3.10.0-693.21.1.el7.x86_64 		0x10b
9
? bvec_alloc
vmlinux 0xffffffff81240452
Build id: 3.10.0-693.21.1.el7.x86_64 		0x92
10
submit_bio
vmlinux 0xffffffff812fee00
Build id: 3.10.0-693.21.1.el7.x86_64 		0x70
11
? bio_alloc_bioset
vmlinux 0xffffffff812406f3
Build id: 3.10.0-693.21.1.el7.x86_64 		0x213
12
xfs_add_to_ioend
xfs 0xffffffffc028af85
Build id: 3.10.0-693.21.1.el7.x86_64 		0xc5
13
xfs_do_writepage
xfs 0xffffffffc028b5f5
Build id: 3.10.0-693.21.1.el7.x86_64 		0x1e5
14
write_cache_pages
vmlinux 0xffffffff811921b4
Build id: 3.10.0-693.21.1.el7.x86_64 		0x254
15
? xfs_aops_discard_page
xfs 0xffffffffc028b410
Build id: 3.10.0-693.21.1.el7.x86_64 		0x150
16
xfs_vm_writepages
xfs 0xffffffffc028b21d
Build id: 3.10.0-693.21.1.el7.x86_64 		0xbd
17
do_writepages
vmlinux 0xffffffff81193531
Build id: 3.10.0-693.21.1.el7.x86_64 		0x21
18
__writeback_single_inode
vmlinux 0xffffffff812326d0
Build id: 3.10.0-693.21.1.el7.x86_64 		0x40
19
writeback_sb_inodes
vmlinux 0xffffffff81233344
Build id: 3.10.0-693.21.1.el7.x86_64 		0x1c4
20
__writeback_inodes_wb
vmlinux 0xffffffff812336af
Build id: 3.10.0-693.21.1.el7.x86_64 		0x9f
21
wb_writeback
vmlinux 0xffffffff81233ee3
Build id: 3.10.0-693.21.1.el7.x86_64 		0x263
22
bdi_writeback_workfn
vmlinux 0xffffffff8123438b
Build id: 3.10.0-693.21.1.el7.x86_64 		0x2cb
23
process_one_work
vmlinux 0xffffffff810abe2f
Build id: 3.10.0-693.21.1.el7.x86_64 		0x17f
24
worker_thread
vmlinux 0xffffffff810acaf6
Build id: 3.10.0-693.21.1.el7.x86_64 		0x126
25
? manage_workers.isra.24
vmlinux 0xffffffff810ac9d0
Build id: 3.10.0-693.21.1.el7.x86_64 		0x2a0
26
kthread
vmlinux 0xffffffff810b4031
Build id: 3.10.0-693.21.1.el7.x86_64 		0xd1
27
? insert_kthread_work
vmlinux 0xffffffff810b3f60
Build id: 3.10.0-693.21.1.el7.x86_64 		0x40
28
ret_from_fork
vmlinux 0xffffffff816c0577
Build id: 3.10.0-693.21.1.el7.x86_64 		0x77
29
? insert_kthread_work
vmlinux 0xffffffff810b3f60
Build id: 3.10.0-693.21.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 25s! [kworker/u28:0:59]
Modules linked in: xt_multiport xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache vmw_vsock_vmci_transport vsock nls_utf8 isofs sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr joydev vmw_balloon sg parport_pc parport i2c_piix4 shpchp vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi libata crct10dif_pclmul mptscsih crct10dif_common crc32c_intel serio_raw mptbase vmxnet3 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 59 Comm: kworker/u28:0 Not tainted 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
Workqueue: writeback bdi_writeback_workfn (flush-253:2)
task: ffff88089bf88000 ti: ffff88089bf90000 task.ti: ffff88089bf90000
RIP: 0010:[<ffffffffc005b3be>]  [<ffffffffc005b3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88089bf936e0  EFLAGS: 00000246
RAX: ffffc90006ac0000 RBX: ffff88018db11c00 RCX: 000000000000000e
RDX: ffff880ab3f67000 RSI: ffff8802d478a000 RDI: 000000000000000e
RBP: ffff88089bf936f0 R08: 0000000000000003 R09: ffff88089bfa30d8
R10: ffff880ab47ddf80 R11: ffff880ab47ddf80 R12: 0000000100000021
R13: ffff880a297cc180 R14: ffff88089bf93654 R15: ffffffff813051fb
FS:  0000000000000000(0000) GS:ffff880abfd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007febb74725a0 CR3: 00000002c264a000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00d172e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00452b0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8146d8f0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8147678f>] scsi_request_fn+0x4df/0x680
[<ffffffff812fd1d9>] __blk_run_queue+0x39/0x50
[<ffffffff81300b33>] blk_queue_bio+0x363/0x3a0
[<ffffffff812feb7b>] generic_make_request+0x10b/0x320
[<ffffffff81240452>] ? bvec_alloc+0x92/0x120
[<ffffffff812fee00>] submit_bio+0x70/0x150
[<ffffffff812406f3>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc028af85>] xfs_add_to_ioend+0xc5/0x150 [xfs]
[<ffffffffc028b5f5>] xfs_do_writepage+0x1e5/0x530 [xfs]
[<ffffffff811921b4>] write_cache_pages+0x254/0x4e0
[<ffffffffc028b410>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc028b21d>] xfs_vm_writepages+0xbd/0xe0 [xfs]
[<ffffffff81193531>] do_writepages+0x21/0x50
[<ffffffff812326d0>] __writeback_single_inode+0x40/0x250
[<ffffffff81233344>] writeback_sb_inodes+0x1c4/0x490
[<ffffffff812336af>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff81233ee3>] wb_writeback+0x263/0x2f0
[<ffffffff8123438b>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff810abe2f>] process_one_work+0x17f/0x440
[<ffffffff810acaf6>] worker_thread+0x126/0x3c0
[<ffffffff810ac9d0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b4031>] kthread+0xd1/0xe0
[<ffffffff810b3f60>] ? insert_kthread_work+0x40/0x40
[<ffffffff816c0577>] ret_from_fork+0x77/0xb0
[<ffffffff810b3f60>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 06
Complete report #10275
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8bf1b2c5
Build id: 3.10.0-862.11.6.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc00e3622
Build id: 3.10.0-862.11.6.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8bca8500
Build id: 3.10.0-862.11.6.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8bca0010
Build id: 3.10.0-862.11.6.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8bca957c
Build id: 3.10.0-862.11.6.el7.x86_64 		0x4cc
6
? deadline_add_request
vmlinux 0xffffffff8bb4122e
Build id: 3.10.0-862.11.6.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffff8bb1d4e9
Build id: 3.10.0-862.11.6.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffff8bb21173
Build id: 3.10.0-862.11.6.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffff8bb1f0cb
Build id: 3.10.0-862.11.6.el7.x86_64 		0x10b
10
? bvec_alloc
vmlinux 0xffffffff8ba5b602
Build id: 3.10.0-862.11.6.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffff8bb1f350
Build id: 3.10.0-862.11.6.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffff8ba5b8a3
Build id: 3.10.0-862.11.6.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc03263c5
Build id: 3.10.0-862.11.6.el7.x86_64 		0xc5
14
xfs_do_writepage
xfs 0xffffffffc0326a27
Build id: 3.10.0-862.11.6.el7.x86_64 		0x1d7
15
write_cache_pages
vmlinux 0xffffffff8b9a2744
Build id: 3.10.0-862.11.6.el7.x86_64 		0x254
16
? xfs_aops_discard_page
xfs 0xffffffffc0326850
Build id: 3.10.0-862.11.6.el7.x86_64 		0x150
17
xfs_vm_writepages
xfs 0xffffffffc032665d
Build id: 3.10.0-862.11.6.el7.x86_64 		0xbd
18
do_writepages
vmlinux 0xffffffff8b9a3ac1
Build id: 3.10.0-862.11.6.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffff8ba4cf00
Build id: 3.10.0-862.11.6.el7.x86_64 		0x40
20
writeback_sb_inodes
vmlinux 0xffffffff8ba4d994
Build id: 3.10.0-862.11.6.el7.x86_64 		0x1c4
21
__writeback_inodes_wb
vmlinux 0xffffffff8ba4dcff
Build id: 3.10.0-862.11.6.el7.x86_64 		0x9f
22
wb_writeback
vmlinux 0xffffffff8ba4e533
Build id: 3.10.0-862.11.6.el7.x86_64 		0x263
23
? get_nr_inodes
vmlinux 0xffffffff8ba3aa4c
Build id: 3.10.0-862.11.6.el7.x86_64 		0x4c
24
bdi_writeback_workfn
vmlinux 0xffffffff8ba4eebb
Build id: 3.10.0-862.11.6.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffff8b8b613f
Build id: 3.10.0-862.11.6.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffff8b8b71d6
Build id: 3.10.0-862.11.6.el7.x86_64 		0x126
27
? manage_workers.isra.24
vmlinux 0xffffffff8b8b70b0
Build id: 3.10.0-862.11.6.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffff8b8bdf21
Build id: 3.10.0-862.11.6.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffff8b8bde50
Build id: 3.10.0-862.11.6.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffff8bf255f7
Build id: 3.10.0-862.11.6.el7.x86_64 		0x21
31
? insert_kthread_work
vmlinux 0xffffffff8b8bde50
Build id: 3.10.0-862.11.6.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [kworker/u4:0:15422]
Modules linked in: binfmt_misc tcp_diag inet_diag talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) vmw_vsock_vmci_transport vsock nf_conntrack_ipv4 nf_defrag_ipv4 xt_owner xt_conntrack nf_conntrack xt_multiport iptable_filter sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev nfit pcspkr sg libnvdimm parport_pc parport i2c_piix4 vmw_vmci shpchp ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sd_mod crc_t10dif sysfillrect sysimgblt fb_sys_fops ttm crct10dif_generic drm ahci ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw vmxnet3 vmw_pvscsi i2c_core
dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 15422 Comm: kworker/u4:0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-862.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff9267b652cf10 ti: ffff92699b760000 task.ti: ffff92699b760000
RIP: 0010:[<ffffffff8bf1b2c5>]  [<ffffffff8bf1b2c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff92699b763740  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffff8bb795b3 RCX: ffffffff8c430880
RDX: ffffbcb700e80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff92699b763740 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9269b429a8d8 R11: ffff9269b40ad800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff926969609c80 R15: ffffffff8bca7259
FS:  0000000000000000(0000) GS:ffff9269bfd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f14d91de170 CR3: 000000022d2b0000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00e3622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8bca8500>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8bca0010>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8bca957c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8bb4122e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8bb1d4e9>] __blk_run_queue+0x39/0x50
[<ffffffff8bb21173>] blk_queue_bio+0x3b3/0x400
[<ffffffff8bb1f0cb>] generic_make_request+0x10b/0x320
[<ffffffff8ba5b602>] ? bvec_alloc+0x92/0x120
[<ffffffff8bb1f350>] submit_bio+0x70/0x150
[<ffffffff8ba5b8a3>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc03263c5>] xfs_add_to_ioend+0xc5/0x150 [xfs]
[<ffffffffc0326a27>] xfs_do_writepage+0x1d7/0x510 [xfs]
[<ffffffff8b9a2744>] write_cache_pages+0x254/0x4e0
[<ffffffffc0326850>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc032665d>] xfs_vm_writepages+0xbd/0xe0 [xfs]
[<ffffffff8b9a3ac1>] do_writepages+0x21/0x50
[<ffffffff8ba4cf00>] __writeback_single_inode+0x40/0x260
[<ffffffff8ba4d994>] writeback_sb_inodes+0x1c4/0x490
[<ffffffff8ba4dcff>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffff8ba4e533>] wb_writeback+0x263/0x2f0
[<ffffffff8ba3aa4c>] ? get_nr_inodes+0x4c/0x70
[<ffffffff8ba4eebb>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffff8b8b613f>] process_one_work+0x17f/0x440
[<ffffffff8b8b71d6>] worker_thread+0x126/0x3c0
[<ffffffff8b8b70b0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff8b8bdf21>] kthread+0xd1/0xe0
[<ffffffff8b8bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8bf255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8b8bde50>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #10231
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffa5f83025
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc00b8632
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffa5ce70b0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffa5cdeab0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffa5ce811c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x4cc
6
? deadline_add_request
vmlinux 0xffffffffa5b7389e
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x6e
7
__blk_run_queue
vmlinux 0xffffffffa5b4f079
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x39
8
blk_queue_bio
vmlinux 0xffffffffa5b52d93
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x3b3
9
generic_make_request
vmlinux 0xffffffffa5b50d27
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x147
10
? bvec_alloc
vmlinux 0xffffffffa5a882c2
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x92
11
submit_bio
vmlinux 0xffffffffa5b50fd0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x70
12
? bio_alloc_bioset
vmlinux 0xffffffffa5a88563
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x213
13
xfs_add_to_ioend
xfs 0xffffffffc0306be5
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x145
14
xfs_do_writepage
xfs 0xffffffffc0307215
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1e5
15
write_cache_pages
vmlinux 0xffffffffa59c9b7c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0307030
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xa0
17
xfs_vm_writepages
xfs 0xffffffffc0306ffb
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x6b
18
do_writepages
vmlinux 0xffffffffa59cabc1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x21
19
__writeback_single_inode
vmlinux 0xffffffffa5a79840
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
20
? wake_up_bit
vmlinux 0xffffffffa58c72a5
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x25
21
writeback_sb_inodes
vmlinux 0xffffffffa5a7a3d4
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1c4
22
__writeback_inodes_wb
vmlinux 0xffffffffa5a7a6df
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x9f
23
wb_writeback
vmlinux 0xffffffffa5a7abc3
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x263
24
bdi_writeback_workfn
vmlinux 0xffffffffa5a7b7bb
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2cb
25
process_one_work
vmlinux 0xffffffffa58be21f
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x17f
26
worker_thread
vmlinux 0xffffffffa58bf336
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x126
27
? manage_workers.isra.26
vmlinux 0xffffffffa58bf210
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2a0
28
kthread
vmlinux 0xffffffffa58c61f1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xd1
29
? insert_kthread_work
vmlinux 0xffffffffa58c6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffa5f8dd1d
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x7
31
? insert_kthread_work
vmlinux 0xffffffffa58c6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [kworker/u16:2:27416]
Modules linked in: veth tcp_lp nls_utf8 isofs fuse xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon edac_mce_amd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_common vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm nfit ata_piix libahci libnvdimm libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks
dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 27416 Comm: kworker/u16:2 Kdump: loaded Tainted: G               ------------ T 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff8a8fa97d0000 ti: ffff8a8b5a2e8000 task.ti: ffff8a8b5a2e8000
RIP: 0010:[<ffffffffa5f83025>]  [<ffffffffa5f83025>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a8b5a2eb750  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffffa5bad9c3 RCX: ffffffffa6434800
RDX: ffffb884c35b0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8a8b5a2eb750 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8a89345228d8 R11: ffff8a89344f7000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff8a8b55fd3680 R15: ffffffffa5ce5e09
FS:  00007fde5f319700(0000) GS:ffff8a8fffcc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 000055859f846590 CR3: 000000038ca80000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc00b8632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffa5ce70b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffa5cdeab0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa5ce811c>] scsi_request_fn+0x4cc/0x680
[<ffffffffa5b7389e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5b4f079>] __blk_run_queue+0x39/0x50
[<ffffffffa5b52d93>] blk_queue_bio+0x3b3/0x400
[<ffffffffa5b50d27>] generic_make_request+0x147/0x380
[<ffffffffa5a882c2>] ? bvec_alloc+0x92/0x120
[<ffffffffa5b50fd0>] submit_bio+0x70/0x150
[<ffffffffa5a88563>] ? bio_alloc_bioset+0x213/0x310
[<ffffffffc0306be5>] xfs_add_to_ioend+0x145/0x1d0 [xfs]
[<ffffffffc0307215>] xfs_do_writepage+0x1e5/0x550 [xfs]
[<ffffffffa59c9b7c>] write_cache_pages+0x21c/0x470
[<ffffffffc0307030>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc0306ffb>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa59cabc1>] do_writepages+0x21/0x50
[<ffffffffa5a79840>] __writeback_single_inode+0x40/0x260
[<ffffffffa58c72a5>] ? wake_up_bit+0x25/0x30
[<ffffffffa5a7a3d4>] writeback_sb_inodes+0x1c4/0x430
[<ffffffffa5a7a6df>] __writeback_inodes_wb+0x9f/0xd0
[<ffffffffa5a7abc3>] wb_writeback+0x263/0x2f0
[<ffffffffa5a7b7bb>] bdi_writeback_workfn+0x2cb/0x460
[<ffffffffa58be21f>] process_one_work+0x17f/0x440
[<ffffffffa58bf336>] worker_thread+0x126/0x3c0
[<ffffffffa58bf210>] ? manage_workers.isra.26+0x2a0/0x2a0
[<ffffffffa58c61f1>] kthread+0xd1/0xe0
[<ffffffffa58c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffa5f8dd1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa58c6120>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48