Info

Permalink
Function
_raw_spin_unlock_irqrestore
First occurrence
2020-10-01
Last occurrence
2025-12-05
State
NEW
Components
kernel
Quality
-53

Statistics

Operating system Count
CentOS 7 723
Architecture Count
x86_64 723
Related packages Count
kernel 723
 0:3.10.0-1127.19.1.el7 136
 0:3.10.0-1160.31.1.el7 119
 0:3.10.0-1160.118.1.el7 41
 0:3.10.0-1160.49.1.el7 39
 0:3.10.0-1062.1.2.el7 27
 0:3.10.0-1062.9.1.el7 27
 0:3.10.0-1127.18.2.el7 24
 0:3.10.0-1160.21.1.el7 21
 0:3.10.0-1160.6.1.el7 21
 0:3.10.0-1160.71.1.el7 21
 0:3.10.0-1160.83.1.el7 18
 0:3.10.0-1160.15.2.el7 17
 0:3.10.0-1160.42.2.el7 17
 0:3.10.0-1160.11.1.el7 16
 0:3.10.0-1160.el7 16
 0:3.10.0-1160.119.1.el7 15
 0:3.10.0-1062.el7 15
 0:3.10.0-1160.66.1.el7 14
 0:3.10.0-1160.45.1.el7 13
 0:3.10.0-1160.53.1.el7 13
 0:3.10.0-1160.80.1.el7 12
 0:3.10.0-1127.10.1.el7 11
 0:3.10.0-1160.25.1.el7 10
 0:3.10.0-1160.88.1.el7 8
 0:3.10.0-1127.el7 5
 0:3.10.0-1160.24.1.el7 5
 0:3.10.0-1160.90.1.el7 5
 0:3.10.0-1062.12.1.el7 5
 0:3.10.0-1062.18.1.el7 5
 0:3.10.0-1160.76.1.el7 4
 0:3.10.0-1160.36.2.el7 3
 0:3.10.0-1160.59.1.el7 3
 0:3.10.0-1160.105.1.el7 2
 0:3.10.0-1160.62.1.el7 2
 0:3.10.0-1160.81.1.el7 2
 0:3.10.0-1160.102.1.el7 2
 0:3.10.0-1160.114.2.el7 2
 0:3.10.0-1127.8.2.el7 1
 0:3.10.0-1160.119.1.el7.tuxcare.els21 1
 0:3.10.0-1062.1.1.el7 1
 0:3.10.0-1127.13.1.el7 1
 0:3.10.0-1160.99.1.el7 1
 0:3.10.0-1160.41.1.el7 1
 0:3.10.0-1160.95.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1361875
Frame # Function Binary Source or offset Line
1
mptspi_qcmd
mptspi 0xffffffffc00ed2e0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x50
2
scsi_dispatch_cmd
vmlinux 0xffffffff93ae4680
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xb0
3
scsi_request_fn
vmlinux 0xffffffff93aeddac
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x4ac
4
__blk_run_queue
vmlinux 0xffffffff93953d39
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x39
5
queue_unplugged
vmlinux 0xffffffff93953dfa
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x2a
6
blk_flush_plug_list
vmlinux 0xffffffff939576f8
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x1d8
7
blk_queue_bio
vmlinux 0xffffffff939578d2
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x182
8
generic_make_request
vmlinux 0xffffffff939559e7
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x147
9
? __bio_add_page
vmlinux 0xffffffff9388baab
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x20b
10
submit_bio
vmlinux 0xffffffff93955c90
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x70
11
_xfs_buf_ioapply
xfs 0xffffffffc0341283
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x2f3
12
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03432e0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x130
13
__xfs_buf_submit
xfs 0xffffffffc0342c72
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x72
14
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03432e0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x130
15
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0343bd0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x10
16
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0372a40
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x90
17
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0343bd0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x10
18
xfsaild
xfs 0xffffffffc0372cca
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x28a
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0372a40
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x90
20
kthread
vmlinux 0xffffffff936c5f91
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xd1
21
? insert_kthread_work
vmlinux 0xffffffff936c5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40
22
ret_from_fork_nospec_begin
vmlinux 0xffffffff93d99df7
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x21
23
? insert_kthread_work
vmlinux 0xffffffff936c5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [java:10001]
[<ffffffffc00ed2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff93ae4680>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff93aeddac>] scsi_request_fn+0x4ac/0x680
[<ffffffff93953d39>] __blk_run_queue+0x39/0x50
[<ffffffff93953dfa>] queue_unplugged+0x2a/0xa0
[<ffffffff939576f8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff939578d2>] blk_queue_bio+0x182/0x400
[<ffffffff939559e7>] generic_make_request+0x147/0x380
[<ffffffff9388baab>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff93955c90>] submit_bio+0x70/0x150
[<ffffffffc0341283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc03432e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0342c72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc03432e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0343bd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0372a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0343bd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0372cca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc0372a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff936c5f91>] kthread+0xd1/0xe0
[<ffffffff936c5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffff93d99df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff936c5ec0>] ? insert_kthread_work+0x40/0x40
Complete report #1090483
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01163be
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc014271e
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01b02e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x50
4
? scsi_dispatch_cmd
vmlinux 0xffffffff9f4e3e6b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xab
5
scsi_dispatch_cmd
vmlinux 0xffffffff9f4e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xb0
6
scsi_request_fn
vmlinux 0xffffffff9f4ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4ac
7
__blk_run_queue
vmlinux 0xffffffff9f353869
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x39
8
queue_unplugged
vmlinux 0xffffffff9f35392a
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2a
9
blk_flush_plug_list
vmlinux 0xffffffff9f357268
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1d8
10
blk_queue_bio
vmlinux 0xffffffff9f357442
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x182
11
generic_make_request
vmlinux 0xffffffff9f3554f7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x147
12
? __bio_add_page
vmlinux 0xffffffff9f28b59b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x20b
13
submit_bio
vmlinux 0xffffffff9f3557a0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x70
14
_xfs_buf_ioapply
xfs 0xffffffffc0389283
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2f3
15
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc038b2e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x130
16
__xfs_buf_submit
xfs 0xffffffffc038ac72
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x72
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc038b2e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x130
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc038bbd0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03baa40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc038bbd0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc03bacca
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x28a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03baa40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffff9f0c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffff9f0c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffff9f794df7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffff9f0c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 29s! [xfsaild/dm-0:599]
Modules linked in: binfmt_misc resguard_linux(OE) secmodel_linux(OE) syshook_linux(OE) secmodel3(OE) xt_set ip_set_hash_net xt_NFLOG nfnetlink_log ip_set xt_CHECKSUM iptable_raw ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock nf_conntrack_netlink nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack sunrpc ext4 mbcache jbd2 iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi libata scsi_transport_spi mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 599 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8d25abbc5280 ti: ffff8d25acfb8000 task.ti: ffff8d25acfb8000
RIP: 0010:[<ffffffffc01163be>]  [<ffffffffc01163be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8d25acfbb940  EFLAGS: 00000246
RAX: ffff9d74c6580000 RBX: ffff8d2a85a1de60 RCX: 000000000000001e
RDX: ffff8d259ffab200 RSI: ffff8d347f4f0000 RDI: 000000000000000e
RBP: ffff8d25acfbb950 R08: 0000000000000003 R09: 000000027457afff
R10: ffff8d259d79e000 R11: ffff8d2b25d956c0 R12: 0000000100000004
R13: ffff8d26174f8d80 R14: ffff8d25acfbb8b4 R15: ffffffff9f35c28b
FS:  0000000000000000(0000) GS:ffff8d347d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f585000b000 CR3: 0000000530210000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc014271e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01b02e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9f4e3e6b>] ? scsi_dispatch_cmd+0xab/0x240
[<ffffffff9f4e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9f4ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffff9f353869>] __blk_run_queue+0x39/0x50
[<ffffffff9f35392a>] queue_unplugged+0x2a/0xa0
[<ffffffff9f357268>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9f357442>] blk_queue_bio+0x182/0x400
[<ffffffff9f3554f7>] generic_make_request+0x147/0x380
[<ffffffff9f28b59b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff9f3557a0>] submit_bio+0x70/0x150
[<ffffffffc0389283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc038b2e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc038ac72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc038b2e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc038bbd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc03baa40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc038bbd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc03bacca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc03baa40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff9f0c5e11>] kthread+0xd1/0xe0
[<ffffffff9f0c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f794df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9f0c5d40>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 12
Complete report #986041
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff943ba385
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc04126a2
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff9410bb80
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff94103030
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff9410cbdc
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff93f687d9
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff93f688aa
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff93f6c2f8
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff93f6c4d2
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff93f6a537
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x157
11
? __bio_add_page
vmlinux 0xffffffff93e9ad1b
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x21b
12
submit_bio
vmlinux 0xffffffff93f6a7e0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc06667d3
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc06688d0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc0668262
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc06688d0
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0669200
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0698e70
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xa0
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0669200
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc06990fa
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x28a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0698e70
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xa0
22
kthread
vmlinux 0xffffffff93ccb621
Build id: 3.10.0-1160.118.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff93ccb550
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff943c51dd
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x7
25
? insert_kthread_work
vmlinux 0xffffffff93ccb550
Build id: 3.10.0-1160.118.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [xfsaild/dm-0:399]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter ipmi_devintf ipmi_msghandler falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_16108(E) vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon pcspkr joydev sg vmw_vmci i2c_piix4 parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic crct10dif_common syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crc32c_intel drm serio_raw ata_piix nfit libata libnvdimm vmxnet3 vmw_pvscsi
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 399 Comm: xfsaild/dm-0 Kdump: loaded Tainted: P            EL ------------   3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9630b5f11080 ti: ffff9630b5ddc000 task.ti: ffff9630b5ddc000
RIP: 0010:[<ffffffff943ba385>]  [<ffffffff943ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9630b5ddf9a8  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff9630b5ddf968 RCX: ffff9631f7dc8060
RDX: ffffa9ce01b80000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9630b5ddf9a8 R08: 0000000000000000 R09: ffff9630810a2100
R10: ffff9630b5dd0000 R11: ffff9630b5de9800 R12: 0000000000000020
R13: ffff9631f0c33080 R14: ffffffff9410a7b9 R15: ffff9630b5ddf920
FS:  0000000000000000(0000) GS:ffff9633bfc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc239f22d70 CR3: 0000000154646000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc04126a2>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9410bb80>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff94103030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff9410cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffff93f687d9>] __blk_run_queue+0x39/0x50
[<ffffffff93f688aa>] queue_unplugged+0x2a/0xa0
[<ffffffff93f6c2f8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff93f6c4d2>] blk_queue_bio+0x182/0x400
[<ffffffff93f6a537>] generic_make_request+0x157/0x390
[<ffffffff93e9ad1b>] ? __bio_add_page+0x21b/0x2c0
[<ffffffff93f6a7e0>] submit_bio+0x70/0x150
[<ffffffffc06667d3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc06688d0>] ? xfs_buf_delwri_submit_buffers+0x130/0x250 [xfs]
[<ffffffffc0668262>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc06688d0>] xfs_buf_delwri_submit_buffers+0x130/0x250 [xfs]
[<ffffffffc0669200>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0698e70>] ? xfs_trans_ail_cursor_first+0xa0/0xa0 [xfs]
[<ffffffffc0669200>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc06990fa>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc0698e70>] ? xfs_trans_ail_cursor_first+0xa0/0xa0 [xfs]
[<ffffffff93ccb621>] kthread+0xd1/0xe0
[<ffffffff93ccb550>] ? insert_kthread_work+0x40/0x40
[<ffffffff943c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff93ccb550>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 cc cc cc cc 66 66 66 66 90 55 48 89 e5 48
Complete report #363747
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff9b581825
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc045f632
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff9b2e4e90
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff9b2dc890
Build id: 3.10.0-1062.1.2.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff9b2e5efc
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x4cc
6
__blk_run_queue
vmlinux 0xffffffff9b14cf59
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff9b14d01a
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff9b150858
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff9b150a32
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff9b14ebf7
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff9b08553b
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff9b14eea0
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc06b1003
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x2f3
14
? xfs_buf_submit
xfs 0xffffffffc06b2b4a
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x1a
15
__xfs_buf_submit
xfs 0xffffffffc06b28c1
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x61
16
xfs_buf_submit
xfs 0xffffffffc06b2b4a
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x1a
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc06b2d9c
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x22c
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc06b3930
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc06e23d0
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc06b3930
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc06e264a
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x27a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc06e23d0
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffff9aec50d1
Build id: 3.10.0-1062.1.2.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffff9aec5000
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffff9b58cd37
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffff9aec5000
Build id: 3.10.0-1062.1.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 36s! [xfsaild/dm-0:694]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 overlay(T) tun bridge stp llc devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev iosf_mbi vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg i2c_piix4 parport_pc vmw_vmci parport ip_tables xfs libcrc32c
sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit libnvdimm ahci ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel serio_raw libata vmw_pvscsi vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 694 Comm: xfsaild/dm-0 Tainted: G        W    L ------------ T 3.10.0-1062.1.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9293f1f53150 ti: ffff9293f1fa0000 task.ti: ffff9293f1fa0000
RIP: 0010:[<ffffffff9b581825>]  [<ffffffff9b581825>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9293f1fa3998  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff9293f1fa3958 RCX: ffffffff9ba34800
RDX: ffffa69c00e70000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9293f1fa3998 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9291f5f680d8 R11: ffff9291f5f62800 R12: 0000000000000020
R13: ffff929379b74c00 R14: ffffffff9b2e3be9 R15: ffff9293f1fa3910
FS:  0000000000000000(0000) GS:ffff9293ffc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f512303c000 CR3: 0000000181810000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc045f632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff9b2e4e90>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff9b2dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9b2e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff9b14cf59>] __blk_run_queue+0x39/0x50
[<ffffffff9b14d01a>] queue_unplugged+0x2a/0xa0
[<ffffffff9b150858>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9b150a32>] blk_queue_bio+0x182/0x400
[<ffffffff9b14ebf7>] generic_make_request+0x147/0x380
[<ffffffff9b08553b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff9b14eea0>] submit_bio+0x70/0x150
[<ffffffffc06b1003>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc06b2b4a>] ? xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc06b28c1>] __xfs_buf_submit+0x61/0x210 [xfs]
[<ffffffffc06b2b4a>] xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc06b2d9c>] xfs_buf_delwri_submit_buffers+0x22c/0x240 [xfs]
[<ffffffffc06b3930>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc06e23d0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc06b3930>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc06e264a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc06e23d0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff9aec50d1>] kthread+0xd1/0xe0
[<ffffffff9aec5000>] ? insert_kthread_work+0x40/0x40
[<ffffffff9b58cd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9aec5000>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #338958
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb3183025
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0200632
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb2ee70b0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb2edeab0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffb2ee811c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x4cc
6
__blk_run_queue
vmlinux 0xffffffffb2d4f079
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffffb2d4f13a
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffb2d52988
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffffb2d52b62
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffffb2d50d27
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffffb2c8767b
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffffb2d50fd0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc043f003
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2f3
14
? xfs_buf_submit
xfs 0xffffffffc0440b4a
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1a
15
__xfs_buf_submit
xfs 0xffffffffc04408c1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x61
16
xfs_buf_submit
xfs 0xffffffffc0440b4a
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1a
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0440d9c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x22c
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0441930
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04703b0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0441930
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc047062a
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x27a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04703b0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffffb2ac61f1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffffb2ac6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffffb318dd37
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffffb2ac6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [xfsaild/dm-1:726]
Modules linked in: tcp_diag inet_diag talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) ip_set nfnetlink xt_connlimit ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter nf_log_ipv4 nf_log_common nf_nat_ftp xt_REDIRECT nf_nat_redirect xt_conntrack nf_conntrack_ftp xt_LOG xt_limit xt_multiport iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc vmw_vmci parport
i2c_piix4 tcp_htcp ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw e1000 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 726 Comm: xfsaild/dm-1 Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff8ca2aa2e5230 ti: ffff8ca2a9a10000 task.ti: ffff8ca2a9a10000
RIP: 0010:[<ffffffffb3183025>]  [<ffffffffb3183025>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8ca2a9a13998  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff8ca2a9a13958 RCX: ffff8ca2bfb29060
RDX: ffffa2a7418b0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8ca2a9a13998 R08: 0000000000000000 R09: ffff8ca0fca42b00
R10: ffff8ca26239e0d8 R11: ffff8ca2bfe7c000 R12: 0000000000000020
R13: ffff8ca2bf616c00 R14: ffffffffb2ee5e09 R15: ffff8ca2a9a13910
FS:  0000000000000000(0000) GS:ffff8ca2bfc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6c534b5fe8 CR3: 0000000342ae4000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc0200632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb2ee70b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb2edeab0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb2ee811c>] scsi_request_fn+0x4cc/0x680
[<ffffffffb2d4f079>] __blk_run_queue+0x39/0x50
[<ffffffffb2d4f13a>] queue_unplugged+0x2a/0xa0
[<ffffffffb2d52988>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb2d52b62>] blk_queue_bio+0x182/0x400
[<ffffffffb2d50d27>] generic_make_request+0x147/0x380
[<ffffffffb2c8767b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffffb2d50fd0>] submit_bio+0x70/0x150
[<ffffffffc043f003>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc0440b4a>] ? xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc04408c1>] __xfs_buf_submit+0x61/0x210 [xfs]
[<ffffffffc0440b4a>] xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc0440d9c>] xfs_buf_delwri_submit_buffers+0x22c/0x240 [xfs]
[<ffffffffc0441930>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc04703b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0441930>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc047062a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc04703b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffb2ac61f1>] kthread+0xd1/0xe0
[<ffffffffb2ac6120>] ? insert_kthread_work+0x40/0x40
[<ffffffffb318dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb2ac6120>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #312738
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffbd98b4d5
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc00fa632
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffbd6ec6b0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffbd6e3fa0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffbd6ed6cc
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffffbd553939
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffffbd5539fa
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffbd557338
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffffbd557512
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffffbd5555c7
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffffbd48b66b
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffffbd555870
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc033f283
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03412e0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc0340c72
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03412e0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0341bd0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0370a40
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0341bd0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc0370cca
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x28a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0370a40
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffffbd2c5da1
Build id: 3.10.0-1160.24.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffffbd2c5cd0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffffbd995df7
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffffbd2c5cd0
Build id: 3.10.0-1160.24.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 29s! [xfsaild/dm-0:500]
Modules linked in: binfmt_misc tcp_diag inet_diag talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) vmw_vsock_vmci_transport vsock nf_conntrack_ipv4 nf_defrag_ipv4 xt_owner xt_conntrack nf_conntrack xt_multiport iptable_filter sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt sd_mod crc_t10dif fb_sys_fops ttm crct10dif_generic drm ahci crct10dif_pclmul crct10dif_common libahci ata_piix crc32c_intel serio_raw libata vmxnet3 vmw_pvscsi drm_panel_orientation_quirks nfit
libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 500 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.24.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff97e83b861080 ti: ffff97e83b86c000 task.ti: ffff97e83b86c000
RIP: 0010:[<ffffffffbd98b4d5>]  [<ffffffffbd98b4d5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff97e83b86f9a8  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff97e83b86f968 RCX: ffffffffbde34f40
RDX: ffffb77880eb0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff97e83b86f9a8 R08: 0000000000000001 R09: 0000000000000000
R10: ffff97e936381800 R11: ffff97e7b66e5000 R12: 0000000000000020
R13: ffff97e8f6b8b380 R14: ffffffffbd6eb369 R15: ffff97e83b86f920
FS:  0000000000000000(0000) GS:ffff97e9bfc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000557602d66ff0 CR3: 000000019d422000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc00fa632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffbd6ec6b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffbd6e3fa0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbd6ed6cc>] scsi_request_fn+0x4ac/0x680
[<ffffffffbd553939>] __blk_run_queue+0x39/0x50
[<ffffffffbd5539fa>] queue_unplugged+0x2a/0xa0
[<ffffffffbd557338>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffbd557512>] blk_queue_bio+0x182/0x400
[<ffffffffbd5555c7>] generic_make_request+0x147/0x380
[<ffffffffbd48b66b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffffbd555870>] submit_bio+0x70/0x150
[<ffffffffc033f283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc03412e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0340c72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc03412e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0341bd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0370a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0341bd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0370cca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc0370a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffbd2c5da1>] kthread+0xd1/0xe0
[<ffffffffbd2c5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffbd995df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbd2c5cd0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #249269
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03353be
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc035771e
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03a22e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8dee3e70
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8deed59c
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8dd53869
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff8dd5392a
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff8dd57268
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff8dd57442
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff8dd554f7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff8dc8b59b
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff8dd557a0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc05ef283
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc05f12e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc05f0c72
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc05f12e0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05f1bd0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0620a40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05f1bd0
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc0620cca
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x28a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0620a40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff8dac5e11
Build id: 3.10.0-1160.15.2.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff8dac5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff8e194df7
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff8dac5d40
Build id: 3.10.0-1160.15.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [xfsaild/dm-0:453]
talpa-deny: Timeout occurred while closing /usr/openv/var/vnetd/inbound_proxy-0.lock on behalf of process vnetd[2192/2192] owned by 0(0)/0(0) <62>
talpa-deny: Timeout occurred while closing /var/lib/pgsql/9.3/data/pg_stat_tmp/db_0.tmp on behalf of process postgres[2065/2065] owned by 1003(1003)/1004(1004) <62>
Modules linked in: tcp_diag inet_diag rfcomm ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) ip6table_filter talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) ip6_tables iptable_filter vmw_vsock_vmci_transport vsock bnep sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 iosf_mbi crc32_pclmul snd_rawmidi btusb ghash_clmulni_intel snd_ac97_codec aesni_intel lrw gf128mul
glue_helper btrtl ablk_helper ac97_bus btbcm ppdev snd_seq cryptd snd_seq_device btintel snd_pcm bluetooth snd_timer snd soundcore rfkill vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul crct10dif_common drm libnvdimm crc32c_intel ata_piix libata mptspi scsi_transport_spi serio_raw e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 453 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff991af2b9d280 ti: ffff9919b5130000 task.ti: ffff9919b5130000
RIP: 0010:[<ffffffffc03353be>]  [<ffffffffc03353be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9919b5133940  EFLAGS: 00000246
RAX: ffffa92ac1c00000 RBX: ffff991b5439b900 RCX: 0000000000000003
RDX: ffff991c10dc7200 RSI: ffff9919b5df3000 RDI: 000000000000000e
RBP: ffff9919b5133950 R08: 0000000000000002 R09: 0000000292fbefff
R10: ffff991c12742000 R11: ffff991c10df79c0 R12: 0000000100000001
R13: ffff9919bb1d9680 R14: ffff9919b51338b4 R15: ffffffff8dd5c28b
FS:  0000000000000000(0000) GS:ffff991c15480000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efc295fe000 CR3: 00000000aea10000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc035771e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a22e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8dee3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8deed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8dd53869>] __blk_run_queue+0x39/0x50
[<ffffffff8dd5392a>] queue_unplugged+0x2a/0xa0
[<ffffffff8dd57268>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8dd57442>] blk_queue_bio+0x182/0x400
[<ffffffff8dd554f7>] generic_make_request+0x147/0x380
[<ffffffff8dc8b59b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff8dd557a0>] submit_bio+0x70/0x150
[<ffffffffc05ef283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc05f12e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc05f0c72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc05f12e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc05f1bd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0620a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc05f1bd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0620cca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc0620a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff8dac5e11>] kthread+0xd1/0xe0
[<ffffffff8dac5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffff8e194df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8dac5d40>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 33 34
Complete report #187897
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8a78b495
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0179632
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8a4ec580
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8a4e3e70
Build id: 3.10.0-1160.21.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8a4ed59c
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8a353809
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff8a3538ca
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff8a357208
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff8a3573e2
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff8a355497
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff8a28b53b
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff8a355740
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc03db283
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03dd2e0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc03dcc72
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc03dd2e0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc03ddbd0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc040ca40
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc03ddbd0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc040ccca
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x28a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc040ca40
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff8a0c5da1
Build id: 3.10.0-1160.21.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff8a0c5cd0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff8a795df7
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff8a0c5cd0
Build id: 3.10.0-1160.21.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 35s! [xfsaild/dm-0:501]
Modules linked in: vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr sg joydev vmw_balloon parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect sysimgblt crc_t10dif fb_sys_fops crct10dif_generic ttm drm ata_piix crct10dif_pclmul crct10dif_common nfit libata crc32c_intel serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 501 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8ee1790e5280 ti: ffff8ee1788cc000 task.ti: ffff8ee1788cc000
RIP: 0010:[<ffffffff8a78b495>]  [<ffffffff8a78b495>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8ee1788cf9a8  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8ee1788cf968 RCX: ffffffff8ac34f40
RDX: ffffb0a081bd0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8ee1788cf9a8 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8edff7ea9800 R11: ffff8edff9d3b000 R12: 0000000000000020
R13: ffff8ee07cbdf680 R14: ffffffff8a4eb239 R15: ffff8ee1788cf920
FS:  0000000000000000(0000) GS:ffff8ee17fc40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f736b7154b1 CR3: 000000031fa10000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc0179632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8a4ec580>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8a4e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8a4ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8a353809>] __blk_run_queue+0x39/0x50
[<ffffffff8a3538ca>] queue_unplugged+0x2a/0xa0
[<ffffffff8a357208>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8a3573e2>] blk_queue_bio+0x182/0x400
[<ffffffff8a355497>] generic_make_request+0x147/0x380
[<ffffffff8a28b53b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff8a355740>] submit_bio+0x70/0x150
[<ffffffffc03db283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc03dd2e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc03dcc72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc03dd2e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc03ddbd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc040ca40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc03ddbd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc040ccca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc040ca40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff8a0c5da1>] kthread+0xd1/0xe0
[<ffffffff8a0c5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff8a795df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8a0c5cd0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #179516
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01893be
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc01ab71e
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01d62e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffad4e2990
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffad4ec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffffad3527a9
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffffad35286a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffad3560a8
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffffad356282
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffffad354437
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffffad28aa5b
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffffad3546e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc04002f3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0402350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc0401ce2
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0402350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0402c40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04319a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0402c40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc0431c1a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x27a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04319a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffffad0c6691
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffffad0c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffffad792d37
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffffad0c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [xfsaild/dm-0:449]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper cryptd vmw_balloon pcspkr joydev sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_piix drm e1000 libata mptspi scsi_transport_spi mptscsih mptbase
drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 2 PID: 449 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 06/22/2012
task: ffff8a6ef56a8000 ti: ffff8a6ef5494000 task.ti: ffff8a6ef5494000
RIP: 0010:[<ffffffffc01893be>]  [<ffffffffc01893be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8a6ef5497940  EFLAGS: 00000246
RAX: ffffa43500f80000 RBX: ffff8a6f35235a00 RCX: 0000000000000000
RDX: ffff8a7074a74800 RSI: ffff8a707463f000 RDI: 000000000000000e
RBP: ffff8a6ef5497950 R08: 0000000000000002 R09: 00000000351cbfff
R10: ffff8a6ef546ac00 R11: ffff8a70780d16c0 R12: 0000000100000001
R13: ffff8a702138ac00 R14: ffff8a6ef54978b4 R15: ffffffffad35b0cb
FS:  0000000000000000(0000) GS:ffff8a707fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7eb05d62a8 CR3: 00000001b7f00000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc01ab71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01d62e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffad4e2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffad4ec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffad3527a9>] __blk_run_queue+0x39/0x50
[<ffffffffad35286a>] queue_unplugged+0x2a/0xa0
[<ffffffffad3560a8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffad356282>] blk_queue_bio+0x182/0x400
[<ffffffffad354437>] generic_make_request+0x147/0x380
[<ffffffffad28aa5b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffffad3546e0>] submit_bio+0x70/0x150
[<ffffffffc04002f3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc0402350>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0401ce2>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc0402350>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0402c40>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc04319a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0402c40>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0431c1a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc04319a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffad0c6691>] kthread+0xd1/0xe0
[<ffffffffad0c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffad792d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffad0c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 19
Complete report #124786
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff879883c5
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc029e632
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff876eb030
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff876e2920
Build id: 3.10.0-1127.10.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff876ec04c
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff87552769
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff8755282a
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff87556068
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff87556242
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff875543f7
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff8748aa6b
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff875546a0
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc04bf2f3
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc04c1350
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc04c0ce2
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc04c1350
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc04c1c40
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04f09a0
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc04c1c40
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc04f0c1a
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x27a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc04f09a0
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff872c6691
Build id: 3.10.0-1127.10.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff872c65c0
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff87992d37
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff872c65c0
Build id: 3.10.0-1127.10.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 32s! [xfsaild/dm-1:404]
Modules linked in: event_collector_1_10_9431(POE) vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd pcspkr sg joydev vmw_balloon vmw_vmci parport_pc parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix serio_raw libata vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 404 Comm: xfsaild/dm-1 Kdump: loaded Tainted: P           OEL ------------ T 3.10.0-1127.10.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 10/22/2013
task: ffff967c778241c0 ti: ffff967c78998000 task.ti: ffff967c78998000
RIP: 0010:[<ffffffff879883c5>]  [<ffffffff879883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff967c7899b9a8  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff967c7899b968 RCX: ffffffff87e2f0a0
RDX: ffffaf25c0410000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff967c7899b9a8 R08: ffffffffffffffff R09: 00000000360dbfff
R10: ffff967c36140c00 R11: ffff967c36acb000 R12: 0000000000000020
R13: ffff967c11445d80 R14: ffffffff876e9ce9 R15: ffff967c7899b920
FS:  0000000000000000(0000) GS:ffff967c7fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f33ffbc8000 CR3: 000000005e94e000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc029e632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff876eb030>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff876e2920>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff876ec04c>] scsi_request_fn+0x4ac/0x680
[<ffffffff87552769>] __blk_run_queue+0x39/0x50
[<ffffffff8755282a>] queue_unplugged+0x2a/0xa0
[<ffffffff87556068>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff87556242>] blk_queue_bio+0x182/0x400
[<ffffffff875543f7>] generic_make_request+0x147/0x380
[<ffffffff8748aa6b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff875546a0>] submit_bio+0x70/0x150
[<ffffffffc04bf2f3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc04c1350>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc04c0ce2>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc04c1350>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc04c1c40>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc04f09a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc04c1c40>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc04f0c1a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc04f09a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff872c6691>] kthread+0xd1/0xe0
[<ffffffff872c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff87992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff872c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #101458
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8d9883c5
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc00d0632
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8d6eb060
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8d6e2950
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff8d6ec07c
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8d552769
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff8d55282a
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff8d556068
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff8d556242
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff8d5543f7
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff8d48aa6b
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff8d5546a0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc033f2f3
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0341350
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc0340ce2
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0341350
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0341c40
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03709a0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0341c40
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc0370c1a
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x27a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03709a0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff8d2c6691
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff8d2c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff8d992d37
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff8d2c65c0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [xfsaild/dm-0:536]
Modules linked in: tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat vmw_vsock_vmci_transport vsock nf_conntrack xt_CHECKSUM iptable_mangle sunrpc coretemp ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr cryptd joydev sg vmw_balloon parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm sd_mod libahci ata_piix crc_t10dif crct10dif_generic libata vmxnet3 crct10dif_pclmul crct10dif_common crc32c_intel nfit serio_raw libnvdimm vmw_pvscsi
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 536 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.13.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8f3142a6c1c0 ti: ffff8f3328360000 task.ti: ffff8f3328360000
RIP: 0010:[<ffffffff8d9883c5>]  [<ffffffff8d9883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8f33283639a8  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8f3328363968 RCX: ffffffff8de34e40
RDX: ffffa27a01b90000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8f33283639a8 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8f332a7c3800 R11: ffff8f332a678800 R12: 0000000000000020
R13: ffff8f3167d98f00 R14: ffffffff8d6e9d19 R15: ffff8f3328363920
FS:  0000000000000000(0000) GS:ffff8f333fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffc07aaa618 CR3: 000000042b660000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00d0632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8d6eb060>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8d6e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d6ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d552769>] __blk_run_queue+0x39/0x50
[<ffffffff8d55282a>] queue_unplugged+0x2a/0xa0
[<ffffffff8d556068>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8d556242>] blk_queue_bio+0x182/0x400
[<ffffffff8d5543f7>] generic_make_request+0x147/0x380
[<ffffffff8d48aa6b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff8d5546a0>] submit_bio+0x70/0x150
[<ffffffffc033f2f3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc0341350>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0340ce2>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc0341350>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0341c40>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc03709a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0341c40>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0370c1a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc03709a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff8d2c6691>] kthread+0xd1/0xe0
[<ffffffff8d2c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d992d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8d2c65c0>] ? insert_kthread_work+0x40/0x40
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #73649
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb5b817c5
Build id: 3.10.0-1062.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc02fb632
Build id: 3.10.0-1062.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb58e4e90
Build id: 3.10.0-1062.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb58dc890
Build id: 3.10.0-1062.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffb58e5efc
Build id: 3.10.0-1062.el7.x86_64 		0x4cc
6
__blk_run_queue
vmlinux 0xffffffffb574cf59
Build id: 3.10.0-1062.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffffb574d01a
Build id: 3.10.0-1062.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffb5750858
Build id: 3.10.0-1062.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffffb5750a32
Build id: 3.10.0-1062.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffffb574ebf7
Build id: 3.10.0-1062.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffffb568553b
Build id: 3.10.0-1062.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffffb574eea0
Build id: 3.10.0-1062.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc059f003
Build id: 3.10.0-1062.el7.x86_64 		0x2f3
14
? xfs_buf_submit
xfs 0xffffffffc05a0b4a
Build id: 3.10.0-1062.el7.x86_64 		0x1a
15
__xfs_buf_submit
xfs 0xffffffffc05a08c1
Build id: 3.10.0-1062.el7.x86_64 		0x61
16
xfs_buf_submit
xfs 0xffffffffc05a0b4a
Build id: 3.10.0-1062.el7.x86_64 		0x1a
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc05a0d9c
Build id: 3.10.0-1062.el7.x86_64 		0x22c
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05a1930
Build id: 3.10.0-1062.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc05d03d0
Build id: 3.10.0-1062.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05a1930
Build id: 3.10.0-1062.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc05d064a
Build id: 3.10.0-1062.el7.x86_64 		0x27a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc05d03d0
Build id: 3.10.0-1062.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffffb54c50d1
Build id: 3.10.0-1062.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffffb54c5000
Build id: 3.10.0-1062.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffffb5b8bd37
Build id: 3.10.0-1062.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffffb54c5000
Build id: 3.10.0-1062.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [xfsaild/dm-6:1514]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_11110(E) falcon_lsm_pinned_10807(E) binfmt_misc falcon_lsm_pinned_10703(E) falcon_lsm_pinned_10404(E) vfat fat isofs ext4 mbcache jbd2 falcon_lsm_pinned_9510(E) falcon_lsm_pinned_8705(E) bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect sysimgblt crc32c_intel fb_sys_fops serio_raw ttm ahci vmxnet3 ata_piix drm libahci vmw_pvscsi libata nfit libnvdimm
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: falcon_lsm_serviceable]
CPU: 2 PID: 1514 Comm: xfsaild/dm-6 Tainted: P            EL ------------   3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9d22a96bd230 ti: ffff9d22a46d0000 task.ti: ffff9d22a46d0000
RIP: 0010:[<ffffffffb5b817c5>]  [<ffffffffb5b817c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9d22a46d3998  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff9d22a46d3958 RCX: ffff9d22bf739060
RDX: ffffb770c1ba0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9d22a46d3998 R08: 0000000000000000 R09: ffff9d21db6cdf00
R10: ffff9d22acaa50d8 R11: ffff9d22aaeea000 R12: 0000000000000020
R13: ffff9d21777a4780 R14: ffffffffb58e3be9 R15: ffff9d22a46d3910
FS:  0000000000000000(0000) GS:ffff9d22bfd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efe11b71000 CR3: 00000003a2a16000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc02fb632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb58e4e90>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb58dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb58e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffb574cf59>] __blk_run_queue+0x39/0x50
[<ffffffffb574d01a>] queue_unplugged+0x2a/0xa0
[<ffffffffb5750858>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb5750a32>] blk_queue_bio+0x182/0x400
[<ffffffffb574ebf7>] generic_make_request+0x147/0x380
[<ffffffffb568553b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffffb574eea0>] submit_bio+0x70/0x150
[<ffffffffc059f003>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc05a0b4a>] ? xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc05a08c1>] __xfs_buf_submit+0x61/0x210 [xfs]
[<ffffffffc05a0b4a>] xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc05a0d9c>] xfs_buf_delwri_submit_buffers+0x22c/0x240 [xfs]
[<ffffffffc05a1930>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc05d03d0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc05a1930>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc05d064a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc05d03d0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffb54c50d1>] kthread+0xd1/0xe0
[<ffffffffb54c5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffb5b8bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb54c5000>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #61735
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc038c3be
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03db71e
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03a72e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff984e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff984ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff983536d9
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff9835379a
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff98356fd8
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff983571b2
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff98355367
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff9828b71b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff98355610
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc0622283
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc06242e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc0623c72
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc06242e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0624bd0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0653a40
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0624bd0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc0653cca
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x28a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc0653a40
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff980c5e71
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff980c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff98793df7
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff980c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 31s! [xfsaild/dm-0:588]
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix serio_raw mptspi scsi_transport_spi libata mptscsih e1000 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 588 Comm: xfsaild/dm-0 Kdump: loaded Tainted: P        W  OEL ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff9cf2f394b180 ti: ffff9cf0f5da4000 task.ti: ffff9cf0f5da4000
RIP: 0010:[<ffffffffc038c3be>]  [<ffffffffc038c3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9cf0f5da7940  EFLAGS: 00000246
RAX: ffffb56f40f40000 RBX: ffff9cf2a8dc3200 RCX: 0000000000000001
RDX: ffff9cf2f5a2aa00 RSI: ffff9cf2313c4000 RDI: 000000000000000e
RBP: ffff9cf0f5da7950 R08: 0000000000000002 R09: 00000001d9f049ff
R10: ffff9cf2f4872000 R11: ffff9cf15d1fc540 R12: 0000000100000001
R13: ffff9cf28aef8a80 R14: ffff9cf0f5da78b4 R15: ffffffff9835bffb
FS:  0000000000000000(0000) GS:ffff9cf2ffd40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fec91053850 CR3: 000000017f610000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc03db71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a72e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff984e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff984ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff983536d9>] __blk_run_queue+0x39/0x50
[<ffffffff9835379a>] queue_unplugged+0x2a/0xa0
[<ffffffff98356fd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff983571b2>] blk_queue_bio+0x182/0x400
[<ffffffff98355367>] generic_make_request+0x147/0x380
[<ffffffff9828b71b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff98355610>] submit_bio+0x70/0x150
[<ffffffffc0622283>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc06242e0>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0623c72>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc06242e0>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc0624bd0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0653a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0624bd0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc0653cca>] xfsaild+0x28a/0x780 [xfs]
[<ffffffffc0653a40>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff980c5e71>] kthread+0xd1/0xe0
[<ffffffff980c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff98793df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff980c5da0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 a3 39
Complete report #41439
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00d63be
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc010d71e
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00f12e0
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb92decd0
Build id: 3.10.0-1062.18.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffffb92e833c
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x4cc
6
__blk_run_queue
vmlinux 0xffffffffb914f289
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffffb914f34a
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffb9152b98
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffffb9152d72
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffffb9150f37
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffffb908793b
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffffb91511e0
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc0331003
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x2f3
14
? xfs_buf_submit
xfs 0xffffffffc0332b4a
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x1a
15
__xfs_buf_submit
xfs 0xffffffffc03328c1
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x61
16
xfs_buf_submit
xfs 0xffffffffc0332b4a
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x1a
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc0332d9c
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x22c
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0333930
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03623b0
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc0333930
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc036262a
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x27a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc03623b0
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffffb8ec6321
Build id: 3.10.0-1062.18.1.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffffb8ec6250
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffffb958dd37
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffffb8ec6250
Build id: 3.10.0-1062.18.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 25s! [xfsaild/dm-0:83486]
Modules linked in: tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter binfmt_misc vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect sysimgblt crc32c_intel fb_sys_fops ttm
serio_raw ata_piix drm mptspi scsi_transport_spi libata mptscsih vmxnet3 mptbase drm_panel_orientation_quirks
CPU: 2 PID: 83486 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff8bac33aa20e0 ti: ffff8babf4d0c000 task.ti: ffff8babf4d0c000
RIP: 0010:[<ffffffffc00d63be>]  [<ffffffffc00d63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8babf4d0f930  EFLAGS: 00000246
RAX: ffffab2501940000 RBX: ffff8bace0501f20 RCX: 0000000000000031
RDX: ffff8bafe844ec00 RSI: ffff8bafe9146000 RDI: 000000000000000e
RBP: ffff8babf4d0f940 R08: 0000000000000003 R09: 000000040c2aebff
R10: ffff8bafe770d0d8 R11: ffff8bac5ed7c8c0 R12: 0000000100000002
R13: ffff8bac64349200 R14: ffff8babf4d0f8a4 R15: ffffffffb9157aeb
FS:  0000000000000000(0000) GS:ffff8bafed680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dee1c CR3: 00000004264ba000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc010d71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00f12e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb92decd0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb92e833c>] scsi_request_fn+0x4cc/0x680
[<ffffffffb914f289>] __blk_run_queue+0x39/0x50
[<ffffffffb914f34a>] queue_unplugged+0x2a/0xa0
[<ffffffffb9152b98>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb9152d72>] blk_queue_bio+0x182/0x400
[<ffffffffb9150f37>] generic_make_request+0x147/0x380
[<ffffffffb908793b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffffb91511e0>] submit_bio+0x70/0x150
[<ffffffffc0331003>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc0332b4a>] ? xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc03328c1>] __xfs_buf_submit+0x61/0x210 [xfs]
[<ffffffffc0332b4a>] xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc0332d9c>] xfs_buf_delwri_submit_buffers+0x22c/0x240 [xfs]
[<ffffffffc0333930>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc03623b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc0333930>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc036262a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc03623b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffb8ec6321>] kthread+0xd1/0xe0
[<ffffffffb8ec6250>] ? insert_kthread_work+0x40/0x40
[<ffffffffb958dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb8ec6250>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 0e
Complete report #22361
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff82783045
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc02f4632
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff824e7110
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff824deb10
Build id: 3.10.0-1062.12.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff824e817c
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x4cc
6
__blk_run_queue
vmlinux 0xffffffff8234f0c9
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff8234f18a
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff823529d8
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff82352bb2
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff82350d77
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff822876fb
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff82351020
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc055c003
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x2f3
14
? xfs_buf_submit
xfs 0xffffffffc055db4a
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x1a
15
__xfs_buf_submit
xfs 0xffffffffc055d8c1
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x61
16
xfs_buf_submit
xfs 0xffffffffc055db4a
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x1a
17
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc055dd9c
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x22c
18
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc055e930
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x10
19
? xfs_trans_ail_cursor_first
xfs 0xffffffffc058d3b0
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x90
20
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc055e930
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x10
21
xfsaild
xfs 0xffffffffc058d62a
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x27a
22
? xfs_trans_ail_cursor_first
xfs 0xffffffffc058d3b0
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x90
23
kthread
vmlinux 0xffffffff820c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64 		0xd1
24
? insert_kthread_work
vmlinux 0xffffffff820c6120
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x40
25
ret_from_fork_nospec_begin
vmlinux 0xffffffff8278dd37
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x21
26
? insert_kthread_work
vmlinux 0xffffffff820c6120
Build id: 3.10.0-1062.12.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [xfsaild/dm-0:649]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack ip_tables br_netfilter overlay(T) tun devlink bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev vmw_balloon glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 binfmt_misc xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod vmwgfx crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci nfit crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel ata_piix serio_raw libata vmxnet3 vmw_pvscsi
drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 1 PID: 649 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8bf376e4b150 ti: ffff8bf37763c000 task.ti: ffff8bf37763c000
RIP: 0010:[<ffffffff82783045>]  [<ffffffff82783045>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8bf37763f998  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff8bf37763f958 RCX: ffff8be4e96db060
RDX: ffff9af4069b0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8bf37763f998 R08: 0000000000000000 R09: ffff8be7bd6b5200
R10: ffff8bf37a3370d8 R11: ffff8bebceda6800 R12: 0000000000000020
R13: ffff8bedc8a27680 R14: ffffffff824e5e69 R15: ffff8bf37763f910
FS:  0000000000000000(0000) GS:ffff8bf3bfc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000043f3dff28 CR3: 0000000fa6296000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc02f4632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff824e7110>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff824deb10>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff824e817c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8234f0c9>] __blk_run_queue+0x39/0x50
[<ffffffff8234f18a>] queue_unplugged+0x2a/0xa0
[<ffffffff823529d8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff82352bb2>] blk_queue_bio+0x182/0x400
[<ffffffff82350d77>] generic_make_request+0x147/0x380
[<ffffffff822876fb>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff82351020>] submit_bio+0x70/0x150
[<ffffffffc055c003>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc055db4a>] ? xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc055d8c1>] __xfs_buf_submit+0x61/0x210 [xfs]
[<ffffffffc055db4a>] xfs_buf_submit+0x1a/0x40 [xfs]
[<ffffffffc055dd9c>] xfs_buf_delwri_submit_buffers+0x22c/0x240 [xfs]
[<ffffffffc055e930>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc058d3b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc055e930>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc058d62a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc058d3b0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff820c61f1>] kthread+0xd1/0xe0
[<ffffffff820c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff8278dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff820c6120>] ? insert_kthread_work+0x40/0x40
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #4608
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00db3be
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc014a71e
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00f62e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9aee2990
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff9aeec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff9ad527a9
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff9ad5286a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff9ad560a8
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff9ad56282
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff9ad54437
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff9ac8aa5b
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff9ad546e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc035a2f3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc035c350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc035bce2
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc035c350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc035cc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc038b9a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc035cc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc038bc1a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x27a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc038b9a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff9aac6691
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff9aac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff9b192d37
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff9aac65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 28s! [xfsaild/dm-0:421]
Modules linked in: vmfsrtx_export(POE) vmfsrtx_bridge(POE) vmfsrtx_bridge_helper(OE) vmfsrtx_dev(POE) vmfsrtx_dev_helper(POE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink fuse ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev parport_pc vmw_balloon pcspkr joydev sg parport vmw_vmci i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic
crct10dif_common sr_mod cdrom ata_generic pata_acpi vmwgfx crc32c_intel drm_kms_helper ata_piix serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm libata mptspi drm scsi_transport_spi nfit mptscsih libnvdimm vmxnet3 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmfsrtx_export]
CPU: 0 PID: 421 Comm: xfsaild/dm-0 Kdump: loaded Tainted: P           OEL ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8c0db5001070 ti: ffff8c0db4af0000 task.ti: ffff8c0db4af0000
RIP: 0010:[<ffffffffc00db3be>]  [<ffffffffc00db3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8c0db4af3940  EFLAGS: 00000246
RAX: ffffb3f4807c0000 RBX: ffff8c0decc3c100 RCX: 0000000000000019
RDX: ffff8c0db5829e00 RSI: ffff8c0db58d3000 RDI: 000000000000000e
RBP: ffff8c0db4af3950 R08: 0000000000000002 R09: 00000000bae23fff
R10: ffff8c0db5835400 R11: ffff8c0eb6dd39c0 R12: 0000000100000001
R13: ffff8c0da16a8480 R14: ffff8c0db4af38b4 R15: ffffffff9ad5b0cb
FS:  0000000000000000(0000) GS:ffff8c0ebae00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7c9c001e48 CR3: 00000000a66be000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc014a71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00f62e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9aee2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9aeec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9ad527a9>] __blk_run_queue+0x39/0x50
[<ffffffff9ad5286a>] queue_unplugged+0x2a/0xa0
[<ffffffff9ad560a8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9ad56282>] blk_queue_bio+0x182/0x400
[<ffffffff9ad54437>] generic_make_request+0x147/0x380
[<ffffffff9ac8aa5b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff9ad546e0>] submit_bio+0x70/0x150
[<ffffffffc035a2f3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc035c350>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc035bce2>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc035c350>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc035cc40>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc038b9a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc035cc40>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc038bc1a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc038b9a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff9aac6691>] kthread+0xd1/0xe0
[<ffffffff9aac65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9b192d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9aac65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 0e
Complete report #4471
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03603be
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc037b71e
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03432e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff85ce2990
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xb0
5
scsi_request_fn
vmlinux 0xffffffff85cec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x4ac
6
__blk_run_queue
vmlinux 0xffffffff85b527a9
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x39
7
queue_unplugged
vmlinux 0xffffffff85b5286a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff85b560a8
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x1d8
9
blk_queue_bio
vmlinux 0xffffffff85b56282
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x182
10
generic_make_request
vmlinux 0xffffffff85b54437
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x147
11
? __bio_add_page
vmlinux 0xffffffff85a8aa5b
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x20b
12
submit_bio
vmlinux 0xffffffff85b546e0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x70
13
_xfs_buf_ioapply
xfs 0xffffffffc05c92f3
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x2f3
14
? xfs_buf_delwri_submit_buffers
xfs 0xffffffffc05cb350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
15
__xfs_buf_submit
xfs 0xffffffffc05cace2
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x72
16
xfs_buf_delwri_submit_buffers
xfs 0xffffffffc05cb350
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x130
17
? xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05cbc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
18
? xfs_trans_ail_cursor_first
xfs 0xffffffffc05fa9a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
19
xfs_buf_delwri_submit_nowait
xfs 0xffffffffc05cbc40
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x10
20
xfsaild
xfs 0xffffffffc05fac1a
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x27a
21
? xfs_trans_ail_cursor_first
xfs 0xffffffffc05fa9a0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x90
22
kthread
vmlinux 0xffffffff858c6691
Build id: 3.10.0-1127.19.1.el7.x86_64 		0xd1
23
? insert_kthread_work
vmlinux 0xffffffff858c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40
24
ret_from_fork_nospec_begin
vmlinux 0xffffffff85f92d37
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x21
25
? insert_kthread_work
vmlinux 0xffffffff858c65c0
Build id: 3.10.0-1127.19.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [xfsaild/dm-0:402]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper vmw_balloon cryptd joydev pcspkr sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod
vmwgfx crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix drm crct10dif_pclmul libnvdimm crct10dif_common libata crc32c_intel mptspi serio_raw scsi_transport_spi mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 402 Comm: xfsaild/dm-0 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9f06747dc1c0 ti: ffff9f06747bc000 task.ti: ffff9f06747bc000
RIP: 0010:[<ffffffffc03603be>]  [<ffffffffc03603be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9f06747bf940  EFLAGS: 00000246
RAX: ffffb58700840000 RBX: ffff9f059746ff60 RCX: 000000000000000c
RDX: ffff9f0575f1b400 RSI: ffff9f06744cf000 RDI: 000000000000000e
RBP: ffff9f06747bf950 R08: 0000000000000003 R09: 000000013454cfff
R10: ffff9f06745e6400 R11: ffff9f05fa3eba80 R12: 0000000100000004
R13: ffff9f05f7c0e780 R14: ffff9f06747bf8b4 R15: ffffffff85b5b0cb
FS:  0000000000000000(0000) GS:ffff9f067fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcbd7248000 CR3: 0000000077570000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc037b71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03432e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff85ce2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff85cec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff85b527a9>] __blk_run_queue+0x39/0x50
[<ffffffff85b5286a>] queue_unplugged+0x2a/0xa0
[<ffffffff85b560a8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff85b56282>] blk_queue_bio+0x182/0x400
[<ffffffff85b54437>] generic_make_request+0x147/0x380
[<ffffffff85a8aa5b>] ? __bio_add_page+0x20b/0x2b0
[<ffffffff85b546e0>] submit_bio+0x70/0x150
[<ffffffffc05c92f3>] _xfs_buf_ioapply+0x2f3/0x460 [xfs]
[<ffffffffc05cb350>] ? xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc05cace2>] __xfs_buf_submit+0x72/0x250 [xfs]
[<ffffffffc05cb350>] xfs_buf_delwri_submit_buffers+0x130/0x240 [xfs]
[<ffffffffc05cbc40>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc05fa9a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffffc05cbc40>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[<ffffffffc05fac1a>] xfsaild+0x27a/0x760 [xfs]
[<ffffffffc05fa9a0>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[<ffffffff858c6691>] kthread+0xd1/0xe0
[<ffffffff858c65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff85f92d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff858c65c0>] ? insert_kthread_work+0x40/0x40
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 e3 36