Info

Permalink
Function
mpt_put_msg_frame
First occurrence
2020-10-17
Last occurrence
2022-04-29
State
NEW
Components
kernel
Quality
-47

Statistics

Operating system Count
CentOS 7 205
Architecture Count
x86_64 205
Related packages Count
kernel 205
 0:3.10.0-1062.9.1.el7 55
 0:3.10.0-1160.62.1.el7 20
 0:3.10.0-1160.24.1.el7 20
 0:3.10.0-1160.118.1.el7 20
 0:3.10.0-1062.el7 19
 0:3.10.0-1160.el7 13
 0:3.10.0-1160.42.2.el7 13
 0:3.10.0-1127.el7 12
 0:3.10.0-957.el7 6
 0:3.10.0-957.27.2.el7 6
 0:3.10.0-1160.53.1.el7 3
 0:3.10.0-957.21.2.el7 2
 0:3.10.0-1127.18.2.el7 2
 0:3.10.0-1160.15.2.el7 2
 0:3.10.0-957.12.1.el7 2
 0:3.10.0-1160.49.1.el7 2
 0:3.10.0-957.21.3.el7 1
 0:3.10.0-1127.10.1.el7 1
 0:3.10.0-1160.6.1.el7 1
 0:3.10.0-1160.80.1.el7 1
 0:3.10.0-1160.36.2.el7 1
 0:3.10.0-1160.45.1.el7 1
 0:3.10.0-1160.83.1.el7 1
 0:3.10.0-1160.88.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1140583
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03f63be
Build id: 3.10.0-1160.62.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc042871e
Build id: 3.10.0-1160.62.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc047a2e0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff896e4640
Build id: 3.10.0-1160.62.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff896edd6c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff895789ce
Build id: 3.10.0-1160.62.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff89553cf9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff89557ac3
Build id: 3.10.0-1160.62.1.el7.x86_64
0x3b3
9
? blk_flush_plug_list
vmlinux 0xffffffff89557710
Build id: 3.10.0-1160.62.1.el7.x86_64
0x230
10
flt_make_request_fn
involflt 0xffffffffc0316d5e
Build id: 3.10.0-1160.62.1.el7.x86_64
0x12e
11
generic_make_request
vmlinux 0xffffffff895559a7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x147
12
? radix_tree_tag_clear
vmlinux 0xffffffff8958cabe
Build id: 3.10.0-1160.62.1.el7.x86_64
0x9e
13
submit_bio
vmlinux 0xffffffff89555c50
Build id: 3.10.0-1160.62.1.el7.x86_64
0x70
14
? __test_set_page_writeback
vmlinux 0xffffffff893c9ef4
Build id: 3.10.0-1160.62.1.el7.x86_64
0x54
15
xfs_submit_ioend.isra.12
xfs 0xffffffffc0611f31
Build id: 3.10.0-1160.62.1.el7.x86_64
0x61
16
xfs_do_writepage
xfs 0xffffffffc0612724
Build id: 3.10.0-1160.62.1.el7.x86_64
0x494
17
write_cache_pages
vmlinux 0xffffffff893ca96c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21c
18
? complete
vmlinux 0xffffffff892d345a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x4a
19
? xfs_vm_writepages
xfs 0xffffffffc0612290
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa0
20
xfs_vm_writepages
xfs 0xffffffffc061225b
Build id: 3.10.0-1160.62.1.el7.x86_64
0x6b
21
do_writepages
vmlinux 0xffffffff893cb9c1
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21
22
__filemap_fdatawrite_range
vmlinux 0xffffffff893bfe45
Build id: 3.10.0-1160.62.1.el7.x86_64
0x65
23
filemap_write_and_wait_range
vmlinux 0xffffffff893bff91
Build id: 3.10.0-1160.62.1.el7.x86_64
0x41
24
xfs_file_fsync
xfs 0xffffffffc061cd66
Build id: 3.10.0-1160.62.1.el7.x86_64
0x66
25
do_fsync
vmlinux 0xffffffff89484215
Build id: 3.10.0-1160.62.1.el7.x86_64
0x55
26
? __audit_syscall_exit
vmlinux 0xffffffff8933ecc6
Build id: 3.10.0-1160.62.1.el7.x86_64
0x1f6
27
SyS_fsync
vmlinux 0xffffffff894844c0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x10
28
system_call_fastpath
vmlinux 0xffffffff89999f92
Build id: 3.10.0-1160.62.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [mysqld:32733]
Modules linked in: ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack vmw_vsock_vmci_transport vsock xt_multiport iptable_filter sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev vmw_balloon cryptd pcspkr sg joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod cdrom crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi scsi_transport_spi nfit libnvdimm mptscsih crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_piix mptbase libata vmxnet3 drm_panel_orientation_quirks involflt(OE) dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 32733 Comm: mysqld Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff971c01fd4200 ti: ffff971c5a96c000 task.ti: ffff971c5a96c000
RIP: 0010:[<ffffffffc03f63be>]  [<ffffffffc03f63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff971c5a96f8b0  EFLAGS: 00000246
RAX: ffffb88440e80000 RBX: ffff971e36b1ae60 RCX: 0000000000000005
RDX: ffff971e1fa36e00 RSI: ffff971e35c34000 RDI: 000000000000000e
RBP: ffff971c5a96f8c0 R08: 0000000000000003 R09: 00000001ca7d1fff
R10: ffff971e1e438c00 R11: ffff971d01fa3c00 R12: 0000000100000004
R13: ffff971d0aa08a80 R14: ffff971c5a96f824 R15: ffffffff8955c6db
FS:  00007f2da7630700(0000) GS:ffff971e3fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000002c70711 CR3: 000000010327a000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc042871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc047a2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff896e4640>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff896edd6c>] scsi_request_fn+0x4ac/0x680
[<ffffffff895789ce>] ? deadline_add_request+0x6e/0x90
[<ffffffff89553cf9>] __blk_run_queue+0x39/0x50
[<ffffffff89557ac3>] blk_queue_bio+0x3b3/0x400
[<ffffffff89557710>] ? blk_flush_plug_list+0x230/0x230
[<ffffffffc0316d5e>] flt_make_request_fn+0x12e/0xe30 [involflt]
[<ffffffff895559a7>] generic_make_request+0x147/0x380
[<ffffffff8958cabe>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff89555c50>] submit_bio+0x70/0x150
[<ffffffff893c9ef4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0611f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0612724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff893ca96c>] write_cache_pages+0x21c/0x470
[<ffffffff892d345a>] ? complete+0x4a/0x60
[<ffffffffc0612290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc061225b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff893cb9c1>] do_writepages+0x21/0x50
[<ffffffff893bfe45>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff893bff91>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc061cd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff89484215>] do_fsync+0x55/0x80
[<ffffffff8933ecc6>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffff894844c0>] SyS_fsync+0x10/0x20
[<ffffffff89999f92>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 40 
Complete report #53130
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8f5883c5
Build id: 3.10.0-1127.10.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc01f3632
Build id: 3.10.0-1127.10.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8f2eb030
Build id: 3.10.0-1127.10.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8f2e2920
Build id: 3.10.0-1127.10.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8f2ec04c
Build id: 3.10.0-1127.10.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8f17716e
Build id: 3.10.0-1127.10.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff8f152769
Build id: 3.10.0-1127.10.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff8f156473
Build id: 3.10.0-1127.10.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff8f1543f7
Build id: 3.10.0-1127.10.1.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8f18b25e
Build id: 3.10.0-1127.10.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff8f1546a0
Build id: 3.10.0-1127.10.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff8efc9744
Build id: 3.10.0-1127.10.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0410fb1
Build id: 3.10.0-1127.10.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc04117a4
Build id: 3.10.0-1127.10.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffff8efca1bc
Build id: 3.10.0-1127.10.1.el7.x86_64
0x21c
16
? complete
vmlinux 0xffffffff8eed3e9a
Build id: 3.10.0-1127.10.1.el7.x86_64
0x4a
17
? xfs_vm_writepages
xfs 0xffffffffc0411310
Build id: 3.10.0-1127.10.1.el7.x86_64
0xa0
18
xfs_vm_writepages
xfs 0xffffffffc04112db
Build id: 3.10.0-1127.10.1.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffff8efcb211
Build id: 3.10.0-1127.10.1.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffff8efbfc35
Build id: 3.10.0-1127.10.1.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffff8efbfd81
Build id: 3.10.0-1127.10.1.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc041bdd6
Build id: 3.10.0-1127.10.1.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffff8f082f67
Build id: 3.10.0-1127.10.1.el7.x86_64
0x67
24
SyS_fsync
vmlinux 0xffffffff8f083250
Build id: 3.10.0-1127.10.1.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffff8f592ed2
Build id: 3.10.0-1127.10.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [kworker/u4:1:6850]
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [mysqld:1802]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod crc_t10dif
ttm crct10dif_generic drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel serio_raw libata nfit vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 1802 Comm: mysqld Kdump: loaded Tainted: G             L ------------   3.10.0-1127.10.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9326f3cae2a0 ti: ffff9326f60c4000 task.ti: ffff9326f60c4000
RIP: 0010:[<ffffffff8f5883c5>]  [<ffffffff8f5883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9326f60c79d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff8f1b1473 RCX: ffff9326f71b0060
RDX: ffffae0240ed0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9326f60c79d0 R08: 0000000000000000 R09: ffff932542340200
R10: ffff9326f3b70000 R11: ffff9324f6504000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff93269c061c80 R15: ffffffff8f2e9ce9
FS:  00007f48861e3700(0000) GS:ffff9326ffd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4e1680d000 CR3: 00000000b8044000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc01f3632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8f2eb030>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8f2e2920>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8f2ec04c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8f17716e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8f152769>] __blk_run_queue+0x39/0x50
[<ffffffff8f156473>] blk_queue_bio+0x3b3/0x400
[<ffffffff8f1543f7>] generic_make_request+0x147/0x380
[<ffffffff8f18b25e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff8f1546a0>] submit_bio+0x70/0x150
[<ffffffff8efc9744>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0410fb1>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc04117a4>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff8efca1bc>] write_cache_pages+0x21c/0x470
[<ffffffff8eed3e9a>] ? complete+0x4a/0x60
[<ffffffffc0411310>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc04112db>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8efcb211>] do_writepages+0x21/0x50
[<ffffffff8efbfc35>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8efbfd81>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc041bdd6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8f082f67>] do_fsync+0x67/0xb0
[<ffffffff8f083250>] SyS_fsync+0x10/0x20
[<ffffffff8f592ed2>] system_call_fastpath+0x25/0x2a
Complete report #1045576
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc032d44e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc036474e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03b02f0
Build id: 3.10.0-1160.118.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa6f03030
Build id: 3.10.0-1160.118.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa6f0cbdc
Build id: 3.10.0-1160.118.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffa6d8e21e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffa6d687d9
Build id: 3.10.0-1160.118.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffa6d6c703
Build id: 3.10.0-1160.118.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffa6d6a537
Build id: 3.10.0-1160.118.1.el7.x86_64
0x157
10
? radix_tree_tag_clear
vmlinux 0xffffffffa6da2bbe
Build id: 3.10.0-1160.118.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffa6d6a7e0
Build id: 3.10.0-1160.118.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffa6bd4fb4
Build id: 3.10.0-1160.118.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc05cd341
Build id: 3.10.0-1160.118.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc05cdb54
Build id: 3.10.0-1160.118.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffa6bd5a5c
Build id: 3.10.0-1160.118.1.el7.x86_64
0x21c
16
? put_prev_entity
vmlinux 0xffffffffa6aec651
Build id: 3.10.0-1160.118.1.el7.x86_64
0x31
17
? xfs_vm_writepages
xfs 0xffffffffc05cd6c0
Build id: 3.10.0-1160.118.1.el7.x86_64
0xb0
18
xfs_vm_writepages
xfs 0xffffffffc05cd67b
Build id: 3.10.0-1160.118.1.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffffa6bd6b01
Build id: 3.10.0-1160.118.1.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffffa6bcaba5
Build id: 3.10.0-1160.118.1.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffffa6bcad11
Build id: 3.10.0-1160.118.1.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc05d8486
Build id: 3.10.0-1160.118.1.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffffa6c93195
Build id: 3.10.0-1160.118.1.el7.x86_64
0x55
24
SyS_fsync
vmlinux 0xffffffffa6c93460
Build id: 3.10.0-1160.118.1.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffffa71c539a
Build id: 3.10.0-1160.118.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 198s! [systemd-journal:24407]
Modules linked in: binfmt_misc cts 8021q garp mrp ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache falcon_lsm_serviceable(PE) falcon_nf_netcontain(E) falcon_kal(E) falcon_lsm_pinned_17005(E) nf_nat_tftp nf_conntrack_tftp nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) tun vmw_vsock_vmci_transport
vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper nfit ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops ttm libata libnvdimm drm crct10dif_pclmul crct10dif_common mptspi crc32c_intel serio_raw scsi_transport_spi mptscsih vmxnet3 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24407 Comm: systemd-journal Kdump: loaded Tainted: P            EL ------------ T 3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff93b1f4e13180 ti: ffff93b02d894000 task.ti: ffff93b02d894000
RIP: 0010:[<ffffffffc032d44e>]  [<ffffffffc032d44e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff93b02d897978  EFLAGS: 00000246
RAX: ffffb51280ec0000 RBX: ffff93b1f50ce5b0 RCX: 000000000000002a
RDX: ffff93aff5f0d000 RSI: ffff93b1f27ed000 RDI: 000000000000000e
RBP: ffff93b02d897988 R08: 0000000000000003 R09: 000000018557afff
R10: ffff93b1f1491000 R11: ffff93b16ed5ba40 R12: ffff93b16ecaad80
R13: ffff93b02d8978ec R14: ffffffffa6d7158b R15: ffff93b02d897930
FS:  00007f3b9d6e0880(0000) GS:ffff93b1ffd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9184515000 CR3: 00000000ac5b0000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc036474e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03b02f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa6f03030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffa6f0cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa6d8e21e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa6d687d9>] __blk_run_queue+0x39/0x50
[<ffffffffa6d6c703>] blk_queue_bio+0x3b3/0x400
[<ffffffffa6d6a537>] generic_make_request+0x157/0x390
[<ffffffffa6da2bbe>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffa6d6a7e0>] submit_bio+0x70/0x150
[<ffffffffa6bd4fb4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc05cd341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc05cdb54>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffa6bd5a5c>] write_cache_pages+0x21c/0x480
[<ffffffffa6aec651>] ? put_prev_entity+0x31/0x400
[<ffffffffc05cd6c0>] ? xfs_vm_writepages+0xb0/0xb0 [xfs]
[<ffffffffc05cd67b>] xfs_vm_writepages+0x6b/0xb0 [xfs]
[<ffffffffa6bd6b01>] do_writepages+0x21/0x50
[<ffffffffa6bcaba5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffffa6bcad11>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc05d8486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa6c93195>] do_fsync+0x55/0x80
[<ffffffffa6c93460>] SyS_fsync+0x10/0x20
[<ffffffffa71c539a>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 
Complete report #990091
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01bd44e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc01a674e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x24e
3
mptsas_qcmd
mptsas 0xffffffffc0222d01
Build id: 3.10.0-1160.118.1.el7.x86_64
0x81
4
? blk_add_timer
vmlinux 0xffffffff9bd72d8c
Build id: 3.10.0-1160.118.1.el7.x86_64
0x7c
5
scsi_dispatch_cmd
vmlinux 0xffffffff9bf03030
Build id: 3.10.0-1160.118.1.el7.x86_64
0xb0
6
scsi_request_fn
vmlinux 0xffffffff9bf0cbdc
Build id: 3.10.0-1160.118.1.el7.x86_64
0x4ac
7
? deadline_add_request
vmlinux 0xffffffff9bd8e21e
Build id: 3.10.0-1160.118.1.el7.x86_64
0x6e
8
__blk_run_queue
vmlinux 0xffffffff9bd687d9
Build id: 3.10.0-1160.118.1.el7.x86_64
0x39
9
blk_queue_bio
vmlinux 0xffffffff9bd6c703
Build id: 3.10.0-1160.118.1.el7.x86_64
0x3b3
10
generic_make_request
vmlinux 0xffffffff9bd6a537
Build id: 3.10.0-1160.118.1.el7.x86_64
0x157
11
? radix_tree_tag_clear
vmlinux 0xffffffff9bda2bbe
Build id: 3.10.0-1160.118.1.el7.x86_64
0x9e
12
submit_bio
vmlinux 0xffffffff9bd6a7e0
Build id: 3.10.0-1160.118.1.el7.x86_64
0x70
13
? __test_set_page_writeback
vmlinux 0xffffffff9bbd4fb4
Build id: 3.10.0-1160.118.1.el7.x86_64
0x54
14
xfs_submit_ioend.isra.12
xfs 0xffffffffc03dc341
Build id: 3.10.0-1160.118.1.el7.x86_64
0x61
15
xfs_do_writepage
xfs 0xffffffffc03dcb54
Build id: 3.10.0-1160.118.1.el7.x86_64
0x494
16
write_cache_pages
vmlinux 0xffffffff9bbd5a5c
Build id: 3.10.0-1160.118.1.el7.x86_64
0x21c
17
? xfs_vm_writepages
xfs 0xffffffffc03dc6c0
Build id: 3.10.0-1160.118.1.el7.x86_64
0xb0
18
xfs_vm_writepages
xfs 0xffffffffc03dc67b
Build id: 3.10.0-1160.118.1.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffff9bbd6b01
Build id: 3.10.0-1160.118.1.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffff9bbcaba5
Build id: 3.10.0-1160.118.1.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffff9bbcad11
Build id: 3.10.0-1160.118.1.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc03e7486
Build id: 3.10.0-1160.118.1.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffff9bc93195
Build id: 3.10.0-1160.118.1.el7.x86_64
0x55
24
? __audit_syscall_exit
vmlinux 0xffffffff9bb46986
Build id: 3.10.0-1160.118.1.el7.x86_64
0x1f6
25
SyS_fdatasync
vmlinux 0xffffffff9bc93483
Build id: 3.10.0-1160.118.1.el7.x86_64
0x13
26
system_call_fastpath
vmlinux 0xffffffff9c1c539a
Build id: 3.10.0-1160.118.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [yum:26997]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr vmw_vmci sg i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul
crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm mptsas e1000 scsi_transport_sas mptscsih ata_piix mptbase libata nfit drm_panel_orientation_quirks libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 26997 Comm: yum Kdump: loaded Tainted: G             L ------------   3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff892b35cf5280 ti: ffff892b23968000 task.ti: ffff892b23968000
RIP: 0010:[<ffffffffc01bd44e>]  [<ffffffffc01bd44e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff892b2396b988  EFLAGS: 00000246
RAX: ffffb6de806b0000 RBX: 0000000017e55f9a RCX: 0000000000000004
RDX: ffff892ab5f87e00 RSI: ffff892bb09dd000 RDI: 000000000000000e
RBP: ffff892b2396b998 R08: 0000000000000002 R09: 000000007b29efff
R10: ffff892ab5f72400 R11: ffff892ae66da840 R12: ffff892a96c0c020
R13: 0000000100000001 R14: ffff892ae3f32280 R15: ffff892b2396b8ec
FS:  00007fa370644740(0000) GS:ffff892bb9640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa3680812b5 CR3: 00000000abf8e000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffc01a674e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc0222d01>] mptsas_qcmd+0x81/0xb0 [mptsas]
[<ffffffff9bd72d8c>] ? blk_add_timer+0x7c/0x130
[<ffffffff9bf03030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff9bf0cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9bd8e21e>] ? deadline_add_request+0x6e/0x90
[<ffffffff9bd687d9>] __blk_run_queue+0x39/0x50
[<ffffffff9bd6c703>] blk_queue_bio+0x3b3/0x400
[<ffffffff9bd6a537>] generic_make_request+0x157/0x390
[<ffffffff9bda2bbe>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff9bd6a7e0>] submit_bio+0x70/0x150
[<ffffffff9bbd4fb4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc03dc341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc03dcb54>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff9bbd5a5c>] write_cache_pages+0x21c/0x480
[<ffffffffc03dc6c0>] ? xfs_vm_writepages+0xb0/0xb0 [xfs]
[<ffffffffc03dc67b>] xfs_vm_writepages+0x6b/0xb0 [xfs]
[<ffffffff9bbd6b01>] do_writepages+0x21/0x50
[<ffffffff9bbcaba5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffff9bbcad11>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc03e7486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff9bc93195>] do_fsync+0x55/0x80
[<ffffffff9bb46986>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffff9bc93483>] SyS_fdatasync+0x13/0x20
[<ffffffff9c1c539a>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 
Complete report #928267
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc027844e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc029874e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc041c2f0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb6902fe0
Build id: 3.10.0-1160.88.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb690cb8c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffb678e08e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffb6768649
Build id: 3.10.0-1160.88.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffb676c573
Build id: 3.10.0-1160.88.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffb676a3a7
Build id: 3.10.0-1160.88.1.el7.x86_64
0x157
10
? radix_tree_tag_clear
vmlinux 0xffffffffb67a2a2e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffb676a650
Build id: 3.10.0-1160.88.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffb65d4ef4
Build id: 3.10.0-1160.88.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0528341
Build id: 3.10.0-1160.88.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0528b54
Build id: 3.10.0-1160.88.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffb65d599c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x21c
16
? put_prev_entity
vmlinux 0xffffffffb64ea151
Build id: 3.10.0-1160.88.1.el7.x86_64
0x31
17
? xfs_vm_writepages
xfs 0xffffffffc05286c0
Build id: 3.10.0-1160.88.1.el7.x86_64
0xb0
18
xfs_vm_writepages
xfs 0xffffffffc052867b
Build id: 3.10.0-1160.88.1.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffffb65d6a41
Build id: 3.10.0-1160.88.1.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffffb65caae5
Build id: 3.10.0-1160.88.1.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffffb65cac51
Build id: 3.10.0-1160.88.1.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc0533486
Build id: 3.10.0-1160.88.1.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffffb66930d5
Build id: 3.10.0-1160.88.1.el7.x86_64
0x55
24
? __audit_syscall_exit
vmlinux 0xffffffffb65468f6
Build id: 3.10.0-1160.88.1.el7.x86_64
0x1f6
25
SyS_fsync
vmlinux 0xffffffffb66933a0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x10
26
system_call_fastpath
vmlinux 0xffffffffb6bc539a
Build id: 3.10.0-1160.88.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [systemd-journal:13807]
Modules linked in: tcp_diag inet_diag ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 iptable_filter xt_conntrack iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle ip_tables ip_set br_netfilter overlay(T) tun bridge stp llc devlink vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi ac97_bus snd_seq crc32_pclmul ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm lrw gf128mul glue_helper snd_timer
ablk_helper snd cryptd soundcore sg ppdev pcspkr joydev vmw_balloon parport_pc parport i2c_piix4 vmw_vmci sch_fq_codel binfmt_misc xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci nfit crct10dif_pclmul mptspi crct10dif_common crc32c_intel scsi_transport_spi libnvdimm libata mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 3 PID: 13807 Comm: systemd-journal Tainted: G             L ------------ T 3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8ffe88fce300 ti: ffff900d8d8ac000 task.ti: ffff900d8d8ac000
RIP: 0010:[<ffffffffc027844e>]  [<ffffffffc027844e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff900d8d8af978  EFLAGS: 00000246
RAX: ffffaecdcc640000 RBX: ffff900fd9e47ea0 RCX: 0000000000000004
RDX: ffff900fd7864600 RSI: ffff900fde9b6000 RDI: 000000000000000e
RBP: ffff900d8d8af988 R08: 0000000000000002 R09: 00000017c71b9fff
R10: ffff900fd7a7dc00 R11: ffff90049d5b96c0 R12: ffff900a6c787d80
R13: ffff900d8d8af8ec R14: ffffffffb67713fb R15: ffff900d8d8af930
FS:  00007f407f166880(0000) GS:ffff900fde6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff744042000 CR3: 0000001752abc000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc029874e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc041c2f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb6902fe0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb690cb8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb678e08e>] ? deadline_add_request+0x6e/0x90
[<ffffffffb6768649>] __blk_run_queue+0x39/0x50
[<ffffffffb676c573>] blk_queue_bio+0x3b3/0x400
[<ffffffffb676a3a7>] generic_make_request+0x157/0x390
[<ffffffffb67a2a2e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffb676a650>] submit_bio+0x70/0x150
[<ffffffffb65d4ef4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0528341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc0528b54>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffb65d599c>] write_cache_pages+0x21c/0x480
[<ffffffffb64ea151>] ? put_prev_entity+0x31/0x400
[<ffffffffc05286c0>] ? xfs_vm_writepages+0xb0/0xb0 [xfs]
[<ffffffffc052867b>] xfs_vm_writepages+0x6b/0xb0 [xfs]
[<ffffffffb65d6a41>] do_writepages+0x21/0x50
[<ffffffffb65caae5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffffb65cac51>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc0533486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffb66930d5>] do_fsync+0x55/0x80
[<ffffffffb65468f6>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffffb66933a0>] SyS_fsync+0x10/0x20
[<ffffffffb6bc539a>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 
Complete report #537680
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff9038b795
Build id: 3.10.0-1160.42.2.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0363632
Build id: 3.10.0-1160.42.2.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff900ecf50
Build id: 3.10.0-1160.42.2.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff900e4840
Build id: 3.10.0-1160.42.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff900edf6c
Build id: 3.10.0-1160.42.2.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8ff78bce
Build id: 3.10.0-1160.42.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff8ff53f19
Build id: 3.10.0-1160.42.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff8ff57cc3
Build id: 3.10.0-1160.42.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff8ff55ba7
Build id: 3.10.0-1160.42.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8ff8ccbe
Build id: 3.10.0-1160.42.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff8ff55e50
Build id: 3.10.0-1160.42.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff8fdc9ee4
Build id: 3.10.0-1160.42.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0651f31
Build id: 3.10.0-1160.42.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0652724
Build id: 3.10.0-1160.42.2.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffff8fdca95c
Build id: 3.10.0-1160.42.2.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0652290
Build id: 3.10.0-1160.42.2.el7.x86_64
0xa0
17
xfs_vm_writepages
xfs 0xffffffffc065225b
Build id: 3.10.0-1160.42.2.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff8fdcb9b1
Build id: 3.10.0-1160.42.2.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff8fdbfe15
Build id: 3.10.0-1160.42.2.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff8fdbff61
Build id: 3.10.0-1160.42.2.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc065cd66
Build id: 3.10.0-1160.42.2.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff8fe84207
Build id: 3.10.0-1160.42.2.el7.x86_64
0x67
23
SyS_fsync
vmlinux 0xffffffff8fe844f0
Build id: 3.10.0-1160.42.2.el7.x86_64
0x10
24
system_call_fastpath
vmlinux 0xffffffff90395f92
Build id: 3.10.0-1160.42.2.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [mysqld:2603]
Modules linked in: arc4 ppp_mppe ppp_async crc_ccitt ppp_generic slhc ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat tun binfmt_misc ipmi_devintf ipmi_msghandler ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel sg lrw gf128mul ppdev glue_helper ablk_helper cryptd vmw_balloon pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sd_mod sysfillrect sysimgblt fb_sys_fops crc_t10dif ttm crct10dif_generic ahci drm e1000e ata_piix libahci nfit crct10dif_pclmul libata crct10dif_common libnvdimm
crc32c_intel serio_raw ptp vmw_pvscsi pps_core drm_panel_orientation_quirks fuse
CPU: 1 PID: 2603 Comm: mysqld Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.42.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9e298c373180 ti: ffff9e2983120000 task.ti: ffff9e2983120000
RIP: 0010:[<ffffffff9038b795>]  [<ffffffff9038b795>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9e29831239d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff8ffb2f63 RCX: ffff9e299ed71060
RDX: ffffa982435f0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9e29831239d0 R08: 0000000000000000 R09: ffff9e28fe42aa00
R10: ffff9e299a72c400 R11: ffff9e21b5c7b800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9e28c92ecd80 R15: ffffffff900ebc09
FS:  00007fe2b564d700(0000) GS:ffff9e299f240000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdcd008d340 CR3: 00000000351f2000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffc0363632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff900ecf50>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff900e4840>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff900edf6c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8ff78bce>] ? deadline_add_request+0x6e/0x90
[<ffffffff8ff53f19>] __blk_run_queue+0x39/0x50
[<ffffffff8ff57cc3>] blk_queue_bio+0x3b3/0x400
[<ffffffff8ff55ba7>] generic_make_request+0x147/0x380
[<ffffffff8ff8ccbe>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff8ff55e50>] submit_bio+0x70/0x150
[<ffffffff8fdc9ee4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0651f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0652724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff8fdca95c>] write_cache_pages+0x21c/0x470
[<ffffffffc0652290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc065225b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8fdcb9b1>] do_writepages+0x21/0x50
[<ffffffff8fdbfe15>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8fdbff61>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc065cd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8fe84207>] do_fsync+0x67/0xb0
[<ffffffff8fe844f0>] SyS_fsync+0x10/0x20
[<ffffffff90395f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #391318
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffb478a3f5
Build id: 3.10.0-1160.15.2.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc023b632
Build id: 3.10.0-1160.15.2.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffb44ec580
Build id: 3.10.0-1160.15.2.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffb44e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb44ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffb437846e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffb4353869
Build id: 3.10.0-1160.15.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffb4357673
Build id: 3.10.0-1160.15.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffb43554f7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffffb438c55e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffb43557a0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffb41c9484
Build id: 3.10.0-1160.15.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc04a6f31
Build id: 3.10.0-1160.15.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc04a7724
Build id: 3.10.0-1160.15.2.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffb41c9efc
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc04a7290
Build id: 3.10.0-1160.15.2.el7.x86_64
0xa0
17
xfs_vm_writepages
xfs 0xffffffffc04a725b
Build id: 3.10.0-1160.15.2.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffffb41caf51
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffffb41bfb45
Build id: 3.10.0-1160.15.2.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffffb41bfc91
Build id: 3.10.0-1160.15.2.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc04b1d66
Build id: 3.10.0-1160.15.2.el7.x86_64
0x66
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 21s! [containerd:1650]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink cmac arc4 md4 nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix libata nfit serio_raw vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 10 PID: 1650 Comm: containerd Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff88e0f5ee6300 ti: ffff88e4dce98000 task.ti: ffff88e4dce98000
RIP: 0010:[<ffffffffb478a3f5>]  [<ffffffffb478a3f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88e4dce9b9d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffffb43b2803 RCX: ffff88e4ed76e060
RDX: ffffb58b81900000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff88e4dce9b9d0 R08: 0000000000000000 R09: ffff88e3531de300
R10: ffff88e23fe52800 R11: ffff88e23cbd7800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff88e167cd8180 R15: ffffffffb44eb239
FS:  00007f88be7fc700(0000) GS:ffff88e4ed880000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f47b091bbc0 CR3: 00000000bbb38000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc023b632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffb44ec580>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffb44e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb44ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb437846e>] ? deadline_add_request+0x6e/0x90
[<ffffffffb4353869>] __blk_run_queue+0x39/0x50
[<ffffffffb4357673>] blk_queue_bio+0x3b3/0x400
[<ffffffffb43554f7>] generic_make_request+0x147/0x380
[<ffffffffb438c55e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffb43557a0>] submit_bio+0x70/0x150
[<ffffffffb41c9484>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc04a6f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc04a7724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffb41c9efc>] write_cache_pages+0x21c/0x470
[<ffffffffc04a7290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc04a725b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffb41caf51>] do_writepages+0x21/0x50
[<ffffffffb41bfb45>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffb41bfc91>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc04b1d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
Complete report #317848
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc02fd3be
Build id: 3.10.0-1160.49.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc031d71e
Build id: 3.10.0-1160.49.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02b82e0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb28e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb28edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffb2778bee
Build id: 3.10.0-1160.49.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffb2753f39
Build id: 3.10.0-1160.49.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffb2757ce3
Build id: 3.10.0-1160.49.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffb2755bc7
Build id: 3.10.0-1160.49.1.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffffb278ccde
Build id: 3.10.0-1160.49.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffb2755e70
Build id: 3.10.0-1160.49.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffb25c9ee4
Build id: 3.10.0-1160.49.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc03f4f31
Build id: 3.10.0-1160.49.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc03f5724
Build id: 3.10.0-1160.49.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffb25ca95c
Build id: 3.10.0-1160.49.1.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc03f5290
Build id: 3.10.0-1160.49.1.el7.x86_64
0xa0
17
xfs_vm_writepages
xfs 0xffffffffc03f525b
Build id: 3.10.0-1160.49.1.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffffb25cb9b1
Build id: 3.10.0-1160.49.1.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffffb25bfe15
Build id: 3.10.0-1160.49.1.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffffb25bff61
Build id: 3.10.0-1160.49.1.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc03ffd66
Build id: 3.10.0-1160.49.1.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffffb2684227
Build id: 3.10.0-1160.49.1.el7.x86_64
0x67
23
? system_call_after_swapgs
vmlinux 0xffffffffb2b95ec9
Build id: 3.10.0-1160.49.1.el7.x86_64
0x96
24
SyS_fsync
vmlinux 0xffffffffb2684510
Build id: 3.10.0-1160.49.1.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffffb2b95f92
Build id: 3.10.0-1160.49.1.el7.x86_64
0x25
26
? system_call_after_swapgs
vmlinux 0xffffffffb2b95ed5
Build id: 3.10.0-1160.49.1.el7.x86_64
0xa2
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 441s! [systemd-journal:8473]
Modules linked in: tcp_lp vboxsf(OE) veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter binfmt_misc overlay(T) sunrpc snd_intel8x0 vboxvideo(OE) snd_ac97_codec ac97_bus ttm snd_seq drm_kms_helper snd_seq_device crc32_pclmul snd_pcm syscopyarea joydev sysfillrect sysimgblt ppdev ghash_clmulni_intel fb_sys_fops drm aesni_intel lrw gf128mul glue_helper snd_timer sg ablk_helper cryptd vboxguest(OE) snd parport_pc drm_panel_orientation_quirks i2c_piix4 video soundcore pcspkr parport ip_tables xfs libcrc32c sd_mod crc_t10dif
crct10dif_generic sr_mod cdrom ata_generic pata_acpi mptspi ata_piix scsi_transport_spi e1000 libata mptscsih crct10dif_pclmul crct10dif_common mptbase serio_raw crc32c_intel dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 8473 Comm: systemd-journal Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff931f7d3f6300 ti: ffff931f62010000 task.ti: ffff931f62010000
RIP: 0010:[<ffffffffc02fd3be>]  [<ffffffffc02fd3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff931f62013968  EFLAGS: 00000246
RAX: ffffadb741bc0000 RBX: ffff931e54af0000 RCX: 0000000000000014
RDX: ffff93217318b400 RSI: ffff932173bd0000 RDI: 000000000000000e
RBP: ffff931f62013978 R08: 0000000000000002 R09: 0000000216313fff
R10: ffff9321774e2c00 R11: ffff93212ad1ddc0 R12: ffff93217612d200
RIP: 0010:[<ffffffffc02fd3be>]  [<ffffffffc02fd3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff931f62013968  EFLAGS: 00000246
RAX: ffffadb741bc0000 RBX: ffff931e54af0000 RCX: 0000000000000014
RDX: ffff93217318b400 RSI: ffff932173bd0000 RDI: 000000000000000e
RBP: ffff931f62013978 R08: 0000000000000002 R09: 0000000216313fff
R10: ffff9321774e2c00 R11: ffff93212ad1ddc0 R12: ffff93217612d200
FS:  00007f7d2dedb880(0000) GS:ffff932187d00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff0d06510f8 CR3: 000000026622a000 CR4: 00000000000606e0
Call Trace:
[<ffffffffc031d71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02b82e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb28e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb28edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb2778bee>] ? deadline_add_request+0x6e/0x90
[<ffffffffb2753f39>] __blk_run_queue+0x39/0x50
[<ffffffffb2757ce3>] blk_queue_bio+0x3b3/0x400
[<ffffffffb2755bc7>] generic_make_request+0x147/0x380
[<ffffffffb278ccde>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffb2755e70>] submit_bio+0x70/0x150
[<ffffffffb25c9ee4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc03f4f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc03f5724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffb25ca95c>] write_cache_pages+0x21c/0x470
[<ffffffffc03f5290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc03f525b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffb25cb9b1>] do_writepages+0x21/0x50
[<ffffffffb25bfe15>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffb25bff61>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc03ffd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffb2684227>] do_fsync+0x67/0xb0
[<ffffffffb2b95ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffb2684510>] SyS_fsync+0x10/0x20
[<ffffffffb2b95f92>] system_call_fastpath+0x25/0x2a
[<ffffffffb2b95ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 30 
Complete report #296245
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8c18b795
Build id: 3.10.0-1160.45.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc03fd632
Build id: 3.10.0-1160.45.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8beecf70
Build id: 3.10.0-1160.45.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8bee4860
Build id: 3.10.0-1160.45.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8beedf8c
Build id: 3.10.0-1160.45.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8bd78bee
Build id: 3.10.0-1160.45.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff8bd53f39
Build id: 3.10.0-1160.45.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff8bd57ce3
Build id: 3.10.0-1160.45.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff8bd55bc7
Build id: 3.10.0-1160.45.1.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8bd8ccde
Build id: 3.10.0-1160.45.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff8bd55e70
Build id: 3.10.0-1160.45.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff8bbc9ee4
Build id: 3.10.0-1160.45.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc067af31
Build id: 3.10.0-1160.45.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc067b724
Build id: 3.10.0-1160.45.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffff8bbca95c
Build id: 3.10.0-1160.45.1.el7.x86_64
0x21c
16
? switch_mm_irqs_off
vmlinux 0xffffffff8ba7dfe9
Build id: 3.10.0-1160.45.1.el7.x86_64
0x109
17
? xfs_vm_writepages
xfs 0xffffffffc067b290
Build id: 3.10.0-1160.45.1.el7.x86_64
0xa0
18
xfs_vm_writepages
xfs 0xffffffffc067b25b
Build id: 3.10.0-1160.45.1.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffff8bbcb9b1
Build id: 3.10.0-1160.45.1.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffff8bbbfe15
Build id: 3.10.0-1160.45.1.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffff8bbbff61
Build id: 3.10.0-1160.45.1.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc0685d66
Build id: 3.10.0-1160.45.1.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffff8bc84227
Build id: 3.10.0-1160.45.1.el7.x86_64
0x67
24
SyS_fsync
vmlinux 0xffffffff8bc84510
Build id: 3.10.0-1160.45.1.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffff8c195f92
Build id: 3.10.0-1160.45.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [mysqld:2091]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nfsv3 nfs_acl nfs lockd grace fscache ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd pcspkr joydev sg parport_pc vmw_vmci parport ip_tables xfs libcrc32c sr_mod cdrom
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea nfit sysfillrect sysimgblt libnvdimm fb_sys_fops ttm ahci sd_mod crc_t10dif crct10dif_generic drm ata_piix libahci libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 2091 Comm: mysqld Kdump: loaded Not tainted 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9d29f3eb0000 ti: ffff9d29de858000 task.ti: ffff9d29de858000
RIP: 0010:[<ffffffff8c18b795>]  [<ffffffff8c18b795>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9d29de85b9d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff8bdb2f83 RCX: ffffffff8c634f40
RDX: ffffb64280e90000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9d29de85b9d0 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9d27f5dd2400 R11: ffff9d29f2a49800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9d29c4b59980 R15: ffffffff8beebc29
FS:  00007f994866d700(0000) GS:ffff9d29ffc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4546fe8000 CR3: 00000000b6ea8000 CR4: 00000000007607f0
PKRU: 55555554
Call Trace:
[<ffffffffc03fd632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8beecf70>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8bee4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8beedf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8bd78bee>] ? deadline_add_request+0x6e/0x90
[<ffffffff8bd53f39>] __blk_run_queue+0x39/0x50
[<ffffffff8bd57ce3>] blk_queue_bio+0x3b3/0x400
[<ffffffff8bd55bc7>] generic_make_request+0x147/0x380
[<ffffffff8bd8ccde>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff8bd55e70>] submit_bio+0x70/0x150
[<ffffffff8bbc9ee4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc067af31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc067b724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff8bbca95c>] write_cache_pages+0x21c/0x470
[<ffffffff8ba7dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffc067b290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc067b25b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8bbcb9b1>] do_writepages+0x21/0x50
[<ffffffff8bbbfe15>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8bbbff61>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0685d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8bc84227>] do_fsync+0x67/0xb0
[<ffffffff8bc84510>] SyS_fsync+0x10/0x20
[<ffffffff8c195f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #255224
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff8d98b4d5
Build id: 3.10.0-1160.24.1.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc03e5632
Build id: 3.10.0-1160.24.1.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff8d6ec6b0
Build id: 3.10.0-1160.24.1.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff8d6e3fa0
Build id: 3.10.0-1160.24.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8d6ed6cc
Build id: 3.10.0-1160.24.1.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8d57853e
Build id: 3.10.0-1160.24.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff8d553939
Build id: 3.10.0-1160.24.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff8d557743
Build id: 3.10.0-1160.24.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff8d5555c7
Build id: 3.10.0-1160.24.1.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8d58c62e
Build id: 3.10.0-1160.24.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff8d555870
Build id: 3.10.0-1160.24.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff8d3c9534
Build id: 3.10.0-1160.24.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0626f31
Build id: 3.10.0-1160.24.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0627724
Build id: 3.10.0-1160.24.1.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffff8d3c9fac
Build id: 3.10.0-1160.24.1.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc0627290
Build id: 3.10.0-1160.24.1.el7.x86_64
0xa0
17
xfs_vm_writepages
xfs 0xffffffffc062725b
Build id: 3.10.0-1160.24.1.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff8d3cb001
Build id: 3.10.0-1160.24.1.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff8d3bfbf5
Build id: 3.10.0-1160.24.1.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff8d3bfd41
Build id: 3.10.0-1160.24.1.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc0631d66
Build id: 3.10.0-1160.24.1.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff8d483cb7
Build id: 3.10.0-1160.24.1.el7.x86_64
0x67
23
SyS_fsync
vmlinux 0xffffffff8d483fa0
Build id: 3.10.0-1160.24.1.el7.x86_64
0x10
24
system_call_fastpath
vmlinux 0xffffffff8d995f92
Build id: 3.10.0-1160.24.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [mysqld:2365]
Modules linked in: talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) vmw_vsock_vmci_transport vsock nf_conntrack_ipv4 nf_defrag_ipv4 xt_owner xt_conntrack nf_conntrack xt_multiport iptable_filter sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper sg ablk_helper vmw_balloon cryptd pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx ahci libahci ata_piix crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel serio_raw syscopyarea sysfillrect libata sysimgblt fb_sys_fops ttm vmxnet3 drm vmw_pvscsi nfit drm_panel_orientation_quirks libnvdimm dm_mirror dm_region_hash
dm_log dm_mod fuse
CPU: 1 PID: 2365 Comm: mysqld Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.24.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8afef3e0a100 ti: ffff8afe3c3c4000 task.ti: ffff8afe3c3c4000
RIP: 0010:[<ffffffff8d98b4d5>]  [<ffffffff8d98b4d5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8afe3c3c79d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff8d5b28d3 RCX: ffffffff8de34f40
RDX: fffface680f40000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8afe3c3c79d0 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8afef5dab000 R11: ffff8afcf660e800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff8afd24277200 R15: ffffffff8d6eb369
FS:  00007f3d927fc700(0000) GS:ffff8afeffd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f14d2c1a99c CR3: 000000017fef6000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc03e5632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8d6ec6b0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8d6e3fa0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d6ed6cc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d57853e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8d553939>] __blk_run_queue+0x39/0x50
[<ffffffff8d557743>] blk_queue_bio+0x3b3/0x400
[<ffffffff8d5555c7>] generic_make_request+0x147/0x380
[<ffffffff8d58c62e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff8d555870>] submit_bio+0x70/0x150
[<ffffffff8d3c9534>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0626f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0627724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff8d3c9fac>] write_cache_pages+0x21c/0x470
[<ffffffffc0627290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc062725b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8d3cb001>] do_writepages+0x21/0x50
[<ffffffff8d3bfbf5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8d3bfd41>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0631d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8d483cb7>] do_fsync+0x67/0xb0
[<ffffffff8d483fa0>] SyS_fsync+0x10/0x20
[<ffffffff8d995f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #234577
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00b63be
Build id: 3.10.0-1160.36.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00d871e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00f02e0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa56e4810
Build id: 3.10.0-1160.36.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa56edf3c
Build id: 3.10.0-1160.36.2.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffa5578b9e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffa5553ee9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffa5557c93
Build id: 3.10.0-1160.36.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffa5555b77
Build id: 3.10.0-1160.36.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffffa558cc8e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffa5555e20
Build id: 3.10.0-1160.36.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffa53c9ec4
Build id: 3.10.0-1160.36.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0354f31
Build id: 3.10.0-1160.36.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0355724
Build id: 3.10.0-1160.36.2.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffa53ca93c
Build id: 3.10.0-1160.36.2.el7.x86_64
0x21c
16
? switch_mm_irqs_off
vmlinux 0xffffffffa527dfe9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x109
17
? xfs_vm_writepages
xfs 0xffffffffc0355290
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa0
18
xfs_vm_writepages
xfs 0xffffffffc035525b
Build id: 3.10.0-1160.36.2.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffffa53cb991
Build id: 3.10.0-1160.36.2.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffffa53bfdf5
Build id: 3.10.0-1160.36.2.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffffa53bff41
Build id: 3.10.0-1160.36.2.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc035fd66
Build id: 3.10.0-1160.36.2.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffffa54841d7
Build id: 3.10.0-1160.36.2.el7.x86_64
0x67
24
SyS_fsync
vmlinux 0xffffffffa54844c0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffffa5995f92
Build id: 3.10.0-1160.36.2.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 24s! [mysqld:1423]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus aesni_intel snd_seq lrw snd_seq_device gf128mul vmw_balloon glue_helper ablk_helper cryptd
snd_pcm joydev pcspkr snd_timer snd sg vmw_vmci soundcore i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata e1000 nfit serio_raw libnvdimm mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 1423 Comm: mysqld Kdump: loaded Tainted: G             L ------------   3.10.0-1160.36.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff98ca26ef8000 ti: ffff98ca2c784000 task.ti: ffff98ca2c784000
RIP: 0010:[<ffffffffc00b63be>]  [<ffffffffc00b63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff98ca2c787968  EFLAGS: 00000246
RAX: ffffa957806c0000 RBX: ffff98ca2c6f2470 RCX: 0000000000000012
RDX: ffff98cab056de00 RSI: ffff98cab139d000 RDI: 000000000000000e
RBP: ffff98ca2c787978 R08: 0000000000000003 R09: 000000001a4f6fff
R10: ffff98cab0550c00 R11: ffff98cab3499500 R12: ffff98c984db7c00
R13: ffff98ca2c7878dc R14: ffffffffa555c8ab R15: ffff98ca2c787920
FS:  00007f6689ffb700(0000) GS:ffff98cab9640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055e9a0c0b2a8 CR3: 00000000af706000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffc00d871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00f02e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa56e4810>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa56edf3c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa5578b9e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5553ee9>] __blk_run_queue+0x39/0x50
[<ffffffffa5557c93>] blk_queue_bio+0x3b3/0x400
[<ffffffffa5555b77>] generic_make_request+0x147/0x380
[<ffffffffa558cc8e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffa5555e20>] submit_bio+0x70/0x150
[<ffffffffa53c9ec4>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0354f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0355724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffa53ca93c>] write_cache_pages+0x21c/0x470
[<ffffffffa527dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffc0355290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc035525b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa53cb991>] do_writepages+0x21/0x50
[<ffffffffa53bfdf5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffa53bff41>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc035fd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa54841d7>] do_fsync+0x67/0xb0
[<ffffffffa54844c0>] SyS_fsync+0x10/0x20
[<ffffffffa5995f92>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 0c 
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #195061
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc02a23be
Build id: 3.10.0-957.12.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc02bd71e
Build id: 3.10.0-957.12.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc029b2e0
Build id: 3.10.0-957.12.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8f2d14f0
Build id: 3.10.0-957.12.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8f2daa8c
Build id: 3.10.0-957.12.1.el7.x86_64
0x4cc
6
? deadline_add_request
vmlinux 0xffffffff8f167c8e
Build id: 3.10.0-957.12.1.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff8f143679
Build id: 3.10.0-957.12.1.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff8f147373
Build id: 3.10.0-957.12.1.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff8f145307
Build id: 3.10.0-957.12.1.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8f17bd6e
Build id: 3.10.0-957.12.1.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff8f1455b0
Build id: 3.10.0-957.12.1.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff8efc1944
Build id: 3.10.0-957.12.1.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc04c2251
Build id: 3.10.0-957.12.1.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc04c2a54
Build id: 3.10.0-957.12.1.el7.x86_64
0x4a4
15
write_cache_pages
vmlinux 0xffffffff8efc23bc
Build id: 3.10.0-957.12.1.el7.x86_64
0x21c
16
? xfs_aops_discard_page
xfs 0xffffffffc04c25b0
Build id: 3.10.0-957.12.1.el7.x86_64
0x150
17
xfs_vm_writepages
xfs 0xffffffffc04c23ab
Build id: 3.10.0-957.12.1.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff8efc34d1
Build id: 3.10.0-957.12.1.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff8efb7d75
Build id: 3.10.0-957.12.1.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff8efb7ec1
Build id: 3.10.0-957.12.1.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc04cd156
Build id: 3.10.0-957.12.1.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff8f0762a7
Build id: 3.10.0-957.12.1.el7.x86_64
0x67
23
? system_call_after_swapgs
vmlinux 0xffffffff8f575cd5
Build id: 3.10.0-957.12.1.el7.x86_64
0xa2
24
SyS_fdatasync
vmlinux 0xffffffff8f0765b3
Build id: 3.10.0-957.12.1.el7.x86_64
0x13
25
system_call_fastpath
vmlinux 0xffffffff8f575d9b
Build id: 3.10.0-957.12.1.el7.x86_64
0x22
26
? system_call_after_swapgs
vmlinux 0xffffffff8f575ce1
Build id: 3.10.0-957.12.1.el7.x86_64
0xae
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [yum:61007]
Modules linked in: ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 cbsensor(POE) xfs libcrc32c sd_mod sr_mod crc_t10dif cdrom crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi vmwgfx vmxnet3 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 0 PID: 61007 Comm: yum Kdump: loaded Tainted: P           OE  ------------   3.10.0-957.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8f4dc92eb0c0 ti: ffff8f4d923a4000 task.ti: ffff8f4d923a4000
RIP: 0010:[<ffffffffc02a23be>]  [<ffffffffc02a23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8f4d923a7968  EFLAGS: 00000246
RAX: ffffba30c0d40000 RBX: ffff8f4dc4a67000 RCX: 0000000000000008
RDX: ffff8f4df167b400 RSI: ffff8f4df1578000 RDI: 000000000000000e
RBP: ffff8f4d923a7978 R08: 0000000000000002 R09: 00000001e51bbfff
R10: ffff8f4df157c0d8 R11: ffff8f4da870ca80 R12: ffff8f4dc929f180
R13: ffff8f4d923a78dc R14: ffffffff8f14bebb R15: ffff8f4d923a7920
FS:  00007f1458f6d740(0000) GS:ffff8f4df5600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000046fa268 CR3: 0000000211ec4000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc02bd71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc029b2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8f2d14f0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8f2daa8c>] scsi_request_fn+0x4cc/0x680
[<ffffffff8f167c8e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8f143679>] __blk_run_queue+0x39/0x50
[<ffffffff8f147373>] blk_queue_bio+0x3b3/0x400
[<ffffffff8f145307>] generic_make_request+0x147/0x380
[<ffffffff8f17bd6e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff8f1455b0>] submit_bio+0x70/0x150
[<ffffffff8efc1944>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc04c2251>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc04c2a54>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffff8efc23bc>] write_cache_pages+0x21c/0x470
[<ffffffffc04c25b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc04c23ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff8efc34d1>] do_writepages+0x21/0x50
[<ffffffff8efb7d75>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8efb7ec1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc04cd156>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8f0762a7>] do_fsync+0x67/0xb0
[<ffffffff8f575cd5>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff8f0765b3>] SyS_fdatasync+0x13/0x20
[<ffffffff8f575d9b>] system_call_fastpath+0x22/0x27
[<ffffffff8f575ce1>] ? system_call_after_swapgs+0xae/0x146
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 2b 
Complete report #169082
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc04013be
Build id: 3.10.0-1160.15.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc048371e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc02fb2e0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff892e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff892ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffff8917846e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff89153869
Build id: 3.10.0-1160.15.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff89157673
Build id: 3.10.0-1160.15.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff891554f7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff8918c55e
Build id: 3.10.0-1160.15.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff891557a0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff88fc9484
Build id: 3.10.0-1160.15.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc056ef31
Build id: 3.10.0-1160.15.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc056f724
Build id: 3.10.0-1160.15.2.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffff88fc9efc
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc056f290
Build id: 3.10.0-1160.15.2.el7.x86_64
0xa0
17
xfs_vm_writepages
xfs 0xffffffffc056f25b
Build id: 3.10.0-1160.15.2.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff88fcaf51
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff88fbfb45
Build id: 3.10.0-1160.15.2.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff88fbfc91
Build id: 3.10.0-1160.15.2.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc0579d66
Build id: 3.10.0-1160.15.2.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff89083be7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x67
23
SyS_fdatasync
vmlinux 0xffffffff89083ef3
Build id: 3.10.0-1160.15.2.el7.x86_64
0x13
24
system_call_fastpath
vmlinux 0xffffffff89594f92
Build id: 3.10.0-1160.15.2.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 24s! [WTCheck.tThread:3278]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper snd_timer ablk_helper snd pcspkr sg vmw_balloon cryptd soundcore joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi
vmwgfx drm_kms_helper syscopyarea sysfillrect nfit sysimgblt mptspi fb_sys_fops scsi_transport_spi ttm mptscsih ata_piix crct10dif_pclmul libata e1000 crct10dif_common mptbase drm libnvdimm crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 3278 Comm: WTCheck.tThread Kdump: loaded Tainted: G               ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9ae139346300 ti: ffff9ade75ec0000 task.ti: ffff9ade75ec0000
RIP: 0010:[<ffffffffc04013be>]  [<ffffffffc04013be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9ade75ec3968  EFLAGS: 00000246
RAX: ffffc07a01980000 RBX: ffff9ae15ca70010 RCX: 0000000000000019
RDX: ffff9ae16e8aa000 RSI: ffff9ae17352c000 RDI: 000000000000000e
RBP: ffff9ade75ec3978 R08: 0000000000000003 R09: 00000002ebfc9fff
R10: ffff9ae16e2cd400 R11: ffff9ae16e36e300 R12: ffff9ae0870ab600
R13: ffff9ade75ec38dc R14: ffffffff8915c28b R15: ffff9ade75ec3920
FS:  00007f1f7257b700(0000) GS:ffff9ae171700000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1f72577000 CR3: 000000031d9be000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffc048371e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02fb2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff892e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff892ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8917846e>] ? deadline_add_request+0x6e/0x90
[<ffffffff89153869>] __blk_run_queue+0x39/0x50
[<ffffffff89157673>] blk_queue_bio+0x3b3/0x400
[<ffffffff891554f7>] generic_make_request+0x147/0x380
[<ffffffff8918c55e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff891557a0>] submit_bio+0x70/0x150
[<ffffffff88fc9484>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc056ef31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc056f724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff88fc9efc>] write_cache_pages+0x21c/0x470
[<ffffffffc056f290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc056f25b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff88fcaf51>] do_writepages+0x21/0x50
[<ffffffff88fbfb45>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff88fbfc91>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0579d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff89083be7>] do_fsync+0x67/0xb0
[<ffffffff89083ef3>] SyS_fdatasync+0x13/0x20
[<ffffffff89594f92>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 f3 40 
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #109784
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffff9416a0f5
Build id: 3.10.0-957.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0244622
Build id: 3.10.0-957.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffff93ed8e90
Build id: 3.10.0-957.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffff93ed0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff93ed9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
? deadline_add_request
vmlinux 0xffffffff93d675fe
Build id: 3.10.0-957.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff93d42ff9
Build id: 3.10.0-957.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff93d46cf3
Build id: 3.10.0-957.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff93d44c87
Build id: 3.10.0-957.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff93d7b6de
Build id: 3.10.0-957.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff93d44f30
Build id: 3.10.0-957.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff93bc1584
Build id: 3.10.0-957.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0464251
Build id: 3.10.0-957.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0464a54
Build id: 3.10.0-957.el7.x86_64
0x4a4
15
write_cache_pages
vmlinux 0xffffffff93bc2034
Build id: 3.10.0-957.el7.x86_64
0x254
16
? xfs_aops_discard_page
xfs 0xffffffffc04645b0
Build id: 3.10.0-957.el7.x86_64
0x150
17
xfs_vm_writepages
xfs 0xffffffffc04643ab
Build id: 3.10.0-957.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff93bc3181
Build id: 3.10.0-957.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff93bb79b5
Build id: 3.10.0-957.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff93bb7b01
Build id: 3.10.0-957.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc046f156
Build id: 3.10.0-957.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff93c75c77
Build id: 3.10.0-957.el7.x86_64
0x67
23
SyS_fsync
vmlinux 0xffffffff93c75f60
Build id: 3.10.0-957.el7.x86_64
0x10
24
system_call_fastpath
vmlinux 0xffffffff94174ddb
Build id: 3.10.0-957.el7.x86_64
0x22
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 25s! [mysqld:13118]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp iosf_mbi kvm_intel ppdev kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq
snd_seq_device snd_pcm snd_timer snd sg soundcore parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ata_generic pata_acpi serio_raw vmwgfx ahci drm_kms_helper libahci e1000 syscopyarea sysfillrect sysimgblt fb_sys_fops vmw_pvscsi ttm ata_piix libata drm nfit libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 13118 Comm: mysqld Kdump: loaded Not tainted 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9fb322f3b0c0 ti: ffff9fb30afcc000 task.ti: ffff9fb30afcc000
RIP: 0010:[<ffffffff9416a0f5>]  [<ffffffff9416a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
sched: RT throttling activated
RSP: 0018:ffff9fb30afcf9c0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff93da10f3 RCX: ffffffff946343c0
RDX: ffffbf2680cb0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9fb30afcf9c0 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9fb331aa28d8 R11: ffff9fb135dcc000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9fb2ed7c7480 R15: ffffffff93ed7be9
FS:  00007f842b1b6700(0000) GS:ffff9fb335880000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f123865c000 CR3: 000000021e690000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffc0244622>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff93ed8e90>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff93ed0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff93ed9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff93d675fe>] ? deadline_add_request+0x6e/0x90
[<ffffffff93d42ff9>] __blk_run_queue+0x39/0x50
[<ffffffff93d46cf3>] blk_queue_bio+0x3b3/0x400
[<ffffffff93d44c87>] generic_make_request+0x147/0x380
[<ffffffff93d7b6de>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff93d44f30>] submit_bio+0x70/0x150
[<ffffffff93bc1584>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0464251>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0464a54>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffff93bc2034>] write_cache_pages+0x254/0x4e0
[<ffffffffc04645b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc04643ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff93bc3181>] do_writepages+0x21/0x50
[<ffffffff93bb79b5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff93bb7b01>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc046f156>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff93c75c77>] do_fsync+0x67/0xb0
[<ffffffff93c75f60>] SyS_fsync+0x10/0x20
[<ffffffff94174ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #77401
Frame # Function Binary Source or offset Line
1
_raw_spin_unlock_irqrestore
vmlinux 0xffffffffa59883c5
Build id: 3.10.0-1127.18.2.el7.x86_64
0x15
2
pvscsi_queue
vmw_pvscsi 0xffffffffc0096632
Build id: 3.10.0-1127.18.2.el7.x86_64
0x4a2
3
? scsi_test_unit_ready
vmlinux 0xffffffffa56eb060
Build id: 3.10.0-1127.18.2.el7.x86_64
0x170
4
scsi_dispatch_cmd
vmlinux 0xffffffffa56e2950
Build id: 3.10.0-1127.18.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa56ec07c
Build id: 3.10.0-1127.18.2.el7.x86_64
0x4ac
6
? deadline_add_request
vmlinux 0xffffffffa557716e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffa5552769
Build id: 3.10.0-1127.18.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffa5556473
Build id: 3.10.0-1127.18.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffa55543f7
Build id: 3.10.0-1127.18.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffffa558b25e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffa55546a0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffa53c9794
Build id: 3.10.0-1127.18.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc1605fb1
Build id: 3.10.0-1127.18.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc16067a4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x494
15
write_cache_pages
vmlinux 0xffffffffa53ca20c
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21c
16
? xfs_vm_writepages
xfs 0xffffffffc1606310
Build id: 3.10.0-1127.18.2.el7.x86_64
0xa0
17
? core_sys_select
vmlinux 0xffffffffa546439d
Build id: 3.10.0-1127.18.2.el7.x86_64
0x26d
18
xfs_vm_writepages
xfs 0xffffffffc16062db
Build id: 3.10.0-1127.18.2.el7.x86_64
0x6b
19
do_writepages
vmlinux 0xffffffffa53cb261
Build id: 3.10.0-1127.18.2.el7.x86_64
0x21
20
__filemap_fdatawrite_range
vmlinux 0xffffffffa53bfc85
Build id: 3.10.0-1127.18.2.el7.x86_64
0x65
21
filemap_write_and_wait_range
vmlinux 0xffffffffa53bfdd1
Build id: 3.10.0-1127.18.2.el7.x86_64
0x41
22
xfs_file_fsync
xfs 0xffffffffc1610dd6
Build id: 3.10.0-1127.18.2.el7.x86_64
0x66
23
do_fsync
vmlinux 0xffffffffa5482f27
Build id: 3.10.0-1127.18.2.el7.x86_64
0x67
24
SyS_fsync
vmlinux 0xffffffffa5483210
Build id: 3.10.0-1127.18.2.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffffa5992ed2
Build id: 3.10.0-1127.18.2.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [etxdb:3073]
Modules linked in: fuse nvidia_uvm(OE) nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter rpcsec_gss_krb5 nfsv4 dns_resolver xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter nfsv3 nfs_acl nfs lockd grace fscache overlay(T) vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd vmw_balloon sg joydev pcspkr i2c_piix4 vmw_vmci parport_pc parport binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) sr_mod cdrom ata_generic pata_acpi
vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect ttm sysimgblt fb_sys_fops ahci ata_piix libahci drm crct10dif_pclmul crct10dif_common nfit crc32c_intel libata serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 3073 Comm: etxdb Kdump: loaded Tainted: P           OE  ------------ T 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8900a7940000 ti: ffff89003300c000 task.ti: ffff89003300c000
RIP: 0010:[<ffffffffa59883c5>]  [<ffffffffa59883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff89003300f9d0  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffffa55b1473 RCX: ffffffffa5e34e40
RDX: ffffaebf19260000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff89003300f9d0 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8900b54cc400 R11: ffff8900b57b7000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff88e143190900 R15: ffffffffa56e9d19
FS:  00002b6a1358d880(0000) GS:ffff89013fc40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001ce1100 CR3: 000000203300a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc0096632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffa56eb060>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffa56e2950>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa56ec07c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa557716e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5552769>] __blk_run_queue+0x39/0x50
[<ffffffffa5556473>] blk_queue_bio+0x3b3/0x400
[<ffffffffa55543f7>] generic_make_request+0x147/0x380
[<ffffffffa558b25e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffa55546a0>] submit_bio+0x70/0x150
[<ffffffffa53c9794>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc1605fb1>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc16067a4>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffffa53ca20c>] write_cache_pages+0x21c/0x470
[<ffffffffc1606310>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffa546439d>] ? core_sys_select+0x26d/0x340
[<ffffffffc16062db>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffa53cb261>] do_writepages+0x21/0x50
[<ffffffffa53bfc85>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffa53bfdd1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc1610dd6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa5482f27>] do_fsync+0x67/0xb0
[<ffffffffa5483210>] SyS_fsync+0x10/0x20
[<ffffffffa5992ed2>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 
Complete report #23087
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03c93be
Build id: 3.10.0-957.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03e471e
Build id: 3.10.0-957.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc05092e0
Build id: 3.10.0-957.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffadad0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffadad9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
? deadline_add_request
vmlinux 0xffffffffad9675fe
Build id: 3.10.0-957.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffffad942ff9
Build id: 3.10.0-957.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffffad946cf3
Build id: 3.10.0-957.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffffad944c87
Build id: 3.10.0-957.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffffad97b6de
Build id: 3.10.0-957.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffffad944f30
Build id: 3.10.0-957.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffffad7c1584
Build id: 3.10.0-957.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc0615251
Build id: 3.10.0-957.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc0615a54
Build id: 3.10.0-957.el7.x86_64
0x4a4
15
write_cache_pages
vmlinux 0xffffffffad7c2034
Build id: 3.10.0-957.el7.x86_64
0x254
16
? xfs_aops_discard_page
xfs 0xffffffffc06155b0
Build id: 3.10.0-957.el7.x86_64
0x150
17
xfs_vm_writepages
xfs 0xffffffffc06153ab
Build id: 3.10.0-957.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffffad7c3181
Build id: 3.10.0-957.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffffad7b79b5
Build id: 3.10.0-957.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffffad7b7b01
Build id: 3.10.0-957.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc0620156
Build id: 3.10.0-957.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffffad875c77
Build id: 3.10.0-957.el7.x86_64
0x67
23
SyS_fdatasync
vmlinux 0xffffffffad875f83
Build id: 3.10.0-957.el7.x86_64
0x13
24
system_call_fastpath
vmlinux 0xffffffffadd74ddb
Build id: 3.10.0-957.el7.x86_64
0x22
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [containerd:26060]
Modules linked in: tcp_diag inet_diag veth ipt_rpfilter xt_multiport xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set ipip tunnel4 ip_tunnel xt_statistic xt_nat ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter sysdigcloud_probe(OE) overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd ppdev snd_ens1371 snd_rawmidi snd_ac97_codec vmw_balloon ac97_bus snd_seq snd_seq_device snd_pcm snd_timer sg snd pcspkr soundcore joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi drm libata scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm
CPU: 0 PID: 26060 Comm: containerd Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffffa08176e99040 ti: ffffa0801ebcc000 task.ti: ffffa0801ebcc000
RIP: 0010:[<ffffffffc03c93be>]  [<ffffffffc03c93be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffffa0801ebcf958  EFLAGS: 00000246
RAX: ffffb06ac0d40000 RBX: ffffa08026d18040 RCX: 0000000000000007
RDX: ffffa0817379da00 RSI: ffffa08173698000 RDI: 000000000000000e
RBP: ffffa0801ebcf968 R08: 0000000000000003 R09: 0000000095ac8fff
R10: ffffa081736990d8 R11: ffffa07ff5092300 R12: ffffa0813cfa3480
R13: ffffa0801ebcf8cc R14: ffffffffad94b83b R15: ffffa0801ebcf910
FS:  00007fc0d5ffb700(0000) GS:ffffa08177600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000005093a4e CR3: 00000000b4ee6000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc03e471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc05092e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffadad0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffadad9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffad9675fe>] ? deadline_add_request+0x6e/0x90
[<ffffffffad942ff9>] __blk_run_queue+0x39/0x50
[<ffffffffad946cf3>] blk_queue_bio+0x3b3/0x400
[<ffffffffad944c87>] generic_make_request+0x147/0x380
[<ffffffffad97b6de>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffffad944f30>] submit_bio+0x70/0x150
[<ffffffffad7c1584>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc0615251>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc0615a54>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffffad7c2034>] write_cache_pages+0x254/0x4e0
[<ffffffffc06155b0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc06153ab>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffffad7c3181>] do_writepages+0x21/0x50
[<ffffffffad7b79b5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffad7b7b01>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0620156>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffad875c77>] do_fsync+0x67/0xb0
[<ffffffffad875f83>] SyS_fdatasync+0x13/0x20
[<ffffffffadd74ddb>] system_call_fastpath+0x22/0x27
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 3d 
Complete report #13428
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03c43be
Build id: 3.10.0-957.27.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03e671e
Build id: 3.10.0-957.27.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03b42e0
Build id: 3.10.0-957.27.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9fed2ee0
Build id: 3.10.0-957.27.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9fedc47c
Build id: 3.10.0-957.27.2.el7.x86_64
0x4cc
6
? deadline_add_request
vmlinux 0xffffffff9fd694de
Build id: 3.10.0-957.27.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff9fd44e59
Build id: 3.10.0-957.27.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff9fd48b63
Build id: 3.10.0-957.27.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff9fd46af7
Build id: 3.10.0-957.27.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff9fd7d5be
Build id: 3.10.0-957.27.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff9fd46da0
Build id: 3.10.0-957.27.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff9fbc2c14
Build id: 3.10.0-957.27.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc064c471
Build id: 3.10.0-957.27.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc064cc74
Build id: 3.10.0-957.27.2.el7.x86_64
0x4a4
15
write_cache_pages
vmlinux 0xffffffff9fbc368c
Build id: 3.10.0-957.27.2.el7.x86_64
0x21c
16
? xfs_aops_discard_page
xfs 0xffffffffc064c7d0
Build id: 3.10.0-957.27.2.el7.x86_64
0x150
17
xfs_vm_writepages
xfs 0xffffffffc064c5cb
Build id: 3.10.0-957.27.2.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff9fbc47a1
Build id: 3.10.0-957.27.2.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff9fbb9045
Build id: 3.10.0-957.27.2.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff9fbb9191
Build id: 3.10.0-957.27.2.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc0656ee6
Build id: 3.10.0-957.27.2.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff9fc77367
Build id: 3.10.0-957.27.2.el7.x86_64
0x67
23
? system_call_after_swapgs
vmlinux 0xffffffffa0176d15
Build id: 3.10.0-957.27.2.el7.x86_64
0xa2
24
SyS_fsync
vmlinux 0xffffffff9fc77650
Build id: 3.10.0-957.27.2.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffffa0176ddb
Build id: 3.10.0-957.27.2.el7.x86_64
0x22
26
? system_call_after_swapgs
vmlinux 0xffffffffa0176d21
Build id: 3.10.0-957.27.2.el7.x86_64
0xae
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 26s! [kworker/u128:0:1987]
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 26s! [mysqld:7595]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables tun devlink ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_hda_codec_generic sb_edac snd_hda_intel coretemp snd_hda_codec iosf_mbi crc32_pclmul snd_hda_core ghash_clmulni_intel snd_hwdep snd_seq ppdev aesni_intel snd_seq_device vmw_balloon lrw gf128mul
glue_helper ablk_helper snd_pcm cryptd joydev snd_timer pcspkr sg snd soundcore vmw_vmci i2c_piix4 parport_pc parport xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul e1000 crct10dif_common libata crc32c_intel mptspi scsi_transport_spi serio_raw mptscsih mptbase drm_panel_orientation_quirks floppy [last unloaded: ip_tables]
CPU: 4 PID: 7595 Comm: mysqld Tainted: G        W    L ------------   3.10.0-957.27.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff94da77bea080 ti: ffff94da2b6c0000 task.ti: ffff94da2b6c0000
RIP: 0010:[<ffffffffc03c43be>]  [<ffffffffc03c43be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff94da2b6c3968  EFLAGS: 00000246
RAX: ffffb330c0740000 RBX: ffff94d9d12cd030 RCX: 0000000000000005
RDX: ffff94d9f594b800 RSI: ffff94d9f5d55000 RDI: 000000000000000e
RBP: ffff94da2b6c3978 R08: 0000000000000003 R09: 000000007c393fff
R10: ffff94da77e790d8 R11: ffff94da5731ed80 R12: ffff94d9dafefc00
R13: ffff94da2b6c38dc R14: ffffffff9fd4d6ab R15: ffff94da2b6c3920
FS:  00007fc379030700(0000) GS:ffff94da7b500000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055e31b381e88 CR3: 00000000ba8fa000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc03e671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03b42e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9fed2ee0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9fedc47c>] scsi_request_fn+0x4cc/0x680
[<ffffffff9fd694de>] ? deadline_add_request+0x6e/0x90
[<ffffffff9fd44e59>] __blk_run_queue+0x39/0x50
[<ffffffff9fd48b63>] blk_queue_bio+0x3b3/0x400
[<ffffffff9fd46af7>] generic_make_request+0x147/0x380
[<ffffffff9fd7d5be>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff9fd46da0>] submit_bio+0x70/0x150
[<ffffffff9fbc2c14>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc064c471>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc064cc74>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffff9fbc368c>] write_cache_pages+0x21c/0x470
[<ffffffffc064c7d0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc064c5cb>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff9fbc47a1>] do_writepages+0x21/0x50
[<ffffffff9fbb9045>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff9fbb9191>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0656ee6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff9fc77367>] do_fsync+0x67/0xb0
[<ffffffffa0176d15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff9fc77650>] SyS_fsync+0x10/0x20
[<ffffffffa0176ddb>] system_call_fastpath+0x22/0x27
[<ffffffffa0176d21>] ? system_call_after_swapgs+0xae/0x146
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 23 3d 
Complete report #9198
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc037b3be
Build id: 3.10.0-957.21.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc039d71e
Build id: 3.10.0-957.21.2.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03b52e0
Build id: 3.10.0-957.21.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9ccd1700
Build id: 3.10.0-957.21.2.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9ccdac9c
Build id: 3.10.0-957.21.2.el7.x86_64
0x4cc
6
? deadline_add_request
vmlinux 0xffffffff9cb67e6e
Build id: 3.10.0-957.21.2.el7.x86_64
0x6e
7
__blk_run_queue
vmlinux 0xffffffff9cb43849
Build id: 3.10.0-957.21.2.el7.x86_64
0x39
8
blk_queue_bio
vmlinux 0xffffffff9cb47553
Build id: 3.10.0-957.21.2.el7.x86_64
0x3b3
9
generic_make_request
vmlinux 0xffffffff9cb454e7
Build id: 3.10.0-957.21.2.el7.x86_64
0x147
10
? radix_tree_tag_clear
vmlinux 0xffffffff9cb7bf4e
Build id: 3.10.0-957.21.2.el7.x86_64
0x9e
11
submit_bio
vmlinux 0xffffffff9cb45790
Build id: 3.10.0-957.21.2.el7.x86_64
0x70
12
? __test_set_page_writeback
vmlinux 0xffffffff9c9c1b44
Build id: 3.10.0-957.21.2.el7.x86_64
0x54
13
xfs_submit_ioend.isra.12
xfs 0xffffffffc05ed471
Build id: 3.10.0-957.21.2.el7.x86_64
0x61
14
xfs_do_writepage
xfs 0xffffffffc05edc74
Build id: 3.10.0-957.21.2.el7.x86_64
0x4a4
15
write_cache_pages
vmlinux 0xffffffff9c9c25bc
Build id: 3.10.0-957.21.2.el7.x86_64
0x21c
16
? xfs_aops_discard_page
xfs 0xffffffffc05ed7d0
Build id: 3.10.0-957.21.2.el7.x86_64
0x150
17
xfs_vm_writepages
xfs 0xffffffffc05ed5cb
Build id: 3.10.0-957.21.2.el7.x86_64
0x6b
18
do_writepages
vmlinux 0xffffffff9c9c36d1
Build id: 3.10.0-957.21.2.el7.x86_64
0x21
19
__filemap_fdatawrite_range
vmlinux 0xffffffff9c9b7f75
Build id: 3.10.0-957.21.2.el7.x86_64
0x65
20
filemap_write_and_wait_range
vmlinux 0xffffffff9c9b80c1
Build id: 3.10.0-957.21.2.el7.x86_64
0x41
21
xfs_file_fsync
xfs 0xffffffffc05f7ee6
Build id: 3.10.0-957.21.2.el7.x86_64
0x66
22
do_fsync
vmlinux 0xffffffff9ca764f7
Build id: 3.10.0-957.21.2.el7.x86_64
0x67
23
? system_call_after_swapgs
vmlinux 0xffffffff9cf75d15
Build id: 3.10.0-957.21.2.el7.x86_64
0xa2
24
SyS_fsync
vmlinux 0xffffffff9ca767e0
Build id: 3.10.0-957.21.2.el7.x86_64
0x10
25
system_call_fastpath
vmlinux 0xffffffff9cf75ddb
Build id: 3.10.0-957.21.2.el7.x86_64
0x22
26
? system_call_after_swapgs
vmlinux 0xffffffff9cf75d21
Build id: 3.10.0-957.21.2.el7.x86_64
0xae
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [kworker/u8:1:16190]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [mysqld:8240]
Modules linked in: tcp_diag udp_diag inet_diag unix_diag xt_conntrack nf_conntrack iptable_filter coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr joydev vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw floppy ata_piix drm e1000 libata mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 8240 Comm: mysqld Kdump: loaded Tainted: G             L ------------   3.10.0-957.21.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff966b7a27b0c0 ti: ffff966b5fea8000 task.ti: ffff966b5fea8000
RIP: 0010:[<ffffffffc037b3be>]  [<ffffffffc037b3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff966b5feab968  EFLAGS: 00000246
RAX: ffffad55c0800000 RBX: ffff966b039c37b0 RCX: 0000000000000013
RDX: ffff966af5e10600 RSI: ffff966bf5abf000 RDI: 000000000000000e
RBP: ffff966b5feab978 R08: 0000000000000002 R09: 000000010580bfff
R10: ffff966bf4ff30d8 R11: ffff966b340aac40 R12: ffff966af640f800
R13: ffff966b5feab8dc R14: ffffffff9cb4c09b R15: ffff966b5feab920
FS:  00007f78a5ffb700(0000) GS:ffff966bffc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6605b80000 CR3: 0000000035932000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc039d71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03b52e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9ccd1700>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9ccdac9c>] scsi_request_fn+0x4cc/0x680
[<ffffffff9cb67e6e>] ? deadline_add_request+0x6e/0x90
[<ffffffff9cb43849>] __blk_run_queue+0x39/0x50
[<ffffffff9cb47553>] blk_queue_bio+0x3b3/0x400
[<ffffffff9cb454e7>] generic_make_request+0x147/0x380
[<ffffffff9cb7bf4e>] ? radix_tree_tag_clear+0x9e/0xe0
[<ffffffff9cb45790>] submit_bio+0x70/0x150
[<ffffffff9c9c1b44>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc05ed471>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc05edc74>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffff9c9c25bc>] write_cache_pages+0x21c/0x470
[<ffffffffc05ed7d0>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc05ed5cb>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff9c9c36d1>] do_writepages+0x21/0x50
[<ffffffff9c9b7f75>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff9c9b80c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc05f7ee6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff9ca764f7>] do_fsync+0x67/0xb0
[<ffffffff9cf75d15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff9ca767e0>] SyS_fsync+0x10/0x20
[<ffffffff9cf75ddb>] system_call_fastpath+0x22/0x27
[<ffffffff9cf75d21>] ? system_call_after_swapgs+0xae/0x146
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 93 38