Info

Permalink
Function
mpt_put_msg_frame
First occurrence
2020-09-25
Last occurrence
2020-10-10
State
NEW
Components
kernel
Quality
-26

Statistics

Operating system Count
CentOS 7 22
Architecture Count
x86_64 22
Related packages Count
kernel 22
 0:3.10.0-1160.88.1.el7 5
 0:3.10.0-862.2.3.el7 4
 0:3.10.0-1160.el7 4
 0:3.10.0-693.5.2.el7 2
 0:3.10.0-1160.66.1.el7 1
 0:3.10.0-1160.92.1.el7 1
 0:3.10.0-1160.71.1.el7 1
 0:3.10.0-1062.el7 1
 0:3.10.0-957.el7 1
 0:3.10.0-1160.102.1.el7 1
 0:3.10.0-693.17.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

No reports in the last 20 days.

No reports in the last 20 weeks.

Report backtrace


Complete report #1400638
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc006c3be
Build id: 3.10.0-693.17.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00d6729
Build id: 3.10.0-693.17.1.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffc00e12b0
Build id: 3.10.0-693.17.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8146782a
Build id: 3.10.0-693.17.1.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff814705ef
Build id: 3.10.0-693.17.1.el7.x86_64
0x4df
6
__blk_run_queue
vmlinux 0xffffffff812f96a3
Build id: 3.10.0-693.17.1.el7.x86_64
0x33
7
queue_unplugged
vmlinux 0xffffffff812f975a
Build id: 3.10.0-693.17.1.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff812fcbe8
Build id: 3.10.0-693.17.1.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffff812fcff4
Build id: 3.10.0-693.17.1.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffff811912ee
Build id: 3.10.0-693.17.1.el7.x86_64
0x1de
11
ondemand_readahead
vmlinux 0xffffffff811915e4
Build id: 3.10.0-693.17.1.el7.x86_64
0x224
12
page_cache_sync_readahead
vmlinux 0xffffffff811918e4
Build id: 3.10.0-693.17.1.el7.x86_64
0x44
13
generic_file_aio_read
vmlinux 0xffffffff81185f6b
Build id: 3.10.0-693.17.1.el7.x86_64
0x29b
14
xfs_file_buffered_aio_read
xfs 0xffffffffc02820d5
Build id: 3.10.0-693.17.1.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 26s! [oracle_1782_kon:1782]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg i2c_piix4 vmw_vmci parport_pc shpchp parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm
mptspi scsi_transport_spi crct10dif_pclmul ata_piix crct10dif_common mptscsih libata crc32c_intel serio_raw mptbase vmxnet3 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 1782 Comm: oracle_1782_kon Tainted: G             L ------------   3.10.0-693.17.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff88000b8c5ee0 ti: ffff88010a8a0000 task.ti: ffff88010a8a0000
RIP: 0010:[<ffffffffc006c3be>]  [<ffffffffc006c3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88010a8a3928  EFLAGS: 00000246
RAX: ffffc90000f40000 RBX: ffff880108596000 RCX: 0000000000000000
RDX: ffff880035e0fa00 RSI: ffff8802337d1000 RDI: 000000000000000e
RBP: ffff88010a8a3938 R08: 0000000000000002 R09: ffff88023f84f0d8
R10: ffff880117c6cc40 R11: ffff880117c6cc40 R12: ffffffff816b9bff
R13: ffffffff816b9c06 R14: ffffffff816b9c0d R15: ffffffff816b9c14
FS:  00007f239c452280(0000) GS:ffff88023fc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000001737b580c CR3: 000000010e938000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00d6729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc00e12b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8146782a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff814705ef>] scsi_request_fn+0x4df/0x680
[<ffffffff812f96a3>] __blk_run_queue+0x33/0x40
[<ffffffff812f975a>] queue_unplugged+0x2a/0xa0
[<ffffffff812fcbe8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812fcff4>] blk_finish_plug+0x14/0x40
[<ffffffff811912ee>] __do_page_cache_readahead+0x1de/0x250
[<ffffffff811915e4>] ondemand_readahead+0x224/0x230
[<ffffffff811918e4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff81185f6b>] generic_file_aio_read+0x29b/0x790
[<ffffffffc02820d5>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
Complete report #928763
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00783be
Build id: 3.10.0-693.5.2.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00a2729
Build id: 3.10.0-693.5.2.el7.x86_64
0x249
3
mptspi_qcmd
mptspi 0xffffffffc00c92b0
Build id: 3.10.0-693.5.2.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff814654da
Build id: 3.10.0-693.5.2.el7.x86_64
0xaa
5
scsi_request_fn
vmlinux 0xffffffff8146e29f
Build id: 3.10.0-693.5.2.el7.x86_64
0x4df
6
__blk_run_queue
vmlinux 0xffffffff812f7293
Build id: 3.10.0-693.5.2.el7.x86_64
0x33
7
queue_unplugged
vmlinux 0xffffffff812f734a
Build id: 3.10.0-693.5.2.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff812fa7d8
Build id: 3.10.0-693.5.2.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffff812fabe4
Build id: 3.10.0-693.5.2.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffff8118f41e
Build id: 3.10.0-693.5.2.el7.x86_64
0x1de
11
ondemand_readahead
vmlinux 0xffffffff8118f606
Build id: 3.10.0-693.5.2.el7.x86_64
0x116
12
page_cache_sync_readahead
vmlinux 0xffffffff8118fa14
Build id: 3.10.0-693.5.2.el7.x86_64
0x44
13
generic_file_aio_read
vmlinux 0xffffffff8118409b
Build id: 3.10.0-693.5.2.el7.x86_64
0x29b
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ruby:5271]
Modules linked in: binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun xt_connmark des_generic arc4 md4 nls_utf8 cifs dns_resolver ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sb_edac edac_core coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd joydev pcspkr vmw_balloon sg vmw_vmci shpchp i2c_piix4 parport_pc
parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul mptspi crct10dif_common crc32c_intel scsi_transport_spi serio_raw mptscsih mptbase vmxnet3 i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 5271 Comm: ruby Tainted: G             L ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff88004dae4f10 ti: ffff88004b394000 task.ti: ffff88004b394000
RIP: 0010:[<ffffffffc00783be>]  [<ffffffffc00783be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88004b397960  EFLAGS: 00000246
RAX: ffffc90000440000 RBX: ffff880052ed9e00 RCX: 0000000000000004
RDX: ffff88003614d200 RSI: ffff880036172000 RDI: 000000000000000e
RBP: ffff88004b397970 R08: 0000000000000002 R09: 000000004be68fff
R10: ffff880078609380 R11: ffff880078609380 R12: 0000000100000001
R13: ffff8800252af900 R14: ffff88004b3978d4 R15: ffffffff812ff1fb
FS:  00007f0548342740(0000) GS:ffff88007fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000003d173f8 CR3: 0000000041ad2000 CR4: 00000000001407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880036172d30 0000000000000038 ffff88004b397a18 ffffffffc00a2729
ffff880036172008 000000004b3979d8 0000000000000004 ffff880078609380
0000000000000010 ffff880036172188 0000006000000004 ffff8800363029b0
Call Trace:
[<ffffffffc00a2729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc00c92b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814654da>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e29f>] scsi_request_fn+0x4df/0x680
[<ffffffff812f7293>] __blk_run_queue+0x33/0x40
[<ffffffff812f734a>] queue_unplugged+0x2a/0xa0
[<ffffffff812fa7d8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812fabe4>] blk_finish_plug+0x14/0x40
[<ffffffff8118f41e>] __do_page_cache_readahead+0x1de/0x250
[<ffffffff8118f606>] ondemand_readahead+0x116/0x230
[<ffffffff8118fa14>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff8118409b>] generic_file_aio_read+0x29b/0x790
Complete report #895017
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc00b044e
Build id: 3.10.0-1160.102.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc011074e
Build id: 3.10.0-1160.102.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00cb2f0
Build id: 3.10.0-1160.102.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff8d703120
Build id: 3.10.0-1160.102.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff8d70cccc
Build id: 3.10.0-1160.102.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffff8d568799
Build id: 3.10.0-1160.102.1.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffff8d56886a
Build id: 3.10.0-1160.102.1.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff8d56c2b8
Build id: 3.10.0-1160.102.1.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffff8d56c724
Build id: 3.10.0-1160.102.1.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffff8d3d6f41
Build id: 3.10.0-1160.102.1.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffff8d3d71f2
Build id: 3.10.0-1160.102.1.el7.x86_64
0x232
12
page_cache_sync_readahead
vmlinux 0xffffffff8d3d7538
Build id: 3.10.0-1160.102.1.el7.x86_64
0x68
13
do_generic_file_read.constprop.52
vmlinux 0xffffffff8d3ca0bc
Build id: 3.10.0-1160.102.1.el7.x86_64
0xdc
14
generic_file_aio_read
vmlinux 0xffffffff8d3caef1
Build id: 3.10.0-1160.102.1.el7.x86_64
0x1d1
15
xfs_file_buffered_aio_read
xfs 0xffffffffc0337675
Build id: 3.10.0-1160.102.1.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 30s! [lfd - (child) p:65177]
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 30s! [mysqld:21081]
Modules linked in: ip6_tables xt_owner nf_log_ipv4 nf_log_common xt_connlimit nf_nat_ftp xt_REDIRECT nf_nat_redirect xt_conntrack nf_conntrack_ftp xt_LOG xt_limit xt_multiport iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul crc32c_intel ghash_clmulni_intel ppdev aesni_intel lrw vmw_balloon gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg e1000 vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod ata_generic crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix libata mptspi nfit scsi_transport_spi mptscsih
serio_raw libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 21081 Comm: mysqld Not tainted 3.10.0-1160.102.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9a2085860000 ti: ffff9a1f12054000 task.ti: ffff9a1f12054000
RIP: 0010:[<ffffffffc00b044e>]  [<ffffffffc00b044e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff9a1f12057920  EFLAGS: 00000246
RAX: ffffa98941900000 RBX: ffff9a1ea7fce700 RCX: 0000000000000007
RDX: ffff9a210525f800 RSI: ffff9a212ed59000 RDI: 000000000000000e
RBP: ffff9a1f12057930 R08: 0000000000000002 R09: 000000025e4e2fff
R10: ffff9a212f6e4000 R11: ffff9a212ef01880 R12: 0000000100000001
R13: ffff9a212fc29680 R14: ffff9a1f12057894 R15: ffffffff8d57154b
FS:  00007f9c2cb6f700(0000) GS:ffff9a2131880000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2d49f261d2 CR3: 000000032fe44000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc011074e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00cb2f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d703120>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff8d70cccc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d568799>] __blk_run_queue+0x39/0x50
[<ffffffff8d56886a>] queue_unplugged+0x2a/0xa0
[<ffffffff8d56c2b8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8d56c724>] blk_finish_plug+0x14/0x50
[<ffffffff8d3d6f41>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff8d3d71f2>] ondemand_readahead+0x232/0x240
[<ffffffff8d3d7538>] page_cache_sync_readahead+0x68/0x80
[<ffffffff8d3ca0bc>] do_generic_file_read.constprop.52+0xdc/0x510
[<ffffffff8d3caef1>] generic_file_aio_read+0x1d1/0x2a0
[<ffffffffc0337675>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
Complete report #788865
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc045944e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc048274e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc04a92f0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa1d02fe0
Build id: 3.10.0-1160.88.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa1d0cb8c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffa1b68649
Build id: 3.10.0-1160.88.1.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffffa1b6871a
Build id: 3.10.0-1160.88.1.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffa1b6c168
Build id: 3.10.0-1160.88.1.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffffa1b6c5d4
Build id: 3.10.0-1160.88.1.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffffa19d6ed1
Build id: 3.10.0-1160.88.1.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffffa19d7182
Build id: 3.10.0-1160.88.1.el7.x86_64
0x232
12
page_cache_sync_readahead
vmlinux 0xffffffffa19d74c8
Build id: 3.10.0-1160.88.1.el7.x86_64
0x68
13
do_generic_file_read.constprop.52
vmlinux 0xffffffffa19ca04c
Build id: 3.10.0-1160.88.1.el7.x86_64
0xdc
14
generic_file_aio_read
vmlinux 0xffffffffa19cae81
Build id: 3.10.0-1160.88.1.el7.x86_64
0x1d1
15
? __do_fault.isra.61
vmlinux 0xffffffffa19fa15a
Build id: 3.10.0-1160.88.1.el7.x86_64
0x8a
16
xfs_file_buffered_aio_read
xfs 0xffffffffc0696675
Build id: 3.10.0-1160.88.1.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [oracle:2436]
NMI watchdog: BUG: soft lockup - CPU#15 stuck for 27s! [oracle:2438]
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 nfit libnvdimm coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm pata_acpi drm mptspi scsi_transport_spi ata_piix mptscsih crct10dif_pclmul crct10dif_common mptbase libata crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 15 PID: 2438 Comm: oracle Kdump: loaded Tainted: G             L ------------   3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff977d79ffa100 ti: ffff977d1390c000 task.ti: ffff977d1390c000
RIP: 0010:[<ffffffffc045944e>]  [<ffffffffc045944e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff977d1390f920  EFLAGS: 00000246
RAX: ffffb4f741c00000 RBX: ffff977cbe444820 RCX: 0000000000000024
RDX: ffff977b79432e00 RSI: ffff977b795c1000 RDI: 000000000000000e
RBP: ffff977d1390f930 R08: 0000000000000003 R09: 0000000351ba7fff
R10: ffff977a78f5d400 R11: ffff977a8276a300 R12: 0000000100000002
R13: ffff977d79897300 R14: ffff977d1390f894 R15: ffffffffa1b713fb
FS:  00007f806efc2740(0000) GS:ffff977d7fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000790de00e CR3: 000000022166e000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc048274e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04a92f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa1d02fe0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffa1d0cb8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa1b68649>] __blk_run_queue+0x39/0x50
[<ffffffffa1b6871a>] queue_unplugged+0x2a/0xa0
[<ffffffffa1b6c168>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffa1b6c5d4>] blk_finish_plug+0x14/0x50
[<ffffffffa19d6ed1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffa19d7182>] ondemand_readahead+0x232/0x240
[<ffffffffa19d74c8>] page_cache_sync_readahead+0x68/0x80
[<ffffffffa19ca04c>] do_generic_file_read.constprop.52+0xdc/0x510
[<ffffffffa19cae81>] generic_file_aio_read+0x1d1/0x2a0
[<ffffffffa19fa15a>] ? __do_fault.isra.61+0x8a/0x100
[<ffffffffc0696675>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
Complete report #584480
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc03823be
Build id: 3.10.0-957.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc03f871e
Build id: 3.10.0-957.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc03f02e0
Build id: 3.10.0-957.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffa66d0960
Build id: 3.10.0-957.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffa66d9efc
Build id: 3.10.0-957.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffffa6542ff9
Build id: 3.10.0-957.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffffa65430ba
Build id: 3.10.0-957.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffa65468e8
Build id: 3.10.0-957.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffffa6546d54
Build id: 3.10.0-957.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffffa63c3601
Build id: 3.10.0-957.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffffa63c379f
Build id: 3.10.0-957.el7.x86_64
0x11f
12
page_cache_sync_readahead
vmlinux 0xffffffffa63c3bc4
Build id: 3.10.0-957.el7.x86_64
0x44
13
generic_file_aio_read
vmlinux 0xffffffffa63b7e12
Build id: 3.10.0-957.el7.x86_64
0x2c2
14
xfs_file_buffered_aio_read
xfs 0xffffffffc05f1345
Build id: 3.10.0-957.el7.x86_64
0x95
15
xfs_file_aio_read
xfs 0xffffffffc05f1b57
Build id: 3.10.0-957.el7.x86_64
0x87
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 43s! [syncd:4952]
Modules linked in: udp_diag tcp_diag inet_diag binfmt_misc vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp ppdev iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr parport_pc parport sg i2c_piix4 vmw_vmci sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom crct10dif_pclmul crct10dif_common crc32c_intel ata_generic pata_acpi serio_raw vmxnet3 vmwgfx drm_kms_helper floppy syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 4952 Comm: syncd Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9bcee0696180 ti: ffff9bcc16140000 task.ti: ffff9bcc16140000
RIP: 0010:[<ffffffffc03823be>]  [<ffffffffc03823be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9bcc16143958  EFLAGS: 00200246
RAX: ffffba6c81bc0000 RBX: ffff9bcd341eb2a0 RCX: 0000000000000020
RDX: ffff9bcc35d7da00 RSI: ffff9bd02993a000 RDI: 000000000000000e
RBP: ffff9bcc16143968 R08: 0000000000000002 R09: 00000001d0cccfff
R10: ffff9bcd79f830d8 R11: ffff9bcd0565ddc0 R12: ffff9bcf0035c180
R13: ffff9bcc161438cc R14: ffffffffa654b83b R15: ffff9bcc16143910
FS:  0000000000000000(0000) GS:ffff9bd03fd00000(0063) knlGS:00000000f12d7b40
CS:  0010 DS: 002b ES: 002b CR0: 0000000080050033
CR2: 00007fa3e10e4000 CR3: 000000000f832000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc03f871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03f02e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa66d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa66d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffa6542ff9>] __blk_run_queue+0x39/0x50
[<ffffffffa65430ba>] queue_unplugged+0x2a/0xa0
[<ffffffffa65468e8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffa6546d54>] blk_finish_plug+0x14/0x40
[<ffffffffa63c3601>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffa63c379f>] ondemand_readahead+0x11f/0x240
[<ffffffffa63c3bc4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffa63b7e12>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc05f1345>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
[<ffffffffc05f1b57>] xfs_file_aio_read+0x87/0x100 [xfs]
Complete report #580665
Frame # Function Binary Source or offset Line
1
mptscsih_qcmd
mptscsih 0xffffffffc018171e
Build id: 3.10.0-1160.el7.x86_64
0x24e
2
mptspi_qcmd
mptspi 0xffffffffc01992e0
Build id: 3.10.0-1160.el7.x86_64
0x50
3
scsi_dispatch_cmd
vmlinux 0xffffffff83ce3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
4
scsi_request_fn
vmlinux 0xffffffff83ced0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
5
__blk_run_queue
vmlinux 0xffffffff83b534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
6
queue_unplugged
vmlinux 0xffffffff83b5359a
Build id: 3.10.0-1160.el7.x86_64
0x2a
7
blk_flush_plug_list
vmlinux 0xffffffff83b56dd8
Build id: 3.10.0-1160.el7.x86_64
0x1d8
8
blk_finish_plug
vmlinux 0xffffffff83b57244
Build id: 3.10.0-1160.el7.x86_64
0x14
9
__do_page_cache_readahead
vmlinux 0xffffffff839cb261
Build id: 3.10.0-1160.el7.x86_64
0x1e1
10
ondemand_readahead
vmlinux 0xffffffff839cb3ff
Build id: 3.10.0-1160.el7.x86_64
0x11f
11
page_cache_sync_readahead
vmlinux 0xffffffff839cb840
Build id: 3.10.0-1160.el7.x86_64
0x60
12
do_generic_file_read.constprop.52
vmlinux 0xffffffff839beedc
Build id: 3.10.0-1160.el7.x86_64
0xdc
13
generic_file_aio_read
vmlinux 0xffffffff839bfcb9
Build id: 3.10.0-1160.el7.x86_64
0x1c9
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 33s! [11-dhclient:101054]
[<ffffffffc018171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01992e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff83ce3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff83ced0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffff83b534d9>] __blk_run_queue+0x39/0x50
[<ffffffff83b5359a>] queue_unplugged+0x2a/0xa0
[<ffffffff83b56dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff83b57244>] blk_finish_plug+0x14/0x40
[<ffffffff839cb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff839cb3ff>] ondemand_readahead+0x11f/0x240
[<ffffffff839cb840>] page_cache_sync_readahead+0x60/0x80
[<ffffffff839beedc>] do_generic_file_read.constprop.52+0xdc/0x510
[<ffffffff839bfcb9>] generic_file_aio_read+0x1c9/0x290
Complete report #557867
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01e93be
Build id: 3.10.0-1062.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc00e171e
Build id: 3.10.0-1062.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc027a2e0
Build id: 3.10.0-1062.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff958dc890
Build id: 3.10.0-1062.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff958e5efc
Build id: 3.10.0-1062.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff9574cf59
Build id: 3.10.0-1062.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffff9574d01a
Build id: 3.10.0-1062.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff95750858
Build id: 3.10.0-1062.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffff95750cc4
Build id: 3.10.0-1062.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffff955c91b1
Build id: 3.10.0-1062.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffff955c9462
Build id: 3.10.0-1062.el7.x86_64
0x232
12
page_cache_sync_readahead
vmlinux 0xffffffff955c9774
Build id: 3.10.0-1062.el7.x86_64
0x44
13
generic_file_aio_read
vmlinux 0xffffffff955bd912
Build id: 3.10.0-1062.el7.x86_64
0x2c2
14
? __mem_cgroup_commit_charge
vmlinux 0xffffffff95639cc2
Build id: 3.10.0-1062.el7.x86_64
0xe2
15
xfs_file_buffered_aio_read
xfs 0xffffffffc02e9b95
Build id: 3.10.0-1062.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [java:2788]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ppdev ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper sg cryptd pcspkr i2c_piix4 joydev vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi mptspi scsi_transport_spi e1000 mptscsih vmwgfx mptbase ata_piix libata drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit libnvdimm crct10dif_pclmul
crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 2788 Comm: java Kdump: loaded Tainted: G             L ------------   3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff951c023d1070 ti: ffff9514354c0000 task.ti: ffff9514354c0000
RIP: 0010:[<ffffffffc01e93be>]  [<ffffffffc01e93be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9514354c3928  EFLAGS: 00000246
RAX: ffffb35a03b40000 RBX: ffff951c1e42cff0 RCX: 0000000000000024
RDX: ffff951c19796200 RSI: ffff951c1e83b000 RDI: 000000000000000e
RBP: ffff9514354c3938 R08: 0000000000000003 R09: 00000007cd0d3fff
R10: ffff951c1e23b0d8 R11: ffff951435eb1880 R12: ffff951be66c6d00
R13: ffff9514354c389c R14: ffffffff957557ab R15: ffff9514354c38e0
FS:  00007fd738a2e700(0000) GS:ffff951c1d640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000001001af1b0 CR3: 00000007d17b2000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00e171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc027a2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff958dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff958e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff9574cf59>] __blk_run_queue+0x39/0x50
[<ffffffff9574d01a>] queue_unplugged+0x2a/0xa0
[<ffffffff95750858>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff95750cc4>] blk_finish_plug+0x14/0x40
[<ffffffff955c91b1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff955c9462>] ondemand_readahead+0x232/0x240
[<ffffffff955c9774>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff955bd912>] generic_file_aio_read+0x2c2/0x790
[<ffffffff95639cc2>] ? __mem_cgroup_commit_charge+0xe2/0x2f0
[<ffffffffc02e9b95>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
Complete report #534749
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01423be
Build id: 3.10.0-1160.71.1.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc016471e
Build id: 3.10.0-1160.71.1.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc015d2e0
Build id: 3.10.0-1160.71.1.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffbcee4680
Build id: 3.10.0-1160.71.1.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffbceeddac
Build id: 3.10.0-1160.71.1.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffbcd53d39
Build id: 3.10.0-1160.71.1.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffffbcd53dfa
Build id: 3.10.0-1160.71.1.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffbcd576f8
Build id: 3.10.0-1160.71.1.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffffbcd57b64
Build id: 3.10.0-1160.71.1.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffffbcbcbe91
Build id: 3.10.0-1160.71.1.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffffbcbcc02f
Build id: 3.10.0-1160.71.1.el7.x86_64
0x11f
12
page_cache_sync_readahead
vmlinux 0xffffffffbcbcc470
Build id: 3.10.0-1160.71.1.el7.x86_64
0x60
13
do_generic_file_read.constprop.52
vmlinux 0xffffffffbcbbf41c
Build id: 3.10.0-1160.71.1.el7.x86_64
0xdc
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ora_mz00_iis:5749]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper pcspkr sg cryptd joydev vmw_balloon parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea
sysfillrect sysimgblt libata fb_sys_fops ttm e1000 nfit crct10dif_pclmul crct10dif_common libnvdimm drm crc32c_intel mptspi scsi_transport_spi mptscsih mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 5749 Comm: ora_mz00_iis Kdump: loaded Tainted: G             L ------------   3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8ade23d21080 ti: ffff8ade3e5cc000 task.ti: ffff8ade3e5cc000
RIP: 0010:[<ffffffffc01423be>]  [<ffffffffc01423be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8ade3e5cf920  EFLAGS: 00000246
RAX: ffffa95dc0d00000 RBX: ffff8ade4a744100 RCX: 000000000000001c
RDX: ffff8adf6e637200 RSI: ffff8adf7432f000 RDI: 000000000000000e
RBP: ffff8ade3e5cf930 R08: 0000000000000003 R09: 000000000b575fff
R10: ffff8ade35f4f000 R11: ffff8adf72a10a80 R12: 0000000100000009
R13: ffff8adf1c55d680 R14: ffff8ade3e5cf894 R15: ffffffffbcd5c71b
FS:  00007f2bbb1073c0(0000) GS:ffff8adf786c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000080ec0a4f CR3: 00000000391ae000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc016471e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc015d2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffbcee4680>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffbceeddac>] scsi_request_fn+0x4ac/0x680
[<ffffffffbcd53d39>] __blk_run_queue+0x39/0x50
[<ffffffffbcd53dfa>] queue_unplugged+0x2a/0xa0
[<ffffffffbcd576f8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffbcd57b64>] blk_finish_plug+0x14/0x40
[<ffffffffbcbcbe91>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffbcbcc02f>] ondemand_readahead+0x11f/0x240
[<ffffffffbcbcc470>] page_cache_sync_readahead+0x60/0x80
[<ffffffffbcbbf41c>] do_generic_file_read.constprop.52+0xdc/0x510
Complete report #467382
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01e23be
Build id: 3.10.0-1160.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc028b71e
Build id: 3.10.0-1160.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc01fd2e0
Build id: 3.10.0-1160.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffffb44e3980
Build id: 3.10.0-1160.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffffb44ed0ac
Build id: 3.10.0-1160.el7.x86_64
0x4ac
6
__blk_run_queue
vmlinux 0xffffffffb43534d9
Build id: 3.10.0-1160.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffffb435359a
Build id: 3.10.0-1160.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffffb4356dd8
Build id: 3.10.0-1160.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffffb4357244
Build id: 3.10.0-1160.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffffb41cb261
Build id: 3.10.0-1160.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffffb41cb3ff
Build id: 3.10.0-1160.el7.x86_64
0x11f
12
page_cache_sync_readahead
vmlinux 0xffffffffb41cb840
Build id: 3.10.0-1160.el7.x86_64
0x60
13
do_generic_file_read.constprop.52
vmlinux 0xffffffffb41beedc
Build id: 3.10.0-1160.el7.x86_64
0xdc
14
generic_file_aio_read
vmlinux 0xffffffffb41bfcb9
Build id: 3.10.0-1160.el7.x86_64
0x1c9
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 175s! [node /ql/static:8110]
Modules linked in: tcp_lp nls_utf8 isofs rfcomm veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev snd_ens1371 snd_rawmidi snd_ac97_codec btusb btrtl btbcm iosf_mbi ac97_bus btintel crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device bluetooth aesni_intel snd_pcm lrw gf128mul glue_helper ablk_helper cryptd joydev snd_timer snd pcspkr vmw_balloon soundcore rfkill sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif ata_generic crct10dif_generic pata_acpi crct10dif_pclmul crct10dif_common vmwgfx crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit ttm serio_raw mptspi ata_piix scsi_transport_spi libnvdimm drm drm_panel_orientation_quirks mptscsih libata e1000 mptbase dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 8110 Comm: node /ql/static Kdump: loaded Tainted: G               ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff88e00369b180 ti: ffff88e010074000 task.ti: ffff88e010074000
RIP: 0010:[<ffffffffc01e23be>]  [<ffffffffc01e23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88e010077920  EFLAGS: 00000246
RAX: ffffa39e00400000 RBX: ffff88e00fa81120 RCX: 0000000000000005
RDX: ffff88e03513d400 RSI: ffff88e035d1f000 RDI: 000000000000000e
RBP: ffff88e010077930 R08: 0000000000000002 R09: 000000003e9c6fff
R10: ffff88e034e8b800 R11: ffff88e002d9fb80 R12: 0000000100000002
R13: ffff88e067fb6280 R14: ffff88e010077894 R15: ffffffffb435bdfb
FS:  00007fda1b49e988(0000) GS:ffff88e07b640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000d9cad9f0000 CR3: 000000003c086000 CR4: 00000000000607e0
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 175s! [kworker/3:2:7933]
Modules linked in: tcp_lp nls_utf8 isofs rfcomm veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev snd_ens1371 snd_rawmidi snd_ac97_codec btusb btrtl btbcm iosf_mbi ac97_bus btintel crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device bluetooth aesni_intel snd_pcm lrw gf128mul glue_helper ablk_helper cryptd joydev snd_timer snd pcspkr vmw_balloon soundcore rfkill sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif ata_generic crct10dif_generic pata_acpi crct10dif_pclmul crct10dif_common vmwgfx crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit ttm serio_raw mptspi ata_piix scsi_transport_spi libnvdimm drm drm_panel_orientation_quirks mptscsih libata e1000 mptbase dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 7933 Comm: kworker/3:2 Kdump: loaded Tainted: G               ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
Workqueue: xfs-sync/dm-0 xfs_log_worker [xfs]
task: ffff88e03a6c3180 ti: ffff88e075f3c000 task.ti: ffff88e075f3c000
RIP: 0010:[<ffffffffc01e23be>]  [<ffffffffc01e23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88e075f3f9c8  EFLAGS: 00000246
RAX: ffffa39e00400000 RBX: ffff88e017780410 RCX: 000000000000001b
RDX: ffff88e03513d400 RSI: ffff88e035d1f000 RDI: 000000000000000e
RBP: ffff88e075f3f9d8 R08: 0000000000000002 R09: 0000000071d3d9ff
R10: ffff88e034e8b800 R11: ffff88e002b63640 R12: ffff88e0011afd80
R13: ffff88e075f3f93c R14: ffffffffb435bdfb R15: ffff88e075f3f980
FS:  0000000000000000(0000) GS:ffff88e07b6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fda18657000 CR3: 0000000067292000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc028b71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01fd2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb44e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb44ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffb43534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb435359a>] queue_unplugged+0x2a/0xa0
[<ffffffffb4356dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb4357244>] blk_finish_plug+0x14/0x40
[<ffffffffb41cb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffb41cb3ff>] ondemand_readahead+0x11f/0x240
[<ffffffffb41cb840>] page_cache_sync_readahead+0x60/0x80
[<ffffffffb41beedc>] do_generic_file_read.constprop.52+0xdc/0x510
[<ffffffffb41bfcb9>] generic_file_aio_read+0x1c9/0x290
Complete report #459590
Frame # Function Binary Source or offset Line
1
mptscsih_qcmd
mptscsih 0xffffffffc038571e
Build id: 3.10.0-1160.66.1.el7.x86_64
0x24e
2
mptspi_qcmd
mptspi 0xffffffffc04b82e0
Build id: 3.10.0-1160.66.1.el7.x86_64
0x50
3
scsi_dispatch_cmd
vmlinux 0xffffffff8d2e4650
Build id: 3.10.0-1160.66.1.el7.x86_64
0xb0
4
scsi_request_fn
vmlinux 0xffffffff8d2edd7c
Build id: 3.10.0-1160.66.1.el7.x86_64
0x4ac
5
__blk_run_queue
vmlinux 0xffffffff8d153d09
Build id: 3.10.0-1160.66.1.el7.x86_64
0x39
6
queue_unplugged
vmlinux 0xffffffff8d153dca
Build id: 3.10.0-1160.66.1.el7.x86_64
0x2a
7
blk_flush_plug_list
vmlinux 0xffffffff8d1576c8
Build id: 3.10.0-1160.66.1.el7.x86_64
0x1d8
8
blk_finish_plug
vmlinux 0xffffffff8d157b34
Build id: 3.10.0-1160.66.1.el7.x86_64
0x14
9
__do_page_cache_readahead
vmlinux 0xffffffff8cfcbe51
Build id: 3.10.0-1160.66.1.el7.x86_64
0x1e1
10
ondemand_readahead
vmlinux 0xffffffff8cfcbfef
Build id: 3.10.0-1160.66.1.el7.x86_64
0x11f
11
page_cache_sync_readahead
vmlinux 0xffffffff8cfcc430
Build id: 3.10.0-1160.66.1.el7.x86_64
0x60
12
do_generic_file_read.constprop.52
vmlinux 0xffffffff8cfbf3dc
Build id: 3.10.0-1160.66.1.el7.x86_64
0xdc
13
generic_file_aio_read
vmlinux 0xffffffff8cfc01b9
Build id: 3.10.0-1160.66.1.el7.x86_64
0x1c9
14
xfs_file_buffered_aio_read
xfs 0xffffffffc05c8f55
Build id: 3.10.0-1160.66.1.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 48s! [vmtoolsd:1502]
Modules linked in: rfcomm xt_CHECKSUM tun devlink ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter vmhgfs(OE) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 snd_ens1371 coretemp snd_rawmidi iosf_mbi snd_ac97_codec crc32_pclmul btusb ac97_bus snd_seq btrtl ghash_clmulni_intel btbcm snd_seq_device btintel
bluetooth snd_pcm ppdev aesni_intel vmw_balloon lrw gf128mul snd_timer glue_helper ablk_helper cryptd snd joydev rfkill pcspkr sg soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ata_piix ttm mptspi crct10dif_pclmul e1000 crct10dif_common drm scsi_transport_spi mptscsih libata nfit libnvdimm crc32c_intel mptbase serio_raw drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 1502 Comm: vmtoolsd Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
Call Trace:
[<ffffffffc038571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04b82e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d2e4650>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d2edd7c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d153d09>] __blk_run_queue+0x39/0x50
[<ffffffff8d153dca>] queue_unplugged+0x2a/0xa0
[<ffffffff8d1576c8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8d157b34>] blk_finish_plug+0x14/0x40
[<ffffffff8cfcbe51>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff8cfcbfef>] ondemand_readahead+0x11f/0x240
[<ffffffff8cfcc430>] page_cache_sync_readahead+0x60/0x80
[<ffffffff8cfbf3dc>] do_generic_file_read.constprop.52+0xdc/0x510
[<ffffffff8cfc01b9>] generic_file_aio_read+0x1c9/0x290
[<ffffffffc05c8f55>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Complete report #1807
Frame # Function Binary Source or offset Line
1
mpt_put_msg_frame
mptbase 0xffffffffc01433be
Build id: 3.10.0-862.2.3.el7.x86_64
0x5e
2
mptscsih_qcmd
mptscsih 0xffffffffc016571e
Build id: 3.10.0-862.2.3.el7.x86_64
0x24e
3
mptspi_qcmd
mptspi 0xffffffffc00f82e0
Build id: 3.10.0-862.2.3.el7.x86_64
0x50
4
scsi_dispatch_cmd
vmlinux 0xffffffff9aa9aff0
Build id: 3.10.0-862.2.3.el7.x86_64
0xb0
5
scsi_request_fn
vmlinux 0xffffffff9aaa455c
Build id: 3.10.0-862.2.3.el7.x86_64
0x4cc
6
__blk_run_queue
vmlinux 0xffffffff9a918a79
Build id: 3.10.0-862.2.3.el7.x86_64
0x39
7
queue_unplugged
vmlinux 0xffffffff9a918b3a
Build id: 3.10.0-862.2.3.el7.x86_64
0x2a
8
blk_flush_plug_list
vmlinux 0xffffffff9a91c2f8
Build id: 3.10.0-862.2.3.el7.x86_64
0x1d8
9
blk_finish_plug
vmlinux 0xffffffff9a91c764
Build id: 3.10.0-862.2.3.el7.x86_64
0x14
10
__do_page_cache_readahead
vmlinux 0xffffffff9a7a11e1
Build id: 3.10.0-862.2.3.el7.x86_64
0x1e1
11
ondemand_readahead
vmlinux 0xffffffff9a7a13d6
Build id: 3.10.0-862.2.3.el7.x86_64
0x116
12
page_cache_sync_readahead
vmlinux 0xffffffff9a7a17f4
Build id: 3.10.0-862.2.3.el7.x86_64
0x44
13
generic_file_aio_read
vmlinux 0xffffffff9a795472
Build id: 3.10.0-862.2.3.el7.x86_64
0x2c2
14
xfs_file_buffered_aio_read
xfs 0xffffffffc0391d05
Build id: 3.10.0-862.2.3.el7.x86_64
0x95
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 41s! [abrt-hook-ccpp:30941]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev parport_pc nfit parport libnvdimm sg shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata serio_raw e1000 mptspi scsi_transport_spi mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 30941 Comm: abrt-hook-ccpp Kdump: loaded Tainted: G        W    L ------------   3.10.0-862.2.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9dea5e74cf10 ti: ffff9de8e68c4000 task.ti: ffff9de8e68c4000
RIP: 0010:[<ffffffffc01433be>]  [<ffffffffc01433be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9de8e68c7928  EFLAGS: 00000246
RAX: ffffba7700e80000 RBX: ffff9de90672f2a0 RCX: 000000000000005b
RDX: ffff9de8f5c04e00 RSI: ffff9dea752cc000 RDI: 000000000000000e
RBP: ffff9de8e68c7938 R08: 0000000000000002 R09: 0000000119ff3fff
R10: ffff9dea752c80d8 R11: ffff9de902c2d880 R12: ffff9dea6ff01380
R13: ffff9de8e68c789c R14: ffffffff9a92126b R15: ffff9de8e68c78e0
FS:  00007f0de43a0900(0000) GS:ffff9dea7fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdd67371eb8 CR3: 00000000a1ff0000 CR4: 00000000000607e0
Call Trace:
[<ffffffffc016571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc00f82e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9aa9aff0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9aaa455c>] scsi_request_fn+0x4cc/0x680
[<ffffffff9a918a79>] __blk_run_queue+0x39/0x50
[<ffffffff9a918b3a>] queue_unplugged+0x2a/0xa0
[<ffffffff9a91c2f8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9a91c764>] blk_finish_plug+0x14/0x40
[<ffffffff9a7a11e1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff9a7a13d6>] ondemand_readahead+0x116/0x230
[<ffffffff9a7a17f4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff9a795472>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc0391d05>] xfs_file_buffered_aio_read+0x95/0x110 [xfs]