Complete report #1984
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04563be
Build id: 3.10.0-957.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc047f71e
Build id: 3.10.0-957.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04772e0
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffab6d0960
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffab6d9efc
Build id: 3.10.0-957.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffffab5675fe
Build id: 3.10.0-957.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffab542ff9
Build id: 3.10.0-957.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffab546cf3
Build id: 3.10.0-957.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffab544c87
Build id: 3.10.0-957.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffab544f30
Build id: 3.10.0-957.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc06d7251
Build id: 3.10.0-957.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc06d73bf
Build id: 3.10.0-957.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffffab3c3181
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffab3b79b5
Build id: 3.10.0-957.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffab3b7b01
Build id: 3.10.0-957.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc06e2156
Build id: 3.10.0-957.el7.x86_64
|
0x66
|
|
17
|
generic_write_sync
|
vmlinux |
0xffffffffab475acf
Build id: 3.10.0-957.el7.x86_64
|
0x4f
|
|
18
|
xfs_file_aio_write
|
xfs |
0xffffffffc06e3d35
Build id: 3.10.0-957.el7.x86_64
|
0x155
|
|
19
|
do_sync_write
|
vmlinux |
0xffffffffab4405b3
Build id: 3.10.0-957.el7.x86_64
|
0x93
|
|
20
|
vfs_write
|
vmlinux |
0xffffffffab4410a0
Build id: 3.10.0-957.el7.x86_64
|
0xc0
|
|
21
|
SyS_pwrite64
|
vmlinux |
0xffffffffab442082
Build id: 3.10.0-957.el7.x86_64
|
0x92
|
|
22
|
system_call_fastpath
|
vmlinux |
0xffffffffab974ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 27s! [oracle:4108]
Modules linked in: binfmt_misc tcp_lp fuse tcp_diag inet_diag tun devlink ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm joydev pcspkr snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect serio_raw sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 drm_panel_orientation_quirks mptspi scsi_transport_spi mptscsih nfit mptbase libnvdimm dm_mirror
dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 1 PID: 4108 Comm: oracle Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8b295b6330c0 ti: ffff8b2917b80000 task.ti: ffff8b2917b80000
RIP: 0010:[<ffffffffc04563be>] [<ffffffffc04563be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8b2917b83978 EFLAGS: 00000246
RAX: ffff9d5641900000 RBX: ffff8b25dfa29eb0 RCX: 000000000000000d
RDX: ffff8b29a7703e00 RSI: ffff8b29a9648000 RDI: 000000000000000e
RBP: ffff8b2917b83988 R08: 0000000000000003 R09: 0000000333108fff
R10: ffff8b29a75500d8 R11: ffff8b28f7ab8e00 R12: ffff8b25dff68900
R13: ffff8b2917b838ec R14: ffffffffab54b83b R15: ffff8b2917b83930
FS: 00007fb9828f0740(0000) GS:ffff8b29ad640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f99d9b3fd68 CR3: 00000003fbf30000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc047f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04772e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffab6d0960>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffab6d9efc>] scsi_request_fn+0x4cc/0x680
[<ffffffffab5675fe>] ? deadline_add_request+0x6e/0x90
[<ffffffffab542ff9>] __blk_run_queue+0x39/0x50
[<ffffffffab546cf3>] blk_queue_bio+0x3b3/0x400
[<ffffffffab544c87>] generic_make_request+0x147/0x380
[<ffffffffab544f30>] submit_bio+0x70/0x150
[<ffffffffc06d7251>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc06d73bf>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffab3c3181>] do_writepages+0x21/0x50
[<ffffffffab3b79b5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffab3b7b01>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc06e2156>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffab475acf>] generic_write_sync+0x4f/0x70
[<ffffffffc06e3d35>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffffab4405b3>] do_sync_write+0x93/0xe0
[<ffffffffab4410a0>] vfs_write+0xc0/0x1f0
[<ffffffffab442082>] SyS_pwrite64+0x92/0xc0
[<ffffffffab974ddb>] system_call_fastpath+0x22/0x27
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 46
Complete report #1155147
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab845
Build id: 3.10.0-693.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0040632
Build id: 3.10.0-693.el7.x86_64
|
0x442
|
|
3
|
|
vmlinux |
0xffffffff8146d190
Build id: 3.10.0-693.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8146540a
Build id: 3.10.0-693.el7.x86_64
|
0xaa
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8146e1cf
Build id: 3.10.0-693.el7.x86_64
|
0x4df
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812f71d3
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
7
|
blk_queue_bio
|
vmlinux |
0xffffffff812faad3
Build id: 3.10.0-693.el7.x86_64
|
0x363
|
|
8
|
generic_make_request
|
vmlinux |
0xffffffff812f8b45
Build id: 3.10.0-693.el7.x86_64
|
0x105
|
|
9
|
submit_bio
|
vmlinux |
0xffffffff812f8dc0
Build id: 3.10.0-693.el7.x86_64
|
0x70
|
|
10
|
xfs_submit_ioend.isra.16
|
xfs |
0xffffffffc02ce5e1
Build id: 3.10.0-693.el7.x86_64
|
0x61
|
|
11
|
xfs_vm_writepages
|
xfs |
0xffffffffc02ce7a1
Build id: 3.10.0-693.el7.x86_64
|
0xd1
|
|
12
|
do_writepages
|
vmlinux |
0xffffffff8118effe
Build id: 3.10.0-693.el7.x86_64
|
0x1e
|
|
13
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff81183c55
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
14
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff81183da1
Build id: 3.10.0-693.el7.x86_64
|
0x41
|
|
15
|
xfs_file_fsync
|
xfs |
0xffffffffc02d9b16
Build id: 3.10.0-693.el7.x86_64
|
0x66
|
|
16
|
generic_write_sync
|
vmlinux |
0xffffffff8123331d
Build id: 3.10.0-693.el7.x86_64
|
0x4d
|
|
17
|
xfs_file_aio_write
|
xfs |
0xffffffffc02dbd55
Build id: 3.10.0-693.el7.x86_64
|
0x155
|
|
18
|
do_sync_write
|
vmlinux |
0xffffffff8120026d
Build id: 3.10.0-693.el7.x86_64
|
0x8d
|
|
19
|
vfs_write
|
vmlinux |
0xffffffff81200d2d
Build id: 3.10.0-693.el7.x86_64
|
0xbd
|
|
20
|
SyS_pwrite64
|
vmlinux |
0xffffffff81201cf2
Build id: 3.10.0-693.el7.x86_64
|
0x92
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 33s! [oracle:31865]
Modules linked in: tcp_lp fuse udp_diag tcp_diag inet_diag xt_comment xt_multiport xt_set ip_set_hash_ip ip_set nf_conntrack_netlink nfnetlink xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack sfesp(OE) ipt_REJECT nf_reject_ipv4 tun bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg nfit i2c_piix4 libnvdimm shpchp parport_pc vmw_vmci parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod vmwgfx crc_t10dif crct10dif_generic
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000e ahci libahci crct10dif_pclmul ata_piix crct10dif_common crc32c_intel serio_raw ptp libata pps_core i2c_core vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 10 PID: 31865 Comm: oracle Tainted: G W OEL ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/09/2019
task: ffff88082aa11fa0 ti: ffff8807f5b68000 task.ti: ffff8807f5b68000
RIP: 0010:[<ffffffff816ab845>] [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8807f5b6ba08 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8807f5b6b9c8 RCX: 000000042c358fff
RDX: ffffc90003190000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8807f5b6ba08 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8801e223f480 R11: ffff88043fe9b800 R12: 0000000000000020
R13: ffff8803fae32400 R14: ffffffff8146bdb9 R15: ffff8807f5b6b980
FS: 00007fb01c1db740(0000) GS:ffff88042fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f22ec037d80 CR3: 000000077666c000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8807f5b6ba80 ffffffffc0040632 ffff88080f395b00 ffff88042c7e1000
ffff88042c2ceec0 ffff88042c298fe8 ffffffff8146d190 0000000000000002
ffff88042c298f54 0000000000000286 ffff8801e223f480 0000000000000000
Call Trace:
[<ffffffffc0040632>] pvscsi_queue+0x442/0x7e1 [vmw_pvscsi]
[<ffffffff8146d190>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8146540a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e1cf>] scsi_request_fn+0x4df/0x680
[<ffffffff812f71d3>] __blk_run_queue+0x33/0x40
[<ffffffff812faad3>] blk_queue_bio+0x363/0x3a0
[<ffffffff812f8b45>] generic_make_request+0x105/0x310
[<ffffffff812f8dc0>] submit_bio+0x70/0x150
[<ffffffffc02ce5e1>] xfs_submit_ioend.isra.16+0x61/0xe0 [xfs]
[<ffffffffc02ce7a1>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff8118effe>] do_writepages+0x1e/0x40
[<ffffffff81183c55>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81183da1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc02d9b16>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff8123331d>] generic_write_sync+0x4d/0x60
[<ffffffffc02dbd55>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffff8120026d>] do_sync_write+0x8d/0xd0
[<ffffffff81200d2d>] vfs_write+0xbd/0x1e0
[<ffffffff81201cf2>] SyS_pwrite64+0x92/0xc0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #946675
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff91dba385
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc01846a2
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffff91b0b630
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff91b02ae0
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff91b0c68c
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff9198db9e
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff91968159
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff9196c083
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff91969eb7
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x157
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff9196a160
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffff918cb070
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x1e0
|
|
12
|
|
vmlinux |
0xffffffff918cb55d
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xed
|
|
13
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc03cf341
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x61
|
|
14
|
xfs_vm_writepages
|
xfs |
0xffffffffc03cf68f
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x7f
|
|
15
|
do_writepages
|
vmlinux |
0xffffffff917d6921
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x21
|
|
16
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff917ca9c5
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x65
|
|
17
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff917cab31
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x41
|
|
18
|
xfs_file_fsync
|
xfs |
0xffffffffc03da486
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x66
|
|
19
|
generic_write_sync
|
vmlinux |
0xffffffff91892a4f
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x4f
|
|
20
|
xfs_file_aio_write
|
xfs |
0xffffffffc03dc235
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x155
|
|
21
|
do_sync_write
|
vmlinux |
0xffffffff9185ae73
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x93
|
|
22
|
vfs_write
|
vmlinux |
0xffffffff9185b940
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0xc0
|
|
23
|
SyS_pwrite64
|
vmlinux |
0xffffffff9185c8c2
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x92
|
|
24
|
unload_network_ops_symbols
|
falcon_lsm_pinned_16108 |
0xffffffffc0654e1b
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x852b
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffff91dc539a
Build id: 3.10.0-1160.83.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ora_ckpt_xe:26639]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_16108(E) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter ipmi_devintf ipmi_msghandler vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon pcspkr joydev sg vmw_vmci parport_pc parport i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_common vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm crc32c_intel drm ata_piix serio_raw nfit libnvdimm libata vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 26639 Comm: ora_ckpt_xe Kdump: loaded Tainted: P EL ------------ 3.10.0-1160.83.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff905192ee6300 ti: ffff9051efbe0000 task.ti: ffff9051efbe0000
RIP: 0010:[<ffffffff91dba385>] [<ffffffff91dba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9051efbe3970 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff919c9843 RCX: ffffffff92234f40
RDX: ffffa9eb41b80000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9051efbe3970 R08: 0000000000000001 R09: 0000000000000000
R10: ffff90526d399c00 R11: ffff90526eadf000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff904fbcaf1500 R15: ffffffff91b0a269
FS: 00007f3c117a0f40(0000) GS:ffff90527fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5b69250d70 CR3: 00000002afc22000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc01846a2>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff91b0b630>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff91b02ae0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff91b0c68c>] scsi_request_fn+0x4ac/0x680
[<ffffffff9198db9e>] ? deadline_add_request+0x6e/0x90
[<ffffffff91968159>] __blk_run_queue+0x39/0x50
[<ffffffff9196c083>] blk_queue_bio+0x3b3/0x400
[<ffffffff91969eb7>] generic_make_request+0x157/0x390
[<ffffffff9196a160>] submit_bio+0x70/0x150
[<ffffffff918cb070>] ? iomap_dirty_actor+0x1e0/0x1e0
[<ffffffff918cb55d>] ? iomap_apply+0xed/0x160
[<ffffffffc03cf341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc03cf68f>] xfs_vm_writepages+0x7f/0xb0 [xfs]
[<ffffffff917d6921>] do_writepages+0x21/0x50
[<ffffffff917ca9c5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffff917cab31>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc03da486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff91892a4f>] generic_write_sync+0x4f/0x70
[<ffffffffc03dc235>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffff9185ae73>] do_sync_write+0x93/0xe0
[<ffffffff9185b940>] vfs_write+0xc0/0x1f0
[<ffffffff9185c8c2>] SyS_pwrite64+0x92/0xc0
[<ffffffffc0654e1b>] unload_network_ops_symbols+0x852b/0xa670 [falcon_lsm_pinned_16108]
[<ffffffff91dc539a>] system_call_fastpath+0x25/0x2a
Code: 07 00 66 66 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 cc cc cc cc 66 66 66 66 90 55 48 89 e5 48
Complete report #880916
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc024644e
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc026674e
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03092f0
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff95f02a90
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff95f0c52c
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff95d8db5e
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff95d68119
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff95d6c043
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff95d69e77
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x157
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff95d6a120
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffff95ccb030
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x1e0
|
|
12
|
|
vmlinux |
0xffffffff95ccb51d
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xed
|
|
13
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc04bd341
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x61
|
|
14
|
xfs_vm_writepages
|
xfs |
0xffffffffc04bd68f
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x7f
|
|
15
|
do_writepages
|
vmlinux |
0xffffffff95bd68e1
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x21
|
|
16
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff95bca985
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x65
|
|
17
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff95bcaaf1
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x41
|
|
18
|
xfs_file_fsync
|
xfs |
0xffffffffc04c8486
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x66
|
|
19
|
generic_write_sync
|
vmlinux |
0xffffffff95c92a0f
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x4f
|
|
20
|
xfs_file_aio_write
|
xfs |
0xffffffffc04ca235
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x155
|
|
21
|
do_sync_write
|
vmlinux |
0xffffffff95c5ae33
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x93
|
|
22
|
vfs_write
|
vmlinux |
0xffffffff95c5b900
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xc0
|
|
23
|
SyS_pwrite64
|
vmlinux |
0xffffffff95c5c882
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x92
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffff961c539a
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 23s! [ora_lg00_toppro:2360]
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul snd_ens1371 glue_helper ablk_helper cryptd vmw_balloon snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm joydev pcspkr snd_timer sg snd soundcore i2c_piix4 vmw_vmci parport_pc parport binfmt_misc fuse ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul ata_piix crct10dif_common drm crc32c_intel mptspi libata scsi_transport_spi nfit e1000 mptscsih serio_raw mptbase libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 8 PID: 2360 Comm: ora_lg00_toppro Kdump: loaded Tainted: G L ------------ 3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8810a56b4200 ti: ffff880c1c980000 task.ti: ffff880c1c980000
RIP: 0010:[<ffffffffc024644e>] [<ffffffffc024644e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff880c1c983978 EFLAGS: 00000246
RAX: ffff95d5031c0000 RBX: ffff880cd29d4010 RCX: 0000000000000002
RDX: ffff8810a4cb5200 RSI: ffff8810a5fd1000 RDI: 000000000000000e
RBP: ffff880c1c983988 R08: 0000000000000003 R09: 000000006dc29fff
R10: ffff8810a583d000 R11: ffff8810a35c6140 R12: ffff880f04c6a580
R13: ffff880c1c9838ec R14: ffffffff95d70ecb R15: ffff880c1c983930
FS: 00007faca4593f40(0000) GS:ffff8810a9800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fff480ce418 CR3: 0000000451956000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc026674e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03092f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff95f02a90>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff95f0c52c>] scsi_request_fn+0x4ac/0x680
[<ffffffff95d8db5e>] ? deadline_add_request+0x6e/0x90
[<ffffffff95d68119>] __blk_run_queue+0x39/0x50
[<ffffffff95d6c043>] blk_queue_bio+0x3b3/0x400
[<ffffffff95d69e77>] generic_make_request+0x157/0x390
[<ffffffff95d6a120>] submit_bio+0x70/0x150
[<ffffffff95ccb030>] ? iomap_dirty_actor+0x1e0/0x1e0
[<ffffffff95ccb51d>] ? iomap_apply+0xed/0x160
[<ffffffffc04bd341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc04bd68f>] xfs_vm_writepages+0x7f/0xb0 [xfs]
[<ffffffff95bd68e1>] do_writepages+0x21/0x50
[<ffffffff95bca985>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffff95bcaaf1>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc04c8486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff95c92a0f>] generic_write_sync+0x4f/0x70
[<ffffffffc04ca235>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffff95c5ae33>] do_sync_write+0x93/0xe0
[<ffffffff95c5b900>] vfs_write+0xc0/0x1f0
[<ffffffff95c5c882>] SyS_pwrite64+0x92/0xc0
[<ffffffff961c539a>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #729876
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb85ba385
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x15
|
|
2
|
ata_scsi_queuecmd
|
libata |
0xffffffffc0369275
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x155
|
|
3
|
|
libata |
0xffffffffc0365e60
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb8302fe0
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb830cb8c
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffb818e08e
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffb8168649
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffb816c573
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffb816a3a7
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x157
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffb816a650
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffffb80cb560
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x1e0
|
|
12
|
|
vmlinux |
0xffffffffb80cba4d
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xed
|
|
13
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc052f341
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x61
|
|
14
|
xfs_vm_writepages
|
xfs |
0xffffffffc052f68f
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x7f
|
|
15
|
do_writepages
|
vmlinux |
0xffffffffb7fd6a41
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x21
|
|
16
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffb7fcaae5
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x65
|
|
17
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffb7fcac51
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x41
|
|
18
|
xfs_file_fsync
|
xfs |
0xffffffffc053a486
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x66
|
|
19
|
generic_write_sync
|
vmlinux |
0xffffffffb8092f3f
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x4f
|
|
20
|
xfs_file_aio_write
|
xfs |
0xffffffffc053c235
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x155
|
|
21
|
do_sync_write
|
vmlinux |
0xffffffffb805b1f3
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x93
|
|
22
|
vfs_write
|
vmlinux |
0xffffffffb805bcc0
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xc0
|
|
23
|
SyS_pwrite64
|
vmlinux |
0xffffffffb805cc42
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x92
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffffb85c539a
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 37s! [ora_j004_ddms:7687]
Modules linked in: tcp_lp vboxsf(OE) sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper joydev cryptd vboxguest(OE) pcspkr i2c_piix4 sg parport_pc video parport sch_fq_codel binfmt_misc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common e1000 crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 7687 Comm: ora_j004_ddms Tainted: G OEL ------------ 3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff977838a92100 ti: ffff97782e594000 task.ti: ffff97782e594000
RIP: 0010:[<ffffffffb85ba385>] [<ffffffffb85ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff97782e597a10 EFLAGS: 00010282
RAX: 0000000000000001 RBX: ffffffffc037530a RCX: 0000d830c289951f
RDX: 000000000000d000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff97782e597a10 R08: 000000000000a000 R09: 000000002fc89fff
R10: ffff977735ca5c00 R11: ffff977735676fc0 R12: ffff9779123d8158
R13: ffffffffc03722fb R14: ffff9779123d8158 R15: ffffffffc0373726
FS: 00007f067c15c280(0000) GS:ffff97791fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000f4deb5f0 CR3: 000000012e614000 CR4: 00000000000206e0
Call Trace:
[<ffffffffc0369275>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc0365e60>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffffb8302fe0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb830cb8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb818e08e>] ? deadline_add_request+0x6e/0x90
[<ffffffffb8168649>] __blk_run_queue+0x39/0x50
[<ffffffffb816c573>] blk_queue_bio+0x3b3/0x400
[<ffffffffb816a3a7>] generic_make_request+0x157/0x390
[<ffffffffb816a650>] submit_bio+0x70/0x150
[<ffffffffb80cb560>] ? iomap_dirty_actor+0x1e0/0x1e0
[<ffffffffb80cba4d>] ? iomap_apply+0xed/0x160
[<ffffffffc052f341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc052f68f>] xfs_vm_writepages+0x7f/0xb0 [xfs]
[<ffffffffb7fd6a41>] do_writepages+0x21/0x50
[<ffffffffb7fcaae5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffffb7fcac51>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc053a486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffb8092f3f>] generic_write_sync+0x4f/0x70
[<ffffffffc053c235>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffffb805b1f3>] do_sync_write+0x93/0xe0
[<ffffffffb805bcc0>] vfs_write+0xc0/0x1f0
[<ffffffffb805cc42>] SyS_pwrite64+0x92/0xc0
[<ffffffffb85c539a>] system_call_fastpath+0x25/0x2a
Code: 1a 42 96 ff 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 fa 41 96 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #536017
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00ed3be
Build id: 3.10.0-693.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc0049729
Build id: 3.10.0-693.el7.x86_64
|
0x249
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01642b0
Build id: 3.10.0-693.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8146540a
Build id: 3.10.0-693.el7.x86_64
|
0xaa
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8146e1cf
Build id: 3.10.0-693.el7.x86_64
|
0x4df
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812f71d3
Build id: 3.10.0-693.el7.x86_64
|
0x33
|
|
7
|
blk_queue_bio
|
vmlinux |
0xffffffff812faad3
Build id: 3.10.0-693.el7.x86_64
|
0x363
|
|
8
|
generic_make_request
|
vmlinux |
0xffffffff812f8b45
Build id: 3.10.0-693.el7.x86_64
|
0x105
|
|
9
|
submit_bio
|
vmlinux |
0xffffffff812f8dc0
Build id: 3.10.0-693.el7.x86_64
|
0x70
|
|
10
|
xfs_submit_ioend.isra.16
|
xfs |
0xffffffffc02865e1
Build id: 3.10.0-693.el7.x86_64
|
0x61
|
|
11
|
xfs_vm_writepages
|
xfs |
0xffffffffc02867a1
Build id: 3.10.0-693.el7.x86_64
|
0xd1
|
|
12
|
do_writepages
|
vmlinux |
0xffffffff8118effe
Build id: 3.10.0-693.el7.x86_64
|
0x1e
|
|
13
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff81183c55
Build id: 3.10.0-693.el7.x86_64
|
0x65
|
|
14
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff81183da1
Build id: 3.10.0-693.el7.x86_64
|
0x41
|
|
15
|
xfs_file_fsync
|
xfs |
0xffffffffc0291b16
Build id: 3.10.0-693.el7.x86_64
|
0x66
|
|
16
|
generic_write_sync
|
vmlinux |
0xffffffff8123331d
Build id: 3.10.0-693.el7.x86_64
|
0x4d
|
|
17
|
xfs_file_aio_write
|
xfs |
0xffffffffc0293d55
Build id: 3.10.0-693.el7.x86_64
|
0x155
|
|
18
|
do_sync_write
|
vmlinux |
0xffffffff8120026d
Build id: 3.10.0-693.el7.x86_64
|
0x8d
|
|
19
|
vfs_write
|
vmlinux |
0xffffffff81200d2d
Build id: 3.10.0-693.el7.x86_64
|
0xbd
|
|
20
|
SyS_pwrite64
|
vmlinux |
0xffffffff81201cf2
Build id: 3.10.0-693.el7.x86_64
|
0x92
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [ora_mmon_pdb:60286]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [ora_ckpt_pdb:60206]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 aesni_intel snd_rawmidi lrw snd_ac97_codec gf128mul glue_helper ac97_bus ablk_helper snd_seq cryptd snd_seq_device btusb snd_pcm btrtl btbcm ppdev snd_timer btintel bluetooth snd soundcore rfkill vmw_balloon pcspkr joydev sg nfit libnvdimm parport_pc parport vmw_vmci i2c_piix4 shpchp nfsd auth_rpcgss nfs_acl
lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih mptbase e1000 crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 60206 Comm: ora_ckpt_pdb Tainted: G L ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff88017cb0dee0 ti: ffff880087b28000 task.ti: ffff880087b28000
RIP: 0010:[<ffffffffc00ed3be>] [<ffffffffc00ed3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff880087b2b9a0 EFLAGS: 00000246
RAX: ffffc90000d00000 RBX: ffff8801ea213fe0 RCX: 0000000000000033
RDX: ffff88003651c800 RSI: ffff88022ce33000 RDI: 000000000000000e
RBP: ffff880087b2b9b0 R08: 0000000000000003 R09: ffff88022cd700d8
R10: ffff8801d264cfc0 R11: ffff8801d264cfc0 R12: 0000000100000080
R13: ffff88021f3d8300 R14: ffff880087b2b914 R15: ffffffff812ff13b
FS: 00007fcbd9f46140(0000) GS:ffff88022f600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcbd9f5e000 CR3: 0000000036704000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88022ce33d30 0000000000000048 ffff880087b2ba58 ffffffffc0049729
ffff88022ce33008 0000000087b2ba18 0000000000000033 ffff8801d264cfc0
00000000000000cc ffff88022ce33188 0000006000000033 ffff880200000007
Call Trace:
[<ffffffffc0049729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc01642b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff8146540a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8146e1cf>] scsi_request_fn+0x4df/0x680
[<ffffffff812f71d3>] __blk_run_queue+0x33/0x40
[<ffffffff812faad3>] blk_queue_bio+0x363/0x3a0
[<ffffffff812f8b45>] generic_make_request+0x105/0x310
[<ffffffff812f8dc0>] submit_bio+0x70/0x150
[<ffffffffc02865e1>] xfs_submit_ioend.isra.16+0x61/0xe0 [xfs]
[<ffffffffc02867a1>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff8118effe>] do_writepages+0x1e/0x40
[<ffffffff81183c55>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81183da1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0291b16>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff8123331d>] generic_write_sync+0x4d/0x60
[<ffffffffc0293d55>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffff8120026d>] do_sync_write+0x8d/0xd0
[<ffffffff81200d2d>] vfs_write+0xbd/0x1e0
[<ffffffff81201cf2>] SyS_pwrite64+0x92/0xc0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 0f
Complete report #160405
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa638a3f5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc01ab632
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffffa60ec580
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa60e3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa60ed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffa5f7846e
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffa5f53869
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffa5f57673
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffa5f554f7
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffa5f557a0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffffa5ebb1e0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x1d0
|
|
12
|
|
vmlinux |
0xffffffffa5ebb6cd
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xed
|
|
13
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0401f31
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x61
|
|
14
|
xfs_vm_writepages
|
xfs |
0xffffffffc040226f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x7f
|
|
15
|
do_writepages
|
vmlinux |
0xffffffffa5dcaf51
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x21
|
|
16
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffa5dbfb45
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x65
|
|
17
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffa5dbfc91
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x41
|
|
18
|
xfs_file_fsync
|
xfs |
0xffffffffc040cd66
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x66
|
|
19
|
generic_write_sync
|
vmlinux |
0xffffffffa5e83a3f
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4f
|
|
20
|
xfs_file_aio_write
|
xfs |
0xffffffffc040eaa5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x155
|
|
21
|
do_sync_write
|
vmlinux |
0xffffffffa5e4d243
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x93
|
|
22
|
vfs_write
|
vmlinux |
0xffffffffa5e4dd30
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xc0
|
|
23
|
SyS_pwrite64
|
vmlinux |
0xffffffffa5e4ecd2
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x92
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffffa6394f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 28s! [ora_ckpt_xe:2232]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd pcspkr joydev sg vmw_vmci parport_pc parport i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx
sd_mod crc_t10dif drm_kms_helper crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit libnvdimm ahci e1000e crct10dif_pclmul crct10dif_common ata_piix libahci crc32c_intel serio_raw ptp pps_core vmw_pvscsi libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 2232 Comm: ora_ckpt_xe Kdump: loaded Tainted: G L ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff95a379fa9080 ti: ffff95a2b3cac000 task.ti: ffff95a2b3cac000
RIP: 0010:[<ffffffffa638a3f5>] [<ffffffffa638a3f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff95a2b3caf9e0 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffffa5fb2803 RCX: ffff95a374bcb060
RDX: ffffabc9407f0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff95a2b3caf9e0 R08: 0000000000000000 R09: ffff95a284f18300
R10: ffff95a274a29400 R11: ffff95a36edae000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff95a366100c00 R15: ffffffffa60eb239
FS: 00007f1a72646980(0000) GS:ffff95a37fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd2ef508000 CR3: 000000013b1a8000 CR4: 00000000007607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffc01ab632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffa60ec580>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffa60e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa60ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa5f7846e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5f53869>] __blk_run_queue+0x39/0x50
[<ffffffffa5f57673>] blk_queue_bio+0x3b3/0x400
[<ffffffffa5f554f7>] generic_make_request+0x147/0x380
[<ffffffffa5f557a0>] submit_bio+0x70/0x150
[<ffffffffa5ebb1e0>] ? iomap_dirty_actor+0x1d0/0x1d0
[<ffffffffa5ebb6cd>] ? iomap_apply+0xed/0x160
[<ffffffffc0401f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc040226f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffa5dcaf51>] do_writepages+0x21/0x50
[<ffffffffa5dbfb45>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffa5dbfc91>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc040cd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa5e83a3f>] generic_write_sync+0x4f/0x70
[<ffffffffc040eaa5>] xfs_file_aio_write+0x155/0x1b0 [xfs]
[<ffffffffa5e4d243>] do_sync_write+0x93/0xe0
[<ffffffffa5e4dd30>] vfs_write+0xc0/0x1f0
[<ffffffffa5e4ecd2>] SyS_pwrite64+0x92/0xc0
[<ffffffffa6394f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48