Complete report #918953
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00cd44e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc00a874e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01462f0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb1903160
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb190cd0c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb1768799
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffffb176886a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffffb176c2b8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffffb176c724
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffb15d6f41
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffffb15d7571
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffffb15cbbe5
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x105
|
|
13
|
__xfs_filemap_fault
|
xfs |
0xffffffffc0306ffe
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x7e
|
|
14
|
|
vmlinux |
0xffffffffb1a8c920
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x70
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc030720c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffffb15fa1ca
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffb15fa77c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffffb1602183
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xa33
|
|
19
|
|
vmlinux |
0xffffffffb1674655
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffffb1bbf7d3
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffffb1bbfb05
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffffb1bbb7b8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [sshd:1227]
Modules linked in: nf_nat_ftp nf_conntrack_ftp ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev nfit iosf_mbi crc32_pclmul libnvdimm ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev vmw_balloon parport_pc parport sg vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_piix mptspi scsi_transport_spi mptscsih libata mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 1227 Comm: sshd Not tainted 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9e61f1406300 ti: ffff9e61f2240000 task.ti: ffff9e61f2240000
RIP: 0010:[<ffffffffc00cd44e>] [<ffffffffc00cd44e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0000:ffff9e61f2243918 EFLAGS: 00000246
RAX: ffffb6bdc0780000 RBX: ffff9e61f217b940 RCX: 000000000000001a
RDX: ffff9e6273724c00 RSI: ffff9e6278696000 RDI: 000000000000000e
RBP: ffff9e61f2243928 R08: 0000000000000002 R09: 0000000130947fff
R10: ffff9e62737e6c00 R11: ffff9e6276acafc0 R12: ffff9e61f2353980
R13: ffff9e61f224388c R14: ffffffffb177154b R15: ffff9e61f22438d0
FS: 00007f4d4eb018c0(0000) GS:ffff9e627fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055b870d7ad70 CR3: 00000000b2038000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc00a874e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01462f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb1903160>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffb190cd0c>] scsi_request_fn+0x4ac/0x680
[<ffffffffb1768799>] __blk_run_queue+0x39/0x50
[<ffffffffb176886a>] queue_unplugged+0x2a/0xa0
[<ffffffffb176c2b8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb176c724>] blk_finish_plug+0x14/0x50
[<ffffffffb15d6f41>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffb15d7571>] ra_submit+0x21/0x30
[<ffffffffb15cbbe5>] filemap_fault+0x105/0x420
[<ffffffffc0306ffe>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffb1a8c920>] ? dst_alloc+0x70/0xb0
[<ffffffffc030720c>] xfs_filemap_fault+0x2c/0x40 [xfs]
[<ffffffffb15fa1ca>] __do_fault.isra.61+0x8a/0x100
[<ffffffffb15fa77c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffb1602183>] handle_mm_fault+0xa33/0x1190
[<ffffffffb1674655>] ? __d_free+0x35/0x40
[<ffffffffb1bbf7d3>] __do_page_fault+0x213/0x510
[<ffffffffb1bbfb05>] do_page_fault+0x35/0x90
[<ffffffffb1bbb7b8>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #254627
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01fd3be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc021871e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02632e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffae2e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffae2ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffae1534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffffae15359a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffffae156dd8
Build id: 3.10.0-1160.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffffae157244
Build id: 3.10.0-1160.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffadfcb261
Build id: 3.10.0-1160.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffffadfcb881
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffffadfc0975
Build id: 3.10.0-1160.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffffae05bff8
Build id: 3.10.0-1160.el7.x86_64
|
0x238
|
|
14
|
|
vmlinux |
0xffffffffadea32d2
Build id: 3.10.0-1160.el7.x86_64
|
0x12
|
|
15
|
__xfs_filemap_fault
|
xfs |
0xffffffffc049b91e
Build id: 3.10.0-1160.el7.x86_64
|
0x7e
|
|
16
|
xfs_filemap_fault
|
xfs |
0xffffffffc049bb1c
Build id: 3.10.0-1160.el7.x86_64
|
0x2c
|
|
17
|
__do_fault.isra.61
|
vmlinux |
0xffffffffadfede3a
Build id: 3.10.0-1160.el7.x86_64
|
0x8a
|
|
18
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffadfee3ec
Build id: 3.10.0-1160.el7.x86_64
|
0x4c
|
|
19
|
|
vmlinux |
0xffffffffae05fb5d
Build id: 3.10.0-1160.el7.x86_64
|
0x3d
|
|
20
|
handle_mm_fault
|
vmlinux |
0xffffffffadff5c30
Build id: 3.10.0-1160.el7.x86_64
|
0xa20
|
|
21
|
__do_page_fault
|
vmlinux |
0xffffffffae58e653
Build id: 3.10.0-1160.el7.x86_64
|
0x213
|
|
22
|
do_page_fault
|
vmlinux |
0xffffffffae58e975
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
23
|
page_fault
|
vmlinux |
0xffffffffae58a778
Build id: 3.10.0-1160.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s! [java:12619]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel ppdev snd_ens1371 btusb btrtl btbcm snd_rawmidi btintel snd_ac97_codec ac97_bus bluetooth snd_seq lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device pcspkr snd_pcm vmw_balloon sg snd_timer rfkill joydev snd soundcore parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod
crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix nfit drm crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel libata mptspi scsi_transport_spi serio_raw e1000 mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 12619 Comm: java Kdump: loaded Not tainted 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff97f4501d0000 ti: ffff97f4529f0000 task.ti: ffff97f4529f0000
RIP: 0010:[<ffffffffc01fd3be>] [<ffffffffc01fd3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff97f4529f3918 EFLAGS: 00000246
RAX: ffffbcfb40440000 RBX: ffff97f4968bf890 RCX: 000000000000001e
RDX: ffff97f4759ac200 RSI: ffff97f4754e1000 RDI: 000000000000000e
RBP: ffff97f4529f3928 R08: 0000000000000003 R09: 0000000053159fff
R10: ffff97f4759ef800 R11: ffff97f4b36e2680 R12: ffff97f446323a80
R13: ffff97f4529f388c R14: ffffffffae15bdfb R15: ffff97f4529f38d0
FS: 0000000000000000(0000) GS:ffff97f4bb600000(0063) knlGS:00000000e109bb40
CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033
CR2: 00000000e375ae27 CR3: 000000001291a000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc021871e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02632e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffae2e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffae2ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffae1534d9>] __blk_run_queue+0x39/0x50
[<ffffffffae15359a>] queue_unplugged+0x2a/0xa0
[<ffffffffae156dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffae157244>] blk_finish_plug+0x14/0x40
[<ffffffffadfcb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffadfcb881>] ra_submit+0x21/0x30
[<ffffffffadfc0975>] filemap_fault+0x105/0x420
[<ffffffffae05bff8>] ? link_path_walk+0x238/0x8b0
[<ffffffffadea32d2>] ? current_fs_time+0x12/0x60
[<ffffffffc049b91e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffc049bb1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffffadfede3a>] __do_fault.isra.61+0x8a/0x100
[<ffffffffadfee3ec>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffae05fb5d>] ? putname+0x3d/0x60
[<ffffffffadff5c30>] handle_mm_fault+0xa20/0xfb0
[<ffffffffae58e653>] __do_page_fault+0x213/0x500
[<ffffffffae58e975>] do_page_fault+0x35/0x90
[<ffffffffae58a778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 20
Complete report #6485
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffa00893be
Build id: 3.10.0-327.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffa00a4729
Build id: 3.10.0-327.el7.x86_64
|
0x249
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffa00fb2b0
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff81417b1a
Build id: 3.10.0-327.el7.x86_64
|
0xaa
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff81420aa1
Build id: 3.10.0-327.el7.x86_64
|
0x501
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812c73e3
Build id: 3.10.0-327.el7.x86_64
|
0x33
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff812c749a
Build id: 3.10.0-327.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff812cbd18
Build id: 3.10.0-327.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff812cc124
Build id: 3.10.0-327.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff81175cee
Build id: 3.10.0-327.el7.x86_64
|
0x1de
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff81176321
Build id: 3.10.0-327.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff8116b7ed
Build id: 3.10.0-327.el7.x86_64
|
0x11d
|
|
13
|
xfs_filemap_fault
|
xfs |
0xffffffffa02471df
Build id: 3.10.0-327.el7.x86_64
|
0x4f
|
|
14
|
__do_fault
|
vmlinux |
0xffffffff81192b2e
Build id: 3.10.0-327.el7.x86_64
|
0x7e
|
|
15
|
handle_mm_fault
|
vmlinux |
0xffffffff81197088
Build id: 3.10.0-327.el7.x86_64
|
0x5b8
|
|
16
|
|
vmlinux |
0xffffffff8122bb37
Build id: 3.10.0-327.el7.x86_64
|
0x67
|
|
17
|
|
vmlinux |
0xffffffff810c14bc
Build id: 3.10.0-327.el7.x86_64
|
0xcc
|
|
18
|
__do_page_fault
|
vmlinux |
0xffffffff81640e22
Build id: 3.10.0-327.el7.x86_64
|
0x152
|
|
19
|
do_page_fault
|
vmlinux |
0xffffffff81641113
Build id: 3.10.0-327.el7.x86_64
|
0x23
|
|
20
|
page_fault
|
vmlinux |
0xffffffff8163d408
Build id: 3.10.0-327.el7.x86_64
|
0x28
|
BUG: soft lockup - CPU#1 stuck for 56s! [gnome-shell:14635]
Modules linked in: rfcomm fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ip6t_REJECT ipt_REJECT xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter bnep vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 coretemp snd_rawmidi snd_ac97_codec ppdev ac97_bus snd_seq snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel lrw gf128mul snd_timer glue_helper btusb ablk_helper pcspkr cryptd bluetooth snd vmw_balloon soundcore rfkill sg parport_pc
parport shpchp vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper ttm drm mptspi e1000 i2c_core ata_piix scsi_transport_spi mptscsih mptbase libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14635 Comm: gnome-shell Tainted: G L ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff880013d2e780 ti: ffff88001baf8000 task.ti: ffff88001baf8000
RIP: 0010:[<ffffffffa00893be>] [<ffffffffa00893be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff88001bafba08 EFLAGS: 00000246
RAX: ffffc90008280000 RBX: ffff88003621f7f0 RCX: 0000000000000005
RDX: ffff8800363d6600 RSI: ffff880036389000 RDI: 000000000000000e
RBP: ffff88001bafba18 R08: 0000000000000003 R09: 0000000019141fff
R10: ffff8800357adc00 R11: ffff8800357adc00 R12: ffff88000f286d00
R13: ffff88001bafb97c R14: ffffffff812d06bb R15: ffff88001bafb9c0
FS: 00007f807ef8ea80(0000) GS:ffff88003d620000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f801a5f2303 CR3: 000000001ba9d000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880036389d30 0000000000000020 ffff88001bafbac0 ffffffffa00a4729
ffff880036389008 000000001bafba80 0000000000000005 ffff8800357adc00
0000000000000014 ffff880036389188 0000006000000005 ffff880035742a10
Call Trace:
[<ffffffffa00a4729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffa00fb2b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff81417b1a>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff81420aa1>] scsi_request_fn+0x501/0x770
[<ffffffff812c73e3>] __blk_run_queue+0x33/0x40
[<ffffffff812c749a>] queue_unplugged+0x2a/0xa0
[<ffffffff812cbd18>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff812cc124>] blk_finish_plug+0x14/0x40
[<ffffffff81175cee>] __do_page_cache_readahead+0x1de/0x250
[<ffffffff81176321>] ra_submit+0x21/0x30
[<ffffffff8116b7ed>] filemap_fault+0x11d/0x430
[<ffffffffa02471df>] xfs_filemap_fault+0x4f/0xa0 [xfs]
[<ffffffff81192b2e>] __do_fault+0x7e/0x510
[<ffffffff81197088>] handle_mm_fault+0x5b8/0xf50
[<ffffffff8122bb37>] ? eventfd_ctx_read+0x67/0x210
[<ffffffff810c14bc>] ? update_curr+0xcc/0x150
[<ffffffff81640e22>] __do_page_fault+0x152/0x420
[<ffffffff81641113>] do_page_fault+0x23/0x80
[<ffffffff8163d408>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 09
Complete report #889806
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03623be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc03e171e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04572e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9489b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff948a45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff94718ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff94718b9a
Build id: 3.10.0-862.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff9471c358
Build id: 3.10.0-862.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff9471c7c4
Build id: 3.10.0-862.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff945a1241
Build id: 3.10.0-862.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff945a18e1
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff94596265
Build id: 3.10.0-862.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffff945aee04
Build id: 3.10.0-862.el7.x86_64
|
0x84
|
|
14
|
|
vmlinux |
0xffffffff94b12062
Build id: 3.10.0-862.el7.x86_64
|
0x12
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc05c785f
Build id: 3.10.0-862.el7.x86_64
|
0x5f
|
|
16
|
__do_fault.isra.58
|
vmlinux |
0xffffffff945c05fa
Build id: 3.10.0-862.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.60
|
vmlinux |
0xffffffff945c0b5c
Build id: 3.10.0-862.el7.x86_64
|
0x4c
|
|
18
|
handle_pte_fault
|
vmlinux |
0xffffffff945c52dc
Build id: 3.10.0-862.el7.x86_64
|
0x2dc
|
|
19
|
|
vmlinux |
0xffffffff9462a2eb
Build id: 3.10.0-862.el7.x86_64
|
0x2b
|
|
20
|
handle_mm_fault
|
vmlinux |
0xffffffff945c747d
Build id: 3.10.0-862.el7.x86_64
|
0x39d
|
|
21
|
__do_page_fault
|
vmlinux |
0xffffffff94b1a597
Build id: 3.10.0-862.el7.x86_64
|
0x197
|
|
22
|
do_page_fault
|
vmlinux |
0xffffffff94b1a925
Build id: 3.10.0-862.el7.x86_64
|
0x35
|
|
23
|
page_fault
|
vmlinux |
0xffffffff94b16768
Build id: 3.10.0-862.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 27s! [systemd-journal:520]
Modules linked in: binfmt_misc target_core_mod xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer crc32_pclmul ghash_clmulni_intel aesni_intel snd lrw gf128mul glue_helper ablk_helper cryptd soundcore sg joydev ppdev vmw_balloon pcspkr nfit libnvdimm parport_pc parport shpchp i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common mptspi crc32c_intel i2c_core ata_piix scsi_transport_spi e1000 mptscsih libata serio_raw mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 520 Comm: systemd-journal Kdump: loaded Tainted: G L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8b4ef5f74f10 ti: ffff8b4f7b200000 task.ti: ffff8b4f7b200000
RIP: 0010:[<ffffffffc03623be>] [<ffffffffc03623be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8b4f7b2038e0 EFLAGS: 00000246
RAX: ffff9ff380700000 RBX: ffff8b4ff5d94500 RCX: 000000000000001a
RDX: ffff8b4ff1f55e00 RSI: ffff8b4ef5c16000 RDI: 000000000000000e
RBP: ffff8b4f7b2038f0 R08: 0000000000000002 R09: 0000000129663fff
R10: ffff8b4ef5c100d8 R11: ffff8b4ff1ef8540 R12: 0000000100000001
R13: ffff8b4f1d744780 R14: ffff8b4f7b203854 R15: ffffffff947212cb
FS: 00007f5539501880(0000) GS:ffff8b4ff96c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f55390e0f80 CR3: 0000000035c64000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc03e171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04572e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9489b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff948a45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff94718ad9>] __blk_run_queue+0x39/0x50
[<ffffffff94718b9a>] queue_unplugged+0x2a/0xa0
[<ffffffff9471c358>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9471c7c4>] blk_finish_plug+0x14/0x40
[<ffffffff945a1241>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff945a18e1>] ra_submit+0x21/0x30
[<ffffffff94596265>] filemap_fault+0x105/0x490
[<ffffffff945aee04>] ? shmem_fault+0x84/0x1f0
[<ffffffff94b12062>] ? down_read+0x12/0x40
[<ffffffffc05c785f>] xfs_filemap_fault+0x5f/0xe0 [xfs]
[<ffffffff945c05fa>] __do_fault.isra.58+0x8a/0x100
[<ffffffff945c0b5c>] do_read_fault.isra.60+0x4c/0x1a0
[<ffffffff945c52dc>] handle_pte_fault+0x2dc/0xc30
[<ffffffff9462a2eb>] ? filename_lookup+0x2b/0xc0
[<ffffffff945c747d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff94b1a597>] __do_page_fault+0x197/0x4f0
[<ffffffff94b1a925>] do_page_fault+0x35/0x90
[<ffffffff94b16768>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 37
Complete report #1188669
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00cf3be
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc00fe71e
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01122e0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff968e4680
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff968eddac
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff96753d39
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff96753dfa
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff967576f8
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff96757b64
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff965cbe91
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff965cc4b1
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff965c0eb5
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffff964d7552
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x92
|
|
14
|
|
vmlinux |
0xffffffff964d7579
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x19
|
|
15
|
__xfs_filemap_fault
|
xfs |
0xffffffffc038791e
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x7e
|
|
16
|
xfs_filemap_fault
|
xfs |
0xffffffffc0387b1c
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x2c
|
|
17
|
__do_fault.isra.61
|
vmlinux |
0xffffffff965ee7fa
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x8a
|
|
18
|
do_read_fault.isra.63
|
vmlinux |
0xffffffff965eedac
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x4c
|
|
19
|
handle_mm_fault
|
vmlinux |
0xffffffff965f65f0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa20
|
|
20
|
|
vmlinux |
0xffffffff966664d5
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x35
|
|
21
|
__do_page_fault
|
vmlinux |
0xffffffff96b94653
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x213
|
|
22
|
do_page_fault
|
vmlinux |
0xffffffff96b94975
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x35
|
|
23
|
page_fault
|
vmlinux |
0xffffffff96b90778
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 36s! [vmtoolsd:793]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw snd_ens1371 gf128mul glue_helper ablk_helper snd_rawmidi cryptd snd_ac97_codec ac97_bus snd_seq joydev pcspkr snd_seq_device snd_pcm snd_timer
sg snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common nfit ata_piix libnvdimm crc32c_intel e1000 libata mptspi scsi_transport_spi mptscsih drm_panel_orientation_quirks serio_raw mptbase dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 793 Comm: vmtoolsd Kdump: loaded Tainted: G L ------------ 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8838f64e6300 ti: ffff8838f6558000 task.ti: ffff8838f6558000
RIP: 0010:[<ffffffffc00cf3be>] [<ffffffffc00cf3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8838f655b918 EFLAGS: 00000246
RAX: ffffb201c0380000 RBX: ffff88388cbfc830 RCX: 0000000000000014
RDX: ffff8838b57f1800 RSI: ffff8838b5813000 RDI: 000000000000000e
RBP: ffff8838f655b928 R08: 0000000000000003 R09: 0000000049c60fff
R10: ffff8838f468c000 R11: ffff8838f43f0e00 R12: ffff88388930b180
R13: ffff8838f655b88c R14: ffffffff9675c71b R15: ffff8838f655b8d0
FS: 00007f55076b5780(0000) GS:ffff8838fb680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f55072744b0 CR3: 0000000000104000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc00fe71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01122e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff968e4680>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff968eddac>] scsi_request_fn+0x4ac/0x680
[<ffffffff96753d39>] __blk_run_queue+0x39/0x50
[<ffffffff96753dfa>] queue_unplugged+0x2a/0xa0
[<ffffffff967576f8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff96757b64>] blk_finish_plug+0x14/0x40
[<ffffffff965cbe91>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff965cc4b1>] ra_submit+0x21/0x30
[<ffffffff965c0eb5>] filemap_fault+0x105/0x420
[<ffffffff964d7552>] ? check_preempt_curr+0x92/0xa0
[<ffffffff964d7579>] ? ttwu_do_wakeup+0x19/0xe0
[<ffffffffc038791e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffc0387b1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffff965ee7fa>] __do_fault.isra.61+0x8a/0x100
[<ffffffff965eedac>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffff965f65f0>] handle_mm_fault+0xa20/0xfb0
[<ffffffff966664d5>] ? __d_free+0x35/0x40
[<ffffffff96b94653>] __do_page_fault+0x213/0x500
[<ffffffff96b94975>] do_page_fault+0x35/0x90
[<ffffffff96b90778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 d3 0d
Complete report #859541
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc037c44e
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc039774e
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03a22f0
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff91b02fe0
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff91b0cb8c
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff91968649
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff9196871a
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff9196c168
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff9196c5d4
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff917d6ed1
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff917d7501
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff917cbb75
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x105
|
|
13
|
__xfs_filemap_fault
|
xfs |
0xffffffffc062fffe
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x7e
|
|
14
|
|
vmlinux |
0xffffffff91c5e655
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xc5
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc063020c
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffff917fa15a
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffff917fa70c
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffff91802113
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0xa33
|
|
19
|
|
vmlinux |
0xffffffff918745e5
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffff91dbf7d3
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffff91dbfb05
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffff91dbb7b8
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 223s! [crond:123700]
Modules linked in: xt_nat tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec ac97_bus vmw_balloon lrw gf128mul snd_seq glue_helper ablk_helper cryptd snd_seq_device snd_pcm joydev pcspkr sg snd_timer snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common nfit crc32c_intel libata libnvdimm mptspi e1000 scsi_transport_spi mptscsih serio_raw drm_panel_orientation_quirks mptbase fuse
CPU: 3 PID: 123700 Comm: crond Kdump: loaded Tainted: G L ------------ 3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9fee4a2b9080 ti: ffff9fedf6db4000 task.ti: ffff9fedf6db4000
RIP: 0010:[<ffffffffc037c44e>] [<ffffffffc037c44e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0000:ffff9fedf6db7918 EFLAGS: 00000246
RAX: ffffb47901940000 RBX: ffff9fed4573d010 RCX: 0000000000000014
RDX: ffff9fee6e3b8200 RSI: ffff9fee6e66d000 RDI: 000000000000000e
RBP: ffff9fedf6db7928 R08: 0000000000000003 R09: 00000004285821ff
R10: ffff9fee691fd800 R11: ffff9fedf6014380 R12: ffff9fee227f9b00
R13: ffff9fedf6db788c R14: ffffffff919713fb R15: ffff9fedf6db78d0
FS: 00007fb564043840(0000) GS:ffff9fee6d6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005570eca5d340 CR3: 00000003b20ce000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc039774e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03a22f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff91b02fe0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff91b0cb8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff91968649>] __blk_run_queue+0x39/0x50
[<ffffffff9196871a>] queue_unplugged+0x2a/0xa0
[<ffffffff9196c168>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff9196c5d4>] blk_finish_plug+0x14/0x50
[<ffffffff917d6ed1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff917d7501>] ra_submit+0x21/0x30
[<ffffffff917cbb75>] filemap_fault+0x105/0x420
[<ffffffffc062fffe>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffff91c5e655>] ? sock_recvmsg+0xc5/0x110
[<ffffffffc063020c>] xfs_filemap_fault+0x2c/0x40 [xfs]
[<ffffffff917fa15a>] __do_fault.isra.61+0x8a/0x100
[<ffffffff917fa70c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffff91802113>] handle_mm_fault+0xa33/0x1190
[<ffffffff918745e5>] ? __d_free+0x35/0x40
[<ffffffff91dbf7d3>] __do_page_fault+0x213/0x510
[<ffffffff91dbfb05>] do_page_fault+0x35/0x90
[<ffffffff91dbb7b8>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #804337
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03c73be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc03e971e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04012e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffae6e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffae6ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffae5534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffffae55359a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffffae556dd8
Build id: 3.10.0-1160.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffffae557244
Build id: 3.10.0-1160.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffae3cb261
Build id: 3.10.0-1160.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffffae3cb881
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffffae3c0975
Build id: 3.10.0-1160.el7.x86_64
|
0x105
|
|
13
|
__xfs_filemap_fault
|
xfs |
0xffffffffc065f91e
Build id: 3.10.0-1160.el7.x86_64
|
0x7e
|
|
14
|
|
vmlinux |
0xffffffffae8344f5
Build id: 3.10.0-1160.el7.x86_64
|
0xc5
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc065fb1c
Build id: 3.10.0-1160.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffffae3ede3a
Build id: 3.10.0-1160.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffae3ee3ec
Build id: 3.10.0-1160.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffffae3f5c30
Build id: 3.10.0-1160.el7.x86_64
|
0xa20
|
|
19
|
|
vmlinux |
0xffffffffae465f05
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffffae98e653
Build id: 3.10.0-1160.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffffae98e975
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffffae98a778
Build id: 3.10.0-1160.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 70s! [crond:54588]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel pcspkr aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nfit libnvdimm ppdev joydev sg vmw_balloon parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix e1000 libata serio_raw mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 54588 Comm: crond Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8e21419a1080 ti: ffff8e2146ab0000 task.ti: ffff8e2146ab0000
RIP: 0010:[<ffffffffc03c73be>] [<ffffffffc03c73be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff8e2146ab3918 EFLAGS: 00000246
RAX: ffff9c97434c0000 RBX: ffff8e24fffa2600 RCX: 0000000000000015
RDX: ffff8e26e5d9c000 RSI: ffff8e22bff54000 RDI: 000000000000000e
RBP: ffff8e2146ab3928 R08: 0000000000000002 R09: 00000001f59e9fff
R10: ffff8e26e54cb000 R11: ffff8e26e4332a00 R12: ffff8e25beea8600
R13: ffff8e2146ab388c R14: ffffffffae55bdfb R15: ffff8e2146ab38d0
FS: 00007fca9f1a5840(0000) GS:ffff8e26e7680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005582a647a2e0 CR3: 0000000002c2a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc03e971e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04012e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffae6e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffae6ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffae5534d9>] __blk_run_queue+0x39/0x50
[<ffffffffae55359a>] queue_unplugged+0x2a/0xa0
[<ffffffffae556dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffae557244>] blk_finish_plug+0x14/0x40
[<ffffffffae3cb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffae3cb881>] ra_submit+0x21/0x30
[<ffffffffae3c0975>] filemap_fault+0x105/0x420
[<ffffffffc065f91e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffae8344f5>] ? sock_recvmsg+0xc5/0x100
[<ffffffffc065fb1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffffae3ede3a>] __do_fault.isra.61+0x8a/0x100
[<ffffffffae3ee3ec>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffae3f5c30>] handle_mm_fault+0xa20/0xfb0
[<ffffffffae465f05>] ? __d_free+0x35/0x40
[<ffffffffae98e653>] __do_page_fault+0x213/0x500
[<ffffffffae98e975>] do_page_fault+0x35/0x90
[<ffffffffae98a778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 3d
Complete report #711899
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff98fba385
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc00e26a2
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffff98d0bb30
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff98d02fe0
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff98d0cb8c
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff98b68649
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff98b6871a
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff98b6c168
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff98b6c5d4
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff989d6ed1
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff989d7501
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff989cbb75
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x105
|
|
13
|
__xfs_filemap_fault
|
xfs |
0xffffffffc0356ffe
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x7e
|
|
14
|
|
vmlinux |
0xffffffff98fb7920
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x320
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc035720c
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffff989fa15a
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffff989fa70c
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffff98a02113
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xa33
|
|
19
|
|
vmlinux |
0xffffffff98a745e5
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffff98fbf7d3
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffff98fbfb05
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffff98fbb7b8
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 32s! [snmptrapd:1383]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables iscsi_target_mod target_core_user uio target_core_pscsi target_core_file target_core_iblock target_core_mod bridge stp llc ip_set vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ppdev joydev vmw_balloon sg i2c_piix4 vmw_vmci parport_pc parport xfs libcrc32c sr_mod cdrom ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif ahci crct10dif_generic nfit libahci drm ata_piix crct10dif_pclmul libnvdimm crct10dif_common libata serio_raw vmw_pvscsi vmxnet3 crc32c_intel drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 3 PID: 1383 Comm: snmptrapd Kdump: loaded Tainted: G L ------------ 3.10.0-1160.90.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8b776e8f0000 ti: ffff8b77784cc000 task.ti: ffff8b77784cc000
RIP: 0010:[<ffffffff98fba385>] [<ffffffff98fba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0000:ffff8b77784cf980 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffff98bc9d43 RCX: ffffffff99434f40
RDX: ffffb692007a0000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8b77784cf980 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8b77f4501400 R11: ffff8b76f5fe8000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff8b77f30d6d80 R15: ffffffff98d0a769
FS: 00007fc0bf406880(0000) GS:ffff8b77ffcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc0bebf4a30 CR3: 00000000aeb6a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc00e26a2>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff98d0bb30>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff98d02fe0>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff98d0cb8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff98b68649>] __blk_run_queue+0x39/0x50
[<ffffffff98b6871a>] queue_unplugged+0x2a/0xa0
[<ffffffff98b6c168>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff98b6c5d4>] blk_finish_plug+0x14/0x50
[<ffffffff989d6ed1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff989d7501>] ra_submit+0x21/0x30
[<ffffffff989cbb75>] filemap_fault+0x105/0x420
[<ffffffffc0356ffe>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffff98fb7920>] ? __schedule+0x320/0x680
[<ffffffffc035720c>] xfs_filemap_fault+0x2c/0x40 [xfs]
[<ffffffff989fa15a>] __do_fault.isra.61+0x8a/0x100
[<ffffffff989fa70c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffff98a02113>] handle_mm_fault+0xa33/0x1190
[<ffffffff98a745e5>] ? __d_free+0x35/0x40
[<ffffffff98fbf7d3>] __do_page_fault+0x213/0x510
[<ffffffff98fbfb05>] do_page_fault+0x35/0x90
[<ffffffff98fbb7b8>] page_fault+0x28/0x30
Code: 07 00 0f 1f 40 00 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #648376
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00f63be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc017d71e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01972e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb98e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb98ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffb97534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffffb975359a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffffb9756dd8
Build id: 3.10.0-1160.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffffb9757244
Build id: 3.10.0-1160.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffb95cb261
Build id: 3.10.0-1160.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffffb95cb881
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffffb95c0975
Build id: 3.10.0-1160.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffffb9aa59db
Build id: 3.10.0-1160.el7.x86_64
|
0x7b
|
|
14
|
__xfs_filemap_fault
|
xfs |
0xffffffffc03a991e
Build id: 3.10.0-1160.el7.x86_64
|
0x7e
|
|
15
|
|
vmlinux |
0xffffffffb94ad8b3
Build id: 3.10.0-1160.el7.x86_64
|
0x83
|
|
16
|
xfs_filemap_fault
|
xfs |
0xffffffffc03a9b1c
Build id: 3.10.0-1160.el7.x86_64
|
0x2c
|
|
17
|
__do_fault.isra.61
|
vmlinux |
0xffffffffb95ede3a
Build id: 3.10.0-1160.el7.x86_64
|
0x8a
|
|
18
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffb95ee3ec
Build id: 3.10.0-1160.el7.x86_64
|
0x4c
|
|
19
|
handle_mm_fault
|
vmlinux |
0xffffffffb95f5c30
Build id: 3.10.0-1160.el7.x86_64
|
0xa20
|
|
20
|
|
vmlinux |
0xffffffffb9665f05
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
21
|
__do_page_fault
|
vmlinux |
0xffffffffb9b8e653
Build id: 3.10.0-1160.el7.x86_64
|
0x213
|
|
22
|
do_page_fault
|
vmlinux |
0xffffffffb9b8e975
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
23
|
|
vmlinux |
0xffffffffb9b8aac9
Build id: 3.10.0-1160.el7.x86_64
|
0xaa
|
|
24
|
page_fault
|
vmlinux |
0xffffffffb9b8a778
Build id: 3.10.0-1160.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 87s! [cmdb_taskserver:2103]
Modules linked in: udp_diag tcp_diag inet_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event kvm_intel snd_ens1371 snd_rawmidi snd_ac97_codec kvm ac97_bus snd_seq snd_seq_device irqbypass snd_pcm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul snd_timer glue_helper snd ablk_helper ppdev cryptd soundcore pcspkr vmw_balloon joydev sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix libata mptspi serio_raw nfit scsi_transport_spi mptscsih e1000 libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 2103 Comm: cmdb_taskserver Kdump: loaded Tainted: G ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9cf26b4f2100 ti: ffff9cf268bb0000 task.ti: ffff9cf268bb0000
RIP: 0010:[<ffffffffc00f63be>] [<ffffffffc00f63be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9cf268bb3918 EFLAGS: 00000246
RAX: ffffbec7c3200000 RBX: ffff9cf1a5f736c0 RCX: 0000000000000010
RDX: ffff9cedb57f8000 RSI: ffff9cedad3fd000 RDI: 000000000000000e
RBP: ffff9cf268bb3928 R08: 0000000000000003 R09: 00000003039cffff
R10: ffff9cedad3e4000 R11: ffff9cedb2bf6a00 R12: ffff9cedda441c80
R13: ffff9cf268bb388c R14: ffffffffb975bdfb R15: ffff9cf268bb38d0
FS: 00007fd5dc973700(0000) GS:ffff9cf27d040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000010550e1 CR3: 0000000631056000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc017d71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01972e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb98e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb98ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffb97534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb975359a>] queue_unplugged+0x2a/0xa0
[<ffffffffb9756dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb9757244>] blk_finish_plug+0x14/0x40
[<ffffffffb95cb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffb95cb881>] ra_submit+0x21/0x30
[<ffffffffb95c0975>] filemap_fault+0x105/0x420
[<ffffffffb9aa59db>] ? ip_output+0x7b/0xf0
[<ffffffffc03a991e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffb94ad8b3>] ? internal_add_timer+0x83/0xe0
[<ffffffffc03a9b1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffffb95ede3a>] __do_fault.isra.61+0x8a/0x100
[<ffffffffb95ee3ec>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffb95f5c30>] handle_mm_fault+0xa20/0xfb0
[<ffffffffb9665f05>] ? __d_free+0x35/0x40
[<ffffffffb9b8e653>] __do_page_fault+0x213/0x500
[<ffffffffb9b8e975>] do_page_fault+0x35/0x90
[<ffffffffb9b8aac9>] ? error_swapgs+0xaa/0xc0
[<ffffffffb9b8a778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 10
Complete report #548928
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01ad3be
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc025571e
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02732e0
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8d6e4690
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8d6eddbc
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8d553d19
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff8d553dda
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff8d5576d8
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff8d557b44
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff8d3cbe71
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff8d3cc491
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff8d3c0e95
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x105
|
|
13
|
__xfs_filemap_fault
|
xfs |
0xffffffffc045391e
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x7e
|
|
14
|
|
vmlinux |
0xffffffff8d2e3f6b
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x8b
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc0453b1c
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffff8d3ee7da
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffff8d3eed8c
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffff8d3f65d0
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa20
|
|
19
|
|
vmlinux |
0xffffffff8d4664b5
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffff8d994653
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffff8d994975
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffff8d990778
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [vmtoolsd:4214]
Modules linked in: nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul snd_ens1371 glue_helper snd_rawmidi ablk_helper snd_ac97_codec ac97_bus cryptd snd_seq
snd_seq_device snd_pcm snd_timer snd sg soundcore joydev vmw_balloon pcspkr parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit crct10dif_pclmul crct10dif_common drm libnvdimm crc32c_intel ata_piix mptspi scsi_transport_spi mptscsih libata serio_raw e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 4214 Comm: vmtoolsd Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9de6fac58000 ti: ffff9de6d1280000 task.ti: ffff9de6d1280000
RIP: 0010:[<ffffffffc01ad3be>] [<ffffffffc01ad3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9de6d1283918 EFLAGS: 00000246
RAX: ffffaed800700000 RBX: ffff9de6c1079c10 RCX: 0000000000000025
RDX: ffff9de7f04fc200 RSI: ffff9de7f59d4000 RDI: 000000000000000e
RBP: ffff9de6d1283928 R08: 0000000000000003 R09: 00000000ab54bfff
R10: ffff9de6f5001000 R11: ffff9de6faf8d340 R12: ffff9de77854e400
R13: ffff9de6d128388c R14: ffffffff8d55c6fb R15: ffff9de6d12838d0
FS: 00007f0d9f269780(0000) GS:ffff9de7f9600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0d9edf7c70 CR3: 0000000010d04000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc025571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02732e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d6e4690>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d6eddbc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d553d19>] __blk_run_queue+0x39/0x50
[<ffffffff8d553dda>] queue_unplugged+0x2a/0xa0
[<ffffffff8d5576d8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8d557b44>] blk_finish_plug+0x14/0x40
[<ffffffff8d3cbe71>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff8d3cc491>] ra_submit+0x21/0x30
[<ffffffff8d3c0e95>] filemap_fault+0x105/0x420
[<ffffffffc045391e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffff8d2e3f6b>] ? put_prev_entity+0x8b/0x400
[<ffffffffc0453b1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffff8d3ee7da>] __do_fault.isra.61+0x8a/0x100
[<ffffffff8d3eed8c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffff8d3f65d0>] handle_mm_fault+0xa20/0xfb0
[<ffffffff8d4664b5>] ? __d_free+0x35/0x40
[<ffffffff8d994653>] __do_page_fault+0x213/0x500
[<ffffffff8d994975>] do_page_fault+0x35/0x90
[<ffffffff8d990778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 b3 1b
Complete report #464738
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffabd8a3f5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc00b1632
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffffabaec580
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffabae3e70
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffabaed59c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffffab953869
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffffab95392a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffffab957268
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffffab9576d4
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffab7cb3d1
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffffab7cb9f1
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffffab7c0b65
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffffabca65fb
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x7b
|
|
14
|
__xfs_filemap_fault
|
xfs |
0xffffffffc033091e
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x7e
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc0330b1c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffffab7edfba
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffab7ee56c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffffab7f5db0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa20
|
|
19
|
|
vmlinux |
0xffffffffab8660e5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x35
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffffabd8f653
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffffabd8f975
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffffabd8b778
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [pilot-discovery:36486]
Modules linked in: xt_statistic xt_owner xt_REDIRECT nf_nat_redirect veth ip6t_MASQUERADE nf_nat_masquerade_ipv6 nf_conntrack_netlink xt_nat ip6table_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_physdev xt_NFLOG nfnetlink_log xt_limit vxlan ip6_udp_tunnel xt_mark udp_tunnel xt_multiport xt_addrtype xt_comment ip_set nfnetlink xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter cmac arc4 md4 nls_utf8 br_netfilter bridge cifs stp ccm llc dns_resolver overlay(T) vmw_vsock_vmci_transport vsock sb_edac coretemp iosf_mbi kvm_intel
kvm irqbypass crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel libata nfit serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 36486 Comm: pilot-discovery Tainted: G L ------------ T 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff96154ef65280 ti: ffff96151b334000 task.ti: ffff96151b334000
RIP: 0010:[<ffffffffabd8a3f5>] [<ffffffffabd8a3f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff96151b337980 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffffffffab9b2803 RCX: ffffffffac234f40
RDX: ffffafe783160000 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff96151b337980 R08: 0000000000000001 R09: 0000000000000000
R10: ffff9611e17ba400 R11: ffff9611dafb7000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff96186b91df80 R15: ffffffffabaeb239
FS: 000000c001d0f490(0000) GS:ffff96189d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000169010 CR3: 0000000699fda000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc00b1632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffabaec580>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffabae3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffabaed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffab953869>] __blk_run_queue+0x39/0x50
[<ffffffffab95392a>] queue_unplugged+0x2a/0xa0
[<ffffffffab957268>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffab9576d4>] blk_finish_plug+0x14/0x40
[<ffffffffab7cb3d1>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffab7cb9f1>] ra_submit+0x21/0x30
[<ffffffffab7c0b65>] filemap_fault+0x105/0x420
[<ffffffffabca65fb>] ? ip_output+0x7b/0xf0
[<ffffffffc033091e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffc0330b1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffffab7edfba>] __do_fault.isra.61+0x8a/0x100
[<ffffffffab7ee56c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffab7f5db0>] handle_mm_fault+0xa20/0xfb0
[<ffffffffab8660e5>] ? __d_free+0x35/0x40
[<ffffffffabd8f653>] __do_page_fault+0x213/0x500
[<ffffffffabd8f975>] do_page_fault+0x35/0x90
[<ffffffffabd8b778>] page_fault+0x28/0x30
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #282056
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc03193be
Build id: 3.10.0-1127.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc035571e
Build id: 3.10.0-1127.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc046a2e0
Build id: 3.10.0-1127.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8dee2850
Build id: 3.10.0-1127.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8deebefc
Build id: 3.10.0-1127.el7.x86_64
|
0x4ac
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff8dd526f9
Build id: 3.10.0-1127.el7.x86_64
|
0x39
|
|
7
|
queue_unplugged
|
vmlinux |
0xffffffff8dd527ba
Build id: 3.10.0-1127.el7.x86_64
|
0x2a
|
|
8
|
blk_flush_plug_list
|
vmlinux |
0xffffffff8dd55ff8
Build id: 3.10.0-1127.el7.x86_64
|
0x1d8
|
|
9
|
blk_finish_plug
|
vmlinux |
0xffffffff8dd56464
Build id: 3.10.0-1127.el7.x86_64
|
0x14
|
|
10
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff8dbcb631
Build id: 3.10.0-1127.el7.x86_64
|
0x1e1
|
|
11
|
ra_submit
|
vmlinux |
0xffffffff8dbcbc81
Build id: 3.10.0-1127.el7.x86_64
|
0x21
|
|
12
|
filemap_fault
|
vmlinux |
0xffffffff8dbc0dc5
Build id: 3.10.0-1127.el7.x86_64
|
0x105
|
|
13
|
|
vmlinux |
0xffffffff8dc50001
Build id: 3.10.0-1127.el7.x86_64
|
0x31
|
|
14
|
__xfs_filemap_fault
|
xfs |
0xffffffffc0560a4e
Build id: 3.10.0-1127.el7.x86_64
|
0x7e
|
|
15
|
xfs_filemap_fault
|
xfs |
0xffffffffc0560c4c
Build id: 3.10.0-1127.el7.x86_64
|
0x2c
|
|
16
|
__do_fault.isra.61
|
vmlinux |
0xffffffff8dbedeea
Build id: 3.10.0-1127.el7.x86_64
|
0x8a
|
|
17
|
do_read_fault.isra.63
|
vmlinux |
0xffffffff8dbee49c
Build id: 3.10.0-1127.el7.x86_64
|
0x4c
|
|
18
|
handle_mm_fault
|
vmlinux |
0xffffffff8dbf5d00
Build id: 3.10.0-1127.el7.x86_64
|
0xa20
|
|
19
|
|
vmlinux |
0xffffffff8dbfe439
Build id: 3.10.0-1127.el7.x86_64
|
0x1b9
|
|
20
|
__do_page_fault
|
vmlinux |
0xffffffff8e18d653
Build id: 3.10.0-1127.el7.x86_64
|
0x213
|
|
21
|
do_page_fault
|
vmlinux |
0xffffffff8e18d975
Build id: 3.10.0-1127.el7.x86_64
|
0x35
|
|
22
|
page_fault
|
vmlinux |
0xffffffff8e189778
Build id: 3.10.0-1127.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 25s! [gnome-shell:2175]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel pcspkr ppdev cryptd joydev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic vmwgfx pata_acpi drm_kms_helper syscopyarea nfit sysfillrect mptspi sysimgblt fb_sys_fops ahci scsi_transport_spi ttm crct10dif_pclmul crct10dif_common libnvdimm ata_piix mptscsih libahci crc32c_intel drm serio_raw e1000
libata mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 2175 Comm: gnome-shell Tainted: G L ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9b70211ee2a0 ti: ffff9b703b8a8000 task.ti: ffff9b703b8a8000
RIP: 0010:[<ffffffffc03193be>] [<ffffffffc03193be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9b703b8ab918 EFLAGS: 00000246
RAX: ffffaa2e80200000 RBX: ffff9b70396d6c00 RCX: 0000000000000019
RDX: ffff9b7035a10e00 RSI: ffff9b70350a0000 RDI: 000000000000000e
RBP: ffff9b703b8ab928 R08: 0000000000000002 R09: 0000000009fc3fff
R10: ffff9b70350ecc00 R11: ffff9b701bb35180 R12: ffff9b7036848600
R13: ffff9b703b8ab88c R14: ffffffff8dd5b01b R15: ffff9b703b8ab8d0
FS: 00007ff98cdada80(0000) GS:ffff9b703d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff98269a3a0 CR3: 0000000017c0c000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc035571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc046a2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8dee2850>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8deebefc>] scsi_request_fn+0x4ac/0x660
[<ffffffff8dd526f9>] __blk_run_queue+0x39/0x50
[<ffffffff8dd527ba>] queue_unplugged+0x2a/0xa0
[<ffffffff8dd55ff8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffff8dd56464>] blk_finish_plug+0x14/0x40
[<ffffffff8dbcb631>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffff8dbcbc81>] ra_submit+0x21/0x30
[<ffffffff8dbc0dc5>] filemap_fault+0x105/0x420
[<ffffffff8dc50001>] ? __sb_end_write+0x31/0x70
[<ffffffffc0560a4e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffc0560c4c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffff8dbedeea>] __do_fault.isra.61+0x8a/0x100
[<ffffffff8dbee49c>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffff8dbf5d00>] handle_mm_fault+0xa20/0xfb0
[<ffffffff8dbfe439>] ? mprotect_fixup+0x1b9/0x3a0
[<ffffffff8e18d653>] __do_page_fault+0x213/0x500
[<ffffffff8e18d975>] do_page_fault+0x35/0x90
[<ffffffff8e189778>] page_fault+0x28/0x30
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 73 32
Complete report #416483
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
scsi_request_fn
|
vmlinux |
0xffffffffb3aecd35
Build id: 3.10.0-1160.el7.x86_64
|
0x135
|
|
2
|
__blk_run_queue
|
vmlinux |
0xffffffffb39534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
|
3
|
queue_unplugged
|
vmlinux |
0xffffffffb395359a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
4
|
blk_flush_plug_list
|
vmlinux |
0xffffffffb3956dd8
Build id: 3.10.0-1160.el7.x86_64
|
0x1d8
|
|
5
|
blk_finish_plug
|
vmlinux |
0xffffffffb3957244
Build id: 3.10.0-1160.el7.x86_64
|
0x14
|
|
6
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffb37cb261
Build id: 3.10.0-1160.el7.x86_64
|
0x1e1
|
|
7
|
ra_submit
|
vmlinux |
0xffffffffb37cb881
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
8
|
filemap_fault
|
vmlinux |
0xffffffffb37c0975
Build id: 3.10.0-1160.el7.x86_64
|
0x105
|
|
9
|
|
vmlinux |
0xffffffffb3ca59db
Build id: 3.10.0-1160.el7.x86_64
|
0x7b
|
|
10
|
__xfs_filemap_fault
|
xfs |
0xffffffffc066b91e
Build id: 3.10.0-1160.el7.x86_64
|
0x7e
|
|
11
|
xfs_filemap_fault
|
xfs |
0xffffffffc066bb1c
Build id: 3.10.0-1160.el7.x86_64
|
0x2c
|
|
12
|
__do_fault.isra.61
|
vmlinux |
0xffffffffb37ede3a
Build id: 3.10.0-1160.el7.x86_64
|
0x8a
|
|
13
|
do_read_fault.isra.63
|
vmlinux |
0xffffffffb37ee3ec
Build id: 3.10.0-1160.el7.x86_64
|
0x4c
|
|
14
|
handle_mm_fault
|
vmlinux |
0xffffffffb37f5c30
Build id: 3.10.0-1160.el7.x86_64
|
0xa20
|
|
15
|
|
vmlinux |
0xffffffffb3865f05
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
16
|
__do_page_fault
|
vmlinux |
0xffffffffb3d8e653
Build id: 3.10.0-1160.el7.x86_64
|
0x213
|
|
17
|
do_page_fault
|
vmlinux |
0xffffffffb3d8e975
Build id: 3.10.0-1160.el7.x86_64
|
0x35
|
|
18
|
page_fault
|
vmlinux |
0xffffffffb3d8a778
Build id: 3.10.0-1160.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [kubelet:2455]
Modules linked in: xt_multiport ipt_rpfilter xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set ipip tunnel4 ip_tunnel xt_statistic ipt_REJECT nf_reject_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle xt_comment xt_mark veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vmw_vmci vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi iosf_mbi snd_ac97_codec crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device ppdev snd_pcm vmw_balloon aesni_intel lrw gf128mul snd_timer glue_helper
ablk_helper snd cryptd soundcore sg pcspkr joydev parport_pc parport i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit e1000 libnvdimm drm ata_piix crct10dif_pclmul mptspi crct10dif_common scsi_transport_spi mptscsih crc32c_intel serio_raw libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 2455 Comm: kubelet Kdump: loaded Tainted: G ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff94ad67eac200 ti: ffff94ad4d988000 task.ti: ffff94ad4d988000
RIP: 0010:[<ffffffffb3aecd35>] [<ffffffffb3aecd35>] scsi_request_fn+0x135/0x680
RSP: 0000:ffff94ad4d98ba30 EFLAGS: 00000206
RAX: 000000010013c180 RBX: ffff94acc6c2a280 RCX: 0000000100141284
RDX: 000000010013fef9 RSI: ffff94acc6c28450 RDI: ffff94ad77a38450
RBP: ffff94ad4d98ba88 R08: ffff94ad77a38598 R09: 0000000000000000
R10: ffff94ad779ce800 R11: ffff94acf7659880 R12: ffff94ad4d98b9c0
R13: ffffffffc0607f0f R14: ffff94ad4d98b9a0 R15: ffff94acf765b9c0
FS: 00007f6d53fff700(0000) GS:ffff94ad7a640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000472a9ae CR3: 00000000b9016000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb39534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb395359a>] queue_unplugged+0x2a/0xa0
[<ffffffffb3956dd8>] blk_flush_plug_list+0x1d8/0x230
[<ffffffffb3957244>] blk_finish_plug+0x14/0x40
[<ffffffffb37cb261>] __do_page_cache_readahead+0x1e1/0x260
[<ffffffffb37cb881>] ra_submit+0x21/0x30
[<ffffffffb37c0975>] filemap_fault+0x105/0x420
[<ffffffffb3ca59db>] ? ip_output+0x7b/0xf0
[<ffffffffc066b91e>] __xfs_filemap_fault+0x7e/0x1d0 [xfs]
[<ffffffffc066bb1c>] xfs_filemap_fault+0x2c/0x30 [xfs]
[<ffffffffb37ede3a>] __do_fault.isra.61+0x8a/0x100
[<ffffffffb37ee3ec>] do_read_fault.isra.63+0x4c/0x1b0
[<ffffffffb37f5c30>] handle_mm_fault+0xa20/0xfb0
[<ffffffffb3865f05>] ? __d_free+0x35/0x40
[<ffffffffb3d8e653>] __do_page_fault+0x213/0x500
[<ffffffffb3d8e975>] do_page_fault+0x35/0x90
[<ffffffffb3d8a778>] page_fault+0x28/0x30
Code: 8b 85 40 04 00 00 a8 02 0f 85 18 01 00 00 4c 89 f7 e8 80 99 e6 ff 49 8b bd 58 04 00 00 c6 07 00 0f 1f 40 00 fb 66 0f 1f 44 00 00 <4d> 8b be 00 01 00 00 4d 85 ff 0f 84 3d 04 00 00 49 8b 85 40 04