Complete report #1256586
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa618b7d5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc00b2632
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffffa5eecf70
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa5ee4860
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa5eedf8c
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffa5d78bee
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffa5d53f39
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffa5d57ce3
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffa5d55bc7
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffa5d55e70
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffffa5ae26ec
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x3c
|
|
12
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0313f31
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc031426f
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x7f
|
|
14
|
do_writepages
|
vmlinux |
0xffffffffa5bcb9b1
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffa5bbfe15
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffa5bbff61
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc031ed66
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffffa5c84227
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x67
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffffa5c84533
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffffa6195f92
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 29s! [mysqld:1819]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter tun bridge stp llc nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr ppdev sg vmw_balloon parport_pc vmw_vmci i2c_piix4 parport auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod nfit crc_t10dif crct10dif_generic ahci libnvdimm drm
libahci ata_piix libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 drm_panel_orientation_quirks vmw_pvscsi fuse
CPU: 3 PID: 1819 Comm: mysqld Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff961537905280 ti: ffff9617215e4000 task.ti: ffff9617215e4000
RIP: 0010:[<ffffffffa618b7d5>] [<ffffffffa618b7d5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9617215e7b60 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffffa5db2f83 RCX: ffff961735029060
RDX: ffffba4d81ba0000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9617215e7b60 R08: 0000000000000000 R09: ffff961588019d00
R10: ffff961730fab800 R11: ffff961730431800 R12: 0000000000000000
R13: 0000000000000020 R14: ffff96159296d800 R15: ffffffffa5eebc29
FS: 00007fab49d1b700(0000) GS:ffff96173fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f90daa44000 CR3: 00000002a4a1c000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffc00b2632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffffa5eecf70>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffffa5ee4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa5eedf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffffa5d78bee>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5d53f39>] __blk_run_queue+0x39/0x50
[<ffffffffa5d57ce3>] blk_queue_bio+0x3b3/0x400
[<ffffffffa5d55bc7>] generic_make_request+0x147/0x380
[<ffffffffa5d55e70>] submit_bio+0x70/0x150
[<ffffffffa5ae26ec>] ? set_next_entity+0x3c/0xe0
[<ffffffffc0313f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc031426f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffa5bcb9b1>] do_writepages+0x21/0x50
[<ffffffffa5bbfe15>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffa5bbff61>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc031ed66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa5c84227>] do_fsync+0x67/0xb0
[<ffffffffa5c84533>] SyS_fdatasync+0x13/0x20
[<ffffffffa6195f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #956730
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff955ba385
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x15
|
|
2
|
ata_scsi_queuecmd
|
libata |
0xffffffffc02ea275
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x155
|
|
3
|
|
libata |
0xffffffffc02e6e60
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff95303120
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9530cccc
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff9518e1de
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff95168799
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff9516c6c3
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff9516a4f7
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x157
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff9516a7a0
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc04ea341
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc04ea68f
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff94fd6ab1
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff94fcab55
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff94fcacc1
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc04f5486
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff95093145
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x55
|
|
18
|
|
vmlinux |
0xffffffff94f46966
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x1f6
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff95093433
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffff955c539a
Build id: 3.10.0-1160.102.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [mysqld:26452]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter binfmt_misc sunrpc sg pcspkr i2c_piix4 joydev virtio_balloon ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi virtio_console bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci ata_piix serio_raw virtio_pci e1000 drm_panel_orientation_quirks virtio_ring virtio libata floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 26452 Comm: mysqld Kdump: loaded Tainted: G L ------------ 3.10.0-1160.102.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.1-0-g3208b098f51a-prebuilt.qemu.org 04/01/2014
task: ffff934c9a186300 ti: ffff934caac28000 task.ti: ffff934caac28000
RIP: 0010:[<ffffffff955ba385>] [<ffffffff955ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff934caac2bba0 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff934caac2bb70 RCX: 0000000000000000
RDX: ffff934ab5c1b818 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff934caac2bba0 R08: 0000000000000000 R09: 00000001faf59fff
R10: ffff934cb9c67c00 R11: ffff934b30483e00 R12: ffffffff94f1ed4b
R13: ffff934caac2bb20 R14: ffff934ab5d18d98 R15: ffff934cab818000
FS: 00007fa0ba8b1700(0000) GS:ffff934cb9d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f734fe809e5 CR3: 000000022ff88000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc02ea275>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc02e6e60>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffff95303120>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff9530cccc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9518e1de>] ? deadline_add_request+0x6e/0x90
[<ffffffff95168799>] __blk_run_queue+0x39/0x50
[<ffffffff9516c6c3>] blk_queue_bio+0x3b3/0x400
[<ffffffff9516a4f7>] generic_make_request+0x157/0x390
[<ffffffff9516a7a0>] submit_bio+0x70/0x150
[<ffffffffc04ea341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc04ea68f>] xfs_vm_writepages+0x7f/0xb0 [xfs]
[<ffffffff94fd6ab1>] do_writepages+0x21/0x50
[<ffffffff94fcab55>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffff94fcacc1>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc04f5486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff95093145>] do_fsync+0x55/0x80
[<ffffffff94f46966>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffff95093433>] SyS_fdatasync+0x13/0x20
[<ffffffff955c539a>] system_call_fastpath+0x25/0x2a
Code: 8a 42 96 ff 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 6a 42 96 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #831082
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa038ea15
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x15
|
|
2
|
ata_scsi_queuecmd
|
libata |
0xffffffffc0421d45
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x155
|
|
3
|
|
libata |
0xffffffffc041ea20
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa00e4680
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa00eddac
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff9ff78a0e
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff9ff53d39
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff9ff57b03
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff9ff559e7
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff9ff55c90
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0621f31
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc062226f
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff9fdcba11
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff9fdbfe95
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff9fdbffe1
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc062cd66
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff9fe84255
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x55
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa0399ec9
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x96
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff9fe84523
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffffa0399f92
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x25
|
|
21
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa0399ed5
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 580s! [hr_agent:2328]
Modules linked in: tcp_diag udp_diag inet_diag unix_diag esmnet(OE) esmfile(OE) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter crc32_pclmul ppdev snd_intel8x0 ghash_clmulni_intel snd_ac97_codec ac97_bus aesni_intel snd_seq lrw gf128mul snd_seq_device glue_helper snd_pcm ablk_helper cryptd sg pcspkr i2c_piix4 parport_pc snd_timer video parport snd soundcore ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci libata crct10dif_pclmul crct10dif_common crc32c_intel e1000 serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 2328 Comm: hr_agent Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff8a08f02dc200 ti: ffff8a090102c000 task.ti: ffff8a090102c000
RIP: 0010:[<ffffffffa038ea15>] [<ffffffffa038ea15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a090102fba0 EFLAGS: 00010282
RAX: 0000000000000001 RBX: ffff8a0875ca4000 RCX: 0000000000000000
RDX: ffff8a0875d42c18 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8a090102fba0 R08: 0000000000000000 R09: 00000000d04c1fff
R10: ffff8a0875f9b000 R11: ffff8a08f02fdc00 R12: ffff8a090102fb48
R13: ffffffff9ffb2da3 R14: ffff8a090102fb48 R15: ffff8a0875820f08
FS: 00007faed17c7700(0000) GS:ffff8a095fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faecc19fe88 CR3: 00000000b0334000 CR4: 00000000000206e0
Call Trace:
[<ffffffffc0421d45>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc041ea20>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffffa00e4680>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa00eddac>] scsi_request_fn+0x4ac/0x680
[<ffffffff9ff78a0e>] ? deadline_add_request+0x6e/0x90
[<ffffffff9ff53d39>] __blk_run_queue+0x39/0x50
[<ffffffff9ff57b03>] blk_queue_bio+0x3b3/0x400
[<ffffffff9ff559e7>] generic_make_request+0x147/0x380
[<ffffffff9ff55c90>] submit_bio+0x70/0x150
[<ffffffffc0621f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc062226f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff9fdcba11>] do_writepages+0x21/0x50
[<ffffffff9fdbfe95>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff9fdbffe1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc062cd66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff9fe84255>] do_fsync+0x55/0x80
[<ffffffffa0399ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff9fe84523>] SyS_fdatasync+0x13/0x20
[<ffffffffa0399f92>] system_call_fastpath+0x25/0x2a
[<ffffffffa0399ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: b6 8b 98 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 96 8b 98 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #274698
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa636b455
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x15
|
|
2
|
ata_scsi_queuecmd
|
libata |
0xffffffffc0463d65
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x155
|
|
3
|
|
libata |
0xffffffffc0460a40
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa60d1700
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa60dac9c
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffffa5f67e6e
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffa5f43849
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffa5f47553
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffa5f454e7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffa5f45790
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc061a471
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc061a5df
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffffa5dc36d1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffa5db7f75
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffa5db80c1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0624ee6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffffa5e764f7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x67
|
|
18
|
SyS_fdatasync
|
vmlinux |
0xffffffffa5e76803
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x13
|
|
19
|
system_call_fastpath
|
vmlinux |
0xffffffffa6375ddb
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 50s! [mysqld:23047]
Modules linked in: xt_TCPMSS xt_state xt_recent xt_multiport xt_LOG xt_mac xt_owner nf_conntrack_irc nf_conntrack_ftp xt_ecn xt_limit xt_length ipt_ULOG ipt_REJECT nf_reject_ipv4 iptable_mangle nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd e1000e pcspkr sg joydev ptp ppdev pps_core vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel ahci serio_raw libahci vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops ttm e1000 libata drm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log
dm_mod
CPU: 5 PID: 23047 Comm: mysqld Kdump: loaded Tainted: G W L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff8a7a5bfc8000 ti: ffff8a79f72a8000 task.ti: ffff8a79f72a8000
RIP: 0010:[<ffffffffa636b455>] [<ffffffffa636b455>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a79f72abb90 EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff8a7a6f594000 RCX: 0000000000000000
RDX: ffff8a7a728b5818 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff8a79f72abb90 R08: 0000000000000000 R09: 000000005cf64fff
R10: ffff8a7a723f98d8 R11: ffff8a7a6f58e840 R12: ffff8a79f72abb38
R13: ffffffffa5fa1973 R14: ffff8a79f72abb38 R15: ffff8a7a6f5a8548
FS: 00007effb37fe700(0000) GS:ffff8a7a75740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff994f95ff0 CR3: 000000021b2bc000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc0463d65>] ata_scsi_queuecmd+0x155/0x450 [libata]
[<ffffffffc0460a40>] ? ata_scsi_invalid_field+0x50/0x50 [libata]
[<ffffffffa60d1700>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa60dac9c>] scsi_request_fn+0x4cc/0x680
[<ffffffffa5f67e6e>] ? deadline_add_request+0x6e/0x90
[<ffffffffa5f43849>] __blk_run_queue+0x39/0x50
[<ffffffffa5f47553>] blk_queue_bio+0x3b3/0x400
[<ffffffffa5f454e7>] generic_make_request+0x147/0x380
[<ffffffffa5f45790>] submit_bio+0x70/0x150
[<ffffffffc061a471>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc061a5df>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffa5dc36d1>] do_writepages+0x21/0x50
[<ffffffffa5db7f75>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffa5db80c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0624ee6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffa5e764f7>] do_fsync+0x67/0xb0
[<ffffffffa5e76803>] SyS_fdatasync+0x13/0x20
[<ffffffffa6375ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #67635
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8cb89845
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc03c7632
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffff8c8ec0e0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8c8e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8c8ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff8c777fce
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff8c753519
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff8c757223
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff8c7551a7
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff8c755450
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc062af31
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc062b26f
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff8c5cae21
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff8c5bf995
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff8c5bfae1
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0635d66
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff8c683a47
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x67
|
|
18
|
SyS_fdatasync
|
vmlinux |
0xffffffff8c683d53
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x13
|
|
19
|
system_call_fastpath
|
vmlinux |
0xffffffff8cb94f92
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 21s! [mysqld:65902]
IPv4: martian source 10.1.134.255 from 10.1.134.110, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 0c 29 98 e5 af 08 00 ........).....
IPv4: martian source 10.1.134.255 from 10.1.134.110, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 0c 29 98 e5 af 08 00 ........).....
IPv4: martian source 10.1.134.255 from 10.1.134.58, on dev ens224
ll header: 00000000: ff ff ff ff ff ff 00 0c 29 f4 e8 4f 08 00 ........)..O..
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper ata_piix syscopyarea ahci sysfillrect sysimgblt fb_sys_fops ttm libahci crct10dif_pclmul crct10dif_common crc32c_intel drm libata nfit libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: CyProtectDrv]
CPU: 5 PID: 65902 Comm: mysqld Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e839ebed280 ti: ffff8e838746c000 task.ti: ffff8e838746c000
RIP: 0010:[<ffffffff8cb89845>] [<ffffffff8cb89845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e838746fb60 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff8c7b2363 RCX: ffffffff8d034f40
RDX: ffffb85e83130000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8e838746fb60 R08: 0000000000000001 R09: 0000000000000000
R10: ffff8e7cd8bfb000 R11: ffff8e7cd83de000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff8e7fd39f6280 R15: ffffffff8c8ead99
FS: 00007f38525c6700(0000) GS:ffff8e839d740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd0e3d23000 CR3: 00000003b2d70000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc03c7632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff8c8ec0e0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8c8e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8c8ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8c777fce>] ? deadline_add_request+0x6e/0x90
[<ffffffff8c753519>] __blk_run_queue+0x39/0x50
[<ffffffff8c757223>] blk_queue_bio+0x3b3/0x400
[<ffffffff8c7551a7>] generic_make_request+0x147/0x380
[<ffffffff8c755450>] submit_bio+0x70/0x150
[<ffffffffc062af31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc062b26f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff8c5cae21>] do_writepages+0x21/0x50
[<ffffffff8c5bf995>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff8c5bfae1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0635d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8c683a47>] do_fsync+0x67/0xb0
[<ffffffff8c683d53>] SyS_fdatasync+0x13/0x20
[<ffffffff8cb94f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #32132
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff87f883c5
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc0062632
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4a2
|
|
3
|
|
vmlinux |
0xffffffff87ceb0a0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff87ce2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff87cec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff87b771ae
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff87b527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff87b564b3
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff87b54437
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff87b546e0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc028cfb1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc028d2ef
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff879cb261
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff879bfc85
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff879bfdd1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0297dd6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff87a82f57
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x67
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff87f92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff87a83263
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffff87f92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
|
21
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff87f92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 34s! [tlsmgr:14479]
Modules linked in: ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev vmw_balloon pcspkr joydev sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea sysfillrect crc_t10dif sysimgblt fb_sys_fops crct10dif_generic ttm crct10dif_common drm ahci ata_piix libahci libata crc32c_intel serio_raw vmxnet3
vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14479 Comm: tlsmgr Kdump: loaded Tainted: G W L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9c10b7985230 ti: ffff9c10b745c000 task.ti: ffff9c10b745c000
RIP: 0010:[<ffffffff87f883c5>] [<ffffffff87f883c5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9c10b745fb60 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffffffff87bb14b3 RCX: ffff9c10b9097060
RDX: ffffb1a300770000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9c10b745fb60 R08: 0000000000000000 R09: ffff9c101ad6b100
R10: ffff9c0fb6712c00 R11: ffff9c0fb661f000 R12: 0000000000000000
R13: 0000000000000020 R14: ffff9c10b7bcbe00 R15: ffffffff87ce9d59
FS: 00007f4aa7616880(0000) GS:ffff9c10bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000e63508 CR3: 0000000138328000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc0062632>] pvscsi_queue+0x4a2/0x920 [vmw_pvscsi]
[<ffffffff87ceb0a0>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff87ce2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff87cec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff87b771ae>] ? deadline_add_request+0x6e/0x90
[<ffffffff87b527a9>] __blk_run_queue+0x39/0x50
[<ffffffff87b564b3>] blk_queue_bio+0x3b3/0x400
[<ffffffff87b54437>] generic_make_request+0x147/0x380
[<ffffffff87b546e0>] submit_bio+0x70/0x150
[<ffffffffc028cfb1>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc028d2ef>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff879cb261>] do_writepages+0x21/0x50
[<ffffffff879bfc85>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff879bfdd1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0297dd6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff87a82f57>] do_fsync+0x67/0xb0
[<ffffffff87f92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff87a83263>] SyS_fdatasync+0x13/0x20
[<ffffffff87f92ed2>] system_call_fastpath+0x25/0x2a
[<ffffffff87f92e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #23418
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816b6545
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x15
|
|
2
|
pvscsi_queue
|
vmw_pvscsi |
0xffffffffc005966a
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x44a
|
|
3
|
|
vmlinux |
0xffffffff81475750
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x170
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8146d8f0
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8147678f
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x4df
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812fd1d9
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x39
|
|
7
|
blk_queue_bio
|
vmlinux |
0xffffffff81300b33
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x363
|
|
8
|
generic_make_request
|
vmlinux |
0xffffffff812feb7b
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x10b
|
|
9
|
submit_bio
|
vmlinux |
0xffffffff812fee00
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x70
|
|
10
|
|
vmlinux |
0xffffffff810295d0
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x150
|
|
11
|
xfs_submit_ioend.isra.16
|
xfs |
0xffffffffc0258071
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc0258231
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xd1
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff81193531
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff811880f5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff81188241
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0263746
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff81238467
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x67
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816c0655
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xa2
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff81238773
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffff816c0715
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1c
|
|
21
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816c0661
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 31s! [mysqld:1612]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 sb_edac edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon joydev nfit sg libnvdimm shpchp parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel
ahci serio_raw libahci ata_piix libata vmxnet3 i2c_core vmw_pvscsi floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 1612 Comm: mysqld Tainted: G W L ------------ 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff88023f6f9fa0 ti: ffff880034aa4000 task.ti: ffff880034aa4000
RIP: 0010:[<ffffffff816b6545>] [<ffffffff816b6545>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880034aa7b58 EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff880034aa7b18 RCX: 0000000000001000
RDX: ffffc90000e90000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff880034aa7b58 R08: 0000000000000000 R09: ffff8802162c9b00
R10: ffff88022b349c00 R11: ffff880035e29800 R12: 0000000000000020
R13: ffff880192acad80 R14: ffffffff81474359 R15: ffff880034aa7ad0
FS: 00007f749ceb3700(0000) GS:ffff88023fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6d3bd31a10 CR3: 0000000228234000 CR4: 00000000000607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc005966a>] pvscsi_queue+0x44a/0x7e9 [vmw_pvscsi]
[<ffffffff81475750>] ? scsi_test_unit_ready+0x170/0x170
[<ffffffff8146d8f0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8147678f>] scsi_request_fn+0x4df/0x680
[<ffffffff812fd1d9>] __blk_run_queue+0x39/0x50
[<ffffffff81300b33>] blk_queue_bio+0x363/0x3a0
[<ffffffff812feb7b>] generic_make_request+0x10b/0x320
[<ffffffff812fee00>] submit_bio+0x70/0x150
[<ffffffff810295d0>] ? __switch_to+0x150/0x500
[<ffffffffc0258071>] xfs_submit_ioend.isra.16+0x61/0xe0 [xfs]
[<ffffffffc0258231>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff81193531>] do_writepages+0x21/0x50
[<ffffffff811880f5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81188241>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0263746>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff81238467>] do_fsync+0x67/0xb0
[<ffffffff816c0655>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff81238773>] SyS_fdatasync+0x13/0x20
[<ffffffff816c0715>] system_call_fastpath+0x1c/0x21
[<ffffffff816c0661>] ? system_call_after_swapgs+0xae/0x146
Complete report #1178850
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc010444e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc01a474e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01b92f0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8ab03030
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8ab0cbdc
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff8a98e21e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff8a9687d9
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff8a96c703
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff8a96a537
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x157
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff8a96a7e0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffff8a85ecb1
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x31
|
|
12
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc037e341
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc037e68f
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x7f
|
|
14
|
do_writepages
|
vmlinux |
0xffffffff8a7d6b01
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff8a7caba5
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff8a7cad11
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc0389486
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffff8a893195
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x55
|
|
19
|
|
vmlinux |
0xffffffff8a746986
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x1f6
|
|
20
|
SyS_fdatasync
|
vmlinux |
0xffffffff8a893483
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x13
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffff8adc539a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 48s! [postmaster:1544]
Modules linked in: tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi crc32_pclmul snd_ac97_codec ghash_clmulni_intel aesni_intel ac97_bus snd_seq lrw gf128mul snd_seq_device glue_helper ablk_helper snd_pcm cryptd snd_timer
ppdev snd soundcore vmw_balloon joydev sg pcspkr parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit libnvdimm crct10dif_pclmul crct10dif_common mptspi scsi_transport_spi mptscsih ata_piix crc32c_intel libata mptbase e1000 serio_raw drm_panel_orientation_quirks fuse
CPU: 0 PID: 1544 Comm: postmaster Kdump: loaded Not tainted 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9371d5e8d280 ti: ffff9371c6d70000 task.ti: ffff9371c6d70000
RIP: 0010:[<ffffffffc010444e>] [<ffffffffc010444e>] mpt_put_msg_frame+0x5e/0x90 [mptbase]
RSP: 0018:ffff9371c6d73b08 EFLAGS: 00010246
RAX: ffff9eb2c3480000 RBX: ffff9371d3aa7010 RCX: 0000000000000010
RDX: ffff936bf5cc5000 RSI: ffff936d302c6000 RDI: 000000000000000e
RBP: ffff9371c6d73b18 R08: 0000000000000003 R09: 00000001df5d3fff
R10: ffff936d3026e400 R11: ffff9371b033a680 R12: ffff9370db2fc180
R13: ffff9371c6d73a7c R14: ffffffff8a97158b R15: ffff9371c6d73ac0
FS: 00007f7d0890f8c0(0000) GS:ffff9371da800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7d795a1000 CR3: 0000000609b54000 CR4: 00000000003607f0
Call Trace:
[<ffffffffc01a474e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01b92f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8ab03030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffff8ab0cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffff8a98e21e>] ? deadline_add_request+0x6e/0x90
[<ffffffff8a9687d9>] __blk_run_queue+0x39/0x50
[<ffffffff8a96c703>] blk_queue_bio+0x3b3/0x400
[<ffffffff8a96a537>] generic_make_request+0x157/0x390
[<ffffffff8a96a7e0>] submit_bio+0x70/0x150
[<ffffffff8a85ecb1>] ? __sb_end_write+0x31/0x70
[<ffffffffc037e341>] xfs_submit_ioend.isra.12+0x61/0xf0 [xfs]
[<ffffffffc037e68f>] xfs_vm_writepages+0x7f/0xb0 [xfs]
[<ffffffff8a7d6b01>] do_writepages+0x21/0x50
[<ffffffff8a7caba5>] __filemap_fdatawrite_range+0x65/0x90
[<ffffffff8a7cad11>] filemap_write_and_wait_range+0x41/0xa0
[<ffffffffc0389486>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff8a893195>] do_fsync+0x55/0x80
[<ffffffff8a746986>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffff8a893483>] SyS_fdatasync+0x13/0x20
[<ffffffff8adc539a>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 14 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 cc cc cc cc 48 8d 76 08 0f b7 c8 44 89 e2 48 c7
Complete report #933947
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc036e3be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc039071e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03892e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9609b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff960a45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff95f3c84e
Build id: 3.10.0-862.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff95f18ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff95f1c763
Build id: 3.10.0-862.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff95f1a6bb
Build id: 3.10.0-862.el7.x86_64
|
0x10b
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff95f1a940
Build id: 3.10.0-862.el7.x86_64
|
0x70
|
|
11
|
?
__test_set_page_writeback
|
vmlinux |
0xffffffff95d9f244
Build id: 3.10.0-862.el7.x86_64
|
0x54
|
|
12
|
xfs_submit_ioend.isra.14
|
xfs |
0xffffffffc05b14b1
Build id: 3.10.0-862.el7.x86_64
|
0x61
|
|
13
|
xfs_do_writepage
|
xfs |
0xffffffffc05b1cf4
Build id: 3.10.0-862.el7.x86_64
|
0x4a4
|
|
14
|
write_cache_pages
|
vmlinux |
0xffffffff95d9faa4
Build id: 3.10.0-862.el7.x86_64
|
0x254
|
|
15
|
|
xfs |
0xffffffffc05b1850
Build id: 3.10.0-862.el7.x86_64
|
0x150
|
|
16
|
xfs_vm_writepages
|
xfs |
0xffffffffc05b165d
Build id: 3.10.0-862.el7.x86_64
|
0xbd
|
|
17
|
do_writepages
|
vmlinux |
0xffffffff95da0e21
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
18
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff95d95075
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
19
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff95d951c1
Build id: 3.10.0-862.el7.x86_64
|
0x41
|
|
20
|
xfs_file_fsync
|
xfs |
0xffffffffc05bbaf6
Build id: 3.10.0-862.el7.x86_64
|
0x66
|
|
21
|
do_fsync
|
vmlinux |
0xffffffff95e4eda7
Build id: 3.10.0-862.el7.x86_64
|
0x67
|
|
22
|
SyS_fdatasync
|
vmlinux |
0xffffffff95e4f0b3
Build id: 3.10.0-862.el7.x86_64
|
0x13
|
|
23
|
system_call_fastpath
|
vmlinux |
0xffffffff9631f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 97s! [WTCheck.tThread:4185]
Modules linked in: ppp_async crc_ccitt ppp_generic slhc nf_conntrack_pptp nf_conntrack_proto_gre tcp_lp fuse tcp_diag inet_diag cfg80211 rfkill xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack ip_vs nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmnet(OE) vmmon(OE) vmw_vsock_vmci_transport vmw_vmci vsock snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel kvm snd_ens1371 irqbypass crc32_pclmul snd_rawmidi ppdev snd_ac97_codec ghash_clmulni_intel vmw_balloon ac97_bus aesni_intel snd_seq lrw gf128mul snd_seq_device glue_helper ablk_helper snd_pcm cryptd snd_timer joydev pcspkr snd sg i2c_piix4 soundcore nfit libnvdimm
shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common libata crc32c_intel mptspi scsi_transport_spi e1000 serio_raw mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 4185 Comm: WTCheck.tThread Kdump: loaded Tainted: G OEL ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9cc0a7f72f70 ti: ffff9cc0afb58000 task.ti: ffff9cc0afb58000
RIP: 0010:[<ffffffffc036e3be>] [<ffffffffc036e3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9cc0afb5b958 EFLAGS: 00000246
RAX: ffffad50834c0000 RBX: ffff9cc0d3af2020 RCX: 0000000000000039
RDX: ffff9cc0d9b78600 RSI: ffff9cc0d9ad6000 RDI: 000000000000000e
RBP: ffff9cc0afb5b968 R08: 0000000000000003 R09: 000000078086efff
R10: ffff9cc0d987d0d8 R11: ffff9cc0d723b100 R12: ffff9cbf01a11680
R13: ffff9cc0afb5b8cc R14: ffffffff95f212cb R15: ffff9cc0afb5b910
FS: 00007f060adc9700(0000) GS:ffff9cc0dd700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6652d93000 CR3: 00000007b8334000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc039071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03892e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9609b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff960a45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff95f3c84e>] ? deadline_add_request+0x6e/0x90
[<ffffffff95f18ad9>] __blk_run_queue+0x39/0x50
[<ffffffff95f1c763>] blk_queue_bio+0x3b3/0x400
[<ffffffff95f1a6bb>] generic_make_request+0x10b/0x320
[<ffffffff95f1a940>] submit_bio+0x70/0x150
[<ffffffff95d9f244>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc05b14b1>] xfs_submit_ioend.isra.14+0x61/0xe0 [xfs]
[<ffffffffc05b1cf4>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffff95d9faa4>] write_cache_pages+0x254/0x4e0
[<ffffffffc05b1850>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc05b165d>] xfs_vm_writepages+0xbd/0xe0 [xfs]
[<ffffffff95da0e21>] do_writepages+0x21/0x50
[<ffffffff95d95075>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff95d951c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc05bbaf6>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff95e4eda7>] do_fsync+0x67/0xb0
[<ffffffff95e4f0b3>] SyS_fdatasync+0x13/0x20
[<ffffffff9631f7d5>] system_call_fastpath+0x1c/0x21
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 c3 37
Complete report #525857
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01963be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc01f571e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01b12e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff84c9b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff84ca45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff84b3c84e
Build id: 3.10.0-862.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff84b18ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff84b1c763
Build id: 3.10.0-862.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff84b1a6bb
Build id: 3.10.0-862.el7.x86_64
|
0x10b
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff84b1a940
Build id: 3.10.0-862.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffff848cee3b
Build id: 3.10.0-862.el7.x86_64
|
0x5b
|
|
12
|
xfs_submit_ioend.isra.14
|
xfs |
0xffffffffc03f54b1
Build id: 3.10.0-862.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc03f5671
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
|
14
|
do_writepages
|
vmlinux |
0xffffffff849a0e21
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff84995075
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff849951c1
Build id: 3.10.0-862.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc03ffaf6
Build id: 3.10.0-862.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffff84a4eda7
Build id: 3.10.0-862.el7.x86_64
|
0x67
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff84a4f0b3
Build id: 3.10.0-862.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffff84f1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [dm_redolog_thd:2417]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache target_core_mod xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink sfs(OE) vmw_vsock_vmci_transport vsock coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg nfit libnvdimm parport_pc parport vmw_vmci i2c_piix4 shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt crct10dif_pclmul
crct10dif_common fb_sys_fops ttm crc32c_intel drm serio_raw i2c_core mptspi scsi_transport_spi e1000 mptscsih ata_piix libata mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 2417 Comm: dm_redolog_thd Kdump: loaded Tainted: G OE ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9781d2603f40 ti: ffff978070ab8000 task.ti: ffff978070ab8000
RIP: 0010:[<ffffffffc01963be>] [<ffffffffc01963be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff978070abbae8 EFLAGS: 00000246
RAX: ffffb33600d00000 RBX: ffff97807ac7c000 RCX: 000000000000000c
RDX: ffff9781bcb82800 RSI: ffff9781bc7cf000 RDI: 000000000000000e
RBP: ffff978070abbaf8 R08: 0000000000000002 R09: 00000001f05b1fff
R10: ffff9781f1fbe0d8 R11: ffff9781df689a40 R12: ffff9781ed7c8780
R13: ffff978070abba5c R14: ffffffff84b212cb R15: ffff978070abbaa0
FS: 00007f8c8ebeb700(0000) GS:ffff9781f5640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc83cb4b000 CR3: 0000000229236000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc01f571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01b12e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff84c9b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff84ca45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff84b3c84e>] ? deadline_add_request+0x6e/0x90
[<ffffffff84b18ad9>] __blk_run_queue+0x39/0x50
[<ffffffff84b1c763>] blk_queue_bio+0x3b3/0x400
[<ffffffff84b1a6bb>] generic_make_request+0x10b/0x320
[<ffffffff84b1a940>] submit_bio+0x70/0x150
[<ffffffff848cee3b>] ? wake_up_q+0x5b/0x80
[<ffffffffc03f54b1>] xfs_submit_ioend.isra.14+0x61/0xe0 [xfs]
[<ffffffffc03f5671>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff849a0e21>] do_writepages+0x21/0x50
[<ffffffff84995075>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff849951c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc03ffaf6>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff84a4eda7>] do_fsync+0x67/0xb0
[<ffffffff84a4f0b3>] SyS_fdatasync+0x13/0x20
[<ffffffff84f1f7d5>] system_call_fastpath+0x1c/0x21
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 43 1a
Complete report #500052
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc019c3be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc022571e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc021d2e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff97a9b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff97aa45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff9793c84e
Build id: 3.10.0-862.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff97918ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff9791c763
Build id: 3.10.0-862.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff9791a6bb
Build id: 3.10.0-862.el7.x86_64
|
0x10b
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff9791a940
Build id: 3.10.0-862.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.14
|
xfs |
0xffffffffc040b4b1
Build id: 3.10.0-862.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc040b671
Build id: 3.10.0-862.el7.x86_64
|
0xd1
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff977a0e21
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff97795075
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff977951c1
Build id: 3.10.0-862.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0415af6
Build id: 3.10.0-862.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff9784eda7
Build id: 3.10.0-862.el7.x86_64
|
0x67
|
|
18
|
SyS_fdatasync
|
vmlinux |
0xffffffff9784f0b3
Build id: 3.10.0-862.el7.x86_64
|
0x13
|
|
19
|
system_call_fastpath
|
vmlinux |
0xffffffff97d1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 21s! [redis-server:11937]
Modules linked in: ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_timer cryptd snd ppdev vmw_balloon soundcore pcspkr joydev sg vmw_vmci i2c_piix4 shpchp nfit libnvdimm parport_pc parport binfmt_misc xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi drm ata_piix crct10dif_pclmul mptscsih libata crct10dif_common crc32c_intel e1000 i2c_core serio_raw mptbase [last unloaded: ip_tables]
CPU: 8 PID: 11937 Comm: redis-server Tainted: G W L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9d7f9e2a6eb0 ti: ffff9d7f35640000 task.ti: ffff9d7f35640000
RIP: 0010:[<ffffffffc019c3be>] [<ffffffffc019c3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9d7f35643ae8 EFLAGS: 00000246
RAX: ffffaf6a462c0000 RBX: ffff9d7a53f8b000 RCX: 0000000000000002
RDX: ffff9d80f8b88c00 RSI: ffff9d80f8278000 RDI: 000000000000000e
RBP: ffff9d7f35643af8 R08: 0000000000000002 R09: 00000006218bffff
R10: ffff9d80f9abb0d8 R11: ffff9d7268fd4000 R12: ffff9d80f0ba8d80
R13: ffff9d7f35643a5c R14: ffffffff979212cb R15: ffff9d7f35643aa0
FS: 00002b6aa3244700(0000) GS:ffff9d80fd800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002ad3b19f4f80 CR3: 0000000e3aa4e000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffc022571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc021d2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff97a9b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff97aa45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffff9793c84e>] ? deadline_add_request+0x6e/0x90
[<ffffffff97918ad9>] __blk_run_queue+0x39/0x50
[<ffffffff9791c763>] blk_queue_bio+0x3b3/0x400
[<ffffffff9791a6bb>] generic_make_request+0x10b/0x320
[<ffffffff9791a940>] submit_bio+0x70/0x150
[<ffffffffc040b4b1>] xfs_submit_ioend.isra.14+0x61/0xe0 [xfs]
[<ffffffffc040b671>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff977a0e21>] do_writepages+0x21/0x50
[<ffffffff97795075>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff977951c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0415af6>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff9784eda7>] do_fsync+0x67/0xb0
[<ffffffff9784f0b3>] SyS_fdatasync+0x13/0x20
[<ffffffff97d1f7d5>] system_call_fastpath+0x1c/0x21
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 a3 1a
Complete report #376710
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc015a3be
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc014c71e
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01eb2e0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff890e3b90
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff890ed2bc
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff88f7818e
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff88f536d9
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff88f573e3
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff88f55367
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff88f55610
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffff88e50bf1
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x31
|
|
12
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0347f31
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc034826f
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x7f
|
|
14
|
do_writepages
|
vmlinux |
0xffffffff88dcafb1
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff88dbfba5
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff88dbfcf1
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc0352d66
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffff88e83be7
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x67
|
|
19
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff89393ec9
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x96
|
|
20
|
SyS_fdatasync
|
vmlinux |
0xffffffff88e83ef3
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x13
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffff89393f92
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x25
|
|
22
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff89393ed5
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 48s! [postmaster:1482]
Modules linked in: tcp_lp vboxsf(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink sunrpc joydev ppdev pcnet32 vboxguest(OE) sg parport_pc iosf_mbi mii pcspkr parport i2c_piix4 video ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea mptspi scsi_transport_spi sysfillrect ata_piix sysimgblt serio_raw fb_sys_fops mptscsih libata ttm mptbase drm e1000 crc32c_intel drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 1482 Comm: postmaster Tainted: G OEL ------------ 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff992718c35280 ti: ffff99270dbc8000 task.ti: ffff99270dbc8000
RIP: 0010:[<ffffffffc015a3be>] [<ffffffffc015a3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff99270dbcbaf8 EFLAGS: 00010246
RAX: ffffab7500780000 RBX: ffff9927101dd010 RCX: 0000000000000018
RDX: ffff99275338bc00 RSI: ffff99274f12b000 RDI: 000000000000000e
RBP: ffff99270dbcbb08 R08: 0000000000000002 R09: 000000002ed0efff
R10: ffff99271b841800 R11: ffff9927038608c0 R12: ffff992752672d80
R13: ffff99270dbcba6c R14: ffffffff88f5bffb R15: ffff99270dbcbab0
FS: 00007faf085be8c0(0000) GS:ffff99275fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fff5ba6a098 CR3: 00000000cdbcc000 CR4: 00000000000206f0
Call Trace:
[<ffffffffc014c71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01eb2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff890e3b90>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff890ed2bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff88f7818e>] ? deadline_add_request+0x6e/0x90
[<ffffffff88f536d9>] __blk_run_queue+0x39/0x50
[<ffffffff88f573e3>] blk_queue_bio+0x3b3/0x400
[<ffffffff88f55367>] generic_make_request+0x147/0x380
[<ffffffff88f55610>] submit_bio+0x70/0x150
[<ffffffff88e50bf1>] ? __sb_end_write+0x31/0x70
[<ffffffffc0347f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc034826f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff88dcafb1>] do_writepages+0x21/0x50
[<ffffffff88dbfba5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff88dbfcf1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0352d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff88e83be7>] do_fsync+0x67/0xb0
[<ffffffff89393ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff88e83ef3>] SyS_fdatasync+0x13/0x20
[<ffffffff89393f92>] system_call_fastpath+0x25/0x2a
[<ffffffff89393ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 83 16
Complete report #376107
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc04133be
Build id: 3.10.0-1160.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc046271e
Build id: 3.10.0-1160.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc047f2e0
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb40e3980
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb40ed0ac
Build id: 3.10.0-1160.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffb3f77f8e
Build id: 3.10.0-1160.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffb3f534d9
Build id: 3.10.0-1160.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffb3f571e3
Build id: 3.10.0-1160.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffb3f55167
Build id: 3.10.0-1160.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffb3f55410
Build id: 3.10.0-1160.el7.x86_64
|
0x70
|
|
11
|
?
fsnotify_grab_connector
|
vmlinux |
0xffffffffb3e979f3
Build id: 3.10.0-1160.el7.x86_64
|
0x43
|
|
12
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc06a6f31
Build id: 3.10.0-1160.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc06a726f
Build id: 3.10.0-1160.el7.x86_64
|
0x7f
|
|
14
|
do_writepages
|
vmlinux |
0xffffffffb3dcade1
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffb3dbf955
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffb3dbfaa1
Build id: 3.10.0-1160.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc06b1d66
Build id: 3.10.0-1160.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffffb3e83a07
Build id: 3.10.0-1160.el7.x86_64
|
0x67
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffffb3e83d13
Build id: 3.10.0-1160.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffffb4393f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 173s! [etcd:76534]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink binfmt_misc vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper joydev ablk_helper cryptd pcspkr vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx nfit libnvdimm drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops ttm libata crct10dif_pclmul crct10dif_common crc32c_intel drm mptspi scsi_transport_spi serio_raw
mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 76534 Comm: etcd Kdump: loaded Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff989bb826b180 ti: ffff989b11408000 task.ti: ffff989b11408000
RIP: 0010:[<ffffffffc04133be>] [<ffffffffc04133be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff989b1140baf8 EFLAGS: 00000246
RAX: ffffbac7c0d00000 RBX: ffff989b60b37000 RCX: 000000000000001c
RDX: ffff989b35eefa00 RSI: ffff989d32a4c000 RDI: 000000000000000e
RBP: ffff989b1140bb08 R08: 0000000000000002 R09: 000000005c338fff
R10: ffff989d3712c000 R11: ffff989c7a3b39c0 R12: ffff989ca99e9500
R13: ffff989b1140ba6c R14: ffffffffb3f5bdfb R15: ffff989b1140bab0
FS: 000000c000190410(0000) GS:ffff989d35640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fefc0944a10 CR3: 000000008ae7e000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc046271e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc047f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb40e3980>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb40ed0ac>] scsi_request_fn+0x4ac/0x680
[<ffffffffb3f77f8e>] ? deadline_add_request+0x6e/0x90
[<ffffffffb3f534d9>] __blk_run_queue+0x39/0x50
[<ffffffffb3f571e3>] blk_queue_bio+0x3b3/0x400
[<ffffffffb3f55167>] generic_make_request+0x147/0x380
[<ffffffffb3f55410>] submit_bio+0x70/0x150
[<ffffffffb3e979f3>] ? fsnotify_grab_connector+0x43/0x70
[<ffffffffc06a6f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc06a726f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffb3dcade1>] do_writepages+0x21/0x50
[<ffffffffb3dbf955>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffb3dbfaa1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc06b1d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffb3e83a07>] do_fsync+0x67/0xb0
[<ffffffffb3e83d13>] SyS_fdatasync+0x13/0x20
[<ffffffffb4393f92>] system_call_fastpath+0x25/0x2a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 13 42
Complete report #325329
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc02733be
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc02c771e
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc032f2e0
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffb9adeb10
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffb9ae817c
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffffb99738ee
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffb994f0c9
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffb9952de3
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffb9950d77
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffb9951020
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x70
|
|
11
|
|
vmlinux |
0xffffffffb96db48b
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x5b
|
|
12
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc04eccd1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x61
|
|
13
|
xfs_vm_writepages
|
xfs |
0xffffffffc04ed00f
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x7f
|
|
14
|
do_writepages
|
vmlinux |
0xffffffffb97cabe1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x21
|
|
15
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffb97bee75
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x65
|
|
16
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffb97befc1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x41
|
|
17
|
xfs_file_fsync
|
xfs |
0xffffffffc04f79a6
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x66
|
|
18
|
do_fsync
|
vmlinux |
0xffffffffb987fbf7
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x67
|
|
19
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb9d8de15
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xa2
|
|
20
|
SyS_fdatasync
|
vmlinux |
0xffffffffb987ff03
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x13
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffffb9d8dede
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x25
|
|
22
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb9d8de21
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 24s! [mysqld:3279]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink iptable_filter ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 xt_conntrack nf_conntrack ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock nfit edac_mce_amd libnvdimm ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport pcc_cpufreq vmw_vmci i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul
crct10dif_common crc32c_intel mptspi libata scsi_transport_spi mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 3279 Comm: mysqld Kdump: loaded Tainted: G L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff88d915fb62a0 ti: ffff88d904bb0000 task.ti: ffff88d904bb0000
RIP: 0010:[<ffffffffc02733be>] [<ffffffffc02733be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88d904bb3af8 EFLAGS: 00000246
RAX: ffffa02b43200000 RBX: ffff88d919a3b000 RCX: 000000000000001f
RDX: ffff88d91e882a00 RSI: ffff88d9263b3000 RDI: 000000000000000e
RBP: ffff88d904bb3b08 R08: 0000000000000002 R09: 0000000549dcbfff
R10: ffff88d91d0900d8 R11: ffff88d91d366f40 R12: ffff88d847b5d680
R13: ffff88d904bb3a6c R14: ffffffffb995792b R15: ffff88d904bb3ab0
FS: 00007f6abdffb700(0000) GS:ffff88d925740000(0000) knlGS:00000000f7798700
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000257f568 CR3: 000000060ebfa000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc02c771e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc032f2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffb9adeb10>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffb9ae817c>] scsi_request_fn+0x4cc/0x680
[<ffffffffb99738ee>] ? deadline_add_request+0x6e/0x90
[<ffffffffb994f0c9>] __blk_run_queue+0x39/0x50
[<ffffffffb9952de3>] blk_queue_bio+0x3b3/0x400
[<ffffffffb9950d77>] generic_make_request+0x147/0x380
[<ffffffffb9951020>] submit_bio+0x70/0x150
[<ffffffffb96db48b>] ? wake_up_q+0x5b/0x80
[<ffffffffc04eccd1>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc04ed00f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffb97cabe1>] do_writepages+0x21/0x50
[<ffffffffb97bee75>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffb97befc1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc04f79a6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffb987fbf7>] do_fsync+0x67/0xb0
[<ffffffffb9d8de15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffffb987ff03>] SyS_fdatasync+0x13/0x20
[<ffffffffb9d8dede>] system_call_fastpath+0x25/0x2a
[<ffffffffb9d8de21>] ? system_call_after_swapgs+0xae/0x146
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 13 28
Complete report #230545
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc02c73be
Build id: 3.10.0-862.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc031271e
Build id: 3.10.0-862.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03f82e0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffab89b050
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffab8a45bc
Build id: 3.10.0-862.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffffab73c84e
Build id: 3.10.0-862.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffab718ad9
Build id: 3.10.0-862.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffab71c763
Build id: 3.10.0-862.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffab71a6bb
Build id: 3.10.0-862.el7.x86_64
|
0x10b
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffab71a940
Build id: 3.10.0-862.el7.x86_64
|
0x70
|
|
11
|
?
__test_set_page_writeback
|
vmlinux |
0xffffffffab59f244
Build id: 3.10.0-862.el7.x86_64
|
0x54
|
|
12
|
xfs_submit_ioend.isra.14
|
xfs |
0xffffffffc051e4b1
Build id: 3.10.0-862.el7.x86_64
|
0x61
|
|
13
|
xfs_do_writepage
|
xfs |
0xffffffffc051ecf4
Build id: 3.10.0-862.el7.x86_64
|
0x4a4
|
|
14
|
write_cache_pages
|
vmlinux |
0xffffffffab59faa4
Build id: 3.10.0-862.el7.x86_64
|
0x254
|
|
15
|
|
xfs |
0xffffffffc051e850
Build id: 3.10.0-862.el7.x86_64
|
0x150
|
|
16
|
xfs_vm_writepages
|
xfs |
0xffffffffc051e65d
Build id: 3.10.0-862.el7.x86_64
|
0xbd
|
|
17
|
do_writepages
|
vmlinux |
0xffffffffab5a0e21
Build id: 3.10.0-862.el7.x86_64
|
0x21
|
|
18
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffab595075
Build id: 3.10.0-862.el7.x86_64
|
0x65
|
|
19
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffab5951c1
Build id: 3.10.0-862.el7.x86_64
|
0x41
|
|
20
|
xfs_file_fsync
|
xfs |
0xffffffffc0528af6
Build id: 3.10.0-862.el7.x86_64
|
0x66
|
|
21
|
do_fsync
|
vmlinux |
0xffffffffab64eda7
Build id: 3.10.0-862.el7.x86_64
|
0x67
|
|
22
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffabb1f715
Build id: 3.10.0-862.el7.x86_64
|
0xa2
|
|
23
|
SyS_fdatasync
|
vmlinux |
0xffffffffab64f0b3
Build id: 3.10.0-862.el7.x86_64
|
0x13
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffffabb1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
|
25
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffabb1f721
Build id: 3.10.0-862.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 29s! [etcd:1453]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 lockd nf_nat_ipv6 grace fscache xt_set xt_mark ip_set_hash_ipportip ip_set_hash_ipport ip_set_bitmap_port ip_set_hash_ipportnet ip_set dummy xt_comment nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle overlay(T) ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack vmw_vsock_vmci_transport vsock sunrpc crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper
cryptd joydev pcspkr ppdev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci shpchp sch_fq_codel ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci ata_piix libahci crct10dif_pclmul mptspi crct10dif_common scsi_transport_spi crc32c_intel libata mptscsih serio_raw i2c_core vmxnet3 mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 1453 Comm: etcd Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff883ab5494f10 ti: ffff883e9e2d0000 task.ti: ffff883e9e2d0000
RIP: 0010:[<ffffffffc02c73be>] [<ffffffffc02c73be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff883e9e2d3958 EFLAGS: 00000246
RAX: ffff9aff01cc0000 RBX: ffff883cf6f8e000 RCX: 0000000000000001
RDX: ffff883eac3a5e00 RSI: ffff883eac0fd000 RDI: 000000000000000e
RBP: ffff883e9e2d3968 R08: 0000000000000002 R09: 000000037dd73fff
R10: ffff883eac2730d8 R11: ffff883e80a84380 R12: ffff883d65ffcd80
R13: ffff883e9e2d38cc R14: ffffffffab7212cb R15: ffff883e9e2d3910
FS: 000000c420250890(0000) GS:ffff883ebfc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efd71256524 CR3: 0000000421b44000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc031271e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03f82e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffab89b050>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffab8a45bc>] scsi_request_fn+0x4cc/0x680
[<ffffffffab73c84e>] ? deadline_add_request+0x6e/0x90
[<ffffffffab718ad9>] __blk_run_queue+0x39/0x50
[<ffffffffab71c763>] blk_queue_bio+0x3b3/0x400
[<ffffffffab71a6bb>] generic_make_request+0x10b/0x320
[<ffffffffab71a940>] submit_bio+0x70/0x150
[<ffffffffab59f244>] ? __test_set_page_writeback+0x54/0x1a0
[<ffffffffc051e4b1>] xfs_submit_ioend.isra.14+0x61/0xe0 [xfs]
[<ffffffffc051ecf4>] xfs_do_writepage+0x4a4/0x510 [xfs]
[<ffffffffab59faa4>] write_cache_pages+0x254/0x4e0
[<ffffffffc051e850>] ? xfs_aops_discard_page+0x150/0x150 [xfs]
[<ffffffffc051e65d>] xfs_vm_writepages+0xbd/0xe0 [xfs]
[<ffffffffab5a0e21>] do_writepages+0x21/0x50
[<ffffffffab595075>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffab5951c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0528af6>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffffab64eda7>] do_fsync+0x67/0xb0
[<ffffffffabb1f715>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffffab64f0b3>] SyS_fdatasync+0x13/0x20
[<ffffffffabb1f7d5>] system_call_fastpath+0x1c/0x21
[<ffffffffabb1f721>] ? system_call_after_swapgs+0xae/0x146
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 2d
Complete report #126594
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01f23be
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc022071e
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc02342e0
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffab0e3e70
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffab0ed59c
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffaaf7840e
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffffaaf53809
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffffaaf57613
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffffaaf55497
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffffaaf55740
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x70
|
|
11
|
|
event_collector_2_1_364486 |
0xffffffffc0607fc0
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x60
|
|
12
|
|
event_collector_2_1_364486 |
0xffffffffc05f7417
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xc7
|
|
13
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0427f31
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x61
|
|
14
|
xfs_vm_writepages
|
xfs |
0xffffffffc042826f
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x7f
|
|
15
|
do_writepages
|
vmlinux |
0xffffffffaadcaef1
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x21
|
|
16
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffffaadbfae5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x65
|
|
17
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffffaadbfc31
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x41
|
|
18
|
xfs_file_fsync
|
xfs |
0xffffffffc0432d66
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x66
|
|
19
|
do_fsync
|
vmlinux |
0xffffffffaae83b87
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x67
|
|
20
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffab395ec9
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x96
|
|
21
|
SyS_fdatasync
|
vmlinux |
0xffffffffaae83e93
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x13
|
|
22
|
system_call_fastpath
|
vmlinux |
0xffffffffab395f92
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x25
|
|
23
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffab395ed5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [kworker/u256:1:107250]
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [postmaster:108264]
Modules linked in: event_collector_2_1_364486(OE) binfmt_misc ip6table_filter ip6_tables iptable_filter talpa_vfshook(OE) talpa_pedconnector(OE) talpa_pedevice(OE) talpa_vcdevice(OE) talpa_core(OE) talpa_linux(OE) talpa_syscallhook(OE) vmw_vsock_vmci_transport vsock crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd pcspkr sg vmw_balloon joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel libata mptspi scsi_transport_spi mptscsih nfit serio_raw mptbase libnvdimm vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log
dm_mod fuse
CPU: 1 PID: 108264 Comm: postmaster Kdump: loaded Tainted: G OEL ------------ 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9ba897f7b180 ti: ffff9ba9e509c000 task.ti: ffff9ba9e509c000
RIP: 0010:[<ffffffffc01f23be>] [<ffffffffc01f23be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff9ba9e509faf8 EFLAGS: 00000246
RAX: ffffb8f900d40000 RBX: ffff9ba9bcfaa010 RCX: 000000000000003a
RDX: ffff9ba87601c200 RSI: ffff9baa7398c000 RDI: 000000000000000e
RBP: ffff9ba9e509fb08 R08: 0000000000000003 R09: 0000000189c5efff
R10: ffff9ba8767bcc00 R11: ffff9ba9b8fa6300 R12: ffff9ba9bafdc180
R13: ffff9ba9e509fa6c R14: ffffffffaaf5c22b R15: ffff9ba9e509fab0
FS: 00007f30052ac8c0(0000) GS:ffff9baa75640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2ff7e3fce0 CR3: 00000001562da000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc022071e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc02342e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffab0e3e70>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffab0ed59c>] scsi_request_fn+0x4ac/0x680
[<ffffffffaaf7840e>] ? deadline_add_request+0x6e/0x90
[<ffffffffaaf53809>] __blk_run_queue+0x39/0x50
[<ffffffffaaf57613>] blk_queue_bio+0x3b3/0x400
[<ffffffffaaf55497>] generic_make_request+0x147/0x380
[<ffffffffaaf55740>] submit_bio+0x70/0x150
[<ffffffffc0607fc0>] ? file_process_status+0x60/0x80 [event_collector_2_1_364486]
[<ffffffffc05f7417>] ? cb_mem_cache_strdup+0xc7/0x230 [event_collector_2_1_364486]
[<ffffffffc0427f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc042826f>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffffaadcaef1>] do_writepages+0x21/0x50
[<ffffffffaadbfae5>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffffaadbfc31>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0432d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffffaae83b87>] do_fsync+0x67/0xb0
[<ffffffffab395ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffaae83e93>] SyS_fdatasync+0x13/0x20
[<ffffffffab395f92>] system_call_fastpath+0x25/0x2a
[<ffffffffab395ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 03 20
Complete report #23193
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc047e3be
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc04a371e
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc04bd2e0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff868e39d0
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff868ed0fc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff86777fce
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff86753519
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff86757223
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff867551a7
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff86755450
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc06baf31
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x61
|
|
12
|
xfs_do_writepage
|
xfs |
0xffffffffc06bb724
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x494
|
|
13
|
write_cache_pages
|
vmlinux |
0xffffffff865c9dcc
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x21c
|
|
14
|
|
xfs |
0xffffffffc06bb290
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa0
|
|
15
|
xfs_vm_writepages
|
xfs |
0xffffffffc06bb25b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6b
|
|
16
|
do_writepages
|
vmlinux |
0xffffffff865cae21
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x21
|
|
17
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff865bf995
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x65
|
|
18
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff865bfae1
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x41
|
|
19
|
xfs_file_fsync
|
xfs |
0xffffffffc06c5d66
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x66
|
|
20
|
do_fsync
|
vmlinux |
0xffffffff86683a47
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x67
|
|
21
|
|
vmlinux |
0xffffffff8653e4a4
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb4
|
|
22
|
SyS_fdatasync
|
vmlinux |
0xffffffff86683d53
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x13
|
|
23
|
tracesys
|
vmlinux |
0xffffffff86b95226
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 28s! [elasticsearch[z:94590]
Modules linked in: tcp_diag inet_diag binfmt_misc ipmi_devintf ipmi_msghandler xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat_ipv4 nf_nat overlay(T) vmw_vsock_vmci_transport vsock iptable_filter coretemp crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon sg joydev pcspkr vmw_vmci parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common libata mptspi scsi_transport_spi mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror
dm_region_hash dm_log dm_mod fuse nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack libcrc32c br_netfilter bridge stp llc
CPU: 0 PID: 94590 Comm: elasticsearch[z Tainted: G L ------------ T 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9b651fab8000 ti: ffff9b652df50000 task.ti: ffff9b652df50000
RIP: 0010:[<ffffffffc047e3be>] [<ffffffffc047e3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9b652df53968 EFLAGS: 00000246
RAX: ffffb51c43500000 RBX: ffff9b6cf6bea000 RCX: 0000000000000006
RDX: ffff9b6d1ddbb200 RSI: ffff9b6d1f238000 RDI: 000000000000000e
RBP: ffff9b652df53978 R08: 0000000000000002 R09: 00000003fe935fff
R10: ffff9b66627c0c00 R11: ffff9b666579f800 R12: ffff9b6c5550b780
R13: ffff9b652df538dc R14: ffffffff8675be3b R15: ffff9b652df53920
FS: 00007f09e5338700(0000) GS:ffff9b6d1d600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f791b22a9e5 CR3: 00000007efc16000 CR4: 00000000000007f0
Call Trace:
[<ffffffffc04a371e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc04bd2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff868e39d0>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff868ed0fc>] scsi_request_fn+0x4ac/0x680
[<ffffffff86777fce>] ? deadline_add_request+0x6e/0x90
[<ffffffff86753519>] __blk_run_queue+0x39/0x50
[<ffffffff86757223>] blk_queue_bio+0x3b3/0x400
[<ffffffff867551a7>] generic_make_request+0x147/0x380
[<ffffffff86755450>] submit_bio+0x70/0x150
[<ffffffffc06baf31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc06bb724>] xfs_do_writepage+0x494/0x550 [xfs]
[<ffffffff865c9dcc>] write_cache_pages+0x21c/0x470
[<ffffffffc06bb290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[<ffffffffc06bb25b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[<ffffffff865cae21>] do_writepages+0x21/0x50
[<ffffffff865bf995>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff865bfae1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc06c5d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff86683a47>] do_fsync+0x67/0xb0
[<ffffffff8653e4a4>] ? __audit_syscall_entry+0xb4/0x110
[<ffffffff86683d53>] SyS_fdatasync+0x13/0x20
[<ffffffff86b95226>] tracesys+0xa6/0xcc
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 c3 48
Complete report #18333
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01c33be
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc01e671e
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01de2e0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff960d1700
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff960dac9c
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff95f67e6e
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff95f43849
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff95f47553
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff95f454e7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff95f45790
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0416471
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc04165df
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff95dc36d1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff95db7f75
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff95db80c1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0420ee6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff95e764f7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x67
|
|
18
|
SyS_fdatasync
|
vmlinux |
0xffffffff95e76803
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x13
|
|
19
|
system_call_fastpath
|
vmlinux |
0xffffffff96375ddb
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 24s! [mysqld:4282]
Modules linked in: tcp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd vmw_balloon pcspkr sg joydev vmw_vmci i2c_piix4 parport_pc parport
ip_tables xfs libcrc32c vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper ata_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm pata_acpi drm nfit mptspi scsi_transport_spi e1000 mptscsih ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common mptbase crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 4282 Comm: mysqld Kdump: loaded Tainted: G W L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff94cd0a2c9040 ti: ffff94cd0126c000 task.ti: ffff94cd0126c000
RIP: 0010:[<ffffffffc01c33be>] [<ffffffffc01c33be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff94cd0126faf8 EFLAGS: 00000246
RAX: ffffaaa401c00000 RBX: ffff94cc9d3883a0 RCX: 000000000000000e
RDX: ffff94cd2a306800 RSI: ffff94cd2acef000 RDI: 000000000000000e
RBP: ffff94cd0126fb08 R08: 0000000000000002 R09: 000000040ea4efff
R10: ffff94cd2afc50d8 R11: ffff94cd28270000 R12: ffff94cd2021a480
R13: ffff94cd0126fa6c R14: ffffffff95f4c09b R15: ffff94cd0126fab0
FS: 00007f4c761e3700(0000) GS:ffff94cd3fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffe61238080 CR3: 0000000416222000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc01e671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01de2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff960d1700>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff960dac9c>] scsi_request_fn+0x4cc/0x680
[<ffffffff95f67e6e>] ? deadline_add_request+0x6e/0x90
[<ffffffff95f43849>] __blk_run_queue+0x39/0x50
[<ffffffff95f47553>] blk_queue_bio+0x3b3/0x400
[<ffffffff95f454e7>] generic_make_request+0x147/0x380
[<ffffffff95f45790>] submit_bio+0x70/0x150
[<ffffffffc0416471>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc04165df>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff95dc36d1>] do_writepages+0x21/0x50
[<ffffffff95db7f75>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff95db80c1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0420ee6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff95e764f7>] do_fsync+0x67/0xb0
[<ffffffff95e76803>] SyS_fdatasync+0x13/0x20
[<ffffffff96375ddb>] system_call_fastpath+0x22/0x27
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 13 1d
Complete report #6891
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00f43be
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc00dd71e
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01272e0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff9a6e2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff9a6ec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff9a5771ae
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff9a5527a9
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x39
|
|
8
|
blk_queue_bio
|
vmlinux |
0xffffffff9a5564b3
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x3b3
|
|
9
|
generic_make_request
|
vmlinux |
0xffffffff9a554437
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x147
|
|
10
|
submit_bio
|
vmlinux |
0xffffffff9a5546e0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x70
|
|
11
|
xfs_submit_ioend.isra.12
|
xfs |
0xffffffffc0307fb1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x61
|
|
12
|
xfs_vm_writepages
|
xfs |
0xffffffffc03082ef
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x7f
|
|
13
|
do_writepages
|
vmlinux |
0xffffffff9a3cb261
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x21
|
|
14
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff9a3bfc85
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x65
|
|
15
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff9a3bfdd1
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x41
|
|
16
|
xfs_file_fsync
|
xfs |
0xffffffffc0312dd6
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x66
|
|
17
|
do_fsync
|
vmlinux |
0xffffffff9a482f57
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x67
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9a992e09
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x96
|
|
19
|
SyS_fdatasync
|
vmlinux |
0xffffffff9a483263
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x13
|
|
20
|
system_call_fastpath
|
vmlinux |
0xffffffff9a992ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x25
|
|
21
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff9a992e15
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [psvc:27084]
Modules linked in: vmfsrtx_dev(POE) vmfsrtx_dev_helper(POE) vmfsrtx_export(POE) vmfsrtx_bridge(POE) vmfsrtx_bridge_helper(OE) fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper vmw_balloon ablk_helper cryptd pcspkr joydev sg parport_pc
parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel nfit libnvdimm mptspi scsi_transport_spi mptscsih serio_raw ata_piix mptbase libata vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: vmfsrtx_dev_helper]
CPU: 0 PID: 27084 Comm: psvc Kdump: loaded Tainted: P W OEL ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8896b2be1070 ti: ffff8895f0da0000 task.ti: ffff8895f0da0000
RIP: 0010:[<ffffffffc00f43be>] [<ffffffffc00f43be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8895f0da3af8 EFLAGS: 00000246
RAX: ffff9bfbc0740000 RBX: ffff88958344c4a0 RCX: 0000000000000003
RDX: ffff8895b5f97600 RSI: ffff8895b5eba000 RDI: 000000000000000e
RBP: ffff8895f0da3b08 R08: 0000000000000002 R09: 00000000b995dfff
R10: ffff8895b5440800 R11: ffff8896b33bda40 R12: ffff8895c65d8300
R13: ffff8895f0da3a6c R14: ffffffff9a55b0cb R15: ffff8895f0da3ab0
FS: 00007f15cda84700(0000) GS:ffff8896bae00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f15f00f5290 CR3: 0000000070e72000 CR4: 00000000000407f0
Call Trace:
[<ffffffffc00dd71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01272e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff9a6e2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff9a6ec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffff9a5771ae>] ? deadline_add_request+0x6e/0x90
[<ffffffff9a5527a9>] __blk_run_queue+0x39/0x50
[<ffffffff9a5564b3>] blk_queue_bio+0x3b3/0x400
[<ffffffff9a554437>] generic_make_request+0x147/0x380
[<ffffffff9a5546e0>] submit_bio+0x70/0x150
[<ffffffffc0307fb1>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[<ffffffffc03082ef>] xfs_vm_writepages+0x7f/0xa0 [xfs]
[<ffffffff9a3cb261>] do_writepages+0x21/0x50
[<ffffffff9a3bfc85>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff9a3bfdd1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc0312dd6>] xfs_file_fsync+0x66/0x1c0 [xfs]
[<ffffffff9a482f57>] do_fsync+0x67/0xb0
[<ffffffff9a992e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff9a483263>] SyS_fdatasync+0x13/0x20
[<ffffffff9a992ed2>] system_call_fastpath+0x25/0x2a
[<ffffffff9a992e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 23 10
Complete report #6461
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc00573be
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc00e9729
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x249
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc00502b0
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff814675aa
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xaa
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8147036f
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x4df
|
|
6
|
__blk_run_queue
|
vmlinux |
0xffffffff812f9373
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x33
|
|
7
|
blk_queue_bio
|
vmlinux |
0xffffffff812fcc73
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x363
|
|
8
|
generic_make_request
|
vmlinux |
0xffffffff812face5
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x105
|
|
9
|
submit_bio
|
vmlinux |
0xffffffff812faf60
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x70
|
|
10
|
xfs_submit_ioend.isra.16
|
xfs |
0xffffffffc0271661
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x61
|
|
11
|
xfs_vm_writepages
|
xfs |
0xffffffffc0271821
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xd1
|
|
12
|
do_writepages
|
vmlinux |
0xffffffff81190d1e
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x1e
|
|
13
|
__filemap_fdatawrite_range
|
vmlinux |
0xffffffff81185955
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x65
|
|
14
|
filemap_write_and_wait_range
|
vmlinux |
0xffffffff81185aa1
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x41
|
|
15
|
xfs_file_fsync
|
xfs |
0xffffffffc027cbb6
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x66
|
|
16
|
do_fsync
|
vmlinux |
0xffffffff81235565
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x65
|
|
17
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816b8930
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x15d
|
|
18
|
SyS_fdatasync
|
vmlinux |
0xffffffff81235863
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x13
|
|
19
|
system_call_fastpath
|
vmlinux |
0xffffffff816b89fd
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x16
|
|
20
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff816b889d
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xca
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 58s! [mysqld:2912]
Modules linked in: ppdev coretemp joydev vmw_balloon pcspkr sg i2c_piix4 vmw_vmci shpchp parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm mptspi scsi_transport_spi mptscsih vmxnet3 ata_piix libata i2c_core mptbase floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 2912 Comm: mysqld Not tainted 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff88001cc18fd0 ti: ffff88007a180000 task.ti: ffff88007a180000
RIP: 0010:[<ffffffffc00573be>] [<ffffffffc00573be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff88007a183af0 EFLAGS: 00000246
RAX: ffffc90000780000 RBX: ffff880002f3d200 RCX: 000000000000001b
RDX: ffff880036660e00 RSI: ffff8800364ff000 RDI: 000000000000000e
RBP: ffff88007a183b00 R08: 0000000000000002 R09: ffff8800364f80d8
R10: ffff88004cf59dc0 R11: ffff88004cf59dc0 R12: ffffffff816b9bff
R13: ffffffff816b9c06 R14: ffffffff816b9c0d R15: ffffffff816b9c14
FS: 00007fd2cfe6c700(0000) GS:ffff88013fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe7db048000 CR3: 00000000237e6000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffc00e9729>] mptscsih_qcmd+0x249/0x820 [mptscsih]
[<ffffffffc00502b0>] mptspi_qcmd+0x50/0xe0 [mptspi]
[<ffffffff814675aa>] scsi_dispatch_cmd+0xaa/0x230
[<ffffffff8147036f>] scsi_request_fn+0x4df/0x680
[<ffffffff812f9373>] __blk_run_queue+0x33/0x40
[<ffffffff812fcc73>] blk_queue_bio+0x363/0x3a0
[<ffffffff812face5>] generic_make_request+0x105/0x310
[<ffffffff812faf60>] submit_bio+0x70/0x150
[<ffffffffc0271661>] xfs_submit_ioend.isra.16+0x61/0xe0 [xfs]
[<ffffffffc0271821>] xfs_vm_writepages+0xd1/0xe0 [xfs]
[<ffffffff81190d1e>] do_writepages+0x1e/0x40
[<ffffffff81185955>] __filemap_fdatawrite_range+0x65/0x80
[<ffffffff81185aa1>] filemap_write_and_wait_range+0x41/0x90
[<ffffffffc027cbb6>] xfs_file_fsync+0x66/0x1e0 [xfs]
[<ffffffff81235565>] do_fsync+0x65/0xa0
[<ffffffff816b8930>] ? system_call_after_swapgs+0x15d/0x214
[<ffffffff81235863>] SyS_fdatasync+0x13/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
[<ffffffff816b889d>] ? system_call_after_swapgs+0xca/0x214
Code: 8b 96 68 01 00 00 0f b7 c8 44 03 a6 b0 01 00 00 44 8b 04 8a 45 09 c4 f6 86 e0 00 00 00 04 75 10 48 8b 83 e8 00 00 00 44 89 60 40 <5b> 41 5c 5d c3 48 8d 76 08 0f b7 c8 44 89 e2 48 c7 c7 78 53 06