Complete report #1075214
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc04048e7
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x7f7
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc042f74e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc044f2f0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffaeb03030
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffaeb0cbdc
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffae98e21e
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x6e
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 32s! [ksmtuned:4624]
RSP: 0018:ffff8bcf21d4f888 EFLAGS: 00000246
RAX: ffffaabf006c0000 RBX: ffff8bcff4526040 RCX: 0000000000000016
RDX: ffff8bcff06fce00 RSI: ffff8bcff5fbc000 RDI: 000000000000000e
RBP: ffff8bcf21d4f898 R08: 0000000000000003 R09: 000000008072cfff
R10: ffff8bcef5b9e800 R11: ffff8bcef4eb7800 R12: ffff8bcf58a1fc00
R13: ffff8bcf21d4f7fc R14: ffffffffae97158b R15: ffff8bcf21d4f840
FS: 0000000000000000(0000) GS:ffff8bcff9680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f83e2241800 CR3: 0000000090d6e000 CR4: 0000000000760fe0
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd btusb btrtl btbcm snd_ens1371 btintel snd_rawmidi snd_ac97_codec bluetooth joydev ac97_bus snd_seq snd_seq_device pcspkr snd_pcm rfkill snd_timer snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel nfit mptspi ata_piix scsi_transport_spi serio_raw mptscsih mptbase e1000 libata drm_panel_orientation_quirks libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 4624 Comm: ksmtuned Tainted: G L ------------ 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8bcfd8d3d280 ti: ffff8bcf06d2c000 task.ti: ffff8bcf06d2c000
RIP: 0010:[<ffffffffc04048e7>] [<ffffffffc04048e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8bcff9603998 EFLAGS: 00000286
RAX: ffffaabf00a03818 RBX: 0000000000000000 RCX: 00000000000000bd
RDX: 00000000000000bd RSI: ffff8bcef5a13b40 RDI: ffff8bcff41a6000
RBP: ffff8bcff9603a18 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff8bcff9603908
R13: ffffffffaedc63f2 R14: ffff8bcff9603a18 R15: ffff8bcff41a6000
FS: 00007f48ec0d7740(0000) GS:ffff8bcff9600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f48eb812d20 CR3: 0000000046c86000 CR4: 0000000000760ff0
PKRU: 00000000
Call Trace:
[<ffffffffc042f74e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc044f2f0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffaeb03030>] scsi_dispatch_cmd+0xb0/0x250
[<ffffffffaeb0cbdc>] scsi_request_fn+0x4ac/0x680
[<ffffffffae98e21e>] ? deadline_add_request+0x6e/0x90
Complete report #18762
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01e43be
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc027f71e
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc03362e0
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8aadc890
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8aae5efc
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff8a9716de
Build id: 3.10.0-1062.1.2.el7.x86_64
|
0x6e
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [kworker/u256:1:99153]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink binfmt_misc vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus coretemp snd_seq iosf_mbi crc32_pclmul snd_seq_device snd_pcm ghash_clmulni_intel aesni_intel ppdev snd_timer snd lrw gf128mul vmw_balloon glue_helper soundcore ablk_helper cryptd sg joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom sd_mod ata_generic
crc_t10dif pata_acpi crct10dif_generic vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit ttm mptspi libnvdimm serio_raw scsi_transport_spi drm mptscsih e1000 ata_piix libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 0 PID: 99153 Comm: kworker/u256:1 Kdump: loaded Tainted: G W L ------------ T 3.10.0-1062.1.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
Workqueue: writeback bdi_writeback_workfn (flush-253:0)
task: ffff9da9e105d230 ti: ffff9da9c5130000 task.ti: ffff9da9c5130000
RIP: 0010:[<ffffffffc01e43be>] [<ffffffffc01e43be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0000:ffff9da9c5133708 EFLAGS: 00000246
RAX: ffffb5b080d00000 RBX: ffff9da9da865000 RCX: 0000000000000007
RDX: ffff9dab6b3cd000 RSI: ffff9dab3c441000 RDI: 000000000000000e
RBP: ffff9da9c5133718 R08: 0000000000000002 R09: 000000013da86fff
R10: ffff9dabf00b90d8 R11: ffff9dab88701340 R12: ffff9daa76d78000
R13: ffff9da9c513367c R14: ffffffff8a9557ab R15: ffff9da9c51336c0
FS: 0000000000000000(0000) GS:ffff9dabf5600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055d30bd9def0 CR3: 0000000235f7e000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc027f71e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc03362e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8aadc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8aae5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff8a9716de>] ? deadline_add_request+0x6e/0x90
Complete report #13956
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc01253be
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc01c671e
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc01892e0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffffa3ae2990
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffffa3aec0bc
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffffa39771ae
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6e
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 26s! [lfd - processin:20072]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag xt_nat xt_mark ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 nf_log_ipv4 nf_log_common nf_nat_ftp xt_REDIRECT nf_nat_redirect nf_conntrack_ftp xt_LOG xt_limit xt_multiport xt_CHECKSUM xt_conntrack tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack vmw_vsock_vmci_transport vsock sunrpc coretemp iosf_mbi ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon cryptd pcspkr joydev sg i2c_piix4 parport_pc parport vmw_vmci ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi scsi_transport_spi crct10dif_pclmul crct10dif_common ata_piix crc32c_intel mptscsih e1000 libata serio_raw mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 20072 Comm: lfd - processin Kdump: loaded Tainted: G L ------------ 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff8e9ab58962a0 ti: ffff8e9aa144c000 task.ti: ffff8e9aa144c000
RIP: 0010:[<ffffffffc01253be>] [<ffffffffc01253be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8e9aa144fac8 EFLAGS: 00000246
RAX: ffffa4bc40f00000 RBX: ffff8e9934fb96a0 RCX: 0000000000000014
RDX: ffff8e9ab7398400 RSI: ffff8e9ab3529000 RDI: 000000000000000e
RBP: ffff8e9aa144fad8 R08: 0000000000000003 R09: 000000019c171fff
R10: ffff8e9ab76c9800 R11: ffff8e9ab62168c0 R12: ffff8e9a8d0f5980
R13: ffff8e9aa144fa3c R14: ffffffffa395b0cb R15: ffff8e9aa144fa80
FS: 00007f0eafc99740(0000) GS:ffff8e9abfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0eaf81cee0 CR3: 0000000174086000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc01c671e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc01892e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffffa3ae2990>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffffa3aec0bc>] scsi_request_fn+0x4ac/0x680
[<ffffffffa39771ae>] ? deadline_add_request+0x6e/0x90
Complete report #331100
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc047f3be
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc04a171e
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc049a2e0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8d6e4860
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8d6edf8c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x4ac
|
|
6
|
|
vmlinux |
0xffffffff8d578bee
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff8d553f39
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x39
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [nagios:2020]
Modules linked in: vmw_vsock_vmci_transport vsock ppdev pcspkr sg joydev vmw_balloon vmw_vmci i2c_piix4 parport_pc parport nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi ata_piix scsi_transport_spi drm libata mptscsih nfit mptbase libnvdimm e1000 serio_raw drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 2020 Comm: nagios Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8e927327e300 ti: ffff8e925fa08000 task.ti: ffff8e925fa08000
RIP: 0010:[<ffffffffc047f3be>] [<ffffffffc047f3be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff8e925fa0b958 EFLAGS: 00000246
RAX: ffffbba8c0d00000 RBX: ffff8e90fae074a0 RCX: 0000000000000000
RDX: ffff8e9075d17000 RSI: ffff8e927576a000 RDI: 000000000000000e
RBP: ffff8e925fa0b968 R08: 0000000000000002 R09: 00000000bba24fff
R10: ffff8e92712a4000 R11: ffff8e90f5fc8c40 R12: ffff8e90f5f42400
R13: ffff8e925fa0b8cc R14: ffffffff8d55c8fb R15: ffff8e925fa0b910
FS: 00007f25f94b0740(0000) GS:ffff8e92756c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f25f94b6000 CR3: 000000021faf8000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc04a171e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc049a2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8d6e4860>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8d6edf8c>] scsi_request_fn+0x4ac/0x680
[<ffffffff8d578bee>] ? deadline_add_request+0x6e/0x90
[<ffffffff8d553f39>] __blk_run_queue+0x39/0x50
Complete report #20231
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mpt_put_msg_frame
|
mptbase |
0xffffffffc02d93be
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x5e
|
|
2
|
mptscsih_qcmd
|
mptscsih |
0xffffffffc032571e
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x24e
|
|
3
|
mptspi_qcmd
|
mptspi |
0xffffffffc034c2e0
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x50
|
|
4
|
scsi_dispatch_cmd
|
vmlinux |
0xffffffff8a8dc890
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0xb0
|
|
5
|
scsi_request_fn
|
vmlinux |
0xffffffff8a8e5efc
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x4cc
|
|
6
|
|
vmlinux |
0xffffffff8a7716de
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x6e
|
|
7
|
__blk_run_queue
|
vmlinux |
0xffffffff8a74cf59
Build id: 3.10.0-1062.1.1.el7.x86_64
|
0x39
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 55s! [xfsaild/dm-0:485]
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 55s! [mysqld:2529]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon ppdev cryptd sg pcspkr joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix libata drm mptspi scsi_transport_spi crct10dif_pclmul crct10dif_common mptscsih crc32c_intel serio_raw mptbase vmxnet3
drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 2529 Comm: mysqld Kdump: loaded Tainted: G L ------------ 3.10.0-1062.1.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff99030b241070 ti: ffff99030b38c000 task.ti: ffff99030b38c000
RIP: 0010:[<ffffffffc02d93be>] [<ffffffffc02d93be>] mpt_put_msg_frame+0x5e/0x80 [mptbase]
RSP: 0018:ffff99030b38faf8 EFLAGS: 00000246
RAX: ffffb36740580000 RBX: 000000002460a4de RCX: 0000000000000014
RDX: ffff9902f625ae00 RSI: ffff99033c76a000 RDI: 000000000000000e
RBP: ffff99030b38fb08 R08: 0000000000000003 R09: 0000000012124fff
R10: ffff9902f63960d8 R11: 000000005d7e7000 R12: ffff9902f630ac30
R13: 000000002460a4de R14: 0000000000000000 R15: 0080000200000000
FS: 00007f655c2ee700(0000) GS:ffff99033fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb7e7df0000 CR3: 0000000068218000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc032571e>] mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc034c2e0>] mptspi_qcmd+0x50/0xf0 [mptspi]
[<ffffffff8a8dc890>] scsi_dispatch_cmd+0xb0/0x240
[<ffffffff8a8e5efc>] scsi_request_fn+0x4cc/0x680
[<ffffffff8a7716de>] ? deadline_add_request+0x6e/0x90
[<ffffffff8a74cf59>] __blk_run_queue+0x39/0x50