Complete report #1219211
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8431e23a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff84281fa8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8428201b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff8440aa0d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5dd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff844278bb
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff842e6096
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x216
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff842c7f6b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff8422dcfd
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xad
|
|
9
|
int_signal
|
vmlinux |
0xffffffff849c56f7
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#30 stuck for 22s! [influxd:4116]
Modules linked in: sctp ip6table_filter ip6_tables iptable_filter tcp_diag udp_diag inet_diag fortiedr_3_10_0_1160(POE) base_driver_3_10_0_1160_2_1_el7_x86_64_2(POE) vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg pcspkr joydev parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptsas nfit libata crct10dif_pclmul scsi_transport_sas crct10dif_common crc32c_intel mptscsih libnvdimm serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 30 PID: 4116 Comm: influxd Kdump: loaded Tainted: P OEL ------------ T 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8a0f29cdb180 ti: ffff8a0ee981c000 task.ti: ffff8a0ee981c000
RIP: 0010:[<ffffffff8431e23a>] [<ffffffff8431e23a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a0ee981fcf8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff8a0b6ea20ae8
RDX: 0000000000000000 RSI: 0000000000000020 RDI: 0000000000000000
RBP: ffff8a0ee981fd30 R08: ffff8a0bf6e84000 R09: ffffffff8459c7b9
R10: ffff8a0f3f99f160 R11: ffffeb6311b81600 R12: 0000000000000292
R13: ffff8a0ee981fca8 R14: ffff8a0f2e5a8640 R15: 0000000000000000
FS: 00007f4ed5ffb700(0000) GS:ffff8a0f3f980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8887ed2a08 CR3: 00000003ac4bc000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff84281fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8428201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff8440aa0d>] change_protection+0x5dd/0x680
[<ffffffff844278bb>] change_prot_numa+0x1b/0x40
[<ffffffff842e6096>] task_numa_work+0x216/0x370
[<ffffffff842c7f6b>] task_work_run+0xbb/0xe0
[<ffffffff8422dcfd>] do_notify_resume+0xad/0xd0
[<ffffffff849c56f7>] int_signal+0x12/0x17
Code: 48 63 35 8e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 f5 84 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1198855
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff815170da
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8147e258
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8147e2cb
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff815feaed
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x5dd
|
|
5
|
|
vmlinux |
0xffffffff8147dfe9
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x109
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffff8161b1ab
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffff814dff02
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x212
|
|
8
|
task_work_run
|
vmlinux |
0xffffffff814c299b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffff8142cc65
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffff81b8c57c
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [snmp_exporter:1735]
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [snmp_exporter:24827]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink tun vmw_vsock_vmci_transport vsock sunrpc vfat fat sb_edac nfit ppdev iosf_mbi libnvdimm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon pcspkr joydev sg parport_pc parport tpm_crb i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm drm ata_piix libata mptspi scsi_transport_spi mptscsih e1000 crct10dif_pclmul crct10dif_common crc32c_intel mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ipmi_msghandler]
CPU: 11 PID: 24827 Comm: snmp_exporter Tainted: G W L ------------ 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.16707776.B64.2008070230 08/07/2020
task: ffff9449e51e5280 ti: ffff944d7bc5c000 task.ti: ffff944d7bc5c000
RIP: 0010:[<ffffffff815170da>] [<ffffffff815170da>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff944d7bc5fcf8 EFLAGS: 00000202
RAX: 0000000000000004 RBX: 00000000000000fc RCX: ffff944b2bd207f0
RDX: 0000000000000004 RSI: 000000000000000c RDI: 0000000000000000
RBP: ffff944d7bc5fd30 R08: ffff944bb2e92400 R09: ffffffff81786bf9
R10: ffff94503fd5f160 R11: ffffebfe59b20e00 R12: 0000000000000292
R13: ffff944d7bc5fca8 R14: ffffffff8147dff5 R15: ffffffff816267ed
FS: 000000c000501490(0000) GS:ffff94503fd40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000262000 CR3: 000000052a2dc000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8147e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8147e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff815feaed>] change_protection+0x5dd/0x680
[<ffffffff8147dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffff8161b1ab>] change_prot_numa+0x1b/0x40
[<ffffffff814dff02>] task_numa_work+0x212/0x370
[<ffffffff814c299b>] task_work_run+0xbb/0xe0
[<ffffffff8142cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff81b8c57c>] retint_signal+0x48/0x8c
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 15 82 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #968254
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa3311a8a
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa327a6a8
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa327a718
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffffa33f465d
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x5cd
|
|
5
|
|
vmlinux |
0xffffffffa3968988
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x338
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffffa340fcfb
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffffa32dbce2
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x202
|
|
8
|
task_work_run
|
vmlinux |
0xffffffffa32be90b
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffffa322bc65
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffffa396c56c
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 2395s! [mysqld:5657]
Modules linked in: gsch(OE) redirfs(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter dsa_filter(POE) vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev sg pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common serio_raw crc32c_intel vmxnet3 ahci libahci vmw_pvscsi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm
drm ata_piix libata nfit libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 5657 Comm: mysqld Kdump: loaded Tainted: P OE ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9e1900bcd140 ti: ffff9e1895a2c000 task.ti: ffff9e1895a2c000
RIP: 0010:[<ffffffffa3311a8a>] [<ffffffffa3311a8a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9e1895a2fcf8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00000000000000fc RCX: ffff9e1ea7c60598
RDX: 0000000000000001 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9e1895a2fd30 R08: ffff9e19f6419000 R09: ffffffffa3575e39
R10: ffff9e1ea7d5f160 R11: fffff19fc4027200 R12: 0000000000000292
R13: 0000000000000018 R14: ffff9e1895a2fc68 R15: 0000000000000292
FS: 00007f6527255700(0000) GS:ffff9e1ea7d40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f41383a2000 CR3: 0000000624bf2000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa327a6a8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa327a718>] flush_tlb_mm_range+0x68/0x140
[<ffffffffa33f465d>] change_protection+0x5cd/0x680
[<ffffffffa3968988>] ? __schedule+0x338/0x860
[<ffffffffa340fcfb>] change_prot_numa+0x1b/0x40
[<ffffffffa32dbce2>] task_numa_work+0x202/0x360
[<ffffffffa32be90b>] task_work_run+0xbb/0xe0
[<ffffffffa322bc65>] do_notify_resume+0xa5/0xc0
[<ffffffffa396c56c>] retint_signal+0x48/0x8c
Code: 48 63 35 3e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 bc f4 a3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #943369
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffbb31e23a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffbb281fa8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffbb28201b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffffbb40aa0d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5dd
|
|
5
|
?
fuse_request_send_nowait_locked
|
fuse |
0xffffffffc0145beb
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5b
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffffbb4278bb
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffffbb2e6096
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x216
|
|
8
|
task_work_run
|
vmlinux |
0xffffffffbb2c7f6b
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffffbb22dcfd
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xad
|
|
10
|
retint_signal
|
vmlinux |
0xffffffffbb9bb5bc
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [combine_data:76302]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc vfat fat ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport tpm_crb vmw_vmci i2c_piix4 ip_tables xfs libcrc32c ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci crct10dif_pclmul
crct10dif_common nfit ata_piix libahci drm crc32c_intel libata libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 9 PID: 76302 Comm: combine_data Not tainted 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.21100432.B64.2301110304 01/11/2023
task: ffff916950075280 ti: ffff905c8f804000 task.ti: ffff905c8f804000
RIP: 0010:[<ffffffffbb31e23a>] [<ffffffffbb31e23a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff905c8f807cf8 EFLAGS: 00000202
RAX: 0000000000000041 RBX: 00000000000000fc RCX: ffff9169517607a0
RDX: 0000000000000041 RSI: 0000000000000078 RDI: 0000000000000040
RBP: ffff905c8f807d30 R08: ffff8ff4dded2808 R09: ffffffffbb59c7b9
R10: ffff90ab1005f160 R11: fffff1a87ce29000 R12: 0000000000000292
R13: ffff905c8f807ca8 R14: 0000000000000092 R15: ffff905c8f807c58
FS: 00007fa3f17fa700(0000) GS:ffff90ab10040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa3d6206ff0 CR3: 000000a21bb0e000 CR4: 0000000000340fe0
Call Trace:
[<ffffffffbb281fa8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffbb28201b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffbb40aa0d>] change_protection+0x5dd/0x680
[<ffffffffc0145beb>] ? fuse_request_send_nowait_locked+0x5b/0xb0 [fuse]
[<ffffffffbb4278bb>] change_prot_numa+0x1b/0x40
[<ffffffffbb2e6096>] task_numa_work+0x216/0x370
[<ffffffffbb2c7f6b>] task_work_run+0xbb/0xe0
[<ffffffffbb22dcfd>] do_notify_resume+0xad/0xd0
[<ffffffffbb9bb5bc>] retint_signal+0x48/0x8c
Code: 48 63 35 8e 18 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 80 19 f5 bb f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d 18 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #871935
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff95b1718a
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff95a7e258
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff95a7e2cb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff95bfeacd
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x5dd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff95c1b1ab
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff95adff52
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x212
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff95ac2acb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff95a2cc65
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffff9619057c
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#41 stuck for 22s! [python:192242]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) nvidia_uvm(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter iscsi_tcp libiscsi_tcp ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_umad i40iw rpcrdma rdma_ucm ib_iser ib_ipoib rdma_cm iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_uverbs ib_core dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) skx_edac intel_powerclamp coretemp
intel_rapl iosf_mbi snd_hda_codec_hdmi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_ssif pcspkr snd_hda_intel sg snd_hda_codec snd_seq snd_hda_core snd_seq_device snd_hwdep snd_pcm snd_timer joydev snd soundcore lpc_ich i2c_i801 mei_me mei ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic mlx5_core i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt i40e fb_sys_fops ttm ahci crct10dif_pclmul crct10dif_common megaraid_sas crc32c_intel libahci drm libata mlxfw devlink ptp pps_core drm_panel_orientation_quirks nfit wmi libnvdimm fuse
CPU: 41 PID: 192242 Comm: python Kdump: loaded Tainted: P OE ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: Kunqian KI4224G-HS2/KQ11DPG-OT-CPU, BIOS 3.4 12/18/2020
task: ffff8842b492e300 ti: ffff882bb0880000 task.ti: ffff882bb0880000
RIP: 0010:[<ffffffff95b1718a>] [<ffffffff95b1718a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff882bb0883cf8 EFLAGS: 00000202
RAX: 000000000000003b RBX: 000000c000000029 RCX: ffff88503f3e0cc8
RDX: 000000000000003b RSI: 00000000000000c0 RDI: 0000000000000000
RBP: ffff882bb0883d30 R08: ffff88125e3d7800 R09: ffffffff95d869f9
R10: ffff88503ef5f160 R11: ffffe9a1ef1eb600 R12: 000000fc95d8694f
R13: 0000000000000292 R14: 0000000000000292 R15: ffff882bb0883ca8
FS: 00007f79b5a34740(0000) GS:ffff88503ef40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000d319d8 CR3: 0000007f446a6000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff95a7e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff95a7e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff95bfeacd>] change_protection+0x5dd/0x680
[<ffffffff95c1b1ab>] change_prot_numa+0x1b/0x40
[<ffffffff95adff52>] task_numa_work+0x212/0x370
[<ffffffff95ac2acb>] task_work_run+0xbb/0xe0
[<ffffffff95a2cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff9619057c>] retint_signal+0x48/0x8c
Code: 48 63 35 be 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 75 96 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 8d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #710689
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffac111a0e
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffac07a628
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffac07a698
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffffac1f45dd
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffffac20fc7b
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffffac0dbc62
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffffac0be88b
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffffac02bc65
Build id: 3.10.0-957.12.2.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffffac76c56c
Build id: 3.10.0-957.12.2.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#76 stuck for 22s! [docker-containe:5703]
Modules linked in: 8021q garp mrp mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack libcrc32c ko2iblnd(OE) br_netfilter bridge stp llc ptlrpc(OE) obdclass(OE) lnet(OE) overlay(T) rpcsec_gss_krb5 nfsv4 dns_resolver nfs libcfs(OE) fscache dm_mirror dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd mgag200 ttm drm_kms_helper hfi1 syscopyarea sysfillrect sysimgblt fb_sys_fops pcspkr drm mei_me sg i2c_i801 lpc_ich ipmi_si joydev
mei drm_panel_orientation_quirks rdmavt ipmi_devintf wmi ipmi_msghandler acpi_power_meter acpi_pad rpcrdma ib_isert iscsi_target_mod target_core_mod crct10dif_generic crct10dif_pclmul ib_iser libiscsi scsi_transport_iscsi ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm ib_core nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables igb i2c_algo_bit dca ptp pps_core ahci libahci libata sd_mod crc_t10dif crct10dif_common
CPU: 76 PID: 5703 Comm: docker-containe Tainted: G OE ------------ T 3.10.0-957.12.2.el7.x86_64 #1
Hardware name: Acer AW570h F3/S7200AP, BIOS S72C610.86B.01.02.0072.041620172102 04/16/2017
task: ffff96ee7f722080 ti: ffff97051976c000 task.ti: ffff97051976c000
RIP: 0010:[<ffffffffac111a0e>] [<ffffffffac111a0e>] smp_call_function_many+0x20e/0x270
RSP: 0000:ffff97051976fcf8 EFLAGS: 00000202
RAX: 0000000000000022 RBX: 000001100000004c RCX: ffff9705dc2a1218
RDX: 0000000000000022 RSI: 0000000000000110 RDI: 0000000000000000
RBP: ffff97051976fd30 R08: ffff96ef5ed77800 R09: ffffffffac375d69
R10: ffff9705dcd1f160 R11: ffffe11c5e0ed200 R12: 000000fcac375cbf
R13: 0000000000000292 R14: 0000000000000292 R15: 0000000000000000
FS: 00007fb6affff700(0000) GS:ffff9705dcd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3fa4e2e000 CR3: 00000017d7b5e000 CR4: 00000000001407e0
Call Trace:
[<ffffffffac07a628>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffac07a698>] flush_tlb_mm_range+0x68/0x140
[<ffffffffac1f45dd>] change_protection+0x5cd/0x680
[<ffffffffac20fc7b>] change_prot_numa+0x1b/0x40
[<ffffffffac0dbc62>] task_numa_work+0x202/0x360
[<ffffffffac0be88b>] task_work_run+0xbb/0xe0
[<ffffffffac02bc65>] do_notify_resume+0xa5/0xc0
[<ffffffffac76c56c>] retint_signal+0x48/0x8c
Code: 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 bc d4 ac f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 4d 83 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #796184
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffae31173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffae27a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffae27a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffffae3f404d
Build id: 3.10.0-957.el7.x86_64
|
0x5cd
|
|
5
|
|
xfs |
0xffffffffc056fa9c
Build id: 3.10.0-957.el7.x86_64
|
0x1c
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffffae40f66b
Build id: 3.10.0-957.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffffae2dba52
Build id: 3.10.0-957.el7.x86_64
|
0x202
|
|
8
|
task_work_run
|
vmlinux |
0xffffffffae2be79b
Build id: 3.10.0-957.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffffae22bc65
Build id: 3.10.0-957.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffffae96b55c
Build id: 3.10.0-957.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#20 stuck for 22s! [node_exporter:10637]
Modules linked in: binfmt_misc tcp_diag inet_diag rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper sg cryptd vmw_balloon pcspkr joydev i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci crc32c_intel libahci drm ata_piix serio_raw libata nfit vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 20 PID: 10637 Comm: node_exporter Kdump: loaded Tainted: G W L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff97ea19f6d140 ti: ffff97e4928d0000 task.ti: ffff97e4928d0000
RIP: 0010:[<ffffffffae31173a>] [<ffffffffae31173a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff97e4928d3cf8 EFLAGS: 00000202
RAX: 0000000000000016 RBX: 00000000000000fc RCX: ffff97ebff860908
RDX: 0000000000000016 RSI: 0000000000000019 RDI: 0000000000000000
RBP: ffff97e4928d3d30 R08: ffff97e636aca400 R09: ffffffffae5755c9
R10: ffff97ebff7df160 R11: fffff2e3ab77fc00 R12: 0000000000000292
R13: ffff97e4928d3ca8 R14: ffff97e532313937 R15: 3834323436333038
FS: 000000c00021d890(0000) GS:ffff97ebff7c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f40a8594930 CR3: 00000002a7758000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffae27a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffae27a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffffae3f404d>] change_protection+0x5cd/0x680
[<ffffffffc056fa9c>] ? xfs_sysfs_object_show+0x1c/0x20 [xfs]
[<ffffffffae40f66b>] change_prot_numa+0x1b/0x40
[<ffffffffae2dba52>] task_numa_work+0x202/0x360
[<ffffffffae2be79b>] task_work_run+0xbb/0xe0
[<ffffffffae22bc65>] do_notify_resume+0xa5/0xc0
[<ffffffffae96b55c>] retint_signal+0x48/0x8c
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 f4 ae f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #725346
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff83716c5a
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff8367e868
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff8367e8db
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff837fe15d
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x5dd
|
|
5
|
|
vmlinux |
0xffffffff837f5d2b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xadb
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffff8381a80b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffff836dfec2
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x212
|
|
8
|
task_work_run
|
vmlinux |
0xffffffff836c275b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffff8362cc65
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffff83d8b57c
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [mysqld:6496]
Modules linked in: uas usb_storage binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi coretemp snd_ac97_codec ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device snd_pcm aesni_intel lrw gf128mul glue_helper snd_timer sg snd soundcore
ablk_helper pcspkr cryptd ppdev joydev vmw_balloon i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci crc32c_intel drm ata_piix libahci mptspi scsi_transport_spi serio_raw libata mptscsih e1000 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 6496 Comm: mysqld Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/31/2013
task: ffff9d548c0c1080 ti: ffff9d5442018000 task.ti: ffff9d5442018000
RIP: 0010:[<ffffffff83716c5a>] [<ffffffff83716c5a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9d544201bcf8 EFLAGS: 00000202
RAX: 000000000000000b RBX: 000000000001b840 RCX: ffff9d54bd8e0630
RDX: 000000000000000b RSI: 0000000000000040 RDI: 0000000000000000
RBP: ffff9d544201bd30 R08: ffff9d54bc36dc00 R09: ffffffff83985fd9
R10: ffff9d54bd69f160 R11: fffffb9640d31e00 R12: ffff9d544201bca8
R13: ffff9d544201bcb0 R14: 0000000000000040 R15: 0000000000000054
FS: 00007f53442d0700(0000) GS:ffff9d54bd680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5214050000 CR3: 000000005c748000 CR4: 00000000000007e0
Call Trace:
[<ffffffff8367e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff8367e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff837fe15d>] change_protection+0x5dd/0x680
[<ffffffff837f5d2b>] ? handle_mm_fault+0xadb/0xfb0
[<ffffffff8381a80b>] change_prot_numa+0x1b/0x40
[<ffffffff836dfec2>] task_numa_work+0x212/0x370
[<ffffffff836c275b>] task_work_run+0xbb/0xe0
[<ffffffff8362cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff83d8b57c>] retint_signal+0x48/0x8c
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 35 84 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #722415
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9a9170da
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9a87e258
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9a87e2cb
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff9a9feaed
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x5dd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff9aa1b1ab
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff9a8dff02
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x212
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff9a8c299b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff9a82cc65
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffff9af9057c
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [java:5597]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 nfit libnvdimm coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi libata scsi_transport_spi crct10dif_pclmul vmxnet3 crct10dif_common mptscsih crc32c_intel serio_raw mptbase drm_panel_orientation_quirks
floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 11 PID: 5597 Comm: java Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9184550de300 ti: ffff91849daa0000 task.ti: ffff91849daa0000
RIP: 0010:[<ffffffff9a9170da>] [<ffffffff9a9170da>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff91849daa3cf8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00000000000000fc RCX: ffff9180b7ca08c8
RDX: 0000000000000001 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff91849daa3d30 R08: ffff91833fc0a400 R09: ffffffff9ab86c09
R10: ffff9184b7d9f160 R11: ffffcfa085e1ec00 R12: 0000000000000292
R13: ffff91849daa3ca8 R14: ffffffff9af8c060 R15: ffffffff9aa28ac9
FS: 00007fd53d7f9700(0000) GS:ffff9184b7d80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000814f50 CR3: 0000000634e08000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9a87e258>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9a87e2cb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9a9feaed>] change_protection+0x5dd/0x680
[<ffffffff9aa1b1ab>] change_prot_numa+0x1b/0x40
[<ffffffff9a8dff02>] task_numa_work+0x212/0x370
[<ffffffff9a8c299b>] task_work_run+0xbb/0xe0
[<ffffffff9a82cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff9af9057c>] retint_signal+0x48/0x8c
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 18 55 9b f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #615156
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa6116c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffa607e868
Build id: 3.10.0-1160.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffa607e8db
Build id: 3.10.0-1160.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffffa61fe11d
Build id: 3.10.0-1160.el7.x86_64
|
0x5dd
|
|
5
|
|
vmlinux |
0xffffffffa60e4306
Build id: 3.10.0-1160.el7.x86_64
|
0x86
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffffa621a7cb
Build id: 3.10.0-1160.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffffa60dfe72
Build id: 3.10.0-1160.el7.x86_64
|
0x212
|
|
8
|
task_work_run
|
vmlinux |
0xffffffffa60c275b
Build id: 3.10.0-1160.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffffa602cc65
Build id: 3.10.0-1160.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffffa678a57c
Build id: 3.10.0-1160.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kubelet:9838]
Modules linked in: tcp_diag inet_diag xt_multiport ipt_rpfilter ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel iptable_raw xt_CT xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport ip_set dummy nf_tables ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter iptable_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment xt_mark veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_nat_ipv4 nf_nat br_netfilter bridge stp llc vmw_vsock_vmci_transport vsock sb_edac iosf_mbi ppdev crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw
gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci parport i2c_piix4 overlay(T) nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif drm_kms_helper crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci nfit crct10dif_pclmul crct10dif_common crc32c_intel libnvdimm libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 6 PID: 9838 Comm: kubelet Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9fccfdb22100 ti: ffff9fca1e1d8000 task.ti: ffff9fca1e1d8000
RIP: 0010:[<ffffffffa6116c0a>] [<ffffffffa6116c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9fca1e1dbcf8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9fce2fc20728
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9fca1e1dbd30 R08: ffff9fcb7976b400 R09: ffffffffa6385f99
R10: ffff9fce2fd9f160 R11: ffffd9630414b600 R12: 0000000000000292
R13: ffff9fca1e1dbca8 R14: ffff9fca1e1dbc98 R15: ffffffffa60dfb48
FS: 00007f3b6ffff700(0000) GS:ffff9fce2fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c02c063368 CR3: 0000000675180000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa607e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffa607e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffa61fe11d>] change_protection+0x5dd/0x680
[<ffffffffa60e4306>] ? update_curr+0x86/0x1e0
[<ffffffffa621a7cb>] change_prot_numa+0x1b/0x40
[<ffffffffa60dfe72>] task_numa_work+0x212/0x370
[<ffffffffa60c275b>] task_work_run+0xbb/0xe0
[<ffffffffa602cc65>] do_notify_resume+0xa5/0xc0
[<ffffffffa678a57c>] retint_signal+0x48/0x8c
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 d5 a6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
[drm:drm_atomic_helper_wait_for_dependencies [drm_kms_helper]] *ERROR* [CRTC:37:crtc-0] flip_done timed out
Complete report #513700
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff89d1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff89c7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff89c7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffff89df404d
Build id: 3.10.0-957.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff89e0f66b
Build id: 3.10.0-957.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff89cdba52
Build id: 3.10.0-957.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff89cbe79b
Build id: 3.10.0-957.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff89c2bc65
Build id: 3.10.0-957.el7.x86_64
|
0xa5
|
|
9
|
int_signal
|
vmlinux |
0xffffffff8a375124
Build id: 3.10.0-957.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [srepd:522652]
Modules linked in: dattobd(OE) sfs(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon joydev sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom
crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm mptspi scsi_transport_spi mptscsih ahci vmxnet3 libahci mptbase drm_panel_orientation_quirks ata_piix libata dm_mirror dm_region_hash dm_log dm_mod [last unloaded: dattobd]
CPU: 14 PID: 522652 Comm: srepd Kdump: loaded Tainted: G W OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff98679fc92080 ti: ffff9866d38f8000 task.ti: ffff9866d38f8000
RIP: 0010:[<ffffffff89d1173a>] [<ffffffff89d1173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9866d38fbcf8 EFLAGS: 00000202
RAX: 0000000000000011 RBX: 00000000000000fc RCX: ffff986df4b20b68
RDX: 0000000000000011 RSI: 0000000000000014 RDI: 0000000000000000
RBP: ffff9866d38fbd30 R08: ffff986ce7c08800 R09: ffffffff89f755c9
R10: ffff986d2ca1f160 R11: fffffb8502718600 R12: 0000000000000292
R13: ffff9866d38fbca8 R14: 000000000001ab80 R15: ffff9866d38fbfd8
FS: 00007f6c3562a700(0000) GS:ffff986d2ca00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000074b2df000 CR3: 00000005182cc000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff89c7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff89c7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff89df404d>] change_protection+0x5cd/0x680
[<ffffffff89e0f66b>] change_prot_numa+0x1b/0x40
[<ffffffff89cdba52>] task_numa_work+0x202/0x360
[<ffffffff89cbe79b>] task_work_run+0xbb/0xe0
[<ffffffff89c2bc65>] do_notify_resume+0xa5/0xc0
[<ffffffff8a375124>] int_signal+0x12/0x17
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 94 8a f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #249832
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff97f07e3a
Build id: 3.10.0-862.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff97e77018
Build id: 3.10.0-862.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff97e77088
Build id: 3.10.0-862.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffff97fcf45d
Build id: 3.10.0-862.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff97fea34b
Build id: 3.10.0-862.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff97ed3ba2
Build id: 3.10.0-862.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff97eb784b
Build id: 3.10.0-862.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff97e2ac55
Build id: 3.10.0-862.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffff9851655c
Build id: 3.10.0-862.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [java:24027]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_12207(E) falcon_lsm_pinned_11711(E) falcon_lsm_pinned_11312(E) binfmt_misc arc4 md4 nls_utf8 cifs ccm dns_resolver vmw_vsock_vmci_transport vsock sb_edac iosf_mbi crc32_pclmul ext4 ghash_clmulni_intel aesni_intel lrw gf128mul mbcache glue_helper jbd2 ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg nfit parport_pc libnvdimm parport shpchp vmw_vmci i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common mptspi scsi_transport_spi crc32c_intel mptscsih serio_raw mptbase vmxnet3 i2c_core floppy
dm_mirror dm_region_hash dm_log dm_mod [last unloaded: falcon_kal]
CPU: 16 PID: 24027 Comm: java Kdump: loaded Tainted: P EL ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9b28d32bcf10 ti: ffff9b2c31890000 task.ti: ffff9b2c31890000
RIP: 0010:[<ffffffff97f07e3a>] [<ffffffff97f07e3a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9b2c31893cf8 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff9b30cfadd168
RDX: 0000000000000003 RSI: 0000000000000018 RDI: 0000000000000000
RBP: ffff9b2c31893d30 R08: ffff9b316ded5000 R09: ffffffff9814a3d9
R10: ffff9b3ccf31bb00 R11: ffffe836b1075c00 R12: 0000000000000292
R13: ffff9b2c31893ca8 R14: 0000000031893c70 R15: ffffffff97ed92c8
FS: 00007faea55e0700(0000) GS:ffff9b3ccf300000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faed8c4c000 CR3: 00000017d6b20000 CR4: 00000000000607e0
Call Trace:
[<ffffffff97e77018>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff97e77088>] flush_tlb_mm_range+0x68/0x140
[<ffffffff97fcf45d>] change_protection+0x5cd/0x680
[<ffffffff97fea34b>] change_prot_numa+0x1b/0x40
[<ffffffff97ed3ba2>] task_numa_work+0x202/0x350
[<ffffffff97eb784b>] task_work_run+0xbb/0xe0
[<ffffffff97e2ac55>] do_notify_resume+0xa5/0xc0
[<ffffffff9851655c>] retint_signal+0x48/0x8c
Code: 48 63 35 8e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 b3 98 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 5d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #211550
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb6111a8e
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb607a6a8
Build id: 3.10.0-957.21.2.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb607a718
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffffb61f465d
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffffb620fcfb
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffffb60dbce2
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffffb60be90b
Build id: 3.10.0-957.21.2.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffffb602bc65
Build id: 3.10.0-957.21.2.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffffb676c56c
Build id: 3.10.0-957.21.2.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#16 stuck for 23s! [R:25085]
Modules linked in: binfmt_misc iptable_filter ext4 mbcache jbd2 iscsi_tcp libiscsi_tcp rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm iTCO_wdt gpio_ich iTCO_vendor_support intel_powerclamp coretemp kvm_intel kvm irqbypass crc32_pclmul vfat fat ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_ssif ocrdma(T) cdc_ether pcspkr usbnet ib_core mii lpc_ich ioatdma sg i2c_i801 i7core_edac dca ipmi_si ipmi_devintf wmi ipmi_msghandler nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi mgag200 i2c_algo_bit drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common serio_raw crc32c_intel drm ata_piix libata drm_panel_orientation_quirks be2net megaraid_sas bnx2 dm_mirror dm_region_hash dm_log dm_mod
CPU: 16 PID: 25085 Comm: R Kdump: loaded Tainted: G I L ------------ T 3.10.0-957.21.2.el7.x86_64 #1
Hardware name: IBM System x3850 X5 -[7143B7U]-/Node 1, Processor Card, BIOS -[G0E188BUS-1.88]- 05/29/2018
task: ffff95230b77c100 ti: ffff952d41900000 task.ti: ffff952d41900000
RIP: 0010:[<ffffffffb6111a8e>] [<ffffffffb6111a8e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff952d41903cf8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00000000000000fc RCX: ffff953eff860940
RDX: 0000000000000001 RSI: 0000000000000050 RDI: 0000000000000000
RBP: ffff952d41903d30 R08: ffff953fffc0b400 R09: ffffffffb6375e39
R10: ffff955eff99f160 R11: ffffe1249c59ae00 R12: 0000000000000292
R13: ffff952d41903ca8 R14: ffffffffb600a704 R15: ffff952d41903e90
FS: 00007f07695e0700(0000) GS:ffff955eff980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000091ad2c0 CR3: 0000001f8a3a8000 CR4: 00000000000207e0
Call Trace:
[<ffffffffb607a6a8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb607a718>] flush_tlb_mm_range+0x68/0x140
[<ffffffffb61f465d>] change_protection+0x5cd/0x680
[<ffffffffb620fcfb>] change_prot_numa+0x1b/0x40
[<ffffffffb60dbce2>] task_numa_work+0x202/0x360
[<ffffffffb60be90b>] task_work_run+0xbb/0xe0
[<ffffffffb602bc65>] do_notify_resume+0xa5/0xc0
[<ffffffffb676c56c>] retint_signal+0x48/0x8c
Code: 82 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 bc d4 b6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 cd 82 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #175493
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff88907e3a
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff88877028
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff88877098
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffff889cf3fd
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff889ea2eb
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff888d3ba2
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff888b784b
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff8882ac55
Build id: 3.10.0-862.2.3.el7.x86_64
|
0xa5
|
|
9
|
int_signal
|
vmlinux |
0xffffffff88f1fadd
Build id: 3.10.0-862.2.3.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 21s! [java:3586]
Modules linked in: binfmt_misc kcare(OE) ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 nf_log_ipv4 nf_log_common ip6table_filter nf_nat_ftp xt_conntrack nf_conntrack_ftp xt_LOG xt_limit ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 xt_REDIRECT nf_nat_redirect iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_owner xt_multiport iptable_filter ext4 mbcache jbd2 loop vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg nfit vmw_vmci libnvdimm shpchp i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi
vmwgfx drm_kms_helper sd_mod crc_t10dif syscopyarea crct10dif_generic sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 i2c_core vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 3586 Comm: java Kdump: loaded Tainted: G OEL ------------ 3.10.0-862.2.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8ece9aa80000 ti: ffff8eceea6f0000 task.ti: ffff8eceea6f0000
RIP: 0010:[<ffffffff88907e3a>] [<ffffffff88907e3a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8eceea6f3cf8 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 00000000000000fc RCX: ffff8ec71fc9d0c8
RDX: 0000000000000002 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff8eceea6f3d30 R08: ffff8ec7b4da6400 R09: ffffffff88b4a379
R10: ffff8ecf3fd1bb00 R11: ffffd5009b49ce00 R12: 0000000000000292
R13: ffff8eceea6f3ca8 R14: 0000000000000000 R15: 0000000000000280
FS: 00007f14693ab700(0000) GS:ffff8ecf3fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f811a2de000 CR3: 0000000783fa8000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff88877028>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff88877098>] flush_tlb_mm_range+0x68/0x140
[<ffffffff889cf3fd>] change_protection+0x5cd/0x680
[<ffffffff889ea2eb>] change_prot_numa+0x1b/0x40
[<ffffffff888d3ba2>] task_numa_work+0x202/0x350
[<ffffffff888b784b>] task_work_run+0xbb/0xe0
[<ffffffff8882ac55>] do_notify_resume+0xa5/0xc0
[<ffffffff88f1fadd>] int_signal+0x12/0x17
Code: 48 63 35 6e b6 c3 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 53 53 89 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 3d b6 c3 00 eb b7 0f b6 4d cc 4c 89
Complete report #137010
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff82f1173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff82e7a598
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff82e7a608
Build id: 3.10.0-957.el7.x86_64
|
0x68
|
|
4
|
change_protection
|
vmlinux |
0xffffffff82ff404d
Build id: 3.10.0-957.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff8300f66b
Build id: 3.10.0-957.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff82edba52
Build id: 3.10.0-957.el7.x86_64
|
0x202
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff82ebe79b
Build id: 3.10.0-957.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff82e2bc65
Build id: 3.10.0-957.el7.x86_64
|
0xa5
|
|
9
|
int_signal
|
vmlinux |
0xffffffff83575124
Build id: 3.10.0-957.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [fwupd:20039]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif
crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel ahci drm serio_raw ata_piix libahci libata vmxnet3 nfit libnvdimm vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 20039 Comm: fwupd Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff88b2c67fd140 ti: ffff88b2b7238000 task.ti: ffff88b2b7238000
RIP: 0010:[<ffffffff82f1173a>] [<ffffffff82f1173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff88b2b723bcf8 EFLAGS: 00000202
RAX: 000000000000000d RBX: 00000000000000fc RCX: ffff88b7eb1a07c8
RDX: 000000000000000d RSI: 0000000000000014 RDI: 0000000000000000
RBP: ffff88b2b723bd30 R08: ffff88b377ea9000 R09: ffffffff831755c9
R10: ffff88b7eb11f160 R11: ffffc268a8539c00 R12: 0000000000000292
R13: ffff88b2b723bca8 R14: ffff88b7b3f43128 R15: 0000000000000000
FS: 00007f933cfb1900(0000) GS:ffff88b7eb100000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd86a606000 CR3: 0000000a1bff2000 CR4: 00000000000007e0
Call Trace:
[<ffffffff82e7a598>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff82e7a608>] flush_tlb_mm_range+0x68/0x140
[<ffffffff82ff404d>] change_protection+0x5cd/0x680
[<ffffffff8300f66b>] change_prot_numa+0x1b/0x40
[<ffffffff82edba52>] task_numa_work+0x202/0x360
[<ffffffff82ebe79b>] task_work_run+0xbb/0xe0
[<ffffffff82e2bc65>] do_notify_resume+0xa5/0xc0
[<ffffffff83575124>] int_signal+0x12/0x17
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 b4 83 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #61635
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff90f16cbe
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20e
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff90e7e608
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff90e7e67b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff90ffc08d
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x5cd
|
|
5
|
|
vmlinux |
0xffffffff90ff3cd2
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb22
|
|
6
|
change_prot_numa
|
vmlinux |
0xffffffff9101864b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1b
|
|
7
|
task_numa_work
|
vmlinux |
0xffffffff90ee04c2
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x212
|
|
8
|
task_work_run
|
vmlinux |
0xffffffff90ec2e5b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xbb
|
|
9
|
do_notify_resume
|
vmlinux |
0xffffffff90e2cc65
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xa5
|
|
10
|
retint_signal
|
vmlinux |
0xffffffff9158457c
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [MonitorEventDis:3325]
Modules linked in: veth rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache xt_statistic xt_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ip_set nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ppdev vmw_balloon sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport auth_rpcgss
sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod drm_kms_helper crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci nfit libahci ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc
CPU: 6 PID: 3325 Comm: MonitorEventDis Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9e2a218520e0 ti: ffff9e2a995a8000 task.ti: ffff9e2a995a8000
RIP: 0010:[<ffffffff90f16cbe>] [<ffffffff90f16cbe>] smp_call_function_many+0x20e/0x270
RSP: 0000:ffff9e2a995abcf8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff9e2aafc20728
RDX: 0000000000000000 RSI: 0000000000000010 RDI: 0000000000000000
RBP: ffff9e2a995abd30 R08: ffff9e27fa379c00 R09: ffffffff911819f9
R10: ffff9e2aafd9f160 R11: ffffe72ecbe39800 R12: 0000000000000292
R13: ffff9e2a995abca8 R14: 000000000001ac80 R15: ffffffff91025f59
FS: 00007f8ca4ece700(0000) GS:ffff9e2aafd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8e20dab480 CR3: 000000079503a000 CR4: 00000000003607e0
Call Trace:
[<ffffffff90e7e608>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff90e7e67b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff90ffc08d>] change_protection+0x5cd/0x670
[<ffffffff90ff3cd2>] ? handle_mm_fault+0xb22/0xfb0
[<ffffffff9101864b>] change_prot_numa+0x1b/0x40
[<ffffffff90ee04c2>] task_numa_work+0x212/0x370
[<ffffffff90ec2e5b>] task_work_run+0xbb/0xe0
[<ffffffff90e2cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff9158457c>] retint_signal+0x48/0x8c
Code: 5c c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 e8 b4 91 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 bd 5c c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #22822
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9ef1761a
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9ee7e8f8
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9ee7e96b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff9effe26d
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x5dd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff9f01a89b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff9eee0952
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x212
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff9eec31cb
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff9ee2cc65
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xa5
|
|
9
|
int_signal
|
vmlinux |
0xffffffff9f59322f
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [kcp:38854]
NMI watchdog: BUG: soft lockup - CPU#21 stuck for 22s! [kcp:2641]
Modules linked in: binfmt_misc xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) iptable_filter sunrpc ext4 mbcache jbd2 sb_edac ppdev iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc virtio_balloon parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi virtio_console virtio_scsi bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci libata crct10dif_pclmul crct10dif_common crc32c_intel virtio_pci ixgbevf virtio_ring serio_raw
virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 21 PID: 2641 Comm: kcp Kdump: loaded Tainted: G W L ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
task: ffff949a2daf3150 ti: ffff949a12780000 task.ti: ffff949a12780000
RIP: 0010:[<ffffffff9ef1761a>] [<ffffffff9ef1761a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff949a12783cf8 EFLAGS: 00000202
RAX: 0000000000000011 RBX: 00000000000000fc RCX: ffff949e3f760980
RDX: 0000000000000011 RSI: 0000000000000018 RDI: 0000000000000000
RBP: ffff949a12783d30 R08: ffff949aba6a1800 R09: ffffffff9f1851b9
R10: ffff949e3f85f160 R11: ffffdc5f9f19a400 R12: 0000000000000292
R13: ffff949a12783ca8 R14: ffff949a12783c60 R15: ffff949e2c9083c0
FS: 00007fa3c376b700(0000) GS:ffff949e3f840000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa3c043b27b CR3: 000000041260c000 CR4: 0000000000160fe0
Call Trace:
[<ffffffff9ee7e8f8>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9ee7e96b>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9effe26d>] change_protection+0x5dd/0x680
[<ffffffff9f01a89b>] change_prot_numa+0x1b/0x40
[<ffffffff9eee0952>] task_numa_work+0x212/0x370
[<ffffffff9eec31cb>] task_work_run+0xbb/0xe0
[<ffffffff9ee2cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff9f59322f>] int_signal+0x12/0x17
Code: 48 63 35 ce 7b c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 a0 10 b5 9f f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 7b c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #20902
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb1316c5a
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffffb127e868
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffffb127e8db
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffffb13fe15d
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x5dd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffffb141a80b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffffb12dfec2
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x212
|
|
7
|
task_work_run
|
vmlinux |
0xffffffffb12c275b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffffb122cc65
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xa5
|
|
9
|
int_signal
|
vmlinux |
0xffffffffb19952ef
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x12
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [QuartzScheduler:3175]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm d2dctl(POE) vhost_net vhost macvtap macvlan xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink mpt3sas mpt2sas raid_class scsi_transport_sas mptctl mptbase rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack bridge stp llc ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter jc42 vfat fat ext4 mbcache jbd2 ipmi_ssif intel_powerclamp coretemp kvm_intel kvm irqbypass iTCO_wdt
crc32_pclmul ghash_clmulni_intel aesni_intel cdc_ether iTCO_vendor_support lrw gf128mul glue_helper usbnet ipmi_si ipmi_devintf ablk_helper cryptd ipmi_msghandler ioatdma pcspkr mii sg i2c_i801 lpc_ich i7core_edac dca acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_generic pata_acpi drm ata_piix libata megaraid_sas crct10dif_pclmul crct10dif_common crc32c_intel serio_raw bnx2 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 3175 Comm: QuartzScheduler Kdump: loaded Tainted: P W OEL ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: IBM System x3650 M3 -[7945K3G]-/69Y4438, BIOS -[D6E164AUS-1.22]- 06/04/2018
task: ffff927e7abdb180 ti: ffff927e933ec000 task.ti: ffff927e933ec000
RIP: 0010:[<ffffffffb1316c5a>] [<ffffffffb1316c5a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff927e933efcf8 EFLAGS: 00000202
RAX: 0000000000000003 RBX: 000000000001b840 RCX: ffff92898fae05b8
RDX: 0000000000000003 RSI: 0000000000000018 RDI: 0000000000000000
RBP: ffff927e933efd30 R08: ffff927e617b0800 R09: ffffffffb1585fd9
R10: ffff92898fb1f160 R11: ffffe6f1a2d4f000 R12: ffff927e933efca8
R13: ffff927e613a0000 R14: 000000000001acc0 R15: ffff927e933effd8
FS: 00007f8b5d300700(0000) GS:ffff92898fb00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f98bdc20000 CR3: 0000000cc73ee000 CR4: 00000000000227e0
Call Trace:
[<ffffffffb127e868>] native_flush_tlb_others+0xb8/0xc0
[<ffffffffb127e8db>] flush_tlb_mm_range+0x6b/0x140
[<ffffffffb13fe15d>] change_protection+0x5dd/0x680
[<ffffffffb141a80b>] change_prot_numa+0x1b/0x40
[<ffffffffb12dfec2>] task_numa_work+0x212/0x370
[<ffffffffb12c275b>] task_work_run+0xbb/0xe0
[<ffffffffb122cc65>] do_notify_resume+0xa5/0xc0
[<ffffffffb19952ef>] int_signal+0x12/0x17
Code: 48 63 35 ce 88 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 f5 b1 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1786
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9e116a6a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20a
|
|
2
|
native_flush_tlb_others
|
vmlinux |
0xffffffff9e07e478
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb8
|
|
3
|
flush_tlb_mm_range
|
vmlinux |
0xffffffff9e07e4eb
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x6b
|
|
4
|
change_protection
|
vmlinux |
0xffffffff9e1fc46d
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x5cd
|
|
5
|
change_prot_numa
|
vmlinux |
0xffffffff9e21851b
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x1b
|
|
6
|
task_numa_work
|
vmlinux |
0xffffffff9e0e0382
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x212
|
|
7
|
task_work_run
|
vmlinux |
0xffffffff9e0c2d2b
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xbb
|
|
8
|
do_notify_resume
|
vmlinux |
0xffffffff9e02cc65
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xa5
|
|
9
|
retint_signal
|
vmlinux |
0xffffffff9e78457c
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x48
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [FileLogWriter.F:27187]
Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dm_mirror dm_region_hash dm_log dm_mod ppdev aesni_intel lrw gf128mul sg glue_helper ablk_helper cryptd pcspkr joydev i2c_piix4 virtio_balloon parport_pc parport ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_common sr_mod cdrom virtio_net virtio_console virtio_scsi
ata_generic pata_acpi bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm ata_piix libata drm_panel_orientation_quirks virtio_pci serio_raw virtio_ring virtio floppy
CPU: 1 PID: 27187 Comm: FileLogWriter.F Tainted: G L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.1-0-ga5cab58e9a3f-prebuilt.qemu.org 04/01/2014
task: ffff9705768da0e0 ti: ffff9707f1034000 task.ti: ffff9707f1034000
RIP: 0010:[<ffffffff9e116a6a>] [<ffffffff9e116a6a>] smp_call_function_many+0x20a/0x270
RSP: 0000:ffff9707f1037cf8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff97055e020400
RDX: 0000000000000000 RSI: 0000000000000004 RDI: 0000000000000000
RBP: ffff9707f1037d30 R08: ffff97055d4c6400 R09: ffffffff9e3817e9
R10: ffff97055e11f160 R11: fffffd88cb596d00 R12: 0000000000000292
R13: ffff9707f1037ca8 R14: 000000000001ac80 R15: ffff9707f1037fd8
FS: 00007fb7a4cd8700(0000) GS:ffff97055e100000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000003e6f0b8 CR3: 00000002d669a000 CR4: 00000000000006e0
Call Trace:
[<ffffffff9e07e478>] native_flush_tlb_others+0xb8/0xc0
[<ffffffff9e07e4eb>] flush_tlb_mm_range+0x6b/0x140
[<ffffffff9e1fc46d>] change_protection+0x5cd/0x670
[<ffffffff9e21851b>] change_prot_numa+0x1b/0x40
[<ffffffff9e0e0382>] task_numa_work+0x212/0x370
[<ffffffff9e0c2d2b>] task_work_run+0xbb/0xe0
[<ffffffff9e02cc65>] do_notify_resume+0xa5/0xc0
[<ffffffff9e78457c>] retint_signal+0x48/0x8c
Code: 48 63 35 fe 5e c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 20 e8 d4 9e f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 cd 5e c4 00 eb b7 0f b6 4d cc 4c 89