Complete report #1339329
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fa3de
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x1be
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8169e6bf
Build id: 3.10.0-693.5.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816abc10
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff811b09c8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x68
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1ea7
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x6b7
|
|
6
|
|
vmlinux |
0xffffffff811b9ea5
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x65
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff816b0074
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x154
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff816b03a5
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff816ac5c8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [vasp_std:8355]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache 8021q garp mrp stp llc ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm ib_core intel_powerclamp coretemp snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep kvm_intel snd_seq snd_seq_device snd_pcm kvm snd_timer irqbypass iTCO_wdt gpio_ich iTCO_vendor_support snd shpchp sg lpc_ich ioatdma i2c_i801 tpm_infineon pcspkr soundcore i7core_edac edac_core dca acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sr_mod sd_mod cdrom crc_t10dif crct10dif_generic crct10dif_common mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm ata_generic pata_acpi drm ata_piix e1000e libata crc32c_intel serio_raw ptp i2c_core pps_core [last unloaded: ipmi_msghandler]
CPU: 0 PID: 8355 Comm: vasp_std Tainted: G L ------------ 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Supermicro X8DTL/X8DTL, BIOS 2.0c 01/05/11
task: ffff88082e271fa0 ti: ffff88082dae0000 task.ti: ffff88082dae0000
RIP: 0010:[<ffffffff810fa3de>] [<ffffffff810fa3de>] native_queued_spin_lock_slowpath+0x1be/0x1e0
RSP: 0000:ffff88082dae3db0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffffea0010b73630
RBP: ffff88082dae3db0 R08: 0000000000000101 R09: ffff88042d681210
R10: 0000000000000000 R11: 80000007f603f124 R12: 0000000000000000
R13: ffffffff81f579c0 R14: ffffffff810cc21b R15: ffff88082dae3d40
FS: 00002acac22b0080(0000) GS:ffff88042fa00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000085085e0 CR3: 000000042e6e3000 CR4: 00000000000007f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88082dae3dc0 ffffffff8169e6bf ffff88082dae3dd0 ffffffff816abc10
ffff88082dae3e20 ffffffff811b09c8 ffff88042e662580 00000000eca00000
0000000017bf985a 00000000085085e0 ffff88042b702360 ffff88042d681210
Call Trace:
[<ffffffff8169e6bf>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816abc10>] _raw_spin_lock+0x20/0x30
[<ffffffff811b09c8>] do_numa_page+0x68/0x1e0
[<ffffffff811b1ea7>] handle_mm_fault+0x6b7/0xfa0
[<ffffffff811b9ea5>] ? change_protection+0x65/0xa0
[<ffffffff816b0074>] __do_page_fault+0x154/0x450
[<ffffffff816b03a5>] do_page_fault+0x35/0x90
[<ffffffff816ac5c8>] page_fault+0x28/0x30
Code: 37 81 fe 00 01 00 00 74 f4 e9 95 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 69 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 f3
Complete report #1383371
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff86b11ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8715bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff8716a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff86be8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff86be9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
|
vmlinux |
0xffffffff86b0ce34
Build id: 3.10.0-957.el7.x86_64
|
0xa4
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff86bebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff8716f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff8716f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffff8716b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [java:5885]
Modules linked in: xt_REDIRECT nf_nat_redirect ip_vs_rr xt_ipvs ip_vs xt_nat veth vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter bridge stp llc overlay(T) iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter tcp_lp fuse symap_custom_3_10_0_957_el7_x86_64(POE) symev_custom_3_10_0_957_el7_x86_64(OE) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg i2c_piix4 vmw_vmci parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul sysimgblt crct10dif_common fb_sys_fops crc32c_intel ttm vmw_pvscsi serio_raw ahci libahci drm ata_piix libata nfit vmxnet3 libnvdimm drm_panel_orientation_quirks floppy
CPU: 3 PID: 5885 Comm: java Kdump: loaded Tainted: P OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff94b010b6d140 ti: ffff94b0b5bc0000 task.ti: ffff94b0b5bc0000
RIP: 0010:[<ffffffff86b11ffd>] [<ffffffff86b11ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff94b0b5bc3cf8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffd4db1b29de80 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffd4db13933130
RBP: ffff94b0b5bc3cf8 R08: ffff94afe4cc4c68 R09: ffff94ae20226370
R10: 0000000000000000 R11: 00003ffffffff000 R12: ffffffff87bbb4c0
R13: ffffffff86adeaeb R14: ffff94b0b5bc3c88 R15: ffff94b0b5bc3fd8
FS: 00007f5863cc2b10(0000) GS:ffff94af2fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000005cdd8d994 CR3: 00000003fc43e000 CR4: 00000000001607e0
Call Trace:
[<ffffffff8715bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8716a400>] _raw_spin_lock+0x20/0x30
[<ffffffff86be8c8f>] do_numa_page+0x9f/0x250
[<ffffffff86be9156>] handle_pte_fault+0x316/0xd10
[<ffffffff86b0ce34>] ? futex_wait_queue_me+0xa4/0x130
[<ffffffff86bebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff8716f5e3>] __do_page_fault+0x203/0x500
[<ffffffff8716f915>] do_page_fault+0x35/0x90
[<ffffffff8716b758>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #1313641
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fa3e0
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x1c0
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8169e6bf
Build id: 3.10.0-693.5.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816abc10
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff811b09c8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x68
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1ea7
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x6b7
|
|
6
|
|
vmlinux |
0xffffffff816a8dda
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x12a
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff816b0074
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x154
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff816b03a5
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff816ac5c8
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [serf:1635]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfsv3 nfs fscache 8021q garp mrp stp llc ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core intel_powerclamp coretemp kvm joydev irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support sg ipmi_si ipmi_devintf dcdbas ipmi_msghandler pcspkr lpc_ich i2c_i801 shpchp i7core_edac ioatdma edac_core nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 mlx4_en sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm drm mlx4_core igb ahci libahci libata crct10dif_pclmul crct10dif_common ptp crc32c_intel pps_core dca i2c_algo_bit i2c_core devlink
CPU: 7 PID: 1635 Comm: serf Tainted: G L ------------ 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Dell C6100 /0D61XP, BIOS 1.57 04/14/2011
task: ffff880c1eb30fd0 ti: ffff88061e6cc000 task.ti: ffff88061e6cc000
RIP: 0010:[<ffffffff810fa3e0>] [<ffffffff810fa3e0>] native_queued_spin_lock_slowpath+0x1c0/0x1e0
RSP: 0000:ffff88061e6cfdb0 EFLAGS: 00000202
RAX: 0000000000000101 RBX: ffffffff810f5e15 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffffea001897bb30
RBP: ffff88061e6cfdb0 R08: 0000000000000101 R09: ffff880625f1a800
R10: 0000000000000000 R11: 80000003d70d8966 R12: ffff880c1eb30fd0
R13: 000000c420021110 R14: 0000000000000000 R15: 0000000000000000
FS: 000000c420021068(0000) GS:ffff880c3f640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c42013c000 CR3: 0000000612b2d000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88061e6cfdc0 ffffffff8169e6bf ffff88061e6cfdd0 ffffffff816abc10
ffff88061e6cfe20 ffffffff811b09c8 ffff880626101f40 000000001e6cfe30
000000006eccc282 000000c42013c000 ffff880c1ff91440 ffff880625f1a800
Call Trace:
[<ffffffff8169e6bf>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816abc10>] _raw_spin_lock+0x20/0x30
[<ffffffff811b09c8>] do_numa_page+0x68/0x1e0
[<ffffffff811b1ea7>] handle_mm_fault+0x6b7/0xfa0
[<ffffffff816a8dda>] ? __schedule+0x12a/0x8b0
[<ffffffff816b0074>] __do_page_fault+0x154/0x450
[<ffffffff816b03a5>] do_page_fault+0x35/0x90
[<ffffffff816ac5c8>] page_fault+0x28/0x30
Code: fe 00 01 00 00 74 f4 e9 95 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 69 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 8b 07 <0f> b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 f3 90 4d
Complete report #1307620
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb711598f
Build id: 3.10.0-1062.el7.x86_64
|
0x1f
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb7773365
Build id: 3.10.0-1062.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb7781ad0
Build id: 3.10.0-1062.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb71eef1f
Build id: 3.10.0-1062.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffb71ef389
Build id: 3.10.0-1062.el7.x86_64
|
0x2b9
|
|
6
|
|
vmlinux |
0xffffffffb71c68f6
Build id: 3.10.0-1062.el7.x86_64
|
0x176
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffb71f200d
Build id: 3.10.0-1062.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffb7786633
Build id: 3.10.0-1062.el7.x86_64
|
0x213
|
|
9
|
|
vmlinux |
0xffffffffb726bca4
Build id: 3.10.0-1062.el7.x86_64
|
0x24
|
|
10
|
do_page_fault
|
vmlinux |
0xffffffffb7786955
Build id: 3.10.0-1062.el7.x86_64
|
0x35
|
|
11
|
|
vmlinux |
0xffffffffb7782ab6
Build id: 3.10.0-1062.el7.x86_64
|
0xa7
|
|
12
|
page_fault
|
vmlinux |
0xffffffffb7782768
Build id: 3.10.0-1062.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#28 stuck for 21s! [kubelet:6975]
Modules linked in: xt_multiport iptable_raw ip_set_hash_net ip_set_hash_ip ipip tunnel4 ip_tunnel veth xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport ip_set dummy ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc overlay(T) ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter ext4 mbcache jbd2 nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel
lrw gf128mul glue_helper ablk_helper joydev cryptd parport_pc vmw_vmci sg parport i2c_piix4 pcspkr sch_fq_codel ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea mptspi sysfillrect sysimgblt scsi_transport_spi fb_sys_fops mptscsih ttm crct10dif_pclmul drm crct10dif_common libata serio_raw crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 28 PID: 6975 Comm: kubelet Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff8fafbff30000 ti: ffff8fae6d3bc000 task.ti: ffff8fae6d3bc000
RIP: 0010:[<ffffffffb711598f>] [<ffffffffb711598f>] native_queued_spin_lock_slowpath+0x1f/0x200
RSP: 0000:ffff8fae6d3bfce8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000001 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffffee5d7079330
RBP: ffff8fae6d3bfce8 R08: ffff8fad41e4c408 R09: ffff8fae6de230b0
R10: 00007fc16853d200 R11: 00003ffffffff000 R12: ffffffffb71de088
R13: ffff8fae6d3bfc70 R14: ffff8fae6d3bfc58 R15: ffff8fae6d3bfe20
FS: 00007fc16affd700(0000) GS:ffff8fb7bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c002c81000 CR3: 00000006ed04a000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb7773365>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb7781ad0>] _raw_spin_lock+0x20/0x30
[<ffffffffb71eef1f>] do_numa_page+0x9f/0x250
[<ffffffffb71ef389>] handle_pte_fault+0x2b9/0xe20
[<ffffffffb71c68f6>] ? __alloc_pages_nodemask+0x176/0x420
[<ffffffffb71f200d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffb7786633>] __do_page_fault+0x213/0x500
[<ffffffffb726bca4>] ? mntput+0x24/0x40
[<ffffffffb7786955>] do_page_fault+0x35/0x90
[<ffffffffb7782ab6>] ? error_swapgs+0xa7/0xbd
[<ffffffffb7782768>] page_fault+0x28/0x30
Code: ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 8b 17 <85> d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f 1f 80
Complete report #1294248
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff9811e23e
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x20e
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff987ac21a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff987ba734
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x24
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff981ff74f
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff98202243
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xaf3
|
|
6
|
|
vmlinux |
0xffffffff9825886a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1ca
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff987bf7d3
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff987bfb05
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff987bb7b8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 29s! [kworker/u61:1:28203]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod ahci crc_t10dif crct10dif_generic drm ata_piix libahci
nfit libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 6 PID: 28203 Comm: kworker/u61:1 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
Workqueue: writeback bdi_writeback_workfn (flush-8:0)
task: ffff8e351504d280 ti: ffff8e401f954000 task.ti: ffff8e401f954000
RIP: 0010:[<ffffffff9811e23e>] [<ffffffff9811e23e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff8e401f957830 EFLAGS: 00000202
RAX: 000000000000000e RBX: 0008000098081b10 RCX: ffff8e418dda0728
RDX: 000000000000000e RSI: 000000000000001e RDI: 0000000000000000
RBP: ffff8e401f957868 R08: ffff8e36604b1c00 R09: ffffffff9839c7b9
R10: ffff8e418db9f160 R11: ffffe25575cb3200 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8e401f9577e0 R15: 0000000000000000
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 28s! [java:24813]
FS: 0000000000000000(0000) GS:ffff8e418db80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000023d6cc000 CR3: 0000000189f1a000 CR4: 00000000007607e0
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul glue_helper ablk_helper cryptd vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm sd_mod ahci crc_t10dif crct10dif_generic drm ata_piix libahci
nfit libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 8 PID: 24813 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8e42041cd280 ti: ffff8e41f8f2c000 task.ti: ffff8e41f8f2c000
RIP: 0010:[<ffffffff9811eb3d>] [<ffffffff9811eb3d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff8e41f8f2fd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000008 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe25544cb9570
RBP: ffff8e41f8f2fd70 R08: ffff8e3632e55bf8 R09: ffff8e36de8403d8
R10: ffff8e41f8f2fe20 R11: ffff8e3e7bbdc1b0 R12: 0000000000000001
R13: ffffffff9920db80 R14: ffffffff980e944b R15: ffff8e41f8f2fd00
FS: 00007fd33e1c5700(0000) GS:ffff8e418dc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd40f77ff18 CR3: 0000000191246000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff987ac21a>] queued_spin_lock_slowpath+0xb/0x13
[<ffffffff987ba734>] _raw_spin_lock+0x24/0x30
[<ffffffff981ff74f>] do_numa_page+0x9f/0x260
[<ffffffff98202243>] handle_mm_fault+0xaf3/0x1190
[<ffffffff9825886a>] ? __check_object_size+0x1ca/0x250
[<ffffffff987bf7d3>] __do_page_fault+0x213/0x510
[<ffffffff987bfb05>] do_page_fault+0x35/0x90
[<ffffffff987bb7b8>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 cc
Complete report #1278742
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9ed11ff6
Build id: 3.10.0-957.el7.x86_64
|
0x16
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9f35bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9f36a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff9ede8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff9ede9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
|
vmlinux |
0xffffffff9ee1b762
Build id: 3.10.0-957.el7.x86_64
|
0x1e2
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff9edebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff9f36f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff9f36f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffff9f36b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [dockerd:26930]
Modules linked in: xt_REDIRECT nf_nat_redirect ip_vs_rr xt_ipvs ip_vs xt_nat veth vxlan ip6_udp_tunnel udp_tunnel xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter binfmt_misc overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi
crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx e1000 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm floppy ata_piix mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 26930 Comm: dockerd Kdump: loaded Tainted: G W L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8b2dd3706180 ti: ffff8b2e15e9c000 task.ti: ffff8b2e15e9c000
RIP: 0010:[<ffffffff9ed11ff6>] [<ffffffff9ed11ff6>] native_queued_spin_lock_slowpath+0x16/0x200
RSP: 0000:ffff8b2e15e9fcf8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffffce849de13700 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffce848fb09130
RBP: ffff8b2e15e9fcf8 R08: ffff8b29ec244540 R09: ffff8b2e23e1d040
R10: 000000c00147c048 R11: 00003ffffffff000 R12: ffffffff9fdbb5c8
R13: ffffffff9ecdeaeb R14: ffff8b2e15e9fc88 R15: ffff8b2e15e9fc88
FS: 00007fee977fe700(0000) GS:ffff8b2e3fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0010a87c8 CR3: 00000006fba78000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9f35bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9f36a400>] _raw_spin_lock+0x20/0x30
[<ffffffff9ede8c8f>] do_numa_page+0x9f/0x250
[<ffffffff9ede9156>] handle_pte_fault+0x316/0xd10
[<ffffffff9ee1b762>] ? kmem_cache_free+0x1e2/0x200
[<ffffffff9edebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff9f36f5e3>] __do_page_fault+0x203/0x500
[<ffffffff9f36f915>] do_page_fault+0x35/0x90
[<ffffffff9f36b758>] page_fault+0x28/0x30
Code: c7 00 00 00 00 00 e9 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d <83> fa 03 74 08 f3 90 8b 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2
Complete report #1233218
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffaa31757d
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffaa9754ee
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffaa983b20
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffaa3f121f
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffaa3f3c8b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffffaa45d9cd
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x4d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffaa988653
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffaa988975
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffaa984778
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 23s! [dockerd:3350]
Modules linked in: binfmt_misc tcp_diag inet_diag ip_vs_rr xt_ipvs ip_vs xt_REDIRECT nf_nat_redirect xt_nat veth vxlan ip6_udp_tunnel udp_tunnel xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter i2dtrack(OE) nbd(E) dattobd(OE) sfs(OE) overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom
sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi libata scsi_transport_spi crct10dif_pclmul crct10dif_common mptscsih serio_raw crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 13 PID: 3350 Comm: dockerd Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff88e1a37241c0 ti: ffff88e1a3734000 task.ti: ffff88e1a3734000
RIP: 0010:[<ffffffffaa31757d>] [<ffffffffaa31757d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff88e1a3737d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 000000000000000d RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffdf5df4a97c30
RBP: ffff88e1a3737d70 R08: ffff88d8aa5f0cb0 R09: ffff88e1ad6375d8
R10: ffff88e1a3737e20 R11: ffff88d01bdef368 R12: 0000000000000000
R13: ffffffffab3d9818 R14: ffffffffaa2e356b R15: ffff88e1a3737d00
FS: 00007fb2d1ffb700(0000) GS:ffff88dd4e480000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c057796000 CR3: 000000090d9b8000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaa9754ee>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffaa983b20>] _raw_spin_lock+0x20/0x30
[<ffffffffaa3f121f>] do_numa_page+0x9f/0x250
[<ffffffffaa3f3c8b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffaa45d9cd>] ? do_filp_open+0x4d/0xb0
[<ffffffffaa988653>] __do_page_fault+0x213/0x500
[<ffffffffaa988975>] do_page_fault+0x35/0x90
[<ffffffffaa984778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #1045834
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff85b0b956
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x16
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8610dadf
Build id: 3.10.0-862.11.6.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff8611b5d0
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff85bc7fdf
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff85bc84a6
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x316
|
|
6
|
|
vmlinux |
0xffffffff85b06610
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x90
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff85bcae3d
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff86120557
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x197
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff861208e5
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffff8611c758
Build id: 3.10.0-862.11.6.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 23s! [Camel (camel-1):4711]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay gsch(OE) redirfs(OE) event_collector_3_0_1194840(OE) ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev dm_thin_pool dm_persistent_data vmw_balloon iosf_mbi dm_bio_prison crc32_pclmul dm_bufio ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd joydev pcspkr sg parport_pc vmw_vmci i2c_piix4 nfit shpchp parport libnvdimm auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel vmxnet3 serio_raw i2c_core vmw_pvscsi dm_mirror dm_region_hash dm_log dm_mod
CPU: 9 PID: 4711 Comm: Camel (camel-1) Kdump: loaded Tainted: G OEL ------------ 3.10.0-862.11.6.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff97eecf2bcf10 ti: ffff97e4fa440000 task.ti: ffff97e4fa440000
RIP: 0010:[<ffffffff85b0b956>] [<ffffffff85b0b956>] native_queued_spin_lock_slowpath+0x16/0x200
RSP: 0000:ffff97e4fa443d00 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffff97eecf2bcf10 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffeab5f50ac470
RBP: ffff97e4fa443d00 R08: ffff97f202b11720 R09: ffff97f78cbed698
R10: 00000000dc495f20 R11: 00003ffffffff000 R12: ffffb40d8cddb748
R13: 0000000000000000 R14: 0000000000000001 R15: ffff97eecf2bcf10
FS: 00007f12da6e6700(0000) GS:ffff97fdcd400000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f12da6e4470 CR3: 0000000018ada000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8610dadf>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8611b5d0>] _raw_spin_lock+0x20/0x30
[<ffffffff85bc7fdf>] do_numa_page+0x9f/0x250
[<ffffffff85bc84a6>] handle_pte_fault+0x316/0xd10
[<ffffffff85b06610>] ? futex_wake+0x90/0x180
[<ffffffff85bcae3d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff86120557>] __do_page_fault+0x197/0x4f0
[<ffffffff861208e5>] do_page_fault+0x35/0x90
[<ffffffff8611c758>] page_fault+0x28/0x30
Code: c7 00 00 00 00 00 e9 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d <83> fa 03 74 08 f3 90 8b 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2
Complete report #1013890
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb8d1eb3d
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb93ac21a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb93ba734
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x24
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb8dff74f
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffb8e02243
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xaf3
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffffb93bf7d3
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x213
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffffb93bfb05
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x35
|
|
8
|
page_fault
|
vmlinux |
0xffffffffb93bb7b8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [Navlink Plugin :10900]
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [diagnostics-jvm:10273]
Modules linked in: bmsensor(OE) ip6table_filter ip6_tables iptable_filter tcp_diag inet_diag binfmt_misc tmhook(OE) vmw_vsock_vmci_transport vsock crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd ppdev pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix crct10dif_pclmul drm crct10dif_common mptspi crc32c_intel libnvdimm scsi_transport_spi libata serio_raw mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: bmsensor]
CPU: 6 PID: 10273 Comm: diagnostics-jvm Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff91bf74b21080 ti: ffff91bd6a420000 task.ti: ffff91bd6a420000
RIP: 0010:[<ffffffffb8d1eb3d>] [<ffffffffb8d1eb3d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff91bd6a423d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000006 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe430a0d657b0
RBP: ffff91bd6a423d70 R08: ffff91bf7595eba8 R09: ffff91bf76762000
R10: ffff91bd6a423e20 R11: ffff91b96f0b56c8 R12: 0000000000000001
R13: ffffffffb9e0dc88 R14: ffffffffb8ce944b R15: ffff91bd6a423d00
FS: 00007f958cffd700(0000) GS:ffff91bb77d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000800175c88 CR3: 000000023750a000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb93ac21a>] queued_spin_lock_slowpath+0xb/0x13
[<ffffffffb93ba734>] _raw_spin_lock+0x24/0x30
[<ffffffffb8dff74f>] do_numa_page+0x9f/0x260
[<ffffffffb8e02243>] handle_mm_fault+0xaf3/0x1190
[<ffffffffb93bf7d3>] __do_page_fault+0x213/0x510
[<ffffffffb93bfb05>] do_page_fault+0x35/0x90
[<ffffffffb93bb7b8>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 cc
Complete report #902638
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb911799d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb977dcf3
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb978baa0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb91f3bef
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffb91f665b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffffb9254355
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x165
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffb9790653
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffb9790975
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffb978c778
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [kubelet:16759]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vxlan ip6_udp_tunnel udp_tunnel veth ipt_rpfilter xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set xt_statistic xt_nat xt_recent ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc netfilter_edr(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp ppdev vmw_balloon
iosf_mbi crc32_pclmul ghash_clmulni_intel joydev aesni_intel sg parport_pc lrw gf128mul glue_helper vmw_vmci ablk_helper parport cryptd i2c_piix4 pcspkr ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect ata_piix sysimgblt fb_sys_fops mptspi ttm scsi_transport_spi crct10dif_pclmul drm crct10dif_common mptscsih libata e1000 serio_raw crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy fuse
CPU: 0 PID: 16759 Comm: kubelet Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa11181319080 ti: ffffa1152ded8000 task.ti: ffffa1152ded8000
RIP: 0010:[<ffffffffb911799d>] [<ffffffffb911799d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffffa1152dedbd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffeb036ff682b0
RBP: ffffa1152dedbd70 R08: ffffa1153da0afc0 R09: ffffa11609240070
R10: ffffa1152dedbe20 R11: ffffa115e1acea20 R12: 0000000000000003
R13: ffffffffba1dbb80 R14: ffffffffb90e31ab R15: ffffa1152dedbd00
FS: 00007f346d2d2700(0000) GS:ffffa10d6fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c001df8160 CR3: 00000003f05da000 CR4: 00000000001607f0
Call Trace:
[<ffffffffb977dcf3>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb978baa0>] _raw_spin_lock+0x20/0x30
[<ffffffffb91f3bef>] do_numa_page+0x9f/0x250
[<ffffffffb91f665b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffb9254355>] ? cp_new_stat+0x165/0x1a0
[<ffffffffb9790653>] __do_page_fault+0x213/0x500
[<ffffffffb9790975>] do_page_fault+0x35/0x90
[<ffffffffb978c778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #897335
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffab111ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffab75bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffab76a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffab1e8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffab1e9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
?
schedule_hrtimeout_range_clock
|
vmlinux |
0xffffffffab766d8e
Build id: 3.10.0-957.el7.x86_64
|
0xbe
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffab1ebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffab76f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffab76f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffffab76b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [java:27149]
Modules linked in: vmw_vsock_vmci_transport vsock coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg parport_pc vmw_vmci parport i2c_piix4 sunrpc sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect ahci sysimgblt crct10dif_pclmul fb_sys_fops crct10dif_common ttm crc32c_intel libahci ata_piix serio_raw drm libata vmxnet3 vmw_pvscsi drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 15 PID: 27149 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8acd76eb6180 ti: ffff8ad2d2274000 task.ti: ffff8ad2d2274000
RIP: 0010:[<ffffffffab111ffd>] [<ffffffffab111ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff8ad2d2277cf8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffe1fec2cfb140 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe1fef072a670
RBP: ffff8ad2d2277cf8 R08: ffff8ad2dca99d38 R09: ffff8ad2e1bdd800
R10: 00000000000003e8 R11: 00003ffffffff000 R12: ffffffffac1bb5c8
R13: ffffffffab0deaeb R14: ffff8ad2d2277c88 R15: 00000000770d2451
FS: 00007fc4ccb83700(0000) GS:ffff8ad2ffdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc4201a7a70 CR3: 0000000bf1e00000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffab75bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffab76a400>] _raw_spin_lock+0x20/0x30
[<ffffffffab1e8c8f>] do_numa_page+0x9f/0x250
[<ffffffffab1e9156>] handle_pte_fault+0x316/0xd10
[<ffffffffab766d8e>] ? schedule_hrtimeout_range_clock+0xbe/0x150
[<ffffffffab1ebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffab76f5e3>] __do_page_fault+0x203/0x500
[<ffffffffab76f915>] do_page_fault+0x35/0x90
[<ffffffffab76b758>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #882631
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb6117aed
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb6780d63
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb678ed00
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb61f3bdf
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffb61f664b
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffffb624b14a
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1ca
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffb6794653
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffb6794975
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffb6790778
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [kube-apiserver:13050]
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kube-apiserver:12808]
Modules linked in: xt_statistic vxlan ip6_udp_tunnel udp_tunnel xt_nat nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter tun overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ppdev ablk_helper cryptd vmw_balloon sg pcspkr joydev vmw_vmci i2c_piix4 parport_pc parport
ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm nfit ata_piix libahci libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 11 PID: 12808 Comm: kube-apiserver Tainted: G L ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff90b442c1e300 ti: ffff90b409cd8000 task.ti: ffff90b409cd8000
RIP: 0010:[<ffffffffb6117aed>] [<ffffffffb6117aed>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff90b409cdbd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff90b409cdbd18 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffff61ceeff55f0
RBP: ffff90b409cdbd70 R08: ffff90b3ffd57b28 R09: ffff90b42da94100
R10: ffff90b409cdbe20 R11: ffff90ae5c3117a0 R12: ffff90b409cdbe70
R13: ffff90ab5ec63320 R14: ffff90a9bfd8b500 R15: ffff90ab5ec63320
FS: 000000c000580490(0000) GS:ffff90b47fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe844165f80 CR3: 0000000beda50000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb6780d63>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb678ed00>] _raw_spin_lock+0x20/0x30
[<ffffffffb61f3bdf>] do_numa_page+0x9f/0x250
[<ffffffffb61f664b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffb624b14a>] ? __check_object_size+0x1ca/0x250
[<ffffffffb6794653>] __do_page_fault+0x213/0x500
[<ffffffffb6794975>] do_page_fault+0x35/0x90
[<ffffffffb6790778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #881924
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9df1799d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9e57dcf3
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9e58baa0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff9dff3bef
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff9dff665b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffff9e067d09
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x29
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff9e590653
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff9e590975
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff9e58c778
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 24s! [kubelet:1542]
Modules linked in: netfilter_edr(OE) unix_diag udp_diag nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache sch_htb binfmt_misc tcp_diag inet_diag veth ipt_rpfilter xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_statistic xt_nat xt_recent xt_mark xt_comment nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash
dm_log dm_mod nfit libnvdimm iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi crct10dif_pclmul mptscsih libata crct10dif_common e1000 serio_raw crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy fuse
CPU: 3 PID: 1542 Comm: kubelet Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9ed7a52c0000 ti: ffff9ed5034e8000 task.ti: ffff9ed5034e8000
RIP: 0010:[<ffffffff9df1799d>] [<ffffffff9df1799d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9ed5034ebd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 000000001fcb5444 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffdf68b0b3f930
RBP: ffff9ed5034ebd70 R08: ffff9ed7acfe49f0 R09: ffff9ed10d0ec0a8
R10: ffff9ed5034ebe20 R11: ffff9ecfadd05bd8 R12: ffff9ed7a52c0000
R13: ffffffff9e191672 R14: ffff9ed5034ebcf0 R15: ffff9ed5034ebd00
FS: 00007f70ce7fc700(0000) GS:ffff9ecfafcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c002b3e020 CR3: 00000007c8dbc000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9e57dcf3>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9e58baa0>] _raw_spin_lock+0x20/0x30
[<ffffffff9dff3bef>] do_numa_page+0x9f/0x250
[<ffffffff9dff665b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff9e067d09>] ? dput+0x29/0x1a0
[<ffffffff9e590653>] __do_page_fault+0x213/0x500
[<ffffffff9e590975>] do_page_fault+0x35/0x90
[<ffffffff9e58c778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #796136
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffae311ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffae95bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffae96a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffae3e8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffae3e9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
handle_mm_fault
|
vmlinux |
0xffffffffae3ebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffae96f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffae96f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffae96b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [java:7194]
Modules linked in: binfmt_misc tcp_diag inet_diag rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper sg cryptd vmw_balloon pcspkr joydev i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci crc32c_intel libahci drm ata_piix serio_raw libata nfit vmxnet3 libnvdimm vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 13 PID: 7194 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff97dff94ac100 ti: ffff97dff0f50000 task.ti: ffff97dff0f50000
RIP: 0010:[<ffffffffae311ffd>] [<ffffffffae311ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff97dff0f53cf8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000002000 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffff2e39a2d7570
RBP: ffff97dff0f53cf8 R08: ffff97e5cb5d5698 R09: ffff97e8b97810f0
R10: 0000000000000000 R11: 00003ffffffff000 R12: ffff97e935bfbc00
R13: ffff97dff0f53e50 R14: 00000000a15dbec6 R15: 7fffffffffffffff
FS: 00007f9692e80700(0000) GS:ffff97ebff600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000c3cd3344 CR3: 000000090de0c000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffae95bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffae96a400>] _raw_spin_lock+0x20/0x30
[<ffffffffae3e8c8f>] do_numa_page+0x9f/0x250
[<ffffffffae3e9156>] handle_pte_fault+0x316/0xd10
[<ffffffffae3ebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffae96f5e3>] __do_page_fault+0x203/0x500
[<ffffffffae96f915>] do_page_fault+0x35/0x90
[<ffffffffae96b758>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #738762
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9e517a4d
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9eb80d83
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9eb8ed20
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff9e5f3c1f
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff9e5f668b
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xadb
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffff9eb94653
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x213
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffff9eb94975
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x35
|
|
8
|
|
vmlinux |
0xffffffff9eb90ac9
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xaa
|
|
9
|
page_fault
|
vmlinux |
0xffffffff9eb90778
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [java:19544]
Modules linked in: tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc vfat fat crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev sg pcspkr parport_pc parport tpm_crb vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix nfit crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi scsi_transport_spi libnvdimm serio_raw
e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 8 PID: 19544 Comm: java Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.9318676.B64.1807270745 07/27/2018
task: ffff9650c6464200 ti: ffff965901544000 task.ti: ffff965901544000
RIP: 0010:[<ffffffff9e517a4d>] [<ffffffff9e517a4d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff965901547d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000009 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe78288befe70
RBP: ffff965901547d70 R08: ffff9652efbf9158 R09: ffff9653cc05c0e8
R10: ffff965901547e20 R11: ffff96516cf7b878 R12: 0000000000000000
R13: ffffffff9f5dcc88 R14: ffffffff9e4e5fcb R15: ffff9651c2ad67c0
FS: 00002af38f2ea700(0000) GS:ffff9660ffc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000003a2b3f0 CR3: 00000007ee914000 CR4: 00000000000007e0
Call Trace:
[<ffffffff9eb80d83>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9eb8ed20>] _raw_spin_lock+0x20/0x30
[<ffffffff9e5f3c1f>] do_numa_page+0x9f/0x250
[<ffffffff9e5f668b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff9eb94653>] __do_page_fault+0x213/0x500
[<ffffffff9eb94975>] do_page_fault+0x35/0x90
[<ffffffff9eb90ac9>] ? error_swapgs+0xaa/0xc0
[<ffffffff9eb90778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #724441
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9a91799d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9af80d73
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9af8ed10
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff9a9f3bef
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff9a9f665b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xadb
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffff9af94653
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x213
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffff9af94975
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x35
|
|
8
|
page_fault
|
vmlinux |
0xffffffff9af90778
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 21s! [containerd:1951]
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [containerd:1491]
Modules linked in: veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 nfit libnvdimm coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi libata scsi_transport_spi crct10dif_pclmul vmxnet3 crct10dif_common mptscsih crc32c_intel serio_raw mptbase drm_panel_orientation_quirks
floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 1491 Comm: containerd Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9184b5b30000 ti: ffff9182b7a84000 task.ti: ffff9182b7a84000
RIP: 0010:[<ffffffff9a91799d>] [<ffffffff9a91799d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9182b7a87d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000086 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffcfa090dd2c70
RBP: ffff9182b7a87d70 R08: ffff9182b74b1248 R09: ffff9182b74b0270
R10: ffff9182b7a87e20 R11: ffff9184b4637950 R12: ffff9182b7a87d30
R13: ffff9186b6938400 R14: 00000010b7a87da0 R15: 0000000000000000
FS: 00007fbdc568a700(0000) GS:ffff9180b7c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbdc9c490a2 CR3: 0000000435c64000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9af80d73>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9af8ed10>] _raw_spin_lock+0x20/0x30
[<ffffffff9a9f3bef>] do_numa_page+0x9f/0x250
[<ffffffff9a9f665b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff9af94653>] __do_page_fault+0x213/0x500
[<ffffffff9af94975>] do_page_fault+0x35/0x90
[<ffffffff9af90778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #694469
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa9711ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa9d5bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffa9d6a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffa97e8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffa97e9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
|
vmlinux |
0xffffffffa9d6770f
Build id: 3.10.0-957.el7.x86_64
|
0x3ff
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffffa97ebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffa9d6f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffa9d6f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffffa9d6b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#27 stuck for 21s! [uwsgi:28097]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul sg glue_helper parport_pc ablk_helper parport vmw_vmci cryptd joydev pcspkr i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom vmwgfx drm_kms_helper sd_mod syscopyarea crc_t10dif ata_generic sysfillrect crct10dif_generic pata_acpi sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ahci ttm
libahci ata_piix drm serio_raw crc32c_intel vmw_pvscsi vmxnet3 libata drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 27 PID: 28097 Comm: uwsgi Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9ec3a3aca080 ti: ffff9ebc746cc000 task.ti: ffff9ebc746cc000
RIP: 0010:[<ffffffffa9711ffd>] [<ffffffffa9711ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9ebc746cfcf8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffebfb3c7a49c0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffebfb3db3c930
RBP: ffff9ebc746cfcf8 R08: ffff9ecbacf24038 R09: ffff9ecb5534c060
R10: 000000000000000f R11: 00003ffffffff000 R12: ffffffffaa7bb5c8
R13: ffffffffa96deaeb R14: ffff9ebc746cfc88 R15: ffff9ec72da50b40
FS: 00007fb1311b2700(0000) GS:ffff9ecc5f4c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001807c30 CR3: 0000000f54338000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa9d5bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa9d6a400>] _raw_spin_lock+0x20/0x30
[<ffffffffa97e8c8f>] do_numa_page+0x9f/0x250
[<ffffffffa97e9156>] handle_pte_fault+0x316/0xd10
[<ffffffffa9d6770f>] ? __schedule+0x3ff/0x890
[<ffffffffa97ebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffa9d6f5e3>] __do_page_fault+0x203/0x500
[<ffffffffa9d6f915>] do_page_fault+0x35/0x90
[<ffffffffa9d6b758>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #691646
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff99917a4d
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff99f80d83
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff99f8ed20
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff999f3c1f
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff999f668b
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffff99897cc4
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x14
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff99f94653
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff99f94975
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff99f90778
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [netdata:2164]
Modules linked in: ext4 mbcache jbd2 loop xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon joydev sg parport_pc i2c_piix4 vmw_vmci parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops sd_mod ttm crc_t10dif crct10dif_generic ahci drm ata_piix libahci nfit libata crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 8 PID: 2164 Comm: netdata Kdump: loaded Tainted: G L ------------ 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff984f612b3180 ti: ffff984f62330000 task.ti: ffff984f62330000
RIP: 0010:[<ffffffff99917a4d>] [<ffffffff99917a4d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff984f62333d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff984f612b31e8 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffd2c920ba66b0
RBP: ffff984f62333d70 R08: ffff984f6e99afb0 R09: ffff984f6212dc28
R10: ffff984f62333e20 R11: ffff984b6b9f4a20 R12: ffff984f62333d08
R13: 00e6d6c4d785e11f R14: ffffffff998e2acc R15: ffff984f62333d08
FS: 00007f8a2cab9700(0000) GS:ffff984f7fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8a30bf6000 CR3: 00000008221d6000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff99f80d83>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff99f8ed20>] _raw_spin_lock+0x20/0x30
[<ffffffff999f3c1f>] do_numa_page+0x9f/0x250
[<ffffffff999f668b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff99897cc4>] ? mmput+0x14/0xf0
[<ffffffff99f94653>] __do_page_fault+0x213/0x500
[<ffffffff99f94975>] do_page_fault+0x35/0x90
[<ffffffff99f90778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #531713
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb3f1799d
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb457dcf3
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb458baa0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb3ff3bef
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffb3ff665b
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xadb
|
|
6
|
?
schedule_hrtimeout_range_clock
|
vmlinux |
0xffffffffb4588536
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xa6
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffb4590653
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffb4590975
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x35
|
|
9
|
|
vmlinux |
0xffffffffb458cac9
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0xaa
|
|
10
|
page_fault
|
vmlinux |
0xffffffffb458c778
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [elasticsearch[L:2728]
Modules linked in: nf_conntrack_ftp udp_diag binfmt_misc tcp_diag inet_diag xt_REDIRECT nf_nat_redirect ip_vs_rr xt_ipvs ip_vs xt_nat veth vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc coretemp ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper vmw_vmci joydev pcspkr cryptd sg parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul mptspi crct10dif_common libata crc32c_intel scsi_transport_spi serio_raw mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 9 PID: 2728 Comm: elasticsearch[L Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff9f27a1413180 ti: ffff9f2bf7874000 task.ti: ffff9f2bf7874000
RIP: 0010:[<ffffffffb3f1799d>] [<ffffffffb3f1799d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9f2bf7877d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000000000009 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe20ff2bda870
RBP: ffff9f2bf7877d70 R08: ffff9f40af6a1078 R09: ffff9f32d7f3d900
R10: ffff9f2bf7877e20 R11: ffff9f3be6bb0948 R12: 0000000000000001
R13: ffffffffb4fdbc88 R14: ffffffffb3ee31ab R15: ffff9f2bf7877d00
FS: 00007f645acee700(0000) GS:ffff9f341fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f642400f974 CR3: 000000015678c000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb457dcf3>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb458baa0>] _raw_spin_lock+0x20/0x30
[<ffffffffb3ff3bef>] do_numa_page+0x9f/0x250
[<ffffffffb3ff665b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffb4588536>] ? schedule_hrtimeout_range_clock+0xa6/0x150
[<ffffffffb4590653>] __do_page_fault+0x213/0x500
[<ffffffffb4590975>] do_page_fault+0x35/0x90
[<ffffffffb458cac9>] ? error_swapgs+0xaa/0xc0
[<ffffffffb458c778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #488462
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff97517b30
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x1d0
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff97b7dd13
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff97b8bac0
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff975f3bcf
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff975f663b
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0xadb
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffff97b90653
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x213
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffff97b90975
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x35
|
|
8
|
page_fault
|
vmlinux |
0xffffffff97b8c778
Build id: 3.10.0-1160.36.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [fio:40791]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iTCO_wdt iTCO_vendor_support vfat fat skx_edac coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg i2c_i801 mei_me mei lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter ip_tables
xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper syscopyarea crct10dif_pclmul sysfillrect crct10dif_common sysimgblt crc32c_intel fb_sys_fops ttm i40e ahci drm libahci nvme libata ptp pps_core nvme_core drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 18 PID: 40791 Comm: fio Kdump: loaded Tainted: G L ------------ 3.10.0-1160.36.2.el7.x86_64 #1
Hardware name: New H3C Technologies Co., Ltd. UniServer R4900 G3/RS33M2C9S, BIOS 2.00.52 08/10/2021
task: ffff94f3b3ca4200 ti: ffff95080e118000 task.ti: ffff95080e118000
RIP: 0010:[<ffffffff97517b30>] [<ffffffff97517b30>] native_queued_spin_lock_slowpath+0x1d0/0x200
RSP: 0000:ffff95080e11bd70 EFLAGS: 00000202
RAX: 0000000001e10101 RBX: 0000000000000001 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffffeb10a03b3a30
RBP: ffff95080e11bd70 R08: 0000000000000101 R09: ffff95085baef018
R10: ffff95080e11be20 R11: ffff95083eaeb888 R12: ffff94f3b3ca4268
R13: ffffffff974e0f3e R14: ffff95080e11bcf8 R15: 000066e3c1257d0c
FS: 00007f3357f02700(0000) GS:ffff95086d280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006a6ce0 CR3: 0000002809b62000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff97b7dd13>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff97b8bac0>] _raw_spin_lock+0x20/0x30
[<ffffffff975f3bcf>] do_numa_page+0x9f/0x250
[<ffffffff975f663b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff97b90653>] __do_page_fault+0x213/0x500
[<ffffffff97b90975>] do_page_fault+0x35/0x90
[<ffffffff97b8c778>] page_fault+0x28/0x30
Code: fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 8b 07 <0f> b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66 0f 1f
Complete report #427333
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffbc31799d
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffbc980d73
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffbc98ed10
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffbc3f3bef
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffbc3f665b
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffffbc27dff5
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x115
|
|
7
|
|
vmlinux |
0xffffffffbc27dfe9
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x109
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffffbc994653
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffffbc994975
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x35
|
|
10
|
page_fault
|
vmlinux |
0xffffffffbc990778
Build id: 3.10.0-1160.59.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#21 stuck for 21s! [llvmpipe-2:10770]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_13501(E) falcon_lsm_pinned_13402(E) tcp_lp falcon_lsm_pinned_13207(E) nf_conntrack_netlink xt_addrtype falcon_lsm_pinned_13109(E) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw ppdev gf128mul vmw_balloon glue_helper ablk_helper cryptd sg joydev parport_pc i2c_piix4 parport vmw_vmci pcspkr nf_conntrack_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ip_tables xfs sr_mod cdrom ata_generic pata_acpi vmwgfx
sd_mod crc_t10dif drm_kms_helper crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm nfit libata crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse nf_defrag_ipv4 br_netfilter bridge stp llc libcrc32c [last unloaded: falcon_kal]
CPU: 21 PID: 10770 Comm: llvmpipe-2 Kdump: loaded Tainted: P EL ------------ T 3.10.0-1160.59.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9b2685cf1080 ti: ffff9b269ed1c000 task.ti: ffff9b269ed1c000
RIP: 0010:[<ffffffffbc31799d>] [<ffffffffbc31799d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9b269ed1fd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff9b269ed1fd90 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffffbde09137130
RBP: ffff9b269ed1fd70 R08: ffff9b25c4dc4b98 R09: ffff9b2686ac8218
R10: ffff9b269ed1fe20 R11: ffff9b269ec1ae58 R12: ffff9b269ed1fcf0
R13: fffffbde07d56840 R14: 0000000000000019 R15: 0000000000000018
FS: 00007fa658fcd700(0000) GS:ffff9b29bf740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000008773168 CR3: 0000000302dec000 CR4: 00000000003607e0
Call Trace:
[<ffffffffbc980d73>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffbc98ed10>] _raw_spin_lock+0x20/0x30
[<ffffffffbc3f3bef>] do_numa_page+0x9f/0x250
[<ffffffffbc3f665b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffbc27dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffffbc27dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffbc994653>] __do_page_fault+0x213/0x500
[<ffffffffbc994975>] do_page_fault+0x35/0x90
[<ffffffffbc990778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #355675
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fa232
Build id: 3.10.0-693.el7.x86_64
|
0x22
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8169e5ff
Build id: 3.10.0-693.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816abb50
Build id: 3.10.0-693.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff811b0a58
Build id: 3.10.0-693.el7.x86_64
|
0x68
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1f37
Build id: 3.10.0-693.el7.x86_64
|
0x6b7
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffff816affb4
Build id: 3.10.0-693.el7.x86_64
|
0x154
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffff816b02e5
Build id: 3.10.0-693.el7.x86_64
|
0x35
|
|
8
|
page_fault
|
vmlinux |
0xffffffff816ac508
Build id: 3.10.0-693.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [2_scheduler:12225]
Modules linked in: bmhook(OE) tmhook(OE) dsa_filter(POE) dsa_filter_hook(OE) vxlan ip6_udp_tunnel udp_tunnel veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_comment xt_ipvs xt_set ip_vs_wlc ip_vs ip_set_hash_net ip_set nfnetlink xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter dummy rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev vmw_balloon sb_edac edac_core iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd joydev pcspkr sg vmw_vmci parport_pc parport shpchp i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm ata_piix vmxnet3 libata vmw_pvscsi i2c_core nfit libnvdimm [last unloaded: dsa_filter_hook]
CPU: 2 PID: 12225 Comm: 2_scheduler Tainted: P B W OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff880815e35ee0 ti: ffff88081c340000 task.ti: ffff88081c340000
RIP: 0010:[<ffffffff810fa232>] [<ffffffff810fa232>] native_queued_spin_lock_slowpath+0x22/0x1e0
RSP: 0000:ffff88081c343db0 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0000000000000002 RCX: 80000002d9f89926
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffea001f8fe530
RBP: ffff88081c343db0 R08: ffff8807e3f94418 R09: ffff88003610b408
R10: 0000000000000000 R11: 80000002d9f89926 R12: 0000000000000000
R13: ffffffff81f579c0 R14: ffffffff810cc20b R15: ffff88081c343d40
FS: 00007efeaea3e700(0000) GS:ffff88042fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007efd10283ef0 CR3: 00000007f8aba000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88081c343dc0 ffffffff8169e5ff ffff88081c343dd0 ffffffff816abb50
ffff88081c343e20 ffffffff811b0a58 ffff88082d161f40 0000000000000001
000000001d07bdfe 00007efd10283ef0 ffff8800ad96f878 ffff88003610b408
Call Trace:
[<ffffffff8169e5ff>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816abb50>] _raw_spin_lock+0x20/0x30
[<ffffffff811b0a58>] do_numa_page+0x68/0x1e0
[<ffffffff811b1f37>] handle_mm_fault+0x6b7/0xfa0
[<ffffffff816affb4>] __do_page_fault+0x154/0x450
[<ffffffff816b02e5>] do_page_fault+0x35/0x90
[<ffffffff816ac508>] page_fault+0x28/0x30
Code: 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 5d c3 f3 90 <eb> ec 0f 1f 40 00 81 fe 00 01 00 00 66 90 0f 84 5c 01 00 00 41
Complete report #205643
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9a7086fd
Build id: 3.10.0-862.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9ad0842a
Build id: 3.10.0-862.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9ad15680
Build id: 3.10.0-862.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff9a7c4e76
Build id: 3.10.0-862.el7.x86_64
|
0x86
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff9a7c52fe
Build id: 3.10.0-862.el7.x86_64
|
0x2fe
|
|
6
|
|
vmlinux |
0xffffffff9a7033a0
Build id: 3.10.0-862.el7.x86_64
|
0x90
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff9a7c747d
Build id: 3.10.0-862.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff9ad1a597
Build id: 3.10.0-862.el7.x86_64
|
0x197
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff9ad1a925
Build id: 3.10.0-862.el7.x86_64
|
0x35
|
|
10
|
|
vmlinux |
0xffffffff9ad16ab6
Build id: 3.10.0-862.el7.x86_64
|
0xa7
|
|
11
|
page_fault
|
vmlinux |
0xffffffff9ad16768
Build id: 3.10.0-862.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [java:7125]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock coretemp ppdev vmw_balloon joydev pcspkr sg i2c_piix4 vmw_vmci shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi libata scsi_transport_spi mptscsih e1000 serio_raw mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 14 PID: 7125 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa09b11480fd0 ti: ffffa099b45bc000 task.ti: ffffa099b45bc000
RIP: 0010:[<ffffffff9a7086fd>] [<ffffffff9a7086fd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffffa099b45bfd00 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffa09b11480fd0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffdf5ca3db63f0
RBP: ffffa099b45bfd00 R08: ffffa0a276d8f648 R09: ffffa09e0be39a00
R10: 00007fd0e4ee9000 R11: 00003ffffffff000 R12: 00000000000139a0
R13: 00000000000139a0 R14: ffffffff9a6bed10 R15: ffffa09b11480fd0
FS: 00007fd09c1f6700(0000) GS:ffffa0a280900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fd0680c9be0 CR3: 000000073437c000 CR4: 00000000000007e0
Call Trace:
[<ffffffff9ad0842a>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9ad15680>] _raw_spin_lock+0x20/0x30
[<ffffffff9a7c4e76>] do_numa_page+0x86/0x210
[<ffffffff9a7c52fe>] handle_pte_fault+0x2fe/0xc30
[<ffffffff9a7033a0>] ? futex_wake+0x90/0x180
[<ffffffff9a7c747d>] handle_mm_fault+0x39d/0x9b0
[<ffffffff9ad1a597>] __do_page_fault+0x197/0x4f0
[<ffffffff9ad1a925>] do_page_fault+0x35/0x90
[<ffffffff9ad16ab6>] ? error_swapgs+0xa7/0xbd
[<ffffffff9ad16768>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #187338
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8491211d
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff84f5cfcb
Build id: 3.10.0-957.10.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff84f6b480
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff849e911f
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff849e95e6
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x316
|
|
6
|
handle_mm_fault
|
vmlinux |
0xffffffff849ec0fd
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x39d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff84f705e3
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x203
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff84f70915
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x35
|
|
9
|
|
vmlinux |
0xffffffff84f6ca96
Build id: 3.10.0-957.10.1.el7.x86_64
|
0xa7
|
|
10
|
page_fault
|
vmlinux |
0xffffffff84f6c758
Build id: 3.10.0-957.10.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [java:3908]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev vmw_vmci sg nfit parport_pc parport libnvdimm i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt crc32c_intel fb_sys_fops ttm serio_raw drm ata_piix e1000 mptspi libata scsi_transport_spi
mptscsih mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3908 Comm: java Kdump: loaded Tainted: G L ------------ 3.10.0-957.10.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9766aaef8000 ti: ffff9765fefcc000 task.ti: ffff9765fefcc000
RIP: 0010:[<ffffffff8491211d>] [<ffffffff8491211d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9765fefcfcf8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: fffffdbecb8d0cc0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffffdbec7fc6570
RBP: ffff9765fefcfcf8 R08: ffff97647f195e68 R09: ffff976695fedb70
R10: 00007f2f22156da0 R11: 00003ffffffff000 R12: ffffffff859bb4c0
R13: ffffffff848deb7b R14: ffff9765fefcfc88 R15: 000000009345952d
FS: 00007f2f0bd0a700(0000) GS:ffff9764b7c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000007addcd010 CR3: 0000000233fd8000 CR4: 00000000000407f0
Call Trace:
[<ffffffff84f5cfcb>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff84f6b480>] _raw_spin_lock+0x20/0x30
[<ffffffff849e911f>] do_numa_page+0x9f/0x250
[<ffffffff849e95e6>] handle_pte_fault+0x316/0xd10
[<ffffffff849ec0fd>] handle_mm_fault+0x39d/0x9b0
[<ffffffff84f705e3>] __do_page_fault+0x203/0x500
[<ffffffff84f70915>] do_page_fault+0x35/0x90
[<ffffffff84f6ca96>] ? error_swapgs+0xa7/0xbd
[<ffffffff84f6c758>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #143008
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffad7121ae
Build id: 3.10.0-957.el7.x86_64
|
0x1ce
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffadd5bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffadd6a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffad7e8c8f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffad7e9156
Build id: 3.10.0-957.el7.x86_64
|
0x316
|
|
6
|
handle_mm_fault
|
vmlinux |
0xffffffffad7ebc6d
Build id: 3.10.0-957.el7.x86_64
|
0x39d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffadd6f5e3
Build id: 3.10.0-957.el7.x86_64
|
0x203
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffadd6f915
Build id: 3.10.0-957.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffadd6b758
Build id: 3.10.0-957.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [java:14065]
Modules linked in: nvidia_uvm(OE) binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) nls_utf8 isofs loop iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ipmi_devintf ablk_helper cryptd ppdev ipmi_msghandler pcspkr joydev vmw_balloon
i2c_piix4 sg parport_pc parport vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 ahci libahci vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libnvdimm libata vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14065 Comm: java Tainted: P OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8d5f52aab0c0 ti: ffff8d5f4d448000 task.ti: ffff8d5f4d448000
RIP: 0010:[<ffffffffad7121ae>] [<ffffffffad7121ae>] native_queued_spin_lock_slowpath+0x1ce/0x200
RSP: 0000:ffff8d5f4d44bcf8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffff8d5fba91ac00 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffffe9c886aa83f0
RBP: ffff8d5f4d44bcf8 R08: 0000000000000101 R09: ffff8d5fcb353440
R10: 0000000000000001 R11: 00003ffffffff000 R12: ffff8d5fba518400
R13: 0000000000000400 R14: 0000000200000000 R15: ffffffffad6e05ac
FS: 00007f1f1619f700(0000) GS:ffff8d5fba900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000791058634 CR3: 00000002949c4000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffadd5bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffadd6a400>] _raw_spin_lock+0x20/0x30
[<ffffffffad7e8c8f>] do_numa_page+0x9f/0x250
[<ffffffffad7e9156>] handle_pte_fault+0x316/0xd10
[<ffffffffad7ebc6d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffadd6f5e3>] __do_page_fault+0x203/0x500
[<ffffffffadd6f915>] do_page_fault+0x35/0x90
[<ffffffffadd6b758>] page_fault+0x28/0x30
Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66
Complete report #61632
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff90f1757d
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff915754ee
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff91583b20
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff90ff121f
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff90ff3c8b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffff9105d9cd
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x4d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff91588653
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff91588975
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff91584778
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [kubelet:923]
Modules linked in: veth rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache xt_statistic xt_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ip_set nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ppdev vmw_balloon sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport auth_rpcgss
sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod drm_kms_helper crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci nfit libahci ata_piix libnvdimm libata crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc
CPU: 11 PID: 923 Comm: kubelet Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9e2eae24b150 ti: ffff9e2e3b2bc000 task.ti: ffff9e2e3b2bc000
RIP: 0010:[<ffffffff90f1757d>] [<ffffffff90f1757d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9e2e3b2bfd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff9e2ebfd6d520 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffe72ee0b70d70
RBP: ffff9e2e3b2bfd70 R08: ffff9e2eadc35620 R09: ffff9e2e73f130c0
R10: ffff9e2e3b2bfe20 R11: ffff9e28c9e3be60 R12: ffff9e2eae24b150
R13: ffffffff91fd9608 R14: ffffffff9105963d R15: ffff9e2e3b2bfd68
FS: 00007fafea7e4700(0000) GS:ffff9e2ebfcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0030c4000 CR3: 000000055ce7c000 CR4: 00000000003607e0
Call Trace:
[<ffffffff915754ee>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff91583b20>] _raw_spin_lock+0x20/0x30
[<ffffffff90ff121f>] do_numa_page+0x9f/0x250
[<ffffffff90ff3c8b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff9105d9cd>] ? do_filp_open+0x4d/0xb0
[<ffffffff91588653>] __do_page_fault+0x213/0x500
[<ffffffff91588975>] do_page_fault+0x35/0x90
[<ffffffff91584778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #61616
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8eb1757d
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8f1754ee
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff8f183b20
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff8ebf121f
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff8ebf3c8b
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffff8eae0148
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x78
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff8f188653
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff8f188975
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff8f184778
Build id: 3.10.0-1062.18.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [dockerd:12704]
Modules linked in: veth rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache xt_statistic xt_nat ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ip_set nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_thin_pool ppdev dm_persistent_data vmw_balloon dm_bio_prison dm_bufio sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport auth_rpcgss
sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm nfit ata_piix libahci crct10dif_pclmul crct10dif_common libata libnvdimm crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc
CPU: 14 PID: 12704 Comm: dockerd Kdump: loaded Tainted: G L ------------ 3.10.0-1062.18.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8c9d0ca13150 ti: ffff8ca0fb70c000 task.ti: ffff8ca0fb70c000
RIP: 0010:[<ffffffff8eb1757d>] [<ffffffff8eb1757d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff8ca0fb70fd70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 000000000000000e RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffff5b14ffcbdf0
RBP: ffff8ca0fb70fd70 R08: ffff8c9cff2f7c80 R09: ffff8c9d09a781a0
R10: ffff8ca0fb70fe20 R11: ffff8c9b88636288 R12: 0000000000000000
R13: ffffffff8fbd9608 R14: ffffffff8eae356b R15: ffff8ca0fb70fd00
FS: 00007fcdd77fe700(0000) GS:ffff8ca13fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c006990000 CR3: 000000042d75a000 CR4: 00000000003607e0
Call Trace:
[<ffffffff8f1754ee>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8f183b20>] _raw_spin_lock+0x20/0x30
[<ffffffff8ebf121f>] do_numa_page+0x9f/0x250
[<ffffffff8ebf3c8b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff8eae0148>] ? __enqueue_entity+0x78/0x80
[<ffffffff8f188653>] __do_page_fault+0x213/0x500
[<ffffffff8f188975>] do_page_fault+0x35/0x90
[<ffffffff8f184778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #55720
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb69086fd
Build id: 3.10.0-862.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb6f0842a
Build id: 3.10.0-862.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb6f15680
Build id: 3.10.0-862.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb69c4e76
Build id: 3.10.0-862.el7.x86_64
|
0x86
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffffb69c52fe
Build id: 3.10.0-862.el7.x86_64
|
0x2fe
|
|
6
|
handle_mm_fault
|
vmlinux |
0xffffffffb69c747d
Build id: 3.10.0-862.el7.x86_64
|
0x39d
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffffb6f1a597
Build id: 3.10.0-862.el7.x86_64
|
0x197
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffffb6f1a925
Build id: 3.10.0-862.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffffb6f16768
Build id: 3.10.0-862.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [csi-provisioner:16277]
Modules linked in: ext4 mbcache jbd2 rbd libceph dns_resolver veth xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter fuse overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd nfit pcspkr sg ppdev joydev vmw_balloon parport_pc libnvdimm parport vmw_vmci shpchp i2c_piix4 ip_tables xfs libcrc32c ata_generic sr_mod pata_acpi cdrom vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci sd_mod crc_t10dif libahci ata_piix crct10dif_generic drm crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 i2c_core vmw_pvscsi floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 16277 Comm: csi-provisioner Tainted: G W L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff9f729def2f70 ti: ffff9f72eab74000 task.ti: ffff9f72eab74000
RIP: 0010:[<ffffffffb69086fd>] [<ffffffffb69086fd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9f72eab77d00 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff9f729def2f70 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffef994a98d830
RBP: ffff9f72eab77d00 R08: ffff9f7366360538 R09: ffff9f74a22f6000
R10: 0000000000000000 R11: 00003ffffffff000 R12: ffffb68541b7a648
R13: ffff9f72eab77da0 R14: 0000000000000001 R15: ffff9f729def2f70
FS: 000000c00008e090(0000) GS:ffff9f72f7cc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0000a7e28 CR3: 00000002a8196000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb6f0842a>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb6f15680>] _raw_spin_lock+0x20/0x30
[<ffffffffb69c4e76>] do_numa_page+0x86/0x210
[<ffffffffb69c52fe>] handle_pte_fault+0x2fe/0xc30
[<ffffffffb69c747d>] handle_mm_fault+0x39d/0x9b0
[<ffffffffb6f1a597>] __do_page_fault+0x197/0x4f0
[<ffffffffb6f1a925>] do_page_fault+0x35/0x90
[<ffffffffb6f16768>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #24708
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8111732d
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8177544a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff81783330
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff811f0ddf
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x9f
|
|
5
|
handle_pte_fault
|
vmlinux |
0xffffffff811f1249
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x2b9
|
|
6
|
|
xfs |
0xffffffffc02de0dc
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xfc
|
|
7
|
handle_mm_fault
|
vmlinux |
0xffffffff811f3ecd
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x39d
|
|
8
|
__do_page_fault
|
vmlinux |
0xffffffff81788653
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x213
|
|
9
|
do_page_fault
|
vmlinux |
0xffffffff81788975
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x35
|
|
10
|
|
vmlinux |
0xffffffff81784ac9
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xaa
|
|
11
|
page_fault
|
vmlinux |
0xffffffff81784778
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#17 stuck for 22s! [snjournald:11987]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter tcp_diag inet_diag vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi libata mptscsih crct10dif_pclmul crct10dif_common serio_raw
crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 17 PID: 11987 Comm: snjournald Kdump: loaded Tainted: G L ------------ 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff9550c5dd9070 ti: ffff955118958000 task.ti: ffff955118958000
RIP: 0010:[<ffffffff8111732d>] [<ffffffff8111732d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff95511895bce8 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 000000000000001d RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffeae188f53770
RBP: ffff95511895bce8 R08: ffff9550fd4dd928 R09: ffff955118625058
R10: 0000000000000001 R11: 00003ffffffff000 R12: ffff95514d8cb5c0
R13: ffff954f12713700 R14: 0000000006f81853 R15: 0000000000000000
FS: 00007f030c804700(0000) GS:ffff954f29900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001725ad8 CR3: 0000000258a0e000 CR4: 00000000000407e0
Call Trace:
[<ffffffff8177544a>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff81783330>] _raw_spin_lock+0x20/0x30
[<ffffffff811f0ddf>] do_numa_page+0x9f/0x250
[<ffffffff811f1249>] handle_pte_fault+0x2b9/0xe20
[<ffffffffc02de0dc>] ? xfs_iunlock+0xfc/0x120 [xfs]
[<ffffffff811f3ecd>] handle_mm_fault+0x39d/0x9b0
[<ffffffff81788653>] __do_page_fault+0x213/0x500
[<ffffffff81788975>] do_page_fault+0x35/0x90
[<ffffffff81784ac9>] ? error_swapgs+0xaa/0xc0
[<ffffffff81784778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #20906
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb131765b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x15b
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb197b43b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb1989b50
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffffb13f32bf
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffffb13f5d2b
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xadb
|
|
6
|
__do_page_fault
|
vmlinux |
0xffffffffb198f653
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x213
|
|
7
|
do_page_fault
|
vmlinux |
0xffffffffb198f975
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x35
|
|
8
|
|
vmlinux |
0xffffffffb198bac9
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0xaa
|
|
9
|
page_fault
|
vmlinux |
0xffffffffb198b778
Build id: 3.10.0-1160.6.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 23s! [QuartzScheduler:3191]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm d2dctl(POE) vhost_net vhost macvtap macvlan xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink mpt3sas mpt2sas raid_class scsi_transport_sas mptctl mptbase rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack bridge stp llc ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter jc42 vfat fat ext4 mbcache jbd2 ipmi_ssif intel_powerclamp coretemp kvm_intel kvm irqbypass iTCO_wdt
crc32_pclmul ghash_clmulni_intel aesni_intel cdc_ether iTCO_vendor_support lrw gf128mul glue_helper usbnet ipmi_si ipmi_devintf ablk_helper cryptd ipmi_msghandler ioatdma pcspkr mii sg i2c_i801 lpc_ich i7core_edac dca acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_generic pata_acpi drm ata_piix libata megaraid_sas crct10dif_pclmul crct10dif_common crc32c_intel serio_raw bnx2 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 2 PID: 3191 Comm: QuartzScheduler Kdump: loaded Tainted: P W OEL ------------ 3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: IBM System x3650 M3 -[7945K3G]-/69Y4438, BIOS -[D6E164AUS-1.22]- 06/04/2018
task: ffff927e90fec200 ti: ffff927e8f7a8000 task.ti: ffff927e8f7a8000
RIP: 0010:[<ffffffffb131765b>] [<ffffffffb131765b>] native_queued_spin_lock_slowpath+0x15b/0x200
RSP: 0000:ffff927e8f7abd70 EFLAGS: 00000202
RAX: 0000000000000101 RBX: 0000000000000002 RCX: 0000000000110000
RDX: 0000000000110101 RSI: 0000000000000101 RDI: ffffe6f1852cfdb0
RBP: ffff927e8f7abd70 R08: ffff92898fa9b8c0 R09: 0000000000000000
R10: ffff927e8f7abe20 R11: ffff927e93fca5e8 R12: 0000000000000000
R13: ffffffffb23dcb80 R14: ffffffffb12e2e7b R15: ffff927e8f7abd00
FS: 00007f8b5c2f0700(0000) GS:ffff92898fa80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8b5d820aef CR3: 0000000cc73ee000 CR4: 00000000000227e0
Call Trace:
[<ffffffffb197b43b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb1989b50>] _raw_spin_lock+0x20/0x30
[<ffffffffb13f32bf>] do_numa_page+0x9f/0x250
[<ffffffffb13f5d2b>] handle_mm_fault+0xadb/0xfb0
[<ffffffffb198f653>] __do_page_fault+0x213/0x500
[<ffffffffb198f975>] do_page_fault+0x35/0x90
[<ffffffffb198bac9>] ? error_swapgs+0xaa/0xc0
[<ffffffffb198b778>] page_fault+0x28/0x30
Code: 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 8b 17 0f b7 c2 <83> f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84 00 00 00 00 00
Complete report #9970
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff81d17edd
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8237a024
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff823886d0
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x20
|
|
4
|
do_numa_page
|
vmlinux |
0xffffffff81df33cf
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x9f
|
|
5
|
handle_mm_fault
|
vmlinux |
0xffffffff81df5e3b
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0xadb
|
|
6
|
|
vmlinux |
0xffffffff81ce0628
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x78
|
|
7
|
__do_page_fault
|
vmlinux |
0xffffffff8238d653
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x213
|
|
8
|
do_page_fault
|
vmlinux |
0xffffffff8238d975
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x35
|
|
9
|
page_fault
|
vmlinux |
0xffffffff82389778
Build id: 3.10.0-1127.19.1.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#40 stuck for 22s! [dockerd:24458]
Modules linked in: tcp_lp fuse veeamsnap(OE) veth binfmt_misc nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nls_utf8 cifs ccm dns_resolver overlay(T) vmw_vsock_vmci_transport vsock ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport nfsd nfs_acl lockd grace auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx sd_mod crc_t10dif crct10dif_generic drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci nfit ata_piix libahci drm crct10dif_pclmul libnvdimm crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 40 PID: 24458 Comm: dockerd Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff9c1274cd5230 ti: ffff9c0f5cdc4000 task.ti: ffff9c0f5cdc4000
RIP: 0010:[<ffffffff81d17edd>] [<ffffffff81d17edd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff9c0f5cdc7d70 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff9c0f5cdc7d00 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: fffff758e29c37b0
RBP: ffff9c0f5cdc7d70 R08: ffff9c07e70de350 R09: ffff9bf059358020
R10: ffff9c0f5cdc7e20 R11: ffff9c1d71dfd0e0 R12: 000000000000006b
R13: 0000000000000041 R14: 0000000000017b28 R15: ffff9c1274cd5230
FS: 00007f705a7fc700(0000) GS:ffff9c1d81c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00086a810 CR3: 0000001ee5308000 CR4: 00000000007607e0
DR0: 00000000006050d4 DR1: 0000000000601048 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600
PKRU: 55555554
Call Trace:
[<ffffffff8237a024>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff823886d0>] _raw_spin_lock+0x20/0x30
[<ffffffff81df33cf>] do_numa_page+0x9f/0x250
[<ffffffff81df5e3b>] handle_mm_fault+0xadb/0xfb0
[<ffffffff81ce0628>] ? __enqueue_entity+0x78/0x80
[<ffffffff8238d653>] __do_page_fault+0x213/0x500
[<ffffffff8238d975>] do_page_fault+0x35/0x90
[<ffffffff82389778>] page_fault+0x28/0x30
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f