Complete report #723587
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffb01356aa
Build id: 3.10.0-1160.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffffb0135660
Build id: 3.10.0-1160.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffb0135969
Build id: 3.10.0-1160.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffb0786c8f
Build id: 3.10.0-1160.el7.x86_64
|
0x3af
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [migration/7:45]
Modules linked in: nfnetlink_queue nfnetlink_log bluetooth rfkill xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink xt_set ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ip_set_hash_net ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev sb_edac coretemp iosf_mbi vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper joydev cryptd pcspkr sg vmw_vmci parport_pc parport
i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul ata_piix crct10dif_common crc32c_intel serio_raw libata mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 45 Comm: migration/7 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/30/2013
task: ffff907468b90000 ti: ffff907468b8c000 task.ti: ffff907468b8c000
RIP: 0010:[<ffffffffb01356aa>] [<ffffffffb01356aa>] multi_cpu_stop+0x4a/0x110
RSP: 0018:ffff907468b8fd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffffb00d6f6c RCX: dead000000000200
RDX: ffff9076f3d960b0 RSI: 0000000000000286 RDI: ffff907c5d847b30
RBP: ffff907468b8fdc0 R08: ffff907c5d847ad0 R09: 0000000000000001
R10: ffff9076f3d80000 R11: 0000000000001831 R12: 0000000000000000
R13: ffff9079ebc12904 R14: 0000000000000000 R15: ffff9073f3c1acc0
FS: 0000000000000000(0000) GS:ffff9076f3d80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f89636927b0 CR3: 00000003132fc000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb0135660>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffb0135969>] cpu_stopper_thread+0x99/0x150
[<ffffffffb0786c8f>] ? __schedule+0x3af/0x860
Complete report #1413790
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff8111850a
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffff811184c0
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811187b7
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x97
|
|
4
|
|
vmlinux |
0xffffffff816ab124
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x424
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810bac3f
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x12f
|
|
6
|
|
vmlinux |
0xffffffff810bab10
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x40
|
|
7
|
kthread
|
vmlinux |
0xffffffff810b252f
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xcf
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [migration/5:34]
Modules linked in: nf_conntrack_ftp ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 nf_log_ipv4 nf_log_common xt_LOG xt_recent xt_conntrack nf_conntrack fuse mpt2sas raid_class mptctl binfmt_misc dell_rbu iptable_filter sunrpc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio squashfs loop iTCO_wdt gpio_ich iTCO_vendor_support dcdbas intel_powerclamp coretemp kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_si pcspkr sg ipmi_devintf acpi_power_meter ipmi_msghandler wmi i7core_edac lpc_ich edac_core shpchp acpi_cpufreq ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptsas mptscsih drm ata_piix crct10dif_pclmul libata crct10dif_common mptbase crc32c_intel serio_raw i2c_core bnx2 scsi_transport_sas dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 34 Comm: migration/5 Tainted: G W I L ------------ 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R710/00NH4P, BIOS 6.4.0 07/23/2013
task: ffff88017640dee0 ti: ffff88017643c000 task.ti: ffff88017643c000
RIP: 0010:[<ffffffff8111850a>] [<ffffffff8111850a>] multi_cpu_stop+0x4a/0x100
RSP: 0000:ffff88017643fd98 EFLAGS: 00000246
RejectedIN=em3 OUT= MAC=d4:ae:52:7a:1a:46:00:09:0f:09:00:19:08:00 SRC=187.222.158.14 DST=10.5.14.229 LEN=40 TOS=0x00 PREC=0x00 TTL=44 ID=45295 DF PROTO=TCP SPT=59759 DPT=443 WINDOW=0 RES=0x00 RST URGP=0
RAX: 0000000000000001 RBX: 0000000000000000 RCX: dead000000000200
RDX: ffff880617a93fb0 RSI: 0000000000000286 RDI: ffff8805ffe0bb70
RBP: ffff88017643fdc0 R08: ffff8805ffe0bb10 R09: 0000000000000001
R10: 000000000000b80d R11: 0000000000000001 R12: ffffffff816b9bff
R13: ffffffff816b9c06 R14: ffffffff816b9c0d R15: ffffffff816b9c14
FS: 0000000000000000(0000) GS:ffff880617a80000(0000) knlGS:0000000000000000
RejectedIN=em3 OUT= MAC=d4:ae:52:7a:1a:46:00:09:0f:09:00:19:08:00 SRC=187.222.158.14 DST=10.5.14.229 LEN=40 TOS=0x00 PREC=0x00 TTL=44 ID=45296 DF PROTO=TCP SPT=59759 DPT=443 WINDOW=0 RES=0x00 RST URGP=0
RejectedIN=em3 OUT= MAC=d4:ae:52:7a:1a:46:00:09:0f:09:00:19:08:00 SRC=187.222.158.14 DST=10.5.14.229 LEN=40 TOS=0x00 PREC=0x00 TTL=44 ID=45297 DF PROTO=TCP SPT=59759 DPT=443 WINDOW=0 RES=0x00 RST URGP=0
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007febcfe10000 CR3: 00000006076de000 CR4: 00000000000207e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff811184c0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811187b7>] cpu_stopper_thread+0x97/0x150
[<ffffffff816ab124>] ? __schedule+0x424/0x9b0
[<ffffffff810bac3f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810bab10>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b252f>] kthread+0xcf/0xe0
Complete report #1322885
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff9c53d78a
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffff9c53d740
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff9c53da59
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff9cbb7920
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x320
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9c4d43fc
Build id: 3.10.0-1160.88.1.el7.x86_64
|
0x14c
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [migration/1:13]
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 nfit libnvdimm coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc vmw_vmci parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic vmwgfx drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi mptscsih libata mptbase crct10dif_pclmul crct10dif_common vmxnet3 crc32c_intel serio_raw drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 13 Comm: migration/1 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff91aab8c3d280 ti: ffff91aab8c5c000 task.ti: ffff91aab8c5c000
RIP: 0010:[<ffffffff9c53d78a>] [<ffffffff9c53d78a>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff91aab8c5fd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff91aab8c5fdc0 RCX: dead000000000200
RDX: ffff91aabbc960b0 RSI: 0000000000000286 RDI: ffff91abacb33b20
RBP: ffff91aab8c5fdc0 R08: ffff91abacb33ac0 R09: 0000000000000001
R10: ffff91aabbc80000 R11: ffff91abbb22ce00 R12: ffff91aab8c82c00
R13: ffffffff9c4e6658 R14: ffff91aab8c5fd10 R15: ffff91aabbd1acc0
FS: 0000000000000000(0000) GS:ffff91aabbc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000c156940 CR3: 0000000276766000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9c53d740>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff9c53da59>] cpu_stopper_thread+0x99/0x160
[<ffffffff9cbb7920>] ? __schedule+0x320/0x680
[<ffffffff9c4d43fc>] smpboot_thread_fn+0x14c/0x1b0
Complete report #1196309
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffb1b3d81a
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffffb1b3d7d0
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffb1b3dae9
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffb21b7920
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x320
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb1ad43fc
Build id: 3.10.0-1160.119.1.el7.x86_64
|
0x14c
|
NMI watchdog: BUG: soft lockup - CPU#68 stuck for 194s! [migration/68:350]
Modules linked in: AliSecGuard(OE) cmac arc4 md4 nls_utf8 cifs ccm dns_resolver tcp_diag inet_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter devlink ebtables ip6table_filter ip6_tables iptable_filter tun vmw_vsock_vmci_transport vsock sunrpc vfat fat ext4 mbcache jbd2 ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport tpm_crb vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci
ata_piix drm crct10dif_pclmul nfit crct10dif_common crc32c_intel libnvdimm libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: AliSecGuard]
CPU: 68 PID: 350 Comm: migration/68 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.17369862.B64.2012240522 12/24/2020
task: ffff88684cffb180 ti: ffff88684c808000 task.ti: ffff88684c808000
RIP: 0010:[<ffffffffb1b3d81a>] [<ffffffffb1b3d81a>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff88684c80bd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: 0000000000000000 RCX: dead000000000200
NMI watchdog: BUG: soft lockup - CPU#75 stuck for 194s! [migration/75:385]
RDX: ffff887cea7160b0 RSI: 0000000000000286 RDI: ffff887e1dec7b20
RBP: ffff88684c80bdc0 R08: ffff887e1dec7ac0 R09: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000001
R13: 0000000000000000 R14: 0000000000000082 R15: ffff887cea7c0000
FS: 0000000000000000(0000) GS:ffff887cea700000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000055b69d000 CR3: 000000066c810000 CR4: 00000000007607e0
Modules linked in: AliSecGuard(OE) cmac arc4 md4 nls_utf8 cifs ccm dns_resolver tcp_diag inet_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter devlink ebtables ip6table_filter ip6_tables iptable_filter tun vmw_vsock_vmci_transport vsock sunrpc vfat fat ext4 mbcache jbd2 ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport tpm_crb vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci
ata_piix drm crct10dif_pclmul nfit crct10dif_common crc32c_intel libnvdimm libata serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: AliSecGuard]
CPU: 75 PID: 385 Comm: migration/75 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.17369862.B64.2012240522 12/24/2020
task: ffff88684c8cb180 ti: ffff88684c8dc000 task.ti: ffff88684c8dc000
RIP: 0010:[<ffffffffb1b3d81a>] [<ffffffffb1b3d81a>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff88684c8dfd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffffb1adcf5c RCX: dead000000000200
RDX: ffff887cea8d60b0 RSI: 0000000000000286 RDI: ffff88691c5f7b20
RBP: ffff88684c8dfdc0 R08: ffff88691c5f7ac0 R09: 0000000000000001
R10: 0000000000000000 R11: 0000000000000010 R12: 0000000000000000
R13: ffff8868080fb984 R14: 0000000000000078 R15: ffff88922a01acc0
FS: 0000000000000000(0000) GS:ffff887cea8c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f05ea272e18 CR3: 000000066c810000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffb1b3d7d0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffb1b3dae9>] cpu_stopper_thread+0x99/0x160
[<ffffffffb21b7920>] ? __schedule+0x320/0x680
[<ffffffffb1ad43fc>] smpboot_thread_fn+0x14c/0x1b0
Complete report #1120079
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff98535cca
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffff98535c80
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff98535f89
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff984d4ca0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xe0
|
|
5
|
|
vmlinux |
0xffffffff98b8c050
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x320
|
|
6
|
smpboot_thread_fn
|
vmlinux |
0xffffffff984cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x144
|
|
7
|
|
vmlinux |
0xffffffff984ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [migration/18:100]
Modules linked in: vhost_net vhost macvtap macvlan xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter fuse openvswitch nf_conntrack_ipv6 nf_nat_ipv6 nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_defrag_ipv6 nf_nat nf_conntrack zram sunrpc ext4 mbcache jbd2 dell_smbios iTCO_wdt iTCO_vendor_support dell_wmi_descriptor dcdbas skx_edac intel_powerclamp coretemp intel_rapl ipmi_ssif iosf_mbi kvm_intel kvm irqbypass pcspkr sg i2c_i801 mei_me mei lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler tpm_crb acpi_power_meter acpi_pad ip_tables xfs libcrc32c dm_crypt drbg ansi_cprng sr_mod cdrom sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32_pclmul crc32c_intel ghash_clmulni_intel drm aesni_intel glue_helper lrw gf128mul ablk_helper ahci tg3 cryptd megaraid_sas libahci libata ptp pps_core drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 18 PID: 100 Comm: migration/18 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/04JN2K, BIOS 2.15.1 06/17/2022
task: ffff955934ab8000 ti: ffff955934aac000 task.ti: ffff955934aac000
RIP: 0010:[<ffffffff98535cca>] [<ffffffff98535cca>] multi_cpu_stop+0x4a/0x110
RSP: 0018:ffff955934aafd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff95602d89acc0 RCX: dead000000000200
RDX: ffff955c2e2560b0 RSI: 0000000000000286 RDI: ffff955de60fb750
RBP: ffff955934aafdc0 R08: ffff955de60fb6f0 R09: 0000000000000001
R10: 0000000000000000 R11: 0000000000000000 R12: ffff95602b4892c0
R13: ffff956000000000 R14: 0000000000000003 R15: ffff956000000000
FS: 0000000000000000(0000) GS:ffff955c2e240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa3bea10000 CR3: 00000006eee10000 CR4: 00000000007627e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 00000000
Call Trace:
[<ffffffff98535c80>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff98535f89>] cpu_stopper_thread+0x99/0x150
[<ffffffff984d4ca0>] ? finish_task_switch+0xe0/0x1c0
[<ffffffff98b8c050>] ? __schedule+0x320/0x680
[<ffffffff984cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff984ce8c0>] ? lg_double_unlock+0x40/0x40
Complete report #902345
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffb9135b7a
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffffb9135b30
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffb9135e39
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffb9788df0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x320
|
|
5
|
|
vmlinux |
0xffffffffb9788df0
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x320
|
|
6
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb90ce8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x144
|
|
7
|
|
vmlinux |
0xffffffffb90ce770
Build id: 3.10.0-1160.49.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [migration/9:55]
Modules linked in: veth ipt_rpfilter xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set xt_statistic xt_nat xt_recent ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc netfilter_edr(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel joydev aesni_intel sg parport_pc
veth ipt_rpfilter xt_set xt_multiport iptable_raw ip_set_hash_ip ip_set_hash_net ip_set xt_statistic xt_nat xt_recent ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc netfilter_edr(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac coretemp ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel joydev aesni_intel sg parport_pc lrw gf128mul glue_helper vmw_vmci ablk_helper parport
cryptd i2c_piix4 pcspkr ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect ata_piix sysimgblt fb_sys_fops mptspi ttm scsi_transport_spi crct10dif_pclmul drm crct10dif_common mptscsih libata e1000 serio_raw crc32c_intel mptbase vmxnet3 drm_panel_orientation_quirks floppy fuse
CPU: 11 PID: 65 Comm: migration/11 Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa10df6596300 ti: ffffa10df65b0000 task.ti: ffffa10df65b0000
RIP: 0010:[<ffffffffb9135b7a>] [<ffffffffb9135b7a>] multi_cpu_stop+0x4a/0x110
RSP: 0018:ffffa10df65b3d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffffb90d718c RCX: dead000000000200
RDX: ffffa1116fcd60b0 RSI: 0000000000000286 RDI: ffffa10c79b5f6e0
RBP: ffffa10df65b3dc0 R08: ffffa10c79b5f680 R09: 0000000000000001
R10: 000000000000b6f7 R11: 00000000000000c5 R12: 0000000000000000
R13: ffffa1196dc56b04 R14: 0000000000000014 R15: ffffa1156fd1acc0
FS: 0000000000000000(0000) GS:ffffa1116fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00044e000 CR3: 000000055c810000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb9135b30>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffb9135e39>] cpu_stopper_thread+0x99/0x150
[<ffffffffb9788df0>] ? __schedule+0x320/0x680
[<ffffffffb9788df0>] ? __schedule+0x320/0x680
[<ffffffffb90ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb90ce770>] ? lg_double_unlock+0x40/0x40
Complete report #640664
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff89d2fd4f
Build id: 3.10.0-957.el7.x86_64
|
0x4f
|
|
2
|
|
vmlinux |
0xffffffff89d2fd00
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff89d30009
Build id: 3.10.0-957.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff8a36770f
Build id: 3.10.0-957.el7.x86_64
|
0x3ff
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff89cca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
6
|
|
vmlinux |
0xffffffff89cca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
7
|
kthread
|
vmlinux |
0xffffffff89cc1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 39s! [migration/6:41]
Modules linked in: dattobd(OE) sfs(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr vmw_balloon joydev sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom
crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm mptspi scsi_transport_spi mptscsih ahci vmxnet3 libahci mptbase drm_panel_orientation_quirks ata_piix libata dm_mirror dm_region_hash dm_log dm_mod [last unloaded: dattobd]
CPU: 6 PID: 41 Comm: migration/6 Kdump: loaded Tainted: G W OEL ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9869aee55140 ti: ffff9869aee64000 task.ti: ffff9869aee64000
RIP: 0010:[<ffffffff89d2fd4f>] [<ffffffff89d2fd4f>] multi_cpu_stop+0x4f/0x110
RSP: 0000:ffff9869aee67d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff98669a6930c0 RCX: dead000000000200
RDX: ffff9869fc815ff0 RSI: 0000000000000282 RDI: ffff98672a8d7ab0
RBP: ffff9869aee67dc0 R08: ffff98672a8d7a80 R09: 0000000000000001
R10: 000000000000bb1e R11: 0000000000000001 R12: 0000000000000013
R13: ffff986ebfd1ab80 R14: 0000000000000013 R15: 0008000000000006
FS: 0000000000000000(0000) GS:ffff9869fc800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6c43e46d48 CR3: 0000000021e10000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 40s! [sdatalogd:305390]
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff89d2fd00>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff89d30009>] cpu_stopper_thread+0x99/0x150
[<ffffffff8a36770f>] ? __schedule+0x3ff/0x890
[<ffffffff89cca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff89cca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff89cc1c31>] kthread+0xd1/0xe0
Complete report #461142
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffa0b26e1e
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x8e
|
|
2
|
|
vmlinux |
0xffffffffa0b26d90
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffa0b27099
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffa11139fc
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x41c
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa0ac3dd4
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x144
|
|
6
|
|
vmlinux |
0xffffffffa0ac3c90
Build id: 3.10.0-862.9.1.el7.x86_64
|
0x40
|
|
7
|
kthread
|
vmlinux |
0xffffffffa0abb621
Build id: 3.10.0-862.9.1.el7.x86_64
|
0xd1
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 56s! [migration/11:65]
Modules linked in: nfnetlink_queue nfnetlink_log nfnetlink bluetooth rfkill fuse nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm joydev irqbypass dcdbas iTCO_wdt ipmi_si iTCO_vendor_support ipmi_devintf mxm_wmi sg pcspkr ipmi_msghandler lpc_ich wmi shpchp acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc sch_fq_codel ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci crc32_pclmul crc32c_intel ghash_clmulni_intel drm libahci aesni_intel
glue_helper libata lrw gf128mul tg3 ablk_helper cryptd megaraid_sas i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 65 Comm: migration/11 Tainted: G W L ------------ 3.10.0-862.9.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R630/02C2CP, BIOS 2.7.1 001/22/2018
task: ffff8c5a3949dee0 ti: ffff8c5a394d0000 task.ti: ffff8c5a394d0000
RIP: 0010:[<ffffffffa0b26e1e>] [<ffffffffa0b26e1e>] multi_cpu_stop+0x8e/0x110
RSP: 0000:ffff8c5a394d3d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff8c5a39009000 RCX: dead000000000200
RDX: ffff8c5dadf54070 RSI: 0000000000000286 RDI: ffff8c5da9b4bac0
RBP: ffff8c5a394d3dc0 R08: ffff8c5da9b4ba90 R09: 0000000000000001
R10: 000000000000b980 R11: 0000000000000001 R12: ffff8c5a394d3d20
R13: ffff8c5da80a5ee0 R14: ffffffffa0acc0a8 R15: ffff8c5a394d3d00
FS: 0000000000000000(0000) GS:ffff8c5dadf40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f24a400b040 CR3: 0000000588d1c000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa0b26d90>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffa0b27099>] cpu_stopper_thread+0x99/0x150
[<ffffffffa11139fc>] ? __schedule+0x41c/0xa20
[<ffffffffa0ac3dd4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa0ac3c90>] ? lg_double_unlock+0x40/0x40
[<ffffffffa0abb621>] kthread+0xd1/0xe0
Complete report #375014
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffb152fd4a
Build id: 3.10.0-957.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffffb152fd00
Build id: 3.10.0-957.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffb1530009
Build id: 3.10.0-957.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffb1b6770f
Build id: 3.10.0-957.el7.x86_64
|
0x3ff
|
|
5
|
?
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb1b5bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
6
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb14ca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
7
|
|
vmlinux |
0xffffffffb14ca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
8
|
kthread
|
vmlinux |
0xffffffffb14c1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
NMI watchdog: BUG: soft lockup - CPU#15 stuck for 23s! [migration/15:85]
Modules linked in: veth br_netfilter tun devlink overlay(T) ip_set nfnetlink bridge stp llc vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr nfit sg ppdev joydev parport_pc vmw_balloon parport libnvdimm i2c_piix4 vmw_vmci xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt crc32c_intel fb_sys_fops ttm serio_raw ata_piix drm libata mptspi scsi_transport_spi mptscsih vmxnet3 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 15 PID: 85 Comm: migration/15 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff883c386f5140 ti: ffff883c38710000 task.ti: ffff883c38710000
RIP: 0010:[<ffffffffb152fd4a>] [<ffffffffb152fd4a>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff883c38713d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff883877d6fc00 RCX: dead000000000200
RDX: ffff88403fdd5ff0 RSI: 0000000000000282 RDI: ffff884003e53b20
RBP: ffff883c38713dc0 R08: ffff884003e53af0 R09: 0000000000000001
R10: 000000000000bd18 R11: 0000000000000001 R12: ffff883c38713d20
R13: ffff883fbb645140 R14: ffffffffb14d2ea0 R15: ffff883fbb645140
FS: 0000000000000000(0000) GS:ffff88403fdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4328205a30 CR3: 00000007d7a10000 CR4: 00000000000607e0
Call Trace:
[<ffffffffb152fd00>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffb1530009>] cpu_stopper_thread+0x99/0x150
[<ffffffffb1b6770f>] ? __schedule+0x3ff/0x890
[<ffffffffb1b5bf4b>] ? queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb14ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb14ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffffb14c1c31>] kthread+0xd1/0xe0
Complete report #308468
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffffa933545a
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffffa9335410
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffffa9335719
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffffa99805a2
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x402
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffffa92cec24
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x144
|
|
6
|
|
vmlinux |
0xffffffffa92ceae0
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0x40
|
|
7
|
kthread
|
vmlinux |
0xffffffffa92c61f1
Build id: 3.10.0-1062.9.1.el7.x86_64
|
0xd1
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 28s! [migration/10:59]
Modules linked in: tcp_lp fuse veth nf_conntrack_netlink nfnetlink xt_statistic xt_nat xt_recent vport_vxlan vxlan ip6_udp_tunnel udp_tunnel openvswitch nf_conntrack_ipv6 nf_nat_ipv6 nf_defrag_ipv6 xt_comment xt_mark br_netfilter tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache scsi_transport_iscsi ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_CHECKSUM overlay(T) iptable_mangle vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr
sg i2c_piix4 vmw_vmci parport_pc parport target_core_user target_core_mod uio dm_multipath dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod syscopyarea crc_t10dif sysfillrect crct10dif_generic sysimgblt fb_sys_fops ahci ttm ata_piix libahci drm nfit crct10dif_pclmul crct10dif_common libata crc32c_intel libnvdimm serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 10 PID: 59 Comm: migration/10 Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/28/2017
task: ffff958f6bed20e0 ti: ffff958f6bee0000 task.ti: ffff958f6bee0000
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 28s! [migration/2:19]
RIP: 0010:[<ffffffffa933545a>] [<ffffffffa933545a>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff958f6bee3d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff95a35d74e2a0 RCX: dead000000000200
RDX: ffff959b89c96070 RSI: 0000000000000286 RDI: ffff95a89cb63aa0
RBP: ffff958f6bee3dc0 R08: ffff95a89cb63a40 R09: 0000000000000001
R10: 0000000000000000 R11: 0000000000000376 R12: 000000000000000c
R13: ffff959b89d1ac80 R14: 000000000000000c R15: 0008000089c9ac80
FS: 0000000000000000(0000) GS:ffff959b89c80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f370c5c72b8 CR3: 0000000499810000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 00000000
Call Trace:
[<ffffffffa9335410>] ? cpu_stop_should_run+0x50/0x50
[<ffffffffa9335719>] cpu_stopper_thread+0x99/0x150
[<ffffffffa99805a2>] ? __schedule+0x402/0x840
[<ffffffffa92cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa92ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa92c61f1>] kthread+0xd1/0xe0
Complete report #233534
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff9d3265ea
Build id: 3.10.0-862.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffff9d3265a0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff9d3268a9
Build id: 3.10.0-862.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff9d91291c
Build id: 3.10.0-862.el7.x86_64
|
0x41c
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff9d2c35e4
Build id: 3.10.0-862.el7.x86_64
|
0x144
|
|
6
|
|
vmlinux |
0xffffffff9d2c34a0
Build id: 3.10.0-862.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [migration/2:19]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport ip_set dummy ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter symap_custom_3_10_0_862_el7_x86_64(POE) devlink nf_conntrack_ipv4 nf_defrag_ipv4 symev_custom_3_10_0_862_el7_x86_64(OE) ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack overlay(T) vmw_vsock_vmci_transport vsock sunrpc xfs libcrc32c ppdev vmw_balloon pcspkr joydev sg vmw_vmci nfit i2c_piix4
shpchp libnvdimm parport_pc parport ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm ata_piix libahci libata vmxnet3 serio_raw i2c_core vmw_pvscsi floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: migration/2 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff93442953af70 ti: ffff93442954c000 task.ti: ffff93442954c000
RIP: 0010:[<ffffffff9d3265ea>] [<ffffffff9d3265ea>] multi_cpu_stop+0x4a/0x110
RSP: 0000:ffff93442954fd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff9352b3693f40 RCX: dead000000000200
RDX: ffff9352bfa94030 RSI: 0000000000000282 RDI: ffff9350aa283ac0
RBP: ffff93442954fdc0 R08: ffff9350aa283a60 R09: 0000000000000001
R10: 000000000000bbca R11: 0000000000000001 R12: ffffffff9d2c8ab7
R13: ffff93442954fd20 R14: 0000000000000000 R15: ffff9353885ef6b4
FS: 0000000000000000(0000) GS:ffff9352bfa80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f06322256f8 CR3: 00000018c440e000 CR4: 00000000000007e0
Call Trace:
[<ffffffff9d3265a0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff9d3268a9>] cpu_stopper_thread+0x99/0x150
[<ffffffff9d91291c>] ? __schedule+0x41c/0xa20
[<ffffffff9d2c35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9d2c34a0>] ? lg_double_unlock+0x40/0x40
Complete report #32637
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff903298ba
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x4a
|
|
2
|
|
vmlinux |
0xffffffff90329870
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff90329b79
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff9091890c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x41c
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff902c6774
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x144
|
|
6
|
|
vmlinux |
0xffffffff902c6630
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#15 stuck for 22s! [migration/15:85]
Modules linked in: vhost_net vhost macvtap macvlan ebt_arp tun nfsv3 nfs_acl nfs lockd grace fscache bnx2fc cnic uio fcoe libfcoe libfc scsi_transport_fc ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm dm_mirror dm_region_hash dm_log rpcrdma sunrpc rdma_ucm sb_edac ib_uverbs intel_powerclamp ib_iser coretemp rdma_cm iw_cm intel_rapl
ib_umad libiscsi ib_ipoib iosf_mbi scsi_transport_iscsi ib_cm kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul mlx4_ib ipmi_si dm_service_time glue_helper iTCO_wdt ablk_helper ipmi_devintf ib_core iTCO_vendor_support ioatdma sg hpilo cryptd hpwdt joydev pcspkr acpi_power_meter ipmi_msghandler wmi dca shpchp lpc_ich pcc_cpufreq dm_multipath dm_mod ip_tables xfs libcrc32c raid1 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit mlx4_en ptp drm_kms_helper pps_core syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm mlx4_core libahci crct10dif_pclmul crct10dif_common mpt2sas libata crc32c_intel serio_raw i2c_core raid_class devlink scsi_transport_sas 8021q garp mrp bridge stp llc bonding
CPU: 15 PID: 85 Comm: migration/15 Kdump: loaded Tainted: G L ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: HP ProLiant SL230s Gen8 /, BIOS P75 05/24/2019
task: ffff8eab6972cf10 ti: ffff8eab6975c000 task.ti: ffff8eab6975c000
RIP: 0010:[<ffffffff903298ba>] [<ffffffff903298ba>] multi_cpu_stop+0x4a/0x110
RSP: 0018:ffff8eab6975fd98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffff8eab69148400 RCX: dead000000000200
RDX: ffff8ebabefd4070 RSI: 0000000000000286 RDI: ffff8eaa95ec7660
RBP: ffff8eab6975fdc0 R08: ffff8eaa95ec7630 R09: 0000000000000001
R10: 000000000000b833 R11: 0000000000000001 R12: ffff8eab6975fd20
R13: ffff8ebabbe88000 R14: ffffffff902cea48 R15: ffff8eab6975fd00
FS: 0000000000000000(0000) GS:ffff8ebabefc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f613e21e550 CR3: 0000000f6426a000 CR4: 00000000000627e0
Call Trace:
[<ffffffff90329870>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff90329b79>] cpu_stopper_thread+0x99/0x150
[<ffffffff9091890c>] ? __schedule+0x41c/0xa20
[<ffffffff902c6774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff902c6630>] ? lg_double_unlock+0x40/0x40
Complete report #12877
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff895265ef
Build id: 3.10.0-862.el7.x86_64
|
0x4f
|
|
2
|
|
vmlinux |
0xffffffff895265a0
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff895268a9
Build id: 3.10.0-862.el7.x86_64
|
0x99
|
|
4
|
|
vmlinux |
0xffffffff89b1291c
Build id: 3.10.0-862.el7.x86_64
|
0x41c
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff894c35e4
Build id: 3.10.0-862.el7.x86_64
|
0x144
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [migration/5:34]
Modules linked in: veth vxlan ip6_udp_tunnel udp_tunnel xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipport ip_set_hash_ipportip ip_set dummy ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc symap_custom_3_10_0_862_el7_x86_64(POE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink nf_conntrack_ipv4 nf_defrag_ipv4 symev_custom_3_10_0_862_el7_x86_64(OE) ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack overlay(T) vmw_vsock_vmci_transport vsock sunrpc xfs libcrc32c ppdev vmw_balloon joydev pcspkr sg parport_pc nfit parport
libnvdimm vmw_vmci shpchp i2c_piix4 ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci ata_piix libahci vmxnet3 libata serio_raw i2c_core vmw_pvscsi floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 5 PID: 34 Comm: migration/5 Kdump: loaded Tainted: P OEL ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffffa10f295cbf40 ti: ffffa10f295e0000 task.ti: ffffa10f295e0000
RIP: 0010:[<ffffffff895265ef>] [<ffffffff895265ef>] multi_cpu_stop+0x4f/0x110
RSP: 0000:ffffa10f295e3d98 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffa12dbf318b00 RCX: dead000000000200
RDX: ffffa11dbfb54030 RSI: 0000000000000282 RDI: ffffa10dcee77ac0
RBP: ffffa10f295e3dc0 R08: ffffa10dcee77a60 R09: 0000000000000001
R10: 000000000000b766 R11: 0000000000000001 R12: ffffffff00000000
R13: ffffa10f295e3d18 R14: ffffa10f29759800 R15: 0000000200000005
FS: 0000000000000000(0000) GS:ffffa11dbfb40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f9ab08f7c80 CR3: 0000000ad9e0e000 CR4: 00000000000007e0
Call Trace:
[<ffffffff895265a0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff895268a9>] cpu_stopper_thread+0x99/0x150
[<ffffffff89b1291c>] ? __schedule+0x41c/0xa20
[<ffffffff894c35e4>] smpboot_thread_fn+0x144/0x1a0