Complete report #415026
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031f3
Build id: 3.10.0-327.el7.x86_64
|
0x83
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a2b8
Build id: 3.10.0-327.el7.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#0 stuck for 22s! [migration/0:8]
Modules linked in: nvidia_uvm(OE) binfmt_misc parastor(OE) dlm(OE) kfnccli(OE) knal_without_rdma(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx4_en(OE) vxlan ip6_udp_tunnel udp_tunnel mlx4_ib(OE) mlx4_core(OE) dm_mirror dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) intel_powerclamp coretemp intel_rapl kvm crc32_pclmul ghash_clmulni_intel snd_hda_codec_realtek aesni_intel snd_hda_codec_generic lrw gf128mul glue_helper ablk_helper cryptd pcspkr snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep sb_edac snd_seq edac_core snd_seq_device snd_pcm i2c_i801 snd_timer lpc_ich mei_me snd mfd_core mei soundcore sg shpchp
ipmi_ssif ipmi_si ipmi_msghandler acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd knem(OE) grace sunrpc ip_tables xfs libcrc32c mlx5_ib(OE) ib_core(OE) raid1 sd_mod crc_t10dif crct10dif_generic mxm_wmi ast syscopyarea crct10dif_pclmul sysfillrect crct10dif_common sysimgblt crc32c_intel drm_kms_helper igb ttm dca i2c_algo_bit drm ahci libahci i2c_core libata mlx5_core(OE) mlx_compat(OE) ptp pps_core wmi
CPU: 0 PID: 8 Comm: migration/0 Tainted: P OEL ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: Inspur NF5568M4/NF5568M4, BIOS 2.0 12/31/2015
task: ffff883f2767d080 ti: ffff883f2701c000 task.ti: ffff883f2701c000
RIP: 0010:[<ffffffff811031f3>] [<ffffffff811031f3>] multi_cpu_stop+0x83/0xf0
RSP: 0000:ffff883f2701fd90 EFLAGS: 00000293
RAX: ffffffff81660fe0 RBX: ffffffff00000000 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000286 RDI: ffff883f04edfb90
RBP: ffff883f2701fdb0 R08: 0000000000000000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000002 R12: 0000000000014780
R13: 0000000000014780 R14: ffffffff810bb685 R15: ffff883f26e8fc00
FS: 0000000000000000(0000) GS:ffff883f7fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffe86e86b98 CR3: 0000007f205a7000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff883f04edfbb8 ffff883f7fc0dd00 ffff883f04edfb90 ffffffff81103170
ffff883f2701fe78 ffffffff811033f8 ffff883f7fc0dd08 0000000000000000
0000000000000000 ffff883f7fc14780 ffff887f22f5f6c0 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a2b8>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 83 fa 04 74 23 f3 90 44 8b 63 20 <41> 39 d4 74 f0 41 83 fc 02 75 c2 fa 66 0f 1f 44 00 00 eb c4 66
Complete report #159351
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031e8
Build id: 3.10.0-327.el7.x86_64
|
0x78
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a2b8
Build id: 3.10.0-327.el7.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#31 stuck for 22s! [migration/31:228]
Modules linked in: rpcsec_gss_krb5 fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bonding vfat fat ipmi_ssif ext4 mbcache jbd2 iTCO_wdt iTCO_vendor_support mxm_wmi ipmi_devintf cdc_ether usbnet mii dm_service_time intel_powerclamp coretemp intel_rapl kvm_intel kvm crc32_pclmul ghash_clmulni_intel pcspkr aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sb_edac edac_core sg i2c_i801 mei_me lpc_ich mei mfd_core shpchp wmi ipmi_si ipmi_msghandler acpi_power_meter acpi_pad nfsd dm_multipath auth_rpcgss nfs_acl lockd grace sunrpc tcp_hybla binfmt_misc ip_tables xfs libcrc32c sd_mod
sr_mod cdrom mgag200 syscopyarea sysfillrect sysimgblt i2c_algo_bit lpfc drm_kms_helper crc32c_intel ttm ahci libahci drm crc_t10dif crct10dif_generic tg3 libata crct10dif_pclmul i2c_core ptp scsi_transport_fc pps_core megaraid_sas scsi_tgt crct10dif_common dm_mirror dm_region_hash dm_log dm_mod
CPU: 31 PID: 228 Comm: migration/31 Tainted: G W L ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: LENOVO System x3650 M5: -[8871AC1]-/01KN176, BIOS -[TCE138G-2.81]- 11/28/2018
task: ffff88086910d080 ti: ffff880869170000 task.ti: ffff880869170000
RIP: 0010:[<ffffffff811031e8>] [<ffffffff811031e8>] multi_cpu_stop+0x78/0xf0
RSP: 0000:ffff880869173d90 EFLAGS: 00000246
RAX: ffffffff8165f1e0 RBX: 0000000000000000 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000286 RDI: ffff880840983b90
RBP: ffff880869173db0 R08: 0000000000000000 R09: 00000000000000b5
R10: 0000000000000001 R11: 0000000000000002 R12: 0000000000014780
R13: 0000000000014780 R14: 0000000000014780 R15: ffff8808691cb000
FS: 0000000000000000(0000) GS:ffff88087fde0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0098cdf648 CR3: 000000000194a000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880840983bb8 ffff88087fdedd00 ffff880840983b90 ffffffff81103170
ffff880869173e78 ffffffff811033f8 ffff88087fdedd08 0000000000000000
0000000000000000 ffff88087fdf4780 ffff880466d15780 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a2b8>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 00 00 00 41 83 fc 03 75 05 45 84 ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 <83> fa 04 74 23 f3 90 44 8b 63 20 41 39 d4 74 f0 41 83 fc 02 75
Complete report #36581
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031f3
Build id: 3.10.0-327.el7.x86_64
|
0x83
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a2b8
Build id: 3.10.0-327.el7.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#6 stuck for 33s! [migration/6:86]
Modules linked in: tcp_diag inet_diag coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr shpchp i2c_piix4 sg vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic vmwgfx drm_kms_helper ttm crct10dif_pclmul pata_acpi crct10dif_common crc32c_intel mptspi serio_raw scsi_transport_spi drm ata_piix mptscsih libata e1000 mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 86 Comm: migration/6 Tainted: G L ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/30/2014
task: ffff880139da2e00 ti: ffff880139de8000 task.ti: ffff880139de8000
RIP: 0010:[<ffffffff811031f3>] [<ffffffff811031f3>] multi_cpu_stop+0x83/0xf0
RSP: 0000:ffff880139debd90 EFLAGS: 00000293
RAX: ffffffff81662160 RBX: ffffffff00000000 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000286 RDI: ffff88013a337b90
RBP: ffff880139debdb0 R08: 0000000000000000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000002 R12: 0000000000014780
R13: 0000000000014780 R14: ffffffff810bb685 R15: ffff880139f42400
FS: 0000000000000000(0000) GS:ffff88013bcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fce82670e80 CR3: 0000000137566000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88013a337bb8 ffff88013bccdd00 ffff88013a337b90 ffffffff81103170
ffff880139debe78 ffffffff811033f8 ffff88013bccdd08 0000000000000000
0000000000000000 ffff88013bcd4780 ffff880137ddf6c0 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a2b8>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 83 fa 04 74 23 f3 90 44 8b 63 20 <41> 39 d4 74 f0 41 83 fc 02 75 c2 fa 66 0f 1f 44 00 00 eb c4 66
Complete report #23886
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031f3
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x83
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a468
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163aab9
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645a58
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x140
|
BUG: soft lockup - CPU#4 stuck for 22s! [migration/4:93]
Modules linked in: dccp_diag dccp tcp_diag udp_diag inet_diag unix_diag osc(OE) mgc(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) usb_storage 8021q garp mrp mpt3sas mpt2sas raid_class scsi_transport_sas mptctl mptbase dell_rbu ksocklnd(OE) lnet(OE) sha512_generic crypto_null libcfs(OE) xt_CHECKSUM tun ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT bonding ipt_REJECT xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter vfat fat coretemp intel_rapl
kvm_intel kvm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ext4 mbcache jbd2 iTCO_wdt pcspkr iTCO_vendor_support sg dcdbas ipmi_devintf ipmi_si ipmi_msghandler wmi acpi_power_meter lpc_ich ntb sb_edac edac_core mei_me mei mfd_core shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom mgag200 syscopyarea sysfillrect sysimgblt drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel ttm ahci libahci drm ixgbe libata igb tg3 mdio dca megaraid_sas i2c_algo_bit ptp i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 93 Comm: migration/4 Tainted: G W OEL ------------ 3.10.0-327.3.1.el7_lustre.x86_64 #1
Hardware name: Dell Inc. PowerEdge R720/0C4Y3R, BIOS 1.6.0 03/07/2013
task: ffff880429a86780 ti: ffff880429ad4000 task.ti: ffff880429ad4000
RIP: 0010:[<ffffffff811031f3>] [<ffffffff811031f3>] multi_cpu_stop+0x83/0xf0
RSP: 0000:ffff880429ad7d90 EFLAGS: 00000293
RAX: ffffffff81660da0 RBX: 0000000000000000 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000286 RDI: ffff88041e03bb90
RBP: ffff880429ad7db0 R08: 0000000000000000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000002 R12: 0000000000014780
R13: 0000000000014780 R14: 0000000000014780 R15: ffff880429660800
FS: 0000000000000000(0000) GS:ffff88042fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001df9360 CR3: 0000000428bd9000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88041e03bbb8 ffff88042fc4dd00 ffff88041e03bb90 ffffffff81103170
ffff880429ad7e78 ffffffff811033f8 ffff88042fc4dd08 0000000000000000
0000000000000000 ffff88042fc54780 ffff880426a2b200 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a468>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163aab9>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645a58>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 83 fa 04 74 23 f3 90 44 8b 63 20 <41> 39 d4 74 f0 41 83 fc 02 75 c2 fa 66 66 90 66 66 90 eb c4 66
Complete report #19117
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031f3
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x83
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a468
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163aab9
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645a58
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.3.1.el7_lustre.x86_64
|
0x140
|
BUG: soft lockup - CPU#2 stuck for 22s! [migration/2:83]
Modules linked in: tcp_lp rpcsec_gss_krb5 usb_storage bnep bluetooth rfkill fuse osc(OE) mgc(OE) lustre(OE) lmv(OE) mdc(OE) lov(OE) fid(OE) fld(OE) ptlrpc(OE) obdclass(OE) 8021q garp mrp mpt3sas mpt2sas raid_class scsi_transport_sas mptctl mptbase dell_rbu ksocklnd(OE) lnet(OE) sha512_generic crypto_null libcfs(OE) xt_CHECKSUM tun ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT bonding ipt_REJECT xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter vfat fat coretemp intel_rapl kvm_intel
kvm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ipmi_devintf cryptd ext4 mbcache jbd2 sg ipmi_si ipmi_msghandler pcspkr dcdbas iTCO_wdt acpi_power_meter iTCO_vendor_support wmi mei_me lpc_ich mei mfd_core ntb shpchp sb_edac edac_core nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic mgag200 syscopyarea sysfillrect sysimgblt drm_kms_helper crct10dif_pclmul ttm crct10dif_common crc32c_intel ahci drm libahci ixgbe libata igb tg3 mdio dca i2c_algo_bit megaraid_sas ptp i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 83 Comm: migration/2 Tainted: G OE ------------ 3.10.0-327.3.1.el7_lustre.x86_64 #1
Hardware name: Dell Inc. PowerEdge R720/0C4Y3R, BIOS 1.6.0 03/07/2013
task: ffff88042999e780 ti: ffff880429adc000 task.ti: ffff880429adc000
RIP: 0010:[<ffffffff811031f3>] [<ffffffff811031f3>] multi_cpu_stop+0x83/0xf0
RSP: 0000:ffff880429adfd90 EFLAGS: 00000293
RAX: ffffffff816603a0 RBX: ffff880429adfdb0 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000286 RDI: ffff88038fc43b90
RBP: ffff880429adfdb0 R08: 0000000000000000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000002 R12: ffff88042fc34780
R13: ffffffff810bd16b R14: ffff880429adfd10 R15: ffff880371d8d080
FS: 0000000000000000(0000) GS:ffff88042fc20000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fef87676f58 CR3: 000000042236d000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88038fc43bb8 ffff88042fc2dd00 ffff88038fc43b90 ffffffff81103170
ffff880429adfe78 ffffffff811033f8 ffff88042fc2dd08 0000000000000000
0000000000000000 ffff88042fc34780 ffff8804288c6a40 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a468>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163aab9>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645a58>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 83 fa 04 74 23 f3 90 44 8b 63 20 <41> 39 d4 74 f0 41 83 fc 02 75 c2 fa 66 66 90 66 66 90 eb c4 66
Complete report #15122
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff811031ef
Build id: 3.10.0-327.el7.x86_64
|
0x7f
|
|
2
|
|
vmlinux |
0xffffffff81103170
Build id: 3.10.0-327.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff811033f8
Build id: 3.10.0-327.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff8163a2b8
Build id: 3.10.0-327.el7.x86_64
|
0x2d8
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff810ae52f
Build id: 3.10.0-327.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8163a909
Build id: 3.10.0-327.el7.x86_64
|
0x29
|
|
7
|
|
vmlinux |
0xffffffff810ae430
Build id: 3.10.0-327.el7.x86_64
|
0x90
|
|
8
|
kthread
|
vmlinux |
0xffffffff810a5aef
Build id: 3.10.0-327.el7.x86_64
|
0xcf
|
|
9
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
|
10
|
ret_from_fork
|
vmlinux |
0xffffffff81645858
Build id: 3.10.0-327.el7.x86_64
|
0x58
|
|
11
|
|
vmlinux |
0xffffffff810a5a20
Build id: 3.10.0-327.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#3 stuck for 22s! [migration/3:85]
Modules linked in: xt_set xt_multiport ip_set_hash_ip ip_set nfnetlink rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ip6t_REJECT ipt_REJECT xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw iptable_filter nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) ext4 mbcache jbd2 sg ipmi_msghandler snd_hda_intel snd_hda_codec ppdev virtio_balloon snd_hda_core snd_hwdep i2c_i801 lpc_ich mfd_core snd_seq snd_seq_device snd_pcm snd_timer
snd soundcore shpchp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc dm_multipath ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif cdrom crct10dif_generic virtio_blk cirrus syscopyarea sysfillrect sysimgblt drm_kms_helper ahci ttm libahci crct10dif_pclmul crct10dif_common virtio_pci virtio_ring crc32c_intel libata e1000 drm serio_raw virtio i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 85 Comm: migration/3 Tainted: P W OE ------------ 3.10.0-327.el7.x86_64 #1
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.8.2-0-g33fbe13 by qemu-project.org 04/01/2014
task: ffff8817fb07ae00 ti: ffff8817fb098000 task.ti: ffff8817fb098000
RIP: 0010:[<ffffffff811031ef>] [<ffffffff811031ef>] multi_cpu_stop+0x7f/0xf0
RSP: 0000:ffff8817fb09bd90 EFLAGS: 00000293
RAX: ffffffff81662b60 RBX: ffffffff00000000 RCX: dead000000200200
RDX: 0000000000000001 RSI: 00000000d248d246 RDI: ffff882ff0e43b70
RBP: ffff8817fb09bdb0 R08: 0000000000000000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000002 R12: 0000000000014780
R13: 0000000000014780 R14: ffffffff810bb685 R15: ffff8817fb329800
FS: 0000000000000000(0000) GS:ffff88181fc60000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000026ec000 CR3: 000000000194a000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff882ff0e43b98 ffff88181fc6dd00 ffff882ff0e43b70 ffffffff81103170
ffff8817fb09be78 ffffffff811033f8 ffff88181fc6dd08 0000000000000000
0000000000000000 ffff88181fc74780 ffff882ffa0aabc0 0000000000000000
Call Trace:
[<ffffffff81103170>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff811033f8>] cpu_stopper_thread+0x88/0x160
[<ffffffff8163a2b8>] ? __schedule+0x2d8/0x900
[<ffffffff810ae52f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8163a909>] ? schedule+0x29/0x70
[<ffffffff810ae430>] ? lg_double_unlock+0x90/0x90
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 75 05 45 84 ed 75 65 f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 0f 8b 43 20 8b 73 10 8d 48 01 89 73 24 89 4b 20 83 fa 04 74 23 f3 90 <44> 8b 63 20 41 39 d4 74 f0 41 83 fc 02 75 c2 fa 66 0f 1f 44 00
Complete report #23464
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff810deb62
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x82
|
|
2
|
|
vmlinux |
0xffffffff810deae0
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff810ded68
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff815e7db5
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x2c5
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8108e30f
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8108e210
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xc0
|
|
7
|
kthread
|
vmlinux |
0xffffffff81085aff
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xcf
|
|
8
|
|
vmlinux |
0xffffffff81085a30
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x140
|
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff815f316c
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x7c
|
|
10
|
|
vmlinux |
0xffffffff81085a30
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#2 stuck for 22s! [migration/2:66]
Modules linked in: iptable_mangle iptable_nat nf_nat_ipv4 nf_nat iptable_filter ip_tables tcp_diag udp_diag inet_diag vfat fat isofs rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache mptctl mptbase ipmi_devintf xt_LOG xt_limit nf_conntrack_ipv4 nf_defrag_ipv4 xt_multiport xt_conntrack nf_conntrack sg coretemp kvm_intel kvm crct10dif_pclmul crc32_pclmul crc32c_intel dm_mirror ghash_clmulni_intel iTCO_wdt dm_region_hash aesni_intel dm_log dm_mod iTCO_vendor_support lrw gf128mul glue_helper ablk_helper cryptd ipmi_si i7core_edac ipmi_msghandler serio_raw pcspkr edac_core lpc_ich shpchp mfd_core acpi_cpufreq mperf nfsd auth_rpcgss nfs_acl lockd sunrpc binfmt_misc ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_common mgag200 syscopyarea sysfillrect sysimgblt drm_kms_helper ahci ttm igb libahci
drm libata ptp pps_core dca i2c_algo_bit hpsa i2c_core [last unloaded: ip_tables]
CPU: 2 PID: 66 Comm: migration/2 Tainted: G I -------------- 3.10.0-123.20.1.el7.x86_64 #1
Hardware name: HP ProLiant DL180 G6 , BIOS O20 09/01/2011
task: ffff880429f40000 ti: ffff880429f48000 task.ti: ffff880429f48000
RIP: 0010:[<ffffffff810deb62>] [<ffffffff810deb62>] multi_cpu_stop+0x82/0xf0
RSP: 0000:ffff880429f49d88 EFLAGS: 00000293
RAX: ffffffff81604b80 RBX: ffff88042fcd35c0 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff8804b2b45ac0
RBP: ffff880429f49da8 R08: ffff8804b2b45a90 R09: 0000000000000000
R10: 000000000000b9a6 R11: 00000000000003e5 R12: ffffffff8109ba2b
R13: ffff880429f49d08 R14: ffff88042815db00 R15: ffffffff81094ea5
FS: 0000000000000000(0000) GS:ffff88042fc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fe77dd1d728 CR3: 00000000018d0000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8804b2b45ae8 ffff88042fc4db20 ffff8804b2b45ac0 ffffffff810deae0
ffff880429f49e70 ffffffff810ded68 ffff88042fc4db28 0000000000000000
0000000000000000 ffff88042fc535c0 ffff8802266ae400 0000000000000000
Call Trace:
[<ffffffff810deae0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff810ded68>] cpu_stopper_thread+0x88/0x160
[<ffffffff815e7db5>] ? __schedule+0x2c5/0x790
[<ffffffff8108e30f>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8108e210>] ? lg_global_unlock+0xc0/0xc0
[<ffffffff81085aff>] kthread+0xcf/0xe0
[<ffffffff81085a30>] ? kthread_create_on_node+0x140/0x140
[<ffffffff815f316c>] ret_from_fork+0x7c/0xb0
[<ffffffff81085a30>] ? kthread_create_on_node+0x140/0x140
Code: 84 ed 75 5d f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 12 8b 43 20 8b 4b 10 8d 50 01 89 4b 24 89 53 20 44 89 e2 83 fa 04 74 20 f3 90 <44> 8b 63 20 41 39 d4 74 f0 41 83 fc 02 75 bf fa 66 66 90 66 66
Complete report #1762
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
multi_cpu_stop
|
vmlinux |
0xffffffff810f2952
Build id: 3.10.0-229.el7.x86_64
|
0x82
|
|
2
|
|
vmlinux |
0xffffffff810f28d0
Build id: 3.10.0-229.el7.x86_64
|
0x50
|
|
3
|
cpu_stopper_thread
|
vmlinux |
0xffffffff810f2b58
Build id: 3.10.0-229.el7.x86_64
|
0x88
|
|
4
|
|
vmlinux |
0xffffffff81609925
Build id: 3.10.0-229.el7.x86_64
|
0x2c5
|
|
5
|
smpboot_thread_fn
|
vmlinux |
0xffffffff8109fdaf
Build id: 3.10.0-229.el7.x86_64
|
0xff
|
|
6
|
|
vmlinux |
0xffffffff8109fcb0
Build id: 3.10.0-229.el7.x86_64
|
0xc0
|
|
7
|
kthread
|
vmlinux |
0xffffffff8109739f
Build id: 3.10.0-229.el7.x86_64
|
0xcf
|
|
8
|
|
vmlinux |
0xffffffff810972d0
Build id: 3.10.0-229.el7.x86_64
|
0x140
|
|
9
|
ret_from_fork
|
vmlinux |
0xffffffff8161497c
Build id: 3.10.0-229.el7.x86_64
|
0x7c
|
|
10
|
|
vmlinux |
0xffffffff810972d0
Build id: 3.10.0-229.el7.x86_64
|
0x140
|
BUG: soft lockup - CPU#8 stuck for 24s! [migration/8:83]
Modules linked in: binfmt_misc coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr vmw_vmci sg i2c_piix4 shpchp parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix mptspi ttm scsi_transport_spi crct10dif_pclmul crct10dif_common crc32c_intel drm serio_raw mptscsih libata i2c_core mptbase vmxnet3 floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 8 PID: 83 Comm: migration/8 Not tainted 3.10.0-229.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff880cb1858b60 ti: ffff880cb1864000 task.ti: ffff880cb1864000
RIP: 0010:[<ffffffff810f2952>] [<ffffffff810f2952>] multi_cpu_stop+0x82/0xf0
RSP: 0000:ffff880cb1867d88 EFLAGS: 00000293
RAX: ffffffff81632ca0 RBX: ffff88073f725b00 RCX: dead000000200200
RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff880cb0c5baf0
RBP: ffff880cb1867da8 R08: ffff880cb0c5bac0 R09: 0000000000000000
R10: 000000000000b848 R11: 0000000000000012 R12: ffffffff810a4e7c
R13: ffff880cb1867d10 R14: ffff880cb1867fd8 R15: 00000000fffffff5
FS: 0000000000000000(0000) GS:ffff880cb5c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f3b117ee8f0 CR3: 000000000190a000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880cb0c5bb18 ffff880cb5c0db80 ffff880cb0c5baf0 ffffffff810f28d0
ffff880cb1867e70 ffffffff810f2b58 ffff880cb5c0db88 0000000000000000
0000000000000000 ffff880cb5c13680 ffff8802b086ea40 0000000000000000
Call Trace:
[<ffffffff810f28d0>] ? cpu_stop_should_run+0x50/0x50
[<ffffffff810f2b58>] cpu_stopper_thread+0x88/0x160
[<ffffffff81609925>] ? __schedule+0x2c5/0x7b0
[<ffffffff8109fdaf>] smpboot_thread_fn+0xff/0x1a0
[<ffffffff8109fcb0>] ? lg_global_unlock+0xc0/0xc0
[<ffffffff8109739f>] kthread+0xcf/0xe0
[<ffffffff810972d0>] ? kthread_create_on_node+0x140/0x140
[<ffffffff8161497c>] ret_from_fork+0x7c/0xb0
[<ffffffff810972d0>] ? kthread_create_on_node+0x140/0x140
Code: 84 ed 75 5d f0 ff 4b 24 0f 94 c1 84 c9 44 89 e2 74 12 8b 43 20 8b 4b 10 8d 50 01 89 4b 24 89 53 20 44 89 e2 83 fa 04 74 20 f3 90 <44> 8b 63 20 41 39 d4 74 f0 41 83 fc 02 75 bf fa 66 0f 1f 44 00