Complete report #1018251
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffffb7d1db82
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x102
|
|
2
|
|
vmlinux |
0xffffffffb7db21e0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffffb7db21e0
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffffb7d1dc9f
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffffb7db163c
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffffb7db7c30
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1b0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffffb7db21c4
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffffb7d327e5
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x1b5
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffffb7d34540
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffffb7d34596
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffffb7d3246f
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x12f
|
|
12
|
|
vmlinux |
0xffffffffb7e5ecf8
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffffb7f1b257
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x27
|
|
14
|
vfs_write
|
vmlinux |
0xffffffffb7e5bd30
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0xc0
|
|
15
|
SyS_write
|
vmlinux |
0xffffffffb7e5cae5
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x55
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffffb83c539a
Build id: 3.10.0-1160.108.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [runc:22337]
Modules linked in: xt_owner xt_REDIRECT nf_nat_redirect xt_multiport ipt_rpfilter iptable_raw ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel rpcsec_gss_krb5 veth auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xt_set ip_set_hash_ipportnet ip_set_bitmap_port ip_set_hash_ipportip ip_set_hash_ipport dummy nf_tables ip_set ip_vs_sh ip_vs_wrr ip_vs_rr ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6_tables xt_mark xt_comment iptable_mangle xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter scini(POE) ip_vs nf_conntrack binfmt_misc vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon
sg parport_pc joydev parport pcspkr vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi scsi_transport_spi mptscsih drm libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod br_netfilter bridge stp llc overlay(T)
CPU: 5 PID: 22337 Comm: runc Tainted: P W OEL ------------ T 3.10.0-1160.108.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffffa05f0aeb2100 ti: ffffa05e00b34000 task.ti: ffffa05e00b34000
RIP: 0010:[<ffffffffb7d1db82>] [<ffffffffb7d1db82>] generic_exec_single+0x102/0x1c0
RSP: 0018:ffffa05e00b37c30 EFLAGS: 00000202
RAX: 000000000000000c RBX: ffffa05e00b37bc0 RCX: 0000000000000034
RDX: 0000000000000fff RSI: 000000000000000c RDI: 0000000000000286
RBP: ffffa05e00b37c80 R08: ffffffffb8414140 R09: ffffffffb7d30f73
R10: ffffa05efbc9f0a0 R11: fffff0f30515ef40 R12: 0000000000000010
R13: ffffffffb7c67006 R14: ffffffffffffff10 R15: 0000000000000286
FS: 00007f4a1704f700(0000) GS:ffffa05efbc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562e6c772e50 CR3: 0000000198270000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb7db21e0>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffb7db21e0>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffb7d1dc9f>] smp_call_function_single+0x5f/0xa0
[<ffffffffb7db163c>] task_function_call+0x4c/0x80
[<ffffffffb7db7c30>] ? perf_cgroup_switch+0x1b0/0x1b0
[<ffffffffb7db21c4>] perf_cgroup_attach+0x44/0x60
[<ffffffffb7d327e5>] cgroup_attach_task+0x1b5/0x410
[<ffffffffb7d34540>] attach_task_by_pid+0x160/0x1a0
[<ffffffffb7d34596>] cgroup_procs_write+0x16/0x20
[<ffffffffb7d3246f>] cgroup_file_write+0x12f/0x2f0
[<ffffffffb7e5ecf8>] ? __sb_start_write+0x58/0x120
[<ffffffffb7f1b257>] ? security_file_permission+0x27/0xb0
[<ffffffffb7e5bd30>] vfs_write+0xc0/0x1f0
[<ffffffffb7e5cae5>] SyS_write+0x55/0xd0
[<ffffffffb83c539a>] system_call_fastpath+0x25/0x2a
Code: 89 de 48 03 14 c5 80 19 95 b8 48 89 df e8 97 67 29 00 84 c0 75 53 45 85 ed 74 16 f6 43 20 01 74 10 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28 00 00
Complete report #830573
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffff8cd1db10
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x100
|
|
2
|
|
vmlinux |
0xffffffff8cdb2170
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffff8cdb2170
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffff8cd1dc2f
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffff8cdb15cc
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffff8cdb7bc0
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x1b0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffff8cdb2154
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffff8cd32775
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x1b5
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffff8cd344d0
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffff8cd34526
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffff8cd323ff
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x12f
|
|
12
|
|
vmlinux |
0xffffffff8ce5ec88
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffff8cf1b107
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x27
|
|
14
|
vfs_write
|
vmlinux |
0xffffffff8ce5bcc0
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0xc0
|
|
15
|
SyS_write
|
vmlinux |
0xffffffff8ce5ca75
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x55
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff8d3c539a
Build id: 3.10.0-1160.92.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [docker-runc:61498]
Modules linked in: xt_multiport ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_bitmap_port ip_set_hash_ipport dummy nf_tables ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_conntrack_netlink br_netfilter tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag mmfs26(OE) mmfslinux(OE) tracedev(OE) overlay(T) ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink xt_set ip_set xt_addrtype iptable_nat nf_nat_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_CHECKSUM xt_conntrack iptable_mangle ipt_rpfilter xt_comment xt_mark iptable_raw sunrpc ext4 mbcache jbd2
ppdev pcspkr joydev virtio_balloon parport_pc parport i2c_piix4 ip_vs_wrr ip_vs_wlc ip_vs_sh ip_vs_sed ip_vs_rr ip_vs_pe_sip nf_conntrack_sip ip_vs_nq ip_vs_lc ip_vs_lblcr ip_vs_lblc ip_vs_ftp nf_nat ip_vs_dh ip_vs nf_conntrack binfmt_misc ip_tables xfs libcrc32c virtio_console virtio_blk virtio_scsi virtio_net net_failover failover ata_generic pata_acpi cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata serio_raw virtio_pci drm_panel_orientation_quirks virtio_ring floppy virtio dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 61498 Comm: docker-runc Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-1160.92.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.7.5-0-ge51488c-20140602_164612-nilsson.home.kraxel.org 04/01/2014
task: ffff9c58db44c200 ti: ffff9c5447b54000 task.ti: ffff9c5447b54000
RIP: 0010:[<ffffffff8cd1db10>] [<ffffffff8cd1db10>] generic_exec_single+0x100/0x1c0
RSP: 0018:ffff9c5447b57c30 EFLAGS: 00000202
RAX: 00000000000000a0 RBX: ffff9c5447b57c00 RCX: 0000000000000020
RDX: 00000000ffffffff RSI: 00000000000000a0 RDI: 0000000000000286
RBP: ffff9c5447b57c80 R08: ffffffff8d413bc8 R09: 0000000180400034
R10: 00000000f2397c01 R11: ffff9c58f2397cc0 R12: 000000007dfad93d
R13: ffff9c58ca7b0de0 R14: 0000000c00000000 R15: ffff9c58ca7b0de0
FS: 00007faf4b167700(0000) GS:ffff9c582cec0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c420200038 CR3: 000000010337e000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8cdb2170>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff8cdb2170>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff8cd1dc2f>] smp_call_function_single+0x5f/0xa0
[<ffffffff8cdb15cc>] task_function_call+0x4c/0x80
[<ffffffff8cdb7bc0>] ? perf_cgroup_switch+0x1b0/0x1b0
[<ffffffff8cdb2154>] perf_cgroup_attach+0x44/0x60
[<ffffffff8cd32775>] cgroup_attach_task+0x1b5/0x410
[<ffffffff8cd344d0>] attach_task_by_pid+0x160/0x1a0
[<ffffffff8cd34526>] cgroup_procs_write+0x16/0x20
[<ffffffff8cd323ff>] cgroup_file_write+0x12f/0x2f0
[<ffffffff8ce5ec88>] ? __sb_start_write+0x58/0x120
[<ffffffff8cf1b107>] ? security_file_permission+0x27/0xb0
[<ffffffff8ce5bcc0>] vfs_write+0xc0/0x1f0
[<ffffffff8ce5ca75>] SyS_write+0x55/0xd0
[<ffffffff8d3c539a>] system_call_fastpath+0x25/0x2a
Code: 00 48 89 de 48 03 14 c5 00 19 95 8d 48 89 df e8 c7 66 29 00 84 c0 75 53 45 85 ed 74 16 f6 43 20 01 74 10 0f 1f 84 00 00 00 00 00 <f3> 90 f6 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28
Complete report #770881
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffffa6b1117a
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xfa
|
|
2
|
|
vmlinux |
0xffffffffa6ba02b0
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffffa6ba02b0
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffffa6b1128f
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffffa6b9f65c
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffffa6ba5a40
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x1a0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffffa6ba0294
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffffa6b253f5
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x1b5
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffffa6b27100
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffffa6b27156
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffffa6b25087
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x127
|
|
12
|
|
vmlinux |
0xffffffffa6c44328
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffffa6cf90e7
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x27
|
|
14
|
vfs_write
|
vmlinux |
0xffffffffa6c41310
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xc0
|
|
15
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa7174d21
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xae
|
|
16
|
SyS_write
|
vmlinux |
0xffffffffa6c4212f
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x7f
|
|
17
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa7174d21
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xae
|
|
18
|
system_call_fastpath
|
vmlinux |
0xffffffffa7174ddb
Build id: 3.10.0-957.1.3.el7.x86_64
|
0x22
|
|
19
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa7174d21
Build id: 3.10.0-957.1.3.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kswapd0:66]
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [docker-runc-lat:29592]
Modules linked in: xt_addrtype ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter binfmt_misc xt_nat veth br_netfilter rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip_set nfnetlink bridge stp llc iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack vmw_vsock_vmci_transport vsock sunrpc btrfs raid6_pq xor sb_edac coretemp ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd
joydev pcspkr sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm floppy drm vmxnet3 ata_piix drm_panel_orientation_quirks libata mptspi scsi_transport_spi mptscsih mptbase dm_mirror dm_region_hash dm_log dm_mod [last unloaded: iptable_filter]
CPU: 2 PID: 29592 Comm: docker-runc-lat Kdump: loaded Tainted: G L ------------ 3.10.0-957.1.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff9b83547ed140 ti: ffff9b7b43ba0000 task.ti: ffff9b7b43ba0000
RIP: 0010:[<ffffffffa6b1117a>] [<ffffffffa6b1117a>] generic_exec_single+0xfa/0x1b0
RSP: 0018:ffff9b7b43ba3c30 EFLAGS: 00000202
RAX: 00000000000008fb RBX: 0000000065cc0d6c RCX: 0000000000000000
RDX: 00000000000008fb RSI: 00000000000000fb RDI: 0000000000000286
RBP: ffff9b7b43ba3c80 R08: 0000000000000001 R09: ffffffffa6b23bcb
R10: ffff9b837fc9f0a0 R11: ffffd8178e58ba80 R12: 0000000c00000000
R13: ffff9b835f0e69a0 R14: 000000020000a068 R15: ffff9b835f0e69a0
FS: 00007f37e5076700(0000) GS:ffff9b837fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c4201803f8 CR3: 00000000487d4000 CR4: 00000000000407e0
Call Trace:
[<ffffffffa6ba02b0>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffa6ba02b0>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffa6b1128f>] smp_call_function_single+0x5f/0xa0
[<ffffffffa6b9f65c>] task_function_call+0x4c/0x80
[<ffffffffa6ba5a40>] ? perf_cgroup_switch+0x1a0/0x1a0
[<ffffffffa6ba0294>] perf_cgroup_attach+0x44/0x60
[<ffffffffa6b253f5>] cgroup_attach_task+0x1b5/0x410
[<ffffffffa6b27100>] attach_task_by_pid+0x160/0x1a0
[<ffffffffa6b27156>] cgroup_procs_write+0x16/0x20
[<ffffffffa6b25087>] cgroup_file_write+0x127/0x2e0
[<ffffffffa6c44328>] ? __sb_start_write+0x58/0x110
[<ffffffffa6cf90e7>] ? security_file_permission+0x27/0xa0
[<ffffffffa6c41310>] vfs_write+0xc0/0x1f0
[<ffffffffa7174d21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffffa6c4212f>] SyS_write+0x7f/0xf0
[<ffffffffa7174d21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffffa7174ddb>] system_call_fastpath+0x22/0x27
[<ffffffffa7174d21>] ? system_call_after_swapgs+0xae/0x146
Complete report #169725
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffff94f16842
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x102
|
|
2
|
|
vmlinux |
0xffffffff94fa7a90
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffff94fa7a90
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffff94f1695f
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffff94fa6f6c
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffff94fad300
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x1a0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffff94fa7a74
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffff94f2ad45
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x1b5
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffff94f2ca50
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffff94f2caa6
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffff94f2a9d7
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x127
|
|
12
|
|
vmlinux |
0xffffffff95050d58
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffff95108d67
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x27
|
|
14
|
vfs_write
|
vmlinux |
0xffffffff9504de00
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0xc0
|
|
15
|
SyS_write
|
vmlinux |
0xffffffff9504ebdf
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x7f
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff95595f92
Build id: 3.10.0-1160.25.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 33s! [Thread Pool Wor:116986]
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [runc:116994]
Modules linked in: veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache overlay nf_conntrack_netbios_ns nf_conntrack_broadcast xt_nat xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash
dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul snd_ens1371 glue_helper snd_rawmidi ablk_helper snd_ac97_codec ppdev cryptd ac97_bus snd_seq joydev snd_seq_device snd_pcm vmw_balloon pcspkr snd_timer snd soundcore sg vmw_vmci i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sch_fq_codel sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi nfit ata_piix crct10dif_pclmul crct10dif_common scsi_transport_spi libata mptscsih crc32c_intel libnvdimm serio_raw drm_panel_orientation_quirks e1000 mptbase fuse
CPU: 2 PID: 116994 Comm: runc Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff8c17e5f18000 ti: ffff8c17d80bc000 task.ti: ffff8c17d80bc000
RIP: 0010:[<ffffffff94f16842>] [<ffffffff94f16842>] generic_exec_single+0x102/0x1c0
RSP: 0018:ffff8c17d80bfc30 EFLAGS: 00000202
RAX: 0000000000000080 RBX: ffff8c17d80bfc00 RCX: 0000000000000001
RDX: ffffffff956137d0 RSI: 0000000000000080 RDI: 0000000000000286
RBP: ffff8c17d80bfc80 R08: ffffffff956137c8 R09: ffffffff94f2951b
R10: ffff8c197569f0a0 R11: ffffd2ecc84c8b80 R12: ffff8c190d54de60
R13: ffff8c1969195400 R14: 0000000000000001 R15: 0000000000000000
FS: 00007fbcf736c700(0000) GS:ffff8c1975680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000559384c11980 CR3: 000000020874c000 CR4: 00000000003607e0
Call Trace:
[<ffffffff94fa7a90>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff94fa7a90>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff94f1695f>] smp_call_function_single+0x5f/0xa0
[<ffffffff94fa6f6c>] task_function_call+0x4c/0x80
[<ffffffff94fad300>] ? perf_cgroup_switch+0x1a0/0x1a0
[<ffffffff94fa7a74>] perf_cgroup_attach+0x44/0x60
[<ffffffff94f2ad45>] cgroup_attach_task+0x1b5/0x410
[<ffffffff94f2ca50>] attach_task_by_pid+0x160/0x1a0
[<ffffffff94f2caa6>] cgroup_procs_write+0x16/0x20
[<ffffffff94f2a9d7>] cgroup_file_write+0x127/0x2e0
[<ffffffff95050d58>] ? __sb_start_write+0x58/0x120
[<ffffffff95108d67>] ? security_file_permission+0x27/0xa0
[<ffffffff9504de00>] vfs_write+0xc0/0x1f0
[<ffffffff9504ebdf>] SyS_write+0x7f/0xf0
[<ffffffff95595f92>] system_call_fastpath+0x25/0x2a
Code: 89 de 48 03 14 c5 a0 15 b5 95 48 89 df e8 c7 6c 28 00 84 c0 75 53 45 85 ed 74 16 f6 43 20 01 74 10 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28 00 00
Complete report #143410
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffffb830779a
Build id: 3.10.0-862.el7.x86_64
|
0xfa
|
|
2
|
|
vmlinux |
0xffffffffb837e570
Build id: 3.10.0-862.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffffb837e570
Build id: 3.10.0-862.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffffb83078af
Build id: 3.10.0-862.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffffb837dc0c
Build id: 3.10.0-862.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffffb8383f80
Build id: 3.10.0-862.el7.x86_64
|
0x1a0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffffb837e554
Build id: 3.10.0-862.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffffb831a9c5
Build id: 3.10.0-862.el7.x86_64
|
0x1b5
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffffb831b780
Build id: 3.10.0-862.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffffb831b7d6
Build id: 3.10.0-862.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffffb831a089
Build id: 3.10.0-862.el7.x86_64
|
0x119
|
|
12
|
|
vmlinux |
0xffffffffb841dc58
Build id: 3.10.0-862.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffffb84d0917
Build id: 3.10.0-862.el7.x86_64
|
0x27
|
|
14
|
vfs_write
|
vmlinux |
0xffffffffb841acd0
Build id: 3.10.0-862.el7.x86_64
|
0xc0
|
|
15
|
SyS_write
|
vmlinux |
0xffffffffb841baff
Build id: 3.10.0-862.el7.x86_64
|
0x7f
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffffb891f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [docker-runc-cur:3330]
Modules linked in: tcp_lp fuse xt_multiport xt_set iptable_raw ip_set_hash_ip ip_set_hash_net veth ip_set nf_conntrack_netlink nfnetlink xt_statistic xt_nat ip_vs_sh ip_vs_wrr ip_vs_rr rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ip_vs nf_conntrack ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi ppdev snd_ac97_codec ac97_bus snd_seq aesni_intel
vmw_balloon snd_seq_device lrw gf128mul glue_helper ablk_helper cryptd snd_pcm vmw_vmci sg pcspkr joydev snd_timer snd nfit libnvdimm soundcore i2c_piix4 shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix libata drm mptspi e1000 scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common mptbase crc32c_intel i2c_core serio_raw floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 3330 Comm: docker-runc-cur Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8e838a29af70 ti: ffff8e81e3154000 task.ti: ffff8e81e3154000
RIP: 0010:[<ffffffffb830779a>] [<ffffffffb830779a>] generic_exec_single+0xfa/0x1b0
RSP: 0018:ffff8e81e3157c30 EFLAGS: 00000202
RAX: 0000000000000080 RBX: ffff8e81e3157c00 RCX: 0000000000000003
RDX: ffffffffb8a13670 RSI: 0000000000000080 RDI: 0000000000000286
RBP: ffff8e81e3157c80 R08: ffffffffb8a13668 R09: ffffffffb8319dfb
R10: ffffd6e1fde43570 R11: fffff6910237ea80 R12: 0000000000000010
R13: ffffffffb825e6c6 R14: ffffffffffffff10 R15: 0000000000000286
FS: 00007f293e573700(0000) GS:ffff8e83dcc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c42019e010 CR3: 00000001e0b1c000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb837e570>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffb837e570>] ? perf_cgroup_attach+0x60/0x60
[<ffffffffb83078af>] smp_call_function_single+0x5f/0xa0
[<ffffffffb837dc0c>] task_function_call+0x4c/0x80
[<ffffffffb8383f80>] ? perf_cgroup_switch+0x1a0/0x1a0
[<ffffffffb837e554>] perf_cgroup_attach+0x44/0x60
[<ffffffffb831a9c5>] cgroup_attach_task+0x1b5/0x410
[<ffffffffb831b780>] attach_task_by_pid+0x160/0x1a0
[<ffffffffb831b7d6>] cgroup_procs_write+0x16/0x20
[<ffffffffb831a089>] cgroup_file_write+0x119/0x2d0
[<ffffffffb841dc58>] ? __sb_start_write+0x58/0x110
[<ffffffffb84d0917>] ? security_file_permission+0x27/0xa0
[<ffffffffb841acd0>] vfs_write+0xc0/0x1f0
[<ffffffffb841baff>] SyS_write+0x7f/0xf0
[<ffffffffb891f7d5>] system_call_fastpath+0x1c/0x21
Code: 80 96 01 00 48 89 de 48 03 14 c5 a0 53 f3 b8 48 89 df e8 4a 87 25 00 84 c0 75 46 45 85 ed 74 11 f6 43 20 01 74 0b 0f 1f 00 f3 90 <f6> 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28 00 00
Complete report #67415
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
generic_exec_single
|
vmlinux |
0xffffffff810f925a
Build id: 3.10.0-514.el7.x86_64
|
0xfa
|
|
2
|
|
vmlinux |
0xffffffff8116c420
Build id: 3.10.0-514.el7.x86_64
|
0x60
|
|
3
|
|
vmlinux |
0xffffffff8116c420
Build id: 3.10.0-514.el7.x86_64
|
0x60
|
|
4
|
smp_call_function_single
|
vmlinux |
0xffffffff810f935f
Build id: 3.10.0-514.el7.x86_64
|
0x5f
|
|
5
|
task_function_call
|
vmlinux |
0xffffffff8116bcbc
Build id: 3.10.0-514.el7.x86_64
|
0x4c
|
|
6
|
|
vmlinux |
0xffffffff81171050
Build id: 3.10.0-514.el7.x86_64
|
0x1c0
|
|
7
|
perf_cgroup_attach
|
vmlinux |
0xffffffff8116c404
Build id: 3.10.0-514.el7.x86_64
|
0x44
|
|
8
|
cgroup_attach_task
|
vmlinux |
0xffffffff8110b882
Build id: 3.10.0-514.el7.x86_64
|
0x1b2
|
|
9
|
attach_task_by_pid
|
vmlinux |
0xffffffff8110c610
Build id: 3.10.0-514.el7.x86_64
|
0x160
|
|
10
|
cgroup_procs_write
|
vmlinux |
0xffffffff8110c666
Build id: 3.10.0-514.el7.x86_64
|
0x16
|
|
11
|
cgroup_file_write
|
vmlinux |
0xffffffff8110a673
Build id: 3.10.0-514.el7.x86_64
|
0x113
|
|
12
|
|
vmlinux |
0xffffffff81200be8
Build id: 3.10.0-514.el7.x86_64
|
0x58
|
|
13
|
?
security_file_permission
|
vmlinux |
0xffffffff812a8b83
Build id: 3.10.0-514.el7.x86_64
|
0x23
|
|
14
|
vfs_write
|
vmlinux |
0xffffffff811fe12d
Build id: 3.10.0-514.el7.x86_64
|
0xbd
|
|
15
|
|
vmlinux |
0xffffffff8120eb4d
Build id: 3.10.0-514.el7.x86_64
|
0x3d
|
|
16
|
SyS_write
|
vmlinux |
0xffffffff811fec4f
Build id: 3.10.0-514.el7.x86_64
|
0x7f
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffff816965c9
Build id: 3.10.0-514.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [docker-runc:51624]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ext4 mbcache jbd2 dm_mirror dm_region_hash dm_log dm_mod intel_powerclamp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd ppdev vmw_balloon pcspkr sg vmw_vmci shpchp i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper mptspi syscopyarea crct10dif_pclmul crct10dif_common sysfillrect scsi_transport_spi sysimgblt fb_sys_fops crc32c_intel ata_piix ttm drm serio_raw mptscsih libata vmxnet3 mptbase i2c_core floppy fjes
CPU: 5 PID: 51624 Comm: docker-runc Tainted: G ------------ T 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8807c5fe0fb0 ti: ffff88011e6d8000 task.ti: ffff88011e6d8000
RIP: 0010:[<ffffffff810f925a>] [<ffffffff810f925a>] generic_exec_single+0xfa/0x1a0
RSP: 0018:ffff88011e6dbc60 EFLAGS: 00000202
RAX: 0000000000000040 RBX: 00000000c101a756 RCX: 0000000000000003
RDX: 0000000000000040 RSI: 0000000000000000 RDI: 0000000000000282
RBP: ffff88011e6dbcb0 R08: ffffffff816afea8 R09: 000000018040003a
R10: 00000000de5d1e01 R11: ffffea0007797440 R12: 0000000c00000000
R13: ffff8805aa1d2750 R14: ffff8805aa1d2750 R15: 0000000000000001
FS: 00007f2ce5144700(0000) GS:ffff88081e740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c420175000 CR3: 000000078677b000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
0000000000000000 0000000000000000 ffffffff8116c420 ffff88011e6dbcf0
0000000000000003 00000000c101a756 ffff8807f2f82028 0000000000000002
ffffffff8116c420 ffff880806ad0200 ffff88011e6dbce0 ffffffff810f935f
Call Trace:
[<ffffffff8116c420>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff8116c420>] ? perf_cgroup_attach+0x60/0x60
[<ffffffff810f935f>] smp_call_function_single+0x5f/0xa0
[<ffffffff8116bcbc>] task_function_call+0x4c/0x80
[<ffffffff81171050>] ? perf_cgroup_switch+0x1c0/0x1c0
[<ffffffff8116c404>] perf_cgroup_attach+0x44/0x60
[<ffffffff8110b882>] cgroup_attach_task+0x1b2/0x400
[<ffffffff8110c610>] attach_task_by_pid+0x160/0x1a0
[<ffffffff8110c666>] cgroup_procs_write+0x16/0x20
[<ffffffff8110a673>] cgroup_file_write+0x113/0x2b0
[<ffffffff81200be8>] ? __sb_start_write+0x58/0x110
[<ffffffff812a8b83>] ? security_file_permission+0x23/0xa0
[<ffffffff811fe12d>] vfs_write+0xbd/0x1e0
[<ffffffff8120eb4d>] ? putname+0x3d/0x60
[<ffffffff811fec4f>] SyS_write+0x7f/0xe0
[<ffffffff816965c9>] system_call_fastpath+0x16/0x1b
Code: 80 77 01 00 48 89 de 48 03 14 c5 40 57 ad 81 48 89 df e8 ea 28 23 00 84 c0 75 46 45 85 ed 74 11 f6 43 20 01 74 0b 0f 1f 00 f3 90 <f6> 43 20 01 75 f8 31 c0 48 8b 7c 24 28 65 48 33 3c 25 28 00 00