Complete report #1424487
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff816ab865
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff811f3339
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff811f3774
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff8110d757
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xe7
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff8110da02
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff8120f636
Build id: 3.10.0-693.2.2.el7.x86_64
|
0xd6
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff812122d1
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x1f1
|
|
8
|
SyS_rmdir
|
vmlinux |
0xffffffff812134d6
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff816b5009
Build id: 3.10.0-693.2.2.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#24 stuck for 23s! [slurmstepd:111348]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 auth_rpcgss nfsv4 dell_rbu dns_resolver dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt dell_smbios iTCO_vendor_support sparse_keymap dcdbas joydev crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables fuse btrfs raid6_pq xor msdos bnxt_en nvme nvme_core qed hpilo dm_thin_pool
dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci libsas qla3xxx hpsa sg e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix sd_mod crc_t10dif mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun crct10dif_pclmul crct10dif_common ahci libahci libata i40e(OE) megaraid_sas tg3(OE) ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 24 PID: 111348 Comm: slurmstepd Tainted: G OE ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 2.21.1 03/07/2024
task: ffff883057408fd0 ti: ffff8830f312c000 task.ti: ffff8830f312c000
RIP: 0010:[<ffffffff816ab865>] [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8830f312fd50 EFLAGS: 00000287
RAX: 0000000000000287 RBX: ffffea01447c45c0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff8830f312fd50 R08: ffff885798de5400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff81190590
R13: 0000000000000000 R14: ffffffff81190eac R15: ffff8830f312fcf8
FS: 00002aaaaaafd180(0000) GS:ffff882f7dd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aacbb8029e0 CR3: 00000058fb7f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8830f312fdd0 ffffffff811f3339 0000000200000001 ffff8824187d3d00
0000000000000001 0000000000001000 ffff8824187d3c40 0000000000000001
0000000100000002 ffff8824187d3c00 0000000000000000 0000000000000000
Call Trace:
[<ffffffff811f3339>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #1421442
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffba832f65
Build id: 3.10.0-957.el7.x86_64
|
0x1e5
|
|
2
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffba8332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
3
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffba72573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
4
|
cgroup_rmdir
|
vmlinux |
0xffffffffba7259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
5
|
vfs_rmdir
|
vmlinux |
0xffffffffba84dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
6
|
do_rmdir
|
vmlinux |
0xffffffffba8530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
7
|
|
vmlinux |
0xffffffffba847034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
8
|
SyS_unlinkat
|
vmlinux |
0xffffffffba854335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffffbad74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#26 stuck for 23s! [runc:183133]
Modules linked in: fuse cmac arc4 md4 nls_utf8 cifs ccm veth xt_nat xt_CHECKSUM nf_conntrack_netlink ipt_MASQUERADE nf_nat_masquerade_ipv4 tun xt_addrtype ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack br_netfilter devlink rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache overlay(T) ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ext4 mbcache jbd2 iTCO_wdt iTCO_vendor_support vfat fat skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm squashfs
irqbypass crc32_pclmul loop ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd megaraid_sas pcspkr joydev sg mei_me i2c_i801 lpc_ich mei ipmi_si ipmi_devintf ipmi_msghandler acpi_cpufreq wmi acpi_pad acpi_power_meter binfmt_misc sch_fq_codel nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops uas ttm igb drm usb_storage ptp pps_core dca i2c_algo_bit drm_panel_orientation_quirks ahci libahci nfit libata libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 26 PID: 183133 Comm: runc Kdump: loaded Tainted: G ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Supermicro SYS-2049U-TR4/X11QPH+, BIOS 3.2a 01/24/2020
task: ffff8910848e5140 ti: ffff891b07a5c000 task.ti: ffff891b07a5c000
RIP: 0010:[<ffffffffba832f65>] [<ffffffffba832f65>] mem_cgroup_reparent_charges+0x1e5/0x3c0
RSP: 0018:ffff891b07a5fd30 EFLAGS: 00000246
RAX: ffff8910256c0c48 RBX: 0000000000000000 RCX: ffff89167500b800
RDX: ffff8920f334c580 RSI: 0000000000000005 RDI: ffff89307ffd9540
RBP: ffff891b07a5fda0 R08: ffff8910256c0c00 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffffdb2cf254ea00
R13: 0000000000000246 R14: ffffffffba7c4930 R15: 0000000000000000
FS: 00007efdea53e740(0000) GS:ffff89103e280000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f669d4f6200 CR3: 00000018426e0000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffba8332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffba72573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffba7259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffba84dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffba8530d1>] do_rmdir+0x1f1/0x220
[<ffffffffba847034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffba854335>] SyS_unlinkat+0x25/0x40
[<ffffffffbad74ddb>] system_call_fastpath+0x22/0x27
Code: 23 9c 58 0f 1f 44 00 00 48 89 45 d0 fa 66 0f 1f 44 00 00 f0 41 0f ba 6f e0 18 19 c0 85 c0 0f 85 c5 01 00 00 48 8b 4d c8 4c 89 f2 <44> 89 d6 48 89 df 44 89 5d b8 44 89 55 c4 e8 68 e8 ff ff 85 c0
Complete report #1414195
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8b16a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff8ac32eb9
Build id: 3.10.0-957.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff8ac332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff8ab2573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff8ab259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff8ac4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff8ac530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffff8ac47034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffff8ac54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff8b174ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [runc:28588]
Modules linked in: udp_diag tcp_diag inet_diag binfmt_misc veth nf_conntrack_netlink br_netfilter overlay(T) nvidia_uvm(OE) tun rpcsec_gss_krb5 auth_rpcgss nfsv4 uio_pci_generic uio dns_resolver vfio_pci vfio_iommu_type1 vfio nfs cuse fuse lockd grace fscache ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 xt_conntrack rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ip_set nfnetlink ebtable_nat ebtable_broute bridge stp ib_umad(OE) llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack ip6table_mangle ip6table_security ip6table_raw ebtable_filter ebtables ip6table_filter ip6_tables sunrpc dm_mirror dm_region_hash dm_log ext4 mbcache jbd2 nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) intel_powerclamp
coretemp kvm_intel kvm vfat fat irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_hda_codec_hdmi cryptd dm_mod pcspkr snd_hda_intel joydev snd_hda_codec snd_hda_core snd_hwdep snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg mei_me mei i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel acpi_cpufreq acpi_power_meter acpi_pad knem(OE) ip_tables xfs libcrc32c mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) rndis_host cdc_ether usbnet mii sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper crct10dif_pclmul crct10dif_common mlx5_core(OE) crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops mlxfw(OE) ttm psample auxiliary(OE) devlink mlx_compat(OE) drm ahci igb libahci nvme libata ptp nvme_core drm_panel_orientation_quirks
pps_core dca i2c_algo_bit wmi xpmem(OE)
CPU: 18 PID: 28588 Comm: runc Kdump: loaded Tainted: P W OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Supermicro SYS-420GP-TNR/X12DPG-OA6, BIOS 1.1b 11/10/2021
task: ffff979367f0d140 ti: ffff971623934000 task.ti: ffff971623934000
RIP: 0010:[<ffffffff8b16a0f5>] [<ffffffff8b16a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff971623937d20 EFLAGS: 00000283
RAX: 0000000000000283 RBX: ffffd798ab0e9680 RCX: ffff971623937c28
RDX: 0000000000000001 RSI: 0000000000000283 RDI: 0000000000000283
RBP: ffff971623937d20 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff8abc4930
R13: 0000000000000000 R14: ffffffff8abc524c R15: ffff971623937cc8
FS: 00007f5e0a20c740(0000) GS:ffff97947ee80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0003b9010 CR3: 000000408df44000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff8ac32eb9>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffff8ac332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8ab2573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff8ab259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff8ac4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff8ac530d1>] do_rmdir+0x1f1/0x220
[<ffffffff8ac47034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff8ac54335>] SyS_unlinkat+0x25/0x40
[<ffffffff8b174ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #1397660
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_lock_irqsave
|
vmlinux |
0xffffffffa3f6a45b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffa3a32e92
Build id: 3.10.0-957.el7.x86_64
|
0x112
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffa3a332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffa392573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffa39259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffa3a4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffa3a530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffa3a47034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffa3a54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffa3f74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#102 stuck for 22s! [runc:217970]
Modules linked in: ses enclosure scsi_transport_sas xt_nat binfmt_misc nfsv4 nfs fscache rpcsec_gss_krb5 veth cmac arc4 md4 nls_utf8 cifs ccm dns_resolver fuse ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ext4 mbcache jbd2 iTCO_wdt iTCO_vendor_support dell_smbios dcdbas
dell_wmi_descriptor skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ixgbe sg mdio dca mei_me i2c_i801 lpc_ich mei wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm megaraid_sas tg3 drm_panel_orientation_quirks ptp pps_core ahci libahci libata nfit libnvdimm uas usb_storage dm_mirror dm_region_hash dm_log dm_mod
CPU: 102 PID: 217970 Comm: runc Kdump: loaded Tainted: G W ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R940xa/08XR9M, BIOS 2.9.4 11/09/2020
task: ffff8b873be62080 ti: ffff8b849b558000 task.ti: ffff8b849b558000
RIP: 0010:[<ffffffffa3f6a45b>] [<ffffffffa3f6a45b>] _raw_spin_lock_irqsave+0xb/0x40
RSP: 0018:ffff8b849b55bd10 EFLAGS: 00000283
RAX: 006fffff00000000 RBX: ffffffffa39c4930 RCX: 0000000000000001
RDX: 0000000000000007 RSI: 0000000000000001 RDI: ffff8c02bffda540
RBP: ffff8b849b55bd20 R08: ffff8bc7aafc3400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffa39c524c
R13: ffff8b849b55bcc8 R14: ffff8c02bffda000 R15: ffff8c02bffda000
FS: 00007fade5c66740(0000) GS:ffff8c41be840000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055cdaf245900 CR3: 0000008da93e8000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa3a32e92>] mem_cgroup_reparent_charges+0x112/0x3c0
[<ffffffffa3a332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa392573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa39259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffa3a4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffa3a530d1>] do_rmdir+0x1f1/0x220
[<ffffffffa3a47034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffa3a54335>] SyS_unlinkat+0x25/0x40
[<ffffffffa3f74ddb>] system_call_fastpath+0x22/0x27
Code: 0f b1 17 85 c0 75 01 c3 55 89 c6 48 89 e5 e8 fd 1a ff ff 5d c3 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 9c <58> 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 31 c0 ba 01 00
Complete report #1193812
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
put_page
|
vmlinux |
0xffffffff827c457e
Build id: 3.10.0-957.el7.x86_64
|
0x2e
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff82832fb2
Build id: 3.10.0-957.el7.x86_64
|
0x232
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff828332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff8272573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff827259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff8284dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff828530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffff82847034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffff82854335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff82d74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [runc:104251]
Modules linked in: xt_nat veth nvidia_uvm(OE) nfsv4 dns_resolver nfs fscache rpcsec_gss_krb5 binfmt_misc nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter iTCO_wdt iTCO_vendor_support vfat fat nvidia_drm(POE) nvidia_modeset(POE) intel_powerclamp coretemp
kvm_intel kvm dm_service_time irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nvidia(POE) snd_hda_codec_hdmi dm_multipath joydev snd_hda_intel cdc_ether usbnet pcspkr mii snd_hda_codec snd_hda_core snd_hwdep snd_seq ipmi_ssif sg snd_seq_device mpt3sas snd_pcm raid_class scsi_transport_sas snd_timer snd i2c_i801 soundcore mei_me mei ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel acpi_cpufreq acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod lpfc ast drm_kms_helper nvmet_fc syscopyarea nvmet sysfillrect sysimgblt fb_sys_fops crc_t10dif crct10dif_generic ttm crct10dif_pclmul crc32c_intel nvme_fc nvme_fabrics nvme_core drm scsi_transport_fc igb ahci scsi_tgt crct10dif_common
libahci ptp megaraid_sas pps_core libata drm_panel_orientation_quirks dca i2c_algo_bit wmi dm_mirror dm_region_hash dm_log dm_mod
CPU: 35 PID: 104251 Comm: runc Tainted: P W OE ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Nettrix X640 G40/60WB32, BIOS NKMH041018 10/08/2021
task: ffff8e7076754100 ti: ffff8e3165c4c000 task.ti: ffff8e3165c4c000
RIP: 0010:[<ffffffff827c457e>] [<ffffffff827c457e>] put_page+0x2e/0x60
RSP: 0018:ffff8e3165c4fd20 EFLAGS: 00000202
RAX: 002fffff0002004c RBX: ffffc50140162440 RCX: 0000000000000001
RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffffc5018aff3980
RBP: ffff8e3165c4fd20 R08: ffff8e32251fb000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff827c4930
R13: 0000000000000000 R14: ffffffff827c524c R15: ffff8e3165c4fcc8
FS: 00007f8026d5d740(0000) GS:ffff8eb07e6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005597317bb000 CR3: 0000004229310000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff82832fb2>] mem_cgroup_reparent_charges+0x232/0x3c0
[<ffffffff828332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8272573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff827259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff8284dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff828530d1>] do_rmdir+0x1f1/0x220
[<ffffffff82847034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff82854335>] SyS_unlinkat+0x25/0x40
[<ffffffff82d74ddb>] system_call_fastpath+0x22/0x27
Code: 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 12 48 8b 57 20 8b 92 90 00 00 00 83 ea 01 83 fa 01 76 22 f6 c4 c0 75 27 f0 ff 4f 1c <0f> 94 c0 84 c0 75 0b 5d c3 66 0f 1f 84 00 00 00 00 00 e8 8b ff
Complete report #1068334
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffaeb6a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffae632eb9
Build id: 3.10.0-957.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffae6332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffae52573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffae5259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffae64dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffae6530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffae647034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffae654335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffaeb74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [runc:36075]
Modules linked in: binfmt_misc xt_nat veth nvidia_uvm(OE) fuse xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_addrtype iptable_filter br_netfilter bridge stp llc mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd squashfs loop snd_hda_codec_hdmi snd_hda_intel snd_hda_codec pcspkr snd_hda_core snd_hwdep joydev snd_seq snd_seq_device snd_pcm snd_timer sg snd i2c_i801 soundcore lpc_ich mei_me mei ioatdma ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter knem(OE) auth_rpcgss sunrpc ip_tables xfs libcrc32c mlx4_ib(OE) mlx4_en(OE) sd_mod crc_t10dif crct10dif_generic mxm_wmi mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb mlx4_core(OE) crct10dif_pclmul crct10dif_common crc32c_intel drm ahci dca libahci i2c_algo_bit mlx5_core(OE) libata mlxfw(OE) drm_panel_orientation_quirks devlink mlx_compat(OE) ptp pps_core wmi overlay(T)
CPU: 2 PID: 36075 Comm: runc Kdump: loaded Tainted: P OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Supermicro SYS-4028GR-TR2-1-EC028/X10DRG-O+-CPU, BIOS 3.1 07/13/2018
task: ffff8e9d66660000 ti: ffff8e96d6960000 task.ti: ffff8e96d6960000
RIP: 0010:[<ffffffffaeb6a0f5>] [<ffffffffaeb6a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8e96d6963d20 EFLAGS: 00000283
RAX: 0000000000000283 RBX: 00ffdfab98e9d540 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000283 RDI: 0000000000000283
RBP: ffff8e96d6963d20 R08: ffff8f13ee0d9000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffae5c4930
R13: 0000000000000000 R14: ffffffffae63168e R15: ffff8e96d6963cd0
FS: 00007f7f8b5cb740(0000) GS:ffff8f143f680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f53b1847eb0 CR3: 0000000118754000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffae632eb9>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffffae6332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffae52573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffae5259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffae64dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffae6530d1>] do_rmdir+0x1f1/0x220
[<ffffffffae647034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffae654335>] SyS_unlinkat+0x25/0x40
[<ffffffffaeb74ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #1043524
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_lock_irqsave
|
vmlinux |
0xffffffffaeb6a45b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffae632e92
Build id: 3.10.0-957.el7.x86_64
|
0x112
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffae6332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffae52573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffae5259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffae64dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffae6530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffae647034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffae654335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffaeb74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 23s! [runc:27577]
Modules linked in: xt_nat veth nvidia_uvm(OE) fuse xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_addrtype iptable_filter br_netfilter bridge stp llc mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul
glue_helper ablk_helper cryptd squashfs loop snd_hda_codec_hdmi snd_hda_intel snd_hda_codec pcspkr snd_hda_core snd_hwdep joydev snd_seq snd_seq_device snd_pcm snd_timer sg snd i2c_i801 soundcore lpc_ich mei_me mei ioatdma ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter knem(OE) auth_rpcgss sunrpc ip_tables xfs libcrc32c mlx4_ib(OE) mlx4_en(OE) sd_mod crc_t10dif crct10dif_generic mxm_wmi mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb mlx4_core(OE) crct10dif_pclmul crct10dif_common crc32c_intel drm ahci dca libahci i2c_algo_bit mlx5_core(OE) libata mlxfw(OE) drm_panel_orientation_quirks devlink mlx_compat(OE) ptp pps_core wmi overlay(T)
CPU: 18 PID: 27577 Comm: runc Kdump: loaded Tainted: P OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Supermicro SYS-4028GR-TR2-1-EC028/X10DRG-O+-CPU, BIOS 3.1 07/13/2018
task: ffff8e97eb158000 ti: ffff8f80a4098000 task.ti: ffff8f80a4098000
RIP: 0010:[<ffffffffaeb6a45b>] [<ffffffffaeb6a45b>] _raw_spin_lock_irqsave+0xb/0x40
RSP: 0018:ffff8f80a409bd10 EFLAGS: 00000287
RAX: 006fffff00000000 RBX: ffffffffae5c4930 RCX: 0000000000000001
RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff8f963ffd8540
RBP: ffff8f80a409bd20 R08: ffff8f13ee0d9000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffae5c524c
R13: ffff8f80a409bcc8 R14: ffff8f963ffd8000 R15: ffff8f963ffd8000
FS: 00007f62ce860740(0000) GS:ffff8f943ef00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc433c7d000 CR3: 000000fd90b1e000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffae632e92>] mem_cgroup_reparent_charges+0x112/0x3c0
[<ffffffffae6332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffae52573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffae5259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffae64dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffae6530d1>] do_rmdir+0x1f1/0x220
[<ffffffffae647034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffae654335>] SyS_unlinkat+0x25/0x40
[<ffffffffaeb74ddb>] system_call_fastpath+0x22/0x27
Code: 0f b1 17 85 c0 75 01 c3 55 89 c6 48 89 e5 e8 fd 1a ff ff 5d c3 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 9c <58> 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 31 c0 ba 01 00
Complete report #1010888
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffaae32edb
Build id: 3.10.0-957.el7.x86_64
|
0x15b
|
|
2
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffaae332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
3
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffaad2573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
4
|
cgroup_rmdir
|
vmlinux |
0xffffffffaad259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
5
|
vfs_rmdir
|
vmlinux |
0xffffffffaae4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
6
|
do_rmdir
|
vmlinux |
0xffffffffaae530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
7
|
SyS_unlinkat
|
vmlinux |
0xffffffffaae54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
8
|
system_call_fastpath
|
vmlinux |
0xffffffffab374ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#82 stuck for 22s! [runc:88635]
Modules linked in: xt_nat xt_addrtype nf_conntrack_netlink xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 iptable_filter xt_conntrack iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle ip_tables cmac arc4 md4 nls_utf8 cifs ccm veth ip_set rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) nfnetlink br_netfilter tun bridge stp llc nfsv3 nfs_acl nfs lockd grace fscache overlay(T) bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) libcrc32c sunrpc dm_mirror dm_region_hash dm_log dm_mod
iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr wdat_wdt joydev sg i2c_i801 mei_me lpc_ich mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm ahci mdio libahci mlx4_core(OE) igb libata drm_panel_orientation_quirks megaraid_sas ptp devlink pps_core dca mlx_compat(OE) i2c_algo_bit [last unloaded: ip_tables]
CPU: 82 PID: 88635 Comm: runc Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff93402ed12080 ti: ffff935436e30000 task.ti: ffff935436e30000
RIP: 0010:[<ffffffffaae32edb>] [<ffffffffaae32edb>] mem_cgroup_reparent_charges+0x15b/0x3c0
RSP: 0018:ffff935436e33d30 EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffd11b8f7be100 RCX: 0000000000000002
RDX: 0000000000000001 RSI: ffffd11b8f7be15c RDI: 000000000d3def85
RBP: ffff935436e33da0 R08: ffff93d05a6c6000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffd11b8f7be080
R13: 0000000000000246 R14: ffffffffaadc4930 R15: 0000000000000000
FS: 00007f92ecab8740(0000) GS:ffff9390bfa80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006e9360 CR3: 000000bfed2f6000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaae332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffaad2573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffaad259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffaae4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffaae530d1>] do_rmdir+0x1f1/0x220
[<ffffffffaae54335>] SyS_unlinkat+0x25/0x40
[<ffffffffab374ddb>] system_call_fastpath+0x22/0x27
Code: c6 4c 89 ef e8 27 72 53 00 48 89 df e8 3f 3d 00 00 41 8b 57 fc 49 89 c6 85 d2 74 ac 8d 4a 01 49 8d 77 fc 89 d0 f0 41 0f b1 4f fc <39> c2 89 c1 0f 85 bb 01 00 00 48 89 df e8 b3 70 f9 ff 85 c0 0f
Complete report #1000707
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8168e70b
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x1b
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff811f1709
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff811f1b44
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff8110d6e7
Build id: 3.10.0-514.21.1.el7.x86_64
|
0xe7
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff8110d982
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff8120a778
Build id: 3.10.0-514.21.1.el7.x86_64
|
0xa8
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff8120f815
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x1a5
|
|
8
|
|
vmlinux |
0xffffffff81203ee4
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x24
|
|
9
|
|
vmlinux |
0xffffffff81692db5
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x35
|
|
10
|
SyS_unlinkat
|
vmlinux |
0xffffffff812109f5
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x25
|
|
11
|
system_call_fastpath
|
vmlinux |
0xffffffff816975c9
Build id: 3.10.0-514.21.1.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#122 stuck for 22s! [runc:64145]
Modules linked in: xt_nat udp_diag tcp_diag inet_diag bluetooth rfkill squashfs loop veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter devlink binfmt_misc overlay(T) nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 mpt2sas raid_class scsi_transport_sas mptctl mptbase ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu sunrpc ext4 mbcache jbd2 vfat fat iTCO_wdt iTCO_vendor_support
dcdbas intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ipmi_devintf sb_edac edac_core sg i2c_i801 lpc_ich mei_me mei shpchp wmi ipmi_si ipmi_msghandler acpi_power_meter dm_multipath sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom crct10dif_pclmul crct10dif_common crc32c_intel mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm qla2xxx ahci ixgbe drm libahci igb i2c_algo_bit libata mdio scsi_transport_fc i2c_core ptp megaraid_sas pps_core scsi_tgt dca fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 122 PID: 64145 Comm: runc Tainted: G W L ------------ T 3.10.0-514.21.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R930/0Y0V4F, BIOS 2.2.0 09/29/2016
task: ffff8840991e8000 ti: ffff88624f324000 task.ti: ffff88624f324000
RIP: 0010:[<ffffffff8168e70b>] [<ffffffff8168e70b>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff88624f327d48 EFLAGS: 00000283
RAX: 0000000000000283 RBX: ffffffff8118f010 RCX: 0000000000000b5c
RDX: 0000000000000b5c RSI: 0000000000000283 RDI: 0000000000000283
RBP: ffff88624f327d50 R08: 0000000000000283 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff8118f93c
R13: ffff88624f327cf8 R14: ffff88207ffda000 R15: ffff88207ffda000
FS: 00007f0867ed7740(0000) GS:ffff885fff980000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000215000 CR3: 000000415d837000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffea0022749380 ffff88624f327dd0 ffffffff811f1709 0000000200000000
ffff88256dd86d00 0000000000000000 0000000000001000 ffff88256dd86c40
0000000000000001 0000000100000002 ffff88256dd86c00 0000000000000000
Call Trace:
[<ffffffff811f1709>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffff811f1b44>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d6e7>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d982>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a778>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f815>] do_rmdir+0x1a5/0x200
[<ffffffff81203ee4>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff81692db5>] ? do_page_fault+0x35/0x90
[<ffffffff812109f5>] SyS_unlinkat+0x25/0x40
[<ffffffff816975c9>] system_call_fastpath+0x16/0x1b
Code: 08 e8 ba 07 a0 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6
Complete report #976743
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffaae33059
Build id: 3.10.0-957.el7.x86_64
|
0x2d9
|
|
2
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffaae332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
3
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffaad2573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
4
|
cgroup_rmdir
|
vmlinux |
0xffffffffaad259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
5
|
vfs_rmdir
|
vmlinux |
0xffffffffaae4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
6
|
do_rmdir
|
vmlinux |
0xffffffffaae530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
7
|
|
vmlinux |
0xffffffffaae47034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
8
|
SyS_unlinkat
|
vmlinux |
0xffffffffaae54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffffab374ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#98 stuck for 22s! [runc:18283]
Modules linked in: xt_nat xt_addrtype nf_conntrack_netlink xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 iptable_filter xt_conntrack iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle ip_tables cmac arc4 md4 nls_utf8 cifs ccm veth ip_set rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) nfnetlink br_netfilter tun bridge stp llc nfsv3 nfs_acl nfs lockd grace fscache overlay(T) bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) libcrc32c sunrpc dm_mirror dm_region_hash dm_log dm_mod
iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr wdat_wdt joydev sg i2c_i801 mei_me lpc_ich mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm ahci mdio libahci mlx4_core(OE) igb libata drm_panel_orientation_quirks megaraid_sas ptp devlink pps_core dca mlx_compat(OE) i2c_algo_bit [last unloaded: ip_tables]
CPU: 98 PID: 18283 Comm: runc Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff934fa6196180 ti: ffff935ee6f7c000 task.ti: ffff935ee6f7c000
RIP: 0010:[<ffffffffaae33059>] [<ffffffffaae33059>] mem_cgroup_reparent_charges+0x2d9/0x3c0
RSP: 0018:ffff935ee6f7fd30 EFLAGS: 00000206
RAX: 0000000000000001 RBX: ffffd11b76052300 RCX: 0000000000000001
RDX: 00000000031ddd86 RSI: 0000000000000001 RDI: ffff93c09d6c2048
RBP: ffff935ee6f7fda0 R08: ffff93d05a6c6000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffd11b76052240
R13: 0000000000000246 R14: ffffffffaadc4930 R15: 0000000000000000
FS: 00007f67bcc06740(0000) GS:ffff93b0bfa80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c002612000 CR3: 000000dff2ed4000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaae332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffaad2573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffaad259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffaae4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffaae530d1>] do_rmdir+0x1f1/0x220
[<ffffffffaae47034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffaae54335>] SyS_unlinkat+0x25/0x40
[<ffffffffab374ddb>] system_call_fastpath+0x22/0x27
Code: 89 c6 e8 ab 70 53 00 e9 46 fe ff ff 66 0f 1f 44 00 00 48 8b 7d b0 49 63 c3 48 89 c6 48 89 45 b8 e8 0d bb ff ff 8b 05 27 c7 b2 00 <44> 8b 55 c4 85 c0 0f 84 26 ff ff ff 48 8b 75 b8 48 8b 7d 98 e8
Complete report #947631
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_lock_irqsave
|
vmlinux |
0xffffffff8416a45c
Build id: 3.10.0-957.el7.x86_64
|
0xc
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff83c32e92
Build id: 3.10.0-957.el7.x86_64
|
0x112
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff83c332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff83b2573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff83b259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff83c4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff83c530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffff83c47034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffff83c54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff84174ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#31 stuck for 23s! [runc:2928]
Modules linked in: xt_multiport iptable_raw ip_set_hash_net ip_set_hash_ip ipip tunnel4 ip_tunnel xt_set veth ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_hash_ipport ip_set_bitmap_port ip_set dummy nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) br_netfilter bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack cambricon_gdrdrv(OE) iTCO_wdt iTCO_vendor_support vfat fat intel_powerclamp coretemp kvm
irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg mei_me i2c_i801 mei wmi ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel acpi_cpufreq acpi_power_meter cambricon_drv(OE) nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops serio_raw ttm drm i40e ahci libahci megaraid_sas ptp libata pps_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 31 PID: 2928 Comm: runc Kdump: loaded Tainted: G W OE ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur NF5468M6/NF5468M6, BIOS 06.01.02 06/28/2022
task: ffff9c4574bab0c0 ti: ffff9c4f026f0000 task.ti: ffff9c4f026f0000
RIP: 0010:[<ffffffff8416a45c>] [<ffffffff8416a45c>] _raw_spin_lock_irqsave+0xc/0x40
RSP: 0018:ffff9c4f026f3d18 EFLAGS: 00000287
RAX: 0000000000000287 RBX: ffffffff83bc4930 RCX: 0000000000000001
RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff9d597ffd6540
RBP: ffff9c4f026f3d20 R08: ffff9bd77d622400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff83bc524c
R13: ffff9c4f026f3cc8 R14: ffff9d597ffd6000 R15: ffff9d597ffd6000
FS: 00007f677aa94740(0000) GS:ffff9cd77f6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000233d000 CR3: 00000179abfcc000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff83c32e92>] mem_cgroup_reparent_charges+0x112/0x3c0
[<ffffffff83c332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff83b2573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff83b259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff83c4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff83c530d1>] do_rmdir+0x1f1/0x220
[<ffffffff83c47034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff83c54335>] SyS_unlinkat+0x25/0x40
[<ffffffff84174ddb>] system_call_fastpath+0x22/0x27
Code: b1 17 85 c0 75 01 c3 55 89 c6 48 89 e5 e8 fd 1a ff ff 5d c3 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 9c 58 <0f> 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 31 c0 ba 01 00 00
Complete report #937536
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb136a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffb0e32eb9
Build id: 3.10.0-957.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffb0e332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffb0d2573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffb0d259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffb0e4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffb0e530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffb0e47034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffb0e54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb1374ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#16 stuck for 22s! [runc:23437]
Modules linked in: xt_multiport iptable_raw ip_set_hash_net ip_set_hash_ip ipip tunnel4 ip_tunnel veth xt_set ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_bitmap_port ip_set_hash_ipport ip_set dummy nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) br_netfilter bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack cambricon_gdrdrv(OE) iTCO_wdt iTCO_vendor_support vfat fat intel_powerclamp coretemp kvm
irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg mei_me i2c_i801 mei wmi ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel acpi_cpufreq acpi_power_meter cambricon_drv(OE) nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper crct10dif_pclmul syscopyarea crct10dif_common sysfillrect crc32c_intel sysimgblt fb_sys_fops ttm serio_raw drm i40e ahci libahci megaraid_sas ptp libata pps_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 16 PID: 23437 Comm: runc Kdump: loaded Tainted: G W OE ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur NF5468M6/NF5468M6, BIOS 06.01.02 06/28/2022
task: ffff8a433250c100 ti: ffff8a30a46c4000 task.ti: ffff8a30a46c4000
RIP: 0010:[<ffffffffb136a0f5>] [<ffffffffb136a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a30a46c7d20 EFLAGS: 00000287
RAX: 0000000000000287 RBX: ffffdbf0754c5400 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff8a30a46c7d20 R08: ffff8a4228b11800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffb0dc4930
R13: 0000000000000000 R14: ffffffffb0dc524c R15: ffff8a30a46c7cc8
FS: 00007fa7a2ef6740(0000) GS:ffff8ac33f680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005618573c10c0 CR3: 000000c94fea2000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb0e32eb9>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffffb0e332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffb0d2573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffb0d259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffb0e4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffb0e530d1>] do_rmdir+0x1f1/0x220
[<ffffffffb0e47034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffb0e54335>] SyS_unlinkat+0x25/0x40
[<ffffffffb1374ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #769483
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa9f6a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffa9a32eb9
Build id: 3.10.0-957.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffa9a332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffa992573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffa99259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffa9a4dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffa9a530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
SyS_unlinkat
|
vmlinux |
0xffffffffa9a54335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffffa9f74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#33 stuck for 23s! [runc:11723]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables nfsv3 nfs_acl nfs lockd grace fscache overlay(T) bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod
iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd wdat_wdt pcspkr joydev sg lpc_ich i2c_i801 mei_me mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm mdio igb mlx4_core(OE) ahci drm_panel_orientation_quirks megaraid_sas ptp pps_core devlink libahci dca i2c_algo_bit mlx_compat(OE) libata
CPU: 33 PID: 11723 Comm: runc Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff9c4f7eaa0000 ti: ffff9c4829590000 task.ti: ffff9c4829590000
RIP: 0010:[<ffffffffa9f6a0f5>] [<ffffffffa9f6a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9c4829593d20 EFLAGS: 00000287
RAX: 0000000000000287 RBX: 00fff8e62f695680 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff9c4829593d20 R08: ffff9c9093bf6000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffa99c4930
R13: 0000000000000000 R14: ffffffffa9a3168e R15: ffff9c4829593cd0
FS: 00007f30c081f740(0000) GS:ffff9c8fbfa40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000563a1c825700 CR3: 0000004b2d5d6000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa9a32eb9>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffffa9a332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa992573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa99259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffa9a4dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffa9a530d1>] do_rmdir+0x1f1/0x220
[<ffffffffa9a54335>] SyS_unlinkat+0x25/0x40
[<ffffffffa9f74ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #490458
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
put_page
|
vmlinux |
0xffffffff971c457e
Build id: 3.10.0-957.el7.x86_64
|
0x2e
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff97232fb2
Build id: 3.10.0-957.el7.x86_64
|
0x232
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff972332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff9712573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff971259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff9724dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff972530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffff97247034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffff97254335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff97774ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#51 stuck for 23s! [runc:51239]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) beegfs(OE) obdclass(OE) lnet(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc libcfs(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache overlay(T) bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) sunrpc dm_mirror dm_region_hash
dm_log dm_mod iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr wdat_wdt joydev sg i2c_i801 lpc_ich mei_me mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm igb mdio ahci mlx4_core(OE) ptp drm_panel_orientation_quirks libahci pps_core dca devlink i2c_algo_bit libata mlx_compat(OE) megaraid_sas
CPU: 51 PID: 51239 Comm: runc Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff891e0026e180 ti: ffff889612bb8000 task.ti: ffff889612bb8000
RIP: 0010:[<ffffffff971c457e>] [<ffffffff971c457e>] put_page+0x2e/0x60
RSP: 0018:ffff889612bbbd20 EFLAGS: 00000202
RAX: 00afffff0000000c RBX: ffffd5e8bfee04c0 RCX: 0000000000000001
RDX: 0000000000000007 RSI: 0000000000000001 RDI: ffffd5e8bfee06c0
RBP: ffff889612bbbd20 R08: ffff892013bf7800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff971c4930
R13: 0000000000000000 R14: ffffffff971c524c R15: ffff889612bbbcc8
FS: 00007f90e4956740(0000) GS:ffff88df3fac0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c001191000 CR3: 000000695ae42000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff97232fb2>] mem_cgroup_reparent_charges+0x232/0x3c0
[<ffffffff972332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9712573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff971259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff9724dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff972530d1>] do_rmdir+0x1f1/0x220
[<ffffffff97247034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff97254335>] SyS_unlinkat+0x25/0x40
[<ffffffff97774ddb>] system_call_fastpath+0x22/0x27
Code: 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 12 48 8b 57 20 8b 92 90 00 00 00 83 ea 01 83 fa 01 76 22 f6 c4 c0 75 27 f0 ff 4f 1c <0f> 94 c0 84 c0 75 0b 5d c3 66 0f 1f 84 00 00 00 00 00 e8 8b ff
Complete report #436579
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff94832edb
Build id: 3.10.0-957.el7.x86_64
|
0x15b
|
|
2
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff948332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
3
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff9472573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
4
|
cgroup_rmdir
|
vmlinux |
0xffffffff947259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
5
|
vfs_rmdir
|
vmlinux |
0xffffffff9484dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
6
|
do_rmdir
|
vmlinux |
0xffffffff948530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
7
|
|
vmlinux |
0xffffffff94847034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
8
|
SyS_unlinkat
|
vmlinux |
0xffffffff94854335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff94d74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#34 stuck for 23s! [runc:88709]
Modules linked in: nf_conntrack_netbios_ns nf_conntrack_broadcast tcp_lp fuse nvidia_uvm(OE) xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iTCO_wdt iTCO_vendor_support vfat fat intel_powerclamp coretemp nvidia_drm(POE) nvidia_modeset(POE) kvm_intel kvm irqbypass crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nvidia(POE) snd_hda_codec_hdmi dm_service_time joydev cdc_ether pcspkr usbnet mii snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep sg snd_seq snd_seq_device snd_pcm mei_me ipmi_ssif snd_timer snd i2c_i801 mei soundcore dm_multipath ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel acpi_cpufreq acpi_pad acpi_power_meter ip_tables xfs libcrc32c sd_mod lpfc ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crc32c_intel nvmet_fc nvmet drm crc_t10dif crct10dif_generic crct10dif_pclmul igb nvme_fc ahci nvme_fabrics nvme_core libahci megaraid_sas scsi_transport_fc ptp libata pps_core drm_panel_orientation_quirks scsi_tgt dca crct10dif_common i2c_algo_bit wmi dm_mirror dm_region_hash
dm_log dm_mod
CPU: 34 PID: 88709 Comm: runc Tainted: P W OE ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Nettrix X640 G40/60WB32, BIOS NKMH041018 10/08/2021
task: ffff8ab824a28000 ti: ffff8af338af8000 task.ti: ffff8af338af8000
RIP: 0010:[<ffffffff94832edb>] [<ffffffff94832edb>] mem_cgroup_reparent_charges+0x15b/0x3c0
RSP: 0018:ffff8af338afbd30 EFLAGS: 00000246
RAX: 0000000000000002 RBX: ffffef38bd6b6bc0 RCX: 0000000000000003
RDX: 0000000000000002 RSI: ffffef38bcd8c45c RDI: 0000000000f36311
RBP: ffff8af338afbda0 R08: ffff8ab339f35800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffef395a56a880
R13: 0000000000000246 R14: ffffffff947c4930 R15: 0000000000000000
FS: 00007f72bb318740(0000) GS:ffff8af33e680000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00093d000 CR3: 0000004157882000 CR4: 0000000000760fe0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff948332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9472573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff947259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff9484dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff948530d1>] do_rmdir+0x1f1/0x220
[<ffffffff94847034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff94854335>] SyS_unlinkat+0x25/0x40
[<ffffffff94d74ddb>] system_call_fastpath+0x22/0x27
Code: c6 4c 89 ef e8 27 72 53 00 48 89 df e8 3f 3d 00 00 41 8b 57 fc 49 89 c6 85 d2 74 ac 8d 4a 01 49 8d 77 fc 89 d0 f0 41 0f b1 4f fc <39> c2 89 c1 0f 85 bb 01 00 00 48 89 df e8 b3 70 f9 ff 85 c0 0f
Complete report #412245
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff97232edb
Build id: 3.10.0-957.el7.x86_64
|
0x15b
|
|
2
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff972332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
3
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff9712573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
4
|
cgroup_rmdir
|
vmlinux |
0xffffffff971259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
5
|
vfs_rmdir
|
vmlinux |
0xffffffff9724dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
6
|
do_rmdir
|
vmlinux |
0xffffffff972530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
7
|
|
vmlinux |
0xffffffff97247034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
8
|
SyS_unlinkat
|
vmlinux |
0xffffffff97254335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
9
|
system_call_fastpath
|
vmlinux |
0xffffffff97774ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#29 stuck for 22s! [migration/29:155]
NMI watchdog: BUG: soft lockup - CPU#97 stuck for 22s! [runc:26083]
Modules linked in: rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) beegfs(OE) obdclass(OE) lnet(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc libcfs(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache overlay(T) bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) sunrpc dm_mirror dm_region_hash
dm_log dm_mod iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr wdat_wdt joydev sg i2c_i801 lpc_ich mei_me mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm igb mdio ahci mlx4_core(OE) ptp drm_panel_orientation_quirks libahci pps_core dca devlink i2c_algo_bit libata mlx_compat(OE) megaraid_sas
CPU: 97 PID: 26083 Comm: runc Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff88ff1998b0c0 ti: ffff8890e9b10000 task.ti: ffff8890e9b10000
RIP: 0010:[<ffffffff97232edb>] [<ffffffff97232edb>] mem_cgroup_reparent_charges+0x15b/0x3c0
RSP: 0018:ffff8890e9b13d30 EFLAGS: 00000246
RAX: 0000000000000002 RBX: 0000000000000000 RCX: 0000000000000003
RDX: 0000000000000002 RSI: ffffd5e8d09ea61c RDI: 0000000005427a98
RBP: ffff8890e9b13da0 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffffd5e8bafcd240
R13: 0000000000000246 R14: ffffffff971c4930 R15: 0000000000000000
FS: 00007f0b4754c740(0000) GS:ffff893f3fa40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000558a26937a60 CR3: 000000dac68f2000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff972332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9712573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff971259e2>] cgroup_rmdir+0x22/0x40
[<ffffffff9724dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffff972530d1>] do_rmdir+0x1f1/0x220
[<ffffffff97247034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff97254335>] SyS_unlinkat+0x25/0x40
[<ffffffff97774ddb>] system_call_fastpath+0x22/0x27
Code: c6 4c 89 ef e8 27 72 53 00 48 89 df e8 3f 3d 00 00 41 8b 57 fc 49 89 c6 85 d2 74 ac 8d 4a 01 49 8d 77 fc 89 d0 f0 41 0f b1 4f fc <39> c2 89 c1 0f 85 bb 01 00 00 48 89 df e8 b3 70 f9 ff 85 c0 0f
Complete report #212828
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9eb15375
Build id: 3.10.0-862.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffff9e60cf99
Build id: 3.10.0-862.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffff9e60d3d4
Build id: 3.10.0-862.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffff9e51c1da
Build id: 3.10.0-862.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffff9e51c482
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffff9e6295ac
Build id: 3.10.0-862.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffff9e62cb31
Build id: 3.10.0-862.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffff9eb1a5bc
Build id: 3.10.0-862.el7.x86_64
|
0x1bc
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffff9e62dd95
Build id: 3.10.0-862.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffff9eb1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [runc:151732]
Modules linked in: veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) nvidia_uvm(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink nfsv3 nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) vfat fat skx_edac coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass joydev i40iw ib_core pcspkr ses enclosure sg lpc_ich i2c_i801 mei_me shpchp
mei ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter nfsd sch_fq_codel binfmt_misc auth_rpcgss nfs_acl lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic 8021q garp mrp stp llc crct10dif_pclmul crct10dif_common crc32_pclmul crc32c_intel ast ghash_clmulni_intel drm_kms_helper aesni_intel igb syscopyarea lrw sysfillrect gf128mul sysimgblt scsi_transport_iscsi glue_helper fb_sys_fops ablk_helper ttm cryptd i40e ahci drm dca smartpqi libahci i2c_algo_bit libata ptp scsi_transport_sas pps_core i2c_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 17 PID: 151732 Comm: runc Kdump: loaded Tainted: P OE ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Inspur NF5468M5/YZMB-01130-10C, BIOS 4.1.14 03/27/2020
task: ffff8a93b1620fd0 ti: ffff8a6941ed8000 task.ti: ffff8a6941ed8000
RIP: 0010:[<ffffffff9eb15375>] [<ffffffff9eb15375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a6941edbd20 EFLAGS: 00000283
RAX: 0000000000000283 RBX: ffffe9cb45172800 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000283 RDI: 0000000000000283
RBP: ffff8a6941edbd20 R08: ffff8a97b46c3000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffff9e5a2400
R13: 0000000000000000 R14: ffffffff9e5a2d1c R15: ffff8a6941edbcc8
FS: 00007ff67c94d740(0000) GS:ffff8ac7bda40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000056000a97cf00 CR3: 0000005ba9238000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff9e60cf99>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffff9e60d3d4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9e51c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffff9e51c482>] cgroup_rmdir+0x22/0x40
[<ffffffff9e6295ac>] vfs_rmdir+0xdc/0x150
[<ffffffff9e62cb31>] do_rmdir+0x1f1/0x220
[<ffffffff9eb1a5bc>] ? __do_page_fault+0x1bc/0x4f0
[<ffffffff9e62dd95>] SyS_unlinkat+0x25/0x40
[<ffffffff9eb1f7d5>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #133633
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb3915375
Build id: 3.10.0-862.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffb340cf99
Build id: 3.10.0-862.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffb340d3d4
Build id: 3.10.0-862.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffb331c1da
Build id: 3.10.0-862.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffb331c482
Build id: 3.10.0-862.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffb34295ac
Build id: 3.10.0-862.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffb342cb31
Build id: 3.10.0-862.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffb3420d84
Build id: 3.10.0-862.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffb342dd95
Build id: 3.10.0-862.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffb391f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#74 stuck for 22s! [runc:56211]
Modules linked in: xt_nat veth vxlan ip6_udp_tunnel udp_tunnel xt_mark nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc nfsd auth_rpcgss nfs_acl lockd grace nls_utf8 isofs loop ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod sunrpc ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm iTCO_wdt iTCO_vendor_support vfat fat i40iw ib_core skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel
kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper joydev ablk_helper cryptd mei_me ses mei ipmi_ssif enclosure scsi_transport_sas pcspkr sg i2c_i801 lpc_ich shpchp ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm i40e drm ahci libahci megaraid_sas libata ptp pps_core i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 74 PID: 56211 Comm: runc Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Inspur NF8480M5/IIMS, BIOS 4.1.11 12/26/2019
task: ffff891b80ed0fd0 ti: ffff891e6de48000 task.ti: ffff891e6de48000
RIP: 0010:[<ffffffffb3915375>] [<ffffffffb3915375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff891e6de4bd20 EFLAGS: 00000287
RAX: 0000000000000287 RBX: ffffc8b195457b00 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff891e6de4bd20 R08: 0000000000000101 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffb33a2400
R13: 0000000000000000 R14: ffffffffb33a2d1c R15: ffff891e6de4bcc8
FS: 00007f883af8c740(0000) GS:ffff89dbfee80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f16d89e5930 CR3: 00000120464d0000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb340cf99>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffffb340d3d4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffb331c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffffb331c482>] cgroup_rmdir+0x22/0x40
[<ffffffffb34295ac>] vfs_rmdir+0xdc/0x150
[<ffffffffb342cb31>] do_rmdir+0x1f1/0x220
[<ffffffffb3420d84>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffb342dd95>] SyS_unlinkat+0x25/0x40
[<ffffffffb391f7d5>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #12922
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_lock_irqsave
|
vmlinux |
0xffffffffa976a45b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffa9232e92
Build id: 3.10.0-957.el7.x86_64
|
0x112
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffa92332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffa912573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffa91259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffa924dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffa92530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffa9247034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffa9254335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffa9774ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [runc:21475]
Modules linked in: beegfs(OE) veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat osc(OE) nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack mgc(OE) nf_conntrack lustre(OE) libcrc32c lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ipt_REJECT nf_reject_ipv4 tun ko2iblnd(OE) bridge stp llc ptlrpc(OE) obdclass(OE) lnet(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter libcfs(OE) nfsv3 nfs_acl nfs lockd grace fscache bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) sunrpc dm_mirror dm_region_hash
dm_log dm_mod iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd wdat_wdt pcspkr joydev sg mei_me i2c_i801 lpc_ich mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm ahci mdio libahci igb libata drm_panel_orientation_quirks ptp pps_core mlx4_core(OE) dca i2c_algo_bit megaraid_sas devlink mlx_compat(OE)
CPU: 22 PID: 21475 Comm: runc Kdump: loaded Tainted: G OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff9de5dd4fd140 ti: ffff9e0f39aac000 task.ti: ffff9e0f39aac000
RIP: 0010:[<ffffffffa976a45b>] [<ffffffffa976a45b>] _raw_spin_lock_irqsave+0xb/0x40
RSP: 0018:ffff9e0f39aafd10 EFLAGS: 00000283
RAX: 00efffff00000000 RBX: ffffffffa91c4930 RCX: 0000000000000001
RDX: 0000000000000009 RSI: 0000000000000001 RDI: ffff9d8fbffda540
RBP: ffff9e0f39aafd20 R08: ffff9ddbd260b800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffa91c524c
R13: ffff9e0f39aafcc8 R14: ffff9d8fbffda000 R15: ffff9d8fbffda000
FS: 00007f5aa049e740(0000) GS:ffff9d4f3fb80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001687028 CR3: 0000003ff80ba000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa9232e92>] mem_cgroup_reparent_charges+0x112/0x3c0
[<ffffffffa92332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa912573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa91259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffa924dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffa92530d1>] do_rmdir+0x1f1/0x220
[<ffffffffa9247034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffa9254335>] SyS_unlinkat+0x25/0x40
[<ffffffffa9774ddb>] system_call_fastpath+0x22/0x27
Code: 0f b1 17 85 c0 75 01 c3 55 89 c6 48 89 e5 e8 fd 1a ff ff 5d c3 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 9c <58> 0f 1f 44 00 00 48 89 c3 fa 66 0f 1f 44 00 00 31 c0 ba 01 00
Complete report #10836
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa976a0f5
Build id: 3.10.0-957.el7.x86_64
|
0x15
|
|
2
|
mem_cgroup_reparent_charges
|
vmlinux |
0xffffffffa9232eb9
Build id: 3.10.0-957.el7.x86_64
|
0x139
|
|
3
|
mem_cgroup_css_offline
|
vmlinux |
0xffffffffa92332f4
Build id: 3.10.0-957.el7.x86_64
|
0x84
|
|
4
|
cgroup_destroy_locked
|
vmlinux |
0xffffffffa912573a
Build id: 3.10.0-957.el7.x86_64
|
0xea
|
|
5
|
cgroup_rmdir
|
vmlinux |
0xffffffffa91259e2
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
6
|
vfs_rmdir
|
vmlinux |
0xffffffffa924dc2c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
7
|
do_rmdir
|
vmlinux |
0xffffffffa92530d1
Build id: 3.10.0-957.el7.x86_64
|
0x1f1
|
|
8
|
|
vmlinux |
0xffffffffa9247034
Build id: 3.10.0-957.el7.x86_64
|
0x24
|
|
9
|
SyS_unlinkat
|
vmlinux |
0xffffffffa9254335
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
10
|
system_call_fastpath
|
vmlinux |
0xffffffffa9774ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [runc:16230]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat osc(OE) nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack mgc(OE) nf_conntrack lustre(OE) libcrc32c lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ipt_REJECT nf_reject_ipv4 tun ko2iblnd(OE) bridge stp llc ptlrpc(OE) obdclass(OE) lnet(OE) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter libcfs(OE) nfsv3 nfs_acl nfs lockd grace fscache bonding rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) sunrpc dm_mirror dm_region_hash dm_log dm_mod
iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd wdat_wdt pcspkr joydev sg mei_me i2c_i801 lpc_ich mei ipmi_si ipmi_devintf ipmi_msghandler knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm ahci mdio libahci igb libata drm_panel_orientation_quirks ptp pps_core mlx4_core(OE) dca i2c_algo_bit megaraid_sas devlink mlx_compat(OE)
CPU: 18 PID: 16230 Comm: runc Kdump: loaded Tainted: G OE ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: Inspur TS860G3/TS860G3, BIOS TS860G3_4.0.07 10/09/2016
task: ffff9def35b45140 ti: ffff9d1fa5354000 task.ti: ffff9d1fa5354000
RIP: 0010:[<ffffffffa976a0f5>] [<ffffffffa976a0f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9d1fa5357d20 EFLAGS: 00000283
RAX: 0000000000000283 RBX: ffffd6d229e74700 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000283 RDI: 0000000000000283
RBP: ffff9d1fa5357d20 R08: 0000000000000101 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffffffffa91c4930
R13: 0000000000000000 R14: ffffffffa91c524c R15: ffff9d1fa5357cc8
FS: 00007fbc704da740(0000) GS:ffff9d4f3fa80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000491000 CR3: 0000002bc367a000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa9232eb9>] mem_cgroup_reparent_charges+0x139/0x3c0
[<ffffffffa92332f4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa912573a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa91259e2>] cgroup_rmdir+0x22/0x40
[<ffffffffa924dc2c>] vfs_rmdir+0xdc/0x150
[<ffffffffa92530d1>] do_rmdir+0x1f1/0x220
[<ffffffffa9247034>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffa9254335>] SyS_unlinkat+0x25/0x40
[<ffffffffa9774ddb>] system_call_fastpath+0x22/0x27
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48