NMI watchdog: BUG: soft lockup - CPU#14 stuck for 22s! [systemd:1]
Modules linked in: joydev vhost_net vhost macvtap macvlan ebt_arp tun fuse geneve ip6_udp_tunnel udp_tunnel bnx2fc cnic uio fcoe libfcoe libfc scsi_transport_fc scsi_tgt scsi_transport_iscsi nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack openvswitch ip_set nfnetlink ebtable_nat ebtable_broute ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ext4 mbcache jbd2 dm_mirror dm_region_hash dm_log dm_service_time dm_multipath skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi
kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support dell_smbios dcdbas sparse_keymap dm_mod pcspkr sg mei_me mei i2c_i801 lpc_ich shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt crc32c_intel fb_sys_fops ttm drm ahci tg3 libahci bnxt_en megaraid_sas libata i2c_core ptp devlink pps_core 8021q garp mrp bridge stp llc bonding
CPU: 14 PID: 1 Comm: systemd Kdump: loaded Tainted: G        W      ------------   3.10.0-862.2.3.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R640/06NR82, BIOS 2.4.8 11/26/2019
task: ffffa0b6a7658000 ti: ffffa07727384000 task.ti: ffffa07727384000
RIP: 0010:[<ffffffff9b1153a5>]  [<ffffffff9b1153a5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffffa07727387c78  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffffa07727387c78 R08: 0000000000000000 R09: fffff2a880000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffffa07727387bf8
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff9aba28cb
FS:  00007f42a94d0940(0000) GS:ffffa0b57cdc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f84808d7390 CR3: 0000007f7c836000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff9aba2ca5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff9aba23a0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff9aba2f34>] __lru_cache_add+0x64/0x90
[<ffffffff9aba32fe>] lru_cache_add+0xe/0x10
[<ffffffff9aba7b9d>] putback_lru_page+0x3d/0xb0
[<ffffffff9ac0d02a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff9ac0d374>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9ab1c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffff9ab1c482>] cgroup_rmdir+0x22/0x40
[<ffffffff9ac2954c>] vfs_rmdir+0xdc/0x150
[<ffffffff9ac2cad1>] do_rmdir+0x1f1/0x220
[<ffffffff9ac1d18e>] ? ____fput+0xe/0x10
[<ffffffff9ac2dd06>] SyS_rmdir+0x16/0x20
[<ffffffff9b11fa11>] tracesys+0x9d/0xc3
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [runc:60500]
Modules linked in: fuse tcp_diag udp_diag inet_diag xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma rdma_ucm ib_uverbs ib_iser rdma_cm
ib_umad iw_cm libiscsi ib_ipoib scsi_transport_iscsi ib_cm mlx4_ib ib_core dm_mirror dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr zfs(POE) zunicode(POE) zavl(POE) icp(POE) zcommon(POE) znvpair(POE) spl(OE) raid0 joydev ses enclosure mei_me mei ioatdma sg i2c_i801 lpc_ich shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c raid1 raid10 sd_mod crc_t10dif crct10dif_generic mlx4_en ast drm_kms_helper syscopyarea sysfillrect mlx4_core sysimgblt fb_sys_fops ttm mpt3sas drm ahci igb libahci crct10dif_pclmul
crct10dif_common crc32c_intel libata raid_class devlink ptp scsi_transport_sas pps_core dca i2c_algo_bit i2c_core
CPU: 2 PID: 60500 Comm: runc Kdump: loaded Tainted: P        W  OE  ------------ T 3.10.0-862.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-6018U-TR4+/X10DRU-i+, BIOS 2.0c 04/21/2017
task: ffff8e3363cd0fd0 ti: ffff8e48c25c4000 task.ti: ffff8e48c25c4000
RIP: 0010:[<ffffffffa0da551b>]  [<ffffffffa0da551b>] release_pages+0x9b/0x430
RSP: 0018:ffff8e48c25c7bf8  EFLAGS: 00000246
RAX: ffff8e64ff495c00 RBX: ffff8e3363cd0fd0 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff8e64ff495bd0
RBP: ffff8e48c25c7c78 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 7fffffffffffffff
R13: ffff8e48c25c7cc8 R14: ffff8e48c25c7c00 R15: 000000009171adba
FS:  00007fb0bdc7d740(0000) GS:ffff8e64ff480000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f18d603d9e0 CR3: 00000025c0918000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa0da59bc>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffffa0da50a0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffffa0da5c34>] __lru_cache_add+0x64/0x90
[<ffffffffa0da5ffe>] lru_cache_add+0xe/0x10
[<ffffffffa0daa89d>] putback_lru_page+0x3d/0xb0
[<ffffffffa0e1144a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffffa0e11794>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa0d1f47a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa0d1f722>] cgroup_rmdir+0x22/0x40
[<ffffffffa0e2da4c>] vfs_rmdir+0xdc/0x150
[<ffffffffa0e30fd1>] do_rmdir+0x1f1/0x220
[<ffffffffa0e25234>] ? SYSC_newfstatat+0x24/0x60
[<ffffffffa0e32235>] SyS_unlinkat+0x25/0x40
[<ffffffffa132579b>] system_call_fastpath+0x22/0x27
Code: 48 8d 75 b0 4c 89 f7 e8 e4 78 1c 00 49 83 c4 08 4c 3b 65 a8 0f 84 96 01 00 00 49 8b 1c 24 41 89 d9 41 83 e1 01 0f 84 0f 03 00 00 <4d> 85 ff 74 0e 41 83 c5 01 41 83 fd 20 0f 84 12 02 00 00 45 84 

NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [slurmstepd:38428]
Modules linked in: squashfs loop overlay(T) nfs_layout_nfsv41_files osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) sha512_ssse3 sha512_generic crypto_null libcfs(OE) dell_rbu rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache dm_mirror dm_region_hash dm_log dm_mod ses enclosure sd_mod sg intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd serio_raw pcspkr uas usb_storage smartpqi scsi_transport_sas nvme hpilo hpwdt lpc_ich shpchp wmi acpi_power_meter rpcrdma ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_iser libiscsi scsi_transport_iscsi ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm
iw_cm nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ipmi_devintf hfi1(OE) i2c_algo_bit i2c_core crct10dif_pclmul rdmavt(OE) crct10dif_common crc32c_intel bnxt_en ib_core ipmi_si fjes ipmi_msghandler
CPU: 6 PID: 38428 Comm: slurmstepd Tainted: G        W  OEL ------------ T 3.10.0-514.10.2.el7.x86_64 #1
Hardware name: HPE ProLiant DL560 Gen10/ProLiant DL560 Gen10, BIOS U34 04/21/2022
task: ffff89f275772f10 ti: ffff89f8eefac000 task.ti: ffff89f8eefac000
RIP: 0010:[<ffffffff8168dceb>]  [<ffffffff8168dceb>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff89f8eefafca0  EFLAGS: 00000246
RAX: 0000000000000800 RBX: ffff89f8eefafcd8 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff89f8eefafca8 R08: 0000000000000000 R09: 0000000000000003
R10: 0000000000000001 R11: 0000000000000001 R12: ffff89f8eefafc28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff8118f05f
FS:  00002b778a193300(0000) GS:ffff88fc3f980000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3179f05000 CR3: 000002a0a8c66000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
000000000000000e ffff89f8eefafcf8 ffffffff8118f435 0000000000000000
ffffffff8118eb20 0000000000000246 ffffea027bc3a840 ffff88fc3f991a20
0000000000000000 0000000000000000 ffffea027bc3a860 ffff89f8eefafd18
Call Trace:
[<ffffffff8118f435>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff8118eb20>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff8118f6c5>] __lru_cache_add+0x65/0x90
[<ffffffff8118fa9e>] lru_cache_add+0xe/0x10
[<ffffffff81194bdd>] putback_lru_page+0x3d/0xb0
[<ffffffff811f130a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1654>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d257>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d4f2>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a278>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f315>] do_rmdir+0x1a5/0x200
[<ffffffff8168b5b0>] ? __schedule+0x3b0/0x990
[<ffffffff812104c6>] SyS_rmdir+0x16/0x20
[<ffffffff81696b09>] system_call_fastpath+0x16/0x1b
Code: 08 e8 6a 0e a0 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6 

NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [systemd:1]
Modules linked in: nvidia_uvm(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 beegfs(OE) dns_resolver nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod vfat fat iTCO_wdt iTCO_vendor_support skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw
gf128mul glue_helper ablk_helper cryptd pcspkr joydev ipmi_ssif sg mei_me mei shpchp lpc_ich i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c nvidia_drm(POE) nvidia_modeset(POE) raid1 sd_mod crc_t10dif crct10dif_generic nvidia(POE) ast i2c_algo_bit mlx5_core crct10dif_pclmul drm_kms_helper crct10dif_common crc32c_intel i40e ttm syscopyarea sysfillrect sysimgblt fb_sys_fops drm devlink ahci libahci ptp pps_core libata i2c_core wmi
CPU: 11 PID: 1 Comm: systemd Tainted: P           OE  ------------   3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-4029GP-TRT2/X11DPG-OT-CPU, BIOS 3.3 02/21/2020
task: ffff880127330000 ti: ffff880127324000 task.ti: ffff880127324000
RIP: 0010:[<ffffffff816adb15>]  [<ffffffff816adb15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880127327c78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880127327c78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880127327bf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff811927df
FS:  00007fbfdd8b6940(0000) GS:ffff883f7e8c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000558d826c1120 CR3: 0000003f7e166000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bb5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff81204fde>] ? ____fput+0xe/0x10
[<ffffffff810aedec>] ? task_work_run+0xac/0xf0
[<ffffffff81215396>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#24 stuck for 23s! [slurmstepd:104802]
Modules linked in: beegfs(OE) nvidia_uvm(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod vfat fat iTCO_wdt iTCO_vendor_support skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw
gf128mul glue_helper ablk_helper cryptd pcspkr joydev ipmi_ssif sg mei_me mei i2c_i801 shpchp lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c nvidia_drm(POE) nvidia_modeset(POE) raid1 sd_mod crc_t10dif crct10dif_generic nvidia(POE) ast i2c_algo_bit mlx5_core crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel i40e ttm syscopyarea sysfillrect sysimgblt fb_sys_fops drm devlink ahci ptp libahci pps_core libata i2c_core wmi [last unloaded: beegfs]
CPU: 24 PID: 104802 Comm: slurmstepd Tainted: P        W  OEL ------------   3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-4029GP-TRT2/X11DPG-OT-CPU, BIOS 3.3 02/21/2020
task: ffff88229f923f40 ti: ffff883f76a5c000 task.ti: ffff883f76a5c000
RIP: 0010:[<ffffffff816adb15>]  [<ffffffff816adb15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff883f76a5fc78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff883f76a5fc78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff883f76a5fbf8
R13: 0000000000000206 R14: 0000000000000010 R15: ffffffff811927df
FS:  00002b49a77aa440(0000) GS:ffff887f7db00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd726540000 CR3: 0000007664234000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bb5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff81215396>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [slurmstepd:166636]
Modules linked in: joydev devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 auth_rpcgss nfsv4 dell_rbu dns_resolver dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul crc32c_intel iTCO_wdt iTCO_vendor_support mgag200 dell_smbios ttm dcdbas sparse_keymap drm_kms_helper ghash_clmulni_intel syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter
bridge stp llc xfs isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common ahci libahci libata i40e(OE) megaraid_sas tg3(OE) ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 18 PID: 166636 Comm: slurmstepd Tainted: G           OE  ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/04JN2K, BIOS 2.21.1 03/07/2024
task: ffff88021882eeb0 ti: ffff880294fc8000 task.ti: ffff880294fc8000
RIP: 0010:[<ffffffff816ab865>]  [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880294fcbca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880294fcbca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880294fcbc28
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff811909d0
FS:  00002aaaaaafd180(0000) GS:ffff882fb8c40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aaab001d000 CR3: 0000000f97229000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880294fcbcf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea00c858c740 ffff882fb8c51ae0 0000000000000000
0000000000000000 ffffea00c858c760 ffff880294fcbd18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816a8fad>] ? __schedule+0x39d/0x8b0
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [docker-runc:169695]
Modules linked in: veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack libcrc32c br_netfilter bridge overlay(T) nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache 8021q garp mrp stp llc bonding rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ip6_tables ib_cm iw_cm i40iw ib_core skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support sg joydev pcspkr mei_me mei lpc_ich i2c_i801 shpchp
ipmi_si ipmi_devintf ipmi_msghandler nfit acpi_cpufreq libnvdimm wmi nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sr_mod sd_mod cdrom crc_t10dif crct10dif_generic uas usb_storage ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe igb drm crct10dif_pclmul i40e crct10dif_common mpt3sas crc32c_intel ahci libahci libata mdio dca raid_class i2c_algo_bit scsi_transport_sas ptp i2c_core pps_core
CPU: 12 PID: 169695 Comm: docker-runc Tainted: G        W    L ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Sugon CX50-G35/60P16B-TC4600T, BIOS 0KLSN218 05/22/2019
task: ffff881bf7f22f70 ti: ffff882d85790000 task.ti: ffff882d85790000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff882d85793ca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff882d85793ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff882d85793c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190aef
FS:  0000000002740880(0063) GS:ffff882fdda80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000867fe0 CR3: 00000022e5cb1000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff882d85793cf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea00a4fb9980 ffff882fdda91ae0 0000000000000000
0000000000000000 ffffea00a4fb99a0 ffff882d85793d18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81206a04>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff816a904d>] ? __schedule+0x39d/0x8b0
[<ffffffff81213475>] SyS_unlinkat+0x25/0x40
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [docker-runc:20593]
Modules linked in: veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype xt_conntrack nf_nat nf_conntrack br_netfilter bridge squashfs loop overlay(T) nfsv3 8021q garp mrp stp llc rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm ib_core amd64_edac_mod edac_mce_amd edac_core kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd hpilo sg sp5100_tco hpwdt i2c_piix4 fam15h_power shpchp k10temp acpi_power_meter ipmi_si ipmi_devintf ipmi_msghandler pcspkr
pcc_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper bnx2x syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_generic pata_acpi drm ahci pata_atiixp libahci libata crct10dif_pclmul hpsa crct10dif_common crc32c_intel mdio libcrc32c ptp serio_raw i2c_core pps_core scsi_transport_sas
CPU: 25 PID: 20593 Comm: docker-runc Tainted: G             L ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: HP ProLiant BL465c Gen8, BIOS A26 11/02/2014
task: ffff88204035cf10 ti: ffff88206f874000 task.ti: ffff88206f874000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88206f877ca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff882fffa21000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88206f877ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88206f877c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190aef
FS:  00007fa903693740(0000) GS:ffff88403eb00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000559bb1fd1b40 CR3: 0000003ff9b15000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88206f877cf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea009dbfa440 ffff88403eb11ae0 0000000000000000
0000000000000000 ffffea009dbfa460 ffff88206f877d18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213475>] SyS_unlinkat+0x25/0x40
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [slurmstepd:70957]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) squashfs loop overlay(T) osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) sha512_ssse3 sha512_generic nfs_layout_nfsv41_files crypto_null libcfs(OE) dell_rbu rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache dm_mirror dm_region_hash dm_log ses enclosure sd_mod sg dm_mod intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd serio_raw pcspkr uas usb_storage nvme smartpqi scsi_transport_sas hpilo hpwdt lpc_ich shpchp wmi acpi_power_meter rpcrdma ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_iser libiscsi scsi_transport_iscsi ib_ipoib rdma_ucm
ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ipmi_devintf hfi1(OE) i2c_algo_bit i2c_core crct10dif_pclmul crct10dif_common rdmavt(OE) crc32c_intel bnxt_en ib_core ipmi_si fjes ipmi_msghandler
CPU: 3 PID: 70957 Comm: slurmstepd Tainted: G        W  OEL ------------ T 3.10.0-514.10.2.el7.x86_64 #1
Hardware name: HPE ProLiant DL560 Gen10/ProLiant DL560 Gen10, BIOS U34 03/16/2023
task: ffff8bc9c2c70000 ti: ffff8a456eb60000 task.ti: ffff8a456eb60000
RIP: 0010:[<ffffffff8118f05f>]  [<ffffffff8118f05f>] release_pages+0x14f/0x430
RSP: 0018:ffff8a456eb63c28  EFLAGS: 00000202
RAX: ffff88fc3fcd1a00 RBX: ffff8bc9c2c70000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff88fc3fcd1a30
RBP: ffff8a456eb63ca8 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8a456eb63cf8
R13: ffff8a456eb63cf0 R14: ffff8bc9c2c70000 R15: ffff8a456eb63fd8
FS:  00002b985c193300(0000) GS:ffff88fc3fcc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000002a4bdf0 CR3: 000003c00c4ae000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000000000000 0000000000000001 0000000000000003 0000000000000000
0000000000000001 ffff88fc3fcd1aa0 ffff8a456eb63c58 ffff8a456eb63c58
ffff8a456eb63c68 ffff8a456eb63c68 00000000364993ed 000000000000000e
Call Trace:
[<ffffffff8118f44c>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffff8118eb20>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff8118f6c5>] __lru_cache_add+0x65/0x90
[<ffffffff8118fa9e>] lru_cache_add+0xe/0x10
[<ffffffff81194bdd>] putback_lru_page+0x3d/0xb0
[<ffffffff811f130a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1654>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d257>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d4f2>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a278>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f315>] do_rmdir+0x1a5/0x200
[<ffffffff81692305>] ? do_page_fault+0x35/0x90
[<ffffffff812104c6>] SyS_rmdir+0x16/0x20
[<ffffffff81696b09>] system_call_fastpath+0x16/0x1b
Code: 31 ed e8 85 ee 4f 00 48 8b 55 98 44 8b 4d 90 48 89 45 a0 49 89 d7 49 8b 06 f6 c4 40 0f 84 9a 02 00 00 f0 41 ff 4e 50 f0 ff 4b 1c <0f> 94 c0 84 c0 0f 84 22 ff ff ff 48 8b 03 4c 8d 73 20 a8 20 0f 

NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [slurmstepd:41784]
Modules linked in: nvidia_uvm(OE) binfmt_misc fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache nvidia_drm(POE) nvidia_modeset(POE) btrfs iTCO_wdt iTCO_vendor_support nvidia(POE) raid456 async_raid6_recov async_memcpy async_pq coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel vfat lrw gf128mul fat glue_helper ablk_helper cryptd raid6_pq async_xor xor async_tx pcspkr cdc_ether joydev usbnet ipmi_ssif mii i2c_i801 sg shpchp ipmi_si ipmi_devintf ipmi_msghandler pinctrl_lewisburg pinctrl_intel
mei_me acpi_cpufreq mei acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic uas usb_storage nouveau ast video mxm_wmi drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea nvme sysfillrect sysimgblt fb_sys_fops nvme_core ttm ahci drm igb libahci libata ptp pps_core dca i2c_algo_bit i2c_core wmi dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 41784 Comm: slurmstepd Kdump: loaded Tainted: P        W  OEL ------------   3.10.0-862.el7.x86_64 #1
Hardware name: N/A X660 G45/60WB32, BIOS NKMH051061 05/12/2023
task: ffff8b1e8437dee0 ti: ffff8b34eea8c000 task.ti: ffff8b34eea8c000
RIP: 0010:[<ffffffff9eb15375>]  [<ffffffff9eb15375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8b34eea8fc78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8b34eea8fc78 R08: 0000000000000000 R09: ffffe05740000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8b34eea8fbf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff9e5a292f
FS:  00002b714c41e400(0000) GS:ffff8b283fcc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002ad4b1faa000 CR3: 000001fad6fd0000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff9e5a2d05>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff9e5a2400>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff9e5a2f94>] __lru_cache_add+0x64/0x90
[<ffffffff9e5a335e>] lru_cache_add+0xe/0x10
[<ffffffff9e5a7bfd>] putback_lru_page+0x3d/0xb0
[<ffffffff9e60d08a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff9e60d3d4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9e51c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffff9e51c482>] cgroup_rmdir+0x22/0x40
[<ffffffff9e6295ac>] vfs_rmdir+0xdc/0x150
[<ffffffff9e62cb31>] do_rmdir+0x1f1/0x220
[<ffffffff9eb1a5bc>] ? __do_page_fault+0x1bc/0x4f0
[<ffffffff9e62dd66>] SyS_rmdir+0x16/0x20
[<ffffffff9eb1f7d5>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [slurmstepd:208383]
Modules linked in: nvidia_uvm(OE) nouveau mxm_wmi video xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver beegfs(OE) nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) vfat fat iTCO_wdt iTCO_vendor_support nvidia(POE) skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel
kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_ssif pcspkr joydev sg mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c raid1 sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper mlx5_core crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea sysfillrect sysimgblt i40e fb_sys_fops ttm drm ahci devlink libahci ptp libata pps_core i2c_core wmi
CPU: 22 PID: 208383 Comm: slurmstepd Tainted: P           OEL ------------   3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-4029GP-TRT2/X11DPG-OT-CPU, BIOS 3.3 02/21/2020
task: ffff887f4e80dee0 ti: ffff887d0bec4000 task.ti: ffff887d0bec4000
RIP: 0010:[<ffffffff816adb15>]  [<ffffffff816adb15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff887d0bec7c78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff887d0bec7c78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff887d0bec7bf8
R13: ffff887d0bec7c78 R14: 0000000000000010 R15: ffffffff811927df
FS:  00002b9a35eec340(0000) GS:ffff887f7da80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5dc1f33180 CR3: 0000007f4e84e000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bb5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff816b3801>] ? __do_page_fault+0x171/0x450
[<ffffffff81215396>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [systemd:1]
Modules linked in: ceph libceph ipt_rpfilter iptable_raw nf_conntrack_netlink xt_multiport xt_set ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6_tables xt_statistic xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark xt_addrtype ipt_REJECT nf_reject_ipv4 xt_comment xt_conntrack iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle ip_tables joydev binfmt_misc ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth tun rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip_set nfnetlink dm_mirror dm_region_hash dm_log dm_mod sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support snd_hda_codec_hdmi snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq pcspkr snd_seq_device snd_pcm ipmi_ssif snd_timer snd i2c_i801 soundcore mei_me sg mei shpchp lpc_ich ipmi_si ipmi_devintf ipmi_msghandler acpi_pad fuse nfsd br_netfilter bridge auth_rpcgss nfs_acl stp lockd llc grace overlay(T) sunrpc xfs libcrc32c nvidia_drm(POE) nvidia_modeset(POE) sd_mod crc_t10dif crct10dif_generic nvidia(POE) ast mxm_wmi drm_kms_helper ahci ttm libahci syscopyarea sysfillrect sysimgblt fb_sys_fops libata crct10dif_pclmul crct10dif_common igb drm crc32c_intel ptp pps_core dca i2c_algo_bit i2c_core wmi [last unloaded: ip_tables]
CPU: 12 PID: 1 Comm: systemd Tainted: P           OEL ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: TYAN B7079F77CV10HR-N/S7079GM2NR-N, BIOS V2.03.B10 09/07/2016
task: ffff880137290000 ti: ffff880137284000 task.ti: ffff880137284000
RIP: 0010:[<ffffffff816ab865>]  [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880137287ca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880137287ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880137287c28
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff81190a0b
FS:  00007f7b484ef900(0000) GS:ffff885fbf000000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3a32a673e0 CR3: 0000002fba7a3000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880137287cf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea0165972940 ffff885fbf011ae0 0000000000000000
0000000000000000 ffffea0165972960 ffff880137287d18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff810ad24c>] ? task_work_run+0xac/0xf0
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#17 stuck for 23s! [docker-runc:4495]
Modules linked in: veeamsnap(OE) ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge overlay(T) nfsv3 nfs_acl nfs lockd grace fscache dm_round_robin binfmt_misc 8021q garp mrp stp llc bonding rpcrdma sunrpc ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib vfat rdma_ucm fat iTCO_wdt iTCO_vendor_support ib_ucm sb_edac edac_core intel_powerclamp ib_uverbs coretemp intel_rapl iosf_mbi kvm_intel ib_umad kvm rdma_cm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ib_cm glue_helper ablk_helper dm_multipath cryptd iw_cm ipmi_ssif pcspkr sg
i2c_i801 lpc_ich hpilo hpwdt ioatdma ipmi_si ipmi_devintf ipmi_msghandler shpchp pcc_cpufreq wmi acpi_power_meter mlx4_ib ib_core ip_tables xfs mlx4_en sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper syscopyarea bnx2x sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul ahci crct10dif_common mlx4_core crc32c_intel drm libahci igb libata dca mdio ptp i2c_algo_bit pps_core i2c_core devlink libcrc32c dm_mirror dm_region_hash dm_log dm_mod
CPU: 17 PID: 4495 Comm: docker-runc Tainted: G           OEL ------------ T 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: HP ProLiant XL170r Gen9/ProLiant XL170r Gen9, BIOS U14 05/17/2022
task: ffff883f32f9af70 ti: ffff88423e2f4000 task.ti: ffff88423e2f4000
RIP: 0010:[<ffffffff816adb15>]  [<ffffffff816adb15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88423e2f7c78  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88423e2f7c78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88423e2f7bf8
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff81192a41
FS:  00007f6d975c2740(0000) GS:ffff887f7eec0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c420fdd010 CR3: 000000419a058000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bb5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff8120878e>] ? SYSC_newstat+0x2e/0x60
[<ffffffff816b3801>] ? __do_page_fault+0x171/0x450
[<ffffffff812153c5>] SyS_unlinkat+0x25/0x40
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [slurmstepd:18605]
Modules linked in: falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14812(E) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache panfs(POE) 8021q garp mrp stp llc falcon_lsm_pinned_14810(E) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) mlx5_fpga_tools(OE) ip6_tables iptable_filter edac_mce_amd edac_core kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel dell_smbios lrw gf128mul glue_helper dcdbas ablk_helper cryptd sparse_keymap sg mlx5_ib(OE) mlx4_ib(OE) mlx4_en(OE) ib_uverbs(OE) pcspkr ipmi_si ipmi_devintf ipmi_msghandler wmi i2c_piix4 acpi_power_meter shpchp mlx5_core(OE) mlxfw(OE) vfio_mdev(OE) binfmt_misc nfsd vfio_iommu_type1 vfio mdev(OE) ptp pps_core auth_rpcgss nfs_acl mlx4_core(OE) lockd devlink grace knem(OE) sunrpc ip_tables ext4 mbcache
jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci crct10dif_pclmul drm crct10dif_common libahci crc32c_intel libata bnxt_en i2c_core megaraid_sas ib_ipoib(OE) ib_cm(OE) ib_umad(OE) ib_core(OE) mlx_compat(OE) [last unloaded: falcon_kal]
CPU: 0 PID: 18605 Comm: slurmstepd Tainted: P        W  OEL ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0PYVT1, BIOS 1.4.8 05/06/2020
task: ffff8919dd606eb0 ti: ffff897dfd260000 task.ti: ffff897dfd260000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff897dfd263c28  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff883c04531000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff897dfd263c28 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff897dfd263ba8
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff81190d4a
FS:  00002b5ba43db300(0000) GS:ffff887e7e200000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000020bd4a0 CR3: 000001427c022000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff897dfd263c78 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea042eb53240 ffff887e7e211ae0 0000000000000000
0000000000000000 ffffea042eb53260 ffff897dfd263c98 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffffc07388fd>] unload_network_ops_symbols+0x432d/0x72a0 [falcon_lsm_pinned_14810]
[<ffffffffc08268fd>] unload_network_ops_symbols+0x432d/0x72a0 [falcon_lsm_pinned_14812]
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [systemd:1]
Modules linked in: tcp_lp vhost_net vhost macvtap macvlan fuse ext4 mbcache jbd2 tq_base(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter iTCO_wdt iTCO_vendor_support ipmi_ssif skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper
cryptd ses enclosure pcspkr sg joydev mei_me i2c_i801 lpc_ich mei shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm i40e smartpqi ahci scsi_transport_sas uas libahci usb_storage ptp libata i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 30 PID: 1 Comm: systemd Tainted: G           OE  ------------   3.10.0-693.el7.x86_64 #1
Hardware name: ZTE              R5300 G4/R5300G4, BIOS 03.18.0300_70716 11/25/2020
task: ffff88606efb0000 ti: ffff88012e804000 task.ti: ffff88012e804000
RIP: 0010:[<ffffffff816ab845>]  [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88012e807c88  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88012e807c88 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88012e807c08
R13: ffff88012e807c88 R14: 0000000000000010 R15: ffffffff81190abf
FS:  00007efca07ba940(0000) GS:ffff88bebba80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001a6464c CR3: 0000005eb6aad000 CR4: 00000000003427e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88012e807cd8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea01bdd86340 ffff88bebba91ae0 0000000000000000
0000000000000000 ffffea01bdd86360 ffff88012e807cf8 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f341a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3764>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f626>] vfs_rmdir+0xd6/0x150
[<ffffffff81212251>] do_rmdir+0x1f1/0x220
[<ffffffff8120321e>] ? ____fput+0xe/0x10
[<ffffffff810ad24c>] ? task_work_run+0xac/0xf0
[<ffffffff81213456>] SyS_rmdir+0x16/0x20
[<ffffffffc06534a9>] hook_sys_rmdir+0x39/0x40 [tq_base]
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#22 stuck for 22s! [slurmstepd:163644]
Modules linked in: 8021q garp mrp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm dm_mirror dm_region_hash dm_log dm_mod skx_edac edac_core iTCO_wdt intel_powerclamp iTCO_vendor_support coretemp intel_rapl iosf_mbi kvm hfi1 irqbypass crc32_pclmul rdmavt ghash_clmulni_intel aesni_intel ib_core lrw gf128mul glue_helper mei_me ablk_helper cryptd joydev pcspkr sg mei
lpc_ich i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ixgbe ttm crc32c_intel ahci mdio drm libahci ptp libata pps_core i2c_core dca
CPU: 22 PID: 163644 Comm: slurmstepd Tainted: G        W      ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Advanced Clustering S2600BPB/S2600BPB, BIOS SE5C620.86B.00.01.0009.C2.101920170742 10/19/2017
task: ffff88474a3e6eb0 ti: ffff8842e1f4c000 task.ti: ffff8842e1f4c000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8842e1f4fca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff882e61a99800 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8842e1f4fca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8842e1f4fc28
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff81190a3b
FS:  00002aea08755d00(0000) GS:ffff885f7be80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007febd8ecf000 CR3: 0000004834786000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8842e1f4fcf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea0049efe700 ffff885f7be91ae0 0000000000000000
0000000000000000 ffffea0049efe720 ffff8842e1f4fd18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#33 stuck for 23s! [systemd:1]
Modules linked in: beegfs(OE) squashfs loop overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) vfat fat iTCO_wdt iTCO_vendor_support nvidia(POE) skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi
kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev ipmi_ssif sg mei_me mei lpc_ich i2c_i801 shpchp ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c raid1 sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper mlx5_core crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea sysfillrect i40e sysimgblt fb_sys_fops ttm drm ahci devlink libahci ptp pps_core libata i2c_core wmi [last unloaded: beegfs]
CPU: 33 PID: 1 Comm: systemd Tainted: P    B      OEL ------------ T 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-4029GP-TRT2/X11DPG-OT-CPU, BIOS 3.3 02/21/2020
task: ffff8801272d8000 ti: ffff8801272e0000 task.ti: ffff8801272e0000
RIP: 0010:[<ffffffff8119269d>]  [<ffffffff8119269d>] release_pages+0xd/0x430
RSP: 0018:ffff8801272e3c68  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8801272e3bf8 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff887f7dd55af0
RBP: ffff8801272e3c78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 0000000000000010
R13: ffffffff811927df R14: ffffffffffffff10 R15: ffff887f7dd55af0
FS:  00007ff77414b940(0000) GS:ffff887f7dd40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9c70fbc000 CR3: 0000003f7e06c000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bcc>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff81204fde>] ? ____fput+0xe/0x10
[<ffffffff810aedec>] ? task_work_run+0xac/0xf0
[<ffffffff81215396>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 8b 04 24 48 85 c0 75 e6 eb 94 48 c7 c0 00 fe ff ff 89 c2 e9 d7 fe ff ff 66 0f 1f 44 00 00 0f 1f 44 00 00 55 48 89 e5 41 57 41 56 <41> 55 41 54 53 48 83 ec 58 89 55 80 65 48 8b 04 25 28 00 00 00 

NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [slurmstepd:48929]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) ip6_udp_tunnel udp_tunnel rdma_ucm ib_uverbs rdma_cm iw_cm ib_cm ib_core mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c br_netfilter bridge 8021q garp mrp stp llc overlay(T) ip6_tables iptable_filter iTCO_wdt iTCO_vendor_support coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ses joydev enclosure cdc_ether scsi_transport_sas usbnet mii sg mei_me mei shpchp i2c_i801 wmi ipmi_si ipmi_devintf
ipmi_msghandler acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm serio_raw ahci libahci mlx5_core(OE) libata megaraid_sas mlxfw(OE) ptp pps_core auxiliary(OE) mlx_compat(OE) i2c_core devlink [last unloaded: tracedev]
CPU: 1 PID: 48929 Comm: slurmstepd Tainted: G        W  OEL ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Lenovo ThinkSystem SR650 V2/7Z73CTO1WW, BIOS AFE112A-1.02 05/10/2021
task: ffff89f806769fa0 ti: ffff891c03cfc000 task.ti: ffff891c03cfc000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff891c03cffca8  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff891c03cffca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff891c03cffc28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190aef
FS:  00002b3240a6cfc0(0000) GS:ffff89f83da40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0287ed000 CR3: 000001f78bab2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff891c03cffcf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea0318fc8980 ffff89f83da51ae0 0000000000000000
0000000000000000 ffffea0318fc89a0 ffff891c03cffd18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [systemd:1]
Modules linked in: veth xt_addrtype br_netfilter overlay(T) nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache devlink nls_utf8 isofs loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter iTCO_wdt iTCO_vendor_support dell_smbios sparse_keymap dcdbas vfat fat skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_ssif joydev sg pcspkr ipmi_si ipmi_devintf ipmi_msghandler mei_me mei shpchp nfit i2c_i801 lpc_ich acpi_power_meter wmi libnvdimm acpi_pad tpm_crb
nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit crct10dif_pclmul crct10dif_common drm_kms_helper crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci tg3 libata megaraid_sas bnxt_en ptp i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 1 Comm: systemd Tainted: G        W    L ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R740/0DY2X0, BIOS 2.8.2 08/27/2020
task: ffff8840a7618000 ti: ffff880127164000 task.ti: ffff880127164000
RIP: 0010:[<ffffffff816ab845>]  [<ffffffff816ab845>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880127167ca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880127167ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880127167c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190abf
FS:  00007f0b0aa9c940(0000) GS:ffff883f7e6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2cd4003108 CR3: 000000ff7bbfa000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff880127167cf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea03547900c0 ffff883f7e6d1ae0 0000000000000000
0000000000000000 ffffea03547900e0 ffff880127167d18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f341a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3764>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f626>] vfs_rmdir+0xd6/0x150
[<ffffffff81212251>] do_rmdir+0x1f1/0x220
[<ffffffff810ad24c>] ? task_work_run+0xac/0xf0
[<ffffffff81213456>] SyS_rmdir+0x16/0x20
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [runc:29395]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ses enclosure ext4 mbcache jbd2 kvm_amd kvm dcdbas irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_devintf pcspkr ipmi_si sp5100_tco shpchp ipmi_msghandler amd64_edac_mod sg edac_mce_amd i2c_piix4 edac_core acpi_power_meter fam15h_power k10temp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt
crct10dif_pclmul fb_sys_fops crct10dif_common ttm crc32c_intel drm ahci serio_raw libahci mpt2sas libata i2c_core raid_class megaraid_sas scsi_transport_sas bnx2 fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 22 PID: 29395 Comm: runc Tainted: G        W      ------------ T 3.10.0-514.26.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R815/0W13NR, BIOS 3.1.1 04/13/2013
task: ffff880108ea8fb0 ti: ffff8875a1954000 task.ti: ffff8875a1954000
RIP: 0010:[<ffffffff8168e92b>]  [<ffffffff8168e92b>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff8875a1957ca0  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8875a1957ca8 R08: 0000000000000000 R09: 0000000000000002
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8875a1957c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff8118f55f
FS:  00007eff180b4740(0000) GS:ffff88801ed00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc8168df500 CR3: 00000071fd96f000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
000000000000000e ffff8875a1957cf8 ffffffff8118f935 0000000000000000
ffffffff8118f020 0000000000000246 ffffea01f680c5c0 ffff88801ed11a20
0000000000000000 0000000000000000 ffffea01f680c5e0 ffff8875a1957d18
Call Trace:
[<ffffffff8118f935>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff8118f020>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff8118fbc5>] __lru_cache_add+0x65/0x90
[<ffffffff8118ff9e>] lru_cache_add+0xe/0x10
[<ffffffff811950dd>] putback_lru_page+0x3d/0xb0
[<ffffffff811f19ca>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1d14>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d6f7>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d992>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a948>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f9e5>] do_rmdir+0x1a5/0x200
[<ffffffff812040b4>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff81210bc5>] SyS_unlinkat+0x25/0x40
[<ffffffff81697809>] system_call_fastpath+0x16/0x1b
Code: 08 e8 9a 05 a0 ff 5d c3 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 53 48 89 f3 66 66 66 66 90 66 83 07 02 48 89 df 57 9d <66> 66 90 66 90 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6 

NMI watchdog: BUG: soft lockup - CPU#25 stuck for 22s! [docker-runc:155771]
Modules linked in: veth xt_nat xt_comment osc(OE) mgc(OE) xt_addrtype br_netfilter overlay(T) sep4_1(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) socperf2_0(OE) xt_CHECKSUM iptable_mangle libcfs(OE) ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter pax(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) vfat fat skx_edac coretemp intel_rapl iosf_mbi iTCO_wdt iTCO_vendor_support kvm joydev irqbypass sg ipmi_si ipmi_devintf ipmi_msghandler
pcspkr mei_me nfit mei lpc_ich i2c_i801 shpchp libnvdimm acpi_power_meter nfsd binfmt_misc auth_rpcgss nfs_acl lockd sch_fq_codel grace knem(OE) ip_tables ext4 mbcache jbd2 mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic ast 8021q i2c_algo_bit garp mrp drm_kms_helper stp llc crct10dif_pclmul crct10dif_common crc32_pclmul scsi_transport_iscsi crc32c_intel syscopyarea ghash_clmulni_intel sysfillrect sysimgblt fb_sys_fops ttm aesni_intel lrw gf128mul mlx4_core(OE) glue_helper ahci ablk_helper cryptd drm i40e libahci mlx_compat(OE) megaraid_sas devlink libata ptp i2c_core pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 25 PID: 155771 Comm: docker-runc Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Inspur NF5280M5/YZMB-00882-104, BIOS 4.0.7 09/06/2018
task: ffff93b835388000 ti: ffff93b8317d4000 task.ti: ffff93b8317d4000
RIP: 0010:[<ffffffff94715375>]  [<ffffffff94715375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff93b8317d7c78  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff93b8317d7c78 R08: 0000000000000000 R09: fffff4ccc0000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff93b8317d7bf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff941a292f
FS:  00007f5668cab740(0000) GS:ffff93f83dac0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f169e9b4de8 CR3: 0000004c3aaf0000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff941a2d05>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff941a2400>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff941a2f94>] __lru_cache_add+0x64/0x90
[<ffffffff941a335e>] lru_cache_add+0xe/0x10
[<ffffffff941a7bfd>] putback_lru_page+0x3d/0xb0
[<ffffffff9420d08a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff9420d3d4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff9411c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffff9411c482>] cgroup_rmdir+0x22/0x40
[<ffffffff942295ac>] vfs_rmdir+0xdc/0x150
[<ffffffff9422cb31>] do_rmdir+0x1f1/0x220
[<ffffffff94220cce>] ? SYSC_newstat+0x2e/0x60
[<ffffffff9422dd95>] SyS_unlinkat+0x25/0x40
[<ffffffff9471f7d5>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#102 stuck for 22s! [systemd:1]
Modules linked in: veth cmac arc4 md4 nls_utf8 cifs ccm xt_addrtype br_netfilter nfsv3 nfs_acl xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter svcrdma(OE) rpcrdma(OE) xprtrdma(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) overlay(T) sunrpc raid0 skx_edac iTCO_wdt iTCO_vendor_support coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw
gf128mul glue_helper ablk_helper cryptd pcspkr nvme joydev nvme_core sg mei_me mei lpc_ich shpchp i2c_i801 ipmi_ssif wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter knem(OE) ip_tables xfs libcrc32c mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mlx4_core(OE) drm i40e ahci megaraid_sas libahci devlink libata mlx_compat(OE) ptp i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 102 PID: 1 Comm: systemd Kdump: loaded Tainted: G        W  OE  ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Sugon I840-G30/80P48-US, BIOS 0THSY016 04/16/2018
task: ffff961b13940000 ti: ffff95fb93628000 task.ti: ffff95fb93628000
RIP: 0010:[<ffffffffb63a292f>]  [<ffffffffb63a292f>] release_pages+0x14f/0x430
RSP: 0018:ffff95fb9362bbf8  EFLAGS: 00000202
RAX: ffff963a3e995c00 RBX: ffff961b13940000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff963a3e995b90
RBP: ffff95fb9362bc78 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 7fffffffffffffff
R13: ffff95fb9362bcc8 R14: ffff95fb9362bb60 R15: 00000000f9385862
FS:  00007fd1b4e67940(0000) GS:ffff963a3e980000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005650dacc6708 CR3: 0000007ff8058000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb63a2d1c>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffffb63a2400>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffffb63a2f94>] __lru_cache_add+0x64/0x90
[<ffffffffb63a335e>] lru_cache_add+0xe/0x10
[<ffffffffb63a7bfd>] putback_lru_page+0x3d/0xb0
[<ffffffffb640d08a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffffb640d3d4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffb631c1da>] cgroup_destroy_locked+0xea/0x370
[<ffffffffb631c482>] cgroup_rmdir+0x22/0x40
[<ffffffffb64295ac>] vfs_rmdir+0xdc/0x150
[<ffffffffb642cb31>] do_rmdir+0x1f1/0x220
[<ffffffffb641d1ee>] ? ____fput+0xe/0x10
[<ffffffffb62b7850>] ? task_work_run+0xc0/0xe0
[<ffffffffb642dd66>] SyS_rmdir+0x16/0x20
[<ffffffffb691f7d5>] system_call_fastpath+0x1c/0x21
Code: 31 ed e8 c5 2d 57 00 48 8b 55 98 44 8b 4d 90 48 89 45 a0 49 89 d7 49 8b 06 f6 c4 40 0f 84 9a 02 00 00 f0 41 ff 4e 50 f0 ff 4b 1c <0f> 94 c0 84 c0 0f 84 22 ff ff ff 48 8b 03 4c 8d 73 20 a8 20 0f 

NMI watchdog: BUG: soft lockup - CPU#19 stuck for 23s! [slurmstepd:251183]
Modules linked in: 8021q garp mrp stp llc nfsv3 nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm ib_core dm_mirror dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi ast i2c_algo_bit kvm ttm ixgbe drm_kms_helper irqbypass crc32_pclmul syscopyarea sysfillrect sysimgblt ghash_clmulni_intel fb_sys_fops ptp drm aesni_intel pps_core lrw gf128mul glue_helper mdio i2c_i801 ablk_helper mei_me dca cryptd joydev sg pcspkr i2c_core shpchp lpc_ich mei ipmi_si wmi ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace
sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ahci libahci libata
CPU: 19 PID: 251183 Comm: slurmstepd Kdump: loaded Not tainted 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: Advanced Clustering S2600BPB/S2600BPB, BIOS SE5C620.86B.02.01.0008.C1.031920191559 03/19/2019
task: ffff9e4524ab6eb0 ti: ffff9e45372a0000 task.ti: ffff9e45372a0000
RIP: 0010:[<ffffffffa01a568f>]  [<ffffffffa01a568f>] release_pages+0x14f/0x430
RSP: 0018:ffff9e45372a3bf8  EFLAGS: 00000202
RAX: ffff9e753be55c00 RBX: ffff9e4524ab6eb0 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff9e753be55bd0
RBP: ffff9e45372a3c78 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 7fffffffffffffff
R13: ffff9e45372a3cc8 R14: ffff9e45372a3b60 R15: 00000000f95f530f
FS:  00002ad2d5984fc0(0000) GS:ffff9e753be40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd375788ad0 CR3: 0000002f5a7a6000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa01a5a7c>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffffa01a5160>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffffa01a5cf4>] __lru_cache_add+0x64/0x90
[<ffffffffa01a60be>] lru_cache_add+0xe/0x10
[<ffffffffa01aa95d>] putback_lru_page+0x3d/0xb0
[<ffffffffa021150a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffffa0211854>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffa011f4aa>] cgroup_destroy_locked+0xea/0x370
[<ffffffffa011f752>] cgroup_rmdir+0x22/0x40
[<ffffffffa022db0c>] vfs_rmdir+0xdc/0x150
[<ffffffffa0231091>] do_rmdir+0x1f1/0x220
[<ffffffffa02322c6>] SyS_rmdir+0x16/0x20
[<ffffffffa072579b>] system_call_fastpath+0x22/0x27
Code: 31 ed e8 35 61 57 00 48 8b 55 98 44 8b 4d 90 48 89 45 a0 49 89 d7 49 8b 06 f6 c4 40 0f 84 9a 02 00 00 f0 41 ff 4e 50 f0 ff 4b 1c <0f> 94 c0 84 c0 0f 84 22 ff ff ff 48 8b 03 4c 8d 73 20 a8 20 0f 

NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [systemd:1]
Modules linked in: uas usb_storage ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack_netlink ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_conntrack iptable_filter iptable_mangle rpcsec_gss_krb5 fuse nf_tables(T) xt_statistic xt_comment xt_nat veth vxlan ip6_udp_tunnel udp_tunnel xt_mark nfnetlink br_netfilter tcp_diag inet_diag ext4 mbcache jbd2 binfmt_misc tun bridge stp llc ebtable_filter ebtables dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop ip6table_filter ip6_tables team_mode_activebackup team_mode_loadbalance team ipt_REJECT nf_reject_ipv4 xt_limit vboxpci(OE) vboxnetadp(OE) vboxnetflt(OE) xt_CHECKSUM xt_addrtype iptable_nat nf_nat_ipv4 nf_nat vboxdrv(OE) sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm joydev irqbypass crc32_pclmul ghash_clmulni_intel
iTCO_wdt aesni_intel lrw iTCO_vendor_support gf128mul glue_helper dcdbas ablk_helper cryptd pcspkr lpc_ich mei_me mei sg shpchp ipmi_si ipmi_devintf ipmi_msghandler wmi acpi_power_meter nfsd auth_rpcgss nf_conntrack_ipv4 nfs_acl lockd nf_defrag_ipv4 nf_conntrack grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ahci crc32c_intel drm libahci libata megaraid_sas tg3 i2c_core ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: xt_conntrack]
CPU: 6 PID: 1 Comm: systemd Tainted: G           OEL ------------ T 3.10.0-693.17.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R730/0H21J3, BIOS 1.2.10 03/09/2015
task: ffff8830b75f8000 ti: ffff880136c48000 task.ti: ffff880136c48000
RIP: 0010:[<ffffffff816adce5>]  [<ffffffff816adce5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff880136c4bc78  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff880136c4bc78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff880136c4bbf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff811929c2
FS:  00007f6dcddfa940(0000) GS:ffff882fbecc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000559800a188d0 CR3: 0000002fbe08e000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192d95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81192490>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81193024>] __lru_cache_add+0x64/0x90
[<ffffffff811933ee>] lru_cache_add+0xe/0x10
[<ffffffff81197c0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f554a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f5894>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f647>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f8f2>] cgroup_rmdir+0x22/0x40
[<ffffffff812116f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214391>] do_rmdir+0x1f1/0x220
[<ffffffff812051de>] ? ____fput+0xe/0x10
[<ffffffff810aefcc>] ? task_work_run+0xac/0xf0
[<ffffffff81215596>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#24 stuck for 22s! [slurmstepd:210650]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log joydev iTCO_wdt iTCO_vendor_support dell_smbios dcdbas sparse_keymap skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat
br_netfilter bridge stp llc xfs isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci megaraid_sas libahci tg3(OE) ptp libata pps_core [last unloaded: tracedev]
CPU: 24 PID: 210650 Comm: slurmstepd Tainted: G    B      OEL ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 2.8.1 06/30/2020
task: ffff883126643f40 ti: ffff8800192d8000 task.ti: ffff8800192d8000
RIP: 0010:[<ffffffff816ab865>]  [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8800192dbca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8800192dbca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8800192dbc28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190abf
FS:  00002aaaaaafd180(0000) GS:ffff882f7dd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000007318cf010 CR3: 0000002aa8809000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8800192dbcf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea00f8ec6480 ffff882f7dd11ae0 0000000000000000
0000000000000000 ffffea00f8ec64a0 ffff8800192dbd18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [docker-runc:2266]
Modules linked in: binfmt_misc nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache xfrm6_mode_tunnel xfrm4_mode_tunnel esp4 drbg ansi_cprng veth vxlan ip6_udp_tunnel udp_tunnel xt_mark xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_multiport iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables
iptable_filter sunrpc joydev amd64_edac_mod sp5100_tco edac_mce_amd edac_core sg k10temp pcspkr kvm_amd ppdev kvm ipmi_ssif irqbypass parport_pc parport ipmi_si i2c_piix4 shpchp ipmi_devintf ipmi_msghandler tpm_infineon acpi_cpufreq ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm igb mpt2sas e1000e mdio serio_raw dca raid_class i2c_algo_bit ptp scsi_transport_sas i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 2266 Comm: docker-runc Tainted: G               ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: empty empty/S8812, BIOS V1.08     07/09/2013
task: ffff880115b44f10 ti: ffff880102c6c000 task.ti: ffff880102c6c000
RIP: 0010:[<ffffffff81190abf>]  [<ffffffff81190abf>] release_pages+0x14f/0x430
RSP: 0018:ffff880102c6fc28  EFLAGS: 00000202
RAX: ffff881fff491b00 RBX: ffff880115b44f10 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff881fff491af0
RBP: ffff880102c6fca8 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 7fffffffffffffff
R13: ffff880102c6fcf8 R14: ffff880115b44f10 R15: ffff880102c6ffd8
FS:  00007fb4dc51d740(0000) GS:ffff881fff480000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562ff2cdf000 CR3: 00000017ea3eb000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
0000000000000000 0000000000000001 ffffea0000000000 0000000000000000
0000000000000001 ffff881fff491b60 ffff880102c6fc58 ffff880102c6fc58
ffff880102c6fc68 ffff880102c6fc68 00000000b544af87 000000000000000e
Call Trace:
[<ffffffff81190eac>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f341a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3764>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f626>] vfs_rmdir+0xd6/0x150
[<ffffffff81212251>] do_rmdir+0x1f1/0x220
[<ffffffff81206a84>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff81213485>] SyS_unlinkat+0x25/0x40
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 31 ed e8 05 b1 51 00 48 8b 55 98 44 8b 4d 90 48 89 45 a0 49 89 d7 49 8b 06 f6 c4 40 0f 84 9a 02 00 00 f0 41 ff 4e 50 f0 ff 4b 1c <0f> 94 c0 84 c0 0f 84 22 ff ff ff 48 8b 03 4c 8d 73 20 a8 20 0f 

NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [docker-runc:20763]
Modules linked in: ip6t_REJECT nf_reject_ipv6 xt_owner xt_REDIRECT nf_nat_redirect vxlan ip6_udp_tunnel udp_tunnel xt_statistic ipt_REJECT nf_reject_ipv4 ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype xt_conntrack nf_nat nf_conntrack br_netfilter overlay(T) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bonding bridge stp llc iTCO_wdt iTCO_vendor_support sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev i2c_i801 lpc_ich mei_me sg mei ioatdma shpchp wmi ipmi_ssif
ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm igb ahci crct10dif_pclmul crct10dif_common libahci crc32c_intel megaraid_sas libata ptp pps_core dca i2c_algo_bit i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 20763 Comm: docker-runc Tainted: G        W    L ------------ T 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-6028R-TR/X10DRi, BIOS 1.1 04/14/2015
task: ffff883f2da93f40 ti: ffff880414068000 task.ti: ffff880414068000
RIP: 0010:[<ffffffff816adb15>]  [<ffffffff816adb15>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88041406bc78  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88041406bc78 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88041406bbf8
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff8119272b
FS:  00007f9bb3d74740(0000) GS:ffff881fffa80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c420483380 CR3: 0000001195f08000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff81192bb5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811922b0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81192e44>] __lru_cache_add+0x64/0x90
[<ffffffff8119320e>] lru_cache_add+0xe/0x10
[<ffffffff81197a2d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff8120878e>] ? SYSC_newstat+0x2e/0x60
[<ffffffff816b3801>] ? __do_page_fault+0x171/0x450
[<ffffffff812153c5>] SyS_unlinkat+0x25/0x40
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#24 stuck for 22s! [slurmstepd:129427]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log dcdbas skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm iTCO_wdt iTCO_vendor_support irqbypass crc32_pclmul crc32c_intel mgag200 ghash_clmulni_intel ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci
libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) megaraid_sas ahci tg3(OE) libahci ptp pps_core libata [last unloaded: ipmi_msghandler]
CPU: 24 PID: 129427 Comm: slurmstepd Tainted: G           OEL ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff880f01c69fa0 ti: ffff88321a2fc000 task.ti: ffff88321a2fc000
RIP: 0010:[<ffffffff81190abf>]  [<ffffffff81190abf>] release_pages+0x14f/0x430
RSP: 0018:ffff88321a2ffc28  EFLAGS: 00000202
RAX: ffff882f7dd11b00 RBX: ffff880f01c69fa0 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 000000000000000e RDI: ffff882f7dd11af0
RBP: ffff88321a2ffca8 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000001 R12: 7fffffffffffffff
R13: ffff88321a2ffcf8 R14: ffff880f01c69fa0 R15: ffff88321a2fffd8
FS:  00002aaaaaafd180(0000) GS:ffff882f7dd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000004821a10 CR3: 00000031fd401000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000000000000 0000000000000001 ffffea0000000000 0000000000000000
0000000000000001 ffff882f7dd11b60 ffff88321a2ffc58 ffff88321a2ffc58
ffff88321a2ffc68 ffff88321a2ffc68 0000000037e1e878 000000000000000e
Call Trace:
[<ffffffff81190eac>] pagevec_lru_move_fn+0x10c/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 31 ed e8 25 b1 51 00 48 8b 55 98 44 8b 4d 90 48 89 45 a0 49 89 d7 49 8b 06 f6 c4 40 0f 84 9a 02 00 00 f0 41 ff 4e 50 f0 ff 4b 1c <0f> 94 c0 84 c0 0f 84 22 ff ff ff 48 8b 03 4c 8d 73 20 a8 20 0f 

NMI watchdog: BUG: soft lockup - CPU#23 stuck for 22s! [slurmstepd:35521]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt iTCO_vendor_support dell_smbios dcdbas sparse_keymap crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt aesni_intel lrw fb_sys_fops gf128mul glue_helper drm ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich wmi nfit ipmi_si libnvdimm ipmi_devintf ipmi_msghandler acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs
isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci megaraid_sas libahci tg3(OE) ptp libata pps_core [last unloaded: ipmi_msghandler]
CPU: 23 PID: 35521 Comm: slurmstepd Tainted: G           OEL ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 2.5.4 01/14/2020
task: ffff885f6a9a8000 ti: ffff88025c240000 task.ti: ffff88025c240000
RIP: 0010:[<ffffffff816ab865>]  [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88025c243ca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88025c243ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88025c243c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190abf
FS:  00002aaaaaafd180(0000) GS:ffff885f7c0c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aaab473d680 CR3: 0000005f57eda000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88025c243cf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea0107dcbc80 ffff885f7c0d1ae0 0000000000000000
0000000000000000 ffffea0107dcbca0 ffff88025c243d18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [slurmstepd:119633]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) ebtable_filter devlink ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log dcdbas skx_edac iTCO_wdt iTCO_vendor_support edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci libsas qla3xxx hpsa e1000e
ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci libahci megaraid_sas libata tg3(OE) ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 2 PID: 119633 Comm: slurmstepd Tainted: G        W  OE  ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff884bdbe48fd0 ti: ffff88306739c000 task.ti: ffff88306739c000
RIP: 0010:[<ffffffff816ab865>]  [<ffffffff816ab865>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88306739fca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff882f4fbc0800 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88306739fca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88306739fc28
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff81190abb
FS:  00002aaaaaafd180(0000) GS:ffff882f7da40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffff7ff7000 CR3: 000000329bb5d000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88306739fcf8 ffffffff81190e95 0000000000000000 ffffffff81190590
0000000000000246 ffffea012556bf40 ffff882f7da51ae0 0000000000000000
0000000000000000 ffffea012556bf60 ffff88306739fd18 ffffffff81191124
Call Trace:
[<ffffffff81190e95>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff81190590>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191124>] __lru_cache_add+0x64/0x90
[<ffffffff811914ee>] lru_cache_add+0xe/0x10
[<ffffffff81195d0d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [slurmstepd:197355]
Modules linked in: squashfs loop overlay(T) nvidia_uvm(OE) beegfs(OE) nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) ipmi_devintf vfio_iommu_type1 vfio nfsv3 nfs_acl opa_vnic hfi1 rdmavt 8021q garp mrp stp llc rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache iTCO_wdt iTCO_vendor_support skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ixgbe cryptd ipmi_ssif ast pcspkr ttm ptp joydev pps_core mdio dca mei_me sg lpc_ich mei shpchp i2c_i801 wmi ipmi_si nfit libnvdimm acpi_cpufreq acpi_power_meter acpi_pad rpcrdma sunrpc ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm
ib_uverbs ib_umad rdma_cm ib_cm iw_cm binfmt_misc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic drm_kms_helper drm i2c_algo_bit crct10dif_pclmul ahci crct10dif_common crc32c_intel ib_core syscopyarea libahci sysfillrect sysimgblt fb_sys_fops i2c_core libata ipmi_msghandler [last unloaded: beegfs]
CPU: 35 PID: 197355 Comm: slurmstepd Tainted: P           OE  ------------ T 3.10.0-862.3.3.el7.x86_64 #1
Hardware name: Supermicro SYS-1029GQ-TVRT/X11DGQ, BIOS 2.0b 02/28/2018
task: ffff90e53e6e9fa0 ti: ffff90f443ef8000 task.ti: ffff90f443ef8000
RIP: 0010:[<ffffffffb7b16475>]  [<ffffffffb7b16475>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff90f443efbc78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff90f443efbc78 R08: 0000000000000000 R09: ffffd75a00000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff90f443efbbf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffffb75a2c6f
FS:  00002b8a988420c0(0000) GS:ffff90f55ddc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdad89589e0 CR3: 0000002ec3ef2000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb75a3045>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffffb75a2740>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffffb75a32d4>] __lru_cache_add+0x64/0x90
[<ffffffffb75a369e>] lru_cache_add+0xe/0x10
[<ffffffffb75a7f3d>] putback_lru_page+0x3d/0xb0
[<ffffffffb760d3ca>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffffb760d714>] mem_cgroup_css_offline+0x84/0x140
[<ffffffffb751c50a>] cgroup_destroy_locked+0xea/0x370
[<ffffffffb751c7b2>] cgroup_rmdir+0x22/0x40
[<ffffffffb76298ec>] vfs_rmdir+0xdc/0x150
[<ffffffffb762ce71>] do_rmdir+0x1f1/0x220
[<ffffffffb762e0a6>] SyS_rmdir+0x16/0x20
[<ffffffffb7b2082f>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [slurmstepd:38838]
Modules linked in: nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core iTCO_wdt iTCO_vendor_support
intel_powerclamp coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg sb_edac edac_core ipmi_ssif hpilo hpwdt lpc_ich ioatdma shpchp ipmi_si ipmi_msghandler acpi_power_meter pcc_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c mlx4_en sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common mgag200 crc32c_intel drm_kms_helper syscopyarea sysfillrect mlx4_core serio_raw sysimgblt fb_sys_fops ttm ahci igb hpsa libahci drm ptp devlink pps_core libata dca i2c_algo_bit mpt2sas i2c_core raid_class scsi_transport_sas wmi fjes dm_mirror dm_region_hash dm_log dm_mod [last unloaded: nvidia]
CPU: 1 PID: 38838 Comm: slurmstepd Tainted: P    B      OE  ------------   3.10.0-514.21.2.el7.x86_64 #1
Hardware name: HP ProLiant SL270s Gen8 SE/, BIOS P75 09/18/2013
task: ffff881f98e28000 ti: ffff882c76770000 task.ti: ffff882c76770000
RIP: 0010:[<ffffffff8168e86b>]  [<ffffffff8168e86b>] _raw_spin_unlock_irqrestore+0x1b/0x40
RSP: 0018:ffff882c76773ca0  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff882c76773ca8 R08: 0000000000000000 R09: 0000000000000003
R10: 0000000000000001 R11: 0000000000000001 R12: ffff882c76773c28
R13: 0000000000000246 R14: 0000000000000010 R15: ffffffff8118f7b1
FS:  00007f8cfb479780(0000) GS:ffff881fbf640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff438443000 CR3: 0000001d5e317000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
000000000000000e ffff882c76773cf8 ffffffff8118f925 0000000000000000
ffffffff8118f010 0000000000000246 ffffea008e1bb3c0 ffff881fbf651a20
0000000000000000 0000000000000000 ffffea008e1bb3e0 ffff882c76773d18
Call Trace:
[<ffffffff8118f925>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff8118f010>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff8118fbb5>] __lru_cache_add+0x65/0x90
[<ffffffff8118ff8e>] lru_cache_add+0xe/0x10
[<ffffffff811950cd>] putback_lru_page+0x3d/0xb0
[<ffffffff811f194a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1c94>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d6e7>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d982>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a8c8>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f965>] do_rmdir+0x1a5/0x200
[<ffffffff81210b16>] SyS_rmdir+0x16/0x20
[<ffffffff81697749>] system_call_fastpath+0x16/0x1b
Code: 08 e8 5a 06 a0 ff 5d c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 f3 0f 1f 44 00 00 66 83 07 02 48 89 df 57 9d <0f> 1f 44 00 00 5b 5d c3 0f 1f 44 00 00 8b 37 f0 66 83 07 02 f6 

NMI watchdog: BUG: soft lockup - CPU#25 stuck for 23s! [docker-runc:5055]
Modules linked in: veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype xt_conntrack nf_nat nf_conntrack br_netfilter bridge overlay(T) nfsv3 8021q garp mrp stp llc rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm ib_core amd64_edac_mod edac_mce_amd edac_core kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel sp5100_tco aesni_intel lrw i2c_piix4 gf128mul glue_helper ablk_helper ipmi_si cryptd ipmi_devintf sg k10temp fam15h_power hpilo hpwdt ipmi_msghandler shpchp pcspkr acpi_power_meter
pcc_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper bnx2x syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_generic drm pata_acpi ahci libahci pata_atiixp libata crct10dif_pclmul crct10dif_common hpsa crc32c_intel mdio libcrc32c serio_raw ptp i2c_core pps_core scsi_transport_sas
CPU: 25 PID: 5055 Comm: docker-runc Tainted: G               ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: HP ProLiant BL465c Gen8, BIOS A26 11/02/2014
task: ffff882f4f808fd0 ti: ffff88282544c000 task.ti: ffff88282544c000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff88282544fca8  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff88282544fca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88282544fc28
R13: 0000000000000206 R14: 0000000000000010 R15: ffffffff81190aef
FS:  00007f5f9eb76740(0000) GS:ffff88403eb00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055589924a330 CR3: 0000003284be3000 CR4: 00000000000407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88282544fcf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea00ff6efe80 ffff88403eb11ae0 0000000000000000
0000000000000000 ffffea00ff6efea0 ffff88282544fd18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81206a04>] ? SYSC_newfstatat+0x24/0x60
[<ffffffff81213475>] SyS_unlinkat+0x25/0x40
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#25 stuck for 22s! [slurmstepd:202618]
Modules linked in: 8021q garp mrp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink nfsv3 nfs fscache ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma rdma_ucm ib_uverbs dm_mirror ib_iser dm_region_hash dm_log rdma_cm opa_vnic iw_cm ib_umad dm_mod libiscsi ib_ipoib scsi_transport_iscsi ib_cm skx_edac intel_powerclamp coretemp intel_rapl iTCO_wdt iosf_mbi iTCO_vendor_support kvm irqbypass crc32_pclmul ghash_clmulni_intel hfi1 aesni_intel lrw gf128mul glue_helper ablk_helper cryptd rdmavt joydev ib_core sg pcspkr mei_me
mei ipmi_si shpchp i2c_i801 lpc_ich ipmi_devintf wmi ipmi_msghandler nfit libnvdimm acpi_cpufreq acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ixgbe ttm ahci crct10dif_pclmul crct10dif_common crc32c_intel mdio drm libahci ptp libata pps_core i2c_core dca
CPU: 25 PID: 202618 Comm: slurmstepd Kdump: loaded Tainted: G             L ------------   3.10.0-862.3.2.el7.x86_64 #1
Hardware name: Advanced Clustering S2600BPB/S2600BPB, BIOS SE5C620.86B.00.01.0013.C1.030920180427 03/09/2018
task: ffff94c382fe4f10 ti: ffff9487a9f28000 task.ti: ffff9487a9f28000
RIP: 0010:[<ffffffff93116475>]  [<ffffffff93116475>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9487a9f2bc78  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000004
RDX: 0000000000000000 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff9487a9f2bc78 R08: 0000000000000000 R09: ffffdec800000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff9487a9f2bbf8
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff92ba2c6f
FS:  00002b529683ad40(0000) GS:ffff94cc3bf40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b07add43338 CR3: 000000478be20000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff92ba3045>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff92ba2740>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff92ba32d4>] __lru_cache_add+0x64/0x90
[<ffffffff92ba369e>] lru_cache_add+0xe/0x10
[<ffffffff92ba7f3d>] putback_lru_page+0x3d/0xb0
[<ffffffff92c0d3ca>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff92c0d714>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff92b1c50a>] cgroup_destroy_locked+0xea/0x370
[<ffffffff92b1c7b2>] cgroup_rmdir+0x22/0x40
[<ffffffff92c298ec>] vfs_rmdir+0xdc/0x150
[<ffffffff92c2ce71>] do_rmdir+0x1f1/0x220
[<ffffffff92c2e0a6>] SyS_rmdir+0x16/0x20
[<ffffffff9312082f>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48 

NMI watchdog: BUG: soft lockup - CPU#47 stuck for 23s! [slurmstepd:77756]
Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) 8021q garp mrp stp llc ip6_tables rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi amd64_edac_mod edac_mce_amd edac_core kvm_amd ib_srpt kvm target_core_mod irqbypass crc32_pclmul ib_srp scsi_transport_srp ghash_clmulni_intel scsi_tgt dcdbas aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm pcspkr sg ccp shpchp i2c_piix4 ib_core ipmi_si ipmi_devintf ipmi_msghandler tpm_crb acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200
i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm crct10dif_pclmul libahci crct10dif_common crc32c_intel tg3 libata megaraid_sas bnxt_en ptp i2c_core pps_core
CPU: 47 PID: 77756 Comm: slurmstepd Tainted: G           OEL ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7415/07YXFK, BIOS 1.8.7 04/02/2019
task: ffff881ff437bf40 ti: ffff8814acfb4000 task.ti: ffff8814acfb4000
RIP: 0010:[<ffffffff816ab905>]  [<ffffffff816ab905>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8814acfb7ca8  EFLAGS: 00000246
RAX: 0000000000000800 RBX: 0000000000000001 RCX: 0000000000000004
RDX: 0000000000000040 RSI: 0000000000000246 RDI: 0000000000000246
RBP: ffff8814acfb7ca8 R08: 0000000000000000 R09: ffffea0000000000
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8814acfb7c28
R13: 0000000000000202 R14: 0000000000000010 R15: ffffffff81190aef
FS:  00002ade4a92efc0(0000) GS:ffff883f7d7c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002ade5080c620 CR3: 000000229f213000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8814acfb7cf8 ffffffff81190ec5 0000000000000000 ffffffff811905c0
0000000000000246 ffffea00b9885480 ffff883f7d7d1ae0 0000000000000000
0000000000000000 ffffea00b98854a0 ffff8814acfb7d18 ffffffff81191154
Call Trace:
[<ffffffff81190ec5>] pagevec_lru_move_fn+0xf5/0x130
[<ffffffff811905c0>] ? lru_deactivate_fn+0x1d0/0x1d0
[<ffffffff81191154>] __lru_cache_add+0x64/0x90
[<ffffffff8119151e>] lru_cache_add+0xe/0x10
[<ffffffff81195d3d>] putback_lru_page+0x3d/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48