NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [slurmstepd:39602]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_core intel_powerclamp coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel
lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support pcspkr sg sb_edac edac_core lpc_ich hpilo hpwdt ioatdma ipmi_ssif shpchp ipmi_si acpi_power_meter pcc_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c mlx4_en sd_mod crc_t10dif crct10dif_generic nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) crct10dif_pclmul crct10dif_common crc32c_intel mgag200 ipmi_msghandler drm_kms_helper mlx4_core serio_raw syscopyarea sysfillrect sysimgblt ttm ahci fb_sys_fops igb libahci hpsa drm devlink ptp libata pps_core dca i2c_algo_bit mpt2sas i2c_core raid_class scsi_transport_sas wmi fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 19 PID: 39602 Comm: slurmstepd Tainted: P    B      OE  ------------   3.10.0-514.21.2.el7.x86_64 #1
Hardware name: HP ProLiant SL270s Gen8 SE/, BIOS P75 09/18/2013
task: ffff881fab333ec0 ti: ffff881fa7398000 task.ti: ffff881fa7398000
RIP: 0010:[<ffffffff81195086>]  [<ffffffff81195086>] page_evictable+0x36/0x40
RSP: 0018:ffff881fa739bd28  EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffff88403ffd9000 RCX: 0000000000000001
RDX: 00000000000200da RSI: 0000000000000001 RDI: ffffea00b9b8a880
RBP: ffff881fa739bd28 R08: ffff883f9dc0f000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88403f651a20
R13: ffffea00b10db9c0 R14: 000000000000000e R15: 0000000015157d52
FS:  00007fed91ea5780(0000) GS:ffff88403f640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000009d4d90 CR3: 0000003f58e50000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff881fa739bd50 ffffffff811950be ffffea00b9b8a880 ffff882859623440
ffff88403ffd9540 ffff881fa739bdd0 ffffffff811f194a 0000000200000001
ffff883f9dc09500 0000000000000001 0000000000001000 ffff883f9dc09440
Call Trace:
[<ffffffff811950be>] putback_lru_page+0x2e/0xb0
[<ffffffff811f194a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1c94>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d6e7>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d982>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a8c8>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f965>] do_rmdir+0x1a5/0x200
[<ffffffff81210b16>] SyS_rmdir+0x16/0x20
[<ffffffff81697749>] system_call_fastpath+0x16/0x1b
Code: 89 fb e8 1e b8 00 00 48 85 c0 74 11 48 8b 90 80 00 00 00 31 c0 f7 c2 00 00 00 10 75 0d 48 8b 03 48 c1 e8 15 83 e0 01 83 f0 01 5b <5d> c3 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 55 

NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [slurmstepd:130847]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core iTCO_wdt intel_powerclamp iTCO_vendor_support coretemp dcdbas intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul crc32c_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ghash_clmulni_intel drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf nfit ipmi_msghandler libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci libsas qla3xxx hpsa e1000e
ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci megaraid_sas tg3(OE) libahci ptp pps_core libata [last unloaded: ipmi_msghandler]
CPU: 19 PID: 130847 Comm: slurmstepd Tainted: G           OE  ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff8808b876bf40 ti: ffff880e2e198000 task.ti: ffff880e2e198000
RIP: 0010:[<ffffffff81195cae>]  [<ffffffff81195cae>] page_evictable+0x1e/0x40
RSP: 0018:ffff880e2e19bd20  EFLAGS: 00000286
RAX: ffff88304d850b48 RBX: ffff88603ffd7000 RCX: 0000000000000001
RDX: 00000000000200da RSI: 0000000000000001 RDI: ffffea0145f1cac0
RBP: ffff880e2e19bd28 R08: ffff885f445fbc00 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff885f7c051ae0
R13: ffffea01306c7440 R14: 000000000000000e R15: 00000000b0f32fec
FS:  00002aaaaaafd180(0000) GS:ffff885f7c040000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00005555560c9000 CR3: 00000051f7015000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffea0145f1cac0 ffff880e2e19bd50 ffffffff81195cfe ffffea0145f1cac0
ffff8848fd266440 ffff88603ffd7540 ffff880e2e19bdd0 ffffffff811f342a
0000000200000001 ffff881e7f610d00 0000000000000001 0000000000001000
Call Trace:
[<ffffffff81195cfe>] putback_lru_page+0x2e/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 02 00 00 00 00 00 5b 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 fb e8 8e b6 00 00 48 85 c0 74 11 48 8b 90 80 00 00 00 <31> c0 f7 c2 00 00 00 10 75 0d 48 8b 03 48 c1 e8 15 83 e0 01 83 

NMI watchdog: BUG: soft lockup - CPU#52 stuck for 22s! [slurmstepd:412252]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log iTCO_wdt iTCO_vendor_support skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul crc32c_intel dell_smbios dcdbas sparse_keymap mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei i2c_i801 shpchp lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter
bridge stp llc xfs isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common ahci libahci libata i40e(OE) megaraid_sas tg3(OE) ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 52 PID: 412252 Comm: slurmstepd Tainted: G           OEL ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/04JN2K, BIOS 2.21.1 03/07/2024
task: ffff882f4932af70 ti: ffff88404cfa4000 task.ti: ffff88404cfa4000
RIP: 0010:[<ffffffff81190359>]  [<ffffffff81190359>] put_page+0x29/0x50
RSP: 0018:ffff88404cfa7d28  EFLAGS: 00000206
RAX: 006fffff0000000c RBX: ffff88607bfd7000 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffea014c2d8400
RBP: ffff88404cfa7d28 R08: ffff882f55896c00 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff882fb9091ae0
R13: ffffea00dfcd8180 R14: 000000000000000e R15: 0000000003235819
FS:  00002aaaaaafd180(0000) GS:ffff882fb9080000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055555557e160 CR3: 00000042fa0a7000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88404cfa7d50 ffffffff81195d22 ffffea014c2d8400 ffff883115644430
ffff88607bfd7540 ffff88404cfa7dd0 ffffffff811f342a 0000000200000001
ffff885f74095d00 0000000000000001 0000000000001000 ffff885f74095c40
Call Trace:
[<ffffffff81195d22>] putback_lru_page+0x52/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816a8fad>] ? __schedule+0x39d/0x8b0
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 00 00 0f 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 18 0c ff ff 5d c3 66 0f 1f 44 00 00 f6 c4 c0 75 14 f0 ff 4f 1c <0f> 94 c0 84 c0 66 90 74 e6 e8 99 ff ff ff 5d c3 0f 1f 80 00 00 

NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [slurmstepd:27990]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) squashfs loop overlay(T) osc(OE) mgc(OE) lustre(OE) lmv(OE) fld(OE) mdc(OE) fid(OE) lov(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) sha512_ssse3 sha512_generic nfs_layout_nfsv41_files crypto_null libcfs(OE) dell_rbu rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache dm_mirror dm_region_hash dm_log ses enclosure sd_mod sg dm_mod intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd serio_raw pcspkr uas usb_storage nvme smartpqi scsi_transport_sas hpilo hpwdt lpc_ich shpchp wmi acpi_power_meter rpcrdma ib_isert iscsi_target_mod target_core_mod crc_t10dif crct10dif_generic ib_iser libiscsi scsi_transport_iscsi ib_ipoib rdma_ucm
ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ipmi_devintf hfi1(OE) i2c_algo_bit i2c_core crct10dif_pclmul crct10dif_common rdmavt(OE) crc32c_intel bnxt_en ib_core ipmi_si fjes ipmi_msghandler
CPU: 1 PID: 27990 Comm: slurmstepd Tainted: G        W  OEL ------------ T 3.10.0-514.10.2.el7.x86_64 #1
Hardware name: HPE ProLiant DL560 Gen10/ProLiant DL560 Gen10, BIOS U34 03/16/2023
task: ffff89fc3c9e4e70 ti: ffff89a315f34000 task.ti: ffff89a315f34000
RIP: 0010:[<ffffffff81194bc6>]  [<ffffffff81194bc6>] putback_lru_page+0x26/0xb0
RSP: 0018:ffff89a315f37d38  EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffff8118f44c RCX: 0000000000000001
RDX: 0000000003243c7b RSI: 0000000000000001 RDI: ffffea0f0f8951c0
RBP: ffff89a315f37d50 R08: ffff88fc39ed7800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8c003ffd8000
R13: ffff8c003ffd8000 R14: ffff88fc3fc51a20 R15: ffffea0e90c52d80
FS:  00002ac0e81a2300(0000) GS:ffff88fc3fc40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000020aa7d0 CR3: 0000035a18ff6000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffea0f0f8951c0 ffff8bfc3cfec440 ffff8c003ffd8540 ffff89a315f37dd0
ffffffff811f130a 0000000200000003 ffff8bfc3db38900 0000000000000003
0000000000001000 ffff8bfc3db38840 0000000000000001 0000000100000003
Call Trace:
[<ffffffff811f130a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f1654>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d257>] cgroup_destroy_locked+0xe7/0x360
[<ffffffff8110d4f2>] cgroup_rmdir+0x22/0x40
[<ffffffff8120a278>] vfs_rmdir+0xa8/0x100
[<ffffffff8120f315>] do_rmdir+0x1a5/0x200
[<ffffffff81692305>] ? do_page_fault+0x35/0x90
[<ffffffff812104c6>] SyS_rmdir+0x16/0x20
[<ffffffff81696b09>] system_call_fastpath+0x16/0x1b
Code: 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 55 41 54 4c 8d 67 02 53 4c 8b 2f 48 89 fb 49 c1 ed 14 41 83 e5 01 f0 41 80 24 24 ef <48> 89 df e8 92 ff ff ff 85 c0 74 2e 48 89 df 48 8b 03 e8 b3 ae 

NMI watchdog: BUG: soft lockup - CPU#28 stuck for 23s! [slurmstepd:73122]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi dcdbas kvm_intel kvm iTCO_wdt iTCO_vendor_support irqbypass crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei i2c_i801 shpchp lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci
libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common ahci libahci i40e(OE) libata megaraid_sas tg3(OE) ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 28 PID: 73122 Comm: slurmstepd Tainted: G           OE  ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff8830ca6caf70 ti: ffff883284950000 task.ti: ffff883284950000
RIP: 0010:[<ffffffff811914ef>]  [<ffffffff811914ef>] lru_cache_add+0xf/0x10
RSP: 0018:ffff883284953d30  EFLAGS: 00000246
RAX: 0000000000000004 RBX: ffffffff81190eac RCX: 0000000000000001
RDX: 0000000000000003 RSI: 0000000000000001 RDI: ffffea0112e2d700
RBP: ffff883284953d50 R08: ffff8845142a0000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88603ffd7000
R13: ffff88603ffd7000 R14: ffff882f7dd91ae0 R15: ffffea01253b74c0
FS:  00002aaaaaafd180(0000) GS:ffff882f7dd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aaaabe2e1a0 CR3: 000000326aa76000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffff81195d0d ffffea0112e2d700 ffff885f2ea07430 ffff88603ffd7540
ffff883284953dd0 ffffffff811f342a 0000000200000001 ffff883b21b43d00
0000000000000001 0000000000001000 ffff883b21b43c40 0000000000000001
Call Trace:
[<ffffffff81195d0d>] ? putback_lru_page+0x3d/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: f8 ff 74 0e 48 63 d0 48 3b 5c d1 10 75 ee f0 80 0b 40 eb ab 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 d2 fb ff ff 5d <c3> 0f 1f 44 00 00 55 48 89 e5 41 56 41 55 41 54 53 4c 8b 27 48 

NMI watchdog: BUG: soft lockup - CPU#45 stuck for 22s! [slurmstepd:302491]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm iTCO_wdt irqbypass iTCO_vendor_support crc32_pclmul crc32c_intel dell_smbios dcdbas sparse_keymap ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs
isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci libahci megaraid_sas tg3(OE) libata ptp pps_core [last unloaded: ipmi_msghandler]
CPU: 45 PID: 302491 Comm: slurmstepd Tainted: G        W  OEL ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 2.8.1 06/30/2020
task: ffff88057d365ee0 ti: ffff88246c4d0000 task.ti: ffff88246c4d0000
RIP: 0010:[<ffffffff81190359>]  [<ffffffff81190359>] put_page+0x29/0x50
RSP: 0018:ffff88246c4d3d28  EFLAGS: 00000206
RAX: 006fffff0000004c RBX: ffff88603ffd7000 RCX: 0000000000000001
RDX: 000000000000000a RSI: 0000000000000001 RDI: ffffea0134856680
RBP: ffff88246c4d3d28 R08: ffff882f597dd400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff885f7c391ae0
R13: ffffea0100fa7100 R14: 000000000000000e R15: 000000003def42bb
FS:  00002aaaaaafd180(0000) GS:ffff885f7c380000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aab2008c000 CR3: 0000001cb78ed000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88246c4d3d50 ffffffff81195d22 ffffea0134856680 ffff8847375b9440
ffff88603ffd7540 ffff88246c4d3dd0 ffffffff811f342a 0000000200000001
ffff88022fd2a100 0000000000000001 0000000000001000 ffff88022fd2a040
Call Trace:
[<ffffffff81195d22>] putback_lru_page+0x52/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 00 00 0f 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 18 0c ff ff 5d c3 66 0f 1f 44 00 00 f6 c4 c0 75 14 f0 ff 4f 1c <0f> 94 c0 84 c0 66 90 74 e6 e8 99 ff ff ff 5d c3 0f 1f 80 00 00 

NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [slurmstepd:18086]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ksocklnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c br_netfilter bridge 8021q garp mrp stp llc overlay(T) ip6_tables iTCO_wdt iTCO_vendor_support iptable_filter coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr ses enclosure cdc_ether scsi_transport_sas usbnet joydev mii sg mei_me mei shpchp i2c_i801 wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace
binfmt_misc sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm serio_raw ahci libahci mlx5_core(OE) libata mlxfw(OE) ptp megaraid_sas pps_core auxiliary(OE) mlx_compat(OE) i2c_core devlink [last unloaded: tracedev]
CPU: 18 PID: 18086 Comm: slurmstepd Tainted: G        W  OEL ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Lenovo ThinkSystem SR650 V2/7Z73CTO1WW, BIOS AFE112A-1.02 05/10/2021
task: ffff89f7db341fa0 ti: ffff89983eeec000 task.ti: ffff89983eeec000
RIP: 0010:[<ffffffff81195d26>]  [<ffffffff81195d26>] putback_lru_page+0x26/0xb0
RSP: 0018:ffff89983eeefd38  EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffff81190edc RCX: 0000000000000001
RDX: 000000000231e403 RSI: 0000000000000001 RDI: ffffea0710f16680
RBP: ffff89983eeefd50 R08: ffff897b7bbe0800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff8a003ffd8000
R13: ffff8a003ffd8000 R14: ffff89f83de91ae0 R15: ffffea02a925e1c0
FS:  00002b9018d4dfc0(0000) GS:ffff89f83de80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3d8b2e6000 CR3: 000001edf9d30000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffea0710f16680 ffff88ecdc0ed440 ffff8a003ffd8540 ffff89983eeefdd0
ffffffff811f339a 0000000200000000 ffff89f831386100 0000000000000000
0000000000001000 ffff89f831386040 0000000000000001 0000000100000003
Call Trace:
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 55 41 54 4c 8d 67 02 53 4c 8b 2f 48 89 fb 49 c1 ed 14 41 83 e5 01 f0 41 80 24 24 ef <48> 89 df e8 92 ff ff ff 85 c0 74 2e 48 89 df 48 8b 03 e8 d3 b7 

NMI watchdog: BUG: soft lockup - CPU#28 stuck for 23s! [slurmstepd:163589]
Modules linked in: 8021q garp mrp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 nfs fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm dm_mirror dm_region_hash dm_log dm_mod skx_edac edac_core iTCO_wdt intel_powerclamp iTCO_vendor_support coretemp intel_rapl iosf_mbi kvm hfi1 irqbypass crc32_pclmul rdmavt ghash_clmulni_intel aesni_intel ib_core lrw gf128mul glue_helper mei_me ablk_helper cryptd joydev pcspkr sg mei
lpc_ich i2c_i801 shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ixgbe ttm crc32c_intel ahci mdio drm libahci ptp libata pps_core i2c_core dca
CPU: 28 PID: 163589 Comm: slurmstepd Tainted: G        W    L ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Advanced Clustering S2600BPB/S2600BPB, BIOS SE5C620.86B.00.01.0009.C2.101920170742 10/19/2017
task: ffff88474a3e1fa0 ti: ffff883adbec4000 task.ti: ffff883adbec4000
RIP: 0010:[<ffffffff81190389>]  [<ffffffff81190389>] put_page+0x29/0x50
RSP: 0018:ffff883adbec7d28  EFLAGS: 00000206
RAX: 002fffff00000048 RBX: ffff88303ffda000 RCX: 0000000000000001
RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffffea00642ad500
RBP: ffff883adbec7d28 R08: ffff882e61a9d400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff885f7c011ae0
R13: ffffea00642a50c0 R14: 000000000000000e R15: 000000000decd5d7
FS:  00002b6778a38d00(0000) GS:ffff885f7c000000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd3ba4aa3e4 CR3: 0000002469745000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff883adbec7d50 ffffffff81195d52 ffffea00642ad500 ffff882f74b70440
ffff88303ffda540 ffff883adbec7dd0 ffffffff811f339a 0000000200000000
ffff882e61a99900 0000000000000000 0000000000001000 ffff882e61a99840
Call Trace:
[<ffffffff81195d52>] putback_lru_page+0x52/0xb0
[<ffffffff811f339a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f36e4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d767>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da12>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f5a6>] vfs_rmdir+0xd6/0x150
[<ffffffff81212241>] do_rmdir+0x1f1/0x220
[<ffffffff81213446>] SyS_rmdir+0x16/0x20
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 00 00 0f 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 f8 0b ff ff 5d c3 66 0f 1f 44 00 00 f6 c4 c0 75 14 f0 ff 4f 1c <0f> 94 c0 84 c0 66 90 74 e6 e8 99 ff ff ff 5d c3 0f 1f 80 00 00 

NMI watchdog: BUG: soft lockup - CPU#33 stuck for 22s! [slurmstepd:41614]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs beegfs(OE) fscache rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw mlx5_ib ib_core dm_mirror dm_region_hash dm_log dm_mod vfat fat nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) iTCO_wdt iTCO_vendor_support skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm nvidia(POE) irqbypass crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev ipmi_ssif sg mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c raid1 sd_mod crc_t10dif crct10dif_generic ast i2c_algo_bit drm_kms_helper mlx5_core crct10dif_pclmul crct10dif_common syscopyarea crc32c_intel sysfillrect sysimgblt fb_sys_fops i40e ttm drm ahci libahci devlink ptp libata pps_core i2c_core wmi
CPU: 33 PID: 41614 Comm: slurmstepd Tainted: P    B      OE  ------------   3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Supermicro SYS-4029GP-TRT2/X11DPG-OT-CPU, BIOS 3.3 02/21/2020
task: ffff887d646d8fd0 ti: ffff887e772c8000 task.ti: ffff887e772c8000
RIP: 0010:[<ffffffff81197a16>]  [<ffffffff81197a16>] putback_lru_page+0x26/0xb0
RSP: 0018:ffff887e772cbd08  EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffff81192bcc RCX: 0000000000000001
RDX: 0000000001e885aa RSI: 0000000000000001 RDI: ffffea014be89900
RBP: ffff887e772cbd20 R08: ffff883edcfb5000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88807ffd7000
R13: ffff88807ffd7000 R14: ffff887f7dd55ae0 R15: ffffea015dface40
FS:  00002ac41161d5c0(0000) GS:ffff887f7dd40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000006dde1c CR3: 0000007ea9358000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff811f536a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f56b4>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110f467>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110f712>] cgroup_rmdir+0x22/0x40
[<ffffffff812114f6>] vfs_rmdir+0xd6/0x150
[<ffffffff81214191>] do_rmdir+0x1f1/0x220
[<ffffffff81215396>] SyS_rmdir+0x16/0x20
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 55 41 54 4c 8d 67 02 53 4c 8b 2f 48 89 fb 49 c1 ed 14 41 83 e5 01 f0 41 80 24 24 ef <48> 89 df e8 92 ff ff ff 85 c0 74 2e 48 89 df 48 8b 03 e8 d3 b7 

NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [slurmstepd:232090]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel iTCO_wdt kvm iTCO_vendor_support joydev irqbypass dcdbas crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper aesni_intel syscopyarea lrw sysfillrect gf128mul sysimgblt glue_helper fb_sys_fops ablk_helper cryptd drm pcspkr mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc
xfs isci libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci megaraid_sas libahci tg3(OE) ptp libata pps_core [last unloaded: ipmi_msghandler]
CPU: 0 PID: 232090 Comm: slurmstepd Tainted: G           OE  ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff880a56274f10 ti: ffff8801034f4000 task.ti: ffff8801034f4000
RIP: 0010:[<ffffffff81190359>]  [<ffffffff81190359>] put_page+0x29/0x50
RSP: 0000:ffff8801034f7d28  EFLAGS: 00000206
RAX: 006fffff0000000c RBX: ffff88603ffd7000 RCX: 0000000000000001
RDX: 0000000000000004 RSI: 0000000000000001 RDI: ffffea00f8a2e540
RBP: ffff8801034f7d28 R08: ffff885f74f74400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff882f7da11ae0
R13: ffffea00e0e26d00 R14: 000000000000000e R15: 00000000298fa1c7
FS:  00002aaaaaafd180(0000) GS:ffff882f7da00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002aaad7491000 CR3: 00000034e2f18000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8801034f7d50 ffffffff81195d22 ffffea00f8a2e540 ffff88591940a430
ffff88603ffd7540 ffff8801034f7dd0 ffffffff811f342a 0000000200000001
ffff882f41bf7d00 0000000000000001 0000000000001000 ffff882f41bf7c40
Call Trace:
[<ffffffff81195d22>] putback_lru_page+0x52/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 00 00 0f 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 18 0c ff ff 5d c3 66 0f 1f 44 00 00 f6 c4 c0 75 14 f0 ff 4f 1c <0f> 94 c0 84 c0 66 90 74 e6 e8 99 ff ff ff 5d c3 0f 1f 80 00 00 

NMI watchdog: BUG: soft lockup - CPU#14 stuck for 23s! [slurmstepd:370756]
Modules linked in: squashfs loop overlay(T) mmfs26(OE) mmfslinux(OE) tracedev(OE) ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log dcdbas skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm iTCO_wdt iTCO_vendor_support irqbypass crc32_pclmul crc32c_intel mgag200 ghash_clmulni_intel ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter acpi_pad binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci
libsas qla3xxx hpsa e1000e ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) megaraid_sas ahci tg3(OE) libahci ptp pps_core libata [last unloaded: ipmi_msghandler]
CPU: 14 PID: 370756 Comm: slurmstepd Tainted: G           OE  ------------ T 3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff883146ea5ee0 ti: ffff883203f94000 task.ti: ffff883203f94000
RIP: 0010:[<ffffffff81190330>]  [<ffffffff81190330>] __put_single_page+0x30/0x30
RSP: 0018:ffff883203f97d30  EFLAGS: 00000246
RAX: 0000000000000002 RBX: ffffffff81190eac RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffffea00d41b02c0
RBP: ffff883203f97d50 R08: ffff885f7845a800 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88603ffd7000
R13: ffff88603ffd7000 R14: ffff882f7dbd1ae0 R15: ffffea00f8db9180
FS:  00002aaaaaafd180(0000) GS:ffff882f7dbc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffff7ff5000 CR3: 000000320e98f000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffff81195d22 ffffea00d41b02c0 ffff884853306440 ffff88603ffd7540
ffff883203f97dd0 ffffffff811f342a 0000000200000001 ffff884db0e28100
0000000000000001 0000000000001000 ffff884db0e28040 0000000000000001
Call Trace:
[<ffffffff81195d22>] ? putback_lru_page+0x52/0xb0
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 55 48 89 e5 53 48 8b 07 48 89 fb a8 20 74 05 e8 97 fe ff ff 48 89 df 31 f6 e8 7d b2 ff ff 5b 5d c3 66 2e 0f 1f 84 00 00 00 00 00 <0f> 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 18 

NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [slurmstepd:131887]
Modules linked in: mmfs26(OE) mmfslinux(OE) tracedev(OE) devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dell_rbu dm_mirror dm_region_hash dm_log skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass dcdbas iTCO_wdt iTCO_vendor_support crc32_pclmul crc32c_intel ghash_clmulni_intel mgag200 ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr mei_me mei i2c_i801 shpchp lpc_ich ipmi_si ipmi_devintf nfit ipmi_msghandler libnvdimm acpi_pad acpi_power_meter binfmt_misc ip_tables bnxt_en nvme nvme_core qed hpilo dm_thin_pool dm_bio_prison dm_persistent_data dm_bufio dm_mod udf crc_itu_t vfat fat br_netfilter bridge stp llc xfs isci libsas qla3xxx hpsa e1000e
ixgbe(OE) igb(OE) i2c_algo_bit i2c_core dca aacraid aic79xx ata_piix mpt2sas raid_class mptspi scsi_transport_spi mptsas mptscsih mptbase scsi_transport_sas arcmsr sata_nv sata_svw bnx2x(OE) mdio libcrc32c bnx2(OE) ext4 mbcache jbd2 sata_sil e1000 nfsv3 nfs_acl nfs lockd grace sunrpc fscache tun sd_mod crc_t10dif crct10dif_generic sg crct10dif_pclmul crct10dif_common i40e(OE) ahci megaraid_sas libahci tg3(OE) ptp pps_core libata [last unloaded: ipmi_msghandler]
CPU: 12 PID: 131887 Comm: slurmstepd Tainted: G           OE  ------------   3.10.0-693.2.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R440/0WKGTH, BIOS 1.3.7 02/09/2018
task: ffff88254c3b8fd0 ti: ffff88306b794000 task.ti: ffff88306b794000
RIP: 0010:[<ffffffff81195cf6>]  [<ffffffff81195cf6>] putback_lru_page+0x26/0xb0
RSP: 0018:ffff88306b797d38  EFLAGS: 00000246
RAX: 0000000000000001 RBX: ffffffff81190eac RCX: 0000000000000001
RDX: 0000000003f41ef8 RSI: 0000000000000001 RDI: ffffea013de2dfc0
RBP: ffff88306b797d50 R08: ffff880134f61400 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88603ffd7000
R13: ffff88603ffd7000 R14: ffff882f7db91ae0 R15: ffffea014f1af540
FS:  00002aaaaaafd180(0000) GS:ffff882f7db80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffff6f10810 CR3: 000000305b746000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffea013de2dfc0 ffff8834e2e49430 ffff88603ffd7540 ffff88306b797dd0
ffffffff811f342a 0000000200000001 ffff8814f6a6fd00 0000000000000001
0000000000001000 ffff8814f6a6fc40 0000000000000001 0000000100000002
Call Trace:
[<ffffffff811f342a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3774>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f636>] vfs_rmdir+0xd6/0x150
[<ffffffff812122d1>] do_rmdir+0x1f1/0x220
[<ffffffff816b0325>] ? do_page_fault+0x35/0x90
[<ffffffff812134d6>] SyS_rmdir+0x16/0x20
[<ffffffff816b5009>] system_call_fastpath+0x16/0x1b
Code: 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 55 41 54 4c 8d 67 02 53 4c 8b 2f 48 89 fb 49 c1 ed 14 41 83 e5 01 f0 41 80 24 24 ef <48> 89 df e8 92 ff ff ff 85 c0 74 2e 48 89 df 48 8b 03 e8 d3 b7 

NMI watchdog: BUG: soft lockup - CPU#63 stuck for 22s! [systemd:1]
Modules linked in: veth xt_addrtype br_netfilter overlay(T) nfsv3 binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) iTCO_wdt iTCO_vendor_support sb_edac edac_core vfat fat intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd cdc_ether usbnet mii ipmi_ssif
pcspkr sg lpc_ich ioatdma shpchp wmi acpi_pad ipmi_si ipmi_devintf ipmi_msghandler nfsd auth_rpcgss nfs_acl lockd grace sunrpc knem(OE) ip_tables xfs libcrc32c mlx4_ib(OE) ib_core(OE) sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe drm igb mlx4_core(OE) devlink mdio mlx_compat(OE) ptp megaraid_sas pps_core dca i2c_algo_bit i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 63 PID: 1 Comm: systemd Tainted: G           OE  ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: LENOVO x3950 X6 -[6241SCW]-/00FN846, BIOS -[A9E128PUS-2.10]- 10/21/2015
task: ffff88a0938f8000 ti: ffff88015356c000 task.ti: ffff88015356c000
RIP: 0010:[<ffffffff81190359>]  [<ffffffff81190359>] put_page+0x29/0x50
RSP: 0018:ffff88015356fd28  EFLAGS: 00000206
RAX: 00afffff0000000c RBX: ffff88603ffda000 RCX: 0000000000000001
RDX: 0000000000000005 RSI: 0000000000000001 RDI: ffffea013048cd00
RBP: ffff88015356fd28 R08: ffff88f8dd247000 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000001 R12: ffff88bfbfad1ae0
R13: ffffea017ceb3480 R14: 000000000000000e R15: 000000005b665206
FS:  00007ffad96a8940(0000) GS:ffff88bfbfac0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000015c9a88 CR3: 0000003fbbee7000 CR4: 00000000001407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88015356fd50 ffffffff81195d22 ffffea013048cd00 ffff88518698c430
ffff88603ffda540 ffff88015356fdd0 ffffffff811f341a 0000000200000002
ffff88a09353c100 0000000000000002 0000000000001000 ffff88a09353c040
Call Trace:
[<ffffffff81195d22>] putback_lru_page+0x52/0xb0
[<ffffffff811f341a>] mem_cgroup_reparent_charges+0x22a/0x3c0
[<ffffffff811f3764>] mem_cgroup_css_offline+0x84/0x140
[<ffffffff8110d757>] cgroup_destroy_locked+0xe7/0x370
[<ffffffff8110da02>] cgroup_rmdir+0x22/0x40
[<ffffffff8120f626>] vfs_rmdir+0xd6/0x150
[<ffffffff81212251>] do_rmdir+0x1f1/0x220
[<ffffffff810ad24c>] ? task_work_run+0xac/0xf0
[<ffffffff81213456>] SyS_rmdir+0x16/0x20
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 00 00 0f 1f 44 00 00 48 8b 07 55 48 89 e5 a9 00 00 00 80 74 0d e8 18 0c ff ff 5d c3 66 0f 1f 44 00 00 f6 c4 c0 75 14 f0 ff 4f 1c <0f> 94 c0 84 c0 66 90 74 e6 e8 99 ff ff ff 5d c3 0f 1f 80 00 00