NMI watchdog: BUG: soft lockup - CPU#47 stuck for 22s! [kworker/47:0:1301]
Modules linked in: binfmt_misc nfsv3 nvidia_uvm(OE) beegfs(OE) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_ib(OE) mlx4_en(OE) mlx4_core(OE) dm_mirror dm_region_hash dm_log dm_mod joydev nvidia_drm(POE) nvidia_modeset(POE) iTCO_wdt iTCO_vendor_support nvidia(POE) vfat fat coretemp kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg mei_me mei shpchp i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc knem(OE) ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) drm_kms_helper syscopyarea sysfillrect
crct10dif_pclmul sysimgblt crct10dif_common fb_sys_fops crc32c_intel ttm serio_raw i40e mlx5_core(OE) mlxfw(OE) drm devlink igb vfio_mdev(OE) vfio_iommu_type1 ahci vfio libahci mdev(OE) mlx_compat(OE) megaraid_sas libata ptp dca pps_core i2c_algo_bit i2c_core wmi
CPU: 47 PID: 1301 Comm: kworker/47:0 Tainted: P        W  OE  ------------   3.10.0-693.el7.x86_64 #1
Hardware name: Inspur NF5468M6/NF5468M6, BIOS 06.01.00 03/01/2022
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff896254530000 ti: ffff89625452c000 task.ti: ffff89625452c000
RIP: 0010:[<ffffffff816abb40>]  [<ffffffff816abb40>] _raw_spin_lock+0x10/0x30
RSP: 0018:ffff89625452fd38  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000005 RCX: 0000000000000034
RDX: 0000000000000001 RSI: ffff886777855040 RDI: ffff886f31e168a8
RBP: ffff89625452fd68 R08: 8028000000000000 R09: 6777855040140000
R10: 986a948da1401005 R11: 0000000000000000 R12: 57f50ddcc9097b80
R13: 0000000000000000 R14: 0000000100000003 R15: ffff89625452fd30
FS:  0000000000000000(0000) GS:ffff89fc7f2c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f98563b5000 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffffc0aeae17 ffff886777855000 ffff88f981a0cd00 0000000000000000
ffff88f981a0ced8 ffff88f981a0cec8 ffff89625452fdd0 ffffffffc0aebe4f
0000000000000000 0000000000000bc0 ffff89625452fda0 0000000000000286
Call Trace:
[<ffffffffc0aeae17>] ? nfs_inode_remove_request+0x47/0xf0 [nfs]
[<ffffffffc0aebe4f>] nfs_commit_release_pages+0x5f/0x230 [nfs]
[<ffffffffc0aea916>] nfs_commit_release+0x16/0x30 [nfs]
[<ffffffffc04bca10>] rpc_free_task+0x30/0x70 [sunrpc]
[<ffffffffc04bca62>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 44 00 00 85 d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 <85> c0 75 01 c3 55 89 c6 48 89 e5 e8 a4 2a ff ff 5d c3 0f 1f 40 

NMI watchdog: BUG: soft lockup - CPU#40 stuck for 22s! [kworker/40:1:208467]
Modules linked in: binfmt_misc xt_nat veth tcp_lp fuse nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache overlay(T) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm dm_mirror i40iw dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd hfi1 joydev rdmavt ib_core pcspkr sg i2c_i801 mei_me ipmi_ssif mei lpc_ich shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 uas usb_storage sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm i40e drm ahci libahci megaraid_sas libata ptp pps_core i2c_core
CPU: 40 PID: 208467 Comm: kworker/40:1 Tainted: G        W    L ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: Sugon I840-G30/80P48-US, BIOS 0THS00229 06/18/2021
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff898061004f10 ti: ffff89806b664000 task.ti: ffff89806b664000
RIP: 0010:[<ffffffff81181345>]  [<ffffffff81181345>] page_waitqueue+0x85/0xb0
RSP: 0018:ffff89806b667d38  EFLAGS: 00000202
RAX: ffff8b003ff8ddb8 RBX: 0000000000000005 RCX: 0000000000000034
RDX: 00000000000003b7 RSI: 0000000000000000 RDI: 000000000000013d
RBP: ffff89806b667d68 R08: 6400000000000000 R09: a82d34f332000000
R10: 57d2b5181b3ccc80 R11: 0000000000000000 R12: ffff8b003ffa3cb8
R13: ffff89806b667d10 R14: 00000001ffffff10 R15: ffffffff810ba588
FS:  0000000000000000(0000) GS:ffff8a3d3e200000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fb1d3e9f110 CR3: 0000023d38a54000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffffc08abe48 ffff8aa758c49d00 ffff88c04cca44c0 0000000000000000
ffff88c04cca4698 ffff88c04cca4688 ffff89806b667dd0 ffffffffc08ace4f
0000000000000000 0000000000000a00 ffff89806b667da0 0000000000000286
Call Trace:
[<ffffffffc08abe48>] ? nfs_inode_remove_request+0x78/0xf0 [nfs]
[<ffffffffc08ace4f>] nfs_commit_release_pages+0x5f/0x230 [nfs]
[<ffffffffc08ab916>] nfs_commit_release+0x16/0x30 [nfs]
[<ffffffffc02f1a10>] rpc_free_task+0x30/0x70 [sunrpc]
[<ffffffffc02f1a62>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffff810a881a>] process_one_work+0x17a/0x440
[<ffffffff810a94e6>] worker_thread+0x126/0x3c0
[<ffffffff810a93c0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 48 c1 e7 3f 48 01 f2 48 29 ca b9 40 00 00 00 2b 88 50 07 00 00 48 01 d7 48 8b 80 40 07 00 00 5d 48 d3 ef 48 8d 14 7f 48 8d 04 d0 <c3> 66 2e 0f 1f 84 00 00 00 00 00 48 c1 e8 36 48 8b 04 c5 60 a2 

NMI watchdog: BUG: soft lockup - CPU#44 stuck for 23s! [kworker/44:1:5372]
Modules linked in: squashfs loop overlay(T) tcp_diag udp_diag inet_diag nf_conntrack_netlink nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack_ftp nf_conntrack libcrc32c rpcsec_gss_krb5 nfsv4 dns_resolver nfsv3 nfs fscache 8021q garp mrp stp llc ip6_tables iptable_filter sb_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel iTCO_wdt iTCO_vendor_support aesni_intel lrw gf128mul glue_helper ablk_helper cryptd mxm_wmi pcspkr sg mei_me ioatdma mei shpchp i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler wmi acpi_power_meter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib nfsd ib_core
auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 mlx4_en sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mlx4_core drm crct10dif_pclmul mpt3sas igb crct10dif_common crc32c_intel ptp pps_core raid_class dca i2c_algo_bit devlink scsi_transport_sas i2c_core
CPU: 44 PID: 5372 Comm: kworker/44:1 Tainted: G               ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Inspur NF5170M4/YZMB-00370-104, BIOS 4.1.12 05/09/2017
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff8813ce4a5ee0 ti: ffff880113ab4000 task.ti: ffff880113ab4000
RIP: 0010:[<ffffffffc069ae3c>]  [<ffffffffc069ae3c>] nfs_inode_remove_request+0x6c/0xf0 [nfs]
RSP: 0018:ffff880113ab7d40  EFLAGS: 00000246
RAX: ffffea01f8e3e740 RBX: 000000003fa49df2 RCX: 0000000000000034
RDX: 006fffff0000082c RSI: ffff88517ab28d40 RDI: ffff884085619ae0
RBP: ffff880113ab7d68 R08: 8028000000000000 R09: 517ab28d40140000
R10: ae6761d16c8f5005 R11: 0000000000000000 R12: 0000000000000005
R13: ffff88517ab28d40 R14: ffff88807ffae5e8 R15: ffff880113ab7d10
FS:  0000000000000000(0000) GS:ffff883f7fd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b007e0f2000 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88517ab28d00 ffff88415bb69080 0000000000000000 ffff88415bb69258
ffff88415bb69248 ffff880113ab7dd0 ffffffffc069be4f ffffffffc069c2a9
ffff884085619a58 ffff8840856199a0 0000000000000000 0000000000000000
Call Trace:
[<ffffffffc069be4f>] nfs_commit_release_pages+0x5f/0x230 [nfs]
[<ffffffffc069c2a9>] ? nfs_write_completion+0x39/0x160 [nfs]
[<ffffffffc069a916>] nfs_commit_release+0x16/0x30 [nfs]
[<ffffffffc0321a20>] rpc_free_task+0x30/0x70 [sunrpc]
[<ffffffffc0321a72>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffff810a882a>] process_one_work+0x17a/0x440
[<ffffffff810a94f6>] worker_thread+0x126/0x3c0
[<ffffffff810a93d0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b099f>] kthread+0xcf/0xe0
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4fd8>] ret_from_fork+0x58/0x90
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
Code: f7 e8 d9 0d 01 c1 49 8b 45 10 48 85 c0 74 3e 48 8b 10 f7 c2 00 00 01 00 75 33 48 c7 40 30 00 00 00 00 49 8b 45 10 f0 80 60 01 f7 <4d> 8b 7d 10 4c 89 ff e8 88 64 ae c0 ba 0b 00 00 00 4c 89 fe 48 

NMI watchdog: BUG: soft lockup - CPU#12 stuck for 24s! [kworker/12:0:158899]
Modules linked in: uas usb_storage tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nfsv3 nfs fscache nls_utf8 isofs loop rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_ib(OE) mlx4_en(OE) mlx4_core(OE) vfat fat skx_edac coretemp intel_rapl iosf_mbi iTCO_wdt iTCO_vendor_support kvm irqbypass joydev ses enclosure pcspkr scsi_transport_sas sg lpc_ich i2c_i801 shpchp mei_me mei ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter nfsd binfmt_misc sch_fq_codel knem(OE) auth_rpcgss nfs_acl
 lockd grace ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) 8021q garp mrp stp llc crct10dif_pclmul crct10dif_common crc32_pclmul crc32c_intel ghash_clmulni_intel mlx5_core(OE) aesni_intel ast i2c_algo_bit lrw gf128mul drm_kms_helper glue_helper ablk_helper vfio_mdev(OE) vfio_iommu_type1 cryptd vfio ahci syscopyarea sysfillrect megaraid_sas sysimgblt mdev(OE) fb_sys_fops mlx_compat(OE) ttm libahci mlxfw(OE) i40e devlink libata drm ptp pps_core i2c_core wmi scsi_transport_iscsi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 158899 Comm: kworker/12:0 Kdump: loaded Tainted: G           OE  ------------   3.10.0-862.el7.x86_64 #1
Hardware name: Inspur NF5280M5/YZMB-00882-104, BIOS 4.1.16 06/23/2020
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff8874583e3f40 ti: ffff886457dc4000 task.ti: ffff886457dc4000
RIP: 0010:[<ffffffff98f15670>]  [<ffffffff98f15670>] _raw_spin_lock+0x10/0x30
RSP: 0018:ffff886457dc7d38  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000005 RCX: 0000000000000034
RDX: 0000000000000001 RSI: ffff8865005eb2c0 RDI: ffff886f9b0968a8
RBP: ffff886457dc7d68 R08: 8028000000000000 R09: 65005eb2c0140000
R10: ffff88747ffd9000 R11: 0000000000000000 R12: 0000000000076819
R13: 0000000080000000 R14: 8028000000000000 R15: ffffffffc0aa9db3
FS:  0000000000000000(0000) GS:ffff88745da00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055f7b0d176b6 CR3: 000000044ea0e000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 00000000
Call Trace:
 [<ffffffffc0aad167>] ? nfs_inode_remove_request+0x47/0xf0 [nfs]
 [<ffffffffc0aae20f>] nfs_commit_release_pages+0x5f/0x230 [nfs]
 [<ffffffff988b037e>] ? move_linked_works+0x5e/0x90
 [<ffffffffc0aacc6c>] nfs_commit_release+0x1c/0x30 [nfs]
 [<ffffffffc0141d73>] rpc_free_task+0x33/0x70 [sunrpc]
 [<ffffffffc0141dc2>] rpc_async_release+0x12/0x20 [sunrpc]
 [<ffffffff988b2dff>] process_one_work+0x17f/0x440
 [<ffffffff988b3ac6>] worker_thread+0x126/0x3c0
 [<ffffffff988b39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
 [<ffffffff988bae31>] kthread+0xd1/0xe0
 [<ffffffff988bad60>] ? insert_kthread_work+0x40/0x40
 [<ffffffff98f1f61d>] ret_from_fork_nospec_begin+0x7/0x21
 [<ffffffff988bad60>] ? insert_kthread_work+0x40/0x40
Code: 44 00 00 85 d2 74 e4 0f 1f 40 00 eb ed 66 0f 1f 44 00 00 b8 01 00 00 00 5d c3 90 0f 1f 44 00 00 31 c0 ba 01 00 00 00 f0 0f b1 17 <85> c0 75 01 c3 55 89 c6 48 89 e5 e8 9f 2d ff ff 5d c3 0f 1f 40