NMI watchdog: BUG: soft lockup - CPU#48 stuck for 22s! [kworker/48:2:5468]
Modules linked in: veth fuse cdc_mbim cdc_wdm cdc_ncm ax88179_178a usbnet mii cmac arc4 md4 nls_utf8 cifs ccm xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_uverbs(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) mlx4_ib(OE) ib_core(OE) mlx4_core(OE) mlx_compat(OE) devlink overlay(T) sunrpc vfat fat sb_edac coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul
ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd mxm_wmi iTCO_wdt iTCO_vendor_support i2c_i801 ses mei_me joydev enclosure mei sg pcspkr lpc_ich shpchp wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter knem(OE) ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci ixgbe igb crct10dif_pclmul crct10dif_common crc32c_intel libata aacraid mpt3sas i2c_algo_bit mdio ptp i2c_core pps_core raid_class dca scsi_transport_sas dm_mirror dm_region_hash dm_log dm_mod [last unloaded: devlink]
CPU: 48 PID: 5468 Comm: kworker/48:2 Kdump: loaded Tainted: G           OE  ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name:   4U Rack Server/60G24-US, BIOS 227 05/07/2018
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff96095be4dee0 ti: ffff95f1b662c000 task.ti: ffff95f1b662c000
RIP: 0010:[<ffffffffc08c14c3>]  [<ffffffffc08c14c3>] nfs_page_group_lock+0x23/0x90 [nfs]
RSP: 0018:ffff95f1b662fcf8  EFLAGS: 00000246
RAX: ffff9609c022c300 RBX: 000000006f732301 RCX: 0000000000000034
RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff9609c022c340
RBP: ffff95f1b662fd08 R08: 8000000000000000 R09: 09c022c340000000
R10: 05938277c308d700 R11: 0000000000000000 R12: 3400000000000000
R13: ffff95f1b662fd08 R14: 000000010001a680 R15: ffff9609c022c280
FS:  0000000000000000(0000) GS:ffff96095fb80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000042b98039370 CR3: 000000219560e000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc08c1b8b>] nfs_page_group_sync_on_bit+0x1b/0x100 [nfs]
[<ffffffffc08c1f26>] nfs_page_group_destroy+0x36/0x70 [nfs]
[<ffffffffc08c1f87>] nfs_unlock_and_release_request+0x27/0x30 [nfs]
[<ffffffffc08c6224>] nfs_commit_release_pages+0x74/0x230 [nfs]
[<ffffffff83d1551e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffc08c4c6c>] nfs_commit_release+0x1c/0x30 [nfs]
[<ffffffffc0587d73>] rpc_free_task+0x33/0x70 [sunrpc]
[<ffffffffc0587dc2>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffff836b2dff>] process_one_work+0x17f/0x440
[<ffffffff836b3ac6>] worker_thread+0x126/0x3c0
[<ffffffff836b39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff836bae31>] kthread+0xd1/0xe0
[<ffffffff836bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffff83d1f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff836bad60>] ? insert_kthread_work+0x40/0x40
Code: c3 5d c3 0f 1f 44 00 00 0f 1f 44 00 00 55 48 89 e5 41 54 41 89 f4 53 48 8b 5f 58 48 3b 5b 58 75 52 48 8d 7b 40 f0 0f ba 6b 40 05 <19> d2 31 c0 85 d2 74 18 45 84 e4 b8 f5 ff ff ff 75 0e f0 0f ba 

NMI watchdog: BUG: soft lockup - CPU#22 stuck for 23s! [kworker/22:1:4918]
Modules linked in: nfsv3 nfs_acl fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx4_ib(OE) mlx4_en(OE) mlx4_core(OE) vfat fat skx_edac coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass iTCO_wdt iTCO_vendor_support joydev pcspkr sg lpc_ich mei_me i2c_i801 shpchp mei ipmi_si
ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_power_meter binfmt_misc sch_fq_codel knem(OE) ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) 8021q garp mrp stp llc crct10dif_pclmul crct10dif_common crc32_pclmul crc32c_intel ghash_clmulni_intel ast mlx5_core(OE) aesni_intel drm_kms_helper lrw gf128mul glue_helper vfio_mdev(OE) ablk_helper vfio_iommu_type1 cryptd syscopyarea sysfillrect vfio sysimgblt fb_sys_fops ttm mdev(OE) mlx_compat(OE) igb mlxfw(OE) drm devlink ahci megaraid_sas i40e libahci dca libata i2c_algo_bit scsi_transport_iscsi ptp i2c_core pps_core wmi sunrpc dm_mirror dm_region_hash dm_log dm_mod
CPU: 22 PID: 4918 Comm: kworker/22:1 Kdump: loaded Tainted: G           OE  ------------   3.10.0-862.el7.x86_64 #1
Hardware name: Inspur NF5180M5/NF5180M5, BIOS 4.1.5 05/21/2019
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff9359e1a70000 ti: ffff935757d8c000 task.ti: ffff935757d8c000
RIP: 0010:[<ffffffffc110ed78>]  [<ffffffffc110ed78>] nfs_free_request+0x88/0x200 [nfs]
RSP: 0018:ffff935757d8fd18  EFLAGS: 00000202
RAX: 002fffff00000000 RBX: ffffffffffffff10 RCX: 0000000000000034
RDX: 0000000000000005 RSI: ffff93831f86d040 RDI: ffff93853bd46528
RBP: ffff935757d8fd30 R08: 8028000000000000 R09: 831f86d040140000
R10: 1d20c3095e6b9bc0 R11: 0000000000000000 R12: a000000000000000
R13: 0000000180400020 R14: 0000000000000034 R15: ffff93873ffba000
FS:  0000000000000000(0000) GS:ffff936edfb80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002adef94f8240 CR3: 0000001c88e0e000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 00000000
Call Trace:
[<ffffffffc110ef44>] nfs_page_group_destroy+0x54/0x70 [nfs]
[<ffffffffc110ef87>] nfs_unlock_and_release_request+0x27/0x30 [nfs]
[<ffffffffc1113224>] nfs_commit_release_pages+0x74/0x230 [nfs]
[<ffffffffc1111c6c>] nfs_commit_release+0x1c/0x30 [nfs]
[<ffffffffc0128d73>] rpc_free_task+0x33/0x70 [sunrpc]
[<ffffffffc0128dc2>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffffb8ab2dff>] process_one_work+0x17f/0x440
[<ffffffffb8ab3ac6>] worker_thread+0x126/0x3c0
[<ffffffffb8ab39a0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffffb8abae31>] kthread+0xd1/0xe0
[<ffffffffb8abad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffb911f61d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb8abad60>] ? insert_kthread_work+0x40/0x40
Code: 10 4c 8b 6b 18 4c 8b 63 20 48 85 ff 74 0d e8 40 34 a9 f7 48 c7 43 10 00 00 00 00 4d 85 e4 74 22 49 8d 7c 24 28 f0 41 ff 4c 24 28 <0f> 94 c0 84 c0 75 41 4c 89 e7 e8 f9 52 ff ff 48 c7 43 20 00 00 

NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [kworker/3:2:29557]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfsv3 nfs fscache 8021q garp mrp stp llc ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx5_ib ib_core xfs libcrc32c skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass vfat fat iTCO_wdt crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_vendor_support joydev sg pcspkr ipmi_si ipmi_devintf ipmi_msghandler mei_me nfit mei i2c_i801 lpc_ich libnvdimm wmi shpchp acpi_cpufreq acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif
crct10dif_generic ast i2c_algo_bit drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul sysimgblt crct10dif_common fb_sys_fops crc32c_intel mlx5_core ttm ixgbe mpt3sas drm ahci libahci libata raid_class scsi_transport_sas devlink mdio ptp pps_core i2c_core dca
CPU: 3 PID: 29557 Comm: kworker/3:2 Tainted: G             L ------------   3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Ace Computers Ace /X11DPT-PS, BIOS 3.4 10/31/2020
Workqueue: nfsiod rpc_async_release [sunrpc]
task: ffff8817c110cf10 ti: ffff880063048000 task.ti: ffff880063048000
RIP: 0010:[<ffffffffc080a283>]  [<ffffffffc080a283>] nfs_page_group_lock+0x23/0x90 [nfs]
RSP: 0000:ffff88006304bcf8  EFLAGS: 00000246
RAX: ffff88176a6e0f80 RBX: ffffffffffffff10 RCX: 0000000000000034
RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff88176a6e0fc0
RBP: ffff88006304bd08 R08: 8000000000000000 R09: 176a6e0fc0000000
R10: e87797cadb83f000 R11: 0000000000000000 R12: ffff88176a6e05c0
R13: 0000000000000005 R14: 0000000100010034 R15: ffff88176a6e0f00
FS:  0000000000000000(0000) GS:ffff8817ddac0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002af6f8bb61e8 CR3: 00000000019f2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff88176a6e0f80 0000000000000006 ffff88006304bd30 ffffffffc080a94b
ffff88176a6e0f80 ffff88176a6e0f80 0000000000000000 ffff88006304bd50
ffffffffc080ace6 ffff88176a6e0f80 ffff880a0c103c80 ffff88006304bd68
Call Trace:
[<ffffffffc080a94b>] nfs_page_group_sync_on_bit+0x1b/0x100 [nfs]
[<ffffffffc080ace6>] nfs_page_group_destroy+0x36/0x70 [nfs]
[<ffffffffc080ad47>] nfs_unlock_and_release_request+0x27/0x30 [nfs]
[<ffffffffc080ee64>] nfs_commit_release_pages+0x74/0x230 [nfs]
[<ffffffffc080d916>] nfs_commit_release+0x16/0x30 [nfs]
[<ffffffffc0370a20>] rpc_free_task+0x30/0x70 [sunrpc]
[<ffffffffc0370a72>] rpc_async_release+0x12/0x20 [sunrpc]
[<ffffffff810a882a>] process_one_work+0x17a/0x440
[<ffffffff810a94f6>] worker_thread+0x126/0x3c0
[<ffffffff810a93d0>] ? manage_workers.isra.24+0x2a0/0x2a0
[<ffffffff810b099f>] kthread+0xcf/0xe0
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4fd8>] ret_from_fork+0x58/0x90
[<ffffffff810b08d0>] ? insert_kthread_work+0x40/0x40
Code: c0 5d c3 0f 1f 44 00 00 0f 1f 44 00 00 55 48 89 e5 41 54 41 89 f4 53 48 8b 5f 58 48 3b 5b 58 75 52 48 8d 7b 40 f0 0f ba 6b 40 05 <19> d2 31 c0 85 d2 74 18 45 84 e4 b8 f5 ff ff ff 75 0e f0 0f ba