NMI watchdog: BUG: soft lockup - CPU#47 stuck for 67s! [kiblnd_sd_05_01:4010]
Modules linked in: overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) crc_t10dif crct10dif_generic rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) dell_rbu sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) vfat fat kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper dell_smbios ablk_helper cryptd dcdbas dell_wmi_descriptor pcspkr i2c_piix4 ipmi_si ipmi_devintf
ipmi_msghandler acpi_power_meter binfmt_misc knem(OE) ip_tables ext4 mbcache jbd2 mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt ahci fb_sys_fops ttm libahci crct10dif_pclmul crct10dif_common drm crc32c_intel mlx5_core(OE) libata tg3 bnxt_en nvme mlxfw(OE) nvme_core psample auxiliary(OE) mlx_compat(OE) devlink ptp drm_panel_orientation_quirks pps_core wmi dm_mirror dm_region_hash dm_log dm_mod xpmem(OE) fuse
CPU: 47 PID: 4010 Comm: kiblnd_sd_05_01 Kdump: loaded Tainted: P           OE  ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0590KW, BIOS 2.12.4 07/26/2023
task: ffff970071432100 ti: ffff97007143c000 task.ti: ffff97007143c000
RIP: 0010:[<ffffffffa0ddf13d>]  [<ffffffffa0ddf13d>] next_zones_zonelist+0x2d/0x60
RSP: 0018:ffff97007143f700  EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffeef5aca3fec0 RCX: ffff97007143f7e0
RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff96a17fedb058
RBP: ffff97007143f700 R08: ffff96017fdda000 R09: 0000000000000000
R10: 0000000000002265 R11: fff00000f0000000 R12: ffff96617f2da4a0
R13: 000000000000000a R14: 0000000000000000 R15: ffff96617f2da000
FS:  0000000000000000(0000) GS:ffff96a0ffdc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002acaa36ed9c8 CR3: 0000009e95210000 CR4: 0000000000760ee0
PKRU: 00000000
Call Trace:
[<ffffffffa0dc88fd>] get_page_from_freelist+0x63d/0xac0
[<ffffffffa0dc8f04>] __alloc_pages_nodemask+0x184/0xbe0
[<ffffffffa0f86a45>] ? cpumask_next_and+0x35/0x50
[<ffffffffa0e193b8>] alloc_pages_current+0x98/0x110
[<ffffffffa0dc393e>] __get_free_pages+0xe/0x40
[<ffffffffa0dc3986>] get_zeroed_page+0x16/0x20
[<ffffffffa11fea32>] iommu_map_page+0x182/0x4c0
[<ffffffffa1200f7d>] map_sg+0x17d/0x2f0
[<ffffffffc08a0227>] kiblnd_map_tx+0x2a7/0x430 [ko2iblnd]
[<ffffffffc08a0d62>] kiblnd_setup_rd_kiov+0x252/0x3c0 [ko2iblnd]
[<ffffffffc08a61be>] kiblnd_recv+0x3be/0x7f0 [ko2iblnd]
[<ffffffffc0cc31e8>] lnet_ni_recv+0xc8/0x330 [lnet]
[<ffffffffc0ccb0e9>] lnet_parse_local+0x4c9/0xd40 [lnet]
[<ffffffffc0ccbfaf>] ? lnet_parse+0x64f/0x11f0 [lnet]
[<ffffffffc0ccc2fa>] lnet_parse+0x99a/0x11f0 [lnet]
[<ffffffffc08e67fb>] ? mlx5_ib_poll_cq+0x40b/0xdd0 [mlx5_ib]
[<ffffffffc08a69a3>] kiblnd_handle_rx+0x213/0x6b0 [ko2iblnd]
[<ffffffffc08adb62>] kiblnd_scheduler+0xfa2/0x11c0 [ko2iblnd]
[<ffffffffa0cdadf0>] ? wake_up_state+0x20/0x20
[<ffffffffc08acbc0>] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd]
[<ffffffffa0cc5e61>] kthread+0xd1/0xe0
[<ffffffffa0cc5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffa1395de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffa0cc5d90>] ? insert_kthread_work+0x40/0x40
Code: 66 66 90 55 48 85 d2 48 89 e5 75 27 3b 77 08 72 08 eb 18 0f 1f 00 48 89 c7 48 8d 47 10 39 70 08 77 f4 4c 8b 47 10 4c 89 01 5d c3 <4c> 8b 07 48 89 f8 eb f3 48 89 f8 eb 17 66 0f 1f 44 00 00 41 8b 

NMI watchdog: BUG: soft lockup - CPU#26 stuck for 67s! [kiblnd_sd_03_01:4047]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) crc_t10dif crct10dif_generic rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) dell_rbu sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) vfat fat kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper dell_smbios cryptd dcdbas dell_wmi_descriptor pcspkr i2c_piix4 ipmi_si ipmi_devintf ipmi_msghandler
acpi_power_meter binfmt_misc knem(OE) ip_tables ext4 mbcache jbd2 mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci crct10dif_pclmul mlx5_core(OE) drm crct10dif_common crc32c_intel libata tg3 bnxt_en nvme mlxfw(OE) psample nvme_core auxiliary(OE) mlx_compat(OE) devlink ptp drm_panel_orientation_quirks pps_core wmi dm_mirror dm_region_hash dm_log dm_mod xpmem(OE) fuse
CPU: 26 PID: 4047 Comm: kiblnd_sd_03_01 Kdump: loaded Tainted: P           OE  ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0590KW, BIOS 2.14.1 12/17/2023
task: ffff89367637e300 ti: ffff893675ea0000 task.ti: ffff893675ea0000
RIP: 0010:[<ffffffff98fc34d6>]  [<ffffffff98fc34d6>] zlc_zone_worth_trying+0x26/0x50
RSP: 0018:ffff893675ea3700  EFLAGS: 00000212
RAX: 0000000000000000 RBX: ffffd77c0d539d40 RCX: ffff8936ff2eb018
RDX: ffff89367637ebb0 RSI: 0000000000000001 RDI: ffff8936ff2db008
RBP: ffff893675ea3700 R08: ffff88d6ffdda000 R09: 0000000000000000
R10: 0000000000002263 R11: ffffffffffffff83 R12: ffff8916ffeda4a0
R13: 000000000000000a R14: 0000000000000000 R15: ffff8916ffeda000
FS:  0000000000000000(0000) GS:ffff89367f080000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efecc507030 CR3: 0000003121e10000 CR4: 0000000000760ee0
PKRU: 00000000
Call Trace:
[<ffffffff98fc8a52>] get_page_from_freelist+0x792/0xac0
[<ffffffff98fc8f04>] __alloc_pages_nodemask+0x184/0xbe0
[<ffffffff99186a45>] ? cpumask_next_and+0x35/0x50
[<ffffffff990193b8>] alloc_pages_current+0x98/0x110
[<ffffffff98fc393e>] __get_free_pages+0xe/0x40
[<ffffffff98fc3986>] get_zeroed_page+0x16/0x20
[<ffffffff993fea32>] iommu_map_page+0x182/0x4c0
[<ffffffff99400f7d>] map_sg+0x17d/0x2f0
[<ffffffffc075f227>] kiblnd_map_tx+0x2a7/0x430 [ko2iblnd]
[<ffffffffc075fd62>] kiblnd_setup_rd_kiov+0x252/0x3c0 [ko2iblnd]
[<ffffffffc07651be>] kiblnd_recv+0x3be/0x7f0 [ko2iblnd]
[<ffffffffc139c1e8>] lnet_ni_recv+0xc8/0x330 [lnet]
[<ffffffffc13a40e9>] lnet_parse_local+0x4c9/0xd40 [lnet]
[<ffffffff98ed757f>] ? ttwu_do_activate+0x6f/0x80
[<ffffffffc13a52fa>] lnet_parse+0x99a/0x11f0 [lnet]
[<ffffffff98ed30c2>] ? __wake_up_common+0x82/0x120
[<ffffffffc057b7fb>] ? mlx5_ib_poll_cq+0x40b/0xdd0 [mlx5_ib]
[<ffffffffc07659a3>] kiblnd_handle_rx+0x213/0x6b0 [ko2iblnd]
[<ffffffffc076cb62>] kiblnd_scheduler+0xfa2/0x11c0 [ko2iblnd]
[<ffffffff98edadf0>] ? wake_up_state+0x20/0x20
[<ffffffffc076bbc0>] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd]
[<ffffffff98ec5e61>] kthread+0xd1/0xe0
[<ffffffff98ec5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff99595de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff98ec5d90>] ? insert_kthread_work+0x40/0x40
Code: 0f 1f 44 00 00 66 66 66 66 90 55 48 8b 0f 48 89 e5 48 85 c9 74 2f 48 83 c7 08 48 29 fe 48 c1 fe 04 48 63 c6 0f b7 04 41 0f a3 02 <19> d2 31 c0 85 d2 74 10 0f a3 b1 00 20 00 00 19 f6 31 c0 85 f6 

NMI watchdog: BUG: soft lockup - CPU#40 stuck for 22s! [kiblnd_sd_05_00:4200]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) crc_t10dif crct10dif_generic rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) dell_rbu sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) vfat fat kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper dell_smbios ablk_helper cryptd dcdbas dell_wmi_descriptor pcspkr cdc_ether usbnet mii i2c_piix4 ipmi_si ipmi_devintf
ipmi_msghandler acpi_power_meter binfmt_misc knem(OE) ip_tables xfs libcrc32c mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci crct10dif_pclmul drm crct10dif_common crc32c_intel mlx5_core(OE) libata tg3 bnxt_en nvme mlxfw(OE) nvme_core psample auxiliary(OE) mlx_compat(OE) devlink ptp drm_panel_orientation_quirks pps_core wmi dm_mirror dm_region_hash dm_log dm_mod xpmem(OE) fuse
CPU: 40 PID: 4200 Comm: kiblnd_sd_05_00 Kdump: loaded Tainted: P           OE  ------------   3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0590KW, BIOS 2.10.2 10/25/2022
task: ffff95015e7d0000 ti: ffff95015e7c4000 task.ti: ffff95015e7c4000
RIP: 0010:[<ffffffff921c82d1>]  [<ffffffff921c82d1>] get_page_from_freelist+0x11/0xac0
RSP: 0018:ffff95015e7c77d0  EFLAGS: 00000202
RAX: ffff9581ffeda000 RBX: 0000000000000041 RCX: ffff9581ffedb000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000028020
RBP: ffff95015e7c77f0 R08: 0000000000000002 R09: 0000000000000141
R10: 0000000000002265 R11: ffffffffffffff83 R12: ffff9581ffedb028
R13: ffff95e17fed9068 R14: ffff9581ffedb000 R15: 0000000000000001
FS:  0000000000000000(0000) GS:ffff95817fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002af518057148 CR3: 0000001f1ca10000 CR4: 0000000000760ee0
PKRU: 00000000
Call Trace:
[<ffffffff921c8f04>] __alloc_pages_nodemask+0x184/0xbe0
[<ffffffff921c7264>] ? free_one_page+0x2d4/0x300
[<ffffffff925fb493>] ? alloc_iova+0x33/0x1c0
[<ffffffff922193b8>] alloc_pages_current+0x98/0x110
[<ffffffff921c393e>] __get_free_pages+0xe/0x40
[<ffffffff921c3986>] get_zeroed_page+0x16/0x20
[<ffffffff925fea32>] iommu_map_page+0x182/0x4c0
[<ffffffff92600f7d>] map_sg+0x17d/0x2f0
[<ffffffffc0934227>] kiblnd_map_tx+0x2a7/0x430 [ko2iblnd]
[<ffffffffc0934d62>] kiblnd_setup_rd_kiov+0x252/0x3c0 [ko2iblnd]
[<ffffffffc093a079>] kiblnd_recv+0x279/0x7f0 [ko2iblnd]
[<ffffffffc1081bcc>] ? lnet_cpt_of_nid+0x5c/0x70 [lnet]
[<ffffffffc109c1e8>] lnet_ni_recv+0xc8/0x330 [lnet]
[<ffffffffc109c9f5>] lnet_recv_put+0x85/0xb0 [lnet]
[<ffffffffc10a41ce>] lnet_parse_local+0x5ae/0xd40 [lnet]
[<ffffffff920e482c>] ? update_curr+0x14c/0x1e0
[<ffffffff920e0f5e>] ? account_entity_dequeue+0xae/0xd0
[<ffffffffc10a52fa>] lnet_parse+0x99a/0x11f0 [lnet]
[<ffffffff920d30c2>] ? __wake_up_common+0x82/0x120
[<ffffffffc06d27fb>] ? mlx5_ib_poll_cq+0x40b/0xdd0 [mlx5_ib]
[<ffffffffc093a9a3>] kiblnd_handle_rx+0x213/0x6b0 [ko2iblnd]
[<ffffffffc0941b62>] kiblnd_scheduler+0xfa2/0x11c0 [ko2iblnd]
[<ffffffff920dadf0>] ? wake_up_state+0x20/0x20
[<ffffffffc0940bc0>] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd]
[<ffffffff920c5e61>] kthread+0xd1/0xe0
[<ffffffff920c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff92795de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff920c5d90>] ? insert_kthread_work+0x40/0x40
Code: e8 e5 ef ff ff 5d f3 c3 66 90 31 f6 e8 99 f6 ff ff 5d eb f2 66 0f 1f 44 00 00 66 66 66 66 90 55 48 89 e5 41 57 41 56 41 55 41 54 <53> 48 81 ec e0 00 00 00 48 8b 45 10 89 7d b0 48 89 b5 08 ff ff 

NMI watchdog: BUG: soft lockup - CPU#35 stuck for 22s! [kiblnd_sd_04_01:4063]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) crc_t10dif crct10dif_generic rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) dell_rbu sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) nvidia(POE) vfat fat kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper dell_smbios ablk_helper cryptd dcdbas dell_wmi_descriptor pcspkr i2c_piix4 ipmi_si ipmi_devintf ipmi_msghandler
acpi_power_meter knem(OE) binfmt_misc ip_tables ext4 mbcache jbd2 mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci crct10dif_pclmul libahci crct10dif_common crc32c_intel drm mlx5_core(OE) libata tg3 bnxt_en nvme nvme_core mlxfw(OE) psample auxiliary(OE) mlx_compat(OE) devlink ptp drm_panel_orientation_quirks pps_core wmi dm_mirror dm_region_hash dm_log dm_mod xpmem(OE) fuse
CPU: 35 PID: 4063 Comm: kiblnd_sd_04_01 Kdump: loaded Tainted: P           OEL ------------   3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0590KW, BIOS 2.10.2 10/25/2022
task: ffff9cc8b5a34200 ti: ffff9cc8b4204000 task.ti: ffff9cc8b4204000
RIP: 0010:[<ffffffffa93c844e>]  [<ffffffffa93c844e>] get_page_from_freelist+0x18e/0xac0
RSP: 0018:ffff9cc8b4207710  EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffff9cc8b4207700 RCX: 0000000000000002
RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff9d493feda000
RBP: ffff9cc8b4207818 R08: 0000000000000041 R09: 0000000000000000
R10: 0000000000002265 R11: ffffffffffffff83 R12: ffffffffa95a681d
R13: ffff9cc8b4207690 R14: ffff9d893feda4a0 R15: 000000000000000a
FS:  0000000000000000(0000) GS:ffff9d48bfcc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f937e9a8000 CR3: 0000010fccc10000 CR4: 0000000000760ee0
PKRU: 00000000
Call Trace:
[<ffffffffa93c8f04>] __alloc_pages_nodemask+0x184/0xbe0
[<ffffffffa9586a45>] ? cpumask_next_and+0x35/0x50
[<ffffffffa97fb493>] ? alloc_iova+0x33/0x1c0
[<ffffffffa94193b8>] alloc_pages_current+0x98/0x110
[<ffffffffa93c393e>] __get_free_pages+0xe/0x40
[<ffffffffa93c3986>] get_zeroed_page+0x16/0x20
[<ffffffffa97fea32>] iommu_map_page+0x182/0x4c0
[<ffffffffa9800f7d>] map_sg+0x17d/0x2f0
[<ffffffffc074b227>] kiblnd_map_tx+0x2a7/0x430 [ko2iblnd]
[<ffffffffc074bd62>] kiblnd_setup_rd_kiov+0x252/0x3c0 [ko2iblnd]
[<ffffffffc07511be>] kiblnd_recv+0x3be/0x7f0 [ko2iblnd]
[<ffffffffc09a01e8>] lnet_ni_recv+0xc8/0x330 [lnet]
[<ffffffffc09a80e9>] lnet_parse_local+0x4c9/0xd40 [lnet]
[<ffffffffa92d757f>] ? ttwu_do_activate+0x6f/0x80
[<ffffffffc09a92fa>] lnet_parse+0x99a/0x11f0 [lnet]
[<ffffffffa92d30c2>] ? __wake_up_common+0x82/0x120
[<ffffffffc057b7fb>] ? mlx5_ib_poll_cq+0x40b/0xdd0 [mlx5_ib]
[<ffffffffc07519a3>] kiblnd_handle_rx+0x213/0x6b0 [ko2iblnd]
[<ffffffffc0758b62>] kiblnd_scheduler+0xfa2/0x11c0 [ko2iblnd]
[<ffffffffa92dadf0>] ? wake_up_state+0x20/0x20
[<ffffffffc0757bc0>] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd]
[<ffffffffa92c5e61>] kthread+0xd1/0xe0
[<ffffffffa92c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffa9995de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffa92c5d90>] ? insert_kthread_work+0x40/0x40
Code: 60 ff ff ff 04 75 46 44 8b 85 60 ff ff ff 8b 8d 18 ff ff ff 45 31 c9 8b 75 c4 4c 89 ff 45 89 c4 41 83 e4 03 49 63 c4 49 8b 1c c7 <49> 8b 87 d8 05 00 00 48 85 c0 48 89 da 4c 0f 49 c8 e8 6c af ff