NMI watchdog: BUG: soft lockup - CPU#105 stuck for 23s! [kiblnd_sd_06_01:4790]
Modules linked in: overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter mgc(OE) lustre(OE) lmv(OE) mdc(OE) fid(OE) osc(OE) lov(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rdma_ucm(OE) rdma_cm(OE) iw_cm(OE) dell_rbu ib_ipoib(OE) ib_cm(OE) ib_umad(OE) sunrpc vfat fat dell_smbios dcdbas dell_wmi_descriptor kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr raid0 sg i2c_piix4 wmi ipmi_si ipmi_devintf ipmi_msghandler tpm_crb acpi_power_meter binfmt_misc knem(OE) ip_tables xfs libcrc32c
sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common mlx5_core(OE) ahci crc32c_intel tg3 bnxt_en libahci mlxfw(OE) psample megaraid_sas libata auxiliary(OE) mlx_compat(OE) devlink nvme drm_panel_orientation_quirks ptp pps_core nvme_core dm_mirror dm_region_hash dm_log dm_mod xpmem(OE) fuse
CPU: 105 PID: 4790 Comm: kiblnd_sd_06_01 Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R7525/0H3K7P, BIOS 2.8.4 06/23/2022
task: ffff9fe6ac8b3180 ti: ffff9fe6ac8c0000 task.ti: ffff9fe6ac8c0000
RIP: 0010:[<ffffffff941fecc5>]  [<ffffffff941fecc5>] iommu_map_page+0x415/0x4c0
RSP: 0018:ffff9fe6ac8c39c0  EFLAGS: 00000246
RAX: ffff9fe6bd377000 RBX: ffff9fe6ac8b3180 RCX: 0000000000000000
RDX: 6000036c91e20e01 RSI: 0000000000000000 RDI: ffffa2d4d1e20000
RBP: ffff9fe6ac8c3a28 R08: ffffa2d551e20000 R09: 0000000000000e00
R10: 00000000000027b0 R11: ffffffffffffff83 R12: 0000000000000002
R13: 0000000000008020 R14: ffffffff94ddc1b0 R15: 000000007f283a14
FS:  0000000000000000(0000) GS:ffffa3663fc40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc62acc8000 CR3: 000003fd16c10000 CR4: 0000000000760ee0
PKRU: 00000000
Call Trace:
[<ffffffff94200f7d>] map_sg+0x17d/0x2f0
[<ffffffffc08c9227>] kiblnd_map_tx+0x2a7/0x430 [ko2iblnd]
[<ffffffffc08c9d62>] kiblnd_setup_rd_kiov+0x252/0x3c0 [ko2iblnd]
[<ffffffffc08cf1be>] kiblnd_recv+0x3be/0x7f0 [ko2iblnd]
[<ffffffffc09ff1e8>] lnet_ni_recv+0xc8/0x330 [lnet]
[<ffffffffc0a070e9>] lnet_parse_local+0x4c9/0xd40 [lnet]
[<ffffffff93cd757f>] ? ttwu_do_activate+0x6f/0x80
[<ffffffffc0a082fa>] lnet_parse+0x99a/0x11f0 [lnet]
[<ffffffff93cd30c2>] ? __wake_up_common+0x82/0x120
[<ffffffffc06697fb>] ? mlx5_ib_poll_cq+0x40b/0xdd0 [mlx5_ib]
[<ffffffff93d07c32>] ? ktime_get+0x52/0xe0
[<ffffffffc08cf9a3>] kiblnd_handle_rx+0x213/0x6b0 [ko2iblnd]
[<ffffffffc08d6b62>] kiblnd_scheduler+0xfa2/0x11c0 [ko2iblnd]
[<ffffffff93cdadf0>] ? wake_up_state+0x20/0x20
[<ffffffffc08d5bc0>] ? kiblnd_cq_event+0x90/0x90 [ko2iblnd]
[<ffffffff93cc5e61>] kthread+0xd1/0xe0
[<ffffffff93cc5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffff94395de4>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff93cc5d90>] ? insert_kthread_work+0x40/0x40
Code: 00 01 8d 58 01 41 89 9c 24 94 00 00 00 e9 94 fc ff ff 0f 1f 40 00 41 83 fe 07 0f 85 50 fd ff ff 48 8b 45 b8 c6 80 b8 00 00 00 01 <e9> 40 fd ff ff 66 0f 1f 44 00 00 41 8b 9c 24 94 00 00 00 e9 63