NMI watchdog: BUG: soft lockup - CPU#137 stuck for 22s! [llama-server:48974]
Modules linked in: tcp_diag inet_diag veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache nvidia_drm(POE) nvidia_modeset(POE) nvidia_uvm(OE) kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr nvidia(POE) vfat fat ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt rpcrdma sunrpc rdma_ucm ib_iser joydev cdc_ether usbnet mii sg ib_umad i2c_piix4
ipmi_si ipmi_devintf ipmi_msghandler rdma_cm ib_ipoib iw_cm pinctrl_amd i2c_designware_platform i2c_designware_core ib_cm libiscsi scsi_transport_iscsi acpi_cpufreq mlx5_ib ib_uverbs ib_core ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea mlx5_core sysfillrect sysimgblt fb_sys_fops ttm igb ahci libahci drm mlxfw dca libata devlink i2c_algo_bit megaraid_sas drm_panel_orientation_quirks ptp pps_core nfit libnvdimm wmi dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 137 PID: 48974 Comm: llama-server Kdump: loaded Tainted: P           OEL ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: ASUSTeK COMPUTER INC. ESC8000A-E11/KMPG-D32 Series, BIOS 2002 08/30/2024
task: ffff90f3c6f03180 ti: ffff91f297dc4000 task.ti: ffff91f297dc4000
RIP: 0010:[<ffffffffa8824296>]  [<ffffffffa8824296>] iommu_unmap_page+0x66/0x110
RSP: 0018:ffff91f297dc7998  EFLAGS: 00000246
RAX: 0000000000000000 RBX: 000000000000001b RCX: 0000000000000027
RDX: 000088815a53f000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffff91f297dc79d0 R08: 0000000000000003 R09: ffff91f297dc7998
R10: 000000000000001b R11: 000ffffffffff000 R12: 0000000000000000
R13: ffff91f297dc7988 R14: 0000000097dc7910 R15: ffffffffc10ed631
FS:  00007f2242355000(0000) GS:ffff90f40ec40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000006563c38 CR3: 0000007dea552000 CR4: 0000000000760ee0
PKRU: 55555554
Call Trace:
[<ffffffffa8826523>] __unmap_single.isra.22+0x63/0x200
[<ffffffffa8826e7f>] unmap_sg+0x5f/0x80
[<ffffffffc10dd3ce>] nv_unmap_dma_map_scatterlist+0x8e/0xc0 [nvidia]
[<ffffffffc10de41b>] nv_dma_unmap_alloc+0x19b/0x1b0 [nvidia]
[<ffffffffc1bd5fd5>] _nv041699rm+0xc5/0x1d0 [nvidia]
[<ffffffffc113cddf>] ? _nv032724rm+0x6f/0x90 [nvidia]
[<ffffffffc11a0dca>] ? _nv039226rm+0x25a/0x290 [nvidia]
[<ffffffffc1be9c80>] ? rm_gpu_ops_free_duped_handle+0x20/0x60 [nvidia]
[<ffffffffc10ed77a>] ? nvUvmInterfaceFreeDupedHandle+0x2a/0x50 [nvidia]
[<ffffffffc4502b2a>] ? uvm_ext_gpu_map_free_internal+0x6a/0x90 [nvidia_uvm]
[<ffffffffc45038be>] ? uvm_ext_gpu_map_free+0xe/0x30 [nvidia_uvm]
[<ffffffffc44b05c4>] ? uvm_deferred_free_object_list+0x64/0x120 [nvidia_uvm]
[<ffffffffc4504a10>] ? uvm_api_unmap_external+0x100/0x220 [nvidia_uvm]
[<ffffffffc449cc5d>] ? uvm_ioctl+0x1a9d/0x1dd0 [nvidia_uvm]
[<ffffffffa83fd4d4>] ? unmap_page_range+0xa94/0xc80
[<ffffffffa83d9685>] ? lru_add_drain+0x15/0x20
[<ffffffffa841610d>] ? free_pages_and_swap_cache+0x1d/0xd0
[<ffffffffa84332fd>] ? __slab_free+0x9d/0x2b0
[<ffffffffc449cfcc>] ? uvm_unlocked_ioctl+0x3c/0x60 [nvidia_uvm]
[<ffffffffc449d0e4>] ? uvm_unlocked_ioctl_entry+0x64/0xd0 [nvidia_uvm]
[<ffffffffa84719d8>] ? do_vfs_ioctl+0x3a8/0x5c0
[<ffffffffa8471c71>] ? SyS_ioctl+0x81/0xa0
[<ffffffffa8346986>] ? __audit_syscall_exit+0x1f6/0x2b0
[<ffffffffa89c539a>] ? system_call_fastpath+0x25/0x2a
Code: 00 00 00 4c 8d a7 98 00 00 00 49 89 fd 49 89 f7 45 31 f6 66 90 41 8b bd 94 00 00 00 48 8d 4d c8 4c 89 fa 4c 89 e6 e8 0a fe ff ff <48> 85 c0 49 89 c1 74 43 48 b8 8f e3 38 8e e3 38 8e e3 45 31 c0