NMI watchdog: BUG: soft lockup - CPU#198 stuck for 22s! [llama-server:207448]
Modules linked in: dm_round_robin overlay(T) tcp_diag udp_diag inet_diag xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 devlink tun iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi binfmt_misc ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vboxnetadp(OE) vboxnetflt(OE) vboxdrv(OE) sunrpc dm_mirror dm_region_hash dm_log snd_hda_codec_hdmi amd64_edac_mod edac_mce_amd kvm_amd kvm dm_service_time irqbypass crc32_pclmul
ghash_clmulni_intel aesni_intel dm_multipath lrw gf128mul glue_helper ablk_helper cryptd pcspkr nvidia_drm(OE) nvidia_modeset(OE) nvidia_uvm(OE) nvidia(OE) snd_hda_intel dm_mod snd_hda_codec snd_hda_core snd_hwdep snd_seq ses enclosure snd_seq_device scsi_transport_sas snd_pcm sg snd_timer snd k10temp i2c_piix4 soundcore ipmi_si ipmi_devintf ipmi_msghandler i2c_designware_platform pinctrl_amd i2c_designware_core acpi_cpufreq sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe ahci drm libahci igb libata mdio megaraid_sas ptp pps_core drm_panel_orientation_quirks dca i2c_algo_bit nfit libnvdimm fuse
CPU: 198 PID: 207448 Comm: llama-server Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: N/A 2U GPU Server/65N32-US, BIOS 0ZZQ002035 05/14/2021
task: ffff96c995ce9080 ti: ffff9730d6ba8000 task.ti: ffff9730d6ba8000
RIP: 0010:[<ffffffff8d4240f7>]  [<ffffffff8d4240f7>] fetch_pte.isra.6+0x57/0x190
RSP: 0018:ffff9730d6bab410  EFLAGS: 00000206
RAX: ffff96aed51ed8a8 RBX: ffff9730d6bab3d0 RCX: 0000000000000027
RDX: 00008afc49320000 RSI: 0000000000000000 RDI: 0000000000000003
RBP: ffff9730d6bab410 R08: 0000000000000002 R09: ffff9730d6bab420
R10: 000000000000001b R11: 000ffffffffff000 R12: 0000000180800035
R13: ffff97093fc00f80 R14: ffffffff8d031fea R15: ffff9730d6bab3f0
FS:  00007f78c9732000(0000) GS:ffff9707bfb80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f62c171d000 CR3: 00000213f8e10000 CR4: 0000000000340fe0
Call Trace:
[<ffffffff8d424296>] iommu_unmap_page+0x66/0x110
[<ffffffff8d426523>] __unmap_single.isra.22+0x63/0x200
[<ffffffff8d426e7f>] unmap_sg+0x5f/0x80
[<ffffffffc0e6597e>] nv_unmap_dma_map_scatterlist+0x8e/0xc0 [nvidia]
[<ffffffffc0e66685>] nv_dma_unmap_pages+0x125/0x130 [nvidia]
[<ffffffffc0e669e5>] nv_dma_unmap_alloc+0x45/0x60 [nvidia]
[<ffffffffc0f525c5>] osIovaUnmap+0xc5/0x1f0 [nvidia]
[<ffffffffc10d42d7>] iovaspaceDestroyMapping_IMPL+0x67/0x90 [nvidia]
[<ffffffffc1047b84>] memdescDestroy.part.0+0x184/0x510 [nvidia]
[<ffffffffc10d5355>] memDestruct_IMPL+0x65/0x160 [nvidia]
[<ffffffffc0eab6a5>] __nvoc_dtor_Memory+0x15/0x30 [nvidia]
[<ffffffffc114ea1c>] __nvoc_objDelete+0x2c/0xf0 [nvidia]
[<ffffffffc1154330>] clientFreeResource_IMPL+0x640/0x7b0 [nvidia]
[<ffffffffc0e6f49a>] ? os_release_spinlock+0x1a/0x20 [nvidia]
[<ffffffffc0f8449e>] ? _rmGpuLocksAcquire+0x37e/0x940 [nvidia]
[<ffffffffc11569b8>] ? _serverFindClientEntry.isra.0+0x58/0xd0 [nvidia]
[<ffffffffc10ff7dc>] ? serverResLock_Prologue+0x1fc/0x260 [nvidia]
[<ffffffffc110241d>] rmclientFreeResource_IMPL+0x4d/0x60 [nvidia]
[<ffffffffc115a3d4>] serverFreeResourceTreeUnderLock+0xb4/0x160 [nvidia]
[<ffffffffc115a7a6>] serverFreeResourceTree+0x326/0x960 [nvidia]
[<ffffffffc0e6f462>] ? os_acquire_spinlock+0x12/0x30 [nvidia]
[<ffffffffc115afa7>] serverFreeDisabledClients+0xb7/0x1c0 [nvidia]
[<ffffffffc0f54db3>] rm_cleanup_file_private+0x173/0x190 [nvidia]
[<ffffffffc0e5f95e>] nvidia_close+0x13e/0x2c0 [nvidia]
[<ffffffff8d05dfcc>] __fput+0xec/0x230
[<ffffffff8d05e1fe>] ____fput+0xe/0x20
[<ffffffff8cec7f6b>] task_work_run+0xbb/0xe0
[<ffffffff8cea6174>] do_exit+0x2d4/0xa30
[<ffffffff8cf1a61f>] ? futex_wait+0x11f/0x290
[<ffffffff8cea694f>] do_group_exit+0x3f/0xa0
[<ffffffff8ceb825e>] get_signal_to_deliver+0x1ce/0x5e0
[<ffffffff8ce2d5b7>] do_signal+0x57/0x6f0
[<ffffffff8cf1c376>] ? do_futex+0x106/0x4d0
[<ffffffff8ceb5da1>] ? __set_task_blocked+0x41/0xa0
[<ffffffff8ce2dcca>] do_notify_resume+0x7a/0xd0
[<ffffffff8d5c56f7>] int_signal+0x12/0x17
Code: 01 48 8b 36 48 89 d0 44 8d 14 ff 41 8d 4a 0c 48 d3 e8 25 ff 01 00 00 48 8d 04 c6 be 01 00 00 00 48 d3 e6 85 ff 49 89 31 48 8b 30 <0f> 8e b2 00 00 00 40 f6 c6 01 0f 84 19 01 00 00 49 89 f0 49 c1