BUG: soft lockup - CPU#1 stuck for 23s! [hfi1_cq0:1471]
Modules linked in: 8021q garp stp mrp llc mpt2sas raid_class scsi_transport_sas mptctl mptbase beegfs(OE) nvidia_uvm(OE) rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache ib_isert iscsi_target_mod target_core_mod ib_ucm nvidia_drm(POE) nvidia_modeset(POE) dell_rbu ext4 mbcache jbd2 dcdbas coretemp kvm crc32_pclmul svcrdma ipmi_devintf xprtrdma ghash_clmulni_intel rdma_ucm ib_uverbs(OE) aesni_intel lrw gf128mul ib_iser glue_helper ib_umad ablk_helper cryptd rdma_cm pcspkr ib_ipoib iw_cm nvidia(POE) ib_cm libiscsi ib_sa scsi_transport_iscsi raid0 drm sg shpchp wmi ipmi_si ipmi_msghandler acpi_power_meter acpi_pad nfsd auth_rpcgss nfs_acl lockd grace ip_tables sd_mod crc_t10dif crct10dif_generic hfi1(OE) rdmavt(OE) crct10dif_pclmul ib_mad crct10dif_common crc32c_intel i40e ib_core igb ib_addr
i2c_algo_bit vxlan i2c_core ip6_udp_tunnel ahci udp_tunnel dca libahci ptp libata megaraid_sas pps_core sunrpc
CPU: 1 PID: 1471 Comm: hfi1_cq0 Tainted: P   M    W  OE  ------------   3.10.0-327.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R740/014X06, BIOS 2.21.2 02/19/2024
task: ffff8817f2b97300 ti: ffff8817f2b54000 task.ti: ffff8817f2b54000
RIP: 0010:[<ffffffff8163cbb2>]  [<ffffffff8163cbb2>] _raw_spin_lock+0x32/0x50
RSP: 0018:ffff8817f2b579a8  EFLAGS: 00000202
RAX: 0000000000006f82 RBX: 0000000000000000 RCX: 0000000000009dca
RDX: 0000000000009dcc RSI: 0000000000009dcc RDI: ffff880a31730f70
RBP: ffff8817f2b579a8 R08: 0000000000000000 R09: 0000000000000000
R10: ffffffffa0333000 R11: 0000000000000000 R12: ffff8817f2b57918
R13: ffffffff8164655d R14: ffff8817f2b579a8 R15: ffff8815844d5900
FS:  0000000000000000(0000) GS:ffff88180da00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8f03023000 CR3: 000000000194a000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff8817f2b57a40 ffffffff8158bf09 ffff88180da57c28 00000000000037eb
ffff88180da34800 ffff88180da34800 0000000000000000 ffff880827fc5c68
ffff8817f2b579f8 0000000000000000 ffff8817f0be951c ffff8817f2b57a48
Call Trace:
[<ffffffff8158bf09>] tcp_v4_rcv+0x809/0x9f0
[<ffffffff81565c54>] ip_local_deliver_finish+0xb4/0x1f0
[<ffffffff81565f39>] ip_local_deliver+0x59/0xd0
[<ffffffff8158b6bc>] ? tcp_v4_early_demux+0x10c/0x150
[<ffffffff815658cd>] ip_rcv_finish+0x7d/0x350
[<ffffffff81566266>] ip_rcv+0x2b6/0x410
[<ffffffff8152a882>] __netif_receive_skb_core+0x582/0x7d0
[<ffffffff8101c829>] ? read_tsc+0x9/0x10
[<ffffffff8152aae8>] __netif_receive_skb+0x18/0x60
[<ffffffff8152ab70>] netif_receive_skb+0x40/0xc0
[<ffffffffa03093f7>] ipoib_cm_handle_rx_wc+0x217/0x810 [ib_ipoib]
[<ffffffffa015b586>] ? handle_one_recv_msg+0x136/0xd00 [ipmi_msghandler]
[<ffffffffa0159425>] ? free_smi_msg+0x15/0x20 [ipmi_msghandler]
[<ffffffffa030158d>] ipoib_poll+0xad/0x180 [ib_ipoib]
[<ffffffff8152afa2>] net_rx_action+0x152/0x240
[<ffffffff81084b0f>] __do_softirq+0xef/0x280
[<ffffffff8164721c>] call_softirq+0x1c/0x30
[<ffffffff81016fc5>] do_softirq+0x65/0xa0
[<ffffffff81084404>] local_bh_enable+0x94/0xa0
[<ffffffffa0329407>] send_complete+0x37/0x50 [rdmavt]
[<ffffffff810a6003>] kthread_worker_fn+0x83/0x1d0
[<ffffffff810a5f80>] ? flush_kthread_work+0x190/0x190
[<ffffffff810a5aef>] kthread+0xcf/0xe0
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
[<ffffffff81645858>] ret_from_fork+0x58/0x90
[<ffffffff810a5a20>] ? kthread_create_on_node+0x140/0x140
Code: 89 e5 b8 00 00 02 00 f0 0f c1 07 89 c2 c1 ea 10 66 39 c2 75 02 5d c3 83 e2 fe 0f b7 f2 b8 00 80 00 00 eb 0c 0f 1f 44 00 00 f3 90 <83> e8 01 74 0a 0f b7 0f 66 39 ca 75 f1 5d c3 0f 1f 80 00 00 00