NMI watchdog: BUG: soft lockup - CPU#3 stuck for 25s! [ksoftirqd/3:24]
Modules linked in: binfmt_misc resguard_linux(OE) secmodel_linux(OE) syshook_linux(OE) secmodel3(OE) xt_set ip_set_hash_net xt_NFLOG nfnetlink_log ip_set xt_CHECKSUM iptable_raw ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat iptable_mangle xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock nf_conntrack_netlink nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack sunrpc ext4 mbcache jbd2 iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev vmw_balloon pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi libata scsi_transport_spi mptscsih serio_raw mptbase vmxnet3 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8d25e834b180 ti: ffff8d25e8358000 task.ti: ffff8d25e8358000
RIP: 0010:[<ffffffff9f11770d>]  [<ffffffff9f11770d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8d25e835ba00  EFLAGS: 00000293
RAX: 0000000000000001 RBX: 0000000d0000000c RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8d25873ee530
RBP: ffff8d25e835ba00 R08: 0000000000000000 R09: 0000000000000000
R10: ffffffff9fd15e40 R11: 0000000000000000 R12: ffffffffc0847402
R13: ffff8d25e835b9f0 R14: ffffffff9f78a59e R15: ffff8d25e835b968
FS:  0000000000000000(0000) GS:ffff8d347d6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007eff5e40d000 CR3: 000000016bb86000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9f77bff5>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9f78a700>] _raw_spin_lock+0x20/0x30
[<ffffffff9f6c7bf0>] tcp_v4_rcv+0x790/0x9e0
[<ffffffff9f69fead>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff9f6a01a0>] ip_local_deliver+0x60/0xe0
[<ffffffff9f69fdf0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff9f69fb10>] ip_rcv_finish+0x90/0x370
[<ffffffff9f6a04e0>] ip_rcv+0x2c0/0x420
[<ffffffff9f69fa80>] ? inet_del_offload+0x40/0x40
[<ffffffff9f655829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff9f6ccd00>] ? tcp4_gro_receive+0x40/0x1a0
[<ffffffff9f6de9d9>] ? inet_gro_receive+0x249/0x290
[<ffffffff9f655b28>] __netif_receive_skb+0x18/0x60
[<ffffffff9f655bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9f656838>] napi_gro_receive+0xd8/0x100
[<ffffffffc01066b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc01070e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff9f6561cf>] net_rx_action+0x26f/0x390
[<ffffffff9f0a4b35>] __do_softirq+0xf5/0x280
[<ffffffff9f0a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffff9f0ce864>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9f0ce720>] ? lg_double_unlock+0x40/0x40
[<ffffffff9f0c5e11>] kthread+0xd1/0xe0
[<ffffffff9f0c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f794df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9f0c5d40>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f 

NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:3]
Modules linked in: ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event sb_edac edac_core iosf_mbi crc32_pclmul snd_ens1371 snd_rawmidi ghash_clmulni_intel snd_ac97_codec ac97_bus aesni_intel lrw snd_seq gf128mul glue_helper ablk_helper snd_seq_device cryptd snd_pcm ppdev nfit vmw_balloon sg snd_timer pcspkr snd joydev vmw_vmci soundcore libnvdimm i2c_piix4
shpchp parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod ata_generic crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi crct10dif_pclmul scsi_transport_spi crct10dif_common mptscsih crc32c_intel libata e1000 mptbase serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff88017cdf9fa0 ti: ffff88017ce74000 task.ti: ffff88017ce74000
RIP: 0010:[<ffffffff810fa232>]  [<ffffffff810fa232>] native_queued_spin_lock_slowpath+0x22/0x1e0
RSP: 0018:ffff88017ce77980  EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000000
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8801f8217770
RBP: ffff88017ce77980 R08: ffff88019dbcfc86 R09: 0000000000000000
R10: 0000000000000000 R11: ffff8800ba2de098 R12: 0000000000000000
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
FS:  0000000000000000(0000) GS:ffff880235600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fe18a78c000 CR3: 00000000b8637000 CR4: 00000000001407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff88017ce77990 ffffffff8169e5ff ffff88017ce779a0 ffffffff816abb50
ffff88017ce77a38 ffffffff815f085c ffff8800b3fc5cc0 ffff88018a2c0c00
ffffffff81b26550 ffff88017ce779e8 ffffffffc046e306 ffff88017ce77a70
Call Trace:
[<ffffffff8169e5ff>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816abb50>] _raw_spin_lock+0x20/0x30
[<ffffffff815f085c>] tcp_v4_rcv+0x74c/0x9a0
[<ffffffffc046e306>] ? ipv4_confirm+0x86/0x100 [nf_conntrack_ipv4]
[<ffffffff815ca054>] ip_local_deliver_finish+0xb4/0x1f0
[<ffffffff815ca339>] ip_local_deliver+0x59/0xd0
[<ffffffff815c9fa0>] ? ip_rcv_finish+0x350/0x350
[<ffffffff815c9cda>] ip_rcv_finish+0x8a/0x350
[<ffffffff815ca666>] ip_rcv+0x2b6/0x410
[<ffffffff815c9c50>] ? inet_del_offload+0x40/0x40
[<ffffffff81586f22>] __netif_receive_skb_core+0x572/0x7c0
[<ffffffff815f5944>] ? tcp4_gro_receive+0x134/0x1b0
[<ffffffff81607087>] ? inet_gro_receive+0x247/0x290
[<ffffffff81587188>] __netif_receive_skb+0x18/0x60
[<ffffffff81587210>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff81588318>] napi_gro_receive+0xd8/0x130
[<ffffffffc005347c>] e1000_clean_rx_irq+0x2ac/0x4f0 [e1000]
[<ffffffffc0053ed8>] e1000_clean+0x278/0x8d0 [e1000]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 ba 01 00 00 00 8b 07 85 c0 75 0a f0 0f b1 17 85 c0 75 f2 5d c3 f3 90 <eb> ec 0f 1f 40 00 81 fe 00 01 00 00 66 90 0f 84 5c 01 00 00 41 

NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [ksoftirqd/6:39]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_conntrack tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack libcrc32c ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon parport_pc pcspkr parport sg joydev i2c_piix4 vmw_vmci sch_fq_codel ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi crct10dif_pclmul mptscsih crct10dif_common libata crc32c_intel serio_raw mptbase vmxnet3 drm_panel_orientation_quirks floppy fuse
CPU: 6 PID: 39 Comm: ksoftirqd/6 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff94bc4476b180 ti: ffff94bc4477c000 task.ti: ffff94bc4477c000
RIP: 0010:[<ffffffff9791776d>]  [<ffffffff9791776d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0000:ffff94bc4477fa00  EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffffff978adaa3 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff94d25eb23e70
RBP: ffff94bc4477fa00 R08: ffff94ce3cf95af8 R09: 0000000000000000
R10: ffffffff98515e40 R11: 0000000000000000 R12: ffff94c341ea7990
R13: ffff94db8ed93940 R14: ffff94db8ed93940 R15: ffff94c341ea7990
FS:  0000000000000000(0000) GS:ffff94db8ed80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f93badfa10e CR3: 00000001ea7fe000 CR4: 00000000000607e0
Call Trace:
[<ffffffff97f7afe5>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff97f896f0>] _raw_spin_lock+0x20/0x30
[<ffffffff97ec6bf0>] tcp_v4_rcv+0x790/0x9e0
[<ffffffff97e9eead>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff97e9f1a0>] ip_local_deliver+0x60/0xe0
[<ffffffff97e9edf0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff97e9eb10>] ip_rcv_finish+0x90/0x370
[<ffffffff97e9f4e0>] ip_rcv+0x2c0/0x420
[<ffffffff97e9ea80>] ? inet_del_offload+0x40/0x40
[<ffffffff97e54829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff97ecbd00>] ? tcp4_gro_receive+0x40/0x1a0
[<ffffffff97edd9d9>] ? inet_gro_receive+0x249/0x290
[<ffffffff97e54b28>] __netif_receive_skb+0x18/0x60
[<ffffffff97e54bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff97e55838>] napi_gro_receive+0xd8/0x100
[<ffffffffc007b6b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc007c0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff97e551cf>] net_rx_action+0x26f/0x390
[<ffffffff978a4b95>] __do_softirq+0xf5/0x280
[<ffffffff978a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff978ce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff978ce780>] ? lg_double_unlock+0x40/0x40
[<ffffffff978c5e71>] kthread+0xd1/0xe0
[<ffffffff978c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff97f93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff978c5da0>] ? insert_kthread_work+0x40/0x40
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f 

NMI watchdog: BUG: soft lockup - CPU#10 stuck for 21s! [ksoftirqd/10:59]
Modules linked in: ip_vs_rr xt_ipvs ip_vs xt_REDIRECT nf_nat_redirect vxlan ip6_udp_tunnel udp_tunnel xt_mark xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM tun devlink rpcsec_gss_krb5 cmac auth_rpcgss nfsv4 arc4 md4 nfs lockd nls_utf8 grace cifs fscache ccm dns_resolver ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash
dm_log dm_mod nfit libnvdimm ppdev snd_intel8x0 iosf_mbi crc32_pclmul snd_ac97_codec ac97_bus ghash_clmulni_intel snd_seq snd_seq_device snd_pcm joydev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg snd_timer parport_pc snd parport i2c_piix4 pcspkr video soundcore ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul e1000 crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks fuse
CPU: 10 PID: 59 Comm: ksoftirqd/10 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff8ad4e4f46300 ti: ffff8ad4e5340000 task.ti: ffff8ad4e5340000
RIP: 0010:[<ffffffff87f8b02b>]  [<ffffffff87f8b02b>] native_safe_halt+0xb/0x20
RSP: 0018:ffff8ad4e5343908  EFLAGS: 00000202
RAX: 0000000000000001 RBX: ffffff00c02b8719 RCX: 0000000000000001
RDX: 0000000000000000 RSI: 0000000000000001 RDI: ffff8ae398e9b904
RBP: ffff8ad4e5343908 R08: ffff8ad4e5343a60 R09: 0000000000000000
R10: ffff8ad4adb9a150 R11: ffffffffc02bd5f8 R12: ffff8ad4e5343920
R13: ffff8ad4e5343878 R14: ffff8ad4adb9a0e0 R15: 0000000000000246
FS:  0000000000000000(0000) GS:ffff8ae398e80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6e5f51e000 CR3: 00000000d7bf2000 CR4: 00000000000606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8786caf5>] kvm_wait+0x65/0x70
[<ffffffff879176c6>] __pv_queued_spin_lock_slowpath+0x216/0x2e0
[<ffffffff87f7d155>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff87f8b870>] _raw_spin_lock+0x20/0x30
[<ffffffff87ec8c90>] tcp_v4_rcv+0x790/0x9e0
[<ffffffffc06482ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff87ea0f1d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff87ea1210>] ip_local_deliver+0x60/0xe0
[<ffffffff87ea0e60>] ? ip_rcv_finish+0x370/0x370
[<ffffffff87ea0b80>] ip_rcv_finish+0x90/0x370
[<ffffffff87ea1550>] ip_rcv+0x2c0/0x420
[<ffffffff87ea0af0>] ? inet_del_offload+0x40/0x40
[<ffffffff87e56829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff87ecde00>] ? tcp4_gro_receive+0xa0/0x1a0
[<ffffffff87edfa99>] ? inet_gro_receive+0x249/0x290
[<ffffffff87e56b28>] __netif_receive_skb+0x18/0x60
[<ffffffff87e56bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff87e57838>] napi_gro_receive+0xd8/0x100
[<ffffffffc025a45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc025b100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc078d02d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffff878aeccc>] ? mod_timer+0x10c/0x230
[<ffffffff87e571cf>] net_rx_action+0x26f/0x390
[<ffffffff878a4b35>] __do_softirq+0xf5/0x280
[<ffffffff878a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffff878ce7f4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff878ce6b0>] ? lg_double_unlock+0x40/0x40
[<ffffffff878c5da1>] kthread+0xd1/0xe0
[<ffffffff878c5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff87f95df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff878c5cd0>] ? insert_kthread_work+0x40/0x40
Code: ff ff 66 0f 1f 44 00 00 0f 00 2d cd d4 07 00 e9 14 ff ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 90 55 48 89 e5 0f 1f 44 00 00 fb f4 <5d> c3 0f 1f 00 0f 00 2d f9 e4 07 00 eb f0 0f 1f 80 00 00 00 00