Info

Permalink
Function
e1000_xmit_frame
First occurrence
2020-11-09
Last occurrence
2020-12-03
State
NEW
Components
kernel
Quality
-81

Statistics

Operating system Count
CentOS 7 52
Architecture Count
x86_64 52
Related packages Count
kernel 52
 0:3.10.0-957.el7 12
 0:3.10.0-1160.el7 7
 0:3.10.0-862.14.4.el7 6
 0:3.10.0-862.el7 6
 0:3.10.0-1160.25.1.el7 6
 0:3.10.0-1160.118.1.el7 6
 0:3.10.0-957.12.2.el7 2
 0:3.10.0-1127.el7 2
 0:3.10.0-1160.80.1.el7 2
 0:3.10.0-1160.6.1.el7 1
 0:3.10.0-1160.66.1.el7 1
 0:3.10.0-1062.9.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

No reports in the last 20 days.

Report backtrace


Complete report #933887
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0369857
Build id: 3.10.0-957.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff95e38c66
Build id: 3.10.0-957.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff95e65cea
Build id: 3.10.0-957.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff95e3bbe1
Build id: 3.10.0-957.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff95e3bdb0
Build id: 3.10.0-957.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffff95e88736
Build id: 3.10.0-957.el7.x86_64 		0x556
7
ip_output
vmlinux 0xffffffff95e88c93
Build id: 3.10.0-957.el7.x86_64 		0x73
8
? __ip_append_data.isra.51
vmlinux 0xffffffff95e881e0
Build id: 3.10.0-957.el7.x86_64 		0xa50
9
ip_local_out_sk
vmlinux 0xffffffff95e86817
Build id: 3.10.0-957.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffff95e896f6
Build id: 3.10.0-957.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffff95e89763
Build id: 3.10.0-957.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffff95e89b38
Build id: 3.10.0-957.el7.x86_64 		0x288
13
? __fib_validate_source.isra.13
vmlinux 0xffffffff95ec6165
Build id: 3.10.0-957.el7.x86_64 		0xf5
14
? local_bh_enable
vmlinux 0xffffffff958a0387
Build id: 3.10.0-957.el7.x86_64 		0x17
15
tcp_v4_send_reset
vmlinux 0xffffffff95ea8476
Build id: 3.10.0-957.el7.x86_64 		0x356
16
? __fib_validate_source.isra.13
vmlinux 0xffffffff95ec6165
Build id: 3.10.0-957.el7.x86_64 		0xf5
17
? __inet_lookup_established
vmlinux 0xffffffff95e8c5e6
Build id: 3.10.0-957.el7.x86_64 		0x46
18
tcp_v4_do_rcv
vmlinux 0xffffffff95ea875e
Build id: 3.10.0-957.el7.x86_64 		0x18e
19
tcp_v4_rcv
vmlinux 0xffffffff95ea9dcc
Build id: 3.10.0-957.el7.x86_64 		0x7bc
20
ip_local_deliver_finish
vmlinux 0xffffffff95e8285d
Build id: 3.10.0-957.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffff95e82b49
Build id: 3.10.0-957.el7.x86_64 		0x59
22
? ip_rcv_finish
vmlinux 0xffffffff95e827a0
Build id: 3.10.0-957.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffff95e824c0
Build id: 3.10.0-957.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffff95e82e79
Build id: 3.10.0-957.el7.x86_64 		0x2b9
25
? inet_del_offload
vmlinux 0xffffffff95e82430
Build id: 3.10.0-957.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffff95e394f9
Build id: 3.10.0-957.el7.x86_64 		0x729
27
? tcp4_gro_receive
vmlinux 0xffffffff95eaee00
Build id: 3.10.0-957.el7.x86_64 		0x80
28
? __getnstimeofday64
vmlinux 0xffffffff959024ef
Build id: 3.10.0-957.el7.x86_64 		0x3f
29
__netif_receive_skb
vmlinux 0xffffffff95e397f8
Build id: 3.10.0-957.el7.x86_64 		0x18
30
netif_receive_skb_internal
vmlinux 0xffffffff95e39880
Build id: 3.10.0-957.el7.x86_64 		0x40
31
napi_gro_receive
vmlinux 0xffffffff95e3a508
Build id: 3.10.0-957.el7.x86_64 		0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc036a43b
Build id: 3.10.0-957.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc036b0dd
Build id: 3.10.0-957.el7.x86_64 		0x27d
34
net_rx_action
vmlinux 0xffffffff95e39e9f
Build id: 3.10.0-957.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffff958a0f05
Build id: 3.10.0-957.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffff958a10c8
Build id: 3.10.0-957.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffff958ca5c4
Build id: 3.10.0-957.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffff958ca480
Build id: 3.10.0-957.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffff958c1c31
Build id: 3.10.0-957.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffff958c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffff95f74c1d
Build id: 3.10.0-957.el7.x86_64 		0x7
42
? insert_kthread_work
vmlinux 0xffffffff958c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 29s! [ksoftirqd/1:14]
Modules linked in: binfmt_misc tcp_lp fuse veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c
sd_mod crc_t10dif sr_mod cdrom crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi e1000 vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi scsi_transport_spi ata_piix mptscsih libata mptbase drm_panel_orientation_quirks nfit libnvdimm
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9766f9890000 ti: ffff9766f9898000 task.ti: ffff9766f9898000
RIP: 0010:[<ffffffffc0369857>]  [<ffffffffc0369857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff9766f989b4c0  EFLAGS: 00000286
RAX: ffffb48501903818 RBX: 0000000000000001 RCX: 00000000000000ac
RDX: 00000000000000ac RSI: ffff9769a92e0a00 RDI: ffff9769ad5f4000
RBP: ffff9766f989b540 R08: 0000000000000100 R09: 0000000002000000
R10: fdcc000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000364bbf602 R14: 00000001a92e0a00 R15: ffffffff95ba04b0
FS:  0000000000000000(0000) GS:ffff9769ad640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdae4651000 CR3: 0000000390658000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffff95e38c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff95e65cea>] sch_direct_xmit+0x11a/0x250
[<ffffffff95e3bbe1>] __dev_queue_xmit+0x491/0x650
[<ffffffff95e3bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffff95e88736>] ip_finish_output+0x556/0x7b0
[<ffffffff95e88c93>] ip_output+0x73/0xe0
[<ffffffff95e881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffff95e86817>] ip_local_out_sk+0x37/0x40
[<ffffffff95e896f6>] ip_send_skb+0x16/0x50
[<ffffffff95e89763>] ip_push_pending_frames+0x33/0x40
[<ffffffff95e89b38>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff95ec6165>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffff958a0387>] ? local_bh_enable+0x17/0x20
[<ffffffff95ea8476>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff95ec6165>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffff95e8c5e6>] ? __inet_lookup_established+0x46/0x130
[<ffffffff95ea875e>] tcp_v4_do_rcv+0x18e/0x350
[<ffffffff95ea9dcc>] tcp_v4_rcv+0x7bc/0x9c0
[<ffffffff95e8285d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff95e82b49>] ip_local_deliver+0x59/0xd0
[<ffffffff95e827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff95e824c0>] ip_rcv_finish+0x90/0x370
[<ffffffff95e82e79>] ip_rcv+0x2b9/0x410
[<ffffffff95e82430>] ? inet_del_offload+0x40/0x40
[<ffffffff95e394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff95eaee00>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff959024ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff95e397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff95e39880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff95e3a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc036a43b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc036b0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffff95e39e9f>] net_rx_action+0x26f/0x390
[<ffffffff958a0f05>] __do_softirq+0xf5/0x280
[<ffffffff958a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff958ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff958ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff958c1c31>] kthread+0xd1/0xe0
[<ffffffff958c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff95f74c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff958c1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #919318
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc01d5867
Build id: 3.10.0-862.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffbbdeb166
Build id: 3.10.0-862.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffbbe16a2a
Build id: 3.10.0-862.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffbbdee051
Build id: 3.10.0-862.el7.x86_64 		0x4a1
5
dev_queue_xmit
vmlinux 0xffffffffbbdee220
Build id: 3.10.0-862.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffbbe37416
Build id: 3.10.0-862.el7.x86_64 		0x546
7
ip_output
vmlinux 0xffffffffbbe37973
Build id: 3.10.0-862.el7.x86_64 		0x73
8
? __ip_append_data.isra.50
vmlinux 0xffffffffbbe36ed0
Build id: 3.10.0-862.el7.x86_64 		0xa50
9
ip_local_out_sk
vmlinux 0xffffffffbbe35567
Build id: 3.10.0-862.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffffbbe383c6
Build id: 3.10.0-862.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffffbbe38433
Build id: 3.10.0-862.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffffbbe38804
Build id: 3.10.0-862.el7.x86_64 		0x284
13
? local_bh_enable
vmlinux 0xffffffffbb899df7
Build id: 3.10.0-862.el7.x86_64 		0x17
14
? ipt_do_table
ip_tables 0xffffffffc04e5a2e
Build id: 3.10.0-862.el7.x86_64 		0x33e
15
? __fib_validate_source.isra.13
vmlinux 0xffffffffbbe747a5
Build id: 3.10.0-862.el7.x86_64 		0xf5
16
tcp_v4_send_reset
vmlinux 0xffffffffbbe56b8a
Build id: 3.10.0-862.el7.x86_64 		0x31a
17
tcp_v4_rcv
vmlinux 0xffffffffbbe58376
Build id: 3.10.0-862.el7.x86_64 		0x596
18
ip_local_deliver_finish
vmlinux 0xffffffffbbe31699
Build id: 3.10.0-862.el7.x86_64 		0xb9
19
ip_local_deliver
vmlinux 0xffffffffbbe31989
Build id: 3.10.0-862.el7.x86_64 		0x59
20
? ip_rcv_finish
vmlinux 0xffffffffbbe315e0
Build id: 3.10.0-862.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffffbbe31300
Build id: 3.10.0-862.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffbbe31cb9
Build id: 3.10.0-862.el7.x86_64 		0x2b9
23
? udp4_lib_lookup
vmlinux 0xffffffffbbe63002
Build id: 3.10.0-862.el7.x86_64 		0x22
24
? inet_del_offload
vmlinux 0xffffffffbbe31270
Build id: 3.10.0-862.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffbbdeb9f9
Build id: 3.10.0-862.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffffbbe5d600
Build id: 3.10.0-862.el7.x86_64 		0xe0
27
? __getnstimeofday64
vmlinux 0xffffffffbb8f8c5f
Build id: 3.10.0-862.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffbbdebd08
Build id: 3.10.0-862.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffbbdebd90
Build id: 3.10.0-862.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffbbdec9b8
Build id: 3.10.0-862.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc01d644b
Build id: 3.10.0-862.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc01d70eb
Build id: 3.10.0-862.el7.x86_64 		0x27b
33
? netif_receive_skb_internal
vmlinux 0xffffffffbbdebd90
Build id: 3.10.0-862.el7.x86_64 		0x40
34
net_rx_action
vmlinux 0xffffffffbbdec3af
Build id: 3.10.0-862.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffffbb89a945
Build id: 3.10.0-862.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffffbb89ab08
Build id: 3.10.0-862.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffffbb8c35e4
Build id: 3.10.0-862.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffffbb8c34a0
Build id: 3.10.0-862.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffffbb8bae31
Build id: 3.10.0-862.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffffbb8bad60
Build id: 3.10.0-862.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffffbbf1f637
Build id: 3.10.0-862.el7.x86_64 		0x21
42
? insert_kthread_work
vmlinux 0xffffffffbb8bad60
Build id: 3.10.0-862.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus crc32_pclmul ghash_clmulni_intel snd_seq aesni_intel lrw gf128mul snd_seq_device snd_pcm glue_helper snd_timer snd ablk_helper cryptd pcspkr ppdev soundcore
nfit vmw_balloon joydev sg libnvdimm parport_pc parport i2c_piix4 vmw_vmci shpchp ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi serio_raw scsi_transport_spi mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9de7f5b69fa0 ti: ffff9de7f5b8c000 task.ti: ffff9de7f5b8c000
RIP: 0010:[<ffffffffc01d5867>]  [<ffffffffc01d5867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9de7f5b8f500  EFLAGS: 00000286
RAX: ffffaca6831c3818 RBX: 0000000000000001 RCX: 000000000000003b
RDX: 000000000000003b RSI: ffff9ded8bb2e140 RDI: ffff9dea75e1c000
RBP: ffff9de7f5b8f580 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000003bd6bca02 R14: 0000000100000202 R15: ffffffffbbb73e00
FS:  0000000000000000(0000) GS:ffff9ded90600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0cee530000 CR3: 000000070877c000 CR4: 00000000007607f0
PKRU: 00000000
Call Trace:
[<ffffffffbbdeb166>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbbe16a2a>] sch_direct_xmit+0x11a/0x250
[<ffffffffbbdee051>] __dev_queue_xmit+0x4a1/0x660
[<ffffffffbbdee220>] dev_queue_xmit+0x10/0x20
[<ffffffffbbe37416>] ip_finish_output+0x546/0x7a0
[<ffffffffbbe37973>] ip_output+0x73/0xe0
[<ffffffffbbe36ed0>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffbbe35567>] ip_local_out_sk+0x37/0x40
[<ffffffffbbe383c6>] ip_send_skb+0x16/0x50
[<ffffffffbbe38433>] ip_push_pending_frames+0x33/0x40
[<ffffffffbbe38804>] ip_send_unicast_reply+0x284/0x2d0
[<ffffffffbb899df7>] ? local_bh_enable+0x17/0x20
[<ffffffffc04e5a2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffbbe747a5>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffffbbe56b8a>] tcp_v4_send_reset+0x31a/0x460
[<ffffffffbbe58376>] tcp_v4_rcv+0x596/0x990
[<ffffffffbbe31699>] ip_local_deliver_finish+0xb9/0x200
[<ffffffffbbe31989>] ip_local_deliver+0x59/0xd0
[<ffffffffbbe315e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbbe31300>] ip_rcv_finish+0x90/0x370
[<ffffffffbbe31cb9>] ip_rcv+0x2b9/0x410
[<ffffffffbbe63002>] ? udp4_lib_lookup+0x22/0x70
[<ffffffffbbe31270>] ? inet_del_offload+0x40/0x40
[<ffffffffbbdeb9f9>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffbbe5d600>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffbb8f8c5f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffbbdebd08>] __netif_receive_skb+0x18/0x60
[<ffffffffbbdebd90>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbbdec9b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc01d644b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01d70eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffbbdebd90>] ? netif_receive_skb_internal+0x40/0xc0
[<ffffffffbbdec3af>] net_rx_action+0x26f/0x390
[<ffffffffbb89a945>] __do_softirq+0xf5/0x280
[<ffffffffbb89ab08>] run_ksoftirqd+0x38/0x50
[<ffffffffbb8c35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbb8c34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbb8bae31>] kthread+0xd1/0xe0
[<ffffffffbb8bad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffbbf1f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbb8bad60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #913316
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04d6877
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x7f7
2
? nf_ct_delete
nf_conntrack 0xffffffffc0994016
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x176
3
? lock_timer_base.isra.37
vmlinux 0xffffffff924adfdb
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2b
4
dev_hard_start_xmit
vmlinux 0xffffffff92a4f346
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x246
5
sch_direct_xmit
vmlinux 0xffffffff92a7d0ba
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x11a
6
__dev_queue_xmit
vmlinux 0xffffffff92a522b1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x491
7
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc09d2020
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x20
8
dev_queue_xmit
vmlinux 0xffffffff92a52480
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x10
9
neigh_resolve_output
vmlinux 0xffffffff92a5efed
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x11d
10
ip_finish_output
vmlinux 0xffffffff92aa057c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x27c
11
ip_output
vmlinux 0xffffffff92aa0efb
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x7b
12
? __ip_append_data.isra.50
vmlinux 0xffffffff92aa0300
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xa60
13
ip_local_out_sk
vmlinux 0xffffffff92a9e927
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x37
14
ip_send_skb
vmlinux 0xffffffff92aa1966
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x16
15
ip_push_pending_frames
vmlinux 0xffffffff92aa19d3
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x33
16
ip_send_unicast_reply
vmlinux 0xffffffff92aa1da8
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x288
17
? _raw_spin_unlock_bh
vmlinux 0xffffffff92b831ce
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x1e
18
tcp_v4_send_reset
vmlinux 0xffffffff92ac0af6
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x356
19
tcp_v4_rcv
vmlinux 0xffffffff92ac224b
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x5bb
20
ip_local_deliver_finish
vmlinux 0xffffffff92a9a86d
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffff92a9ab60
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x60
22
? ip_rcv_finish
vmlinux 0xffffffff92a9a7b0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffff92a9a4d0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffff92a9aea0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2c0
25
? inet_del_offload
vmlinux 0xffffffff92a9a440
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffff92a4fbd9
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x729
27
? tcp4_gro_receive
vmlinux 0xffffffff92ac7600
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x110
28
? __getnstimeofday64
vmlinux 0xffffffff925076bf
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x3f
29
__netif_receive_skb
vmlinux 0xffffffff92a4fed8
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x18
30
netif_receive_skb_internal
vmlinux 0xffffffff92a4ff60
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
31
napi_gro_receive
vmlinux 0xffffffff92a50be8
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc04d745b
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc04d8100
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x280
34
? __queue_work
vmlinux 0xffffffff924bc16c
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x13c
35
net_rx_action
vmlinux 0xffffffff92a5057f
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x26f
36
__do_softirq
vmlinux 0xffffffff924a5305
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xf5
37
run_ksoftirqd
vmlinux 0xffffffff924a54c8
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x38
38
smpboot_thread_fn
vmlinux 0xffffffff924cec24
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x144
39
? lg_double_unlock
vmlinux 0xffffffff924ceae0
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
40
kthread
vmlinux 0xffffffff924c61f1
Build id: 3.10.0-1062.9.1.el7.x86_64 		0xd1
41
? insert_kthread_work
vmlinux 0xffffffff924c6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffff92b8dd37
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x21
43
? insert_kthread_work
vmlinux 0xffffffff924c6120
Build id: 3.10.0-1062.9.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ksoftirqd/3:24]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus ppdev vmw_balloon snd_seq iosf_mbi snd_seq_device snd_pcm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper snd_timer sg ablk_helper pcspkr cryptd snd joydev soundcore parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul syscopyarea crct10dif_common
sysfillrect sysimgblt crc32c_intel fb_sys_fops mptspi ttm scsi_transport_spi mptscsih drm ata_piix e1000 mptbase serio_raw libata drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/31/2013
task: ffff8ec7fca3a0e0 ti: ffff8ec7fca54000 task.ti: ffff8ec7fca54000
RIP: 0010:[<ffffffffc04d6877>]  [<ffffffffc04d6877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8ec7fca57490  EFLAGS: 00000286
RAX: ffffb572c0e03818 RBX: 0000000000000001 RCX: 0000000000000087
RDX: 0000000000000087 RSI: ffff8ec6b5852100 RDI: ffff8ec7fce2e000
RBP: ffff8ec7fca57510 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000001abb1a802 R14: 0000000190f779ed R15: ffffffff927acd80
FS:  0000000000000000(0000) GS:ffff8ec8b66c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055ec3db91108 CR3: 00000000b797a000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc0994016>] ? nf_ct_delete+0x176/0x270 [nf_conntrack]
[<ffffffff924adfdb>] ? lock_timer_base.isra.37+0x2b/0x50
[<ffffffff92a4f346>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff92a7d0ba>] sch_direct_xmit+0x11a/0x250
[<ffffffff92a522b1>] __dev_queue_xmit+0x491/0x650
[<ffffffffc09d2020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffff92a52480>] dev_queue_xmit+0x10/0x20
[<ffffffff92a5efed>] neigh_resolve_output+0x11d/0x220
[<ffffffff92aa057c>] ip_finish_output+0x27c/0x8d0
[<ffffffff92aa0efb>] ip_output+0x7b/0xf0
[<ffffffff92aa0300>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff92a9e927>] ip_local_out_sk+0x37/0x40
[<ffffffff92aa1966>] ip_send_skb+0x16/0x50
[<ffffffff92aa19d3>] ip_push_pending_frames+0x33/0x40
[<ffffffff92aa1da8>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff92b831ce>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff92ac0af6>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff92ac224b>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffff92a9a86d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff92a9ab60>] ip_local_deliver+0x60/0xe0
[<ffffffff92a9a7b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff92a9a4d0>] ip_rcv_finish+0x90/0x370
[<ffffffff92a9aea0>] ip_rcv+0x2c0/0x420
[<ffffffff92a9a440>] ? inet_del_offload+0x40/0x40
[<ffffffff92a4fbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff92ac7600>] ? tcp4_gro_receive+0x110/0x1a0
[<ffffffff925076bf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff92a4fed8>] __netif_receive_skb+0x18/0x60
[<ffffffff92a4ff60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff92a50be8>] napi_gro_receive+0xd8/0x100
[<ffffffffc04d745b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04d8100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff924bc16c>] ? __queue_work+0x13c/0x3f0
[<ffffffff92a5057f>] net_rx_action+0x26f/0x390
[<ffffffff924a5305>] __do_softirq+0xf5/0x280
[<ffffffff924a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffff924cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff924ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffff924c61f1>] kthread+0xd1/0xe0
[<ffffffff924c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff92b8dd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff924c6120>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #865477
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03fc877
Build id: 3.10.0-1160.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff93c54386
Build id: 3.10.0-1160.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff93c8242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff93c57301
Build id: 3.10.0-1160.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff93c574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffff93ca545b
Build id: 3.10.0-1160.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffff93ca59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffff93ca4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffff93ca3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffff93ca6446
Build id: 3.10.0-1160.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffff93ca64b3
Build id: 3.10.0-1160.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffff93ca6888
Build id: 3.10.0-1160.el7.x86_64 		0x288
13
? kmem_cache_free
vmlinux 0xffffffff93826642
Build id: 3.10.0-1160.el7.x86_64 		0x1e2
14
? inet6_destroy_sock
vmlinux 0xffffffff93d17201
Build id: 3.10.0-1160.el7.x86_64 		0x41
15
tcp_v4_send_reset
vmlinux 0xffffffff93cc5546
Build id: 3.10.0-1160.el7.x86_64 		0x356
16
tcp_v4_do_rcv
vmlinux 0xffffffff93cc582e
Build id: 3.10.0-1160.el7.x86_64 		0x18e
17
tcp_v4_rcv
vmlinux 0xffffffff93cc6ffc
Build id: 3.10.0-1160.el7.x86_64 		0x7dc
18
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc0a532ce
Build id: 3.10.0-1160.el7.x86_64 		0x4e
19
ip_local_deliver_finish
vmlinux 0xffffffff93c9f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
20
ip_local_deliver
vmlinux 0xffffffff93c9f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
21
? ip_rcv_finish
vmlinux 0xffffffff93c9f1d0
Build id: 3.10.0-1160.el7.x86_64 		0x370
22
ip_rcv_finish
vmlinux 0xffffffff93c9eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
23
ip_rcv
vmlinux 0xffffffff93c9f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
24
? packet_rcv
vmlinux 0xffffffff93d5fb02
Build id: 3.10.0-1160.el7.x86_64 		0x232
25
? inet_del_offload
vmlinux 0xffffffff93c9ee60
Build id: 3.10.0-1160.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffff93c54c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
27
? tcp4_gro_receive
vmlinux 0xffffffff93ccc100
Build id: 3.10.0-1160.el7.x86_64 		0x80
28
? inet_gro_receive
vmlinux 0xffffffff93cddd99
Build id: 3.10.0-1160.el7.x86_64 		0x249
29
__netif_receive_skb
vmlinux 0xffffffff93c54f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
30
netif_receive_skb_internal
vmlinux 0xffffffff93c54fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
31
napi_gro_receive
vmlinux 0xffffffff93c55c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc03fd45b
Build id: 3.10.0-1160.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc03fe100
Build id: 3.10.0-1160.el7.x86_64 		0x280
34
net_rx_action
vmlinux 0xffffffff93c555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffff936a4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffff936a4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffff936ce674
Build id: 3.10.0-1160.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffff936ce530
Build id: 3.10.0-1160.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffff936c5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffff936c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffff93d93ddd
Build id: 3.10.0-1160.el7.x86_64 		0x7
42
? insert_kthread_work
vmlinux 0xffffffff936c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper btusb btrtl btbcm btintel bluetooth snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq rfkill cryptd snd_seq_device snd_pcm snd_timer ppdev snd soundcore sg vmw_balloon joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops mptspi ttm ata_piix crct10dif_pclmul scsi_transport_spi crct10dif_common nfit crc32c_intel mptscsih drm libnvdimm serio_raw libata mptbase drm_panel_orientation_quirks e1000 fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Not tainted 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff901bf8775280 ti: ffff901bf0ff8000 task.ti: ffff901bf0ff8000
RIP: 0010:[<ffffffffc03fc877>]  [<ffffffffc03fc877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff901bf0ffb490  EFLAGS: 00000286
RAX: ffffa775c09c3818 RBX: 0000000000000001 RCX: 00000000000000e4
RDX: 00000000000000e4 RSI: ffff901af5c4da40 RDI: ffff901bf934a000
RBP: ffff901bf0ffb510 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000013538ae02 R14: 00000001f5c4da40 R15: ffffffff939b1650
FS:  0000000000000000(0000) GS:ffff901bf9600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000c9c00010 CR3: 0000000065410000 CR4: 0000000000760ff0
PKRU: 00000000
Call Trace:
[<ffffffff93c54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff93c8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff93c57301>] __dev_queue_xmit+0x491/0x650
[<ffffffff93c574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff93ca545b>] ip_finish_output+0x67b/0x8d0
[<ffffffff93ca59db>] ip_output+0x7b/0xf0
[<ffffffff93ca4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff93ca3407>] ip_local_out_sk+0x37/0x40
[<ffffffff93ca6446>] ip_send_skb+0x16/0x50
[<ffffffff93ca64b3>] ip_push_pending_frames+0x33/0x40
[<ffffffff93ca6888>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff93826642>] ? kmem_cache_free+0x1e2/0x200
[<ffffffff93d17201>] ? inet6_destroy_sock+0x41/0x90
[<ffffffff93cc5546>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff93cc582e>] tcp_v4_do_rcv+0x18e/0x350
[<ffffffff93cc6ffc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc0a532ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff93c9f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff93c9f580>] ip_local_deliver+0x60/0xe0
[<ffffffff93c9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff93c9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffff93c9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffff93d5fb02>] ? packet_rcv+0x232/0x430
[<ffffffff93c9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffff93c54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff93ccc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff93cddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffff93c54f18>] __netif_receive_skb+0x18/0x60
[<ffffffff93c54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff93c55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc03fd45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03fe100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff93c555bf>] net_rx_action+0x26f/0x390
[<ffffffff936a4b95>] __do_softirq+0xf5/0x280
[<ffffffff936a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff936ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff936ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff936c5c21>] kthread+0xd1/0xe0
[<ffffffff936c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff93d93ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff936c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #829850
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc020e877
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffa0a58f86
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffa0a8709a
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffa0a5bf01
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffa0a5c0d0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffa0aaac5b
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffffa0aab1db
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffa0aaa5e0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffffa0aa8c07
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffffa0aabc46
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffffa0aabcb3
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffffa0aac088
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x288
13
? ipt_do_table
ip_tables 0xffffffffc032e721
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x361
14
tcp_v4_send_reset
vmlinux 0xffffffffa0acadd6
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x356
15
tcp_v4_rcv
vmlinux 0xffffffffa0acc66b
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x5bb
16
ip_local_deliver_finish
vmlinux 0xffffffffa0aa4a8d
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xbd
17
ip_local_deliver
vmlinux 0xffffffffa0aa4d80
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x60
18
? ip_rcv_finish
vmlinux 0xffffffffa0aa49d0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x370
19
ip_rcv_finish
vmlinux 0xffffffffa0aa46f0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x90
20
ip_rcv
vmlinux 0xffffffffa0aa50c0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x2c0
21
? inet_del_offload
vmlinux 0xffffffffa0aa4660
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x40
22
__netif_receive_skb_core
vmlinux 0xffffffffa0a59819
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x729
23
? tcp4_gro_receive
vmlinux 0xffffffffa0ad1a00
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xf0
24
? __getnstimeofday64
vmlinux 0xffffffffa050803f
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x3f
25
__netif_receive_skb
vmlinux 0xffffffffa0a59b18
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x18
26
netif_receive_skb_internal
vmlinux 0xffffffffa0a59ba0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x40
27
napi_gro_receive
vmlinux 0xffffffffa0a5a828
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xd8
28
e1000_clean_rx_irq
e1000 0xffffffffc020f45b
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x2bb
29
e1000_clean
e1000 0xffffffffc0210100
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x280
30
? switch_mm_irqs_off
vmlinux 0xffffffffa047dfe9
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x109
31
? switch_mm_irqs_off
vmlinux 0xffffffffa047dff5
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x115
32
? switch_mm_irqs_off
vmlinux 0xffffffffa047dfe9
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x109
33
? switch_mm_irqs_off
vmlinux 0xffffffffa047dff5
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x115
34
? switch_mm_irqs_off
vmlinux 0xffffffffa047dfe9
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x109
35
net_rx_action
vmlinux 0xffffffffa0a5a1bf
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x26f
36
__do_softirq
vmlinux 0xffffffffa04a4d85
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xf5
37
run_ksoftirqd
vmlinux 0xffffffffa04a4f48
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x38
38
smpboot_thread_fn
vmlinux 0xffffffffa04ce9e4
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x144
39
? lg_double_unlock
vmlinux 0xffffffffa04ce8a0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x40
40
kthread
vmlinux 0xffffffffa04c5f91
Build id: 3.10.0-1160.66.1.el7.x86_64 		0xd1
41
? insert_kthread_work
vmlinux 0xffffffffa04c5ec0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffa0b99ddd
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x7
43
? insert_kthread_work
vmlinux 0xffffffffa04c5ec0
Build id: 3.10.0-1160.66.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ksoftirqd/3:24]
Modules linked in: veth tcp_lp rfcomm xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle xt_conntrack ipt_REJECT nf_reject_ipv4 bridge stp llc ebtable_filter ebtables devlink ip6t_MASQUERADE nf_nat_masquerade_ipv6 ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark overlay(T) tun nf_tables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev iosf_mbi btusb btrtl btbcm crc32_pclmul ghash_clmulni_intel aesni_intel btintel lrw bluetooth gf128mul glue_helper ablk_helper rfkill cryptd vmw_balloon sg pcspkr joydev ahci libahci i2c_piix4 vmw_vmci parport_pc parport
binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel nfit drm_kms_helper libnvdimm syscopyarea serio_raw sysfillrect sysimgblt fb_sys_fops ttm ata_piix libata drm mptspi e1000 scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9e076543b180 ti: ffff9e0765468000 task.ti: ffff9e0765468000
RIP: 0010:[<ffffffffc020e877>]  [<ffffffffc020e877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff9e076546b4d0  EFLAGS: 00000286
RAX: ffffc2fac0703818 RBX: 0000000000000001 RCX: 00000000000000a1
RDX: 00000000000000a1 RSI: ffff9e06755dc080 RDI: ffff9e076383c000
RBP: ffff9e076546b550 R08: 0000000000000100 R09: 0000000002000000
R10: fa88000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000010d465602 R14: 00000001755dc080 R15: ffffffffa07b20a0
FS:  0000000000000000(0000) GS:ffff9e0768ac0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f45bf785eb0 CR3: 00000001213ee000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa0a58f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa0a8709a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa0a5bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffffa0a5c0d0>] dev_queue_xmit+0x10/0x20
[<ffffffffa0aaac5b>] ip_finish_output+0x67b/0x8d0
[<ffffffffa0aab1db>] ip_output+0x7b/0xf0
[<ffffffffa0aaa5e0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa0aa8c07>] ip_local_out_sk+0x37/0x40
[<ffffffffa0aabc46>] ip_send_skb+0x16/0x50
[<ffffffffa0aabcb3>] ip_push_pending_frames+0x33/0x40
[<ffffffffa0aac088>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffc032e721>] ? ipt_do_table+0x361/0x740 [ip_tables]
[<ffffffffa0acadd6>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffa0acc66b>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffffa0aa4a8d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa0aa4d80>] ip_local_deliver+0x60/0xe0
[<ffffffffa0aa49d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa0aa46f0>] ip_rcv_finish+0x90/0x370
[<ffffffffa0aa50c0>] ip_rcv+0x2c0/0x420
[<ffffffffa0aa4660>] ? inet_del_offload+0x40/0x40
[<ffffffffa0a59819>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa0ad1a00>] ? tcp4_gro_receive+0xf0/0x1a0
[<ffffffffa050803f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa0a59b18>] __netif_receive_skb+0x18/0x60
[<ffffffffa0a59ba0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa0a5a828>] napi_gro_receive+0xd8/0x100
[<ffffffffc020f45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0210100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa047dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffa047dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffffa047dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffa047dff5>] ? switch_mm_irqs_off+0x115/0x290
[<ffffffffa047dfe9>] ? switch_mm_irqs_off+0x109/0x290
[<ffffffffa0a5a1bf>] net_rx_action+0x26f/0x390
[<ffffffffa04a4d85>] __do_softirq+0xf5/0x280
[<ffffffffa04a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffa04ce9e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa04ce8a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa04c5f91>] kthread+0xd1/0xe0
[<ffffffffa04c5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa0b99ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa04c5ec0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #789590
Frame # Function Binary Source or offset Line
1
vmxnet3_xmit_frame
vmxnet3 0xffffffffc04583dc
Build id: 3.10.0-1160.el7.x86_64 		0xaec
2
dev_hard_start_xmit
vmlinux 0xffffffff81a54386
Build id: 3.10.0-1160.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff81a8242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff81a57301
Build id: 3.10.0-1160.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff81a574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
6
neigh_resolve_output
vmlinux 0xffffffff81a641fd
Build id: 3.10.0-1160.el7.x86_64 		0x11d
7
ip_finish_output
vmlinux 0xffffffff81aa505c
Build id: 3.10.0-1160.el7.x86_64 		0x27c
8
ip_output
vmlinux 0xffffffff81aa59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff81aa4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
10
ip_local_out_sk
vmlinux 0xffffffff81aa3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
11
ip_send_skb
vmlinux 0xffffffff81aa6446
Build id: 3.10.0-1160.el7.x86_64 		0x16
12
ip_push_pending_frames
vmlinux 0xffffffff81aa64b3
Build id: 3.10.0-1160.el7.x86_64 		0x33
13
ip_send_unicast_reply
vmlinux 0xffffffff81aa6888
Build id: 3.10.0-1160.el7.x86_64 		0x288
14
tcp_v4_send_reset
vmlinux 0xffffffff81ac5546
Build id: 3.10.0-1160.el7.x86_64 		0x356
15
tcp_v4_rcv
vmlinux 0xffffffff81ac6ddb
Build id: 3.10.0-1160.el7.x86_64 		0x5bb
16
ip_local_deliver_finish
vmlinux 0xffffffff81a9f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
17
ip_local_deliver
vmlinux 0xffffffff81a9f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
18
? ip_route_input_noref
vmlinux 0xffffffff81a9dba6
Build id: 3.10.0-1160.el7.x86_64 		0x46
19
ip_rcv_finish
vmlinux 0xffffffff81a9eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
20
ip_rcv
vmlinux 0xffffffff81a9f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
21
__netif_receive_skb_core
vmlinux 0xffffffff81a54c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
22
? tcp4_gro_receive
vmlinux 0xffffffff81acc100
Build id: 3.10.0-1160.el7.x86_64 		0x80
23
? __getnstimeofday64
vmlinux 0xffffffff815078df
Build id: 3.10.0-1160.el7.x86_64 		0x3f
24
__netif_receive_skb
vmlinux 0xffffffff81a54f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
25
netif_receive_skb_internal
vmlinux 0xffffffff81a54fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
26
napi_gro_receive
vmlinux 0xffffffff81a55c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
27
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc04596b4
Build id: 3.10.0-1160.el7.x86_64 		0x924
28
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc045a0e6
Build id: 3.10.0-1160.el7.x86_64 		0x36
29
net_rx_action
vmlinux 0xffffffff81a555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
30
__do_softirq
vmlinux 0xffffffff814a4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
31
run_ksoftirqd
vmlinux 0xffffffff814a4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
32
smpboot_thread_fn
vmlinux 0xffffffff814ce674
Build id: 3.10.0-1160.el7.x86_64 		0x144
33
? lg_double_unlock
vmlinux 0xffffffff814ce530
Build id: 3.10.0-1160.el7.x86_64 		0x40
34
kthread
vmlinux 0xffffffff814c5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
35
? insert_kthread_work
vmlinux 0xffffffff814c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
36
ret_from_fork_nospec_begin
vmlinux 0xffffffff81b93df7
Build id: 3.10.0-1160.el7.x86_64 		0x21
37
? insert_kthread_work
vmlinux 0xffffffff814c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 28s! [ksoftirqd/1:14]
Modules linked in: overlay vmw_vsock_vmci_transport vsock iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd sg pcspkr joydev vmw_balloon parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c ata_generic sr_mod cdrom pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci libahci drm ata_piix sd_mod crc_t10dif libata crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel nfit serio_raw libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8e69fe350000 ti: ffff8e69fe358000 task.ti: ffff8e69fe358000
RIP: 0010:[<ffffffffc04583dc>]  [<ffffffffc04583dc>] vmxnet3_xmit_frame+0xaec/0xfd0 [vmxnet3]
RSP: 0018:ffff8e69fe35b520  EFLAGS: 00000286
RAX: 0000000000000000 RBX: 0000002200000000 RCX: 00000000000000bb
RDX: ffffb2dec1900600 RSI: 0000000000000293 RDI: 0000000000000293
RBP: ffff8e69fe35b5b0 R08: 0000d7c49b18018d R09: 0000000000004036
R10: dd73539800000000 R11: ffff8e69fdbcaba0 R12: ffff8e69fe35b558
R13: 000000005e892b21 R14: 0000000000000000 R15: 0000000000000000
FS:  0000000000000000(0000) GS:ffff8e6cad640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000012566a8 CR3: 0000000421fca000 CR4: 00000000003607e0
Call Trace:
[<ffffffff81a54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff81a8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff81a57301>] __dev_queue_xmit+0x491/0x650
[<ffffffff81a574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff81a641fd>] neigh_resolve_output+0x11d/0x220
[<ffffffff81aa505c>] ip_finish_output+0x27c/0x8d0
[<ffffffff81aa59db>] ip_output+0x7b/0xf0
[<ffffffff81aa4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff81aa3407>] ip_local_out_sk+0x37/0x40
[<ffffffff81aa6446>] ip_send_skb+0x16/0x50
[<ffffffff81aa64b3>] ip_push_pending_frames+0x33/0x40
[<ffffffff81aa6888>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff81ac5546>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff81ac6ddb>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffff81a9f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff81a9f580>] ip_local_deliver+0x60/0xe0
[<ffffffff81a9dba6>] ? ip_route_input_noref+0x46/0x220
[<ffffffff81a9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffff81a9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffff81a54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff81acc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff815078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff81a54f18>] __netif_receive_skb+0x18/0x60
[<ffffffff81a54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff81a55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc04596b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc045a0e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffff81a555bf>] net_rx_action+0x26f/0x390
[<ffffffff814a4b95>] __do_softirq+0xf5/0x280
[<ffffffff814a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff814ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff814ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff814c5c21>] kthread+0xd1/0xe0
[<ffffffff814c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff81b93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff814c5b50>] ? insert_kthread_work+0x40/0x40
Code: ff c7 02 00 00 00 00 41 8b 94 24 e8 00 00 00 48 8b 4d d0 8d 14 d5 00 06 00 00 48 63 d2 48 03 91 b8 23 00 00 41 8b 4c 24 34 89 0a <e9> d0 f7 ff ff 0f 1f 80 00 00 00 00 48 c7 45 c8 00 40 00 00 41
Complete report #782366
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc048f877
Build id: 3.10.0-1160.el7.x86_64 		0x7f7
2
? nf_ct_delete
nf_conntrack 0xffffffffc094b016
Build id: 3.10.0-1160.el7.x86_64 		0x176
3
? lock_timer_base.isra.38
vmlinux 0xffffffffa56ad9fb
Build id: 3.10.0-1160.el7.x86_64 		0x2b
4
dev_hard_start_xmit
vmlinux 0xffffffffa5c54386
Build id: 3.10.0-1160.el7.x86_64 		0x246
5
sch_direct_xmit
vmlinux 0xffffffffa5c8242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
6
__dev_queue_xmit
vmlinux 0xffffffffa5c57301
Build id: 3.10.0-1160.el7.x86_64 		0x491
7
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0988020
Build id: 3.10.0-1160.el7.x86_64 		0x20
8
dev_queue_xmit
vmlinux 0xffffffffa5c574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
9
neigh_resolve_output
vmlinux 0xffffffffa5c641fd
Build id: 3.10.0-1160.el7.x86_64 		0x11d
10
ip_finish_output
vmlinux 0xffffffffa5ca505c
Build id: 3.10.0-1160.el7.x86_64 		0x27c
11
ip_output
vmlinux 0xffffffffa5ca59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
12
? __ip_append_data.isra.50
vmlinux 0xffffffffa5ca4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
13
ip_local_out_sk
vmlinux 0xffffffffa5ca3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
14
ip_send_skb
vmlinux 0xffffffffa5ca6446
Build id: 3.10.0-1160.el7.x86_64 		0x16
15
ip_push_pending_frames
vmlinux 0xffffffffa5ca64b3
Build id: 3.10.0-1160.el7.x86_64 		0x33
16
ip_send_unicast_reply
vmlinux 0xffffffffa5ca6888
Build id: 3.10.0-1160.el7.x86_64 		0x288
17
? _raw_spin_unlock_bh
vmlinux 0xffffffffa5d8992e
Build id: 3.10.0-1160.el7.x86_64 		0x1e
18
tcp_v4_send_reset
vmlinux 0xffffffffa5cc5546
Build id: 3.10.0-1160.el7.x86_64 		0x356
19
tcp_v4_rcv
vmlinux 0xffffffffa5cc6ddb
Build id: 3.10.0-1160.el7.x86_64 		0x5bb
20
ip_local_deliver_finish
vmlinux 0xffffffffa5c9f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffffa5c9f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
22
? ip_rcv_finish
vmlinux 0xffffffffa5c9f1d0
Build id: 3.10.0-1160.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffffa5c9eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffffa5c9f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
25
? inet_del_offload
vmlinux 0xffffffffa5c9ee60
Build id: 3.10.0-1160.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffa5c54c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
27
? tcp4_gro_receive
vmlinux 0xffffffffa5ccc100
Build id: 3.10.0-1160.el7.x86_64 		0x80
28
? __getnstimeofday64
vmlinux 0xffffffffa57078df
Build id: 3.10.0-1160.el7.x86_64 		0x3f
29
__netif_receive_skb
vmlinux 0xffffffffa5c54f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
30
netif_receive_skb_internal
vmlinux 0xffffffffa5c54fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
31
napi_gro_receive
vmlinux 0xffffffffa5c55c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc049045b
Build id: 3.10.0-1160.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc0491100
Build id: 3.10.0-1160.el7.x86_64 		0x280
34
? nf_ct_delete
nf_conntrack 0xffffffffc094b016
Build id: 3.10.0-1160.el7.x86_64 		0x176
35
net_rx_action
vmlinux 0xffffffffa5c555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
36
__do_softirq
vmlinux 0xffffffffa56a4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
37
run_ksoftirqd
vmlinux 0xffffffffa56a4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
38
smpboot_thread_fn
vmlinux 0xffffffffa56ce674
Build id: 3.10.0-1160.el7.x86_64 		0x144
39
? lg_double_unlock
vmlinux 0xffffffffa56ce530
Build id: 3.10.0-1160.el7.x86_64 		0x40
40
kthread
vmlinux 0xffffffffa56c5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
41
? insert_kthread_work
vmlinux 0xffffffffa56c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffa5d93df7
Build id: 3.10.0-1160.el7.x86_64 		0x21
43
? insert_kthread_work
vmlinux 0xffffffffa56c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec sb_edac ac97_bus snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel lrw snd_timer snd gf128mul ppdev glue_helper ablk_helper vmw_balloon cryptd
soundcore pcspkr joydev vmw_vmci sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit ata_piix libnvdimm libata e1000 mptspi scsi_transport_spi mptscsih mptbase crct10dif_pclmul crct10dif_common crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffffa039b936d280 ti: ffffa039ba760000 task.ti: ffffa039ba760000
RIP: 0010:[<ffffffffc048f877>]  [<ffffffffc048f877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffffa039ba763490  EFLAGS: 00000286
RAX: ffffb430c1c83818 RBX: 0000000000000001 RCX: 00000000000000fd
RDX: 00000000000000fd RSI: ffffa03c6a967140 RDI: ffffa039a77b6000
RBP: ffffa039ba763510 R08: 0000000000000100 R09: 0000000002000000
R10: 85fe0000145095f9 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000414d4e002 R14: 0000000100000010 R15: ffffffffa59b1650
FS:  0000000000000000(0000) GS:ffffa03c6d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000563d7a7fab98 CR3: 0000000254010000 CR4: 00000000001607f0
Call Trace:
[<ffffffffc094b016>] ? nf_ct_delete+0x176/0x270 [nf_conntrack]
[<ffffffffa56ad9fb>] ? lock_timer_base.isra.38+0x2b/0x50
[<ffffffffa5c54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa5c8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa5c57301>] __dev_queue_xmit+0x491/0x650
[<ffffffffc0988020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffa5c574d0>] dev_queue_xmit+0x10/0x20
[<ffffffffa5c641fd>] neigh_resolve_output+0x11d/0x220
[<ffffffffa5ca505c>] ip_finish_output+0x27c/0x8d0
[<ffffffffa5ca59db>] ip_output+0x7b/0xf0
[<ffffffffa5ca4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa5ca3407>] ip_local_out_sk+0x37/0x40
[<ffffffffa5ca6446>] ip_send_skb+0x16/0x50
[<ffffffffa5ca64b3>] ip_push_pending_frames+0x33/0x40
[<ffffffffa5ca6888>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffa5d8992e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffa5cc5546>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffa5cc6ddb>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffffa5c9f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa5c9f580>] ip_local_deliver+0x60/0xe0
[<ffffffffa5c9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa5c9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffa5c9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffa5c9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffa5c54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa5ccc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffa57078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa5c54f18>] __netif_receive_skb+0x18/0x60
[<ffffffffa5c54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa5c55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc049045b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0491100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc094b016>] ? nf_ct_delete+0x176/0x270 [nf_conntrack]
[<ffffffffa5c555bf>] net_rx_action+0x26f/0x390
[<ffffffffa56a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa56a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa56ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa56ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa56c5c21>] kthread+0xd1/0xe0
[<ffffffffa56c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa5d93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa56c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #656376
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04b2877
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff8ac543e6
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff8ac8248a
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff8ac57361
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff8ac57530
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffff8aca54cb
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffff8aca5a4b
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffff8aca4e50
Build id: 3.10.0-1160.6.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffff8aca3477
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffff8aca64b6
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffff8aca6523
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffff8aca68f8
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x288
13
? ip_build_and_send_pkt
vmlinux 0xffffffff8aca35e2
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x162
14
? local_bh_enable
vmlinux 0xffffffff8a6a4017
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x17
15
tcp_v4_send_reset
vmlinux 0xffffffff8acc55d6
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x356
16
? __inet_lookup_established
vmlinux 0xffffffff8aca9356
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x46
17
tcp_v4_do_rcv
vmlinux 0xffffffff8acc58be
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x18e
18
tcp_v4_rcv
vmlinux 0xffffffff8acc708c
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x7dc
19
ip_local_deliver_finish
vmlinux 0xffffffff8ac9f2fd
Build id: 3.10.0-1160.6.1.el7.x86_64 		0xbd
20
ip_local_deliver
vmlinux 0xffffffff8ac9f5f0
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x60
21
? ip_rcv_finish
vmlinux 0xffffffff8ac9f240
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x370
22
ip_rcv_finish
vmlinux 0xffffffff8ac9ef60
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x90
23
ip_rcv
vmlinux 0xffffffff8ac9f930
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x2c0
24
? inet_del_offload
vmlinux 0xffffffff8ac9eed0
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffff8ac54c79
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x729
26
__netif_receive_skb
vmlinux 0xffffffff8ac54f78
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x18
27
netif_receive_skb_internal
vmlinux 0xffffffff8ac55000
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x40
28
? tcp4_gro_receive
vmlinux 0xffffffff8accc230
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x120
29
napi_gro_complete
vmlinux 0xffffffff8ac5516f
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x7f
30
dev_gro_receive
vmlinux 0xffffffff8ac55a4e
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x30e
31
napi_gro_receive
vmlinux 0xffffffff8ac55be8
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x38
32
e1000_clean_rx_irq
e1000 0xffffffffc04b345b
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc04b4100
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x280
34
net_rx_action
vmlinux 0xffffffff8ac5561f
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffff8a6a4b95
Build id: 3.10.0-1160.6.1.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffff8a6a4d58
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffff8a6ce674
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffff8a6ce530
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffff8a6c5c21
Build id: 3.10.0-1160.6.1.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffff8a6c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffff8ad94df7
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x21
42
? insert_kthread_work
vmlinux 0xffffffff8a6c5b50
Build id: 3.10.0-1160.6.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: tcp_diag inet_diag cfg80211 rfkill xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper snd_seq_midi snd_seq_midi_event cryptd pcspkr joydev sg snd_ens1371 vmw_balloon snd_rawmidi snd_ac97_codec parport_pc parport ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm crct10dif_pclmul crct10dif_common ata_piix libnvdimm crc32c_intel e1000 mptspi libata scsi_transport_spi mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.6.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8df672375280 ti: ffff8df672f74000 task.ti: ffff8df672f74000
RIP: 0010:[<ffffffffc04b2877>]  [<ffffffffc04b2877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff8df672f77400  EFLAGS: 00000286
RAX: ffffa60983583818 RBX: 0000000000000001 RCX: 0000000000000030
RDX: 0000000000000030 RSI: ffff8df66abec280 RDI: ffff8df6457ce000
RBP: ffff8df672f77480 R08: 0000000000000000 R09: 0000000002000000
R10: 1e00000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000707850402 R14: 00000001e2152800 R15: ffffffff8a9b1690
FS:  0000000000000000(0000) GS:ffff8dfd1d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3e3199efb8 CR3: 00000001a9c10000 CR4: 00000000003607f0
Call Trace:
[<ffffffff8ac543e6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8ac8248a>] sch_direct_xmit+0x11a/0x250
[<ffffffff8ac57361>] __dev_queue_xmit+0x491/0x650
[<ffffffff8ac57530>] dev_queue_xmit+0x10/0x20
[<ffffffff8aca54cb>] ip_finish_output+0x67b/0x8d0
[<ffffffff8aca5a4b>] ip_output+0x7b/0xf0
[<ffffffff8aca4e50>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff8aca3477>] ip_local_out_sk+0x37/0x40
[<ffffffff8aca64b6>] ip_send_skb+0x16/0x50
[<ffffffff8aca6523>] ip_push_pending_frames+0x33/0x40
[<ffffffff8aca68f8>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff8aca35e2>] ? ip_build_and_send_pkt+0x162/0x250
[<ffffffff8a6a4017>] ? local_bh_enable+0x17/0x20
[<ffffffff8acc55d6>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff8aca9356>] ? __inet_lookup_established+0x46/0x130
[<ffffffff8acc58be>] tcp_v4_do_rcv+0x18e/0x350
[<ffffffff8acc708c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffff8ac9f2fd>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff8ac9f5f0>] ip_local_deliver+0x60/0xe0
[<ffffffff8ac9f240>] ? ip_rcv_finish+0x370/0x370
[<ffffffff8ac9ef60>] ip_rcv_finish+0x90/0x370
[<ffffffff8ac9f930>] ip_rcv+0x2c0/0x420
[<ffffffff8ac9eed0>] ? inet_del_offload+0x40/0x40
[<ffffffff8ac54c79>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff8ac54f78>] __netif_receive_skb+0x18/0x60
[<ffffffff8ac55000>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8accc230>] ? tcp4_gro_receive+0x120/0x1a0
[<ffffffff8ac5516f>] napi_gro_complete+0x7f/0x100
[<ffffffff8ac55a4e>] dev_gro_receive+0x30e/0x470
[<ffffffff8ac55be8>] napi_gro_receive+0x38/0x100
[<ffffffffc04b345b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04b4100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff8ac5561f>] net_rx_action+0x26f/0x390
[<ffffffff8a6a4b95>] __do_softirq+0xf5/0x280
[<ffffffff8a6a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8a6ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8a6ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8a6c5c21>] kthread+0xd1/0xe0
[<ffffffff8a6c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8ad94df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8a6c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #630606
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc015b8e7
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb927edc6
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb92add72
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x122
4
__dev_queue_xmit
vmlinux 0xffffffffb9281dd1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffb9281fa0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffb92d260b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffffb92d2b9b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb92d1f90
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb92d0587
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffffb92d3636
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffffb92d36a3
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffffb92d3a98
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x288
13
? local_bh_enable
vmlinux 0xffffffffb8ca8967
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x17
14
tcp_v4_send_reset
vmlinux 0xffffffffb92f2e86
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x356
15
? __inet_lookup_established
vmlinux 0xffffffffb92d6576
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x46
16
tcp_v4_do_rcv
vmlinux 0xffffffffb92f317e
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x19e
17
tcp_v4_rcv
vmlinux 0xffffffffb92f498c
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7dc
18
ip_local_deliver_finish
vmlinux 0xffffffffb92cc34d
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffffb92cc660
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x60
20
? ip_rcv_finish
vmlinux 0xffffffffb92cc290
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x380
21
ip_rcv_finish
vmlinux 0xffffffffb92cbfa0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffb92cc9a0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2c0
23
? inet_del_offload
vmlinux 0xffffffffb92cbf10
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
24
__netif_receive_skb_core
vmlinux 0xffffffffb927f669
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x739
25
? tcp4_gro_receive
vmlinux 0xffffffffb92f9b00
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x30
26
? inet_gro_receive
vmlinux 0xffffffffb930bcf9
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x249
27
__netif_receive_skb
vmlinux 0xffffffffb927f968
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x18
28
netif_receive_skb_internal
vmlinux 0xffffffffb927fa00
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
29
napi_gro_receive
vmlinux 0xffffffffb92806b8
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd8
30
e1000_clean_rx_irq
e1000 0xffffffffc015c4cb
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2bb
31
e1000_clean
e1000 0xffffffffc015d180
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x280
32
? tcp_retransmit_timer
vmlinux 0xffffffffb92efb32
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x3f2
33
net_rx_action
vmlinux 0xffffffffb9280037
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x277
34
__do_softirq
vmlinux 0xffffffffb8ca9545
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffb8ca9718
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffb8cd42ac
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x14c
37
? lg_double_unlock
vmlinux 0xffffffffb8cd4160
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffb8ccb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffb8ccb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffb93c51dd
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7
41
? insert_kthread_work
vmlinux 0xffffffffb8ccb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 iosf_mbi crc32_pclmul snd_rawmidi ghash_clmulni_intel snd_ac97_codec ac97_bus aesni_intel snd_seq lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device ppdev snd_pcm snd_timer snd sg soundcore vmw_balloon joydev pcspkr parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm drm crct10dif_pclmul crct10dif_common nfit ata_piix crc32c_intel libnvdimm mptspi scsi_transport_spi mptscsih libata mptbase serio_raw e1000 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9d1b7afe5280 ti: ffff9d1b7a8a4000 task.ti: ffff9d1b7a8a4000
RIP: 0010:[<ffffffffc015b8e7>]  [<ffffffffc015b8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9d1b7a8a7490  EFLAGS: 00000286
RAX: ffffb71a80d43818 RBX: 0000000000000001 RCX: 0000000000000068
RDX: 0000000000000068 RSI: ffff9d1c2fa37140 RDI: ffff9d1c33ad6000
RBP: ffff9d1b7a8a7510 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffc06fa2fd
R13: ffff9d1b7a8a74d0 R14: 0000000100000009 R15: ffffffffb8fc8a90
FS:  0000000000000000(0000) GS:ffff9d1c34a00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c000a07010 CR3: 00000000b2c64000 CR4: 00000000001607f0
Call Trace:
[<ffffffffb927edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb92add72>] sch_direct_xmit+0x122/0x250
[<ffffffffb9281dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb9281fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffb92d260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb92d2b9b>] ip_output+0x7b/0x100
[<ffffffffb92d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb92d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffb92d3636>] ip_send_skb+0x16/0x50
[<ffffffffb92d36a3>] ip_push_pending_frames+0x33/0x40
[<ffffffffb92d3a98>] ip_send_unicast_reply+0x288/0x300
[<ffffffffb8ca8967>] ? local_bh_enable+0x17/0x20
[<ffffffffb92f2e86>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffb92d6576>] ? __inet_lookup_established+0x46/0x130
[<ffffffffb92f317e>] tcp_v4_do_rcv+0x19e/0x360
[<ffffffffb92f498c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffb92cc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffffb92cc660>] ip_local_deliver+0x60/0xe0
[<ffffffffb92cc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffffb92cbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffffb92cc9a0>] ip_rcv+0x2c0/0x420
[<ffffffffb92cbf10>] ? inet_del_offload+0x40/0x40
[<ffffffffb927f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffffb92f9b00>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffffb930bcf9>] ? inet_gro_receive+0x249/0x290
[<ffffffffb927f968>] __netif_receive_skb+0x18/0x70
[<ffffffffb927fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb92806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc015c4cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc015d180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb92efb32>] ? tcp_retransmit_timer+0x3f2/0x7a0
[<ffffffffb9280037>] net_rx_action+0x277/0x3a0
[<ffffffffb8ca9545>] __do_softirq+0xf5/0x290
[<ffffffffb8ca9718>] run_ksoftirqd+0x38/0x60
[<ffffffffb8cd42ac>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb8cd4160>] ? lg_double_unlock+0x40/0x40
[<ffffffffb8ccb4d1>] kthread+0xd1/0xe0
[<ffffffffb8ccb400>] ? insert_kthread_work+0x40/0x40
[<ffffffffb93c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb8ccb400>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #567599
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc031b8e7
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7f7
2
? nf_ct_delete
nf_conntrack 0xffffffffc08590b6
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x176
3
? lock_timer_base.isra.38
vmlinux 0xffffffffa8cb287b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2b
4
dev_hard_start_xmit
vmlinux 0xffffffffa927edc6
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x246
5
sch_direct_xmit
vmlinux 0xffffffffa92add72
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x122
6
__dev_queue_xmit
vmlinux 0xffffffffa9281dd1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x491
7
dev_queue_xmit
vmlinux 0xffffffffa9281fa0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x10
8
neigh_resolve_output
vmlinux 0xffffffffa928f03d
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x11d
9
ip_finish_output
vmlinux 0xffffffffa92d220c
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x27c
10
ip_output
vmlinux 0xffffffffa92d2b9b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7b
11
? __ip_append_data.isra.50
vmlinux 0xffffffffa92d1f90
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xa60
12
ip_local_out_sk
vmlinux 0xffffffffa92d0587
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x37
13
ip_send_skb
vmlinux 0xffffffffa92d3636
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x16
14
ip_push_pending_frames
vmlinux 0xffffffffa92d36a3
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x33
15
ip_send_unicast_reply
vmlinux 0xffffffffa92d3a98
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x288
16
? _raw_spin_unlock_bh
vmlinux 0xffffffffa93ba56e
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x1e
17
tcp_v4_send_reset
vmlinux 0xffffffffa92f2e86
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x356
18
tcp_v4_rcv
vmlinux 0xffffffffa92f476b
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x5bb
19
ip_local_deliver_finish
vmlinux 0xffffffffa92cc34d
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xbd
20
ip_local_deliver
vmlinux 0xffffffffa92cc660
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x60
21
? ip_rcv_finish
vmlinux 0xffffffffa92cc290
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x380
22
ip_rcv_finish
vmlinux 0xffffffffa92cbfa0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x90
23
ip_rcv
vmlinux 0xffffffffa92cc9a0
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2c0
24
? inet_del_offload
vmlinux 0xffffffffa92cbf10
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffa927f669
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x739
26
? tcp4_gro_receive
vmlinux 0xffffffffa92f9b00
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x30
27
? __getnstimeofday64
vmlinux 0xffffffffa8d0e8df
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffa927f968
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffa927fa00
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffa92806b8
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc031c4cb
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc031d180
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x280
33
net_rx_action
vmlinux 0xffffffffa9280037
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x277
34
__do_softirq
vmlinux 0xffffffffa8ca9545
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffa8ca9718
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffa8cd42ac
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x14c
37
? lg_double_unlock
vmlinux 0xffffffffa8cd4160
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffa8ccb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffa8ccb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffa93c51dd
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x7
41
? insert_kthread_work
vmlinux 0xffffffffa8ccb400
Build id: 3.10.0-1160.80.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: binfmt_misc nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock bnep sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel btusb btrtl btbcm snd_ens1371 ppdev btintel snd_rawmidi bluetooth snd_ac97_codec ac97_bus aesni_intel lrw gf128mul snd_seq vmw_balloon glue_helper ablk_helper cryptd snd_seq_device snd_pcm joydev pcspkr rfkill snd_timer sg snd soundcore i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs
libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit ata_piix e1000 drm mptspi libnvdimm scsi_transport_spi libata mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff98e3f9aa8000 ti: ffff98e3f9aa4000 task.ti: ffff98e3f9aa4000
RIP: 0010:[<ffffffffc031b8e7>]  [<ffffffffc031b8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff98e3f9aa7490  EFLAGS: 00010286
RAX: ffffa90240383818 RBX: 0000000400000010 RCX: 00000000000000ba
RDX: 00000000000000ba RSI: ffff98e3b4dd6100 RDI: ffff98e3f7c28000
RBP: ffff98e3f9aa7510 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff98e3f9aa7488
R13: ffffffffa92c2965 R14: ffff98e3f9aa7400 R15: ffff98e3d54fc424
FS:  0000000000000000(0000) GS:ffff98e3fb640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000bcae79169c3 CR3: 0000000075652000 CR4: 00000000003607e0
Call Trace:
[<ffffffffc08590b6>] ? nf_ct_delete+0x176/0x280 [nf_conntrack]
[<ffffffffa8cb287b>] ? lock_timer_base.isra.38+0x2b/0x60
[<ffffffffa927edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa92add72>] sch_direct_xmit+0x122/0x250
[<ffffffffa9281dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffa9281fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffa928f03d>] neigh_resolve_output+0x11d/0x220
[<ffffffffa92d220c>] ip_finish_output+0x27c/0x8d0
[<ffffffffa92d2b9b>] ip_output+0x7b/0x100
[<ffffffffa92d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa92d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffa92d3636>] ip_send_skb+0x16/0x50
[<ffffffffa92d36a3>] ip_push_pending_frames+0x33/0x40
[<ffffffffa92d3a98>] ip_send_unicast_reply+0x288/0x300
[<ffffffffa93ba56e>] ? _raw_spin_unlock_bh+0x1e/0x30
[<ffffffffa92f2e86>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffa92f476b>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffffa92cc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffffa92cc660>] ip_local_deliver+0x60/0xe0
[<ffffffffa92cc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffffa92cbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffffa92cc9a0>] ip_rcv+0x2c0/0x420
[<ffffffffa92cbf10>] ? inet_del_offload+0x40/0x40
[<ffffffffa927f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffffa92f9b00>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffffa8d0e8df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa927f968>] __netif_receive_skb+0x18/0x70
[<ffffffffa927fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa92806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc031c4cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc031d180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa9280037>] net_rx_action+0x277/0x3a0
[<ffffffffa8ca9545>] __do_softirq+0xf5/0x290
[<ffffffffa8ca9718>] run_ksoftirqd+0x38/0x60
[<ffffffffa8cd42ac>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffa8cd4160>] ? lg_double_unlock+0x40/0x40
[<ffffffffa8ccb4d1>] kthread+0xd1/0xe0
[<ffffffffa8ccb400>] ? insert_kthread_work+0x40/0x40
[<ffffffffa93c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa8ccb400>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #527926
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc015a877
Build id: 3.10.0-1160.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffbd054386
Build id: 3.10.0-1160.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffbd08242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
4
__qdisc_run
vmlinux 0xffffffffbd0825ee
Build id: 3.10.0-1160.el7.x86_64 		0x8e
5
__dev_queue_xmit
vmlinux 0xffffffffbd057319
Build id: 3.10.0-1160.el7.x86_64 		0x4a9
6
dev_queue_xmit
vmlinux 0xffffffffbd0574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
7
ip_finish_output
vmlinux 0xffffffffbd0a545b
Build id: 3.10.0-1160.el7.x86_64 		0x67b
8
ip_output
vmlinux 0xffffffffbd0a59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffffbd0a4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
10
ip_local_out_sk
vmlinux 0xffffffffbd0a3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
11
ip_send_skb
vmlinux 0xffffffffbd0a6446
Build id: 3.10.0-1160.el7.x86_64 		0x16
12
ip_push_pending_frames
vmlinux 0xffffffffbd0a64b3
Build id: 3.10.0-1160.el7.x86_64 		0x33
13
ip_send_unicast_reply
vmlinux 0xffffffffbd0a6888
Build id: 3.10.0-1160.el7.x86_64 		0x288
14
? ipt_do_table
ip_tables 0xffffffffc019b719
Build id: 3.10.0-1160.el7.x86_64 		0x359
15
tcp_v4_send_reset
vmlinux 0xffffffffbd0c5546
Build id: 3.10.0-1160.el7.x86_64 		0x356
16
tcp_v4_rcv
vmlinux 0xffffffffbd0c6ddb
Build id: 3.10.0-1160.el7.x86_64 		0x5bb
17
ip_local_deliver_finish
vmlinux 0xffffffffbd09f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
18
ip_local_deliver
vmlinux 0xffffffffbd09f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
19
? ip_rcv_finish
vmlinux 0xffffffffbd09f1d0
Build id: 3.10.0-1160.el7.x86_64 		0x370
20
ip_rcv_finish
vmlinux 0xffffffffbd09eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
21
ip_rcv
vmlinux 0xffffffffbd09f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
22
? packet_rcv
vmlinux 0xffffffffbd15fb02
Build id: 3.10.0-1160.el7.x86_64 		0x232
23
? inet_del_offload
vmlinux 0xffffffffbd09ee60
Build id: 3.10.0-1160.el7.x86_64 		0x40
24
__netif_receive_skb_core
vmlinux 0xffffffffbd054c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
25
? tcp4_gro_receive
vmlinux 0xffffffffbd0cc100
Build id: 3.10.0-1160.el7.x86_64 		0x80
26
? inet_gro_receive
vmlinux 0xffffffffbd0ddd99
Build id: 3.10.0-1160.el7.x86_64 		0x249
27
__netif_receive_skb
vmlinux 0xffffffffbd054f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
28
netif_receive_skb_internal
vmlinux 0xffffffffbd054fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
29
napi_gro_receive
vmlinux 0xffffffffbd055c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
30
e1000_clean_rx_irq
e1000 0xffffffffc015b45b
Build id: 3.10.0-1160.el7.x86_64 		0x2bb
31
e1000_clean
e1000 0xffffffffc015c100
Build id: 3.10.0-1160.el7.x86_64 		0x280
32
net_rx_action
vmlinux 0xffffffffbd0555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
33
__do_softirq
vmlinux 0xffffffffbcaa4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
34
run_ksoftirqd
vmlinux 0xffffffffbcaa4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
35
smpboot_thread_fn
vmlinux 0xffffffffbcace674
Build id: 3.10.0-1160.el7.x86_64 		0x144
36
? lg_double_unlock
vmlinux 0xffffffffbcace530
Build id: 3.10.0-1160.el7.x86_64 		0x40
37
kthread
vmlinux 0xffffffffbcac5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
38
? insert_kthread_work
vmlinux 0xffffffffbcac5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
39
ret_from_fork_nospec_begin
vmlinux 0xffffffffbd193df7
Build id: 3.10.0-1160.el7.x86_64 		0x21
40
? insert_kthread_work
vmlinux 0xffffffffbcac5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev
glue_helper ablk_helper cryptd pcspkr vmw_balloon joydev sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul drm crct10dif_common mptspi e1000 nfit scsi_transport_spi ata_piix crc32c_intel libnvdimm mptscsih libata mptbase serio_raw drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8b697d765280 ti: ffff8b697e364000 task.ti: ffff8b697e364000
RIP: 0010:[<ffffffffc015a877>]  [<ffffffffc015a877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8b697e367478  EFLAGS: 00000286
RAX: ffffb77540d03818 RBX: 0000000000000042 RCX: 00000000000000cc
RDX: 00000000000000cc RSI: ffff8b6835c9d6c0 RDI: ffff8b6977be6000
RBP: ffff8b697e3674f8 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 00000002325f3cce
R13: 000000017e3674e8 R14: ffffffffbcdb1650 R15: ffff8b697e367430
FS:  0000000000000000(0000) GS:ffff8b6a35600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5218186000 CR3: 000000023095e000 CR4: 00000000003607f0
Call Trace:
[<ffffffffbd054386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbd08242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffbd0825ee>] __qdisc_run+0x8e/0x360
[<ffffffffbd057319>] __dev_queue_xmit+0x4a9/0x650
[<ffffffffbd0574d0>] dev_queue_xmit+0x10/0x20
[<ffffffffbd0a545b>] ip_finish_output+0x67b/0x8d0
[<ffffffffbd0a59db>] ip_output+0x7b/0xf0
[<ffffffffbd0a4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffbd0a3407>] ip_local_out_sk+0x37/0x40
[<ffffffffbd0a6446>] ip_send_skb+0x16/0x50
[<ffffffffbd0a64b3>] ip_push_pending_frames+0x33/0x40
[<ffffffffbd0a6888>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffc019b719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffbd0c5546>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffbd0c6ddb>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffffbd09f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffbd09f580>] ip_local_deliver+0x60/0xe0
[<ffffffffbd09f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbd09eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffbd09f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffbd15fb02>] ? packet_rcv+0x232/0x430
[<ffffffffbd09ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffbd054c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbd0cc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffbd0ddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffffbd054f18>] __netif_receive_skb+0x18/0x60
[<ffffffffbd054fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbd055c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc015b45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc015c100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffbd0555bf>] net_rx_action+0x26f/0x390
[<ffffffffbcaa4b95>] __do_softirq+0xf5/0x280
[<ffffffffbcaa4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffbcace674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbcace530>] ? lg_double_unlock+0x40/0x40
[<ffffffffbcac5c21>] kthread+0xd1/0xe0
[<ffffffffbcac5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffbd193df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbcac5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #500046
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03c0857
Build id: 3.10.0-957.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff88038c66
Build id: 3.10.0-957.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff88065cea
Build id: 3.10.0-957.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff8803bbe1
Build id: 3.10.0-957.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff8803bdb0
Build id: 3.10.0-957.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffff88088736
Build id: 3.10.0-957.el7.x86_64 		0x556
7
ip_output
vmlinux 0xffffffff88088c93
Build id: 3.10.0-957.el7.x86_64 		0x73
8
? __ip_append_data.isra.51
vmlinux 0xffffffff880881e0
Build id: 3.10.0-957.el7.x86_64 		0xa50
9
ip_local_out_sk
vmlinux 0xffffffff88086817
Build id: 3.10.0-957.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffff880896f6
Build id: 3.10.0-957.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffff88089763
Build id: 3.10.0-957.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffff88089b38
Build id: 3.10.0-957.el7.x86_64 		0x288
13
? inet6_destroy_sock
vmlinux 0xffffffff880f9111
Build id: 3.10.0-957.el7.x86_64 		0x41
14
tcp_v4_send_reset
vmlinux 0xffffffff880a8476
Build id: 3.10.0-957.el7.x86_64 		0x356
15
tcp_v4_do_rcv
vmlinux 0xffffffff880a875e
Build id: 3.10.0-957.el7.x86_64 		0x18e
16
tcp_v4_rcv
vmlinux 0xffffffff880a9dcc
Build id: 3.10.0-957.el7.x86_64 		0x7bc
17
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc08e22ce
Build id: 3.10.0-957.el7.x86_64 		0x4e
18
ip_local_deliver_finish
vmlinux 0xffffffff8808285d
Build id: 3.10.0-957.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffff88082b49
Build id: 3.10.0-957.el7.x86_64 		0x59
20
? ip_rcv_finish
vmlinux 0xffffffff880827a0
Build id: 3.10.0-957.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffff880824c0
Build id: 3.10.0-957.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffff88082e79
Build id: 3.10.0-957.el7.x86_64 		0x2b9
23
? udp4_lib_lookup
vmlinux 0xffffffff880b4902
Build id: 3.10.0-957.el7.x86_64 		0x52
24
? inet_del_offload
vmlinux 0xffffffff88082430
Build id: 3.10.0-957.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffff880394f9
Build id: 3.10.0-957.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffff880aee00
Build id: 3.10.0-957.el7.x86_64 		0x80
27
? __getnstimeofday64
vmlinux 0xffffffff87b024ef
Build id: 3.10.0-957.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffff880397f8
Build id: 3.10.0-957.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffff88039880
Build id: 3.10.0-957.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffff8803a508
Build id: 3.10.0-957.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc03c143b
Build id: 3.10.0-957.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc03c20dd
Build id: 3.10.0-957.el7.x86_64 		0x27d
33
net_rx_action
vmlinux 0xffffffff88039e9f
Build id: 3.10.0-957.el7.x86_64 		0x26f
34
__do_softirq
vmlinux 0xffffffff87aa0f05
Build id: 3.10.0-957.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffff87aa10c8
Build id: 3.10.0-957.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffff87aca5c4
Build id: 3.10.0-957.el7.x86_64 		0x144
37
? lg_double_unlock
vmlinux 0xffffffff87aca480
Build id: 3.10.0-957.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffff87ac1c31
Build id: 3.10.0-957.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffff87ac1b60
Build id: 3.10.0-957.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffff88174c37
Build id: 3.10.0-957.el7.x86_64 		0x21
41
? insert_kthread_work
vmlinux 0xffffffff87ac1b60
Build id: 3.10.0-957.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
Modules linked in: tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw snd_ens1371 gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer joydev snd soundcore pcspkr sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
vmwgfx drm_kms_helper serio_raw syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 mptspi scsi_transport_spi mptscsih drm_panel_orientation_quirks mptbase nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff9b01bcf72080 ti: ffff9b01bc800000 task.ti: ffff9b01bc800000
RIP: 0010:[<ffffffffc03c0857>]  [<ffffffffc03c0857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9b01bc8034c0  EFLAGS: 00000286
RAX: ffffa7dd40d43818 RBX: 0000000000000001 RCX: 000000000000003e
RDX: 000000000000003e RSI: ffff9b023cef9340 RDI: ffff9b0274c1e000
RBP: ffff9b01bc803540 R08: 0000000000000100 R09: 0000000002000000
R10: 754c000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000226ed9002 R14: 0000000100000097 R15: ffffffff87da04b0
FS:  0000000000000000(0000) GS:ffff9b0275600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0dfcd23c80 CR3: 00000001fcc10000 CR4: 00000000001607f0
Call Trace:
[<ffffffff88038c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff88065cea>] sch_direct_xmit+0x11a/0x250
[<ffffffff8803bbe1>] __dev_queue_xmit+0x491/0x650
[<ffffffff8803bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffff88088736>] ip_finish_output+0x556/0x7b0
[<ffffffff88088c93>] ip_output+0x73/0xe0
[<ffffffff880881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffff88086817>] ip_local_out_sk+0x37/0x40
[<ffffffff880896f6>] ip_send_skb+0x16/0x50
[<ffffffff88089763>] ip_push_pending_frames+0x33/0x40
[<ffffffff88089b38>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff880f9111>] ? inet6_destroy_sock+0x41/0x90
[<ffffffff880a8476>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff880a875e>] tcp_v4_do_rcv+0x18e/0x350
[<ffffffff880a9dcc>] tcp_v4_rcv+0x7bc/0x9c0
[<ffffffffc08e22ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff8808285d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff88082b49>] ip_local_deliver+0x59/0xd0
[<ffffffff880827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff880824c0>] ip_rcv_finish+0x90/0x370
[<ffffffff88082e79>] ip_rcv+0x2b9/0x410
[<ffffffff880b4902>] ? udp4_lib_lookup+0x52/0x70
[<ffffffff88082430>] ? inet_del_offload+0x40/0x40
[<ffffffff880394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff880aee00>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff87b024ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff880397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff88039880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8803a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc03c143b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03c20dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffff88039e9f>] net_rx_action+0x26f/0x390
[<ffffffff87aa0f05>] __do_softirq+0xf5/0x280
[<ffffffff87aa10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff87aca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff87aca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff87ac1c31>] kthread+0xd1/0xe0
[<ffffffff87ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff88174c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff87ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #267292
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04d5877
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffa2655f96
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffa26840aa
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffa2658f11
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffa26590e0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffa26a70eb
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffffa26a766b
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffa26a6a70
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffffa26a5097
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffffa26a80d6
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffffa26a8143
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffffa26a8518
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x288
13
? _raw_spin_unlock_bh
vmlinux 0xffffffffa278b70e
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x1e
14
tcp_v4_send_reset
vmlinux 0xffffffffa26c7226
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x356
15
tcp_v4_rcv
vmlinux 0xffffffffa26c8abb
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x5bb
16
ip_local_deliver_finish
vmlinux 0xffffffffa26a0f1d
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xbd
17
ip_local_deliver
vmlinux 0xffffffffa26a1210
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x60
18
? ip_rcv_finish
vmlinux 0xffffffffa26a0e60
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x370
19
ip_rcv_finish
vmlinux 0xffffffffa26a0b80
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x90
20
ip_rcv
vmlinux 0xffffffffa26a1550
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x2c0
21
? inet_del_offload
vmlinux 0xffffffffa26a0af0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
22
__netif_receive_skb_core
vmlinux 0xffffffffa2656829
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x729
23
? tcp4_gro_receive
vmlinux 0xffffffffa26cde00
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xa0
24
? __getnstimeofday64
vmlinux 0xffffffffa2107bbf
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x3f
25
__netif_receive_skb
vmlinux 0xffffffffa2656b28
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x18
26
netif_receive_skb_internal
vmlinux 0xffffffffa2656bb0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
27
napi_gro_receive
vmlinux 0xffffffffa2657838
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xd8
28
e1000_clean_rx_irq
e1000 0xffffffffc04d645b
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x2bb
29
e1000_clean
e1000 0xffffffffc04d7100
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x280
30
? update_curr
vmlinux 0xffffffffa20e46dc
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x14c
31
net_rx_action
vmlinux 0xffffffffa26571cf
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x26f
32
__do_softirq
vmlinux 0xffffffffa20a4b35
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xf5
33
run_ksoftirqd
vmlinux 0xffffffffa20a4cf8
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x38
34
smpboot_thread_fn
vmlinux 0xffffffffa20ce7f4
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x144
35
? lg_double_unlock
vmlinux 0xffffffffa20ce6b0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
36
kthread
vmlinux 0xffffffffa20c5da1
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xd1
37
? insert_kthread_work
vmlinux 0xffffffffa20c5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
38
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2795df7
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x21
39
? insert_kthread_work
vmlinux 0xffffffffa20c5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: uas usb_storage tcp_lp binfmt_misc veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) nls_utf8 udf crc_itu_t xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink lp vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event nfit libnvdimm coretemp snd_ens1371 iosf_mbi snd_rawmidi snd_ac97_codec crc32_pclmul ghash_clmulni_intel ac97_bus ppdev snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon snd_timer snd soundcore sg pcspkr
joydev parport_pc parport i2c_piix4 vmw_vmci sch_fq_codel ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel serio_raw e1000 ata_piix mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks floppy fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/20/2014
task: ffff92b1fdf70000 ti: ffff92b1fdf78000 task.ti: ffff92b1fdf78000
RIP: 0010:[<ffffffffc04d5877>]  [<ffffffffc04d5877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff92b1fdf7b4d0  EFLAGS: 00000286
RAX: ffffb97541c03818 RBX: 0000000000000001 RCX: 00000000000000d0
RDX: 00000000000000d0 RSI: ffff92b4ac987e00 RDI: ffff92b1e4f96000
RBP: ffff92b1fdf7b550 R08: 0000000000000100 R09: 0000000002000000
R10: b0ff00001450d50c R11: 0000000000000000 R12: 0000000000000202
R13: 000000035b8b8802 R14: 00000001a20a3fb7 R15: ffffffffa23b1c00
FS:  0000000000000000(0000) GS:ffff92b4ad640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8c4f895390 CR3: 00000000bbaa4000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa2655f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa26840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffa2658f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffa26590e0>] dev_queue_xmit+0x10/0x20
[<ffffffffa26a70eb>] ip_finish_output+0x67b/0x8d0
[<ffffffffa26a766b>] ip_output+0x7b/0xf0
[<ffffffffa26a6a70>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa26a5097>] ip_local_out_sk+0x37/0x40
[<ffffffffa26a80d6>] ip_send_skb+0x16/0x50
[<ffffffffa26a8143>] ip_push_pending_frames+0x33/0x40
[<ffffffffa26a8518>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffa278b70e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffa26c7226>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffa26c8abb>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffffa26a0f1d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa26a1210>] ip_local_deliver+0x60/0xe0
[<ffffffffa26a0e60>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa26a0b80>] ip_rcv_finish+0x90/0x370
[<ffffffffa26a1550>] ip_rcv+0x2c0/0x420
[<ffffffffa26a0af0>] ? inet_del_offload+0x40/0x40
[<ffffffffa2656829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa26cde00>] ? tcp4_gro_receive+0xa0/0x1a0
[<ffffffffa2107bbf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa2656b28>] __netif_receive_skb+0x18/0x60
[<ffffffffa2656bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa2657838>] napi_gro_receive+0xd8/0x100
[<ffffffffc04d645b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04d7100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa20e46dc>] ? update_curr+0x14c/0x1e0
[<ffffffffa26571cf>] net_rx_action+0x26f/0x390
[<ffffffffa20a4b35>] __do_softirq+0xf5/0x280
[<ffffffffa20a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffa20ce7f4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa20ce6b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa20c5da1>] kthread+0xd1/0xe0
[<ffffffffa20c5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2795df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa20c5cd0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #255088
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0100857
Build id: 3.10.0-957.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffbe638c66
Build id: 3.10.0-957.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffbe665cea
Build id: 3.10.0-957.el7.x86_64 		0x11a
4
__qdisc_run
vmlinux 0xffffffffbe665eae
Build id: 3.10.0-957.el7.x86_64 		0x8e
5
__dev_queue_xmit
vmlinux 0xffffffffbe63bbf9
Build id: 3.10.0-957.el7.x86_64 		0x4a9
6
dev_queue_xmit
vmlinux 0xffffffffbe63bdb0
Build id: 3.10.0-957.el7.x86_64 		0x10
7
ip_finish_output
vmlinux 0xffffffffbe688736
Build id: 3.10.0-957.el7.x86_64 		0x556
8
ip_output
vmlinux 0xffffffffbe688c93
Build id: 3.10.0-957.el7.x86_64 		0x73
9
? __ip_append_data.isra.51
vmlinux 0xffffffffbe6881e0
Build id: 3.10.0-957.el7.x86_64 		0xa50
10
ip_local_out_sk
vmlinux 0xffffffffbe686817
Build id: 3.10.0-957.el7.x86_64 		0x37
11
ip_send_skb
vmlinux 0xffffffffbe6896f6
Build id: 3.10.0-957.el7.x86_64 		0x16
12
ip_push_pending_frames
vmlinux 0xffffffffbe689763
Build id: 3.10.0-957.el7.x86_64 		0x33
13
ip_send_unicast_reply
vmlinux 0xffffffffbe689b38
Build id: 3.10.0-957.el7.x86_64 		0x288
14
? local_bh_enable
vmlinux 0xffffffffbe0a0387
Build id: 3.10.0-957.el7.x86_64 		0x17
15
? ipt_do_table
ip_tables 0xffffffffc02286fe
Build id: 3.10.0-957.el7.x86_64 		0x33e
16
tcp_v4_send_reset
vmlinux 0xffffffffbe6a8476
Build id: 3.10.0-957.el7.x86_64 		0x356
17
tcp_v4_rcv
vmlinux 0xffffffffbe6a9baa
Build id: 3.10.0-957.el7.x86_64 		0x59a
18
ip_local_deliver_finish
vmlinux 0xffffffffbe68285d
Build id: 3.10.0-957.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffffbe682b49
Build id: 3.10.0-957.el7.x86_64 		0x59
20
? ip_rcv_finish
vmlinux 0xffffffffbe6827a0
Build id: 3.10.0-957.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffffbe6824c0
Build id: 3.10.0-957.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffbe682e79
Build id: 3.10.0-957.el7.x86_64 		0x2b9
23
? udp4_lib_lookup
vmlinux 0xffffffffbe6b4902
Build id: 3.10.0-957.el7.x86_64 		0x52
24
? inet_del_offload
vmlinux 0xffffffffbe682430
Build id: 3.10.0-957.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffbe6394f9
Build id: 3.10.0-957.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffffbe6aee00
Build id: 3.10.0-957.el7.x86_64 		0x80
27
? __getnstimeofday64
vmlinux 0xffffffffbe1024ef
Build id: 3.10.0-957.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffbe6397f8
Build id: 3.10.0-957.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffbe639880
Build id: 3.10.0-957.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffbe63a508
Build id: 3.10.0-957.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc010143b
Build id: 3.10.0-957.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc01020dd
Build id: 3.10.0-957.el7.x86_64 		0x27d
33
? __queue_work
vmlinux 0xffffffffbe0b7c0c
Build id: 3.10.0-957.el7.x86_64 		0x13c
34
net_rx_action
vmlinux 0xffffffffbe639e9f
Build id: 3.10.0-957.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffffbe0a0f05
Build id: 3.10.0-957.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffffbe0a10c8
Build id: 3.10.0-957.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffffbe0ca5c4
Build id: 3.10.0-957.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffffbe0ca480
Build id: 3.10.0-957.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffffbe0c1c31
Build id: 3.10.0-957.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffffbe0c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffffbe774c1d
Build id: 3.10.0-957.el7.x86_64 		0x7
42
? insert_kthread_work
vmlinux 0xffffffffbe0c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#16 stuck for 21s! [ksoftirqd/16:89]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables overlay(T) xt_conntrack iptable_filter iptable_nat nf_conntrack_ipv4 vmw_vsock_vmci_transport nf_defrag_ipv4 vsock nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device ppdev iosf_mbi snd_pcm crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper snd_timer ablk_helper i2c_piix4 snd cryptd soundcore pcspkr sg parport_pc joydev vmw_vmci parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ata_piix ttm mptspi scsi_transport_spi drm nfit crct10dif_pclmul crct10dif_common libata mptscsih serio_raw e1000 crc32c_intel libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 16 PID: 89 Comm: ksoftirqd/16 Kdump: loaded Tainted: G             L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9b8ee8ca5140 ti: ffff9b8ee8ce0000 task.ti: ffff9b8ee8ce0000
RIP: 0010:[<ffffffffc0100857>]  [<ffffffffc0100857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9b8ee8ce34a8  EFLAGS: 00000286
RAX: ffffb521c65c3818 RBX: 000000000000003c RCX: 0000000000000065
RDX: 0000000000000065 RSI: ffff9b9d7620ed00 RDI: ffff9b9dbf60a000
RBP: ffff9b8ee8ce3528 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000fe9e55002
R13: 00000001c65c3818 R14: ffffffffbe3a04b0 R15: ffff9b8ee8ce3460
FS:  0000000000000000(0000) GS:ffff9b9d7da00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5bd043a000 CR3: 0000000f9de10000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffbe638c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbe665cea>] sch_direct_xmit+0x11a/0x250
[<ffffffffbe665eae>] __qdisc_run+0x8e/0x360
[<ffffffffbe63bbf9>] __dev_queue_xmit+0x4a9/0x650
[<ffffffffbe63bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffffbe688736>] ip_finish_output+0x556/0x7b0
[<ffffffffbe688c93>] ip_output+0x73/0xe0
[<ffffffffbe6881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffffbe686817>] ip_local_out_sk+0x37/0x40
[<ffffffffbe6896f6>] ip_send_skb+0x16/0x50
[<ffffffffbe689763>] ip_push_pending_frames+0x33/0x40
[<ffffffffbe689b38>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffbe0a0387>] ? local_bh_enable+0x17/0x20
[<ffffffffc02286fe>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffbe6a8476>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffbe6a9baa>] tcp_v4_rcv+0x59a/0x9c0
[<ffffffffbe68285d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffbe682b49>] ip_local_deliver+0x59/0xd0
[<ffffffffbe6827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbe6824c0>] ip_rcv_finish+0x90/0x370
[<ffffffffbe682e79>] ip_rcv+0x2b9/0x410
[<ffffffffbe6b4902>] ? udp4_lib_lookup+0x52/0x70
[<ffffffffbe682430>] ? inet_del_offload+0x40/0x40
[<ffffffffbe6394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbe6aee00>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffbe1024ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffbe6397f8>] __netif_receive_skb+0x18/0x60
[<ffffffffbe639880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbe63a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc010143b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01020dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffffbe0b7c0c>] ? __queue_work+0x13c/0x3e0
[<ffffffffbe639e9f>] net_rx_action+0x26f/0x390
[<ffffffffbe0a0f05>] __do_softirq+0xf5/0x280
[<ffffffffbe0a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffffbe0ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbe0ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffffbe0c1c31>] kthread+0xd1/0xe0
[<ffffffffbe0c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffbe774c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffbe0c1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #229426
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03ea877
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff8e455f96
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff8e4840aa
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff8e458f11
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffff8e4590e0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffff8e4a70eb
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffff8e4a766b
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffff8e4a6a70
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffff8e4a5097
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffff8e4a80d6
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffff8e4a8143
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffff8e4a8518
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x288
13
? _raw_spin_unlock_bh
vmlinux 0xffffffff8e58b70e
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x1e
14
tcp_v4_send_reset
vmlinux 0xffffffff8e4c7226
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x356
15
tcp_v4_rcv
vmlinux 0xffffffff8e4c8abb
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x5bb
16
ip_local_deliver_finish
vmlinux 0xffffffff8e4a0f1d
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xbd
17
ip_local_deliver
vmlinux 0xffffffff8e4a1210
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x60
18
? ip_rcv_finish
vmlinux 0xffffffff8e4a0e60
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x370
19
ip_rcv_finish
vmlinux 0xffffffff8e4a0b80
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x90
20
ip_rcv
vmlinux 0xffffffff8e4a1550
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x2c0
21
? inet_del_offload
vmlinux 0xffffffff8e4a0af0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
22
__netif_receive_skb_core
vmlinux 0xffffffff8e456829
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x729
23
? tcp4_gro_receive
vmlinux 0xffffffff8e4cde00
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xa0
24
? __getnstimeofday64
vmlinux 0xffffffff8df07bbf
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x3f
25
__netif_receive_skb
vmlinux 0xffffffff8e456b28
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x18
26
netif_receive_skb_internal
vmlinux 0xffffffff8e456bb0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
27
napi_gro_receive
vmlinux 0xffffffff8e457838
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xd8
28
e1000_clean_rx_irq
e1000 0xffffffffc03eb45b
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x2bb
29
e1000_clean
e1000 0xffffffffc03ec100
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x280
30
? update_curr
vmlinux 0xffffffff8dee46dc
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x14c
31
net_rx_action
vmlinux 0xffffffff8e4571cf
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x26f
32
__do_softirq
vmlinux 0xffffffff8dea4b35
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xf5
33
run_ksoftirqd
vmlinux 0xffffffff8dea4cf8
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x38
34
smpboot_thread_fn
vmlinux 0xffffffff8dece7f4
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x144
35
? lg_double_unlock
vmlinux 0xffffffff8dece6b0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
36
kthread
vmlinux 0xffffffff8dec5da1
Build id: 3.10.0-1160.25.1.el7.x86_64 		0xd1
37
? insert_kthread_work
vmlinux 0xffffffff8dec5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40
38
ret_from_fork_nospec_begin
vmlinux 0xffffffff8e595df7
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x21
39
? insert_kthread_work
vmlinux 0xffffffff8dec5cd0
Build id: 3.10.0-1160.25.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ksoftirqd/4:30]
Modules linked in: tcp_diag inet_diag binfmt_misc tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc vfat fat sb_edac nfit libnvdimm iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul vmw_balloon glue_helper ablk_helper cryptd sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci sch_fq_codel ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptsas drm ata_piix scsi_transport_sas mptscsih
libata mptbase crct10dif_pclmul crct10dif_common crc32c_intel e1000 serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 30 Comm: ksoftirqd/4 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.25.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.0.B64.1704110547 04/11/2017
task: ffff9f4e39f6e300 ti: ffff9f4e3a34c000 task.ti: ffff9f4e3a34c000
RIP: 0010:[<ffffffffc03ea877>]  [<ffffffffc03ea877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9f4e3a34f4d0  EFLAGS: 00000286
RAX: ffffb67343643818 RBX: 0000000000000001 RCX: 000000000000000b
RDX: 000000000000000b RSI: ffff9f54ffde8840 RDI: ffff9f4e3bb7e000
RBP: ffff9f4e3a34f550 R08: 0000000000000000 R09: 0000000002000000
R10: b216000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000821d3d202 R14: 000000018dea3fb7 R15: ffffffff8e1b1c00
FS:  0000000000000000(0000) GS:ffff9f50efd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd8db5ed560 CR3: 00000002e2a10000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8e455f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8e4840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffff8e458f11>] __dev_queue_xmit+0x491/0x650
[<ffffffff8e4590e0>] dev_queue_xmit+0x10/0x20
[<ffffffff8e4a70eb>] ip_finish_output+0x67b/0x8d0
[<ffffffff8e4a766b>] ip_output+0x7b/0xf0
[<ffffffff8e4a6a70>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff8e4a5097>] ip_local_out_sk+0x37/0x40
[<ffffffff8e4a80d6>] ip_send_skb+0x16/0x50
[<ffffffff8e4a8143>] ip_push_pending_frames+0x33/0x40
[<ffffffff8e4a8518>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffff8e58b70e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff8e4c7226>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffff8e4c8abb>] tcp_v4_rcv+0x5bb/0x9e0
[<ffffffff8e4a0f1d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff8e4a1210>] ip_local_deliver+0x60/0xe0
[<ffffffff8e4a0e60>] ? ip_rcv_finish+0x370/0x370
[<ffffffff8e4a0b80>] ip_rcv_finish+0x90/0x370
[<ffffffff8e4a1550>] ip_rcv+0x2c0/0x420
[<ffffffff8e4a0af0>] ? inet_del_offload+0x40/0x40
[<ffffffff8e456829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff8e4cde00>] ? tcp4_gro_receive+0xa0/0x1a0
[<ffffffff8df07bbf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff8e456b28>] __netif_receive_skb+0x18/0x60
[<ffffffff8e456bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8e457838>] napi_gro_receive+0xd8/0x100
[<ffffffffc03eb45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03ec100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff8dee46dc>] ? update_curr+0x14c/0x1e0
[<ffffffff8e4571cf>] net_rx_action+0x26f/0x390
[<ffffffff8dea4b35>] __do_softirq+0xf5/0x280
[<ffffffff8dea4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffff8dece7f4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8dece6b0>] ? lg_double_unlock+0x40/0x40
[<ffffffff8dec5da1>] kthread+0xd1/0xe0
[<ffffffff8dec5cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffff8e595df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8dec5cd0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #171135
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc056c857
Build id: 3.10.0-957.12.2.el7.x86_64 		0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffa3222717
Build id: 3.10.0-957.12.2.el7.x86_64 		0x37
3
dev_hard_start_xmit
vmlinux 0xffffffffa3239ce6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x246
4
sch_direct_xmit
vmlinux 0xffffffffa3266d6a
Build id: 3.10.0-957.12.2.el7.x86_64 		0x11a
5
__dev_queue_xmit
vmlinux 0xffffffffa323cc61
Build id: 3.10.0-957.12.2.el7.x86_64 		0x491
6
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0ab5020
Build id: 3.10.0-957.12.2.el7.x86_64 		0x20
7
dev_queue_xmit
vmlinux 0xffffffffa323ce30
Build id: 3.10.0-957.12.2.el7.x86_64 		0x10
8
neigh_resolve_output
vmlinux 0xffffffffa324969d
Build id: 3.10.0-957.12.2.el7.x86_64 		0x11d
9
ip_finish_output
vmlinux 0xffffffffa32894dc
Build id: 3.10.0-957.12.2.el7.x86_64 		0x27c
10
ip_output
vmlinux 0xffffffffa3289d13
Build id: 3.10.0-957.12.2.el7.x86_64 		0x73
11
? __ip_append_data.isra.51
vmlinux 0xffffffffa3289260
Build id: 3.10.0-957.12.2.el7.x86_64 		0xa50
12
ip_local_out_sk
vmlinux 0xffffffffa3287897
Build id: 3.10.0-957.12.2.el7.x86_64 		0x37
13
ip_send_skb
vmlinux 0xffffffffa328a776
Build id: 3.10.0-957.12.2.el7.x86_64 		0x16
14
ip_push_pending_frames
vmlinux 0xffffffffa328a7e3
Build id: 3.10.0-957.12.2.el7.x86_64 		0x33
15
ip_send_unicast_reply
vmlinux 0xffffffffa328abb8
Build id: 3.10.0-957.12.2.el7.x86_64 		0x288
16
? local_bh_enable
vmlinux 0xffffffffa2ca0477
Build id: 3.10.0-957.12.2.el7.x86_64 		0x17
17
? local_bh_enable
vmlinux 0xffffffffa2ca0477
Build id: 3.10.0-957.12.2.el7.x86_64 		0x17
18
? ipt_do_table
ip_tables 0xffffffffc058d6fe
Build id: 3.10.0-957.12.2.el7.x86_64 		0x33e
19
tcp_v4_send_reset
vmlinux 0xffffffffa32a94f6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x356
20
tcp_v4_rcv
vmlinux 0xffffffffa32aac2a
Build id: 3.10.0-957.12.2.el7.x86_64 		0x59a
21
ip_local_deliver_finish
vmlinux 0xffffffffa32838dd
Build id: 3.10.0-957.12.2.el7.x86_64 		0xbd
22
ip_local_deliver
vmlinux 0xffffffffa3283bc9
Build id: 3.10.0-957.12.2.el7.x86_64 		0x59
23
? ip_rcv_finish
vmlinux 0xffffffffa3283820
Build id: 3.10.0-957.12.2.el7.x86_64 		0x370
24
ip_rcv_finish
vmlinux 0xffffffffa3283540
Build id: 3.10.0-957.12.2.el7.x86_64 		0x90
25
ip_rcv
vmlinux 0xffffffffa3283ef9
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2b9
26
? udp4_lib_lookup_skb
vmlinux 0xffffffffa32b5a02
Build id: 3.10.0-957.12.2.el7.x86_64 		0x62
27
? inet_del_offload
vmlinux 0xffffffffa32834b0
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffa323a579
Build id: 3.10.0-957.12.2.el7.x86_64 		0x729
29
? __getnstimeofday64
vmlinux 0xffffffffa2d0278f
Build id: 3.10.0-957.12.2.el7.x86_64 		0x3f
30
__netif_receive_skb
vmlinux 0xffffffffa323a878
Build id: 3.10.0-957.12.2.el7.x86_64 		0x18
31
netif_receive_skb_internal
vmlinux 0xffffffffa323a900
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
32
napi_gro_receive
vmlinux 0xffffffffa323b588
Build id: 3.10.0-957.12.2.el7.x86_64 		0xd8
33
e1000_clean_rx_irq
e1000 0xffffffffc056d43b
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2bb
34
e1000_clean
e1000 0xffffffffc056e0dd
Build id: 3.10.0-957.12.2.el7.x86_64 		0x27d
35
? update_curr
vmlinux 0xffffffffa2ce02dc
Build id: 3.10.0-957.12.2.el7.x86_64 		0x14c
36
net_rx_action
vmlinux 0xffffffffa323af1f
Build id: 3.10.0-957.12.2.el7.x86_64 		0x26f
37
__do_softirq
vmlinux 0xffffffffa2ca0ff5
Build id: 3.10.0-957.12.2.el7.x86_64 		0xf5
38
run_ksoftirqd
vmlinux 0xffffffffa2ca11b8
Build id: 3.10.0-957.12.2.el7.x86_64 		0x38
39
smpboot_thread_fn
vmlinux 0xffffffffa2cca6b4
Build id: 3.10.0-957.12.2.el7.x86_64 		0x144
40
? lg_double_unlock
vmlinux 0xffffffffa2cca570
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
41
kthread
vmlinux 0xffffffffa2cc1d21
Build id: 3.10.0-957.12.2.el7.x86_64 		0xd1
42
? insert_kthread_work
vmlinux 0xffffffffa2cc1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffffa3375c37
Build id: 3.10.0-957.12.2.el7.x86_64 		0x21
44
? insert_kthread_work
vmlinux 0xffffffffa2cc1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
Modules linked in: nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) binfmt_misc vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event coretemp iosf_mbi kvm_intel kvm snd_ens1371 irqbypass ppdev crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_rawmidi cryptd snd_ac97_codec ac97_bus vmw_balloon snd_seq joydev snd_seq_device pcspkr snd_pcm snd_timer snd soundcore sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper e1000 mptspi scsi_transport_spi mptscsih syscopyarea mptbase sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata drm_panel_orientation_quirks nfit libnvdimm
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------ T 3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9366fa35a080 ti: ffff9366f9c58000 task.ti: ffff9366f9c58000
RIP: 0010:[<ffffffffc056c857>]  [<ffffffffc056c857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9366f9c5b4c0  EFLAGS: 00000286
RAX: ffffad5e81bc3818 RBX: 0000000000000001 RCX: 0000000000000061
RDX: 0000000000000061 RSI: ffff9365b5481100 RDI: ffff9369ad50e000
RBP: ffff9366f9c5b540 R08: 0000000000000100 R09: 0000000002000000
R10: ed80000004500000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000004222c4002 R14: 0000000100000202 R15: ffffffffa2fa0c60
FS:  0000000000000000(0000) GS:ffff9369ad600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000560e32d7d018 CR3: 00000000bb5ba000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa3222717>] ? kfree_skbmem+0x37/0x90
[<ffffffffa3239ce6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa3266d6a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa323cc61>] __dev_queue_xmit+0x491/0x650
[<ffffffffc0ab5020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffa323ce30>] dev_queue_xmit+0x10/0x20
[<ffffffffa324969d>] neigh_resolve_output+0x11d/0x220
[<ffffffffa32894dc>] ip_finish_output+0x27c/0x7b0
[<ffffffffa3289d13>] ip_output+0x73/0xe0
[<ffffffffa3289260>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffffa3287897>] ip_local_out_sk+0x37/0x40
[<ffffffffa328a776>] ip_send_skb+0x16/0x50
[<ffffffffa328a7e3>] ip_push_pending_frames+0x33/0x40
[<ffffffffa328abb8>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffa2ca0477>] ? local_bh_enable+0x17/0x20
[<ffffffffa2ca0477>] ? local_bh_enable+0x17/0x20
[<ffffffffc058d6fe>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffa32a94f6>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffa32aac2a>] tcp_v4_rcv+0x59a/0x9c0
[<ffffffffa32838dd>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa3283bc9>] ip_local_deliver+0x59/0xd0
[<ffffffffa3283820>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa3283540>] ip_rcv_finish+0x90/0x370
[<ffffffffa3283ef9>] ip_rcv+0x2b9/0x410
[<ffffffffa32b5a02>] ? udp4_lib_lookup_skb+0x62/0x70
[<ffffffffa32834b0>] ? inet_del_offload+0x40/0x40
[<ffffffffa323a579>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa2d0278f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa323a878>] __netif_receive_skb+0x18/0x60
[<ffffffffa323a900>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa323b588>] napi_gro_receive+0xd8/0x100
[<ffffffffc056d43b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc056e0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffffa2ce02dc>] ? update_curr+0x14c/0x1e0
[<ffffffffa323af1f>] net_rx_action+0x26f/0x390
[<ffffffffa2ca0ff5>] __do_softirq+0xf5/0x280
[<ffffffffa2ca11b8>] run_ksoftirqd+0x38/0x50
[<ffffffffa2cca6b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa2cca570>] ? lg_double_unlock+0x40/0x40
[<ffffffffa2cc1d21>] kthread+0xd1/0xe0
[<ffffffffa2cc1c50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa3375c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa2cc1c50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #23225
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03b3867
Build id: 3.10.0-862.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb13eb166
Build id: 3.10.0-862.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb1416a2a
Build id: 3.10.0-862.el7.x86_64 		0x11a
4
__qdisc_run
vmlinux 0xffffffffb1416bee
Build id: 3.10.0-862.el7.x86_64 		0x8e
5
__dev_queue_xmit
vmlinux 0xffffffffb13ee069
Build id: 3.10.0-862.el7.x86_64 		0x4b9
6
dev_queue_xmit
vmlinux 0xffffffffb13ee220
Build id: 3.10.0-862.el7.x86_64 		0x10
7
neigh_resolve_output
vmlinux 0xffffffffb13f9c5d
Build id: 3.10.0-862.el7.x86_64 		0x11d
8
ip_finish_output
vmlinux 0xffffffffb143717c
Build id: 3.10.0-862.el7.x86_64 		0x2ac
9
ip_output
vmlinux 0xffffffffb1437973
Build id: 3.10.0-862.el7.x86_64 		0x73
10
? __ip_append_data.isra.50
vmlinux 0xffffffffb1436ed0
Build id: 3.10.0-862.el7.x86_64 		0xa50
11
ip_local_out_sk
vmlinux 0xffffffffb1435567
Build id: 3.10.0-862.el7.x86_64 		0x37
12
ip_send_skb
vmlinux 0xffffffffb14383c6
Build id: 3.10.0-862.el7.x86_64 		0x16
13
ip_push_pending_frames
vmlinux 0xffffffffb1438433
Build id: 3.10.0-862.el7.x86_64 		0x33
14
ip_send_unicast_reply
vmlinux 0xffffffffb1438804
Build id: 3.10.0-862.el7.x86_64 		0x284
15
? local_bh_enable
vmlinux 0xffffffffb0e99df7
Build id: 3.10.0-862.el7.x86_64 		0x17
16
? local_bh_enable
vmlinux 0xffffffffb0e99df7
Build id: 3.10.0-862.el7.x86_64 		0x17
17
? ipt_do_table
ip_tables 0xffffffffc0473a2e
Build id: 3.10.0-862.el7.x86_64 		0x33e
18
? __fib_validate_source.isra.13
vmlinux 0xffffffffb14747a5
Build id: 3.10.0-862.el7.x86_64 		0xf5
19
tcp_v4_send_reset
vmlinux 0xffffffffb1456b8a
Build id: 3.10.0-862.el7.x86_64 		0x31a
20
tcp_v4_rcv
vmlinux 0xffffffffb1458376
Build id: 3.10.0-862.el7.x86_64 		0x596
21
ip_local_deliver_finish
vmlinux 0xffffffffb1431699
Build id: 3.10.0-862.el7.x86_64 		0xb9
22
ip_local_deliver
vmlinux 0xffffffffb1431989
Build id: 3.10.0-862.el7.x86_64 		0x59
23
? ip_rcv_finish
vmlinux 0xffffffffb14315e0
Build id: 3.10.0-862.el7.x86_64 		0x370
24
ip_rcv_finish
vmlinux 0xffffffffb1431300
Build id: 3.10.0-862.el7.x86_64 		0x90
25
ip_rcv
vmlinux 0xffffffffb1431cb9
Build id: 3.10.0-862.el7.x86_64 		0x2b9
26
? udp4_lib_lookup
vmlinux 0xffffffffb1463002
Build id: 3.10.0-862.el7.x86_64 		0x22
27
? inet_del_offload
vmlinux 0xffffffffb1431270
Build id: 3.10.0-862.el7.x86_64 		0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffb13eb9f9
Build id: 3.10.0-862.el7.x86_64 		0x729
29
? tcp4_gro_receive
vmlinux 0xffffffffb145d600
Build id: 3.10.0-862.el7.x86_64 		0xe0
30
? inet_gro_receive
vmlinux 0xffffffffb146edb9
Build id: 3.10.0-862.el7.x86_64 		0x249
31
__netif_receive_skb
vmlinux 0xffffffffb13ebd08
Build id: 3.10.0-862.el7.x86_64 		0x18
32
netif_receive_skb_internal
vmlinux 0xffffffffb13ebd90
Build id: 3.10.0-862.el7.x86_64 		0x40
33
napi_gro_receive
vmlinux 0xffffffffb13ec9b8
Build id: 3.10.0-862.el7.x86_64 		0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc03b444b
Build id: 3.10.0-862.el7.x86_64 		0x2bb
35
e1000_clean
e1000 0xffffffffc03b50eb
Build id: 3.10.0-862.el7.x86_64 		0x27b
36
net_rx_action
vmlinux 0xffffffffb13ec3af
Build id: 3.10.0-862.el7.x86_64 		0x26f
37
__do_softirq
vmlinux 0xffffffffb0e9a945
Build id: 3.10.0-862.el7.x86_64 		0xf5
38
run_ksoftirqd
vmlinux 0xffffffffb0e9ab08
Build id: 3.10.0-862.el7.x86_64 		0x38
39
smpboot_thread_fn
vmlinux 0xffffffffb0ec35e4
Build id: 3.10.0-862.el7.x86_64 		0x144
40
? lg_double_unlock
vmlinux 0xffffffffb0ec34a0
Build id: 3.10.0-862.el7.x86_64 		0x40
41
kthread
vmlinux 0xffffffffb0ebae31
Build id: 3.10.0-862.el7.x86_64 		0xd1
42
? insert_kthread_work
vmlinux 0xffffffffb0ebad60
Build id: 3.10.0-862.el7.x86_64 		0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffffb151f637
Build id: 3.10.0-862.el7.x86_64 		0x21
44
? insert_kthread_work
vmlinux 0xffffffffb0ebad60
Build id: 3.10.0-862.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
<EOI> 
Modules linked in:
xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq cryptd snd_seq_device snd_pcm snd_timer snd soundcore ppdev nfit vmw_balloon sg joydev pcspkr libnvdimm parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi scsi_transport_spi mptscsih e1000 mptbase serio_raw i2c_core
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9608f7549fa0 ti: ffff9608f75d8000 task.ti: ffff9608f75d8000
RIP: 0010:[<ffffffffc03b3867>]  [<ffffffffc03b3867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff9608f75db468  EFLAGS: 00000286
RAX: ffffa99b80983818 RBX: 0000000000000001 RCX: 00000000000000ee
RDX: 00000000000000ee RSI: ffff9608f1e86140 RDI: ffff9608f2ed0000
RBP: ffff9608f75db4e8 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000001333f70c6 R14: 00000001f33f70e8 R15: ffffffffb1173e00
FS:  0000000000000000(0000) GS:ffff9608f9600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000983db0 CR3: 00000000933fe000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb13eb166>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb1416a2a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb1416bee>] __qdisc_run+0x8e/0x360
[<ffffffffb13ee069>] __dev_queue_xmit+0x4b9/0x660
[<ffffffffb13ee220>] dev_queue_xmit+0x10/0x20
[<ffffffffb13f9c5d>] neigh_resolve_output+0x11d/0x220
[<ffffffffb143717c>] ip_finish_output+0x2ac/0x7a0
[<ffffffffb1437973>] ip_output+0x73/0xe0
[<ffffffffb1436ed0>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffb1435567>] ip_local_out_sk+0x37/0x40
[<ffffffffb14383c6>] ip_send_skb+0x16/0x50
[<ffffffffb1438433>] ip_push_pending_frames+0x33/0x40
[<ffffffffb1438804>] ip_send_unicast_reply+0x284/0x2d0
[<ffffffffb0e99df7>] ? local_bh_enable+0x17/0x20
[<ffffffffb0e99df7>] ? local_bh_enable+0x17/0x20
[<ffffffffc0473a2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffb14747a5>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffffb1456b8a>] tcp_v4_send_reset+0x31a/0x460
[<ffffffffb1458376>] tcp_v4_rcv+0x596/0x990
[<ffffffffb1431699>] ip_local_deliver_finish+0xb9/0x200
[<ffffffffb1431989>] ip_local_deliver+0x59/0xd0
[<ffffffffb14315e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb1431300>] ip_rcv_finish+0x90/0x370
[<ffffffffb1431cb9>] ip_rcv+0x2b9/0x410
[<ffffffffb1463002>] ? udp4_lib_lookup+0x22/0x70
[<ffffffffb1431270>] ? inet_del_offload+0x40/0x40
[<ffffffffb13eb9f9>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffb145d600>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffb146edb9>] ? inet_gro_receive+0x249/0x290
[<ffffffffb13ebd08>] __netif_receive_skb+0x18/0x60
[<ffffffffb13ebd90>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb13ec9b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc03b444b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03b50eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffb13ec3af>] net_rx_action+0x26f/0x390
[<ffffffffb0e9a945>] __do_softirq+0xf5/0x280
[<ffffffffb0e9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffffb0ec35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb0ec34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb0ebae31>] kthread+0xd1/0xe0
[<ffffffffb0ebad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffb151f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb0ebad60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #17460
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc025d857
Build id: 3.10.0-957.12.2.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb0439ce6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb0466d6a
Build id: 3.10.0-957.12.2.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb043cc61
Build id: 3.10.0-957.12.2.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffb043ce30
Build id: 3.10.0-957.12.2.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffb04897b6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x556
7
ip_output
vmlinux 0xffffffffb0489d13
Build id: 3.10.0-957.12.2.el7.x86_64 		0x73
8
? __ip_append_data.isra.51
vmlinux 0xffffffffb0489260
Build id: 3.10.0-957.12.2.el7.x86_64 		0xa50
9
ip_local_out_sk
vmlinux 0xffffffffb0487897
Build id: 3.10.0-957.12.2.el7.x86_64 		0x37
10
ip_send_skb
vmlinux 0xffffffffb048a776
Build id: 3.10.0-957.12.2.el7.x86_64 		0x16
11
ip_push_pending_frames
vmlinux 0xffffffffb048a7e3
Build id: 3.10.0-957.12.2.el7.x86_64 		0x33
12
ip_send_unicast_reply
vmlinux 0xffffffffb048abb8
Build id: 3.10.0-957.12.2.el7.x86_64 		0x288
13
? _raw_spin_unlock_bh
vmlinux 0xffffffffb056b53e
Build id: 3.10.0-957.12.2.el7.x86_64 		0x1e
14
? nf_nat_setup_info
nf_nat 0xffffffffc06be3b7
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2f7
15
tcp_v4_send_reset
vmlinux 0xffffffffb04a94f6
Build id: 3.10.0-957.12.2.el7.x86_64 		0x356
16
? internal_add_timer
vmlinux 0xffffffffafea9b9d
Build id: 3.10.0-957.12.2.el7.x86_64 		0x7d
17
tcp_v4_rcv
vmlinux 0xffffffffb04aac2a
Build id: 3.10.0-957.12.2.el7.x86_64 		0x59a
18
ip_local_deliver_finish
vmlinux 0xffffffffb04838dd
Build id: 3.10.0-957.12.2.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffffb0483bc9
Build id: 3.10.0-957.12.2.el7.x86_64 		0x59
20
? ip_rcv_finish
vmlinux 0xffffffffb0483820
Build id: 3.10.0-957.12.2.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffffb0483540
Build id: 3.10.0-957.12.2.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffb0483ef9
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2b9
23
? udp4_lib_lookup_skb
vmlinux 0xffffffffb04b5a02
Build id: 3.10.0-957.12.2.el7.x86_64 		0x62
24
? inet_del_offload
vmlinux 0xffffffffb04834b0
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffb043a579
Build id: 3.10.0-957.12.2.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffffb04aff00
Build id: 3.10.0-957.12.2.el7.x86_64 		0x100
27
? inet_gro_receive
vmlinux 0xffffffffb04c1889
Build id: 3.10.0-957.12.2.el7.x86_64 		0x249
28
__netif_receive_skb
vmlinux 0xffffffffb043a878
Build id: 3.10.0-957.12.2.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffb043a900
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffb043b588
Build id: 3.10.0-957.12.2.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc025e43b
Build id: 3.10.0-957.12.2.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc025f0dd
Build id: 3.10.0-957.12.2.el7.x86_64 		0x27d
33
? update_curr
vmlinux 0xffffffffafee02dc
Build id: 3.10.0-957.12.2.el7.x86_64 		0x14c
34
net_rx_action
vmlinux 0xffffffffb043af1f
Build id: 3.10.0-957.12.2.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffffafea0ff5
Build id: 3.10.0-957.12.2.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffffafea11b8
Build id: 3.10.0-957.12.2.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffffafeca6b4
Build id: 3.10.0-957.12.2.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffffafeca570
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffffafec1d21
Build id: 3.10.0-957.12.2.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffffafec1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffffb0575c1d
Build id: 3.10.0-957.12.2.el7.x86_64 		0x7
42
? insert_kthread_work
vmlinux 0xffffffffafec1c50
Build id: 3.10.0-957.12.2.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:3]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev pcspkr vmw_balloon sg parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif cdrom crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ata_generic pata_acpi serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi ata_piix mptscsih drm mptbase e1000 libata
drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-957.12.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8c54ba35a080 ti: ffff8c54b9c58000 task.ti: ffff8c54b9c58000
RIP: 0010:[<ffffffffc025d857>]  [<ffffffffc025d857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff8c54b9c5b500  EFLAGS: 00000286
RAX: ffffa20201943818 RBX: 0000000000000001 RCX: 00000000000000da
RDX: 00000000000000da RSI: ffff8c5769712600 RDI: ffff8c5627b40000
RBP: ffff8c54b9c5b580 R08: 0000000000000100 R09: 0000000002000000
R10: d3be000014504281 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000410af6002 R14: 00000001afea0477 R15: ffffffffb01a0c60
FS:  0000000000000000(0000) GS:ffff8c576d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000019df430 CR3: 00000002e7210000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb0439ce6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb0466d6a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb043cc61>] __dev_queue_xmit+0x491/0x650
[<ffffffffb043ce30>] dev_queue_xmit+0x10/0x20
[<ffffffffb04897b6>] ip_finish_output+0x556/0x7b0
[<ffffffffb0489d13>] ip_output+0x73/0xe0
[<ffffffffb0489260>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffffb0487897>] ip_local_out_sk+0x37/0x40
[<ffffffffb048a776>] ip_send_skb+0x16/0x50
[<ffffffffb048a7e3>] ip_push_pending_frames+0x33/0x40
[<ffffffffb048abb8>] ip_send_unicast_reply+0x288/0x2f0
[<ffffffffb056b53e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffc06be3b7>] ? nf_nat_setup_info+0x2f7/0x320 [nf_nat]
[<ffffffffb04a94f6>] tcp_v4_send_reset+0x356/0x4b0
[<ffffffffafea9b9d>] ? internal_add_timer+0x7d/0xc0
[<ffffffffb04aac2a>] tcp_v4_rcv+0x59a/0x9c0
[<ffffffffb04838dd>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb0483bc9>] ip_local_deliver+0x59/0xd0
[<ffffffffb0483820>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb0483540>] ip_rcv_finish+0x90/0x370
[<ffffffffb0483ef9>] ip_rcv+0x2b9/0x410
[<ffffffffb04b5a02>] ? udp4_lib_lookup_skb+0x62/0x70
[<ffffffffb04834b0>] ? inet_del_offload+0x40/0x40
[<ffffffffb043a579>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb04aff00>] ? tcp4_gro_receive+0x100/0x1a0
[<ffffffffb04c1889>] ? inet_gro_receive+0x249/0x290
[<ffffffffb043a878>] __netif_receive_skb+0x18/0x60
[<ffffffffb043a900>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb043b588>] napi_gro_receive+0xd8/0x100
[<ffffffffc025e43b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc025f0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffffafee02dc>] ? update_curr+0x14c/0x1e0
[<ffffffffb043af1f>] net_rx_action+0x26f/0x390
[<ffffffffafea0ff5>] __do_softirq+0xf5/0x280
[<ffffffffafea11b8>] run_ksoftirqd+0x38/0x50
[<ffffffffafeca6b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffafeca570>] ? lg_double_unlock+0x40/0x40
[<ffffffffafec1d21>] kthread+0xd1/0xe0
[<ffffffffafec1c50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb0575c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffafec1c50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #14156
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03fe867
Build id: 3.10.0-862.14.4.el7.x86_64 		0x7f7
2
? nf_ct_delete
nf_conntrack 0xffffffffc08d7056
Build id: 3.10.0-862.14.4.el7.x86_64 		0x176
3
? lock_timer_base.isra.34
vmlinux 0xffffffffbdea64cb
Build id: 3.10.0-862.14.4.el7.x86_64 		0x2b
4
dev_hard_start_xmit
vmlinux 0xffffffffbe3f02e6
Build id: 3.10.0-862.14.4.el7.x86_64 		0x246
5
sch_direct_xmit
vmlinux 0xffffffffbe41bc0a
Build id: 3.10.0-862.14.4.el7.x86_64 		0x11a
6
__dev_queue_xmit
vmlinux 0xffffffffbe3f31d1
Build id: 3.10.0-862.14.4.el7.x86_64 		0x4a1
7
dev_queue_xmit
vmlinux 0xffffffffbe3f33a0
Build id: 3.10.0-862.14.4.el7.x86_64 		0x10
8
neigh_resolve_output
vmlinux 0xffffffffbe3fee2d
Build id: 3.10.0-862.14.4.el7.x86_64 		0x11d
9
ip_finish_output
vmlinux 0xffffffffbe43c33c
Build id: 3.10.0-862.14.4.el7.x86_64 		0x2ac
10
ip_output
vmlinux 0xffffffffbe43cb33
Build id: 3.10.0-862.14.4.el7.x86_64 		0x73
11
? __ip_append_data.isra.50
vmlinux 0xffffffffbe43c090
Build id: 3.10.0-862.14.4.el7.x86_64 		0xa50
12
ip_local_out_sk
vmlinux 0xffffffffbe43a727
Build id: 3.10.0-862.14.4.el7.x86_64 		0x37
13
ip_send_skb
vmlinux 0xffffffffbe43d586
Build id: 3.10.0-862.14.4.el7.x86_64 		0x16
14
ip_push_pending_frames
vmlinux 0xffffffffbe43d5f3
Build id: 3.10.0-862.14.4.el7.x86_64 		0x33
15
ip_send_unicast_reply
vmlinux 0xffffffffbe43d9c4
Build id: 3.10.0-862.14.4.el7.x86_64 		0x284
16
? _raw_spin_unlock_bh
vmlinux 0xffffffffbe51b5ee
Build id: 3.10.0-862.14.4.el7.x86_64 		0x1e
17
? nf_nat_setup_info
nf_nat 0xffffffffc09073b7
Build id: 3.10.0-862.14.4.el7.x86_64 		0x2f7
18
tcp_v4_send_reset
vmlinux 0xffffffffbe45bf6a
Build id: 3.10.0-862.14.4.el7.x86_64 		0x31a
19
tcp_v4_rcv
vmlinux 0xffffffffbe45d756
Build id: 3.10.0-862.14.4.el7.x86_64 		0x596
20
ip_local_deliver_finish
vmlinux 0xffffffffbe43685d
Build id: 3.10.0-862.14.4.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffffbe436b49
Build id: 3.10.0-862.14.4.el7.x86_64 		0x59
22
? ip_rcv_finish
vmlinux 0xffffffffbe4367a0
Build id: 3.10.0-862.14.4.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffffbe4364c0
Build id: 3.10.0-862.14.4.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffffbe436e79
Build id: 3.10.0-862.14.4.el7.x86_64 		0x2b9
25
? inet_del_offload
vmlinux 0xffffffffbe436430
Build id: 3.10.0-862.14.4.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffbe3f0b79
Build id: 3.10.0-862.14.4.el7.x86_64 		0x729
27
? tcp4_gro_receive
vmlinux 0xffffffffbe462a00
Build id: 3.10.0-862.14.4.el7.x86_64 		0xe0
28
? __getnstimeofday64
vmlinux 0xffffffffbdefbeff
Build id: 3.10.0-862.14.4.el7.x86_64 		0x3f
29
__netif_receive_skb
vmlinux 0xffffffffbe3f0e88
Build id: 3.10.0-862.14.4.el7.x86_64 		0x18
30
netif_receive_skb_internal
vmlinux 0xffffffffbe3f0f10
Build id: 3.10.0-862.14.4.el7.x86_64 		0x40
31
napi_gro_receive
vmlinux 0xffffffffbe3f1b38
Build id: 3.10.0-862.14.4.el7.x86_64 		0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc03ff44b
Build id: 3.10.0-862.14.4.el7.x86_64 		0x2bb
33
e1000_clean
e1000 0xffffffffc04000eb
Build id: 3.10.0-862.14.4.el7.x86_64 		0x27b
34
? try_to_wake_up
vmlinux 0xffffffffbded1d8c
Build id: 3.10.0-862.14.4.el7.x86_64 		0x18c
35
net_rx_action
vmlinux 0xffffffffbe3f152f
Build id: 3.10.0-862.14.4.el7.x86_64 		0x26f
36
__do_softirq
vmlinux 0xffffffffbde9dba5
Build id: 3.10.0-862.14.4.el7.x86_64 		0xf5
37
run_ksoftirqd
vmlinux 0xffffffffbde9dd68
Build id: 3.10.0-862.14.4.el7.x86_64 		0x38
38
smpboot_thread_fn
vmlinux 0xffffffffbdec6774
Build id: 3.10.0-862.14.4.el7.x86_64 		0x144
39
? lg_double_unlock
vmlinux 0xffffffffbdec6630
Build id: 3.10.0-862.14.4.el7.x86_64 		0x40
40
kthread
vmlinux 0xffffffffbdebdf21
Build id: 3.10.0-862.14.4.el7.x86_64 		0xd1
41
? insert_kthread_work
vmlinux 0xffffffffbdebde50
Build id: 3.10.0-862.14.4.el7.x86_64 		0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffbe5255f7
Build id: 3.10.0-862.14.4.el7.x86_64 		0x21
43
? insert_kthread_work
vmlinux 0xffffffffbdebde50
Build id: 3.10.0-862.14.4.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [ksoftirqd/1:14]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security fuse iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec aesni_intel ac97_bus snd_seq lrw gf128mul glue_helper ablk_helper cryptd
snd_seq_device ppdev snd_pcm vmw_balloon pcspkr joydev snd_timer snd vmw_vmci sg soundcore i2c_piix4 shpchp nfit libnvdimm parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 crct10dif_pclmul crct10dif_common mptspi scsi_transport_spi crc32c_intel mptscsih mptbase serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff900b371b3f40 ti: ffff900b371cc000 task.ti: ffff900b371cc000
RIP: 0010:[<ffffffffc03fe867>]  [<ffffffffc03fe867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff900b371cf4c0  EFLAGS: 00000286
RAX: ffffa466006c3818 RBX: ffff900b3283a000 RCX: 000000000000008f
RDX: 000000000000008f RSI: ffff900a355b4680 RDI: ffff900b3283a000
RBP: ffff900b371cf540 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000000
R13: ffff900b31a1f400 R14: ffff900b371cf4a4 R15: ffffffffbe3f02e6
FS:  0000000000000000(0000) GS:ffff900b39640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [kworker/2:0:113246]
CR2: 00007f6c0e047000 CR3: 00000000b6a38000 CR4: 00000000001607e0
Call Trace:
[<ffffffffc08d7056>] ? nf_ct_delete+0x176/0x270 [nf_conntrack]
[<ffffffffbdea64cb>] ? lock_timer_base.isra.34+0x2b/0x50
[<ffffffffbe3f02e6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbe41bc0a>] sch_direct_xmit+0x11a/0x250
[<ffffffffbe3f31d1>] __dev_queue_xmit+0x4a1/0x660
[<ffffffffbe3f33a0>] dev_queue_xmit+0x10/0x20
[<ffffffffbe3fee2d>] neigh_resolve_output+0x11d/0x220
[<ffffffffbe43c33c>] ip_finish_output+0x2ac/0x7a0
[<ffffffffbe43cb33>] ip_output+0x73/0xe0
[<ffffffffbe43c090>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffbe43a727>] ip_local_out_sk+0x37/0x40
[<ffffffffbe43d586>] ip_send_skb+0x16/0x50
[<ffffffffbe43d5f3>] ip_push_pending_frames+0x33/0x40
[<ffffffffbe43d9c4>] ip_send_unicast_reply+0x284/0x2d0
[<ffffffffbe51b5ee>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffc09073b7>] ? nf_nat_setup_info+0x2f7/0x320 [nf_nat]
[<ffffffffbe45bf6a>] tcp_v4_send_reset+0x31a/0x460
[<ffffffffbe45d756>] tcp_v4_rcv+0x596/0x9c0
[<ffffffffbe43685d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffbe436b49>] ip_local_deliver+0x59/0xd0
[<ffffffffbe4367a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbe4364c0>] ip_rcv_finish+0x90/0x370
[<ffffffffbe436e79>] ip_rcv+0x2b9/0x410
[<ffffffffbe436430>] ? inet_del_offload+0x40/0x40
[<ffffffffbe3f0b79>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffbe462a00>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffbdefbeff>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffbe3f0e88>] __netif_receive_skb+0x18/0x60
[<ffffffffbe3f0f10>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbe3f1b38>] napi_gro_receive+0xd8/0x100
[<ffffffffc03ff44b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04000eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffbded1d8c>] ? try_to_wake_up+0x18c/0x350
[<ffffffffbe3f152f>] net_rx_action+0x26f/0x390
[<ffffffffbde9dba5>] __do_softirq+0xf5/0x280
[<ffffffffbde9dd68>] run_ksoftirqd+0x38/0x50
[<ffffffffbdec6774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbdec6630>] ? lg_double_unlock+0x40/0x40
[<ffffffffbdebdf21>] kthread+0xd1/0xe0
[<ffffffffbdebde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffbe5255f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffbdebde50>] ? insert_kthread_work+0x40/0x40
Complete report #229573
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04ab867
Build id: 3.10.0-862.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff9ffeb166
Build id: 3.10.0-862.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffa0016a2a
Build id: 3.10.0-862.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff9ffee051
Build id: 3.10.0-862.el7.x86_64 		0x4a1
5
? nf_nat_ipv4_out
nf_nat_ipv4 0xffffffffc098d8d8
Build id: 3.10.0-862.el7.x86_64 		0x48
6
dev_queue_xmit
vmlinux 0xffffffff9ffee220
Build id: 3.10.0-862.el7.x86_64 		0x10
7
ip_finish_output
vmlinux 0xffffffffa0037416
Build id: 3.10.0-862.el7.x86_64 		0x546
8
ip_output
vmlinux 0xffffffffa0037973
Build id: 3.10.0-862.el7.x86_64 		0x73
9
? __ip_append_data.isra.50
vmlinux 0xffffffffa0036ed0
Build id: 3.10.0-862.el7.x86_64 		0xa50
10
ip_local_out_sk
vmlinux 0xffffffffa0035567
Build id: 3.10.0-862.el7.x86_64 		0x37
11
ip_send_skb
vmlinux 0xffffffffa00383c6
Build id: 3.10.0-862.el7.x86_64 		0x16
12
ip_push_pending_frames
vmlinux 0xffffffffa0038433
Build id: 3.10.0-862.el7.x86_64 		0x33
13
ip_send_unicast_reply
vmlinux 0xffffffffa0038804
Build id: 3.10.0-862.el7.x86_64 		0x284
14
? local_bh_enable
vmlinux 0xffffffff9fa99df7
Build id: 3.10.0-862.el7.x86_64 		0x17
15
? local_bh_enable
vmlinux 0xffffffff9fa99df7
Build id: 3.10.0-862.el7.x86_64 		0x17
16
? ipt_do_table
ip_tables 0xffffffffc04cca2e
Build id: 3.10.0-862.el7.x86_64 		0x33e
17
? __fib_validate_source.isra.13
vmlinux 0xffffffffa00747a5
Build id: 3.10.0-862.el7.x86_64 		0xf5
18
tcp_v4_send_reset
vmlinux 0xffffffffa0056b8a
Build id: 3.10.0-862.el7.x86_64 		0x31a
19
tcp_v4_rcv
vmlinux 0xffffffffa0058376
Build id: 3.10.0-862.el7.x86_64 		0x596
20
ip_local_deliver_finish
vmlinux 0xffffffffa0031699
Build id: 3.10.0-862.el7.x86_64 		0xb9
21
ip_local_deliver
vmlinux 0xffffffffa0031989
Build id: 3.10.0-862.el7.x86_64 		0x59
22
? ip_rcv_finish
vmlinux 0xffffffffa00315e0
Build id: 3.10.0-862.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffffa0031300
Build id: 3.10.0-862.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffffa0031cb9
Build id: 3.10.0-862.el7.x86_64 		0x2b9
25
? udp4_lib_lookup
vmlinux 0xffffffffa0063002
Build id: 3.10.0-862.el7.x86_64 		0x22
26
? inet_del_offload
vmlinux 0xffffffffa0031270
Build id: 3.10.0-862.el7.x86_64 		0x40
27
__netif_receive_skb_core
vmlinux 0xffffffff9ffeb9f9
Build id: 3.10.0-862.el7.x86_64 		0x729
28
? tcp4_gro_receive
vmlinux 0xffffffffa005d600
Build id: 3.10.0-862.el7.x86_64 		0xe0
29
? inet_gro_receive
vmlinux 0xffffffffa006edb9
Build id: 3.10.0-862.el7.x86_64 		0x249
30
__netif_receive_skb
vmlinux 0xffffffff9ffebd08
Build id: 3.10.0-862.el7.x86_64 		0x18
31
netif_receive_skb_internal
vmlinux 0xffffffff9ffebd90
Build id: 3.10.0-862.el7.x86_64 		0x40
32
napi_gro_receive
vmlinux 0xffffffff9ffec9b8
Build id: 3.10.0-862.el7.x86_64 		0xd8
33
e1000_clean_rx_irq
e1000 0xffffffffc04ac44b
Build id: 3.10.0-862.el7.x86_64 		0x2bb
34
e1000_clean
e1000 0xffffffffc04ad0eb
Build id: 3.10.0-862.el7.x86_64 		0x27b
35
? mod_timer
vmlinux 0xffffffff9faa445d
Build id: 3.10.0-862.el7.x86_64 		0x14d
36
net_rx_action
vmlinux 0xffffffff9ffec3af
Build id: 3.10.0-862.el7.x86_64 		0x26f
37
__do_softirq
vmlinux 0xffffffff9fa9a945
Build id: 3.10.0-862.el7.x86_64 		0xf5
38
run_ksoftirqd
vmlinux 0xffffffff9fa9ab08
Build id: 3.10.0-862.el7.x86_64 		0x38
39
smpboot_thread_fn
vmlinux 0xffffffff9fac35e4
Build id: 3.10.0-862.el7.x86_64 		0x144
40
? lg_double_unlock
vmlinux 0xffffffff9fac34a0
Build id: 3.10.0-862.el7.x86_64 		0x40
41
kthread
vmlinux 0xffffffff9fabae31
Build id: 3.10.0-862.el7.x86_64 		0xd1
42
? insert_kthread_work
vmlinux 0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64 		0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffffa011f637
Build id: 3.10.0-862.el7.x86_64 		0x21
44
? insert_kthread_work
vmlinux 0xffffffff9fabad60
Build id: 3.10.0-862.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:3]
Modules linked in: esn_cfs(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm coretemp snd_timer snd soundcore iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel ppdev lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_balloon parport_pc parport shpchp vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul
syscopyarea crct10dif_common sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm ata_piix drm mptspi serio_raw scsi_transport_spi mptscsih e1000 mptbase libata i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2012
task: ffff8870bb5a1fa0 ti: ffff8870bb600000 task.ti: ffff8870bb600000
RIP: 0010:[<ffffffffc04ab867>]  [<ffffffffc04ab867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8870bb603500  EFLAGS: 00000286
RAX: ffffb426003c3818 RBX: ffff88707516d100 RCX: 000000000000002a
RDX: 000000000000002a RSI: ffff88707516d100 RDI: ffff8870b9f70000
RBP: ffff8870bb603580 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: 00000000000000fc
R13: ffffb426003c3818 R14: 0000000000000100 R15: 0000000002000000
FS:  0000000000000000(0000) GS:ffff8870bc600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f52bf812310 CR3: 00000000796d2000 CR4: 00000000001607f0
Call Trace:
[<ffffffff9ffeb166>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa0016a2a>] sch_direct_xmit+0x11a/0x250
[<ffffffff9ffee051>] __dev_queue_xmit+0x4a1/0x660
[<ffffffffc098d8d8>] ? nf_nat_ipv4_out+0x48/0xf0 [nf_nat_ipv4]
[<ffffffff9ffee220>] dev_queue_xmit+0x10/0x20
[<ffffffffa0037416>] ip_finish_output+0x546/0x7a0
[<ffffffffa0037973>] ip_output+0x73/0xe0
[<ffffffffa0036ed0>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffa0035567>] ip_local_out_sk+0x37/0x40
[<ffffffffa00383c6>] ip_send_skb+0x16/0x50
[<ffffffffa0038433>] ip_push_pending_frames+0x33/0x40
[<ffffffffa0038804>] ip_send_unicast_reply+0x284/0x2d0
[<ffffffff9fa99df7>] ? local_bh_enable+0x17/0x20
[<ffffffff9fa99df7>] ? local_bh_enable+0x17/0x20
[<ffffffffc04cca2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffa00747a5>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffffa0056b8a>] tcp_v4_send_reset+0x31a/0x460
[<ffffffffa0058376>] tcp_v4_rcv+0x596/0x990
[<ffffffffa0031699>] ip_local_deliver_finish+0xb9/0x200
[<ffffffffa0031989>] ip_local_deliver+0x59/0xd0
[<ffffffffa00315e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa0031300>] ip_rcv_finish+0x90/0x370
[<ffffffffa0031cb9>] ip_rcv+0x2b9/0x410
[<ffffffffa0063002>] ? udp4_lib_lookup+0x22/0x70
[<ffffffffa0031270>] ? inet_del_offload+0x40/0x40
[<ffffffff9ffeb9f9>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffa005d600>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffa006edb9>] ? inet_gro_receive+0x249/0x290
[<ffffffff9ffebd08>] __netif_receive_skb+0x18/0x60
[<ffffffff9ffebd90>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9ffec9b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc04ac44b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04ad0eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffff9faa445d>] ? mod_timer+0x14d/0x230
[<ffffffff9ffec3af>] net_rx_action+0x26f/0x390
[<ffffffff9fa9a945>] __do_softirq+0xf5/0x280
[<ffffffff9fa9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffff9fac35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9fac34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9fabae31>] kthread+0xd1/0xe0
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa011f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9fabad60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00