Info

Permalink
Function
e1000_xmit_frame
First occurrence
2020-11-03
Last occurrence
2025-08-06
State
NEW
Components
kernel
Quality
-89

Statistics

Operating system Count
CentOS 7 87
Architecture Count
x86_64 87
Related packages Count
kernel 87
 0:3.10.0-1160.41.1.el7 51
 0:3.10.0-693.el7 6
 0:3.10.0-1160.el7 5
 0:3.10.0-1160.102.1.el7 4
 0:3.10.0-1160.11.1.el7 4
 0:3.10.0-1127.el7 2
 0:3.10.0-1160.71.1.el7 2
 0:3.10.0-1160.66.1.el7 2
 0:3.10.0-1062.4.1.el7 1
 0:3.10.0-1062.el7 1
 0:3.10.0-1160.105.1.el7 1
 0:3.10.0-1160.59.1.el7 1
 0:3.10.0-1160.76.1.el7 1
 0:3.10.0-1160.81.1.el7 1
 0:3.10.0-1160.45.1.el7 1
 0:3.10.0-957.el7 1
 0:3.10.0-1160.88.1.el7 1
 0:3.10.0-862.el7 1
 0:3.10.0-1160.90.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1016791
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0065877
Build id: 3.10.0-1160.el7.x86_64 		0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffab43c6f7
Build id: 3.10.0-1160.el7.x86_64 		0x37
3
dev_hard_start_xmit
vmlinux 0xffffffffab454386
Build id: 3.10.0-1160.el7.x86_64 		0x246
4
sch_direct_xmit
vmlinux 0xffffffffab48242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
5
__dev_queue_xmit
vmlinux 0xffffffffab457301
Build id: 3.10.0-1160.el7.x86_64 		0x491
6
dev_queue_xmit
vmlinux 0xffffffffab4574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
7
ip_finish_output
vmlinux 0xffffffffab4a545b
Build id: 3.10.0-1160.el7.x86_64 		0x67b
8
ip_output
vmlinux 0xffffffffab4a59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffffab4a4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
10
ip_local_out_sk
vmlinux 0xffffffffab4a3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
11
ip_queue_xmit
vmlinux 0xffffffffab4a37a4
Build id: 3.10.0-1160.el7.x86_64 		0x144
12
tcp_transmit_skb
vmlinux 0xffffffffab4bde94
Build id: 3.10.0-1160.el7.x86_64 		0x4e4
13
tcp_send_ack
vmlinux 0xffffffffab4c00af
Build id: 3.10.0-1160.el7.x86_64 		0x11f
14
__tcp_ack_snd_check
vmlinux 0xffffffffab4b4c7b
Build id: 3.10.0-1160.el7.x86_64 		0x6b
15
tcp_rcv_established
vmlinux 0xffffffffab4ba957
Build id: 3.10.0-1160.el7.x86_64 		0x417
16
tcp_v4_do_rcv
vmlinux 0xffffffffab4c57aa
Build id: 3.10.0-1160.el7.x86_64 		0x10a
17
tcp_v4_rcv
vmlinux 0xffffffffab4c6ffc
Build id: 3.10.0-1160.el7.x86_64 		0x7dc
18
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc06f92ce
Build id: 3.10.0-1160.el7.x86_64 		0x4e
19
ip_local_deliver_finish
vmlinux 0xffffffffab49f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
20
ip_local_deliver
vmlinux 0xffffffffab49f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
21
? ip_rcv_finish
vmlinux 0xffffffffab49f1d0
Build id: 3.10.0-1160.el7.x86_64 		0x370
22
ip_rcv_finish
vmlinux 0xffffffffab49eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
23
ip_rcv
vmlinux 0xffffffffab49f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
24
? packet_rcv
vmlinux 0xffffffffab55f902
Build id: 3.10.0-1160.el7.x86_64 		0x32
25
? inet_del_offload
vmlinux 0xffffffffab49ee60
Build id: 3.10.0-1160.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffab454c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
27
? __getnstimeofday64
vmlinux 0xffffffffaaf078df
Build id: 3.10.0-1160.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffab454f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffab454fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffab455c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc006645b
Build id: 3.10.0-1160.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc0067100
Build id: 3.10.0-1160.el7.x86_64 		0x280
33
? mod_timer
vmlinux 0xffffffffaaeaeb7c
Build id: 3.10.0-1160.el7.x86_64 		0x10c
34
net_rx_action
vmlinux 0xffffffffab4555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
35
__do_softirq
vmlinux 0xffffffffaaea4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
36
run_ksoftirqd
vmlinux 0xffffffffaaea4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
37
smpboot_thread_fn
vmlinux 0xffffffffaaece674
Build id: 3.10.0-1160.el7.x86_64 		0x144
38
? lg_double_unlock
vmlinux 0xffffffffaaece530
Build id: 3.10.0-1160.el7.x86_64 		0x40
39
kthread
vmlinux 0xffffffffaaec5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
40
? insert_kthread_work
vmlinux 0xffffffffaaec5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffffab593ddd
Build id: 3.10.0-1160.el7.x86_64 		0x7
42
? insert_kthread_work
vmlinux 0xffffffffaaec5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ksoftirqd/4:29]
Modules linked in: ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt rpcrdma ib_ipoib ib_iser libiscsi scsi_transport_iscsi ib_umad rdma_rxe ip6_udp_tunnel udp_tunnel rdma_ucm rdma_cm iw_cm ib_cm ib_uverbs ib_core mmfs26(OE) mmfslinux(OE) tracedev(OE) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul snd_rawmidi snd_ac97_codec glue_helper ablk_helper
ac97_bus snd_seq snd_seq_device snd_pcm btusb btrtl btbcm btintel bluetooth cryptd snd_timer rfkill snd pcspkr ppdev soundcore joydev vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix nfit mptspi scsi_transport_spi libnvdimm mptscsih crct10dif_pclmul crct10dif_common libata mptbase e1000 crc32c_intel serio_raw drm_panel_orientation_quirks fuse
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8b0ff1389080 ti: ffff8b0ff1398000 task.ti: ffff8b0ff1398000
RIP: 0010:[<ffffffffc0065877>]  [<ffffffffc0065877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8b0ff139b550  EFLAGS: 00000286
RAX: ffffb53c40f83818 RBX: 0000000000000001 RCX: 0000000000000003
RDX: 0000000000000003 RSI: ffff8b0eb5f31140 RDI: ffff8b0ff7fb0000
RBP: ffff8b0ff139b5d0 R08: 0000000000000000 R09: 0000000022100000
R10: c269280008450008 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000232bf80da R14: 00000001b5f31140 R15: ffffffffab1b1650
FS:  0000000000000000(0000) GS:ffff8b10b5700000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000a5f0d0d0000 CR3: 00000001763fe000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffab43c6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffffab454386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffab48242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffab457301>] __dev_queue_xmit+0x491/0x650
[<ffffffffab4574d0>] dev_queue_xmit+0x10/0x20
[<ffffffffab4a545b>] ip_finish_output+0x67b/0x8d0
[<ffffffffab4a59db>] ip_output+0x7b/0xf0
[<ffffffffab4a4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffab4a3407>] ip_local_out_sk+0x37/0x40
[<ffffffffab4a37a4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffab4bde94>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffab4c00af>] tcp_send_ack+0x11f/0x170
[<ffffffffab4b4c7b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffab4ba957>] tcp_rcv_established+0x417/0x770
[<ffffffffab4c57aa>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffab4c6ffc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc06f92ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffab49f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffab49f580>] ip_local_deliver+0x60/0xe0
[<ffffffffab49f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffab49eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffab49f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffab55f902>] ? packet_rcv+0x32/0x430
[<ffffffffab49ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffab454c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffaaf078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffab454f18>] __netif_receive_skb+0x18/0x60
[<ffffffffab454fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffab455c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc006645b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0067100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffaaeaeb7c>] ? mod_timer+0x10c/0x230
[<ffffffffab4555bf>] net_rx_action+0x26f/0x390
[<ffffffffaaea4b95>] __do_softirq+0xf5/0x280
[<ffffffffaaea4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffaaece674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffaaece530>] ? lg_double_unlock+0x40/0x40
[<ffffffffaaec5c21>] kthread+0xd1/0xe0
[<ffffffffaaec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffab593ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffaaec5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #733155
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc01fd8e7
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff9147edc6
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff914add72
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x122
4
__dev_queue_xmit
vmlinux 0xffffffff91481dd1
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x491
5
? nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc075e7de
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x1de
6
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc076d020
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x20
7
dev_queue_xmit
vmlinux 0xffffffff91481fa0
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x10
8
neigh_resolve_output
vmlinux 0xffffffff9148f03d
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x11d
9
ip_finish_output
vmlinux 0xffffffff914d220c
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x27c
10
ip_output
vmlinux 0xffffffff914d2b9b
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x7b
11
? __ip_append_data.isra.50
vmlinux 0xffffffff914d1f90
Build id: 3.10.0-1160.90.1.el7.x86_64 		0xa60
12
ip_local_out_sk
vmlinux 0xffffffff914d0587
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x37
13
ip_queue_xmit
vmlinux 0xffffffff914d0924
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x144
14
tcp_transmit_skb
vmlinux 0xffffffff914eb5f4
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x4e4
15
tcp_send_ack
vmlinux 0xffffffff914ed83f
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x11f
16
__tcp_ack_snd_check
vmlinux 0xffffffff914e1c6b
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x6b
17
tcp_rcv_established
vmlinux 0xffffffff914e7dc7
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x267
18
tcp_v4_do_rcv
vmlinux 0xffffffff914f30f2
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x112
19
tcp_v4_rcv
vmlinux 0xffffffff914f498c
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x7dc
20
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc07672ee
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x4e
21
ip_local_deliver_finish
vmlinux 0xffffffff914cc34d
Build id: 3.10.0-1160.90.1.el7.x86_64 		0xbd
22
ip_local_deliver
vmlinux 0xffffffff914cc660
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x60
23
? ip_rcv_finish
vmlinux 0xffffffff914cc290
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x380
24
ip_rcv_finish
vmlinux 0xffffffff914cbfa0
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x90
25
ip_rcv
vmlinux 0xffffffff914cc9a0
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x2c0
26
? packet_rcv
vmlinux 0xffffffff9158b402
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x252
27
? inet_del_offload
vmlinux 0xffffffff914cbf10
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x40
28
__netif_receive_skb_core
vmlinux 0xffffffff9147f669
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x739
29
? tcp4_gro_receive
vmlinux 0xffffffff914f9b00
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x30
30
? __getnstimeofday64
vmlinux 0xffffffff90f0ea2f
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x3f
31
__netif_receive_skb
vmlinux 0xffffffff9147f968
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x18
32
netif_receive_skb_internal
vmlinux 0xffffffff9147fa00
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x40
33
napi_gro_receive
vmlinux 0xffffffff914806b8
Build id: 3.10.0-1160.90.1.el7.x86_64 		0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc01fe4cb
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x2bb
35
e1000_clean
e1000 0xffffffffc01ff180
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x280
36
net_rx_action
vmlinux 0xffffffff91480037
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x277
37
__do_softirq
vmlinux 0xffffffff90ea9595
Build id: 3.10.0-1160.90.1.el7.x86_64 		0xf5
38
run_ksoftirqd
vmlinux 0xffffffff90ea9768
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x38
39
smpboot_thread_fn
vmlinux 0xffffffff90ed43fc
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x14c
40
? lg_double_unlock
vmlinux 0xffffffff90ed42b0
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x40
41
kthread
vmlinux 0xffffffff90ecb621
Build id: 3.10.0-1160.90.1.el7.x86_64 		0xd1
42
? insert_kthread_work
vmlinux 0xffffffff90ecb550
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffff915c51dd
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x7
44
? insert_kthread_work
vmlinux 0xffffffff90ecb550
Build id: 3.10.0-1160.90.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink iscsi_target_mod target_core_mod sfs(OE) nbd(E) iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd crc32_pclmul soundcore ghash_clmulni_intel
aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg vmw_balloon pcspkr joydev parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata nfit crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel mptspi scsi_transport_spi mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.90.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9d6a793ad280 ti: ffff9d6a78ce0000 task.ti: ffff9d6a78ce0000
RIP: 0010:[<ffffffffc01fd8e7>]  [<ffffffffc01fd8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9d6a78ce3510  EFLAGS: 00000286
RAX: ffffae34c1983818 RBX: 0000000000000001 RCX: 0000000000000059
RDX: 0000000000000059 RSI: ffff9d6935da6140 RDI: ffff9d6bafd06000
RBP: ffff9d6a78ce3590 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000029323c8c2 R14: 00000001ab4d7c00 R15: ffffffff911c8fd0
FS:  0000000000000000(0000) GS:ffff9d6bb1200000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f098133f000 CR3: 00000000ac398000 CR4: 0000000000760ff0
PKRU: 00000000
Call Trace:
[<ffffffff9147edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff914add72>] sch_direct_xmit+0x122/0x250
[<ffffffff91481dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffc075e7de>] ? nf_nat_ipv4_fn+0x1de/0x230 [nf_nat_ipv4]
[<ffffffffc076d020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffff91481fa0>] dev_queue_xmit+0x10/0x20
[<ffffffff9148f03d>] neigh_resolve_output+0x11d/0x220
[<ffffffff914d220c>] ip_finish_output+0x27c/0x8d0
[<ffffffff914d2b9b>] ip_output+0x7b/0x100
[<ffffffff914d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff914d0587>] ip_local_out_sk+0x37/0x40
[<ffffffff914d0924>] ip_queue_xmit+0x144/0x3d0
[<ffffffff914eb5f4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff914ed83f>] tcp_send_ack+0x11f/0x180
[<ffffffff914e1c6b>] __tcp_ack_snd_check+0x6b/0xa0
[<ffffffff914e7dc7>] tcp_rcv_established+0x267/0x7a0
[<ffffffff914f30f2>] tcp_v4_do_rcv+0x112/0x360
[<ffffffff914f498c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc07672ee>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff914cc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffff914cc660>] ip_local_deliver+0x60/0xe0
[<ffffffff914cc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffff914cbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffff914cc9a0>] ip_rcv+0x2c0/0x420
[<ffffffff9158b402>] ? packet_rcv+0x252/0x440
[<ffffffff914cbf10>] ? inet_del_offload+0x40/0x40
[<ffffffff9147f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffff914f9b00>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffff90f0ea2f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9147f968>] __netif_receive_skb+0x18/0x70
[<ffffffff9147fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff914806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc01fe4cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01ff180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff91480037>] net_rx_action+0x277/0x3a0
[<ffffffff90ea9595>] __do_softirq+0xf5/0x290
[<ffffffff90ea9768>] run_ksoftirqd+0x38/0x60
[<ffffffff90ed43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffff90ed42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffff90ecb621>] kthread+0xd1/0xe0
[<ffffffff90ecb550>] ? insert_kthread_work+0x40/0x40
[<ffffffff915c51dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff90ecb550>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #672638
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc011a877
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb5a53f96
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb5a8203a
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb5a56f11
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffb5a570e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffb5aa507b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffffb5aa55fb
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb5aa4a00
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb5aa3027
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x37
10
ip_queue_xmit
vmlinux 0xffffffffb5aa33c4
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x144
11
tcp_transmit_skb
vmlinux 0xffffffffb5abdad4
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x4e4
12
tcp_send_ack
vmlinux 0xffffffffb5abfcef
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x11f
13
__tcp_ack_snd_check
vmlinux 0xffffffffb5ab489b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x6b
14
tcp_rcv_established
vmlinux 0xffffffffb5aba3c4
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x264
15
tcp_v4_do_rcv
vmlinux 0xffffffffb5ac53ea
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x10a
16
tcp_v4_rcv
vmlinux 0xffffffffb5ac6c3c
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x7dc
17
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc05e82ce
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x4e
18
ip_local_deliver_finish
vmlinux 0xffffffffb5a9eead
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffffb5a9f1a0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x60
20
? ip_rcv_finish
vmlinux 0xffffffffb5a9edf0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffffb5a9eb10
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffb5a9f4e0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2c0
23
? inet_del_offload
vmlinux 0xffffffffb5a9ea80
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
24
__netif_receive_skb_core
vmlinux 0xffffffffb5a54829
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x729
25
? tcp4_gro_receive
vmlinux 0xffffffffb5acbd00
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
26
? __getnstimeofday64
vmlinux 0xffffffffb5507b7f
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x3f
27
__netif_receive_skb
vmlinux 0xffffffffb5a54b28
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x18
28
netif_receive_skb_internal
vmlinux 0xffffffffb5a54bb0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
29
napi_gro_receive
vmlinux 0xffffffffb5a55838
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xd8
30
e1000_clean_rx_irq
e1000 0xffffffffc011b45b
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x2bb
31
e1000_clean
e1000 0xffffffffc011c100
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x280
32
? update_curr
vmlinux 0xffffffffb54e466c
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x14c
33
net_rx_action
vmlinux 0xffffffffb5a551cf
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x26f
34
__do_softirq
vmlinux 0xffffffffb54a4b95
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffb54a4d58
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffb54ce8c4
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x144
37
? lg_double_unlock
vmlinux 0xffffffffb54ce780
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffb54c5e71
Build id: 3.10.0-1160.11.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffb54c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffb5b93df7
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x21
41
? insert_kthread_work
vmlinux 0xffffffffb54c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: tcp_lp xt_nat veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi sb_edac snd_ac97_codec ppdev iosf_mbi ac97_bus crc32_pclmul snd_seq ghash_clmulni_intel snd_seq_device aesni_intel snd_pcm vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd snd_timer snd joydev pcspkr sg soundcore vmw_vmci i2c_piix4 parport_pc parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif
crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix drm crc32c_intel mptspi nfit scsi_transport_spi libata mptscsih libnvdimm e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff8b313936d280 ti: ffff8b313a760000 task.ti: ffff8b313a760000
RIP: 0010:[<ffffffffc011a877>]  [<ffffffffc011a877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8b313a763550  EFLAGS: 00000286
RAX: ffffb1e041943818 RBX: ffffffffb6504bc0 RCX: 0000000000000070
RDX: 0000000000000070 RSI: ffff8b33ebe91140 RDI: ffff8b3127fb6000
RBP: ffff8b313a7635d0 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff8b313a763598
R13: 0000000000001018 R14: ffff8b3127fb6000 R15: 0000000000001018
FS:  0000000000000000(0000) GS:ffff8b33ed600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000021a1028 CR3: 000000013977c000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb5a53f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb5a8203a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb5a56f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb5a570e0>] dev_queue_xmit+0x10/0x20
[<ffffffffb5aa507b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb5aa55fb>] ip_output+0x7b/0xf0
[<ffffffffb5aa4a00>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb5aa3027>] ip_local_out_sk+0x37/0x40
[<ffffffffb5aa33c4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb5abdad4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb5abfcef>] tcp_send_ack+0x11f/0x170
[<ffffffffb5ab489b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffb5aba3c4>] tcp_rcv_established+0x264/0x770
[<ffffffffb5ac53ea>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffb5ac6c3c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc05e82ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffb5a9eead>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb5a9f1a0>] ip_local_deliver+0x60/0xe0
[<ffffffffb5a9edf0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb5a9eb10>] ip_rcv_finish+0x90/0x370
[<ffffffffb5a9f4e0>] ip_rcv+0x2c0/0x420
[<ffffffffb5a9ea80>] ? inet_del_offload+0x40/0x40
[<ffffffffb5a54829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb5acbd00>] ? tcp4_gro_receive+0x40/0x1a0
[<ffffffffb5507b7f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb5a54b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb5a54bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb5a55838>] napi_gro_receive+0xd8/0x100
[<ffffffffc011b45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc011c100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb54e466c>] ? update_curr+0x14c/0x1e0
[<ffffffffb5a551cf>] net_rx_action+0x26f/0x390
[<ffffffffb54a4b95>] __do_softirq+0xf5/0x280
[<ffffffffb54a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffb54ce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb54ce780>] ? lg_double_unlock+0x40/0x40
[<ffffffffb54c5e71>] kthread+0xd1/0xe0
[<ffffffffb54c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb5b93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb54c5da0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #611373
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0365857
Build id: 3.10.0-957.el7.x86_64 		0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffa6e21697
Build id: 3.10.0-957.el7.x86_64 		0x37
3
dev_hard_start_xmit
vmlinux 0xffffffffa6e38c66
Build id: 3.10.0-957.el7.x86_64 		0x246
4
sch_direct_xmit
vmlinux 0xffffffffa6e65cea
Build id: 3.10.0-957.el7.x86_64 		0x11a
5
__dev_queue_xmit
vmlinux 0xffffffffa6e3bbe1
Build id: 3.10.0-957.el7.x86_64 		0x491
6
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc082a2ce
Build id: 3.10.0-957.el7.x86_64 		0x4e
7
dev_queue_xmit
vmlinux 0xffffffffa6e3bdb0
Build id: 3.10.0-957.el7.x86_64 		0x10
8
ip_finish_output
vmlinux 0xffffffffa6e88736
Build id: 3.10.0-957.el7.x86_64 		0x556
9
ip_output
vmlinux 0xffffffffa6e88c93
Build id: 3.10.0-957.el7.x86_64 		0x73
10
? __ip_append_data.isra.51
vmlinux 0xffffffffa6e881e0
Build id: 3.10.0-957.el7.x86_64 		0xa50
11
ip_local_out_sk
vmlinux 0xffffffffa6e86817
Build id: 3.10.0-957.el7.x86_64 		0x37
12
ip_queue_xmit
vmlinux 0xffffffffa6e86bb4
Build id: 3.10.0-957.el7.x86_64 		0x144
13
tcp_transmit_skb
vmlinux 0xffffffffa6ea0f64
Build id: 3.10.0-957.el7.x86_64 		0x4e4
14
tcp_send_ack
vmlinux 0xffffffffa6ea2fef
Build id: 3.10.0-957.el7.x86_64 		0x11f
15
__tcp_ack_snd_check
vmlinux 0xffffffffa6e9796b
Build id: 3.10.0-957.el7.x86_64 		0x6b
16
tcp_rcv_established
vmlinux 0xffffffffa6e9daf7
Build id: 3.10.0-957.el7.x86_64 		0x417
17
tcp_v4_do_rcv
vmlinux 0xffffffffa6ea86da
Build id: 3.10.0-957.el7.x86_64 		0x10a
18
tcp_v4_rcv
vmlinux 0xffffffffa6ea9dcc
Build id: 3.10.0-957.el7.x86_64 		0x7bc
19
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc082a2ce
Build id: 3.10.0-957.el7.x86_64 		0x4e
20
ip_local_deliver_finish
vmlinux 0xffffffffa6e8285d
Build id: 3.10.0-957.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffffa6e82b49
Build id: 3.10.0-957.el7.x86_64 		0x59
22
? ip_rcv_finish
vmlinux 0xffffffffa6e827a0
Build id: 3.10.0-957.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffffa6e824c0
Build id: 3.10.0-957.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffffa6e82e79
Build id: 3.10.0-957.el7.x86_64 		0x2b9
25
? inet_del_offload
vmlinux 0xffffffffa6e82430
Build id: 3.10.0-957.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffa6e394f9
Build id: 3.10.0-957.el7.x86_64 		0x729
27
? __getnstimeofday64
vmlinux 0xffffffffa69024ef
Build id: 3.10.0-957.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffa6e397f8
Build id: 3.10.0-957.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffa6e39880
Build id: 3.10.0-957.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffa6e3a508
Build id: 3.10.0-957.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc036643b
Build id: 3.10.0-957.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc03670dd
Build id: 3.10.0-957.el7.x86_64 		0x27d
33
? update_curr
vmlinux 0xffffffffa68e00bc
Build id: 3.10.0-957.el7.x86_64 		0x14c
34
? __schedule
vmlinux 0xffffffffa6f678fa
Build id: 3.10.0-957.el7.x86_64 		0x5ea
35
net_rx_action
vmlinux 0xffffffffa6e39e9f
Build id: 3.10.0-957.el7.x86_64 		0x26f
36
__do_softirq
vmlinux 0xffffffffa68a0f05
Build id: 3.10.0-957.el7.x86_64 		0xf5
37
run_ksoftirqd
vmlinux 0xffffffffa68a10c8
Build id: 3.10.0-957.el7.x86_64 		0x38
38
smpboot_thread_fn
vmlinux 0xffffffffa68ca5c4
Build id: 3.10.0-957.el7.x86_64 		0x144
39
? lg_double_unlock
vmlinux 0xffffffffa68ca480
Build id: 3.10.0-957.el7.x86_64 		0x40
40
kthread
vmlinux 0xffffffffa68c1c31
Build id: 3.10.0-957.el7.x86_64 		0xd1
41
? insert_kthread_work
vmlinux 0xffffffffa68c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffa6f74c1d
Build id: 3.10.0-957.el7.x86_64 		0x7
43
? insert_kthread_work
vmlinux 0xffffffffa68c1b60
Build id: 3.10.0-957.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 24s! [ksoftirqd/1:14]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ppdev iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_midi snd_seq_midi_event joydev pcspkr snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus
snd_seq snd_seq_device snd_pcm snd_timer snd soundcore sg vmw_vmci parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm e1000 ata_piix mptspi libata scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8f5d771e0000 ti: ffff8f5d771e8000 task.ti: ffff8f5d771e8000
RIP: 0010:[<ffffffffc0365857>]  [<ffffffffc0365857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff8f5d771eb580  EFLAGS: 00010286
RAX: ffff9e9a006c3818 RBX: ffff8f5d79653900 RCX: 0000000000000085
RDX: 0000000000000085 RSI: ffff8f5c75e04cc0 RDI: ffff8f5d755fc000
RBP: ffff8f5d771eb600 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff8f5d771eb5c8
R13: ffffffffa6ba0f58 R14: ffffffffffffff10 R15: ffff8f5d76952098
FS:  0000000000000000(0000) GS:ffff8f5d79640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd87d905860 CR3: 00000001338f6000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa6e21697>] ? kfree_skbmem+0x37/0x90
[<ffffffffa6e38c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa6e65cea>] sch_direct_xmit+0x11a/0x250
[<ffffffffa6e3bbe1>] __dev_queue_xmit+0x491/0x650
[<ffffffffc082a2ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffa6e3bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffffa6e88736>] ip_finish_output+0x556/0x7b0
[<ffffffffa6e88c93>] ip_output+0x73/0xe0
[<ffffffffa6e881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffffa6e86817>] ip_local_out_sk+0x37/0x40
[<ffffffffa6e86bb4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffa6ea0f64>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffa6ea2fef>] tcp_send_ack+0x11f/0x170
[<ffffffffa6e9796b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffa6e9daf7>] tcp_rcv_established+0x417/0x770
[<ffffffffa6ea86da>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffa6ea9dcc>] tcp_v4_rcv+0x7bc/0x9c0
[<ffffffffc082a2ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffa6e8285d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa6e82b49>] ip_local_deliver+0x59/0xd0
[<ffffffffa6e827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa6e824c0>] ip_rcv_finish+0x90/0x370
[<ffffffffa6e82e79>] ip_rcv+0x2b9/0x410
[<ffffffffa6e82430>] ? inet_del_offload+0x40/0x40
[<ffffffffa6e394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa69024ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa6e397f8>] __netif_receive_skb+0x18/0x60
[<ffffffffa6e39880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa6e3a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc036643b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03670dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffffa68e00bc>] ? update_curr+0x14c/0x1e0
[<ffffffffa6f678fa>] ? __schedule+0x5ea/0x890
[<ffffffffa6e39e9f>] net_rx_action+0x26f/0x390
[<ffffffffa68a0f05>] __do_softirq+0xf5/0x280
[<ffffffffa68a10c8>] run_ksoftirqd+0x38/0x50
[<ffffffffa68ca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa68ca480>] ? lg_double_unlock+0x40/0x40
[<ffffffffa68c1c31>] kthread+0xd1/0xe0
[<ffffffffa68c1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffffa6f74c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa68c1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #544083
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02db877
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffa8458f86
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffa848709a
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffa845bf01
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffa845c0d0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x10
6
neigh_resolve_output
vmlinux 0xffffffffa8468dfd
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x11d
7
ip_finish_output
vmlinux 0xffffffffa84aa85c
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x27c
8
ip_output
vmlinux 0xffffffffa84ab1db
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffffa84aa5e0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xa60
10
ip_local_out_sk
vmlinux 0xffffffffa84a8c07
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x37
11
ip_queue_xmit
vmlinux 0xffffffffa84a8fa4
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x144
12
tcp_transmit_skb
vmlinux 0xffffffffa84c3734
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x4e4
13
tcp_send_ack
vmlinux 0xffffffffa84c594f
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x11f
14
__tcp_ack_snd_check
vmlinux 0xffffffffa84b9f3b
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x6b
15
tcp_rcv_established
vmlinux 0xffffffffa84bfff7
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x267
16
tcp_v4_do_rcv
vmlinux 0xffffffffa84cb04a
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x10a
17
tcp_v4_rcv
vmlinux 0xffffffffa84cc89c
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x7dc
18
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc07e42ce
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x4e
19
ip_local_deliver_finish
vmlinux 0xffffffffa84a4a8d
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xbd
20
ip_local_deliver
vmlinux 0xffffffffa84a4d80
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x60
21
? ip_rcv_finish
vmlinux 0xffffffffa84a49d0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x370
22
ip_rcv_finish
vmlinux 0xffffffffa84a46f0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x90
23
ip_rcv
vmlinux 0xffffffffa84a50c0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x2c0
24
? inet_del_offload
vmlinux 0xffffffffa84a4660
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffa8459819
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffffa84d1a00
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xe0
27
? __getnstimeofday64
vmlinux 0xffffffffa7f0803f
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffa8459b18
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffa8459ba0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffa845a828
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc02dc45b
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc02dd100
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x280
33
net_rx_action
vmlinux 0xffffffffa845a1bf
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x26f
34
__do_softirq
vmlinux 0xffffffffa7ea4d85
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffa7ea4f48
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffa7ece9e4
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x144
37
? lg_double_unlock
vmlinux 0xffffffffa7ece8a0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffa7ec5f91
Build id: 3.10.0-1160.71.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffa7ec5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffa8599ddd
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x7
41
? insert_kthread_work
vmlinux 0xffffffffa7ec5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [ksoftirqd/7:44]
Modules linked in: nls_utf8 isofs veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec lrw gf128mul glue_helper ac97_bus ablk_helper snd_seq ppdev vmw_balloon cryptd snd_seq_device joydev pcspkr snd_pcm snd_timer snd soundcore sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c
sr_mod cdrom sd_mod crc_t10dif crct10dif_generic vmwgfx ata_generic pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi drm ahci scsi_transport_spi nfit libahci crct10dif_pclmul ata_piix crct10dif_common libata drm_panel_orientation_quirks libnvdimm crc32c_intel mptscsih e1000 serio_raw mptbase dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff97d4f9b82100 ti: ffff97d4f9b94000 task.ti: ffff97d4f9b94000
RIP: 0010:[<ffffffffc02db877>]  [<ffffffffc02db877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff97d4f9b97510  EFLAGS: 00010286
RAX: ffffaea5403c3818 RBX: ffff97d4f9099098 RCX: 00000000000000e9
RDX: 00000000000000e9 RSI: ffff97d4f44cfec0 RDI: ffff97d4f7e84000
RBP: ffff97d4f9b97590 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffc07c1fad
R13: ffff97d4f9b97550 R14: ffff97d4f9b97500 R15: 0000000000001680
FS:  0000000000000000(0000) GS:ffff97d4fb7c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0000f5010 CR3: 0000000015e10000 CR4: 00000000003607e0
Call Trace:
[<ffffffffa8458f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa848709a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa845bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffffa845c0d0>] dev_queue_xmit+0x10/0x20
[<ffffffffa8468dfd>] neigh_resolve_output+0x11d/0x220
[<ffffffffa84aa85c>] ip_finish_output+0x27c/0x8d0
[<ffffffffa84ab1db>] ip_output+0x7b/0xf0
[<ffffffffa84aa5e0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa84a8c07>] ip_local_out_sk+0x37/0x40
[<ffffffffa84a8fa4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffa84c3734>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffa84c594f>] tcp_send_ack+0x11f/0x170
[<ffffffffa84b9f3b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffa84bfff7>] tcp_rcv_established+0x267/0x7a0
[<ffffffffa84cb04a>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffa84cc89c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc07e42ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffa84a4a8d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa84a4d80>] ip_local_deliver+0x60/0xe0
[<ffffffffa84a49d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa84a46f0>] ip_rcv_finish+0x90/0x370
[<ffffffffa84a50c0>] ip_rcv+0x2c0/0x420
[<ffffffffa84a4660>] ? inet_del_offload+0x40/0x40
[<ffffffffa8459819>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa84d1a00>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffa7f0803f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa8459b18>] __netif_receive_skb+0x18/0x60
[<ffffffffa8459ba0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa845a828>] napi_gro_receive+0xd8/0x100
[<ffffffffc02dc45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02dd100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa845a1bf>] net_rx_action+0x26f/0x390
[<ffffffffa7ea4d85>] __do_softirq+0xf5/0x280
[<ffffffffa7ea4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffa7ece9e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa7ece8a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa7ec5f91>] kthread+0xd1/0xe0
[<ffffffffa7ec5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa8599ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa7ec5ec0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #371322
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0228877
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb8c55f96
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb8c840aa
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb8c58f11
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x491
5
dev_queue_xmit
vmlinux 0xffffffffb8c590e0
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x10
6
ip_finish_output
vmlinux 0xffffffffb8ca7c6b
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x67b
7
ip_output
vmlinux 0xffffffffb8ca81eb
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb8ca75f0
Build id: 3.10.0-1160.41.1.el7.x86_64 		0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb8ca5c17
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x37
10
ip_queue_xmit
vmlinux 0xffffffffb8ca5fb4
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x144
11
tcp_transmit_skb
vmlinux 0xffffffffb8cc06f4
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x4e4
12
tcp_send_ack
vmlinux 0xffffffffb8cc290f
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x11f
13
__tcp_ack_snd_check
vmlinux 0xffffffffb8cb6f3b
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x6b
14
tcp_rcv_established
vmlinux 0xffffffffb8cbcfb7
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x267
15
tcp_v4_do_rcv
vmlinux 0xffffffffb8cc800a
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x10a
16
tcp_v4_rcv
vmlinux 0xffffffffb8cc985c
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x7dc
17
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc08382ce
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x4e
18
ip_local_deliver_finish
vmlinux 0xffffffffb8ca1a9d
Build id: 3.10.0-1160.41.1.el7.x86_64 		0xbd
19
ip_local_deliver
vmlinux 0xffffffffb8ca1d90
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x60
20
? ip_rcv_finish
vmlinux 0xffffffffb8ca19e0
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x370
21
ip_rcv_finish
vmlinux 0xffffffffb8ca1700
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x90
22
ip_rcv
vmlinux 0xffffffffb8ca20d0
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x2c0
23
? packet_rcv
vmlinux 0xffffffffb8d62502
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x2f2
24
? inet_del_offload
vmlinux 0xffffffffb8ca1670
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffffb8c56829
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x729
26
? tcp4_gro_receive
vmlinux 0xffffffffb8ccea00
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x120
27
? __getnstimeofday64
vmlinux 0xffffffffb8707f0f
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffb8c56b28
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffb8c56bb0
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffb8c57838
Build id: 3.10.0-1160.41.1.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc022945b
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc022a100
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x280
33
net_rx_action
vmlinux 0xffffffffb8c571cf
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x26f
34
__do_softirq
vmlinux 0xffffffffb86a4bf5
Build id: 3.10.0-1160.41.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffb86a4db8
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffb86ce8b4
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x144
37
? lg_double_unlock
vmlinux 0xffffffffb86ce770
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffb86c5e61
Build id: 3.10.0-1160.41.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffb86c5d90
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffb8d95df7
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x21
41
? insert_kthread_work
vmlinux 0xffffffffb86c5d90
Build id: 3.10.0-1160.41.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: lp veeamsnap(OE) tcp_lp xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 iosf_mbi crc32_pclmul snd_rawmidi snd_ac97_codec ghash_clmulni_intel aesni_intel ac97_bus lrw snd_seq gf128mul glue_helper ablk_helper snd_seq_device snd_pcm
ppdev cryptd pcspkr snd_timer joydev snd soundcore vmw_balloon sg parport_pc parport i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx nfit libnvdimm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix serio_raw mptspi scsi_transport_spi libata e1000 mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G           OEL ------------   3.10.0-1160.41.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff944bbb438000 ti: ffff944bbb434000 task.ti: ffff944bbb434000
RIP: 0010:[<ffffffffc0228877>]  [<ffffffffc0228877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff944bbb437550  EFLAGS: 00000286
RAX: ffffaeeb40f83818 RBX: 0000000000000001 RCX: 0000000000000084
RDX: 0000000000000084 RSI: ffff944bf3356140 RDI: ffff944bf863e000
RBP: ffff944bbb4375d0 R08: 0000000000000100 R09: 0000000022100000
R10: 0f01a8c05f1e0640 R11: 0000000000000000 R12: 0000000000000202
R13: 000000016aeb3cc2 R14: 00000001f4757510 R15: ffffffffb89b2290
FS:  0000000000000000(0000) GS:ffff944bf7640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3e3c6c4000 CR3: 00000000b915a000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb8c55f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb8c840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffb8c58f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb8c590e0>] dev_queue_xmit+0x10/0x20
[<ffffffffb8ca7c6b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb8ca81eb>] ip_output+0x7b/0xf0
[<ffffffffb8ca75f0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb8ca5c17>] ip_local_out_sk+0x37/0x40
[<ffffffffb8ca5fb4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb8cc06f4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb8cc290f>] tcp_send_ack+0x11f/0x170
[<ffffffffb8cb6f3b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffb8cbcfb7>] tcp_rcv_established+0x267/0x7a0
[<ffffffffb8cc800a>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffb8cc985c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc08382ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffb8ca1a9d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb8ca1d90>] ip_local_deliver+0x60/0xe0
[<ffffffffb8ca19e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb8ca1700>] ip_rcv_finish+0x90/0x370
[<ffffffffb8ca20d0>] ip_rcv+0x2c0/0x420
[<ffffffffb8d62502>] ? packet_rcv+0x2f2/0x430
[<ffffffffb8ca1670>] ? inet_del_offload+0x40/0x40
[<ffffffffb8c56829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb8ccea00>] ? tcp4_gro_receive+0x120/0x1a0
[<ffffffffb8707f0f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb8c56b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb8c56bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb8c57838>] napi_gro_receive+0xd8/0x100
[<ffffffffc022945b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc022a100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb8c571cf>] net_rx_action+0x26f/0x390
[<ffffffffb86a4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb86a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb86ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb86ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb86c5e61>] kthread+0xd1/0xe0
[<ffffffffb86c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb8d95df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb86c5d90>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #346516
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc034f877
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb9c55f96
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffb9c840aa
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb9c58f11
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x491
5
? local_bh_enable
vmlinux 0xffffffffb96a4077
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x17
6
dev_queue_xmit
vmlinux 0xffffffffb9c590e0
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x10
7
neigh_resolve_output
vmlinux 0xffffffffb9c65e0d
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x11d
8
ip_finish_output
vmlinux 0xffffffffb9ca786c
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x27c
9
ip_output
vmlinux 0xffffffffb9ca81eb
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x7b
10
? __ip_append_data.isra.50
vmlinux 0xffffffffb9ca75f0
Build id: 3.10.0-1160.45.1.el7.x86_64 		0xa60
11
ip_local_out_sk
vmlinux 0xffffffffb9ca5c17
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x37
12
ip_queue_xmit
vmlinux 0xffffffffb9ca5fb4
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x144
13
tcp_transmit_skb
vmlinux 0xffffffffb9cc06f4
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x4e4
14
tcp_send_ack
vmlinux 0xffffffffb9cc290f
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x11f
15
__tcp_ack_snd_check
vmlinux 0xffffffffb9cb6f3b
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x6b
16
tcp_rcv_established
vmlinux 0xffffffffb9cbd175
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x425
17
tcp_v4_do_rcv
vmlinux 0xffffffffb9cc800a
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x10a
18
tcp_v4_rcv
vmlinux 0xffffffffb9cc985c
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x7dc
19
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc091b2ce
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x4e
20
ip_local_deliver_finish
vmlinux 0xffffffffb9ca1a9d
Build id: 3.10.0-1160.45.1.el7.x86_64 		0xbd
21
ip_local_deliver
vmlinux 0xffffffffb9ca1d90
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x60
22
? ip_rcv_finish
vmlinux 0xffffffffb9ca19e0
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x370
23
ip_rcv_finish
vmlinux 0xffffffffb9ca1700
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x90
24
ip_rcv
vmlinux 0xffffffffb9ca20d0
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x2c0
25
? inet_del_offload
vmlinux 0xffffffffb9ca1670
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffb9c56829
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x729
27
? __getnstimeofday64
vmlinux 0xffffffffb9707f0f
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x3f
28
__netif_receive_skb
vmlinux 0xffffffffb9c56b28
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x18
29
netif_receive_skb_internal
vmlinux 0xffffffffb9c56bb0
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x40
30
napi_gro_receive
vmlinux 0xffffffffb9c57838
Build id: 3.10.0-1160.45.1.el7.x86_64 		0xd8
31
e1000_clean_rx_irq
e1000 0xffffffffc035045b
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x2bb
32
e1000_clean
e1000 0xffffffffc0351100
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x280
33
net_rx_action
vmlinux 0xffffffffb9c571cf
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x26f
34
__do_softirq
vmlinux 0xffffffffb96a4bf5
Build id: 3.10.0-1160.45.1.el7.x86_64 		0xf5
35
run_ksoftirqd
vmlinux 0xffffffffb96a4db8
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x38
36
smpboot_thread_fn
vmlinux 0xffffffffb96ce8b4
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x144
37
? lg_double_unlock
vmlinux 0xffffffffb96ce770
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x40
38
kthread
vmlinux 0xffffffffb96c5e61
Build id: 3.10.0-1160.45.1.el7.x86_64 		0xd1
39
? insert_kthread_work
vmlinux 0xffffffffb96c5d90
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x40
40
ret_from_fork_nospec_begin
vmlinux 0xffffffffb9d95df7
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x21
41
? insert_kthread_work
vmlinux 0xffffffffb96c5d90
Build id: 3.10.0-1160.45.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 26s! [ksoftirqd/3:24]
Modules linked in: xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_12905(E) ext4 mbcache jbd2 snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq iosf_mbi snd_seq_device crc32_pclmul snd_pcm ghash_clmulni_intel aesni_intel snd_timer lrw gf128mul snd glue_helper ablk_helper soundcore cryptd ppdev joydev virtio_rng pcspkr parport_pc i2c_piix4 parport virtio_balloon ip_tables xfs libcrc32c ata_generic virtio_blk virtio_console pata_acpi qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops
crct10dif_pclmul crct10dif_common ttm crc32c_intel drm ata_piix floppy serio_raw e1000 virtio_pci virtio_ring virtio libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Tainted: P            E  ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7860+a7792d29 04/01/2014
task: ffff964d38535280 ti: ffff964d3856c000 task.ti: ffff964d3856c000
RIP: 0010:[<ffffffffc034f877>]  [<ffffffffc034f877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff964d3856f510  EFLAGS: 00000286
RAX: ffffa6d3807c3818 RBX: ffff964d3856f4d8 RCX: 000000000000009f
RDX: 000000000000009f RSI: ffff964c3665fe80 RDI: ffff964d36c50000
RBP: ffff964d3856f590 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff964d3856f4c0
R13: 0000000000c091ba R14: ffff964c05bf7a00 R15: ffffffffb9707f0f
FS:  0000000000000000(0000) GS:ffff964d3fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd14c2ce000 CR3: 00000000a0f0a000 CR4: 00000000001606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb9c55f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb9c840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9c58f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb96a4077>] ? local_bh_enable+0x17/0x20
[<ffffffffb9c590e0>] dev_queue_xmit+0x10/0x20
[<ffffffffb9c65e0d>] neigh_resolve_output+0x11d/0x220
[<ffffffffb9ca786c>] ip_finish_output+0x27c/0x8d0
[<ffffffffb9ca81eb>] ip_output+0x7b/0xf0
[<ffffffffb9ca75f0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb9ca5c17>] ip_local_out_sk+0x37/0x40
[<ffffffffb9ca5fb4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb9cc06f4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb9cc290f>] tcp_send_ack+0x11f/0x170
[<ffffffffb9cb6f3b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffb9cbd175>] tcp_rcv_established+0x425/0x7a0
[<ffffffffb9cc800a>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffb9cc985c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc091b2ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffb9ca1a9d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb9ca1d90>] ip_local_deliver+0x60/0xe0
[<ffffffffb9ca19e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb9ca1700>] ip_rcv_finish+0x90/0x370
[<ffffffffb9ca20d0>] ip_rcv+0x2c0/0x420
[<ffffffffb9ca1670>] ? inet_del_offload+0x40/0x40
[<ffffffffb9c56829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb9707f0f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb9c56b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb9c56bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb9c57838>] napi_gro_receive+0xd8/0x100
[<ffffffffc035045b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0351100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb9c571cf>] net_rx_action+0x26f/0x390
[<ffffffffb96a4bf5>] __do_softirq+0xf5/0x280
[<ffffffffb96a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffb96ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb96ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffffb96c5e61>] kthread+0xd1/0xe0
[<ffffffffb96c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffb9d95df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb96c5d90>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #258066
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc001eee7
Build id: 3.10.0-693.el7.x86_64 		0x947
2
? kfree_skbmem
vmlinux 0xffffffff815710d7
Build id: 3.10.0-693.el7.x86_64 		0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff81586771
Build id: 3.10.0-693.el7.x86_64 		0x171
4
sch_direct_xmit
vmlinux 0xffffffff815af50a
Build id: 3.10.0-693.el7.x86_64 		0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff81589566
Build id: 3.10.0-693.el7.x86_64 		0x226
6
dev_queue_xmit
vmlinux 0xffffffff815898a0
Build id: 3.10.0-693.el7.x86_64 		0x10
7
ip_finish_output
vmlinux 0xffffffff815cfd3e
Build id: 3.10.0-693.el7.x86_64 		0x52e
8
ip_output
vmlinux 0xffffffff815d0293
Build id: 3.10.0-693.el7.x86_64 		0x73
9
? __ip_append_data.isra.48
vmlinux 0xffffffff815cf810
Build id: 3.10.0-693.el7.x86_64 		0xa00
10
ip_local_out_sk
vmlinux 0xffffffff815cdf31
Build id: 3.10.0-693.el7.x86_64 		0x31
11
ip_queue_xmit
vmlinux 0xffffffff815ce2a3
Build id: 3.10.0-693.el7.x86_64 		0x143
12
tcp_transmit_skb
vmlinux 0xffffffff815e7c0f
Build id: 3.10.0-693.el7.x86_64 		0x4cf
13
tcp_send_ack
vmlinux 0xffffffff815e9c2c
Build id: 3.10.0-693.el7.x86_64 		0x11c
14
__tcp_ack_snd_check
vmlinux 0xffffffff815defab
Build id: 3.10.0-693.el7.x86_64 		0x6b
15
tcp_rcv_established
vmlinux 0xffffffff815e4694
Build id: 3.10.0-693.el7.x86_64 		0x264
16
tcp_v4_do_rcv
vmlinux 0xffffffff815ef15a
Build id: 3.10.0-693.el7.x86_64 		0x10a
17
tcp_v4_rcv
vmlinux 0xffffffff815f08a8
Build id: 3.10.0-693.el7.x86_64 		0x798
18
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc05f9306
Build id: 3.10.0-693.el7.x86_64 		0x86
19
ip_local_deliver_finish
vmlinux 0xffffffff815ca054
Build id: 3.10.0-693.el7.x86_64 		0xb4
20
ip_local_deliver
vmlinux 0xffffffff815ca339
Build id: 3.10.0-693.el7.x86_64 		0x59
21
? ip_rcv_finish
vmlinux 0xffffffff815c9fa0
Build id: 3.10.0-693.el7.x86_64 		0x350
22
ip_rcv_finish
vmlinux 0xffffffff815c9cda
Build id: 3.10.0-693.el7.x86_64 		0x8a
23
ip_rcv
vmlinux 0xffffffff815ca666
Build id: 3.10.0-693.el7.x86_64 		0x2b6
24
? inet_del_offload
vmlinux 0xffffffff815c9c50
Build id: 3.10.0-693.el7.x86_64 		0x40
25
__netif_receive_skb_core
vmlinux 0xffffffff81586f22
Build id: 3.10.0-693.el7.x86_64 		0x572
26
? __getnstimeofday64
vmlinux 0xffffffff810ea8ca
Build id: 3.10.0-693.el7.x86_64 		0x3a
27
__netif_receive_skb
vmlinux 0xffffffff81587188
Build id: 3.10.0-693.el7.x86_64 		0x18
28
netif_receive_skb_internal
vmlinux 0xffffffff81587210
Build id: 3.10.0-693.el7.x86_64 		0x40
29
napi_gro_receive
vmlinux 0xffffffff81588318
Build id: 3.10.0-693.el7.x86_64 		0xd8
30
e1000_clean_rx_irq
e1000 0xffffffffc002047c
Build id: 3.10.0-693.el7.x86_64 		0x2ac
31
e1000_clean
e1000 0xffffffffc0020ed8
Build id: 3.10.0-693.el7.x86_64 		0x278
32
net_rx_action
vmlinux 0xffffffff8158799d
Build id: 3.10.0-693.el7.x86_64 		0x16d
33
__do_softirq
vmlinux 0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64 		0xef
34
run_ksoftirqd
vmlinux 0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64 		0x38
35
smpboot_thread_fn
vmlinux 0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64 		0x12f
36
? lg_double_unlock
vmlinux 0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64 		0x40
37
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64 		0xcf
38
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64 		0x40
39
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64 		0x58
40
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [ksoftirqd/3:23]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device joydev snd_pcm snd_timer snd iosf_mbi crc32_pclmul ghash_clmulni_intel soundcore aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd ppdev vmw_balloon sg pcspkr nfit libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix mptspi scsi_transport_spi mptscsih libata syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel mptbase e1000 serio_raw i2c_core
CPU: 3 PID: 23 Comm: ksoftirqd/3 Tainted: G             L ------------   3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff88017cb0cf10 ti: ffff88017cb28000 task.ti: ffff88017cb28000
RIP: 0010:[<ffffffffc001eee7>]  [<ffffffffc001eee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff88017cb2b5a8  EFLAGS: 00000286
RAX: ffffc90000d03818 RBX: ffff88017cb2b530 RCX: 000000000000001f
RDX: 000000000000001f RSI: ffff8802319d3e40 RDI: ffff88023fde2000
RBP: ffff88017cb2b628 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff8800b65b8d10
R13: ffff88017cb34000 R14: ffffffff8169e5ff R15: ffffffffc05c874d
FS:  0000000000000000(0000) GS:ffff8802356c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000032d44c5c2000 CR3: 0000000225ecb000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffff815710d7 ffff880235b2c098 0000001d81573004 0000000900000000
000000002cf0d800 ffff880100000000 ffffffff00000000 0000000100001000
ffff88022b73a300 ffff88023fde2000 ffff880235b2c000 ffff88023fde2000
Call Trace:
[<ffffffff815710d7>] ? kfree_skbmem+0x37/0x90
[<ffffffff81586771>] dev_hard_start_xmit+0x171/0x3b0
[<ffffffff815af50a>] sch_direct_xmit+0x11a/0x240
[<ffffffff81589566>] __dev_queue_xmit+0x226/0x550
[<ffffffff815898a0>] dev_queue_xmit+0x10/0x20
[<ffffffff815cfd3e>] ip_finish_output+0x52e/0x780
[<ffffffff815d0293>] ip_output+0x73/0xe0
[<ffffffff815cf810>] ? __ip_append_data.isra.48+0xa00/0xa00
[<ffffffff815cdf31>] ip_local_out_sk+0x31/0x40
[<ffffffff815ce2a3>] ip_queue_xmit+0x143/0x3a0
[<ffffffff815e7c0f>] tcp_transmit_skb+0x4cf/0x9b0
[<ffffffff815e9c2c>] tcp_send_ack+0x11c/0x170
[<ffffffff815defab>] __tcp_ack_snd_check+0x6b/0xa0
[<ffffffff815e4694>] tcp_rcv_established+0x264/0x760
[<ffffffff815ef15a>] tcp_v4_do_rcv+0x10a/0x340
[<ffffffff815f08a8>] tcp_v4_rcv+0x798/0x9a0
[<ffffffffc05f9306>] ? ipv4_confirm+0x86/0x100 [nf_conntrack_ipv4]
[<ffffffff815ca054>] ip_local_deliver_finish+0xb4/0x1f0
[<ffffffff815ca339>] ip_local_deliver+0x59/0xd0
[<ffffffff815c9fa0>] ? ip_rcv_finish+0x350/0x350
[<ffffffff815c9cda>] ip_rcv_finish+0x8a/0x350
[<ffffffff815ca666>] ip_rcv+0x2b6/0x410
[<ffffffff815c9c50>] ? inet_del_offload+0x40/0x40
[<ffffffff81586f22>] __netif_receive_skb_core+0x572/0x7c0
[<ffffffff810ea8ca>] ? __getnstimeofday64+0x3a/0xd0
[<ffffffff81587188>] __netif_receive_skb+0x18/0x60
[<ffffffff81587210>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff81588318>] napi_gro_receive+0xd8/0x130
[<ffffffffc002047c>] e1000_clean_rx_irq+0x2ac/0x4f0 [e1000]
[<ffffffffc0020ed8>] e1000_clean+0x278/0x8d0 [e1000]
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #51725
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0417877
Build id: 3.10.0-1160.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff98e54386
Build id: 3.10.0-1160.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffff98e8242a
Build id: 3.10.0-1160.el7.x86_64 		0x11a
4
__qdisc_run
vmlinux 0xffffffff98e825ee
Build id: 3.10.0-1160.el7.x86_64 		0x8e
5
__dev_queue_xmit
vmlinux 0xffffffff98e57088
Build id: 3.10.0-1160.el7.x86_64 		0x218
6
? nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc09fb7ae
Build id: 3.10.0-1160.el7.x86_64 		0x1de
7
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc0a0d020
Build id: 3.10.0-1160.el7.x86_64 		0x20
8
dev_queue_xmit
vmlinux 0xffffffff98e574d0
Build id: 3.10.0-1160.el7.x86_64 		0x10
9
neigh_resolve_output
vmlinux 0xffffffff98e641fd
Build id: 3.10.0-1160.el7.x86_64 		0x11d
10
ip_finish_output
vmlinux 0xffffffff98ea505c
Build id: 3.10.0-1160.el7.x86_64 		0x27c
11
ip_output
vmlinux 0xffffffff98ea59db
Build id: 3.10.0-1160.el7.x86_64 		0x7b
12
? __ip_append_data.isra.50
vmlinux 0xffffffff98ea4de0
Build id: 3.10.0-1160.el7.x86_64 		0xa60
13
ip_local_out_sk
vmlinux 0xffffffff98ea3407
Build id: 3.10.0-1160.el7.x86_64 		0x37
14
ip_queue_xmit
vmlinux 0xffffffff98ea37a4
Build id: 3.10.0-1160.el7.x86_64 		0x144
15
tcp_transmit_skb
vmlinux 0xffffffff98ebde94
Build id: 3.10.0-1160.el7.x86_64 		0x4e4
16
tcp_send_ack
vmlinux 0xffffffff98ec00af
Build id: 3.10.0-1160.el7.x86_64 		0x11f
17
__tcp_ack_snd_check
vmlinux 0xffffffff98eb4c7b
Build id: 3.10.0-1160.el7.x86_64 		0x6b
18
tcp_rcv_established
vmlinux 0xffffffff98eba7a4
Build id: 3.10.0-1160.el7.x86_64 		0x264
19
tcp_v4_do_rcv
vmlinux 0xffffffff98ec57aa
Build id: 3.10.0-1160.el7.x86_64 		0x10a
20
tcp_v4_rcv
vmlinux 0xffffffff98ec6ffc
Build id: 3.10.0-1160.el7.x86_64 		0x7dc
21
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc09f62ce
Build id: 3.10.0-1160.el7.x86_64 		0x4e
22
ip_local_deliver_finish
vmlinux 0xffffffff98e9f28d
Build id: 3.10.0-1160.el7.x86_64 		0xbd
23
ip_local_deliver
vmlinux 0xffffffff98e9f580
Build id: 3.10.0-1160.el7.x86_64 		0x60
24
? ip_rcv_finish
vmlinux 0xffffffff98e9f1d0
Build id: 3.10.0-1160.el7.x86_64 		0x370
25
ip_rcv_finish
vmlinux 0xffffffff98e9eef0
Build id: 3.10.0-1160.el7.x86_64 		0x90
26
ip_rcv
vmlinux 0xffffffff98e9f8c0
Build id: 3.10.0-1160.el7.x86_64 		0x2c0
27
? inet_del_offload
vmlinux 0xffffffff98e9ee60
Build id: 3.10.0-1160.el7.x86_64 		0x40
28
__netif_receive_skb_core
vmlinux 0xffffffff98e54c19
Build id: 3.10.0-1160.el7.x86_64 		0x729
29
? tcp4_gro_receive
vmlinux 0xffffffff98ecc100
Build id: 3.10.0-1160.el7.x86_64 		0x80
30
? __getnstimeofday64
vmlinux 0xffffffff989078df
Build id: 3.10.0-1160.el7.x86_64 		0x3f
31
__netif_receive_skb
vmlinux 0xffffffff98e54f18
Build id: 3.10.0-1160.el7.x86_64 		0x18
32
netif_receive_skb_internal
vmlinux 0xffffffff98e54fa0
Build id: 3.10.0-1160.el7.x86_64 		0x40
33
napi_gro_receive
vmlinux 0xffffffff98e55c28
Build id: 3.10.0-1160.el7.x86_64 		0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc041845b
Build id: 3.10.0-1160.el7.x86_64 		0x2bb
35
e1000_clean
e1000 0xffffffffc0419100
Build id: 3.10.0-1160.el7.x86_64 		0x280
36
? nf_ct_delete
nf_conntrack 0xffffffffc09cf02d
Build id: 3.10.0-1160.el7.x86_64 		0x18d
37
net_rx_action
vmlinux 0xffffffff98e555bf
Build id: 3.10.0-1160.el7.x86_64 		0x26f
38
__do_softirq
vmlinux 0xffffffff988a4b95
Build id: 3.10.0-1160.el7.x86_64 		0xf5
39
run_ksoftirqd
vmlinux 0xffffffff988a4d58
Build id: 3.10.0-1160.el7.x86_64 		0x38
40
smpboot_thread_fn
vmlinux 0xffffffff988ce674
Build id: 3.10.0-1160.el7.x86_64 		0x144
41
? lg_double_unlock
vmlinux 0xffffffff988ce530
Build id: 3.10.0-1160.el7.x86_64 		0x40
42
kthread
vmlinux 0xffffffff988c5c21
Build id: 3.10.0-1160.el7.x86_64 		0xd1
43
? insert_kthread_work
vmlinux 0xffffffff988c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40
44
ret_from_fork_nospec_begin
vmlinux 0xffffffff98f93df7
Build id: 3.10.0-1160.el7.x86_64 		0x21
45
? insert_kthread_work
vmlinux 0xffffffff988c5b50
Build id: 3.10.0-1160.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod ext4 mbcache jbd2 iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper pcspkr joydev cryptd ppdev sg vmw_balloon parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul ata_piix crct10dif_common drm mptspi nfit crc32c_intel
libata scsi_transport_spi libnvdimm mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff907dfd765280 ti: ffff907dfe364000 task.ti: ffff907dfe364000
RIP: 0010:[<ffffffffc0417877>]  [<ffffffffc0417877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff907dfe3674b8  EFLAGS: 00000286
RAX: ffff9c2700d03818 RBX: 0000000000000042 RCX: 0000000000000012
RDX: 0000000000000012 RSI: ffff907eb1f29b80 RDI: ffff907df779a000
RBP: ffff907dfe367538 R08: 0000000000000000 R09: 0000000022100000
R10: 3400004500088847 R11: 0000000000000000 R12: 00000002145b38ce
R13: 0000000124f32f00 R14: ffffffff98bb1650 R15: ffff907dfe367470
FS:  0000000000000000(0000) GS:ffff907eb5600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0a5485e300 CR3: 0000000029810000 CR4: 00000000003607f0
Call Trace:
[<ffffffff98e54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff98e8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff98e825ee>] __qdisc_run+0x8e/0x360
[<ffffffff98e57088>] __dev_queue_xmit+0x218/0x650
[<ffffffffc09fb7ae>] ? nf_nat_ipv4_fn+0x1de/0x230 [nf_nat_ipv4]
[<ffffffffc0a0d020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffff98e574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff98e641fd>] neigh_resolve_output+0x11d/0x220
[<ffffffff98ea505c>] ip_finish_output+0x27c/0x8d0
[<ffffffff98ea59db>] ip_output+0x7b/0xf0
[<ffffffff98ea4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff98ea3407>] ip_local_out_sk+0x37/0x40
[<ffffffff98ea37a4>] ip_queue_xmit+0x144/0x3c0
[<ffffffff98ebde94>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff98ec00af>] tcp_send_ack+0x11f/0x170
[<ffffffff98eb4c7b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffff98eba7a4>] tcp_rcv_established+0x264/0x770
[<ffffffff98ec57aa>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffff98ec6ffc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc09f62ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff98e9f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff98e9f580>] ip_local_deliver+0x60/0xe0
[<ffffffff98e9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff98e9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffff98e9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffff98e9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffff98e54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff98ecc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff989078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff98e54f18>] __netif_receive_skb+0x18/0x60
[<ffffffff98e54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff98e55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc041845b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0419100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc09cf02d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffff98e555bf>] net_rx_action+0x26f/0x390
[<ffffffff988a4b95>] __do_softirq+0xf5/0x280
[<ffffffff988a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff988ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff988ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff988c5c21>] kthread+0xd1/0xe0
[<ffffffff988c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff98f93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff988c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #12972
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0367877
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffa5e4d346
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x246
3
sch_direct_xmit
vmlinux 0xffffffffa5e7b0ba
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffa5e502b1
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x491
5
? nf_nat_ipv4_fn
nf_nat_ipv4 0xffffffffc09bb7ae
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x1de
6
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc09b6020
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x20
7
dev_queue_xmit
vmlinux 0xffffffffa5e50480
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x10
8
neigh_resolve_output
vmlinux 0xffffffffa5e5cfed
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x11d
9
ip_finish_output
vmlinux 0xffffffffa5e9e57c
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x27c
10
ip_output
vmlinux 0xffffffffa5e9eefb
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x7b
11
? __ip_append_data.isra.50
vmlinux 0xffffffffa5e9e300
Build id: 3.10.0-1062.4.1.el7.x86_64 		0xa60
12
ip_local_out_sk
vmlinux 0xffffffffa5e9c927
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x37
13
ip_queue_xmit
vmlinux 0xffffffffa5e9ccc4
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x144
14
tcp_transmit_skb
vmlinux 0xffffffffa5eb7364
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x4e4
15
tcp_send_ack
vmlinux 0xffffffffa5eb955f
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x11f
16
__tcp_ack_snd_check
vmlinux 0xffffffffa5eae15b
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x6b
17
tcp_rcv_established
vmlinux 0xffffffffa5eb3c74
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x264
18
tcp_v4_do_rcv
vmlinux 0xffffffffa5ebed3a
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x10a
19
tcp_v4_rcv
vmlinux 0xffffffffa5ec044c
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x7dc
20
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc09a52ce
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x4e
21
ip_local_deliver_finish
vmlinux 0xffffffffa5e9886d
Build id: 3.10.0-1062.4.1.el7.x86_64 		0xbd
22
ip_local_deliver
vmlinux 0xffffffffa5e98b60
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x60
23
? ip_rcv_finish
vmlinux 0xffffffffa5e987b0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x370
24
ip_rcv_finish
vmlinux 0xffffffffa5e984d0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x90
25
ip_rcv
vmlinux 0xffffffffa5e98ea0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x2c0
26
? packet_rcv
vmlinux 0xffffffffa5f58d02
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x312
27
? inet_del_offload
vmlinux 0xffffffffa5e98440
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffa5e4dbd9
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x729
29
? tcp4_gro_receive
vmlinux 0xffffffffa5ec5500
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x30
30
? __getnstimeofday64
vmlinux 0xffffffffa5905e2f
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x3f
31
__netif_receive_skb
vmlinux 0xffffffffa5e4ded8
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x18
32
netif_receive_skb_internal
vmlinux 0xffffffffa5e4df60
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x40
33
napi_gro_receive
vmlinux 0xffffffffa5e4ebe8
Build id: 3.10.0-1062.4.1.el7.x86_64 		0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc036845b
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x2bb
35
e1000_clean
e1000 0xffffffffc0369100
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x280
36
? update_curr
vmlinux 0xffffffffa58e382c
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x14c
37
net_rx_action
vmlinux 0xffffffffa5e4e57f
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x26f
38
__do_softirq
vmlinux 0xffffffffa58a42c5
Build id: 3.10.0-1062.4.1.el7.x86_64 		0xf5
39
run_ksoftirqd
vmlinux 0xffffffffa58a4488
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x38
40
smpboot_thread_fn
vmlinux 0xffffffffa58cdbe4
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x144
41
? lg_double_unlock
vmlinux 0xffffffffa58cdaa0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x40
42
kthread
vmlinux 0xffffffffa58c51b1
Build id: 3.10.0-1062.4.1.el7.x86_64 		0xd1
43
? insert_kthread_work
vmlinux 0xffffffffa58c50e0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x40
44
ret_from_fork_nospec_begin
vmlinux 0xffffffffa5f8bd1d
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x7
45
? insert_kthread_work
vmlinux 0xffffffffa58c50e0
Build id: 3.10.0-1062.4.1.el7.x86_64 		0x40 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter overlay vmw_vsock_vmci_transport vsock sunrpc iosf_mbi kvm_intel kvm snd_seq_midi snd_seq_midi_event irqbypass crc32_pclmul ppdev ghash_clmulni_intel vmw_balloon snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device aesni_intel lrw gf128mul glue_helper snd_pcm ablk_helper cryptd snd_timer snd joydev soundcore pcspkr sg i2c_piix4 vmw_vmci nfit libnvdimm parport_pc
parport binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common mptspi libata crc32c_intel scsi_transport_spi e1000 mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Not tainted 3.10.0-1062.4.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff99757a3ad230 ti: ffff997579cc0000 task.ti: ffff997579cc0000
RIP: 0010:[<ffffffffc0367877>]  [<ffffffffc0367877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff997579cc3510  EFLAGS: 00000286
RAX: ffffb45301c43818 RBX: 0000000000000001 RCX: 0000000000000028
RDX: 0000000000000028 RSI: ffff9978273e8ec0 RDI: ffff99782cdb6000
RBP: ffff997579cc3590 R08: 0000000000000100 R09: 0000000022100000
R10: c902010ab2450640 R11: 0000000000000000 R12: 0000000000000202
R13: 00000003b962b8c2 R14: 00000001273e8ec0 R15: ffffffffa5bab430
FS:  0000000000000000(0000) GS:ffff99782d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055cc1bd39010 CR3: 00000000babd6000 CR4: 00000000003607f0
Call Trace:
[<ffffffffa5e4d346>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa5e7b0ba>] sch_direct_xmit+0x11a/0x250
[<ffffffffa5e502b1>] __dev_queue_xmit+0x491/0x650
[<ffffffffc09bb7ae>] ? nf_nat_ipv4_fn+0x1de/0x230 [nf_nat_ipv4]
[<ffffffffc09b6020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffa5e50480>] dev_queue_xmit+0x10/0x20
[<ffffffffa5e5cfed>] neigh_resolve_output+0x11d/0x220
[<ffffffffa5e9e57c>] ip_finish_output+0x27c/0x8d0
[<ffffffffa5e9eefb>] ip_output+0x7b/0xf0
[<ffffffffa5e9e300>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa5e9c927>] ip_local_out_sk+0x37/0x40
[<ffffffffa5e9ccc4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffa5eb7364>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffa5eb955f>] tcp_send_ack+0x11f/0x170
[<ffffffffa5eae15b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffa5eb3c74>] tcp_rcv_established+0x264/0x770
[<ffffffffa5ebed3a>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffa5ec044c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc09a52ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffa5e9886d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa5e98b60>] ip_local_deliver+0x60/0xe0
[<ffffffffa5e987b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa5e984d0>] ip_rcv_finish+0x90/0x370
[<ffffffffa5e98ea0>] ip_rcv+0x2c0/0x420
[<ffffffffa5f58d02>] ? packet_rcv+0x312/0x430
[<ffffffffa5e98440>] ? inet_del_offload+0x40/0x40
[<ffffffffa5e4dbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa5ec5500>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffffa5905e2f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffa5e4ded8>] __netif_receive_skb+0x18/0x60
[<ffffffffa5e4df60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa5e4ebe8>] napi_gro_receive+0xd8/0x100
[<ffffffffc036845b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0369100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa58e382c>] ? update_curr+0x14c/0x1e0
[<ffffffffa5e4e57f>] net_rx_action+0x26f/0x390
[<ffffffffa58a42c5>] __do_softirq+0xf5/0x280
[<ffffffffa58a4488>] run_ksoftirqd+0x38/0x50
[<ffffffffa58cdbe4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa58cdaa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffa58c51b1>] kthread+0xd1/0xe0
[<ffffffffa58c50e0>] ? insert_kthread_work+0x40/0x40
[<ffffffffa5f8bd1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffa58c50e0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00