Info

Permalink
Function
e1000_xmit_frame
First occurrence
2020-10-22
Last occurrence
2020-10-22
State
NEW
Components
kernel
Quality
-96

Statistics

Operating system Count
CentOS 7 37
Architecture Count
x86_64 37
Related packages Count
kernel 37
 0:3.10.0-1160.49.1.el7 15
 0:3.10.0-1160.el7 6
 0:3.10.0-862.el7 4
 0:3.10.0-1062.el7 3
 0:3.10.0-1160.119.1.el7 2
 0:3.10.0-693.el7 2
 0:3.10.0-862.14.4.el7 1
 0:3.10.0-1160.15.2.el7 1
 0:3.10.0-693.21.1.el7 1
 0:3.10.0-1160.80.1.el7 1
 0:3.10.0-1160.88.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

No reports in the last 20 days.

Report backtrace


Complete report #1392886
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc002eee7
Build id: 3.10.0-693.el7.x86_64
0x947
2
dev_hard_start_xmit
vmlinux 0xffffffff81586771
Build id: 3.10.0-693.el7.x86_64
0x171
3
sch_direct_xmit
vmlinux 0xffffffff815af50a
Build id: 3.10.0-693.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff81589566
Build id: 3.10.0-693.el7.x86_64
0x226
5
dev_queue_xmit
vmlinux 0xffffffff815898a0
Build id: 3.10.0-693.el7.x86_64
0x10
6
neigh_resolve_output
vmlinux 0xffffffff8159527b
Build id: 3.10.0-693.el7.x86_64
0x11b
7
ip_finish_output
vmlinux 0xffffffff815cfaa7
Build id: 3.10.0-693.el7.x86_64
0x297
8
ip_output
vmlinux 0xffffffff815d0293
Build id: 3.10.0-693.el7.x86_64
0x73
9
? __ip_append_data.isra.48
vmlinux 0xffffffff815cf810
Build id: 3.10.0-693.el7.x86_64
0xa00
10
ip_local_out_sk
vmlinux 0xffffffff815cdf31
Build id: 3.10.0-693.el7.x86_64
0x31
11
ip_build_and_send_pkt
vmlinux 0xffffffff815ce0a2
Build id: 3.10.0-693.el7.x86_64
0x162
12
tcp_v4_send_synack
vmlinux 0xffffffff815eff64
Build id: 3.10.0-693.el7.x86_64
0x74
13
? __fib_validate_source.isra.12
vmlinux 0xffffffff8160cbd6
Build id: 3.10.0-693.el7.x86_64
0x2b6
14
tcp_rtx_synack
vmlinux 0xffffffff815e637b
Build id: 3.10.0-693.el7.x86_64
0x3b
15
? local_bh_enable
vmlinux 0xffffffff81090007
Build id: 3.10.0-693.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc0050d06
Build id: 3.10.0-693.el7.x86_64
0x336
17
? local_bh_enable
vmlinux 0xffffffff81090007
Build id: 3.10.0-693.el7.x86_64
0x17
18
? ipt_do_table
ip_tables 0xffffffffc0050d06
Build id: 3.10.0-693.el7.x86_64
0x336
19
inet_rtx_syn_ack
vmlinux 0xffffffff815d54b7
Build id: 3.10.0-693.el7.x86_64
0x17
20
tcp_check_req
vmlinux 0xffffffff815f1729
Build id: 3.10.0-693.el7.x86_64
0x239
21
tcp_v4_do_rcv
vmlinux 0xffffffff815ef26f
Build id: 3.10.0-693.el7.x86_64
0x21f
22
tcp_v4_rcv
vmlinux 0xffffffff815f08a8
Build id: 3.10.0-693.el7.x86_64
0x798
23
ip_local_deliver_finish
vmlinux 0xffffffff815ca054
Build id: 3.10.0-693.el7.x86_64
0xb4
24
ip_local_deliver
vmlinux 0xffffffff815ca339
Build id: 3.10.0-693.el7.x86_64
0x59
25
? ip_rcv_finish
vmlinux 0xffffffff815c9fa0
Build id: 3.10.0-693.el7.x86_64
0x350
26
ip_rcv_finish
vmlinux 0xffffffff815c9cda
Build id: 3.10.0-693.el7.x86_64
0x8a
27
ip_rcv
vmlinux 0xffffffff815ca666
Build id: 3.10.0-693.el7.x86_64
0x2b6
28
? inet_del_offload
vmlinux 0xffffffff815c9c50
Build id: 3.10.0-693.el7.x86_64
0x40
29
__netif_receive_skb_core
vmlinux 0xffffffff81586f22
Build id: 3.10.0-693.el7.x86_64
0x572
30
? tcp4_gro_receive
vmlinux 0xffffffff815f5944
Build id: 3.10.0-693.el7.x86_64
0x134
31
? inet_gro_receive
vmlinux 0xffffffff81607087
Build id: 3.10.0-693.el7.x86_64
0x247
32
__netif_receive_skb
vmlinux 0xffffffff81587188
Build id: 3.10.0-693.el7.x86_64
0x18
33
netif_receive_skb_internal
vmlinux 0xffffffff81587210
Build id: 3.10.0-693.el7.x86_64
0x40
34
napi_gro_receive
vmlinux 0xffffffff81588318
Build id: 3.10.0-693.el7.x86_64
0xd8
35
e1000_clean_rx_irq
e1000 0xffffffffc003047c
Build id: 3.10.0-693.el7.x86_64
0x2ac
36
e1000_clean
e1000 0xffffffffc0030ed8
Build id: 3.10.0-693.el7.x86_64
0x278
37
? __wake_up_common
vmlinux 0xffffffff810ba588
Build id: 3.10.0-693.el7.x86_64
0x58
38
net_rx_action
vmlinux 0xffffffff8158799d
Build id: 3.10.0-693.el7.x86_64
0x16d
39
__do_softirq
vmlinux 0xffffffff81090b3f
Build id: 3.10.0-693.el7.x86_64
0xef
40
run_ksoftirqd
vmlinux 0xffffffff81090d08
Build id: 3.10.0-693.el7.x86_64
0x38
41
smpboot_thread_fn
vmlinux 0xffffffff810b909f
Build id: 3.10.0-693.el7.x86_64
0x12f
42
? lg_double_unlock
vmlinux 0xffffffff810b8f70
Build id: 3.10.0-693.el7.x86_64
0x40
43
kthread
vmlinux 0xffffffff810b098f
Build id: 3.10.0-693.el7.x86_64
0xcf
44
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
45
ret_from_fork
vmlinux 0xffffffff816b4f18
Build id: 3.10.0-693.el7.x86_64
0x58
46
? insert_kthread_work
vmlinux 0xffffffff810b08c0
Build id: 3.10.0-693.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 25s! [ksoftirqd/3:23]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 snd_ens1371 iosf_mbi snd_rawmidi snd_ac97_codec crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq ppdev nfit snd_seq_device libnvdimm aesni_intel vmw_balloon snd_pcm lrw gf128mul glue_helper ablk_helper cryptd snd_timer joydev pcspkr sg snd i2c_piix4 vmw_vmci soundcore shpchp parport_pc parport fuse ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel libata mptspi scsi_transport_spi serio_raw mptscsih e1000 mptbase i2c_core
CPU: 3 PID: 23 Comm: ksoftirqd/3 Tainted: G             L ------------   3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff88017d30cf10 ti: ffff88017d328000 task.ti: ffff88017d328000
RIP: 0010:[<ffffffffc002eee7>]  [<ffffffffc002eee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff88017d32b4c0  EFLAGS: 00000286
RAX: ffffc90000d03818 RBX: ffff8800654c204c RCX: 000000000000005d
RDX: 000000000000005d RSI: ffff880036211e80 RDI: ffff8801bf874000
RBP: ffff88017d32b540 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 00000000d3569145
R13: 0000000000000286 R14: 0000000000000010 R15: ffffffffc002eee7
FS:  0000000000000000(0000) GS:ffff8801b76c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9bf84ee258 CR3: 00000001a3eb8000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff880138f6242a ffff8801b7c6c098 0000005b81586771 0000000900000000
00000000ace29b00 ffff880100000000 ffffffff00000000 0000000100001000
ffff880188accc00 ffff8801bf874000 ffff8801b7c6c000 ffff8801bf874000
Call Trace:
[<ffffffff81586771>] dev_hard_start_xmit+0x171/0x3b0
[<ffffffff815af50a>] sch_direct_xmit+0x11a/0x240
[<ffffffff81589566>] __dev_queue_xmit+0x226/0x550
[<ffffffff815898a0>] dev_queue_xmit+0x10/0x20
[<ffffffff8159527b>] neigh_resolve_output+0x11b/0x220
[<ffffffff815cfaa7>] ip_finish_output+0x297/0x780
[<ffffffff815d0293>] ip_output+0x73/0xe0
[<ffffffff815cf810>] ? __ip_append_data.isra.48+0xa00/0xa00
[<ffffffff815cdf31>] ip_local_out_sk+0x31/0x40
[<ffffffff815ce0a2>] ip_build_and_send_pkt+0x162/0x220
[<ffffffff815eff64>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff8160cbd6>] ? __fib_validate_source.isra.12+0x2b6/0x400
[<ffffffff815e637b>] tcp_rtx_synack+0x3b/0x70
[<ffffffff81090007>] ? local_bh_enable+0x17/0x20
[<ffffffffc0050d06>] ? ipt_do_table+0x336/0x710 [ip_tables]
[<ffffffff81090007>] ? local_bh_enable+0x17/0x20
[<ffffffffc0050d06>] ? ipt_do_table+0x336/0x710 [ip_tables]
[<ffffffff815d54b7>] inet_rtx_syn_ack+0x17/0x30
[<ffffffff815f1729>] tcp_check_req+0x239/0x580
[<ffffffff815ef26f>] tcp_v4_do_rcv+0x21f/0x340
[<ffffffff815f08a8>] tcp_v4_rcv+0x798/0x9a0
[<ffffffff815ca054>] ip_local_deliver_finish+0xb4/0x1f0
[<ffffffff815ca339>] ip_local_deliver+0x59/0xd0
[<ffffffff815c9fa0>] ? ip_rcv_finish+0x350/0x350
[<ffffffff815c9cda>] ip_rcv_finish+0x8a/0x350
[<ffffffff815ca666>] ip_rcv+0x2b6/0x410
[<ffffffff815c9c50>] ? inet_del_offload+0x40/0x40
[<ffffffff81586f22>] __netif_receive_skb_core+0x572/0x7c0
[<ffffffff815f5944>] ? tcp4_gro_receive+0x134/0x1b0
[<ffffffff81607087>] ? inet_gro_receive+0x247/0x290
[<ffffffff81587188>] __netif_receive_skb+0x18/0x60
[<ffffffff81587210>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff81588318>] napi_gro_receive+0xd8/0x130
[<ffffffffc003047c>] e1000_clean_rx_irq+0x2ac/0x4f0 [e1000]
[<ffffffffc0030ed8>] e1000_clean+0x278/0x8d0 [e1000]
[<ffffffff810ba588>] ? __wake_up_common+0x58/0x90
[<ffffffff8158799d>] net_rx_action+0x16d/0x380
[<ffffffff81090b3f>] __do_softirq+0xef/0x280
[<ffffffff81090d08>] run_ksoftirqd+0x38/0x50
[<ffffffff810b909f>] smpboot_thread_fn+0x12f/0x180
[<ffffffff810b8f70>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b098f>] kthread+0xcf/0xe0
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
[<ffffffff816b4f18>] ret_from_fork+0x58/0x90
[<ffffffff810b08c0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #1160732
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc00c18e7
Build id: 3.10.0-1160.119.1.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffb3466c7f
Build id: 3.10.0-1160.119.1.el7.x86_64
0x3f
3
dev_hard_start_xmit
vmlinux 0xffffffffb347edc6
Build id: 3.10.0-1160.119.1.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffffb34add72
Build id: 3.10.0-1160.119.1.el7.x86_64
0x122
5
__dev_queue_xmit
vmlinux 0xffffffffb3481dd1
Build id: 3.10.0-1160.119.1.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffffb3481fa0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffffb34d260b
Build id: 3.10.0-1160.119.1.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffffb34d2b9b
Build id: 3.10.0-1160.119.1.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffffb34d1f90
Build id: 3.10.0-1160.119.1.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffffb34d0587
Build id: 3.10.0-1160.119.1.el7.x86_64
0x37
11
ip_build_and_send_pkt
vmlinux 0xffffffffb34d06f2
Build id: 3.10.0-1160.119.1.el7.x86_64
0x162
12
tcp_v4_send_synack
vmlinux 0xffffffffb34f3fe4
Build id: 3.10.0-1160.119.1.el7.x86_64
0x74
13
? tcp_openreq_init_rwin
vmlinux 0xffffffffb34f5c8c
Build id: 3.10.0-1160.119.1.el7.x86_64
0xcc
14
tcp_conn_request
vmlinux 0xffffffffb34e0a65
Build id: 3.10.0-1160.119.1.el7.x86_64
0x525
15
? get_unique_tuple
nf_nat 0xffffffffc066608c
Build id: 3.10.0-1160.119.1.el7.x86_64
0x59c
16
tcp_v4_conn_request
vmlinux 0xffffffffb34f18a5
Build id: 3.10.0-1160.119.1.el7.x86_64
0x55
17
tcp_rcv_state_process
vmlinux 0xffffffffb34e874c
Build id: 3.10.0-1160.119.1.el7.x86_64
0x1ac
18
? __inet_lookup_established
vmlinux 0xffffffffb34d6576
Build id: 3.10.0-1160.119.1.el7.x86_64
0x46
19
tcp_v4_do_rcv
vmlinux 0xffffffffb34f316b
Build id: 3.10.0-1160.119.1.el7.x86_64
0x18b
20
tcp_v4_rcv
vmlinux 0xffffffffb34f498c
Build id: 3.10.0-1160.119.1.el7.x86_64
0x7dc
21
ip_local_deliver_finish
vmlinux 0xffffffffb34cc34d
Build id: 3.10.0-1160.119.1.el7.x86_64
0xbd
22
ip_local_deliver
vmlinux 0xffffffffb34cc660
Build id: 3.10.0-1160.119.1.el7.x86_64
0x60
23
? ip_rcv_finish
vmlinux 0xffffffffb34cc290
Build id: 3.10.0-1160.119.1.el7.x86_64
0x380
24
ip_rcv_finish
vmlinux 0xffffffffb34cbfa0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x90
25
ip_rcv
vmlinux 0xffffffffb34cc9a0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x2c0
26
? packet_create
vmlinux 0xffffffffb358b102
Build id: 3.10.0-1160.119.1.el7.x86_64
0x1f2
27
? inet_del_offload
vmlinux 0xffffffffb34cbf10
Build id: 3.10.0-1160.119.1.el7.x86_64
0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffb347f669
Build id: 3.10.0-1160.119.1.el7.x86_64
0x739
29
? __getnstimeofday64
vmlinux 0xffffffffb2f0eabf
Build id: 3.10.0-1160.119.1.el7.x86_64
0x3f
30
__netif_receive_skb
vmlinux 0xffffffffb347f968
Build id: 3.10.0-1160.119.1.el7.x86_64
0x18
31
netif_receive_skb_internal
vmlinux 0xffffffffb347fa00
Build id: 3.10.0-1160.119.1.el7.x86_64
0x40
32
napi_gro_receive
vmlinux 0xffffffffb34806b8
Build id: 3.10.0-1160.119.1.el7.x86_64
0xd8
33
e1000_clean_rx_irq
e1000 0xffffffffc00c24cb
Build id: 3.10.0-1160.119.1.el7.x86_64
0x2bb
34
e1000_clean
e1000 0xffffffffc00c3180
Build id: 3.10.0-1160.119.1.el7.x86_64
0x280
35
? __wake_up_common_lock
vmlinux 0xffffffffb2ed8f31
Build id: 3.10.0-1160.119.1.el7.x86_64
0x91
36
net_rx_action
vmlinux 0xffffffffb3480037
Build id: 3.10.0-1160.119.1.el7.x86_64
0x277
37
__do_softirq
vmlinux 0xffffffffb2ea9595
Build id: 3.10.0-1160.119.1.el7.x86_64
0xf5
38
run_ksoftirqd
vmlinux 0xffffffffb2ea9768
Build id: 3.10.0-1160.119.1.el7.x86_64
0x38
39
smpboot_thread_fn
vmlinux 0xffffffffb2ed43fc
Build id: 3.10.0-1160.119.1.el7.x86_64
0x14c
40
? lg_double_unlock
vmlinux 0xffffffffb2ed42b0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x40
41
kthread
vmlinux 0xffffffffb2ecb621
Build id: 3.10.0-1160.119.1.el7.x86_64
0xd1
42
? insert_kthread_work
vmlinux 0xffffffffb2ecb550
Build id: 3.10.0-1160.119.1.el7.x86_64
0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffffb35c51f7
Build id: 3.10.0-1160.119.1.el7.x86_64
0x21
44
? insert_kthread_work
vmlinux 0xffffffffb2ecb550
Build id: 3.10.0-1160.119.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm nfit mptspi ata_piix scsi_transport_spi libnvdimm crct10dif_pclmul crct10dif_common mptscsih libata e1000
mptbase crc32c_intel serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff95b9b06b5280 ti: ffff95b9b0774000 task.ti: ffff95b9b0774000
RIP: 0010:[<ffffffffc00c18e7>]  [<ffffffffc00c18e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff95b9b0777440  EFLAGS: 00000286
RAX: ffffb3e0034c3818 RBX: 0000000000000001 RCX: 00000000000000ac
RDX: 00000000000000ac RSI: ffff95bc8491c900 RDI: ffff95be743f4000
RBP: ffff95b9b07774c0 R08: 0000000000000100 R09: 0000000022100000
R10: 340000450008e375 R11: 0000000000000000 R12: 0000000000000202
R13: 00000003db9da8ce R14: 00000001b3f35bc0 R15: ffffffffb31c9150
FS:  0000000000000000(0000) GS:ffff95be75200000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002398ebabf054 CR3: 0000000444610000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb3466c7f>] ? kfree_skbmem+0x3f/0xa0
[<ffffffffb347edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb34add72>] sch_direct_xmit+0x122/0x250
[<ffffffffb3481dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb3481fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffb34d260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb34d2b9b>] ip_output+0x7b/0x100
[<ffffffffb34d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb34d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffb34d06f2>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffb34f3fe4>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffb34f5c8c>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffffb34e0a65>] tcp_conn_request+0x525/0x760
[<ffffffffc066608c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffffb34f18a5>] tcp_v4_conn_request+0x55/0x60
[<ffffffffb34e874c>] tcp_rcv_state_process+0x1ac/0xf50
[<ffffffffb34d6576>] ? __inet_lookup_established+0x46/0x130
[<ffffffffb34f316b>] tcp_v4_do_rcv+0x18b/0x360
[<ffffffffb34f498c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffb34cc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffffb34cc660>] ip_local_deliver+0x60/0xe0
[<ffffffffb34cc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffffb34cbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffffb34cc9a0>] ip_rcv+0x2c0/0x420
[<ffffffffb358b102>] ? packet_create+0x1f2/0x2a0
[<ffffffffb34cbf10>] ? inet_del_offload+0x40/0x40
[<ffffffffb347f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffffb2f0eabf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb347f968>] __netif_receive_skb+0x18/0x70
[<ffffffffb347fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb34806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc00c24cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc00c3180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb2ed8f31>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffffb3480037>] net_rx_action+0x277/0x3a0
[<ffffffffb2ea9595>] __do_softirq+0xf5/0x290
[<ffffffffb2ea9768>] run_ksoftirqd+0x38/0x60
[<ffffffffb2ed43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffffb2ed42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb2ecb621>] kthread+0xd1/0xe0
[<ffffffffb2ecb550>] ? insert_kthread_work+0x40/0x40
[<ffffffffb35c51f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb2ecb550>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 
Complete report #1101562
Frame # Function Binary Source or offset Line
1
dev_hard_start_xmit
vmlinux 0xffffffffa2a54386
Build id: 3.10.0-1160.el7.x86_64
0x246
2
sch_direct_xmit
vmlinux 0xffffffffa2a8242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
3
__dev_queue_xmit
vmlinux 0xffffffffa2a57301
Build id: 3.10.0-1160.el7.x86_64
0x491
4
dev_queue_xmit
vmlinux 0xffffffffa2a574d0
Build id: 3.10.0-1160.el7.x86_64
0x10
5
ip_finish_output
vmlinux 0xffffffffa2aa545b
Build id: 3.10.0-1160.el7.x86_64
0x67b
6
ip_output
vmlinux 0xffffffffa2aa59db
Build id: 3.10.0-1160.el7.x86_64
0x7b
7
? __ip_append_data.isra.50
vmlinux 0xffffffffa2aa4de0
Build id: 3.10.0-1160.el7.x86_64
0xa60
8
ip_local_out_sk
vmlinux 0xffffffffa2aa3407
Build id: 3.10.0-1160.el7.x86_64
0x37
9
ip_build_and_send_pkt
vmlinux 0xffffffffa2aa3572
Build id: 3.10.0-1160.el7.x86_64
0x162
10
tcp_v4_send_synack
vmlinux 0xffffffffa2ac6664
Build id: 3.10.0-1160.el7.x86_64
0x74
11
? get_unique_tuple
nf_nat 0xffffffffc093001c
Build id: 3.10.0-1160.el7.x86_64
0x59c
12
tcp_rtx_synack
vmlinux 0xffffffffa2abc4f1
Build id: 3.10.0-1160.el7.x86_64
0x41
13
? local_bh_enable
vmlinux 0xffffffffa24a4017
Build id: 3.10.0-1160.el7.x86_64
0x17
14
? ipt_do_table
ip_tables 0xffffffffc0528719
Build id: 3.10.0-1160.el7.x86_64
0x359
15
? local_bh_enable
vmlinux 0xffffffffa24a4017
Build id: 3.10.0-1160.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc0528719
Build id: 3.10.0-1160.el7.x86_64
0x359
17
inet_rtx_syn_ack
vmlinux 0xffffffffa2aab02d
Build id: 3.10.0-1160.el7.x86_64
0x1d
18
tcp_check_req
vmlinux 0xffffffffa2ac7d39
Build id: 3.10.0-1160.el7.x86_64
0x239
19
tcp_v4_do_rcv
vmlinux 0xffffffffa2ac58cf
Build id: 3.10.0-1160.el7.x86_64
0x22f
20
tcp_v4_rcv
vmlinux 0xffffffffa2ac6ffc
Build id: 3.10.0-1160.el7.x86_64
0x7dc
21
ip_local_deliver_finish
vmlinux 0xffffffffa2a9f28d
Build id: 3.10.0-1160.el7.x86_64
0xbd
22
ip_local_deliver
vmlinux 0xffffffffa2a9f580
Build id: 3.10.0-1160.el7.x86_64
0x60
23
? ip_rcv_finish
vmlinux 0xffffffffa2a9f1d0
Build id: 3.10.0-1160.el7.x86_64
0x370
24
ip_rcv_finish
vmlinux 0xffffffffa2a9eef0
Build id: 3.10.0-1160.el7.x86_64
0x90
25
ip_rcv
vmlinux 0xffffffffa2a9f8c0
Build id: 3.10.0-1160.el7.x86_64
0x2c0
26
? next_arg
vmlinux 0xffffffffa2785d02
Build id: 3.10.0-1160.el7.x86_64
0xe2
27
? inet_del_offload
vmlinux 0xffffffffa2a9ee60
Build id: 3.10.0-1160.el7.x86_64
0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffa2a54c19
Build id: 3.10.0-1160.el7.x86_64
0x729
29
? tcp4_gro_receive
vmlinux 0xffffffffa2acc100
Build id: 3.10.0-1160.el7.x86_64
0x80
30
? inet_gro_receive
vmlinux 0xffffffffa2addd99
Build id: 3.10.0-1160.el7.x86_64
0x249
31
__netif_receive_skb
vmlinux 0xffffffffa2a54f18
Build id: 3.10.0-1160.el7.x86_64
0x18
32
netif_receive_skb_internal
vmlinux 0xffffffffa2a54fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
33
napi_gro_receive
vmlinux 0xffffffffa2a55c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc044345b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
35
e1000_clean
e1000 0xffffffffc0444100
Build id: 3.10.0-1160.el7.x86_64
0x280
36
? update_curr
vmlinux 0xffffffffa24e43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
37
net_rx_action
vmlinux 0xffffffffa2a555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
38
__do_softirq
vmlinux 0xffffffffa24a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
39
run_ksoftirqd
vmlinux 0xffffffffa24a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
40
smpboot_thread_fn
vmlinux 0xffffffffa24ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
41
? lg_double_unlock
vmlinux 0xffffffffa24ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
42
kthread
vmlinux 0xffffffffa24c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
43
? insert_kthread_work
vmlinux 0xffffffffa24c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
44
ret_from_fork_nospec_begin
vmlinux 0xffffffffa2b93df7
Build id: 3.10.0-1160.el7.x86_64
0x21
45
? insert_kthread_work
vmlinux 0xffffffffa24c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 40s! [X:1009]
[<ffffffffa2a54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa2a8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa2a57301>] __dev_queue_xmit+0x491/0x650
[<ffffffffa2a574d0>] dev_queue_xmit+0x10/0x20
[<ffffffffa2aa545b>] ip_finish_output+0x67b/0x8d0
[<ffffffffa2aa59db>] ip_output+0x7b/0xf0
[<ffffffffa2aa4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa2aa3407>] ip_local_out_sk+0x37/0x40
[<ffffffffa2aa3572>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffa2ac6664>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffc093001c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffffa2abc4f1>] tcp_rtx_synack+0x41/0x70
[<ffffffffa24a4017>] ? local_bh_enable+0x17/0x20
[<ffffffffc0528719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffa24a4017>] ? local_bh_enable+0x17/0x20
[<ffffffffc0528719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffa2aab02d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffa2ac7d39>] tcp_check_req+0x239/0x5b0
[<ffffffffa2ac58cf>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffa2ac6ffc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffa2a9f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffa2a9f580>] ip_local_deliver+0x60/0xe0
[<ffffffffa2a9f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffa2a9eef0>] ip_rcv_finish+0x90/0x370
[<ffffffffa2a9f8c0>] ip_rcv+0x2c0/0x420
[<ffffffffa2785d02>] ? next_arg+0xe2/0x110
[<ffffffffa2a9ee60>] ? inet_del_offload+0x40/0x40
[<ffffffffa2a54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffa2acc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffffa2addd99>] ? inet_gro_receive+0x249/0x290
[<ffffffffa2a54f18>] __netif_receive_skb+0x18/0x60
[<ffffffffa2a54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffa2a55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc044345b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0444100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffa24e43cc>] ? update_curr+0x14c/0x1e0
[<ffffffffa2a555bf>] net_rx_action+0x26f/0x390
[<ffffffffa24a4b95>] __do_softirq+0xf5/0x280
[<ffffffffa24a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffa24ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffa24ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffa24c5c21>] kthread+0xd1/0xe0
[<ffffffffa24c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffa2b93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffa24c5b50>] ? insert_kthread_work+0x40/0x40
Complete report #859540
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03bc8e7
Build id: 3.10.0-1160.88.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff91c7edc6
Build id: 3.10.0-1160.88.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff91cadd72
Build id: 3.10.0-1160.88.1.el7.x86_64
0x122
4
__qdisc_run
vmlinux 0xffffffff91cadf2e
Build id: 3.10.0-1160.88.1.el7.x86_64
0x8e
5
__dev_queue_xmit
vmlinux 0xffffffff91c81de9
Build id: 3.10.0-1160.88.1.el7.x86_64
0x4a9
6
dev_queue_xmit
vmlinux 0xffffffff91c81fa0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x10
7
neigh_resolve_output
vmlinux 0xffffffff91c8f03d
Build id: 3.10.0-1160.88.1.el7.x86_64
0x11d
8
ip_finish_output
vmlinux 0xffffffff91cd220c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x27c
9
ip_output
vmlinux 0xffffffff91cd2b9b
Build id: 3.10.0-1160.88.1.el7.x86_64
0x7b
10
? __ip_append_data.isra.50
vmlinux 0xffffffff91cd1f90
Build id: 3.10.0-1160.88.1.el7.x86_64
0xa60
11
ip_local_out_sk
vmlinux 0xffffffff91cd0587
Build id: 3.10.0-1160.88.1.el7.x86_64
0x37
12
ip_build_and_send_pkt
vmlinux 0xffffffff91cd06f2
Build id: 3.10.0-1160.88.1.el7.x86_64
0x162
13
tcp_v4_send_synack
vmlinux 0xffffffff91cf3fe4
Build id: 3.10.0-1160.88.1.el7.x86_64
0x74
14
? tcp_openreq_init_rwin
vmlinux 0xffffffff91cf5c8c
Build id: 3.10.0-1160.88.1.el7.x86_64
0xcc
15
tcp_conn_request
vmlinux 0xffffffff91ce0b17
Build id: 3.10.0-1160.88.1.el7.x86_64
0x5d7
16
? get_unique_tuple
nf_nat 0xffffffffc097c08c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x59c
17
? ipt_do_table
ip_tables 0xffffffffc0715761
Build id: 3.10.0-1160.88.1.el7.x86_64
0x361
18
tcp_v4_conn_request
vmlinux 0xffffffff91cf18a5
Build id: 3.10.0-1160.88.1.el7.x86_64
0x55
19
tcp_rcv_state_process
vmlinux 0xffffffff91ce874c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x1ac
20
? __inet_lookup_established
vmlinux 0xffffffff91cd6576
Build id: 3.10.0-1160.88.1.el7.x86_64
0x46
21
tcp_v4_do_rcv
vmlinux 0xffffffff91cf316b
Build id: 3.10.0-1160.88.1.el7.x86_64
0x18b
22
tcp_v4_rcv
vmlinux 0xffffffff91cf498c
Build id: 3.10.0-1160.88.1.el7.x86_64
0x7dc
23
ip_local_deliver_finish
vmlinux 0xffffffff91ccc34d
Build id: 3.10.0-1160.88.1.el7.x86_64
0xbd
24
ip_local_deliver
vmlinux 0xffffffff91ccc660
Build id: 3.10.0-1160.88.1.el7.x86_64
0x60
25
? ip_rcv_finish
vmlinux 0xffffffff91ccc290
Build id: 3.10.0-1160.88.1.el7.x86_64
0x380
26
ip_rcv_finish
vmlinux 0xffffffff91ccbfa0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x90
27
ip_rcv
vmlinux 0xffffffff91ccc9a0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x2c0
28
? __slab_free
vmlinux 0xffffffff91833202
Build id: 3.10.0-1160.88.1.el7.x86_64
0x62
29
? inet_del_offload
vmlinux 0xffffffff91ccbf10
Build id: 3.10.0-1160.88.1.el7.x86_64
0x40
30
__netif_receive_skb_core
vmlinux 0xffffffff91c7f669
Build id: 3.10.0-1160.88.1.el7.x86_64
0x739
31
? tcp4_gro_receive
vmlinux 0xffffffff91cf9b00
Build id: 3.10.0-1160.88.1.el7.x86_64
0x30
32
? inet_gro_receive
vmlinux 0xffffffff91d0bcf9
Build id: 3.10.0-1160.88.1.el7.x86_64
0x249
33
__netif_receive_skb
vmlinux 0xffffffff91c7f968
Build id: 3.10.0-1160.88.1.el7.x86_64
0x18
34
netif_receive_skb_internal
vmlinux 0xffffffff91c7fa00
Build id: 3.10.0-1160.88.1.el7.x86_64
0x40
35
napi_gro_receive
vmlinux 0xffffffff91c806b8
Build id: 3.10.0-1160.88.1.el7.x86_64
0xd8
36
e1000_clean_rx_irq
e1000 0xffffffffc03bd4cb
Build id: 3.10.0-1160.88.1.el7.x86_64
0x2bb
37
e1000_clean
e1000 0xffffffffc03be180
Build id: 3.10.0-1160.88.1.el7.x86_64
0x280
38
? try_to_wake_up
vmlinux 0xffffffff916e0e92
Build id: 3.10.0-1160.88.1.el7.x86_64
0x192
39
net_rx_action
vmlinux 0xffffffff91c80037
Build id: 3.10.0-1160.88.1.el7.x86_64
0x277
40
__do_softirq
vmlinux 0xffffffff916a9595
Build id: 3.10.0-1160.88.1.el7.x86_64
0xf5
41
run_ksoftirqd
vmlinux 0xffffffff916a9768
Build id: 3.10.0-1160.88.1.el7.x86_64
0x38
42
smpboot_thread_fn
vmlinux 0xffffffff916d43fc
Build id: 3.10.0-1160.88.1.el7.x86_64
0x14c
43
? lg_double_unlock
vmlinux 0xffffffff916d42b0
Build id: 3.10.0-1160.88.1.el7.x86_64
0x40
44
kthread
vmlinux 0xffffffff916cb621
Build id: 3.10.0-1160.88.1.el7.x86_64
0xd1
45
? insert_kthread_work
vmlinux 0xffffffff916cb550
Build id: 3.10.0-1160.88.1.el7.x86_64
0x40
46
ret_from_fork_nospec_begin
vmlinux 0xffffffff91dc51f7
Build id: 3.10.0-1160.88.1.el7.x86_64
0x21
47
? insert_kthread_work
vmlinux 0xffffffff916cb550
Build id: 3.10.0-1160.88.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 155s! [ksoftirqd/0:6]
Modules linked in: xt_nat tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev snd_ens1371 snd_rawmidi aesni_intel snd_ac97_codec ac97_bus vmw_balloon lrw gf128mul snd_seq glue_helper ablk_helper cryptd snd_seq_device snd_pcm joydev pcspkr sg snd_timer snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common nfit crc32c_intel libata libnvdimm mptspi e1000 scsi_transport_spi mptscsih serio_raw drm_panel_orientation_quirks mptbase fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.88.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9febb63ad280 ti: ffff9febb5cd8000 task.ti: ffff9febb5cd8000
RIP: 0010:[<ffffffffc03bc8e7>]  [<ffffffffc03bc8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9febb5cdb3a8  EFLAGS: 00000286
RAX: ffffb47901983818 RBX: 000000000000003c RCX: 0000000000000086
RDX: 0000000000000086 RSI: ffff9fee67e36140 RDI: ffff9fee6c64e000
RBP: ffff9febb5cdb428 R08: 0000000000000100 R09: 0000000022100000
R10: 6e9d280000450008 R11: 0000000000000000 R12: 00000003e2f030da
R13: 0000000100000000 R14: ffffffff919c8fd0 R15: ffff9febb5cdb360
FS:  0000000000000000(0000) GS:ffff9fee6d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbf3c99d000 CR3: 00000003b63dc000 CR4: 00000000000007f0
Call Trace:
[<ffffffff91c7edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff91cadd72>] sch_direct_xmit+0x122/0x250
[<ffffffff91cadf2e>] __qdisc_run+0x8e/0x360
[<ffffffff91c81de9>] __dev_queue_xmit+0x4a9/0x650
[<ffffffff91c81fa0>] dev_queue_xmit+0x10/0x20
[<ffffffff91c8f03d>] neigh_resolve_output+0x11d/0x220
[<ffffffff91cd220c>] ip_finish_output+0x27c/0x8d0
[<ffffffff91cd2b9b>] ip_output+0x7b/0x100
[<ffffffff91cd1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff91cd0587>] ip_local_out_sk+0x37/0x40
[<ffffffff91cd06f2>] ip_build_and_send_pkt+0x162/0x250
[<ffffffff91cf3fe4>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff91cf5c8c>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffff91ce0b17>] tcp_conn_request+0x5d7/0x760
[<ffffffffc097c08c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffffc0715761>] ? ipt_do_table+0x361/0x750 [ip_tables]
[<ffffffff91cf18a5>] tcp_v4_conn_request+0x55/0x60
[<ffffffff91ce874c>] tcp_rcv_state_process+0x1ac/0xf50
[<ffffffff91cd6576>] ? __inet_lookup_established+0x46/0x130
[<ffffffff91cf316b>] tcp_v4_do_rcv+0x18b/0x360
[<ffffffff91cf498c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffff91ccc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffff91ccc660>] ip_local_deliver+0x60/0xe0
[<ffffffff91ccc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffff91ccbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffff91ccc9a0>] ip_rcv+0x2c0/0x420
[<ffffffff91833202>] ? __slab_free+0x62/0x2b0
[<ffffffff91ccbf10>] ? inet_del_offload+0x40/0x40
[<ffffffff91c7f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffff91cf9b00>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffff91d0bcf9>] ? inet_gro_receive+0x249/0x290
[<ffffffff91c7f968>] __netif_receive_skb+0x18/0x70
[<ffffffff91c7fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff91c806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc03bd4cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03be180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff916e0e92>] ? try_to_wake_up+0x192/0x3a0
[<ffffffff91c80037>] net_rx_action+0x277/0x3a0
[<ffffffff916a9595>] __do_softirq+0xf5/0x290
[<ffffffff916a9768>] run_ksoftirqd+0x38/0x60
[<ffffffff916d43fc>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffff916d42b0>] ? lg_double_unlock+0x40/0x40
[<ffffffff916cb621>] kthread+0xd1/0xe0
[<ffffffff916cb550>] ? insert_kthread_work+0x40/0x40
[<ffffffff91dc51f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff916cb550>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 
Complete report #823379
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0359877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff91254386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff9128242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff91257301
Build id: 3.10.0-1160.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffff912574d0
Build id: 3.10.0-1160.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffff912a545b
Build id: 3.10.0-1160.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffff912a59db
Build id: 3.10.0-1160.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffff912a4de0
Build id: 3.10.0-1160.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffff912a3407
Build id: 3.10.0-1160.el7.x86_64
0x37
10
ip_build_and_send_pkt
vmlinux 0xffffffff912a3572
Build id: 3.10.0-1160.el7.x86_64
0x162
11
tcp_v4_send_synack
vmlinux 0xffffffff912c6664
Build id: 3.10.0-1160.el7.x86_64
0x74
12
? tcp_openreq_init_rwin
vmlinux 0xffffffff912c82ec
Build id: 3.10.0-1160.el7.x86_64
0xcc
13
tcp_conn_request
vmlinux 0xffffffff912b37d5
Build id: 3.10.0-1160.el7.x86_64
0x525
14
? get_unique_tuple
nf_nat 0xffffffffc086e01c
Build id: 3.10.0-1160.el7.x86_64
0x59c
15
tcp_v4_conn_request
vmlinux 0xffffffff912c3fdd
Build id: 3.10.0-1160.el7.x86_64
0x4d
16
tcp_rcv_state_process
vmlinux 0xffffffff912bb0dc
Build id: 3.10.0-1160.el7.x86_64
0x1ac
17
? __inet_lookup_established
vmlinux 0xffffffff912a92e6
Build id: 3.10.0-1160.el7.x86_64
0x46
18
tcp_v4_do_rcv
vmlinux 0xffffffff912c581b
Build id: 3.10.0-1160.el7.x86_64
0x17b
19
tcp_v4_rcv
vmlinux 0xffffffff912c6ffc
Build id: 3.10.0-1160.el7.x86_64
0x7dc
20
ip_local_deliver_finish
vmlinux 0xffffffff9129f28d
Build id: 3.10.0-1160.el7.x86_64
0xbd
21
ip_local_deliver
vmlinux 0xffffffff9129f580
Build id: 3.10.0-1160.el7.x86_64
0x60
22
? ip_rcv_finish
vmlinux 0xffffffff9129f1d0
Build id: 3.10.0-1160.el7.x86_64
0x370
23
ip_rcv_finish
vmlinux 0xffffffff9129eef0
Build id: 3.10.0-1160.el7.x86_64
0x90
24
ip_rcv
vmlinux 0xffffffff9129f8c0
Build id: 3.10.0-1160.el7.x86_64
0x2c0
25
? inet_del_offload
vmlinux 0xffffffff9129ee60
Build id: 3.10.0-1160.el7.x86_64
0x40
26
__netif_receive_skb_core
vmlinux 0xffffffff91254c19
Build id: 3.10.0-1160.el7.x86_64
0x729
27
? tcp4_gro_receive
vmlinux 0xffffffff912cc100
Build id: 3.10.0-1160.el7.x86_64
0x80
28
? inet_gro_receive
vmlinux 0xffffffff912ddd99
Build id: 3.10.0-1160.el7.x86_64
0x249
29
__netif_receive_skb
vmlinux 0xffffffff91254f18
Build id: 3.10.0-1160.el7.x86_64
0x18
30
netif_receive_skb_internal
vmlinux 0xffffffff91254fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
31
napi_gro_receive
vmlinux 0xffffffff91255c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc035a45b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
33
e1000_clean
e1000 0xffffffffc035b100
Build id: 3.10.0-1160.el7.x86_64
0x280
34
net_rx_action
vmlinux 0xffffffff912555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
35
__do_softirq
vmlinux 0xffffffff90ca4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
36
run_ksoftirqd
vmlinux 0xffffffff90ca4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
37
smpboot_thread_fn
vmlinux 0xffffffff90cce674
Build id: 3.10.0-1160.el7.x86_64
0x144
38
? lg_double_unlock
vmlinux 0xffffffff90cce530
Build id: 3.10.0-1160.el7.x86_64
0x40
39
kthread
vmlinux 0xffffffff90cc5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
40
? insert_kthread_work
vmlinux 0xffffffff90cc5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
41
ret_from_fork_nospec_begin
vmlinux 0xffffffff91393ddd
Build id: 3.10.0-1160.el7.x86_64
0x7
42
? insert_kthread_work
vmlinux 0xffffffff90cc5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 62s! [ksoftirqd/7:44]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc kvm_intel kvm nfit irqbypass libnvdimm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev pcspkr sg joydev vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix drm libata mptspi serio_raw scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff99b899b82100 ti: ffff99b899b94000 task.ti: ffff99b899b94000
RIP: 0010:[<ffffffffc0359877>]  [<ffffffffc0359877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff99b899b97440  EFLAGS: 00000286
RAX: ffffb801035c3818 RBX: 0000000000000001 RCX: 00000000000000fe
RDX: 00000000000000fe RSI: ffff99b8fa794bc0 RDI: ffff99b8a17be000
RBP: ffff99b899b974c0 R08: 0000000000000100 R09: 0000000022100000
R10: cd9c064000400000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000005a922a4c6 R14: 00000001e922a4e8 R15: ffffffff90fb1650
FS:  0000000000000000(0000) GS:ffff99bce77c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff0c1d00000 CR3: 000000000950e000 CR4: 00000000003607e0
Call Trace:
[<ffffffff91254386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9128242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff91257301>] __dev_queue_xmit+0x491/0x650
[<ffffffff912574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff912a545b>] ip_finish_output+0x67b/0x8d0
[<ffffffff912a59db>] ip_output+0x7b/0xf0
[<ffffffff912a4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff912a3407>] ip_local_out_sk+0x37/0x40
[<ffffffff912a3572>] ip_build_and_send_pkt+0x162/0x250
[<ffffffff912c6664>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff912c82ec>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffff912b37d5>] tcp_conn_request+0x525/0x760
[<ffffffffc086e01c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffff912c3fdd>] tcp_v4_conn_request+0x4d/0x50
[<ffffffff912bb0dc>] tcp_rcv_state_process+0x1ac/0xf50
[<ffffffff912a92e6>] ? __inet_lookup_established+0x46/0x130
[<ffffffff912c581b>] tcp_v4_do_rcv+0x17b/0x350
[<ffffffff912c6ffc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffff9129f28d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff9129f580>] ip_local_deliver+0x60/0xe0
[<ffffffff9129f1d0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff9129eef0>] ip_rcv_finish+0x90/0x370
[<ffffffff9129f8c0>] ip_rcv+0x2c0/0x420
[<ffffffff9129ee60>] ? inet_del_offload+0x40/0x40
[<ffffffff91254c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff912cc100>] ? tcp4_gro_receive+0x80/0x1a0
[<ffffffff912ddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffff91254f18>] __netif_receive_skb+0x18/0x60
[<ffffffff91254fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff91255c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc035a45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc035b100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff912555bf>] net_rx_action+0x26f/0x390
[<ffffffff90ca4b95>] __do_softirq+0xf5/0x280
[<ffffffff90ca4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff90cce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff90cce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff90cc5c21>] kthread+0xd1/0xe0
[<ffffffff90cc5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff91393ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff90cc5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #754251
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc014a867
Build id: 3.10.0-862.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb51eb166
Build id: 3.10.0-862.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb5216a2a
Build id: 3.10.0-862.el7.x86_64
0x11a
4
__qdisc_run
vmlinux 0xffffffffb5216bee
Build id: 3.10.0-862.el7.x86_64
0x8e
5
__dev_queue_xmit
vmlinux 0xffffffffb51eddc8
Build id: 3.10.0-862.el7.x86_64
0x218
6
dev_queue_xmit
vmlinux 0xffffffffb51ee220
Build id: 3.10.0-862.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffffb5237416
Build id: 3.10.0-862.el7.x86_64
0x546
8
ip_output
vmlinux 0xffffffffb5237973
Build id: 3.10.0-862.el7.x86_64
0x73
9
? __ip_append_data.isra.50
vmlinux 0xffffffffb5236ed0
Build id: 3.10.0-862.el7.x86_64
0xa50
10
ip_local_out_sk
vmlinux 0xffffffffb5235567
Build id: 3.10.0-862.el7.x86_64
0x37
11
ip_build_and_send_pkt
vmlinux 0xffffffffb52356d2
Build id: 3.10.0-862.el7.x86_64
0x162
12
tcp_v4_send_synack
vmlinux 0xffffffffb5257c24
Build id: 3.10.0-862.el7.x86_64
0x74
13
tcp_rtx_synack
vmlinux 0xffffffffb524ded1
Build id: 3.10.0-862.el7.x86_64
0x41
14
? local_bh_enable
vmlinux 0xffffffffb4c99df7
Build id: 3.10.0-862.el7.x86_64
0x17
15
? ipt_do_table
ip_tables 0xffffffffc047ca2e
Build id: 3.10.0-862.el7.x86_64
0x33e
16
? __fib_validate_source.isra.13
vmlinux 0xffffffffb52747a5
Build id: 3.10.0-862.el7.x86_64
0xf5
17
inet_rtx_syn_ack
vmlinux 0xffffffffb523cd8d
Build id: 3.10.0-862.el7.x86_64
0x1d
18
tcp_check_req
vmlinux 0xffffffffb5259419
Build id: 3.10.0-862.el7.x86_64
0x239
19
tcp_v4_do_rcv
vmlinux 0xffffffffb5256eff
Build id: 3.10.0-862.el7.x86_64
0x22f
20
tcp_v4_rcv
vmlinux 0xffffffffb5258568
Build id: 3.10.0-862.el7.x86_64
0x788
21
ip_local_deliver_finish
vmlinux 0xffffffffb5231699
Build id: 3.10.0-862.el7.x86_64
0xb9
22
ip_local_deliver
vmlinux 0xffffffffb5231989
Build id: 3.10.0-862.el7.x86_64
0x59
23
? ip_rcv_finish
vmlinux 0xffffffffb52315e0
Build id: 3.10.0-862.el7.x86_64
0x370
24
ip_rcv_finish
vmlinux 0xffffffffb5231300
Build id: 3.10.0-862.el7.x86_64
0x90
25
ip_rcv
vmlinux 0xffffffffb5231cb9
Build id: 3.10.0-862.el7.x86_64
0x2b9
26
? inet_del_offload
vmlinux 0xffffffffb5231270
Build id: 3.10.0-862.el7.x86_64
0x40
27
__netif_receive_skb_core
vmlinux 0xffffffffb51eb9f9
Build id: 3.10.0-862.el7.x86_64
0x729
28
? tcp4_gro_receive
vmlinux 0xffffffffb525d600
Build id: 3.10.0-862.el7.x86_64
0xe0
29
? inet_gro_receive
vmlinux 0xffffffffb526edb9
Build id: 3.10.0-862.el7.x86_64
0x249
30
__netif_receive_skb
vmlinux 0xffffffffb51ebd08
Build id: 3.10.0-862.el7.x86_64
0x18
31
netif_receive_skb_internal
vmlinux 0xffffffffb51ebd90
Build id: 3.10.0-862.el7.x86_64
0x40
32
napi_gro_receive
vmlinux 0xffffffffb51ec9b8
Build id: 3.10.0-862.el7.x86_64
0xd8
33
e1000_clean_rx_irq
e1000 0xffffffffc014b44b
Build id: 3.10.0-862.el7.x86_64
0x2bb
34
e1000_clean
e1000 0xffffffffc014c0eb
Build id: 3.10.0-862.el7.x86_64
0x27b
35
? nf_ct_delete
nf_conntrack 0xffffffffc068b04d
Build id: 3.10.0-862.el7.x86_64
0x18d
36
? mod_timer
vmlinux 0xffffffffb4ca445d
Build id: 3.10.0-862.el7.x86_64
0x14d
37
net_rx_action
vmlinux 0xffffffffb51ec3af
Build id: 3.10.0-862.el7.x86_64
0x26f
38
__do_softirq
vmlinux 0xffffffffb4c9a945
Build id: 3.10.0-862.el7.x86_64
0xf5
39
run_ksoftirqd
vmlinux 0xffffffffb4c9ab08
Build id: 3.10.0-862.el7.x86_64
0x38
40
smpboot_thread_fn
vmlinux 0xffffffffb4cc35e4
Build id: 3.10.0-862.el7.x86_64
0x144
41
? lg_double_unlock
vmlinux 0xffffffffb4cc34a0
Build id: 3.10.0-862.el7.x86_64
0x40
42
kthread
vmlinux 0xffffffffb4cbae31
Build id: 3.10.0-862.el7.x86_64
0xd1
43
? insert_kthread_work
vmlinux 0xffffffffb4cbad60
Build id: 3.10.0-862.el7.x86_64
0x40
44
ret_from_fork_nospec_begin
vmlinux 0xffffffffb531f637
Build id: 3.10.0-862.el7.x86_64
0x21
45
? insert_kthread_work
vmlinux 0xffffffffb4cbad60
Build id: 3.10.0-862.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:3]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac coretemp iosf_mbi snd_ens1371 crc32_pclmul snd_rawmidi ghash_clmulni_intel aesni_intel snd_ac97_codec lrw ac97_bus snd_seq gf128mul glue_helper snd_seq_device ablk_helper cryptd snd_pcm snd_timer snd soundcore
sg pcspkr joydev ppdev vmw_balloon nfit parport_pc parport libnvdimm shpchp i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm mptspi libata scsi_transport_spi mptscsih crct10dif_pclmul crct10dif_common crc32c_intel mptbase e1000 serio_raw i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff98b5fa3d1fa0 ti: ffff98b5fa3f4000 task.ti: ffff98b5fa3f4000
RIP: 0010:[<ffffffffc014a867>]  [<ffffffffc014a867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff98b5fa3f7488  EFLAGS: 00000286
RAX: ffffbfddc1943818 RBX: 000000000000004a RCX: 00000000000000e9
RDX: 00000000000000e9 RSI: ffff98b8a9a9d000 RDI: ffff98b5f9bc6000
RBP: ffff98b5fa3f7508 R08: 0000000000000100 R09: 0000000022100000
R10: 397f064000400000 R11: 0000000000000000 R12: 000000026bb080c6
R13: 000000010000004a R14: ffffffffb4f73e00 R15: ffff98b5fa3f7440
FS:  0000000000000000(0000) GS:ffff98b8ad600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000026e2728 CR3: 000000012e80e000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb51eb166>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb5216a2a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb5216bee>] __qdisc_run+0x8e/0x360
[<ffffffffb51eddc8>] __dev_queue_xmit+0x218/0x660
[<ffffffffb51ee220>] dev_queue_xmit+0x10/0x20
[<ffffffffb5237416>] ip_finish_output+0x546/0x7a0
[<ffffffffb5237973>] ip_output+0x73/0xe0
[<ffffffffb5236ed0>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffb5235567>] ip_local_out_sk+0x37/0x40
[<ffffffffb52356d2>] ip_build_and_send_pkt+0x162/0x220
[<ffffffffb5257c24>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffb524ded1>] tcp_rtx_synack+0x41/0x70
[<ffffffffb4c99df7>] ? local_bh_enable+0x17/0x20
[<ffffffffc047ca2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffb52747a5>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffffb523cd8d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffb5259419>] tcp_check_req+0x239/0x590
[<ffffffffb5256eff>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffb5258568>] tcp_v4_rcv+0x788/0x990
[<ffffffffb5231699>] ip_local_deliver_finish+0xb9/0x200
[<ffffffffb5231989>] ip_local_deliver+0x59/0xd0
[<ffffffffb52315e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb5231300>] ip_rcv_finish+0x90/0x370
[<ffffffffb5231cb9>] ip_rcv+0x2b9/0x410
[<ffffffffb5231270>] ? inet_del_offload+0x40/0x40
[<ffffffffb51eb9f9>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffb525d600>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffb526edb9>] ? inet_gro_receive+0x249/0x290
[<ffffffffb51ebd08>] __netif_receive_skb+0x18/0x60
[<ffffffffb51ebd90>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb51ec9b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc014b44b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc014c0eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffc068b04d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffffb4ca445d>] ? mod_timer+0x14d/0x230
[<ffffffffb51ec3af>] net_rx_action+0x26f/0x390
[<ffffffffb4c9a945>] __do_softirq+0xf5/0x280
[<ffffffffb4c9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffffb4cc35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb4cc34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb4cbae31>] kthread+0xd1/0xe0
[<ffffffffb4cbad60>] ? insert_kthread_work+0x40/0x40
[<ffffffffb531f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb4cbad60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #586313
Frame # Function Binary Source or offset Line
1
memcpy
vmlinux 0xffffffff9dbad7a8
Build id: 3.10.0-1160.80.1.el7.x86_64
0x68
2
? swiotlb_tbl_map_single
vmlinux 0xffffffff9dbc8a90
Build id: 3.10.0-1160.80.1.el7.x86_64
0x280
3
map_single
vmlinux 0xffffffff9dbc952c
Build id: 3.10.0-1160.80.1.el7.x86_64
0x2c
4
swiotlb_map_page
vmlinux 0xffffffff9dbc9706
Build id: 3.10.0-1160.80.1.el7.x86_64
0x66
5
e1000_xmit_frame
e1000 0xffffffffc045043f
Build id: 3.10.0-1160.80.1.el7.x86_64
0x34f
6
dev_hard_start_xmit
vmlinux 0xffffffff9de7edc6
Build id: 3.10.0-1160.80.1.el7.x86_64
0x246
7
sch_direct_xmit
vmlinux 0xffffffff9deadd72
Build id: 3.10.0-1160.80.1.el7.x86_64
0x122
8
__dev_queue_xmit
vmlinux 0xffffffff9de81dd1
Build id: 3.10.0-1160.80.1.el7.x86_64
0x491
9
dev_queue_xmit
vmlinux 0xffffffff9de81fa0
Build id: 3.10.0-1160.80.1.el7.x86_64
0x10
10
ip_finish_output
vmlinux 0xffffffff9ded260b
Build id: 3.10.0-1160.80.1.el7.x86_64
0x67b
11
ip_output
vmlinux 0xffffffff9ded2b9b
Build id: 3.10.0-1160.80.1.el7.x86_64
0x7b
12
? __ip_append_data.isra.50
vmlinux 0xffffffff9ded1f90
Build id: 3.10.0-1160.80.1.el7.x86_64
0xa60
13
ip_local_out_sk
vmlinux 0xffffffff9ded0587
Build id: 3.10.0-1160.80.1.el7.x86_64
0x37
14
ip_build_and_send_pkt
vmlinux 0xffffffff9ded06f2
Build id: 3.10.0-1160.80.1.el7.x86_64
0x162
15
tcp_v4_send_synack
vmlinux 0xffffffff9def3fe4
Build id: 3.10.0-1160.80.1.el7.x86_64
0x74
16
? tcp_openreq_init_rwin
vmlinux 0xffffffff9def5c8c
Build id: 3.10.0-1160.80.1.el7.x86_64
0xcc
17
tcp_conn_request
vmlinux 0xffffffff9dee0a65
Build id: 3.10.0-1160.80.1.el7.x86_64
0x525
18
? get_unique_tuple
nf_nat 0xffffffffc09ac08c
Build id: 3.10.0-1160.80.1.el7.x86_64
0x59c
19
tcp_v4_conn_request
vmlinux 0xffffffff9def18a5
Build id: 3.10.0-1160.80.1.el7.x86_64
0x55
20
tcp_rcv_state_process
vmlinux 0xffffffff9dee874c
Build id: 3.10.0-1160.80.1.el7.x86_64
0x1ac
21
? __inet_lookup_established
vmlinux 0xffffffff9ded6576
Build id: 3.10.0-1160.80.1.el7.x86_64
0x46
22
tcp_v4_do_rcv
vmlinux 0xffffffff9def316b
Build id: 3.10.0-1160.80.1.el7.x86_64
0x18b
23
tcp_v4_rcv
vmlinux 0xffffffff9def498c
Build id: 3.10.0-1160.80.1.el7.x86_64
0x7dc
24
ip_local_deliver_finish
vmlinux 0xffffffff9decc34d
Build id: 3.10.0-1160.80.1.el7.x86_64
0xbd
25
ip_local_deliver
vmlinux 0xffffffff9decc660
Build id: 3.10.0-1160.80.1.el7.x86_64
0x60
26
? ip_rcv_finish
vmlinux 0xffffffff9decc290
Build id: 3.10.0-1160.80.1.el7.x86_64
0x380
27
ip_rcv_finish
vmlinux 0xffffffff9decbfa0
Build id: 3.10.0-1160.80.1.el7.x86_64
0x90
28
ip_rcv
vmlinux 0xffffffff9decc9a0
Build id: 3.10.0-1160.80.1.el7.x86_64
0x2c0
29
? inet_del_offload
vmlinux 0xffffffff9decbf10
Build id: 3.10.0-1160.80.1.el7.x86_64
0x40
30
__netif_receive_skb_core
vmlinux 0xffffffff9de7f669
Build id: 3.10.0-1160.80.1.el7.x86_64
0x739
31
? tcp4_gro_receive
vmlinux 0xffffffff9def9b00
Build id: 3.10.0-1160.80.1.el7.x86_64
0x30
32
? __getnstimeofday64
vmlinux 0xffffffff9d90e8df
Build id: 3.10.0-1160.80.1.el7.x86_64
0x3f
33
__netif_receive_skb
vmlinux 0xffffffff9de7f968
Build id: 3.10.0-1160.80.1.el7.x86_64
0x18
34
netif_receive_skb_internal
vmlinux 0xffffffff9de7fa00
Build id: 3.10.0-1160.80.1.el7.x86_64
0x40
35
napi_gro_receive
vmlinux 0xffffffff9de806b8
Build id: 3.10.0-1160.80.1.el7.x86_64
0xd8
36
e1000_clean_rx_irq
e1000 0xffffffffc04514cb
Build id: 3.10.0-1160.80.1.el7.x86_64
0x2bb
37
e1000_clean
e1000 0xffffffffc0452180
Build id: 3.10.0-1160.80.1.el7.x86_64
0x280
38
? sched_clock_cpu
vmlinux 0xffffffff9d8e41f5
Build id: 3.10.0-1160.80.1.el7.x86_64
0x85
39
net_rx_action
vmlinux 0xffffffff9de80037
Build id: 3.10.0-1160.80.1.el7.x86_64
0x277
40
__do_softirq
vmlinux 0xffffffff9d8a9545
Build id: 3.10.0-1160.80.1.el7.x86_64
0xf5
41
run_ksoftirqd
vmlinux 0xffffffff9d8a9718
Build id: 3.10.0-1160.80.1.el7.x86_64
0x38
42
smpboot_thread_fn
vmlinux 0xffffffff9d8d42ac
Build id: 3.10.0-1160.80.1.el7.x86_64
0x14c
43
? lg_double_unlock
vmlinux 0xffffffff9d8d4160
Build id: 3.10.0-1160.80.1.el7.x86_64
0x40
44
kthread
vmlinux 0xffffffff9d8cb4d1
Build id: 3.10.0-1160.80.1.el7.x86_64
0xd1
45
? insert_kthread_work
vmlinux 0xffffffff9d8cb400
Build id: 3.10.0-1160.80.1.el7.x86_64
0x40
46
ret_from_fork_nospec_begin
vmlinux 0xffffffff9dfc51f7
Build id: 3.10.0-1160.80.1.el7.x86_64
0x21
47
? insert_kthread_work
vmlinux 0xffffffff9d8cb400
Build id: 3.10.0-1160.80.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [ksoftirqd/0:6]
Modules linked in: binfmt_misc xt_connmark xt_addrtype xt_TPROXY ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle xt_mark xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul ppdev glue_helper ablk_helper vmw_balloon cryptd joydev pcspkr sg parport_pc i2c_piix4 parport vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi crct10dif_pclmul libata crct10dif_common scsi_transport_spi crc32c_intel mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff9bfd3be43180 ti: ffff9bfd3be50000 task.ti: ffff9bfd3be50000
RIP: 0010:[<ffffffff9dbad7a8>]  [<ffffffff9dbad7a8>] memcpy+0x68/0x130
RSP: 0018:ffff9bfd3be53398  EFLAGS: 00010206
RAX: ffff9bfc7ea1b000 RBX: ffff9bfd3be53328 RCX: 0000000000000001
RDX: 000000000000000a RSI: ffff9bfd51c714f0 RDI: ffff9bfc7ea1b04a
RBP: ffff9bfd3be533f8 R08: 070303012ac75fdb R09: 18a9d26c0a080204
R10: b405040200008e2d R11: 207112a0d814e318 R12: ffff9bfd6a0c7b5d
R13: 000000000000000e R14: ffff9bfd6a0c7b38 R15: 0000000000000000
FS:  0000000000000000(0000) GS:ffff9bfd79c00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000067c37c CR3: 000000033740a000 CR4: 00000000000007f0
Call Trace:
[<ffffffff9dbc8a90>] ? swiotlb_tbl_map_single+0x280/0x2e0
[<ffffffff9dbc952c>] map_single+0x2c/0x40
[<ffffffff9dbc9706>] swiotlb_map_page+0x66/0xf0
[<ffffffffc045043f>] e1000_xmit_frame+0x34f/0x1120 [e1000]
[<ffffffff9de7edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9deadd72>] sch_direct_xmit+0x122/0x250
[<ffffffff9de81dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffff9de81fa0>] dev_queue_xmit+0x10/0x20
[<ffffffff9ded260b>] ip_finish_output+0x67b/0x8d0
[<ffffffff9ded2b9b>] ip_output+0x7b/0x100
[<ffffffff9ded1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff9ded0587>] ip_local_out_sk+0x37/0x40
[<ffffffff9ded06f2>] ip_build_and_send_pkt+0x162/0x250
[<ffffffff9def3fe4>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff9def5c8c>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffff9dee0a65>] tcp_conn_request+0x525/0x760
[<ffffffffc09ac08c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffff9def18a5>] tcp_v4_conn_request+0x55/0x60
[<ffffffff9dee874c>] tcp_rcv_state_process+0x1ac/0xf50
[<ffffffff9ded6576>] ? __inet_lookup_established+0x46/0x130
[<ffffffff9def316b>] tcp_v4_do_rcv+0x18b/0x360
[<ffffffff9def498c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffff9decc34d>] ip_local_deliver_finish+0xbd/0x210
[<ffffffff9decc660>] ip_local_deliver+0x60/0xe0
[<ffffffff9decc290>] ? ip_rcv_finish+0x380/0x380
[<ffffffff9decbfa0>] ip_rcv_finish+0x90/0x380
[<ffffffff9decc9a0>] ip_rcv+0x2c0/0x420
[<ffffffff9decbf10>] ? inet_del_offload+0x40/0x40
[<ffffffff9de7f669>] __netif_receive_skb_core+0x739/0xa20
[<ffffffff9def9b00>] ? tcp4_gro_receive+0x30/0x1a0
[<ffffffff9d90e8df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9de7f968>] __netif_receive_skb+0x18/0x70
[<ffffffff9de7fa00>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9de806b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc04514cb>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0452180>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff9d8e41f5>] ? sched_clock_cpu+0x85/0xc0
[<ffffffff9de80037>] net_rx_action+0x277/0x3a0
[<ffffffff9d8a9545>] __do_softirq+0xf5/0x290
[<ffffffff9d8a9718>] run_ksoftirqd+0x38/0x60
[<ffffffff9d8d42ac>] smpboot_thread_fn+0x14c/0x1b0
[<ffffffff9d8d4160>] ? lg_double_unlock+0x40/0x40
[<ffffffff9d8cb4d1>] kthread+0xd1/0xe0
[<ffffffff9d8cb400>] ? insert_kthread_work+0x40/0x40
[<ffffffff9dfc51f7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9d8cb400>] ? insert_kthread_work+0x40/0x40
Code: d4 83 c2 20 eb 44 48 01 d6 48 01 d7 48 83 ea 20 0f 1f 00 48 83 ea 20 4c 8b 46 f8 4c 8b 4e f0 4c 8b 56 e8 4c 8b 5e e0 48 8d 76 e0 <4c> 89 47 f8 4c 89 4f f0 4c 89 57 e8 4c 89 5f e0 48 8d 7f e0 73 
Complete report #506444
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc01a1877
Build id: 3.10.0-1160.49.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffade55f96
Build id: 3.10.0-1160.49.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffade840aa
Build id: 3.10.0-1160.49.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffade58f11
Build id: 3.10.0-1160.49.1.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffade590e0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffadea7c6b
Build id: 3.10.0-1160.49.1.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffadea81eb
Build id: 3.10.0-1160.49.1.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffadea75f0
Build id: 3.10.0-1160.49.1.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffadea5c17
Build id: 3.10.0-1160.49.1.el7.x86_64
0x37
10
ip_build_and_send_pkt
vmlinux 0xffffffffadea5d82
Build id: 3.10.0-1160.49.1.el7.x86_64
0x162
11
tcp_v4_send_synack
vmlinux 0xffffffffadec8ec4
Build id: 3.10.0-1160.49.1.el7.x86_64
0x74
12
tcp_rtx_synack
vmlinux 0xffffffffadebed31
Build id: 3.10.0-1160.49.1.el7.x86_64
0x41
13
? local_bh_enable
vmlinux 0xffffffffad8a4077
Build id: 3.10.0-1160.49.1.el7.x86_64
0x17
14
? ipt_do_table
ip_tables 0xffffffffc04e6721
Build id: 3.10.0-1160.49.1.el7.x86_64
0x361
15
inet_rtx_syn_ack
vmlinux 0xffffffffadead83d
Build id: 3.10.0-1160.49.1.el7.x86_64
0x1d
16
tcp_check_req
vmlinux 0xffffffffadeca599
Build id: 3.10.0-1160.49.1.el7.x86_64
0x239
17
tcp_v4_do_rcv
vmlinux 0xffffffffadec812f
Build id: 3.10.0-1160.49.1.el7.x86_64
0x22f
18
tcp_v4_rcv
vmlinux 0xffffffffadec985c
Build id: 3.10.0-1160.49.1.el7.x86_64
0x7dc
19
ip_local_deliver_finish
vmlinux 0xffffffffadea1a9d
Build id: 3.10.0-1160.49.1.el7.x86_64
0xbd
20
ip_local_deliver
vmlinux 0xffffffffadea1d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x60
21
? ip_rcv_finish
vmlinux 0xffffffffadea19e0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x370
22
ip_rcv_finish
vmlinux 0xffffffffadea1700
Build id: 3.10.0-1160.49.1.el7.x86_64
0x90
23
ip_rcv
vmlinux 0xffffffffadea20d0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x2c0
24
? packet_rcv
vmlinux 0xffffffffadf62502
Build id: 3.10.0-1160.49.1.el7.x86_64
0x2f2
25
? inet_del_offload
vmlinux 0xffffffffadea1670
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffade56829
Build id: 3.10.0-1160.49.1.el7.x86_64
0x729
27
? tcp4_gro_receive
vmlinux 0xffffffffadecea00
Build id: 3.10.0-1160.49.1.el7.x86_64
0x120
28
? __getnstimeofday64
vmlinux 0xffffffffad907f0f
Build id: 3.10.0-1160.49.1.el7.x86_64
0x3f
29
__netif_receive_skb
vmlinux 0xffffffffade56b28
Build id: 3.10.0-1160.49.1.el7.x86_64
0x18
30
netif_receive_skb_internal
vmlinux 0xffffffffade56bb0
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
31
napi_gro_receive
vmlinux 0xffffffffade57838
Build id: 3.10.0-1160.49.1.el7.x86_64
0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc01a245b
Build id: 3.10.0-1160.49.1.el7.x86_64
0x2bb
33
e1000_clean
e1000 0xffffffffc01a3100
Build id: 3.10.0-1160.49.1.el7.x86_64
0x280
34
? nf_ct_delete
nf_conntrack 0xffffffffc069b02d
Build id: 3.10.0-1160.49.1.el7.x86_64
0x18d
35
net_rx_action
vmlinux 0xffffffffade571cf
Build id: 3.10.0-1160.49.1.el7.x86_64
0x26f
36
__do_softirq
vmlinux 0xffffffffad8a4bf5
Build id: 3.10.0-1160.49.1.el7.x86_64
0xf5
37
run_ksoftirqd
vmlinux 0xffffffffad8a4db8
Build id: 3.10.0-1160.49.1.el7.x86_64
0x38
38
smpboot_thread_fn
vmlinux 0xffffffffad8ce8b4
Build id: 3.10.0-1160.49.1.el7.x86_64
0x144
39
? lg_double_unlock
vmlinux 0xffffffffad8ce770
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
40
kthread
vmlinux 0xffffffffad8c5e61
Build id: 3.10.0-1160.49.1.el7.x86_64
0xd1
41
? insert_kthread_work
vmlinux 0xffffffffad8c5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffadf95df7
Build id: 3.10.0-1160.49.1.el7.x86_64
0x21
43
? insert_kthread_work
vmlinux 0xffffffffad8c5d90
Build id: 3.10.0-1160.49.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 21s! [ksoftirqd/4:29]
Modules linked in: tun bridge stp llc nf_log_ipv4 nf_log_common xt_CHECKSUM ipt_MASQUERADE devlink ebtable_filter nf_nat_masquerade_ipv4 ebtables nf_nat_ftp xt_REDIRECT nf_nat_redirect xt_conntrack nf_conntrack_ftp xt_LOG ip6table_filter xt_limit xt_multiport iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm sb_edac coretemp snd_timer snd iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper soundcore cryptd sg ppdev
joydev pcspkr vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul crct10dif_common nfit crc32c_intel libnvdimm mptspi e1000 scsi_transport_spi mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.49.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9675ba4c1080 ti: ffff9675ba4c8000 task.ti: ffff9675ba4c8000
RIP: 0010:[<ffffffffc01a1877>]  [<ffffffffc01a1877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9675ba4cb4b0  EFLAGS: 00000286
RAX: ffffa36200d03818 RBX: ffff967591c4fc90 RCX: 000000000000003d
RDX: 000000000000003d RSI: ffff9676705107c0 RDI: ffff9675bae64000
RBP: ffff9675ba4cb530 R08: 0000000000000000 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffc06a3052
R13: ffff9675ba4cb4f0 R14: ffff967591c4fd0c R15: ffff9674ba1618e8
FS:  0000000000000000(0000) GS:ffff967675700000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa0940ff740 CR3: 0000000225c10000 CR4: 00000000001607e0
Call Trace:
[<ffffffffade55f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffade840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffade58f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffade590e0>] dev_queue_xmit+0x10/0x20
[<ffffffffadea7c6b>] ip_finish_output+0x67b/0x8d0
[<ffffffffadea81eb>] ip_output+0x7b/0xf0
[<ffffffffadea75f0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffadea5c17>] ip_local_out_sk+0x37/0x40
[<ffffffffadea5d82>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffadec8ec4>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffadebed31>] tcp_rtx_synack+0x41/0x70
[<ffffffffad8a4077>] ? local_bh_enable+0x17/0x20
[<ffffffffc04e6721>] ? ipt_do_table+0x361/0x740 [ip_tables]
[<ffffffffadead83d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffadeca599>] tcp_check_req+0x239/0x5b0
[<ffffffffadec812f>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffadec985c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffadea1a9d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffadea1d90>] ip_local_deliver+0x60/0xe0
[<ffffffffadea19e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffadea1700>] ip_rcv_finish+0x90/0x370
[<ffffffffadea20d0>] ip_rcv+0x2c0/0x420
[<ffffffffadf62502>] ? packet_rcv+0x2f2/0x430
[<ffffffffadea1670>] ? inet_del_offload+0x40/0x40
[<ffffffffade56829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffadecea00>] ? tcp4_gro_receive+0x120/0x1a0
[<ffffffffad907f0f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffade56b28>] __netif_receive_skb+0x18/0x60
[<ffffffffade56bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffade57838>] napi_gro_receive+0xd8/0x100
[<ffffffffc01a245b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01a3100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc069b02d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffffade571cf>] net_rx_action+0x26f/0x390
[<ffffffffad8a4bf5>] __do_softirq+0xf5/0x280
[<ffffffffad8a4db8>] run_ksoftirqd+0x38/0x50
[<ffffffffad8ce8b4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffad8ce770>] ? lg_double_unlock+0x40/0x40
[<ffffffffad8c5e61>] kthread+0xd1/0xe0
[<ffffffffad8c5d90>] ? insert_kthread_work+0x40/0x40
[<ffffffffadf95df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffad8c5d90>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #405340
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0218877
Build id: 3.10.0-1062.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffbcc4d346
Build id: 3.10.0-1062.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffbcc7b0ba
Build id: 3.10.0-1062.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffbcc502b1
Build id: 3.10.0-1062.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffbcc50480
Build id: 3.10.0-1062.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffbcc9e97b
Build id: 3.10.0-1062.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffbcc9eefb
Build id: 3.10.0-1062.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffbcc9e300
Build id: 3.10.0-1062.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffbcc9c927
Build id: 3.10.0-1062.el7.x86_64
0x37
10
ip_build_and_send_pkt
vmlinux 0xffffffffbcc9ca92
Build id: 3.10.0-1062.el7.x86_64
0x162
11
tcp_v4_send_synack
vmlinux 0xffffffffbccbfa54
Build id: 3.10.0-1062.el7.x86_64
0x74
12
? tcp_openreq_init_rwin
vmlinux 0xffffffffbccc16dc
Build id: 3.10.0-1062.el7.x86_64
0xcc
13
tcp_conn_request
vmlinux 0xffffffffbccaccb5
Build id: 3.10.0-1062.el7.x86_64
0x525
14
? get_unique_tuple
nf_nat 0xffffffffc061b01c
Build id: 3.10.0-1062.el7.x86_64
0x59c
15
tcp_v4_conn_request
vmlinux 0xffffffffbccbd50d
Build id: 3.10.0-1062.el7.x86_64
0x4d
16
tcp_rcv_state_process
vmlinux 0xffffffffbccb45ac
Build id: 3.10.0-1062.el7.x86_64
0x1ac
17
? __inet_lookup_established
vmlinux 0xffffffffbcca27f6
Build id: 3.10.0-1062.el7.x86_64
0x46
18
tcp_v4_do_rcv
vmlinux 0xffffffffbccbed4b
Build id: 3.10.0-1062.el7.x86_64
0x17b
19
tcp_v4_rcv
vmlinux 0xffffffffbccc03ec
Build id: 3.10.0-1062.el7.x86_64
0x7dc
20
ip_local_deliver_finish
vmlinux 0xffffffffbcc9886d
Build id: 3.10.0-1062.el7.x86_64
0xbd
21
ip_local_deliver
vmlinux 0xffffffffbcc98b60
Build id: 3.10.0-1062.el7.x86_64
0x60
22
? ip_rcv_finish
vmlinux 0xffffffffbcc987b0
Build id: 3.10.0-1062.el7.x86_64
0x370
23
ip_rcv_finish
vmlinux 0xffffffffbcc984d0
Build id: 3.10.0-1062.el7.x86_64
0x90
24
ip_rcv
vmlinux 0xffffffffbcc98ea0
Build id: 3.10.0-1062.el7.x86_64
0x2c0
25
? inet_del_offload
vmlinux 0xffffffffbcc98440
Build id: 3.10.0-1062.el7.x86_64
0x40
26
__netif_receive_skb_core
vmlinux 0xffffffffbcc4dbd9
Build id: 3.10.0-1062.el7.x86_64
0x729
27
? tcp4_gro_receive
vmlinux 0xffffffffbccc5500
Build id: 3.10.0-1062.el7.x86_64
0x90
28
? __getnstimeofday64
vmlinux 0xffffffffbc705d5f
Build id: 3.10.0-1062.el7.x86_64
0x3f
29
__netif_receive_skb
vmlinux 0xffffffffbcc4ded8
Build id: 3.10.0-1062.el7.x86_64
0x18
30
netif_receive_skb_internal
vmlinux 0xffffffffbcc4df60
Build id: 3.10.0-1062.el7.x86_64
0x40
31
napi_gro_receive
vmlinux 0xffffffffbcc4ebe8
Build id: 3.10.0-1062.el7.x86_64
0xd8
32
e1000_clean_rx_irq
e1000 0xffffffffc021945b
Build id: 3.10.0-1062.el7.x86_64
0x2bb
33
e1000_clean
e1000 0xffffffffc021a100
Build id: 3.10.0-1062.el7.x86_64
0x280
34
? update_curr
vmlinux 0xffffffffbc6e375c
Build id: 3.10.0-1062.el7.x86_64
0x14c
35
net_rx_action
vmlinux 0xffffffffbcc4e57f
Build id: 3.10.0-1062.el7.x86_64
0x26f
36
__do_softirq
vmlinux 0xffffffffbc6a41e5
Build id: 3.10.0-1062.el7.x86_64
0xf5
37
run_ksoftirqd
vmlinux 0xffffffffbc6a43a8
Build id: 3.10.0-1062.el7.x86_64
0x38
38
smpboot_thread_fn
vmlinux 0xffffffffbc6cdb04
Build id: 3.10.0-1062.el7.x86_64
0x144
39
? lg_double_unlock
vmlinux 0xffffffffbc6cd9c0
Build id: 3.10.0-1062.el7.x86_64
0x40
40
kthread
vmlinux 0xffffffffbc6c50d1
Build id: 3.10.0-1062.el7.x86_64
0xd1
41
? insert_kthread_work
vmlinux 0xffffffffbc6c5000
Build id: 3.10.0-1062.el7.x86_64
0x40
42
ret_from_fork_nospec_begin
vmlinux 0xffffffffbcd8bd24
Build id: 3.10.0-1062.el7.x86_64
0xe
43
? insert_kthread_work
vmlinux 0xffffffffbc6c5000
Build id: 3.10.0-1062.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 29s! [ksoftirqd/0:6]
Modules linked in: tcp_lp nls_utf8 isofs fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc crc32_pclmul ghash_clmulni_intel aesni_intel ppdev vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport pcc_cpufreq i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi drm e1000 nfit mptscsih
ata_piix serio_raw libnvdimm mptbase libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff96923a635230 ti: ffff96923a6d0000 task.ti: ffff96923a6d0000
RIP: 0010:[<ffffffffc0218877>]  [<ffffffffc0218877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff96923a6d3440  EFLAGS: 00000286
RAX: ffffb4c400383818 RBX: ffffffffbd701c00 RCX: 0000000000000021
RDX: 0000000000000021 RSI: ffff9691f537f640 RDI: ffff96923a2ea000
RBP: ffff96923a6d34c0 R08: 0000000000000000 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff96923a6d3488
R13: ffff96921d5718cc R14: ffff9692375450e8 R15: 000000003a198203
FS:  0000000000000000(0000) GS:ffff96923b600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f4987d20200 CR3: 000000005ca10000 CR4: 00000000003407f0
Call Trace:
[<ffffffffbcc4d346>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbcc7b0ba>] sch_direct_xmit+0x11a/0x250
[<ffffffffbcc502b1>] __dev_queue_xmit+0x491/0x650
[<ffffffffbcc50480>] dev_queue_xmit+0x10/0x20
[<ffffffffbcc9e97b>] ip_finish_output+0x67b/0x8d0
[<ffffffffbcc9eefb>] ip_output+0x7b/0xf0
[<ffffffffbcc9e300>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffbcc9c927>] ip_local_out_sk+0x37/0x40
[<ffffffffbcc9ca92>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffbccbfa54>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffbccc16dc>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffffbccaccb5>] tcp_conn_request+0x525/0x760
[<ffffffffc061b01c>] ? get_unique_tuple+0x59c/0x670 [nf_nat]
[<ffffffffbccbd50d>] tcp_v4_conn_request+0x4d/0x50
[<ffffffffbccb45ac>] tcp_rcv_state_process+0x1ac/0xf50
[<ffffffffbcca27f6>] ? __inet_lookup_established+0x46/0x130
[<ffffffffbccbed4b>] tcp_v4_do_rcv+0x17b/0x350
[<ffffffffbccc03ec>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffbcc9886d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffbcc98b60>] ip_local_deliver+0x60/0xe0
[<ffffffffbcc987b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffbcc984d0>] ip_rcv_finish+0x90/0x370
[<ffffffffbcc98ea0>] ip_rcv+0x2c0/0x420
[<ffffffffbcc98440>] ? inet_del_offload+0x40/0x40
[<ffffffffbcc4dbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbccc5500>] ? tcp4_gro_receive+0x90/0x1a0
[<ffffffffbc705d5f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffbcc4ded8>] __netif_receive_skb+0x18/0x60
[<ffffffffbcc4df60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbcc4ebe8>] napi_gro_receive+0xd8/0x100
[<ffffffffc021945b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc021a100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffbc6e375c>] ? update_curr+0x14c/0x1e0
[<ffffffffbcc4e57f>] net_rx_action+0x26f/0x390
[<ffffffffbc6a41e5>] __do_softirq+0xf5/0x280
[<ffffffffbc6a43a8>] run_ksoftirqd+0x38/0x50
[<ffffffffbc6cdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbc6cd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffbc6c50d1>] kthread+0xd1/0xe0
[<ffffffffbc6c5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffbcd8bd24>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffffbc6c5000>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #403702
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0248877
Build id: 3.10.0-1062.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb244d346
Build id: 3.10.0-1062.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb247b0ba
Build id: 3.10.0-1062.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb24502b1
Build id: 3.10.0-1062.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb2450480
Build id: 3.10.0-1062.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb249e97b
Build id: 3.10.0-1062.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb249eefb
Build id: 3.10.0-1062.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb249e300
Build id: 3.10.0-1062.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb249c927
Build id: 3.10.0-1062.el7.x86_64
0x37
10
ip_build_and_send_pkt
vmlinux 0xffffffffb249ca92
Build id: 3.10.0-1062.el7.x86_64
0x162
11
tcp_v4_send_synack
vmlinux 0xffffffffb24bfa54
Build id: 3.10.0-1062.el7.x86_64
0x74
12
tcp_rtx_synack
vmlinux 0xffffffffb24b59c1
Build id: 3.10.0-1062.el7.x86_64
0x41
13
? local_bh_enable
vmlinux 0xffffffffb1ea3667
Build id: 3.10.0-1062.el7.x86_64
0x17
14
? ipt_do_table
ip_tables 0xffffffffc0417719
Build id: 3.10.0-1062.el7.x86_64
0x359
15
? local_bh_enable
vmlinux 0xffffffffb1ea3667
Build id: 3.10.0-1062.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc0417719
Build id: 3.10.0-1062.el7.x86_64
0x359
17
inet_rtx_syn_ack
vmlinux 0xffffffffb24a452d
Build id: 3.10.0-1062.el7.x86_64
0x1d
18
tcp_check_req
vmlinux 0xffffffffb24c1129
Build id: 3.10.0-1062.el7.x86_64
0x239
19
tcp_v4_do_rcv
vmlinux 0xffffffffb24bedff
Build id: 3.10.0-1062.el7.x86_64
0x22f
20
tcp_v4_rcv
vmlinux 0xffffffffb24c03ec
Build id: 3.10.0-1062.el7.x86_64
0x7dc
21
ip_local_deliver_finish
vmlinux 0xffffffffb249886d
Build id: 3.10.0-1062.el7.x86_64
0xbd
22
ip_local_deliver
vmlinux 0xffffffffb2498b60
Build id: 3.10.0-1062.el7.x86_64
0x60
23
? ip_rcv_finish
vmlinux 0xffffffffb24987b0
Build id: 3.10.0-1062.el7.x86_64
0x370
24
ip_rcv_finish
vmlinux 0xffffffffb24984d0
Build id: 3.10.0-1062.el7.x86_64
0x90
25
ip_rcv
vmlinux 0xffffffffb2498ea0
Build id: 3.10.0-1062.el7.x86_64
0x2c0
26
? packet_setsockopt
vmlinux 0xffffffffb2558902
Build id: 3.10.0-1062.el7.x86_64
0xa52
27
? inet_del_offload
vmlinux 0xffffffffb2498440
Build id: 3.10.0-1062.el7.x86_64
0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffb244dbd9
Build id: 3.10.0-1062.el7.x86_64
0x729
29
? tcp4_gro_receive
vmlinux 0xffffffffb24c5500
Build id: 3.10.0-1062.el7.x86_64
0x90
30
? __getnstimeofday64
vmlinux 0xffffffffb1f05d5f
Build id: 3.10.0-1062.el7.x86_64
0x3f
31
__netif_receive_skb
vmlinux 0xffffffffb244ded8
Build id: 3.10.0-1062.el7.x86_64
0x18
32
netif_receive_skb_internal
vmlinux 0xffffffffb244df60
Build id: 3.10.0-1062.el7.x86_64
0x40
33
napi_gro_receive
vmlinux 0xffffffffb244ebe8
Build id: 3.10.0-1062.el7.x86_64
0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc024945b
Build id: 3.10.0-1062.el7.x86_64
0x2bb
35
e1000_clean
e1000 0xffffffffc024a100
Build id: 3.10.0-1062.el7.x86_64
0x280
36
? nf_ct_delete
nf_conntrack 0xffffffffc080b02d
Build id: 3.10.0-1062.el7.x86_64
0x18d
37
? tcp_init_xmit_timers
vmlinux 0xffffffffb24bada0
Build id: 3.10.0-1062.el7.x86_64
0x30
38
net_rx_action
vmlinux 0xffffffffb244e57f
Build id: 3.10.0-1062.el7.x86_64
0x26f
39
__do_softirq
vmlinux 0xffffffffb1ea41e5
Build id: 3.10.0-1062.el7.x86_64
0xf5
40
run_ksoftirqd
vmlinux 0xffffffffb1ea43a8
Build id: 3.10.0-1062.el7.x86_64
0x38
41
smpboot_thread_fn
vmlinux 0xffffffffb1ecdb04
Build id: 3.10.0-1062.el7.x86_64
0x144
42
? lg_double_unlock
vmlinux 0xffffffffb1ecd9c0
Build id: 3.10.0-1062.el7.x86_64
0x40
43
kthread
vmlinux 0xffffffffb1ec50d1
Build id: 3.10.0-1062.el7.x86_64
0xd1
44
? insert_kthread_work
vmlinux 0xffffffffb1ec5000
Build id: 3.10.0-1062.el7.x86_64
0x40
45
ret_from_fork_nospec_begin
vmlinux 0xffffffffb258bd37
Build id: 3.10.0-1062.el7.x86_64
0x21
46
? insert_kthread_work
vmlinux 0xffffffffb1ec5000
Build id: 3.10.0-1062.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [ksoftirqd/2:19]
Modules linked in: binfmt_misc vsock_diag tcp_diag udp_diag inet_diag unix_diag veth tcp_lp nls_utf8 isofs fuse xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 snd_seq_midi snd_seq_midi_event ppdev vmw_balloon crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus aesni_intel snd_seq lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm pcspkr snd_timer snd joydev sg soundcore vmw_vmci i2c_piix4 parport_pc
parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel drm nfit libnvdimm ata_piix serio_raw e1000 mptspi scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff916f740d5230 ti: ffff916f740f8000 task.ti: ffff916f740f8000
RIP: 0010:[<ffffffffc0248877>]  [<ffffffffc0248877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff916f740fb4b0  EFLAGS: 00000286
RAX: ffffb32143203818 RBX: ffff91760d3b4a90 RCX: 0000000000000089
RDX: 0000000000000089 RSI: ffff917618014700 RDI: ffff91761ce70000
RBP: ffff916f740fb530 R08: 0000000000000000 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffc0813032
R13: ffff916f740fb4f0 R14: ffff91760d3b4b0c R15: ffff916e03b6e4f0
FS:  0000000000000000(0000) GS:ffff91761d680000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000e3279000 CR3: 0000000035726000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb244d346>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb247b0ba>] sch_direct_xmit+0x11a/0x250
[<ffffffffb24502b1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb2450480>] dev_queue_xmit+0x10/0x20
[<ffffffffb249e97b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb249eefb>] ip_output+0x7b/0xf0
[<ffffffffb249e300>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb249c927>] ip_local_out_sk+0x37/0x40
[<ffffffffb249ca92>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffb24bfa54>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffb24b59c1>] tcp_rtx_synack+0x41/0x70
[<ffffffffb1ea3667>] ? local_bh_enable+0x17/0x20
[<ffffffffc0417719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffb1ea3667>] ? local_bh_enable+0x17/0x20
[<ffffffffc0417719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffb24a452d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffb24c1129>] tcp_check_req+0x239/0x5b0
[<ffffffffb24bedff>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffb24c03ec>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffb249886d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb2498b60>] ip_local_deliver+0x60/0xe0
[<ffffffffb24987b0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb24984d0>] ip_rcv_finish+0x90/0x370
[<ffffffffb2498ea0>] ip_rcv+0x2c0/0x420
[<ffffffffb2558902>] ? packet_setsockopt+0xa52/0xae0
[<ffffffffb2498440>] ? inet_del_offload+0x40/0x40
[<ffffffffb244dbd9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb24c5500>] ? tcp4_gro_receive+0x90/0x1a0
[<ffffffffb1f05d5f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb244ded8>] __netif_receive_skb+0x18/0x60
[<ffffffffb244df60>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb244ebe8>] napi_gro_receive+0xd8/0x100
[<ffffffffc024945b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc024a100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc080b02d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffffb24bada0>] ? tcp_init_xmit_timers+0x30/0x30
[<ffffffffb244e57f>] net_rx_action+0x26f/0x390
[<ffffffffb1ea41e5>] __do_softirq+0xf5/0x280
[<ffffffffb1ea43a8>] run_ksoftirqd+0x38/0x50
[<ffffffffb1ecdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb1ecd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb1ec50d1>] kthread+0xd1/0xe0
[<ffffffffb1ec5000>] ? insert_kthread_work+0x40/0x40
[<ffffffffb258bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb1ec5000>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #372553
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc008cee7
Build id: 3.10.0-693.21.1.el7.x86_64
0x947
2
? kfree_skbmem
vmlinux 0xffffffff8157a297
Build id: 3.10.0-693.21.1.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff8158fa64
Build id: 3.10.0-693.21.1.el7.x86_64
0x194
4
sch_direct_xmit
vmlinux 0xffffffff815b8d2a
Build id: 3.10.0-693.21.1.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff815928b6
Build id: 3.10.0-693.21.1.el7.x86_64
0x226
6
dev_queue_xmit
vmlinux 0xffffffff81592bf0
Build id: 3.10.0-693.21.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff815d98c6
Build id: 3.10.0-693.21.1.el7.x86_64
0x546
8
ip_output
vmlinux 0xffffffff815d9e23
Build id: 3.10.0-693.21.1.el7.x86_64
0x73
9
? __ip_append_data.isra.50
vmlinux 0xffffffff815d9380
Build id: 3.10.0-693.21.1.el7.x86_64
0xa50
10
ip_local_out_sk
vmlinux 0xffffffff815d7a57
Build id: 3.10.0-693.21.1.el7.x86_64
0x37
11
ip_build_and_send_pkt
vmlinux 0xffffffff815d7bc2
Build id: 3.10.0-693.21.1.el7.x86_64
0x162
12
tcp_v4_send_synack
vmlinux 0xffffffff815f9e44
Build id: 3.10.0-693.21.1.el7.x86_64
0x74
13
? ip_queue_xmit
vmlinux 0xffffffff815d7dc3
Build id: 3.10.0-693.21.1.el7.x86_64
0x143
14
tcp_rtx_synack
vmlinux 0xffffffff815f0191
Build id: 3.10.0-693.21.1.el7.x86_64
0x41
15
? local_bh_enable
vmlinux 0xffffffff810934e7
Build id: 3.10.0-693.21.1.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc003ac8e
Build id: 3.10.0-693.21.1.el7.x86_64
0x33e
17
? __fib_validate_source.isra.12
vmlinux 0xffffffff81616d36
Build id: 3.10.0-693.21.1.el7.x86_64
0x2b6
18
inet_rtx_syn_ack
vmlinux 0xffffffff815df09d
Build id: 3.10.0-693.21.1.el7.x86_64
0x1d
19
tcp_check_req
vmlinux 0xffffffff815fb629
Build id: 3.10.0-693.21.1.el7.x86_64
0x239
20
tcp_v4_do_rcv
vmlinux 0xffffffff815f911f
Build id: 3.10.0-693.21.1.el7.x86_64
0x22f
21
tcp_v4_rcv
vmlinux 0xffffffff815fa798
Build id: 3.10.0-693.21.1.el7.x86_64
0x798
22
ip_local_deliver_finish
vmlinux 0xffffffff815d3b29
Build id: 3.10.0-693.21.1.el7.x86_64
0xb9
23
ip_local_deliver
vmlinux 0xffffffff815d3e19
Build id: 3.10.0-693.21.1.el7.x86_64
0x59
24
? ip_rcv_finish
vmlinux 0xffffffff815d3a70
Build id: 3.10.0-693.21.1.el7.x86_64
0x370
25
ip_rcv_finish
vmlinux 0xffffffff815d3790
Build id: 3.10.0-693.21.1.el7.x86_64
0x90
26
ip_rcv
vmlinux 0xffffffff815d4149
Build id: 3.10.0-693.21.1.el7.x86_64
0x2b9
27
? inet_del_offload
vmlinux 0xffffffff815d3700
Build id: 3.10.0-693.21.1.el7.x86_64
0x40
28
__netif_receive_skb_core
vmlinux 0xffffffff81590234
Build id: 3.10.0-693.21.1.el7.x86_64
0x594
29
? __getnstimeofday64
vmlinux 0xffffffff810ee42f
Build id: 3.10.0-693.21.1.el7.x86_64
0x3f
30
__netif_receive_skb
vmlinux 0xffffffff81590498
Build id: 3.10.0-693.21.1.el7.x86_64
0x18
31
netif_receive_skb_internal
vmlinux 0xffffffff81590520
Build id: 3.10.0-693.21.1.el7.x86_64
0x40
32
napi_gro_receive
vmlinux 0xffffffff81591648
Build id: 3.10.0-693.21.1.el7.x86_64
0xd8
33
e1000_clean_rx_irq
e1000 0xffffffffc008e47f
Build id: 3.10.0-693.21.1.el7.x86_64
0x2af
34
e1000_clean
e1000 0xffffffffc008eeeb
Build id: 3.10.0-693.21.1.el7.x86_64
0x27b
35
? nf_ct_delete
nf_conntrack 0xffffffffc05ac08d
Build id: 3.10.0-693.21.1.el7.x86_64
0x18d
36
net_rx_action
vmlinux 0xffffffff81590cc3
Build id: 3.10.0-693.21.1.el7.x86_64
0x173
37
__do_softirq
vmlinux 0xffffffff81094035
Build id: 3.10.0-693.21.1.el7.x86_64
0xf5
38
run_ksoftirqd
vmlinux 0xffffffff810941f8
Build id: 3.10.0-693.21.1.el7.x86_64
0x38
39
smpboot_thread_fn
vmlinux 0xffffffff810bc774
Build id: 3.10.0-693.21.1.el7.x86_64
0x144
40
? lg_double_unlock
vmlinux 0xffffffff810bc630
Build id: 3.10.0-693.21.1.el7.x86_64
0x40
41
kthread
vmlinux 0xffffffff810b4031
Build id: 3.10.0-693.21.1.el7.x86_64
0xd1
42
? insert_kthread_work
vmlinux 0xffffffff810b3f60
Build id: 3.10.0-693.21.1.el7.x86_64
0x40
43
ret_from_fork
vmlinux 0xffffffff816c0577
Build id: 3.10.0-693.21.1.el7.x86_64
0x77
44
? insert_kthread_work
vmlinux 0xffffffff810b3f60
Build id: 3.10.0-693.21.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:3]
Modules linked in: nls_utf8 isofs fuse vmw_vsock_vmci_transport vsock ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter snd_seq_midi snd_seq_midi_event sb_edac edac_core coretemp snd_ens1371 iosf_mbi snd_rawmidi crc32_pclmul snd_ac97_codec ghash_clmulni_intel ac97_bus aesni_intel lrw snd_seq snd_seq_device gf128mul glue_helper snd_pcm snd_timer snd ablk_helper cryptd soundcore ppdev pcspkr vmw_balloon joydev vmw_vmci sg i2c_piix4 shpchp nfit libnvdimm parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops
ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix mptspi libata serio_raw scsi_transport_spi mptscsih e1000 mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G             L ------------   3.10.0-693.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff88017cf11fa0 ti: ffff88017cf94000 task.ti: ffff88017cf94000
RIP: 0010:[<ffffffffc008cee7>]  [<ffffffffc008cee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff88017cf97500  EFLAGS: 00000286
RAX: ffffc90000d03818 RBX: ffffffff8109bff3 RCX: 000000000000004c
RDX: 000000000000004c RSI: ffff8802307c1e80 RDI: ffff88017c6e0000
RBP: ffff88017cf97580 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffff81e9f1c0
R13: ffff88020d7ac6d0 R14: ffffffff81e9f1c0 R15: ffff88017cf97568
FS:  0000000000000000(0000) GS:ffff880235600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1a4c1e16a0 CR3: 0000000001a02000 CR4: 00000000001607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8157a297>] ? kfree_skbmem+0x37/0x90
[<ffffffff8158fa64>] dev_hard_start_xmit+0x194/0x3d0
[<ffffffff815b8d2a>] sch_direct_xmit+0x11a/0x240
[<ffffffff815928b6>] __dev_queue_xmit+0x226/0x550
[<ffffffff81592bf0>] dev_queue_xmit+0x10/0x20
[<ffffffff815d98c6>] ip_finish_output+0x546/0x7a0
[<ffffffff815d9e23>] ip_output+0x73/0xe0
[<ffffffff815d9380>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffff815d7a57>] ip_local_out_sk+0x37/0x40
[<ffffffff815d7bc2>] ip_build_and_send_pkt+0x162/0x220
[<ffffffff815f9e44>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff815d7dc3>] ? ip_queue_xmit+0x143/0x3a0
[<ffffffff815f0191>] tcp_rtx_synack+0x41/0x70
[<ffffffff810934e7>] ? local_bh_enable+0x17/0x20
[<ffffffffc003ac8e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffff81616d36>] ? __fib_validate_source.isra.12+0x2b6/0x400
[<ffffffff815df09d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffff815fb629>] tcp_check_req+0x239/0x590
[<ffffffff815f911f>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffff815fa798>] tcp_v4_rcv+0x798/0x9a0
[<ffffffff815d3b29>] ip_local_deliver_finish+0xb9/0x200
[<ffffffff815d3e19>] ip_local_deliver+0x59/0xd0
[<ffffffff815d3a70>] ? ip_rcv_finish+0x370/0x370
[<ffffffff815d3790>] ip_rcv_finish+0x90/0x370
[<ffffffff815d4149>] ip_rcv+0x2b9/0x410
[<ffffffff815d3700>] ? inet_del_offload+0x40/0x40
[<ffffffff81590234>] __netif_receive_skb_core+0x594/0x7e0
[<ffffffff810ee42f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff81590498>] __netif_receive_skb+0x18/0x60
[<ffffffff81590520>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff81591648>] napi_gro_receive+0xd8/0x130
[<ffffffffc008e47f>] e1000_clean_rx_irq+0x2af/0x500 [e1000]
[<ffffffffc008eeeb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffc05ac08d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffff81590cc3>] net_rx_action+0x173/0x380
[<ffffffff81094035>] __do_softirq+0xf5/0x280
[<ffffffff810941f8>] run_ksoftirqd+0x38/0x50
[<ffffffff810bc774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff810bc630>] ? lg_double_unlock+0x40/0x40
[<ffffffff810b4031>] kthread+0xd1/0xe0
[<ffffffff810b3f60>] ? insert_kthread_work+0x40/0x40
[<ffffffff816c0577>] ret_from_fork+0x77/0xb0
[<ffffffff810b3f60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #154683
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02b8877
Build id: 3.10.0-1160.15.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb9654f96
Build id: 3.10.0-1160.15.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb968303a
Build id: 3.10.0-1160.15.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb9657f11
Build id: 3.10.0-1160.15.2.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb96580e0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb96a607b
Build id: 3.10.0-1160.15.2.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb96a65fb
Build id: 3.10.0-1160.15.2.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb96a5a00
Build id: 3.10.0-1160.15.2.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb96a4027
Build id: 3.10.0-1160.15.2.el7.x86_64
0x37
10
ip_build_and_send_pkt
vmlinux 0xffffffffb96a4192
Build id: 3.10.0-1160.15.2.el7.x86_64
0x162
11
tcp_v4_send_synack
vmlinux 0xffffffffb96c72a4
Build id: 3.10.0-1160.15.2.el7.x86_64
0x74
12
tcp_rtx_synack
vmlinux 0xffffffffb96bd111
Build id: 3.10.0-1160.15.2.el7.x86_64
0x41
13
? local_bh_enable
vmlinux 0xffffffffb90a3fb7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x17
14
? ipt_do_table
ip_tables 0xffffffffc0495719
Build id: 3.10.0-1160.15.2.el7.x86_64
0x359
15
? local_bh_enable
vmlinux 0xffffffffb90a3fb7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc0495719
Build id: 3.10.0-1160.15.2.el7.x86_64
0x359
17
inet_rtx_syn_ack
vmlinux 0xffffffffb96abc4d
Build id: 3.10.0-1160.15.2.el7.x86_64
0x1d
18
tcp_check_req
vmlinux 0xffffffffb96c8979
Build id: 3.10.0-1160.15.2.el7.x86_64
0x239
19
tcp_v4_do_rcv
vmlinux 0xffffffffb96c650f
Build id: 3.10.0-1160.15.2.el7.x86_64
0x22f
20
tcp_v4_rcv
vmlinux 0xffffffffb96c7c3c
Build id: 3.10.0-1160.15.2.el7.x86_64
0x7dc
21
ip_local_deliver_finish
vmlinux 0xffffffffb969fead
Build id: 3.10.0-1160.15.2.el7.x86_64
0xbd
22
ip_local_deliver
vmlinux 0xffffffffb96a01a0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x60
23
? ip_rcv_finish
vmlinux 0xffffffffb969fdf0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x370
24
ip_rcv_finish
vmlinux 0xffffffffb969fb10
Build id: 3.10.0-1160.15.2.el7.x86_64
0x90
25
ip_rcv
vmlinux 0xffffffffb96a04e0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x2c0
26
? packet_rcv
vmlinux 0xffffffffb9760802
Build id: 3.10.0-1160.15.2.el7.x86_64
0x2b2
27
? inet_del_offload
vmlinux 0xffffffffb969fa80
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
28
__netif_receive_skb_core
vmlinux 0xffffffffb9655829
Build id: 3.10.0-1160.15.2.el7.x86_64
0x729
29
? tcp4_gro_receive
vmlinux 0xffffffffb96ccd00
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
30
? __getnstimeofday64
vmlinux 0xffffffffb9107b1f
Build id: 3.10.0-1160.15.2.el7.x86_64
0x3f
31
__netif_receive_skb
vmlinux 0xffffffffb9655b28
Build id: 3.10.0-1160.15.2.el7.x86_64
0x18
32
netif_receive_skb_internal
vmlinux 0xffffffffb9655bb0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
33
napi_gro_receive
vmlinux 0xffffffffb9656838
Build id: 3.10.0-1160.15.2.el7.x86_64
0xd8
34
e1000_clean_rx_irq
e1000 0xffffffffc02b945b
Build id: 3.10.0-1160.15.2.el7.x86_64
0x2bb
35
e1000_clean
e1000 0xffffffffc02ba100
Build id: 3.10.0-1160.15.2.el7.x86_64
0x280
36
net_rx_action
vmlinux 0xffffffffb96561cf
Build id: 3.10.0-1160.15.2.el7.x86_64
0x26f
37
__do_softirq
vmlinux 0xffffffffb90a4b35
Build id: 3.10.0-1160.15.2.el7.x86_64
0xf5
38
run_ksoftirqd
vmlinux 0xffffffffb90a4cf8
Build id: 3.10.0-1160.15.2.el7.x86_64
0x38
39
smpboot_thread_fn
vmlinux 0xffffffffb90ce864
Build id: 3.10.0-1160.15.2.el7.x86_64
0x144
40
? lg_double_unlock
vmlinux 0xffffffffb90ce720
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
41
kthread
vmlinux 0xffffffffb90c5e11
Build id: 3.10.0-1160.15.2.el7.x86_64
0xd1
42
? insert_kthread_work
vmlinux 0xffffffffb90c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
43
ret_from_fork_nospec_begin
vmlinux 0xffffffffb9794df7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21
44
? insert_kthread_work
vmlinux 0xffffffffb90c5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: nf_conntrack_netlink xt_set ip_set_hash_net ip_set nfnetlink iptable_raw vxlan ip6_udp_tunnel udp_tunnel xt_mark xfrm6_mode_tunnel xfrm4_mode_tunnel esp4 drbg ansi_cprng veth xt_nat xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac snd_ens1371 snd_rawmidi coretemp snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq aesni_intel lrw snd_seq_device gf128mul snd_pcm glue_helper
ablk_helper cryptd snd_timer snd ppdev sg soundcore pcspkr vmw_balloon joydev vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt fb_sys_fops crc32c_intel ttm drm ata_piix nfit mptspi serio_raw libata libnvdimm scsi_transport_spi mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff949eb2375280 ti: ffff949eb2f74000 task.ti: ffff949eb2f74000
RIP: 0010:[<ffffffffc02b8877>]  [<ffffffffc02b8877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff949eb2f774b0  EFLAGS: 00000286
RAX: ffffadc3031c3818 RBX: 0000000000000001 RCX: 00000000000000a8
RDX: 00000000000000a8 RSI: ffff949ea5bfb980 RDI: ffff949e85386000
RBP: ffff949eb2f77530 R08: 0000000000000100 R09: 0000000022100000
R10: 26ac064000400000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000059021fcc6 R14: 0000000107f14ecc R15: ffffffffb93b1b30
FS:  0000000000000000(0000) GS:ffff94a55d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe3d5040000 CR3: 00000007401d4000 CR4: 00000000000607f0
Call Trace:
[<ffffffffb9654f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb968303a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9657f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb96580e0>] dev_queue_xmit+0x10/0x20
[<ffffffffb96a607b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb96a65fb>] ip_output+0x7b/0xf0
[<ffffffffb96a5a00>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb96a4027>] ip_local_out_sk+0x37/0x40
[<ffffffffb96a4192>] ip_build_and_send_pkt+0x162/0x250
[<ffffffffb96c72a4>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffb96bd111>] tcp_rtx_synack+0x41/0x70
[<ffffffffb90a3fb7>] ? local_bh_enable+0x17/0x20
[<ffffffffc0495719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffb90a3fb7>] ? local_bh_enable+0x17/0x20
[<ffffffffc0495719>] ? ipt_do_table+0x359/0x740 [ip_tables]
[<ffffffffb96abc4d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffb96c8979>] tcp_check_req+0x239/0x5b0
[<ffffffffb96c650f>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffb96c7c3c>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffb969fead>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb96a01a0>] ip_local_deliver+0x60/0xe0
[<ffffffffb969fdf0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb969fb10>] ip_rcv_finish+0x90/0x370
[<ffffffffb96a04e0>] ip_rcv+0x2c0/0x420
[<ffffffffb9760802>] ? packet_rcv+0x2b2/0x430
[<ffffffffb969fa80>] ? inet_del_offload+0x40/0x40
[<ffffffffb9655829>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb96ccd00>] ? tcp4_gro_receive+0x40/0x1a0
[<ffffffffb9107b1f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb9655b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb9655bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb9656838>] napi_gro_receive+0xd8/0x100
[<ffffffffc02b945b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02ba100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb96561cf>] net_rx_action+0x26f/0x390
[<ffffffffb90a4b35>] __do_softirq+0xf5/0x280
[<ffffffffb90a4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffb90ce864>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb90ce720>] ? lg_double_unlock+0x40/0x40
[<ffffffffb90c5e11>] kthread+0xd1/0xe0
[<ffffffffb90c5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffb9794df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb90c5d40>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #10570
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0131867
Build id: 3.10.0-862.14.4.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb4df02e6
Build id: 3.10.0-862.14.4.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb4e1bc0a
Build id: 3.10.0-862.14.4.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb4df31d1
Build id: 3.10.0-862.14.4.el7.x86_64
0x4a1
5
? iptable_nat_ipv4_fn
iptable_nat 0xffffffffc063f020
Build id: 3.10.0-862.14.4.el7.x86_64
0x20
6
dev_queue_xmit
vmlinux 0xffffffffb4df33a0
Build id: 3.10.0-862.14.4.el7.x86_64
0x10
7
neigh_resolve_output
vmlinux 0xffffffffb4dfee2d
Build id: 3.10.0-862.14.4.el7.x86_64
0x11d
8
ip_finish_output
vmlinux 0xffffffffb4e3c33c
Build id: 3.10.0-862.14.4.el7.x86_64
0x2ac
9
ip_output
vmlinux 0xffffffffb4e3cb33
Build id: 3.10.0-862.14.4.el7.x86_64
0x73
10
? __ip_append_data.isra.50
vmlinux 0xffffffffb4e3c090
Build id: 3.10.0-862.14.4.el7.x86_64
0xa50
11
ip_local_out_sk
vmlinux 0xffffffffb4e3a727
Build id: 3.10.0-862.14.4.el7.x86_64
0x37
12
ip_build_and_send_pkt
vmlinux 0xffffffffb4e3a892
Build id: 3.10.0-862.14.4.el7.x86_64
0x162
13
tcp_v4_send_synack
vmlinux 0xffffffffb4e5d004
Build id: 3.10.0-862.14.4.el7.x86_64
0x74
14
tcp_rtx_synack
vmlinux 0xffffffffb4e532f1
Build id: 3.10.0-862.14.4.el7.x86_64
0x41
15
? local_bh_enable
vmlinux 0xffffffffb489d057
Build id: 3.10.0-862.14.4.el7.x86_64
0x17
16
? ipt_do_table
ip_tables 0xffffffffc028ea2e
Build id: 3.10.0-862.14.4.el7.x86_64
0x33e
17
? local_bh_enable
vmlinux 0xffffffffb489d057
Build id: 3.10.0-862.14.4.el7.x86_64
0x17
18
? ipt_do_table
ip_tables 0xffffffffc028ea2e
Build id: 3.10.0-862.14.4.el7.x86_64
0x33e
19
? __fib_validate_source.isra.13
vmlinux 0xffffffffb4e79c15
Build id: 3.10.0-862.14.4.el7.x86_64
0xf5
20
inet_rtx_syn_ack
vmlinux 0xffffffffb4e41f6d
Build id: 3.10.0-862.14.4.el7.x86_64
0x1d
21
tcp_check_req
vmlinux 0xffffffffb4e5e819
Build id: 3.10.0-862.14.4.el7.x86_64
0x239
22
tcp_v4_do_rcv
vmlinux 0xffffffffb4e5c2df
Build id: 3.10.0-862.14.4.el7.x86_64
0x22f
23
tcp_v4_rcv
vmlinux 0xffffffffb4e5d976
Build id: 3.10.0-862.14.4.el7.x86_64
0x7b6
24
ip_local_deliver_finish
vmlinux 0xffffffffb4e3685d
Build id: 3.10.0-862.14.4.el7.x86_64
0xbd
25
ip_local_deliver
vmlinux 0xffffffffb4e36b49
Build id: 3.10.0-862.14.4.el7.x86_64
0x59
26
? ip_rcv_finish
vmlinux 0xffffffffb4e367a0
Build id: 3.10.0-862.14.4.el7.x86_64
0x370
27
ip_rcv_finish
vmlinux 0xffffffffb4e364c0
Build id: 3.10.0-862.14.4.el7.x86_64
0x90
28
ip_rcv
vmlinux 0xffffffffb4e36e79
Build id: 3.10.0-862.14.4.el7.x86_64
0x2b9
29
? inet_del_offload
vmlinux 0xffffffffb4e36430
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
30
__netif_receive_skb_core
vmlinux 0xffffffffb4df0b79
Build id: 3.10.0-862.14.4.el7.x86_64
0x729
31
? tcp4_gro_receive
vmlinux 0xffffffffb4e62a00
Build id: 3.10.0-862.14.4.el7.x86_64
0xe0
32
? __getnstimeofday64
vmlinux 0xffffffffb48fbeff
Build id: 3.10.0-862.14.4.el7.x86_64
0x3f
33
__netif_receive_skb
vmlinux 0xffffffffb4df0e88
Build id: 3.10.0-862.14.4.el7.x86_64
0x18
34
netif_receive_skb_internal
vmlinux 0xffffffffb4df0f10
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
35
napi_gro_receive
vmlinux 0xffffffffb4df1b38
Build id: 3.10.0-862.14.4.el7.x86_64
0xd8
36
e1000_clean_rx_irq
e1000 0xffffffffc013244b
Build id: 3.10.0-862.14.4.el7.x86_64
0x2bb
37
e1000_clean
e1000 0xffffffffc01330eb
Build id: 3.10.0-862.14.4.el7.x86_64
0x27b
38
? update_curr
vmlinux 0xffffffffb48db2fc
Build id: 3.10.0-862.14.4.el7.x86_64
0x14c
39
? account_entity_dequeue
vmlinux 0xffffffffb48d7c6e
Build id: 3.10.0-862.14.4.el7.x86_64
0xae
40
? __schedule
vmlinux 0xffffffffb4f18b04
Build id: 3.10.0-862.14.4.el7.x86_64
0x614
41
net_rx_action
vmlinux 0xffffffffb4df152f
Build id: 3.10.0-862.14.4.el7.x86_64
0x26f
42
__do_softirq
vmlinux 0xffffffffb489dba5
Build id: 3.10.0-862.14.4.el7.x86_64
0xf5
43
run_ksoftirqd
vmlinux 0xffffffffb489dd68
Build id: 3.10.0-862.14.4.el7.x86_64
0x38
44
smpboot_thread_fn
vmlinux 0xffffffffb48c6774
Build id: 3.10.0-862.14.4.el7.x86_64
0x144
45
? lg_double_unlock
vmlinux 0xffffffffb48c6630
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
46
kthread
vmlinux 0xffffffffb48bdf21
Build id: 3.10.0-862.14.4.el7.x86_64
0xd1
47
? insert_kthread_work
vmlinux 0xffffffffb48bde50
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
48
ret_from_fork_nospec_begin
vmlinux 0xffffffffb4f255dd
Build id: 3.10.0-862.14.4.el7.x86_64
0x7
49
? insert_kthread_work
vmlinux 0xffffffffb48bde50
Build id: 3.10.0-862.14.4.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 60s! [ksoftirqd/0:3]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel snd_seq_midi snd_seq_midi_event lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer ppdev snd vmw_balloon soundcore pcspkr nfit sg joydev libnvdimm parport_pc parport vmw_vmci shpchp i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic sd_mod crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm mptspi serio_raw scsi_transport_spi e1000 mptscsih ata_piix libata i2c_core mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Tainted: G        W    L ------------   3.10.0-862.14.4.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff9eb4ba3d1fa0 ti: ffff9eb4ba3f4000 task.ti: ffff9eb4ba3f4000
RIP: 0010:[<ffffffffc0131867>]  [<ffffffffc0131867>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9eb4ba3f74a0  EFLAGS: 00010286
RAX: ffffc1d8c1903818 RBX: ffff9eb375dd6500 RCX: 00000000000000d0
RDX: 00000000000000d0 RSI: ffff9eb375dd6500 RDI: ffff9eb76d47c000
RBP: ffff9eb4ba3f7520 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000001
R13: 000000000000004a R14: ffff9eb76f3b0098 R15: 0000000000200000
FS:  0000000000000000(0000) GS:ffff9eb76d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f17e4316024 CR3: 0000000426ec6000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb4df02e6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb4e1bc0a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb4df31d1>] __dev_queue_xmit+0x4a1/0x660
[<ffffffffc063f020>] ? iptable_nat_ipv4_fn+0x20/0x20 [iptable_nat]
[<ffffffffb4df33a0>] dev_queue_xmit+0x10/0x20
[<ffffffffb4dfee2d>] neigh_resolve_output+0x11d/0x220
[<ffffffffb4e3c33c>] ip_finish_output+0x2ac/0x7a0
[<ffffffffb4e3cb33>] ip_output+0x73/0xe0
[<ffffffffb4e3c090>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffffb4e3a727>] ip_local_out_sk+0x37/0x40
[<ffffffffb4e3a892>] ip_build_and_send_pkt+0x162/0x220
[<ffffffffb4e5d004>] tcp_v4_send_synack+0x74/0xd0
[<ffffffffb4e532f1>] tcp_rtx_synack+0x41/0x70
[<ffffffffb489d057>] ? local_bh_enable+0x17/0x20
[<ffffffffc028ea2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffb489d057>] ? local_bh_enable+0x17/0x20
[<ffffffffc028ea2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffb4e79c15>] ? __fib_validate_source.isra.13+0xf5/0x380
[<ffffffffb4e41f6d>] inet_rtx_syn_ack+0x1d/0x30
[<ffffffffb4e5e819>] tcp_check_req+0x239/0x590
[<ffffffffb4e5c2df>] tcp_v4_do_rcv+0x22f/0x350
[<ffffffffb4e5d976>] tcp_v4_rcv+0x7b6/0x9c0
[<ffffffffb4e3685d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb4e36b49>] ip_local_deliver+0x59/0xd0
[<ffffffffb4e367a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb4e364c0>] ip_rcv_finish+0x90/0x370
[<ffffffffb4e36e79>] ip_rcv+0x2b9/0x410
[<ffffffffb4e36430>] ? inet_del_offload+0x40/0x40
[<ffffffffb4df0b79>] __netif_receive_skb_core+0x729/0xa20
[<ffffffffb4e62a00>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffffb48fbeff>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb4df0e88>] __netif_receive_skb+0x18/0x60
[<ffffffffb4df0f10>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb4df1b38>] napi_gro_receive+0xd8/0x100
[<ffffffffc013244b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01330eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffb48db2fc>] ? update_curr+0x14c/0x1e0
[<ffffffffb48d7c6e>] ? account_entity_dequeue+0xae/0xd0
[<ffffffffb4f18b04>] ? __schedule+0x614/0xa20
[<ffffffffb4df152f>] net_rx_action+0x26f/0x390
[<ffffffffb489dba5>] __do_softirq+0xf5/0x280
[<ffffffffb489dd68>] run_ksoftirqd+0x38/0x50
[<ffffffffb48c6774>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb48c6630>] ? lg_double_unlock+0x40/0x40
[<ffffffffb48bdf21>] kthread+0xd1/0xe0
[<ffffffffb48bde50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb4f255dd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb48bde50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #20267
Frame # Function Binary Source or offset Line
1
memcpy
vmlinux 0xffffffff9d15a0c8
Build id: 3.10.0-862.el7.x86_64
0x68
2
? swiotlb_tbl_map_single
vmlinux 0xffffffff9d173e00
Build id: 3.10.0-862.el7.x86_64
0x280
3
map_single
vmlinux 0xffffffff9d1746fc
Build id: 3.10.0-862.el7.x86_64
0x2c
4
swiotlb_map_page
vmlinux 0xffffffff9d1748c6
Build id: 3.10.0-862.el7.x86_64
0x66
5
e1000_xmit_frame
e1000 0xffffffffc019b3bf
Build id: 3.10.0-862.el7.x86_64
0x34f
6
dev_hard_start_xmit
vmlinux 0xffffffff9d3eb166
Build id: 3.10.0-862.el7.x86_64
0x246
7
sch_direct_xmit
vmlinux 0xffffffff9d416a2a
Build id: 3.10.0-862.el7.x86_64
0x11a
8
__dev_queue_xmit
vmlinux 0xffffffff9d3ee051
Build id: 3.10.0-862.el7.x86_64
0x4a1
9
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc05fa2ce
Build id: 3.10.0-862.el7.x86_64
0x4e
10
dev_queue_xmit
vmlinux 0xffffffff9d3ee220
Build id: 3.10.0-862.el7.x86_64
0x10
11
ip_finish_output
vmlinux 0xffffffff9d437416
Build id: 3.10.0-862.el7.x86_64
0x546
12
ip_output
vmlinux 0xffffffff9d437973
Build id: 3.10.0-862.el7.x86_64
0x73
13
? __ip_append_data.isra.50
vmlinux 0xffffffff9d436ed0
Build id: 3.10.0-862.el7.x86_64
0xa50
14
ip_local_out_sk
vmlinux 0xffffffff9d435567
Build id: 3.10.0-862.el7.x86_64
0x37
15
ip_build_and_send_pkt
vmlinux 0xffffffff9d4356d2
Build id: 3.10.0-862.el7.x86_64
0x162
16
tcp_v4_send_synack
vmlinux 0xffffffff9d457c24
Build id: 3.10.0-862.el7.x86_64
0x74
17
? tcp_openreq_init_rwin
vmlinux 0xffffffff9d45983c
Build id: 3.10.0-862.el7.x86_64
0xcc
18
tcp_conn_request
vmlinux 0xffffffff9d4451e5
Build id: 3.10.0-862.el7.x86_64
0x525
19
? _raw_spin_unlock_bh
vmlinux 0xffffffff9d51551e
Build id: 3.10.0-862.el7.x86_64
0x1e
20
tcp_v4_conn_request
vmlinux 0xffffffff9d4553dd
Build id: 3.10.0-862.el7.x86_64
0x4d
21
tcp_rcv_state_process
vmlinux 0xffffffff9d44cad6
Build id: 3.10.0-862.el7.x86_64
0x1a6
22
? __internal_add_timer
vmlinux 0xffffffff9cea2f83
Build id: 3.10.0-862.el7.x86_64
0x113
23
? __inet_lookup_established
vmlinux 0xffffffff9d43b086
Build id: 3.10.0-862.el7.x86_64
0x46
24
tcp_v4_do_rcv
vmlinux 0xffffffff9d456e4b
Build id: 3.10.0-862.el7.x86_64
0x17b
25
tcp_v4_rcv
vmlinux 0xffffffff9d458568
Build id: 3.10.0-862.el7.x86_64
0x788
26
ip_local_deliver_finish
vmlinux 0xffffffff9d431699
Build id: 3.10.0-862.el7.x86_64
0xb9
27
ip_local_deliver
vmlinux 0xffffffff9d431989
Build id: 3.10.0-862.el7.x86_64
0x59
28
? ip_rcv_finish
vmlinux 0xffffffff9d4315e0
Build id: 3.10.0-862.el7.x86_64
0x370
29
ip_rcv_finish
vmlinux 0xffffffff9d431300
Build id: 3.10.0-862.el7.x86_64
0x90
30
ip_rcv
vmlinux 0xffffffff9d431cb9
Build id: 3.10.0-862.el7.x86_64
0x2b9
31
? inet_del_offload
vmlinux 0xffffffff9d431270
Build id: 3.10.0-862.el7.x86_64
0x40
32
__netif_receive_skb_core
vmlinux 0xffffffff9d3eb9f9
Build id: 3.10.0-862.el7.x86_64
0x729
33
? tcp4_gro_receive
vmlinux 0xffffffff9d45d600
Build id: 3.10.0-862.el7.x86_64
0xe0
34
? __getnstimeofday64
vmlinux 0xffffffff9cef8c5f
Build id: 3.10.0-862.el7.x86_64
0x3f
35
__netif_receive_skb
vmlinux 0xffffffff9d3ebd08
Build id: 3.10.0-862.el7.x86_64
0x18
36
netif_receive_skb_internal
vmlinux 0xffffffff9d3ebd90
Build id: 3.10.0-862.el7.x86_64
0x40
37
napi_gro_receive
vmlinux 0xffffffff9d3ec9b8
Build id: 3.10.0-862.el7.x86_64
0xd8
38
e1000_clean_rx_irq
e1000 0xffffffffc019c44b
Build id: 3.10.0-862.el7.x86_64
0x2bb
39
e1000_clean
e1000 0xffffffffc019d0eb
Build id: 3.10.0-862.el7.x86_64
0x27b
40
? nf_ct_delete
nf_conntrack 0xffffffffc05cb04d
Build id: 3.10.0-862.el7.x86_64
0x18d
41
? __queue_work
vmlinux 0xffffffff9ceb0cbc
Build id: 3.10.0-862.el7.x86_64
0x13c
42
net_rx_action
vmlinux 0xffffffff9d3ec3af
Build id: 3.10.0-862.el7.x86_64
0x26f
43
__do_softirq
vmlinux 0xffffffff9ce9a945
Build id: 3.10.0-862.el7.x86_64
0xf5
44
run_ksoftirqd
vmlinux 0xffffffff9ce9ab08
Build id: 3.10.0-862.el7.x86_64
0x38
45
smpboot_thread_fn
vmlinux 0xffffffff9cec35e4
Build id: 3.10.0-862.el7.x86_64
0x144
46
? lg_double_unlock
vmlinux 0xffffffff9cec34a0
Build id: 3.10.0-862.el7.x86_64
0x40
47
kthread
vmlinux 0xffffffff9cebae31
Build id: 3.10.0-862.el7.x86_64
0xd1
48
? insert_kthread_work
vmlinux 0xffffffff9cebad60
Build id: 3.10.0-862.el7.x86_64
0x40
49
ret_from_fork_nospec_begin
vmlinux 0xffffffff9d51f637
Build id: 3.10.0-862.el7.x86_64
0x21
50
? insert_kthread_work
vmlinux 0xffffffff9cebad60
Build id: 3.10.0-862.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 24s! [ksoftirqd/4:30]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock ppdev coretemp vmw_balloon joydev pcspkr sg vmw_vmci i2c_piix4 shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata mptspi scsi_transport_spi serio_raw e1000 mptscsih mptbase i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 30 Comm: ksoftirqd/4 Kdump: loaded Tainted: G        W    L ------------   3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff95deec7dcf10 ti: ffff95deec7f4000 task.ti: ffff95deec7f4000
RIP: 0010:[<ffffffff9d15a0c8>]  [<ffffffff9d15a0c8>] memcpy+0x68/0x110
RSP: 0018:ffff95deec7f73c8  EFLAGS: 00010206
RAX: ffff95dcbd295000 RBX: ffff95e095b67140 RCX: 0000000000000001
RDX: 000000000000000a RSI: ffff95dff00168f0 RDI: ffff95dcbd29504a
RBP: ffff95deec7f7428 R08: 07030301f9c07f03 R09: 7014190e0a080204
R10: b405040200005ddc R11: 207112a0318db6a6 R12: 0000000000000001
R13: 0000000000000004 R14: ffff95e5009d1098 R15: 0000000000200000
FS:  0000000000000000(0000) GS:ffff95e097000000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fbc2032a000 CR3: 00000008f07b2000 CR4: 00000000000007e0
Call Trace:
[<ffffffff9d173e00>] ? swiotlb_tbl_map_single+0x280/0x2d0
[<ffffffff9d1746fc>] map_single+0x2c/0x30
[<ffffffff9d1748c6>] swiotlb_map_page+0x66/0x150
[<ffffffffc019b3bf>] e1000_xmit_frame+0x34f/0x1120 [e1000]
[<ffffffff9d3eb166>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9d416a2a>] sch_direct_xmit+0x11a/0x250
[<ffffffff9d3ee051>] __dev_queue_xmit+0x4a1/0x660
[<ffffffffc05fa2ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff9d3ee220>] dev_queue_xmit+0x10/0x20
[<ffffffff9d437416>] ip_finish_output+0x546/0x7a0
[<ffffffff9d437973>] ip_output+0x73/0xe0
[<ffffffff9d436ed0>] ? __ip_append_data.isra.50+0xa50/0xa50
[<ffffffff9d435567>] ip_local_out_sk+0x37/0x40
[<ffffffff9d4356d2>] ip_build_and_send_pkt+0x162/0x220
[<ffffffff9d457c24>] tcp_v4_send_synack+0x74/0xd0
[<ffffffff9d45983c>] ? tcp_openreq_init_rwin+0xcc/0x210
[<ffffffff9d4451e5>] tcp_conn_request+0x525/0x760
[<ffffffff9d51551e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff9d4553dd>] tcp_v4_conn_request+0x4d/0x50
[<ffffffff9d44cad6>] tcp_rcv_state_process+0x1a6/0xf40
[<ffffffff9cea2f83>] ? __internal_add_timer+0x113/0x130
[<ffffffff9d43b086>] ? __inet_lookup_established+0x46/0x130
[<ffffffff9d456e4b>] tcp_v4_do_rcv+0x17b/0x350
[<ffffffff9d458568>] tcp_v4_rcv+0x788/0x990
[<ffffffff9d431699>] ip_local_deliver_finish+0xb9/0x200
[<ffffffff9d431989>] ip_local_deliver+0x59/0xd0
[<ffffffff9d4315e0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff9d431300>] ip_rcv_finish+0x90/0x370
[<ffffffff9d431cb9>] ip_rcv+0x2b9/0x410
[<ffffffff9d431270>] ? inet_del_offload+0x40/0x40
[<ffffffff9d3eb9f9>] __netif_receive_skb_core+0x729/0xa20
[<ffffffff9d45d600>] ? tcp4_gro_receive+0xe0/0x1a0
[<ffffffff9cef8c5f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9d3ebd08>] __netif_receive_skb+0x18/0x60
[<ffffffff9d3ebd90>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9d3ec9b8>] napi_gro_receive+0xd8/0x100
[<ffffffffc019c44b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc019d0eb>] e1000_clean+0x27b/0x8e0 [e1000]
[<ffffffffc05cb04d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffff9ceb0cbc>] ? __queue_work+0x13c/0x3e0
[<ffffffff9d3ec3af>] net_rx_action+0x26f/0x390
[<ffffffff9ce9a945>] __do_softirq+0xf5/0x280
[<ffffffff9ce9ab08>] run_ksoftirqd+0x38/0x50
[<ffffffff9cec35e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9cec34a0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9cebae31>] kthread+0xd1/0xe0
[<ffffffff9cebad60>] ? insert_kthread_work+0x40/0x40
[<ffffffff9d51f637>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9cebad60>] ? insert_kthread_work+0x40/0x40
Code: d4 83 c2 20 eb 44 48 01 d6 48 01 d7 48 83 ea 20 0f 1f 00 48 83 ea 20 4c 8b 46 f8 4c 8b 4e f0 4c 8b 56 e8 4c 8b 5e e0 48 8d 76 e0 <4c> 89 47 f8 4c 89 4f f0 4c 89 57 e8 4c 89 5f e0 48 8d 7f e0 73