Info

Permalink
Function
e1000_xmit_frame
First occurrence
2020-11-17
Last occurrence
2020-11-17
State
NEW
Components
kernel
Quality
-63

Statistics

Operating system Count
CentOS 7 74
Architecture Count
x86_64 74
Related packages Count
kernel 74
 0:3.10.0-1160.el7 49
 0:3.10.0-1160.62.1.el7 5
 0:3.10.0-957.el7 4
 0:3.10.0-1160.102.1.el7 4
 0:3.10.0-1160.36.2.el7 3
 0:3.10.0-1160.11.1.el7 2
 0:3.10.0-1160.25.1.el7 2
 0:3.10.0-1127.19.1.el7 1
 0:3.10.0-1160.15.2.el7 1
 0:3.10.0-957.27.2.el7 1
 0:3.10.0-1160.53.1.el7 1
 0:3.10.0-1160.76.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1015884
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02dd877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb0454386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb048242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__qdisc_run
vmlinux 0xffffffffb04825ee
Build id: 3.10.0-1160.el7.x86_64
0x8e
5
__dev_queue_xmit
vmlinux 0xffffffffb0457088
Build id: 3.10.0-1160.el7.x86_64
0x218
6
dev_queue_xmit_sk
vmlinux 0xffffffffb04574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
7
arp_xmit
vmlinux 0xffffffffb04d48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
8
? arp_create
vmlinux 0xffffffffb04d44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
9
arp_send_dst.part.16
vmlinux 0xffffffffb04d4975
Build id: 3.10.0-1160.el7.x86_64
0x45
10
arp_process
vmlinux 0xffffffffb04d5957
Build id: 3.10.0-1160.el7.x86_64
0x877
11
? __skb_clone
vmlinux 0xffffffffb043caee
Build id: 3.10.0-1160.el7.x86_64
0x2e
12
arp_rcv
vmlinux 0xffffffffb04d5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
13
? cpumask_next_and
vmlinux 0xffffffffb0185de5
Build id: 3.10.0-1160.el7.x86_64
0x35
14
? find_busiest_group
vmlinux 0xffffffffafee8054
Build id: 3.10.0-1160.el7.x86_64
0x144
15
__netif_receive_skb_core
vmlinux 0xffffffffb0454bc5
Build id: 3.10.0-1160.el7.x86_64
0x6d5
16
? __getnstimeofday64
vmlinux 0xffffffffaff078df
Build id: 3.10.0-1160.el7.x86_64
0x3f
17
__netif_receive_skb
vmlinux 0xffffffffb0454f18
Build id: 3.10.0-1160.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffb0454fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffb0455c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc02de45b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc02df100
Build id: 3.10.0-1160.el7.x86_64
0x280
22
net_rx_action
vmlinux 0xffffffffb04555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffffafea4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffffafea4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffffafece674
Build id: 3.10.0-1160.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffffafece530
Build id: 3.10.0-1160.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffffafec5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffffafec5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffffb0593df7
Build id: 3.10.0-1160.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffffafec5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ksoftirqd/1:14]
Modules linked in: vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event ppdev snd_ens1371 iosf_mbi crc32_pclmul ghash_clmulni_intel snd_rawmidi snd_ac97_codec aesni_intel lrw ac97_bus gf128mul snd_seq glue_helper ablk_helper cryptd vmw_balloon snd_seq_device snd_pcm pcspkr joydev snd_timer snd soundcore sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect crct10dif_pclmul crct10dif_common sysimgblt fb_sys_fops ttm crc32c_intel ata_piix serio_raw drm libata mptspi nfit scsi_transport_spi mptscsih e1000 mptbase libnvdimm drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff996bb97a8000 ti: ffff996bb97a4000 task.ti: ffff996bb97a4000
RIP: 0010:[<ffffffffc02dd877>]  [<ffffffffc02dd877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff996bb97a7768  EFLAGS: 00000286
RAX: ffffafffc03c3818 RBX: ffffffffb0b67e40 RCX: 000000000000001d
RDX: 000000000000001d RSI: ffff996b76752440 RDI: ffff996bb5b8a000
RBP: ffff996bb97a77e8 R08: 0000000000000000 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff996bb97a7750
R13: ffff996bb97a7788 R14: 0000000000000000 R15: 0000000000000000
FS:  0000000000000000(0000) GS:ffff996bbb640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f06df5961c5 CR3: 000000005b95c000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb0454386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb048242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb04825ee>] __qdisc_run+0x8e/0x360
[<ffffffffb0457088>] __dev_queue_xmit+0x218/0x650
[<ffffffffb04574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb04d48ba>] arp_xmit+0x3a/0xb0
[<ffffffffb04d44f3>] ? arp_create+0x1c3/0x230
[<ffffffffb04d4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb04d5957>] arp_process+0x877/0x8e0
[<ffffffffb043caee>] ? __skb_clone+0x2e/0x150
[<ffffffffb04d5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffffb0185de5>] ? cpumask_next_and+0x35/0x50
[<ffffffffafee8054>] ? find_busiest_group+0x144/0x990
[<ffffffffb0454bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffaff078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb0454f18>] __netif_receive_skb+0x18/0x60
[<ffffffffb0454fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb0455c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc02de45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02df100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb04555bf>] net_rx_action+0x26f/0x390
[<ffffffffafea4b95>] __do_softirq+0xf5/0x280
[<ffffffffafea4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffafece674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffafece530>] ? lg_double_unlock+0x40/0x40
[<ffffffffafec5c21>] kthread+0xd1/0xe0
[<ffffffffafec5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb0593df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffafec5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #973404
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0385877
Build id: 3.10.0-1160.76.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff9f458f86
Build id: 3.10.0-1160.76.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff9f48709a
Build id: 3.10.0-1160.76.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff9f45bf01
Build id: 3.10.0-1160.76.1.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffff9f45c0f3
Build id: 3.10.0-1160.76.1.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffff9f4da15a
Build id: 3.10.0-1160.76.1.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffff9f4d9d93
Build id: 3.10.0-1160.76.1.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffff9f4da215
Build id: 3.10.0-1160.76.1.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffff9f4db1f7
Build id: 3.10.0-1160.76.1.el7.x86_64
0x877
10
? kmem_cache_alloc
vmlinux 0xffffffff9f02917b
Build id: 3.10.0-1160.76.1.el7.x86_64
0x19b
11
? __skb_clone
vmlinux 0xffffffff9f441ade
Build id: 3.10.0-1160.76.1.el7.x86_64
0x2e
12
arp_rcv
vmlinux 0xffffffff9f4db3ad
Build id: 3.10.0-1160.76.1.el7.x86_64
0x12d
13
? inet_del_offload
vmlinux 0xffffffff9f4a4660
Build id: 3.10.0-1160.76.1.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffff9f4597c5
Build id: 3.10.0-1160.76.1.el7.x86_64
0x6d5
15
? __getnstimeofday64
vmlinux 0xffffffff9ef07fbf
Build id: 3.10.0-1160.76.1.el7.x86_64
0x3f
16
__netif_receive_skb
vmlinux 0xffffffff9f459b18
Build id: 3.10.0-1160.76.1.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffff9f459ba0
Build id: 3.10.0-1160.76.1.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffff9f45a828
Build id: 3.10.0-1160.76.1.el7.x86_64
0xd8
19
e1000_clean_rx_irq
e1000 0xffffffffc038645b
Build id: 3.10.0-1160.76.1.el7.x86_64
0x2bb
20
e1000_clean
e1000 0xffffffffc0387100
Build id: 3.10.0-1160.76.1.el7.x86_64
0x280
21
? nf_ct_delete
nf_conntrack 0xffffffffc08b202d
Build id: 3.10.0-1160.76.1.el7.x86_64
0x18d
22
net_rx_action
vmlinux 0xffffffff9f45a1bf
Build id: 3.10.0-1160.76.1.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffff9eea4d85
Build id: 3.10.0-1160.76.1.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffff9eea4f48
Build id: 3.10.0-1160.76.1.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffff9eece9e4
Build id: 3.10.0-1160.76.1.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffff9eece8a0
Build id: 3.10.0-1160.76.1.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffff9eec5f91
Build id: 3.10.0-1160.76.1.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffff9eec5ec0
Build id: 3.10.0-1160.76.1.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffff9f599df7
Build id: 3.10.0-1160.76.1.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffff9eec5ec0
Build id: 3.10.0-1160.76.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 23s! [ksoftirqd/1:14]
Modules linked in: nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache nf_conntrack_netlink xt_addrtype br_netfilter vmw_vsock_vmci_transport vsock xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 dm_thin_pool fuse dm_persistent_data dm_bio_prison dm_bufio loop tun devlink overlay ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus coretemp snd_seq
snd_seq_device iosf_mbi crc32_pclmul ppdev snd_pcm ghash_clmulni_intel aesni_intel lrw vmw_balloon gf128mul glue_helper ablk_helper cryptd snd_timer snd pcspkr joydev soundcore sg vmw_vmci i2c_piix4 parport_pc parport sch_fq_codel nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm drm serio_raw mptspi scsi_transport_spi mptscsih mptbase e1000 ata_piix libata drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/31/2013
task: ffff981cba9f6300 ti: ffff981cbaa10000 task.ti: ffff981cbaa10000
RIP: 0010:[<ffffffffc0385877>]  [<ffffffffc0385877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff981cbaa137c0  EFLAGS: 00000286
RAX: ffffb9f880e03818 RBX: ffff981d7393c100 RCX: 00000000000000fa
RDX: 00000000000000fa RSI: ffff981d711b6340 RDI: ffff981d74d9c000
RBP: ffff981cbaa13840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffdb6806868680
R13: ffff981c90d09080 R14: ffff981d7665acc0 R15: 0000000000002d46
FS:  0000000000000000(0000) GS:ffff981d76640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000556658c5ce48 CR3: 00000002047f0000 CR4: 00000000001607e0
Call Trace:
[<ffffffff9f458f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9f48709a>] sch_direct_xmit+0x11a/0x250
[<ffffffff9f45bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffff9f45c0f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff9f4da15a>] arp_xmit+0x3a/0xb0
[<ffffffff9f4d9d93>] ? arp_create+0x1c3/0x230
[<ffffffff9f4da215>] arp_send_dst.part.16+0x45/0x50
[<ffffffff9f4db1f7>] arp_process+0x877/0x8e0
[<ffffffff9f02917b>] ? kmem_cache_alloc+0x19b/0x1f0
[<ffffffff9f441ade>] ? __skb_clone+0x2e/0x150
[<ffffffff9f4db3ad>] arp_rcv+0x12d/0x1b0
[<ffffffff9f4a4660>] ? inet_del_offload+0x40/0x40
[<ffffffff9f4597c5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff9ef07fbf>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9f459b18>] __netif_receive_skb+0x18/0x60
[<ffffffff9f459ba0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9f45a828>] napi_gro_receive+0xd8/0x100
[<ffffffffc038645b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0387100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc08b202d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffff9f45a1bf>] net_rx_action+0x26f/0x390
[<ffffffff9eea4d85>] __do_softirq+0xf5/0x280
[<ffffffff9eea4f48>] run_ksoftirqd+0x38/0x50
[<ffffffff9eece9e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9eece8a0>] ? lg_double_unlock+0x40/0x40
[<ffffffff9eec5f91>] kthread+0xd1/0xe0
[<ffffffff9eec5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffff9f599df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9eec5ec0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #826061
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc020a857
Build id: 3.10.0-957.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff9e238c66
Build id: 3.10.0-957.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff9e265cea
Build id: 3.10.0-957.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff9e23bbe1
Build id: 3.10.0-957.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffff9e23bdd3
Build id: 3.10.0-957.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffff9e2b73f3
Build id: 3.10.0-957.el7.x86_64
0x33
7
? arp_create
vmlinux 0xffffffff9e2b7033
Build id: 3.10.0-957.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffff9e2b74a5
Build id: 3.10.0-957.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffff9e2b8487
Build id: 3.10.0-957.el7.x86_64
0x877
10
? ip_frag_mem
vmlinux 0xffffffff9e284520
Build id: 3.10.0-957.el7.x86_64
0x20
11
? __skb_clone
vmlinux 0xffffffff9e221c8e
Build id: 3.10.0-957.el7.x86_64
0x2e
12
arp_rcv
vmlinux 0xffffffff9e2b8635
Build id: 3.10.0-957.el7.x86_64
0x125
13
? inet_del_offload
vmlinux 0xffffffff9e282430
Build id: 3.10.0-957.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffff9e2394a5
Build id: 3.10.0-957.el7.x86_64
0x6d5
15
? __getnstimeofday64
vmlinux 0xffffffff9dd024ef
Build id: 3.10.0-957.el7.x86_64
0x3f
16
__netif_receive_skb
vmlinux 0xffffffff9e2397f8
Build id: 3.10.0-957.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffff9e239880
Build id: 3.10.0-957.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffff9e23a508
Build id: 3.10.0-957.el7.x86_64
0xd8
19
e1000_clean_rx_irq
e1000 0xffffffffc020b43b
Build id: 3.10.0-957.el7.x86_64
0x2bb
20
e1000_clean
e1000 0xffffffffc020c0dd
Build id: 3.10.0-957.el7.x86_64
0x27d
21
net_rx_action
vmlinux 0xffffffff9e239e9f
Build id: 3.10.0-957.el7.x86_64
0x26f
22
__do_softirq
vmlinux 0xffffffff9dca0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
23
run_ksoftirqd
vmlinux 0xffffffff9dca10c8
Build id: 3.10.0-957.el7.x86_64
0x38
24
smpboot_thread_fn
vmlinux 0xffffffff9dcca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
25
? lg_double_unlock
vmlinux 0xffffffff9dcca480
Build id: 3.10.0-957.el7.x86_64
0x40
26
kthread
vmlinux 0xffffffff9dcc1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
27
? insert_kthread_work
vmlinux 0xffffffff9dcc1b60
Build id: 3.10.0-957.el7.x86_64
0x40
28
ret_from_fork_nospec_begin
vmlinux 0xffffffff9e374c37
Build id: 3.10.0-957.el7.x86_64
0x21
29
? insert_kthread_work
vmlinux 0xffffffff9dcc1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 37s! [ksoftirqd/4:29]
Modules linked in: veth nfnetlink_queue nfnetlink_log bluetooth rfkill xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ppdev ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper vmw_balloon ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq joydev snd_seq_device pcspkr snd_pcm snd_timer snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic
crct10dif_pclmul crct10dif_common crc32c_intel pata_acpi vmwgfx serio_raw drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix e1000 libata mptspi scsi_transport_spi nfit mptscsih libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G             L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8f15782e1040 ti: ffff8f15782e8000 task.ti: ffff8f15782e8000
RIP: 0010:[<ffffffffc020a857>]  [<ffffffffc020a857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff8f15782eb7d0  EFLAGS: 00000286
RAX: ffffbb9f831c3818 RBX: 0000000000000001 RCX: 0000000000000052
RDX: 0000000000000052 RSI: ffff8f184167e840 RDI: ffff8f1577eee000
RBP: ffff8f15782eb850 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000004166a9a02 R14: 000000012aa8b600 R15: ffffffff9dfa04b0
FS:  0000000000000000(0000) GS:ffff8f1845100000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efbfdc28000 CR3: 000000025ba10000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9e238c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9e265cea>] sch_direct_xmit+0x11a/0x250
[<ffffffff9e23bbe1>] __dev_queue_xmit+0x491/0x650
[<ffffffff9e23bdd3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff9e2b73f3>] arp_xmit+0x33/0xa0
[<ffffffff9e2b7033>] ? arp_create+0x1c3/0x230
[<ffffffff9e2b74a5>] arp_send_dst.part.16+0x45/0x50
[<ffffffff9e2b8487>] arp_process+0x877/0x8e0
[<ffffffff9e284520>] ? ip_frag_mem+0x20/0x20
[<ffffffff9e221c8e>] ? __skb_clone+0x2e/0x150
[<ffffffff9e2b8635>] arp_rcv+0x125/0x1b0
[<ffffffff9e282430>] ? inet_del_offload+0x40/0x40
[<ffffffff9e2394a5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff9dd024ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9e2397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff9e239880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9e23a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc020b43b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc020c0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffff9e239e9f>] net_rx_action+0x26f/0x390
[<ffffffff9dca0f05>] __do_softirq+0xf5/0x280
[<ffffffff9dca10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff9dcca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff9dcca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff9dcc1c31>] kthread+0xd1/0xe0
[<ffffffff9dcc1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff9e374c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff9dcc1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #777550
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc01cb857
Build id: 3.10.0-957.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff86438c66
Build id: 3.10.0-957.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff86465cea
Build id: 3.10.0-957.el7.x86_64
0x11a
4
__qdisc_run
vmlinux 0xffffffff86465eae
Build id: 3.10.0-957.el7.x86_64
0x8e
5
__dev_queue_xmit
vmlinux 0xffffffff8643bbf9
Build id: 3.10.0-957.el7.x86_64
0x4a9
6
dev_queue_xmit_sk
vmlinux 0xffffffff8643bdd3
Build id: 3.10.0-957.el7.x86_64
0x13
7
arp_xmit
vmlinux 0xffffffff864b73f3
Build id: 3.10.0-957.el7.x86_64
0x33
8
? arp_create
vmlinux 0xffffffff864b7033
Build id: 3.10.0-957.el7.x86_64
0x1c3
9
arp_send_dst.part.16
vmlinux 0xffffffff864b74a5
Build id: 3.10.0-957.el7.x86_64
0x45
10
arp_process
vmlinux 0xffffffff864b8487
Build id: 3.10.0-957.el7.x86_64
0x877
11
? __skb_clone
vmlinux 0xffffffff86421c8e
Build id: 3.10.0-957.el7.x86_64
0x2e
12
arp_rcv
vmlinux 0xffffffff864b8635
Build id: 3.10.0-957.el7.x86_64
0x125
13
? packet_rcv
vmlinux 0xffffffff86540839
Build id: 3.10.0-957.el7.x86_64
0x319
14
__netif_receive_skb_core
vmlinux 0xffffffff864394a5
Build id: 3.10.0-957.el7.x86_64
0x6d5
15
? inet_gro_receive
vmlinux 0xffffffff864c06c9
Build id: 3.10.0-957.el7.x86_64
0x249
16
__netif_receive_skb
vmlinux 0xffffffff864397f8
Build id: 3.10.0-957.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffff86439880
Build id: 3.10.0-957.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffff8643a508
Build id: 3.10.0-957.el7.x86_64
0xd8
19
e1000_clean_rx_irq
e1000 0xffffffffc01cc43b
Build id: 3.10.0-957.el7.x86_64
0x2bb
20
e1000_clean
e1000 0xffffffffc01cd0dd
Build id: 3.10.0-957.el7.x86_64
0x27d
21
? update_curr
vmlinux 0xffffffff85ee00bc
Build id: 3.10.0-957.el7.x86_64
0x14c
22
net_rx_action
vmlinux 0xffffffff86439e9f
Build id: 3.10.0-957.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffff85ea0f05
Build id: 3.10.0-957.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffff85ea10c8
Build id: 3.10.0-957.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffff85eca5c4
Build id: 3.10.0-957.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffff85eca480
Build id: 3.10.0-957.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffff85ec1c31
Build id: 3.10.0-957.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffff85ec1b60
Build id: 3.10.0-957.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffff86574c37
Build id: 3.10.0-957.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffff85ec1b60
Build id: 3.10.0-957.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 37s! [ksoftirqd/0:3]
Modules linked in: binfmt_misc veth xt_nat xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock fuse vmhgfs(OE) overlay(T) sunrpc sb_edac ppdev coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec joydev ac97_bus snd_seq pcspkr snd_seq_device snd_pcm sg snd_timer snd soundcore i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif crct10dif_generic
cdrom vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm crct10dif_common crc32c_intel drm serio_raw ata_generic pata_acpi e1000 mptspi ata_piix scsi_transport_spi mptscsih libata mptbase nfit drm_panel_orientation_quirks libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3 Comm: ksoftirqd/0 Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff971034462080 ti: ffff9710344f0000 task.ti: ffff9710344f0000
RIP: 0010:[<ffffffffc01cb857>]  [<ffffffffc01cb857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9710344f3778  EFLAGS: 00000286
RAX: ffffa87f83483818 RBX: 0000000000000042 RCX: 0000000000000087
RDX: 0000000000000087 RSI: ffff9716d92a0dc0 RDI: ffff9716dcff6000
RBP: ffff9710344f37f8 R08: 0000000000000100 R09: 0000000022100000
R10: 34000045000893f5 R11: 0000000000000000 R12: 00000006c21c0cce
R13: 000000018643bdb0 R14: ffffffff861a04b0 R15: ffff9710344f3730
FS:  0000000000000000(0000) GS:ffff9716dd600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007effcf4f6740 CR3: 00000001a4410000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff86438c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff86465cea>] sch_direct_xmit+0x11a/0x250
[<ffffffff86465eae>] __qdisc_run+0x8e/0x360
[<ffffffff8643bbf9>] __dev_queue_xmit+0x4a9/0x650
[<ffffffff8643bdd3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff864b73f3>] arp_xmit+0x33/0xa0
[<ffffffff864b7033>] ? arp_create+0x1c3/0x230
[<ffffffff864b74a5>] arp_send_dst.part.16+0x45/0x50
[<ffffffff864b8487>] arp_process+0x877/0x8e0
[<ffffffff86421c8e>] ? __skb_clone+0x2e/0x150
[<ffffffff864b8635>] arp_rcv+0x125/0x1b0
[<ffffffff86540839>] ? packet_rcv+0x319/0x430
[<ffffffff864394a5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff864c06c9>] ? inet_gro_receive+0x249/0x290
[<ffffffff864397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff86439880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8643a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc01cc43b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01cd0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffff85ee00bc>] ? update_curr+0x14c/0x1e0
[<ffffffff86439e9f>] net_rx_action+0x26f/0x390
[<ffffffff85ea0f05>] __do_softirq+0xf5/0x280
[<ffffffff85ea10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff85eca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff85eca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff85ec1c31>] kthread+0xd1/0xe0
[<ffffffff85ec1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff86574c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff85ec1b60>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #706806
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04e0877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff84054386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff8408242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__qdisc_run
vmlinux 0xffffffff840825ee
Build id: 3.10.0-1160.el7.x86_64
0x8e
5
__dev_queue_xmit
vmlinux 0xffffffff84057319
Build id: 3.10.0-1160.el7.x86_64
0x4a9
6
dev_queue_xmit_sk
vmlinux 0xffffffff840574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
7
arp_xmit
vmlinux 0xffffffff840d48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
8
? arp_create
vmlinux 0xffffffff840d44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
9
arp_send_dst.part.16
vmlinux 0xffffffff840d4975
Build id: 3.10.0-1160.el7.x86_64
0x45
10
arp_process
vmlinux 0xffffffff840d5957
Build id: 3.10.0-1160.el7.x86_64
0x877
11
? __wake_up_common_lock
vmlinux 0xffffffff83ad3101
Build id: 3.10.0-1160.el7.x86_64
0x91
12
? task_rq_unlock
vmlinux 0xffffffff83ad2ba0
Build id: 3.10.0-1160.el7.x86_64
0x20
13
? __skb_clone
vmlinux 0xffffffff8403caee
Build id: 3.10.0-1160.el7.x86_64
0x2e
14
arp_rcv
vmlinux 0xffffffff840d5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
15
? packet_rcv
vmlinux 0xffffffff8415fbe9
Build id: 3.10.0-1160.el7.x86_64
0x319
16
__netif_receive_skb_core
vmlinux 0xffffffff84054bc5
Build id: 3.10.0-1160.el7.x86_64
0x6d5
17
? inet_gro_receive
vmlinux 0xffffffff840ddd99
Build id: 3.10.0-1160.el7.x86_64
0x249
18
__netif_receive_skb
vmlinux 0xffffffff84054f18
Build id: 3.10.0-1160.el7.x86_64
0x18
19
netif_receive_skb_internal
vmlinux 0xffffffff84054fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
20
napi_gro_receive
vmlinux 0xffffffff84055c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
21
e1000_clean_rx_irq
e1000 0xffffffffc04e145b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
22
e1000_clean
e1000 0xffffffffc04e2100
Build id: 3.10.0-1160.el7.x86_64
0x280
23
? update_curr
vmlinux 0xffffffff83ae43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
24
net_rx_action
vmlinux 0xffffffff840555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
25
__do_softirq
vmlinux 0xffffffff83aa4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
26
run_ksoftirqd
vmlinux 0xffffffff83aa4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
27
smpboot_thread_fn
vmlinux 0xffffffff83ace674
Build id: 3.10.0-1160.el7.x86_64
0x144
28
? lg_double_unlock
vmlinux 0xffffffff83ace530
Build id: 3.10.0-1160.el7.x86_64
0x40
29
kthread
vmlinux 0xffffffff83ac5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
30
? insert_kthread_work
vmlinux 0xffffffff83ac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
31
ret_from_fork_nospec_begin
vmlinux 0xffffffff84193ddd
Build id: 3.10.0-1160.el7.x86_64
0x7
32
? insert_kthread_work
vmlinux 0xffffffff83ac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 63s! [ksoftirqd/0:6]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc nfit kvm_intel libnvdimm kvm irqbypass pcspkr joydev crc32_pclmul ghash_clmulni_intel sg aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata mptspi serio_raw scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff909b927b5280 ti: ffff909b92f94000 task.ti: ffff909b92f94000
RIP: 0010:[<ffffffffc04e0877>]  [<ffffffffc04e0877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff909b92f97768  EFLAGS: 00000286
RAX: ffffbb51031c3818 RBX: ffff90a03f3db180 RCX: 0000000000000018
RDX: 0000000000000018 RSI: ffff90a0411f1c00 RDI: ffff909b9dba6000
RBP: ffff909b92f977e8 R08: 0000000000000000 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 000800003f3db1e8
R13: 00000000000000fd R14: 0000000000000086 R15: 0000000000000086
FS:  0000000000000000(0000) GS:ffff90a041e00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc2e9475000 CR3: 00000001f41ae000 CR4: 00000000003607f0
Call Trace:
[<ffffffff84054386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8408242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff840825ee>] __qdisc_run+0x8e/0x360
[<ffffffff84057319>] __dev_queue_xmit+0x4a9/0x650
[<ffffffff840574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff840d48ba>] arp_xmit+0x3a/0xb0
[<ffffffff840d44f3>] ? arp_create+0x1c3/0x230
[<ffffffff840d4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffff840d5957>] arp_process+0x877/0x8e0
[<ffffffff83ad3101>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffff83ad2ba0>] ? task_rq_unlock+0x20/0x20
[<ffffffff8403caee>] ? __skb_clone+0x2e/0x150
[<ffffffff840d5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffff8415fbe9>] ? packet_rcv+0x319/0x430
[<ffffffff84054bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff840ddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffff84054f18>] __netif_receive_skb+0x18/0x60
[<ffffffff84054fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff84055c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc04e145b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc04e2100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff83ae43cc>] ? update_curr+0x14c/0x1e0
[<ffffffff840555bf>] net_rx_action+0x26f/0x390
[<ffffffff83aa4b95>] __do_softirq+0xf5/0x280
[<ffffffff83aa4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff83ace674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff83ace530>] ? lg_double_unlock+0x40/0x40
[<ffffffff83ac5c21>] kthread+0xd1/0xe0
[<ffffffff83ac5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff84193ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffff83ac5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #645250
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc011633f
Build id: 3.10.0-1160.el7.x86_64
0x2bf
2
dev_hard_start_xmit
vmlinux 0xffffffff9a054386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff9a08242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff9a057301
Build id: 3.10.0-1160.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffff9a0574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffff9a0d48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffff9a0d44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffff9a0d4975
Build id: 3.10.0-1160.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffff9a0d5957
Build id: 3.10.0-1160.el7.x86_64
0x877
10
? __skb_clone
vmlinux 0xffffffff9a03caee
Build id: 3.10.0-1160.el7.x86_64
0x2e
11
arp_rcv
vmlinux 0xffffffff9a0d5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
12
? inet_del_offload
vmlinux 0xffffffff9a09ee60
Build id: 3.10.0-1160.el7.x86_64
0x40
13
__netif_receive_skb_core
vmlinux 0xffffffff9a054bc5
Build id: 3.10.0-1160.el7.x86_64
0x6d5
14
? __getnstimeofday64
vmlinux 0xffffffff99b078df
Build id: 3.10.0-1160.el7.x86_64
0x3f
15
__netif_receive_skb
vmlinux 0xffffffff9a054f18
Build id: 3.10.0-1160.el7.x86_64
0x18
16
netif_receive_skb_internal
vmlinux 0xffffffff9a054fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
17
napi_gro_receive
vmlinux 0xffffffff9a055c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
18
e1000_clean_rx_irq
e1000 0xffffffffc011745b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
19
e1000_clean
e1000 0xffffffffc0118100
Build id: 3.10.0-1160.el7.x86_64
0x280
20
? update_curr
vmlinux 0xffffffff99ae43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
21
net_rx_action
vmlinux 0xffffffff9a0555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
22
__do_softirq
vmlinux 0xffffffff99aa4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
23
run_ksoftirqd
vmlinux 0xffffffff99aa4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
24
smpboot_thread_fn
vmlinux 0xffffffff99ace674
Build id: 3.10.0-1160.el7.x86_64
0x144
25
? lg_double_unlock
vmlinux 0xffffffff99ace530
Build id: 3.10.0-1160.el7.x86_64
0x40
26
kthread
vmlinux 0xffffffff99ac5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
27
? insert_kthread_work
vmlinux 0xffffffff99ac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
28
ret_from_fork_nospec_begin
vmlinux 0xffffffff9a193df7
Build id: 3.10.0-1160.el7.x86_64
0x21
29
? insert_kthread_work
vmlinux 0xffffffff99ac5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [ksoftirqd/3:24]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec crc32_pclmul ghash_clmulni_intel ac97_bus aesni_intel lrw gf128mul glue_helper snd_seq btusb btrtl snd_seq_device btbcm btintel bluetooth snd_pcm ablk_helper cryptd snd_timer rfkill snd soundcore ppdev vmw_balloon joydev sg pcspkr parport_pc parport i2c_piix4 vmw_vmci ip_tables
xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata crct10dif_pclmul mptspi crct10dif_common nfit crc32c_intel scsi_transport_spi e1000 libnvdimm mptscsih serio_raw mptbase drm_panel_orientation_quirks fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffffa002f1f8b180 ti: ffffa002f1f98000 task.ti: ffffa002f1f98000
RIP: 0010:[<ffffffffc011633f>]  [<ffffffffc011633f>] e1000_xmit_frame+0x2bf/0x1120 [e1000]
RSP: 0018:ffffa002f1f9b7c0  EFLAGS: 00000246
RAX: 0000000100dd7777 RBX: ffffffff9a0a6446 RCX: 000000000000003c
RDX: 000000000000028f RSI: 0000000000000000 RDI: 000000000000003c
RBP: ffffa002f1f9b840 R08: 0000000000000000 R09: 0000000000000000
R10: ffffa002f8be8000 R11: 0000000000000000 R12: ffffa00422372000
R13: ffffffff9a715e40 R14: ffffffff9a0a3407 R15: ffffa002f1f9b738
FS:  0000000000000000(0000) GS:ffffa004216c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055d404581000 CR3: 00000001797c8000 CR4: 0000000000360fe0
Call Trace:
[<ffffffff9a054386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9a08242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff9a057301>] __dev_queue_xmit+0x491/0x650
[<ffffffff9a0574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff9a0d48ba>] arp_xmit+0x3a/0xb0
[<ffffffff9a0d44f3>] ? arp_create+0x1c3/0x230
[<ffffffff9a0d4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffff9a0d5957>] arp_process+0x877/0x8e0
[<ffffffff9a03caee>] ? __skb_clone+0x2e/0x150
[<ffffffff9a0d5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffff9a09ee60>] ? inet_del_offload+0x40/0x40
[<ffffffff9a054bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff99b078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff9a054f18>] __netif_receive_skb+0x18/0x60
[<ffffffff9a054fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff9a055c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc011745b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0118100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff99ae43cc>] ? update_curr+0x14c/0x1e0
[<ffffffff9a0555bf>] net_rx_action+0x26f/0x390
[<ffffffff99aa4b95>] __do_softirq+0xf5/0x280
[<ffffffff99aa4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff99ace674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff99ace530>] ? lg_double_unlock+0x40/0x40
[<ffffffff99ac5c21>] kthread+0xd1/0xe0
[<ffffffff99ac5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff9a193df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff99ac5b50>] ? insert_kthread_work+0x40/0x40
Code: 02 0f 84 45 0b 00 00 44 89 e9 48 8b 45 c8 8b 75 a8 80 b8 50 0b 00 00 00 0f 85 5e 0b 00 00 48 8b 05 c7 5c 65 da 66 45 89 6c 24 18 <41> c6 44 24 1c 00 49 89 44 24 10 48 8b 45 c0 48 03 b0 e8 00 00 
Complete report #591737
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc03ca877
Build id: 3.10.0-1160.62.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb9c58f86
Build id: 3.10.0-1160.62.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb9c8709a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb9c5bf01
Build id: 3.10.0-1160.62.1.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb9c5c0f3
Build id: 3.10.0-1160.62.1.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb9cda14a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb9cd9d83
Build id: 3.10.0-1160.62.1.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb9cda205
Build id: 3.10.0-1160.62.1.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb9cdb1e7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffffb96d3341
Build id: 3.10.0-1160.62.1.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffffb96d2db0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffffb9c41ade
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffffb9cdb39d
Build id: 3.10.0-1160.62.1.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffffb9d65599
Build id: 3.10.0-1160.62.1.el7.x86_64
0x319
15
__netif_receive_skb_core
vmlinux 0xffffffffb9c597c5
Build id: 3.10.0-1160.62.1.el7.x86_64
0x6d5
16
? __getnstimeofday64
vmlinux 0xffffffffb970805f
Build id: 3.10.0-1160.62.1.el7.x86_64
0x3f
17
__netif_receive_skb
vmlinux 0xffffffffb9c59b18
Build id: 3.10.0-1160.62.1.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffb9c59ba0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffb9c5a828
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc03cb45b
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc03cc100
Build id: 3.10.0-1160.62.1.el7.x86_64
0x280
22
? __queue_work
vmlinux 0xffffffffb96bbf2c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x13c
23
net_rx_action
vmlinux 0xffffffffb9c5a1bf
Build id: 3.10.0-1160.62.1.el7.x86_64
0x26f
24
__do_softirq
vmlinux 0xffffffffb96a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
0xf5
25
run_ksoftirqd
vmlinux 0xffffffffb96a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
0x38
26
smpboot_thread_fn
vmlinux 0xffffffffb96cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
0x144
27
? lg_double_unlock
vmlinux 0xffffffffb96ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
28
kthread
vmlinux 0xffffffffb96c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd1
29
? insert_kthread_work
vmlinux 0xffffffffb96c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffb9d99df7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21
31
? insert_kthread_work
vmlinux 0xffffffffb96c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [ksoftirqd/0:6]
Modules linked in: binfmt_misc veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq sb_edac snd_seq_device snd_pcm snd_timer snd iosf_mbi crc32_pclmul soundcore ghash_clmulni_intel pcspkr aesni_intel lrw gf128mul glue_helper ablk_helper joydev ppdev sg cryptd vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common
crc32c_intel ata_piix libata e1000 nfit mptspi scsi_transport_spi serio_raw mptscsih libnvdimm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff92c2b63ad280 ti: ffff92c2b5ce0000 task.ti: ffff92c2b5ce0000
RIP: 0010:[<ffffffffc03ca877>]  [<ffffffffc03ca877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff92c2b5ce37c0  EFLAGS: 00000286
RAX: ffffa41281c43818 RBX: 0000000000000001 RCX: 0000000000000007
RDX: 0000000000000007 RSI: ffff92c56984e1c0 RDI: ffff92c56c71e000
RBP: ffff92c2b5ce3840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000021506da02 R14: 00000001c082e4e0 R15: ffffffffb99b2090
FS:  0000000000000000(0000) GS:ffff92c56d600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c00b490000 CR3: 00000001fe964000 CR4: 00000000001607f0
Call Trace:
[<ffffffffb9c58f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb9c8709a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9c5bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffffb9c5c0f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb9cda14a>] arp_xmit+0x3a/0xb0
[<ffffffffb9cd9d83>] ? arp_create+0x1c3/0x230
[<ffffffffb9cda205>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb9cdb1e7>] arp_process+0x877/0x8e0
[<ffffffffb96d3341>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffffb96d2db0>] ? task_rq_unlock+0x20/0x20
[<ffffffffb9c41ade>] ? __skb_clone+0x2e/0x150
[<ffffffffb9cdb39d>] arp_rcv+0x12d/0x1b0
[<ffffffffb9d65599>] ? packet_rcv+0x319/0x430
[<ffffffffb9c597c5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb970805f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb9c59b18>] __netif_receive_skb+0x18/0x60
[<ffffffffb9c59ba0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb9c5a828>] napi_gro_receive+0xd8/0x100
[<ffffffffc03cb45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc03cc100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb96bbf2c>] ? __queue_work+0x13c/0x3f0
[<ffffffffb9c5a1bf>] net_rx_action+0x26f/0x390
[<ffffffffb96a4d85>] __do_softirq+0xf5/0x280
[<ffffffffb96a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffb96cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb96ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb96c5fb1>] kthread+0xd1/0xe0
[<ffffffffb96c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb9d99df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb96c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #536636
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0250877
Build id: 3.10.0-1160.36.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb0a55f96
Build id: 3.10.0-1160.36.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb0a840aa
Build id: 3.10.0-1160.36.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb0a58f11
Build id: 3.10.0-1160.36.2.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb0a59103
Build id: 3.10.0-1160.36.2.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb0ad717a
Build id: 3.10.0-1160.36.2.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb0ad6db3
Build id: 3.10.0-1160.36.2.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb0ad7235
Build id: 3.10.0-1160.36.2.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb0ad8217
Build id: 3.10.0-1160.36.2.el7.x86_64
0x877
10
? __skb_clone
vmlinux 0xffffffffb0a3eaee
Build id: 3.10.0-1160.36.2.el7.x86_64
0x2e
11
arp_rcv
vmlinux 0xffffffffb0ad83cd
Build id: 3.10.0-1160.36.2.el7.x86_64
0x12d
12
? packet_rcv
vmlinux 0xffffffffb0b62502
Build id: 3.10.0-1160.36.2.el7.x86_64
0x2d2
13
? inet_del_offload
vmlinux 0xffffffffb0aa16d0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x40
14
__netif_receive_skb_core
vmlinux 0xffffffffb0a567d5
Build id: 3.10.0-1160.36.2.el7.x86_64
0x6d5
15
? __getnstimeofday64
vmlinux 0xffffffffb0507eef
Build id: 3.10.0-1160.36.2.el7.x86_64
0x3f
16
__netif_receive_skb
vmlinux 0xffffffffb0a56b28
Build id: 3.10.0-1160.36.2.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffffb0a56bb0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffffb0a57838
Build id: 3.10.0-1160.36.2.el7.x86_64
0xd8
19
e1000_clean_rx_irq
e1000 0xffffffffc025145b
Build id: 3.10.0-1160.36.2.el7.x86_64
0x2bb
20
e1000_clean
e1000 0xffffffffc0252100
Build id: 3.10.0-1160.36.2.el7.x86_64
0x280
21
? nf_ct_delete
nf_conntrack 0xffffffffc08aa02d
Build id: 3.10.0-1160.36.2.el7.x86_64
0x18d
22
net_rx_action
vmlinux 0xffffffffb0a571cf
Build id: 3.10.0-1160.36.2.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffffb04a4bd5
Build id: 3.10.0-1160.36.2.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffffb04a4d98
Build id: 3.10.0-1160.36.2.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffffb04ce894
Build id: 3.10.0-1160.36.2.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffffb04ce750
Build id: 3.10.0-1160.36.2.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffffb04c5e41
Build id: 3.10.0-1160.36.2.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffffb04c5d70
Build id: 3.10.0-1160.36.2.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffffb0b95ddd
Build id: 3.10.0-1160.36.2.el7.x86_64
0x7
30
? insert_kthread_work
vmlinux 0xffffffffb04c5d70
Build id: 3.10.0-1160.36.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [ksoftirqd/4:29]
Modules linked in: rfcomm veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio loop xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink bnep vmw_vsock_vmci_transport vsock nfit snd_seq_midi snd_seq_midi_event libnvdimm snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi ac97_bus snd_seq crc32_pclmul snd_seq_device snd_pcm btusb ghash_clmulni_intel aesni_intel btrtl snd_timer btbcm snd btintel lrw bluetooth gf128mul glue_helper ppdev soundcore ablk_helper cryptd rfkill pcspkr joydev sg vmw_balloon parport_pc parport
i2c_piix4 vmw_vmci nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul ata_piix crct10dif_common drm crc32c_intel mptspi libata scsi_transport_spi mptscsih e1000 serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 29 Comm: ksoftirqd/4 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.36.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8c057b5b9080 ti: ffff8c057b5c0000 task.ti: ffff8c057b5c0000
RIP: 0010:[<ffffffffc0250877>]  [<ffffffffc0250877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8c057b5c37c0  EFLAGS: 00000286
RAX: ffff9a2bc0d03818 RBX: 726ea8c07b5c3778 RCX: 00000000000000e3
RDX: 00000000000000e3 RSI: ffff8c05b6146140 RDI: ffff8c05bc034000
RBP: ffff8c057b5c3840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff8c05907bae00
R13: ffffffffb0aa78d4 R14: ffff8c057b5c3778 R15: ffffffffb04a4057
FS:  0000000000000000(0000) GS:ffff8c05bb500000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000401010 CR3: 0000000124e5e000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb0a55f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb0a840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffb0a58f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb0a59103>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb0ad717a>] arp_xmit+0x3a/0xb0
[<ffffffffb0ad6db3>] ? arp_create+0x1c3/0x230
[<ffffffffb0ad7235>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb0ad8217>] arp_process+0x877/0x8e0
[<ffffffffb0a3eaee>] ? __skb_clone+0x2e/0x150
[<ffffffffb0ad83cd>] arp_rcv+0x12d/0x1b0
[<ffffffffb0b62502>] ? packet_rcv+0x2d2/0x430
[<ffffffffb0aa16d0>] ? inet_del_offload+0x40/0x40
[<ffffffffb0a567d5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb0507eef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb0a56b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb0a56bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb0a57838>] napi_gro_receive+0xd8/0x100
[<ffffffffc025145b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0252100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffc08aa02d>] ? nf_ct_delete+0x18d/0x270 [nf_conntrack]
[<ffffffffb0a571cf>] net_rx_action+0x26f/0x390
[<ffffffffb04a4bd5>] __do_softirq+0xf5/0x280
[<ffffffffb04a4d98>] run_ksoftirqd+0x38/0x50
[<ffffffffb04ce894>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb04ce750>] ? lg_double_unlock+0x40/0x40
[<ffffffffb04c5e41>] kthread+0xd1/0xe0
[<ffffffffb04c5d70>] ? insert_kthread_work+0x40/0x40
[<ffffffffb0b95ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb04c5d70>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #515289
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02d7877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb1854386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb188242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb1857301
Build id: 3.10.0-1160.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb18574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb18d48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb18d44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb18d4975
Build id: 3.10.0-1160.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb18d5957
Build id: 3.10.0-1160.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffffb12d3101
Build id: 3.10.0-1160.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffffb12d2ba0
Build id: 3.10.0-1160.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffffb183caee
Build id: 3.10.0-1160.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffffb18d5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffffb195fbe9
Build id: 3.10.0-1160.el7.x86_64
0x319
15
__netif_receive_skb_core
vmlinux 0xffffffffb1854bc5
Build id: 3.10.0-1160.el7.x86_64
0x6d5
16
? inet_gro_receive
vmlinux 0xffffffffb18ddd99
Build id: 3.10.0-1160.el7.x86_64
0x249
17
__netif_receive_skb
vmlinux 0xffffffffb1854f18
Build id: 3.10.0-1160.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffb1854fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffb1855c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc02d845b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc02d9100
Build id: 3.10.0-1160.el7.x86_64
0x280
22
? update_curr
vmlinux 0xffffffffb12e43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
23
? __schedule
vmlinux 0xffffffffb1986e82
Build id: 3.10.0-1160.el7.x86_64
0x5a2
24
net_rx_action
vmlinux 0xffffffffb18555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
25
__do_softirq
vmlinux 0xffffffffb12a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
26
run_ksoftirqd
vmlinux 0xffffffffb12a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
27
smpboot_thread_fn
vmlinux 0xffffffffb12ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
28
? lg_double_unlock
vmlinux 0xffffffffb12ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
29
kthread
vmlinux 0xffffffffb12c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
30
? insert_kthread_work
vmlinux 0xffffffffb12c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
31
ret_from_fork_nospec_begin
vmlinux 0xffffffffb1993ddd
Build id: 3.10.0-1160.el7.x86_64
0x7
32
? insert_kthread_work
vmlinux 0xffffffffb12c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 107s! [ksoftirqd/0:6]
Modules linked in: cmac arc4 md4 nls_utf8 cifs ccm dns_resolver ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw nfit gf128mul ppdev libnvdimm glue_helper ablk_helper sg vmw_balloon vmw_vmci cryptd pcspkr joydev i2c_piix4 parport_pc parport ip_tables xfs libcrc32c vmwgfx sr_mod cdrom sd_mod crc_t10dif
crct10dif_generic ata_generic pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 crct10dif_pclmul mptspi crct10dif_common scsi_transport_spi crc32c_intel mptscsih mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff8af350fb5280 ti: ffff8af3513f4000 task.ti: ffff8af3513f4000
RIP: 0010:[<ffffffffc02d7877>]  [<ffffffffc02d7877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8af3513f77c0  EFLAGS: 00000286
RAX: ffffad4a034c3818 RBX: 0000000000000000 RCX: 0000000000000067
RDX: 0000000000000067 RSI: ffff8af8245ff280 RDI: ffff8af35bf9a000
RBP: ffff8af3513f7840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffb12da830
R13: ffff8af3513f7778 R14: 0000000000000092 R15: 0000000000000092
FS:  0000000000000000(0000) GS:ffff8af825600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f927affd000 CR3: 000000017ff5e000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb1854386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb188242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb1857301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb18574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb18d48ba>] arp_xmit+0x3a/0xb0
[<ffffffffb18d44f3>] ? arp_create+0x1c3/0x230
[<ffffffffb18d4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb18d5957>] arp_process+0x877/0x8e0
[<ffffffffb12d3101>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffffb12d2ba0>] ? task_rq_unlock+0x20/0x20
[<ffffffffb183caee>] ? __skb_clone+0x2e/0x150
[<ffffffffb18d5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffffb195fbe9>] ? packet_rcv+0x319/0x430
[<ffffffffb1854bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb18ddd99>] ? inet_gro_receive+0x249/0x290
[<ffffffffb1854f18>] __netif_receive_skb+0x18/0x60
[<ffffffffb1854fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb1855c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc02d845b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02d9100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb12e43cc>] ? update_curr+0x14c/0x1e0
[<ffffffffb1986e82>] ? __schedule+0x5a2/0x860
[<ffffffffb18555bf>] net_rx_action+0x26f/0x390
[<ffffffffb12a4b95>] __do_softirq+0xf5/0x280
[<ffffffffb12a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffb12ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb12ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffb12c5c21>] kthread+0xd1/0xe0
[<ffffffffb12c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffb1993ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffb12c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #486318
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc014b877
Build id: 3.10.0-1160.62.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb8c58f86
Build id: 3.10.0-1160.62.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb8c8709a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb8c5bf01
Build id: 3.10.0-1160.62.1.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb8c5c0f3
Build id: 3.10.0-1160.62.1.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb8cda14a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb8cd9d83
Build id: 3.10.0-1160.62.1.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb8cda205
Build id: 3.10.0-1160.62.1.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb8cdb1e7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffffb86d3341
Build id: 3.10.0-1160.62.1.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffffb86d2db0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffffb8c41ade
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffffb8cdb39d
Build id: 3.10.0-1160.62.1.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffffb8d65599
Build id: 3.10.0-1160.62.1.el7.x86_64
0x319
15
__netif_receive_skb_core
vmlinux 0xffffffffb8c597c5
Build id: 3.10.0-1160.62.1.el7.x86_64
0x6d5
16
? inet_gro_receive
vmlinux 0xffffffffb8ce3649
Build id: 3.10.0-1160.62.1.el7.x86_64
0x249
17
__netif_receive_skb
vmlinux 0xffffffffb8c59b18
Build id: 3.10.0-1160.62.1.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffffb8c59ba0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffffb8c5a828
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc014c45b
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc014d100
Build id: 3.10.0-1160.62.1.el7.x86_64
0x280
22
net_rx_action
vmlinux 0xffffffffb8c5a1bf
Build id: 3.10.0-1160.62.1.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffffb86a4d85
Build id: 3.10.0-1160.62.1.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffffb86a4f48
Build id: 3.10.0-1160.62.1.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffffb86cea04
Build id: 3.10.0-1160.62.1.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffffb86ce8c0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffffb86c5fb1
Build id: 3.10.0-1160.62.1.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffffb86c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffffb8d99df7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x21
30
? insert_kthread_work
vmlinux 0xffffffffb86c5ee0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [ksoftirqd/0:6]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw sg gf128mul glue_helper ablk_helper cryptd ppdev pcspkr joydev vmw_balloon parport_pc parport i2c_piix4
vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix nfit mptspi libnvdimm libata scsi_transport_spi serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff90ecb65b5280 ti: ffff90ecb6678000 task.ti: ffff90ecb6678000
RIP: 0010:[<ffffffffc014b877>]  [<ffffffffc014b877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff90ecb667b7c0  EFLAGS: 00000286
RAX: ffffbb79406c3818 RBX: ffff90ecb667b758 RCX: 00000000000000fe
RDX: 00000000000000fe RSI: ffff90ebb5df0040 RDI: ffff90ecb5c70000
RBP: ffff90ecb667b840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff90ecb975acc0
R13: ffffffffb86d667e R14: ffff90ecb667b740 R15: ffffffffb86594b2
FS:  0000000000000000(0000) GS:ffff90ecb9600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6dc7692000 CR3: 00000001343e8000 CR4: 00000000003607f0
Call Trace:
[<ffffffffb8c58f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb8c8709a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb8c5bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffffb8c5c0f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb8cda14a>] arp_xmit+0x3a/0xb0
[<ffffffffb8cd9d83>] ? arp_create+0x1c3/0x230
[<ffffffffb8cda205>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb8cdb1e7>] arp_process+0x877/0x8e0
[<ffffffffb86d3341>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffffb86d2db0>] ? task_rq_unlock+0x20/0x20
[<ffffffffb8c41ade>] ? __skb_clone+0x2e/0x150
[<ffffffffb8cdb39d>] arp_rcv+0x12d/0x1b0
[<ffffffffb8d65599>] ? packet_rcv+0x319/0x430
[<ffffffffb8c597c5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb8ce3649>] ? inet_gro_receive+0x249/0x290
[<ffffffffb8c59b18>] __netif_receive_skb+0x18/0x60
[<ffffffffb8c59ba0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb8c5a828>] napi_gro_receive+0xd8/0x100
[<ffffffffc014c45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc014d100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb8c5a1bf>] net_rx_action+0x26f/0x390
[<ffffffffb86a4d85>] __do_softirq+0xf5/0x280
[<ffffffffb86a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffb86cea04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb86ce8c0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb86c5fb1>] kthread+0xd1/0xe0
[<ffffffffb86c5ee0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb8d99df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb86c5ee0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #282797
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02b3877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff8d254386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff8d28242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff8d257301
Build id: 3.10.0-1160.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffff8d2574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffff8d2d48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffff8d2d44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffff8d2d4975
Build id: 3.10.0-1160.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffff8d2d5957
Build id: 3.10.0-1160.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffff8ccd3101
Build id: 3.10.0-1160.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffff8ccd2ba0
Build id: 3.10.0-1160.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffff8d23caee
Build id: 3.10.0-1160.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffff8d2d5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffff8d35fbe9
Build id: 3.10.0-1160.el7.x86_64
0x319
15
__netif_receive_skb_core
vmlinux 0xffffffff8d254bc5
Build id: 3.10.0-1160.el7.x86_64
0x6d5
16
? __getnstimeofday64
vmlinux 0xffffffff8cd078df
Build id: 3.10.0-1160.el7.x86_64
0x3f
17
__netif_receive_skb
vmlinux 0xffffffff8d254f18
Build id: 3.10.0-1160.el7.x86_64
0x18
18
netif_receive_skb_internal
vmlinux 0xffffffff8d254fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
19
napi_gro_receive
vmlinux 0xffffffff8d255c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
20
e1000_clean_rx_irq
e1000 0xffffffffc02b445b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
21
e1000_clean
e1000 0xffffffffc02b5100
Build id: 3.10.0-1160.el7.x86_64
0x280
22
? update_curr
vmlinux 0xffffffff8cce43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
23
net_rx_action
vmlinux 0xffffffff8d2555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
24
__do_softirq
vmlinux 0xffffffff8cca4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
25
run_ksoftirqd
vmlinux 0xffffffff8cca4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
26
smpboot_thread_fn
vmlinux 0xffffffff8ccce674
Build id: 3.10.0-1160.el7.x86_64
0x144
27
? lg_double_unlock
vmlinux 0xffffffff8ccce530
Build id: 3.10.0-1160.el7.x86_64
0x40
28
kthread
vmlinux 0xffffffff8ccc5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
29
? insert_kthread_work
vmlinux 0xffffffff8ccc5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffff8d393df7
Build id: 3.10.0-1160.el7.x86_64
0x21
31
? insert_kthread_work
vmlinux 0xffffffff8ccc5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [ksoftirqd/0:6]
Modules linked in: veth xt_nat nf_conntrack_netlink xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype tun br_netfilter devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus iosf_mbi snd_seq crc32_pclmul snd_seq_device ghash_clmulni_intel
snd_pcm ppdev aesni_intel lrw snd_timer gf128mul glue_helper vmw_balloon ablk_helper cryptd snd soundcore joydev pcspkr vmw_vmci sg i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm ata_piix mptspi nfit scsi_transport_spi libata libnvdimm serio_raw mptscsih e1000 mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff95c5773b5280 ti: ffff95c5777f4000 task.ti: ffff95c5777f4000
RIP: 0010:[<ffffffffc02b3877>]  [<ffffffffc02b3877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff95c5777f77c0  EFLAGS: 00000286
RAX: ffffbe4940403818 RBX: ffff95c565421200 RCX: 00000000000000d9
RDX: 00000000000000d9 RSI: ffff95c5359c4140 RDI: ffff95c5769be000
RBP: ffff95c5777f7840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff95c50ae7eb40
R13: ffff95c502cbd100 R14: 00000000c2064a67 R15: 0100a8c0777f7750
FS:  0000000000000000(0000) GS:ffff95c57b600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000e896e0 CR3: 0000000071bf2000 CR4: 00000000003607f0
Call Trace:
[<ffffffff8d254386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8d28242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff8d257301>] __dev_queue_xmit+0x491/0x650
[<ffffffff8d2574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff8d2d48ba>] arp_xmit+0x3a/0xb0
[<ffffffff8d2d44f3>] ? arp_create+0x1c3/0x230
[<ffffffff8d2d4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffff8d2d5957>] arp_process+0x877/0x8e0
[<ffffffff8ccd3101>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffff8ccd2ba0>] ? task_rq_unlock+0x20/0x20
[<ffffffff8d23caee>] ? __skb_clone+0x2e/0x150
[<ffffffff8d2d5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffff8d35fbe9>] ? packet_rcv+0x319/0x430
[<ffffffff8d254bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff8cd078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff8d254f18>] __netif_receive_skb+0x18/0x60
[<ffffffff8d254fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8d255c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc02b445b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02b5100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff8cce43cc>] ? update_curr+0x14c/0x1e0
[<ffffffff8d2555bf>] net_rx_action+0x26f/0x390
[<ffffffff8cca4b95>] __do_softirq+0xf5/0x280
[<ffffffff8cca4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8ccce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8ccce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff8ccc5c21>] kthread+0xd1/0xe0
[<ffffffff8ccc5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff8d393df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8ccc5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #128649
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc02ee877
Build id: 3.10.0-1160.11.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff8ee53f96
Build id: 3.10.0-1160.11.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff8ee8203a
Build id: 3.10.0-1160.11.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff8ee56f11
Build id: 3.10.0-1160.11.1.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffff8ee57103
Build id: 3.10.0-1160.11.1.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffff8eed44fa
Build id: 3.10.0-1160.11.1.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffff8eed4133
Build id: 3.10.0-1160.11.1.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffff8eed45b5
Build id: 3.10.0-1160.11.1.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffff8eed5597
Build id: 3.10.0-1160.11.1.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffff8e8d3351
Build id: 3.10.0-1160.11.1.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffff8e8d2df0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffff8ee3caee
Build id: 3.10.0-1160.11.1.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffff8eed574d
Build id: 3.10.0-1160.11.1.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffff8ef5f859
Build id: 3.10.0-1160.11.1.el7.x86_64
0x319
15
? __netif_receive_skb_core
vmlinux 0xffffffff8ee54588
Build id: 3.10.0-1160.11.1.el7.x86_64
0x488
16
__netif_receive_skb_core
vmlinux 0xffffffff8ee547d5
Build id: 3.10.0-1160.11.1.el7.x86_64
0x6d5
17
? __getnstimeofday64
vmlinux 0xffffffff8e907b7f
Build id: 3.10.0-1160.11.1.el7.x86_64
0x3f
18
__netif_receive_skb
vmlinux 0xffffffff8ee54b28
Build id: 3.10.0-1160.11.1.el7.x86_64
0x18
19
netif_receive_skb_internal
vmlinux 0xffffffff8ee54bb0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
20
napi_gro_receive
vmlinux 0xffffffff8ee55838
Build id: 3.10.0-1160.11.1.el7.x86_64
0xd8
21
e1000_clean_rx_irq
e1000 0xffffffffc02ef45b
Build id: 3.10.0-1160.11.1.el7.x86_64
0x2bb
22
e1000_clean
e1000 0xffffffffc02f0100
Build id: 3.10.0-1160.11.1.el7.x86_64
0x280
23
? update_curr
vmlinux 0xffffffff8e8e466c
Build id: 3.10.0-1160.11.1.el7.x86_64
0x14c
24
net_rx_action
vmlinux 0xffffffff8ee551cf
Build id: 3.10.0-1160.11.1.el7.x86_64
0x26f
25
__do_softirq
vmlinux 0xffffffff8e8a4b95
Build id: 3.10.0-1160.11.1.el7.x86_64
0xf5
26
run_ksoftirqd
vmlinux 0xffffffff8e8a4d58
Build id: 3.10.0-1160.11.1.el7.x86_64
0x38
27
smpboot_thread_fn
vmlinux 0xffffffff8e8ce8c4
Build id: 3.10.0-1160.11.1.el7.x86_64
0x144
28
? lg_double_unlock
vmlinux 0xffffffff8e8ce780
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
29
kthread
vmlinux 0xffffffff8e8c5e71
Build id: 3.10.0-1160.11.1.el7.x86_64
0xd1
30
? insert_kthread_work
vmlinux 0xffffffff8e8c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
31
ret_from_fork_nospec_begin
vmlinux 0xffffffff8ef93df7
Build id: 3.10.0-1160.11.1.el7.x86_64
0x21
32
? insert_kthread_work
vmlinux 0xffffffff8e8c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 21s! [ksoftirqd/0:6]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag bridge stp llc ip_set nfnetlink vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw snd_seq_midi snd_seq_midi_event gf128mul glue_helper ablk_helper cryptd joydev ppdev pcspkr sg vmw_balloon parport_pc parport snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore i2c_piix4 vmw_vmci xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common drm crc32c_intel ata_piix mptspi nfit e1000 scsi_transport_spi libata mptscsih libnvdimm mptbase serio_raw drm_panel_orientation_quirks
dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff90fd3936d280 ti: ffff90fd3a760000 task.ti: ffff90fd3a760000
RIP: 0010:[<ffffffffc02ee877>]  [<ffffffffc02ee877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff90fd3a7637c0  EFLAGS: 00000286
RAX: ffffb5ddc1d03818 RBX: 0000000000000001 RCX: 00000000000000f4
RDX: 00000000000000f4 RSI: ffff90ffecb10280 RDI: ffff90fd293a6000
RBP: ffff90fd3a763840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000042a5b4a02 R14: 000000018ea27619 R15: ffffffff8ebb1850
FS:  0000000000000000(0000) GS:ffff90ffed600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000563bb33c9948 CR3: 00000000ba092000 CR4: 00000000001607f0
Call Trace:
[<ffffffff8ee53f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8ee8203a>] sch_direct_xmit+0x11a/0x250
[<ffffffff8ee56f11>] __dev_queue_xmit+0x491/0x650
[<ffffffff8ee57103>] dev_queue_xmit_sk+0x13/0x20
[<ffffffff8eed44fa>] arp_xmit+0x3a/0xb0
[<ffffffff8eed4133>] ? arp_create+0x1c3/0x230
[<ffffffff8eed45b5>] arp_send_dst.part.16+0x45/0x50
[<ffffffff8eed5597>] arp_process+0x877/0x8e0
[<ffffffff8e8d3351>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffff8e8d2df0>] ? task_rq_unlock+0x20/0x20
[<ffffffff8ee3caee>] ? __skb_clone+0x2e/0x150
[<ffffffff8eed574d>] arp_rcv+0x12d/0x1b0
[<ffffffff8ef5f859>] ? packet_rcv+0x319/0x430
[<ffffffff8ee54588>] ? __netif_receive_skb_core+0x488/0xa10
[<ffffffff8ee547d5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffff8e907b7f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffff8ee54b28>] __netif_receive_skb+0x18/0x60
[<ffffffff8ee54bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8ee55838>] napi_gro_receive+0xd8/0x100
[<ffffffffc02ef45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc02f0100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffff8e8e466c>] ? update_curr+0x14c/0x1e0
[<ffffffff8ee551cf>] net_rx_action+0x26f/0x390
[<ffffffff8e8a4b95>] __do_softirq+0xf5/0x280
[<ffffffff8e8a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff8e8ce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff8e8ce780>] ? lg_double_unlock+0x40/0x40
[<ffffffff8e8c5e71>] kthread+0xd1/0xe0
[<ffffffff8e8c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffff8ef93df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff8e8c5da0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #116769
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0468857
Build id: 3.10.0-957.27.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffaf03ace6
Build id: 3.10.0-957.27.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffaf067d6a
Build id: 3.10.0-957.27.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffaf03dc61
Build id: 3.10.0-957.27.2.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffaf03de53
Build id: 3.10.0-957.27.2.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffaf0b9593
Build id: 3.10.0-957.27.2.el7.x86_64
0x33
7
? arp_create
vmlinux 0xffffffffaf0b91d3
Build id: 3.10.0-957.27.2.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffaf0b9645
Build id: 3.10.0-957.27.2.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffaf0ba627
Build id: 3.10.0-957.27.2.el7.x86_64
0x877
10
? __skb_clone
vmlinux 0xffffffffaf023d0e
Build id: 3.10.0-957.27.2.el7.x86_64
0x2e
11
arp_rcv
vmlinux 0xffffffffaf0ba7d5
Build id: 3.10.0-957.27.2.el7.x86_64
0x125
12
? packet_rcv
vmlinux 0xffffffffaf142b99
Build id: 3.10.0-957.27.2.el7.x86_64
0x319
13
__netif_receive_skb_core
vmlinux 0xffffffffaf03b525
Build id: 3.10.0-957.27.2.el7.x86_64
0x6d5
14
? __getnstimeofday64
vmlinux 0xffffffffaeb038ef
Build id: 3.10.0-957.27.2.el7.x86_64
0x3f
15
__netif_receive_skb
vmlinux 0xffffffffaf03b878
Build id: 3.10.0-957.27.2.el7.x86_64
0x18
16
netif_receive_skb_internal
vmlinux 0xffffffffaf03b900
Build id: 3.10.0-957.27.2.el7.x86_64
0x40
17
napi_gro_receive
vmlinux 0xffffffffaf03c588
Build id: 3.10.0-957.27.2.el7.x86_64
0xd8
18
e1000_clean_rx_irq
e1000 0xffffffffc046943b
Build id: 3.10.0-957.27.2.el7.x86_64
0x2bb
19
e1000_clean
e1000 0xffffffffc046a0dd
Build id: 3.10.0-957.27.2.el7.x86_64
0x27d
20
? update_curr
vmlinux 0xffffffffaeae143c
Build id: 3.10.0-957.27.2.el7.x86_64
0x14c
21
net_rx_action
vmlinux 0xffffffffaf03bf1f
Build id: 3.10.0-957.27.2.el7.x86_64
0x26f
22
__do_softirq
vmlinux 0xffffffffaeaa2155
Build id: 3.10.0-957.27.2.el7.x86_64
0xf5
23
run_ksoftirqd
vmlinux 0xffffffffaeaa2318
Build id: 3.10.0-957.27.2.el7.x86_64
0x38
24
smpboot_thread_fn
vmlinux 0xffffffffaeacb814
Build id: 3.10.0-957.27.2.el7.x86_64
0x144
25
? lg_double_unlock
vmlinux 0xffffffffaeacb6d0
Build id: 3.10.0-957.27.2.el7.x86_64
0x40
26
kthread
vmlinux 0xffffffffaeac2e81
Build id: 3.10.0-957.27.2.el7.x86_64
0xd1
27
? insert_kthread_work
vmlinux 0xffffffffaeac2db0
Build id: 3.10.0-957.27.2.el7.x86_64
0x40
28
ret_from_fork_nospec_begin
vmlinux 0xffffffffaf176c1d
Build id: 3.10.0-957.27.2.el7.x86_64
0x7
29
? insert_kthread_work
vmlinux 0xffffffffaeac2db0
Build id: 3.10.0-957.27.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [ksoftirqd/3:24]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq iosf_mbi crc32_pclmul snd_seq_device snd_pcm ppdev ghash_clmulni_intel aesni_intel snd_timer lrw gf128mul glue_helper vmw_balloon snd ablk_helper soundcore cryptd sg pcspkr joydev parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm crc32c_intel drm mptspi ata_piix scsi_transport_spi serio_raw mptscsih libata mptbase nfit e1000 libnvdimm drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G             L ------------   3.10.0-957.27.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff92c73800a080 ti: ffff92c738020000 task.ti: ffff92c738020000
RIP: 0010:[<ffffffffc0468857>]  [<ffffffffc0468857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff92c7380237d0  EFLAGS: 00000286
RAX: ffffa080c0803818 RBX: 0000000000000000 RCX: 000000000000005d
RDX: 000000000000005d RSI: ffff92c635fd64c0 RDI: ffff92c7364a6000
RBP: ffff92c738023850 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff92c7380237e0
R13: 000000000000d500 R14: 000000000000002c R15: ffff92c6b262fb20
FS:  0000000000000000(0000) GS:ffff92c73a6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2fb74e4000 CR3: 00000000b8166000 CR4: 00000000001607e0
Call Trace:
[<ffffffffaf03ace6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffaf067d6a>] sch_direct_xmit+0x11a/0x250
[<ffffffffaf03dc61>] __dev_queue_xmit+0x491/0x650
[<ffffffffaf03de53>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffaf0b9593>] arp_xmit+0x33/0xa0
[<ffffffffaf0b91d3>] ? arp_create+0x1c3/0x230
[<ffffffffaf0b9645>] arp_send_dst.part.16+0x45/0x50
[<ffffffffaf0ba627>] arp_process+0x877/0x8e0
[<ffffffffaf023d0e>] ? __skb_clone+0x2e/0x150
[<ffffffffaf0ba7d5>] arp_rcv+0x125/0x1b0
[<ffffffffaf142b99>] ? packet_rcv+0x319/0x430
[<ffffffffaf03b525>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffaeb038ef>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffaf03b878>] __netif_receive_skb+0x18/0x60
[<ffffffffaf03b900>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffaf03c588>] napi_gro_receive+0xd8/0x100
[<ffffffffc046943b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc046a0dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffffaeae143c>] ? update_curr+0x14c/0x1e0
[<ffffffffaf03bf1f>] net_rx_action+0x26f/0x390
[<ffffffffaeaa2155>] __do_softirq+0xf5/0x280
[<ffffffffaeaa2318>] run_ksoftirqd+0x38/0x50
[<ffffffffaeacb814>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffaeacb6d0>] ? lg_double_unlock+0x40/0x40
[<ffffffffaeac2e81>] kthread+0xd1/0xe0
[<ffffffffaeac2db0>] ? insert_kthread_work+0x40/0x40
[<ffffffffaf176c1d>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffaeac2db0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #67411
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc01ee877
Build id: 3.10.0-1160.15.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb9454f96
Build id: 3.10.0-1160.15.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb948303a
Build id: 3.10.0-1160.15.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb9457f11
Build id: 3.10.0-1160.15.2.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb9458103
Build id: 3.10.0-1160.15.2.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb94d54fa
Build id: 3.10.0-1160.15.2.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb94d5133
Build id: 3.10.0-1160.15.2.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb94d55b5
Build id: 3.10.0-1160.15.2.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb94d6597
Build id: 3.10.0-1160.15.2.el7.x86_64
0x877
10
? __wake_up_common_lock
vmlinux 0xffffffffb8ed32f1
Build id: 3.10.0-1160.15.2.el7.x86_64
0x91
11
? task_rq_unlock
vmlinux 0xffffffffb8ed2d90
Build id: 3.10.0-1160.15.2.el7.x86_64
0x20
12
? __skb_clone
vmlinux 0xffffffffb943daee
Build id: 3.10.0-1160.15.2.el7.x86_64
0x2e
13
arp_rcv
vmlinux 0xffffffffb94d674d
Build id: 3.10.0-1160.15.2.el7.x86_64
0x12d
14
? packet_rcv
vmlinux 0xffffffffb9560869
Build id: 3.10.0-1160.15.2.el7.x86_64
0x319
15
? __netif_receive_skb_core
vmlinux 0xffffffffb9455588
Build id: 3.10.0-1160.15.2.el7.x86_64
0x488
16
__netif_receive_skb_core
vmlinux 0xffffffffb94557d5
Build id: 3.10.0-1160.15.2.el7.x86_64
0x6d5
17
? __getnstimeofday64
vmlinux 0xffffffffb8f07b1f
Build id: 3.10.0-1160.15.2.el7.x86_64
0x3f
18
__netif_receive_skb
vmlinux 0xffffffffb9455b28
Build id: 3.10.0-1160.15.2.el7.x86_64
0x18
19
netif_receive_skb_internal
vmlinux 0xffffffffb9455bb0
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
20
napi_gro_receive
vmlinux 0xffffffffb9456838
Build id: 3.10.0-1160.15.2.el7.x86_64
0xd8
21
e1000_clean_rx_irq
e1000 0xffffffffc01ef45b
Build id: 3.10.0-1160.15.2.el7.x86_64
0x2bb
22
e1000_clean
e1000 0xffffffffc01f0100
Build id: 3.10.0-1160.15.2.el7.x86_64
0x280
23
net_rx_action
vmlinux 0xffffffffb94561cf
Build id: 3.10.0-1160.15.2.el7.x86_64
0x26f
24
__do_softirq
vmlinux 0xffffffffb8ea4b35
Build id: 3.10.0-1160.15.2.el7.x86_64
0xf5
25
run_ksoftirqd
vmlinux 0xffffffffb8ea4cf8
Build id: 3.10.0-1160.15.2.el7.x86_64
0x38
26
smpboot_thread_fn
vmlinux 0xffffffffb8ece864
Build id: 3.10.0-1160.15.2.el7.x86_64
0x144
27
? lg_double_unlock
vmlinux 0xffffffffb8ece720
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
28
kthread
vmlinux 0xffffffffb8ec5e11
Build id: 3.10.0-1160.15.2.el7.x86_64
0xd1
29
? insert_kthread_work
vmlinux 0xffffffffb8ec5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
30
ret_from_fork_nospec_begin
vmlinux 0xffffffffb9594df7
Build id: 3.10.0-1160.15.2.el7.x86_64
0x21
31
? insert_kthread_work
vmlinux 0xffffffffb8ec5d40
Build id: 3.10.0-1160.15.2.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 33s! [ksoftirqd/0:6]
Modules linked in: cifs ccm dns_resolver ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter snd_intel8x0 intel_powerclamp snd_ac97_codec ac97_bus iosf_mbi snd_seq crc32_pclmul snd_seq_device snd_pcm ghash_clmulni_intel aesni_intel lrw gf128mul snd_timer ppdev glue_helper snd ablk_helper cryptd soundcore pcspkr sg joydev parport_pc parport i2c_piix4 video ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi ata_piix crct10dif_pclmul crct10dif_common crc32c_intel serio_raw libata e1000 dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G        W    L ------------   3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff9bd13cb45280 ti: ffff9bd13cb74000 task.ti: ffff9bd13cb74000
RIP: 0010:[<ffffffffc01ee877>]  [<ffffffffc01ee877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9bd13cb777c0  EFLAGS: 00000286
RAX: ffffc18500483818 RBX: ffff9bd13cb45280 RCX: 000000000000008d
RDX: 000000000000008d RSI: ffff9bd0f66b4e00 RDI: ffff9bd1383ee000
RBP: ffff9bd13cb77840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff9bd13cb45280
R13: ffff9bd13fc1f120 R14: ffffffffb90277bc R15: ffff9bd13cb777f0
FS:  0000000000000000(0000) GS:ffff9bd13fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f3a92418000 CR3: 000000007686e000 CR4: 00000000000606f0
Call Trace:
[<ffffffffb9454f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb948303a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9457f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb9458103>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb94d54fa>] arp_xmit+0x3a/0xb0
[<ffffffffb94d5133>] ? arp_create+0x1c3/0x230
[<ffffffffb94d55b5>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb94d6597>] arp_process+0x877/0x8e0
[<ffffffffb8ed32f1>] ? __wake_up_common_lock+0x91/0xc0
[<ffffffffb8ed2d90>] ? task_rq_unlock+0x20/0x20
[<ffffffffb943daee>] ? __skb_clone+0x2e/0x150
[<ffffffffb94d674d>] arp_rcv+0x12d/0x1b0
[<ffffffffb9560869>] ? packet_rcv+0x319/0x430
[<ffffffffb9455588>] ? __netif_receive_skb_core+0x488/0xa10
[<ffffffffb94557d5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb8f07b1f>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb9455b28>] __netif_receive_skb+0x18/0x60
[<ffffffffb9455bb0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb9456838>] napi_gro_receive+0xd8/0x100
[<ffffffffc01ef45b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc01f0100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb94561cf>] net_rx_action+0x26f/0x390
[<ffffffffb8ea4b35>] __do_softirq+0xf5/0x280
[<ffffffffb8ea4cf8>] run_ksoftirqd+0x38/0x50
[<ffffffffb8ece864>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb8ece720>] ? lg_double_unlock+0x40/0x40
[<ffffffffb8ec5e11>] kthread+0xd1/0xe0
[<ffffffffb8ec5d40>] ? insert_kthread_work+0x40/0x40
[<ffffffffb9594df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb8ec5d40>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #16439
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0413877
Build id: 3.10.0-1127.19.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb9253386
Build id: 3.10.0-1127.19.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb928131a
Build id: 3.10.0-1127.19.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb9256301
Build id: 3.10.0-1127.19.1.el7.x86_64
0x491
5
dev_queue_xmit_sk
vmlinux 0xffffffffb92564f3
Build id: 3.10.0-1127.19.1.el7.x86_64
0x13
6
arp_xmit
vmlinux 0xffffffffb92d369a
Build id: 3.10.0-1127.19.1.el7.x86_64
0x3a
7
? arp_create
vmlinux 0xffffffffb92d32d3
Build id: 3.10.0-1127.19.1.el7.x86_64
0x1c3
8
arp_send_dst.part.16
vmlinux 0xffffffffb92d3755
Build id: 3.10.0-1127.19.1.el7.x86_64
0x45
9
arp_process
vmlinux 0xffffffffb92d4737
Build id: 3.10.0-1127.19.1.el7.x86_64
0x877
10
? __skb_clone
vmlinux 0xffffffffb923baee
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2e
11
arp_rcv
vmlinux 0xffffffffb92d48ed
Build id: 3.10.0-1127.19.1.el7.x86_64
0x12d
12
? update_curr
vmlinux 0xffffffffb8ce4eac
Build id: 3.10.0-1127.19.1.el7.x86_64
0x14c
13
__netif_receive_skb_core
vmlinux 0xffffffffb9253bc5
Build id: 3.10.0-1127.19.1.el7.x86_64
0x6d5
14
? __getnstimeofday64
vmlinux 0xffffffffb8d082af
Build id: 3.10.0-1127.19.1.el7.x86_64
0x3f
15
__netif_receive_skb
vmlinux 0xffffffffb9253f18
Build id: 3.10.0-1127.19.1.el7.x86_64
0x18
16
netif_receive_skb_internal
vmlinux 0xffffffffb9253fa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
17
napi_gro_receive
vmlinux 0xffffffffb9254c28
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd8
18
e1000_clean_rx_irq
e1000 0xffffffffc041445b
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2bb
19
e1000_clean
e1000 0xffffffffc0415100
Build id: 3.10.0-1127.19.1.el7.x86_64
0x280
20
net_rx_action
vmlinux 0xffffffffb92545bf
Build id: 3.10.0-1127.19.1.el7.x86_64
0x26f
21
__do_softirq
vmlinux 0xffffffffb8ca5695
Build id: 3.10.0-1127.19.1.el7.x86_64
0xf5
22
run_ksoftirqd
vmlinux 0xffffffffb8ca5858
Build id: 3.10.0-1127.19.1.el7.x86_64
0x38
23
smpboot_thread_fn
vmlinux 0xffffffffb8ccf0e4
Build id: 3.10.0-1127.19.1.el7.x86_64
0x144
24
? lg_double_unlock
vmlinux 0xffffffffb8ccefa0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
25
kthread
vmlinux 0xffffffffb8cc6691
Build id: 3.10.0-1127.19.1.el7.x86_64
0xd1
26
? insert_kthread_work
vmlinux 0xffffffffb8cc65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
27
ret_from_fork_nospec_begin
vmlinux 0xffffffffb9392d37
Build id: 3.10.0-1127.19.1.el7.x86_64
0x21
28
? insert_kthread_work
vmlinux 0xffffffffb8cc65c0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 33s! [ksoftirqd/1:14]
Modules linked in: ip6table_filter ip6_tables iptable_filter ppdev pcspkr sg joydev parport_pc parport i2c_piix4 virtio_balloon ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 virtio_pci virtio_ring serio_raw virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.7.5-0-ge51488c-20140602_164612-nilsson.home.kraxel.org 04/01/2014
task: ffff9023fc1a8000 ti: ffff9023fc1b0000 task.ti: ffff9023fc1b0000
RIP: 0010:[<ffffffffc0413877>]  [<ffffffffc0413877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff9023fc1b37c0  EFLAGS: 00010286
RAX: ffffada8c0403818 RBX: ffff9023b5de0500 RCX: 0000000000000025
RDX: 0000000000000025 RSI: ffff9023b65703c0 RDI: ffff9023fb29a000
RBP: ffff9023fc1b3840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffb9915c80
R13: ffff9023f82a4c00 R14: ffff9023fc1b38b0 R15: 000000009942e083
FS:  0000000000000000(0000) GS:ffff9023ffd00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1599467de2 CR3: 0000000026c10000 CR4: 00000000000006e0
Call Trace:
[<ffffffffb9253386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb928131a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9256301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb92564f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffb92d369a>] arp_xmit+0x3a/0xb0
[<ffffffffb92d32d3>] ? arp_create+0x1c3/0x230
[<ffffffffb92d3755>] arp_send_dst.part.16+0x45/0x50
[<ffffffffb92d4737>] arp_process+0x877/0x8e0
[<ffffffffb923baee>] ? __skb_clone+0x2e/0x150
[<ffffffffb92d48ed>] arp_rcv+0x12d/0x1b0
[<ffffffffb8ce4eac>] ? update_curr+0x14c/0x1e0
[<ffffffffb9253bc5>] __netif_receive_skb_core+0x6d5/0xa10
[<ffffffffb8d082af>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffb9253f18>] __netif_receive_skb+0x18/0x60
[<ffffffffb9253fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb9254c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc041445b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0415100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffb92545bf>] net_rx_action+0x26f/0x390
[<ffffffffb8ca5695>] __do_softirq+0xf5/0x280
[<ffffffffb8ca5858>] run_ksoftirqd+0x38/0x50
[<ffffffffb8ccf0e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb8ccefa0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb8cc6691>] kthread+0xd1/0xe0
[<ffffffffb8cc65c0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb9392d37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb8cc65c0>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #540426
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0227877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffbda3c6f7
Build id: 3.10.0-1160.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffffbda54386
Build id: 3.10.0-1160.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffffbda8242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffffbda57301
Build id: 3.10.0-1160.el7.x86_64
0x491
6
dev_queue_xmit_sk
vmlinux 0xffffffffbda574f3
Build id: 3.10.0-1160.el7.x86_64
0x13
7
arp_xmit
vmlinux 0xffffffffbdad48ba
Build id: 3.10.0-1160.el7.x86_64
0x3a
8
? arp_create
vmlinux 0xffffffffbdad44f3
Build id: 3.10.0-1160.el7.x86_64
0x1c3
9
arp_send_dst.part.16
vmlinux 0xffffffffbdad4975
Build id: 3.10.0-1160.el7.x86_64
0x45
10
arp_process
vmlinux 0xffffffffbdad5957
Build id: 3.10.0-1160.el7.x86_64
0x877
11
? tcp_v4_early_demux
vmlinux 0xffffffffbdac67d2
Build id: 3.10.0-1160.el7.x86_64
0x112
12
arp_rcv
vmlinux 0xffffffffbdad5b0d
Build id: 3.10.0-1160.el7.x86_64
0x12d
13
? packet_rcv
vmlinux 0xffffffffbdb5f91c
Build id: 3.10.0-1160.el7.x86_64
0x4c
14
__netif_receive_skb_core
vmlinux 0xffffffffbda54c19
Build id: 3.10.0-1160.el7.x86_64
0x729
15
? __getnstimeofday64
vmlinux 0xffffffffbd5078df
Build id: 3.10.0-1160.el7.x86_64
0x3f
16
__netif_receive_skb
vmlinux 0xffffffffbda54f18
Build id: 3.10.0-1160.el7.x86_64
0x18
17
netif_receive_skb_internal
vmlinux 0xffffffffbda54fa0
Build id: 3.10.0-1160.el7.x86_64
0x40
18
napi_gro_receive
vmlinux 0xffffffffbda55c28
Build id: 3.10.0-1160.el7.x86_64
0xd8
19
e1000_clean_rx_irq
e1000 0xffffffffc022845b
Build id: 3.10.0-1160.el7.x86_64
0x2bb
20
e1000_clean
e1000 0xffffffffc0229100
Build id: 3.10.0-1160.el7.x86_64
0x280
21
? update_curr
vmlinux 0xffffffffbd4e43cc
Build id: 3.10.0-1160.el7.x86_64
0x14c
22
net_rx_action
vmlinux 0xffffffffbda555bf
Build id: 3.10.0-1160.el7.x86_64
0x26f
23
__do_softirq
vmlinux 0xffffffffbd4a4b95
Build id: 3.10.0-1160.el7.x86_64
0xf5
24
run_ksoftirqd
vmlinux 0xffffffffbd4a4d58
Build id: 3.10.0-1160.el7.x86_64
0x38
25
smpboot_thread_fn
vmlinux 0xffffffffbd4ce674
Build id: 3.10.0-1160.el7.x86_64
0x144
26
? lg_double_unlock
vmlinux 0xffffffffbd4ce530
Build id: 3.10.0-1160.el7.x86_64
0x40
27
kthread
vmlinux 0xffffffffbd4c5c21
Build id: 3.10.0-1160.el7.x86_64
0xd1
28
? insert_kthread_work
vmlinux 0xffffffffbd4c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
29
ret_from_fork_nospec_begin
vmlinux 0xffffffffbdb93ddd
Build id: 3.10.0-1160.el7.x86_64
0x7
30
? insert_kthread_work
vmlinux 0xffffffffbd4c5b50
Build id: 3.10.0-1160.el7.x86_64
0x40
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 25s! [ksoftirqd/2:19]
Modules linked in: tun devlink bridge stp llc ip_set nfnetlink bnep vmw_vsock_vmci_transport vsock sunrpc crc32_pclmul ghash_clmulni_intel aesni_intel btusb lrw gf128mul ppdev btrtl glue_helper btbcm ablk_helper btintel cryptd bluetooth vmw_balloon rfkill sg pcspkr joydev parport_pc parport snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd soundcore i2c_piix4 vmw_vmci xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crc32c_intel nfit libnvdimm drm ata_piix serio_raw e1000 mptspi scsi_transport_spi libata mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash
dm_log dm_mod fuse [last unloaded: ip_tables]
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffffa0bcf83f5280 ti: ffffa0bcf8798000 task.ti: ffffa0bcf8798000
RIP: 0010:[<ffffffffc0227877>]  [<ffffffffc0227877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffffa0bcf879b7c0  EFLAGS: 00000286
RAX: ffffbe4180383818 RBX: 0000000000000010 RCX: 0000000000000068
RDX: 0000000000000068 RSI: ffffa0bcb4d6b140 RDI: ffffa0bcf59da000
RBP: ffffa0bcf879b840 R08: 0000000000000100 R09: 0000000002000000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffffffffff10
R13: ffffa0bcf59da000 R14: ffffa0bcb4d6b140 R15: 0000000000000062
FS:  0000000000000000(0000) GS:ffffa0bcf8c80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000ee6fd0 CR3: 000000007b810000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffffbda3c6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffffbda54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbda8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffffbda57301>] __dev_queue_xmit+0x491/0x650
[<ffffffffbda574f3>] dev_queue_xmit_sk+0x13/0x20
[<ffffffffbdad48ba>] arp_xmit+0x3a/0xb0
[<ffffffffbdad44f3>] ? arp_create+0x1c3/0x230
[<ffffffffbdad4975>] arp_send_dst.part.16+0x45/0x50
[<ffffffffbdad5957>] arp_process+0x877/0x8e0
[<ffffffffbdac67d2>] ? tcp_v4_early_demux+0x112/0x160
[<ffffffffbdad5b0d>] arp_rcv+0x12d/0x1b0
[<ffffffffbdb5f91c>] ? packet_rcv+0x4c/0x430
[<ffffffffbda54c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffbd5078df>] ? __getnstimeofday64+0x3f/0xd0
[<ffffffffbda54f18>] __netif_receive_skb+0x18/0x60
[<ffffffffbda54fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffbda55c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc022845b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc0229100>] e1000_clean+0x280/0x8e0 [e1000]
[<ffffffffbd4e43cc>] ? update_curr+0x14c/0x1e0
[<ffffffffbda555bf>] net_rx_action+0x26f/0x390
[<ffffffffbd4a4b95>] __do_softirq+0xf5/0x280
[<ffffffffbd4a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffbd4ce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffbd4ce530>] ? lg_double_unlock+0x40/0x40
[<ffffffffbd4c5c21>] kthread+0xd1/0xe0
[<ffffffffbd4c5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffffbdb93ddd>] ret_from_fork_nospec_begin+0x7/0x21
[<ffffffffbd4c5b50>] ? insert_kthread_work+0x40/0x40
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00