Info

Permalink
Function
e1000_xmit_frame
First occurrence
2020-10-21
Last occurrence
2020-12-17
State
NEW
Components
kernel
Quality
-68

Statistics

Operating system Count
CentOS 7 115
Architecture Count
x86_64 115
Related packages Count
kernel 115
 0:3.10.0-693.el7 96
 0:3.10.0-1160.el7 9
 0:3.10.0-1127.el7 3
 0:3.10.0-1127.18.2.el7 2
 0:3.10.0-1160.119.1.el7 2
 0:3.10.0-1160.11.1.el7 1
 0:3.10.0-1160.45.1.el7 1
 0:3.10.0-1160.108.1.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #1199217
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc00bd8e7
Build id: 3.10.0-1160.119.1.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffffb7266c7f
Build id: 3.10.0-1160.119.1.el7.x86_64
0x3f
3
dev_hard_start_xmit
vmlinux 0xffffffffb727edc6
Build id: 3.10.0-1160.119.1.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffffb72add72
Build id: 3.10.0-1160.119.1.el7.x86_64
0x122
5
__dev_queue_xmit
vmlinux 0xffffffffb7281dd1
Build id: 3.10.0-1160.119.1.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffffb7281fa0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffffb72d260b
Build id: 3.10.0-1160.119.1.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffffb72d2b9b
Build id: 3.10.0-1160.119.1.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffffb72d1f90
Build id: 3.10.0-1160.119.1.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffffb72d0587
Build id: 3.10.0-1160.119.1.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffffb72d3636
Build id: 3.10.0-1160.119.1.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffffb72fc3ac
Build id: 3.10.0-1160.119.1.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffffb72d02b0
Build id: 3.10.0-1160.119.1.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffffb72fd9a1
Build id: 3.10.0-1160.119.1.el7.x86_64
0x5e1
15
? udp_poll
vmlinux 0xffffffffb72fed49
Build id: 3.10.0-1160.119.1.el7.x86_64
0x19
16
inet_sendmsg
vmlinux 0xffffffffb730b4e9
Build id: 3.10.0-1160.119.1.el7.x86_64
0x69
17
sock_sendmsg
vmlinux 0xffffffffb725e4f6
Build id: 3.10.0-1160.119.1.el7.x86_64
0xb6
18
? page_fault
vmlinux 0xffffffffb73bb7b8
Build id: 3.10.0-1160.119.1.el7.x86_64
0x28
19
___sys_sendmsg
vmlinux 0xffffffffb725f307
Build id: 3.10.0-1160.119.1.el7.x86_64
0x2f7
20
? release_pages
vmlinux 0xffffffffb6dd892e
Build id: 3.10.0-1160.119.1.el7.x86_64
0x24e
21
? __ip_route_output_key_hash
vmlinux 0xffffffffb72c8c63
Build id: 3.10.0-1160.119.1.el7.x86_64
0x223
22
? ip4_datagram_release_cb
vmlinux 0xffffffffb72f9cdf
Build id: 3.10.0-1160.119.1.el7.x86_64
0x6f
23
? _raw_spin_unlock_bh
vmlinux 0xffffffffb73ba56e
Build id: 3.10.0-1160.119.1.el7.x86_64
0x1e
24
? udp_lib_rehash
vmlinux 0xffffffffb72fcecf
Build id: 3.10.0-1160.119.1.el7.x86_64
0x11f
25
__sys_sendmmsg
vmlinux 0xffffffffb7260c4f
Build id: 3.10.0-1160.119.1.el7.x86_64
0x9f
26
? SYSC_connect
vmlinux 0xffffffffb725ea90
Build id: 3.10.0-1160.119.1.el7.x86_64
0x90
27
? ip_setsockopt
vmlinux 0xffffffffb72d5bcc
Build id: 3.10.0-1160.119.1.el7.x86_64
0x2c
28
SyS_sendmmsg
vmlinux 0xffffffffb7260d62
Build id: 3.10.0-1160.119.1.el7.x86_64
0x12
29
system_call_fastpath
vmlinux 0xffffffffb73c539a
Build id: 3.10.0-1160.119.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 42s! [timer:73372]
Modules linked in: binfmt_misc tcp_lp xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 ppdev snd_rawmidi crc32_pclmul snd_ac97_codec ac97_bus ghash_clmulni_intel vmw_balloon snd_seq aesni_intel snd_seq_device lrw gf128mul snd_pcm glue_helper ablk_helper cryptd snd_timer sg snd pcspkr joydev soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm drm nfit libnvdimm ata_piix libata mptspi crct10dif_pclmul crct10dif_common crc32c_intel scsi_transport_spi e1000 mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 73372 Comm: timer Kdump: loaded Tainted: G             L ------------   3.10.0-1160.119.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff935965712100 ti: ffff93596cd60000 task.ti: ffff93596cd60000
RIP: 0010:[<ffffffffc00bd8e7>]  [<ffffffffc00bd8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff93596cd636a0  EFLAGS: 00000286
RAX: ffffa94681983818 RBX: 0000000000000000 RCX: 0000000000000044
RDX: 0000000000000044 RSI: ffff93599c7143c0 RDI: ffff9359a014e000
RBP: ffff93596cd63720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff93596cd63708
R13: 000100006cd63678 R14: ffffffffc066f99d R15: ffff93596cd63638
FS:  00007efe257fa700(0000) GS:ffff9359a1bc0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efe257f7fec CR3: 0000000241818000 CR4: 0000000000760fe0
PKRU: 55555554
Call Trace:
[<ffffffffb7266c7f>] ? kfree_skbmem+0x3f/0xa0
[<ffffffffb727edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb72add72>] sch_direct_xmit+0x122/0x250
[<ffffffffb7281dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb7281fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffb72d260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb72d2b9b>] ip_output+0x7b/0x100
[<ffffffffb72d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb72d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffb72d3636>] ip_send_skb+0x16/0x50
[<ffffffffb72fc3ac>] udp_send_skb+0xac/0x2b0
[<ffffffffb72d02b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb72fd9a1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb72fed49>] ? udp_poll+0x19/0xa0
[<ffffffffb730b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffffb725e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffffb73bb7b8>] ? page_fault+0x28/0x30
[<ffffffffb725f307>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffb6dd892e>] ? release_pages+0x24e/0x430
[<ffffffffb72c8c63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffb72f9cdf>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffb73ba56e>] ? _raw_spin_unlock_bh+0x1e/0x30
[<ffffffffb72fcecf>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffb7260c4f>] __sys_sendmmsg+0x9f/0x1a0
[<ffffffffb725ea90>] ? SYSC_connect+0x90/0x130
[<ffffffffb72d5bcc>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb7260d62>] SyS_sendmmsg+0x12/0x30
[<ffffffffb73c539a>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc 
Complete report #1067486
Frame # Function Binary Source or offset Line
1
? kfree_skbmem
vmlinux 0xffffffffb5466c7f
Build id: 3.10.0-1160.108.1.el7.x86_64
0x3f
2
dev_hard_start_xmit
vmlinux 0xffffffffb547edc6
Build id: 3.10.0-1160.108.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb54add72
Build id: 3.10.0-1160.108.1.el7.x86_64
0x122
4
__dev_queue_xmit
vmlinux 0xffffffffb5481dd1
Build id: 3.10.0-1160.108.1.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb5481fa0
Build id: 3.10.0-1160.108.1.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb54d260b
Build id: 3.10.0-1160.108.1.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb54d2b9b
Build id: 3.10.0-1160.108.1.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb54d1f90
Build id: 3.10.0-1160.108.1.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb54d0587
Build id: 3.10.0-1160.108.1.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffffb54d3636
Build id: 3.10.0-1160.108.1.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffffb54fc3ac
Build id: 3.10.0-1160.108.1.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffffb54d02b0
Build id: 3.10.0-1160.108.1.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffffb54fd9a1
Build id: 3.10.0-1160.108.1.el7.x86_64
0x5e1
14
? udp_poll
vmlinux 0xffffffffb54fed49
Build id: 3.10.0-1160.108.1.el7.x86_64
0x19
15
inet_sendmsg
vmlinux 0xffffffffb550b4e9
Build id: 3.10.0-1160.108.1.el7.x86_64
0x69
16
sock_sendmsg
vmlinux 0xffffffffb545e4f6
Build id: 3.10.0-1160.108.1.el7.x86_64
0xb6
17
? page_fault
vmlinux 0xffffffffb55bb7b8
Build id: 3.10.0-1160.108.1.el7.x86_64
0x28
18
___sys_sendmsg
vmlinux 0xffffffffb545f307
Build id: 3.10.0-1160.108.1.el7.x86_64
0x2f7
19
? release_pages
vmlinux 0xffffffffb4fd88de
Build id: 3.10.0-1160.108.1.el7.x86_64
0x24e
20
? __ip_route_output_key_hash
vmlinux 0xffffffffb54c8c63
Build id: 3.10.0-1160.108.1.el7.x86_64
0x223
21
? ip4_datagram_release_cb
vmlinux 0xffffffffb54f9cdf
Build id: 3.10.0-1160.108.1.el7.x86_64
0x6f
22
? _raw_spin_unlock_bh
vmlinux 0xffffffffb55ba56e
Build id: 3.10.0-1160.108.1.el7.x86_64
0x1e
23
? udp_lib_rehash
vmlinux 0xffffffffb54fcecf
Build id: 3.10.0-1160.108.1.el7.x86_64
0x11f
24
__sys_sendmmsg
vmlinux 0xffffffffb5460c4f
Build id: 3.10.0-1160.108.1.el7.x86_64
0x9f
25
? SYSC_connect
vmlinux 0xffffffffb545ea90
Build id: 3.10.0-1160.108.1.el7.x86_64
0x90
26
? ip_setsockopt
vmlinux 0xffffffffb54d5bcc
Build id: 3.10.0-1160.108.1.el7.x86_64
0x2c
27
SyS_sendmmsg
vmlinux 0xffffffffb5460d62
Build id: 3.10.0-1160.108.1.el7.x86_64
0x12
28
system_call_fastpath
vmlinux 0xffffffffb55c539a
Build id: 3.10.0-1160.108.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 31s! [mysqld:118021]
FS:  00007f6f15f72700(0000) GS:ffff8d6ead6c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f6f297a7000 CR3: 00000003de752000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb5466c7f>] ? kfree_skbmem+0x3f/0xa0
[<ffffffffb547edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb54add72>] sch_direct_xmit+0x122/0x250
[<ffffffffb5481dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb5481fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffb54d260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb54d2b9b>] ip_output+0x7b/0x100
[<ffffffffb54d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb54d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffb54d3636>] ip_send_skb+0x16/0x50
[<ffffffffb54fc3ac>] udp_send_skb+0xac/0x2b0
[<ffffffffb54d02b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb54fd9a1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb54fed49>] ? udp_poll+0x19/0xa0
[<ffffffffb550b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffffb545e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffffb55bb7b8>] ? page_fault+0x28/0x30
[<ffffffffb545f307>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffb4fd88de>] ? release_pages+0x24e/0x430
[<ffffffffb54c8c63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffb54f9cdf>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffb55ba56e>] ? _raw_spin_unlock_bh+0x1e/0x30
[<ffffffffb54fcecf>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffb5460c4f>] __sys_sendmmsg+0x9f/0x1a0
[<ffffffffb545ea90>] ? SYSC_connect+0x90/0x130
[<ffffffffb54d5bcc>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb5460d62>] SyS_sendmmsg+0x12/0x30
[<ffffffffb55c539a>] system_call_fastpath+0x25/0x2a
Complete report #1033039
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0164877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
? __alloc_skb
vmlinux 0xffffffff9283e13d
Build id: 3.10.0-1160.el7.x86_64
0x8d
3
dev_hard_start_xmit
vmlinux 0xffffffff92854386
Build id: 3.10.0-1160.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff9288242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff92857301
Build id: 3.10.0-1160.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff928574d0
Build id: 3.10.0-1160.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff928a545b
Build id: 3.10.0-1160.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff928a59db
Build id: 3.10.0-1160.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff928a4de0
Build id: 3.10.0-1160.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff928a3407
Build id: 3.10.0-1160.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff928a6446
Build id: 3.10.0-1160.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff928ce85c
Build id: 3.10.0-1160.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff928a3140
Build id: 3.10.0-1160.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff928cfdc1
Build id: 3.10.0-1160.el7.x86_64
0x5e1
15
? hrtimer_get_res
vmlinux 0xffffffff922c9860
Build id: 3.10.0-1160.el7.x86_64
0x50
16
? sock_has_perm
vmlinux 0xffffffff9250be65
Build id: 3.10.0-1160.el7.x86_64
0x75
17
inet_sendmsg
vmlinux 0xffffffff928dd589
Build id: 3.10.0-1160.el7.x86_64
0x69
18
? selinux_socket_sendmsg
vmlinux 0xffffffff9250bf93
Build id: 3.10.0-1160.el7.x86_64
0x23
19
sock_sendmsg
vmlinux 0xffffffff928343a6
Build id: 3.10.0-1160.el7.x86_64
0xb6
20
? ___slab_alloc
vmlinux 0xffffffff92427439
Build id: 3.10.0-1160.el7.x86_64
0x229
21
? __radix_tree_lookup
vmlinux 0xffffffff9258cfe4
Build id: 3.10.0-1160.el7.x86_64
0x84
22
___sys_sendmsg
vmlinux 0xffffffff92835177
Build id: 3.10.0-1160.el7.x86_64
0x2f7
23
? __ip_route_output_key_hash
vmlinux 0xffffffff9289bc63
Build id: 3.10.0-1160.el7.x86_64
0x223
24
? ip4_datagram_release_cb
vmlinux 0xffffffff928cc28f
Build id: 3.10.0-1160.el7.x86_64
0x6f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffff9298992e
Build id: 3.10.0-1160.el7.x86_64
0x1e
26
? udp_lib_rehash
vmlinux 0xffffffff928cf33f
Build id: 3.10.0-1160.el7.x86_64
0x11f
27
? _raw_spin_unlock_bh
vmlinux 0xffffffff9298992e
Build id: 3.10.0-1160.el7.x86_64
0x1e
28
__sys_sendmmsg
vmlinux 0xffffffff92836a1f
Build id: 3.10.0-1160.el7.x86_64
0x9f
29
? SYSC_connect
vmlinux 0xffffffff9283496d
Build id: 3.10.0-1160.el7.x86_64
0xed
30
? ip_setsockopt
vmlinux 0xffffffff928a895c
Build id: 3.10.0-1160.el7.x86_64
0x2c
31
SyS_sendmmsg
vmlinux 0xffffffff92836b22
Build id: 3.10.0-1160.el7.x86_64
0x12
32
system_call_fastpath
vmlinux 0xffffffff92993f92
Build id: 3.10.0-1160.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 58s! [ntpdate:86887]
Modules linked in: vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec iosf_mbi crc32_pclmul ac97_bus ghash_clmulni_intel snd_seq snd_seq_device snd_pcm aesni_intel lrw gf128mul snd_timer glue_helper ppdev snd ablk_helper cryptd soundcore vmw_balloon pcspkr sg joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt libata fb_sys_fops ttm mptspi crct10dif_pclmul crct10dif_common nfit scsi_transport_spi drm crc32c_intel libnvdimm mptscsih mptbase serio_raw e1000 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 86887 Comm: ntpdate Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff8853634c3180 ti: ffff88536eae0000 task.ti: ffff88536eae0000
RIP: 0010:[<ffffffffc0164877>]  [<ffffffffc0164877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff88536eae36a0  EFLAGS: 00000286
RAX: ffffb852c0983818 RBX: ffff8852f64d5980 RCX: 000000000000005b
RDX: 000000000000005b RSI: ffff8852f64d5980 RDI: ffff8853ef8ae000
RBP: ffff88536eae3720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 000000000000000d
R13: ffffb852c0983818 R14: 0000000000000000 R15: 0000000022100000
FS:  00007f33c0a43740(0000) GS:ffff8853f9600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f33bf1b9cc0 CR3: 00000000a34ec000 CR4: 00000000001607f0
Call Trace:
[<ffffffff9283e13d>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff92854386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9288242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff92857301>] __dev_queue_xmit+0x491/0x650
[<ffffffff928574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff928a545b>] ip_finish_output+0x67b/0x8d0
[<ffffffff928a59db>] ip_output+0x7b/0xf0
[<ffffffff928a4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff928a3407>] ip_local_out_sk+0x37/0x40
[<ffffffff928a6446>] ip_send_skb+0x16/0x50
[<ffffffff928ce85c>] udp_send_skb+0xac/0x2b0
[<ffffffff928a3140>] ? ip_copy_metadata+0x170/0x170
[<ffffffff928cfdc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff922c9860>] ? hrtimer_get_res+0x50/0x50
[<ffffffff9250be65>] ? sock_has_perm+0x75/0x90
[<ffffffff928dd589>] inet_sendmsg+0x69/0xb0
[<ffffffff9250bf93>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff928343a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff92427439>] ? ___slab_alloc+0x229/0x520
[<ffffffff9258cfe4>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffff92835177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff9289bc63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffff928cc28f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff9298992e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff928cf33f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff9298992e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff92836a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff9283496d>] ? SYSC_connect+0xed/0x130
[<ffffffff928a895c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff92836b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff92993f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #698878
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0277877
Build id: 3.10.0-1160.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff90c54386
Build id: 3.10.0-1160.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff90c8242a
Build id: 3.10.0-1160.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff90c57301
Build id: 3.10.0-1160.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffff90c574d0
Build id: 3.10.0-1160.el7.x86_64
0x10
6
neigh_resolve_output
vmlinux 0xffffffff90c641fd
Build id: 3.10.0-1160.el7.x86_64
0x11d
7
ip_finish_output
vmlinux 0xffffffff90ca505c
Build id: 3.10.0-1160.el7.x86_64
0x27c
8
ip_output
vmlinux 0xffffffff90ca59db
Build id: 3.10.0-1160.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff90ca4de0
Build id: 3.10.0-1160.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff90ca3407
Build id: 3.10.0-1160.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff90ca6446
Build id: 3.10.0-1160.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff90cce85c
Build id: 3.10.0-1160.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff90ca3140
Build id: 3.10.0-1160.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff90ccfdc1
Build id: 3.10.0-1160.el7.x86_64
0x5e1
15
? sock_has_perm
vmlinux 0xffffffff9090be65
Build id: 3.10.0-1160.el7.x86_64
0x75
16
inet_sendmsg
vmlinux 0xffffffff90cdd589
Build id: 3.10.0-1160.el7.x86_64
0x69
17
? selinux_socket_sendmsg
vmlinux 0xffffffff9090bf93
Build id: 3.10.0-1160.el7.x86_64
0x23
18
sock_sendmsg
vmlinux 0xffffffff90c343a6
Build id: 3.10.0-1160.el7.x86_64
0xb6
19
? ___slab_alloc
vmlinux 0xffffffff90827439
Build id: 3.10.0-1160.el7.x86_64
0x229
20
? __radix_tree_lookup
vmlinux 0xffffffff9098cfe4
Build id: 3.10.0-1160.el7.x86_64
0x84
21
___sys_sendmsg
vmlinux 0xffffffff90c35177
Build id: 3.10.0-1160.el7.x86_64
0x2f7
22
? __xfs_filemap_fault
xfs 0xffffffffc04ef92e
Build id: 3.10.0-1160.el7.x86_64
0x8e
23
? __ip_route_output_key_hash
vmlinux 0xffffffff90c9bc63
Build id: 3.10.0-1160.el7.x86_64
0x223
24
? ip4_datagram_release_cb
vmlinux 0xffffffff90ccc28f
Build id: 3.10.0-1160.el7.x86_64
0x6f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffff90d8992e
Build id: 3.10.0-1160.el7.x86_64
0x1e
26
? udp_lib_rehash
vmlinux 0xffffffff90ccf33f
Build id: 3.10.0-1160.el7.x86_64
0x11f
27
? _raw_spin_unlock_bh
vmlinux 0xffffffff90d8992e
Build id: 3.10.0-1160.el7.x86_64
0x1e
28
__sys_sendmmsg
vmlinux 0xffffffff90c36a1f
Build id: 3.10.0-1160.el7.x86_64
0x9f
29
? SYSC_connect
vmlinux 0xffffffff90c34910
Build id: 3.10.0-1160.el7.x86_64
0x90
30
? ip_setsockopt
vmlinux 0xffffffff90ca895c
Build id: 3.10.0-1160.el7.x86_64
0x2c
31
SyS_sendmmsg
vmlinux 0xffffffff90c36b22
Build id: 3.10.0-1160.el7.x86_64
0x12
32
system_call_fastpath
vmlinux 0xffffffff90d93f92
Build id: 3.10.0-1160.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 31s! [reporter-urepor:26075]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink binfmt_misc vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache iosf_mbi crc32_pclmul jbd2 ghash_clmulni_intel ppdev snd_ens1371 aesni_intel snd_rawmidi snd_ac97_codec lrw gf128mul glue_helper ablk_helper vmw_balloon cryptd ac97_bus snd_seq snd_seq_device snd_pcm pcspkr snd_timer snd soundcore sg joydev parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea
sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel nfit drm libnvdimm ata_piix libata serio_raw mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 26075 Comm: reporter-urepor Kdump: loaded Tainted: G             L ------------   3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff919fef10a100 ti: ffff919ec1db4000 task.ti: ffff919ec1db4000
RIP: 0010:[<ffffffffc0277877>]  [<ffffffffc0277877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff919ec1db7660  EFLAGS: 00000286
RAX: ffff9e7280703818 RBX: 0000000000000001 RCX: 00000000000000a3
RDX: 00000000000000a3 RSI: ffff919ef5f0fc00 RDI: ffff919ff7fbe000
RBP: ffff919ec1db76e0 R08: 0000000000000100 R09: 0000000022100000
R10: 037463656a6f7270 R11: 0000000000000000 R12: 0000000000000202
R13: 000000012fa35402 R14: 00000001f9600000 R15: ffffffff909b1650
FS:  00007fdf0c874700(0000) GS:ffff919ff9600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fdf170b3560 CR3: 00000000b511a000 CR4: 00000000003607f0
Call Trace:
[<ffffffff90c54386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff90c8242a>] sch_direct_xmit+0x11a/0x250
[<ffffffff90c57301>] __dev_queue_xmit+0x491/0x650
[<ffffffff90c574d0>] dev_queue_xmit+0x10/0x20
[<ffffffff90c641fd>] neigh_resolve_output+0x11d/0x220
[<ffffffff90ca505c>] ip_finish_output+0x27c/0x8d0
[<ffffffff90ca59db>] ip_output+0x7b/0xf0
[<ffffffff90ca4de0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff90ca3407>] ip_local_out_sk+0x37/0x40
[<ffffffff90ca6446>] ip_send_skb+0x16/0x50
[<ffffffff90cce85c>] udp_send_skb+0xac/0x2b0
[<ffffffff90ca3140>] ? ip_copy_metadata+0x170/0x170
[<ffffffff90ccfdc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff9090be65>] ? sock_has_perm+0x75/0x90
[<ffffffff90cdd589>] inet_sendmsg+0x69/0xb0
[<ffffffff9090bf93>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff90c343a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff90827439>] ? ___slab_alloc+0x229/0x520
[<ffffffff9098cfe4>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffff90c35177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffc04ef92e>] ? __xfs_filemap_fault+0x8e/0x1d0 [xfs]
[<ffffffff90c9bc63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffff90ccc28f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff90d8992e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff90ccf33f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff90d8992e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff90c36a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff90c34910>] ? SYSC_connect+0x90/0x130
[<ffffffff90ca895c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff90c36b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff90d93f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #579854
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0087ee7
Build id: 3.10.0-693.el7.x86_64
0x947
2
? kfree_skbmem
vmlinux 0xffffffff815710d7
Build id: 3.10.0-693.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff81586771
Build id: 3.10.0-693.el7.x86_64
0x171
4
sch_direct_xmit
vmlinux 0xffffffff815af50a
Build id: 3.10.0-693.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff81589566
Build id: 3.10.0-693.el7.x86_64
0x226
6
dev_queue_xmit
vmlinux 0xffffffff815898a0
Build id: 3.10.0-693.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff815cfd3e
Build id: 3.10.0-693.el7.x86_64
0x52e
8
ip_output
vmlinux 0xffffffff815d0293
Build id: 3.10.0-693.el7.x86_64
0x73
9
? __ip_append_data.isra.48
vmlinux 0xffffffff815cf810
Build id: 3.10.0-693.el7.x86_64
0xa00
10
ip_local_out_sk
vmlinux 0xffffffff815cdf31
Build id: 3.10.0-693.el7.x86_64
0x31
11
ip_send_skb
vmlinux 0xffffffff815d0cd6
Build id: 3.10.0-693.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff815f81b0
Build id: 3.10.0-693.el7.x86_64
0x160
13
? ip_copy_metadata
vmlinux 0xffffffff815cdc80
Build id: 3.10.0-693.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff815f92cc
Build id: 3.10.0-693.el7.x86_64
0x32c
15
? sock_has_perm
vmlinux 0xffffffff812b4d65
Build id: 3.10.0-693.el7.x86_64
0x75
16
inet_sendmsg
vmlinux 0xffffffff81606893
Build id: 3.10.0-693.el7.x86_64
0x63
17
? selinux_socket_sendmsg
vmlinux 0xffffffff812b4e93
Build id: 3.10.0-693.el7.x86_64
0x23
18
sock_sendmsg
vmlinux 0xffffffff8156a580
Build id: 3.10.0-693.el7.x86_64
0xb0
19
? zone_statistics
vmlinux 0xffffffff811a31b8
Build id: 3.10.0-693.el7.x86_64
0x88
20
? get_page_from_freelist
vmlinux 0xffffffff8118c49a
Build id: 3.10.0-693.el7.x86_64
0x4fa
21
___sys_sendmsg
vmlinux 0xffffffff8156ad37
Build id: 3.10.0-693.el7.x86_64
0x2b7
22
? __ip_route_output_key_hash
vmlinux 0xffffffff815c68d5
Build id: 3.10.0-693.el7.x86_64
0x225
23
? ip4_datagram_release_cb
vmlinux 0xffffffff815f5a29
Build id: 3.10.0-693.el7.x86_64
0x69
24
? _raw_spin_unlock_bh
vmlinux 0xffffffff816ab9ee
Build id: 3.10.0-693.el7.x86_64
0x1e
25
? udp_lib_rehash
vmlinux 0xffffffff815f7c3d
Build id: 3.10.0-693.el7.x86_64
0xfd
26
__sys_sendmmsg
vmlinux 0xffffffff8156b6ef
Build id: 3.10.0-693.el7.x86_64
0x9f
27
? __do_page_fault
vmlinux 0xffffffff816affd1
Build id: 3.10.0-693.el7.x86_64
0x171
28
? do_page_fault
vmlinux 0xffffffff816b02e5
Build id: 3.10.0-693.el7.x86_64
0x35
29
SyS_sendmmsg
vmlinux 0xffffffff8156b7f2
Build id: 3.10.0-693.el7.x86_64
0x12
30
system_call_fastpath
vmlinux 0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
0x16
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [urlgrabber-ext-:4434]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul snd_ens1371 glue_helper ablk_helper snd_rawmidi cryptd snd_ac97_codec ac97_bus snd_seq vmw_balloon snd_seq_device pcspkr snd_pcm joydev snd_timer snd soundcore
vmw_vmci sg nfit i2c_piix4 libnvdimm parport_pc parport shpchp nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix libata serio_raw e1000 mptspi scsi_transport_spi mptscsih mptbase i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 4434 Comm: urlgrabber-ext- Not tainted 3.10.0-693.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8800a4826eb0 ti: ffff880081cd8000 task.ti: ffff880081cd8000
RIP: 0010:[<ffffffffc0087ee7>]  [<ffffffffc0087ee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff880081cdb708  EFLAGS: 00000286
RAX: ffffc90000983818 RBX: ffff8800b14c3d68 RCX: 0000000000000092
RDX: 0000000000000092 RSI: ffff8800361f9140 RDI: ffff88012f524000
RBP: ffff880081cdb788 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff880081cdb708
R13: 0000002000000002 R14: ffffffff815736b9 R15: ffffffff00000000
FS:  00007f697a41e700(0000) GS:ffff880133600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f697eca4a90 CR3: 00000000b14c7000 CR4: 00000000003407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffff815710d7 ffff880130fb6098 0000009081573004 0000000900000000
00000000ab3d1000 ffff880000000000 ffffffff00000000 0000000100001000
ffff88012efe7600 ffff88012f524000 ffff880130fb6000 ffff88012f524000
Call Trace:
[<ffffffff815710d7>] ? kfree_skbmem+0x37/0x90
[<ffffffff81586771>] dev_hard_start_xmit+0x171/0x3b0
[<ffffffff815af50a>] sch_direct_xmit+0x11a/0x240
[<ffffffff81589566>] __dev_queue_xmit+0x226/0x550
[<ffffffff815898a0>] dev_queue_xmit+0x10/0x20
[<ffffffff815cfd3e>] ip_finish_output+0x52e/0x780
[<ffffffff815d0293>] ip_output+0x73/0xe0
[<ffffffff815cf810>] ? __ip_append_data.isra.48+0xa00/0xa00
[<ffffffff815cdf31>] ip_local_out_sk+0x31/0x40
[<ffffffff815d0cd6>] ip_send_skb+0x16/0x50
[<ffffffff815f81b0>] udp_send_skb+0x160/0x290
[<ffffffff815cdc80>] ? ip_copy_metadata+0x170/0x170
[<ffffffff815f92cc>] udp_sendmsg+0x32c/0xa00
[<ffffffff812b4d65>] ? sock_has_perm+0x75/0x90
[<ffffffff81606893>] inet_sendmsg+0x63/0xb0
[<ffffffff812b4e93>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff8156a580>] sock_sendmsg+0xb0/0xf0
[<ffffffff811a31b8>] ? zone_statistics+0x88/0xa0
[<ffffffff8118c49a>] ? get_page_from_freelist+0x4fa/0x9e0
[<ffffffff8156ad37>] ___sys_sendmsg+0x2b7/0x3c0
[<ffffffff815c68d5>] ? __ip_route_output_key_hash+0x225/0x9b0
[<ffffffff815f5a29>] ? ip4_datagram_release_cb+0x69/0x1b0
[<ffffffff816ab9ee>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff815f7c3d>] ? udp_lib_rehash+0xfd/0x130
[<ffffffff8156b6ef>] __sys_sendmmsg+0x9f/0x190
[<ffffffff816affd1>] ? __do_page_fault+0x171/0x450
[<ffffffff816b02e5>] ? do_page_fault+0x35/0x90
[<ffffffff8156b7f2>] SyS_sendmmsg+0x12/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #317015
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0125877
Build id: 3.10.0-1160.45.1.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffff86e3e6f7
Build id: 3.10.0-1160.45.1.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff86e55f96
Build id: 3.10.0-1160.45.1.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff86e840aa
Build id: 3.10.0-1160.45.1.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff86e58f11
Build id: 3.10.0-1160.45.1.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff86e590e0
Build id: 3.10.0-1160.45.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff86ea7c6b
Build id: 3.10.0-1160.45.1.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff86ea81eb
Build id: 3.10.0-1160.45.1.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff86ea75f0
Build id: 3.10.0-1160.45.1.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff86ea5c17
Build id: 3.10.0-1160.45.1.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff86ea8c56
Build id: 3.10.0-1160.45.1.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff86ed10bc
Build id: 3.10.0-1160.45.1.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff86ea5950
Build id: 3.10.0-1160.45.1.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff86ed2621
Build id: 3.10.0-1160.45.1.el7.x86_64
0x5e1
15
? hrtimer_get_res
vmlinux 0xffffffff868c9aa0
Build id: 3.10.0-1160.45.1.el7.x86_64
0x50
16
? schedule_hrtimeout_range_clock
vmlinux 0xffffffff86f88536
Build id: 3.10.0-1160.45.1.el7.x86_64
0xa6
17
? list_del
vmlinux 0xffffffff86ba681d
Build id: 3.10.0-1160.45.1.el7.x86_64
0xd
18
? udp_poll
vmlinux 0xffffffff86ed3969
Build id: 3.10.0-1160.45.1.el7.x86_64
0x19
19
inet_sendmsg
vmlinux 0xffffffff86edfe09
Build id: 3.10.0-1160.45.1.el7.x86_64
0x69
20
sock_sendmsg
vmlinux 0xffffffff86e363a6
Build id: 3.10.0-1160.45.1.el7.x86_64
0xb6
21
? page_frag_free
vmlinux 0xffffffff869c7428
Build id: 3.10.0-1160.45.1.el7.x86_64
0x68
22
? skb_release_data
vmlinux 0xffffffff86e3f732
Build id: 3.10.0-1160.45.1.el7.x86_64
0x142
23
___sys_sendmsg
vmlinux 0xffffffff86e37177
Build id: 3.10.0-1160.45.1.el7.x86_64
0x2f7
24
? __ip_route_output_key_hash
vmlinux 0xffffffff86e9e483
Build id: 3.10.0-1160.45.1.el7.x86_64
0x223
25
? ip4_datagram_release_cb
vmlinux 0xffffffff86eceaef
Build id: 3.10.0-1160.45.1.el7.x86_64
0x6f
26
? _raw_spin_unlock_bh
vmlinux 0xffffffff86f8b93e
Build id: 3.10.0-1160.45.1.el7.x86_64
0x1e
27
? udp_lib_rehash
vmlinux 0xffffffff86ed1b9f
Build id: 3.10.0-1160.45.1.el7.x86_64
0x11f
28
? _raw_spin_unlock_bh
vmlinux 0xffffffff86f8b93e
Build id: 3.10.0-1160.45.1.el7.x86_64
0x1e
29
__sys_sendmmsg
vmlinux 0xffffffff86e38a1f
Build id: 3.10.0-1160.45.1.el7.x86_64
0x9f
30
? SYSC_connect
vmlinux 0xffffffff86e36910
Build id: 3.10.0-1160.45.1.el7.x86_64
0x90
31
? ip_setsockopt
vmlinux 0xffffffff86eab16c
Build id: 3.10.0-1160.45.1.el7.x86_64
0x2c
32
SyS_sendmmsg
vmlinux 0xffffffff86e38b22
Build id: 3.10.0-1160.45.1.el7.x86_64
0x12
33
system_call_fastpath
vmlinux 0xffffffff86f95f92
Build id: 3.10.0-1160.45.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [node:99212]
Modules linked in: tcp_lp xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel ppdev aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd
joydev pcspkr sg vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix mptspi drm serio_raw libata scsi_transport_spi e1000 mptscsih nfit libnvdimm mptbase drm_panel_orientation_quirks fuse
CPU: 1 PID: 99212 Comm: node Kdump: loaded Tainted: G             L ------------ T 3.10.0-1160.45.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff90c0f187b180 ti: ffff90c009750000 task.ti: ffff90c009750000
RIP: 0010:[<ffffffffc0125877>]  [<ffffffffc0125877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff90c0097536a0  EFLAGS: 00000286
RAX: ffffa76800983818 RBX: 0000000000000000 RCX: 00000000000000a4
RDX: 00000000000000a4 RSI: ffff90bff5ff9a80 RDI: ffff90c0f449c000
RBP: ffff90c009753720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff90c009753708
R13: 0000400009753678 R14: ffffffffc066b8dd R15: ffff90c009753638
FS:  00007eff796f2700(0000) GS:ffff90c0f9640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa8588fe5b0 CR3: 00000000aafea000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff86e3e6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffff86e55f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff86e840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffff86e58f11>] __dev_queue_xmit+0x491/0x650
[<ffffffff86e590e0>] dev_queue_xmit+0x10/0x20
[<ffffffff86ea7c6b>] ip_finish_output+0x67b/0x8d0
[<ffffffff86ea81eb>] ip_output+0x7b/0xf0
[<ffffffff86ea75f0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff86ea5c17>] ip_local_out_sk+0x37/0x40
[<ffffffff86ea8c56>] ip_send_skb+0x16/0x50
[<ffffffff86ed10bc>] udp_send_skb+0xac/0x2b0
[<ffffffff86ea5950>] ? ip_copy_metadata+0x170/0x170
[<ffffffff86ed2621>] udp_sendmsg+0x5e1/0xa70
[<ffffffff868c9aa0>] ? hrtimer_get_res+0x50/0x50
[<ffffffff86f88536>] ? schedule_hrtimeout_range_clock+0xa6/0x150
[<ffffffff86ba681d>] ? list_del+0xd/0x30
[<ffffffff86ed3969>] ? udp_poll+0x19/0x90
[<ffffffff86edfe09>] inet_sendmsg+0x69/0xb0
[<ffffffff86e363a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff869c7428>] ? page_frag_free+0x68/0x80
[<ffffffff86e3f732>] ? skb_release_data+0x142/0x150
[<ffffffff86e37177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff86e9e483>] ? __ip_route_output_key_hash+0x223/0x7e0
[<ffffffff86eceaef>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff86f8b93e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff86ed1b9f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff86f8b93e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff86e38a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff86e36910>] ? SYSC_connect+0x90/0x130
[<ffffffff86eab16c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff86e38b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff86f95f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #130340
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0366877
Build id: 3.10.0-1127.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffff9023b6f7
Build id: 3.10.0-1127.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff90253386
Build id: 3.10.0-1127.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff9028131a
Build id: 3.10.0-1127.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff90256301
Build id: 3.10.0-1127.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff902564d0
Build id: 3.10.0-1127.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff902a429b
Build id: 3.10.0-1127.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff902a481b
Build id: 3.10.0-1127.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff902a3c20
Build id: 3.10.0-1127.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff902a2247
Build id: 3.10.0-1127.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff902a5286
Build id: 3.10.0-1127.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff902cd64c
Build id: 3.10.0-1127.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff902a1f80
Build id: 3.10.0-1127.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff902cebb1
Build id: 3.10.0-1127.el7.x86_64
0x5e1
15
? page_add_new_anon_rmap
vmlinux 0xffffffff8fe01048
Build id: 3.10.0-1127.el7.x86_64
0xb8
16
? handle_mm_fault
vmlinux 0xffffffff8fdf5edd
Build id: 3.10.0-1127.el7.x86_64
0xbfd
17
? sock_has_perm
vmlinux 0xffffffff8ff0b155
Build id: 3.10.0-1127.el7.x86_64
0x75
18
inet_sendmsg
vmlinux 0xffffffff902dc379
Build id: 3.10.0-1127.el7.x86_64
0x69
19
? selinux_socket_sendmsg
vmlinux 0xffffffff8ff0b283
Build id: 3.10.0-1127.el7.x86_64
0x23
20
sock_sendmsg
vmlinux 0xffffffff902333a6
Build id: 3.10.0-1127.el7.x86_64
0xb6
21
? free_pcppages_bulk
vmlinux 0xffffffff8fdc751e
Build id: 3.10.0-1127.el7.x86_64
0x16e
22
___sys_sendmsg
vmlinux 0xffffffff90234177
Build id: 3.10.0-1127.el7.x86_64
0x2f7
23
? update_curr
vmlinux 0xffffffff8fce4eac
Build id: 3.10.0-1127.el7.x86_64
0x14c
24
? __ip_route_output_key_hash
vmlinux 0xffffffff9029aacf
Build id: 3.10.0-1127.el7.x86_64
0x20f
25
? ip4_datagram_release_cb
vmlinux 0xffffffff902cb05f
Build id: 3.10.0-1127.el7.x86_64
0x6f
26
? _raw_spin_unlock_bh
vmlinux 0xffffffff9038854e
Build id: 3.10.0-1127.el7.x86_64
0x1e
27
? udp_lib_rehash
vmlinux 0xffffffff902ce12f
Build id: 3.10.0-1127.el7.x86_64
0x11f
28
? _raw_spin_unlock_bh
vmlinux 0xffffffff9038854e
Build id: 3.10.0-1127.el7.x86_64
0x1e
29
__sys_sendmmsg
vmlinux 0xffffffff90235a1f
Build id: 3.10.0-1127.el7.x86_64
0x9f
30
? SYSC_connect
vmlinux 0xffffffff90233910
Build id: 3.10.0-1127.el7.x86_64
0x90
31
? ip_setsockopt
vmlinux 0xffffffff902a779c
Build id: 3.10.0-1127.el7.x86_64
0x2c
32
SyS_sendmmsg
vmlinux 0xffffffff90235b22
Build id: 3.10.0-1127.el7.x86_64
0x12
33
system_call_fastpath
vmlinux 0xffffffff90392ed2
Build id: 3.10.0-1127.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [DNS Resolver #6:3584]
Modules linked in: tcp_lp nls_utf8 isofs rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport bnep vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev snd_ens1371 coretemp iosf_mbi snd_rawmidi crc32_pclmul snd_ac97_codec ghash_clmulni_intel ac97_bus aesni_intel snd_seq btusb vmw_balloon lrw btrtl gf128mul glue_helper btbcm
ablk_helper cryptd btintel snd_seq_device bluetooth snd_pcm joydev pcspkr snd_timer snd rfkill sg soundcore vmw_vmci i2c_piix4 parport_pc parport fuse ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod cdrom crct10dif_generic ata_generic pata_acpi vmwgfx mptspi ata_piix e1000 scsi_transport_spi mptscsih drm_kms_helper mptbase syscopyarea sysfillrect libata sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel nfit libnvdimm drm serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 3584 Comm: DNS Resolver #6 Kdump: loaded Tainted: G             L ------------   3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff94ecfd73d230 ti: ffff94ed65118000 task.ti: ffff94ed65118000
RIP: 0010:[<ffffffffc0366877>]  [<ffffffffc0366877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff94ed6511b6a0  EFLAGS: 00000286
RAX: ffffac0900703818 RBX: 0000000000000000 RCX: 0000000000000038
RDX: 0000000000000038 RSI: ffff94ecf4e25200 RDI: ffff94edc1f16000
RBP: ffff94ed6511b720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff94ed6511b708
R13: 000040006511b678 R14: ffffffffc07db8dd R15: ffff94ed6511b638
FS:  00002b76097d7700(0000) GS:ffff94ed7ac40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b75fa304000 CR3: 000000006da32000 CR4: 00000000003607e0
Call Trace:
[<ffffffff9023b6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffff90253386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9028131a>] sch_direct_xmit+0x11a/0x250
[<ffffffff90256301>] __dev_queue_xmit+0x491/0x650
[<ffffffff902564d0>] dev_queue_xmit+0x10/0x20
[<ffffffff902a429b>] ip_finish_output+0x67b/0x8d0
[<ffffffff902a481b>] ip_output+0x7b/0xf0
[<ffffffff902a3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff902a2247>] ip_local_out_sk+0x37/0x40
[<ffffffff902a5286>] ip_send_skb+0x16/0x50
[<ffffffff902cd64c>] udp_send_skb+0xac/0x2b0
[<ffffffff902a1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffff902cebb1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff8fe01048>] ? page_add_new_anon_rmap+0xb8/0x170
[<ffffffff8fdf5edd>] ? handle_mm_fault+0xbfd/0xfb0
[<ffffffff8ff0b155>] ? sock_has_perm+0x75/0x90
[<ffffffff902dc379>] inet_sendmsg+0x69/0xb0
[<ffffffff8ff0b283>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff902333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff8fdc751e>] ? free_pcppages_bulk+0x16e/0x390
[<ffffffff90234177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff8fce4eac>] ? update_curr+0x14c/0x1e0
[<ffffffff9029aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffff902cb05f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff9038854e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff902ce12f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff9038854e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff90235a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff90233910>] ? SYSC_connect+0x90/0x130
[<ffffffff902a779c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff90235b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff90392ed2>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #115208
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc035b877
Build id: 3.10.0-1127.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffff9103b6f7
Build id: 3.10.0-1127.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff91053386
Build id: 3.10.0-1127.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff9108131a
Build id: 3.10.0-1127.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff91056301
Build id: 3.10.0-1127.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff910564d0
Build id: 3.10.0-1127.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff910a429b
Build id: 3.10.0-1127.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff910a481b
Build id: 3.10.0-1127.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff910a3c20
Build id: 3.10.0-1127.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff910a2247
Build id: 3.10.0-1127.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff910a5286
Build id: 3.10.0-1127.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff910cd64c
Build id: 3.10.0-1127.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff910a1f80
Build id: 3.10.0-1127.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff910cebb1
Build id: 3.10.0-1127.el7.x86_64
0x5e1
15
? hrtimer_get_res
vmlinux 0xffffffff90aca2d0
Build id: 3.10.0-1127.el7.x86_64
0x50
16
? sock_has_perm
vmlinux 0xffffffff90d0b155
Build id: 3.10.0-1127.el7.x86_64
0x75
17
inet_sendmsg
vmlinux 0xffffffff910dc379
Build id: 3.10.0-1127.el7.x86_64
0x69
18
? selinux_socket_sendmsg
vmlinux 0xffffffff90d0b283
Build id: 3.10.0-1127.el7.x86_64
0x23
19
sock_sendmsg
vmlinux 0xffffffff910333a6
Build id: 3.10.0-1127.el7.x86_64
0xb6
20
? do_page_fault
vmlinux 0xffffffff9118d975
Build id: 3.10.0-1127.el7.x86_64
0x35
21
___sys_sendmsg
vmlinux 0xffffffff91034177
Build id: 3.10.0-1127.el7.x86_64
0x2f7
22
? __ip_route_output_key_hash
vmlinux 0xffffffff9109aacf
Build id: 3.10.0-1127.el7.x86_64
0x20f
23
? ip4_datagram_release_cb
vmlinux 0xffffffff910cb05f
Build id: 3.10.0-1127.el7.x86_64
0x6f
24
? _raw_spin_unlock_bh
vmlinux 0xffffffff9118854e
Build id: 3.10.0-1127.el7.x86_64
0x1e
25
? udp_lib_rehash
vmlinux 0xffffffff910ce12f
Build id: 3.10.0-1127.el7.x86_64
0x11f
26
__sys_sendmmsg
vmlinux 0xffffffff91035a1f
Build id: 3.10.0-1127.el7.x86_64
0x9f
27
? ktime_get_ts64
vmlinux 0xffffffff90b06ce2
Build id: 3.10.0-1127.el7.x86_64
0x52
28
SyS_sendmmsg
vmlinux 0xffffffff91035b22
Build id: 3.10.0-1127.el7.x86_64
0x12
29
system_call_fastpath
vmlinux 0xffffffff91192ed2
Build id: 3.10.0-1127.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 111s! [yumBackend.py:3634]
Modules linked in: tcp_lp nls_utf8 isofs rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event coretemp iosf_mbi crc32_pclmul ppdev snd_ens1371 snd_rawmidi snd_ac97_codec ghash_clmulni_intel ac97_bus aesni_intel btusb snd_seq vmw_balloon btrtl btbcm btintel lrw gf128mul glue_helper
bluetooth ablk_helper snd_seq_device cryptd snd_pcm snd_timer joydev snd rfkill pcspkr sg soundcore vmw_vmci i2c_piix4 parport_pc parport fuse ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit drm libnvdimm mptspi crct10dif_pclmul crct10dif_common scsi_transport_spi ata_piix mptscsih crc32c_intel serio_raw libata e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 3634 Comm: yumBackend.py Kdump: loaded Not tainted 3.10.0-1127.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff9f1574a83150 ti: ffff9f15854c0000 task.ti: ffff9f15854c0000
RIP: 0010:[<ffffffffc035b877>]  [<ffffffffc035b877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9f15854c36a0  EFLAGS: 00000286
RAX: ffffc26d40703818 RBX: 0000000000000000 RCX: 000000000000007b
RDX: 000000000000007b RSI: ffff9f15359f2940 RDI: ffff9f15b4d96000
RBP: ffff9f15854c3720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff9f15854c3708
R13: 00004000854c3678 R14: ffffffffc09148dd R15: ffff9f15854c3638
FS:  00007f008678b700(0000) GS:ffff9f15bac00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2115952000 CR3: 000000006f3b6000 CR4: 00000000003607f0
Call Trace:
[<ffffffff9103b6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffff91053386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9108131a>] sch_direct_xmit+0x11a/0x250
[<ffffffff91056301>] __dev_queue_xmit+0x491/0x650
[<ffffffff910564d0>] dev_queue_xmit+0x10/0x20
[<ffffffff910a429b>] ip_finish_output+0x67b/0x8d0
[<ffffffff910a481b>] ip_output+0x7b/0xf0
[<ffffffff910a3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff910a2247>] ip_local_out_sk+0x37/0x40
[<ffffffff910a5286>] ip_send_skb+0x16/0x50
[<ffffffff910cd64c>] udp_send_skb+0xac/0x2b0
[<ffffffff910a1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffff910cebb1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff90aca2d0>] ? hrtimer_get_res+0x50/0x50
[<ffffffff90d0b155>] ? sock_has_perm+0x75/0x90
[<ffffffff910dc379>] inet_sendmsg+0x69/0xb0
[<ffffffff90d0b283>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff910333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff9118d975>] ? do_page_fault+0x35/0x90
[<ffffffff91034177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff9109aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffff910cb05f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff9118854e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff910ce12f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff91035a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff90b06ce2>] ? ktime_get_ts64+0x52/0xf0
[<ffffffff91035b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff91192ed2>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #99666
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc0089ee7
Build id: 3.10.0-693.el7.x86_64
0x947
2
? ipt_do_table
ip_tables 0xffffffffc0189d06
Build id: 3.10.0-693.el7.x86_64
0x336
3
dev_hard_start_xmit
vmlinux 0xffffffff81586771
Build id: 3.10.0-693.el7.x86_64
0x171
4
sch_direct_xmit
vmlinux 0xffffffff815af50a
Build id: 3.10.0-693.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff81589566
Build id: 3.10.0-693.el7.x86_64
0x226
6
dev_queue_xmit
vmlinux 0xffffffff815898a0
Build id: 3.10.0-693.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff815cfd3e
Build id: 3.10.0-693.el7.x86_64
0x52e
8
ip_output
vmlinux 0xffffffff815d0293
Build id: 3.10.0-693.el7.x86_64
0x73
9
? __ip_append_data.isra.48
vmlinux 0xffffffff815cf810
Build id: 3.10.0-693.el7.x86_64
0xa00
10
ip_local_out_sk
vmlinux 0xffffffff815cdf31
Build id: 3.10.0-693.el7.x86_64
0x31
11
ip_send_skb
vmlinux 0xffffffff815d0cd6
Build id: 3.10.0-693.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff815f81b0
Build id: 3.10.0-693.el7.x86_64
0x160
13
? ip_copy_metadata
vmlinux 0xffffffff815cdc80
Build id: 3.10.0-693.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff815f92cc
Build id: 3.10.0-693.el7.x86_64
0x32c
15
? lru_cache_add
vmlinux 0xffffffff811914ee
Build id: 3.10.0-693.el7.x86_64
0xe
16
? page_add_new_anon_rmap
vmlinux 0xffffffff811bc3d8
Build id: 3.10.0-693.el7.x86_64
0xb8
17
? udp_poll
vmlinux 0xffffffff815fa449
Build id: 3.10.0-693.el7.x86_64
0x19
18
inet_sendmsg
vmlinux 0xffffffff81606893
Build id: 3.10.0-693.el7.x86_64
0x63
19
sock_sendmsg
vmlinux 0xffffffff8156a580
Build id: 3.10.0-693.el7.x86_64
0xb0
20
? get_page_from_freelist
vmlinux 0xffffffff8118c49a
Build id: 3.10.0-693.el7.x86_64
0x4fa
21
___sys_sendmsg
vmlinux 0xffffffff8156ad37
Build id: 3.10.0-693.el7.x86_64
0x2b7
22
? __ip_route_output_key_hash
vmlinux 0xffffffff815c68d5
Build id: 3.10.0-693.el7.x86_64
0x225
23
? ip4_datagram_release_cb
vmlinux 0xffffffff815f5a29
Build id: 3.10.0-693.el7.x86_64
0x69
24
? _raw_spin_unlock_bh
vmlinux 0xffffffff816ab9ee
Build id: 3.10.0-693.el7.x86_64
0x1e
25
? udp_lib_rehash
vmlinux 0xffffffff815f7c3d
Build id: 3.10.0-693.el7.x86_64
0xfd
26
? _raw_spin_unlock_bh
vmlinux 0xffffffff816ab9ee
Build id: 3.10.0-693.el7.x86_64
0x1e
27
__sys_sendmmsg
vmlinux 0xffffffff8156b6ef
Build id: 3.10.0-693.el7.x86_64
0x9f
28
? SYSC_connect
vmlinux 0xffffffff8156a497
Build id: 3.10.0-693.el7.x86_64
0xe7
29
? __fd_install
vmlinux 0xffffffff812201ba
Build id: 3.10.0-693.el7.x86_64
0x4a
30
SyS_sendmmsg
vmlinux 0xffffffff8156b7f2
Build id: 3.10.0-693.el7.x86_64
0x12
31
system_call_fastpath
vmlinux 0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
0x16
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 30s! [freshclam:31487]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nls_utf8 isofs ext4 mbcache jbd2 pcspkr sg ppdev joydev shpchp parport_pc parport i2c_piix4 virtio_balloon nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod cdrom crct10dif_generic crct10dif_common ata_generic pata_acpi virtio_scsi bochs_drm drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 serio_raw virtio_pci virtio_ring virtio i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 31487 Comm: freshclam Tainted: G             L ------------   3.10.0-693.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.10.2-0-g5f4c7b1-prebuilt.qemu-project.org 04/01/2014
task: ffff880134c10000 ti: ffff880005db4000 task.ti: ffff880005db4000
RIP: 0010:[<ffffffffc0089ee7>]  [<ffffffffc0089ee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff880005db7708  EFLAGS: 00000286
RAX: ffffc90000783818 RBX: ffff880005db7fd8 RCX: 000000000000003c
RDX: 000000000000003c RSI: ffff880035e97940 RDI: ffff8801387de000
RBP: ffff880005db7788 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffffffff816a8f8d
R13: ffff880005db76c8 R14: 0000000000000000 R15: ffff880134cdcc80
FS:  00007fac6695e840(0000) GS:ffff88013fc00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007fb3d7295000 CR3: 0000000013e83000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffffffffc0189d06 ffff880139563098 0000003ac0189d06 0000000900000000
00000000b6151300 ffff880000000000 ffffffff00000000 0000000100001000
ffff8800a6706300 ffff8801387de000 ffff880139563000 ffff8801387de000
Call Trace:
[<ffffffffc0189d06>] ? ipt_do_table+0x336/0x710 [ip_tables]
[<ffffffff81586771>] dev_hard_start_xmit+0x171/0x3b0
[<ffffffff815af50a>] sch_direct_xmit+0x11a/0x240
[<ffffffff81589566>] __dev_queue_xmit+0x226/0x550
[<ffffffff815898a0>] dev_queue_xmit+0x10/0x20
[<ffffffff815cfd3e>] ip_finish_output+0x52e/0x780
[<ffffffff815d0293>] ip_output+0x73/0xe0
[<ffffffff815cf810>] ? __ip_append_data.isra.48+0xa00/0xa00
[<ffffffff815cdf31>] ip_local_out_sk+0x31/0x40
[<ffffffff815d0cd6>] ip_send_skb+0x16/0x50
[<ffffffff815f81b0>] udp_send_skb+0x160/0x290
[<ffffffff815cdc80>] ? ip_copy_metadata+0x170/0x170
[<ffffffff815f92cc>] udp_sendmsg+0x32c/0xa00
[<ffffffff811914ee>] ? lru_cache_add+0xe/0x10
[<ffffffff811bc3d8>] ? page_add_new_anon_rmap+0xb8/0x170
[<ffffffff815fa449>] ? udp_poll+0x19/0x90
[<ffffffff81606893>] inet_sendmsg+0x63/0xb0
[<ffffffff8156a580>] sock_sendmsg+0xb0/0xf0
[<ffffffff8118c49a>] ? get_page_from_freelist+0x4fa/0x9e0
[<ffffffff8156ad37>] ___sys_sendmsg+0x2b7/0x3c0
[<ffffffff815c68d5>] ? __ip_route_output_key_hash+0x225/0x9b0
[<ffffffff815f5a29>] ? ip4_datagram_release_cb+0x69/0x1b0
[<ffffffff816ab9ee>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff815f7c3d>] ? udp_lib_rehash+0xfd/0x130
[<ffffffff816ab9ee>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff8156b6ef>] __sys_sendmmsg+0x9f/0x190
[<ffffffff8156a497>] ? SYSC_connect+0xe7/0x120
[<ffffffff812201ba>] ? __fd_install+0x4a/0x60
[<ffffffff8156b7f2>] SyS_sendmmsg+0x12/0x30
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #40429
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc04fa877
Build id: 3.10.0-1160.11.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb1453f96
Build id: 3.10.0-1160.11.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb148203a
Build id: 3.10.0-1160.11.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb1456f11
Build id: 3.10.0-1160.11.1.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb14570e0
Build id: 3.10.0-1160.11.1.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb14a507b
Build id: 3.10.0-1160.11.1.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb14a55fb
Build id: 3.10.0-1160.11.1.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb14a4a00
Build id: 3.10.0-1160.11.1.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb14a3027
Build id: 3.10.0-1160.11.1.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffffb14a6066
Build id: 3.10.0-1160.11.1.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffffb14ce49c
Build id: 3.10.0-1160.11.1.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffffb14a2d60
Build id: 3.10.0-1160.11.1.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffffb14cfa01
Build id: 3.10.0-1160.11.1.el7.x86_64
0x5e1
14
? zone_statistics
vmlinux 0xffffffffb0fe0638
Build id: 3.10.0-1160.11.1.el7.x86_64
0x88
15
? udp_poll
vmlinux 0xffffffffb14d0d49
Build id: 3.10.0-1160.11.1.el7.x86_64
0x19
16
inet_sendmsg
vmlinux 0xffffffffb14dd1c9
Build id: 3.10.0-1160.11.1.el7.x86_64
0x69
17
sock_sendmsg
vmlinux 0xffffffffb14343a6
Build id: 3.10.0-1160.11.1.el7.x86_64
0xb6
18
? __radix_tree_lookup
vmlinux 0xffffffffb118d1e4
Build id: 3.10.0-1160.11.1.el7.x86_64
0x84
19
___sys_sendmsg
vmlinux 0xffffffffb1435177
Build id: 3.10.0-1160.11.1.el7.x86_64
0x2f7
20
? __xfs_filemap_fault
xfs 0xffffffffc06e792e
Build id: 3.10.0-1160.11.1.el7.x86_64
0x8e
21
? __ip_route_output_key_hash
vmlinux 0xffffffffb149b883
Build id: 3.10.0-1160.11.1.el7.x86_64
0x223
22
? ip4_datagram_release_cb
vmlinux 0xffffffffb14cbecf
Build id: 3.10.0-1160.11.1.el7.x86_64
0x6f
23
? _raw_spin_unlock_bh
vmlinux 0xffffffffb158958e
Build id: 3.10.0-1160.11.1.el7.x86_64
0x1e
24
? udp_lib_rehash
vmlinux 0xffffffffb14cef7f
Build id: 3.10.0-1160.11.1.el7.x86_64
0x11f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffffb158958e
Build id: 3.10.0-1160.11.1.el7.x86_64
0x1e
26
__sys_sendmmsg
vmlinux 0xffffffffb1436a1f
Build id: 3.10.0-1160.11.1.el7.x86_64
0x9f
27
? SYSC_connect
vmlinux 0xffffffffb143496d
Build id: 3.10.0-1160.11.1.el7.x86_64
0xed
28
? ip_setsockopt
vmlinux 0xffffffffb14a857c
Build id: 3.10.0-1160.11.1.el7.x86_64
0x2c
29
SyS_sendmmsg
vmlinux 0xffffffffb1436b22
Build id: 3.10.0-1160.11.1.el7.x86_64
0x12
30
system_call_fastpath
vmlinux 0xffffffffb1593f92
Build id: 3.10.0-1160.11.1.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 27s! [php:4741]
Modules linked in: xt_comment ipt_REJECT nf_reject_ipv4 nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter vmw_vsock_vmci_transport vsock sunrpc kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul vmw_balloon glue_helper ablk_helper cryptd pcspkr joydev sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi nfit vmwgfx libnvdimm drm_kms_helper ata_piix crct10dif_pclmul crct10dif_common libata syscopyarea sysfillrect sysimgblt fb_sys_fops crc32c_intel ttm mptspi scsi_transport_spi e1000 mptscsih drm serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 4741 Comm: php Tainted: G             L ------------   3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff93ced10d9080 ti: ffff93ce66c1c000 task.ti: ffff93ce66c1c000
RIP: 0010:[<ffffffffc04fa877>]  [<ffffffffc04fa877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff93ce66c1f6a0  EFLAGS: 00000286
RAX: ffff9eedc0fc3818 RBX: 0000000000000001 RCX: 000000000000003b
RDX: 000000000000003b RSI: ffff93cd75eb91c0 RDI: ffff93cebdbc6000
RBP: ffff93ce66c1f720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000126df4602 R14: 00000001facdee80 R15: ffffffffb11b1850
FS:  00007fcd2c8fa880(0000) GS:ffff93cefb4c0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcd2c0b6560 CR3: 000000017a7c4000 CR4: 00000000003407e0
Call Trace:
[<ffffffffb1453f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb148203a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb1456f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffb14570e0>] dev_queue_xmit+0x10/0x20
[<ffffffffb14a507b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb14a55fb>] ip_output+0x7b/0xf0
[<ffffffffb14a4a00>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb14a3027>] ip_local_out_sk+0x37/0x40
[<ffffffffb14a6066>] ip_send_skb+0x16/0x50
[<ffffffffb14ce49c>] udp_send_skb+0xac/0x2b0
[<ffffffffb14a2d60>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb14cfa01>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb0fe0638>] ? zone_statistics+0x88/0xa0
[<ffffffffb14d0d49>] ? udp_poll+0x19/0x90
[<ffffffffb14dd1c9>] inet_sendmsg+0x69/0xb0
[<ffffffffb14343a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffb118d1e4>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffffb1435177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffc06e792e>] ? __xfs_filemap_fault+0x8e/0x1d0 [xfs]
[<ffffffffb149b883>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffb14cbecf>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffb158958e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb14cef7f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffb158958e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb1436a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffffb143496d>] ? SYSC_connect+0xed/0x130
[<ffffffffb14a857c>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb1436b22>] SyS_sendmmsg+0x12/0x30
[<ffffffffb1593f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #10332
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc009b877
Build id: 3.10.0-1127.18.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb5453386
Build id: 3.10.0-1127.18.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb548131a
Build id: 3.10.0-1127.18.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb5456301
Build id: 3.10.0-1127.18.2.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb54564d0
Build id: 3.10.0-1127.18.2.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb54a429b
Build id: 3.10.0-1127.18.2.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb54a481b
Build id: 3.10.0-1127.18.2.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb54a3c20
Build id: 3.10.0-1127.18.2.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb54a2247
Build id: 3.10.0-1127.18.2.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffffb54a5286
Build id: 3.10.0-1127.18.2.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffffb54cd65c
Build id: 3.10.0-1127.18.2.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffffb54a1f80
Build id: 3.10.0-1127.18.2.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffffb54cebc1
Build id: 3.10.0-1127.18.2.el7.x86_64
0x5e1
14
? page_add_new_anon_rmap
vmlinux 0xffffffffb50010d8
Build id: 3.10.0-1127.18.2.el7.x86_64
0xb8
15
? handle_mm_fault
vmlinux 0xffffffffb4ff5f5d
Build id: 3.10.0-1127.18.2.el7.x86_64
0xbfd
16
? __d_instantiate
vmlinux 0xffffffffb5065f7d
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2d
17
? zone_statistics
vmlinux 0xffffffffb4fe05c8
Build id: 3.10.0-1127.18.2.el7.x86_64
0x88
18
? udp_poll
vmlinux 0xffffffffb54cff09
Build id: 3.10.0-1127.18.2.el7.x86_64
0x19
19
inet_sendmsg
vmlinux 0xffffffffb54dc389
Build id: 3.10.0-1127.18.2.el7.x86_64
0x69
20
sock_sendmsg
vmlinux 0xffffffffb54333a6
Build id: 3.10.0-1127.18.2.el7.x86_64
0xb6
21
? __radix_tree_lookup
vmlinux 0xffffffffb518c1c4
Build id: 3.10.0-1127.18.2.el7.x86_64
0x84
22
___sys_sendmsg
vmlinux 0xffffffffb5434177
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2f7
23
? __xfs_filemap_fault
xfs 0xffffffffc030199e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x8e
24
? __ip_route_output_key_hash
vmlinux 0xffffffffb549aacf
Build id: 3.10.0-1127.18.2.el7.x86_64
0x20f
25
? ip4_datagram_release_cb
vmlinux 0xffffffffb54cb06f
Build id: 3.10.0-1127.18.2.el7.x86_64
0x6f
26
? _raw_spin_unlock_bh
vmlinux 0xffffffffb558856e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x1e
27
? udp_lib_rehash
vmlinux 0xffffffffb54ce13f
Build id: 3.10.0-1127.18.2.el7.x86_64
0x11f
28
? _raw_spin_unlock_bh
vmlinux 0xffffffffb558856e
Build id: 3.10.0-1127.18.2.el7.x86_64
0x1e
29
__sys_sendmmsg
vmlinux 0xffffffffb5435a1f
Build id: 3.10.0-1127.18.2.el7.x86_64
0x9f
30
? SYSC_connect
vmlinux 0xffffffffb5433910
Build id: 3.10.0-1127.18.2.el7.x86_64
0x90
31
? ip_setsockopt
vmlinux 0xffffffffb54a779c
Build id: 3.10.0-1127.18.2.el7.x86_64
0x2c
32
SyS_sendmmsg
vmlinux 0xffffffffb5435b22
Build id: 3.10.0-1127.18.2.el7.x86_64
0x12
33
system_call_fastpath
vmlinux 0xffffffffb5592ed2
Build id: 3.10.0-1127.18.2.el7.x86_64
0x25
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 37s! [reporter-urepor:63778]
Modules linked in: tcp_lp fuse xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus aesni_intel snd_seq ppdev lrw gf128mul glue_helper ablk_helper cryptd snd_seq_device snd_pcm vmw_balloon
joydev pcspkr snd_timer snd soundcore sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common ata_piix crc32c_intel drm serio_raw libata mptspi nfit libnvdimm e1000 scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 63778 Comm: reporter-urepor Tainted: G             L ------------   3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff9dad6173e2a0 ti: ffff9dad15300000 task.ti: ffff9dad15300000
RIP: 0010:[<ffffffffc009b877>]  [<ffffffffc009b877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9dad153036a0  EFLAGS: 00000286
RAX: ffffc149c0683818 RBX: 0000000000000000 RCX: 000000000000007d
RDX: 000000000000007d RSI: ffff9dad358a1c00 RDI: ffff9dad79a1a000
RBP: ffff9dad15303720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff9dad15303708
R13: 0000400015303678 R14: ffffffffc05a68dd R15: ffff9dad15303638
FS:  00007f2b4bbdb700(0000) GS:ffff9dad7c640000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2b56420560 CR3: 00000000152d2000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb5453386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb548131a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb5456301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb54564d0>] dev_queue_xmit+0x10/0x20
[<ffffffffb54a429b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb54a481b>] ip_output+0x7b/0xf0
[<ffffffffb54a3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb54a2247>] ip_local_out_sk+0x37/0x40
[<ffffffffb54a5286>] ip_send_skb+0x16/0x50
[<ffffffffb54cd65c>] udp_send_skb+0xac/0x2b0
[<ffffffffb54a1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb54cebc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb50010d8>] ? page_add_new_anon_rmap+0xb8/0x170
[<ffffffffb4ff5f5d>] ? handle_mm_fault+0xbfd/0xfb0
[<ffffffffb5065f7d>] ? __d_instantiate+0x2d/0xe0
[<ffffffffb4fe05c8>] ? zone_statistics+0x88/0xa0
[<ffffffffb54cff09>] ? udp_poll+0x19/0x90
[<ffffffffb54dc389>] inet_sendmsg+0x69/0xb0
[<ffffffffb54333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffb518c1c4>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffffb5434177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffc030199e>] ? __xfs_filemap_fault+0x8e/0x1d0 [xfs]
[<ffffffffb549aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffffb54cb06f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffb558856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb54ce13f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffb558856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb5435a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffffb5433910>] ? SYSC_connect+0x90/0x130
[<ffffffffb54a779c>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb5435b22>] SyS_sendmmsg+0x12/0x30
[<ffffffffb5592ed2>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00