Info

Permalink
Function
e1000_xmit_frame
First occurrence
2021-01-21
Last occurrence
2026-08-31
State
NEW
Components
kernel
Quality
-91

Statistics

Operating system Count
CentOS 7 8
Architecture Count
x86_64 8
Related packages Count
kernel 8
 0:3.10.0-1127.19.1.el7 3
 0:3.10.0-1160.76.1.el7 1
 0:3.10.0-1160.62.1.el7 1
 0:3.10.0-957.21.3.el7 1
 0:3.10.0-1062.9.1.el7 1
 0:3.10.0-1160.36.2.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

No reports in the last 20 days.

Report backtrace


Complete report #1421364
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc00d8877
Build id: 3.10.0-1160.36.2.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffa9855f96
Build id: 3.10.0-1160.36.2.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffa98840aa
Build id: 3.10.0-1160.36.2.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffa9858f11
Build id: 3.10.0-1160.36.2.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffa98590e0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffa98a7ccb
Build id: 3.10.0-1160.36.2.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffa98a824b
Build id: 3.10.0-1160.36.2.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffa98a7650
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffa98a5c77
Build id: 3.10.0-1160.36.2.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffffa98a8cb6
Build id: 3.10.0-1160.36.2.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffffa98d111c
Build id: 3.10.0-1160.36.2.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffffa98a59b0
Build id: 3.10.0-1160.36.2.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffffa98d2681
Build id: 3.10.0-1160.36.2.el7.x86_64
0x5e1
14
? __pollwait
vmlinux 0xffffffffa946457f
Build id: 3.10.0-1160.36.2.el7.x86_64
0x7f
15
? list_del
vmlinux 0xffffffffa95a67cd
Build id: 3.10.0-1160.36.2.el7.x86_64
0xd
16
? udp_poll
vmlinux 0xffffffffa98d39c9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x19
17
inet_sendmsg
vmlinux 0xffffffffa98dfe69
Build id: 3.10.0-1160.36.2.el7.x86_64
0x69
18
sock_sendmsg
vmlinux 0xffffffffa98363a6
Build id: 3.10.0-1160.36.2.el7.x86_64
0xb6
19
___sys_sendmsg
vmlinux 0xffffffffa9837177
Build id: 3.10.0-1160.36.2.el7.x86_64
0x2f7
20
? __xfs_filemap_fault
xfs 0xffffffffc032992e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x8e
21
? __ip_route_output_key_hash
vmlinux 0xffffffffa989e4e3
Build id: 3.10.0-1160.36.2.el7.x86_64
0x223
22
? ip4_datagram_release_cb
vmlinux 0xffffffffa98ceb4f
Build id: 3.10.0-1160.36.2.el7.x86_64
0x6f
23
? _raw_spin_unlock_bh
vmlinux 0xffffffffa998b95e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x1e
24
? udp_lib_rehash
vmlinux 0xffffffffa98d1bff
Build id: 3.10.0-1160.36.2.el7.x86_64
0x11f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffffa998b95e
Build id: 3.10.0-1160.36.2.el7.x86_64
0x1e
26
__sys_sendmmsg
vmlinux 0xffffffffa9838a1f
Build id: 3.10.0-1160.36.2.el7.x86_64
0x9f
27
? SYSC_connect
vmlinux 0xffffffffa9836910
Build id: 3.10.0-1160.36.2.el7.x86_64
0x90
28
? ip_setsockopt
vmlinux 0xffffffffa98ab1cc
Build id: 3.10.0-1160.36.2.el7.x86_64
0x2c
29
? system_call_after_swapgs
vmlinux 0xffffffffa9995ed5
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa2
30
? system_call_after_swapgs
vmlinux 0xffffffffa9995ec9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x96
31
? system_call_after_swapgs
vmlinux 0xffffffffa9995ed5
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa2
32
? system_call_after_swapgs
vmlinux 0xffffffffa9995ec9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x96
33
? system_call_after_swapgs
vmlinux 0xffffffffa9995ed5
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa2
34
? system_call_after_swapgs
vmlinux 0xffffffffa9995ec9
Build id: 3.10.0-1160.36.2.el7.x86_64
0x96
35
SyS_sendmmsg
vmlinux 0xffffffffa9838b22
Build id: 3.10.0-1160.36.2.el7.x86_64
0x12
36
system_call_fastpath
vmlinux 0xffffffffa9995f92
Build id: 3.10.0-1160.36.2.el7.x86_64
0x25
37
? system_call_after_swapgs
vmlinux 0xffffffffa9995ed5
Build id: 3.10.0-1160.36.2.el7.x86_64
0xa2
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 34s! [reporter-urepor:26190]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon joydev pcspkr sg i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic
ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper nfit libnvdimm syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw ata_piix drm libata e1000 mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 0 PID: 26190 Comm: reporter-urepor Kdump: loaded Tainted: G             L ------------   3.10.0-1160.36.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8fa1a8d50000 ti: ffff8fa05baa4000 task.ti: ffff8fa05baa4000
RIP: 0010:[<ffffffffc00d8877>]  [<ffffffffc00d8877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8fa05baa76a0  EFLAGS: 00000286
RAX: ffffa5c2410c3818 RBX: 0000000000000001 RCX: 0000000000000046
RDX: 0000000000000046 RSI: ffff8fa035c9d980 RDI: ffff8fa2344b8000
RBP: ffff8fa05baa7720 R08: 0000000000000100 R09: 0000000022100000
R10: 037463656a6f7270 R11: 0000000000000000 R12: 0000000000000202
R13: 00000001a8eff402 R14: 0000000133166a40 R15: ffffffffa95b2260
FS:  00007f85c8451700(0000) GS:ffff8fa235600000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f85c844dba8 CR3: 0000000176f8c000 CR4: 00000000000007f0
Call Trace:
[<ffffffffa9855f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa98840aa>] sch_direct_xmit+0x11a/0x250
[<ffffffffa9858f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffa98590e0>] dev_queue_xmit+0x10/0x20
[<ffffffffa98a7ccb>] ip_finish_output+0x67b/0x8d0
[<ffffffffa98a824b>] ip_output+0x7b/0xf0
[<ffffffffa98a7650>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa98a5c77>] ip_local_out_sk+0x37/0x40
[<ffffffffa98a8cb6>] ip_send_skb+0x16/0x50
[<ffffffffa98d111c>] udp_send_skb+0xac/0x2b0
[<ffffffffa98a59b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffa98d2681>] udp_sendmsg+0x5e1/0xa70
[<ffffffffa946457f>] ? __pollwait+0x7f/0xf0
[<ffffffffa95a67cd>] ? list_del+0xd/0x30
[<ffffffffa98d39c9>] ? udp_poll+0x19/0x90
[<ffffffffa98dfe69>] inet_sendmsg+0x69/0xb0
[<ffffffffa98363a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffa9837177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffc032992e>] ? __xfs_filemap_fault+0x8e/0x1d0 [xfs]
[<ffffffffa989e4e3>] ? __ip_route_output_key_hash+0x223/0x7e0
[<ffffffffa98ceb4f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffa998b95e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffa98d1bff>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffa998b95e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffa9838a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffffa9836910>] ? SYSC_connect+0x90/0x130
[<ffffffffa98ab1cc>] ? ip_setsockopt+0x2c/0x90
[<ffffffffa9995ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffa9995ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffa9995ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffa9995ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffa9995ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffa9995ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffa9838b22>] SyS_sendmmsg+0x12/0x30
[<ffffffffa9995f92>] system_call_fastpath+0x25/0x2a
[<ffffffffa9995ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #1220618
Frame # Function Binary Source or offset Line
1
vmxnet3_xmit_frame
vmxnet3 0xffffffffc01b43dc
Build id: 3.10.0-1062.9.1.el7.x86_64
0xaec
2
? nf_nat_setup_info
nf_nat 0xffffffffc05e63e7
Build id: 3.10.0-1062.9.1.el7.x86_64
0x2f7
3
dev_hard_start_xmit
vmlinux 0xffffffff9d44f346
Build id: 3.10.0-1062.9.1.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff9d47d0ba
Build id: 3.10.0-1062.9.1.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff9d4522b1
Build id: 3.10.0-1062.9.1.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff9d452480
Build id: 3.10.0-1062.9.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff9d4a097b
Build id: 3.10.0-1062.9.1.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff9d4a0efb
Build id: 3.10.0-1062.9.1.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff9d4a0300
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff9d49e927
Build id: 3.10.0-1062.9.1.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff9d4a1966
Build id: 3.10.0-1062.9.1.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff9d4c9cec
Build id: 3.10.0-1062.9.1.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff9d49e660
Build id: 3.10.0-1062.9.1.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff9d4cb251
Build id: 3.10.0-1062.9.1.el7.x86_64
0x5e1
15
? hrtimer_get_res
vmlinux 0xffffffff9cec9e30
Build id: 3.10.0-1062.9.1.el7.x86_64
0x50
16
? schedule_hrtimeout_range_clock
vmlinux 0xffffffff9d57fc06
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa6
17
? list_del
vmlinux 0xffffffff9d1a12ed
Build id: 3.10.0-1062.9.1.el7.x86_64
0xd
18
? udp_poll
vmlinux 0xffffffff9d4cc5c9
Build id: 3.10.0-1062.9.1.el7.x86_64
0x19
19
inet_sendmsg
vmlinux 0xffffffff9d4d89b9
Build id: 3.10.0-1062.9.1.el7.x86_64
0x69
20
sock_sendmsg
vmlinux 0xffffffff9d42f3a6
Build id: 3.10.0-1062.9.1.el7.x86_64
0xb6
21
? do_page_fault
vmlinux 0xffffffff9d588975
Build id: 3.10.0-1062.9.1.el7.x86_64
0x35
22
___sys_sendmsg
vmlinux 0xffffffff9d430177
Build id: 3.10.0-1062.9.1.el7.x86_64
0x2f7
23
? __ip_route_output_key_hash
vmlinux 0xffffffff9d497257
Build id: 3.10.0-1062.9.1.el7.x86_64
0x207
24
? ip4_datagram_release_cb
vmlinux 0xffffffff9d4c76ff
Build id: 3.10.0-1062.9.1.el7.x86_64
0x6f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffff9d5831ce
Build id: 3.10.0-1062.9.1.el7.x86_64
0x1e
26
? udp_lib_rehash
vmlinux 0xffffffff9d4ca7cf
Build id: 3.10.0-1062.9.1.el7.x86_64
0x11f
27
? _raw_spin_unlock_bh
vmlinux 0xffffffff9d5831ce
Build id: 3.10.0-1062.9.1.el7.x86_64
0x1e
28
__sys_sendmmsg
vmlinux 0xffffffff9d431a1f
Build id: 3.10.0-1062.9.1.el7.x86_64
0x9f
29
? ktime_get_ts64
vmlinux 0xffffffff9cf06142
Build id: 3.10.0-1062.9.1.el7.x86_64
0x52
30
? system_call_after_swapgs
vmlinux 0xffffffff9d58de21
Build id: 3.10.0-1062.9.1.el7.x86_64
0xae
31
? system_call_after_swapgs
vmlinux 0xffffffff9d58de15
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa2
32
? system_call_after_swapgs
vmlinux 0xffffffff9d58de21
Build id: 3.10.0-1062.9.1.el7.x86_64
0xae
33
? system_call_after_swapgs
vmlinux 0xffffffff9d58de15
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa2
34
? system_call_after_swapgs
vmlinux 0xffffffff9d58de21
Build id: 3.10.0-1062.9.1.el7.x86_64
0xae
35
? system_call_after_swapgs
vmlinux 0xffffffff9d58de15
Build id: 3.10.0-1062.9.1.el7.x86_64
0xa2
36
SyS_sendmmsg
vmlinux 0xffffffff9d431b22
Build id: 3.10.0-1062.9.1.el7.x86_64
0x12
37
system_call_fastpath
vmlinux 0xffffffff9d58dede
Build id: 3.10.0-1062.9.1.el7.x86_64
0x25
38
? system_call_after_swapgs
vmlinux 0xffffffff9d58de21
Build id: 3.10.0-1062.9.1.el7.x86_64
0xae
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 112s! [php:6211]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev vmw_balloon nfit aesni_intel lrw gf128mul libnvdimm glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif
ata_generic crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi serio_raw mptscsih mptbase vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 6211 Comm: php Kdump: loaded Tainted: G             L ------------   3.10.0-1062.9.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff8fc69ba2c1c0 ti: ffff8fc84f220000 task.ti: ffff8fc84f220000
RIP: 0010:[<ffffffffc01b43dc>]  [<ffffffffc01b43dc>] vmxnet3_xmit_frame+0xaec/0xfd0 [vmxnet3]
RSP: 0018:ffff8fc84f223690  EFLAGS: 00000282
RAX: 0000000000000000 RBX: ffff8fc84f223708 RCX: 00000000000000b2
RDX: ffffb1bf0189e608 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff8fc84f223720 R08: 3536336563696666 R09: 0000000000a04051
R10: 0335363365636966 R11: ffff8fc85d690b10 R12: ffffffffc065e8cd
R13: ffff8fc84f223638 R14: ffffffff9cea4787 R15: ffff8fc84f2235f8
FS:  00007f075366e880(0000) GS:ffff8fc87fc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f64a5cd2000 CR3: 000000035cfea000 CR4: 00000000000407e0
Call Trace:
[<ffffffffc05e63e7>] ? nf_nat_setup_info+0x2f7/0x320 [nf_nat]
[<ffffffff9d44f346>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff9d47d0ba>] sch_direct_xmit+0x11a/0x250
[<ffffffff9d4522b1>] __dev_queue_xmit+0x491/0x650
[<ffffffff9d452480>] dev_queue_xmit+0x10/0x20
[<ffffffff9d4a097b>] ip_finish_output+0x67b/0x8d0
[<ffffffff9d4a0efb>] ip_output+0x7b/0xf0
[<ffffffff9d4a0300>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff9d49e927>] ip_local_out_sk+0x37/0x40
[<ffffffff9d4a1966>] ip_send_skb+0x16/0x50
[<ffffffff9d4c9cec>] udp_send_skb+0xac/0x2b0
[<ffffffff9d49e660>] ? ip_copy_metadata+0x170/0x170
[<ffffffff9d4cb251>] udp_sendmsg+0x5e1/0xa70
[<ffffffff9cec9e30>] ? hrtimer_get_res+0x50/0x50
[<ffffffff9d57fc06>] ? schedule_hrtimeout_range_clock+0xa6/0x150
[<ffffffff9d1a12ed>] ? list_del+0xd/0x30
[<ffffffff9d4cc5c9>] ? udp_poll+0x19/0x90
[<ffffffff9d4d89b9>] inet_sendmsg+0x69/0xb0
[<ffffffff9d42f3a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff9d588975>] ? do_page_fault+0x35/0x90
[<ffffffff9d430177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff9d497257>] ? __ip_route_output_key_hash+0x207/0x7b0
[<ffffffff9d4c76ff>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff9d5831ce>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff9d4ca7cf>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff9d5831ce>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff9d431a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff9cf06142>] ? ktime_get_ts64+0x52/0xf0
[<ffffffff9d58de21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffff9d58de15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff9d58de21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffff9d58de15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff9d58de21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffff9d58de15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffff9d431b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff9d58dede>] system_call_fastpath+0x25/0x2a
[<ffffffff9d58de21>] ? system_call_after_swapgs+0xae/0x146
Code: ff c7 02 00 00 00 00 41 8b 94 24 e8 00 00 00 48 8b 4d d0 8d 14 d5 00 06 00 00 48 63 d2 48 03 91 b8 23 00 00 41 8b 4c 24 34 89 0a <e9> d0 f7 ff ff 0f 1f 80 00 00 00 00 48 c7 45 c8 00 40 00 00 41 
Complete report #752295
Frame # Function Binary Source or offset Line
1
vmxnet3_xmit_frame
vmxnet3 0xffffffffc0546bcc
Build id: 3.10.0-957.21.3.el7.x86_64
0xaec
2
? nf_nat_setup_info
nf_nat 0xffffffffc0a6b3b7
Build id: 3.10.0-957.21.3.el7.x86_64
0x2f7
3
dev_hard_start_xmit
vmlinux 0xffffffffac039ce6
Build id: 3.10.0-957.21.3.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffffac066d6a
Build id: 3.10.0-957.21.3.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffffac03cc61
Build id: 3.10.0-957.21.3.el7.x86_64
0x491
6
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc0a502ce
Build id: 3.10.0-957.21.3.el7.x86_64
0x4e
7
dev_queue_xmit
vmlinux 0xffffffffac03ce30
Build id: 3.10.0-957.21.3.el7.x86_64
0x10
8
ip_finish_output
vmlinux 0xffffffffac0897b6
Build id: 3.10.0-957.21.3.el7.x86_64
0x556
9
ip_output
vmlinux 0xffffffffac089d13
Build id: 3.10.0-957.21.3.el7.x86_64
0x73
10
? __ip_append_data.isra.51
vmlinux 0xffffffffac089260
Build id: 3.10.0-957.21.3.el7.x86_64
0xa50
11
ip_local_out_sk
vmlinux 0xffffffffac087897
Build id: 3.10.0-957.21.3.el7.x86_64
0x37
12
ip_send_skb
vmlinux 0xffffffffac08a776
Build id: 3.10.0-957.21.3.el7.x86_64
0x16
13
udp_send_skb
vmlinux 0xffffffffac0b26ac
Build id: 3.10.0-957.21.3.el7.x86_64
0xac
14
? ip_copy_metadata
vmlinux 0xffffffffac0875e0
Build id: 3.10.0-957.21.3.el7.x86_64
0x170
15
udp_sendmsg
vmlinux 0xffffffffac0b3995
Build id: 3.10.0-957.21.3.el7.x86_64
0x335
16
? handle_mm_fault
vmlinux 0xffffffffabbec27d
Build id: 3.10.0-957.21.3.el7.x86_64
0x39d
17
? udp_poll
vmlinux 0xffffffffac0b4f59
Build id: 3.10.0-957.21.3.el7.x86_64
0x19
18
inet_sendmsg
vmlinux 0xffffffffac0c1199
Build id: 3.10.0-957.21.3.el7.x86_64
0x69
19
? selinux_socket_sendmsg
vmlinux 0xffffffffabcfcd83
Build id: 3.10.0-957.21.3.el7.x86_64
0x23
20
sock_sendmsg
vmlinux 0xffffffffac01a3a6
Build id: 3.10.0-957.21.3.el7.x86_64
0xb6
21
? __find_get_page
vmlinux 0xffffffffabbb600e
Build id: 3.10.0-957.21.3.el7.x86_64
0x1e
22
___sys_sendmsg
vmlinux 0xffffffffac01b177
Build id: 3.10.0-957.21.3.el7.x86_64
0x2f7
23
? __do_fault.isra.59
vmlinux 0xffffffffabbe47ea
Build id: 3.10.0-957.21.3.el7.x86_64
0x8a
24
? __local_bh_enable_ip
vmlinux 0xffffffffabaa04cb
Build id: 3.10.0-957.21.3.el7.x86_64
0x9b
25
? ip4_datagram_release_cb
vmlinux 0xffffffffac0b012f
Build id: 3.10.0-957.21.3.el7.x86_64
0x6f
26
? _raw_spin_unlock_bh
vmlinux 0xffffffffac16b5fe
Build id: 3.10.0-957.21.3.el7.x86_64
0x1e
27
? udp_lib_rehash
vmlinux 0xffffffffac0b2faf
Build id: 3.10.0-957.21.3.el7.x86_64
0x11f
28
? _raw_spin_unlock_bh
vmlinux 0xffffffffac16b5fe
Build id: 3.10.0-957.21.3.el7.x86_64
0x1e
29
__sys_sendmmsg
vmlinux 0xffffffffac01ca1f
Build id: 3.10.0-957.21.3.el7.x86_64
0x9f
30
? SYSC_connect
vmlinux 0xffffffffac01a96d
Build id: 3.10.0-957.21.3.el7.x86_64
0xed
31
? ip_setsockopt
vmlinux 0xffffffffac08ccc0
Build id: 3.10.0-957.21.3.el7.x86_64
0x30
32
? system_call_after_swapgs
vmlinux 0xffffffffac175d21
Build id: 3.10.0-957.21.3.el7.x86_64
0xae
33
? system_call_after_swapgs
vmlinux 0xffffffffac175d15
Build id: 3.10.0-957.21.3.el7.x86_64
0xa2
34
? system_call_after_swapgs
vmlinux 0xffffffffac175d21
Build id: 3.10.0-957.21.3.el7.x86_64
0xae
35
? system_call_after_swapgs
vmlinux 0xffffffffac175d15
Build id: 3.10.0-957.21.3.el7.x86_64
0xa2
36
? system_call_after_swapgs
vmlinux 0xffffffffac175d21
Build id: 3.10.0-957.21.3.el7.x86_64
0xae
37
? system_call_after_swapgs
vmlinux 0xffffffffac175d15
Build id: 3.10.0-957.21.3.el7.x86_64
0xa2
38
SyS_sendmmsg
vmlinux 0xffffffffac01cb22
Build id: 3.10.0-957.21.3.el7.x86_64
0x12
39
system_call_fastpath
vmlinux 0xffffffffac175ddb
Build id: 3.10.0-957.21.3.el7.x86_64
0x22
40
? system_call_after_swapgs
vmlinux 0xffffffffac175d21
Build id: 3.10.0-957.21.3.el7.x86_64
0xae
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 38s! [vacp:26002]
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 38s! [smbd:13350]
Modules linked in: arc4 md4 xt_CHECKSUM iptable_mangle rpcsec_gss_krb5 nfsv4 ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nfs fscache nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter nls_utf8 cifs ccm dns_resolver vmw_vsock_vmci_transport vsock ppdev vmw_balloon coretemp crc32_pclmul ghash_clmulni_intel ext4 mbcache aesni_intel jbd2 lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport i2c_piix4 vmw_vmci nfsd nfs_acl auth_rpcgss lockd grace binfmt_misc sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi
vmxnet3 vmwgfx drm_kms_helper floppy syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi scsi_transport_spi ata_piix mptscsih mptbase libata drm_panel_orientation_quirks involflt(POE) dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 13350 Comm: smbd Kdump: loaded Tainted: P           OEL ------------   3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff9e789e6830c0 ti: ffff9e808d3f8000 task.ti: ffff9e808d3f8000
RIP: 0010:[<ffffffffc0546bcc>]  [<ffffffffc0546bcc>] vmxnet3_xmit_frame+0xaec/0xfd0 [vmxnet3]
RSP: 0018:ffff9e808d3fb6a0  EFLAGS: 00000286
RAX: 0000000000000000 RBX: ffff9e808d3fb718 RCX: 00000000000000c2
RDX: ffffb66d835aa600 RSI: 0000000000000287 RDI: 0000000000000287
RBP: ffff9e808d3fb730 R08: 0075650261736503 R09: 0000000000a0004c
R10: 6e09000000000000 R11: ffff9e802a7e6c10 R12: ffffffffc0a298bd
R13: ffff9e808d3fb648 R14: ffffffffabaa04f7 R15: ffff9e808d3fb608
FS:  00007ffa05938900(0000) GS:ffff9e80bfc80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff9f6b19560 CR3: 000000079def6000 CR4: 00000000000007e0
Call Trace:
[<ffffffffc0a6b3b7>] ? nf_nat_setup_info+0x2f7/0x320 [nf_nat]
[<ffffffffac039ce6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffac066d6a>] sch_direct_xmit+0x11a/0x250
[<ffffffffac03cc61>] __dev_queue_xmit+0x491/0x650
[<ffffffffc0a502ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffac03ce30>] dev_queue_xmit+0x10/0x20
[<ffffffffac0897b6>] ip_finish_output+0x556/0x7b0
[<ffffffffac089d13>] ip_output+0x73/0xe0
[<ffffffffac089260>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffffac087897>] ip_local_out_sk+0x37/0x40
[<ffffffffac08a776>] ip_send_skb+0x16/0x50
[<ffffffffac0b26ac>] udp_send_skb+0xac/0x2b0
[<ffffffffac0875e0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffac0b3995>] udp_sendmsg+0x335/0xa10
[<ffffffffabbec27d>] ? handle_mm_fault+0x39d/0x9b0
[<ffffffffac0b4f59>] ? udp_poll+0x19/0x90
[<ffffffffac0c1199>] inet_sendmsg+0x69/0xb0
[<ffffffffabcfcd83>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffffac01a3a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffabbb600e>] ? __find_get_page+0x1e/0xa0
[<ffffffffac01b177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffabbe47ea>] ? __do_fault.isra.59+0x8a/0x100
[<ffffffffabaa04cb>] ? __local_bh_enable_ip+0x9b/0xb0
[<ffffffffac0b012f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffac16b5fe>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffac0b2faf>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffac16b5fe>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffac01ca1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffffac01a96d>] ? SYSC_connect+0xed/0x130
[<ffffffffac08ccc0>] ? ip_setsockopt+0x30/0xa0
[<ffffffffac175d21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffffac175d15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffffac175d21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffffac175d15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffffac175d21>] ? system_call_after_swapgs+0xae/0x146
[<ffffffffac175d15>] ? system_call_after_swapgs+0xa2/0x146
[<ffffffffac01cb22>] SyS_sendmmsg+0x12/0x30
[<ffffffffac175ddb>] system_call_fastpath+0x22/0x27
[<ffffffffac175d21>] ? system_call_after_swapgs+0xae/0x146
Code: ff c7 02 00 00 00 00 41 8b 94 24 e8 00 00 00 48 8b 4d d0 8d 14 d5 00 06 00 00 48 63 d2 48 03 91 b8 23 00 00 41 8b 4c 24 34 89 0a <e9> d0 f7 ff ff 0f 1f 80 00 00 00 00 48 c7 45 c8 00 40 00 00 41 
Complete report #428628
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc00f6877
Build id: 3.10.0-1160.62.1.el7.x86_64
0x7f7
2
? kfree_skbmem
vmlinux 0xffffffff878416e7
Build id: 3.10.0-1160.62.1.el7.x86_64
0x37
3
dev_hard_start_xmit
vmlinux 0xffffffff87858f86
Build id: 3.10.0-1160.62.1.el7.x86_64
0x246
4
sch_direct_xmit
vmlinux 0xffffffff8788709a
Build id: 3.10.0-1160.62.1.el7.x86_64
0x11a
5
__dev_queue_xmit
vmlinux 0xffffffff8785bf01
Build id: 3.10.0-1160.62.1.el7.x86_64
0x491
6
dev_queue_xmit
vmlinux 0xffffffff8785c0d0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x10
7
ip_finish_output
vmlinux 0xffffffff878aac5b
Build id: 3.10.0-1160.62.1.el7.x86_64
0x67b
8
ip_output
vmlinux 0xffffffff878ab1db
Build id: 3.10.0-1160.62.1.el7.x86_64
0x7b
9
? __ip_append_data.isra.50
vmlinux 0xffffffff878aa5e0
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa60
10
ip_local_out_sk
vmlinux 0xffffffff878a8c07
Build id: 3.10.0-1160.62.1.el7.x86_64
0x37
11
ip_send_skb
vmlinux 0xffffffff878abc46
Build id: 3.10.0-1160.62.1.el7.x86_64
0x16
12
udp_send_skb
vmlinux 0xffffffff878d40ec
Build id: 3.10.0-1160.62.1.el7.x86_64
0xac
13
? ip_copy_metadata
vmlinux 0xffffffff878a8940
Build id: 3.10.0-1160.62.1.el7.x86_64
0x170
14
udp_sendmsg
vmlinux 0xffffffff878d5651
Build id: 3.10.0-1160.62.1.el7.x86_64
0x5e1
15
? hrtimer_get_res
vmlinux 0xffffffff872c9bf0
Build id: 3.10.0-1160.62.1.el7.x86_64
0x50
16
? schedule_hrtimeout_range_clock
vmlinux 0xffffffff8798b796
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa6
17
? udp_poll
vmlinux 0xffffffff878d6999
Build id: 3.10.0-1160.62.1.el7.x86_64
0x19
18
inet_sendmsg
vmlinux 0xffffffff878e2e39
Build id: 3.10.0-1160.62.1.el7.x86_64
0x69
19
sock_sendmsg
vmlinux 0xffffffff87839396
Build id: 3.10.0-1160.62.1.el7.x86_64
0xb6
20
? __radix_tree_lookup
vmlinux 0xffffffff8758da24
Build id: 3.10.0-1160.62.1.el7.x86_64
0x84
21
___sys_sendmsg
vmlinux 0xffffffff8783a167
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2f7
22
? __ip_route_output_key_hash
vmlinux 0xffffffff878a1473
Build id: 3.10.0-1160.62.1.el7.x86_64
0x223
23
? ip4_datagram_release_cb
vmlinux 0xffffffff878d1b1f
Build id: 3.10.0-1160.62.1.el7.x86_64
0x6f
24
? _raw_spin_unlock_bh
vmlinux 0xffffffff8798eb9e
Build id: 3.10.0-1160.62.1.el7.x86_64
0x1e
25
? udp_lib_rehash
vmlinux 0xffffffff878d4bcf
Build id: 3.10.0-1160.62.1.el7.x86_64
0x11f
26
__sys_sendmmsg
vmlinux 0xffffffff8783ba0f
Build id: 3.10.0-1160.62.1.el7.x86_64
0x9f
27
? SYSC_connect
vmlinux 0xffffffff87839900
Build id: 3.10.0-1160.62.1.el7.x86_64
0x90
28
? ip_setsockopt
vmlinux 0xffffffff878ae15c
Build id: 3.10.0-1160.62.1.el7.x86_64
0x2c
29
? system_call_after_swapgs
vmlinux 0xffffffff87999ed5
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa2
30
? system_call_after_swapgs
vmlinux 0xffffffff87999ec9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x96
31
? system_call_after_swapgs
vmlinux 0xffffffff87999ed5
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa2
32
? system_call_after_swapgs
vmlinux 0xffffffff87999ec9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x96
33
? system_call_after_swapgs
vmlinux 0xffffffff87999ed5
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa2
34
? system_call_after_swapgs
vmlinux 0xffffffff87999ec9
Build id: 3.10.0-1160.62.1.el7.x86_64
0x96
35
SyS_sendmmsg
vmlinux 0xffffffff8783bb12
Build id: 3.10.0-1160.62.1.el7.x86_64
0x12
36
system_call_fastpath
vmlinux 0xffffffff87999f92
Build id: 3.10.0-1160.62.1.el7.x86_64
0x25
37
? system_call_after_swapgs
vmlinux 0xffffffff87999ed5
Build id: 3.10.0-1160.62.1.el7.x86_64
0xa2
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 43s! [reporter-urepor:7141]
Modules linked in: tcp_lp nls_utf8 isofs veth vboxsf(OE) xt_addrtype br_netfilter overlay(T) xt_CHECKSUM tun devlink xt_REDIRECT nf_nat_redirect xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_intel8x0 snd_ac97_codec ac97_bus snd_seq snd_seq_device joydev snd_pcm ppdev snd_timer vboxguest(OE) pcspkr snd sg soundcore parport_pc
video parport i2c_piix4 ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci libahci ata_piix serio_raw e1000 libata crc32c_intel drm_panel_orientation_quirks fuse
CPU: 3 PID: 7141 Comm: reporter-urepor Kdump: loaded Tainted: G           OEL ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff890599676300 ti: ffff890599790000 task.ti: ffff890599790000
RIP: 0010:[<ffffffffc00f6877>]  [<ffffffffc00f6877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8905997936a0  EFLAGS: 00010286
RAX: ffffacc8c0783818 RBX: ffff890696c4a330 RCX: 00000000000000eb
RDX: 00000000000000eb RSI: ffff8905b5e3b600 RDI: ffff890697226000
RBP: ffff890599793720 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff890696c4a2c0
R13: 0000000052615f7d R14: 0000000000000000 R15: 0000002000000000
FS:  00007f12690b4700(0000) GS:ffff89069fd80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f45f4122078 CR3: 000000001ad6a000 CR4: 00000000000206e0
Call Trace:
[<ffffffff878416e7>] ? kfree_skbmem+0x37/0x90
[<ffffffff87858f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff8788709a>] sch_direct_xmit+0x11a/0x250
[<ffffffff8785bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffff8785c0d0>] dev_queue_xmit+0x10/0x20
[<ffffffff878aac5b>] ip_finish_output+0x67b/0x8d0
[<ffffffff878ab1db>] ip_output+0x7b/0xf0
[<ffffffff878aa5e0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff878a8c07>] ip_local_out_sk+0x37/0x40
[<ffffffff878abc46>] ip_send_skb+0x16/0x50
[<ffffffff878d40ec>] udp_send_skb+0xac/0x2b0
[<ffffffff878a8940>] ? ip_copy_metadata+0x170/0x170
[<ffffffff878d5651>] udp_sendmsg+0x5e1/0xa70
[<ffffffff872c9bf0>] ? hrtimer_get_res+0x50/0x50
[<ffffffff8798b796>] ? schedule_hrtimeout_range_clock+0xa6/0x150
[<ffffffff878d6999>] ? udp_poll+0x19/0x90
[<ffffffff878e2e39>] inet_sendmsg+0x69/0xb0
[<ffffffff87839396>] sock_sendmsg+0xb6/0xf0
[<ffffffff8758da24>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffff8783a167>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff878a1473>] ? __ip_route_output_key_hash+0x223/0x7e0
[<ffffffff878d1b1f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff8798eb9e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff878d4bcf>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff8783ba0f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff87839900>] ? SYSC_connect+0x90/0x130
[<ffffffff878ae15c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff87999ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff87999ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff87999ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff87999ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff87999ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff87999ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff8783bb12>] SyS_sendmmsg+0x12/0x30
[<ffffffff87999f92>] system_call_fastpath+0x25/0x2a
[<ffffffff87999ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #58073
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc041a877
Build id: 3.10.0-1127.19.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffff93c53386
Build id: 3.10.0-1127.19.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffff93c8131a
Build id: 3.10.0-1127.19.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffff93c56301
Build id: 3.10.0-1127.19.1.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffff93c564d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffff93ca429b
Build id: 3.10.0-1127.19.1.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffff93ca481b
Build id: 3.10.0-1127.19.1.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffff93ca3c20
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffff93ca2247
Build id: 3.10.0-1127.19.1.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffff93ca5286
Build id: 3.10.0-1127.19.1.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffff93ccd65c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffff93ca1f80
Build id: 3.10.0-1127.19.1.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffff93ccebc1
Build id: 3.10.0-1127.19.1.el7.x86_64
0x5e1
14
? udp_poll
vmlinux 0xffffffff93ccff09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x19
15
inet_sendmsg
vmlinux 0xffffffff93cdc389
Build id: 3.10.0-1127.19.1.el7.x86_64
0x69
16
sock_sendmsg
vmlinux 0xffffffff93c333a6
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb6
17
? do_async_page_fault
vmlinux 0xffffffff93d8cfa2
Build id: 3.10.0-1127.19.1.el7.x86_64
0x22
18
___sys_sendmsg
vmlinux 0xffffffff93c34177
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2f7
19
? release_pages
vmlinux 0xffffffff937cd02e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x24e
20
? __ip_route_output_key_hash
vmlinux 0xffffffff93c9aacf
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20f
21
? ip4_datagram_release_cb
vmlinux 0xffffffff93ccb06f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x6f
22
? _raw_spin_unlock_bh
vmlinux 0xffffffff93d8856e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x1e
23
? udp_lib_rehash
vmlinux 0xffffffff93cce13f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x11f
24
? _raw_spin_unlock_bh
vmlinux 0xffffffff93d8856e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x1e
25
__sys_sendmmsg
vmlinux 0xffffffff93c35a1f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x9f
26
? SYSC_connect
vmlinux 0xffffffff93c33910
Build id: 3.10.0-1127.19.1.el7.x86_64
0x90
27
? ip_setsockopt
vmlinux 0xffffffff93ca779c
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2c
28
? system_call_after_swapgs
vmlinux 0xffffffff93d92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
29
? system_call_after_swapgs
vmlinux 0xffffffff93d92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
30
? system_call_after_swapgs
vmlinux 0xffffffff93d92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
31
? system_call_after_swapgs
vmlinux 0xffffffff93d92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
32
? system_call_after_swapgs
vmlinux 0xffffffff93d92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
33
? system_call_after_swapgs
vmlinux 0xffffffff93d92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
34
SyS_sendmmsg
vmlinux 0xffffffff93c35b22
Build id: 3.10.0-1127.19.1.el7.x86_64
0x12
35
system_call_fastpath
vmlinux 0xffffffff93d92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
0x25
36
? system_call_after_swapgs
vmlinux 0xffffffff93d92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [nscd:538]
Modules linked in: ip6table_filter ip6_tables xt_time nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter tcp_diag inet_diag cifs ccm dns_resolver ext4 mbcache jbd2 loop dm_mirror dm_region_hash dm_log dm_mod i2c_piix4 sg pcspkr joydev ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt virtio_blk fb_sys_fops ttm drm ata_piix e1000 libata serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy
CPU: 4 PID: 538 Comm: nscd Kdump: loaded Tainted: G             L ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.13.0-48-gd9c812dda519-prebuilt.qemu.org 04/01/2014
task: ffff8b52551c62a0 ti: ffff8b5255164000 task.ti: ffff8b5255164000
RIP: 0010:[<ffffffffc041a877>]  [<ffffffffc041a877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff8b52551676a0  EFLAGS: 00000286
RAX: ffffa0d101c03818 RBX: 0000000000000001 RCX: 00000000000000e5
RDX: 00000000000000e5 RSI: ffff8b52549a4e80 RDI: ffff8b5254f1c000
RBP: ffff8b5255167720 R08: 0000000000000100 R09: 0000000022100000
R10: 7703000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000003217df402 R14: 0000000100000020 R15: ffffffff939b0870
FS:  00007f2ad02b9700(0000) GS:ffff8b5267d00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2ae410d000 CR3: 0000000036290000 CR4: 00000000000006e0
Call Trace:
[<ffffffff93c53386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff93c8131a>] sch_direct_xmit+0x11a/0x250
[<ffffffff93c56301>] __dev_queue_xmit+0x491/0x650
[<ffffffff93c564d0>] dev_queue_xmit+0x10/0x20
[<ffffffff93ca429b>] ip_finish_output+0x67b/0x8d0
[<ffffffff93ca481b>] ip_output+0x7b/0xf0
[<ffffffff93ca3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff93ca2247>] ip_local_out_sk+0x37/0x40
[<ffffffff93ca5286>] ip_send_skb+0x16/0x50
[<ffffffff93ccd65c>] udp_send_skb+0xac/0x2b0
[<ffffffff93ca1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffff93ccebc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff93ccff09>] ? udp_poll+0x19/0x90
[<ffffffff93cdc389>] inet_sendmsg+0x69/0xb0
[<ffffffff93c333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff93d8cfa2>] ? do_async_page_fault+0x22/0xf0
[<ffffffff93c34177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffff937cd02e>] ? release_pages+0x24e/0x430
[<ffffffff93c9aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffff93ccb06f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffff93d8856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff93cce13f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffff93d8856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffff93c35a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffff93c33910>] ? SYSC_connect+0x90/0x130
[<ffffffff93ca779c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff93d92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff93d92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff93d92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff93d92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff93d92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffff93d92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffff93c35b22>] SyS_sendmmsg+0x12/0x30
[<ffffffff93d92ed2>] system_call_fastpath+0x25/0x2a
[<ffffffff93d92e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00 
Complete report #45775
Frame # Function Binary Source or offset Line
1
e1000_xmit_frame
e1000 0xffffffffc023b877
Build id: 3.10.0-1127.19.1.el7.x86_64
0x7f7
2
dev_hard_start_xmit
vmlinux 0xffffffffb7a53386
Build id: 3.10.0-1127.19.1.el7.x86_64
0x246
3
sch_direct_xmit
vmlinux 0xffffffffb7a8131a
Build id: 3.10.0-1127.19.1.el7.x86_64
0x11a
4
__dev_queue_xmit
vmlinux 0xffffffffb7a56301
Build id: 3.10.0-1127.19.1.el7.x86_64
0x491
5
dev_queue_xmit
vmlinux 0xffffffffb7a564d0
Build id: 3.10.0-1127.19.1.el7.x86_64
0x10
6
ip_finish_output
vmlinux 0xffffffffb7aa429b
Build id: 3.10.0-1127.19.1.el7.x86_64
0x67b
7
ip_output
vmlinux 0xffffffffb7aa481b
Build id: 3.10.0-1127.19.1.el7.x86_64
0x7b
8
? __ip_append_data.isra.50
vmlinux 0xffffffffb7aa3c20
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa60
9
ip_local_out_sk
vmlinux 0xffffffffb7aa2247
Build id: 3.10.0-1127.19.1.el7.x86_64
0x37
10
ip_send_skb
vmlinux 0xffffffffb7aa5286
Build id: 3.10.0-1127.19.1.el7.x86_64
0x16
11
udp_send_skb
vmlinux 0xffffffffb7acd65c
Build id: 3.10.0-1127.19.1.el7.x86_64
0xac
12
? ip_copy_metadata
vmlinux 0xffffffffb7aa1f80
Build id: 3.10.0-1127.19.1.el7.x86_64
0x170
13
udp_sendmsg
vmlinux 0xffffffffb7acebc1
Build id: 3.10.0-1127.19.1.el7.x86_64
0x5e1
14
? page_add_new_anon_rmap
vmlinux 0xffffffffb76010d8
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb8
15
? handle_mm_fault
vmlinux 0xffffffffb75f5f5d
Build id: 3.10.0-1127.19.1.el7.x86_64
0xbfd
16
? sock_has_perm
vmlinux 0xffffffffb770b1d5
Build id: 3.10.0-1127.19.1.el7.x86_64
0x75
17
inet_sendmsg
vmlinux 0xffffffffb7adc389
Build id: 3.10.0-1127.19.1.el7.x86_64
0x69
18
? selinux_socket_sendmsg
vmlinux 0xffffffffb770b303
Build id: 3.10.0-1127.19.1.el7.x86_64
0x23
19
sock_sendmsg
vmlinux 0xffffffffb7a333a6
Build id: 3.10.0-1127.19.1.el7.x86_64
0xb6
20
? do_page_fault
vmlinux 0xffffffffb7b8d975
Build id: 3.10.0-1127.19.1.el7.x86_64
0x35
21
___sys_sendmsg
vmlinux 0xffffffffb7a34177
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2f7
22
? sidtab_context_to_sid
vmlinux 0xffffffffb7719c65
Build id: 3.10.0-1127.19.1.el7.x86_64
0x2b5
23
? __ip_route_output_key_hash
vmlinux 0xffffffffb7a9aacf
Build id: 3.10.0-1127.19.1.el7.x86_64
0x20f
24
? ip4_datagram_release_cb
vmlinux 0xffffffffb7acb06f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x6f
25
? _raw_spin_unlock_bh
vmlinux 0xffffffffb7b8856e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x1e
26
? udp_lib_rehash
vmlinux 0xffffffffb7ace13f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x11f
27
? _raw_spin_unlock_bh
vmlinux 0xffffffffb7b8856e
Build id: 3.10.0-1127.19.1.el7.x86_64
0x1e
28
__sys_sendmmsg
vmlinux 0xffffffffb7a35a1f
Build id: 3.10.0-1127.19.1.el7.x86_64
0x9f
29
? SYSC_connect
vmlinux 0xffffffffb7a33910
Build id: 3.10.0-1127.19.1.el7.x86_64
0x90
30
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
31
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
32
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
33
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e09
Build id: 3.10.0-1127.19.1.el7.x86_64
0x96
34
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
35
SyS_sendmmsg
vmlinux 0xffffffffb7a35b22
Build id: 3.10.0-1127.19.1.el7.x86_64
0x12
36
system_call_fastpath
vmlinux 0xffffffffb7b92ed2
Build id: 3.10.0-1127.19.1.el7.x86_64
0x25
37
? system_call_after_swapgs
vmlinux 0xffffffffb7b92e15
Build id: 3.10.0-1127.19.1.el7.x86_64
0xa2
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [kworker/4:1:3450]
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [httpd:29216]
ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi libata mptscsih serio_raw mptbase e1000 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 5 PID: 29216 Comm: httpd Kdump: loaded Tainted: G        W  OEL ------------   3.10.0-1127.19.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff89aad72720e0 ti: ffff89a8c21fc000 task.ti: ffff89a8c21fc000
RIP: 0010:[<ffffffffc023b877>]  [<ffffffffc023b877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff89a8c21ff6a0  EFLAGS: 00000286
RAX: ffffaf6b40f43818 RBX: 0000000000000001 RCX: 00000000000000b3
RDX: 00000000000000b3 RSI: ffff89a8f5ff9580 RDI: ffff89aaf5e18000
RBP: ffff89a8c21ff720 R08: 0000000000000100 R09: 0000000022100000
R10: 6d6f630379617768 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000135a1dc02 R14: 00000001c06bc8dd R15: ffffffffb77b0870
FS:  00007f60c07f88c0(0000) GS:ffff89aaffd40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f60c080e000 CR3: 0000000002ef0000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb7a53386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb7a8131a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb7a56301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb7a564d0>] dev_queue_xmit+0x10/0x20
[<ffffffffb7aa429b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb7aa481b>] ip_output+0x7b/0xf0
[<ffffffffb7aa3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb7aa2247>] ip_local_out_sk+0x37/0x40
[<ffffffffb7aa5286>] ip_send_skb+0x16/0x50
[<ffffffffb7acd65c>] udp_send_skb+0xac/0x2b0
[<ffffffffb7aa1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb7acebc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb76010d8>] ? page_add_new_anon_rmap+0xb8/0x170
[<ffffffffb75f5f5d>] ? handle_mm_fault+0xbfd/0xfb0
[<ffffffffb770b1d5>] ? sock_has_perm+0x75/0x90
[<ffffffffb7adc389>] inet_sendmsg+0x69/0xb0
[<ffffffffb770b303>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffffb7a333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffb7b8d975>] ? do_page_fault+0x35/0x90
[<ffffffffb7a34177>] ___sys_sendmsg+0x2f7/0x400
[<ffffffffb7719c65>] ? sidtab_context_to_sid+0x2b5/0x7c0
[<ffffffffb7a9aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffffb7acb06f>] ? ip4_datagram_release_cb+0x6f/0x1c0
[<ffffffffb7b8856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb7ace13f>] ? udp_lib_rehash+0x11f/0x150
[<ffffffffb7b8856e>] ? _raw_spin_unlock_bh+0x1e/0x20
[<ffffffffb7a35a1f>] __sys_sendmmsg+0x9f/0x190
[<ffffffffb7a33910>] ? SYSC_connect+0x90/0x130
[<ffffffffb7b92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffb7b92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffb7b92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffb7b92e09>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffb7b92e15>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffb7a35b22>] SyS_sendmmsg+0x12/0x30
[<ffffffffb7b92ed2>] system_call_fastpath+0x25/0x2a
[<ffffffffb7b92e15>] ? system_call_after_swapgs+0xa2/0x13a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00