Complete report #1107126
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc02688e7
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffff98a7edc6
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffff98aadd72
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x122
|
|
4
|
__qdisc_run
|
vmlinux |
0xffffffff98aadf2e
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x8e
|
|
5
|
__dev_queue_xmit
|
vmlinux |
0xffffffff98a81de9
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x4a9
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffff98a81fa0
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x10
|
|
7
|
ip_finish_output
|
vmlinux |
0xffffffff98ad260b
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x67b
|
|
8
|
ip_output
|
vmlinux |
0xffffffff98ad2b9b
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x7b
|
|
9
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffff98ad1f90
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xa60
|
|
10
|
ip_local_out_sk
|
vmlinux |
0xffffffff98ad0587
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x37
|
|
11
|
ip_send_skb
|
vmlinux |
0xffffffff98ad3636
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x16
|
|
12
|
udp_send_skb
|
vmlinux |
0xffffffff98afc3ac
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xac
|
|
13
|
|
vmlinux |
0xffffffff98ad02b0
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x170
|
|
14
|
udp_sendmsg
|
vmlinux |
0xffffffff98afd9a1
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x5e1
|
|
15
|
|
vmlinux |
0xffffffff987a3c02
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x22
|
|
16
|
|
vmlinux |
0xffffffff985c85fe
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x1e
|
|
17
|
|
vmlinux |
0xffffffff985cbcad
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x17d
|
|
18
|
inet_sendmsg
|
vmlinux |
0xffffffff98b0b4e9
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x69
|
|
19
|
sock_sendmsg
|
vmlinux |
0xffffffff98a5e4f6
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xb6
|
|
20
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffff98ac8c63
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x223
|
|
21
|
SYSC_sendto
|
vmlinux |
0xffffffff98a5ec51
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x121
|
|
22
|
|
vmlinux |
0xffffffff9850d512
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x52
|
|
23
|
SyS_sendto
|
vmlinux |
0xffffffff98a607ee
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0xe
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffff98bc539a
Build id: 3.10.0-1160.118.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 24s! [pool:971]
Modules linked in: overlay ipt_MASQUERADE nf_nat_masquerade_ipv4 ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq crc32_pclmul snd_seq_device ghash_clmulni_intel snd_pcm aesni_intel snd_timer lrw gf128mul glue_helper snd ablk_helper soundcore sg cryptd ppdev vmw_balloon joydev pcspkr parport_pc parport
i2c_piix4 vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix nfit crct10dif_pclmul crct10dif_common drm crc32c_intel libata libnvdimm mptspi scsi_transport_spi mptscsih serio_raw e1000 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 971 Comm: pool Not tainted 3.10.0-1160.118.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9d03b0710000 ti: ffff9d03b3804000 task.ti: ffff9d03b3804000
RIP: 0010:[<ffffffffc02688e7>] [<ffffffffc02688e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9d03b38077e8 EFLAGS: 00000286
RAX: ffffbc2200983818 RBX: 0000000000000945 RCX: 000000000000006a
RDX: 000000000000006a RSI: ffff9d03b052de80 RDI: ffff9d03b420e000
RBP: ffff9d03b3807868 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff9d03b6332800
R13: 0000000000000000 R14: 0000000000000002 R15: 0000000400000000
FS: 00007f03a3aca700(0000) GS:ffff9d03b97c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9b3dee1f90 CR3: 000000013054c000 CR4: 00000000003607e0
Call Trace:
[<ffffffff98a7edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff98aadd72>] sch_direct_xmit+0x122/0x250
[<ffffffff98aadf2e>] __qdisc_run+0x8e/0x360
[<ffffffff98a81de9>] __dev_queue_xmit+0x4a9/0x650
[<ffffffff98a81fa0>] dev_queue_xmit+0x10/0x20
[<ffffffff98ad260b>] ip_finish_output+0x67b/0x8d0
[<ffffffff98ad2b9b>] ip_output+0x7b/0x100
[<ffffffff98ad1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff98ad0587>] ip_local_out_sk+0x37/0x40
[<ffffffff98ad3636>] ip_send_skb+0x16/0x50
[<ffffffff98afc3ac>] udp_send_skb+0xac/0x2b0
[<ffffffff98ad02b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffff98afd9a1>] udp_sendmsg+0x5e1/0xa70
[<ffffffff987a3c02>] ? radix_tree_lookup_slot+0x22/0x50
[<ffffffff985c85fe>] ? __find_get_page+0x1e/0xa0
[<ffffffff985cbcad>] ? filemap_fault+0x17d/0x420
[<ffffffff98b0b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffff98a5e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffff98ac8c63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffff98a5ec51>] SYSC_sendto+0x121/0x1d0
[<ffffffff9850d512>] ? ktime_get_ts64+0x52/0xf0
[<ffffffff98a607ee>] SyS_sendto+0xe/0x20
[<ffffffff98bc539a>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #1012862
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc022b8e7
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffffb387edc6
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffffb38add72
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x122
|
|
4
|
__dev_queue_xmit
|
vmlinux |
0xffffffffb3881dd1
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x491
|
|
5
|
dev_queue_xmit
|
vmlinux |
0xffffffffb3881fa0
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x10
|
|
6
|
ip_finish_output
|
vmlinux |
0xffffffffb38d260b
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x67b
|
|
7
|
ip_output
|
vmlinux |
0xffffffffb38d2b9b
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x7b
|
|
8
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffffb38d1f90
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xa60
|
|
9
|
ip_local_out_sk
|
vmlinux |
0xffffffffb38d0587
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x37
|
|
10
|
ip_send_skb
|
vmlinux |
0xffffffffb38d3636
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x16
|
|
11
|
udp_send_skb
|
vmlinux |
0xffffffffb38fc3ac
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xac
|
|
12
|
|
vmlinux |
0xffffffffb38d02b0
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x170
|
|
13
|
udp_sendmsg
|
vmlinux |
0xffffffffb38fd9a1
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x5e1
|
|
14
|
?
__mem_cgroup_uncharge_common
|
vmlinux |
0xffffffffb344aced
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xcd
|
|
15
|
inet_sendmsg
|
vmlinux |
0xffffffffb390b4e9
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x69
|
|
16
|
sock_sendmsg
|
vmlinux |
0xffffffffb385e4f6
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xb6
|
|
17
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffffb38c8c63
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x223
|
|
18
|
SYSC_sendto
|
vmlinux |
0xffffffffb385ec51
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x121
|
|
19
|
|
vmlinux |
0xffffffffb38d5bcc
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x2c
|
|
20
|
SyS_sendto
|
vmlinux |
0xffffffffb38607ee
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0xe
|
|
21
|
system_call_fastpath
|
vmlinux |
0xffffffffb39c539a
Build id: 3.10.0-1160.90.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 40s! [traceroute:47444]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink iptable_filter vmw_vsock_vmci_transport vsock ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 xt_conntrack nf_conntrack ip6table_filter ip6_tables fuse snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel lrw snd_ens1371 snd_rawmidi snd_ac97_codec gf128mul glue_helper ac97_bus snd_seq ablk_helper cryptd ppdev snd_seq_device ext4 snd_pcm mbcache jbd2 snd_timer snd soundcore vmw_balloon pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic
sr_mod cdrom ata_generic vmwgfx pata_acpi drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm nfit libnvdimm drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi serio_raw e1000 scsi_transport_spi mptscsih ata_piix libata drm_panel_orientation_quirks mptbase dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 47444 Comm: traceroute Kdump: loaded Tainted: G L ------------ 3.10.0-1160.90.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff91dac1080000 ti: ffff91d965e38000 task.ti: ffff91d965e38000
RIP: 0010:[<ffffffffc022b8e7>] [<ffffffffc022b8e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff91d965e3b840 EFLAGS: 00010286
RAX: ffffa05201943818 RBX: 0000000000000001 RCX: 0000000000000033
RDX: 0000000000000033 RSI: ffff91dbe8409bc0 RDI: ffff91dbec7d8000
RBP: ffff91d965e3b8c0 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 0000000000000202
R13: 000000020d240802 R14: 00000001c082a99d R15: ffffffffb35c8fd0
FS: 00007fe4f1a2d740(0000) GS:ffff91dbed640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fe4f1a4c000 CR3: 00000001a5e36000 CR4: 0000000000360fe0
Call Trace:
[<ffffffffb387edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb38add72>] sch_direct_xmit+0x122/0x250
[<ffffffffb3881dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffb3881fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffb38d260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb38d2b9b>] ip_output+0x7b/0x100
[<ffffffffb38d1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb38d0587>] ip_local_out_sk+0x37/0x40
[<ffffffffb38d3636>] ip_send_skb+0x16/0x50
[<ffffffffb38fc3ac>] udp_send_skb+0xac/0x2b0
[<ffffffffb38d02b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb38fd9a1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb344aced>] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
[<ffffffffb390b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffffb385e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffffb38c8c63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffb385ec51>] SYSC_sendto+0x121/0x1d0
[<ffffffffb38d5bcc>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb38607ee>] SyS_sendto+0xe/0x20
[<ffffffffb39c539a>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #584751
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc04e18e7
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffffafe7edc6
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffffafeadd72
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x122
|
|
4
|
__dev_queue_xmit
|
vmlinux |
0xffffffffafe81dd1
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x491
|
|
5
|
dev_queue_xmit
|
vmlinux |
0xffffffffafe81fa0
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x10
|
|
6
|
ip_finish_output
|
vmlinux |
0xffffffffafed260b
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x67b
|
|
7
|
ip_output
|
vmlinux |
0xffffffffafed2b9b
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x7b
|
|
8
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffffafed1f90
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xa60
|
|
9
|
ip_local_out_sk
|
vmlinux |
0xffffffffafed0587
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x37
|
|
10
|
ip_send_skb
|
vmlinux |
0xffffffffafed3636
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x16
|
|
11
|
udp_send_skb
|
vmlinux |
0xffffffffafefc3ac
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xac
|
|
12
|
|
vmlinux |
0xffffffffafed02b0
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x170
|
|
13
|
udp_sendmsg
|
vmlinux |
0xffffffffafefd9a1
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x5e1
|
|
14
|
|
vmlinux |
0xffffffffafba3542
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x22
|
|
15
|
|
vmlinux |
0xffffffffaf9c83de
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x1e
|
|
16
|
|
vmlinux |
0xffffffffaf9cba8d
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x17d
|
|
17
|
inet_sendmsg
|
vmlinux |
0xffffffffaff0b4e9
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x69
|
|
18
|
sock_sendmsg
|
vmlinux |
0xffffffffafe5e4f6
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xb6
|
|
19
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffffafec8c63
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x223
|
|
20
|
SYSC_sendto
|
vmlinux |
0xffffffffafe5ec51
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x121
|
|
21
|
|
vmlinux |
0xffffffffafed5bcc
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x2c
|
|
22
|
SyS_sendto
|
vmlinux |
0xffffffffafe607ee
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0xe
|
|
23
|
system_call_fastpath
|
vmlinux |
0xffffffffaffc539a
Build id: 3.10.0-1160.80.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 32s! [smtpd:21703]
Modules linked in: ip6t_REJECT nf_reject_ipv6 nf_log_ipv6 ip6table_filter nf_log_ipv4 nf_log_common nf_nat_ftp xt_REDIRECT nf_nat_redirect xt_conntrack nf_conntrack_ftp xt_LOG xt_limit xt_multiport iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 ip6table_mangle ip6table_raw ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 nf_nat nf_conntrack ip6_tables iptable_mangle iptable_raw ipt_REJECT nf_reject_ipv4 iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi snd_ens1371 crc32_pclmul snd_rawmidi snd_ac97_codec ghash_clmulni_intel ppdev ac97_bus snd_seq snd_seq_device snd_pcm aesni_intel vmw_balloon lrw gf128mul glue_helper ablk_helper cryptd snd_timer snd joydev pcspkr sg soundcore vmw_vmci
i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod crct10dif_generic cdrom ata_generic pata_acpi vmwgfx e1000 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm crct10dif_pclmul crct10dif_common crc32c_intel drm mptspi ata_piix scsi_transport_spi serio_raw mptscsih mptbase nfit libata libnvdimm drm_panel_orientation_quirks fuse
CPU: 2 PID: 21703 Comm: smtpd Kdump: loaded Tainted: G L ------------ 3.10.0-1160.80.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9259d18a2100 ti: ffff925985864000 task.ti: ffff925985864000
RIP: 0010:[<ffffffffc04e18e7>] [<ffffffffc04e18e7>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff925985867840 EFLAGS: 00000286
RAX: ffffae9f41903818 RBX: 0000000000000000 RCX: 00000000000000db
RDX: 00000000000000db RSI: ffff925975c26500 RDI: ffff925be3824000
RBP: ffff9259858678c0 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff9259858678a8
R13: 0001000085867818 R14: ffffffffc089899d R15: ffff9259858677d8
FS: 00007f511afae880(0000) GS:ffff925be4880000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5118d794e0 CR3: 0000000028e1c000 CR4: 00000000001607e0
Call Trace:
[<ffffffffafe7edc6>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffafeadd72>] sch_direct_xmit+0x122/0x250
[<ffffffffafe81dd1>] __dev_queue_xmit+0x491/0x650
[<ffffffffafe81fa0>] dev_queue_xmit+0x10/0x20
[<ffffffffafed260b>] ip_finish_output+0x67b/0x8d0
[<ffffffffafed2b9b>] ip_output+0x7b/0x100
[<ffffffffafed1f90>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffafed0587>] ip_local_out_sk+0x37/0x40
[<ffffffffafed3636>] ip_send_skb+0x16/0x50
[<ffffffffafefc3ac>] udp_send_skb+0xac/0x2b0
[<ffffffffafed02b0>] ? ip_copy_metadata+0x170/0x170
[<ffffffffafefd9a1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffafba3542>] ? radix_tree_lookup_slot+0x22/0x50
[<ffffffffaf9c83de>] ? __find_get_page+0x1e/0xa0
[<ffffffffaf9cba8d>] ? filemap_fault+0x17d/0x420
[<ffffffffaff0b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffffafe5e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffffafec8c63>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffafe5ec51>] SYSC_sendto+0x121/0x1d0
[<ffffffffafed5bcc>] ? ip_setsockopt+0x2c/0x90
[<ffffffffafe607ee>] SyS_sendto+0xe/0x20
[<ffffffffaffc539a>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 cc cc cc cc
Complete report #487720
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc0167877
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffffa0458f86
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffffa048709a
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x11a
|
|
4
|
__dev_queue_xmit
|
vmlinux |
0xffffffffa045bf01
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x491
|
|
5
|
|
vmlinux |
0xffffffff9fea4207
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x17
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffffa045c0d0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x10
|
|
7
|
neigh_resolve_output
|
vmlinux |
0xffffffffa0468dfd
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x11d
|
|
8
|
ip_finish_output
|
vmlinux |
0xffffffffa04aa85c
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x27c
|
|
9
|
ip_output
|
vmlinux |
0xffffffffa04ab1db
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x7b
|
|
10
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffffa04aa5e0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xa60
|
|
11
|
ip_local_out_sk
|
vmlinux |
0xffffffffa04a8c07
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x37
|
|
12
|
ip_send_skb
|
vmlinux |
0xffffffffa04abc46
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x16
|
|
13
|
udp_send_skb
|
vmlinux |
0xffffffffa04d40fc
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xac
|
|
14
|
|
vmlinux |
0xffffffffa04a8940
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x170
|
|
15
|
udp_sendmsg
|
vmlinux |
0xffffffffa04d5661
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x5e1
|
|
16
|
?
__mem_cgroup_uncharge_common
|
vmlinux |
0xffffffffa003dbcd
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xcd
|
|
17
|
inet_sendmsg
|
vmlinux |
0xffffffffa04e2e49
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x69
|
|
18
|
sock_sendmsg
|
vmlinux |
0xffffffffa0439396
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xb6
|
|
19
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffffa04a1473
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x223
|
|
20
|
SYSC_sendto
|
vmlinux |
0xffffffffa0439ac1
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x121
|
|
21
|
|
vmlinux |
0xffffffffa04ae15c
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x2c
|
|
22
|
SyS_sendto
|
vmlinux |
0xffffffffa043b5de
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xe
|
|
23
|
system_call_fastpath
|
vmlinux |
0xffffffffa0599f92
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [ManagementAgent:1439]
Modules linked in: rfcomm xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter devlink ebtables iptable_filter vmw_vsock_vmci_transport vsock fuse ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 xt_conntrack nf_conntrack ip6table_filter ip6_tables bnep snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ext4 ghash_clmulni_intel ppdev mbcache jbd2 vmw_balloon aesni_intel snd_ens1371 btusb lrw gf128mul glue_helper btrtl ablk_helper btbcm cryptd btintel snd_rawmidi bluetooth snd_ac97_codec ac97_bus snd_seq snd_seq_device joydev snd_pcm pcspkr snd_timer rfkill snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd
grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom sd_mod ata_generic crc_t10dif crct10dif_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi ata_piix nfit scsi_transport_spi e1000 libnvdimm mptscsih libata serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ipmi_msghandler]
CPU: 1 PID: 1439 Comm: ManagementAgent Kdump: loaded Not tainted 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/19/2017
task: ffff9cae68d50000 ti: ffff9cae69354000 task.ti: ffff9cae69354000
RIP: 0010:[<ffffffffc0167877>] [<ffffffffc0167877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff9cae69357800 EFLAGS: 00000286
RAX: ffffaa7fc1943818 RBX: 0000000000000001 RCX: 00000000000000f7
RDX: 00000000000000f7 RSI: ffff9cadb964e140 RDI: ffff9cae6bcb0000
RBP: ffff9cae69357880 R08: 0000000000000100 R09: 0000000022100000
R10: 6404000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 0000000112af9202 R14: 0000000100000001 R15: ffffffffa01b20d0
FS: 00007f009feae700(0000) GS:ffff9cae6b840000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f00a45b5000 CR3: 00000002a8dd6000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffa0458f86>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffa048709a>] sch_direct_xmit+0x11a/0x250
[<ffffffffa045bf01>] __dev_queue_xmit+0x491/0x650
[<ffffffff9fea4207>] ? local_bh_enable+0x17/0x20
[<ffffffffa045c0d0>] dev_queue_xmit+0x10/0x20
[<ffffffffa0468dfd>] neigh_resolve_output+0x11d/0x220
[<ffffffffa04aa85c>] ip_finish_output+0x27c/0x8d0
[<ffffffffa04ab1db>] ip_output+0x7b/0xf0
[<ffffffffa04aa5e0>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffa04a8c07>] ip_local_out_sk+0x37/0x40
[<ffffffffa04abc46>] ip_send_skb+0x16/0x50
[<ffffffffa04d40fc>] udp_send_skb+0xac/0x2b0
[<ffffffffa04a8940>] ? ip_copy_metadata+0x170/0x170
[<ffffffffa04d5661>] udp_sendmsg+0x5e1/0xa70
[<ffffffffa003dbcd>] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
[<ffffffffa04e2e49>] inet_sendmsg+0x69/0xb0
[<ffffffffa0439396>] sock_sendmsg+0xb6/0xf0
[<ffffffffa04a1473>] ? __ip_route_output_key_hash+0x223/0x7e0
[<ffffffffa0439ac1>] SYSC_sendto+0x121/0x1c0
[<ffffffffa04ae15c>] ? ip_setsockopt+0x2c/0x90
[<ffffffffa043b5de>] SyS_sendto+0xe/0x10
[<ffffffffa0599f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #78446
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc025e877
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x7f7
|
|
2
|
|
vmlinux |
0xffffffff99e3d6f7
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x37
|
|
3
|
dev_hard_start_xmit
|
vmlinux |
0xffffffff99e54f96
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x246
|
|
4
|
sch_direct_xmit
|
vmlinux |
0xffffffff99e8303a
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x11a
|
|
5
|
__dev_queue_xmit
|
vmlinux |
0xffffffff99e57f11
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x491
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffff99e580e0
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x10
|
|
7
|
ip_finish_output
|
vmlinux |
0xffffffff99ea607b
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x67b
|
|
8
|
ip_output
|
vmlinux |
0xffffffff99ea65fb
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x7b
|
|
9
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffff99ea5a00
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xa60
|
|
10
|
ip_local_out_sk
|
vmlinux |
0xffffffff99ea4027
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x37
|
|
11
|
ip_send_skb
|
vmlinux |
0xffffffff99ea7066
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x16
|
|
12
|
udp_send_skb
|
vmlinux |
0xffffffff99ecf49c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xac
|
|
13
|
|
vmlinux |
0xffffffff99ea3d60
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x170
|
|
14
|
udp_sendmsg
|
vmlinux |
0xffffffff99ed0a01
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x5e1
|
|
15
|
?
__mem_cgroup_uncharge_common
|
vmlinux |
0xffffffff99a3d37d
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xcd
|
|
16
|
|
vmlinux |
0xffffffff99b0c1d5
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x75
|
|
17
|
inet_sendmsg
|
vmlinux |
0xffffffff99ede1c9
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x69
|
|
18
|
|
vmlinux |
0xffffffff99b0c303
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x23
|
|
19
|
sock_sendmsg
|
vmlinux |
0xffffffff99e353a6
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xb6
|
|
20
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffff99e9c883
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x223
|
|
21
|
SYSC_sendto
|
vmlinux |
0xffffffff99e35ad1
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x121
|
|
22
|
|
vmlinux |
0xffffffff99ea957c
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x2c
|
|
23
|
SyS_sendto
|
vmlinux |
0xffffffff99e375ee
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0xe
|
|
24
|
system_call_fastpath
|
vmlinux |
0xffffffff99f94f92
Build id: 3.10.0-1160.15.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 61s! [ThreadPoolForeg:3474]
Modules linked in: tcp_lp rfcomm xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev btusb aesni_intel btrtl btbcm vmw_balloon btintel lrw gf128mul glue_helper snd_ens1371 ablk_helper bluetooth cryptd snd_rawmidi snd_ac97_codec joydev
pcspkr ac97_bus snd_seq snd_seq_device snd_pcm snd_timer rfkill snd sg soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx crct10dif_pclmul crct10dif_common crc32c_intel drm_kms_helper syscopyarea sysfillrect ahci sysimgblt serio_raw fb_sys_fops ttm nfit libnvdimm libahci ata_piix drm libata mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 3474 Comm: ThreadPoolForeg Kdump: loaded Tainted: G L ------------ 3.10.0-1160.15.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9401732ed280 ti: ffff940162758000 task.ti: ffff940162758000
RIP: 0010:[<ffffffffc025e877>] [<ffffffffc025e877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff94016275b840 EFLAGS: 00000286
RAX: ffffb45d40383818 RBX: 0000000000000000 RCX: 0000000000000037
RDX: 0000000000000037 RSI: ffff940134d3c140 RDI: ffff940173308000
RBP: ffff94016275b8c0 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff94016275b8a8
R13: 000040006275b818 R14: ffffffffc08298dd R15: 000000206275b7d8
FS: 00007f7b9031c700(0000) GS:ffff94017b640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7ba7ea9000 CR3: 000000004a39e000 CR4: 00000000003607e0
Call Trace:
[<ffffffff99e3d6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffff99e54f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff99e8303a>] sch_direct_xmit+0x11a/0x250
[<ffffffff99e57f11>] __dev_queue_xmit+0x491/0x650
[<ffffffff99e580e0>] dev_queue_xmit+0x10/0x20
[<ffffffff99ea607b>] ip_finish_output+0x67b/0x8d0
[<ffffffff99ea65fb>] ip_output+0x7b/0xf0
[<ffffffff99ea5a00>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffff99ea4027>] ip_local_out_sk+0x37/0x40
[<ffffffff99ea7066>] ip_send_skb+0x16/0x50
[<ffffffff99ecf49c>] udp_send_skb+0xac/0x2b0
[<ffffffff99ea3d60>] ? ip_copy_metadata+0x170/0x170
[<ffffffff99ed0a01>] udp_sendmsg+0x5e1/0xa70
[<ffffffff99a3d37d>] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
[<ffffffff99b0c1d5>] ? sock_has_perm+0x75/0x90
[<ffffffff99ede1c9>] inet_sendmsg+0x69/0xb0
[<ffffffff99b0c303>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff99e353a6>] sock_sendmsg+0xb6/0xf0
[<ffffffff99e9c883>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffff99e35ad1>] SYSC_sendto+0x121/0x1c0
[<ffffffff99ea957c>] ? ip_setsockopt+0x2c/0x90
[<ffffffff99e375ee>] SyS_sendto+0xe/0x10
[<ffffffff99f94f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #57022
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc0228877
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x7f7
|
|
2
|
|
vmlinux |
0xffffffffbcc3c6f7
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x37
|
|
3
|
dev_hard_start_xmit
|
vmlinux |
0xffffffffbcc53f96
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x246
|
|
4
|
sch_direct_xmit
|
vmlinux |
0xffffffffbcc8203a
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x11a
|
|
5
|
__dev_queue_xmit
|
vmlinux |
0xffffffffbcc56f11
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x491
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffffbcc570e0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x10
|
|
7
|
ip_finish_output
|
vmlinux |
0xffffffffbcca507b
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x67b
|
|
8
|
ip_output
|
vmlinux |
0xffffffffbcca55fb
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x7b
|
|
9
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffffbcca4a00
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xa60
|
|
10
|
ip_local_out_sk
|
vmlinux |
0xffffffffbcca3027
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x37
|
|
11
|
ip_send_skb
|
vmlinux |
0xffffffffbcca6066
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x16
|
|
12
|
udp_send_skb
|
vmlinux |
0xffffffffbccce49c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xac
|
|
13
|
|
vmlinux |
0xffffffffbcca2d60
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x170
|
|
14
|
udp_sendmsg
|
vmlinux |
0xffffffffbcccfa01
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x5e1
|
|
15
|
?
__mem_cgroup_uncharge_common
|
vmlinux |
0xffffffffbc83d37d
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xcd
|
|
16
|
inet_sendmsg
|
vmlinux |
0xffffffffbccdd1c9
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x69
|
|
17
|
sock_sendmsg
|
vmlinux |
0xffffffffbcc343a6
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xb6
|
|
18
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffffbcc9b883
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x223
|
|
19
|
SYSC_sendto
|
vmlinux |
0xffffffffbcc34ad1
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x121
|
|
20
|
|
vmlinux |
0xffffffffbcca857c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x2c
|
|
21
|
SyS_sendto
|
vmlinux |
0xffffffffbcc365ee
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xe
|
|
22
|
system_call_fastpath
|
vmlinux |
0xffffffffbcd93f92
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 24s! [ERAServer:2945]
Modules linked in: binfmt_misc eset_rtp(OE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event iosf_mbi crc32_pclmul ghash_clmulni_intel snd_ens1371 ppdev snd_rawmidi snd_ac97_codec ac97_bus snd_seq aesni_intel snd_seq_device vmw_balloon snd_pcm lrw gf128mul glue_helper ablk_helper
cryptd pcspkr joydev snd_timer sg vmw_vmci snd soundcore i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops nfit ttm libnvdimm drm crct10dif_pclmul crct10dif_common crc32c_intel ata_piix mptspi libata scsi_transport_spi mptscsih e1000 mptbase serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 2945 Comm: ERAServer Kdump: loaded Tainted: G OE ------------ 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff919e43f64200 ti: ffff919f2f1a0000 task.ti: ffff919f2f1a0000
RIP: 0010:[<ffffffffc0228877>] [<ffffffffc0228877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff919f2f1a3840 EFLAGS: 00000286
RAX: ffffa071806c3818 RBX: 0000000000000000 RCX: 000000000000003c
RDX: 000000000000003c RSI: ffff919e35e16a80 RDI: ffff919f2fffa000
RBP: ffff919f2f1a38c0 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: ffff919f2f1a38a8
R13: 000040002f1a3818 R14: ffffffffc074d8dd R15: ffff919f2f1a37d8
FS: 00007fd29635c700(0000) GS:ffff919f39640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fcfca01c000 CR3: 000000012f8b0000 CR4: 00000000003607e0
Call Trace:
[<ffffffffbcc3c6f7>] ? kfree_skbmem+0x37/0x90
[<ffffffffbcc53f96>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffbcc8203a>] sch_direct_xmit+0x11a/0x250
[<ffffffffbcc56f11>] __dev_queue_xmit+0x491/0x650
[<ffffffffbcc570e0>] dev_queue_xmit+0x10/0x20
[<ffffffffbcca507b>] ip_finish_output+0x67b/0x8d0
[<ffffffffbcca55fb>] ip_output+0x7b/0xf0
[<ffffffffbcca4a00>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffbcca3027>] ip_local_out_sk+0x37/0x40
[<ffffffffbcca6066>] ip_send_skb+0x16/0x50
[<ffffffffbccce49c>] udp_send_skb+0xac/0x2b0
[<ffffffffbcca2d60>] ? ip_copy_metadata+0x170/0x170
[<ffffffffbcccfa01>] udp_sendmsg+0x5e1/0xa70
[<ffffffffbc83d37d>] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
[<ffffffffbccdd1c9>] inet_sendmsg+0x69/0xb0
[<ffffffffbcc343a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffbcc9b883>] ? __ip_route_output_key_hash+0x223/0x7f0
[<ffffffffbcc34ad1>] SYSC_sendto+0x121/0x1c0
[<ffffffffbcca857c>] ? ip_setsockopt+0x2c/0x90
[<ffffffffbcc365ee>] SyS_sendto+0xe/0x10
[<ffffffffbcd93f92>] system_call_fastpath+0x25/0x2a
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #11339
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc00e6877
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffffb9453386
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffffb948131a
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x11a
|
|
4
|
__dev_queue_xmit
|
vmlinux |
0xffffffffb9456301
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x491
|
|
5
|
|
vmlinux |
0xffffffffb8ea4b17
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x17
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffffb94564d0
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x10
|
|
7
|
neigh_resolve_output
|
vmlinux |
0xffffffffb946312d
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x11d
|
|
8
|
ip_finish_output
|
vmlinux |
0xffffffffb94a3e9c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x27c
|
|
9
|
ip_output
|
vmlinux |
0xffffffffb94a481b
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x7b
|
|
10
|
?
__ip_append_data.isra.50
|
vmlinux |
0xffffffffb94a3c20
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xa60
|
|
11
|
ip_local_out_sk
|
vmlinux |
0xffffffffb94a2247
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x37
|
|
12
|
ip_send_skb
|
vmlinux |
0xffffffffb94a5286
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x16
|
|
13
|
udp_send_skb
|
vmlinux |
0xffffffffb94cd65c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xac
|
|
14
|
|
vmlinux |
0xffffffffb94a1f80
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x170
|
|
15
|
udp_sendmsg
|
vmlinux |
0xffffffffb94cebc1
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x5e1
|
|
16
|
?
__mem_cgroup_uncharge_common
|
vmlinux |
0xffffffffb903d2dd
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xcd
|
|
17
|
inet_sendmsg
|
vmlinux |
0xffffffffb94dc389
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x69
|
|
18
|
sock_sendmsg
|
vmlinux |
0xffffffffb94333a6
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xb6
|
|
19
|
?
__ip_route_output_key_hash
|
vmlinux |
0xffffffffb949aacf
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x20f
|
|
20
|
SYSC_sendto
|
vmlinux |
0xffffffffb9433ad1
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x121
|
|
21
|
|
vmlinux |
0xffffffffb94a779c
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0x2c
|
|
22
|
SyS_sendto
|
vmlinux |
0xffffffffb94355ee
Build id: 3.10.0-1127.18.2.el7.x86_64
|
0xe
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 74s! [java:2895]
Modules linked in: veth xt_nat nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) fuse vmw_vsock_vmci_transport vmw_vmci vsock sunrpc snd_seq_midi snd_seq_midi_event sb_edac ppdev iosf_mbi crc32_pclmul snd_ens1371 snd_rawmidi ghash_clmulni_intel snd_ac97_codec ac97_bus vmw_balloon snd_seq aesni_intel snd_seq_device snd_pcm lrw gf128mul glue_helper ablk_helper cryptd snd_timer pcspkr joydev snd soundcore sg i2c_piix4 parport_pc parport btrfs raid6_pq xor vfat fat ext4 mbcache jbd2 replicator(OE) binfmt_misc
ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea nfit sysfillrect sysimgblt fb_sys_fops ata_piix ttm libnvdimm libata drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi mptscsih mptbase e1000 serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 2895 Comm: java Tainted: G OEL ------------ T 3.10.0-1127.18.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff94198e145230 ti: ffff94197f06c000 task.ti: ffff94197f06c000
RIP: 0010:[<ffffffffc00e6877>] [<ffffffffc00e6877>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0018:ffff94197f06f800 EFLAGS: 00000286
RAX: ffffabb500d43818 RBX: 0000000000000001 RCX: 0000000000000005
RDX: 0000000000000005 RSI: ffff9417b5dd0900 RDI: ffff9419b4714000
RBP: ffff94197f06f880 R08: 0000000000000100 R09: 0000000022100000
R10: 7405000000000000 R11: 0000000000000000 R12: 0000000000000202
R13: 00000002311a0802 R14: 00000001b4f3da00 R15: ffffffffb91b0830
FS: 00007f1bc9b89700(0000) GS:ffff9419b56c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1bee4d0000 CR3: 0000000234fa6000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb9453386>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffffb948131a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb9456301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb8ea4b17>] ? local_bh_enable+0x17/0x20
[<ffffffffb94564d0>] dev_queue_xmit+0x10/0x20
[<ffffffffb946312d>] neigh_resolve_output+0x11d/0x220
[<ffffffffb94a3e9c>] ip_finish_output+0x27c/0x8d0
[<ffffffffb94a481b>] ip_output+0x7b/0xf0
[<ffffffffb94a3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb94a2247>] ip_local_out_sk+0x37/0x40
[<ffffffffb94a5286>] ip_send_skb+0x16/0x50
[<ffffffffb94cd65c>] udp_send_skb+0xac/0x2b0
[<ffffffffb94a1f80>] ? ip_copy_metadata+0x170/0x170
[<ffffffffb94cebc1>] udp_sendmsg+0x5e1/0xa70
[<ffffffffb903d2dd>] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
[<ffffffffb94dc389>] inet_sendmsg+0x69/0xb0
[<ffffffffb94333a6>] sock_sendmsg+0xb6/0xf0
[<ffffffffb949aacf>] ? __ip_route_output_key_hash+0x20f/0x7d0
[<ffffffffb9433ad1>] SYSC_sendto+0x121/0x1c0
[<ffffffffb94a779c>] ? ip_setsockopt+0x2c/0x90
[<ffffffffb94355ee>] SyS_sendto+0xe/0x10