Complete report #1114344
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc0089ee7
Build id: 3.10.0-693.el7.x86_64
|
0x947
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffff81586771
Build id: 3.10.0-693.el7.x86_64
|
0x171
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffff815af50a
Build id: 3.10.0-693.el7.x86_64
|
0x11a
|
|
4
|
__dev_queue_xmit
|
vmlinux |
0xffffffff81589566
Build id: 3.10.0-693.el7.x86_64
|
0x226
|
|
5
|
dev_queue_xmit
|
vmlinux |
0xffffffff815898a0
Build id: 3.10.0-693.el7.x86_64
|
0x10
|
|
6
|
neigh_resolve_output
|
vmlinux |
0xffffffff8159527b
Build id: 3.10.0-693.el7.x86_64
|
0x11b
|
|
7
|
ip_finish_output
|
vmlinux |
0xffffffff815cfaa7
Build id: 3.10.0-693.el7.x86_64
|
0x297
|
|
8
|
ip_output
|
vmlinux |
0xffffffff815d0293
Build id: 3.10.0-693.el7.x86_64
|
0x73
|
|
9
|
?
__ip_append_data.isra.48
|
vmlinux |
0xffffffff815cf810
Build id: 3.10.0-693.el7.x86_64
|
0xa00
|
|
10
|
ip_local_out_sk
|
vmlinux |
0xffffffff815cdf31
Build id: 3.10.0-693.el7.x86_64
|
0x31
|
|
11
|
ip_queue_xmit
|
vmlinux |
0xffffffff815ce2a3
Build id: 3.10.0-693.el7.x86_64
|
0x143
|
|
12
|
tcp_transmit_skb
|
vmlinux |
0xffffffff815e7c0f
Build id: 3.10.0-693.el7.x86_64
|
0x4cf
|
|
13
|
tcp_send_ack
|
vmlinux |
0xffffffff815e9c2c
Build id: 3.10.0-693.el7.x86_64
|
0x11c
|
|
14
|
__tcp_ack_snd_check
|
vmlinux |
0xffffffff815defab
Build id: 3.10.0-693.el7.x86_64
|
0x6b
|
|
15
|
tcp_rcv_established
|
vmlinux |
0xffffffff815e4694
Build id: 3.10.0-693.el7.x86_64
|
0x264
|
|
16
|
|
vmlinux |
0xffffffff816a8f8d
Build id: 3.10.0-693.el7.x86_64
|
0x39d
|
|
17
|
tcp_v4_do_rcv
|
vmlinux |
0xffffffff815ef15a
Build id: 3.10.0-693.el7.x86_64
|
0x10a
|
|
18
|
release_sock
|
vmlinux |
0xffffffff8156e921
Build id: 3.10.0-693.el7.x86_64
|
0xa1
|
|
19
|
tcp_sendmsg
|
vmlinux |
0xffffffff815db106
Build id: 3.10.0-693.el7.x86_64
|
0xd6
|
|
20
|
inet_sendmsg
|
vmlinux |
0xffffffff81606893
Build id: 3.10.0-693.el7.x86_64
|
0x63
|
|
21
|
sock_aio_write
|
vmlinux |
0xffffffff815697c7
Build id: 3.10.0-693.el7.x86_64
|
0x157
|
|
22
|
do_sync_write
|
vmlinux |
0xffffffff8120026d
Build id: 3.10.0-693.el7.x86_64
|
0x8d
|
|
23
|
vfs_write
|
vmlinux |
0xffffffff81200e25
Build id: 3.10.0-693.el7.x86_64
|
0x1b5
|
|
24
|
SyS_write
|
vmlinux |
0xffffffff81201b3f
Build id: 3.10.0-693.el7.x86_64
|
0x7f
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 31s! [smtpd:31237]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter nls_utf8 isofs ext4 mbcache jbd2 pcspkr sg ppdev joydev shpchp parport_pc parport i2c_piix4 virtio_balloon nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sd_mod crc_t10dif sr_mod cdrom crct10dif_generic crct10dif_common ata_generic pata_acpi virtio_scsi bochs_drm drm_kms_helper
syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata e1000 serio_raw virtio_pci virtio_ring virtio i2c_core floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 31237 Comm: smtpd Tainted: G L ------------ 3.10.0-693.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.10.2-0-g5f4c7b1-prebuilt.qemu-project.org 04/01/2014
task: ffff8800b7d14f10 ti: ffff880029d4c000 task.ti: ffff880029d4c000
RIP: 0010:[<ffffffffc0089ee7>] [<ffffffffc0089ee7>] e1000_xmit_frame+0x947/0x1040 [e1000]
RSP: 0018:ffff880029d4f7f0 EFLAGS: 00000286
RAX: ffffc90000783818 RBX: 0000000000000202 RCX: 00000000000000cc
RDX: 00000000000000cc RSI: ffff880035ec3240 RDI: ffff8801387da000
RBP: ffff880029d4f870 R08: 0000000000000100 R09: 0000000022100000
R10: 0000000080000000 R11: 0000000000000000 R12: 00000001ffffffff
R13: ffffffff813492f0 R14: ffff880029d4f7b8 R15: 0000000000000202
FS: 00007fe8688ca840(0000) GS:ffff88013fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000556e36d298d0 CR3: 00000000a695a000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8800d2ec5cf7 ffff880139563098 000000ca00000202 0000000900000000
00000000382a2700 ffff880000000000 ffffffff00000000 0000000100001000
ffff8800161a0000 ffff8801387da000 ffff880139563000 ffff8801387da000
Call Trace:
[<ffffffff81586771>] dev_hard_start_xmit+0x171/0x3b0
[<ffffffff815af50a>] sch_direct_xmit+0x11a/0x240
[<ffffffff81589566>] __dev_queue_xmit+0x226/0x550
[<ffffffff815898a0>] dev_queue_xmit+0x10/0x20
[<ffffffff8159527b>] neigh_resolve_output+0x11b/0x220
[<ffffffff815cfaa7>] ip_finish_output+0x297/0x780
[<ffffffff815d0293>] ip_output+0x73/0xe0
[<ffffffff815cf810>] ? __ip_append_data.isra.48+0xa00/0xa00
[<ffffffff815cdf31>] ip_local_out_sk+0x31/0x40
[<ffffffff815ce2a3>] ip_queue_xmit+0x143/0x3a0
[<ffffffff815e7c0f>] tcp_transmit_skb+0x4cf/0x9b0
[<ffffffff815e9c2c>] tcp_send_ack+0x11c/0x170
[<ffffffff815defab>] __tcp_ack_snd_check+0x6b/0xa0
[<ffffffff815e4694>] tcp_rcv_established+0x264/0x760
[<ffffffff816a8f8d>] ? __schedule+0x39d/0x8b0
[<ffffffff815ef15a>] tcp_v4_do_rcv+0x10a/0x340
[<ffffffff8156e921>] release_sock+0xa1/0x170
[<ffffffff815db106>] tcp_sendmsg+0xd6/0xc30
[<ffffffff81606893>] inet_sendmsg+0x63/0xb0
[<ffffffff815697c7>] sock_aio_write+0x157/0x180
[<ffffffff8120026d>] do_sync_write+0x8d/0xd0
[<ffffffff81200e25>] vfs_write+0x1b5/0x1e0
[<ffffffff81201b3f>] SyS_write+0x7f/0xe0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00
Complete report #785993
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
e1000_xmit_frame
|
e1000 |
0xffffffffc06f0857
Build id: 3.10.0-957.el7.x86_64
|
0x7f7
|
|
2
|
dev_hard_start_xmit
|
vmlinux |
0xffffffff95838c66
Build id: 3.10.0-957.el7.x86_64
|
0x246
|
|
3
|
sch_direct_xmit
|
vmlinux |
0xffffffff95865cea
Build id: 3.10.0-957.el7.x86_64
|
0x11a
|
|
4
|
__qdisc_run
|
vmlinux |
0xffffffff95865eae
Build id: 3.10.0-957.el7.x86_64
|
0x8e
|
|
5
|
__dev_queue_xmit
|
vmlinux |
0xffffffff9583bbf9
Build id: 3.10.0-957.el7.x86_64
|
0x4a9
|
|
6
|
dev_queue_xmit
|
vmlinux |
0xffffffff9583bdb0
Build id: 3.10.0-957.el7.x86_64
|
0x10
|
|
7
|
neigh_resolve_output
|
vmlinux |
0xffffffff9584861d
Build id: 3.10.0-957.el7.x86_64
|
0x11d
|
|
8
|
ip_finish_output
|
vmlinux |
0xffffffff9588845c
Build id: 3.10.0-957.el7.x86_64
|
0x27c
|
|
9
|
ip_output
|
vmlinux |
0xffffffff95888c93
Build id: 3.10.0-957.el7.x86_64
|
0x73
|
|
10
|
?
__ip_append_data.isra.51
|
vmlinux |
0xffffffff958881e0
Build id: 3.10.0-957.el7.x86_64
|
0xa50
|
|
11
|
ip_local_out_sk
|
vmlinux |
0xffffffff95886817
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
12
|
ip_queue_xmit
|
vmlinux |
0xffffffff95886bb4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
13
|
tcp_transmit_skb
|
vmlinux |
0xffffffff958a0f64
Build id: 3.10.0-957.el7.x86_64
|
0x4e4
|
|
14
|
tcp_send_ack
|
vmlinux |
0xffffffff958a2fef
Build id: 3.10.0-957.el7.x86_64
|
0x11f
|
|
15
|
__tcp_ack_snd_check
|
vmlinux |
0xffffffff9589796b
Build id: 3.10.0-957.el7.x86_64
|
0x6b
|
|
16
|
tcp_rcv_established
|
vmlinux |
0xffffffff9589d944
Build id: 3.10.0-957.el7.x86_64
|
0x264
|
|
17
|
tcp_v4_do_rcv
|
vmlinux |
0xffffffff958a86da
Build id: 3.10.0-957.el7.x86_64
|
0x10a
|
|
18
|
release_sock
|
vmlinux |
0xffffffff9581e906
Build id: 3.10.0-957.el7.x86_64
|
0xa6
|
|
19
|
tcp_sendmsg
|
vmlinux |
0xffffffff958940a0
Build id: 3.10.0-957.el7.x86_64
|
0xe0
|
|
20
|
inet_sendmsg
|
vmlinux |
0xffffffff958bfeb9
Build id: 3.10.0-957.el7.x86_64
|
0x69
|
|
21
|
sock_aio_write
|
vmlinux |
0xffffffff9581857d
Build id: 3.10.0-957.el7.x86_64
|
0x15d
|
|
22
|
do_sync_write
|
vmlinux |
0xffffffff954405b3
Build id: 3.10.0-957.el7.x86_64
|
0x93
|
|
23
|
vfs_write
|
vmlinux |
0xffffffff954411a5
Build id: 3.10.0-957.el7.x86_64
|
0x1c5
|
|
24
|
SyS_write
|
vmlinux |
0xffffffff95441ebf
Build id: 3.10.0-957.el7.x86_64
|
0x7f
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffff95974ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 21s! [nginx:26509]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ext4 mbcache jbd2 crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd vmw_balloon e1000 joydev pcspkr sg nfit i2c_piix4 vmw_vmci libnvdimm parport_pc parport binfmt_misc ip_tables xfs libcrc32c sd_mod sr_mod crc_t10dif crct10dif_generic cdrom crct10dif_pclmul crct10dif_common crc32c_intel serio_raw ata_generic pata_acpi vmwgfx floppy drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi scsi_transport_spi
mptscsih mptbase drm_panel_orientation_quirks ata_piix libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 3 PID: 26509 Comm: nginx Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/13/2018
task: ffff958a24048000 ti: ffff958a719f4000 task.ti: ffff958a719f4000
RIP: 0010:[<ffffffffc06f0857>] [<ffffffffc06f0857>] e1000_xmit_frame+0x7f7/0x1120 [e1000]
RSP: 0000:ffff958a719f7738 EFLAGS: 00000286
RAX: ffffaafb80f83818 RBX: 0000000000000042 RCX: 0000000000000068
RDX: 0000000000000068 RSI: ffff958b3961ccc0 RDI: ffff958b37a2a000
RBP: ffff958a719f77b8 R08: 0000000000000100 R09: 0000000022100000
R10: 3400004500087cd2 R11: 0000000000000000 R12: 00000001366438ce
R13: 0000000135234cc0 R14: ffffffff955a04b0 R15: ffff958a719f76f0
FS: 00007f7190ce9740(0000) GS:ffff958b3fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff0e1e39000 CR3: 0000000094166000 CR4: 00000000003607e0
Call Trace:
[<ffffffff95838c66>] dev_hard_start_xmit+0x246/0x3b0
[<ffffffff95865cea>] sch_direct_xmit+0x11a/0x250
[<ffffffff95865eae>] __qdisc_run+0x8e/0x360
[<ffffffff9583bbf9>] __dev_queue_xmit+0x4a9/0x650
[<ffffffff9583bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffff9584861d>] neigh_resolve_output+0x11d/0x220
[<ffffffff9588845c>] ip_finish_output+0x27c/0x7b0
[<ffffffff95888c93>] ip_output+0x73/0xe0
[<ffffffff958881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffff95886817>] ip_local_out_sk+0x37/0x40
[<ffffffff95886bb4>] ip_queue_xmit+0x144/0x3c0
[<ffffffff958a0f64>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff958a2fef>] tcp_send_ack+0x11f/0x170
[<ffffffff9589796b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffff9589d944>] tcp_rcv_established+0x264/0x770
[<ffffffff958a86da>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffff9581e906>] release_sock+0xa6/0x170
[<ffffffff958940a0>] tcp_sendmsg+0xe0/0xc60
[<ffffffff958bfeb9>] inet_sendmsg+0x69/0xb0
[<ffffffff9581857d>] sock_aio_write+0x15d/0x180
[<ffffffff954405b3>] do_sync_write+0x93/0xe0
[<ffffffff954411a5>] vfs_write+0x1c5/0x1f0
[<ffffffff95441ebf>] SyS_write+0x7f/0xf0
[<ffffffff95974ddb>] system_call_fastpath+0x22/0x27
Code: 00 00 20 74 10 48 8b 83 00 03 00 00 f6 80 98 00 00 00 03 74 16 48 8b 5d c8 41 0f b7 47 2a 41 8b 57 18 48 03 83 90 0c 00 00 89 10 <48> 83 c4 58 31 c0 5b 41 5c 41 5d 41 5e 41 5f 5d c3 0f 1f 84 00