Info

Permalink
Function
sch_direct_xmit
First occurrence
2020-11-03
Last occurrence
2020-11-03
State
NEW
Components
kernel
Quality
-72

Statistics

Operating system Count
CentOS 7 7
Architecture Count
x86_64 7
Related packages Count
kernel 7
 0:3.10.0-1127.13.1.el7 4
 0:3.10.0-1160.el7 2
 0:3.10.0-957.el7 1

Packages names with count less than the total count of reports are most probably not the cause of the problem.

History

Daily:

Weekly:

Monthly:

Report backtrace


Complete report #37527
Frame # Function Binary Source or offset Line
1
sch_direct_xmit
vmlinux 0xffffffffb128131a
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x11a
2
__dev_queue_xmit
vmlinux 0xffffffffb1256301
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x491
3
dev_queue_xmit
vmlinux 0xffffffffb12564d0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x10
4
ip_finish_output
vmlinux 0xffffffffb12a429b
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x67b
5
ip_output
vmlinux 0xffffffffb12a481b
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x7b
6
? __ip_append_data.isra.50
vmlinux 0xffffffffb12a3c20
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xa60
7
ip_local_out_sk
vmlinux 0xffffffffb12a2247
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x37
8
ip_queue_xmit
vmlinux 0xffffffffb12a25e4
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x144
9
tcp_transmit_skb
vmlinux 0xffffffffb12bccd4
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x4e4
10
tcp_send_ack
vmlinux 0xffffffffb12beedf
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x11f
11
__tcp_ack_snd_check
vmlinux 0xffffffffb12b3abb
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x6b
12
tcp_rcv_established
vmlinux 0xffffffffb12b95e4
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x264
13
tcp_v4_do_rcv
vmlinux 0xffffffffb12c46ca
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x10a
14
tcp_v4_rcv
vmlinux 0xffffffffb12c5ddc
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x7dc
15
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc06c12ce
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x4e
16
ip_local_deliver_finish
vmlinux 0xffffffffb129e0fd
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xbd
17
ip_local_deliver
vmlinux 0xffffffffb129e3f0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x60
18
? ip_rcv_finish
vmlinux 0xffffffffb129e040
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x370
19
ip_rcv_finish
vmlinux 0xffffffffb129dd60
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x90
20
ip_rcv
vmlinux 0xffffffffb129e730
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x2c0
21
? inet_del_offload
vmlinux 0xffffffffb129dcd0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
22
__netif_receive_skb_core
vmlinux 0xffffffffb1253c19
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x729
23
__netif_receive_skb
vmlinux 0xffffffffb1253f18
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x18
24
netif_receive_skb_internal
vmlinux 0xffffffffb1253fa0
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x40
25
napi_gro_receive
vmlinux 0xffffffffb1254c28
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xd8
26
vmxnet3_rq_rx_complete
vmxnet3 0xffffffffc01f16b4
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x924
27
vmxnet3_poll_rx_only
vmxnet3 0xffffffffc01f20e6
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x36
28
net_rx_action
vmlinux 0xffffffffb12545bf
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x26f
29
__do_softirq
vmlinux 0xffffffffb0ca5695
Build id: 3.10.0-1127.13.1.el7.x86_64 		0xf5
30
call_softirq
vmlinux 0xffffffffb139642c
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x1c
31
do_softirq
vmlinux 0xffffffffb0c2f715
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x65
32
irq_exit
vmlinux 0xffffffffb0ca5a15
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x105
33
smp_apic_timer_interrupt
vmlinux 0xffffffffb13979c8
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x48
34
apic_timer_interrupt
vmlinux 0xffffffffb1393efa
Build id: 3.10.0-1127.13.1.el7.x86_64 		0x16a 
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 49s! [beam.smp:1972]
[<ffffffffb128131a>] sch_direct_xmit+0x11a/0x250
[<ffffffffb1256301>] __dev_queue_xmit+0x491/0x650
[<ffffffffb12564d0>] dev_queue_xmit+0x10/0x20
[<ffffffffb12a429b>] ip_finish_output+0x67b/0x8d0
[<ffffffffb12a481b>] ip_output+0x7b/0xf0
[<ffffffffb12a3c20>] ? __ip_append_data.isra.50+0xa60/0xa60
[<ffffffffb12a2247>] ip_local_out_sk+0x37/0x40
[<ffffffffb12a25e4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb12bccd4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb12beedf>] tcp_send_ack+0x11f/0x170
[<ffffffffb12b3abb>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffffb12b95e4>] tcp_rcv_established+0x264/0x770
[<ffffffffb12c46ca>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffffb12c5ddc>] tcp_v4_rcv+0x7dc/0x9e0
[<ffffffffc06c12ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffffb129e0fd>] ip_local_deliver_finish+0xbd/0x200
[<ffffffffb129e3f0>] ip_local_deliver+0x60/0xe0
[<ffffffffb129e040>] ? ip_rcv_finish+0x370/0x370
[<ffffffffb129dd60>] ip_rcv_finish+0x90/0x370
[<ffffffffb129e730>] ip_rcv+0x2c0/0x420
[<ffffffffb129dcd0>] ? inet_del_offload+0x40/0x40
[<ffffffffb1253c19>] __netif_receive_skb_core+0x729/0xa10
[<ffffffffb1253f18>] __netif_receive_skb+0x18/0x60
[<ffffffffb1253fa0>] netif_receive_skb_internal+0x40/0xc0
[<ffffffffb1254c28>] napi_gro_receive+0xd8/0x100
[<ffffffffc01f16b4>] vmxnet3_rq_rx_complete+0x924/0xef0 [vmxnet3]
[<ffffffffc01f20e6>] vmxnet3_poll_rx_only+0x36/0xa0 [vmxnet3]
[<ffffffffb12545bf>] net_rx_action+0x26f/0x390
[<ffffffffb0ca5695>] __do_softirq+0xf5/0x280
[<ffffffffb139642c>] call_softirq+0x1c/0x30
[<ffffffffb0c2f715>] do_softirq+0x65/0xa0
[<ffffffffb0ca5a15>] irq_exit+0x105/0x110
[<ffffffffb13979c8>] smp_apic_timer_interrupt+0x48/0x60
[<ffffffffb1393efa>] apic_timer_interrupt+0x16a/0x170
Complete report #12987
Frame # Function Binary Source or offset Line
1
dev_queue_xmit
vmlinux 0xffffffff8383bdb0
Build id: 3.10.0-957.el7.x86_64 		0x10
2
neigh_resolve_output
vmlinux 0xffffffff8384861d
Build id: 3.10.0-957.el7.x86_64 		0x11d
3
ip_finish_output
vmlinux 0xffffffff8388845c
Build id: 3.10.0-957.el7.x86_64 		0x27c
4
ip_output
vmlinux 0xffffffff83888c93
Build id: 3.10.0-957.el7.x86_64 		0x73
5
? __ip_append_data.isra.51
vmlinux 0xffffffff838881e0
Build id: 3.10.0-957.el7.x86_64 		0xa50
6
ip_local_out_sk
vmlinux 0xffffffff83886817
Build id: 3.10.0-957.el7.x86_64 		0x37
7
ip_queue_xmit
vmlinux 0xffffffff83886bb4
Build id: 3.10.0-957.el7.x86_64 		0x144
8
tcp_transmit_skb
vmlinux 0xffffffff838a0f64
Build id: 3.10.0-957.el7.x86_64 		0x4e4
9
tcp_send_ack
vmlinux 0xffffffff838a2fef
Build id: 3.10.0-957.el7.x86_64 		0x11f
10
__tcp_ack_snd_check
vmlinux 0xffffffff8389796b
Build id: 3.10.0-957.el7.x86_64 		0x6b
11
tcp_rcv_established
vmlinux 0xffffffff8389d944
Build id: 3.10.0-957.el7.x86_64 		0x264
12
tcp_v4_do_rcv
vmlinux 0xffffffff838a86da
Build id: 3.10.0-957.el7.x86_64 		0x10a
13
tcp_v4_rcv
vmlinux 0xffffffff838a9dcc
Build id: 3.10.0-957.el7.x86_64 		0x7bc
14
? ipv4_confirm
nf_conntrack_ipv4 0xffffffffc05b12ce
Build id: 3.10.0-957.el7.x86_64 		0x4e
15
ip_local_deliver_finish
vmlinux 0xffffffff8388285d
Build id: 3.10.0-957.el7.x86_64 		0xbd
16
ip_local_deliver
vmlinux 0xffffffff83882b49
Build id: 3.10.0-957.el7.x86_64 		0x59
17
? ip_rcv_finish
vmlinux 0xffffffff838827a0
Build id: 3.10.0-957.el7.x86_64 		0x370
18
ip_rcv_finish
vmlinux 0xffffffff838824c0
Build id: 3.10.0-957.el7.x86_64 		0x90
19
ip_rcv
vmlinux 0xffffffff83882e79
Build id: 3.10.0-957.el7.x86_64 		0x2b9
20
? inet_del_offload
vmlinux 0xffffffff83882430
Build id: 3.10.0-957.el7.x86_64 		0x40
21
__netif_receive_skb_core
vmlinux 0xffffffff838394f9
Build id: 3.10.0-957.el7.x86_64 		0x729
22
__netif_receive_skb
vmlinux 0xffffffff838397f8
Build id: 3.10.0-957.el7.x86_64 		0x18
23
netif_receive_skb_internal
vmlinux 0xffffffff83839880
Build id: 3.10.0-957.el7.x86_64 		0x40
24
napi_gro_receive
vmlinux 0xffffffff8383a508
Build id: 3.10.0-957.el7.x86_64 		0xd8
25
e1000_clean_rx_irq
e1000 0xffffffffc006543b
Build id: 3.10.0-957.el7.x86_64 		0x2bb
26
e1000_clean
e1000 0xffffffffc00660dd
Build id: 3.10.0-957.el7.x86_64 		0x27d
27
? __queue_work
vmlinux 0xffffffff832b7c0c
Build id: 3.10.0-957.el7.x86_64 		0x13c
28
net_rx_action
vmlinux 0xffffffff83839e9f
Build id: 3.10.0-957.el7.x86_64 		0x26f
29
__do_softirq
vmlinux 0xffffffff832a0f05
Build id: 3.10.0-957.el7.x86_64 		0xf5
30
call_softirq
vmlinux 0xffffffff8397832c
Build id: 3.10.0-957.el7.x86_64 		0x1c
31
do_softirq
vmlinux 0xffffffff8322e675
Build id: 3.10.0-957.el7.x86_64 		0x65
32
irq_exit
vmlinux 0xffffffff832a1285
Build id: 3.10.0-957.el7.x86_64 		0x105
33
smp_apic_timer_interrupt
vmlinux 0xffffffff839796c8
Build id: 3.10.0-957.el7.x86_64 		0x48
34
apic_timer_interrupt
vmlinux 0xffffffff83975df2
Build id: 3.10.0-957.el7.x86_64 		0x162 
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 27s! [kworker/u256:1:62319]
[<ffffffff8383bdb0>] dev_queue_xmit+0x10/0x20
[<ffffffff8384861d>] neigh_resolve_output+0x11d/0x220
[<ffffffff8388845c>] ip_finish_output+0x27c/0x7b0
[<ffffffff83888c93>] ip_output+0x73/0xe0
[<ffffffff838881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffff83886817>] ip_local_out_sk+0x37/0x40
[<ffffffff83886bb4>] ip_queue_xmit+0x144/0x3c0
[<ffffffff838a0f64>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff838a2fef>] tcp_send_ack+0x11f/0x170
[<ffffffff8389796b>] __tcp_ack_snd_check+0x6b/0x90
[<ffffffff8389d944>] tcp_rcv_established+0x264/0x770
[<ffffffff838a86da>] tcp_v4_do_rcv+0x10a/0x350
[<ffffffff838a9dcc>] tcp_v4_rcv+0x7bc/0x9c0
[<ffffffffc05b12ce>] ? ipv4_confirm+0x4e/0x100 [nf_conntrack_ipv4]
[<ffffffff8388285d>] ip_local_deliver_finish+0xbd/0x200
[<ffffffff83882b49>] ip_local_deliver+0x59/0xd0
[<ffffffff838827a0>] ? ip_rcv_finish+0x370/0x370
[<ffffffff838824c0>] ip_rcv_finish+0x90/0x370
[<ffffffff83882e79>] ip_rcv+0x2b9/0x410
[<ffffffff83882430>] ? inet_del_offload+0x40/0x40
[<ffffffff838394f9>] __netif_receive_skb_core+0x729/0xa10
[<ffffffff838397f8>] __netif_receive_skb+0x18/0x60
[<ffffffff83839880>] netif_receive_skb_internal+0x40/0xc0
[<ffffffff8383a508>] napi_gro_receive+0xd8/0x100
[<ffffffffc006543b>] e1000_clean_rx_irq+0x2bb/0x510 [e1000]
[<ffffffffc00660dd>] e1000_clean+0x27d/0x8e0 [e1000]
[<ffffffff832b7c0c>] ? __queue_work+0x13c/0x3e0
[<ffffffff83839e9f>] net_rx_action+0x26f/0x390
[<ffffffff832a0f05>] __do_softirq+0xf5/0x280
[<ffffffff8397832c>] call_softirq+0x1c/0x30
[<ffffffff8322e675>] do_softirq+0x65/0xa0
[<ffffffff832a1285>] irq_exit+0x105/0x110
[<ffffffff839796c8>] smp_apic_timer_interrupt+0x48/0x60
[<ffffffff83975df2>] apic_timer_interrupt+0x162/0x170