Complete report #1423270
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb2d8e9f5
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffffb26aecc4
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x74
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc0549e48
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x78
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc0552cc8
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x3b8
|
|
5
|
|
vmlinux |
0xffffffffb26a4207
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x17
|
|
6
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc054aeaf
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x17f
|
|
7
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc054c5ad
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x3cd
|
|
8
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc057d42f
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x4f
|
|
9
|
nf_iterate
|
vmlinux |
0xffffffffb2c99aa8
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x98
|
|
10
|
nf_hook_slow
|
vmlinux |
0xffffffffb2c99b98
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xa8
|
|
11
|
__ip_local_out_sk
|
vmlinux |
0xffffffffb2ca8bc2
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x102
|
|
12
|
|
vmlinux |
0xffffffffb2ca8c02
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x32
|
|
13
|
|
vmlinux |
0xffffffffb2ca81a0
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x1c0
|
|
14
|
ip_local_out_sk
|
vmlinux |
0xffffffffb2ca8beb
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x1b
|
|
15
|
ip_queue_xmit
|
vmlinux |
0xffffffffb2ca8fa4
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x144
|
|
16
|
tcp_transmit_skb
|
vmlinux |
0xffffffffb2cc3724
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x4e4
|
|
17
|
tcp_write_xmit
|
vmlinux |
0xffffffffb2cc3daa
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x18a
|
|
18
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffffb2cc4bee
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x2e
|
|
19
|
tcp_push
|
vmlinux |
0xffffffffb2cb2bfc
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xec
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffffb2cb6642
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xd2
|
|
21
|
inet_sendmsg
|
vmlinux |
0xffffffffb2ce2e39
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x69
|
|
22
|
sock_sendmsg
|
vmlinux |
0xffffffffb2c39396
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xb6
|
|
23
|
|
vmlinux |
0xffffffffb26d7579
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x19
|
|
24
|
SYSC_sendto
|
vmlinux |
0xffffffffb2c39ac1
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x121
|
|
25
|
|
vmlinux |
0xffffffffb28638c0
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x3a0
|
|
26
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb2d99ed5
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xa2
|
|
27
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb2d99ec9
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0x96
|
|
28
|
SyS_sendto
|
vmlinux |
0xffffffffb2c3b5de
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xe
|
|
29
|
tracesys
|
vmlinux |
0xffffffffb2d9a226
Build id: 3.10.0-1160.66.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 30s! [datax-web, admi:7789]
Modules linked in: veth xt_nat nf_conntrack_netlink br_netfilter xt_CHECKSUM iptable_mangle tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables overlay(T) ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack vmw_vsock_vmci_transport vsock sunrpc coretemp crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg ppdev pcspkr joydev vmw_balloon parport_pc parport vmw_vmci i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod vmwgfx crc_t10dif crct10dif_generic drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptsas scsi_transport_sas crct10dif_pclmul crct10dif_common
libata mptscsih crc32c_intel serio_raw e1000 mptbase drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 4 PID: 7789 Comm: datax-web, admi Kdump: loaded Tainted: G W ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 06/22/2012
task: ffff92ef7eafb180 ti: ffff92eeaf6a0000 task.ti: ffff92eeaf6a0000
RIP: 0010:[<ffffffffb2d8e9f5>] [<ffffffffb2d8e9f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff92eeaf6a3798 EFLAGS: 00000282
RAX: 000000000001acc0 RBX: ffff92f02fd13940 RCX: ffff92f02fd00000
RDX: 0000000000000004 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff92eeaf6a3798 R08: ffff92f42aeec6d0 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000003 R12: ffff92eeaf6a37d8
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000003
FS: 00007f8621070b20(0000) GS:ffff92f02fd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fec6fa38000 CR3: 00000007f6fd4000 CR4: 00000000000007e0
Call Trace:
[<ffffffffb26aecc4>] mod_timer_pending+0x74/0x1a0
[<ffffffffc0549e48>] __nf_ct_refresh_acct+0x78/0xd0 [nf_conntrack]
[<ffffffffc0552cc8>] tcp_packet+0x3b8/0xa60 [nf_conntrack]
[<ffffffffb26a4207>] ? local_bh_enable+0x17/0x20
[<ffffffffc054aeaf>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc054c5ad>] nf_conntrack_in+0x3cd/0x670 [nf_conntrack]
[<ffffffffc057d42f>] ipv4_conntrack_local+0x4f/0x60 [nf_conntrack_ipv4]
[<ffffffffb2c99aa8>] nf_iterate+0x98/0xe0
[<ffffffffb2c99b98>] nf_hook_slow+0xa8/0x110
[<ffffffffb2ca8bc2>] __ip_local_out_sk+0x102/0x110
[<ffffffffb2ca8c02>] ? ip_local_out_sk+0x32/0x40
[<ffffffffb2ca81a0>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffffb2ca8beb>] ip_local_out_sk+0x1b/0x40
[<ffffffffb2ca8fa4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb2cc3724>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb2cc3daa>] tcp_write_xmit+0x18a/0xd40
[<ffffffffb2cc4bee>] __tcp_push_pending_frames+0x2e/0xc0
[<ffffffffb2cb2bfc>] tcp_push+0xec/0x120
[<ffffffffb2cb6642>] tcp_sendmsg+0xd2/0xc60
[<ffffffffb2ce2e39>] inet_sendmsg+0x69/0xb0
[<ffffffffb2c39396>] sock_sendmsg+0xb6/0xf0
[<ffffffffb26d7579>] ? ttwu_do_wakeup+0x19/0xe0
[<ffffffffb2c39ac1>] SYSC_sendto+0x121/0x1c0
[<ffffffffb28638c0>] ? do_vfs_ioctl+0x3a0/0x5b0
[<ffffffffb2d99ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffb2d99ec9>] ? system_call_after_swapgs+0x96/0x13a
[<ffffffffb2c3b5de>] SyS_sendto+0xe/0x10
[<ffffffffb2d9a226>] tracesys+0xa6/0xcc
Code: 07 00 66 66 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 c6 07 00 66 66 66 90 48 89 f7 57 9d <66> 66 90 66 90 5d c3 0f 1f 40 00 66 66 66 66 90 55 48 89 e5 48
Complete report #981493
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff8b3ba385
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffff8acb3994
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x74
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc07fdee0
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x80
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc0807157
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x3e7
|
|
5
|
|
vmlinux |
0xffffffff8aca89b7
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x17
|
|
6
|
|
vmlinux |
0xffffffff8aca89b7
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x17
|
|
7
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc07fef7f
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x17f
|
|
8
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc080069d
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x3cd
|
|
9
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc0830467
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x57
|
|
10
|
nf_iterate
|
vmlinux |
0xffffffff8b2c0ef8
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x98
|
|
11
|
nf_hook_slow
|
vmlinux |
0xffffffff8b2c0fe8
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xa8
|
|
12
|
__ip_local_out_sk
|
vmlinux |
0xffffffff8b2d0542
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x102
|
|
13
|
|
vmlinux |
0xffffffff8b2cfb10
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x1c0
|
|
14
|
ip_local_out_sk
|
vmlinux |
0xffffffff8b2d056b
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x1b
|
|
15
|
ip_queue_xmit
|
vmlinux |
0xffffffff8b2d0924
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x144
|
|
16
|
tcp_transmit_skb
|
vmlinux |
0xffffffff8b2eb5f4
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x4e4
|
|
17
|
tcp_write_xmit
|
vmlinux |
0xffffffff8b2ebc7a
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x18a
|
|
18
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffff8b2ecace
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x2e
|
|
19
|
tcp_push
|
vmlinux |
0xffffffff8b2da7cc
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xec
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffff8b2de272
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xd2
|
|
21
|
|
vmlinux |
0xffffffff8aecbadd
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xed
|
|
22
|
inet_sendmsg
|
vmlinux |
0xffffffff8b30b4e9
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x69
|
|
23
|
|
vmlinux |
0xffffffff8af1f8e3
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x23
|
|
24
|
sock_sendmsg
|
vmlinux |
0xffffffff8b25e4f6
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xb6
|
|
25
|
?
xfs_file_buffered_aio_write
|
xfs |
0xffffffffc051a0ac
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x29c
|
|
26
|
SYSC_sendto
|
vmlinux |
0xffffffff8b25ec51
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x121
|
|
27
|
|
vmlinux |
0xffffffff8ae719a8
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x3a8
|
|
28
|
SyS_sendto
|
vmlinux |
0xffffffff8b2607ee
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xe
|
|
29
|
tracesys
|
vmlinux |
0xffffffff8b3c562e
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 8065s! [com.alibaba.nac:3804]
Modules linked in: xt_nat veth nf_conntrack_netlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc snd_intel8x0 crc32_pclmul snd_ac97_codec ghash_clmulni_intel ac97_bus joydev ppdev snd_seq snd_seq_device aesni_intel snd_pcm lrw gf128mul glue_helper ablk_helper cryptd vboxguest(OE) snd_timer pcspkr sg snd video parport_pc
soundcore parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx serio_raw drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ahci ttm libahci crct10dif_pclmul crct10dif_common crc32c_intel ata_piix drm e1000 libata drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 3804 Comm: com.alibaba.nac Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-1160.114.2.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff9d78d08ab180 ti: ffff9d78d0f94000 task.ti: ffff9d78d0f94000
RIP: 0010:[<ffffffff8b3ba385>] [<ffffffff8b3ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9d78d0f97778 EFLAGS: 00010282
RAX: 0000000000000001 RBX: ffff9d78d9f37b00 RCX: ffff9d78d9d80000
RDX: 0000000000000003 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9d78d0f97778 R08: ffff9d78d9d95298 R09: 0000000000000090
R10: 0000000000000000 R11: 0000000000000001 R12: ffff9d78d0f97720
R13: ffff9d78d9c93940 R14: 0000000000000046 R15: ffff9d78d0f976e0
FS: 00007f2671fe9700(0000) GS:ffff9d78d9d80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa318d87d70 CR3: 00000000cc12a000 CR4: 00000000000206e0
Call Trace:
[<ffffffff8acb3994>] mod_timer_pending+0x74/0x1a0
[<ffffffffc07fdee0>] __nf_ct_refresh_acct+0x80/0xe0 [nf_conntrack]
[<ffffffffc0807157>] tcp_packet+0x3e7/0xc40 [nf_conntrack]
[<ffffffff8aca89b7>] ? local_bh_enable+0x17/0x20
[<ffffffff8aca89b7>] ? local_bh_enable+0x17/0x20
[<ffffffffc07fef7f>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc080069d>] nf_conntrack_in+0x3cd/0x670 [nf_conntrack]
[<ffffffffc0830467>] ipv4_conntrack_local+0x57/0x60 [nf_conntrack_ipv4]
[<ffffffff8b2c0ef8>] nf_iterate+0x98/0xe0
[<ffffffff8b2c0fe8>] nf_hook_slow+0xa8/0x110
[<ffffffff8b2d0542>] __ip_local_out_sk+0x102/0x110
[<ffffffff8b2cfb10>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffff8b2d056b>] ip_local_out_sk+0x1b/0x40
[<ffffffff8b2d0924>] ip_queue_xmit+0x144/0x3d0
[<ffffffff8b2eb5f4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff8b2ebc7a>] tcp_write_xmit+0x18a/0xd40
[<ffffffff8b2ecace>] __tcp_push_pending_frames+0x2e/0xd0
[<ffffffff8b2da7cc>] tcp_push+0xec/0x120
[<ffffffff8b2de272>] tcp_sendmsg+0xd2/0xc60
[<ffffffff8aecbadd>] ? iomap_apply+0xed/0x160
[<ffffffff8b30b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffff8af1f8e3>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffff8b25e4f6>] sock_sendmsg+0xb6/0x100
[<ffffffffc051a0ac>] ? xfs_file_buffered_aio_write+0x29c/0x2d0 [xfs]
[<ffffffff8b25ec51>] SYSC_sendto+0x121/0x1d0
[<ffffffff8ae719a8>] ? do_vfs_ioctl+0x3a8/0x5c0
[<ffffffff8b2607ee>] SyS_sendto+0xe/0x20
[<ffffffff8b3c562e>] tracesys+0xa6/0xcc
Code: aa 42 96 ff 66 90 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 8a 42 96 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #903006
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb65ba385
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffffb5eb3994
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x74
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc07caee0
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x80
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc07d4018
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x3b8
|
|
5
|
|
ip_tables |
0xffffffffc027a761
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x361
|
|
6
|
|
vmlinux |
0xffffffffb5ea89b7
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x17
|
|
7
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc07cbf7f
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x17f
|
|
8
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc07cd69d
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x3cd
|
|
9
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc0869467
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x57
|
|
10
|
nf_iterate
|
vmlinux |
0xffffffffb64c0ef8
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x98
|
|
11
|
nf_hook_slow
|
vmlinux |
0xffffffffb64c0fe8
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0xa8
|
|
12
|
__ip_local_out_sk
|
vmlinux |
0xffffffffb64d0542
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x102
|
|
13
|
|
vmlinux |
0xffffffffb64cfb10
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x1c0
|
|
14
|
ip_local_out_sk
|
vmlinux |
0xffffffffb64d056b
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x1b
|
|
15
|
ip_queue_xmit
|
vmlinux |
0xffffffffb64d0924
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x144
|
|
16
|
tcp_transmit_skb
|
vmlinux |
0xffffffffb64eb5f4
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x4e4
|
|
17
|
tcp_write_xmit
|
vmlinux |
0xffffffffb64ebc7a
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x18a
|
|
18
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffffb64ecace
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x2e
|
|
19
|
tcp_push
|
vmlinux |
0xffffffffb64da7cc
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0xec
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffffb64de272
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0xd2
|
|
21
|
|
vmlinux |
0xffffffffb607864a
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x7a
|
|
22
|
inet_sendmsg
|
vmlinux |
0xffffffffb650b4e9
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x69
|
|
23
|
sock_aio_write
|
vmlinux |
0xffffffffb645d675
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x165
|
|
24
|
|
vmlinux |
0xffffffffb61aee08
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x48
|
|
25
|
do_sync_readv_writev
|
vmlinux |
0xffffffffb605b2bb
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x7b
|
|
26
|
do_readv_writev
|
vmlinux |
0xffffffffb605ce7e
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0xce
|
|
27
|
vfs_writev
|
vmlinux |
0xffffffffb605d0c5
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x35
|
|
28
|
SyS_writev
|
vmlinux |
0xffffffffb605d259
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x59
|
|
29
|
system_call_fastpath
|
vmlinux |
0xffffffffb65c539a
Build id: 3.10.0-1160.95.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 600s! [4_scheduler:1725]
Modules linked in: ipt_REJECT nf_reject_ipv4 xt_multiport ipt_rpfilter xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_mark xt_comment ip6table_filter ip6table_mangle iptable_mangle ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables nf_tables veth xt_nat xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat xt_addrtype iptable_filter br_netfilter bridge stp llc overlay(T) ip_vs nf_conntrack vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq sb_edac iosf_mbi btusb crc32_pclmul snd_seq_device ghash_clmulni_intel btrtl btbcm snd_pcm aesni_intel
lrw btintel gf128mul snd_timer glue_helper snd ablk_helper bluetooth soundcore cryptd ppdev joydev vmw_vmci i2c_piix4 rfkill sg pcspkr vmw_balloon parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common nfit crc32c_intel libata nvme serio_raw libnvdimm e1000 nvme_core drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod synosnap(OE)
CPU: 1 PID: 1725 Comm: 4_scheduler Kdump: loaded Tainted: G W OEL ------------ T 3.10.0-1160.95.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 02/27/2020
task: ffff9215ff9dd280 ti: ffff9215ffd48000 task.ti: ffff9215ffd48000
RIP: 0010:[<ffffffffb65ba385>] [<ffffffffb65ba385>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9215ffd4b728 EFLAGS: 00000282
RAX: 000000000001acc0 RBX: 0000000000000100 RCX: ffff921601e40000
RDX: 0000000000000001 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff9215ffd4b728 R08: ffff921601e54d98 R09: 000000000000000c
R10: 0000000000000001 R11: 0000000000000003 R12: 000000450008cdc9
R13: 0000000000000000 R14: 0000000000000042 R15: ffff9215ffd4b788
FS: 00007f28c25f7700(0000) GS:ffff921601e40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f63799a4000 CR3: 0000000601324000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb5eb3994>] mod_timer_pending+0x74/0x1a0
[<ffffffffc07caee0>] __nf_ct_refresh_acct+0x80/0xe0 [nf_conntrack]
[<ffffffffc07d4018>] tcp_packet+0x3b8/0xa70 [nf_conntrack]
[<ffffffffc027a761>] ? ipt_do_table+0x361/0x750 [ip_tables]
[<ffffffffb5ea89b7>] ? local_bh_enable+0x17/0x20
[<ffffffffc07cbf7f>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc07cd69d>] nf_conntrack_in+0x3cd/0x670 [nf_conntrack]
[<ffffffffc0869467>] ipv4_conntrack_local+0x57/0x60 [nf_conntrack_ipv4]
[<ffffffffb64c0ef8>] nf_iterate+0x98/0xe0
[<ffffffffb64c0fe8>] nf_hook_slow+0xa8/0x110
[<ffffffffb64d0542>] __ip_local_out_sk+0x102/0x110
[<ffffffffb64cfb10>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffffb64d056b>] ip_local_out_sk+0x1b/0x40
[<ffffffffb64d0924>] ip_queue_xmit+0x144/0x3d0
[<ffffffffb64eb5f4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb64ebc7a>] tcp_write_xmit+0x18a/0xd40
[<ffffffffb64ecace>] __tcp_push_pending_frames+0x2e/0xd0
[<ffffffffb64da7cc>] tcp_push+0xec/0x120
[<ffffffffb64de272>] tcp_sendmsg+0xd2/0xc60
[<ffffffffb607864a>] ? __d_lookup+0x7a/0x160
[<ffffffffb650b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffffb645d675>] sock_aio_write+0x165/0x180
[<ffffffffb61aee08>] ? lockref_put_or_lock+0x48/0x80
[<ffffffffb605b2bb>] do_sync_readv_writev+0x7b/0xd0
[<ffffffffb605ce7e>] do_readv_writev+0xce/0x270
[<ffffffffb605d0c5>] vfs_writev+0x35/0x70
[<ffffffffb605d259>] SyS_writev+0x59/0x100
[<ffffffffb65c539a>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 cc cc cc cc 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 cc cc cc cc 0f 1f 44 00 00 55 48 89 e5 48
Complete report #659090
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffb058e9f5
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffffafeaece4
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x74
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc07e2e48
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x78
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc07ebcc8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3b8
|
|
5
|
|
vmlinux |
0xffffffffafea4207
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x17
|
|
6
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc07e3eaf
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x17f
|
|
7
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc07e55ad
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x3cd
|
|
8
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc080942f
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4f
|
|
9
|
nf_iterate
|
vmlinux |
0xffffffffb0499aa8
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x98
|
|
10
|
nf_hook_slow
|
vmlinux |
0xffffffffb0499b98
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa8
|
|
11
|
__ip_local_out_sk
|
vmlinux |
0xffffffffb04a8bc2
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x102
|
|
12
|
|
vmlinux |
0xffffffffb04a81a0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1c0
|
|
13
|
ip_local_out_sk
|
vmlinux |
0xffffffffb04a8beb
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1b
|
|
14
|
ip_queue_xmit
|
vmlinux |
0xffffffffb04a8fa4
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x144
|
|
15
|
tcp_transmit_skb
|
vmlinux |
0xffffffffb04c3724
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x4e4
|
|
16
|
tcp_write_xmit
|
vmlinux |
0xffffffffb04c3daa
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x18a
|
|
17
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffffb04c4bee
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x2e
|
|
18
|
tcp_push
|
vmlinux |
0xffffffffb04b2bfc
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xec
|
|
19
|
tcp_sendmsg
|
vmlinux |
0xffffffffb04b6642
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xd2
|
|
20
|
|
vmlinux |
0xffffffffb043ea08
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x128
|
|
21
|
inet_sendmsg
|
vmlinux |
0xffffffffb04e2e39
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x69
|
|
22
|
|
vmlinux |
0xffffffffb04e2fe0
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x80
|
|
23
|
sock_aio_write
|
vmlinux |
0xffffffffb043856d
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x15d
|
|
24
|
do_sync_write
|
vmlinux |
0xffffffffb004da03
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x93
|
|
25
|
vfs_write
|
vmlinux |
0xffffffffb004e595
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x1c5
|
|
26
|
SyS_write
|
vmlinux |
0xffffffffb004f215
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0x55
|
|
27
|
tracesys
|
vmlinux |
0xffffffffb059a226
Build id: 3.10.0-1160.62.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 504s! [kube-apiserver:19702]
Modules linked in: dummy vport_vxlan openvswitch ip_set_hash_net xt_statistic nf_tables ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs ip_set_hash_ip xt_set ip_set ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_nat ip6table_filter ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables veth xt_comment vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc vboxsf(OE) binfmt_misc overlay(T) lockd grace sunrpc crc32_pclmul joydev ppdev ghash_clmulni_intel snd_intel8x0 snd_ac97_codec ac97_bus aesni_intel snd_seq lrw gf128mul snd_seq_device glue_helper ablk_helper cryptd snd_pcm
pcspkr sg snd_timer snd parport_pc vboxguest(OE) i2c_piix4 parport soundcore video ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm serio_raw ata_piix libahci crct10dif_pclmul crct10dif_common crc32c_intel libata e1000 drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 19702 Comm: kube-apiserver Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.62.1.el7.x86_64 #1
Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
task: ffff8a2bd5418000 ti: ffff8a28cf724000 task.ti: ffff8a28cf724000
RIP: 0010:[<ffffffffb058e9f5>] [<ffffffffb058e9f5>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff8a28cf7277f8 EFLAGS: 00000282
RAX: 0000000000000001 RBX: 0000000000000000 RCX: ffff8a2c1fc00000
RDX: 0000000000000000 RSI: 0000000000000282 RDI: 0000000000000282
RBP: ffff8a28cf7277f8 R08: ffffffffb0f06078 R09: 000000000000000c
R10: 0000000000000000 R11: 0000000000000003 R12: ffffffffb045802c
R13: ffff8a28cf727778 R14: ffff8a2c0e9cf8f8 R15: ffff8a2c0e9cf8f8
FS: 000000c00259c890(0000) GS:ffff8a2c1fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00001001dbb4fbe8 CR3: 00000003d3a48000 CR4: 00000000000606f0
Call Trace:
[<ffffffffafeaece4>] mod_timer_pending+0x74/0x1a0
[<ffffffffc07e2e48>] __nf_ct_refresh_acct+0x78/0xd0 [nf_conntrack]
[<ffffffffc07ebcc8>] tcp_packet+0x3b8/0xa60 [nf_conntrack]
[<ffffffffafea4207>] ? local_bh_enable+0x17/0x20
[<ffffffffc07e3eaf>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc07e55ad>] nf_conntrack_in+0x3cd/0x670 [nf_conntrack]
[<ffffffffc080942f>] ipv4_conntrack_local+0x4f/0x60 [nf_conntrack_ipv4]
[<ffffffffb0499aa8>] nf_iterate+0x98/0xe0
[<ffffffffb0499b98>] nf_hook_slow+0xa8/0x110
[<ffffffffb04a8bc2>] __ip_local_out_sk+0x102/0x110
[<ffffffffb04a81a0>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffffb04a8beb>] ip_local_out_sk+0x1b/0x40
[<ffffffffb04a8fa4>] ip_queue_xmit+0x144/0x3c0
[<ffffffffb04c3724>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffb04c3daa>] tcp_write_xmit+0x18a/0xd40
[<ffffffffb04c4bee>] __tcp_push_pending_frames+0x2e/0xc0
[<ffffffffb04b2bfc>] tcp_push+0xec/0x120
[<ffffffffb04b6642>] tcp_sendmsg+0xd2/0xc60
[<ffffffffb043ea08>] ? release_sock+0x128/0x180
[<ffffffffb04e2e39>] inet_sendmsg+0x69/0xb0
[<ffffffffb04e2fe0>] ? inet_recvmsg+0x80/0xb0
[<ffffffffb043856d>] sock_aio_write+0x15d/0x180
[<ffffffffb004da03>] do_sync_write+0x93/0xe0
[<ffffffffb004e595>] vfs_write+0x1c5/0x1f0
[<ffffffffb004f215>] SyS_write+0x55/0xd0
[<ffffffffb059a226>] tracesys+0xa6/0xcc
Code: f6 8b 98 ff 66 90 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 e8 d6 8b 98 ff 66 90 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #579366
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffff9398b495
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffff932aea94
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x74
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc065fe48
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x78
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc0668cc8
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3b8
|
|
5
|
|
vmlinux |
0xffffffff932a3fb7
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x17
|
|
6
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc0660eaf
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x17f
|
|
7
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc06625ad
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x3cd
|
|
8
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc058a42f
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x4f
|
|
9
|
nf_iterate
|
vmlinux |
0xffffffff93896ab8
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x98
|
|
10
|
nf_hook_slow
|
vmlinux |
0xffffffff93896ba8
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xa8
|
|
11
|
__ip_local_out_sk
|
vmlinux |
0xffffffff938a5052
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x102
|
|
12
|
|
vmlinux |
0xffffffff938a4630
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x1c0
|
|
13
|
ip_local_out_sk
|
vmlinux |
0xffffffff938a507b
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x1b
|
|
14
|
ip_queue_xmit
|
vmlinux |
0xffffffff938a5434
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x144
|
|
15
|
tcp_transmit_skb
|
vmlinux |
0xffffffff938bfb44
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x4e4
|
|
16
|
tcp_write_xmit
|
vmlinux |
0xffffffff938c01ca
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x18a
|
|
17
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffff938c100e
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x2e
|
|
18
|
tcp_push
|
vmlinux |
0xffffffff938af08c
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xec
|
|
19
|
tcp_sendmsg
|
vmlinux |
0xffffffff938b2ad2
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0xd2
|
|
20
|
inet_sendmsg
|
vmlinux |
0xffffffff938df259
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x69
|
|
21
|
sock_aio_write
|
vmlinux |
0xffffffff9383557d
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x15d
|
|
22
|
do_sync_write
|
vmlinux |
0xffffffff9344d1e3
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x93
|
|
23
|
vfs_write
|
vmlinux |
0xffffffff9344ddd5
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x1c5
|
|
24
|
SyS_write
|
vmlinux |
0xffffffff9344eaaf
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x7f
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffff93995f92
Build id: 3.10.0-1160.21.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 37s! [mgmailerd:27797]
Modules linked in: ipt_REJECT nf_reject_ipv4 binfmt_misc nfsv3 nfs_acl nfs lockd grace fscache xt_multiport nf_log_ipv4 nf_log_common xt_LOG nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c iptable_filter vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev vmw_balloon pcspkr joydev nfit sg libnvdimm vmw_vmci i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix crct10dif_pclmul crct10dif_common libata mptspi crc32c_intel scsi_transport_spi serio_raw mptscsih mptbase vmxnet3 drm_panel_orientation_quirks
floppy fuse
CPU: 7 PID: 27797 Comm: mgmailerd Tainted: G L ------------ 3.10.0-1160.21.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9c3e45048000 ti: ffff9c3e428a0000 task.ti: ffff9c3e428a0000
RIP: 0010:[<ffffffff9398b495>] [<ffffffff9398b495>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9c3e428a37f8 EFLAGS: 00000286
RAX: 000000000001acc0 RBX: 0002354a9cc04a3c RCX: ffff9c407fdc0000
RDX: 0000000000000007 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9c3e428a37f8 R08: ffff9c407fdd50c8 R09: 000000000000000c
R10: 0000000000000000 R11: 0000000000000003 R12: ffff9c3e428a37a0
R13: 0000000000000000 R14: 0000000000000001 R15: ffffffff934863fd
FS: 00007fe474762740(0000) GS:ffff9c407fdc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa9e2cd8000 CR3: 00000000037e4000 CR4: 00000000001607e0
Call Trace:
[<ffffffff932aea94>] mod_timer_pending+0x74/0x1a0
[<ffffffffc065fe48>] __nf_ct_refresh_acct+0x78/0xd0 [nf_conntrack]
[<ffffffffc0668cc8>] tcp_packet+0x3b8/0xa60 [nf_conntrack]
[<ffffffff932a3fb7>] ? local_bh_enable+0x17/0x20
[<ffffffffc0660eaf>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc06625ad>] nf_conntrack_in+0x3cd/0x670 [nf_conntrack]
[<ffffffffc058a42f>] ipv4_conntrack_local+0x4f/0x60 [nf_conntrack_ipv4]
[<ffffffff93896ab8>] nf_iterate+0x98/0xe0
[<ffffffff93896ba8>] nf_hook_slow+0xa8/0x110
[<ffffffff938a5052>] __ip_local_out_sk+0x102/0x110
[<ffffffff938a4630>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffff938a507b>] ip_local_out_sk+0x1b/0x40
[<ffffffff938a5434>] ip_queue_xmit+0x144/0x3c0
[<ffffffff938bfb44>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffff938c01ca>] tcp_write_xmit+0x18a/0xd40
[<ffffffff938c100e>] __tcp_push_pending_frames+0x2e/0xc0
[<ffffffff938af08c>] tcp_push+0xec/0x120
[<ffffffff938b2ad2>] tcp_sendmsg+0xd2/0xc60
[<ffffffff938df259>] inet_sendmsg+0x69/0xb0
[<ffffffff9383557d>] sock_aio_write+0x15d/0x180
[<ffffffff9344d1e3>] do_sync_write+0x93/0xe0
[<ffffffff9344ddd5>] vfs_write+0x1c5/0x1f0
[<ffffffff9344eaaf>] SyS_write+0x7f/0xf0
[<ffffffff93995f92>] system_call_fastpath+0x25/0x2a
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48
Complete report #222772
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
_raw_spin_unlock_irqrestore
|
vmlinux |
0xffffffffa7b15375
Build id: 3.10.0-862.el7.x86_64
|
0x15
|
|
2
|
mod_timer_pending
|
vmlinux |
0xffffffffa74a41e6
Build id: 3.10.0-862.el7.x86_64
|
0x76
|
|
3
|
__nf_ct_refresh_acct
|
nf_conntrack |
0xffffffffc062c960
Build id: 3.10.0-862.el7.x86_64
|
0xb0
|
|
4
|
tcp_packet
|
nf_conntrack |
0xffffffffc0635468
Build id: 3.10.0-862.el7.x86_64
|
0x3b8
|
|
5
|
|
ip_tables |
0xffffffffc00eea2e
Build id: 3.10.0-862.el7.x86_64
|
0x33e
|
|
6
|
|
vmlinux |
0xffffffffa7499df7
Build id: 3.10.0-862.el7.x86_64
|
0x17
|
|
7
|
?
__nf_conntrack_find_get
|
nf_conntrack |
0xffffffffc062d4ff
Build id: 3.10.0-862.el7.x86_64
|
0x17f
|
|
8
|
nf_conntrack_in
|
nf_conntrack |
0xffffffffc062f15d
Build id: 3.10.0-862.el7.x86_64
|
0x3bd
|
|
9
|
ipv4_conntrack_local
|
nf_conntrack_ipv4 |
0xffffffffc066566f
Build id: 3.10.0-862.el7.x86_64
|
0x4f
|
|
10
|
nf_iterate
|
vmlinux |
0xffffffffa7a273c8
Build id: 3.10.0-862.el7.x86_64
|
0x98
|
|
11
|
nf_hook_slow
|
vmlinux |
0xffffffffa7a274b8
Build id: 3.10.0-862.el7.x86_64
|
0xa8
|
|
12
|
__ip_local_out_sk
|
vmlinux |
0xffffffffa7a35526
Build id: 3.10.0-862.el7.x86_64
|
0xf6
|
|
13
|
|
vmlinux |
0xffffffffa7a34be0
Build id: 3.10.0-862.el7.x86_64
|
0x1c0
|
|
14
|
ip_local_out_sk
|
vmlinux |
0xffffffffa7a3554b
Build id: 3.10.0-862.el7.x86_64
|
0x1b
|
|
15
|
ip_queue_xmit
|
vmlinux |
0xffffffffa7a358d3
Build id: 3.10.0-862.el7.x86_64
|
0x143
|
|
16
|
tcp_transmit_skb
|
vmlinux |
0xffffffffa7a4f7b4
Build id: 3.10.0-862.el7.x86_64
|
0x4e4
|
|
17
|
tcp_write_xmit
|
vmlinux |
0xffffffffa7a4fe22
Build id: 3.10.0-862.el7.x86_64
|
0x172
|
|
18
|
__tcp_push_pending_frames
|
vmlinux |
0xffffffffa7a50c2e
Build id: 3.10.0-862.el7.x86_64
|
0x2e
|
|
19
|
tcp_push
|
vmlinux |
0xffffffffa7a3f12c
Build id: 3.10.0-862.el7.x86_64
|
0xec
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffffa7a42ae2
Build id: 3.10.0-862.el7.x86_64
|
0xd2
|
|
21
|
inet_sendmsg
|
vmlinux |
0xffffffffa7a6e5a9
Build id: 3.10.0-862.el7.x86_64
|
0x69
|
|
22
|
|
vmlinux |
0xffffffffa76d36b3
Build id: 3.10.0-862.el7.x86_64
|
0x23
|
|
23
|
sock_aio_write
|
vmlinux |
0xffffffffa79cb84d
Build id: 3.10.0-862.el7.x86_64
|
0x15d
|
|
24
|
|
vmlinux |
0xffffffffa74be870
Build id: 3.10.0-862.el7.x86_64
|
0x50
|
|
25
|
do_sync_write
|
vmlinux |
0xffffffffa761a1f3
Build id: 3.10.0-862.el7.x86_64
|
0x93
|
|
26
|
vfs_write
|
vmlinux |
0xffffffffa761add5
Build id: 3.10.0-862.el7.x86_64
|
0x1c5
|
|
27
|
SyS_write
|
vmlinux |
0xffffffffa761baff
Build id: 3.10.0-862.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffffa7b1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 4579s! [java:10532]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 iosf_mbi crc32_pclmul mbcache jbd2 ghash_clmulni_intel aesni_intel snd_ens1371 ppdev lrw gf128mul glue_helper snd_rawmidi ablk_helper cryptd snd_ac97_codec vmw_balloon ac97_bus snd_seq joydev snd_seq_device pcspkr
snd_pcm snd_timer snd sg soundcore nfit vmw_vmci libnvdimm i2c_piix4 shpchp parport_pc parport ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel mptspi scsi_transport_spi serio_raw mptscsih e1000 mptbase ata_piix libata i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 10532 Comm: java Not tainted 3.10.0-862.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/29/2019
task: ffff9e401c0b5ee0 ti: ffff9e3fc30c4000 task.ti: ffff9e3fc30c4000
RIP: 0010:[<ffffffffa7b15375>] [<ffffffffa7b15375>] _raw_spin_unlock_irqrestore+0x15/0x20
RSP: 0018:ffff9e3fc30c7828 EFLAGS: 00000286
RAX: 0000000104ca0348 RBX: 0000000000000100 RCX: ffff9e402ac2ca90
RDX: ffffffffa84c12a8 RSI: 0000000000000286 RDI: 0000000000000286
RBP: ffff9e3fc30c7828 R08: ffffffffa84c12a8 R09: 000000000000000c
R10: 0000000000000000 R11: 0000000000000003 R12: 0000000080000000
R13: 0000000000000000 R14: 00000000b09b2f46 R15: ffff9e3fc30c7888
FS: 00007fee95f6d700(0000) GS:ffff9e403b600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f835c919000 CR3: 0000000077968000 CR4: 00000000003607f0
Call Trace:
[<ffffffffa74a41e6>] mod_timer_pending+0x76/0x1a0
[<ffffffffc062c960>] __nf_ct_refresh_acct+0xb0/0xc0 [nf_conntrack]
[<ffffffffc0635468>] tcp_packet+0x3b8/0xa60 [nf_conntrack]
[<ffffffffc00eea2e>] ? ipt_do_table+0x33e/0x710 [ip_tables]
[<ffffffffa7499df7>] ? local_bh_enable+0x17/0x20
[<ffffffffc062d4ff>] ? __nf_conntrack_find_get+0x17f/0x370 [nf_conntrack]
[<ffffffffc062f15d>] nf_conntrack_in+0x3bd/0x660 [nf_conntrack]
[<ffffffffc066566f>] ipv4_conntrack_local+0x4f/0x60 [nf_conntrack_ipv4]
[<ffffffffa7a273c8>] nf_iterate+0x98/0xe0
[<ffffffffa7a274b8>] nf_hook_slow+0xa8/0x110
[<ffffffffa7a35526>] __ip_local_out_sk+0xf6/0x100
[<ffffffffa7a34be0>] ? ip_forward_options+0x1c0/0x1c0
[<ffffffffa7a3554b>] ip_local_out_sk+0x1b/0x40
[<ffffffffa7a358d3>] ip_queue_xmit+0x143/0x3a0
[<ffffffffa7a4f7b4>] tcp_transmit_skb+0x4e4/0x9e0
[<ffffffffa7a4fe22>] tcp_write_xmit+0x172/0xd00
[<ffffffffa7a50c2e>] __tcp_push_pending_frames+0x2e/0xc0
[<ffffffffa7a3f12c>] tcp_push+0xec/0x120
[<ffffffffa7a42ae2>] tcp_sendmsg+0xd2/0xc80
[<ffffffffa7a6e5a9>] inet_sendmsg+0x69/0xb0
[<ffffffffa76d36b3>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffffa79cb84d>] sock_aio_write+0x15d/0x180
[<ffffffffa74be870>] ? hrtimer_get_res+0x50/0x50
[<ffffffffa761a1f3>] do_sync_write+0x93/0xe0
[<ffffffffa761add5>] vfs_write+0x1c5/0x1f0
[<ffffffffa761baff>] SyS_write+0x7f/0xf0
[<ffffffffa7b1f7d5>] system_call_fastpath+0x1c/0x21
Code: 07 00 0f 1f 40 00 5d c3 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 c6 07 00 0f 1f 40 00 48 89 f7 57 9d <0f> 1f 44 00 00 5d c3 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 48