Complete report #1420722
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff869174cd
Build id: 3.10.0-1160.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff86f7b37b
Build id: 3.10.0-1160.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffff86f89ad3
Build id: 3.10.0-1160.el7.x86_64
|
0x33
|
|
4
|
lock_sock_nested
|
vmlinux |
0xffffffff86e385e0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
|
5
|
tcp_sendmsg
|
vmlinux |
0xffffffff86eb0dab
Build id: 3.10.0-1160.el7.x86_64
|
0x3b
|
|
6
|
|
vmlinux |
0xffffffff868e4759
Build id: 3.10.0-1160.el7.x86_64
|
0xa9
|
|
7
|
|
vmlinux |
0xffffffff868e4306
Build id: 3.10.0-1160.el7.x86_64
|
0x86
|
|
8
|
|
vmlinux |
0xffffffff868dfb48
Build id: 3.10.0-1160.el7.x86_64
|
0x78
|
|
9
|
|
vmlinux |
0xffffffff868e54c1
Build id: 3.10.0-1160.el7.x86_64
|
0x101
|
|
10
|
inet_sendmsg
|
vmlinux |
0xffffffff86edd589
Build id: 3.10.0-1160.el7.x86_64
|
0x69
|
|
11
|
|
vmlinux |
0xffffffff868d7229
Build id: 3.10.0-1160.el7.x86_64
|
0x19
|
|
12
|
sock_aio_write
|
vmlinux |
0xffffffff86e3357d
Build id: 3.10.0-1160.el7.x86_64
|
0x15d
|
|
13
|
do_sync_write
|
vmlinux |
0xffffffff86a4d063
Build id: 3.10.0-1160.el7.x86_64
|
0x93
|
|
14
|
vfs_write
|
vmlinux |
0xffffffff86a4dc55
Build id: 3.10.0-1160.el7.x86_64
|
0x1c5
|
|
15
|
SyS_write
|
vmlinux |
0xffffffff86a4e92f
Build id: 3.10.0-1160.el7.x86_64
|
0x7f
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff86f93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 46s! [flanneld:9908]
Modules linked in: xt_recent xt_statistic veth vxlan ip6_udp_tunnel udp_tunnel ip6t_MASQUERADE nf_nat_masquerade_ipv6 xt_nat xt_mark nf_tables rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle xt_comment nf_conntrack_netlink nfnetlink xt_addrtype xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul snd_ens1371 ghash_clmulni_intel snd_rawmidi snd_ac97_codec aesni_intel ppdev lrw ac97_bus gf128mul glue_helper
vmw_balloon snd_seq ablk_helper cryptd snd_seq_device snd_pcm joydev snd_timer pcspkr snd soundcore vmw_vmci sg i2c_piix4 parport_pc parport br_netfilter bridge stp llc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom ata_generic vmwgfx drm_kms_helper pata_acpi syscopyarea sysfillrect sysimgblt crct10dif_pclmul mptspi nfit scsi_transport_spi fb_sys_fops libnvdimm crct10dif_common ttm mptscsih crc32c_intel ata_piix serio_raw e1000 libata drm mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 0 PID: 9908 Comm: flanneld Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8900272bd280 ti: ffff89003a220000 task.ti: ffff89003a220000
RIP: 0010:[<ffffffff869174cd>] [<ffffffff869174cd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff89003a223be0 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffff89003a223bb0 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff88ffeaa38070
RBP: ffff89003a223be0 R08: 0000000000000000 R09: 0000000000000001
R10: ffff89003a223d78 R11: 0000000000000202 R12: ffff89007565ad40
R13: ffff88feea6b5200 R14: ffff89007565ad40 R15: ffff88feea6b5200
FS: 00007f8c24dcdb38(0000) GS:ffff890075600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c0002ff010 CR3: 0000000092cee000 CR4: 0000000000760ff0
PKRU: 55555554
Call Trace:
[<ffffffff86f7b37b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff86f89ad3>] _raw_spin_lock_bh+0x33/0x40
[<ffffffff86e385e0>] lock_sock_nested+0x20/0x50
[<ffffffff86eb0dab>] tcp_sendmsg+0x3b/0xc60
[<ffffffff868e4759>] ? update_cfs_shares+0xa9/0xf0
[<ffffffff868e4306>] ? update_curr+0x86/0x1e0
[<ffffffff868dfb48>] ? __enqueue_entity+0x78/0x80
[<ffffffff868e54c1>] ? check_preempt_wakeup+0x101/0x220
[<ffffffff86edd589>] inet_sendmsg+0x69/0xb0
[<ffffffff868d7229>] ? ttwu_do_wakeup+0x19/0xe0
[<ffffffff86e3357d>] sock_aio_write+0x15d/0x180
[<ffffffff86a4d063>] do_sync_write+0x93/0xe0
[<ffffffff86a4dc55>] vfs_write+0x1c5/0x1f0
[<ffffffff86a4e92f>] SyS_write+0x7f/0xf0
[<ffffffff86f93f92>] system_call_fastpath+0x25/0x2a
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #989008
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_safe_halt
|
vmlinux |
0xffffffffaf38924b
Build id: 3.10.0-1160.el7.x86_64
|
0xb
|
|
2
|
kvm_wait
|
vmlinux |
0xffffffffaec6caf5
Build id: 3.10.0-1160.el7.x86_64
|
0x65
|
|
3
|
__pv_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffaed1746e
Build id: 3.10.0-1160.el7.x86_64
|
0x29e
|
|
4
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffaf37b37b
Build id: 3.10.0-1160.el7.x86_64
|
0xb
|
|
5
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffffaf389ad3
Build id: 3.10.0-1160.el7.x86_64
|
0x33
|
|
6
|
lock_sock_nested
|
vmlinux |
0xffffffffaf2385e0
Build id: 3.10.0-1160.el7.x86_64
|
0x20
|
|
7
|
tcp_sendmsg
|
vmlinux |
0xffffffffaf2b0dab
Build id: 3.10.0-1160.el7.x86_64
|
0x3b
|
|
8
|
|
vmlinux |
0xffffffffaece4759
Build id: 3.10.0-1160.el7.x86_64
|
0xa9
|
|
9
|
|
vmlinux |
0xffffffffaece4306
Build id: 3.10.0-1160.el7.x86_64
|
0x86
|
|
10
|
|
vmlinux |
0xffffffffaecdfb48
Build id: 3.10.0-1160.el7.x86_64
|
0x78
|
|
11
|
|
vmlinux |
0xffffffffaece54ea
Build id: 3.10.0-1160.el7.x86_64
|
0x12a
|
|
12
|
inet_sendmsg
|
vmlinux |
0xffffffffaf2dd589
Build id: 3.10.0-1160.el7.x86_64
|
0x69
|
|
13
|
|
vmlinux |
0xffffffffaecd7229
Build id: 3.10.0-1160.el7.x86_64
|
0x19
|
|
14
|
sock_aio_write
|
vmlinux |
0xffffffffaf23357d
Build id: 3.10.0-1160.el7.x86_64
|
0x15d
|
|
15
|
do_sync_write
|
vmlinux |
0xffffffffaee4d063
Build id: 3.10.0-1160.el7.x86_64
|
0x93
|
|
16
|
vfs_write
|
vmlinux |
0xffffffffaee4dc55
Build id: 3.10.0-1160.el7.x86_64
|
0x1c5
|
|
17
|
SyS_write
|
vmlinux |
0xffffffffaee4e92f
Build id: 3.10.0-1160.el7.x86_64
|
0x7f
|
|
18
|
system_call_fastpath
|
vmlinux |
0xffffffffaf393f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [frpc:1167]
Modules linked in: tcp_diag inet_diag xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core nfit snd_hwdep libnvdimm snd_seq snd_seq_device snd_pcm ppdev snd_timer iosf_mbi crc32_pclmul ghash_clmulni_intel snd parport_pc virtio_balloon aesni_intel i2c_piix4 parport lrw gf128mul joydev soundcore glue_helper pcspkr ablk_helper cryptd ip_tables xfs libcrc32c ata_generic pata_acpi qxl drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm virtio_console virtio_blk drm ata_piix libata 8139too serio_raw floppy crct10dif_pclmul crct10dif_common
virtio_pci 8139cp crc32c_intel virtio_ring mii virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 1167 Comm: frpc Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2011
task: ffff97e7083b2100 ti: ffff97e5f5df4000 task.ti: ffff97e5f5df4000
RIP: 0010:[<ffffffffaf38924b>] [<ffffffffaf38924b>] native_safe_halt+0xb/0x20
RSP: 0018:ffff97e5f5df7b88 EFLAGS: 00000202
RAX: 0000000000000005 RBX: ffffffffaece4306 RCX: 0000000000000000
RDX: 0000000000000005 RSI: 0000000000000005 RDI: ffff97e913c34630
RBP: ffff97e5f5df7b88 R08: 0000000000000000 R09: 0000000000000098
R10: 0000000000000000 R11: 0000000000000001 R12: ffff97e73376cc00
R13: 0000000200000006 R14: 0000000000000000 R15: ffff97e73fd11800
FS: 000000c00017f210(0000) GS:ffff97edffd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7f52f95000 CR3: 00000001463ea000 CR4: 00000000003606e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaec6caf5>] kvm_wait+0x65/0x70
[<ffffffffaed1746e>] __pv_queued_spin_lock_slowpath+0x29e/0x2e0
[<ffffffffaf37b37b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffaf389ad3>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffaf2385e0>] lock_sock_nested+0x20/0x50
[<ffffffffaf2b0dab>] tcp_sendmsg+0x3b/0xc60
[<ffffffffaece4759>] ? update_cfs_shares+0xa9/0xf0
[<ffffffffaece4306>] ? update_curr+0x86/0x1e0
[<ffffffffaecdfb48>] ? __enqueue_entity+0x78/0x80
[<ffffffffaece54ea>] ? check_preempt_wakeup+0x12a/0x220
[<ffffffffaf2dd589>] inet_sendmsg+0x69/0xb0
[<ffffffffaecd7229>] ? ttwu_do_wakeup+0x19/0xe0
[<ffffffffaf23357d>] sock_aio_write+0x15d/0x180
[<ffffffffaee4d063>] do_sync_write+0x93/0xe0
[<ffffffffaee4dc55>] vfs_write+0x1c5/0x1f0
[<ffffffffaee4e92f>] SyS_write+0x7f/0xf0
[<ffffffffaf393f92>] system_call_fastpath+0x25/0x2a
Code: ff ff 66 0f 1f 44 00 00 0f 00 2d ad f2 07 00 e9 14 ff ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 90 55 48 89 e5 0f 1f 44 00 00 fb f4 <5d> c3 0f 1f 00 0f 00 2d d9 02 08 00 eb f0 0f 1f 80 00 00 00 00
Complete report #149441
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb5b11ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb615bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffffb616a443
Build id: 3.10.0-957.el7.x86_64
|
0x33
|
|
4
|
lock_sock_nested
|
vmlinux |
0xffffffffb601d550
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
5
|
tcp_sendmsg
|
vmlinux |
0xffffffffb6093ffb
Build id: 3.10.0-957.el7.x86_64
|
0x3b
|
|
6
|
|
vmlinux |
0xffffffffb5ae0449
Build id: 3.10.0-957.el7.x86_64
|
0xa9
|
|
7
|
|
vmlinux |
0xffffffffb5adfff6
Build id: 3.10.0-957.el7.x86_64
|
0x86
|
|
8
|
|
vmlinux |
0xffffffffb5adb6e8
Build id: 3.10.0-957.el7.x86_64
|
0x78
|
|
9
|
|
vmlinux |
0xffffffffb5cfc3a5
Build id: 3.10.0-957.el7.x86_64
|
0x75
|
|
10
|
inet_sendmsg
|
vmlinux |
0xffffffffb60bfeb9
Build id: 3.10.0-957.el7.x86_64
|
0x69
|
|
11
|
|
vmlinux |
0xffffffffb5cfc4d3
Build id: 3.10.0-957.el7.x86_64
|
0x23
|
|
12
|
sock_aio_write
|
vmlinux |
0xffffffffb601857d
Build id: 3.10.0-957.el7.x86_64
|
0x15d
|
|
13
|
do_sync_write
|
vmlinux |
0xffffffffb5c405b3
Build id: 3.10.0-957.el7.x86_64
|
0x93
|
|
14
|
vfs_write
|
vmlinux |
0xffffffffb5c411a5
Build id: 3.10.0-957.el7.x86_64
|
0x1c5
|
|
15
|
SyS_write
|
vmlinux |
0xffffffffb5c41ebf
Build id: 3.10.0-957.el7.x86_64
|
0x7f
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffffb6174ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [frpc:21314]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ppdev crc32_pclmul ghash_clmulni_intel vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq joydev pcspkr snd_seq_device snd_pcm snd_timer sg snd soundcore vmw_vmci i2c_piix4 parport_pc parport ip_tables ext4 mbcache jbd2 sd_mod sr_mod crc_t10dif cdrom crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
serio_raw vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt e1000 fb_sys_fops ttm mptspi drm nfit ata_piix libnvdimm scsi_transport_spi mptscsih libata mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 21314 Comm: frpc Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/22/2020
task: ffff9e8206fd2080 ti: ffff9e822fab8000 task.ti: ffff9e822fab8000
RIP: 0010:[<ffffffffb5b11ffd>] [<ffffffffb5b11ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff9e822fabbbe0 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffffffb61678ee RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9e8226600830
RBP: ffff9e822fabbbe0 R08: 0000000000000000 R09: 0000000000000000
R10: ffff9e822fabbd78 R11: 0000000000000000 R12: ffffffffb61678ee
R13: ffffffffb61678fa R14: ffffffffb61678ee R15: ffffffffb61678fa
FS: 000000c42014a890(0000) GS:ffff9e8233780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fbdf746f000 CR3: 00000002afe10000 CR4: 00000000003607e0
Call Trace:
[<ffffffffb615bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb616a443>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffb601d550>] lock_sock_nested+0x20/0x50
[<ffffffffb6093ffb>] tcp_sendmsg+0x3b/0xc60
[<ffffffffb5ae0449>] ? update_cfs_shares+0xa9/0xf0
[<ffffffffb5adfff6>] ? update_curr+0x86/0x1e0
[<ffffffffb5adb6e8>] ? __enqueue_entity+0x78/0x80
[<ffffffffb5cfc3a5>] ? sock_has_perm+0x75/0x90
[<ffffffffb60bfeb9>] inet_sendmsg+0x69/0xb0
[<ffffffffb5cfc4d3>] ? selinux_socket_sendmsg+0x23/0x30
[<ffffffffb601857d>] sock_aio_write+0x15d/0x180
[<ffffffffb5c405b3>] do_sync_write+0x93/0xe0
[<ffffffffb5c411a5>] vfs_write+0x1c5/0x1f0
[<ffffffffb5c41ebf>] SyS_write+0x7f/0xf0
[<ffffffffb6174ddb>] system_call_fastpath+0x22/0x27
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #983748
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa1917a4d
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa1f80d83
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffffa1f8ed63
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x33
|
|
4
|
lock_sock_nested
|
vmlinux |
0xffffffffa1e3d5d0
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x20
|
|
5
|
tcp_sendmsg
|
vmlinux |
0xffffffffa1eb65bb
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x3b
|
|
6
|
|
vmlinux |
0xffffffffa18e2e59
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa9
|
|
7
|
|
vmlinux |
0xffffffffa18dfc28
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x78
|
|
8
|
|
vmlinux |
0xffffffffa18e45ff
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x2ef
|
|
9
|
|
vmlinux |
0xffffffffa18e3dc1
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x101
|
|
10
|
inet_sendmsg
|
vmlinux |
0xffffffffa1ee2e49
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x69
|
|
11
|
|
vmlinux |
0xffffffffa18d74e9
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x19
|
|
12
|
sock_aio_write
|
vmlinux |
0xffffffffa1e3856d
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x15d
|
|
13
|
do_sync_write
|
vmlinux |
0xffffffffa1a4da23
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x93
|
|
14
|
vfs_write
|
vmlinux |
0xffffffffa1a4e5b5
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x1c5
|
|
15
|
SyS_write
|
vmlinux |
0xffffffffa1a4f235
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x55
|
|
16
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa1f99ed5
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa2
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffffa1f99f92
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0x25
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffa1f99ed5
Build id: 3.10.0-1160.76.1.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#5 stuck for 23s! [xdd:49006]
Modules linked in: tcp_lp veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype br_netfilter binfmt_misc overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event snd_ens1371 sb_edac snd_rawmidi snd_ac97_codec coretemp ac97_bus snd_seq iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw snd_seq_device gf128mul snd_pcm
glue_helper snd_timer ablk_helper cryptd snd sg soundcore joydev ppdev pcspkr vmw_vmci vmw_balloon i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm mptspi ata_piix scsi_transport_spi crct10dif_pclmul crct10dif_common nfit mptscsih libata crc32c_intel mptbase libnvdimm e1000 serio_raw drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 5 PID: 49006 Comm: xdd Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.76.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff8ccb2bd5e300 ti: ffff8cc9c1efc000 task.ti: ffff8cc9c1efc000
RIP: 0010:[<ffffffffa1917a4d>] [<ffffffffa1917a4d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff8cc9c1effbe0 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffffffa18e2e59 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8cc89e820ff0
RBP: ffff8cc9c1effbe0 R08: 0000000000000000 R09: 0000000000000001
R10: ffff8cc9c1effd78 R11: 0000000000000212 R12: ffff8ccbe39ebe00
R13: ffff8ccbe39ebe00 R14: 000000000000006e R15: ffff8ccbe8f5ad40
FS: 00007f99c8471700(0000) GS:ffff8ccbe8f40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001633888 CR3: 00000002ddfca000 CR4: 00000000000607e0
Call Trace:
[<ffffffffa1f80d83>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa1f8ed63>] _raw_spin_lock_bh+0x33/0x40
[<ffffffffa1e3d5d0>] lock_sock_nested+0x20/0x50
[<ffffffffa1eb65bb>] tcp_sendmsg+0x3b/0xc60
[<ffffffffa18e2e59>] ? update_cfs_shares+0xa9/0xf0
[<ffffffffa18dfc28>] ? __enqueue_entity+0x78/0x80
[<ffffffffa18e45ff>] ? enqueue_entity+0x2ef/0xbe0
[<ffffffffa18e3dc1>] ? check_preempt_wakeup+0x101/0x220
[<ffffffffa1ee2e49>] inet_sendmsg+0x69/0xb0
[<ffffffffa18d74e9>] ? ttwu_do_wakeup+0x19/0xe0
[<ffffffffa1e3856d>] sock_aio_write+0x15d/0x180
[<ffffffffa1a4da23>] do_sync_write+0x93/0xe0
[<ffffffffa1a4e5b5>] vfs_write+0x1c5/0x1f0
[<ffffffffa1a4f235>] SyS_write+0x55/0xd0
[<ffffffffa1f99ed5>] ? system_call_after_swapgs+0xa2/0x13a
[<ffffffffa1f99f92>] system_call_fastpath+0x25/0x2a
[<ffffffffa1f99ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 66 66 66 66 90 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 0f
Complete report #958328
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8351eb5d
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff83bac21a
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock_bh
|
vmlinux |
0xffffffff83bba777
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x37
|
|
4
|
lock_sock_nested
|
vmlinux |
0xffffffff83a629b0
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x20
|
|
5
|
tcp_sendmsg
|
vmlinux |
0xffffffff83ade1db
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x3b
|
|
6
|
|
vmlinux |
0xffffffff834eaea9
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0xa9
|
|
7
|
|
vmlinux |
0xffffffff834e5d58
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x78
|
|
8
|
|
vmlinux |
0xffffffff834ecdc7
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x2f7
|
|
9
|
|
vmlinux |
0xffffffff834ebe81
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x101
|
|
10
|
inet_sendmsg
|
vmlinux |
0xffffffff83b0b4e9
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x69
|
|
11
|
|
vmlinux |
0xffffffff834dd239
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x19
|
|
12
|
sock_aio_write
|
vmlinux |
0xffffffff83a5d675
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x165
|
|
13
|
do_sync_write
|
vmlinux |
0xffffffff8365b283
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x93
|
|
14
|
vfs_write
|
vmlinux |
0xffffffff8365be55
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x1c5
|
|
15
|
SyS_write
|
vmlinux |
0xffffffff8365cb05
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x55
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff83bc539a
Build id: 3.10.0-1160.114.2.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [ngrokd:2249]
Modules linked in: binfmt_misc nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc dm_mirror dm_region_hash dm_log dm_mod snd_seq_midi snd_seq_midi_event iosf_mbi kvm_intel squashfs kvm loop irqbypass crc32_pclmul ghash_clmulni_intel ppdev aesni_intel btusb vmw_balloon lrw btrtl btbcm gf128mul glue_helper btintel ablk_helper cryptd snd_ens1371 snd_rawmidi bluetooth snd_ac97_codec ac97_bus snd_seq joydev snd_seq_device pcspkr snd_pcm rfkill snd_timer sg vmw_vmci snd soundcore
i2c_piix4 parport_pc parport ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common syscopyarea sysfillrect sysimgblt nfit fb_sys_fops ttm libnvdimm crc32c_intel drm serio_raw ata_piix libata e1000 mptspi scsi_transport_spi mptscsih mptbase drm_panel_orientation_quirks fuse
CPU: 3 PID: 2249 Comm: ngrokd Tainted: G L ------------ T 3.10.0-1160.114.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9a82b563a100 ti: ffff9a82dfd6c000 task.ti: ffff9a82dfd6c000
RIP: 0010:[<ffffffff8351eb5d>] [<ffffffff8351eb5d>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff9a82dfd6fbe0 EFLAGS: 00000293
RAX: 0000000000000001 RBX: ffffffff8379c5f5 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff9a82f3acb1f0
RBP: ffff9a82dfd6fbe0 R08: 0000000000000000 R09: 0000000000000001
R10: ffff9a82dfd6fd78 R11: 0000000000000206 R12: 0000000000000002
R13: 00000000d4743dbc R14: 0000000000000000 R15: 0000000100000000
FS: 00007f9272a7e700(0000) GS:ffff9a82fb6c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa9a4ee2000 CR3: 000000006900c000 CR4: 00000000001607e0
Call Trace:
[<ffffffff83bac21a>] queued_spin_lock_slowpath+0xb/0x13
[<ffffffff83bba777>] _raw_spin_lock_bh+0x37/0x40
[<ffffffff83a629b0>] lock_sock_nested+0x20/0x50
[<ffffffff83ade1db>] tcp_sendmsg+0x3b/0xc60
[<ffffffff834eaea9>] ? update_cfs_shares+0xa9/0x100
[<ffffffff834e5d58>] ? __enqueue_entity+0x78/0x80
[<ffffffff834ecdc7>] ? enqueue_entity+0x2f7/0xbf0
[<ffffffff834ebe81>] ? check_preempt_wakeup+0x101/0x230
[<ffffffff83b0b4e9>] inet_sendmsg+0x69/0xb0
[<ffffffff834dd239>] ? ttwu_do_wakeup+0x19/0xf0
[<ffffffff83a5d675>] sock_aio_write+0x165/0x180
[<ffffffff8365b283>] do_sync_write+0x93/0xe0
[<ffffffff8365be55>] vfs_write+0x1c5/0x1f0
[<ffffffff8365cb05>] SyS_write+0x55/0xd0
[<ffffffff83bc539a>] system_call_fastpath+0x25/0x2a
Code: 47 fe ff ff 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 66 90 b9 01 00 00 00 8b 17 85 d2 74 0d 83 fa 03 74 08 f3 90 <8b> 17 85 d2 75 f3 89 d0 f0 0f b1 0f 39 c2 75 e3 5d 66 90 c3 cc