Complete report #1417537
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
console_unlock
|
vmlinux |
0xffffffff8108a3ed
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x46d
|
|
2
|
vprintk_emit
|
vmlinux |
0xffffffff8108a7f4
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x3c4
|
|
3
|
vprintk_default
|
vmlinux |
0xffffffff8108ab69
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x29
|
|
4
|
printk
|
vmlinux |
0xffffffff8169de96
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x5d
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffff811de8e7
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x177
|
|
6
|
kfree_skbmem
|
vmlinux |
0xffffffff815710d7
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x37
|
|
7
|
__kfree_skb
|
vmlinux |
0xffffffff81572dda
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x1a
|
|
8
|
tcp_recvmsg
|
vmlinux |
0xffffffff815d8966
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x306
|
|
9
|
|
vmlinux |
0xffffffff8105ab43
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x13
|
|
10
|
inet_recvmsg
|
vmlinux |
0xffffffff81606aea
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x7a
|
|
11
|
sock_aio_read.part.7
|
vmlinux |
0xffffffff81569936
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x146
|
|
12
|
|
vmlinux |
0xffffffff8118296b
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x2b
|
|
13
|
sock_aio_read
|
vmlinux |
0xffffffff81569971
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x21
|
|
14
|
do_sync_read
|
vmlinux |
0xffffffff8120011d
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x8d
|
|
15
|
vfs_read
|
vmlinux |
0xffffffff81200bc5
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x145
|
|
16
|
SyS_read
|
vmlinux |
0xffffffff812019df
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x7f
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffff816b5089
Build id: 3.10.0-693.5.2.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#32 stuck for 22s! [serf:2201]
Modules linked in: veth tcp_diag inet_diag ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack libcrc32c br_netfilter bridge overlay(T) nfsv3 rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache 8021q garp mrp stp llc bonding ip6_tables rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm i40iw ib_core skx_edac edac_core coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support pcspkr sg joydev mei_me mei nfit
i2c_i801 lpc_ich ipmi_si ipmi_devintf ipmi_msghandler libnvdimm shpchp wmi acpi_cpufreq nfsd auth_rpcgss nfs_acl lockd grace sunrpc binfmt_misc ip_tables ext4 mbcache jbd2 sd_mod sr_mod crc_t10dif cdrom crct10dif_generic uas usb_storage ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ixgbe crct10dif_pclmul drm crct10dif_common i40e crc32c_intel igb mpt3sas ahci libahci libata mdio raid_class dca scsi_transport_sas ptp i2c_algo_bit i2c_core pps_core
CPU: 32 PID: 2201 Comm: serf Tainted: G W L ------------ T 3.10.0-693.5.2.el7.x86_64 #1
Hardware name: Sugon CX50-G35/60P16B-TC4600T, BIOS 0KLSN218 05/22/2019
task: ffff8817dd1a0000 ti: ffff8817d0700000 task.ti: ffff8817d0700000
RIP: 0010:[<ffffffff8108a3ed>] [<ffffffff8108a3ed>] console_unlock+0x46d/0x4b0
RSP: 0018:ffff8817d0703b28 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffff881700000024 RCX: 00000000ffffffff
RDX: ffff8817d0703ab0 RSI: 0000000000000004 RDI: 0000000000000202
RBP: ffff8817d0703b58 R08: 0000000000000000 R09: ffff8817db4caf00
R10: 00000000000003ff R11: 0000000000000001 R12: ffffffff81408e99
R13: ffff8817d0703af0 R14: ffffffff816b05fa R15: 00000000ed6858e9
FS: 000000c420164468(0000) GS:ffff882fddd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b6b0b066000 CR3: 00000017d172f000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffff81e86e50 00000000ffffffff 0000000000000000 0000000000000000
0000000000000092 0000000000000006 ffff8817d0703bc8 ffffffff8108a7f4
ffffffff81d86542 ffff881700000092 0000000000000000 ffff8817d0703be8
Call Trace:
[<ffffffff8108a7f4>] vprintk_emit+0x3c4/0x510
[<ffffffff8108ab69>] vprintk_default+0x29/0x40
[<ffffffff8169de96>] printk+0x5d/0x74
[<ffffffff811de8e7>] kmem_cache_free+0x177/0x200
[<ffffffff815710d7>] kfree_skbmem+0x37/0x90
[<ffffffff81572dda>] __kfree_skb+0x1a/0x20
[<ffffffff815d8966>] tcp_recvmsg+0x306/0xb50
[<ffffffff8105ab43>] ? x2apic_send_IPI_mask+0x13/0x20
[<ffffffff81606aea>] inet_recvmsg+0x7a/0xa0
[<ffffffff81569936>] sock_aio_read.part.7+0x146/0x160
[<ffffffff8118296b>] ? unlock_page+0x2b/0x30
[<ffffffff81569971>] sock_aio_read+0x21/0x30
[<ffffffff8120011d>] do_sync_read+0x8d/0xd0
[<ffffffff81200bc5>] vfs_read+0x145/0x170
[<ffffffff812019df>] SyS_read+0x7f/0xe0
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: df 00 48 83 05 44 ca df 00 01 48 8b 7d d0 83 e0 1f 89 05 2f ca df 00 c6 07 00 0f 1f 40 00 44 89 f7 e8 c8 e7 ff ff 48 89 df 57 9d <0f> 1f 44 00 00 e9 16 fc ff ff 48 c7 05 1e cb df 00 00 00 00 00
Complete report #361123
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
console_unlock
|
vmlinux |
0xffffffff8108a3dd
Build id: 3.10.0-693.el7.x86_64
|
0x46d
|
|
2
|
vprintk_emit
|
vmlinux |
0xffffffff8108a7e4
Build id: 3.10.0-693.el7.x86_64
|
0x3c4
|
|
3
|
vprintk_default
|
vmlinux |
0xffffffff8108ab59
Build id: 3.10.0-693.el7.x86_64
|
0x29
|
|
4
|
printk
|
vmlinux |
0xffffffff8169ddd6
Build id: 3.10.0-693.el7.x86_64
|
0x5d
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffff811de967
Build id: 3.10.0-693.el7.x86_64
|
0x177
|
|
6
|
kfree_skbmem
|
vmlinux |
0xffffffff815710d7
Build id: 3.10.0-693.el7.x86_64
|
0x37
|
|
7
|
__kfree_skb
|
vmlinux |
0xffffffff81572dda
Build id: 3.10.0-693.el7.x86_64
|
0x1a
|
|
8
|
tcp_recvmsg
|
vmlinux |
0xffffffff815d8dd0
Build id: 3.10.0-693.el7.x86_64
|
0x7e0
|
|
9
|
inet_recvmsg
|
vmlinux |
0xffffffff81606a3a
Build id: 3.10.0-693.el7.x86_64
|
0x7a
|
|
10
|
sock_aio_read.part.7
|
vmlinux |
0xffffffff81569936
Build id: 3.10.0-693.el7.x86_64
|
0x146
|
|
11
|
|
vmlinux |
0xffffffff8118295b
Build id: 3.10.0-693.el7.x86_64
|
0x2b
|
|
12
|
sock_aio_read
|
vmlinux |
0xffffffff81569971
Build id: 3.10.0-693.el7.x86_64
|
0x21
|
|
13
|
do_sync_read
|
vmlinux |
0xffffffff8120019d
Build id: 3.10.0-693.el7.x86_64
|
0x8d
|
|
14
|
vfs_read
|
vmlinux |
0xffffffff81200c45
Build id: 3.10.0-693.el7.x86_64
|
0x145
|
|
15
|
SyS_read
|
vmlinux |
0xffffffff81201a5f
Build id: 3.10.0-693.el7.x86_64
|
0x7f
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff816b4fc9
Build id: 3.10.0-693.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [jdog-monitor.1.:28346]
Modules linked in: ipt_REJECT nf_reject_ipv4 xt_REDIRECT nf_nat_redirect ip_vs_rr xt_ipvs ip_vs binfmt_misc vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx5_fpga_tools(OE) mlx5_ib(OE) ib_uverbs(OE) mlx5_core(OE) mlxfw(OE) mlx4_en(OE) ptp pps_core mlx4_ib(OE) ib_core(OE) mlx4_core(OE) mlx_compat(OE) devlink ext4 mbcache jbd2 dm_mirror dm_region_hash dm_log dm_mod edac_core ppdev parport_pc virtio_balloon parport i2c_piix4 iosf_mbi crc32_pclmul ghash_clmulni_intel
aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev knem(OE) ip_tables xfs libcrc32c ata_generic pata_acpi virtio_net virtio_console virtio_blk bochs_drm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix virtio_pci crct10dif_pclmul crct10dif_common libata floppy crc32c_intel virtio_ring i2c_core serio_raw virtio
CPU: 6 PID: 28346 Comm: jdog-monitor.1. Tainted: G W OEL ------------ T 3.10.0-693.el7.x86_64 #1
Hardware name: JD JCloud Iaas Jvirt, BIOS 1.10.2-1.el7 04/01/2014
task: ffff880ff7350000 ti: ffff8800a88b8000 task.ti: ffff8800a88b8000
RIP: 0010:[<ffffffff8108a3dd>] [<ffffffff8108a3dd>] console_unlock+0x46d/0x4b0
RSP: 0018:ffff8800a88bbb28 EFLAGS: 00000202
RAX: 0000000000000000 RBX: ffff880f00000025 RCX: 00000000ffffffff
RDX: ffff8800a88bbab0 RSI: 0000000000000004 RDI: 0000000000000202
RBP: ffff8800a88bbb58 R08: 0000000000000000 R09: ffff880ff7ee6f00
R10: 0000000000008000 R11: 0000000000000000 R12: ffffffff81408dd7
R13: ffff8800a88bbaf0 R14: ffff8800a88bbab0 R15: 00000000038713f8
FS: 00007fe6a9b4c700(0000) GS:ffff880fff380000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f02378f9400 CR3: 000000040f142000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffffffff81e86e50 00000000ffffffff 0000000000000000 0000000000000000
0000000000000093 0000000000000006 ffff8800a88bbbc8 ffffffff8108a7e4
ffffffff81d86542 0000000100000093 0000000000000000 ffff8800a88bbbe8
Call Trace:
[<ffffffff8108a7e4>] vprintk_emit+0x3c4/0x510
[<ffffffff8108ab59>] vprintk_default+0x29/0x40
[<ffffffff8169ddd6>] printk+0x5d/0x74
[<ffffffff811de967>] kmem_cache_free+0x177/0x200
[<ffffffff815710d7>] kfree_skbmem+0x37/0x90
[<ffffffff81572dda>] __kfree_skb+0x1a/0x20
[<ffffffff815d8dd0>] tcp_recvmsg+0x7e0/0xb50
[<ffffffff81606a3a>] inet_recvmsg+0x7a/0xa0
[<ffffffff81569936>] sock_aio_read.part.7+0x146/0x160
[<ffffffff8118295b>] ? unlock_page+0x2b/0x30
[<ffffffff81569971>] sock_aio_read+0x21/0x30
[<ffffffff8120019d>] do_sync_read+0x8d/0xd0
[<ffffffff81200c45>] vfs_read+0x145/0x170
[<ffffffff81201a5f>] SyS_read+0x7f/0xe0
[<ffffffff816b4fc9>] system_call_fastpath+0x16/0x1b
Code: df 00 48 83 05 54 ca df 00 01 48 8b 7d d0 83 e0 1f 89 05 3f ca df 00 ff 14 25 50 31 a1 81 44 89 f7 e8 c8 e7 ff ff 48 89 df 57 9d <0f> 1f 44 00 00 e9 16 fc ff ff 48 c7 05 2e cb df 00 00 00 00 00
Complete report #3834
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
console_unlock
|
vmlinux |
0xffffffff810884b4
Build id: 3.10.0-514.el7.x86_64
|
0x3c4
|
|
2
|
vprintk_emit
|
vmlinux |
0xffffffff810888b4
Build id: 3.10.0-514.el7.x86_64
|
0x3c4
|
|
3
|
vprintk_default
|
vmlinux |
0xffffffff81088c29
Build id: 3.10.0-514.el7.x86_64
|
0x29
|
|
4
|
printk
|
vmlinux |
0xffffffff8167f554
Build id: 3.10.0-514.el7.x86_64
|
0x5e
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffff811dc5da
Build id: 3.10.0-514.el7.x86_64
|
0x17a
|
|
6
|
kfree_skbmem
|
vmlinux |
0xffffffff8155bf77
Build id: 3.10.0-514.el7.x86_64
|
0x37
|
|
7
|
__kfree_skb
|
vmlinux |
0xffffffff8155df3a
Build id: 3.10.0-514.el7.x86_64
|
0x1a
|
|
8
|
tcp_recvmsg
|
vmlinux |
0xffffffff815bf3b0
Build id: 3.10.0-514.el7.x86_64
|
0x7e0
|
|
9
|
|
vmlinux |
0xffffffff810c95a8
Build id: 3.10.0-514.el7.x86_64
|
0x78
|
|
10
|
inet_recvmsg
|
vmlinux |
0xffffffff815ec86b
Build id: 3.10.0-514.el7.x86_64
|
0x7b
|
|
11
|
sock_aio_read.part.7
|
vmlinux |
0xffffffff81554576
Build id: 3.10.0-514.el7.x86_64
|
0x146
|
|
12
|
?
ttwu_do_activate.constprop.90
|
vmlinux |
0xffffffff810c1b1d
Build id: 3.10.0-514.el7.x86_64
|
0x5d
|
|
13
|
sock_aio_read
|
vmlinux |
0xffffffff815545b1
Build id: 3.10.0-514.el7.x86_64
|
0x21
|
|
14
|
do_sync_read
|
vmlinux |
0xffffffff811fd7ed
Build id: 3.10.0-514.el7.x86_64
|
0x8d
|
|
15
|
vfs_read
|
vmlinux |
0xffffffff811fe04d
Build id: 3.10.0-514.el7.x86_64
|
0x14d
|
|
16
|
SyS_read
|
vmlinux |
0xffffffff811feb6f
Build id: 3.10.0-514.el7.x86_64
|
0x7f
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffff816965c9
Build id: 3.10.0-514.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 23s! [dockerd:5374]
Modules linked in: xt_REDIRECT nf_nat_redirect ip_vs_rr xt_ipvs ip_vs xt_nat veth vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_nat_ipv4 nf_nat br_netfilter bridge stp llc overlay(T) ipmi_devintf gpio_ich iTCO_wdt iTCO_vendor_support dcdbas intel_powerclamp coretemp kvm irqbypass ipmi_ssif pcspkr sg ipmi_si ipmi_msghandler lpc_ich i7core_edac edac_core shpchp wmi acpi_power_meter nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic crct10dif_common sr_mod cdrom ata_generic pata_acpi mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crc32c_intel drm serio_raw
mptsas libata scsi_transport_sas mptscsih i2c_core mptbase bnx2 fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 10 PID: 5374 Comm: dockerd Tainted: G W I L ------------ T 3.10.0-514.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R410/0WWR83, BIOS 1.11.0 07/20/2012
task: ffff88082e306dd0 ti: ffff8807fcaf0000 task.ti: ffff8807fcaf0000
RIP: 0010:[<ffffffff810884b4>] [<ffffffff810884b4>] console_unlock+0x3c4/0x400
RSP: 0018:ffff8807fcaf3b20 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff81d37123 RCX: 00000000ffffffff
RDX: ffff8807fcaf3aa8 RSI: 0000000000000004 RDI: 0000000000000246
RBP: ffff8807fcaf3b48 R08: 0000000000000000 R09: ffff88082cea6f00
R10: 00000000000003ff R11: 0000000000000001 R12: 00000021fcaf3ab8
R13: ffffffff813f7cb5 R14: ffff8807fcaf3ae8 R15: ffffffff8169201a
FS: 00007f0ac3fff700(0000) GS:ffff88083f740000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9aeeffbe68 CR3: 000000080ea89000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
00000000ffffffff 0000000000000000 0000000000000000 0000000000000092
0000000000000006 ffff8807fcaf3bb8 ffffffff810888b4 ffffffff81d374a2
ffff880100000092 0000000000000000 ffff8807fcaf3bd8 ffffffff818de3d8
Call Trace:
[<ffffffff810888b4>] vprintk_emit+0x3c4/0x510
[<ffffffff81088c29>] vprintk_default+0x29/0x40
[<ffffffff8167f554>] printk+0x5e/0x75
[<ffffffff811dc5da>] kmem_cache_free+0x17a/0x1f0
[<ffffffff8155bf77>] kfree_skbmem+0x37/0x90
[<ffffffff8155df3a>] __kfree_skb+0x1a/0x20
[<ffffffff815bf3b0>] tcp_recvmsg+0x7e0/0xb50
[<ffffffff810c95a8>] ? __enqueue_entity+0x78/0x80
[<ffffffff815ec86b>] inet_recvmsg+0x7b/0xa0
[<ffffffff81554576>] sock_aio_read.part.7+0x146/0x160
[<ffffffff810c1b1d>] ? ttwu_do_activate.constprop.90+0x5d/0x70
[<ffffffff815545b1>] sock_aio_read+0x21/0x30
[<ffffffff811fd7ed>] do_sync_read+0x8d/0xd0
[<ffffffff811fe04d>] vfs_read+0x14d/0x170
[<ffffffff811feb6f>] SyS_read+0x7f/0xe0
[<ffffffff816965c9>] system_call_fastpath+0x16/0x1b
Code: 0d de f8 da 00 48 83 05 da f8 da 00 01 83 e0 1f 89 05 c9 f8 da 00 e8 6c 53 60 00 44 89 f7 4c 89 fe e8 41 e6 ff ff 48 89 df 57 9d <66> 66 90 66 90 e9 b2 fc ff ff 48 c7 05 b7 f9 da 00 00 00 00 00