Complete report #1418484
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9fb088ae
Build id: 3.10.0-862.el7.x86_64
|
0x1ce
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa010842a
Build id: 3.10.0-862.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffa0115680
Build id: 3.10.0-862.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0682437
Build id: 3.10.0-862.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc066e365
Build id: 3.10.0-862.el7.x86_64
|
0x7b5
|
|
6
|
|
vmlinux |
0xffffffff9fb9b26a
Build id: 3.10.0-862.el7.x86_64
|
0x8a
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc063e998
Build id: 3.10.0-862.el7.x86_64
|
0x98
|
|
8
|
_ext4_get_block
|
ext4 |
0xffffffffc063f1bf
Build id: 3.10.0-862.el7.x86_64
|
0x1df
|
|
9
|
ext4_get_block
|
ext4 |
0xffffffffc063f216
Build id: 3.10.0-862.el7.x86_64
|
0x16
|
|
10
|
do_mpage_readpage
|
vmlinux |
0xffffffff9fc5ea8b
Build id: 3.10.0-862.el7.x86_64
|
0x4eb
|
|
11
|
|
ext4 |
0xffffffffc063f200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
12
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff9fb94624
Build id: 3.10.0-862.el7.x86_64
|
0xe4
|
|
13
|
|
vmlinux |
0xffffffff9fba335e
Build id: 3.10.0-862.el7.x86_64
|
0xe
|
|
14
|
mpage_readpages
|
vmlinux |
0xffffffff9fc5edf2
Build id: 3.10.0-862.el7.x86_64
|
0x112
|
|
15
|
|
ext4 |
0xffffffffc063f200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
16
|
|
ext4 |
0xffffffffc063f200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
17
|
|
vmlinux |
0xffffffff9fbe8528
Build id: 3.10.0-862.el7.x86_64
|
0x98
|
|
18
|
ext4_readpages
|
ext4 |
0xffffffffc063cb7c
Build id: 3.10.0-862.el7.x86_64
|
0x3c
|
|
19
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff9fba122f
Build id: 3.10.0-862.el7.x86_64
|
0x1cf
|
|
20
|
ondemand_readahead
|
vmlinux |
0xffffffff9fba1436
Build id: 3.10.0-862.el7.x86_64
|
0x116
|
|
21
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff9fba1854
Build id: 3.10.0-862.el7.x86_64
|
0x44
|
|
22
|
generic_file_aio_read
|
vmlinux |
0xffffffff9fb954d2
Build id: 3.10.0-862.el7.x86_64
|
0x2c2
|
|
23
|
ext4_file_read
|
ext4 |
0xffffffffc0638e53
Build id: 3.10.0-862.el7.x86_64
|
0x43
|
|
24
|
do_sync_read
|
vmlinux |
0xffffffff9fc1a113
Build id: 3.10.0-862.el7.x86_64
|
0x93
|
|
25
|
vfs_read
|
vmlinux |
0xffffffff9fc1ab3f
Build id: 3.10.0-862.el7.x86_64
|
0x9f
|
|
26
|
SyS_read
|
vmlinux |
0xffffffff9fc1ba0f
Build id: 3.10.0-862.el7.x86_64
|
0x7f
|
|
27
|
system_call_fastpath
|
vmlinux |
0xffffffffa011f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 23s! [sfavsrv:62139]
Modules linked in: tcp_lp fuse udp_diag tcp_diag inet_diag nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables binfmt_misc iptable_filter overlay(T) rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt sunrpc ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm iTCO_wdt iTCO_vendor_support skx_edac coretemp intel_rapl iosf_mbi kvm_intel vfat fat kvm ext4 irqbypass crc32_pclmul ghash_clmulni_intel mbcache jbd2 aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd i40iw pcspkr ib_core ses enclosure joydev scsi_transport_sas sg i2c_i801 mei_me mei lpc_ich i2c_core shpchp nfit libnvdimm acpi_power_meter acpi_cpufreq ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ahci crct10dif_pclmul i40e crct10dif_common crc32c_intel libahci libata megaraid_sas ptp pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 62139 Comm: sfavsrv Kdump: loaded Tainted: G W L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: Huawei 2288H V5/BC11SPSCB0, BIOS 7.79 10/20/2020
task: ffff9eb2ceb3eeb0 ti: ffff9e8f89cf4000 task.ti: ffff9e8f89cf4000
RIP: 0010:[<ffffffff9fb088ae>] [<ffffffff9fb088ae>] native_queued_spin_lock_slowpath+0x1ce/0x200
RSP: 0018:ffff9e8f89cf7780 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff9e85f6002480
RBP: ffff9e8f89cf7780 R08: 0000000000000101 R09: ffffffffc066e34a
R10: ffff9e8500adba60 R11: ffffdecafc4dd340 R12: ffffffffc0681813
R13: ffffffff00000000 R14: 0000000000000000 R15: 00011220417c9710
FS: 00007f267cff9700(0000) GS:ffff9e8500ac0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f791800b1c8 CR3: 00000040be512000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa010842a>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa0115680>] _raw_spin_lock+0x20/0x30
[<ffffffffc0682437>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc066e365>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffff9fb9b26a>] ? __rmqueue+0x8a/0x460
[<ffffffffc063e998>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc063f1bf>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc063f216>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff9fc5ea8b>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc063f200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9fb94624>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffff9fba335e>] ? lru_cache_add+0xe/0x10
[<ffffffff9fc5edf2>] mpage_readpages+0x112/0x170
[<ffffffffc063f200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc063f200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9fbe8528>] ? alloc_pages_current+0x98/0x110
[<ffffffffc063cb7c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff9fba122f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff9fba1436>] ondemand_readahead+0x116/0x230
[<ffffffff9fba1854>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff9fb954d2>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc0638e53>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff9fc1a113>] do_sync_read+0x93/0xe0
[<ffffffff9fc1ab3f>] vfs_read+0x9f/0x170
[<ffffffff9fc1ba0f>] SyS_read+0x7f/0xf0
[<ffffffffa011f7d5>] system_call_fastpath+0x1c/0x21
Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66
Complete report #1412785
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fdfb2
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff816a8ff4
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816b6850
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc02729d7
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc025e8d5
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x7a5
|
|
6
|
ext4_map_blocks
|
ext4 |
0xffffffffc022f168
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x98
|
|
7
|
_ext4_get_block
|
ext4 |
0xffffffffc022f99f
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1df
|
|
8
|
?
__mem_cgroup_commit_charge
|
vmlinux |
0xffffffff811f7862
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xe2
|
|
9
|
ext4_get_block
|
ext4 |
0xffffffffc022f9f6
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x16
|
|
10
|
do_mpage_readpage
|
vmlinux |
0xffffffff81247dab
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x4eb
|
|
11
|
|
ext4 |
0xffffffffc022f9e0
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x220
|
|
12
|
|
vmlinux |
0xffffffff81195a4e
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xe
|
|
13
|
mpage_readpages
|
vmlinux |
0xffffffff812480eb
Build id: 3.10.0-693.21.1.el7.x86_64
|
0xeb
|
|
14
|
|
ext4 |
0xffffffffc022f9e0
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x220
|
|
15
|
|
ext4 |
0xffffffffc022f9e0
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x220
|
|
16
|
|
vmlinux |
0xffffffff81186da8
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x98
|
|
17
|
ext4_readpages
|
ext4 |
0xffffffffc022d27c
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x3c
|
|
18
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff8119391f
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1cf
|
|
19
|
ondemand_readahead
|
vmlinux |
0xffffffff81193b26
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x116
|
|
20
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff81193f44
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x44
|
|
21
|
generic_file_aio_read
|
vmlinux |
0xffffffff8118852b
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x29b
|
|
22
|
do_sync_read
|
vmlinux |
0xffffffff81204ac3
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x93
|
|
23
|
vfs_read
|
vmlinux |
0xffffffff812054ef
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x9f
|
|
24
|
SyS_read
|
vmlinux |
0xffffffff812063bf
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x7f
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffff816c0715
Build id: 3.10.0-693.21.1.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#19 stuck for 22s! [sfavsrv:156607]
Modules linked in: udp_diag tcp_diag inet_diag binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bonding dm_mirror dm_region_hash dm_log dm_mod skx_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support vfat fat pcspkr cdc_ether usbnet mii uas sg usb_storage i2c_i801 shpchp lpc_ich mei_me mei wmi ipmi_ssif ipmi_si ipmi_devintf ipmi_msghandler nfit libnvdimm acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl
lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops i40e ttm igb drm ahci libahci dca ptp i2c_algo_bit pps_core libata megaraid_sas(OE) i2c_core
CPU: 19 PID: 156607 Comm: sfavsrv Tainted: G OEL ------------ 3.10.0-693.21.1.el7.x86_64 #1
Hardware name: Lenovo ThinkSystem SR850 -[7X19CTO1WW]-/-[7X19CTO1WW]-, BIOS -[TEE164L-2.80]- 10/23/2020
task: ffff882959bb8fd0 ti: ffff882c5351c000 task.ti: ffff882c5351c000
RIP: 0010:[<ffffffff810fdfb2>] [<ffffffff810fdfb2>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff882c5351f7f0 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffffffffff03 RCX: 0000000000990000
RDX: ffff883ffe3d9600 RSI: 0000000001190001 RDI: ffff8840a2477c80
RBP: ffff882c5351f7f0 R08: ffff882fde359600 R09: 0000000000000000
R10: ffff882fde35b960 R11: ffffea0064a4e280 R12: ffffffffc0271db3
R13: ffffffff00000000 R14: 0000000000000000 R15: 0000000000000002
FS: 00007f3e9d7fc700(0000) GS:ffff882fde340000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa8086fdfd8 CR3: 0000002a8fe72000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff816a8ff4>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816b6850>] _raw_spin_lock+0x20/0x30
[<ffffffffc02729d7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc025e8d5>] ext4_ext_map_blocks+0x7a5/0xf50 [ext4]
[<ffffffffc022f168>] ext4_map_blocks+0x98/0x6f0 [ext4]
[<ffffffffc022f99f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffff811f7862>] ? __mem_cgroup_commit_charge+0xe2/0x2f0
[<ffffffffc022f9f6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff81247dab>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc022f9e0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff81195a4e>] ? lru_cache_add+0xe/0x10
[<ffffffff812480eb>] mpage_readpages+0xeb/0x150
[<ffffffffc022f9e0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc022f9e0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff81186da8>] ? __page_cache_alloc+0x98/0xb0
[<ffffffffc022d27c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff8119391f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff81193b26>] ondemand_readahead+0x116/0x230
[<ffffffff81193f44>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff8118852b>] generic_file_aio_read+0x29b/0x7b0
[<ffffffff81204ac3>] do_sync_read+0x93/0xe0
[<ffffffff812054ef>] vfs_read+0x9f/0x170
[<ffffffff812063bf>] SyS_read+0x7f/0xe0
[<ffffffff816c0715>] system_call_fastpath+0x1c/0x21
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 00 96 01 00 48 03 14 c5 a0 10 b2 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #1177201
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fc032
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x112
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff816a070f
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816ade20
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc06028d7
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc05ee805
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x795
|
|
6
|
|
vmlinux |
0xffffffff811a4df3
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x63
|
|
7
|
|
vmlinux |
0xffffffff8118e1ba
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x4fa
|
|
8
|
ext4_map_blocks
|
ext4 |
0xffffffffc05bf138
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x98
|
|
9
|
|
vmlinux |
0xffffffff8118b98a
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x8a
|
|
10
|
_ext4_get_block
|
ext4 |
0xffffffffc05bf96f
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x1df
|
|
11
|
?
__mem_cgroup_commit_charge
|
vmlinux |
0xffffffff811f4d72
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xe2
|
|
12
|
ext4_get_block
|
ext4 |
0xffffffffc05bf9c6
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x16
|
|
13
|
do_mpage_readpage
|
vmlinux |
0xffffffff81244e48
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x4e8
|
|
14
|
|
ext4 |
0xffffffffc05bf9b0
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x220
|
|
15
|
|
vmlinux |
0xffffffff8119320e
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xe
|
|
16
|
mpage_readpages
|
vmlinux |
0xffffffff8124518b
Build id: 3.10.0-693.11.6.el7.x86_64
|
0xeb
|
|
17
|
|
ext4 |
0xffffffffc05bf9b0
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x220
|
|
18
|
|
ext4 |
0xffffffffc05bf9b0
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x220
|
|
19
|
|
vmlinux |
0xffffffff811d3038
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x98
|
|
20
|
ext4_readpages
|
ext4 |
0xffffffffc05bd25c
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x3c
|
|
21
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff811910fc
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x1cc
|
|
22
|
ondemand_readahead
|
vmlinux |
0xffffffff811912f6
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x116
|
|
23
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff81191704
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x44
|
|
24
|
generic_file_aio_read
|
vmlinux |
0xffffffff81185d8b
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x29b
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffff81201f5d
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x8d
|
|
26
|
vfs_read
|
vmlinux |
0xffffffff8120295c
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x9c
|
|
27
|
SyS_pread64
|
vmlinux |
0xffffffff812039f2
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x92
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffff816b89fd
Build id: 3.10.0-693.11.6.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#47 stuck for 22s! [linux_client:154415]
Modules linked in: xt_multiport ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter arc4 rpcsec_gss_krb5 md4 nfsv4 nls_utf8 cifs nfs dns_resolver fscache mxm_wmi iTCO_wdt iTCO_vendor_support dcdbas ext4 mbcache jbd2 sb_edac edac_core intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr lpc_ich mei_me sg mei ipmi_si ipmi_devintf ipmi_msghandler shpchp wmi acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc nf_conntrack_ipv4 nf_defrag_ipv4 nf_conntrack ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb crct10dif_pclmul drm crct10dif_common crc32c_intel ahci
libahci libata ptp megaraid_sas pps_core dca i2c_algo_bit i2c_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 47 PID: 154415 Comm: linux_client Tainted: G L ------------ 3.10.0-693.11.6.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge T630/0NT78X, BIOS 2.5.4 08/17/2017
task: ffff882ed6a50000 ti: ffff88135eefc000 task.ti: ffff88135eefc000
RIP: 0010:[<ffffffff810fc032>] [<ffffffff810fc032>] native_queued_spin_lock_slowpath+0x112/0x1e0
RSP: 0018:ffff88135eeff7e8 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000001790000
RDX: ffff88301e1995c0 RSI: 0000000000e90101 RDI: ffff883019982480
RBP: ffff88135eeff7e8 R08: ffff88301e3d95c0 R09: 0000000000000000
R10: ffff88301e3db920 R11: ffffea0074b4a000 R12: ffff88100a7d4180
R13: 000000006f6ce64d R14: 0000000000000086 R15: 0000000000000040
FS: 00007f0f6f7ee700(0000) GS:ffff88301e3c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0f8c544000 CR3: 0000001267236000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff816a070f>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816ade20>] _raw_spin_lock+0x20/0x30
[<ffffffffc06028d7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc05ee805>] ext4_ext_map_blocks+0x795/0xf40 [ext4]
[<ffffffff811a4df3>] ? zone_statistics+0x63/0xa0
[<ffffffff8118e1ba>] ? get_page_from_freelist+0x4fa/0x9e0
[<ffffffffc05bf138>] ext4_map_blocks+0x98/0x6f0 [ext4]
[<ffffffff8118b98a>] ? __rmqueue+0x8a/0x460
[<ffffffffc05bf96f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffff811f4d72>] ? __mem_cgroup_commit_charge+0xe2/0x2f0
[<ffffffffc05bf9c6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff81244e48>] do_mpage_readpage+0x4e8/0x740
[<ffffffffc05bf9b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff8119320e>] ? lru_cache_add+0xe/0x10
[<ffffffff8124518b>] mpage_readpages+0xeb/0x150
[<ffffffffc05bf9b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc05bf9b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff811d3038>] ? alloc_pages_current+0x98/0x110
[<ffffffffc05bd25c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff811910fc>] __do_page_cache_readahead+0x1cc/0x250
[<ffffffff811912f6>] ondemand_readahead+0x116/0x230
[<ffffffff81191704>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff81185d8b>] generic_file_aio_read+0x29b/0x790
[<ffffffff81201f5d>] do_sync_read+0x8d/0xd0
[<ffffffff8120295c>] vfs_read+0x9c/0x170
[<ffffffff812039f2>] SyS_pread64+0x92/0xc0
[<ffffffff816b89fd>] system_call_fastpath+0x16/0x1b
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 95 01 00 48 03 14 c5 20 8e b1 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #1132589
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb2d12456
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x126
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffb335d28b
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffb336b760
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0b31bc7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc0b1d905
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7b5
|
|
6
|
|
vmlinux |
0xffffffffb2db99fe
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6e
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc0aecb38
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x98
|
|
8
|
|
vmlinux |
0xffffffffb2dc11c6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x176
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc0aed35f
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc0aed3b6
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffffb2e863ab
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x4eb
|
|
12
|
|
ext4 |
0xffffffffc0aed3a0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x220
|
|
13
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffffb2db73a4
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xe4
|
|
14
|
|
vmlinux |
0xffffffffb2dc5dde
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xe
|
|
15
|
mpage_readpages
|
vmlinux |
0xffffffffb2e86712
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x112
|
|
16
|
|
ext4 |
0xffffffffc0aed3a0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x220
|
|
17
|
|
ext4 |
0xffffffffc0aed3a0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x220
|
|
18
|
|
vmlinux |
0xffffffffb2e0e308
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x98
|
|
19
|
ext4_readpages
|
ext4 |
0xffffffffc0aead2c
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x3c
|
|
20
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffb2dc3b3f
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x1cf
|
|
21
|
ondemand_readahead
|
vmlinux |
0xffffffffb2dc3cef
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x11f
|
|
22
|
page_cache_sync_readahead
|
vmlinux |
0xffffffffb2dc4114
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x44
|
|
23
|
generic_file_aio_read
|
vmlinux |
0xffffffffb2db83d2
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x2c2
|
|
24
|
ext4_file_read
|
ext4 |
0xffffffffc0ae6d93
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x43
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffffb2e40cc3
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x93
|
|
26
|
vfs_read
|
vmlinux |
0xffffffffb2e416ff
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x9f
|
|
27
|
SyS_read
|
vmlinux |
0xffffffffb2e425bf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffffb3375ddb
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#8 stuck for 22s! [python:31586]
Modules linked in: binfmt_misc devlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc dm_mirror dm_region_hash dm_log iTCO_wdt iTCO_vendor_support ext4 mbcache jbd2 dm_mod skx_edac intel_powerclamp coretemp snd_hda_codec_hdmi intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel snd_hda_codec_realtek lrw gf128mul glue_helper snd_hda_codec_generic ablk_helper cryptd snd_hda_intel pcspkr snd_hda_codec snd_hda_core joydev snd_hwdep snd_seq snd_seq_device snd_pcm snd_timer snd i2c_i801 sg lpc_ich mei_me soundcore mei wmi ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_cpufreq acpi_pad ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt
fb_sys_fops ttm crct10dif_pclmul ahci crct10dif_common drm crc32c_intel libahci libata i40e(OE) ptp pps_core drm_panel_orientation_quirks nfit libnvdimm fuse
CPU: 8 PID: 31586 Comm: python Tainted: G OEL ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: Supermicro SYS-7039A-I/X11DAi-N, BIOS 3.1a 06/11/2019
task: ffff9d9078ebb0c0 ti: ffff9dbe81d90000 task.ti: ffff9dbe81d90000
RIP: 0010:[<ffffffffb2d12456>] [<ffffffffb2d12456>] native_queued_spin_lock_slowpath+0x126/0x200
RSP: 0018:ffff9dbe81d93778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 00000000bb500c1d RCX: 0000000000410000
RDX: ffff9dc07dc5b780 RSI: 0000000000e90101 RDI: ffff9dc072fc5c80
RBP: ffff9dbe81d93778 R08: ffff9d907f81b780 R09: 0000000000000000
R10: ffff9d907f81f0c0 R11: ffffee9f3de4e640 R12: ffffffffc0b30fa3
R13: ffffffff7fc00150 R14: ffffffffb2e1ac69 R15: ffff9dbe81d93798
FS: 00002b3571d7c8c0(0000) GS:ffff9d907f800000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00002b36bc52c000 CR3: 000000581ff10000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffb335d28b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffb336b760>] _raw_spin_lock+0x20/0x30
[<ffffffffc0b31bc7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc0b1d905>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffffb2db99fe>] ? mempool_alloc+0x6e/0x170
[<ffffffffc0aecb38>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffb2dc11c6>] ? __alloc_pages_nodemask+0x176/0x420
[<ffffffffc0aed35f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc0aed3b6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffffb2e863ab>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc0aed3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffb2db73a4>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffffb2dc5dde>] ? lru_cache_add+0xe/0x10
[<ffffffffb2e86712>] mpage_readpages+0x112/0x170
[<ffffffffc0aed3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc0aed3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffb2e0e308>] ? alloc_pages_current+0x98/0x110
[<ffffffffc0aead2c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffffb2dc3b3f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffffb2dc3cef>] ondemand_readahead+0x11f/0x240
[<ffffffffb2dc4114>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffb2db83d2>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc0ae6d93>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffffb2e40cc3>] do_sync_read+0x93/0xe0
[<ffffffffb2e416ff>] vfs_read+0x9f/0x170
[<ffffffffb2e425bf>] SyS_read+0x7f/0xf0
[<ffffffffb3375ddb>] system_call_fastpath+0x22/0x27
Code: 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 a0 bc 94 b3 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2
Complete report #995453
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff82908802
Build id: 3.10.0-862.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff82f0842a
Build id: 3.10.0-862.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff82f15680
Build id: 3.10.0-862.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc05fc437
Build id: 3.10.0-862.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc05e8365
Build id: 3.10.0-862.el7.x86_64
|
0x7b5
|
|
6
|
|
vmlinux |
0xffffffff8299b26a
Build id: 3.10.0-862.el7.x86_64
|
0x8a
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc05b8998
Build id: 3.10.0-862.el7.x86_64
|
0x98
|
|
8
|
_ext4_get_block
|
ext4 |
0xffffffffc05b91bf
Build id: 3.10.0-862.el7.x86_64
|
0x1df
|
|
9
|
ext4_get_block
|
ext4 |
0xffffffffc05b9216
Build id: 3.10.0-862.el7.x86_64
|
0x16
|
|
10
|
do_mpage_readpage
|
vmlinux |
0xffffffff82a5ea8b
Build id: 3.10.0-862.el7.x86_64
|
0x4eb
|
|
11
|
|
ext4 |
0xffffffffc05b9200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
12
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff82994624
Build id: 3.10.0-862.el7.x86_64
|
0xe4
|
|
13
|
|
vmlinux |
0xffffffff829a335e
Build id: 3.10.0-862.el7.x86_64
|
0xe
|
|
14
|
mpage_readpages
|
vmlinux |
0xffffffff82a5edf2
Build id: 3.10.0-862.el7.x86_64
|
0x112
|
|
15
|
|
ext4 |
0xffffffffc05b9200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
16
|
|
ext4 |
0xffffffffc05b9200
Build id: 3.10.0-862.el7.x86_64
|
0x220
|
|
17
|
|
vmlinux |
0xffffffff829e8528
Build id: 3.10.0-862.el7.x86_64
|
0x98
|
|
18
|
ext4_readpages
|
ext4 |
0xffffffffc05b6b7c
Build id: 3.10.0-862.el7.x86_64
|
0x3c
|
|
19
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff829a122f
Build id: 3.10.0-862.el7.x86_64
|
0x1cf
|
|
20
|
ondemand_readahead
|
vmlinux |
0xffffffff829a1436
Build id: 3.10.0-862.el7.x86_64
|
0x116
|
|
21
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff829a1854
Build id: 3.10.0-862.el7.x86_64
|
0x44
|
|
22
|
generic_file_aio_read
|
vmlinux |
0xffffffff829954d2
Build id: 3.10.0-862.el7.x86_64
|
0x2c2
|
|
23
|
ext4_file_read
|
ext4 |
0xffffffffc05b2e53
Build id: 3.10.0-862.el7.x86_64
|
0x43
|
|
24
|
do_sync_read
|
vmlinux |
0xffffffff82a1a113
Build id: 3.10.0-862.el7.x86_64
|
0x93
|
|
25
|
vfs_read
|
vmlinux |
0xffffffff82a1ab3f
Build id: 3.10.0-862.el7.x86_64
|
0x9f
|
|
26
|
SyS_read
|
vmlinux |
0xffffffff82a1ba0f
Build id: 3.10.0-862.el7.x86_64
|
0x7f
|
|
27
|
system_call_fastpath
|
vmlinux |
0xffffffff82f1f7d5
Build id: 3.10.0-862.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 23s! [java:21750]
Modules linked in: nfnetlink_queue nfnetlink_log bluetooth rfkill ip_set binfmt_misc nfnetlink br_netfilter bridge stp llc overlay(T) vfat fat ext4 mbcache jbd2 skx_edac coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd iTCO_wdt iTCO_vendor_support mei_me mei sg i2c_i801 joydev megaraid_sas pcspkr i2c_core shpchp lpc_ich nfit acpi_power_meter libnvdimm acpi_cpufreq xfs libcrc32c sd_mod crc_t10dif crct10dif_generic mlx5_core crct10dif_pclmul crct10dif_common crc32c_intel ahci i40e libahci libata mlxfw devlink ptp pps_core dm_mirror dm_region_hash dm_log dm_mod [last unloaded: ip_tables]
CPU: 4 PID: 21750 Comm: java Kdump: loaded Tainted: G L ------------ T 3.10.0-862.el7.x86_64 #1
Hardware name: XFUSION 2288H V5/BC11SPSCB10, BIOS 8.27 03/08/2022
task: ffff8cef782a1fa0 ti: ffff8cf3323b8000 task.ti: ffff8cf3323b8000
RIP: 0010:[<ffffffff82908802>] [<ffffffff82908802>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff8cf3323bb780 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8cf3323bb7b0 RCX: 0000000000210000
RDX: ffff8ccb70459700 RSI: 0000000002090001 RDI: ffff8cf38ba00c80
RBP: ffff8cf3323bb780 R08: ffff8ccb6fb19700 R09: 0000000000000000
R10: ffff8ccb6fb1ba60 R11: ffffdddc61a74900 R12: 00000002a4801a0f
R13: 000001ff0043a6ff R14: 0000000000000000 R15: ffffffff82996afe
FS: 00007f6d9a5eb700(0000) GS:ffff8ccb6fb00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5be4338ff0 CR3: 000000545ce30000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff82f0842a>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff82f15680>] _raw_spin_lock+0x20/0x30
[<ffffffffc05fc437>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc05e8365>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffff8299b26a>] ? __rmqueue+0x8a/0x460
[<ffffffffc05b8998>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc05b91bf>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc05b9216>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff82a5ea8b>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc05b9200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff82994624>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffff829a335e>] ? lru_cache_add+0xe/0x10
[<ffffffff82a5edf2>] mpage_readpages+0x112/0x170
[<ffffffffc05b9200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc05b9200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff829e8528>] ? alloc_pages_current+0x98/0x110
[<ffffffffc05b6b7c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff829a122f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff829a1436>] ondemand_readahead+0x116/0x230
[<ffffffff829a1854>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff829954d2>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc05b2e53>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff82a1a113>] do_sync_read+0x93/0xe0
[<ffffffff82a1ab3f>] vfs_read+0x9f/0x170
[<ffffffff82a1ba0f>] SyS_read+0x7f/0xf0
[<ffffffff82f1f7d5>] system_call_fastpath+0x1c/0x21
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 00 97 01 00 48 03 14 c5 a0 53 53 83 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #947002
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff810fa336
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x116
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8169e6cf
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff816abc20
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc02948c7
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc02807f5
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x795
|
|
6
|
|
vmlinux |
0xffffffff810b4dc8
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x28
|
|
7
|
?
schedule_hrtimeout_range_clock
|
vmlinux |
0xffffffff816a873e
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xbe
|
|
8
|
ext4_map_blocks
|
ext4 |
0xffffffffc0251138
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x98
|
|
9
|
|
vmlinux |
0xffffffff8133d92d
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xd
|
|
10
|
_ext4_get_block
|
ext4 |
0xffffffffc025196f
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x1df
|
|
11
|
?
__mem_cgroup_commit_charge
|
vmlinux |
0xffffffff811f2db2
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xe2
|
|
12
|
ext4_get_block
|
ext4 |
0xffffffffc02519c6
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x16
|
|
13
|
do_mpage_readpage
|
vmlinux |
0xffffffff81242d28
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x4e8
|
|
14
|
|
ext4 |
0xffffffffc02519b0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x220
|
|
15
|
|
vmlinux |
0xffffffff8119151e
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xe
|
|
16
|
mpage_readpages
|
vmlinux |
0xffffffff8124306b
Build id: 3.10.0-693.11.1.el7.x86_64
|
0xeb
|
|
17
|
|
ext4 |
0xffffffffc02519b0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x220
|
|
18
|
|
ext4 |
0xffffffffc02519b0
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x220
|
|
19
|
|
vmlinux |
0xffffffff811d1078
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x98
|
|
20
|
ext4_readpages
|
ext4 |
0xffffffffc024f25c
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x3c
|
|
21
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff8118f40c
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x1cc
|
|
22
|
ondemand_readahead
|
vmlinux |
0xffffffff8118f606
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x116
|
|
23
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff8118fa14
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x44
|
|
24
|
generic_file_aio_read
|
vmlinux |
0xffffffff8118409b
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x29b
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffff811fff9d
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x8d
|
|
26
|
vfs_read
|
vmlinux |
0xffffffff8120099c
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x9c
|
|
27
|
SyS_read
|
vmlinux |
0xffffffff8120185f
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffff816b5089
Build id: 3.10.0-693.11.1.el7.x86_64
|
0x16
|
NMI watchdog: BUG: soft lockup - CPU#18 stuck for 22s! [afl-showmap:28988]
Modules linked in: veth nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) nf_conntrack_ftp udp_diag tcp_diag inet_diag mpt3sas mpt2sas raid_class xt_CHECKSUM scsi_transport_sas iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 mptctl nf_nat mptbase nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6_tables iptable_filter dm_mirror dm_region_hash dm_log dm_mod nvidia_drm(POE) sb_edac edac_core nvidia_modeset(POE) intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm iTCO_wdt iTCO_vendor_support irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nvidia(POE) snd_hda_codec_realtek snd_hda_codec_generic snd_hda_intel
snd_hda_codec snd_hda_core snd_hwdep snd_seq joydev snd_seq_device snd_pcm pcspkr sg snd_timer mei_me snd mei ioatdma i2c_i801 shpchp lpc_ich soundcore ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm igb drm ahci libahci mxm_wmi libata crct10dif_pclmul crct10dif_common ptp crc32c_intel pps_core megaraid_sas dca i2c_algo_bit i2c_core wmi
CPU: 18 PID: 28988 Comm: afl-showmap Tainted: P OEL ------------ T 3.10.0-693.11.1.el7.x86_64 #1
Hardware name: Powerleader PR4764GW/X10DRG-Q, BIOS 2.0a 08/29/2016
task: ffff880cdf929fa0 ti: ffff881df06a8000 task.ti: ffff881df06a8000
RIP: 0010:[<ffffffff810fa336>] [<ffffffff810fa336>] native_queued_spin_lock_slowpath+0x116/0x1e0
RSP: 0018:ffff881df06ab820 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffffffff810cd711 RCX: 0000000000910000
RDX: ffff883fff097880 RSI: 0000000000c10101 RDI: ffff883ffaa14480
RBP: ffff881df06ab820 R08: ffff883ffef17880 R09: 0000000000000000
R10: ffff883ffef19be0 R11: ffffea00a079d680 R12: ffffffff810cdb29
R13: ffff881df06ab7b0 R14: ffff883c9bb81800 R15: ffff883c9bb81800
FS: 00007fc1c570e0c0(0000) GS:ffff883ffef00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff243737390 CR3: 0000003fef2c2000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
ffff881df06ab830 ffffffff8169e6cf ffff881df06ab840 ffffffff816abc20
ffff881df06ab870 ffffffffc02948c7 ffff882baa5e9d08 0000000000000001
ffff88281e75a840 0000000000000001 ffff881df06ab958 ffffffffc02807f5
Call Trace:
[<ffffffff8169e6cf>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff816abc20>] _raw_spin_lock+0x20/0x30
[<ffffffffc02948c7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc02807f5>] ext4_ext_map_blocks+0x795/0xf40 [ext4]
[<ffffffff810b4dc8>] ? hrtimer_cancel+0x28/0x40
[<ffffffff816a873e>] ? schedule_hrtimeout_range_clock+0xbe/0x150
[<ffffffffc0251138>] ext4_map_blocks+0x98/0x6f0 [ext4]
[<ffffffff8133d92d>] ? list_del+0xd/0x30
[<ffffffffc025196f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffff811f2db2>] ? __mem_cgroup_commit_charge+0xe2/0x2f0
[<ffffffffc02519c6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff81242d28>] do_mpage_readpage+0x4e8/0x740
[<ffffffffc02519b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff8119151e>] ? lru_cache_add+0xe/0x10
[<ffffffff8124306b>] mpage_readpages+0xeb/0x150
[<ffffffffc02519b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc02519b0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff811d1078>] ? alloc_pages_current+0x98/0x110
[<ffffffffc024f25c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff8118f40c>] __do_page_cache_readahead+0x1cc/0x250
[<ffffffff8118f606>] ondemand_readahead+0x116/0x230
[<ffffffff8118fa14>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff8118409b>] generic_file_aio_read+0x29b/0x790
[<ffffffff811fff9d>] do_sync_read+0x8d/0xd0
[<ffffffff8120099c>] vfs_read+0x9c/0x170
[<ffffffff8120185f>] SyS_read+0x7f/0xe0
[<ffffffff816b5089>] system_call_fastpath+0x16/0x1b
Code: 0d 48 98 83 e2 30 48 81 c2 80 78 01 00 48 03 14 c5 e0 fd b0 81 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2
Complete report #861514
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9df11ffd
Build id: 3.10.0-957.el7.x86_64
|
0x1d
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9e55bf4b
Build id: 3.10.0-957.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9e56a400
Build id: 3.10.0-957.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0480bb7
Build id: 3.10.0-957.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc046c8f5
Build id: 3.10.0-957.el7.x86_64
|
0x7b5
|
|
6
|
|
mptscsih |
0xffffffffc022471e
Build id: 3.10.0-957.el7.x86_64
|
0x24e
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc043bb38
Build id: 3.10.0-957.el7.x86_64
|
0x98
|
|
8
|
|
vmlinux |
0xffffffff9dfc5839
Build id: 3.10.0-957.el7.x86_64
|
0x59
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc043c35f
Build id: 3.10.0-957.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc043c3b6
Build id: 3.10.0-957.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffff9e085b4b
Build id: 3.10.0-957.el7.x86_64
|
0x4eb
|
|
12
|
|
ext4 |
0xffffffffc043c3a0
Build id: 3.10.0-957.el7.x86_64
|
0x220
|
|
13
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff9dfb6de4
Build id: 3.10.0-957.el7.x86_64
|
0xe4
|
|
14
|
|
vmlinux |
0xffffffff9dfc588e
Build id: 3.10.0-957.el7.x86_64
|
0xe
|
|
15
|
mpage_readpages
|
vmlinux |
0xffffffff9e085eb2
Build id: 3.10.0-957.el7.x86_64
|
0x112
|
|
16
|
|
ext4 |
0xffffffffc043c3a0
Build id: 3.10.0-957.el7.x86_64
|
0x220
|
|
17
|
|
ext4 |
0xffffffffc043c3a0
Build id: 3.10.0-957.el7.x86_64
|
0x220
|
|
18
|
|
vmlinux |
0xffffffff9e00dcf8
Build id: 3.10.0-957.el7.x86_64
|
0x98
|
|
19
|
ext4_readpages
|
ext4 |
0xffffffffc0439d2c
Build id: 3.10.0-957.el7.x86_64
|
0x3c
|
|
20
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff9dfc35ef
Build id: 3.10.0-957.el7.x86_64
|
0x1cf
|
|
21
|
ondemand_readahead
|
vmlinux |
0xffffffff9dfc379f
Build id: 3.10.0-957.el7.x86_64
|
0x11f
|
|
22
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff9dfc3bc4
Build id: 3.10.0-957.el7.x86_64
|
0x44
|
|
23
|
generic_file_aio_read
|
vmlinux |
0xffffffff9dfb7e12
Build id: 3.10.0-957.el7.x86_64
|
0x2c2
|
|
24
|
ext4_file_read
|
ext4 |
0xffffffffc0435d93
Build id: 3.10.0-957.el7.x86_64
|
0x43
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffff9e0404d3
Build id: 3.10.0-957.el7.x86_64
|
0x93
|
|
26
|
vfs_read
|
vmlinux |
0xffffffff9e040f0f
Build id: 3.10.0-957.el7.x86_64
|
0x9f
|
|
27
|
SyS_read
|
vmlinux |
0xffffffff9e041dcf
Build id: 3.10.0-957.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffff9e574ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [Plex Script Hos:114580]
Modules linked in: xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink binfmt_misc ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack libcrc32c iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd vmw_balloon pcspkr joydev btusb btrtl btbcm btintel bluetooth rfkill sg snd_ens1371 snd_rawmidi
snd_ac97_codec ac97_bus snd_seq parport_pc parport snd_seq_device snd_pcm snd_timer snd soundcore vmw_vmci i2c_piix4 ip_tables ext4 mbcache jbd2 sd_mod sr_mod crc_t10dif cdrom crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops ttm libata drm mptspi scsi_transport_spi e1000 mptscsih mptbase drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 114580 Comm: Plex Script Hos Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff94bbf9109040 ti: ffff94bbf9330000 task.ti: ffff94bbf9330000
RIP: 0010:[<ffffffff9df11ffd>] [<ffffffff9df11ffd>] native_queued_spin_lock_slowpath+0x1d/0x200
RSP: 0018:ffff94bbf9333778 EFLAGS: 00000293
RAX: 0000000000000001 RBX: 000000001c33a069 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff94bb7547ec80
RBP: ffff94bbf9333778 R08: 000000000001f0c0 R09: ffffffffc046c8da
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 22s! [Plex Script Hos:19257]
R10: ffff94bcf761f0c0 R11: fffffd9bc2bb4ac0 R12: ffff94bbf9333760
R13: ffffffff9dfb92e5 R14: ffff94bbf93336e0 R15: ffff94bbf9109040
FS: 00007f9d24830700(0000) GS:ffff94bcf7600000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9d35974000 CR3: 00000001a8fd4000 CR4: 00000000007407f0
PKRU: 55555554
Call Trace:
[<ffffffff9e55bf4b>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9e56a400>] _raw_spin_lock+0x20/0x30
[<ffffffffc0480bb7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc046c8f5>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffffc022471e>] ? mptscsih_qcmd+0x24e/0x830 [mptscsih]
[<ffffffffc043bb38>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffff9dfc5839>] ? mark_page_accessed+0x59/0xa0
[<ffffffffc043c35f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc043c3b6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff9e085b4b>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc043c3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9dfb6de4>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffff9dfc588e>] ? lru_cache_add+0xe/0x10
[<ffffffff9e085eb2>] mpage_readpages+0x112/0x170
[<ffffffffc043c3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc043c3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9e00dcf8>] ? alloc_pages_current+0x98/0x110
[<ffffffffc0439d2c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff9dfc35ef>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff9dfc379f>] ondemand_readahead+0x11f/0x240
[<ffffffff9dfc3bc4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff9dfb7e12>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc0435d93>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff9e0404d3>] do_sync_read+0x93/0xe0
[<ffffffff9e040f0f>] vfs_read+0x9f/0x170
[<ffffffff9e041dcf>] SyS_read+0x7f/0xf0
[<ffffffff9e574ddb>] system_call_fastpath+0x22/0x27
Complete report #291998
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9ef15b66
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x126
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9f5733cb
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff9f581430
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0612187
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc05fdd45
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x7b5
|
|
6
|
|
vmlinux |
0xffffffff9eee7514
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x144
|
|
7
|
|
vmlinux |
0xffffffff9eee3766
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x86
|
|
8
|
ext4_map_blocks
|
ext4 |
0xffffffffc05ccc38
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x98
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc05cd45f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc05cd4b6
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffff9f08eb3b
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x4fb
|
|
12
|
|
ext4 |
0xffffffffc05cd4a0
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x220
|
|
13
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff9efbc9b4
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xe4
|
|
14
|
|
vmlinux |
0xffffffff9efcba2e
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0xe
|
|
15
|
mpage_readpages
|
vmlinux |
0xffffffff9f08eeb2
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x112
|
|
16
|
|
ext4 |
0xffffffffc05cd4a0
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x220
|
|
17
|
|
ext4 |
0xffffffffc05cd4a0
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x220
|
|
18
|
|
vmlinux |
0xffffffff9f015258
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x98
|
|
19
|
ext4_readpages
|
ext4 |
0xffffffffc05cad9c
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x3c
|
|
20
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff9efc975f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x1cf
|
|
21
|
ondemand_readahead
|
vmlinux |
0xffffffff9efc990f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x11f
|
|
22
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff9efc9d34
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x44
|
|
23
|
generic_file_aio_read
|
vmlinux |
0xffffffff9efbd9e2
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x2c2
|
|
24
|
ext4_file_read
|
ext4 |
0xffffffffc05c6dc3
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x43
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffff9f048353
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x93
|
|
26
|
vfs_read
|
vmlinux |
0xffffffff9f048d8f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x9f
|
|
27
|
SyS_read
|
vmlinux |
0xffffffff9f049c4f
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffff9f58bede
Build id: 3.10.0-1062.4.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#119 stuck for 22s! [ruby:150590]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter devlink ebtables ip6table_filter ip6_tables iptable_filter ext4 mbcache jbd2 iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi sunrpc iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev i2c_i801 ses lpc_ich enclosure mei_me scsi_transport_sas mei sg ipmi_si ipmi_devintf ipmi_msghandler acpi_pad pcc_cpufreq binfmt_misc ip_tables xfs libcrc32c raid1 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul
crct10dif_common crc32c_intel mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci ixgbe drm libahci igb libata aacraid mdio ptp pps_core dca drm_panel_orientation_quirks i2c_algo_bit wmi dm_mirror dm_region_hash dm_log dm_mod
CPU: 119 PID: 150590 Comm: ruby Tainted: G L ------------ 3.10.0-1062.4.1.el7.x86_64 #1
Hardware name: POWERLEADER DaPengWan/Default string, BIOS 5.11 12/05/2017
task: ffff9201e46420e0 ti: ffff921d756f4000 task.ti: ffff921d756f4000
RIP: 0010:[<ffffffff9ef15b66>] [<ffffffff9ef15b66>] native_queued_spin_lock_slowpath+0x126/0x200
RSP: 0018:ffff921d756f7778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff921d756f77a0 RCX: 0000000003b90000
RDX: ffff929d7fc5b800 RSI: 0000000004490101 RDI: ffff91de53741c80
RBP: ffff921d756f7778 R08: ffff927d7fcdb800 R09: 0000000000000000
R10: ffff927d7fcdf0c0 R11: fffff3d49397f380 R12: ffff927dfffda000
R13: ffffffff9efc6854 R14: ffff921d756f7800 R15: 0000000000000001
FS: 00007f7cfbe44740(0000) GS:ffff927d7fcc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f7cfb663480 CR3: 0000009f356ac000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9f5733cb>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff9f581430>] _raw_spin_lock+0x20/0x30
[<ffffffffc0612187>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc05fdd45>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffff9eee7514>] ? find_busiest_group+0x144/0x990
[<ffffffff9eee3766>] ? update_curr+0x86/0x1e0
[<ffffffffc05ccc38>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc05cd45f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc05cd4b6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff9f08eb3b>] do_mpage_readpage+0x4fb/0x760
[<ffffffffc05cd4a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9efbc9b4>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffff9efcba2e>] ? lru_cache_add+0xe/0x10
[<ffffffff9f08eeb2>] mpage_readpages+0x112/0x170
[<ffffffffc05cd4a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc05cd4a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff9f015258>] ? alloc_pages_current+0x98/0x110
[<ffffffffc05cad9c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff9efc975f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff9efc990f>] ondemand_readahead+0x11f/0x240
[<ffffffff9efc9d34>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff9efbd9e2>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc05c6dc3>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff9f048353>] do_sync_read+0x93/0xe0
[<ffffffff9f048d8f>] vfs_read+0x9f/0x170
[<ffffffff9f049c4f>] SyS_read+0x7f/0xf0
[<ffffffff9f58bede>] system_call_fastpath+0x25/0x2a
Code: 0d 48 98 83 e2 30 48 81 c2 00 b8 01 00 48 03 14 c5 e0 d6 b4 9f 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2
Complete report #99801
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffba313532
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffba95e2cb
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffba96c7a0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc030fbc7
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc02fb905
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x7b5
|
|
6
|
?
megasas_build_ldio_fusion
|
megaraid_sas |
0xffffffffc0176996
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x2e6
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc02cab38
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x98
|
|
8
|
|
vmlinux |
0xffffffffba3bf39e
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xee
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc02cb35f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc02cb3b6
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffffba48721b
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x4eb
|
|
12
|
|
ext4 |
0xffffffffc02cb3a0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x220
|
|
13
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffffba3b8474
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xe4
|
|
14
|
|
vmlinux |
0xffffffffba3c6eae
Build id: 3.10.0-957.27.2.el7.x86_64
|
0xe
|
|
15
|
mpage_readpages
|
vmlinux |
0xffffffffba487582
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x112
|
|
16
|
|
ext4 |
0xffffffffc02cb3a0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x220
|
|
17
|
|
ext4 |
0xffffffffc02cb3a0
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x220
|
|
18
|
|
vmlinux |
0xffffffffba40f438
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x98
|
|
19
|
ext4_readpages
|
ext4 |
0xffffffffc02c8d2c
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x3c
|
|
20
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffba3c4c0f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x1cf
|
|
21
|
ondemand_readahead
|
vmlinux |
0xffffffffba3c4dbf
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x11f
|
|
22
|
page_cache_sync_readahead
|
vmlinux |
0xffffffffba3c51e4
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x44
|
|
23
|
generic_file_aio_read
|
vmlinux |
0xffffffffba3b94a2
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x2c2
|
|
24
|
ext4_file_read
|
ext4 |
0xffffffffc02c4d93
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x43
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffffba441b33
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x93
|
|
26
|
vfs_read
|
vmlinux |
0xffffffffba44256f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x9f
|
|
27
|
SyS_read
|
vmlinux |
0xffffffffba44342f
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffffba976ddb
Build id: 3.10.0-957.27.2.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 23s! [ParalInputsProc:335021]
Modules linked in: binfmt_misc nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack libcrc32c xt_comment iptable_filter ip_gre gre ip_tunnel bonding dcdbas iTCO_wdt iTCO_vendor_support skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm ipmi_ssif irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg lpc_ich i2c_i801 mei_me mei ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter tcp_yeah tcp_vegas ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common ttm crc32c_intel drm ahci ixgbe libahci megaraid_sas igb libata mdio ptp pps_core dca i2c_algo_bit drm_panel_orientation_quirks nfit
libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 12 PID: 335021 Comm: ParalInputsProc Kdump: loaded Tainted: G L ------------ 3.10.0-957.27.2.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R740xd/08D89F, BIOS 1.4.8 05/21/2018
task: ffff927a11d730c0 ti: ffff927ab31b0000 task.ti: ffff927ab31b0000
RIP: 0010:[<ffffffffba313532>] [<ffffffffba313532>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff927ab31b3778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 00000000fb18279c RCX: 0000000000610000
RDX: ffff92997c3db780 RSI: 0000000001790001 RDI: ffff92797d1f4c80
RBP: ffff927ab31b3778 R08: ffff92797cb9b780 R09: 0000000000000000
R10: ffff92797cb9f0c0 R11: ffffd8607fe72340 R12: ffff92797ca80000
R13: ffffffffba22a621 R14: ffff927ab31b3728 R15: ffffffffba2e220e
FS: 00007f4cbe368700(0000) GS:ffff92797cb80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f465d02d00f CR3: 0000002331e26000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffba95e2cb>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffba96c7a0>] _raw_spin_lock+0x20/0x30
[<ffffffffc030fbc7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc02fb905>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffffc0176996>] ? megasas_build_ldio_fusion+0x2e6/0x970 [megaraid_sas]
[<ffffffffc02cab38>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffba3bf39e>] ? __rmqueue+0xee/0x460
[<ffffffffc02cb35f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc02cb3b6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffffba48721b>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc02cb3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffba3b8474>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffffba3c6eae>] ? lru_cache_add+0xe/0x10
[<ffffffffba487582>] mpage_readpages+0x112/0x170
[<ffffffffc02cb3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc02cb3a0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffba40f438>] ? alloc_pages_current+0x98/0x110
[<ffffffffc02c8d2c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffffba3c4c0f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffffba3c4dbf>] ondemand_readahead+0x11f/0x240
[<ffffffffba3c51e4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffba3b94a2>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc02c4d93>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffffba441b33>] do_sync_read+0x93/0xe0
[<ffffffffba44256f>] vfs_read+0x9f/0x170
[<ffffffffba44342f>] SyS_read+0x7f/0xf0
[<ffffffffba976ddb>] system_call_fastpath+0x22/0x27
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 80 b7 01 00 48 03 14 c5 a0 bf f4 ba 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #59902
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff92b17fd2
Build id: 3.10.0-1127.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9317a004
Build id: 3.10.0-1127.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff931886b0
Build id: 3.10.0-1127.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc05b78a7
Build id: 3.10.0-1127.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc05a3305
Build id: 3.10.0-1127.el7.x86_64
|
0x7d5
|
|
6
|
|
vmlinux |
0xffffffff92d8c104
Build id: 3.10.0-1127.el7.x86_64
|
0x84
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc0572068
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
8
|
|
vmlinux |
0xffffffff92c8615c
Build id: 3.10.0-1127.el7.x86_64
|
0xbc
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc057288f
Build id: 3.10.0-1127.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc05728e6
Build id: 3.10.0-1127.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffff92c9381b
Build id: 3.10.0-1127.el7.x86_64
|
0x4fb
|
|
12
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
13
|
|
vmlinux |
0xffffffff92be0232
Build id: 3.10.0-1127.el7.x86_64
|
0x32
|
|
14
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff92bbefd7
Build id: 3.10.0-1127.el7.x86_64
|
0xc7
|
|
15
|
|
vmlinux |
0xffffffff92bcd8ee
Build id: 3.10.0-1127.el7.x86_64
|
0xe
|
|
16
|
mpage_readpages
|
vmlinux |
0xffffffff92c93b92
Build id: 3.10.0-1127.el7.x86_64
|
0x112
|
|
17
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
18
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
19
|
|
vmlinux |
0xffffffff92c18e18
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
20
|
ext4_readpages
|
ext4 |
0xffffffffc05701cc
Build id: 3.10.0-1127.el7.x86_64
|
0x3c
|
|
21
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff92bcb61f
Build id: 3.10.0-1127.el7.x86_64
|
0x1cf
|
|
22
|
ondemand_readahead
|
vmlinux |
0xffffffff92bcb7cf
Build id: 3.10.0-1127.el7.x86_64
|
0x11f
|
|
23
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff92bcbbf4
Build id: 3.10.0-1127.el7.x86_64
|
0x44
|
|
24
|
generic_file_aio_read
|
vmlinux |
0xffffffff92bc0032
Build id: 3.10.0-1127.el7.x86_64
|
0x2c2
|
|
25
|
ext4_file_read
|
ext4 |
0xffffffffc056bdc3
Build id: 3.10.0-1127.el7.x86_64
|
0x43
|
|
26
|
do_sync_read
|
vmlinux |
0xffffffff92c4c583
Build id: 3.10.0-1127.el7.x86_64
|
0x93
|
|
27
|
vfs_read
|
vmlinux |
0xffffffff92c4cfbf
Build id: 3.10.0-1127.el7.x86_64
|
0x9f
|
|
28
|
SyS_read
|
vmlinux |
0xffffffff92c4de2f
Build id: 3.10.0-1127.el7.x86_64
|
0x7f
|
|
29
|
system_call_fastpath
|
vmlinux |
0xffffffff93192ed2
Build id: 3.10.0-1127.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#12 stuck for 22s! [python:29067]
Modules linked in: nvidia_uvm(OE) nvidia(POE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink xt_nat xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm snd_hda_codec_hdmi irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr i2c_i801
lpc_ich joydev mei_me snd_hda_intel mei snd_hda_codec snd_hda_core snd_hwdep snd_seq sg snd_seq_device snd_pcm snd_timer snd ioatdma soundcore ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mxm_wmi ahci drm igb libahci crct10dif_pclmul libata crct10dif_common megaraid_sas crc32c_intel ptp pps_core dca i2c_algo_bit drm_panel_orientation_quirks wmi dm_mirror dm_region_hash dm_log dm_mod [last unloaded: nvidia]
CPU: 12 PID: 29067 Comm: python Kdump: loaded Tainted: P OEL ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: Supermicro SYS-4028GR-TR2/X10DRG-O+-CPU, BIOS 3.0b 04/12/2018
task: ffff94cbc1389070 ti: ffff94bda864c000 task.ti: ffff94bda864c000
RIP: 0010:[<ffffffff92b17fd2>] [<ffffffff92b17fd2>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff94bda864f778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 00014a79cbede1de RCX: 0000000000610000
RDX: ffff94dc7fa9b8c0 RSI: 0000000000a10001 RDI: ffff94dc7a452480
RBP: ffff94bda864f778 R08: ffff94fc7f09b8c0 R09: 0000000000000000
R10: ffff94fc7f09f0c0 R11: ffffe9696d3aa080 R12: ffff94fc7f31ad40
R13: ffff94cbc13890d8 R14: 0000000097eece07 R15: 0000000a00000005
FS: 00007f1928d1f740(0000) GS:ffff94fc7f080000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000559ce3b16248 CR3: 000000157a0e4000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9317a004>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff931886b0>] _raw_spin_lock+0x20/0x30
[<ffffffffc05b78a7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc05a3305>] ext4_ext_map_blocks+0x7d5/0xfa0 [ext4]
[<ffffffff92d8c104>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffffc0572068>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffff92c8615c>] ? __find_get_block+0xbc/0x120
[<ffffffffc057288f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc05728e6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff92c9381b>] do_mpage_readpage+0x4fb/0x760
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff92be0232>] ? __inc_zone_page_state+0x32/0x60
[<ffffffff92bbefd7>] ? __add_to_page_cache_locked+0xc7/0x1a0
[<ffffffff92bcd8ee>] ? lru_cache_add+0xe/0x10
[<ffffffff92c93b92>] mpage_readpages+0x112/0x170
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff92c18e18>] ? alloc_pages_current+0x98/0x110
[<ffffffffc05701cc>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff92bcb61f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff92bcb7cf>] ondemand_readahead+0x11f/0x240
[<ffffffff92bcbbf4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff92bc0032>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc056bdc3>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff92c4c583>] do_sync_read+0x93/0xe0
[<ffffffff92c4cfbf>] vfs_read+0x9f/0x170
[<ffffffff92c4de2f>] SyS_read+0x7f/0xf0
[<ffffffff93192ed2>] system_call_fastpath+0x25/0x2a
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 0f 75 93 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #59106
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff92b17fd2
Build id: 3.10.0-1127.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff9317a004
Build id: 3.10.0-1127.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff931886b0
Build id: 3.10.0-1127.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc05b78a7
Build id: 3.10.0-1127.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc05a3305
Build id: 3.10.0-1127.el7.x86_64
|
0x7d5
|
|
6
|
ext4_map_blocks
|
ext4 |
0xffffffffc0572068
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
7
|
_ext4_get_block
|
ext4 |
0xffffffffc057288f
Build id: 3.10.0-1127.el7.x86_64
|
0x1df
|
|
8
|
ext4_get_block
|
ext4 |
0xffffffffc05728e6
Build id: 3.10.0-1127.el7.x86_64
|
0x16
|
|
9
|
do_mpage_readpage
|
vmlinux |
0xffffffff92c9381b
Build id: 3.10.0-1127.el7.x86_64
|
0x4fb
|
|
10
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
11
|
|
vmlinux |
0xffffffff92be0232
Build id: 3.10.0-1127.el7.x86_64
|
0x32
|
|
12
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff92bbefd7
Build id: 3.10.0-1127.el7.x86_64
|
0xc7
|
|
13
|
|
vmlinux |
0xffffffff92bcd8ee
Build id: 3.10.0-1127.el7.x86_64
|
0xe
|
|
14
|
mpage_readpages
|
vmlinux |
0xffffffff92c93b92
Build id: 3.10.0-1127.el7.x86_64
|
0x112
|
|
15
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
16
|
|
ext4 |
0xffffffffc05728d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
17
|
|
vmlinux |
0xffffffff92c18e18
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
18
|
ext4_readpages
|
ext4 |
0xffffffffc05701cc
Build id: 3.10.0-1127.el7.x86_64
|
0x3c
|
|
19
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff92bcb61f
Build id: 3.10.0-1127.el7.x86_64
|
0x1cf
|
|
20
|
ondemand_readahead
|
vmlinux |
0xffffffff92bcb7cf
Build id: 3.10.0-1127.el7.x86_64
|
0x11f
|
|
21
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff92bcbbf4
Build id: 3.10.0-1127.el7.x86_64
|
0x44
|
|
22
|
generic_file_aio_read
|
vmlinux |
0xffffffff92bc0032
Build id: 3.10.0-1127.el7.x86_64
|
0x2c2
|
|
23
|
ext4_file_read
|
ext4 |
0xffffffffc056bdc3
Build id: 3.10.0-1127.el7.x86_64
|
0x43
|
|
24
|
do_sync_read
|
vmlinux |
0xffffffff92c4c583
Build id: 3.10.0-1127.el7.x86_64
|
0x93
|
|
25
|
vfs_read
|
vmlinux |
0xffffffff92c4cfbf
Build id: 3.10.0-1127.el7.x86_64
|
0x9f
|
|
26
|
SyS_read
|
vmlinux |
0xffffffff92c4de2f
Build id: 3.10.0-1127.el7.x86_64
|
0x7f
|
|
27
|
system_call_fastpath
|
vmlinux |
0xffffffff93192ed2
Build id: 3.10.0-1127.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#10 stuck for 22s! [python:824]
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 22s! [python:816]
Modules linked in: nvidia_uvm(OE) nvidia(POE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink xt_nat xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm snd_hda_codec_hdmi irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr i2c_i801
lpc_ich joydev mei_me snd_hda_intel mei snd_hda_codec snd_hda_core snd_hwdep snd_seq sg snd_seq_device snd_pcm snd_timer snd ioatdma soundcore ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mxm_wmi ahci drm igb libahci crct10dif_pclmul libata crct10dif_common megaraid_sas crc32c_intel ptp pps_core dca i2c_algo_bit drm_panel_orientation_quirks wmi dm_mirror dm_region_hash dm_log dm_mod [last unloaded: nvidia]
CPU: 11 PID: 816 Comm: python Kdump: loaded Tainted: P OEL ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: Supermicro SYS-4028GR-TR2/X10DRG-O+-CPU, BIOS 3.0b 04/12/2018
task: ffff94fc6880b150 ti: ffff94ca8b2b8000 task.ti: ffff94ca8b2b8000
RIP: 0010:[<ffffffff92b17fd2>] [<ffffffff92b17fd2>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff94ca8b2bb778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000e120ebc5a29e RCX: 0000000000590000
RDX: ffff94dc7fa5b8c0 RSI: 0000000000490001 RDI: ffff94dc7a452480
RBP: ffff94ca8b2bb778 R08: ffff94fc7f05b8c0 R09: 0000000000000000
R10: ffff94fc7f05f0c0 R11: ffffe9697904bac0 R12: ffff94fc7f25ad40
R13: ffff94fc6880b1b8 R14: 00000000c3bae84b R15: 0000000000000000
FS: 00007f366caf7740(0000) GS:ffff94fc7f040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f35d8dd8570 CR3: 000000168448e000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9317a004>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff931886b0>] _raw_spin_lock+0x20/0x30
[<ffffffffc05b78a7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc05a3305>] ext4_ext_map_blocks+0x7d5/0xfa0 [ext4]
[<ffffffffc0572068>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc057288f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc05728e6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff92c9381b>] do_mpage_readpage+0x4fb/0x760
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff92be0232>] ? __inc_zone_page_state+0x32/0x60
[<ffffffff92bbefd7>] ? __add_to_page_cache_locked+0xc7/0x1a0
[<ffffffff92bcd8ee>] ? lru_cache_add+0xe/0x10
[<ffffffff92c93b92>] mpage_readpages+0x112/0x170
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc05728d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff92c18e18>] ? alloc_pages_current+0x98/0x110
[<ffffffffc05701cc>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff92bcb61f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff92bcb7cf>] ondemand_readahead+0x11f/0x240
[<ffffffff92bcbbf4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff92bc0032>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc056bdc3>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff92c4c583>] do_sync_read+0x93/0xe0
[<ffffffff92c4cfbf>] vfs_read+0x9f/0x170
[<ffffffff92c4de2f>] SyS_read+0x7f/0xf0
[<ffffffff93192ed2>] system_call_fastpath+0x25/0x2a
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 0f 75 93 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #52928
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffad917fd6
Build id: 3.10.0-1127.el7.x86_64
|
0x126
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffadf7a004
Build id: 3.10.0-1127.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffadf886b0
Build id: 3.10.0-1127.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc03a28a7
Build id: 3.10.0-1127.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc038e305
Build id: 3.10.0-1127.el7.x86_64
|
0x7d5
|
|
6
|
|
vmlinux |
0xffffffffad8e4eac
Build id: 3.10.0-1127.el7.x86_64
|
0x14c
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc035d068
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
8
|
_ext4_get_block
|
ext4 |
0xffffffffc035d88f
Build id: 3.10.0-1127.el7.x86_64
|
0x1df
|
|
9
|
ext4_get_block
|
ext4 |
0xffffffffc035d8e6
Build id: 3.10.0-1127.el7.x86_64
|
0x16
|
|
10
|
do_mpage_readpage
|
vmlinux |
0xffffffffada9381b
Build id: 3.10.0-1127.el7.x86_64
|
0x4fb
|
|
11
|
|
ext4 |
0xffffffffc035d8d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
12
|
|
vmlinux |
0xffffffffad9e0232
Build id: 3.10.0-1127.el7.x86_64
|
0x32
|
|
13
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffffad9befd7
Build id: 3.10.0-1127.el7.x86_64
|
0xc7
|
|
14
|
|
vmlinux |
0xffffffffad9cd8ee
Build id: 3.10.0-1127.el7.x86_64
|
0xe
|
|
15
|
mpage_readpages
|
vmlinux |
0xffffffffada93b92
Build id: 3.10.0-1127.el7.x86_64
|
0x112
|
|
16
|
|
ext4 |
0xffffffffc035d8d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
17
|
|
ext4 |
0xffffffffc035d8d0
Build id: 3.10.0-1127.el7.x86_64
|
0x220
|
|
18
|
|
vmlinux |
0xffffffffada18e18
Build id: 3.10.0-1127.el7.x86_64
|
0x98
|
|
19
|
ext4_readpages
|
ext4 |
0xffffffffc035b1cc
Build id: 3.10.0-1127.el7.x86_64
|
0x3c
|
|
20
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffad9cb61f
Build id: 3.10.0-1127.el7.x86_64
|
0x1cf
|
|
21
|
ondemand_readahead
|
vmlinux |
0xffffffffad9cb7cf
Build id: 3.10.0-1127.el7.x86_64
|
0x11f
|
|
22
|
page_cache_sync_readahead
|
vmlinux |
0xffffffffad9cbbf4
Build id: 3.10.0-1127.el7.x86_64
|
0x44
|
|
23
|
generic_file_aio_read
|
vmlinux |
0xffffffffad9c0032
Build id: 3.10.0-1127.el7.x86_64
|
0x2c2
|
|
24
|
ext4_file_read
|
ext4 |
0xffffffffc0356dc3
Build id: 3.10.0-1127.el7.x86_64
|
0x43
|
|
25
|
do_sync_read
|
vmlinux |
0xffffffffada4c583
Build id: 3.10.0-1127.el7.x86_64
|
0x93
|
|
26
|
vfs_read
|
vmlinux |
0xffffffffada4cfbf
Build id: 3.10.0-1127.el7.x86_64
|
0x9f
|
|
27
|
SyS_read
|
vmlinux |
0xffffffffada4de2f
Build id: 3.10.0-1127.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffffadf92ed2
Build id: 3.10.0-1127.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#13 stuck for 22s! [python:14290]
Modules linked in: nvidia_uvm(OE) nvidia(POE) xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink xt_nat xt_mark ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi snd_hda_codec_hdmi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr lpc_ich
i2c_i801 snd_hda_intel joydev snd_hda_codec mei_me snd_hda_core mei snd_hwdep snd_seq snd_seq_device snd_pcm sg snd_timer snd ioatdma soundcore ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm igb mxm_wmi ahci megaraid_sas crct10dif_pclmul libahci crct10dif_common crc32c_intel libata ptp pps_core dca i2c_algo_bit drm_panel_orientation_quirks wmi dm_mirror dm_region_hash dm_log dm_mod [last unloaded: nvidia]
CPU: 13 PID: 14290 Comm: python Kdump: loaded Tainted: P OEL ------------ 3.10.0-1127.el7.x86_64 #1
Hardware name: Supermicro SYS-4028GR-TR2/X10DRG-O+-CPU, BIOS 3.0b 04/12/2018
task: ffff93ed1863a0e0 ti: ffff93fb633cc000 task.ti: ffff93fb633cc000
RIP: 0010:[<ffffffffad917fd6>] [<ffffffffad917fd6>] native_queued_spin_lock_slowpath+0x126/0x200
RSP: 0018:ffff93fb633cf778 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000400 RCX: 0000000000690000
RDX: ffff93e8ffc9b8c0 RSI: 0000000000e10101 RDI: ffff9408fdd87c80
RBP: ffff93fb633cf778 R08: ffff9408ff0db8c0 R09: 0000000000000000
R10: ffff9408ff0df0c0 R11: fffff272eedd4540 R12: 0000000000001800
R13: 0000000000000217 R14: ffff93e9d3161418 R15: 0000000000000000
FS: 00007f2f2cfa3740(0000) GS:ffff9408ff0c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f2ebba7f670 CR3: 000000298dc3e000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffadf7a004>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffadf886b0>] _raw_spin_lock+0x20/0x30
[<ffffffffc03a28a7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc038e305>] ext4_ext_map_blocks+0x7d5/0xfa0 [ext4]
[<ffffffffad8e4eac>] ? update_curr+0x14c/0x1e0
[<ffffffffc035d068>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc035d88f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc035d8e6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffffada9381b>] do_mpage_readpage+0x4fb/0x760
[<ffffffffc035d8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffad9e0232>] ? __inc_zone_page_state+0x32/0x60
[<ffffffffad9befd7>] ? __add_to_page_cache_locked+0xc7/0x1a0
[<ffffffffad9cd8ee>] ? lru_cache_add+0xe/0x10
[<ffffffffada93b92>] mpage_readpages+0x112/0x170
[<ffffffffc035d8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc035d8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffada18e18>] ? alloc_pages_current+0x98/0x110
[<ffffffffc035b1cc>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffffad9cb61f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffffad9cb7cf>] ondemand_readahead+0x11f/0x240
[<ffffffffad9cbbf4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffad9c0032>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc0356dc3>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffffada4c583>] do_sync_read+0x93/0xe0
[<ffffffffada4cfbf>] vfs_read+0x9f/0x170
[<ffffffffada4de2f>] SyS_read+0x7f/0xf0
[<ffffffffadf92ed2>] system_call_fastpath+0x25/0x2a
Code: 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 e0 0f 55 ae 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 41 8b 40 08 <85> c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2
Complete report #11844
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa1b08b66
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x156
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa2109450
Build id: 3.10.0-862.3.2.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffa2116780
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0528437
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc0514365
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x7b5
|
|
6
|
ext4_map_blocks
|
ext4 |
0xffffffffc04e4998
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x98
|
|
7
|
_ext4_get_block
|
ext4 |
0xffffffffc04e51bf
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x1df
|
|
8
|
ext4_get_block
|
ext4 |
0xffffffffc04e5216
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x16
|
|
9
|
do_mpage_readpage
|
vmlinux |
0xffffffffa1c5edcb
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x4eb
|
|
10
|
|
ext4 |
0xffffffffc04e5200
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x220
|
|
11
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffffa1b94964
Build id: 3.10.0-862.3.2.el7.x86_64
|
0xe4
|
|
12
|
|
vmlinux |
0xffffffffa1ba369e
Build id: 3.10.0-862.3.2.el7.x86_64
|
0xe
|
|
13
|
mpage_readpages
|
vmlinux |
0xffffffffa1c5f132
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x112
|
|
14
|
|
ext4 |
0xffffffffc04e5200
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x220
|
|
15
|
|
ext4 |
0xffffffffc04e5200
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x220
|
|
16
|
|
vmlinux |
0xffffffffa1be8868
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x98
|
|
17
|
ext4_readpages
|
ext4 |
0xffffffffc04e2b7c
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x3c
|
|
18
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffa1ba156f
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x1cf
|
|
19
|
ondemand_readahead
|
vmlinux |
0xffffffffa1ba1776
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x116
|
|
20
|
page_cache_sync_readahead
|
vmlinux |
0xffffffffa1ba1b94
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x44
|
|
21
|
generic_file_aio_read
|
vmlinux |
0xffffffffa1b95812
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x2c2
|
|
22
|
ext4_file_read
|
ext4 |
0xffffffffc04dee53
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x43
|
|
23
|
do_sync_read
|
vmlinux |
0xffffffffa1c1a453
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x93
|
|
24
|
vfs_read
|
vmlinux |
0xffffffffa1c1ae7f
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x9f
|
|
25
|
SyS_read
|
vmlinux |
0xffffffffa1c1bd4f
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x7f
|
|
26
|
system_call_fastpath
|
vmlinux |
0xffffffffa212082f
Build id: 3.10.0-862.3.2.el7.x86_64
|
0x1c
|
NMI watchdog: BUG: soft lockup - CPU#37 stuck for 22s! [rsync:15109]
Modules linked in: binfmt_misc xt_nat veth ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dm_round_robin iscsi_tcp libiscsi_tcp bonding ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma sunrpc dm_mirror dm_region_hash dm_log rdma_ucm ib_uverbs ib_iser rdma_cm iw_cm ib_umad ib_ipoib libiscsi scsi_transport_iscsi ib_cm iTCO_wdt iTCO_vendor_support sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm vfat fat irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper
cryptd mlx4_ib ipmi_ssif ib_core pcspkr dm_multipath joydev dm_mod lpc_ich sg i2c_i801 hpilo hpwdt ioatdma shpchp dca wmi ipmi_si ipmi_devintf ipmi_msghandler pcc_cpufreq acpi_power_meter ip_tables ext4 mbcache jbd2 mlx4_en sd_mod crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea bnx2x sysfillrect sysimgblt fb_sys_fops ttm mlx4_core drm crct10dif_pclmul crct10dif_common crc32c_intel tg3 hpsa mdio i2c_core libcrc32c ptp devlink scsi_transport_sas pps_core
CPU: 37 PID: 15109 Comm: rsync Kdump: loaded Tainted: G L ------------ T 3.10.0-862.3.2.el7.x86_64 #1
Hardware name: HP ProLiant DL380 Gen9/ProLiant DL380 Gen9, BIOS P89 12/27/2015
task: ffff9022757c4f10 ti: ffff8fcffcbb0000 task.ti: ffff8fcffcbb0000
RIP: 0010:[<ffffffffa1b08b66>] [<ffffffffa1b08b66>] native_queued_spin_lock_slowpath+0x156/0x200
RSP: 0018:ffff8fcffcbb3780 EFLAGS: 00000202
RAX: 0000000000000101 RBX: ffffffffa1adc96e RCX: 0000000001290000
RDX: 0000000001290101 RSI: 0000000000000101 RDI: ffff90227e017c80
RBP: ffff8fcffcbb3780 R08: ffff90227f459700 R09: 0000000000000000
R10: ffff90227f45ba60 R11: ffffdd7f0663f480 R12: 0000000000000400
R13: 0000000000000000 R14: ffffffffa1bf55d9 R15: ffff8fcffcbb37a8
FS: 00007fdbe6d86740(0000) GS:ffff90227f440000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007efd3815b9c8 CR3: 0000005f687f6000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa2109450>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa2116780>] _raw_spin_lock+0x20/0x30
[<ffffffffc0528437>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc0514365>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffffc04e4998>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc04e51bf>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc04e5216>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffffa1c5edcb>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc04e5200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffa1b94964>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffffa1ba369e>] ? lru_cache_add+0xe/0x10
[<ffffffffa1c5f132>] mpage_readpages+0x112/0x170
[<ffffffffc04e5200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc04e5200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffa1be8868>] ? alloc_pages_current+0x98/0x110
[<ffffffffc04e2b7c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffffa1ba156f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffffa1ba1776>] ondemand_readahead+0x116/0x230
[<ffffffffa1ba1b94>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffa1b95812>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc04dee53>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffffa1c1a453>] do_sync_read+0x93/0xe0
[<ffffffffa1c1ae7f>] vfs_read+0x9f/0x170
[<ffffffffa1c1bd4f>] SyS_read+0x7f/0xf0
[<ffffffffa212082f>] system_call_fastpath+0x1c/0x21
Code: 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 17 0f b7 c2 85 c0 74 21 83 f8 03 75 10 eb 1a 66 2e 0f 1f 84 00 00 00 00 00 85 c0 74 0c f3 90 <8b> 17 0f b7 c2 83 f8 03 75 f0 be 01 00 00 00 eb 15 66 0f 1f 84
Complete report #4915
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa190ba92
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x122
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffffa1f0dc5f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffffa1f1b750
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc0a37487
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc0a233b5
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x7b5
|
|
6
|
|
vmlinux |
0xffffffffa1b55f34
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x84
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc09f3998
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x98
|
|
8
|
_ext4_get_block
|
ext4 |
0xffffffffc09f41bf
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1df
|
|
9
|
ext4_get_block
|
ext4 |
0xffffffffc09f4216
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x16
|
|
10
|
do_mpage_readpage
|
vmlinux |
0xffffffffa1a6310b
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x4eb
|
|
11
|
|
ext4 |
0xffffffffc09f4200
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x220
|
|
12
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffffa19979b4
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xe4
|
|
13
|
|
vmlinux |
0xffffffffa19a60be
Build id: 3.10.0-862.14.4.el7.x86_64
|
0xe
|
|
14
|
mpage_readpages
|
vmlinux |
0xffffffffa1a63472
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x112
|
|
15
|
|
ext4 |
0xffffffffc09f4200
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x220
|
|
16
|
|
ext4 |
0xffffffffc09f4200
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x220
|
|
17
|
|
vmlinux |
0xffffffffa19ec058
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x98
|
|
18
|
ext4_readpages
|
ext4 |
0xffffffffc09f1b7c
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x3c
|
|
19
|
__do_page_cache_readahead
|
vmlinux |
0xffffffffa19a3f8f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x1cf
|
|
20
|
ondemand_readahead
|
vmlinux |
0xffffffffa19a4196
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x116
|
|
21
|
page_cache_sync_readahead
|
vmlinux |
0xffffffffa19a45b4
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x44
|
|
22
|
generic_file_aio_read
|
vmlinux |
0xffffffffa1998862
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x2c2
|
|
23
|
ext4_file_read
|
ext4 |
0xffffffffc09ede53
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x43
|
|
24
|
do_sync_read
|
vmlinux |
0xffffffffa1a1e683
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x93
|
|
25
|
vfs_read
|
vmlinux |
0xffffffffa1a1f0af
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x9f
|
|
26
|
SyS_read
|
vmlinux |
0xffffffffa1a1ff7f
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x7f
|
|
27
|
system_call_fastpath
|
vmlinux |
0xffffffffa1f2579b
Build id: 3.10.0-862.14.4.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#30 stuck for 23s! [foremost:81887]
Modules linked in: fuse nfsd nfsv3 nfs_acl nfnetlink_queue nfnetlink_log xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache nf_conntrack_netbios_ns nf_conntrack_broadcast ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter sunrpc vfat fat ext4 mbcache jbd2 iTCO_wdt iTCO_vendor_support dcdbas skx_edac intel_powerclamp coretemp intel_rapl
iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ipmi_ssif i2c_i801 pcspkr ipmi_si nfit sg ipmi_devintf ipmi_msghandler lpc_ich mei_me libnvdimm mei shpchp acpi_power_meter acpi_pad ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul ttm crct10dif_common crc32c_intel drm ahci i40e libahci libata megaraid_sas ptp i2c_core pps_core dm_mirror dm_region_hash dm_log dm_mod
CPU: 30 PID: 81887 Comm: foremost Kdump: loaded Tainted: G L ------------ 3.10.0-862.14.4.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R640/008R9M, BIOS 1.2.71 12/04/2017
task: ffff97f5f0a74f10 ti: ffff97f65fdb0000 task.ti: ffff97f65fdb0000
RIP: 0010:[<ffffffffa190ba92>] [<ffffffffa190ba92>] native_queued_spin_lock_slowpath+0x122/0x200
RSP: 0018:ffff97f65fdb3780 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000f10000
RDX: ffff97f33de59740 RSI: 0000000001110101 RDI: ffff97f4ecf64c80
RBP: ffff97f65fdb3780 R08: ffff97f33ddd9740 R09: 0000000000000000
R10: ffff97f33dddbaa0 R11: ffffdbb544a12a40 R12: 0000000000000000
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
FS: 00007f4720a74740(0000) GS:ffff97f33ddc0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f471a0a1008 CR3: 00000003bce3c000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffffa1f0dc5f>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffffa1f1b750>] _raw_spin_lock+0x20/0x30
[<ffffffffc0a37487>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc0a233b5>] ext4_ext_map_blocks+0x7b5/0xf60 [ext4]
[<ffffffffa1b55f34>] ? __radix_tree_lookup+0x84/0xf0
[<ffffffffc09f3998>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffffc09f41bf>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc09f4216>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffffa1a6310b>] do_mpage_readpage+0x4eb/0x740
[<ffffffffc09f4200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffa19979b4>] ? __add_to_page_cache_locked+0xe4/0x190
[<ffffffffa19a60be>] ? lru_cache_add+0xe/0x10
[<ffffffffa1a63472>] mpage_readpages+0x112/0x170
[<ffffffffc09f4200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc09f4200>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffa19ec058>] ? alloc_pages_current+0x98/0x110
[<ffffffffc09f1b7c>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffffa19a3f8f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffffa19a4196>] ondemand_readahead+0x116/0x230
[<ffffffffa19a45b4>] page_cache_sync_readahead+0x44/0xb0
[<ffffffffa1998862>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc09ede53>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffffa1a1e683>] do_sync_read+0x93/0xe0
[<ffffffffa1a1f0af>] vfs_read+0x9f/0x170
[<ffffffffa1a1ff7f>] SyS_read+0x7f/0xf0
[<ffffffffa1f2579b>] system_call_fastpath+0x22/0x27
Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 40 97 01 00 48 03 14 c5 a0 54 53 a2 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b
Complete report #2699
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
native_queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8c91803e
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x1ce
|
|
2
|
queued_spin_lock_slowpath
|
vmlinux |
0xffffffff8cf7a024
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xb
|
|
3
|
_raw_spin_lock
|
vmlinux |
0xffffffff8cf886d0
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x20
|
|
4
|
ext4_es_lru_add
|
ext4 |
0xffffffffc03218a7
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x57
|
|
5
|
ext4_ext_map_blocks
|
ext4 |
0xffffffffc030d305
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x7d5
|
|
6
|
|
vmlinux |
0xffffffff8c9c8ed6
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x166
|
|
7
|
ext4_map_blocks
|
ext4 |
0xffffffffc02dc068
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x98
|
|
8
|
|
vmlinux |
0xffffffff8cb85019
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x39
|
|
9
|
_ext4_get_block
|
ext4 |
0xffffffffc02dc88f
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x1df
|
|
10
|
ext4_get_block
|
ext4 |
0xffffffffc02dc8e6
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x16
|
|
11
|
do_mpage_readpage
|
vmlinux |
0xffffffff8ca9387b
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x4fb
|
|
12
|
|
ext4 |
0xffffffffc02dc8d0
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x220
|
|
13
|
|
vmlinux |
0xffffffff8c9e0292
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x32
|
|
14
|
?
__add_to_page_cache_locked
|
vmlinux |
0xffffffff8c9bf037
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xc7
|
|
15
|
|
vmlinux |
0xffffffff8c9cd94e
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xe
|
|
16
|
mpage_readpages
|
vmlinux |
0xffffffff8ca93bf2
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x112
|
|
17
|
|
ext4 |
0xffffffffc02dc8d0
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x220
|
|
18
|
|
ext4 |
0xffffffffc02dc8d0
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x220
|
|
19
|
|
vmlinux |
0xffffffff8ca18e78
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x98
|
|
20
|
ext4_readpages
|
ext4 |
0xffffffffc02da1cc
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x3c
|
|
21
|
__do_page_cache_readahead
|
vmlinux |
0xffffffff8c9cb67f
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x1cf
|
|
22
|
ondemand_readahead
|
vmlinux |
0xffffffff8c9cb82f
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x11f
|
|
23
|
page_cache_sync_readahead
|
vmlinux |
0xffffffff8c9cbc54
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x44
|
|
24
|
generic_file_aio_read
|
vmlinux |
0xffffffff8c9c0092
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x2c2
|
|
25
|
ext4_file_read
|
ext4 |
0xffffffffc02d5dc3
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x43
|
|
26
|
|
vmlinux |
0xffffffff8c9cd94e
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0xe
|
|
27
|
do_sync_read
|
vmlinux |
0xffffffff8ca4c5e3
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x93
|
|
28
|
vfs_read
|
vmlinux |
0xffffffff8ca4d01f
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x9f
|
|
29
|
SyS_pread64
|
vmlinux |
0xffffffff8ca4e082
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x92
|
|
30
|
system_call_fastpath
|
vmlinux |
0xffffffff8cf92ed2
Build id: 3.10.0-1127.10.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#51 stuck for 22s! [clamscan:320565]
Modules linked in: tcp_lp joydev binfmt_misc tcp_diag inet_diag unix_diag fuse nfsv3 nfs_acl nfs lockd grace fscache xt_CHECKSUM tun ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype xt_nat ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack libcrc32c ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter rpcrdma ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib sunrpc rdma_ucm ib_uverbs
ib_umad rdma_cm ib_cm iw_cm dm_mirror dm_region_hash dm_log dm_mod iTCO_wdt iTCO_vendor_support bnxt_re ib_core skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel snd_hda_codec_hdmi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_hda_intel snd_hda_codec snd_hda_core snd_hwdep snd_seq snd_seq_device snd_pcm dell_smbios dcdbas dell_wmi_descriptor snd_timer pcspkr snd soundcore ipmi_ssif cdc_ether usbnet mii sg lpc_ich i2c_i801 mei_me mei ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter acpi_pad ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic nouveau video mxm_wmi i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm crct10dif_pclmul crct10dif_common
crc32c_intel libahci bnxt_en libata megaraid_sas drm_panel_orientation_quirks tg3 devlink ptp pps_core wmi nfit libnvdimm
CPU: 51 PID: 320565 Comm: clamscan Tainted: G L ------------ 3.10.0-1127.10.1.el7.x86_64 #1
Hardware name: Dell Inc. PowerEdge R640/0W23H8, BIOS 2.3.10 08/15/2019
task: ffff93f04f3ec1c0 ti: ffff93ead6da0000 task.ti: ffff93ead6da0000
RIP: 0010:[<ffffffff8c91803e>] [<ffffffff8c91803e>] native_queued_spin_lock_slowpath+0x1ce/0x200
RSP: 0018:ffff93ead6da3770 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 0000000000000002 RCX: 0000000000000001
RDX: 0000000000000101 RSI: 0000000000000001 RDI: ffff93c97d599480
RBP: ffff93ead6da3770 R08: 0000000000000101 R09: ffffffffc030d2ea
R10: ffff93f09d25f0c0 R11: ffffe97ffbe4fe40 R12: ffff93c93ffd9800
R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000020
FS: 00007fd3ad07f840(0000) GS:ffff93f09d240000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fd36a0a0000 CR3: 0000004a34ae8000 CR4: 00000000007607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
PKRU: 55555554
Call Trace:
[<ffffffff8cf7a024>] queued_spin_lock_slowpath+0xb/0xf
[<ffffffff8cf886d0>] _raw_spin_lock+0x20/0x30
[<ffffffffc03218a7>] ext4_es_lru_add+0x57/0x90 [ext4]
[<ffffffffc030d305>] ext4_ext_map_blocks+0x7d5/0xfa0 [ext4]
[<ffffffff8c9c8ed6>] ? __alloc_pages_nodemask+0x166/0x450
[<ffffffffc02dc068>] ext4_map_blocks+0x98/0x6e0 [ext4]
[<ffffffff8cb85019>] ? cpumask_any_but+0x39/0x50
[<ffffffffc02dc88f>] _ext4_get_block+0x1df/0x220 [ext4]
[<ffffffffc02dc8e6>] ext4_get_block+0x16/0x20 [ext4]
[<ffffffff8ca9387b>] do_mpage_readpage+0x4fb/0x760
[<ffffffffc02dc8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff8c9e0292>] ? __inc_zone_page_state+0x32/0x60
[<ffffffff8c9bf037>] ? __add_to_page_cache_locked+0xc7/0x1a0
[<ffffffff8c9cd94e>] ? lru_cache_add+0xe/0x10
[<ffffffff8ca93bf2>] mpage_readpages+0x112/0x170
[<ffffffffc02dc8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffffc02dc8d0>] ? _ext4_get_block+0x220/0x220 [ext4]
[<ffffffff8ca18e78>] ? alloc_pages_current+0x98/0x110
[<ffffffffc02da1cc>] ext4_readpages+0x3c/0x40 [ext4]
[<ffffffff8c9cb67f>] __do_page_cache_readahead+0x1cf/0x260
[<ffffffff8c9cb82f>] ondemand_readahead+0x11f/0x240
[<ffffffff8c9cbc54>] page_cache_sync_readahead+0x44/0xb0
[<ffffffff8c9c0092>] generic_file_aio_read+0x2c2/0x790
[<ffffffffc02d5dc3>] ext4_file_read+0x43/0x130 [ext4]
[<ffffffff8c9cd94e>] ? lru_cache_add+0xe/0x10
[<ffffffff8ca4c5e3>] do_sync_read+0x93/0xe0
[<ffffffff8ca4d01f>] vfs_read+0x9f/0x170
[<ffffffff8ca4e082>] SyS_pread64+0x92/0xc0
[<ffffffff8cf92ed2>] system_call_fastpath+0x25/0x2a
Code: 37 81 fe 00 01 00 00 74 f4 e9 93 fe ff ff 0f 1f 80 00 00 00 00 83 fa 01 75 11 0f 1f 00 e9 68 fe ff ff 0f 1f 00 85 c0 74 0c f3 90 <8b> 07 0f b6 c0 83 f8 03 75 f0 b8 01 00 00 00 66 89 07 5d c3 66