Complete report #1424587
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
isolate_freepages_block
|
vmlinux |
0xffffffff811a6885
Build id: 3.10.0-514.el7.x86_64
|
0xa5
|
|
2
|
compaction_alloc
|
vmlinux |
0xffffffff811a6cb5
Build id: 3.10.0-514.el7.x86_64
|
0x215
|
|
3
|
migrate_pages
|
vmlinux |
0xffffffff811e3969
Build id: 3.10.0-514.el7.x86_64
|
0x219
|
|
4
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff811a6aa0
Build id: 3.10.0-514.el7.x86_64
|
0x2c0
|
|
5
|
compact_zone
|
vmlinux |
0xffffffff811a7bb9
Build id: 3.10.0-514.el7.x86_64
|
0x2d9
|
|
6
|
compact_zone_order
|
vmlinux |
0xffffffff811a7dac
Build id: 3.10.0-514.el7.x86_64
|
0x9c
|
|
7
|
try_to_compact_pages
|
vmlinux |
0xffffffff811a8161
Build id: 3.10.0-514.el7.x86_64
|
0x121
|
|
8
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff8168139f
Build id: 3.10.0-514.el7.x86_64
|
0xac
|
|
9
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff81681873
Build id: 3.10.0-514.el7.x86_64
|
0x3ea
|
|
10
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8118af55
Build id: 3.10.0-514.el7.x86_64
|
0x405
|
|
11
|
alloc_pages_vma
|
vmlinux |
0xffffffff811d20ba
Build id: 3.10.0-514.el7.x86_64
|
0x9a
|
|
12
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff811eb5e3
Build id: 3.10.0-514.el7.x86_64
|
0x123
|
|
13
|
|
vmlinux |
0xffffffff8118f4c5
Build id: 3.10.0-514.el7.x86_64
|
0x65
|
|
14
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1347
Build id: 3.10.0-514.el7.x86_64
|
0xc37
|
|
15
|
__do_page_fault
|
vmlinux |
0xffffffff81691a94
Build id: 3.10.0-514.el7.x86_64
|
0x154
|
|
16
|
do_page_fault
|
vmlinux |
0xffffffff81691dc5
Build id: 3.10.0-514.el7.x86_64
|
0x35
|
|
17
|
page_fault
|
vmlinux |
0xffffffff8168e088
Build id: 3.10.0-514.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#23 stuck for 23s! [python:57317]
Modules linked in: tcp_lp fuse xt_CHECKSUM iptable_mangle ipt_REJECT nf_reject_ipv4 tun devlink ebtable_filter ebtables ip6table_filter ip6_tables nfsv3 nfs_acl nfs lockd grace fscache xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack xt_addrtype iptable_filter br_netfilter bridge stp llc overlay(T) sunrpc iTCO_wdt iTCO_vendor_support mxm_wmi intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd uas usb_storage pcspkr sg sb_edac edac_core i2c_i801 mei_me mei lpc_ich shpchp ipmi_devintf ipmi_si fjes ipmi_msghandler wmi acpi_pad acpi_power_meter ip_tables xfs libcrc32c sr_mod
cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common crc32c_intel ast drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm igb ixgbe i2c_algo_bit ahci mdio i2c_core libahci ptp pps_core dca libata dm_mirror dm_region_hash dm_log dm_mod
CPU: 23 PID: 57317 Comm: python Tainted: G ------------ T 3.10.0-514.el7.x86_64 #1
Hardware name: Supermicro X10DRi/X10DRi, BIOS 3.1a 04/16/2019
task: ffff88193bca1f60 ti: ffff883006fb8000 task.ti: ffff883006fb8000
RIP: 0010:[<ffffffff811a6885>] [<ffffffff811a6885>] isolate_freepages_block+0xa5/0x2c0
RSP: 0000:ffff883006fbb888 EFLAGS: 00000212
RAX: 00000000ffffffff RBX: 0000000000000003 RCX: ffff883006fbba70
RDX: 000000000237ee00 RSI: ffff883006fbbac8 RDI: ffff883006fbba70
RBP: ffff883006fbb918 R08: 0000000000000000 R09: ffff88307ffd7000
R10: 0000000001880800 R11: 0000000003080000 R12: ffff88307fbcce00
R13: ffff88307ffd7000 R14: 0000000001880800 R15: 0000000003080000
FS: 00007f8f1c3e1440(0000) GS:ffff88301ed40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8e61600000 CR3: 0000001c2b3d9000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000003080000 0000000001880800 ffff883006fbb968 0000000000000000
ffff883006fbba70 ffff883006fbbab0 ffff883006fbbac8 ffff883006fbba70
ffff88307ffd7000 0000000000000000 000000000237ee00 0000000000000001
Call Trace:
[<ffffffff811a6cb5>] compaction_alloc+0x215/0x290
[<ffffffff811e3969>] migrate_pages+0x219/0x790
[<ffffffff811a6aa0>] ? isolate_freepages_block+0x2c0/0x2c0
[<ffffffff811a7bb9>] compact_zone+0x2d9/0x430
[<ffffffff811a7dac>] compact_zone_order+0x9c/0xf0
[<ffffffff811a8161>] try_to_compact_pages+0x121/0x1a0
[<ffffffff8168139f>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff81681873>] __alloc_pages_slowpath+0x3ea/0x725
[<ffffffff8118af55>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff811d20ba>] alloc_pages_vma+0x9a/0x150
[<ffffffff811eb5e3>] do_huge_pmd_anonymous_page+0x123/0x520
[<ffffffff8118f4c5>] ? __lru_cache_add+0x65/0x90
[<ffffffff811b1347>] handle_mm_fault+0xc37/0xfe0
[<ffffffff81691a94>] __do_page_fault+0x154/0x450
[<ffffffff81691dc5>] do_page_fault+0x35/0x90
[<ffffffff8168e088>] page_fault+0x28/0x30
Code: a0 48 89 4d 90 45 89 c5 83 f8 80 74 2e 45 84 ed 0f 85 10 01 00 00 49 83 c4 01 48 83 c3 40 4c 39 65 c0 0f 86 fe 00 00 00 8b 43 18 <41> 83 c7 01 4d 85 f6 4c 0f 44 f3 83 f8 80 75 d2 48 8b 45 a8 0f
Complete report #425502
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
copy_page
|
vmlinux |
0xffffffff81325cd3
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x23
|
|
2
|
|
vmlinux |
0xffffffff811e3b2e
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x3ae
|
|
3
|
migrate_page
|
vmlinux |
0xffffffff811e3c6d
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x3d
|
|
4
|
move_to_new_page
|
vmlinux |
0xffffffff811e3d0d
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x8d
|
|
5
|
migrate_pages
|
vmlinux |
0xffffffff811e46a9
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x699
|
|
6
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff811a71b0
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x2c0
|
|
7
|
compact_zone
|
vmlinux |
0xffffffff811a82c9
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x2d9
|
|
8
|
compact_zone_order
|
vmlinux |
0xffffffff811a84bc
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x9c
|
|
9
|
try_to_compact_pages
|
vmlinux |
0xffffffff811a8871
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x121
|
|
10
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff81682526
Build id: 3.10.0-514.26.2.el7.x86_64
|
0xac
|
|
11
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff81682cb4
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x6a4
|
|
12
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8118b655
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x405
|
|
13
|
alloc_pages_vma
|
vmlinux |
0xffffffff811d297a
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x9a
|
|
14
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff811ebea3
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x123
|
|
15
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1b2f
Build id: 3.10.0-514.26.2.el7.x86_64
|
0xd7f
|
|
16
|
|
vmlinux |
0xffffffff811b7f65
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x305
|
|
17
|
__do_page_fault
|
vmlinux |
0xffffffff81692cc4
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x154
|
|
18
|
do_page_fault
|
vmlinux |
0xffffffff81692ff5
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x35
|
|
19
|
page_fault
|
vmlinux |
0xffffffff8168f208
Build id: 3.10.0-514.26.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [certbot:23721]
Modules linked in: intel_powerclamp iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul glue_helper ablk_helper cryptd nfit vmw_balloon sg libnvdimm pcspkr i2c_piix4 parport_pc shpchp parport vmw_vmci ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmwgfx vmxnet3 drm_kms_helper mptspi syscopyarea sysfillrect scsi_transport_spi sysimgblt fb_sys_fops mptscsih ttm mptbase drm ata_piix i2c_core libata floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 23721 Comm: certbot Tainted: G L ------------ 3.10.0-514.26.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff8800a4a8ce70 ti: ffff8800a8fb8000 task.ti: ffff8800a8fb8000
RIP: 0010:[<ffffffff81325cd3>] [<ffffffff81325cd3>] copy_page+0x23/0xe0
RSP: 0018:ffff8800a8fbb868 EFLAGS: 00010206
RAX: 00000000a4c9a000 RBX: 0000000000000000 RCX: 000000000000003a
RDX: 000000000001a088 RSI: ffff8800045c9000 RDI: ffff8800a4c9a000
RBP: ffff8800a8fbb8c0 R08: ffff8800bf424118 R09: ffff8800bf424000
R10: ffff8800a8fbb860 R11: 0000000000000000 R12: ffffea0000117300
R13: 0000000000000297 R14: ffff8800a8fbb7f8 R15: ffff8800bfa1a030
FS: 00007f6bdf201740(0000) GS:ffff8800bfa00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 0000000002460338 CR3: 00000000a7a21000 CR4: 00000000000407f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
ffff8800a8fb8000 ffffea0000117240 ffffffff811e3b2e ffff8800acf60f40
ffff8800b2b84710 0000000000000000 ffffea0000117240 ffffea0002932680
ffff8800b2b846f8 ffffea0002932680 0000000000000000 ffff8800a8fbb8e8
Call Trace:
[<ffffffff811e3b2e>] ? migrate_page_copy+0x3ae/0x4b0
[<ffffffff811e3c6d>] migrate_page+0x3d/0x50
[<ffffffff811e3d0d>] move_to_new_page+0x8d/0x250
[<ffffffff811e46a9>] migrate_pages+0x699/0x790
[<ffffffff811a71b0>] ? isolate_freepages_block+0x2c0/0x2c0
[<ffffffff811a82c9>] compact_zone+0x2d9/0x430
[<ffffffff811a84bc>] compact_zone_order+0x9c/0xf0
[<ffffffff811a8871>] try_to_compact_pages+0x121/0x1a0
[<ffffffff81682526>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff81682cb4>] __alloc_pages_slowpath+0x6a4/0x725
[<ffffffff8118b655>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff811d297a>] alloc_pages_vma+0x9a/0x150
[<ffffffff811ebea3>] do_huge_pmd_anonymous_page+0x123/0x520
[<ffffffff811b1b2f>] handle_mm_fault+0xd7f/0x1000
[<ffffffff811b7f65>] ? do_mmap_pgoff+0x305/0x3c0
[<ffffffff81692cc4>] __do_page_fault+0x154/0x450
[<ffffffff81692ff5>] do_page_fault+0x35/0x90
[<ffffffff8168f208>] page_fault+0x28/0x30
Code: c3 0f 1f 80 00 00 00 00 48 83 ec 10 48 89 1c 24 4c 89 64 24 08 b9 3b 00 00 00 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 48 ff c9 <48> 8b 06 48 8b 5e 08 48 8b 56 10 4c 8b 46 18 4c 8b 4e 20 4c 8b
Complete report #1336132
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
compaction_alloc
|
vmlinux |
0xffffffff81164048
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xf8
|
|
2
|
migrate_pages
|
vmlinux |
0xffffffff8119a974
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xd4
|
|
3
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff81163f50
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x380
|
|
4
|
compact_zone
|
vmlinux |
0xffffffff81164ee9
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x299
|
|
5
|
compact_zone_order
|
vmlinux |
0xffffffff811650ec
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x9c
|
|
6
|
try_to_compact_pages
|
vmlinux |
0xffffffff81165471
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xf1
|
|
7
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff815ddecf
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xac
|
|
8
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8114b2b9
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x6e9
|
|
9
|
alloc_pages_vma
|
vmlinux |
0xffffffff8118b96a
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x9a
|
|
10
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff8119fabb
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x10b
|
|
11
|
handle_mm_fault
|
vmlinux |
0xffffffff8116c104
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x184
|
|
12
|
|
vmlinux |
0xffffffff81172d76
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x186
|
|
13
|
__do_page_fault
|
vmlinux |
0xffffffff815ee286
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x156
|
|
14
|
|
vmlinux |
0xffffffff81173505
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x305
|
|
15
|
|
vmlinux |
0xffffffff8115e0a9
Build id: 3.10.0-123.20.1.el7.x86_64
|
0xb9
|
|
16
|
do_page_fault
|
vmlinux |
0xffffffff815ee68a
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x1a
|
|
17
|
page_fault
|
vmlinux |
0xffffffff815ea8c8
Build id: 3.10.0-123.20.1.el7.x86_64
|
0x28
|
BUG: soft lockup - CPU#3 stuck for 22s! [systemd-journal:345]
Modules linked in: xt_addrtype xt_conntrack ipt_MASQUERADE iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack bridge stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c loop iptable_filter ip_tables sg dm_mirror i2c_piix4 dm_region_hash dm_log mperf serio_raw virtio_balloon pcspkr dm_mod ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi virtio_blk virtio_net syscopyarea sysfillrect sysimgblt drm_kms_helper ata_piix ttm drm libata virtio_pci virtio_ring i2c_core virtio floppy
CPU: 3 PID: 345 Comm: systemd-journal Not tainted 3.10.0-123.20.1.el7.x86_64 #1
Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011
task: ffff8805fadba220 ti: ffff880036daa000 task.ti: ffff880036daa000
RIP: 0010:[<ffffffff81164048>] [<ffffffff81164048>] compaction_alloc+0xf8/0x240
RSP: 0000:ffff880036dab908 EFLAGS: 00010202
RAX: ffff88061ff72ba0 RBX: 00000000002eb600 RCX: 0000000000000000
RDX: ffff88061ff72000 RSI: 0000000000000001 RDI: 0000000000000070
RBP: ffff880036dab948 R08: ffff88061fb70320 R09: ffff88061ffc3e80
R10: 000000000010ee40 R11: 0000000000620000 R12: 0000000000000000
R13: ffff880036dab8f8 R14: ffff880036dab888 R15: ffffffff81163ec3
FS: 00007fc881b3c880(0000) GS:ffff88061fd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fc87a347000 CR3: 00000005fd15b000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
000000000010ee40 ffff88061ffc3e80 0000000000620000 ffffea00043b0fc0
ffff880036daba60 ffffea00043b0f80 ffffea00043b0fe0 ffff8805fadba220
ffff880036dab9e8 ffffffff8119a974 ffff8805fadba220 0001880500000000
Call Trace:
[<ffffffff8119a974>] migrate_pages+0xd4/0x600
[<ffffffff81163f50>] ? isolate_freepages_block+0x380/0x380
[<ffffffff81164ee9>] compact_zone+0x299/0x400
[<ffffffff811650ec>] compact_zone_order+0x9c/0xf0
[<ffffffff81165471>] try_to_compact_pages+0xf1/0x120
[<ffffffff815ddecf>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff8114b2b9>] __alloc_pages_nodemask+0x6e9/0xb10
[<ffffffff8118b96a>] alloc_pages_vma+0x9a/0x140
[<ffffffff8119fabb>] do_huge_pmd_anonymous_page+0x10b/0x410
[<ffffffff8116c104>] handle_mm_fault+0x184/0xd70
[<ffffffff81172d76>] ? mmap_region+0x186/0x610
[<ffffffff815ee286>] __do_page_fault+0x156/0x540
[<ffffffff81173505>] ? do_mmap_pgoff+0x305/0x3c0
[<ffffffff8115e0a9>] ? vm_mmap_pgoff+0xb9/0xe0
[<ffffffff815ee68a>] do_page_fault+0x1a/0x70
[<ffffffff815ea8c8>] page_fault+0x28/0x30
Code: ff ff 07 00 77 4b 4c 89 e2 48 c1 ea 16 48 8b 14 d5 c0 9a dc 81 48 85 d2 74 37 83 e0 7f 48 c1 e0 05 48 01 d0 74 2b f6 00 02 74 26 <49> 8b 45 00 48 89 c2 48 c1 e8 34 83 e0 03 48 c1 ea 36 48 69 c0
Complete report #1416250
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
compaction_alloc
|
vmlinux |
0xffffffff811a6bc8
Build id: 3.10.0-514.el7.x86_64
|
0x128
|
|
2
|
migrate_pages
|
vmlinux |
0xffffffff811e3969
Build id: 3.10.0-514.el7.x86_64
|
0x219
|
|
3
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff811a6aa0
Build id: 3.10.0-514.el7.x86_64
|
0x2c0
|
|
4
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff811a6aa0
Build id: 3.10.0-514.el7.x86_64
|
0x2c0
|
|
5
|
compact_zone
|
vmlinux |
0xffffffff811a7bb9
Build id: 3.10.0-514.el7.x86_64
|
0x2d9
|
|
6
|
compact_zone_order
|
vmlinux |
0xffffffff811a7dac
Build id: 3.10.0-514.el7.x86_64
|
0x9c
|
|
7
|
try_to_compact_pages
|
vmlinux |
0xffffffff811a8161
Build id: 3.10.0-514.el7.x86_64
|
0x121
|
|
8
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff8168139f
Build id: 3.10.0-514.el7.x86_64
|
0xac
|
|
9
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff81681873
Build id: 3.10.0-514.el7.x86_64
|
0x3ea
|
|
10
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8118af55
Build id: 3.10.0-514.el7.x86_64
|
0x405
|
|
11
|
alloc_pages_vma
|
vmlinux |
0xffffffff811d20ba
Build id: 3.10.0-514.el7.x86_64
|
0x9a
|
|
12
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff811eb5e3
Build id: 3.10.0-514.el7.x86_64
|
0x123
|
|
13
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1347
Build id: 3.10.0-514.el7.x86_64
|
0xc37
|
|
14
|
|
vmlinux |
0xffffffff811b76d5
Build id: 3.10.0-514.el7.x86_64
|
0x305
|
|
15
|
__do_page_fault
|
vmlinux |
0xffffffff81691a94
Build id: 3.10.0-514.el7.x86_64
|
0x154
|
|
16
|
do_page_fault
|
vmlinux |
0xffffffff81691dc5
Build id: 3.10.0-514.el7.x86_64
|
0x35
|
|
17
|
page_fault
|
vmlinux |
0xffffffff8168e088
Build id: 3.10.0-514.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [rpm:3230]
Modules linked in: nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink overlay(T) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc intel_powerclamp nfit iosf_mbi libnvdimm crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev sg cryptd pcspkr vmw_balloon shpchp parport_pc i2c_piix4 parport
vmw_vmci fuse ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops crct10dif_pclmul crct10dif_common crc32c_intel ttm serio_raw drm ahci ata_piix libahci vmxnet3 libata vmw_pvscsi i2c_core floppy fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 3230 Comm: rpm Tainted: G L ------------ T 3.10.0-514.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/09/2019
task: ffff8802ae638000 ti: ffff88023bf88000 task.ti: ffff88023bf88000
RIP: 0010:[<ffffffff811a6bc8>] [<ffffffff811a6bc8>] compaction_alloc+0x128/0x290
RSP: 0000:ffff88023bf8b928 EFLAGS: 00010202
RAX: ffff8802bffd65e0 RBX: 0000000000000000 RCX: 0000000000000000
RDX: ffff8802bffd6000 RSI: 0000000000000003 RDI: 0000000000000004
RBP: ffff88023bf8b968 R08: 0000000000000000 R09: ffff8802bffd8000
R10: 0000000000100200 R11: 00000000002c0000 R12: 0000000000218400
R13: 00000000002c0000 R14: ffffffff811a6a64 R15: ffff88023bf8b918
FS: 00007faf44a53840(0000) GS:ffff8802bfd80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007faf38bffff0 CR3: 0000000216222000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
0000000000100200 00000000002c0000 ffff8802bffd8000 ffffea00040007c0
ffff88023bf8ba80 ffffea0004000780 ffff88023bf8ba70 0000000000000000
ffff88023bf8ba08 ffffffff811e3969 ffff8802ae638000 ffff8802ae638000
Call Trace:
[<ffffffff811e3969>] migrate_pages+0x219/0x790
[<ffffffff811a6aa0>] ? isolate_freepages_block+0x2c0/0x2c0
[<ffffffff811a6aa0>] ? isolate_freepages_block+0x2c0/0x2c0
[<ffffffff811a7bb9>] compact_zone+0x2d9/0x430
[<ffffffff811a7dac>] compact_zone_order+0x9c/0xf0
[<ffffffff811a8161>] try_to_compact_pages+0x121/0x1a0
[<ffffffff8168139f>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff81681873>] __alloc_pages_slowpath+0x3ea/0x725
[<ffffffff8118af55>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff811d20ba>] alloc_pages_vma+0x9a/0x150
[<ffffffff811eb5e3>] do_huge_pmd_anonymous_page+0x123/0x520
[<ffffffff811b1347>] handle_mm_fault+0xc37/0xfe0
[<ffffffff811b76d5>] ? do_mmap_pgoff+0x305/0x3c0
[<ffffffff81691a94>] __do_page_fault+0x154/0x450
[<ffffffff81691dc5>] do_page_fault+0x35/0x90
[<ffffffff8168e088>] page_fault+0x28/0x30
Code: ff ff 07 00 77 d1 4c 89 e2 48 c1 ea 16 48 8b 14 d5 00 85 f4 81 48 85 d2 74 bd 83 e0 7f 48 c1 e0 05 48 01 d0 74 b1 f6 00 02 74 ac <49> 8b 45 00 a9 00 00 00 80 75 85 48 89 c2 48 c1 e8 29 48 c1 ea
Complete report #16710
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
compaction_alloc
|
vmlinux |
0xffffffff811a72c8
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x128
|
|
2
|
migrate_pages
|
vmlinux |
0xffffffff811e41a9
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x219
|
|
3
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff811a71a0
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x2c0
|
|
4
|
compact_zone
|
vmlinux |
0xffffffff811a82b9
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x2d9
|
|
5
|
compact_zone_order
|
vmlinux |
0xffffffff811a84ac
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x9c
|
|
6
|
try_to_compact_pages
|
vmlinux |
0xffffffff811a8861
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x121
|
|
7
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff81682466
Build id: 3.10.0-514.21.2.el7.x86_64
|
0xac
|
|
8
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff8168293a
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x3ea
|
|
9
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8118b645
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x405
|
|
10
|
alloc_pages_vma
|
vmlinux |
0xffffffff811d28fa
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x9a
|
|
11
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff811ebe23
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x123
|
|
12
|
handle_mm_fault
|
vmlinux |
0xffffffff811b1b6f
Build id: 3.10.0-514.21.2.el7.x86_64
|
0xdaf
|
|
13
|
|
vmlinux |
0xffffffff811b7f15
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x305
|
|
14
|
__do_page_fault
|
vmlinux |
0xffffffff81692c04
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x154
|
|
15
|
do_page_fault
|
vmlinux |
0xffffffff81692f35
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x35
|
|
16
|
page_fault
|
vmlinux |
0xffffffff8168f148
Build id: 3.10.0-514.21.2.el7.x86_64
|
0x28
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 22s! [beam.smp:2836]
Modules linked in: tcp_lp nls_utf8 isofs fuse binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmhgfs(OE) vmw_vsock_vmci_transport vsock snd_seq_midi snd_seq_midi_event intel_powerclamp coretemp iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd snd_ens1371 snd_rawmidi snd_ac97_codec ac97_bus snd_seq snd_seq_device snd_pcm snd_timer snd ppdev soundcore nfit libnvdimm vmw_balloon i2c_piix4 sg pcspkr vmw_vmci shpchp parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod sd_mod
cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel vmwgfx drm_kms_helper syscopyarea ata_piix sysfillrect mptspi sysimgblt fb_sys_fops scsi_transport_spi serio_raw ttm mptscsih e1000 libata drm mptbase i2c_core fjes dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 2836 Comm: beam.smp Tainted: G OEL ------------ 3.10.0-514.21.2.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff880225f02f10 ti: ffff88021cf58000 task.ti: ffff88021cf58000
RIP: 0010:[<ffffffff811a72c8>] [<ffffffff811a72c8>] compaction_alloc+0x128/0x290
RSP: 0000:ffff88021cf5b928 EFLAGS: 00010202
RAX: ffff88023ffd67e0 RBX: 0000000200000001 RCX: 0000000000000004
RDX: ffff88023ffd6000 RSI: 0000000000000001 RDI: 000000000000002c
RBP: ffff88021cf5b968 R08: ffff88023fbd5ee0 R09: ffff88023ffd8000
R10: 0000000000221c00 R11: 0000000000240000 R12: ffffea0008870000
R13: 0000000000221cc2 R14: ffffffff811a7164 R15: ffff88021cf5b918
FS: 00007f0c50efe700(0000) GS:ffff880235640000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0bd5000000 CR3: 000000021f70c000 CR4: 00000000003407e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Stack:
0000000000221c00 ffff88023ffd8000 0000000000240000 ffffea00041693c0
0000000000000000 ffffea0004169380 ffffea0008878000 0000000000000000
ffff88021cf5ba08 ffffffff811e41a9 ffff880223fb2640 ffff880225f02f10
Call Trace:
[<ffffffff811e41a9>] migrate_pages+0x219/0x790
[<ffffffff811a71a0>] ? isolate_freepages_block+0x2c0/0x2c0
[<ffffffff811a82b9>] compact_zone+0x2d9/0x430
[<ffffffff811a84ac>] compact_zone_order+0x9c/0xf0
[<ffffffff811a8861>] try_to_compact_pages+0x121/0x1a0
[<ffffffff81682466>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff8168293a>] __alloc_pages_slowpath+0x3ea/0x725
[<ffffffff8118b645>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff811d28fa>] alloc_pages_vma+0x9a/0x150
[<ffffffff811ebe23>] do_huge_pmd_anonymous_page+0x123/0x520
[<ffffffff811b1b6f>] handle_mm_fault+0xdaf/0x1040
[<ffffffff811b7f15>] ? do_mmap_pgoff+0x305/0x3c0
[<ffffffff81692c04>] __do_page_fault+0x154/0x450
[<ffffffff81692f35>] do_page_fault+0x35/0x90
[<ffffffff8168f148>] page_fault+0x28/0x30
Code: ff ff 07 00 77 d1 4c 89 e2 48 c1 ea 16 48 8b 14 d5 80 c5 f4 81 48 85 d2 74 bd 83 e0 7f 48 c1 e0 05 48 01 d0 74 b1 f6 00 02 74 ac <49> 8b 45 00 a9 00 00 00 80 75 85 48 89 c2 48 c1 e8 29 48 c1 ea
Complete report #11293
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
isolate_freepages_block
|
vmlinux |
0xffffffff8118df8c
Build id: 3.10.0-327.el7.x86_64
|
0xac
|
|
2
|
compaction_alloc
|
vmlinux |
0xffffffff8118e42f
Build id: 3.10.0-327.el7.x86_64
|
0x1cf
|
|
3
|
migrate_pages
|
vmlinux |
0xffffffff811c71f9
Build id: 3.10.0-327.el7.x86_64
|
0x1e9
|
|
4
|
?
isolate_freepages_block
|
vmlinux |
0xffffffff8118e260
Build id: 3.10.0-327.el7.x86_64
|
0x380
|
|
5
|
compact_zone
|
vmlinux |
0xffffffff8118f259
Build id: 3.10.0-327.el7.x86_64
|
0x299
|
|
6
|
compact_zone_order
|
vmlinux |
0xffffffff8118f45c
Build id: 3.10.0-327.el7.x86_64
|
0x9c
|
|
7
|
try_to_compact_pages
|
vmlinux |
0xffffffff8118f811
Build id: 3.10.0-327.el7.x86_64
|
0x121
|
|
8
|
__alloc_pages_direct_compact
|
vmlinux |
0xffffffff816305c8
Build id: 3.10.0-327.el7.x86_64
|
0xac
|
|
9
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff811734e8
Build id: 3.10.0-327.el7.x86_64
|
0x788
|
|
10
|
alloc_pages_vma
|
vmlinux |
0xffffffff811b78ca
Build id: 3.10.0-327.el7.x86_64
|
0x9a
|
|
11
|
do_huge_pmd_anonymous_page
|
vmlinux |
0xffffffff811cc2d3
Build id: 3.10.0-327.el7.x86_64
|
0x123
|
|
12
|
handle_mm_fault
|
vmlinux |
0xffffffff81196c78
Build id: 3.10.0-327.el7.x86_64
|
0x1a8
|
|
13
|
|
vmlinux |
0xffffffff8119e695
Build id: 3.10.0-327.el7.x86_64
|
0x305
|
|
14
|
__do_page_fault
|
vmlinux |
0xffffffff81640e22
Build id: 3.10.0-327.el7.x86_64
|
0x152
|
|
15
|
do_page_fault
|
vmlinux |
0xffffffff81641113
Build id: 3.10.0-327.el7.x86_64
|
0x23
|
|
16
|
page_fault
|
vmlinux |
0xffffffff8163d408
Build id: 3.10.0-327.el7.x86_64
|
0x28
|
BUG: soft lockup - CPU#0 stuck for 22s! [1_scheduler:15690]
Modules linked in: tcp_diag inet_diag rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache fuse xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter dm_mirror dm_region_hash dm_log dm_mod ppdev sg virtio_balloon pcspkr i2c_piix4 parport_pc parport nfsd auth_rpcgss nfs_acl lockd grace sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_blk virtio_net cirrus syscopyarea sysfillrect sysimgblt drm_kms_helper ttm serio_raw virtio_pci virtio_ring virtio drm ata_piix i2c_core libata floppy
CPU: 0 PID: 15690 Comm: 1_scheduler Not tainted 3.10.0-327.el7.x86_64 #1
Hardware name: Red Hat KVM, BIOS 0.5.1 01/01/2007
task: ffff8803cebe7300 ti: ffff8803bb5cc000 task.ti: ffff8803bb5cc000
RIP: 0010:[<ffffffff8118df8c>] [<ffffffff8118df8c>] isolate_freepages_block+0xac/0x380
RSP: 0000:ffff8803bb5cf8f8 EFLAGS: 00010286
RAX: 00000000ffffffff RBX: ffffffff8152c671 RCX: ffff8803bb5cfb48
RDX: 0000000000240200 RSI: 0000000000240000 RDI: ffff8803bb5cfaf0
RBP: ffff8803bb5cf990 R08: 0000000000000000 R09: ffff88041ffcc000
R10: 0000000009000000 R11: 0000000000420000 R12: ffff8803d27a7cf8
R13: 000000000000009c R14: ffff880404edc000 R15: ffff8803d27a7cf8
FS: 00007febe95ff700(0000) GS:ffff88041fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007feb9e400000 CR3: 00000003b686e000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Stack:
000000000041fc00 ffffea00107edbc0 000000000041fb6f 00000000d27a7cf8
0000000000240000 ffff8803bb5cfaf0 0000000000420000 0000000000000000
ffff8803bb5cfb30 ffff8803bb5cfb48 ffff8803bb5cfaf0 0000000000000000
Call Trace:
[<ffffffff8118e42f>] compaction_alloc+0x1cf/0x240
[<ffffffff811c71f9>] migrate_pages+0x1e9/0x790
[<ffffffff8118e260>] ? isolate_freepages_block+0x380/0x380
[<ffffffff8118f259>] compact_zone+0x299/0x400
[<ffffffff8118f45c>] compact_zone_order+0x9c/0xf0
[<ffffffff8118f811>] try_to_compact_pages+0x121/0x1a0
[<ffffffff816305c8>] __alloc_pages_direct_compact+0xac/0x196
[<ffffffff811734e8>] __alloc_pages_nodemask+0x788/0xb90
[<ffffffff811b78ca>] alloc_pages_vma+0x9a/0x140
[<ffffffff811cc2d3>] do_huge_pmd_anonymous_page+0x123/0x510
[<ffffffff81196c78>] handle_mm_fault+0x1a8/0xf50
[<ffffffff8119e695>] ? do_mmap_pgoff+0x305/0x3c0
[<ffffffff81640e22>] __do_page_fault+0x152/0x420
[<ffffffff81641113>] do_page_fault+0x23/0x80
[<ffffffff8163d408>] page_fault+0x28/0x30
Code: c0 00 4d 89 ee 48 89 4d b0 41 89 c5 eb 1d 90 49 83 c7 01 48 83 c3 40 4d 39 fc 0f 86 07 01 00 00 41 83 c5 01 4d 85 f6 4c 0f 44 f3 <8b> 43 18 83 f8 80 75 dc 48 8b 45 b8 0f b6 55 c0 48 8d 75 c8 4c