Complete report #1419019
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
put_cpu_partial
|
vmlinux |
0xffffffff81e25b1e
Build id: 3.10.0-1160.el7.x86_64
|
0xae
|
|
2
|
__slab_free
|
vmlinux |
0xffffffff81e25f20
Build id: 3.10.0-1160.el7.x86_64
|
0x1b0
|
|
3
|
|
vmlinux |
0xffffffff81e25e0d
Build id: 3.10.0-1160.el7.x86_64
|
0x9d
|
|
4
|
|
vmlinux |
0xffffffff81e8d6bc
Build id: 3.10.0-1160.el7.x86_64
|
0x8c
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffff81e26642
Build id: 3.10.0-1160.el7.x86_64
|
0x1e2
|
|
6
|
mempool_free_slab
|
vmlinux |
0xffffffff81dc1067
Build id: 3.10.0-1160.el7.x86_64
|
0x17
|
|
7
|
mempool_free
|
vmlinux |
0xffffffff81dc147f
Build id: 3.10.0-1160.el7.x86_64
|
0x4f
|
|
8
|
sg_pool_free
|
vmlinux |
0xffffffff81fb91d5
Build id: 3.10.0-1160.el7.x86_64
|
0x45
|
|
9
|
__sg_free_table
|
vmlinux |
0xffffffff81f9b218
Build id: 3.10.0-1160.el7.x86_64
|
0x78
|
|
10
|
|
vmlinux |
0xffffffff81fb9190
Build id: 3.10.0-1160.el7.x86_64
|
0x90
|
|
11
|
sg_free_table_chained
|
vmlinux |
0xffffffff81fb90a7
Build id: 3.10.0-1160.el7.x86_64
|
0x27
|
|
12
|
scsi_release_buffers
|
vmlinux |
0xffffffff820e9b5a
Build id: 3.10.0-1160.el7.x86_64
|
0x6a
|
|
13
|
scsi_end_request
|
vmlinux |
0xffffffff820ed40d
Build id: 3.10.0-1160.el7.x86_64
|
0x12d
|
|
14
|
scsi_io_completion
|
vmlinux |
0xffffffff820ed688
Build id: 3.10.0-1160.el7.x86_64
|
0x168
|
|
15
|
scsi_finish_command
|
vmlinux |
0xffffffff820e297c
Build id: 3.10.0-1160.el7.x86_64
|
0xdc
|
|
16
|
scsi_softirq_done
|
vmlinux |
0xffffffff820ecbd0
Build id: 3.10.0-1160.el7.x86_64
|
0x130
|
|
17
|
blk_done_softirq
|
vmlinux |
0xffffffff81f5d226
Build id: 3.10.0-1160.el7.x86_64
|
0x96
|
|
18
|
__do_softirq
|
vmlinux |
0xffffffff81ca4b95
Build id: 3.10.0-1160.el7.x86_64
|
0xf5
|
|
19
|
run_ksoftirqd
|
vmlinux |
0xffffffff81ca4d58
Build id: 3.10.0-1160.el7.x86_64
|
0x38
|
|
20
|
smpboot_thread_fn
|
vmlinux |
0xffffffff81cce674
Build id: 3.10.0-1160.el7.x86_64
|
0x144
|
|
21
|
|
vmlinux |
0xffffffff81cce530
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
22
|
kthread
|
vmlinux |
0xffffffff81cc5c21
Build id: 3.10.0-1160.el7.x86_64
|
0xd1
|
|
23
|
|
vmlinux |
0xffffffff81cc5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
|
24
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff82393df7
Build id: 3.10.0-1160.el7.x86_64
|
0x21
|
|
25
|
|
vmlinux |
0xffffffff81cc5b50
Build id: 3.10.0-1160.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 21s! [ksoftirqd/7:44]
Modules linked in: tcp_diag inet_diag xt_comment xt_mark xt_owner ip6table_mangle ip_set_bitmap_port ip_set_hash_net xt_set iptable_raw ip_set_hash_ipport ip_set dummy nf_conntrack_netlink nfnetlink xt_addrtype br_netfilter overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc iosf_mbi crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw vmw_balloon gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr vmw_vmci parport_pc i2c_piix4 parport ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack ip_tables xfs libcrc32c sr_mod cdrom ata_generic
pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt sd_mod fb_sys_fops ttm crc_t10dif crct10dif_generic ahci drm ata_piix libahci nfit libata libnvdimm crct10dif_pclmul crct10dif_common crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 7 PID: 44 Comm: ksoftirqd/7 Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff8eeca5352100 ti: ffff8eeca5364000 task.ti: ffff8eeca5364000
RIP: 0010:[<ffffffff81e25b1e>] [<ffffffff81e25b1e>] put_cpu_partial+0xae/0x110
RSP: 0000:ffff8eeca5367b20 EFLAGS: 00000202
RAX: 000000000000009f RBX: ffff8efb1c7ee800 RCX: 0000000100200019
RDX: ffff8eecbfc00e10 RSI: ffffd1f982b739a0 RDI: 0000000000000202
RBP: ffff8eeca5367b40 R08: ffff8eecbfc00e10 R09: 0000000100200019
R10: 0000000000000001 R11: ffff8eed0e3efe00 R12: ffffffff81e24b02
R13: ffff8eeca5367b10 R14: 0000000000000000 R15: ffff8eecbfc00e00
FS: 0000000000000000(0000) GS:ffff8efb3f3c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000048d180 CR3: 00000004e7c10000 CR4: 00000000007607e0
PKRU: 00000000
Call Trace:
[<ffffffff81e25f20>] __slab_free+0x1b0/0x290
[<ffffffff81e25e0d>] ? __slab_free+0x9d/0x290
[<ffffffff81e8d6bc>] ? bio_endio+0x8c/0x130
[<ffffffff81e26642>] kmem_cache_free+0x1e2/0x200
[<ffffffff81dc1067>] mempool_free_slab+0x17/0x20
[<ffffffff81dc147f>] mempool_free+0x4f/0xa0
[<ffffffff81fb91d5>] sg_pool_free+0x45/0x50
[<ffffffff81f9b218>] __sg_free_table+0x78/0xa0
[<ffffffff81fb9190>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffff81fb90a7>] sg_free_table_chained+0x27/0x30
[<ffffffff820e9b5a>] scsi_release_buffers+0x6a/0x70
[<ffffffff820ed40d>] scsi_end_request+0x12d/0x1e0
[<ffffffff820ed688>] scsi_io_completion+0x168/0x720
[<ffffffff820e297c>] scsi_finish_command+0xdc/0x140
[<ffffffff820ecbd0>] scsi_softirq_done+0x130/0x160
[<ffffffff81f5d226>] blk_done_softirq+0x96/0xc0
[<ffffffff81ca4b95>] __do_softirq+0xf5/0x280
[<ffffffff81ca4d58>] run_ksoftirqd+0x38/0x50
[<ffffffff81cce674>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff81cce530>] ? lg_double_unlock+0x40/0x40
[<ffffffff81cc5c21>] kthread+0xd1/0xe0
[<ffffffff81cc5b50>] ? insert_kthread_work+0x40/0x40
[<ffffffff82393df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff81cc5b50>] ? insert_kthread_work+0x40/0x40
Code: 1f 44 00 00 49 89 c6 fa 66 0f 1f 44 00 00 49 8b 34 24 65 48 03 35 d3 b6 1e 7e 4c 89 e7 48 83 c6 18 e8 17 ef ff ff 4c 89 f7 57 9d <0f> 1f 44 00 00 ba 01 00 00 00 31 ff 31 c9 e9 74 ff ff ff 0f 1f
Complete report #1108662
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
cmpxchg_double_slab.isra.51
|
vmlinux |
0xffffffffb361b700
Build id: 3.10.0-957.21.3.el7.x86_64
|
-
|
|
2
|
|
vmlinux |
0xffffffffb361b9a1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x81
|
|
3
|
|
xfs |
0xffffffffc046165b
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x7b
|
|
4
|
kmem_cache_free
|
vmlinux |
0xffffffffb361bdf2
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x1e2
|
|
5
|
mempool_free_slab
|
vmlinux |
0xffffffffb35b98c7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x17
|
|
6
|
mempool_free
|
vmlinux |
0xffffffffb35b9cdf
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x4f
|
|
7
|
sg_pool_free
|
vmlinux |
0xffffffffb37a87f5
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x45
|
|
8
|
__sg_free_table
|
vmlinux |
0xffffffffb378ae68
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x78
|
|
9
|
|
vmlinux |
0xffffffffb37a87b0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x90
|
|
10
|
sg_free_table_chained
|
vmlinux |
0xffffffffb37a86c7
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x27
|
|
11
|
scsi_release_buffers
|
vmlinux |
0xffffffffb38d791a
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x6a
|
|
12
|
scsi_end_request
|
vmlinux |
0xffffffffb38dafdd
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x12d
|
|
13
|
scsi_io_completion
|
vmlinux |
0xffffffffb38db258
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x168
|
|
14
|
scsi_finish_command
|
vmlinux |
0xffffffffb38d070c
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xdc
|
|
15
|
scsi_softirq_done
|
vmlinux |
0xffffffffb38da7a2
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x132
|
|
16
|
blk_done_softirq
|
vmlinux |
0xffffffffb374d546
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x96
|
|
17
|
__do_softirq
|
vmlinux |
0xffffffffb34a1075
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xf5
|
|
18
|
run_ksoftirqd
|
vmlinux |
0xffffffffb34a1238
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x38
|
|
19
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb34ca734
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x144
|
|
20
|
|
vmlinux |
0xffffffffb34ca5f0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
21
|
kthread
|
vmlinux |
0xffffffffb34c1da1
Build id: 3.10.0-957.21.3.el7.x86_64
|
0xd1
|
|
22
|
|
vmlinux |
0xffffffffb34c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
|
23
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb3b75c37
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x21
|
|
24
|
|
vmlinux |
0xffffffffb34c1cd0
Build id: 3.10.0-957.21.3.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [ksoftirqd/2:19]
Modules linked in: vsock_diag tcp_diag udp_diag inet_diag unix_diag binfmt_misc xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun devlink ip6t_rpfilter ipt_REJECT nf_reject_ipv4 ip6t_REJECT nf_reject_ipv6 xt_conntrack ip_set nfnetlink ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc sb_edac ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr joydev sg parport_pc parport vmw_vmci i2c_piix4 ip_tables
xfs libcrc32c sr_mod cdrom ata_generic sd_mod pata_acpi crc_t10dif crct10dif_generic vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm crct10dif_pclmul crct10dif_common libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 2 PID: 19 Comm: ksoftirqd/2 Kdump: loaded Tainted: G W L ------------ 3.10.0-957.21.3.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff8b3df7488000 ti: ffff8b3df7490000 task.ti: ffff8b3df7490000
RIP: 0010:[<ffffffffb361b700>] [<ffffffffb361b700>] cmpxchg_double_slab.isra.51+0x0/0xf0
RSP: 0018:ffff8b3df7493b38 EFLAGS: 00000246
RAX: 0000000000000000 RBX: ffff8b3df7493b68 RCX: 0000000100100010
RDX: 0000000000000000 RSI: fffff4d14e903a80 RDI: 0000000040000000
RBP: ffff8b3df7493be8 R08: ffff8b40240ea600 R09: 000000018010000f
R10: ffff8b41bfd1f120 R11: fffff4d14e903a80 R12: 0000000000000000
R13: 000000010010000f R14: ffff8b3dffc01700 R15: 0000000180120011
FS: 0000000000000000(0000) GS:ffff8b41bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa730b4e210 CR3: 00000002a7010000 CR4: 00000000003607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb361b9a1>] ? __slab_free+0x81/0x2f0
[<ffffffffc046165b>] ? xfs_buf_bio_end_io+0x7b/0x80 [xfs]
[<ffffffffb361bdf2>] kmem_cache_free+0x1e2/0x200
[<ffffffffb35b98c7>] mempool_free_slab+0x17/0x20
[<ffffffffb35b9cdf>] mempool_free+0x4f/0xa0
[<ffffffffb37a87f5>] sg_pool_free+0x45/0x50
[<ffffffffb378ae68>] __sg_free_table+0x78/0xa0
[<ffffffffb37a87b0>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffffb37a86c7>] sg_free_table_chained+0x27/0x30
[<ffffffffb38d791a>] scsi_release_buffers+0x6a/0x70
[<ffffffffb38dafdd>] scsi_end_request+0x12d/0x1e0
[<ffffffffb38db258>] scsi_io_completion+0x168/0x6a0
[<ffffffffb38d070c>] scsi_finish_command+0xdc/0x140
[<ffffffffb38da7a2>] scsi_softirq_done+0x132/0x160
[<ffffffffb374d546>] blk_done_softirq+0x96/0xc0
[<ffffffffb34a1075>] __do_softirq+0xf5/0x280
[<ffffffffb34a1238>] run_ksoftirqd+0x38/0x50
[<ffffffffb34ca734>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb34ca5f0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb34c1da1>] kthread+0xd1/0xe0
[<ffffffffb34c1cd0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb3b75c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb34c1cd0>] ? insert_kthread_work+0x40/0x40
Code: 92 48 c7 c7 40 8d 09 b4 e8 fe b9 54 00 48 83 c4 08 b8 01 00 00 00 5b 41 5c 41 5d 41 5e 5d c3 66 90 66 2e 0f 1f 84 00 00 00 00 00 <55> 49 89 d2 48 89 e5 53 48 83 e4 f0 48 83 ec 40 f7 c7 00 00 00
Complete report #1106403
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
kmem_cache_free
|
vmlinux |
0xffffffffb6426ee5
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x85
|
|
2
|
mempool_free_slab
|
vmlinux |
0xffffffffb63c15a7
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x17
|
|
3
|
mempool_free
|
vmlinux |
0xffffffffb63c19bf
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x4f
|
|
4
|
sg_pool_free
|
vmlinux |
0xffffffffb65b9c55
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x45
|
|
5
|
__sg_free_table
|
vmlinux |
0xffffffffb659bc98
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x78
|
|
6
|
|
vmlinux |
0xffffffffb65b9c10
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x90
|
|
7
|
sg_free_table_chained
|
vmlinux |
0xffffffffb65b9b27
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x27
|
|
8
|
scsi_release_buffers
|
vmlinux |
0xffffffffb66ea85a
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x6a
|
|
9
|
scsi_end_request
|
vmlinux |
0xffffffffb66ee10d
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x12d
|
|
10
|
scsi_io_completion
|
vmlinux |
0xffffffffb66ee388
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x168
|
|
11
|
scsi_finish_command
|
vmlinux |
0xffffffffb66e367c
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xdc
|
|
12
|
scsi_softirq_done
|
vmlinux |
0xffffffffb66ed8d0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x130
|
|
13
|
blk_done_softirq
|
vmlinux |
0xffffffffb655db56
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x96
|
|
14
|
__do_softirq
|
vmlinux |
0xffffffffb62a4d85
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xf5
|
|
15
|
run_ksoftirqd
|
vmlinux |
0xffffffffb62a4f48
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x38
|
|
16
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb62ce9e4
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x144
|
|
17
|
|
vmlinux |
0xffffffffb62ce8a0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x40
|
|
18
|
kthread
|
vmlinux |
0xffffffffb62c5f91
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0xd1
|
|
19
|
|
vmlinux |
0xffffffffb62c5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x40
|
|
20
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb6999df7
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x21
|
|
21
|
|
vmlinux |
0xffffffffb62c5ec0
Build id: 3.10.0-1160.71.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 21s! [ksoftirqd/1:14]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink vmw_vsock_vmci_transport vsock sunrpc sb_edac coretemp ppdev iosf_mbi vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper parport_pc sg pcspkr cryptd joydev parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix mptspi scsi_transport_spi libata mptscsih crct10dif_pclmul crct10dif_common crc32c_intel
mptbase serio_raw vmxnet3 drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G L ------------ 3.10.0-1160.71.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/14/2014
task: ffff9663f62fa100 ti: ffff9663f6344000 task.ti: ffff9663f6344000
RIP: 0010:[<ffffffffb6426ee5>] [<ffffffffb6426ee5>] kmem_cache_free+0x85/0x200
RSP: 0018:ffff9663f6347bf8 EFLAGS: 00010246
RAX: 0000000000000000 RBX: 00000000fffffffb RCX: 0000000000000000
RDX: 0000000010576650 RSI: ffff966394be6a00 RDI: ffff9663ffc03700
RBP: ffff9663f6347c10 R08: 0000000000000000 R09: ffffffffb63c15a7
R10: ffff9666afc5f120 R11: fffffca58452f980 R12: ffff9663f6347c10
R13: ffff9665d27aa900 R14: 0000000000000000 R15: ffff9666afd26000
FS: 0000000000000000(0000) GS:ffff9666afc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f76b0436370 CR3: 000000042a138000 CR4: 00000000000407e0
Call Trace:
[<ffffffffb63c15a7>] mempool_free_slab+0x17/0x20
[<ffffffffb63c19bf>] mempool_free+0x4f/0xa0
[<ffffffffb65b9c55>] sg_pool_free+0x45/0x50
[<ffffffffb659bc98>] __sg_free_table+0x78/0xa0
[<ffffffffb65b9c10>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffffb65b9b27>] sg_free_table_chained+0x27/0x30
[<ffffffffb66ea85a>] scsi_release_buffers+0x6a/0x70
[<ffffffffb66ee10d>] scsi_end_request+0x12d/0x1e0
[<ffffffffb66ee388>] scsi_io_completion+0x168/0x720
[<ffffffffb66e367c>] scsi_finish_command+0xdc/0x140
[<ffffffffb66ed8d0>] scsi_softirq_done+0x130/0x160
[<ffffffffb655db56>] blk_done_softirq+0x96/0xc0
[<ffffffffb62a4d85>] __do_softirq+0xf5/0x280
[<ffffffffb62a4f48>] run_ksoftirqd+0x38/0x50
[<ffffffffb62ce9e4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb62ce8a0>] ? lg_double_unlock+0x40/0x40
[<ffffffffb62c5f91>] kthread+0xd1/0xe0
[<ffffffffb62c5ec0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb6999df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb62c5ec0>] ? insert_kthread_work+0x40/0x40
Code: e6 80 0f 85 59 01 00 00 49 89 c3 4c 8b 17 65 4c 03 15 10 a3 be 49 49 8b 52 08 4d 39 5a 10 0f 85 50 01 00 00 48 63 47 20 49 8b 0a <48> 89 0c 03 48 8d 4a 01 4c 8b 07 49 8b 02 65 49 0f c7 08 0f 94
Complete report #537823
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__slab_free
|
vmlinux |
0xffffffff86c1b32a
Build id: 3.10.0-957.el7.x86_64
|
0x9a
|
|
2
|
|
vmlinux |
0xffffffff86c56de3
Build id: 3.10.0-957.el7.x86_64
|
0x73
|
|
3
|
kmem_cache_free
|
vmlinux |
0xffffffff86c1b762
Build id: 3.10.0-957.el7.x86_64
|
0x1e2
|
|
4
|
mempool_free_slab
|
vmlinux |
0xffffffff86bb9307
Build id: 3.10.0-957.el7.x86_64
|
0x17
|
|
5
|
mempool_free
|
vmlinux |
0xffffffff86bb971f
Build id: 3.10.0-957.el7.x86_64
|
0x4f
|
|
6
|
sg_pool_free
|
vmlinux |
0xffffffff86da7f75
Build id: 3.10.0-957.el7.x86_64
|
0x45
|
|
7
|
__sg_free_table
|
vmlinux |
0xffffffff86d8a5e8
Build id: 3.10.0-957.el7.x86_64
|
0x78
|
|
8
|
|
vmlinux |
0xffffffff86da7f30
Build id: 3.10.0-957.el7.x86_64
|
0x90
|
|
9
|
sg_free_table_chained
|
vmlinux |
0xffffffff86da7e47
Build id: 3.10.0-957.el7.x86_64
|
0x27
|
|
10
|
scsi_release_buffers
|
vmlinux |
0xffffffff86ed6b7a
Build id: 3.10.0-957.el7.x86_64
|
0x6a
|
|
11
|
scsi_end_request
|
vmlinux |
0xffffffff86eda23d
Build id: 3.10.0-957.el7.x86_64
|
0x12d
|
|
12
|
scsi_io_completion
|
vmlinux |
0xffffffff86eda4b8
Build id: 3.10.0-957.el7.x86_64
|
0x168
|
|
13
|
scsi_finish_command
|
vmlinux |
0xffffffff86ecf96c
Build id: 3.10.0-957.el7.x86_64
|
0xdc
|
|
14
|
scsi_softirq_done
|
vmlinux |
0xffffffff86ed9a02
Build id: 3.10.0-957.el7.x86_64
|
0x132
|
|
15
|
blk_done_softirq
|
vmlinux |
0xffffffff86d4cce6
Build id: 3.10.0-957.el7.x86_64
|
0x96
|
|
16
|
__do_softirq
|
vmlinux |
0xffffffff86aa0f05
Build id: 3.10.0-957.el7.x86_64
|
0xf5
|
|
17
|
run_ksoftirqd
|
vmlinux |
0xffffffff86aa10c8
Build id: 3.10.0-957.el7.x86_64
|
0x38
|
|
18
|
smpboot_thread_fn
|
vmlinux |
0xffffffff86aca5c4
Build id: 3.10.0-957.el7.x86_64
|
0x144
|
|
19
|
|
vmlinux |
0xffffffff86aca480
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
20
|
kthread
|
vmlinux |
0xffffffff86ac1c31
Build id: 3.10.0-957.el7.x86_64
|
0xd1
|
|
21
|
|
vmlinux |
0xffffffff86ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
22
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff87174c37
Build id: 3.10.0-957.el7.x86_64
|
0x21
|
|
23
|
|
vmlinux |
0xffffffff86ac1b60
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#11 stuck for 31s! [ksoftirqd/11:64]
Modules linked in: veth xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack libcrc32c br_netfilter bridge stp llc overlay(T) vmw_vsock_vmci_transport vsock ppdev sb_edac iosf_mbi crc32_pclmul vmw_balloon ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci i2c_piix4 nfit libnvdimm parport_pc parport auth_rpcgss sunrpc ip_tables ext4 mbcache jbd2 sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper crct10dif_pclmul crct10dif_common crc32c_intel syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix serio_raw libata mptspi scsi_transport_spi mptscsih vmxnet3 mptbase drm_panel_orientation_quirks
dm_mirror dm_region_hash dm_log dm_mod
CPU: 11 PID: 64 Comm: ksoftirqd/11 Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 12/12/2018
task: ffff9154b4291040 ti: ffff9154b4298000 task.ti: ffff9154b4298000
RIP: 0010:[<ffffffff86c1b32a>] [<ffffffff86c1b32a>] __slab_free+0x9a/0x2f0
RSP: 0000:ffff9154b429bb40 EFLAGS: 00000246
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000180400030
RDX: ffff9153f8001000 RSI: ffffeca102e00000 RDI: ffff9154bfc07700
RBP: ffff9154b429bbe8 R08: 0000000000000001 R09: ffffffff86bb9307
R10: ffff915b5d8df120 R11: ffffeca102e00000 R12: 0008000000000000
R13: 00000000000000fd R14: 0000000000000086 R15: 0000000000000086
FS: 0000000000000000(0000) GS:ffff915b5d8c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ff963a1e5f5 CR3: 0000000118a10000 CR4: 00000000000607e0
Call Trace:
[<ffffffff86c56de3>] ? pollwake+0x73/0x90
[<ffffffff86c1b762>] kmem_cache_free+0x1e2/0x200
[<ffffffff86bb9307>] mempool_free_slab+0x17/0x20
[<ffffffff86bb971f>] mempool_free+0x4f/0xa0
[<ffffffff86da7f75>] sg_pool_free+0x45/0x50
[<ffffffff86d8a5e8>] __sg_free_table+0x78/0xa0
[<ffffffff86da7f30>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffff86da7e47>] sg_free_table_chained+0x27/0x30
[<ffffffff86ed6b7a>] scsi_release_buffers+0x6a/0x70
[<ffffffff86eda23d>] scsi_end_request+0x12d/0x1e0
[<ffffffff86eda4b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff86ecf96c>] scsi_finish_command+0xdc/0x140
[<ffffffff86ed9a02>] scsi_softirq_done+0x132/0x160
[<ffffffff86d4cce6>] blk_done_softirq+0x96/0xc0
[<ffffffff86aa0f05>] __do_softirq+0xf5/0x280
[<ffffffff86aa10c8>] run_ksoftirqd+0x38/0x50
[<ffffffff86aca5c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff86aca480>] ? lg_double_unlock+0x40/0x40
[<ffffffff86ac1c31>] kthread+0xd1/0xe0
[<ffffffff86ac1b60>] ? insert_kthread_work+0x40/0x40
[<ffffffff87174c37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff86ac1b60>] ? insert_kthread_work+0x40/0x40
Code: 24 48 4c 8b 44 24 18 4c 89 e2 48 89 de e8 5f fd ff ff 84 c0 0f 85 9f 00 00 00 4d 85 ed 0f 85 7b 01 00 00 49 63 47 20 48 8b 4b 18 <4c> 8b 63 10 48 8b 74 24 10 48 89 4c 24 48 4c 89 24 06 89 c8 c1
Complete report #519050
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__slab_free
|
vmlinux |
0xffffffff8862249a
Build id: 3.10.0-1062.el7.x86_64
|
0x8a
|
|
2
|
|
vmlinux |
0xffffffff8868677d
Build id: 3.10.0-1062.el7.x86_64
|
0x7d
|
|
3
|
|
vmlinux |
0xffffffff88604c04
Build id: 3.10.0-1062.el7.x86_64
|
0x34
|
|
4
|
|
vmlinux |
0xffffffff88687697
Build id: 3.10.0-1062.el7.x86_64
|
0x67
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffff88622912
Build id: 3.10.0-1062.el7.x86_64
|
0x1e2
|
|
6
|
mempool_free_slab
|
vmlinux |
0xffffffff885bee07
Build id: 3.10.0-1062.el7.x86_64
|
0x17
|
|
7
|
mempool_free
|
vmlinux |
0xffffffff885bf21f
Build id: 3.10.0-1062.el7.x86_64
|
0x4f
|
|
8
|
sg_pool_free
|
vmlinux |
0xffffffff887b2685
Build id: 3.10.0-1062.el7.x86_64
|
0x45
|
|
9
|
__sg_free_table
|
vmlinux |
0xffffffff88794818
Build id: 3.10.0-1062.el7.x86_64
|
0x78
|
|
10
|
|
vmlinux |
0xffffffff887b2640
Build id: 3.10.0-1062.el7.x86_64
|
0x90
|
|
11
|
sg_free_table_chained
|
vmlinux |
0xffffffff887b2557
Build id: 3.10.0-1062.el7.x86_64
|
0x27
|
|
12
|
scsi_release_buffers
|
vmlinux |
0xffffffff888e2b1a
Build id: 3.10.0-1062.el7.x86_64
|
0x6a
|
|
13
|
scsi_end_request
|
vmlinux |
0xffffffff888e623d
Build id: 3.10.0-1062.el7.x86_64
|
0x12d
|
|
14
|
scsi_io_completion
|
vmlinux |
0xffffffff888e64b8
Build id: 3.10.0-1062.el7.x86_64
|
0x168
|
|
15
|
|
vmlinux |
0xffffffff88a4ded8
Build id: 3.10.0-1062.el7.x86_64
|
0x18
|
|
16
|
scsi_finish_command
|
vmlinux |
0xffffffff888db89c
Build id: 3.10.0-1062.el7.x86_64
|
0xdc
|
|
17
|
scsi_softirq_done
|
vmlinux |
0xffffffff888e5a02
Build id: 3.10.0-1062.el7.x86_64
|
0x132
|
|
18
|
blk_done_softirq
|
vmlinux |
0xffffffff88756c56
Build id: 3.10.0-1062.el7.x86_64
|
0x96
|
|
19
|
__do_softirq
|
vmlinux |
0xffffffff884a41e5
Build id: 3.10.0-1062.el7.x86_64
|
0xf5
|
|
20
|
run_ksoftirqd
|
vmlinux |
0xffffffff884a43a8
Build id: 3.10.0-1062.el7.x86_64
|
0x38
|
|
21
|
smpboot_thread_fn
|
vmlinux |
0xffffffff884cdb04
Build id: 3.10.0-1062.el7.x86_64
|
0x144
|
|
22
|
|
vmlinux |
0xffffffff884cd9c0
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
|
23
|
kthread
|
vmlinux |
0xffffffff884c50d1
Build id: 3.10.0-1062.el7.x86_64
|
0xd1
|
|
24
|
|
vmlinux |
0xffffffff884c5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
|
25
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff88b8bd37
Build id: 3.10.0-1062.el7.x86_64
|
0x21
|
|
26
|
|
vmlinux |
0xffffffff884c5000
Build id: 3.10.0-1062.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#0 stuck for 32s! [ksoftirqd/0:6]
Modules linked in: xt_recent veth nf_tables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 rpcsec_gss_krb5 nfsv4 dns_resolver nfs lockd grace fscache xt_CHECKSUM tun ebtable_filter ebtables ip6table_filter devlink ip6_tables nf_conntrack_netlink xt_statistic xt_nat ipt_MASQUERADE nf_nat_masquerade_ipv4 ipt_REJECT nf_reject_ipv4 xt_mark xt_addrtype xt_comment xt_conntrack iptable_filter iptable_nat nf_nat_ipv4 nf_nat iptable_mangle ip_set nfnetlink nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack vmw_vsock_vmci_transport vsock dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio sb_edac ppdev iosf_mbi vmw_balloon crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper parport_pc pcspkr joydev sg i2c_piix4 cryptd parport
vmw_vmci br_netfilter auth_rpcgss bridge stp sunrpc llc ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper ata_piix syscopyarea sysfillrect sysimgblt fb_sys_fops ttm libata mptspi crct10dif_pclmul crct10dif_common scsi_transport_spi drm serio_raw crc32c_intel mptscsih vmxnet3 mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 0 PID: 6 Comm: ksoftirqd/0 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 09/21/2015
task: ffff955bba2ab150 ti: ffff955bba2b8000 task.ti: ffff955bba2b8000
RIP: 0010:[<ffffffff8862249a>] [<ffffffff8862249a>] __slab_free+0x8a/0x320
RSP: 0000:ffff955bba2bbb50 EFLAGS: 00000282
RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000100200020
RDX: ffff9560ff5d6f00 RSI: ffffc6ed1afd7580 RDI: 0000000040000000
RBP: ffff955bba2bbbe8 R08: 0000000000000001 R09: 000000018020001f
R10: ffff955e6fc1f120 R11: ffff9560ff5d6f00 R12: ffffffff884d2601
R13: ffff955bba2bbb28 R14: 0000000000000286 R15: 0000000000000000
FS: 0000000000000000(0000) GS:ffff955e6fc00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffc88ec4624 CR3: 0000000385e10000 CR4: 00000000003607f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8868677d>] ? bio_put+0x7d/0xa0
[<ffffffff88604c04>] ? end_swap_bio_read+0x34/0x70
[<ffffffff88687697>] ? bio_endio+0x67/0xb0
[<ffffffff88622912>] kmem_cache_free+0x1e2/0x200
[<ffffffff885bee07>] mempool_free_slab+0x17/0x20
[<ffffffff885bf21f>] mempool_free+0x4f/0xa0
[<ffffffff887b2685>] sg_pool_free+0x45/0x50
[<ffffffff88794818>] __sg_free_table+0x78/0xa0
[<ffffffff887b2640>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffff887b2557>] sg_free_table_chained+0x27/0x30
[<ffffffff888e2b1a>] scsi_release_buffers+0x6a/0x70
[<ffffffff888e623d>] scsi_end_request+0x12d/0x1e0
[<ffffffff888e64b8>] scsi_io_completion+0x168/0x6a0
[<ffffffff88a4ded8>] ? __netif_receive_skb+0x18/0x60
[<ffffffff888db89c>] scsi_finish_command+0xdc/0x140
[<ffffffff888e5a02>] scsi_softirq_done+0x132/0x160
[<ffffffff88756c56>] blk_done_softirq+0x96/0xc0
[<ffffffff884a41e5>] __do_softirq+0xf5/0x280
[<ffffffff884a43a8>] run_ksoftirqd+0x38/0x50
[<ffffffff884cdb04>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff884cd9c0>] ? lg_double_unlock+0x40/0x40
[<ffffffff884c50d1>] kthread+0xd1/0xe0
[<ffffffff884c5000>] ? insert_kthread_work+0x40/0x40
[<ffffffff88b8bd37>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffff884c5000>] ? insert_kthread_work+0x40/0x40
Code: 89 c8 49 8b 7c 24 08 c1 e8 18 41 89 c7 41 c0 ef 07 66 45 85 f6 66 44 89 74 24 48 45 0f b6 ff 74 50 4d 85 ed 74 4b 4c 8b 4c 24 48 <4c> 8b 44 24 08 4c 89 ea 48 89 de 4c 89 5c 24 20 e8 41 fd ff ff
Complete report #382468
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
cmpxchg_double_slab.isra.51
|
vmlinux |
0xffffffff954235aa
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x2a
|
|
2
|
|
vmlinux |
0xffffffff9548893d
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x7d
|
|
3
|
|
xfs |
0xffffffffc03f87aa
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x1ba
|
|
4
|
__slab_free
|
vmlinux |
0xffffffff95423b3d
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x9d
|
|
5
|
|
vmlinux |
0xffffffff95423b3d
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x9d
|
|
6
|
|
vmlinux |
0xffffffff95489857
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x67
|
|
7
|
kmem_cache_free
|
vmlinux |
0xffffffff95424372
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x1e2
|
|
8
|
mempool_free_slab
|
vmlinux |
0xffffffff953c07c7
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x17
|
|
9
|
mempool_free
|
vmlinux |
0xffffffff953c0bdf
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x4f
|
|
10
|
sg_pool_free
|
vmlinux |
0xffffffff955b4895
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x45
|
|
11
|
__sg_free_table
|
vmlinux |
0xffffffff95596a28
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x78
|
|
12
|
|
vmlinux |
0xffffffff955b4850
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x90
|
|
13
|
sg_free_table_chained
|
vmlinux |
0xffffffff955b4767
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x27
|
|
14
|
scsi_release_buffers
|
vmlinux |
0xffffffff956e4d9a
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x6a
|
|
15
|
scsi_end_request
|
vmlinux |
0xffffffff956e84bd
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x12d
|
|
16
|
scsi_io_completion
|
vmlinux |
0xffffffff956e8738
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x168
|
|
17
|
scsi_finish_command
|
vmlinux |
0xffffffff956ddb1c
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xdc
|
|
18
|
scsi_softirq_done
|
vmlinux |
0xffffffff956e7c82
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x132
|
|
19
|
blk_done_softirq
|
vmlinux |
0xffffffff95558dd6
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x96
|
|
20
|
__do_softirq
|
vmlinux |
0xffffffff952a5305
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xf5
|
|
21
|
run_ksoftirqd
|
vmlinux |
0xffffffff952a54c8
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x38
|
|
22
|
smpboot_thread_fn
|
vmlinux |
0xffffffff952cec24
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x144
|
|
23
|
|
vmlinux |
0xffffffff952ceae0
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
|
24
|
kthread
|
vmlinux |
0xffffffff952c61f1
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xd1
|
|
25
|
|
vmlinux |
0xffffffff952c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
|
26
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffff9598dd24
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0xe
|
|
27
|
|
vmlinux |
0xffffffff952c6120
Build id: 3.10.0-1062.12.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 32s! [ksoftirqd/1:14]
Modules linked in: binfmt_misc xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink iptable_filter ip6t_REJECT nf_reject_ipv6 nf_conntrack_ipv6 nf_defrag_ipv6 xt_conntrack nf_conntrack ip6table_filter ip6_tables vmw_vsock_vmci_transport vsock nfit libnvdimm ppdev vmw_balloon pcspkr edac_mce_amd crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul joydev glue_helper ablk_helper cryptd sg parport_pc parport vmw_vmci i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod sd_mod cdrom crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix crct10dif_pclmul
drm crct10dif_common crc32c_intel libata mptspi scsi_transport_spi e1000 mptscsih serio_raw mptbase drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 1 PID: 14 Comm: ksoftirqd/1 Kdump: loaded Tainted: G L ------------ 3.10.0-1062.12.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 04/05/2016
task: ffff90d2f60c8000 ti: ffff90d2f60c4000 task.ti: ffff90d2f60c4000
RIP: 0010:[<ffffffff954235aa>] [<ffffffff954235aa>] cmpxchg_double_slab.isra.51+0x2a/0xf0
RSP: 0000:ffff90d2f60c7af0 EFLAGS: 00000246
RAX: ffff90d34ec30b00 RBX: ffff90d7a1470e00 RCX: 0000000100400031
RDX: 0000000100400032 RSI: ffffd8e6473b0c00 RDI: 0000000040000000
RBP: ffff90d2f60c7b40 R08: ffff90d34ec32000 R09: 0000000100400031
R10: ffff90d34ec30b00 R11: ffff90d34ec32000 R12: ffff90d2ffc07300
R13: ffff90d5b2179000 R14: ffffffff963c4160 R15: ffff90d5bf8739c0
FS: 00007fc7693d0740(0000) GS:ffff90d7a5640000(0000) knlGS:00000000f771a700
CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00000000020c8df0 CR3: 000000004ba10000 CR4: 00000000000407e0
Call Trace:
[<ffffffff9548893d>] ? bio_put+0x7d/0xa0
[<ffffffffc03f87aa>] ? xfs_destroy_ioend+0x1ba/0x260 [xfs]
[<ffffffff95423b3d>] __slab_free+0x9d/0x290
[<ffffffff95423b3d>] ? __slab_free+0x9d/0x290
[<ffffffff95489857>] ? bio_endio+0x67/0xb0
[<ffffffff95424372>] kmem_cache_free+0x1e2/0x200
[<ffffffff953c07c7>] mempool_free_slab+0x17/0x20
[<ffffffff953c0bdf>] mempool_free+0x4f/0xa0
[<ffffffff955b4895>] sg_pool_free+0x45/0x50
[<ffffffff95596a28>] __sg_free_table+0x78/0xa0
[<ffffffff955b4850>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffff955b4767>] sg_free_table_chained+0x27/0x30
[<ffffffff956e4d9a>] scsi_release_buffers+0x6a/0x70
[<ffffffff956e84bd>] scsi_end_request+0x12d/0x1e0
[<ffffffff956e8738>] scsi_io_completion+0x168/0x6a0
[<ffffffff956ddb1c>] scsi_finish_command+0xdc/0x140
[<ffffffff956e7c82>] scsi_softirq_done+0x132/0x160
[<ffffffff95558dd6>] blk_done_softirq+0x96/0xc0
[<ffffffff952a5305>] __do_softirq+0xf5/0x280
[<ffffffff952a54c8>] run_ksoftirqd+0x38/0x50
[<ffffffff952cec24>] smpboot_thread_fn+0x144/0x1a0
[<ffffffff952ceae0>] ? lg_double_unlock+0x40/0x40
[<ffffffff952c61f1>] kthread+0xd1/0xe0
[<ffffffff952c6120>] ? insert_kthread_work+0x40/0x40
[<ffffffff9598dd24>] ret_from_fork_nospec_begin+0xe/0x21
[<ffffffff952c6120>] ? insert_kthread_work+0x40/0x40
Code: 00 55 49 89 d2 48 89 e5 53 48 83 e4 f0 48 83 ec 40 f7 c7 00 00 00 40 74 30 48 89 ca 4c 89 d0 4c 89 c3 4c 89 c9 f0 48 0f c7 4e 10 <0f> 94 c0 41 89 c2 b8 01 00 00 00 45 84 d2 75 04 f3 90 31 c0 48
Complete report #57961
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__slab_free
|
vmlinux |
0xffffffffb6225faa
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x5a
|
|
2
|
|
vmlinux |
0xffffffffb6225fed
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x9d
|
|
3
|
|
vmlinux |
0xffffffffb6208a5e
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x2e
|
|
4
|
|
vmlinux |
0xffffffffb628d89c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x8c
|
|
5
|
kmem_cache_free
|
vmlinux |
0xffffffffb6226822
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x1e2
|
|
6
|
mempool_free_slab
|
vmlinux |
0xffffffffb61c12b7
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x17
|
|
7
|
mempool_free
|
vmlinux |
0xffffffffb61c16cf
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x4f
|
|
8
|
sg_pool_free
|
vmlinux |
0xffffffffb63b93d5
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x45
|
|
9
|
__sg_free_table
|
vmlinux |
0xffffffffb639b418
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x78
|
|
10
|
|
vmlinux |
0xffffffffb63b9390
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x90
|
|
11
|
sg_free_table_chained
|
vmlinux |
0xffffffffb63b92a7
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x27
|
|
12
|
scsi_release_buffers
|
vmlinux |
0xffffffffb64e9d6a
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x6a
|
|
13
|
scsi_end_request
|
vmlinux |
0xffffffffb64ed61d
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x12d
|
|
14
|
scsi_io_completion
|
vmlinux |
0xffffffffb64ed898
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x168
|
|
15
|
|
vmlinux |
0xffffffffb60d2df0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x20
|
|
16
|
scsi_finish_command
|
vmlinux |
0xffffffffb64e2b8c
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xdc
|
|
17
|
scsi_softirq_done
|
vmlinux |
0xffffffffb64ecde0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x130
|
|
18
|
blk_done_softirq
|
vmlinux |
0xffffffffb635d426
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x96
|
|
19
|
__do_softirq
|
vmlinux |
0xffffffffb60a4b95
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xf5
|
|
20
|
run_ksoftirqd
|
vmlinux |
0xffffffffb60a4d58
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x38
|
|
21
|
smpboot_thread_fn
|
vmlinux |
0xffffffffb60ce8c4
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x144
|
|
22
|
|
vmlinux |
0xffffffffb60ce780
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
|
23
|
kthread
|
vmlinux |
0xffffffffb60c5e71
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0xd1
|
|
24
|
|
vmlinux |
0xffffffffb60c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
|
25
|
ret_from_fork_nospec_begin
|
vmlinux |
0xffffffffb6793df7
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x21
|
|
26
|
|
vmlinux |
0xffffffffb60c5da0
Build id: 3.10.0-1160.11.1.el7.x86_64
|
0x40
|
NMI watchdog: BUG: soft lockup - CPU#3 stuck for 38s! [ksoftirqd/3:24]
Modules linked in: snd_usb_audio snd_usbmidi_lib snd_hwdep tcp_lp rfcomm xt_CHECKSUM tun devlink ipt_MASQUERADE nf_nat_masquerade_ipv4 xt_addrtype br_netfilter vmhgfs(OE) ip6t_rpfilter ip6t_REJECT nf_reject_ipv6 ipt_REJECT nf_reject_ipv4 xt_conntrack ebtable_nat ebtable_broute bridge stp llc ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat iptable_mangle iptable_security iptable_raw nf_conntrack ip_set nfnetlink ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter overlay(T) bnep vmw_vsock_vmci_transport vsock sunrpc snd_seq_midi snd_seq_midi_event ext4 mbcache jbd2 coretemp snd_ens1371 snd_rawmidi iosf_mbi snd_ac97_codec ac97_bus snd_seq crc32_pclmul snd_seq_device
snd_pcm ghash_clmulni_intel ppdev btusb btrtl btbcm btintel bluetooth vmw_balloon aesni_intel lrw gf128mul glue_helper ablk_helper snd_timer cryptd joydev snd sg pcspkr rfkill soundcore i2c_piix4 vmw_vmci parport_pc parport ip_tables xfs libcrc32c sr_mod cdrom sd_mod crc_t10dif crct10dif_generic ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm mptspi scsi_transport_spi mptscsih drm ata_piix e1000 nfit mptbase crct10dif_pclmul crct10dif_common libnvdimm crc32c_intel libata serio_raw drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod fuse
CPU: 3 PID: 24 Comm: ksoftirqd/3 Kdump: loaded Tainted: G OE ------------ T 3.10.0-1160.11.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 07/02/2015
task: ffff95445efba100 ti: ffff95445efd0000 task.ti: ffff95445efd0000
RIP: 0010:[<ffffffffb6225faa>] [<ffffffffb6225faa>] __slab_free+0x5a/0x290
RSP: 0000:ffff95445efd3b50 EFLAGS: 00010206
RAX: 0000000000000000 RBX: ffff95480a7bdf00 RCX: 0000000180080006
RDX: ffff9544667b4000 RSI: fffff5738599ec00 RDI: ffff95447fc03300
RBP: ffff95445efd3be8 R08: 0000000000000001 R09: ffffffffb61c12b7
R10: ffff9548176df1a0 R11: ffff9544667b4000 R12: ffff95445efd3b68
R13: ffff95480a7b8930 R14: ffff95445efd3b20 R15: 0000000180100000
FS: 0000000000000000(0000) GS:ffff9548176c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f32411938e0 CR3: 000000000fc10000 CR4: 00000000001607e0
Call Trace:
[<ffffffffb6225fed>] ? __slab_free+0x9d/0x290
[<ffffffffb6208a5e>] ? end_swap_bio_write+0x2e/0x80
[<ffffffffb628d89c>] ? bio_endio+0x8c/0x130
[<ffffffffb6226822>] kmem_cache_free+0x1e2/0x200
[<ffffffffb61c12b7>] mempool_free_slab+0x17/0x20
[<ffffffffb61c16cf>] mempool_free+0x4f/0xa0
[<ffffffffb63b93d5>] sg_pool_free+0x45/0x50
[<ffffffffb639b418>] __sg_free_table+0x78/0xa0
[<ffffffffb63b9390>] ? sg_alloc_table_chained+0x90/0x90
[<ffffffffb63b92a7>] sg_free_table_chained+0x27/0x30
[<ffffffffb64e9d6a>] scsi_release_buffers+0x6a/0x70
[<ffffffffb64ed61d>] scsi_end_request+0x12d/0x1e0
[<ffffffffb64ed898>] scsi_io_completion+0x168/0x720
[<ffffffffb60d2df0>] ? task_rq_unlock+0x20/0x20
[<ffffffffb64e2b8c>] scsi_finish_command+0xdc/0x140
[<ffffffffb64ecde0>] scsi_softirq_done+0x130/0x160
[<ffffffffb635d426>] blk_done_softirq+0x96/0xc0
[<ffffffffb60a4b95>] __do_softirq+0xf5/0x280
[<ffffffffb60a4d58>] run_ksoftirqd+0x38/0x50
[<ffffffffb60ce8c4>] smpboot_thread_fn+0x144/0x1a0
[<ffffffffb60ce780>] ? lg_double_unlock+0x40/0x40
[<ffffffffb60c5e71>] kthread+0xd1/0xe0
[<ffffffffb60c5da0>] ? insert_kthread_work+0x40/0x40
[<ffffffffb6793df7>] ret_from_fork_nospec_begin+0x21/0x21
[<ffffffffb60c5da0>] ? insert_kthread_work+0x40/0x40
Code: 89 54 24 08 0f 85 86 01 00 00 66 89 5c 24 28 66 90 49 8b 4e 18 49 63 45 20 49 8b 5e 10 41 89 cc 66 44 2b 64 24 28 48 89 4c 24 48 <49> 89 1c 03 89 c8 49 8b 7d 08 c1 e8 18 41 89 c7 41 c0 ef 07 66