Complete report #864307
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb59170da
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb59c78e0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffb5917229
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffb59c47c5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffb59c964b
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_current
|
vmlinux |
0xffffffffb5a193b8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x98
|
|
7
|
new_slab
|
vmlinux |
0xffffffffb5a27aa3
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x393
|
|
8
|
___slab_alloc
|
vmlinux |
0xffffffffb5a27fbc
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x3cc
|
|
9
|
|
vmlinux |
0xffffffffb5e401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
10
|
|
vmlinux |
0xffffffffb5ea81eb
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x7b
|
|
11
|
|
vmlinux |
0xffffffffb5e401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
12
|
__slab_alloc
|
vmlinux |
0xffffffffb5f7fea5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
|
13
|
__kmalloc_node_track_caller
|
vmlinux |
0xffffffffb5a2b1c8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb8
|
|
14
|
|
vmlinux |
0xffffffffb58aed8c
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x10c
|
|
15
|
|
vmlinux |
0xffffffffb5e401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
16
|
__kmalloc_reserve.isra.32
|
vmlinux |
0xffffffffb5e3f111
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x31
|
|
17
|
|
vmlinux |
0xffffffffb5e4017d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x5d
|
|
18
|
__alloc_skb
|
vmlinux |
0xffffffffb5e401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
19
|
sk_stream_alloc_skb
|
vmlinux |
0xffffffffb5eb2e62
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x52
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffffb5eb3d08
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x788
|
|
21
|
inet_sendmsg
|
vmlinux |
0xffffffffb5edfe49
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x69
|
|
22
|
sock_aio_write
|
vmlinux |
0xffffffffb5e3557d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x15d
|
|
23
|
|
vmlinux |
0xffffffffb59c0179
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x1c9
|
|
24
|
do_sync_write
|
vmlinux |
0xffffffffb5a4daa3
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x93
|
|
25
|
vfs_write
|
vmlinux |
0xffffffffb5a4e695
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x1c5
|
|
26
|
SyS_write
|
vmlinux |
0xffffffffb5a4f36f
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x7f
|
|
27
|
system_call_fastpath
|
vmlinux |
0xffffffffb5f95f92
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#1 stuck for 29s! [bareos-fd:17339]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev sg joydev vmw_balloon pcspkr vmw_vmci parport_pc i2c_piix4 parport auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif syscopyarea sysfillrect crct10dif_generic sysimgblt fb_sys_fops ttm ahci libahci
drm crct10dif_pclmul ata_piix crct10dif_common nfit crc32c_intel serio_raw libata libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 1 PID: 17339 Comm: bareos-fd Kdump: loaded Tainted: G W L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9f7cf0cb2100 ti: ffff9f7a836fc000 task.ti: ffff9f7a836fc000
RIP: 0010:[<ffffffffb59170da>] [<ffffffffb59170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9f7a836ff7a8 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 00000000000000fc RCX: ffff9f7cffd60600
RDX: 0000000000000005 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff9f7a836ff7e0 R08: ffff9f7bb9375000 R09: ffffffffb5b86bf9
R10: ffff9f7cffc5f160 R11: ffffe4d48acbba00 R12: 0000000000000296
R13: ffff9f7a836ff758 R14: 0000000001320122 R15: 00000000e4620b50
FS: 00007fcf1f144700(0000) GS:ffff9f7cffc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa428741000 CR3: 00000002b277e000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffffb59c78e0>] ? drain_pages+0xb0/0xb0
[<ffffffffb5917229>] on_each_cpu_mask+0x29/0x70
[<ffffffffb59c47c5>] drain_all_pages+0xb5/0xc0
[<ffffffffb59c964b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffffb5a193b8>] alloc_pages_current+0x98/0x110
[<ffffffffb5a27aa3>] new_slab+0x393/0x4e0
[<ffffffffb5a27fbc>] ___slab_alloc+0x3cc/0x520
[<ffffffffb5e401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffb5ea81eb>] ? ip_output+0x7b/0xf0
[<ffffffffb5e401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffb5f7fea5>] __slab_alloc+0x40/0x5c
[<ffffffffb5a2b1c8>] __kmalloc_node_track_caller+0xb8/0x290
[<ffffffffb58aed8c>] ? mod_timer+0x10c/0x230
[<ffffffffb5e401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffb5e3f111>] __kmalloc_reserve.isra.32+0x31/0x90
[<ffffffffb5e4017d>] ? __alloc_skb+0x5d/0x2d0
[<ffffffffb5e401ad>] __alloc_skb+0x8d/0x2d0
[<ffffffffb5eb2e62>] sk_stream_alloc_skb+0x52/0x1b0
[<ffffffffb5eb3d08>] tcp_sendmsg+0x788/0xc60
[<ffffffffb5edfe49>] inet_sendmsg+0x69/0xb0
[<ffffffffb5e3557d>] sock_aio_write+0x15d/0x180
[<ffffffffb59c0179>] ? generic_file_aio_read+0x1c9/0x290
[<ffffffffb5a4daa3>] do_sync_write+0x93/0xe0
[<ffffffffb5a4e695>] vfs_write+0x1c5/0x1f0
[<ffffffffb5a4f36f>] SyS_write+0x7f/0xf0
[<ffffffffb5f95f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 55 b6 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #814237
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff92d170da
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff92dc78e0
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff92d17229
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff92dc47c5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff92dc964b
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_current
|
vmlinux |
0xffffffff92e193b8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x98
|
|
7
|
new_slab
|
vmlinux |
0xffffffff92e27aa3
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x393
|
|
8
|
___slab_alloc
|
vmlinux |
0xffffffff92e27fbc
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x3cc
|
|
9
|
|
vmlinux |
0xffffffff932401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
10
|
|
vmlinux |
0xffffffff92dc9422
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x6a2
|
|
11
|
|
vmlinux |
0xffffffff9324017d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x5d
|
|
12
|
|
vmlinux |
0xffffffff932401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
13
|
__slab_alloc
|
vmlinux |
0xffffffff9337fea5
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x40
|
|
14
|
__kmalloc_node_track_caller
|
vmlinux |
0xffffffff92e2b1c8
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0xb8
|
|
15
|
|
vmlinux |
0xffffffff932401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
16
|
__kmalloc_reserve.isra.32
|
vmlinux |
0xffffffff9323f111
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x31
|
|
17
|
|
vmlinux |
0xffffffff9324017d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x5d
|
|
18
|
__alloc_skb
|
vmlinux |
0xffffffff932401ad
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x8d
|
|
19
|
sk_stream_alloc_skb
|
vmlinux |
0xffffffff932b2e62
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x52
|
|
20
|
tcp_sendmsg
|
vmlinux |
0xffffffff932b3d08
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x788
|
|
21
|
|
vmlinux |
0xffffffff92dbe002
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x92
|
|
22
|
inet_sendmsg
|
vmlinux |
0xffffffff932dfe49
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x69
|
|
23
|
sock_aio_write
|
vmlinux |
0xffffffff9323557d
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x15d
|
|
24
|
|
vmlinux |
0xffffffff92dc0179
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x1c9
|
|
25
|
do_sync_write
|
vmlinux |
0xffffffff92e4daa3
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x93
|
|
26
|
vfs_write
|
vmlinux |
0xffffffff92e4e695
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x1c5
|
|
27
|
SyS_write
|
vmlinux |
0xffffffff92e4f36f
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x7f
|
|
28
|
system_call_fastpath
|
vmlinux |
0xffffffff93395f92
Build id: 3.10.0-1160.53.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#2 stuck for 25s! [bareos-fd:20407]
Modules linked in: xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables ip6table_filter devlink ip6_tables iptable_filter nfsv3 nfs_acl nfs lockd grace fscache vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod iosf_mbi kvm_intel kvm irqbypass crc32_pclmul ghash_clmulni_intel ppdev aesni_intel lrw gf128mul vmw_balloon glue_helper ablk_helper cryptd joydev pcspkr sg vmw_vmci parport_pc parport i2c_piix4 auth_rpcgss sunrpc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper syscopyarea sysfillrect sysimgblt sd_mod fb_sys_fops ttm crc_t10dif crct10dif_generic ahci ata_piix
libahci nfit crct10dif_pclmul crct10dif_common drm libnvdimm libata crc32c_intel serio_raw vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse
CPU: 2 PID: 20407 Comm: bareos-fd Kdump: loaded Tainted: G L ------------ 3.10.0-1160.53.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9d10f92ac200 ti: ffff9d11ea0ac000 task.ti: ffff9d11ea0ac000
RIP: 0010:[<ffffffff92d170da>] [<ffffffff92d170da>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9d11ea0af7a8 EFLAGS: 00000202
RAX: 0000000000000001 RBX: 00000000000000fc RCX: ffff9d123fc60628
RDX: 0000000000000001 RSI: 0000000000000006 RDI: 0000000000000000
RBP: ffff9d11ea0af7e0 R08: ffff9d10f9391400 R09: ffffffff92f86bf9
R10: ffff9d123fc9f160 R11: fffff34940970400 R12: 0000000000000296
R13: ffff9d11ea0af758 R14: 0000000001320122 R15: 000000001dd9a73e
FS: 00007f134c5c9700(0000) GS:ffff9d123fc80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f1344091000 CR3: 00000002a2474000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff92dc78e0>] ? drain_pages+0xb0/0xb0
[<ffffffff92d17229>] on_each_cpu_mask+0x29/0x70
[<ffffffff92dc47c5>] drain_all_pages+0xb5/0xc0
[<ffffffff92dc964b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffff92e193b8>] alloc_pages_current+0x98/0x110
[<ffffffff92e27aa3>] new_slab+0x393/0x4e0
[<ffffffff92e27fbc>] ___slab_alloc+0x3cc/0x520
[<ffffffff932401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff92dc9422>] ? __alloc_pages_nodemask+0x6a2/0xbe0
[<ffffffff9324017d>] ? __alloc_skb+0x5d/0x2d0
[<ffffffff932401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff9337fea5>] __slab_alloc+0x40/0x5c
[<ffffffff92e2b1c8>] __kmalloc_node_track_caller+0xb8/0x290
[<ffffffff932401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff9323f111>] __kmalloc_reserve.isra.32+0x31/0x90
[<ffffffff9324017d>] ? __alloc_skb+0x5d/0x2d0
[<ffffffff932401ad>] __alloc_skb+0x8d/0x2d0
[<ffffffff932b2e62>] sk_stream_alloc_skb+0x52/0x1b0
[<ffffffff932b3d08>] tcp_sendmsg+0x788/0xc60
[<ffffffff92dbe002>] ? __page_cache_alloc+0x92/0xb0
[<ffffffff932dfe49>] inet_sendmsg+0x69/0xb0
[<ffffffff9323557d>] sock_aio_write+0x15d/0x180
[<ffffffff92dc0179>] ? generic_file_aio_read+0x1c9/0x290
[<ffffffff92e4daa3>] do_sync_write+0x93/0xe0
[<ffffffff92e4e695>] vfs_write+0x1c5/0x1f0
[<ffffffff92e4f36f>] SyS_write+0x7f/0xf0
[<ffffffff93395f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 85 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 95 93 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 85 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #773028
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8b51173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff8b5bf550
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffff8b511889
Build id: 3.10.0-957.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffff8b5bc435
Build id: 3.10.0-957.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_slowpath
|
vmlinux |
0xffffffff8bb5d228
Build id: 3.10.0-957.el7.x86_64
|
0x510
|
|
6
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffff8b5c0e95
Build id: 3.10.0-957.el7.x86_64
|
0x405
|
|
7
|
alloc_pages_current
|
vmlinux |
0xffffffff8b60dcf8
Build id: 3.10.0-957.el7.x86_64
|
0x98
|
|
8
|
new_slab
|
vmlinux |
0xffffffff8b618d05
Build id: 3.10.0-957.el7.x86_64
|
0x2c5
|
|
9
|
___slab_alloc
|
vmlinux |
0xffffffff8b61a77c
Build id: 3.10.0-957.el7.x86_64
|
0x3ac
|
|
10
|
|
vmlinux |
0xffffffff8ba2413d
Build id: 3.10.0-957.el7.x86_64
|
0x8d
|
|
11
|
?
__ip_append_data.isra.51
|
vmlinux |
0xffffffff8ba881e0
Build id: 3.10.0-957.el7.x86_64
|
0xa50
|
|
12
|
|
vmlinux |
0xffffffff8ba86817
Build id: 3.10.0-957.el7.x86_64
|
0x37
|
|
13
|
|
vmlinux |
0xffffffff8ba2413d
Build id: 3.10.0-957.el7.x86_64
|
0x8d
|
|
14
|
__slab_alloc
|
vmlinux |
0xffffffff8bb5e58c
Build id: 3.10.0-957.el7.x86_64
|
0x40
|
|
15
|
__kmalloc_node_track_caller
|
vmlinux |
0xffffffff8b61f1f8
Build id: 3.10.0-957.el7.x86_64
|
0xb8
|
|
16
|
|
vmlinux |
0xffffffff8ba2413d
Build id: 3.10.0-957.el7.x86_64
|
0x8d
|
|
17
|
__kmalloc_reserve.isra.32
|
vmlinux |
0xffffffff8ba22e81
Build id: 3.10.0-957.el7.x86_64
|
0x31
|
|
18
|
|
vmlinux |
0xffffffff8ba2410d
Build id: 3.10.0-957.el7.x86_64
|
0x5d
|
|
19
|
__alloc_skb
|
vmlinux |
0xffffffff8ba2413d
Build id: 3.10.0-957.el7.x86_64
|
0x8d
|
|
20
|
sk_stream_alloc_skb
|
vmlinux |
0xffffffff8ba938a2
Build id: 3.10.0-957.el7.x86_64
|
0x52
|
|
21
|
tcp_sendmsg
|
vmlinux |
0xffffffff8ba94748
Build id: 3.10.0-957.el7.x86_64
|
0x788
|
|
22
|
|
vmlinux |
0xffffffff8ba1e980
Build id: 3.10.0-957.el7.x86_64
|
0x120
|
|
23
|
|
vmlinux |
0xffffffff8ba91aaa
Build id: 3.10.0-957.el7.x86_64
|
0x62a
|
|
24
|
inet_sendmsg
|
vmlinux |
0xffffffff8babfeb9
Build id: 3.10.0-957.el7.x86_64
|
0x69
|
|
25
|
sock_aio_write
|
vmlinux |
0xffffffff8ba1857d
Build id: 3.10.0-957.el7.x86_64
|
0x15d
|
|
26
|
do_sync_write
|
vmlinux |
0xffffffff8b6405b3
Build id: 3.10.0-957.el7.x86_64
|
0x93
|
|
27
|
vfs_write
|
vmlinux |
0xffffffff8b6411a5
Build id: 3.10.0-957.el7.x86_64
|
0x1c5
|
|
28
|
SyS_write
|
vmlinux |
0xffffffff8b641ebf
Build id: 3.10.0-957.el7.x86_64
|
0x7f
|
|
29
|
system_call_fastpath
|
vmlinux |
0xffffffff8bb74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 22s! [oracle:124745]
Modules linked in: nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache tcp_diag inet_diag xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6table_filter ip6_tables iptable_filter vmw_vsock_vmci_transport vsock sunrpc ppdev vmw_balloon iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd joydev pcspkr sg parport_pc parport vmw_vmci i2c_piix4 binfmt_misc ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi sd_mod crc_t10dif crct10dif_generic vmwgfx crct10dif_pclmul drm_kms_helper crct10dif_common crc32c_intel syscopyarea sysfillrect
sysimgblt fb_sys_fops ttm serio_raw ahci drm ata_piix libahci libata vmxnet3 vmw_pvscsi drm_panel_orientation_quirks nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
CPU: 7 PID: 124745 Comm: oracle Kdump: loaded Tainted: G L ------------ 3.10.0-957.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
task: ffff9c0236a9d140 ti: ffff9bef6f25c000 task.ti: ffff9bef6f25c000
RIP: 0010:[<ffffffff8b51173a>] [<ffffffff8b51173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff9bef6f25f740 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 000800008b5bf550 RCX: ffff9c116f6206a0
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffff9bef6f25f778 R08: ffff9bef8358d000 R09: ffffffff8b7755c9
R10: ffff9c116f7df160 R11: fffffe7050c67400 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9bef6f25f6f0 R15: 0000000001320122
FS: 00007fb1b1465740(0000) GS:ffff9c116f7c0000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007fa984ae21e8 CR3: 00000004098bc000 CR4: 00000000007607e0
PKRU: 55555554
Call Trace:
[<ffffffff8b5bf550>] ? drain_pages+0xb0/0xb0
[<ffffffff8b511889>] on_each_cpu_mask+0x29/0x70
[<ffffffff8b5bc435>] drain_all_pages+0xb5/0xc0
[<ffffffff8bb5d228>] __alloc_pages_slowpath+0x510/0x724
[<ffffffff8b5c0e95>] __alloc_pages_nodemask+0x405/0x420
[<ffffffff8b60dcf8>] alloc_pages_current+0x98/0x110
[<ffffffff8b618d05>] new_slab+0x2c5/0x390
[<ffffffff8b61a77c>] ___slab_alloc+0x3ac/0x4f0
[<ffffffff8ba2413d>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff8ba881e0>] ? __ip_append_data.isra.51+0xa50/0xa50
[<ffffffff8ba86817>] ? ip_local_out_sk+0x37/0x40
[<ffffffff8ba2413d>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff8bb5e58c>] __slab_alloc+0x40/0x5c
[<ffffffff8b61f1f8>] __kmalloc_node_track_caller+0xb8/0x290
[<ffffffff8ba2413d>] ? __alloc_skb+0x8d/0x2d0
[<ffffffff8ba22e81>] __kmalloc_reserve.isra.32+0x31/0x90
[<ffffffff8ba2410d>] ? __alloc_skb+0x5d/0x2d0
[<ffffffff8ba2413d>] __alloc_skb+0x8d/0x2d0
[<ffffffff8ba938a2>] sk_stream_alloc_skb+0x52/0x1b0
[<ffffffff8ba94748>] tcp_sendmsg+0x788/0xc60
[<ffffffff8ba1e980>] ? release_sock+0x120/0x170
[<ffffffff8ba91aaa>] ? tcp_recvmsg+0x62a/0xb30
[<ffffffff8babfeb9>] inet_sendmsg+0x69/0xb0
[<ffffffff8ba1857d>] sock_aio_write+0x15d/0x180
[<ffffffff8b6405b3>] do_sync_write+0x93/0xe0
[<ffffffff8b6411a5>] vfs_write+0x1c5/0x1f0
[<ffffffff8b641ebf>] SyS_write+0x7f/0xf0
[<ffffffff8bb74ddb>] system_call_fastpath+0x22/0x27
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 14 8c f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #327932
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffa2f16bda
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffa2fc73e0
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu_mask
|
vmlinux |
0xffffffffa2f16d29
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x29
|
|
4
|
drain_all_pages
|
vmlinux |
0xffffffffa2fc42c5
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xb5
|
|
5
|
__alloc_pages_nodemask
|
vmlinux |
0xffffffffa2fc914b
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x8cb
|
|
6
|
alloc_pages_current
|
vmlinux |
0xffffffffa3018ea8
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x98
|
|
7
|
new_slab
|
vmlinux |
0xffffffffa3027593
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x393
|
|
8
|
___slab_alloc
|
vmlinux |
0xffffffffa3027aac
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x3cc
|
|
9
|
|
vmlinux |
0xffffffffa34401ad
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x8d
|
|
10
|
|
vmlinux |
0xffffffffa34401ad
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x8d
|
|
11
|
__slab_alloc
|
vmlinux |
0xffffffffa357fed5
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x40
|
|
12
|
__kmalloc_node_track_caller
|
vmlinux |
0xffffffffa302acb8
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xb8
|
|
13
|
|
vmlinux |
0xffffffffa34401ad
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x8d
|
|
14
|
__kmalloc_reserve.isra.32
|
vmlinux |
0xffffffffa343f111
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x31
|
|
15
|
|
vmlinux |
0xffffffffa344017d
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x5d
|
|
16
|
__alloc_skb
|
vmlinux |
0xffffffffa34401ad
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x8d
|
|
17
|
sk_stream_alloc_skb
|
vmlinux |
0xffffffffa34b2ed2
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x52
|
|
18
|
tcp_sendmsg
|
vmlinux |
0xffffffffa34b3d78
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x788
|
|
19
|
inet_sendmsg
|
vmlinux |
0xffffffffa34dfe79
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x69
|
|
20
|
sock_aio_write
|
vmlinux |
0xffffffffa343557d
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x15d
|
|
21
|
?
__mem_cgroup_commit_charge
|
vmlinux |
0xffffffffa303e802
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0xe2
|
|
22
|
do_sync_write
|
vmlinux |
0xffffffffa304d593
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x93
|
|
23
|
vfs_write
|
vmlinux |
0xffffffffa304e185
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x1c5
|
|
24
|
SyS_write
|
vmlinux |
0xffffffffa304ee5f
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x7f
|
|
25
|
system_call_fastpath
|
vmlinux |
0xffffffffa3595f92
Build id: 3.10.0-1160.31.1.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 22s! [kube-apiserver:29195]
Modules linked in: ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables ipt_rpfilter xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set ipip tunnel4 ip_tunnel xt_nat xt_statistic ipt_REJECT nf_reject_ipv4 iptable_mangle xt_comment xt_mark xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink xt_addrtype iptable_filter iptable_nat nf_nat_ipv4 nf_nat rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache overlay(T) vmw_vsock_vmci_transport vsock dm_mirror dm_region_hash dm_log dm_mod sb_edac iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd ppdev joydev sg pcspkr vmw_balloon parport_pc parport vmw_vmci i2c_piix4 nfsd auth_rpcgss nf_conntrack_ipv4 nfs_acl nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs lockd
grace sunrpc nf_conntrack libcrc32c ip_tables ext4 mbcache jbd2 sr_mod cdrom ata_generic pata_acpi vmwgfx drm_kms_helper sd_mod crc_t10dif crct10dif_generic syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm crct10dif_pclmul crct10dif_common crc32c_intel ahci ata_piix libahci libata serio_raw nfit libnvdimm vmxnet3 vmw_pvscsi drm_panel_orientation_quirks fuse br_netfilter bridge stp llc
CPU: 6 PID: 29195 Comm: kube-apiserver Kdump: loaded Tainted: G W L ------------ T 3.10.0-1160.31.1.el7.x86_64 #1
Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 05/28/2020
task: ffffa0840c7ec200 ti: ffffa0841af90000 task.ti: ffffa0841af90000
RIP: 0010:[<ffffffffa2f16bda>] [<ffffffffa2f16bda>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffffa0841af937a8 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffffa0841d6206c8
RDX: 0000000000000000 RSI: 0000000000000080 RDI: 0000000000000000
RBP: ffffa0841af937e0 R08: ffffa07d6c57e800 R09: ffffffffa31866a9
R10: ffffa0841d79f160 R11: ffffeac1e06cb800 R12: 0000000000000296
R13: ffffa0841af93758 R14: ffffffffa2f32fb5 R15: 000000005c86933f
FS: 000000c000581090(0000) GS:ffffa0841d780000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000c01de67000 CR3: 000000080f4c8000 CR4: 00000000001607e0
Call Trace:
[<ffffffffa2fc73e0>] ? drain_pages+0xb0/0xb0
[<ffffffffa2f16d29>] on_each_cpu_mask+0x29/0x70
[<ffffffffa2fc42c5>] drain_all_pages+0xb5/0xc0
[<ffffffffa2fc914b>] __alloc_pages_nodemask+0x8cb/0xbe0
[<ffffffffa3018ea8>] alloc_pages_current+0x98/0x110
[<ffffffffa3027593>] new_slab+0x393/0x4e0
[<ffffffffa3027aac>] ___slab_alloc+0x3cc/0x520
[<ffffffffa34401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffa34401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffa357fed5>] __slab_alloc+0x40/0x5c
[<ffffffffa302acb8>] __kmalloc_node_track_caller+0xb8/0x290
[<ffffffffa34401ad>] ? __alloc_skb+0x8d/0x2d0
[<ffffffffa343f111>] __kmalloc_reserve.isra.32+0x31/0x90
[<ffffffffa344017d>] ? __alloc_skb+0x5d/0x2d0
[<ffffffffa34401ad>] __alloc_skb+0x8d/0x2d0
[<ffffffffa34b2ed2>] sk_stream_alloc_skb+0x52/0x1b0
[<ffffffffa34b3d78>] tcp_sendmsg+0x788/0xc60
[<ffffffffa34dfe79>] inet_sendmsg+0x69/0xb0
[<ffffffffa343557d>] sock_aio_write+0x15d/0x180
[<ffffffffa303e802>] ? __mem_cgroup_commit_charge+0xe2/0x2f0
[<ffffffffa304d593>] do_sync_write+0x93/0xe0
[<ffffffffa304e185>] vfs_write+0x1c5/0x1f0
[<ffffffffa304ee5f>] SyS_write+0x7f/0xf0
[<ffffffffa3595f92>] system_call_fastpath+0x25/0x2a
Code: 48 63 35 ce 8a c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 15 b5 a3 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 9d 8a c4 00 eb b7 0f b6 4d cc 4c 89