Complete report #1420980
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffaf71e07e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x20e
|
|
2
|
|
vmlinux |
0xffffffffaf681b60
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffaf71e13d
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffffaf6821b9
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x59
|
|
5
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffffaf810800
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x1a0
|
|
6
|
free_vmap_area_noflush
|
vmlinux |
0xffffffffaf810aa0
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x80
|
|
7
|
remove_vm_area
|
vmlinux |
0xffffffffaf8125c0
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x60
|
|
8
|
__vunmap
|
vmlinux |
0xffffffffaf81260e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x2e
|
|
9
|
vfree
|
vmlinux |
0xffffffffaf812766
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x36
|
|
10
|
do_ipt_get_ctl
|
ip_tables |
0xffffffffc03e8e49
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x359
|
|
11
|
nf_getsockopt
|
vmlinux |
0xffffffffafcc274f
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x4f
|
|
12
|
ip_getsockopt
|
vmlinux |
0xffffffffafcd4740
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x70
|
|
13
|
raw_getsockopt
|
vmlinux |
0xffffffffafcfad8d
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x2d
|
|
14
|
sock_common_getsockopt
|
vmlinux |
0xffffffffafc6183a
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x1a
|
|
15
|
SyS_getsockopt
|
vmlinux |
0xffffffffafc609dd
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0x7d
|
|
16
|
tracesys
|
vmlinux |
0xffffffffafdc562e
Build id: 3.10.0-1160.81.1.el7.x86_64
|
0xa6
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [iptables:30492]
Modules linked in: ipt_rpfilter iptable_raw ip_set_hash_ip ipip ip_set_hash_net tunnel4 ip_tunnel xt_multiport kindling_falcolib_probe(OE) veth xt_nat nf_conntrack_netlink xt_addrtype xt_physdev xt_set ip_set_hash_ipportnet ip_set_hash_ipportip ip_set_bitmap_port ip_set_hash_ipport ip_set dummy ip6table_mangle ip6t_MASQUERADE nf_nat_masquerade_ipv6 ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables devlink iptable_filter overlay(T) nf_conntrack_ipv4 nf_defrag_ipv4 br_netfilter bridge stp llc ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack sunrpc squashfs loop bochs_drm ttm
drm_kms_helper virtio_net syscopyarea sysfillrect net_failover sysimgblt failover sg fb_sys_fops drm i2c_piix4 drm_panel_orientation_quirks aesni_intel virtio_balloon virtio_pci lrw joydev virtio_ring gf128mul virtio glue_helper floppy ablk_helper pcspkr cryptd ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic sr_mod cdrom crct10dif_common ata_generic pata_acpi ahci ata_piix libahci libata crc32c_intel serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse [last unloaded: kindling_falcolib_probe]
CPU: 4 PID: 30492 Comm: iptables Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1160.81.1.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014
task: ffff9de61a52d280 ti: ffff9de61a580000 task.ti: ffff9de61a580000
RIP: 0010:[<ffffffffaf71e07e>] [<ffffffffaf71e07e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9de61a583c40 EFLAGS: 00000202
RAX: 0000000000000005 RBX: 000c0000af681b60 RCX: ffff9de9bfd60398
RDX: 0000000000000005 RSI: 0000000000000008 RDI: 0000000000000000
RBP: ffff9de61a583c78 R08: ffff9de2ecbd9c00 R09: ffffffffaf99c139
R10: ffff9de9bfd1f160 R11: fffff1e5d1c6b600 R12: 0000000000000282
R13: 0000000000000282 R14: ffff9de61a583bf0 R15: 800000063f681867
FS: 00007f9af6a46500(0000) GS:ffff9de9bfd00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f9af6a44000 CR3: 000000049a4a2000 CR4: 00000000000406e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffaf681b60>] ? flush_tlb_func+0xb0/0xb0
[<ffffffffaf71e13d>] on_each_cpu+0x2d/0x60
[<ffffffffaf6821b9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffffaf810800>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffffaf810aa0>] free_vmap_area_noflush+0x80/0x90
[<ffffffffaf8125c0>] remove_vm_area+0x60/0x80
[<ffffffffaf81260e>] __vunmap+0x2e/0x110
[<ffffffffaf812766>] vfree+0x36/0x80
[<ffffffffc03e8e49>] do_ipt_get_ctl+0x359/0x430 [ip_tables]
[<ffffffffafcc274f>] nf_getsockopt+0x4f/0x70
[<ffffffffafcd4740>] ip_getsockopt+0x70/0xb0
[<ffffffffafcfad8d>] raw_getsockopt+0x2d/0x60
[<ffffffffafc6183a>] sock_common_getsockopt+0x1a/0x20
[<ffffffffafc609dd>] SyS_getsockopt+0x7d/0xf0
[<ffffffffafdc562e>] tracesys+0xa6/0xcc
Code: 19 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 00 19 35 b0 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 9d 19 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee
Complete report #1405089
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb471173a
Build id: 3.10.0-957.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb467a480
Build id: 3.10.0-957.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffb47117fd
Build id: 3.10.0-957.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffffb467a7f9
Build id: 3.10.0-957.el7.x86_64
|
0x59
|
|
5
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffffb47fa020
Build id: 3.10.0-957.el7.x86_64
|
0x1a0
|
|
6
|
free_vmap_area_noflush
|
vmlinux |
0xffffffffb47fa2bc
Build id: 3.10.0-957.el7.x86_64
|
0x7c
|
|
7
|
remove_vm_area
|
vmlinux |
0xffffffffb47fbd90
Build id: 3.10.0-957.el7.x86_64
|
0x60
|
|
8
|
__vunmap
|
vmlinux |
0xffffffffb47fbdda
Build id: 3.10.0-957.el7.x86_64
|
0x2a
|
|
9
|
vfree
|
vmlinux |
0xffffffffb47fbf26
Build id: 3.10.0-957.el7.x86_64
|
0x36
|
|
10
|
ip_set_sockfn_get
|
ip_set |
0xffffffffc086c19a
Build id: 3.10.0-957.el7.x86_64
|
0x1fa
|
|
11
|
nf_getsockopt
|
vmlinux |
0xffffffffb4c79b6e
Build id: 3.10.0-957.el7.x86_64
|
0x6e
|
|
12
|
ip_getsockopt
|
vmlinux |
0xffffffffb4c8a7c3
Build id: 3.10.0-957.el7.x86_64
|
0x83
|
|
13
|
raw_getsockopt
|
vmlinux |
0xffffffffb4caffa5
Build id: 3.10.0-957.el7.x86_64
|
0x25
|
|
14
|
sock_common_getsockopt
|
vmlinux |
0xffffffffb4c1c4fa
Build id: 3.10.0-957.el7.x86_64
|
0x1a
|
|
15
|
SyS_getsockopt
|
vmlinux |
0xffffffffb4c1b7bd
Build id: 3.10.0-957.el7.x86_64
|
0x7d
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffffb4d74ddb
Build id: 3.10.0-957.el7.x86_64
|
0x22
|
|
17
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb4d74d21
Build id: 3.10.0-957.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [iptables-save:42665]
Modules linked in: xt_multiport ip_set_hash_ip ip_set_hash_net ipip tunnel4 ip_tunnel veth xt_addrtype xt_set ip_set_hash_ipportnet ip_set_hash_ipport ip_set_bitmap_port ip_set_hash_ipportip ip_set dummy iptable_raw xt_CT rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 xt_comment xt_mark nf_conntrack_netlink nfnetlink overlay(T) xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_conntrack ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter devlink sunrpc ppdev parport_pc parport sg virtio_balloon i2c_piix4 joydev pcspkr br_netfilter bridge stp llc nf_conntrack_ipv4 nf_defrag_ipv4 ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs nf_conntrack
ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix floppy libata drm_panel_orientation_quirks serio_raw virtio_pci virtio_ring virtio dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 42665 Comm: iptables-save Kdump: loaded Tainted: G L ------------ T 3.10.0-957.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff8d2dd7a29040 ti: ffff8d21fb248000 task.ti: ffff8d21fb248000
RIP: 0010:[<ffffffffb471173a>] [<ffffffffb471173a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8d21fb24bc90 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000c0000b467a480 RCX: ffff8d313d8a0320
RDX: 0000000000000002 RSI: 0000000000000030 RDI: 0000000000000000
RBP: ffff8d21fb24bcc8 R08: ffff8d22a9524000 R09: ffffffffb49755c9
R10: ffff8d313d91f160 R11: fffff19c7874de00 R12: 0000000000000286
R13: 0000000000000286 R14: ffff8d21fb24bc40 R15: 0000000000000000
FS: 00007f0f6d6a6280(0000) GS:ffff8d313d900000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f0f6b396a5d CR3: 00000000357da000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb467a480>] ? flush_tlb_func+0xb0/0xb0
[<ffffffffb47117fd>] on_each_cpu+0x2d/0x60
[<ffffffffb467a7f9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffffb47fa020>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffffb47fa2bc>] free_vmap_area_noflush+0x7c/0x90
[<ffffffffb47fbd90>] remove_vm_area+0x60/0x80
[<ffffffffb47fbdda>] __vunmap+0x2a/0x100
[<ffffffffb47fbf26>] vfree+0x36/0x70
[<ffffffffc086c19a>] ip_set_sockfn_get+0x1fa/0x300 [ip_set]
[<ffffffffb4c79b6e>] nf_getsockopt+0x6e/0x90
[<ffffffffb4c8a7c3>] ip_getsockopt+0x83/0xd0
[<ffffffffb4caffa5>] raw_getsockopt+0x25/0x50
[<ffffffffb4c1c4fa>] sock_common_getsockopt+0x1a/0x20
[<ffffffffb4c1b7bd>] SyS_getsockopt+0x7d/0xf0
[<ffffffffb4d74ddb>] system_call_fastpath+0x22/0x27
[<ffffffffb4d74d21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 4e 83 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 60 b9 34 b5 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 1d 83 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1176016
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff8cf150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff8ce7d280
Build id: 3.10.0-1062.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff8cf1518d
Build id: 3.10.0-1062.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffff8ce7d5f9
Build id: 3.10.0-1062.el7.x86_64
|
0x59
|
|
5
|
|
vmlinux |
0xffffffff8d000454
Build id: 3.10.0-1062.el7.x86_64
|
0x84
|
|
6
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffff8d000570
Build id: 3.10.0-1062.el7.x86_64
|
0x1a0
|
|
7
|
free_vmap_area_noflush
|
vmlinux |
0xffffffff8d00080c
Build id: 3.10.0-1062.el7.x86_64
|
0x7c
|
|
8
|
remove_vm_area
|
vmlinux |
0xffffffff8d0022e0
Build id: 3.10.0-1062.el7.x86_64
|
0x60
|
|
9
|
__vunmap
|
vmlinux |
0xffffffff8d00232a
Build id: 3.10.0-1062.el7.x86_64
|
0x2a
|
|
10
|
vfree
|
vmlinux |
0xffffffff8d002476
Build id: 3.10.0-1062.el7.x86_64
|
0x36
|
|
11
|
do_ipt_get_ctl
|
ip_tables |
0xffffffffc04b9d99
Build id: 3.10.0-1062.el7.x86_64
|
0x349
|
|
12
|
nf_getsockopt
|
vmlinux |
0xffffffff8d48faee
Build id: 3.10.0-1062.el7.x86_64
|
0x6e
|
|
13
|
ip_getsockopt
|
vmlinux |
0xffffffff8d4a0a20
Build id: 3.10.0-1062.el7.x86_64
|
0x70
|
|
14
|
raw_getsockopt
|
vmlinux |
0xffffffff8d4c6695
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
15
|
sock_common_getsockopt
|
vmlinux |
0xffffffff8d4304fa
Build id: 3.10.0-1062.el7.x86_64
|
0x1a
|
|
16
|
SyS_getsockopt
|
vmlinux |
0xffffffff8d42f7bd
Build id: 3.10.0-1062.el7.x86_64
|
0x7d
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffff8d58bede
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
18
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff8d58be21
Build id: 3.10.0-1062.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [iptables:83241]
Modules linked in: udp_diag vxlan ip6_udp_tunnel udp_tunnel ipt_rpfilter xt_multiport xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set xt_nat xt_statistic ipt_REJECT nf_reject_ipv4 veth xt_length xt_NFQUEUE tcp_diag inet_diag ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 iptable_mangle xt_comment xt_mark ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc binfmt_misc iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi overlay(T) ip6table_filter ip6_tables iptable_filter sunrpc ppdev joydev pcspkr virtio_balloon sg parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic
pata_acpi virtio_console virtio_net virtio_scsi virtio_blk cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata virtio_pci virtio_ring serio_raw floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod
CPU: 4 PID: 83241 Comm: iptables Kdump: loaded Tainted: G L ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff90a99e72b150 ti: ffff90a9c2ef0000 task.ti: ffff90a9c2ef0000
RIP: 0010:[<ffffffff8cf150ca>] [<ffffffff8cf150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff90a9c2ef3c38 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff90abfce20428
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff90a9c2ef3c70 R08: ffff909d695d3800 R09: ffffffff8d17f629
R10: ffff90abfcf1f160 R11: ffffcc06f058fa00 R12: 0000000000000292
R13: ffff90a9c2ef3be8 R14: ffff90a9c2ef3c38 R15: ffffffff8d024329
FS: 00007ff02a6aed48(0000) GS:ffff90abfcf00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000563919b95000 CR3: 0000000ea36e2000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff8ce7d280>] ? flush_tlb_func+0xb0/0xb0
[<ffffffff8cf1518d>] on_each_cpu+0x2d/0x60
[<ffffffff8ce7d5f9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffff8d000454>] ? __purge_vmap_area_lazy+0x84/0x210
[<ffffffff8d000570>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffff8d00080c>] free_vmap_area_noflush+0x7c/0x90
[<ffffffff8d0022e0>] remove_vm_area+0x60/0x80
[<ffffffff8d00232a>] __vunmap+0x2a/0x100
[<ffffffff8d002476>] vfree+0x36/0x70
[<ffffffffc04b9d99>] do_ipt_get_ctl+0x349/0x420 [ip_tables]
[<ffffffff8d48faee>] nf_getsockopt+0x6e/0x90
[<ffffffff8d4a0a20>] ip_getsockopt+0x70/0xb0
[<ffffffff8d4c6695>] raw_getsockopt+0x25/0x50
[<ffffffff8d4304fa>] sock_common_getsockopt+0x1a/0x20
[<ffffffff8d42f7bd>] SyS_getsockopt+0x7d/0xf0
[<ffffffff8d58bede>] system_call_fastpath+0x25/0x2a
[<ffffffff8d58be21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 b4 8d f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #1026346
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb77150ca
Build id: 3.10.0-1062.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffffb767d280
Build id: 3.10.0-1062.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffb771518d
Build id: 3.10.0-1062.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffffb767d5f9
Build id: 3.10.0-1062.el7.x86_64
|
0x59
|
|
5
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffffb7800570
Build id: 3.10.0-1062.el7.x86_64
|
0x1a0
|
|
6
|
free_vmap_area_noflush
|
vmlinux |
0xffffffffb780080c
Build id: 3.10.0-1062.el7.x86_64
|
0x7c
|
|
7
|
remove_vm_area
|
vmlinux |
0xffffffffb78022e0
Build id: 3.10.0-1062.el7.x86_64
|
0x60
|
|
8
|
__vunmap
|
vmlinux |
0xffffffffb780232a
Build id: 3.10.0-1062.el7.x86_64
|
0x2a
|
|
9
|
vfree
|
vmlinux |
0xffffffffb7802476
Build id: 3.10.0-1062.el7.x86_64
|
0x36
|
|
10
|
do_ipt_get_ctl
|
ip_tables |
0xffffffffc0405d99
Build id: 3.10.0-1062.el7.x86_64
|
0x349
|
|
11
|
nf_getsockopt
|
vmlinux |
0xffffffffb7c8faee
Build id: 3.10.0-1062.el7.x86_64
|
0x6e
|
|
12
|
ip_getsockopt
|
vmlinux |
0xffffffffb7ca0a20
Build id: 3.10.0-1062.el7.x86_64
|
0x70
|
|
13
|
raw_getsockopt
|
vmlinux |
0xffffffffb7cc6695
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
14
|
sock_common_getsockopt
|
vmlinux |
0xffffffffb7c304fa
Build id: 3.10.0-1062.el7.x86_64
|
0x1a
|
|
15
|
SyS_getsockopt
|
vmlinux |
0xffffffffb7c2f7bd
Build id: 3.10.0-1062.el7.x86_64
|
0x7d
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffffb7d8bede
Build id: 3.10.0-1062.el7.x86_64
|
0x25
|
|
17
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffffb7d8be21
Build id: 3.10.0-1062.el7.x86_64
|
0xae
|
NMI watchdog: BUG: soft lockup - CPU#9 stuck for 22s! [iptables:103978]
Modules linked in: resguard_linux(OE) secmodel_linux(OE) livepatch_for_syshook(OE) ipt_rpfilter xt_multiport xt_set iptable_raw ip_set_hash_ip ip_set_hash_net ip_set vxlan ip6_udp_tunnel udp_tunnel veth xt_statistic xt_nat xt_mark ipt_REJECT nf_reject_ipv4 xt_comment iptable_mangle ip_vs_sh ip_vs_wrr ip_vs_rr ip_vs syshook_linux(OE) xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack br_netfilter bridge stp llc iscsi_tcp libiscsi_tcp libiscsi overlay(T) scsi_transport_iscsi sunrpc ppdev pcspkr joydev virtio_balloon sg parport_pc parport i2c_piix4 ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console virtio_scsi virtio_blk
cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ata_piix drm libata virtio_pci virtio_ring serio_raw floppy virtio drm_panel_orientation_quirks dm_mirror dm_region_hash dm_log dm_mod [last unloaded: secmodel_linux]
CPU: 9 PID: 103978 Comm: iptables Kdump: loaded Tainted: G OEL ------------ T 3.10.0-1062.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff8a480a6de2a0 ti: ffff8a45e72d8000 task.ti: ffff8a45e72d8000
RIP: 0010:[<ffffffffb77150ca>] [<ffffffffb77150ca>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8a45e72dbc38 EFLAGS: 00000202
RAX: 0000000000000000 RBX: 00000000000000fc RCX: ffff8a49bce204f0
RDX: 0000000000000000 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff8a45e72dbc70 R08: ffff8a3b295d4c00 R09: ffffffffb797f629
R10: ffff8a49bd05f160 R11: ffffeaddb8599600 R12: 0000000000000292
R13: ffff8a45e72dbbe8 R14: ffff8a45e72dbc38 R15: ffffffffb7824329
FS: 00007f5309cdbd48(0000) GS:ffff8a49bd040000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f5309c25000 CR3: 0000000cc826a000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb767d280>] ? flush_tlb_func+0xb0/0xb0
[<ffffffffb771518d>] on_each_cpu+0x2d/0x60
[<ffffffffb767d5f9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffffb7800570>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffffb780080c>] free_vmap_area_noflush+0x7c/0x90
[<ffffffffb78022e0>] remove_vm_area+0x60/0x80
[<ffffffffb780232a>] __vunmap+0x2a/0x100
[<ffffffffb7802476>] vfree+0x36/0x70
[<ffffffffc0405d99>] do_ipt_get_ctl+0x349/0x420 [ip_tables]
[<ffffffffb7c8faee>] nf_getsockopt+0x6e/0x90
[<ffffffffb7ca0a20>] ip_getsockopt+0x70/0xb0
[<ffffffffb7cc6695>] raw_getsockopt+0x25/0x50
[<ffffffffb7c304fa>] sock_common_getsockopt+0x1a/0x20
[<ffffffffb7c2f7bd>] SyS_getsockopt+0x7d/0xf0
[<ffffffffb7d8bede>] system_call_fastpath+0x25/0x2a
[<ffffffffb7d8be21>] ? system_call_after_swapgs+0xae/0x146
Code: 48 63 35 3e 68 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 d7 34 b8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 0d 68 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #770365
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffff92916c0a
Build id: 3.10.0-1160.el7.x86_64
|
0x20a
|
|
2
|
|
vmlinux |
0xffffffff9287e750
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffff92916ccd
Build id: 3.10.0-1160.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffff9287eac9
Build id: 3.10.0-1160.el7.x86_64
|
0x59
|
|
5
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffff92a04180
Build id: 3.10.0-1160.el7.x86_64
|
0x1a0
|
|
6
|
free_vmap_area_noflush
|
vmlinux |
0xffffffff92a0441c
Build id: 3.10.0-1160.el7.x86_64
|
0x7c
|
|
7
|
remove_vm_area
|
vmlinux |
0xffffffff92a05ef0
Build id: 3.10.0-1160.el7.x86_64
|
0x60
|
|
8
|
__vunmap
|
vmlinux |
0xffffffff92a05f3a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
9
|
vfree
|
vmlinux |
0xffffffff92a06086
Build id: 3.10.0-1160.el7.x86_64
|
0x36
|
|
10
|
do_ipt_get_ctl
|
ip_tables |
0xffffffffc04ead99
Build id: 3.10.0-1160.el7.x86_64
|
0x349
|
|
11
|
nf_getsockopt
|
vmlinux |
0xffffffff92e9655f
Build id: 3.10.0-1160.el7.x86_64
|
0x4f
|
|
12
|
ip_getsockopt
|
vmlinux |
0xffffffff92ea7500
Build id: 3.10.0-1160.el7.x86_64
|
0x70
|
|
13
|
raw_getsockopt
|
vmlinux |
0xffffffff92ecd2a5
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
|
14
|
sock_common_getsockopt
|
vmlinux |
0xffffffff92e374fa
Build id: 3.10.0-1160.el7.x86_64
|
0x1a
|
|
15
|
SyS_getsockopt
|
vmlinux |
0xffffffff92e367bd
Build id: 3.10.0-1160.el7.x86_64
|
0x7d
|
|
16
|
system_call_fastpath
|
vmlinux |
0xffffffff92f93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
|
17
|
?
system_call_after_swapgs
|
vmlinux |
0xffffffff92f93ed5
Build id: 3.10.0-1160.el7.x86_64
|
0xa2
|
NMI watchdog: BUG: soft lockup - CPU#6 stuck for 23s! [iptables:34543]
Modules linked in: nls_utf8 isofs tcp_diag udp_diag inet_diag ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables veth fuse xt_physdev sunrpc nf_tables vxlan ip6_udp_tunnel udp_tunnel xt_statistic xt_nat ipt_REJECT nf_reject_ipv4 xt_comment xt_mark ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 xt_addrtype iptable_filter xt_conntrack nf_nat nf_conntrack br_netfilter bridge stp llc overlay(T) ext4 mbcache jbd2 ppdev joydev virtio_balloon sg i2c_piix4 parport_pc parport pcspkr ip_tables xfs libcrc32c sr_mod cdrom ata_generic pata_acpi virtio_net virtio_console net_failover virtio_scsi virtio_blk failover cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ata_piix libata
serio_raw virtio_pci virtio_ring virtio drm_panel_orientation_quirks floppy dm_mirror dm_region_hash dm_log dm_mod
CPU: 6 PID: 34543 Comm: iptables Kdump: loaded Tainted: G L ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.11.1-0-g0551a4be2c-prebuilt.qemu-project.org 04/01/2014
task: ffff8d4d97259080 ti: ffff8d4d9723c000 task.ti: ffff8d4d9723c000
RIP: 0010:[<ffffffff92916c0a>] [<ffffffff92916c0a>] smp_call_function_many+0x20a/0x270
RSP: 0018:ffff8d4d9723fc40 EFLAGS: 00000202
RAX: 0000000000000002 RBX: 000c00009287e750 RCX: ffff8d512cea04c8
RDX: 0000000000000002 RSI: 00000000000000a0 RDI: 0000000000000000
RBP: ffff8d4d9723fc78 R08: ffff8d4e79b43c00 R09: ffffffff92b85f99
R10: ffff8d512cf9f160 R11: ffffdb48cb52b800 R12: 0000000000000282
R13: 0000000000000282 R14: ffff8d4d9723fbf0 R15: 80000003081c2867
FS: 00007f8846aa1d48(0000) GS:ffff8d512cf80000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f88469e7000 CR3: 000000011a30a000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffff9287e750>] ? flush_tlb_func+0xb0/0xb0
[<ffffffff92916ccd>] on_each_cpu+0x2d/0x60
[<ffffffff9287eac9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffff92a04180>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffff92a0441c>] free_vmap_area_noflush+0x7c/0x90
[<ffffffff92a05ef0>] remove_vm_area+0x60/0x80
[<ffffffff92a05f3a>] __vunmap+0x2a/0x100
[<ffffffff92a06086>] vfree+0x36/0x70
[<ffffffffc04ead99>] do_ipt_get_ctl+0x349/0x420 [ip_tables]
[<ffffffff92e9655f>] nf_getsockopt+0x4f/0x70
[<ffffffff92ea7500>] ip_getsockopt+0x70/0xb0
[<ffffffff92ecd2a5>] raw_getsockopt+0x25/0x50
[<ffffffff92e374fa>] sock_common_getsockopt+0x1a/0x20
[<ffffffff92e367bd>] SyS_getsockopt+0x7d/0xf0
[<ffffffff92f93f92>] system_call_fastpath+0x25/0x2a
[<ffffffff92f93ed5>] ? system_call_after_swapgs+0xa2/0x13a
Code: 48 63 35 1e 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 55 93 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 <f6> 41 20 01 75 f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89
Complete report #533731
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
smp_call_function_many
|
vmlinux |
0xffffffffb7716c0e
Build id: 3.10.0-1160.el7.x86_64
|
0x20e
|
|
2
|
|
vmlinux |
0xffffffffb767e750
Build id: 3.10.0-1160.el7.x86_64
|
0xb0
|
|
3
|
on_each_cpu
|
vmlinux |
0xffffffffb7716ccd
Build id: 3.10.0-1160.el7.x86_64
|
0x2d
|
|
4
|
flush_tlb_kernel_range
|
vmlinux |
0xffffffffb767eac9
Build id: 3.10.0-1160.el7.x86_64
|
0x59
|
|
5
|
|
vmlinux |
0xffffffffb7804064
Build id: 3.10.0-1160.el7.x86_64
|
0x84
|
|
6
|
__purge_vmap_area_lazy
|
vmlinux |
0xffffffffb7804180
Build id: 3.10.0-1160.el7.x86_64
|
0x1a0
|
|
7
|
free_vmap_area_noflush
|
vmlinux |
0xffffffffb780441c
Build id: 3.10.0-1160.el7.x86_64
|
0x7c
|
|
8
|
remove_vm_area
|
vmlinux |
0xffffffffb7805ef0
Build id: 3.10.0-1160.el7.x86_64
|
0x60
|
|
9
|
__vunmap
|
vmlinux |
0xffffffffb7805f3a
Build id: 3.10.0-1160.el7.x86_64
|
0x2a
|
|
10
|
vfree
|
vmlinux |
0xffffffffb7806086
Build id: 3.10.0-1160.el7.x86_64
|
0x36
|
|
11
|
ip_set_sockfn_get
|
ip_set |
0xffffffffc08e714b
Build id: 3.10.0-1160.el7.x86_64
|
0x18b
|
|
12
|
nf_getsockopt
|
vmlinux |
0xffffffffb7c9655f
Build id: 3.10.0-1160.el7.x86_64
|
0x4f
|
|
13
|
ip_getsockopt
|
vmlinux |
0xffffffffb7ca7500
Build id: 3.10.0-1160.el7.x86_64
|
0x70
|
|
14
|
raw_getsockopt
|
vmlinux |
0xffffffffb7ccd2a5
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
|
15
|
sock_common_getsockopt
|
vmlinux |
0xffffffffb7c374fa
Build id: 3.10.0-1160.el7.x86_64
|
0x1a
|
|
16
|
SyS_getsockopt
|
vmlinux |
0xffffffffb7c367bd
Build id: 3.10.0-1160.el7.x86_64
|
0x7d
|
|
17
|
system_call_fastpath
|
vmlinux |
0xffffffffb7d93f92
Build id: 3.10.0-1160.el7.x86_64
|
0x25
|
NMI watchdog: BUG: soft lockup - CPU#7 stuck for 67s! [iptables-restor:62283]
Modules linked in: rpcsec_gss_krb5 nfsv4 dns_resolver nfs fscache xt_owner tcp_diag udp_diag inet_diag xt_multiport ipt_rpfilter iptable_raw ip_set_hash_net ip_set_hash_ip ipip tunnel4 ip_tunnel xt_set ip_set_hash_ipportnet ip_set_hash_ipport ip_set_bitmap_port ip_set_hash_ipportip ip_set dummy ip_vs_sh ip_vs_wrr ip_vs_rr xt_ipvs ip_vs xt_REDIRECT nf_nat_redirect ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6_tables xt_comment vxlan ip6_udp_tunnel udp_tunnel iptable_mangle xt_mark xt_nat veth xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 nf_conntrack_netlink nfnetlink xt_addrtype iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack overlay(T) dm_mirror dm_region_hash dm_log dm_mod sb_edac intel_powerclamp coretemp intel_rapl iosf_mbi
ipmi_ssif kvm_intel kvm iTCO_wdt iTCO_vendor_support irqbypass cdc_ether crc32_pclmul usbnet ghash_clmulni_intel aesni_intel lrw pcspkr gf128mul glue_helper ablk_helper cryptd mii lpc_ich i2c_i801 mei_me sg mei wmi acpi_power_meter ipmi_si ipmi_devintf ipmi_msghandler acpi_pad nfsd auth_rpcgss nfs_acl lockd br_netfilter grace bridge sunrpc stp llc ip_tables xfs libcrc32c sd_mod sr_mod cdrom crc_t10dif crct10dif_generic mgag200 i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm ahci drm libahci crct10dif_pclmul crct10dif_common tg3 crc32c_intel libata megaraid_sas ptp pps_core drm_panel_orientation_quirks
CPU: 7 PID: 62283 Comm: iptables-restor Tainted: G ------------ T 3.10.0-1160.el7.x86_64 #1
Hardware name: LENOVO System x3650 M5: -[5462I37]-/00MW307, BIOS -[TCE106KUS-1.10]- 04/17/2015
task: ffff9d3af7240000 ti: ffff9d3db21a8000 task.ti: ffff9d3db21a8000
RIP: 0010:[<ffffffffb7716c0e>] [<ffffffffb7716c0e>] smp_call_function_many+0x20e/0x270
RSP: 0018:ffff9d3db21abc98 EFLAGS: 00000202
RAX: 000000000000000b RBX: 00000000000000fc RCX: ffff9d3dffd60750
RDX: 000000000000000b RSI: 000000000000000c RDI: 0000000000000000
RBP: ffff9d3db21abcd0 R08: ffff9d3a78b7e800 R09: ffffffffb7985f99
R10: ffff9d3dffc5f160 R11: ffffeb299096fa00 R12: 0000000000000292
R13: ffff9d3db21abc48 R14: ffff9d3dfffda008 R15: 00000000ef45f441
FS: 00007f890fc06500(0000) GS:ffff9d3dffc40000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000562d1a45a9e8 CR3: 000000083e642000 CR4: 00000000001607e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
[<ffffffffb767e750>] ? flush_tlb_func+0xb0/0xb0
[<ffffffffb7716ccd>] on_each_cpu+0x2d/0x60
[<ffffffffb767eac9>] flush_tlb_kernel_range+0x59/0xa0
[<ffffffffb7804064>] ? __purge_vmap_area_lazy+0x84/0x210
[<ffffffffb7804180>] __purge_vmap_area_lazy+0x1a0/0x210
[<ffffffffb780441c>] free_vmap_area_noflush+0x7c/0x90
[<ffffffffb7805ef0>] remove_vm_area+0x60/0x80
[<ffffffffb7805f3a>] __vunmap+0x2a/0x100
[<ffffffffb7806086>] vfree+0x36/0x70
[<ffffffffc08e714b>] ip_set_sockfn_get+0x18b/0x310 [ip_set]
[<ffffffffb7c9655f>] nf_getsockopt+0x4f/0x70
[<ffffffffb7ca7500>] ip_getsockopt+0x70/0xb0
[<ffffffffb7ccd2a5>] raw_getsockopt+0x25/0x50
[<ffffffffb7c374fa>] sock_common_getsockopt+0x1a/0x20
[<ffffffffb7c367bd>] SyS_getsockopt+0x7d/0xf0
[<ffffffffb7d93f92>] system_call_fastpath+0x25/0x2a
Code: 89 c4 00 89 c2 39 f0 0f 8d 7d fe ff ff 48 98 49 8b 0f 48 03 0c c5 e0 13 35 b8 f6 41 20 01 74 cd 0f 1f 44 00 00 f3 90 f6 41 20 01 <75> f8 48 63 35 ed 88 c4 00 eb b7 0f b6 4d cc 4c 89 f2 4c 89 ee